diff --git a/.agents/skills/reverse-engineer-anything/references/native-and-artifacts.md b/.agents/skills/reverse-engineer-anything/references/native-and-artifacts.md index 35576998f9..5e854285d2 100644 --- a/.agents/skills/reverse-engineer-anything/references/native-and-artifacts.md +++ b/.agents/skills/reverse-engineer-anything/references/native-and-artifacts.md @@ -708,6 +708,13 @@ cache one decoded block and support retained filtered/paged views. No decryption or SerializedFile object decoding is provided. See [UnityFS](https://github.com/morluto/rea/blob/main/docs/unityfs.md). +WeChat mini-program packages (`.wxapkg` files with `0xBE`/`0xED` marks) list +members with declared names and absolute carrier offsets; member bytes are +stored verbatim. `V1MMWX` PC-client packages report `encrypted` directory +status with unknown coverage and are refused on extraction; no decryption is +attempted. See +[wxapkg](https://github.com/morluto/rea/blob/main/docs/wxapkg.md). + U-Boot legacy uImages use the same inspection/extraction workflows. Prefer a retained summary and selected inventory views for header facts or component IDs. Header addresses/CPU/OS are declarations; payloads remain as stored. diff --git a/docs/cli.md b/docs/cli.md index 28af2b71e4..3380b0e18f 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -281,6 +281,17 @@ Format versions 6 through 8 retain header, block and directory facts with none/LZ4/LZ4HC block decompression. See [UnityFS](unityfs.md) for encryption and LZMA coverage boundaries. +Inspect a WeChat mini-program package without WeChat or an unpacker: + +```sh +rea inspect-artifact app.wxapkg --detail summary --evidence_output evidence.json --json +rea extract-artifact app.wxapkg --json +``` + +Plaintext packages retain header, table and member facts with verbatim member +bytes. `V1MMWX` packages are detected as encrypted coverage and refused on +extraction. See [wxapkg](wxapkg.md) for layout and coverage boundaries. + Inspect an ESP32-family app or bootloader image without esptool: ```sh diff --git a/docs/index.md b/docs/index.md index bd9cbc36a5..7cc44bdc1d 100644 --- a/docs/index.md +++ b/docs/index.md @@ -47,19 +47,19 @@ For runtime instrumentation, see the [Frida guide](./frida-instrumentation.md). ## Choose a guide -| Task | Guides | -| --------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| Identify a desktop installation | [Runtime candidates, evidence and follow-up files](desktop-applications.md) | -| Inspect native functions, references, or types | [Native binaries](native-investigation.md), [Go metadata](go-binaries.md), [Rust dependencies](rust-binaries.md), [CPython pyc files](python-pyc.md), [PE certificates](pe-certificates.md), [PE imports](pe-imports.md), [PE TLS/load config](pe-load-config.md), [ELF notes](elf-notes.md), [Linux kernel modules](kernel-modules.md), [Apple IPS](apple-ips.md), [Erlang BEAM](erlang-beam.md), [offline ELF layout](binary-diagnostics.md), [recorded crashes](recorded-crashes.md) | -| Select a native provider or import format | [Ghidra semantics](ghidra-provider.md), [Windows](windows-ghidra-p0.md), [DOS](ghidra-dos.md), [raw bytes](raw-binary.md), [GameCube DOL](ghidra-dol.md), [MIPS](ghidra-mips.md), [PSP](ghidra-psp.md), [NativeAOT](ghidra-nativeaot.md), [IDA](ida-provider.md), [Kuna](kuna-provider.md), [JEB](jeb-analysis.md) | -| Analyze JavaScript or Electron artifacts | [Static analysis](javascript-artifact-reconstruction.md), [tracing and comparisons](javascript-application-workflows.md), [application graph](javascript-application-graph.md), [source recovery](javascript-recovery.md) | -| Compare static JavaScript with runtime observations | [Runtime attribution](javascript-runtime-observation.md), [reconciliation](javascript-runtime-reconciliation.md), [Electron observation](electron-observation.md), [Electron fuses](electron-fuses.md) | -| Inspect assemblies, packages, or firmware | [Godot PCK](godot-pck.md), [UnityFS](unityfs.md), [U-Boot uImage](uimage.md), [.NET](managed-code-analysis.md), [Python packages](python-packaged-applications.md), [Android](android-analysis.md), [APK signatures](apk-signature.md), [Apktool resources](apktool-resource-analysis.md), [ADB acquisition](adb-device-analysis.md), [Apple](apple-application-analysis.md), [Harmony](harmony-analysis.md), [firmware](firmware-analysis.md), [ESP images](esp-image.md), [ESP flash images](esp-flash-image.md), [WASM](wasm-artifacts.md), [EVM](evm-bytecode.md) | -| Observe or compare a browser scenario | [CDP observation](browser-observation.md), [scenarios](browser-scenario-contract.md), [runtime evidence](web-runtime.md), [historical captures](web-network-captures.md) | -| Serve recorded behavior as an agent environment | [Agent environments](agent-environments.md) | -| Trace captured website source | [Source locations](web-source-location.md), [module traces](website-module-trace.md), [script export](website-script-export.md) | -| Capture and compare a command | [Process capture](process-capture.md) | -| Use CLI/MCP and retain evidence | [CLI](cli.md), [MCP contracts](mcp-contracts.md), [agent prompts](mcp-prompts.md), [obligation ledgers](reconstruction-obligation-ledgers.md) | +| Task | Guides | +| --------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| Identify a desktop installation | [Runtime candidates, evidence and follow-up files](desktop-applications.md) | +| Inspect native functions, references, or types | [Native binaries](native-investigation.md), [Go metadata](go-binaries.md), [Rust dependencies](rust-binaries.md), [CPython pyc files](python-pyc.md), [PE certificates](pe-certificates.md), [PE imports](pe-imports.md), [PE TLS/load config](pe-load-config.md), [ELF notes](elf-notes.md), [Linux kernel modules](kernel-modules.md), [Apple IPS](apple-ips.md), [Erlang BEAM](erlang-beam.md), [offline ELF layout](binary-diagnostics.md), [recorded crashes](recorded-crashes.md) | +| Select a native provider or import format | [Ghidra semantics](ghidra-provider.md), [Windows](windows-ghidra-p0.md), [DOS](ghidra-dos.md), [raw bytes](raw-binary.md), [GameCube DOL](ghidra-dol.md), [MIPS](ghidra-mips.md), [PSP](ghidra-psp.md), [NativeAOT](ghidra-nativeaot.md), [IDA](ida-provider.md), [Kuna](kuna-provider.md), [JEB](jeb-analysis.md) | +| Analyze JavaScript or Electron artifacts | [Static analysis](javascript-artifact-reconstruction.md), [tracing and comparisons](javascript-application-workflows.md), [application graph](javascript-application-graph.md), [source recovery](javascript-recovery.md) | +| Compare static JavaScript with runtime observations | [Runtime attribution](javascript-runtime-observation.md), [reconciliation](javascript-runtime-reconciliation.md), [Electron observation](electron-observation.md), [Electron fuses](electron-fuses.md) | +| Inspect assemblies, packages, or firmware | [Godot PCK](godot-pck.md), [UnityFS](unityfs.md), [wxapkg](wxapkg.md), [U-Boot uImage](uimage.md), [.NET](managed-code-analysis.md), [Python packages](python-packaged-applications.md), [Android](android-analysis.md), [APK signatures](apk-signature.md), [Apktool resources](apktool-resource-analysis.md), [ADB acquisition](adb-device-analysis.md), [Apple](apple-application-analysis.md), [Harmony](harmony-analysis.md), [firmware](firmware-analysis.md), [ESP images](esp-image.md), [ESP flash images](esp-flash-image.md), [WASM](wasm-artifacts.md), [EVM](evm-bytecode.md) | +| Observe or compare a browser scenario | [CDP observation](browser-observation.md), [scenarios](browser-scenario-contract.md), [runtime evidence](web-runtime.md), [historical captures](web-network-captures.md) | +| Serve recorded behavior as an agent environment | [Agent environments](agent-environments.md) | +| Trace captured website source | [Source locations](web-source-location.md), [module traces](website-module-trace.md), [script export](website-script-export.md) | +| Capture and compare a command | [Process capture](process-capture.md) | +| Use CLI/MCP and retain evidence | [CLI](cli.md), [MCP contracts](mcp-contracts.md), [agent prompts](mcp-prompts.md), [obligation ledgers](reconstruction-obligation-ledgers.md) | For development, use [testing](testing.md), [client verification](client-verification.md), [tool design](tool-design.md), [admission limits and recovery](design/limits-and-recovery.md), diff --git a/docs/wxapkg.md b/docs/wxapkg.md new file mode 100644 index 0000000000..e898350d83 --- /dev/null +++ b/docs/wxapkg.md @@ -0,0 +1,84 @@ +# WeChat mini-program wxapkg archives + +`inspect-artifact` and `extract-artifact` read plaintext WeChat mini-program +package (`.wxapkg`) containers without running WeChat or an unpacker. The +plaintext form starts with a `0xBE` mark and an 18-byte big-endian header: +an unknown `info` field producers leave zero, the member-table byte length, +the member-data byte length, a `0xED` mark and the member count. Producers +either count only the name/offset/size rows in that table length or include +the four-byte member count as well; both declarations are accepted. The table +region then carries one name/offset/size row per member; member bytes are +stored verbatim at absolute file offsets. + +Inspection retains the header facts: `info`, index and data lengths, member +count and directory status. Members retain declared source paths, absolute +carrier offsets and declared sizes, so `source_location` reports real file +positions. Member reads only examine their declared range in bounded 64 KiB +chunks. Carrier freshness is checked when a member is opened and after its +stream completes. Reader close cancels and joins outstanding streams before +releasing its descriptor. + +Encrypted PC-client packages start with the `V1MMWX` magic instead of the +plain header. They keep their format identity but report `encrypted` +directory status with unknown coverage: member identities and bytes remain +unknown and no decryption is attempted, so extraction refuses with a typed +error rather than producing ciphertext. Declared tables or data regions +that exceed the file, member extents outside the declared data region, +truncated rows, invalid UTF-8 names, and counts that cannot fit their table +are malformed-input errors. + +## Resource bounds + +These are resource policies, not format maxima. The encoded member table is +limited to 16 MiB and its buffer, parsed rows, retained entries and escaped +path projections share a conservative 64 MiB admission budget, reserved +before allocation; a declared member name is limited to 64 KiB. A reader +admits four active member streams. The shared archive plaintext-work ceiling +is the larger of 256 MiB and 500 times carrier size: every declared member +extent is charged, including aliases that overlap one stored range. A +separate member-I/O budget allows four times the larger of 1,024 and the sum +of the member count and declared 64 KiB plaintext chunks, bounding repeated +tiny-member reads independently of output bytes. + +## CLI + +```sh +rea inspect-artifact app.wxapkg --detail summary \ + --evidence_output evidence/package.json --json +rea extract-artifact app.wxapkg --json +``` + +Detection is byte-based on the `0xBE`/`0xED` marks or the `V1MMWX` magic, so +`.wxapkg`, `__APP__.wxapkg` and extensionless cache files all classify as +`wxapkg` containers. Compact delivery keeps complete Evidence in the +requested bundle. + +## Verification scope + +Synthetic fixtures cover plaintext headers, multiple members, empty members, +chunked reads, unicode and dotted names, malformed marks, truncated tables, +invalid UTF-8, table and directory-budget admission, member extents inside +the header, the index, or beyond the declared data region, a data region +declared past the file, a one-byte member whose table length includes the +member count, an overstated first row consuming its successor, V1MMWX +packages, cancellation inside member reads, paused iteration, close during +I/O, cleanup retry, stream capacity and carrier freshness. Compiled +full/compact MCP tests cover inventory and extraction. + +Real producer verification used plaintext `0xBE` packages from a local macOS +WeChat mini-program cache (`__APP__.wxapkg` main packages and +`__PLUGINCODE__.wxapkg` plugin packages across three distinct app +directories): every member extracted byte-identical to its declared carrier +range, including `__plugin__/`-namespaced members and the producer's +root-prefixed (`/name`) member convention. A cached `V1MMWX` PC package +reports the encrypted directory state with unknown member facts and refuses +extraction. + +`wxml`/`wxss` source recovery, compiled bundle semantics, subpackage +relationships and runtime behavior are outside this container reader's +verification scope. + +Artifact comparison pairs members by their package-relative paths and +compares their bytes. It does not compare `wxapkg` or `wxapkg-entry` +metadata: header fields repeat the package bytes, `index` is layout, and +`source_path` keeps a producer root prefix that the occurrence path strips. diff --git a/src/application/BinaryTargetResolver.ts b/src/application/BinaryTargetResolver.ts index d69504c5d7..68845303df 100644 --- a/src/application/BinaryTargetResolver.ts +++ b/src/application/BinaryTargetResolver.ts @@ -85,6 +85,10 @@ import { hasDiskImageSignature } from "../domain/apple/diskImageFormat.js"; import { hasGodotPckHeader } from "../domain/godotPckFormat.js"; import { hasUImageMagic } from "../domain/uimageFormat.js"; import { hasUnityFsHeader } from "../domain/unityfsFormat.js"; +import { + hasWxapkgEncryptedMagic, + hasWxapkgHeader, +} from "../domain/wxapkgFormat.js"; import { detectHexRecordEncoding } from "../domain/firmware/hexRecords.js"; import { hasUf2Block } from "../domain/firmware/uf2.js"; import { @@ -777,6 +781,7 @@ const detectArtifactFormat = async ( if (hasZipSignature(magic)) return zipRootPackageFormat(lower); if (hasGodotPckHeader(magic)) return "godot-pck"; if (hasUnityFsHeader(magic)) return "unityfs"; + if (hasWxapkgEncryptedMagic(magic) || hasWxapkgHeader(magic)) return "wxapkg"; if (hasUImageMagic(magic)) return "uimage"; const hexRecords = detectHexRecordEncoding(magic); if (hexRecords !== undefined) return hexRecords; @@ -883,6 +888,7 @@ const isArchiveFormat = ( "ines", "godot-pck", "unityfs", + "wxapkg", "uimage", "dmg", "pkg", diff --git a/src/application/binary/TargetFormatVocabularies.test.ts b/src/application/binary/TargetFormatVocabularies.test.ts index 8aef2e212f..299093c9d1 100644 --- a/src/application/binary/TargetFormatVocabularies.test.ts +++ b/src/application/binary/TargetFormatVocabularies.test.ts @@ -120,6 +120,7 @@ const TARGETS: TargetFixtures = { ines: { ...archive, format: "ines" }, "godot-pck": { ...archive, format: "godot-pck" }, unityfs: { ...archive, format: "unityfs" }, + wxapkg: { ...archive, format: "wxapkg" }, uimage: { ...archive, format: "uimage" }, uf2: { ...archive, format: "uf2" }, dfuse: { ...archive, format: "dfuse" }, diff --git a/src/artifacts/ArtifactReader.ts b/src/artifacts/ArtifactReader.ts index 8709121b3d..ae1621a577 100644 --- a/src/artifacts/ArtifactReader.ts +++ b/src/artifacts/ArtifactReader.ts @@ -199,6 +199,7 @@ export interface ArtifactReader { | "ines" | "godot-pck" | "unityfs" + | "wxapkg" | "uimage" | "pyinstaller" | "dotnet-bundle" diff --git a/src/artifacts/WxapkgArtifactReader.boundaries.test.ts b/src/artifacts/WxapkgArtifactReader.boundaries.test.ts new file mode 100644 index 0000000000..5d8b69eb9c --- /dev/null +++ b/src/artifacts/WxapkgArtifactReader.boundaries.test.ts @@ -0,0 +1,310 @@ +import { once } from "node:events"; +import { writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { buffer, text } from "node:stream/consumers"; +import { setImmediate } from "node:timers/promises"; +import { describe, expect, it, vi } from "vitest"; +import { createTestTempDirectory } from "../../tests/fixtures/temporaryDirectory.js"; +import { createDeferred } from "../../tests/fixtures/binarySession.js"; +import { awaitAcknowledgement } from "../../tests/fixtures/lifecycleAcknowledgement.js"; +import { buildWxapkg } from "../../tests/fixtures/wxapkg.js"; +import { openRegularFile } from "../filesystem/RegularFile.js"; +import { OwnedFileHandle } from "../filesystem/OwnedFileHandle.js"; +import { WxapkgArtifactReader } from "./WxapkgArtifactReader.js"; + +const fixture = async (bytes: Buffer): Promise => { + const path = join( + await createTestTempDirectory("rea-wxapkg-boundary-"), + "package.wxapkg", + ); + await writeFile(path, bytes); + return path; +}; +const entries = async (reader: WxapkgArtifactReader) => { + const result = []; + for await (const entry of reader.entries()) result.push(entry); + return result; +}; + +describe("wxapkg allocation and malformed-input boundaries", () => { + it("admits the encoded table budget before reading it", async () => { + const bytes = buildWxapkg([{ path: "a", data: "x" }]); + bytes.writeUInt32BE(17 * 1024 * 1024, 5); + const file = Buffer.alloc(18 + 17 * 1024 * 1024); + bytes.copy(file); + const path = await fixture(file); + const admitted = await openRegularFile(path, { symlinks: "reject" }); + const reader = new WxapkgArtifactReader(path, admitted); + const reads = vi.spyOn(admitted.handle, "read"); + try { + await expect(entries(reader)).rejects.toMatchObject({ + reason: "limit", + boundary: "capacity", + }); + expect(reads).toHaveBeenCalledTimes(1); // bounded header only + } finally { + await reader.close(); + reads.mockRestore(); + await admitted.handle.close(); + } + }); + + it("reserves row storage before constructing a large member array", async () => { + // 140k minimal rows fit the encoded table budget but exceed the retained + // directory representation budget mid-parse. + const fileCount = 140_000; + const tableBytes = fileCount * 13; + const file = Buffer.alloc(18 + tableBytes); + file[0] = 0xbe; + file.writeUInt32BE(0, 1); + file.writeUInt32BE(tableBytes, 5); + file.writeUInt32BE(0, 9); + file[13] = 0xed; + file.writeUInt32BE(fileCount, 14); + for (let index = 0; index < fileCount; index++) { + const row = 18 + index * 13; + file.writeUInt32BE(1, row); + file[row + 4] = 0x61; + // Zero-size members at the data start stay admissible so parsing + // reaches the representation budget. + file.writeUInt32BE(18 + tableBytes, row + 5); + file.writeUInt32BE(0, row + 9); + } + const reader = new WxapkgArtifactReader(await fixture(file)); + try { + await expect(entries(reader)).rejects.toMatchObject({ + reason: "limit", + boundary: "capacity", + }); + } finally { + await reader.close(); + } + }); + + it("charges overlapping readable members, not only their unique range", async () => { + const size = 1024 * 1024; + const members = Array.from({ length: 600 }, (_, index) => ({ + path: `alias-${String(index)}`, + data: index === 0 ? Buffer.alloc(size, 0x61) : Buffer.alloc(0), + size, + })); + const reader = new WxapkgArtifactReader( + await fixture(buildWxapkg(members, { aliasOffsets: true })), + ); + try { + await expect(entries(reader)).rejects.toMatchObject({ reason: "limit" }); + } finally { + await reader.close(); + } + }); + + it("types a short later row after an overstated first name consumes its bytes", async () => { + const bytes = buildWxapkg([ + { path: "a".repeat(40), data: "x" }, + { path: "b", data: "y" }, + ]); + // The first row's declared name consumes most of the second row. The + // minimum total table-size check alone cannot establish each later row. + bytes.writeUInt32BE(50, 18); + // Keep the first member admissible at the position the parser reads after + // the inflated name, so the failure must come from the short later row. + const dataStart = 18 + bytes.readUInt32BE(5); + bytes.writeUInt32BE(dataStart, 18 + 4 + 50); + bytes.writeUInt32BE(1, 18 + 4 + 50 + 4); + const reader = new WxapkgArtifactReader(await fixture(bytes)); + try { + await expect(entries(reader)).rejects.toMatchObject({ + reason: "format", + message: expect.stringContaining("truncated"), + }); + } finally { + await reader.close(); + } + }); +}); + +describe("wxapkg stream ownership", () => { + it("retains descriptor ownership after cleanup fails and permits retry", async () => { + const reader = new WxapkgArtifactReader( + await fixture(buildWxapkg([{ path: "x", data: "bytes" }])), + ); + const [entry] = await entries(reader); + const stream = await reader.open(entry!); + const failure = new Error("descriptor close denied"); + const close = vi + .spyOn(OwnedFileHandle.prototype, "close") + .mockRejectedValueOnce(failure); + try { + await expect(reader.close()).rejects.toBe(failure); + expect(stream.closed).toBe(true); + await expect(reader.open(entry!)).rejects.toMatchObject({ + reason: "unavailable", + }); + await reader.close(); + expect(close).toHaveBeenCalledTimes(2); + } finally { + close.mockRestore(); + await reader.close(); + } + }); + + it("joins an outstanding positional read before close completes", async () => { + const path = await fixture(buildWxapkg([{ path: "x", data: "bytes" }])); + const admitted = await openRegularFile(path, { symlinks: "reject" }); + const reader = new WxapkgArtifactReader(path, admitted); + const reached = createDeferred(), + release = createDeferred(); + const nativeRead = admitted.handle.read.bind(admitted.handle); + const [entry] = await entries(reader); + const reads = vi.spyOn(admitted.handle, "read").mockImplementation((async ( + ...args: unknown[] + ) => { + reached.resolve(); + await release.promise; + return Reflect.apply(nativeRead, admitted.handle, args); + }) as typeof admitted.handle.read); + try { + const stream = await reader.open(entry!); + stream.read(0); + await awaitAcknowledgement( + reached.promise, + once(stream, "close"), + "positional member read", + ); + let closed = false; + const closing = reader.close().then(() => { + closed = true; + }); + await setImmediate(); + expect(closed).toBe(false); + release.resolve(); + await closing; + expect(stream.closed).toBe(true); + } finally { + release.resolve(); + await reader.close(); + reads.mockRestore(); + await admitted.handle.close(); + } + }); + + it("preserves cancellation inside a member read", async () => { + const data = Buffer.alloc(256 * 1024, 0x61); + const path = await fixture(buildWxapkg([{ path: "x", data }])); + const admitted = await openRegularFile(path, { symlinks: "reject" }); + const reader = new WxapkgArtifactReader(path, admitted); + const controller = new AbortController(); + const reason = new Error("cancel during member read"); + const nativeRead = admitted.handle.read.bind(admitted.handle); + const [entry] = await entries(reader); + const reads = vi.spyOn(admitted.handle, "read").mockImplementation((async ( + ...args: unknown[] + ) => { + const result = await Reflect.apply(nativeRead, admitted.handle, args); + globalThis.setImmediate(() => { + controller.abort(reason); + }); + return result; + }) as typeof admitted.handle.read); + try { + await expect( + buffer(await reader.open(entry!, controller.signal)), + ).rejects.toMatchObject({ reason: "cancelled", cause: reason }); + } finally { + await reader.close(); + reads.mockRestore(); + await admitted.handle.close(); + } + }); +}); + +describe("wxapkg stream capacity and freshness", () => { + it("bounds active streams and releases capacity after abandonment", async () => { + const reader = new WxapkgArtifactReader( + await fixture(buildWxapkg([{ path: "x", data: "bytes" }])), + ); + try { + const [entry] = await entries(reader); + const streams = await Promise.all( + Array.from({ length: 4 }, () => reader.open(entry!)), + ); + await expect(reader.open(entry!)).rejects.toMatchObject({ + reason: "limit", + boundary: "capacity", + }); + const closed = new Promise((resolve) => { + streams[0]!.once("close", resolve); + }); + streams[0]!.destroy(); + await closed; + expect(await text(await reader.open(entry!))).toBe("bytes"); + } finally { + await reader.close(); + } + }); + + it("bounds repeated member I/O operations, including empty members", async () => { + const reader = new WxapkgArtifactReader( + await fixture( + buildWxapkg([ + { path: "x", data: "b" }, + { path: "empty.wxss", data: "" }, + ]), + ), + ); + try { + const [entry, emptyEntry] = await entries(reader); + await expect( + (async () => { + for (let index = 0; index < 5000; index++) + await buffer(await reader.open(entry!)); + })(), + ).rejects.toMatchObject({ reason: "limit", boundary: "capacity" }); + await expect( + (async () => { + for (let index = 0; index < 5000; index++) + await buffer(await reader.open(emptyEntry!)); + })(), + ).rejects.toMatchObject({ reason: "limit", boundary: "capacity" }); + } finally { + await reader.close(); + } + }); + + it("validates freshness before serving member bytes", async () => { + const bytes = buildWxapkg([{ path: "x", data: "before" }]); + const path = await fixture(bytes); + const reader = new WxapkgArtifactReader(path); + try { + const [entry] = await entries(reader); + expect(await text(await reader.open(entry!))).toBe("before"); + await writeFile(path, Buffer.concat([bytes, Buffer.from([0])])); + await expect(reader.open(entry!)).rejects.toMatchObject({ + reason: "integrity", + }); + } finally { + await reader.close(); + } + }); + + it("does not turn a paused inventory into complete coverage when closed", async () => { + const reader = new WxapkgArtifactReader( + await fixture( + buildWxapkg([ + { path: "x", data: "bytes" }, + { path: "y", data: "bytes" }, + ]), + ), + ); + const iterator = reader.entries()[Symbol.asyncIterator](); + try { + expect((await iterator.next()).done).toBe(false); + await reader.close(); + await expect(iterator.next()).rejects.toMatchObject({ + reason: "unavailable", + }); + } finally { + await reader.close(); + } + }); +}); diff --git a/src/artifacts/WxapkgArtifactReader.test.ts b/src/artifacts/WxapkgArtifactReader.test.ts new file mode 100644 index 0000000000..51205da61a --- /dev/null +++ b/src/artifacts/WxapkgArtifactReader.test.ts @@ -0,0 +1,468 @@ +import { writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { text } from "node:stream/consumers"; + +import { describe, expect, it } from "vitest"; + +import { createTestTempDirectory } from "../../tests/fixtures/temporaryDirectory.js"; +import { + buildWxapkg, + wxapkgEncryptedBytes, + type WxapkgMember, +} from "../../tests/fixtures/wxapkg.js"; + +import type { + ArtifactEntry, + ArtifactTraversalOptions, +} from "./ArtifactReader.js"; +import { hasWxapkgHeader } from "../domain/wxapkgFormat.js"; +import { WxapkgArtifactReader } from "./WxapkgArtifactReader.js"; + +const writePackage = async (bytes: Buffer): Promise => { + const path = join( + await createTestTempDirectory("rea-wxapkg-"), + "package.wxapkg", + ); + await writeFile(path, bytes); + return path; +}; + +const list = async ( + reader: WxapkgArtifactReader, + options?: ArtifactTraversalOptions, +): Promise => { + const entries: ArtifactEntry[] = []; + for await (const entry of reader.entries(undefined, options)) + entries.push(entry); + return entries; +}; + +const withReader = async ( + bytes: Buffer, + use: (reader: WxapkgArtifactReader) => Promise, +): Promise => { + const reader = new WxapkgArtifactReader(await writePackage(bytes)); + try { + return await use(reader); + } finally { + await reader.close(); + } +}; + +const members: readonly WxapkgMember[] = [ + { path: "app-config.json", data: "{}" }, + { path: "pages/index/index.js", data: "module.exports={};" }, + { path: "empty.wxss", data: "" }, +]; + +describe("WxapkgArtifactReader", () => { + it("lists members and reads their declared bytes", async () => { + await withReader(buildWxapkg(members), async (reader) => { + const entries = await list(reader); + expect(entries.map(({ path }) => path)).toEqual([ + "app-config.json", + "pages/index/index.js", + "empty.wxss", + ]); + expect(await text(await reader.open(entries[0]!))).toBe("{}"); + expect(await text(await reader.open(entries[1]!))).toBe( + "module.exports={};", + ); + expect(await text(await reader.open(entries[2]!))).toBe(""); + }); + }); + + it("preserves member provenance on entries and metadata", async () => { + await withReader(buildWxapkg(members), async (reader) => { + const entries = await list(reader); + const entry = entries[1]!; + expect(entry).toMatchObject({ + path: "pages/index/index.js", + sourcePath: "pages/index/index.js", + kind: "file", + declaredSize: 18, + compressedSize: null, + encrypted: false, + declaredIntegrity: null, + unpacked: false, + adapterKey: "wxapkg-entry:1", + formatMetadata: { + kind: "wxapkg-entry", + index: 1, + source_path: "pages/index/index.js", + }, + }); + // Member offsets are absolute carrier positions after the table. + expect(entry.byteOffset).toBeGreaterThanOrEqual(18); + expect(reader.formatMetadata()).toMatchObject({ + kind: "wxapkg", + file_count: 3, + directory_status: "available", + }); + }); + }); + + it("reads a member larger than the chunk size", async () => { + const data = Buffer.alloc(200 * 1024, 0x61); + await withReader( + buildWxapkg([{ path: "big.bin", data }]), + async (reader) => { + const [entry] = await list(reader); + const chunks: Buffer[] = []; + for await (const chunk of await reader.open(entry!)) + chunks.push(chunk as Buffer); + expect(Buffer.concat(chunks).equals(data)).toBe(true); + }, + ); + }); + + it("accepts consecutive dots and unicode names", async () => { + await withReader( + buildWxapkg([ + { path: "data..bin", data: "a" }, + { path: "页面/配置.json", data: "b" }, + ]), + async (reader) => { + const entries = await list(reader); + expect(entries[0]?.path).toBe("data..bin"); + expect(entries[1]?.path).toBe("页面/配置.json"); + expect(await text(await reader.open(entries[1]!))).toBe("b"); + }, + ); + }); + + it("translates producer root prefixes while keeping declared names", async () => { + await withReader( + buildWxapkg([ + { path: "/app-config.json", data: "{}" }, + { path: "/pages/index/index.js", data: "x=1;" }, + ]), + async (reader) => { + const entries = await list(reader); + expect(entries.map((entry) => entry.path)).toEqual([ + "app-config.json", + "pages/index/index.js", + ]); + expect(entries[1]).toMatchObject({ + sourcePath: "/pages/index/index.js", + formatMetadata: { source_path: "/pages/index/index.js" }, + }); + expect(await text(await reader.open(entries[0]!))).toBe("{}"); + }, + ); + }); + + it("exposes no members for a V1MMWX-encrypted package", async () => { + await withReader(wxapkgEncryptedBytes(), async (reader) => { + const coverage: { + status?: string; + limitations?: readonly string[]; + } = {}; + const entries = await list(reader, { + onCoverage: (observation) => { + coverage.status = observation.status; + coverage.limitations = observation.limitations; + }, + }); + expect(entries).toEqual([]); + expect(coverage.status).toBe("unknown"); + expect(coverage.limitations?.join(" ")).toContain("V1MMWX"); + expect(reader.formatMetadata()).toMatchObject({ + kind: "wxapkg", + directory_status: "encrypted", + file_count: null, + info: null, + }); + }); + }); +}); + +describe("WxapkgArtifactReader malformed inputs", () => { + it.each([ + ["wrong first mark", { firstMark: 0x00 }], + ["wrong last mark", { lastMark: 0x00 }], + ["file count above the row count", { declaredFileCount: 4 }], + ["index length below the rows", { declaredIndexLength: 10 }], + ["declared name exceeds the row", { nameLength: 1024 }], + ["zero-length declared name", { nameLength: 0 }], + ])("rejects %s as a typed failure", async (_name, options) => { + await withReader(buildWxapkg(members, options), async (reader) => { + await expect(list(reader)).rejects.toMatchObject({ + reason: expect.stringMatching(/^(format|limit)$/), + }); + }); + }); + + it("rejects a member offset into the header region", async () => { + await withReader( + buildWxapkg([{ path: "evil.bin", data: "x", offset: 4 }]), + async (reader) => { + await expect(list(reader)).rejects.toMatchObject({ + reason: "format", + }); + }, + ); + }); + + it("rejects a member extent beyond the file", async () => { + await withReader( + buildWxapkg([{ path: "evil.bin", data: "x", size: 1 << 20 }]), + async (reader) => { + await expect(list(reader)).rejects.toMatchObject({ + reason: "format", + }); + }, + ); + }); + + it("rejects a member offset into the index region", async () => { + // Offset 18 is inside the declared table, not the data region. + await withReader( + buildWxapkg([{ path: "evil.bin", data: "x", offset: 18, size: 1 }]), + async (reader) => { + await expect(list(reader)).rejects.toMatchObject({ reason: "format" }); + }, + ); + }); +}); + +describe("WxapkgArtifactReader data region", () => { + it("rejects a member range that covers unused index bytes", async () => { + const bytes = buildWxapkg([{ path: "a", data: "Z" }], { + tablePadding: 8, + }); + // The row ends at byte 31; the next eight bytes are still the index. + bytes.writeUInt32BE(31, 23); + bytes.writeUInt32BE(1, 27); + await withReader(bytes, async (reader) => { + await expect(list(reader)).rejects.toMatchObject({ reason: "format" }); + }); + }); + + it("reads member bytes that follow unused index bytes", async () => { + await withReader( + buildWxapkg([{ path: "a", data: "Z" }], { tablePadding: 8 }), + async (reader) => { + const [entry] = await list(reader); + expect(entry).toMatchObject({ byteOffset: 39, declaredSize: 1 }); + expect(await text(await reader.open(entry!))).toBe("Z"); + }, + ); + }); + + it("rejects a member extent beyond the declared data region", async () => { + await withReader( + buildWxapkg([{ path: "x", data: "0123456789" }], { + declaredDataLength: 5, + }), + async (reader) => { + await expect(list(reader)).rejects.toMatchObject({ reason: "format" }); + }, + ); + }); + + it("rejects a data region declared past the file", async () => { + await withReader( + buildWxapkg([{ path: "x", data: "ok" }], { + declaredDataLength: 1 << 24, + }), + async (reader) => { + await expect(list(reader)).rejects.toMatchObject({ reason: "format" }); + }, + ); + }); +}); + +describe("WxapkgArtifactReader index length", () => { + it("reads a one-byte member when indexLength counts only the rows", async () => { + const bytes = buildWxapkg([{ path: "a", data: "Z" }]); + expect(bytes).toHaveLength(32); + expect(bytes.readUInt32BE(5)).toBe(13); + await withReader(bytes, async (reader) => { + const [entry] = await list(reader); + expect(entry).toMatchObject({ byteOffset: 31, declaredSize: 1 }); + expect(reader.formatMetadata()).toMatchObject({ + index_length: 13, + data_length: 1, + file_count: 1, + }); + expect(await text(await reader.open(entry!))).toBe("Z"); + }); + }); + + it("reads a one-byte member when indexLength includes the member count", async () => { + const bytes = buildWxapkg([{ path: "a", data: "Z" }], { + indexLengthIncludesFileCount: true, + }); + expect(bytes).toHaveLength(32); + expect(bytes.readUInt32BE(5)).toBe(17); + await withReader(bytes, async (reader) => { + const [entry] = await list(reader); + expect(entry).toMatchObject({ + path: "a", + byteOffset: 31, + declaredSize: 1, + }); + expect(reader.formatMetadata()).toMatchObject({ + index_length: 17, + data_length: 1, + file_count: 1, + }); + expect(await text(await reader.open(entry!))).toBe("Z"); + }); + }); + + it("reads a count-inclusive member when exactly four bytes trail the package", async () => { + const bytes = Buffer.concat([ + buildWxapkg([{ path: "a", data: "Z" }], { + indexLengthIncludesFileCount: true, + }), + Buffer.alloc(4, 0xee), + ]); + expect(bytes).toHaveLength(36); + await withReader(bytes, async (reader) => { + const [entry] = await list(reader); + expect(entry).toMatchObject({ byteOffset: 31, declaredSize: 1 }); + expect(await text(await reader.open(entry!))).toBe("Z"); + }); + }); + + it("rejects a count-inclusive member that covers trailing bytes", async () => { + const bytes = Buffer.concat([ + buildWxapkg([{ path: "a", data: "Z" }], { + indexLengthIncludesFileCount: true, + }), + Buffer.alloc(4, 0xee), + ]); + bytes.writeUInt32BE(5, 27); + await withReader(bytes, async (reader) => { + await expect(list(reader)).rejects.toMatchObject({ reason: "format" }); + }); + }); + + it("keeps a count-inclusive member inside the data region when bytes trail the package", async () => { + const bytes = Buffer.concat([ + buildWxapkg([{ path: "a", data: "Z" }], { + indexLengthIncludesFileCount: true, + }), + Buffer.alloc(8, 0xee), + ]); + await withReader(bytes, async (reader) => { + const [entry] = await list(reader); + expect(entry).toMatchObject({ byteOffset: 31, declaredSize: 1 }); + expect(await text(await reader.open(entry!))).toBe("Z"); + }); + }); + + it("rejects an index byte of a count-inclusive package", async () => { + await withReader( + buildWxapkg([{ path: "a", data: "Z", offset: 18, size: 1 }], { + indexLengthIncludesFileCount: true, + }), + async (reader) => { + await expect(list(reader)).rejects.toMatchObject({ reason: "format" }); + }, + ); + }); +}); + +describe("WxapkgArtifactReader damaged packages", () => { + it("keeps a damaged package's format identity as a typed failure", async () => { + // Marks hold but the declared count cannot fit the declared table. + const bytes = buildWxapkg(members, { + declaredFileCount: 2, + declaredIndexLength: 12, + }); + expect(hasWxapkgHeader(bytes)).toBe(true); + await withReader(bytes, async (reader) => { + await expect(list(reader)).rejects.toMatchObject({ reason: "format" }); + }); + }); + + it("rejects a truncated package before the declared table ends", async () => { + const bytes = buildWxapkg(members); + await withReader(bytes.subarray(0, 30), async (reader) => { + await expect(list(reader)).rejects.toMatchObject({ reason: "format" }); + }); + }); + + it("rejects a non-UTF-8 member name as a typed format failure", async () => { + const bytes = buildWxapkg(members); + bytes[26] = 0xff; // first byte of the first member name + await withReader(bytes, async (reader) => { + await expect(list(reader)).rejects.toMatchObject({ reason: "format" }); + }); + }); + + it("rejects a forged foreign entry with an integrity failure", async () => { + await withReader(buildWxapkg(members), async (reader) => { + const [entry] = await list(reader); + await expect( + reader.open({ ...entry!, adapterKey: "wxapkg-entry:9" }), + ).rejects.toMatchObject({ reason: "integrity" }); + }); + }); +}); + +describe("WxapkgArtifactReader lifecycle", () => { + it("aborts member reads when the caller cancels", async () => { + const data = Buffer.alloc(300 * 1024, 0x62); + await withReader( + buildWxapkg([{ path: "big.bin", data }]), + async (reader) => { + const [entry] = await list(reader); + const controller = new AbortController(); + const stream = await reader.open(entry!, controller.signal); + let delivered = 0; + await expect( + (async () => { + for await (const chunk of stream) { + delivered += (chunk as Buffer).length; + controller.abort(); + } + })(), + ).rejects.toThrow(); + expect(delivered).toBeLessThan(data.length); + }, + ); + }); + + it("survives a destroyed member stream and serves the next entry", async () => { + await withReader(buildWxapkg(members), async (reader) => { + const entries = await list(reader); + const abandoned = await reader.open(entries[0]!); + abandoned.destroy(); + await expect(text(await reader.open(entries[1]!))).resolves.toBe( + "module.exports={};", + ); + }); + }); + + it("fails streams and later reads after close", async () => { + const reader = new WxapkgArtifactReader( + await writePackage(buildWxapkg(members)), + ); + const entries: ArtifactEntry[] = []; + for await (const entry of reader.entries()) entries.push(entry); + const stream = await reader.open(entries[0]!); + const consumed = text(stream).catch((error: unknown) => error); + await reader.close(); + expect(await consumed).toBeInstanceOf(Error); + await expect(reader.open(entries[1]!)).rejects.toMatchObject({ + reason: "unavailable", + }); + }); + + it("bounds concurrent member streams", async () => { + await withReader(buildWxapkg(members), async (reader) => { + const [entry] = await list(reader); + const streams = await Promise.all( + [0, 1, 2, 3].map(() => reader.open(entry!)), + ); + await expect(reader.open(entry!)).rejects.toMatchObject({ + reason: "limit", + }); + for (const stream of streams) stream.destroy(); + }); + }); +}); diff --git a/src/artifacts/WxapkgArtifactReader.ts b/src/artifacts/WxapkgArtifactReader.ts new file mode 100644 index 0000000000..fc7c57a3b1 --- /dev/null +++ b/src/artifacts/WxapkgArtifactReader.ts @@ -0,0 +1,311 @@ +import { artifactExpansionBudgetBytes } from "../domain/artifactExpansionBudget.js"; +import type { FileHandle } from "node:fs/promises"; +import { setImmediate } from "node:timers/promises"; +import { Readable } from "node:stream"; +import { + ArtifactReaderFailure, + artifactFixedLimitFailure, + copyArtifactEntry, + sameArtifactEntry, + type ArtifactEntry, + type ArtifactReader, + type ArtifactTraversalOptions, +} from "./ArtifactReader.js"; +import { ArtifactResourceScope } from "./ArtifactResourceScope.js"; +import { assertArtifactReadActive } from "./ArtifactFileOperation.js"; +import { readArtifactFileBytes } from "./ArtifactFileRange.js"; +import type { + ArtifactFormatMetadata, + WxapkgMetadata, +} from "../domain/artifactFormatMetadata.js"; +import type { StableRegularFileDescriptor } from "../filesystem/RegularFile.js"; +import { ArtifactFileSource } from "./ArtifactFileSource.js"; +import { + readWxapkgFileTable, + readWxapkgHeader, + wxapkgFormatFailure, + type WxapkgMember, +} from "./WxapkgStructures.js"; + +const CHUNK_BYTES = 64 * 1024; +/** Bound active member streams; each owns a file-range generator. */ +const MAX_MEMBER_STREAMS = 4; + +type WxapkgEntry = ArtifactEntry & { + readonly declaredSize: number; + readonly formatMetadata: Extract< + ArtifactFormatMetadata, + { kind: "wxapkg-entry" } + >; +}; + +interface WxapkgEntryState { + readonly entry: WxapkgEntry; + readonly member: WxapkgMember; +} + +/** One descriptor, one admitted member table and bounded raw-range member reads. */ +export class WxapkgArtifactReader implements ArtifactReader { + readonly format = "wxapkg" as const; + readonly #scope = new ArtifactResourceScope(); + readonly #entries = new Map(); + readonly #streams = new Map(); + readonly #source: ArtifactFileSource; + #prepared: Promise | undefined; + #header: WxapkgMetadata | undefined; + #remainingReadOperations = 0; + #admittedReadOperations = 0; + #closing = false; + + constructor(path: string, admitted?: StableRegularFileDescriptor) { + this.#source = new ArtifactFileSource(path, "wxapkg package", admitted); + } + + formatMetadata(): WxapkgMetadata | undefined { + return this.#header === undefined ? undefined : { ...this.#header }; + } + + async prepareContainer( + expectedSha256: string, + signal?: AbortSignal, + ): Promise { + await this.#scope.run(async () => { + this.#assertOpen(signal); + await this.#source.bindSha256(expectedSha256, signal); + }); + } + + async *entries( + signal?: AbortSignal, + options?: ArtifactTraversalOptions, + ): AsyncIterable { + await this.#scope.run(async () => { + this.#assertOpen(signal); + this.#prepared ??= this.#prepare(signal); + await this.#prepared; + await this.#source.read(signal); + }); + this.#assertOpen(signal); + if (this.#header?.directory_status === "encrypted") { + options?.onCoverage?.({ + status: "unknown", + limitations: [ + "wxapkg package is V1MMWX-encrypted; member identities and contents remain unknown.", + ], + }); + return; + } + let index = 0; + for (const { entry } of this.#entries.values()) { + if (index++ % 256 === 0) await setImmediate(); + this.#assertOpen(signal); + yield copyArtifactEntry(entry); + } + // A close while paused at yield must not turn a cleared map into completion. + this.#assertOpen(signal); + } + + async open(entry: ArtifactEntry, signal?: AbortSignal): Promise { + return this.#scope.run(async () => { + this.#assertOpen(signal); + const state = this.#entries.get(entry.adapterKey); + if (state === undefined || !sameArtifactEntry(entry, state.entry)) + throw new ArtifactReaderFailure( + "integrity", + "wxapkg entry does not match its owning inventory", + ); + // Every logical open spends the work budget, including empty members + // whose streams perform only the carrier checks below. + this.#reserveMemberRead(); + // Validate carrier freshness before handing out member bytes. + const { handle } = await this.#source.read(signal); + this.#assertOpen(signal); + if (this.#streams.size >= MAX_MEMBER_STREAMS) + throw new ArtifactReaderFailure( + "limit", + "wxapkg reader already owns four active member streams", + { boundary: "capacity" }, + ); + const controller = new AbortController(); + const output = Readable.from( + this.#memberChunks(handle, state, controller.signal), + { + objectMode: false, + highWaterMark: CHUNK_BYTES, + }, + ); + const onAbort = (): void => { + controller.abort(signal?.reason); + output.destroy( + new ArtifactReaderFailure( + "cancelled", + "wxapkg member read cancelled", + { cause: signal?.reason }, + ), + ); + }; + output.on("error", () => undefined); + this.#streams.set(output, controller); + const completion = new Promise((resolve) => { + output.once("close", () => { + signal?.removeEventListener("abort", onAbort); + this.#streams.delete(output); + resolve(); + }); + }); + // Readable.from waits for its async iterator to retire before close. Keep + // that whole lifetime owned, including a suspended positional read. + void this.#scope + .run(() => completion) + .catch((cause: unknown) => { + controller.abort(cause); + output.destroy( + cause instanceof Error + ? cause + : new Error("wxapkg stream ownership failed", { cause }), + ); + }); + signal?.addEventListener("abort", onAbort, { once: true }); + if (signal?.aborted === true) onAbort(); + return output; + }); + } + + async close(): Promise { + this.#closing = true; + for (const [stream, controller] of this.#streams) { + const cause = new ArtifactReaderFailure( + "unavailable", + "wxapkg reader closed", + ); + controller.abort(cause); + stream.destroy(cause); + } + await this.#scope.close(); + await this.#source.close(); + this.#entries.clear(); + } + + provenance(): readonly [] { + return []; + } + + #assertOpen(signal?: AbortSignal): void { + assertArtifactReadActive(signal); + if (this.#closing) + throw new ArtifactReaderFailure("unavailable", "wxapkg reader is closed"); + } + + async #prepare(signal?: AbortSignal): Promise { + const { handle, initial } = await this.#source.read(signal); + const header = await readWxapkgHeader(handle, initial.size, signal); + if (header === undefined) + throw wxapkgFormatFailure("no complete wxapkg header"); + this.#header = { + kind: "wxapkg", + info: header.layout.status === "available" ? header.info : null, + index_length: + header.layout.status === "available" ? header.indexLength : null, + data_length: + header.layout.status === "available" ? header.dataLength : null, + file_count: + header.layout.status === "available" ? header.fileCount : null, + directory_status: header.layout.status, + }; + if (header.layout.status !== "available") return; + const { members } = await readWxapkgFileTable( + handle, + header, + initial.size, + signal, + ); + const budget = artifactExpansionBudgetBytes(initial.size); + let declaredBytes = 0n; + for (let index = 0; index < members.length; index++) { + if (index % 256 === 0) await setImmediate(); + this.#assertOpen(signal); + const member = members[index]!; + // Overlapping member ranges are valid; each declared path causes a read. + declaredBytes += BigInt(member.size); + if (declaredBytes > BigInt(budget)) + throw new ArtifactReaderFailure( + "limit", + `wxapkg members declare ${String(declaredBytes)} bytes, exceeding the ${String(budget)}-byte plaintext-work budget`, + { boundary: "memory" }, + ); + const adapterKey = `wxapkg-entry:${String(index)}`; + this.#entries.set(adapterKey, { + entry: { + path: member.path, + sourcePath: member.declaredPath, + kind: "file", + declaredSize: member.size, + compressedSize: null, + executable: false, + encrypted: false, + // Member offsets are absolute carrier positions. + byteOffset: member.offset, + declaredIntegrity: null, + unpacked: false, + limitations: [], + adapterKey, + formatMetadata: { + kind: "wxapkg-entry", + index, + source_path: member.declaredPath, + declared_offset: member.offset, + }, + }, + member, + }); + } + // Header, table and per-chunk member reads; a small-reader floor applies. + this.#admittedReadOperations = + 4 * + Math.max( + 1024, + members.length + Math.ceil(Number(declaredBytes) / CHUNK_BYTES), + ); + this.#remainingReadOperations = this.#admittedReadOperations; + await this.#source.read(signal); + } + + async *#memberChunks( + handle: FileHandle, + state: WxapkgEntryState, + signal: AbortSignal, + ): AsyncGenerator { + const { offset, size } = state.member; + let delivered = 0, + chunks = 0; + for (let position = offset; position < offset + size;) { + if (chunks++ % 64 === 0) await setImmediate(); + this.#assertOpen(signal); + this.#reserveMemberRead(); + const length = Math.min(CHUNK_BYTES, offset + size - position); + const chunk = await readArtifactFileBytes( + handle, + position, + length, + offset + size, + signal, + ); + position += chunk.length; + delivered += chunk.length; + yield chunk; + } + if (delivered !== size) + throw wxapkgFormatFailure("member ended before its declared size"); + await this.#source.read(signal); + } + + #reserveMemberRead(): void { + if (this.#remainingReadOperations <= 0) + throw artifactFixedLimitFailure( + "wxapkg directory-and-chunk I/O work", + this.#admittedReadOperations, + "operations", + ); + this.#remainingReadOperations--; + } +} diff --git a/src/artifacts/WxapkgStructures.ts b/src/artifacts/WxapkgStructures.ts new file mode 100644 index 0000000000..3437ead7a7 --- /dev/null +++ b/src/artifacts/WxapkgStructures.ts @@ -0,0 +1,214 @@ +import type { FileHandle } from "node:fs/promises"; +import { + ArtifactReaderFailure, + artifactFixedLimitFailure, +} from "./ArtifactReader.js"; +import { readArtifactFileBytes } from "./ArtifactFileRange.js"; +import { assertArtifactReadActive } from "./ArtifactFileOperation.js"; +import { setImmediate } from "node:timers/promises"; +import { + hasWxapkgEncryptedMagic, + readWxapkgHeaderFacts, + resolveWxapkgIndexLayouts, + selectWxapkgIndexLayout, + WXAPKG_MIN_ROW_BYTES, + type WxapkgIndexLayout, + type WxapkgIndexLayoutFailure, +} from "../domain/wxapkgFormat.js"; + +/** The member table is a flat name/offset/size row list. */ +const MAX_TABLE_BYTES = 16 * 1024 * 1024; +/** Encoded retained directory representation; paths are charged separately. */ +const DIRECTORY_BUDGET_BYTES = 64 * 1024 * 1024; +/** Member names are relative package paths; larger claims are malformed. */ +const MAX_MEMBER_NAME_BYTES = 64 * 1024; +/** Detecting prefix covers the fixed header, table head and encrypted magic. */ +const HEADER_PREFIX_BYTES = 64; + +const UTF8 = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }); + +export const wxapkgFormatFailure = (detail: string): ArtifactReaderFailure => + new ArtifactReaderFailure("format", `wxapkg is malformed: ${detail}`); + +/** Reserve buffers, JS rows and escaped name projections before allocating. */ +class WxapkgDirectoryBudget { + #remaining = DIRECTORY_BUDGET_BYTES; + reserve(bytes: number): void { + if (!Number.isSafeInteger(bytes) || bytes < 0 || bytes > this.#remaining) + throw artifactFixedLimitFailure( + "wxapkg directory representation", + DIRECTORY_BUDGET_BYTES, + "bytes", + ); + this.#remaining -= bytes; + } +} + +export interface WxapkgHeader { + readonly info: number; + readonly indexLength: number; + readonly dataLength: number; + readonly fileCount: number; + readonly layout: + | { readonly status: "encrypted" } + | { + readonly status: "available"; + readonly indexLayouts: readonly WxapkgIndexLayout[]; + }; +} + +export interface WxapkgMember { + readonly offset: number; + readonly size: number; + /** Package-relative path; declared root prefixes are translated here. */ + readonly path: string; + /** Declared member name as stored in the table. */ + readonly declaredPath: string; +} + +export interface WxapkgFileTable { + readonly members: readonly WxapkgMember[]; +} + +/** Decode header fields once; encrypted variants and absent signatures are distinct. */ +export const readWxapkgHeader = async ( + handle: FileHandle, + size: number, + signal?: AbortSignal, +): Promise => { + const prefix = await readArtifactFileBytes( + handle, + 0, + Math.min(size, HEADER_PREFIX_BYTES), + size, + signal, + ); + if (hasWxapkgEncryptedMagic(prefix)) + return { + info: 0, + indexLength: 0, + dataLength: 0, + fileCount: 0, + layout: { status: "encrypted" }, + }; + const facts = readWxapkgHeaderFacts(prefix); + if (facts === undefined) return undefined; + const indexes = resolveWxapkgIndexLayouts(facts, size); + if (!indexes.ok) throw wxapkgIndexLayoutFailure(indexes.error); + return { + ...facts, + layout: { status: "available", indexLayouts: indexes.value }, + }; +}; + +const wxapkgIndexLayoutFailure = ( + failure: WxapkgIndexLayoutFailure, +): ArtifactReaderFailure => + wxapkgFormatFailure( + failure._tag === "DataRegionExceedsFile" + ? "declared data region exceeds the file" + : "declared member table exceeds the file", + ); + +export const readWxapkgFileTable = async ( + handle: FileHandle, + header: WxapkgHeader, + size: number, + signal?: AbortSignal, +): Promise => { + const layout = header.layout; + if (layout.status !== "available") + throw new ArtifactReaderFailure( + "unavailable", + "wxapkg directory is encrypted", + ); + const indexLayouts = layout.indexLayouts; + let readBytes = 0; + let tableStart: number | undefined; + for (const indexLayout of indexLayouts) { + if (tableStart === undefined) tableStart = indexLayout.tableStart; + if (indexLayout.tableBytes > readBytes) readBytes = indexLayout.tableBytes; + } + if (tableStart === undefined) + throw wxapkgFormatFailure("declared member table exceeds the file"); + if (readBytes > MAX_TABLE_BYTES) + throw artifactFixedLimitFailure( + "wxapkg member table", + MAX_TABLE_BYTES, + "bytes", + ); + const budget = new WxapkgDirectoryBudget(); + budget.reserve(readBytes); + const table = await readArtifactFileBytes( + handle, + tableStart, + readBytes, + size, + signal, + ); + // Loose lower bound already checked in facts; precise rows are checked here. + if (table.length < header.fileCount * WXAPKG_MIN_ROW_BYTES) + throw wxapkgFormatFailure("member table is truncated"); + const members: WxapkgMember[] = []; + let cursor = 0; + for (let index = 0; index < header.fileCount; index++) { + if (index % 256 === 0) await setImmediate(); + assertArtifactReadActive(signal); + if (cursor + 4 > table.length) + throw wxapkgFormatFailure("member row is truncated"); + const nameLength = table.readUInt32BE(cursor); + cursor += 4; + if ( + nameLength === 0 || + nameLength > MAX_MEMBER_NAME_BYTES || + cursor + nameLength + 8 > table.length + ) + throw wxapkgFormatFailure("member row exceeds the table"); + // Row storage plus the reader's owned entries/map; charge before decode. + budget.reserve(512 + nameLength * 24); + let declaredPath: string; + try { + declaredPath = UTF8.decode(table.subarray(cursor, cursor + nameLength)); + } catch (cause: unknown) { + throw new ArtifactReaderFailure( + "format", + "wxapkg member path is not valid UTF-8", + { + cause, + }, + ); + } + cursor += nameLength; + const offset = table.readUInt32BE(cursor); + const sizeMember = table.readUInt32BE(cursor + 4); + cursor += 8; + // Producer names are root-prefixed; translate to package-relative once. + const path = declaredPath.startsWith("/") + ? declaredPath.slice(1) + : declaredPath; + if (path.length === 0) + throw wxapkgFormatFailure("member name has no package-relative path"); + members.push({ offset, size: sizeMember, path, declaredPath }); + } + const candidates = indexLayouts.filter((index) => index.tableBytes >= cursor); + if (candidates.length === 0) + throw wxapkgFormatFailure("member table is truncated"); + // Rows and member ranges choose the convention. EOF is not a signal: four + // trailing bytes can make the other convention end on the carrier. + const index = selectWxapkgIndexLayout(candidates, cursor, members); + if (index === undefined) { + const outside = members.find((member) => { + const extentEnd = member.offset + member.size; + return candidates.every( + (candidate) => + member.offset < candidate.dataStart || + extentEnd > candidate.dataEnd || + extentEnd < member.offset, + ); + }); + throw wxapkgFormatFailure( + `member exceeds the data region: ${outside?.path ?? ""}`, + ); + } + return { members }; +}; diff --git a/src/artifacts/artifactFormats.test.ts b/src/artifacts/artifactFormats.test.ts index da351dc0c8..acbd1fe124 100644 --- a/src/artifacts/artifactFormats.test.ts +++ b/src/artifacts/artifactFormats.test.ts @@ -30,6 +30,7 @@ type FormatsWithReaders = | "ines" | "godot-pck" | "unityfs" + | "wxapkg" | "uimage" | "mach-o-universal" | "directory" diff --git a/src/artifacts/artifactFormats.ts b/src/artifacts/artifactFormats.ts index aeb7dc8bff..92c343e202 100644 --- a/src/artifacts/artifactFormats.ts +++ b/src/artifacts/artifactFormats.ts @@ -40,6 +40,7 @@ import { PyInstallerArtifactReader } from "./PyInstallerArtifactReader.js"; import { Uf2ArtifactReader } from "./Uf2ArtifactReader.js"; import { UImageArtifactReader } from "./UImageArtifactReader.js"; import { UnityFsArtifactReader } from "./UnityFsArtifactReader.js"; +import { WxapkgArtifactReader } from "./WxapkgArtifactReader.js"; import { ZipArtifactReader } from "./ZipArtifactReader.js"; /** How inventory describes a format when no reader was constructed. */ @@ -286,6 +287,16 @@ const NON_ZIP_FORMATS = { createReader: (path, options) => new UnityFsArtifactReader(path, sourceOf(options)), }), + wxapkg: container("wxapkg", { + roleContainer: true, + retainClassifiedSource: true, + extractable: true, + absentInventory: "unavailable", + readerLimitation: + "wxapkg inventory establishes packaged member names and stored bytes; wxml/wxss sources, compiled bundles and runtime behavior are not reconstructed. V1MMWX-encrypted packages are unsupported.", + createReader: (path, options) => + new WxapkgArtifactReader(path, sourceOf(options)), + }), uimage: container("uimage", { roleContainer: true, retainClassifiedSource: true, diff --git a/src/artifacts/extraction/ArtifactExtraction.ts b/src/artifacts/extraction/ArtifactExtraction.ts index d51a52c083..16e9745b75 100644 --- a/src/artifacts/extraction/ArtifactExtraction.ts +++ b/src/artifacts/extraction/ArtifactExtraction.ts @@ -215,6 +215,15 @@ export const materializeArtifactInventoryInScope = async ( input.selectedPath ?? input.inputPath, `UnityFS directory is ${containerMetadata.directory_status}; its complete contents cannot be extracted`, ); + if ( + containerMetadata?.kind === "wxapkg" && + containerMetadata.directory_status !== "available" + ) + throw new AnalysisUnsupportedTargetError( + "extract_artifact", + input.selectedPath ?? input.inputPath, + `wxapkg directory is ${containerMetadata.directory_status}; its complete contents cannot be extracted`, + ); const selectedOccurrences = snapshot.occurrences.filter( (occurrence) => (occurrence.entry_kind === "file" || occurrence.entry_kind === "slice") && diff --git a/src/artifacts/inventory/ArtifactGraphConstruction.ts b/src/artifacts/inventory/ArtifactGraphConstruction.ts index b0c77906f8..27561c6772 100644 --- a/src/artifacts/inventory/ArtifactGraphConstruction.ts +++ b/src/artifacts/inventory/ArtifactGraphConstruction.ts @@ -2,6 +2,10 @@ import { artifactFormatDescriptor } from "../artifactFormats.js"; import { hasGodotPckHeader } from "../../domain/godotPckFormat.js"; import { hasUImageMagic } from "../../domain/uimageFormat.js"; import { hasUnityFsHeader } from "../../domain/unityfsFormat.js"; +import { + hasWxapkgEncryptedMagic, + hasWxapkgHeader, +} from "../../domain/wxapkgFormat.js"; import { detectHexRecordEncoding } from "../../domain/firmware/hexRecords.js"; import { recognizeEspImage } from "../../domain/firmware/espImage.js"; import { hasUf2Block } from "../../domain/firmware/uf2.js"; @@ -296,7 +300,8 @@ export const rootOccurrenceFor = ( executable: metadata.executable, encrypted: (metadata.formatMetadata?.kind === "godot-pck" || - metadata.formatMetadata?.kind === "unityfs") && + metadata.formatMetadata?.kind === "unityfs" || + metadata.formatMetadata?.kind === "wxapkg") && metadata.formatMetadata.directory_status === "encrypted", hash_status: "verified", source_location: null, @@ -607,6 +612,8 @@ export const classifyArtifactBytes = ( if (hasZipSignature(prefix)) return "zip"; if (hasGodotPckHeader(prefix)) return "godot-pck"; if (hasUnityFsHeader(prefix)) return "unityfs"; + if (hasWxapkgEncryptedMagic(prefix) || hasWxapkgHeader(prefix)) + return "wxapkg"; if (hasUImageMagic(prefix)) return "uimage"; const hexRecords = detectHexRecordEncoding(prefix); if (hexRecords !== undefined) return hexRecords; diff --git a/src/domain/analysisSnapshot.ts b/src/domain/analysisSnapshot.ts index 7aba9dd72f..913c998ff6 100644 --- a/src/domain/analysisSnapshot.ts +++ b/src/domain/analysisSnapshot.ts @@ -76,6 +76,7 @@ const formatSchema = z.enum([ "ines", "godot-pck", "unityfs", + "wxapkg", "dmg", "pkg", "intel-hex", diff --git a/src/domain/analysisView/artifactInventoryView.ts b/src/domain/analysisView/artifactInventoryView.ts index 50fa587ce1..cd368e68a3 100644 --- a/src/domain/analysisView/artifactInventoryView.ts +++ b/src/domain/analysisView/artifactInventoryView.ts @@ -277,6 +277,7 @@ export const projectArtifactInventoryView = ( view.kind === "inventory" || inventory.manifest.root_format === "godot-pck" || inventory.manifest.root_format === "unityfs" || + inventory.manifest.root_format === "wxapkg" || inventory.manifest.root_format === "pyinstaller" || inventory.manifest.root_format === "node-sea" || inventory.manifest.root_format === "intel-hex" || diff --git a/src/domain/artifactFormatMetadata.ts b/src/domain/artifactFormatMetadata.ts index a53a6c0524..ab07a138ab 100644 --- a/src/domain/artifactFormatMetadata.ts +++ b/src/domain/artifactFormatMetadata.ts @@ -63,6 +63,24 @@ export const unityFsEntryMetadataSchema = z.strictObject({ flags: uint32, }); +/** Parsed source fields owned by the wxapkg adapter, independent of member bytes. */ +export const wxapkgMetadataSchema = z.strictObject({ + kind: z.literal("wxapkg"), + info: uint32.nullable(), + index_length: uint32.nullable(), + data_length: uint32.nullable(), + file_count: uint32.nullable(), + directory_status: z.enum(["available", "encrypted"]), +}); +export type WxapkgMetadata = z.infer; + +export const wxapkgEntryMetadataSchema = z.strictObject({ + kind: z.literal("wxapkg-entry"), + index: uint32, + source_path: z.string(), + declared_offset: offset, +}); + /** Source cookie fields and derived directory counts without Python execution. */ export const pyInstallerMetadataSchema = z.strictObject({ kind: z.literal("pyinstaller"), @@ -765,6 +783,8 @@ export const artifactFormatMetadataSchema = z.discriminatedUnion("kind", [ godotPckEntryMetadataSchema, unityFsMetadataSchema, unityFsEntryMetadataSchema, + wxapkgMetadataSchema, + wxapkgEntryMetadataSchema, pyInstallerMetadataSchema, pyInstallerEntryMetadataSchema, dotnetBundleEntryMetadataSchema, @@ -837,6 +857,14 @@ const declarationsByKind: { flags: metadata.flags, }), }, + wxapkg: { + notCompared: + "Its fields come from the package's own bytes, which content comparison covers.", + }, + "wxapkg-entry": { + notCompared: + "index is layout. source_path keeps the declared name, including a producer root prefix the occurrence path strips.", + }, pyinstaller: { notCompared: "Its fields come from the archive's own bytes, which content comparison covers.", diff --git a/src/domain/artifactGraph.ts b/src/domain/artifactGraph.ts index 9528ba6885..675e7c828e 100644 --- a/src/domain/artifactGraph.ts +++ b/src/domain/artifactGraph.ts @@ -67,6 +67,7 @@ export const ARTIFACT_FORMAT_IDS = [ "ines", "godot-pck", "unityfs", + "wxapkg", "uimage", "javascript-bundle", "source-map", @@ -113,6 +114,7 @@ const artifactContentFormatSchema = artifactFormatSchema.extract([ "zip", "godot-pck", "unityfs", + "wxapkg", "uimage", "intel-hex", "srec", diff --git a/src/domain/binaryTargetTypes.ts b/src/domain/binaryTargetTypes.ts index 743a92c9a2..22e6f3e848 100644 --- a/src/domain/binaryTargetTypes.ts +++ b/src/domain/binaryTargetTypes.ts @@ -293,6 +293,7 @@ export type BinaryTarget = | "ines" | "godot-pck" | "unityfs" + | "wxapkg" | "uimage" | "dmg" | "pkg" diff --git a/src/domain/evidenceSubject.ts b/src/domain/evidenceSubject.ts index 5c0994b32c..d9876742e3 100644 --- a/src/domain/evidenceSubject.ts +++ b/src/domain/evidenceSubject.ts @@ -27,6 +27,7 @@ export const evidenceSubjectFormatSchema = z.enum([ "ines", "godot-pck", "unityfs", + "wxapkg", "uimage", "dmg", "pkg", diff --git a/src/domain/liveBinaryTargetSummary.ts b/src/domain/liveBinaryTargetSummary.ts index ab1efe39f5..45b6d9c0da 100644 --- a/src/domain/liveBinaryTargetSummary.ts +++ b/src/domain/liveBinaryTargetSummary.ts @@ -73,6 +73,7 @@ export const liveBinaryTargetSummaryVariants = [ "ines", "godot-pck", "unityfs", + "wxapkg", "uimage", "dmg", "pkg", diff --git a/src/domain/wxapkgFormat.ts b/src/domain/wxapkgFormat.ts new file mode 100644 index 0000000000..d1cecb0e39 --- /dev/null +++ b/src/domain/wxapkgFormat.ts @@ -0,0 +1,198 @@ +import { err, ok, type Result } from "./result.js"; + +/** + * WeChat mini-program package (wxapkg): an 18-byte big-endian header, a + * name/offset/size member table, then verbatim member data. Members are stored + * uncompressed at absolute file offsets. All multi-byte fields are big-endian. + */ +export const WXAPKG_FIRST_MARK = 0xbe; +export const WXAPKG_LAST_MARK = 0xed; +export const WXAPKG_HEADER_BYTES = 18; + +/** Minimum fixed row fields: name length, offset and size without name bytes. */ +export const WXAPKG_MIN_ROW_BYTES = 12; + +/** + * The member count is stored in the last four header bytes. Some producers + * include those bytes in `indexLength`; others count only the rows that follow. + */ +const WXAPKG_MEMBER_COUNT_BYTES = 4; + +/** + * Encrypted PC-client packages start with this ASCII magic instead of the + * plain header. Their member data is ciphertext; no decryption is supported. + */ +export const WXAPKG_ENCRYPTED_MAGIC = Buffer.from("V1MMWX", "latin1"); + +/** Whether bytes start with the encrypted PC-client magic. */ +export const hasWxapkgEncryptedMagic = (bytes: Uint8Array): boolean => + bytes.length >= WXAPKG_ENCRYPTED_MAGIC.length && + Buffer.from( + bytes.buffer, + bytes.byteOffset, + WXAPKG_ENCRYPTED_MAGIC.length, + ).equals(WXAPKG_ENCRYPTED_MAGIC); + +/** Declared header fields; support and extent admission belong to callers. */ +export interface WxapkgHeaderFacts { + /** Unknown header field after the first mark; producers write zero. */ + readonly info: number; + /** + * Declared index length. This is either the row bytes or those rows plus + * the 4-byte member count; {@link resolveWxapkgIndexLayouts} admits both. + */ + readonly indexLength: number; + /** Declared byte length of the member data region. */ + readonly dataLength: number; + readonly fileCount: number; +} + +/** + * Decode declared fields after the caller admits a prefix covering the fixed + * header. Undefined means the admitted bytes cannot carry a plausible header; + * both mark checks and minimal row accounting must hold. + */ +export const readWxapkgHeaderFacts = ( + bytes: Uint8Array, +): WxapkgHeaderFacts | undefined => { + if ( + bytes.length < WXAPKG_HEADER_BYTES || + bytes[0] !== WXAPKG_FIRST_MARK || + bytes[13] !== WXAPKG_LAST_MARK + ) + return undefined; + const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength); + const info = view.getUint32(1); + const indexLength = view.getUint32(5); + const dataLength = view.getUint32(9); + const fileCount = view.getUint32(14); + // Every row carries at least its fixed fields; a table too small for the + // declared count cannot be a wxapkg index. + if (fileCount === 0 || indexLength < fileCount * WXAPKG_MIN_ROW_BYTES) + return undefined; + return { info, indexLength, dataLength, fileCount }; +}; + +/** + * Whether bytes carry the plain wxapkg marks. Recognition is intentionally + * separate from admission: a damaged package keeps its format identity and the + * reader rejects impossible table accounting as a typed format failure. + */ +export const hasWxapkgHeader = (bytes: Uint8Array): boolean => + bytes.length >= WXAPKG_HEADER_BYTES && + bytes[0] === WXAPKG_FIRST_MARK && + bytes[13] === WXAPKG_LAST_MARK; + +/** Row table and declared data region for one index-length convention. */ +export interface WxapkgIndexLayout { + /** Absolute offset of the first name/offset/size row. */ + readonly tableStart: number; + /** Row bytes to read. The member count, when included, stays in the header. */ + readonly tableBytes: number; + /** Inclusive start of the declared member-data region. */ + readonly dataStart: number; + /** Exclusive end of the declared member-data region. */ + readonly dataEnd: number; +} + +export type WxapkgIndexLayoutFailure = + | { readonly _tag: "IndexExceedsFile" } + | { readonly _tag: "DataRegionExceedsFile" }; + +const spanEnd = (start: number, length: number): number | undefined => { + if ( + !Number.isSafeInteger(start) || + !Number.isSafeInteger(length) || + length < 0 + ) + return undefined; + const end = start + length; + return Number.isSafeInteger(end) ? end : undefined; +}; + +const indexLayout = ( + facts: WxapkgHeaderFacts, + fileSize: number, + includedCountBytes: number, +): WxapkgIndexLayout | "index-outside" | "data-outside" | "too-small" => { + if ( + !Number.isSafeInteger(facts.indexLength) || + !Number.isSafeInteger(facts.fileCount) || + facts.fileCount < 0 || + facts.indexLength < includedCountBytes + ) + return "too-small"; + const tableBytes = facts.indexLength - includedCountBytes; + if (tableBytes < facts.fileCount * WXAPKG_MIN_ROW_BYTES) return "too-small"; + const dataStart = spanEnd(WXAPKG_HEADER_BYTES, tableBytes); + if (dataStart === undefined || dataStart > fileSize) return "index-outside"; + const dataEnd = spanEnd(dataStart, facts.dataLength); + if (dataEnd === undefined || dataEnd > fileSize) return "data-outside"; + return { tableStart: WXAPKG_HEADER_BYTES, tableBytes, dataStart, dataEnd }; +}; + +/** + * Admit every index-length convention whose index and declared data region + * both lie inside the carrier. Trailing bytes can make both fit, so callers + * keep every candidate until the parsed rows and member ranges decide. + */ +export const resolveWxapkgIndexLayouts = ( + facts: WxapkgHeaderFacts, + fileSize: number, +): Result => { + if (!Number.isSafeInteger(fileSize) || fileSize < 0) + return err({ _tag: "IndexExceedsFile" }); + const described = [WXAPKG_MEMBER_COUNT_BYTES, 0].map((included) => + indexLayout(facts, fileSize, included), + ); + const admissible = described.filter( + (layout): layout is WxapkgIndexLayout => typeof layout !== "string", + ); + if (admissible.length === 0) + return err({ + _tag: described.some((layout) => layout === "data-outside") + ? "DataRegionExceedsFile" + : "IndexExceedsFile", + }); + return ok(admissible); +}; + +const memberFitsIndex = ( + layout: WxapkgIndexLayout, + member: { readonly offset: number; readonly size: number }, +): boolean => { + const extentEnd = member.offset + member.size; + return ( + member.offset >= layout.dataStart && + extentEnd <= layout.dataEnd && + extentEnd >= member.offset + ); +}; + +/** + * Choose the admitted index that contains the parsed rows and every member + * range. An index that ends on the last parsed row wins a tie. Neither the + * carrier end nor the shorter table is enough on its own: four trailing bytes + * can make the other convention land on EOF. + */ +export const selectWxapkgIndexLayout = ( + layouts: readonly WxapkgIndexLayout[], + consumedRowBytes: number, + members: readonly { readonly offset: number; readonly size: number }[], +): WxapkgIndexLayout | undefined => { + const fitting = layouts.filter( + (layout) => + layout.tableBytes >= consumedRowBytes && + members.every((member) => memberFitsIndex(layout, member)), + ); + const exactRows = fitting.filter( + (layout) => layout.tableBytes === consumedRowBytes, + ); + const pool = exactRows.length > 0 ? exactRows : fitting; + let selected: WxapkgIndexLayout | undefined; + for (const layout of pool) { + if (selected === undefined || layout.tableBytes < selected.tableBytes) + selected = layout; + } + return selected; +}; diff --git a/tests/boundary/mcp/wxapkgMcp.test.ts b/tests/boundary/mcp/wxapkgMcp.test.ts new file mode 100644 index 0000000000..3d1901421a --- /dev/null +++ b/tests/boundary/mcp/wxapkgMcp.test.ts @@ -0,0 +1,87 @@ +import { readFile, rm, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { expect, it } from "vitest"; +import { parseEvidence } from "../../../src/domain/evidence.js"; +import { artifactInspectionResultSchema } from "../../../src/domain/artifactInspection.js"; +import { artifactExtractionResultSchema } from "../../../src/domain/artifactGraph.js"; +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; +import { connectLocalToolsMcp } from "../../fixtures/localToolsMcp.js"; +import { buildWxapkg, wxapkgEncryptedBytes } from "../../fixtures/wxapkg.js"; + +it.each(["full", "compact"] as const)( + "retains wxapkg facts and encrypted coverage through compiled MCP (%s)", + async (profile) => { + const root = await createTestTempDirectory("rea-wxapkg-mcp-"); + const path = join(root, "package.wxapkg"); + const payload = Buffer.from("module.exports={};"); + await writeFile( + path, + buildWxapkg([ + { path: "app-config.json", data: "{}" }, + { path: "pages/index/index.js", data: payload }, + ]), + ); + const { call } = await connectLocalToolsMcp({ + REA_MCP_INPUT_SCHEMA_PROFILE: profile, + }); + let outputRoot: string | undefined; + try { + expect((await call("open_binary", { path })).isError).not.toBe(true); + const reply = await call("inspect_artifact", {}); + expect(reply.isError, JSON.stringify(reply)).not.toBe(true); + const inspection = artifactInspectionResultSchema.parse( + parseEvidence(reply.structuredContent).normalized_result, + ); + const observed = inspection.substeps[0]!.evidence.normalized_result; + expect(observed.occurrences).toContainEqual( + expect.objectContaining({ + logical_path: "pages/index/index.js", + hash_status: "verified", + source_location: expect.objectContaining({ + offset: expect.any(Number), + }), + }), + ); + const extracted = await call("extract_artifact", {}); + expect(extracted.isError, JSON.stringify(extracted)).not.toBe(true); + const result = artifactExtractionResultSchema.parse( + parseEvidence(extracted.structuredContent).normalized_result, + ); + outputRoot = result.output_root; + expect( + (await readFile(join(outputRoot, "pages/index/index.js"))).equals( + payload, + ), + ).toBe(true); + await call("close_binary", {}); + // Encrypted PC-client packages remain represented without member guesses. + await writeFile(path, wxapkgEncryptedBytes()); + await call("open_binary", { path }); + const unknownReply = await call("inspect_artifact", {}); + expect(unknownReply.isError, JSON.stringify(unknownReply)).not.toBe(true); + const unknown = artifactInspectionResultSchema.parse( + parseEvidence(unknownReply.structuredContent).normalized_result, + ).substeps[0]!.evidence.normalized_result; + expect(unknown.coverage).toMatchObject({ status: "unknown" }); + expect(unknown.occurrences).toMatchObject([ + { + artifact_format: "wxapkg", + encrypted: true, + format_metadata: { + kind: "wxapkg", + directory_status: "encrypted", + file_count: null, + }, + }, + ]); + const refused = await call("extract_artifact", {}); + expect(refused.isError).toBe(true); + expect(JSON.stringify(refused)).toContain("cannot be extracted"); + await call("close_binary", {}); + } finally { + if (outputRoot !== undefined) + await rm(outputRoot, { recursive: true, force: true }); + } + }, + 60_000, +); diff --git a/tests/fixtures/wxapkg.ts b/tests/fixtures/wxapkg.ts new file mode 100644 index 0000000000..58fa5211d0 --- /dev/null +++ b/tests/fixtures/wxapkg.ts @@ -0,0 +1,114 @@ +/** Synthetic WeChat mini-program package (wxapkg) builders for reader tests. */ + +export interface WxapkgMember { + readonly path: string; + readonly data: string | Buffer; + /** Override the declared member offset (absolute file position). */ + readonly offset?: number; + /** Override the declared member byte size. */ + readonly size?: number; +} + +export interface WxapkgOptions { + readonly firstMark?: number; + readonly lastMark?: number; + readonly info?: number; + readonly declaredFileCount?: number; + readonly declaredIndexLength?: number; + readonly declaredDataLength?: number; + /** + * Count the 4-byte member count inside `indexLength`. Row bytes and member + * offsets stay where they are; only the declared length grows by four. + */ + readonly indexLengthIncludesFileCount?: boolean; + /** Override the encoded row's name-length field for every member. */ + readonly nameLength?: number; + /** Bytes appended to the table region beyond the declared rows. */ + readonly tablePadding?: number; + /** Declare every member after the first at the first member's offset. */ + readonly aliasOffsets?: boolean; + /** Replace the whole package prefix (encrypted variants). */ + readonly magic?: Buffer; + /** Truncate the final package to this many bytes. */ + readonly truncate?: number; +} + +const utf8 = (value: string): Buffer => Buffer.from(value, "utf8"); +const u32 = (value: number): Buffer => { + const out = Buffer.alloc(4); + out.writeUInt32BE(value); + return out; +}; + +/** Encrypted PC-client package prefix, without plaintext members. */ +export const wxapkgEncryptedBytes = (): Buffer => + Buffer.concat([Buffer.from("V1MMWX", "latin1"), Buffer.alloc(64)]); + +/** Plain wxapkg: 18-byte header, name/offset/size rows, then verbatim data. */ +export const buildWxapkg = ( + members: readonly WxapkgMember[], + options: WxapkgOptions = {}, +): Buffer => { + if (options.magic !== undefined) + return Buffer.concat([ + options.magic, + ...members.map(({ data }) => + typeof data === "string" ? utf8(data) : data, + ), + ]); + const names = members.map(({ path }) => utf8(path)); + // Encoded rows carry the real name bytes regardless of any declared override. + const rowBytes = names.reduce((sum, name) => sum + 12 + name.length, 0); + const rowRegion = rowBytes + (options.tablePadding ?? 0); + const indexLength = + options.declaredIndexLength ?? + rowRegion + (options.indexLengthIncludesFileCount === true ? 4 : 0); + // A declared length override repositions the data. Including the member + // count changes only the length field; the rows still begin at byte 18. + const dataStart = + options.declaredIndexLength !== undefined + ? 18 + indexLength + : 18 + rowRegion; + let cursor = dataStart; + const rows: Buffer[] = []; + const payload: Buffer[] = []; + let firstOffset: number | undefined; + members.forEach((member, index) => { + const name = names[index]!; + const data = + typeof member.data === "string" ? utf8(member.data) : member.data; + const aliased = + options.aliasOffsets === true && index > 0 && firstOffset !== undefined; + const offset = aliased ? firstOffset! : (member.offset ?? cursor); + const size = member.size ?? data.length; + firstOffset ??= offset; + rows.push( + u32(options.nameLength ?? name.length), + name, + u32(offset), + u32(size), + ); + if (!aliased) { + payload.push(data); + if (offset === cursor) cursor = offset + size; + } + }); + const dataLength = + options.declaredDataLength ?? Math.max(0, cursor - dataStart); + const header = Buffer.concat([ + Buffer.from([options.firstMark ?? 0xbe]), + u32(options.info ?? 0), + u32(indexLength), + u32(dataLength), + Buffer.from([options.lastMark ?? 0xed]), + u32(options.declaredFileCount ?? members.length), + ]); + const table = Buffer.concat([ + ...rows, + Buffer.alloc(options.tablePadding ?? 0), + ]); + let bytes = Buffer.concat([header, table, ...payload]); + if (options.truncate !== undefined) + bytes = bytes.subarray(0, options.truncate); + return bytes; +};