Compare commits

..
76 changed files with 447 additions and 3523 deletions

No files matched your search

+3 -14
View File
@@ -25,7 +25,7 @@ jobs:
node-version-file: .nvmrc
- name: Validate the SDK pin against the provider's verified release
run: node scripts/verify-ghidra-bridge-sources.mjs --check-sdk-pin >> "$GITHUB_ENV"
# The bridge probe imports only source-owned ARM, Thumb and MIPS bytes and never
# The bridge probe imports only source-owned ARM/Thumb bytes and never
# executes target code. All SDK/project state stays in runner storage.
- name: Acquire checksum-verified Ghidra SDK in isolated runner storage
shell: bash
@@ -46,7 +46,7 @@ jobs:
java_home_variable="JAVA_HOME_${GHIDRA_JAVA_MAJOR}_X64"
test -n "${!java_home_variable}"
JAVA_HOME="${!java_home_variable}" npm run verify:ghidra:bridge-sources -- --verified-sdk
- name: Inspect real ARM, Thumb and MIPS instruction context through the bridge
- name: Inspect real ARM and Thumb instruction context through the bridge
shell: bash
run: |
java_home_variable="JAVA_HOME_${GHIDRA_JAVA_MAJOR}_X64"
@@ -58,21 +58,10 @@ jobs:
-import "${probe_root}/arm-thumb.bin" -loader BinaryLoader -processor ARM:LE:32:v7 \
-noanalysis -readOnly -deleteProject \
-scriptPath "${GITHUB_WORKSPACE}/tests/conformance/ghidra;${GITHUB_WORKSPACE}/bridge/ghidra" \
-postScript ReaInstructionContextProbe.java "${GITHUB_WORKSPACE}/bridge/ghidra" \
-postScript ReaArmContextProbe.java "${GITHUB_WORKSPACE}/bridge/ghidra" \
> "${probe_root}/output.log" 2>&1 || { cat "${probe_root}/output.log"; exit 1; }
cat "${probe_root}/output.log"
grep -F 'REA_ARM_CONTEXT_OK arm=0 thumb=1 unavailable=null (injected provider value)' "${probe_root}/output.log"
mkdir -p "${probe_root}/mips-project"
# MIPS32 addiu v0,a0,3; MIPS16 li v0,90. These bytes are never executed.
printf '\003\000\202\044\132\152\000\000' > "${probe_root}/mips.bin"
timeout 180s "${GHIDRA_INSTALL_DIR}/support/analyzeHeadless" "${probe_root}/mips-project" MipsContext \
-import "${probe_root}/mips.bin" -loader BinaryLoader -processor MIPS:LE:32:default \
-noanalysis -readOnly -deleteProject \
-scriptPath "${GITHUB_WORKSPACE}/tests/conformance/ghidra;${GITHUB_WORKSPACE}/bridge/ghidra" \
-postScript ReaInstructionContextProbe.java "${GITHUB_WORKSPACE}/bridge/ghidra" mips \
> "${probe_root}/mips-output.log" 2>&1 || { cat "${probe_root}/mips-output.log"; exit 1; }
cat "${probe_root}/mips-output.log"
grep -F 'REA_MIPS_CONTEXT_OK standard=0 mips16=1' "${probe_root}/mips-output.log"
# Each probe imports source-owned bytes at 0x1000, away from the Cortex-M
# vector symbols that ARMCortex.pspec defines at 0, and decodes them from
# the language's default context, so M-profile Thumb state must hold at
-12
View File
@@ -135,25 +135,13 @@ def inspect_elf(path, cache):
from pwnlib.context import context
with context.local(cache_dir=str(cache), log_level="warning"):
from elftools.common.exceptions import ELFError
from elftools.common.utils import struct_parse
from elftools.elf.elffile import ELFFile
from pwnlib.elf import ELF
try:
headers = ELFFile(io.BytesIO(content))
section_count = headers.num_sections()
if section_count:
if headers.header.e_shentsize < headers.structs.Elf_Shdr.sizeof():
raise LayoutFailure("format", "ELF section-header entry is shorter than the required structure.")
location(headers.header.e_shoff, section_count * headers.header.e_shentsize, len(content))
# Upstream string/symbol/relocation access reads stored stream bytes,
# while this profile reports logical names and file coordinates.
# Admit those encodings before LayoutELF populates convenience maps.
for index in range(section_count):
raw_header = struct_parse(headers.structs.Elf_Shdr, headers.stream,
stream_pos=headers.header.e_shoff + index * headers.header.e_shentsize)
if raw_header.sh_flags & 0x800 and raw_header.sh_type in (
"SHT_STRTAB", "SHT_SYMTAB", "SHT_DYNSYM", "SHT_REL", "SHT_RELA", "SHT_RELR", "SHT_DYNAMIC"):
raise LayoutFailure("unsupported-target", f"ELF section {index} ({raw_header.sh_type}) is SHF_COMPRESSED; logical metadata and its source coordinates are not decoded by this profile.")
segment_count = headers.num_segments()
if segment_count:
location(headers.header.e_phoff, segment_count * headers.header.e_phentsize, len(content))
-87
View File
@@ -1,87 +0,0 @@
"""Producer admission regressions; no pwntools imports or target execution."""
from contextlib import nullcontext
import importlib.metadata
from pathlib import Path
import runpy
import sys
import tempfile
import types
import unittest
from unittest.mock import patch
layout = runpy.run_path(str(Path(__file__).with_name("layout_impl.py")))
class UpstreamReached(RuntimeError):
pass
class LayoutStorageTests(unittest.TestCase):
def inspect(self, section_type, flags, section_stride=64):
class Headers:
header = types.SimpleNamespace(e_shoff=64, e_shentsize=section_stride, e_shstrndx=0)
structs = types.SimpleNamespace(Elf_Shdr=types.SimpleNamespace(sizeof=lambda: 64))
stream = object()
def num_sections(self): return 2
def num_segments(self): return 0
def get_shstrndx(self): return 0
def _get_section_header(self, index):
return types.SimpleNamespace(sh_type="SHT_NULL" if index == 0 else section_type,
sh_flags=0 if index == 0 else flags)
class Elf:
def __init__(self, *_args, **_kwargs):
raise UpstreamReached("No artifact decoding or emulation is allowed in this test")
modules = {name: types.ModuleType(name) for name in (
"pwnlib", "pwnlib.context", "pwnlib.elf", "elftools", "elftools.common",
"elftools.common.exceptions", "elftools.common.utils", "elftools.elf", "elftools.elf.elffile")}
modules["pwnlib.context"].context = types.SimpleNamespace(local=lambda **_kwargs: nullcontext())
modules["pwnlib.elf"].ELF = Elf
modules["elftools.common.exceptions"].ELFError = type("ELFError", (Exception,), {})
modules["elftools.elf.elffile"].ELFFile = lambda _stream: Headers()
modules["elftools.common.utils"].struct_parse = lambda _struct, _stream, stream_pos: Headers()._get_section_header((stream_pos - 64) // max(section_stride, 1))
content = bytearray(64 + 2 * max(section_stride, 64))
content[:7] = b"\x7fELF\x02\x01\x01"
content[16:18] = (1).to_bytes(2, "little") # ET_REL
content[18:20] = (62).to_bytes(2, "little")
content[40:48] = (64).to_bytes(8, "little")
content[58:60] = section_stride.to_bytes(2, "little")
content[60:62] = (2).to_bytes(2, "little")
with tempfile.TemporaryDirectory() as directory:
snapshot = Path(directory) / "object.snapshot"
snapshot.write_bytes(content)
with patch.dict(sys.modules, modules), patch.object(importlib.metadata, "version", side_effect=layout["PACKAGES"].__getitem__):
return layout["inspect_elf"](snapshot, Path(directory) / "cache")
def test_compressed_interpreted_storage_refuses_before_upstream_decoding(self):
for section_type in ("SHT_STRTAB", "SHT_SYMTAB", "SHT_DYNSYM", "SHT_REL", "SHT_RELA", "SHT_RELR", "SHT_DYNAMIC"):
with self.subTest(section_type=section_type):
with self.assertRaises(layout["LayoutFailure"]) as raised:
self.inspect(section_type, 0x800)
self.assertEqual(raised.exception.reason, "unsupported-target")
self.assertIn("SHF_COMPRESSED", str(raised.exception))
with self.assertRaises(UpstreamReached):
self.inspect(section_type, 0)
def test_uninterpreted_compressed_payload_does_not_change_admission(self):
with self.assertRaises(UpstreamReached):
self.inspect("SHT_PROGBITS", 0x800)
def test_short_section_header_stride_refuses_before_raw_header_scan(self):
for stride in (0, 1, 63):
with self.subTest(stride=stride):
with self.assertRaises(layout["LayoutFailure"]) as raised:
self.inspect("SHT_PROGBITS", 0, stride)
self.assertEqual(raised.exception.reason, "format")
self.assertIn("section-header entry", str(raised.exception))
for stride in (64, 96):
with self.subTest(stride=stride):
with self.assertRaises(UpstreamReached):
self.inspect("SHT_PROGBITS", 0, stride)
if __name__ == "__main__":
unittest.main()
-12
View File
@@ -44,11 +44,6 @@ before dependency seeks. Structural inventory and the convenience-map engine
still parse the snapshot separately; eliminating that remaining duplication
requires preserving the upstream GOT/PLT and mitigation facts.
The layout profile returns `unsupported_target` for `SHF_COMPRESSED` string,
symbol, relocation or dynamic metadata sections before upstream convenience-map
decoding. Their logical names and entry coordinates cannot be read from the
compressed file bytes. Other compressed payload sections remain opaque.
## Interpreting results
- Artifact path, SHA-256 and size identify original selected bytes. All parsing
@@ -483,13 +478,6 @@ section directory with neither a dynamic symbol table nor a dynamic segment
reports a complete empty inventory. Mach-O exports are reported by
`inspect_macho`.
ELF export tables marked `SHF_COMPRESSED` are not decompressed. A compressed
dynamic symbol table or its string table reports `unsupported` coverage rather
than treating stored bytes as symbols or names. Compressed version tables or
their string tables keep readable export rows with `partial` coverage and
unresolved versions. When version definition storage is compressed, requirement
names also remain unknown because unseen definitions may override them.
`max_file_bytes` defaults to 256 MiB (maximum 512 MiB; CLI `--max-file-bytes`).
Export directories in PE headers are read like those in sections. An export or
dynamic symbol table above 262144 entries, more than 262144 rows, or more than
-7
View File
@@ -173,12 +173,5 @@ on one page share a fixed 16 MiB text allowance; a page that exceeds it fails
with a resource error whose recovery is a smaller `limit`, because
`max_ranges` and `max_file_bytes` do not raise it.
Compressed symbol or name tables and symbol-table links that do not name a
file-backed `SHT_STRTAB` section leave symbol names unknown and coverage
incomplete; decoded FDE ranges remain available.
A compressed dynamic section cannot supply `DT_PLTGOT`. Without another
observed GOT base, affected data-relative unwind pointers remain unresolved.
Malformed ELF headers fail without partial success. This workflow does not
execute the image, evaluate CFA programs, or follow indirect pointers.
@@ -150,11 +150,6 @@ A symbol name that cannot be read, because its string table is missing or its
offset lies outside the table, is reported as no name and makes the report
`incomplete` with a counted limitation. Name offset 0 is the empty name.
ELF sections marked `SHF_COMPRESSED` are not decoded as initializer slots,
symbols, names or relocation records. Affected facts remain unresolved and
coverage is incomplete; other readable tables remain available. An explicit
`DT_SYMTAB` must identify the decoded symbol table used for dynamic relocations.
The dynamic relocation tables are read in steps that yield to other work and
observe cancellation, so a large `DT_RELR` table does not block the server.
-1
View File
@@ -26,7 +26,6 @@ for new work, roadmap granularity and producer-owned contracts.
- [Native call source coordinates](native/call-source-locations/spec.md) ([plan](native/call-source-locations/plan.md), [tasks](native/call-source-locations/tasks.md), [verification](native/call-source-locations/verification.md))
- [Go static string-header literal recovery](native/go-string-literals/spec.md) ([plan](native/go-string-literals/plan.md), [tasks](native/go-string-literals/tasks.md))
- [Windows Shell Link shortcut inspection](native/windows-shortcut/spec.md) ([plan](native/windows-shortcut/plan.md), [tasks](native/windows-shortcut/tasks.md))
- [NTFS $MFT and USN Journal inspection](native/ntfs-mft/spec.md) ([plan](native/ntfs-mft/plan.md), [tasks](native/ntfs-mft/tasks.md))
- [Solidity terminal metadata inspection](solidity/cbor-metadata/spec.md) ([plan](solidity/cbor-metadata/plan.md), [tasks](solidity/cbor-metadata/tasks.md), [verification](solidity/cbor-metadata/verification.md))
- [Inspect a V8 code cache container](v8/code-cache-inspection/spec.md) ([plan](v8/code-cache-inspection/plan.md), [tasks](v8/code-cache-inspection/tasks.md))
- [Deno compile executable inventory and extraction](artifacts/deno-standalone/spec.md) ([plan](artifacts/deno-standalone/plan.md), [tasks](artifacts/deno-standalone/tasks.md))
-39
View File
@@ -1,39 +0,0 @@
# Implementation plan
## Boundary and contract
Follow the existing artifact readers. `src/domain/windows/ntfsMft.ts` owns FILE
record bytes and coverage; `src/domain/artifactFormatMetadata.ts` owns the
`ntfs-mft` result schema. `NtfsMftArtifactReader` reads the admitted file
through `ArtifactFileSource` and exposes its facts through `formatMetadata()`.
A record stream is its own content, so the root occurrence carries every fact
and no members are enumerated.
Recognition lives in `hasMftFileRecordSignature` and is applied in
`classifyArtifactBytes`, next to the other unambiguous magic probes.
## Incremental work and validation
1. Define the first-step contract, limits and observations; add parser fixtures.
2. Implement the FILE record header and update-sequence fixup parse with
explicit limitations.
3. Add the reader, format descriptor and byte classification.
4. Decode attribute records, then `$FILE_NAME` attributes so a record's parent
reference, names and timestamps become explicit; keep hard links separate.
5. Parse `$UsnJrnl:$J` records and correlate them to records by full file
reference, including sequence numbers, reporting unresolved references.
6. Document the workflow in a native Windows artifact guide.
## Effects and compatibility
The workflow reads one caller-selected local file. It adds no target execution,
injection, mutation, protection recovery, network access, volume mount or live
filesystem access, so no maintainer scope decision is required.
`inspect_artifact` routing for every existing format is unchanged, and the new
format id appears only for bytes whose first record carries the `FILE`
signature.
## References
- https://learn.microsoft.com/en-us/windows/win32/fileio/master-file-table
- https://github.com/libyal/libfsntfs/blob/main/documentation/New%20Technologies%20File%20System%20(NTFS).asciidoc
-51
View File
@@ -1,51 +0,0 @@
# NTFS $MFT and USN Journal inspection
This specification records an implementation increment and its original problem
statement. For current behavior and limitations, see the artifact inventory
guide.
## Problem and outcome
REA has no reader for acquired NTFS metadata artifacts. An analyst holding a
captured `$MFT` stream cannot ask which FILE records it contains, whether their
update-sequence fixups survived acquisition, or which records are directories,
extensions of another record, or no longer in use. Add offline, read-only
inventory of a selected `$MFT` file, delivered in steps so each one merges on
its own.
## Requirements
- Recognize a `$MFT` stream by a plausible first `FILE` record header under
any file name. CLI and MCP target admission route it to the artifact reader.
- Validate each FILE record header: the `FILE` signature, sequence number, flags,
used and total byte counts, base record reference, and the update-sequence
array's offset, count and array bounds.
- Verify the update-sequence fixups: every sector trailer the array covers must
hold the array's sequence number. A mismatch keeps the record's header facts
and is reported per record and in the metadata count.
- Report a missing, misplaced or short update-sequence array as unverified
rather than as a clean record, and a trailing partial record as a byte count.
- Report the first invalid record signature and the number of invalid slots.
Continue at the next slot when the stream declared a supported record size.
With a fallback record size, stop at the first invalid signature.
- Bound the whole file at a fixed 512 MiB ceiling and report the ceiling as a
`limit` failure with its boundary.
- Return the first 32 decoded record headers with their source offsets and the
number of omitted headers. Aggregate counts cover the full parsed stream.
- Parse only the records' headers and fixups. Attributes, filenames, data
streams and the `$UsnJrnl:$J` journal are not decoded in this step, so no path,
file size, file reference correlation or change history is claimed.
## Acceptance
Synthesized fixtures cover a clean record, a failed fixup, a missing update
sequence array, an array shorter than the record's sector count, extension and
directory records, a record whose signature is wrong mid-stream, a trailing
partial record, an undeclared record size, a 4 KiB record, a stream shorter than
one header, bytes without the `FILE` signature, and the file-size ceiling.
Inventory tests cover the root occurrence's metadata and the coverage status
when a fixup fails, and classification tests cover several file names for the
same bytes.
Not covered in this step: real `$MFT` files written by a Windows host, USN
journal records, attribute decoding, and any correlation between the two.
-11
View File
@@ -1,11 +0,0 @@
# Tasks
- [x] Define the first-step scope, contract and limits.
- [x] Parse FILE record headers and update-sequence fixups with synthesized
fixtures and explicit limitations.
- [x] Add the reader, format descriptor and byte classification.
- [ ] Decode attribute records, then `$FILE_NAME` attributes with parent
references, names, hard links and timestamps.
- [ ] Parse `$UsnJrnl:$J` records and correlate them to records by full file
reference.
- [ ] Verify against real `$MFT` and journal captures from a Windows host.
+1 -1
View File
@@ -636,7 +636,7 @@ const PATH_SCOPES = {
"src/ghidra/GhidraJavaProperties.ts",
"src/ghidra/GhidraReleasePolicy.ts",
"scripts/verify-ghidra-bridge-sources.mjs",
"tests/conformance/ghidra/ReaInstructionContextProbe.java",
"tests/conformance/ghidra/ReaArmContextProbe.java",
"tests/conformance/ghidra/ReaComputedCallCoverageProbe.java",
"tests/conformance/ghidra/ReaDecodeProbe.java",
".github/workflows/ghidra-bridge-sources.yml",
+13 -14
View File
@@ -50,7 +50,7 @@ transport.stderr?.on("data", (bytes) => {
const ajv = new Ajv2020({ strict: false, validateFormats: false });
const measurements = [];
const contract = toolContract("inspect_web_network_capture");
const report = await withVerifierCleanup(async () => {
await withVerifierCleanup(async () => {
try {
await client.connect(transport);
} catch (cause) {
@@ -196,7 +196,7 @@ const report = await withVerifierCleanup(async () => {
});
}
await client.ping();
return {
const report = {
status: "passed",
profile: "aioquic-1.5.0-json-0.3",
measurements,
@@ -206,6 +206,17 @@ const report = await withVerifierCleanup(async () => {
"No PCAP or keylog is read by this verifier; repeated STREAM coverage is not proof of retransmission or network loss.",
],
};
await writeFile(reportPath, JSON.stringify(report, null, 2) + "\n", {
flag: "wx",
mode: 0o600,
});
console.log(
JSON.stringify({
status: report.status,
cases: measurements.length,
report: reportPath,
}),
);
}, [
async () => {
await closeVerifierResources([
@@ -214,15 +225,3 @@ const report = await withVerifierCleanup(async () => {
]);
},
]);
await writeFile(reportPath, JSON.stringify(report, null, 2) + "\n", {
flag: "wx",
mode: 0o600,
});
console.log(
JSON.stringify({
status: report.status,
cases: measurements.length,
report: reportPath,
}),
);
-3
View File
@@ -97,7 +97,6 @@ import {
isDfuSeFile,
} from "../domain/firmware/dfuse.js";
import { hasFitImagesNode } from "../domain/firmware/fitImage.js";
import { hasMftFileRecordHeader } from "../domain/windows/ntfsMft.js";
import {
locateEspFlashLayout,
type EspFlashSelection,
@@ -817,7 +816,6 @@ const detectArtifactFormat = async (
});
if (esp === "esp-image") return "esp-image";
if (hasFitImagesNode(magic)) return "fit";
if (hasMftFileRecordHeader(magic)) return "ntfs-mft";
// Binary plists, including keyed archives, often use other extensions.
if (hasBinaryPlistSignature(magic)) return "plist";
const named = namedArtifactFormat(lower);
@@ -901,7 +899,6 @@ const isArchiveFormat = (
"dfuse",
"esp-flash-image",
"fit",
"ntfs-mft",
].includes(format);
/** Whether the leading bytes begin any executable header family, valid or not. */
@@ -3,7 +3,10 @@ import { createHash } from "node:crypto";
import { expect, it } from "vitest";
import { parseReferenceSourceEntries } from "./ReferenceSourceImportEntries.js";
import { historicalSourceLanguages } from "../domain/referenceSourceGraph.js";
import {
historicalSourceInventoryHasPartialFacts,
historicalSourceLanguages,
} from "../domain/referenceSourceGraph.js";
import type { ReferenceSourceRead } from "../reference/ReferenceSourceReaderTypes.js";
it("projects an unsupported non-regular entry as unknown kind instead of file", () => {
@@ -98,6 +101,100 @@ it("keeps source entries when a module has an empty specifier", () => {
expect(parsed.limitations[0]).toMatch(/not recorded.*relationship/iu);
});
it("resolves immediately invoked loaders from their proven source bases", () => {
const files = [
[
"main.cjs",
'(() => require)()("./dep.cjs"); (() => () => {})()("./local.cjs");',
],
[
"main.mjs",
[
'import {createRequire} from "node:module";',
'createRequire(new URL("./sub/base.cjs", import.meta.url))("./dep.cjs");',
'createRequire(new URL("./sub/base.cjs", import.meta.url)).call(null, "./member.cjs");',
].join("\n"),
],
["dep.cjs", ""],
["sub/dep.cjs", ""],
["member.cjs", ""],
["sub/member.cjs", ""],
] as const;
const parsed = parseReferenceSourceEntries(
{
root: "/selected/source",
entries: files.map(([path, content]) => ({
status: "read",
kind: "file",
path,
bytes: Buffer.from(content),
size: Buffer.byteLength(content),
})),
bytesRead: files.reduce(
(sum, [, content]) => sum + Buffer.byteLength(content),
0,
),
limitations: [],
},
[],
);
expect(
parsed.relationships.filter(({ kind }) => kind === "requires"),
).toEqual([
{
from_path: "main.cjs",
to: "dep.cjs",
kind: "requires",
resolution: "internal",
parse_state: "parsed",
},
{
from_path: "main.mjs",
to: "sub/dep.cjs",
kind: "requires",
resolution: "internal",
parse_state: "parsed",
},
]);
});
it.each([
["function load(name) { require(name); }", true],
["function load(require, name) { require(name); }", false],
["require(7); require();", false],
] as const)(
"reports unresolved CommonJS target coverage for %s",
(source, partial) => {
const bytes = Buffer.from(source);
const parsed = parseReferenceSourceEntries(
{
root: "/selected/source",
entries: [
{
status: "read",
kind: "file",
path: "main.cjs",
bytes,
size: bytes.length,
},
],
bytesRead: bytes.length,
limitations: [],
},
[],
);
expect(parsed.relationships).toEqual([]);
expect(
historicalSourceInventoryHasPartialFacts({
...parsed,
parse_failures: parsed.parseFailures,
exclusions: [],
}),
).toBe(partial);
expect(parsed.entries[0]?.limitations).toEqual(parsed.limitations);
},
);
it("retains Kotlin source identity and language across the inventory projection", () => {
const files = [
[
@@ -135,7 +135,6 @@ const TARGETS: TargetFixtures = {
espFlash: { selected: true, flashOffset: null, partitionTableOffset: null },
},
fit: { ...archive, format: "fit" },
"ntfs-mft": { ...archive, format: "ntfs-mft" },
plist: { ...artifact, format: "plist" },
javascript: { ...artifact, format: "javascript" },
"source-map": { ...artifact, format: "source-map" },
-1
View File
@@ -215,7 +215,6 @@ export interface ArtifactReader {
| "dfuse"
| "esp-flash-image"
| "fit"
| "ntfs-mft"
| "file";
entries(
signal?: AbortSignal,
+9 -38
View File
@@ -18,10 +18,7 @@ import {
type ArtifactTraversalOptions,
} from "./ArtifactReader.js";
import { ArtifactResourceScope } from "./ArtifactResourceScope.js";
import {
artifactReadCancelledBy,
assertArtifactReadActive,
} from "./ArtifactFileOperation.js";
import { assertArtifactReadActive } from "./ArtifactFileOperation.js";
import { ArtifactFileSource } from "./ArtifactFileSource.js";
import { readFileHandleChunks } from "../filesystem/readFileHandleChunks.js";
import type { StableRegularFileDescriptor } from "../filesystem/RegularFile.js";
@@ -67,14 +64,12 @@ export const regionFileName = (start: number, end: number): string =>
*/
export class DecodedRegionArtifactReader<Image> implements ArtifactReader {
readonly #scope = new ArtifactResourceScope();
#entries = new Map<string, ArtifactEntry>();
#regions = new Map<string, Uint8Array>();
readonly #entries = new Map<string, ArtifactEntry>();
readonly #regions = new Map<string, Uint8Array>();
readonly #streams = new Set<Readable>();
readonly #source: ArtifactFileSource;
readonly #decoding: RegionDecoding<Image>;
#prepared:
| { signal: AbortSignal | undefined; promise: Promise<void> }
| undefined;
#prepared: Promise<void> | undefined;
/** Whole-file facts only; the region map is the sole owner of region bytes. */
#described: Omit<DecodedRegions, "regions"> | undefined;
#closing = false;
@@ -115,27 +110,8 @@ export class DecodedRegionArtifactReader<Image> implements ArtifactReader {
): AsyncIterable<ArtifactEntry> {
await this.#scope.run(async () => {
this.#assertOpen(signal);
for (;;) {
const pending = (this.#prepared ??= {
signal,
promise: this.#prepare(signal),
});
try {
await pending.promise;
break;
} catch (cause: unknown) {
if (this.#prepared === pending) this.#prepared = undefined;
if (
pending.signal !== signal &&
signal?.aborted !== true &&
artifactReadCancelledBy(cause, pending.signal)
) {
this.#assertOpen(signal);
continue;
}
throw cause;
}
}
this.#prepared ??= this.#prepare(signal);
await this.#prepared;
await this.#source.read(signal);
});
const coverage = this.#described?.coverage;
@@ -254,8 +230,6 @@ export class DecodedRegionArtifactReader<Image> implements ArtifactReader {
async #prepare(signal?: AbortSignal): Promise<void> {
const described = this.#decoding.describe(await this.#decode(signal));
const entries = new Map<string, ArtifactEntry>();
const regions = new Map<string, Uint8Array>();
let index = 0;
for (const region of described.regions) {
if (index % 256 === 0) await setImmediate();
@@ -274,16 +248,13 @@ export class DecodedRegionArtifactReader<Image> implements ArtifactReader {
adapterKey: `${this.#decoding.keyPrefix}:${String(index)}`,
formatMetadata: region.formatMetadata,
} satisfies ArtifactEntry;
entries.set(entry.adapterKey, entry);
regions.set(entry.adapterKey, region.bytes);
this.#entries.set(entry.adapterKey, entry);
this.#regions.set(entry.adapterKey, region.bytes);
index += 1;
}
const { regions: _regions, ...facts } = described;
await this.#source.read(signal);
this.#assertOpen(signal);
this.#entries = entries;
this.#regions = regions;
this.#described = facts;
await this.#source.read(signal);
}
async #decode(signal?: AbortSignal): Promise<Image> {
-40
View File
@@ -1,4 +1,3 @@
import { artifactPreparationRecoveryCases } from "../../tests/fixtures/artifactReaderLifecycle.js";
import { getEventListeners, once } from "node:events";
import { readFile, truncate, writeFile } from "node:fs/promises";
import { join } from "node:path";
@@ -405,38 +404,6 @@ it("keeps the cancellation reason before admission", async () => {
}
});
it("does not publish FIT entries when final source verification fails", async () => {
const path = await writeFit("partial-publication.itb", FIT_EMBEDDED);
const control = new FitArtifactReader(path);
const reader = new FitArtifactReader(path);
const expected = await firstEntry(control);
const nativeRead = ArtifactFileSource.prototype.read;
const physical = Object.assign(
new Error("final source verification failed"),
{ code: "EIO" },
);
let calls = 0;
const read = vi
.spyOn(ArtifactFileSource.prototype, "read")
.mockImplementation(function (this: ArtifactFileSource, ...args) {
calls += 1;
if (calls === 4) return Promise.reject(physical);
return nativeRead.apply(this, args);
});
try {
await expect(firstEntry(reader)).rejects.toBe(physical);
await expect(reader.open(expected)).rejects.toMatchObject({
reason: "integrity",
});
read.mockRestore();
expect((await firstEntry(reader)).path).toBe(expected.path);
} finally {
read.mockRestore();
await reader.close();
await control.close();
}
});
it("keeps the cancellation reason during hash verification", async () => {
const reader = new FitArtifactReader(
await writeFit("hashing.itb", LARGE_FIT),
@@ -463,8 +430,6 @@ it("keeps the cancellation reason during hash verification", async () => {
reader.entries(controller.signal)[Symbol.asyncIterator]().next(),
).rejects.toMatchObject({ reason: "cancelled", cause: reason });
expect(observer).toHaveBeenCalledWith("sha256");
observer.mockRestore();
expect((await firstEntry(reader)).kind).toBe("file");
} finally {
observer.mockRestore();
await reader.close();
@@ -604,8 +569,3 @@ it("preserves a final validation failure when cancellation overlaps it", async (
await reader.close();
}
});
artifactPreparationRecoveryCases({
image: () => FIT_EMBEDDED,
createReader: (path, admitted) => new FitArtifactReader(path, admitted),
});
+7 -34
View File
@@ -13,10 +13,7 @@ import {
type ArtifactTraversalOptions,
} from "./ArtifactReader.js";
import { ArtifactResourceScope } from "./ArtifactResourceScope.js";
import {
artifactReadCancelledBy,
assertArtifactReadActive,
} from "./ArtifactFileOperation.js";
import { assertArtifactReadActive } from "./ArtifactFileOperation.js";
import { ArtifactFileSource } from "./ArtifactFileSource.js";
import {
createFitDigest,
@@ -33,13 +30,11 @@ const FDT_HEADER_BYTES = 40;
export class FitArtifactReader implements ArtifactReader {
readonly format = "fit" as const;
readonly #scope = new ArtifactResourceScope();
#entries = new Map<string, ArtifactEntry>();
readonly #entries = new Map<string, ArtifactEntry>();
readonly #streams = new Set<Readable>();
readonly #path: string;
readonly #source: ArtifactFileSource;
#prepared:
| { signal: AbortSignal | undefined; promise: Promise<void> }
| undefined;
#prepared: Promise<void> | undefined;
#metadata: FitMetadata | undefined;
#coverage: ArtifactReaderTraversal | undefined;
#closing = false;
@@ -71,27 +66,8 @@ export class FitArtifactReader implements ArtifactReader {
): AsyncIterable<ArtifactEntry> {
await this.#scope.run(async () => {
this.#assertOpen(signal);
for (;;) {
const pending = (this.#prepared ??= {
signal,
promise: this.#prepare(signal),
});
try {
await pending.promise;
break;
} catch (cause: unknown) {
if (this.#prepared === pending) this.#prepared = undefined;
if (
pending.signal !== signal &&
signal?.aborted !== true &&
artifactReadCancelledBy(cause, pending.signal)
) {
this.#assertOpen(signal);
continue;
}
throw cause;
}
}
this.#prepared ??= this.#prepare(signal);
await this.#prepared;
await this.#source.read(signal);
});
if (this.#coverage !== undefined)
@@ -231,7 +207,6 @@ export class FitArtifactReader implements ArtifactReader {
this.#path,
initial.size,
);
const entries = new Map<string, ArtifactEntry>();
let index = 0;
for (const image of structure.images) {
if (index % 256 === 0) await setImmediate();
@@ -253,17 +228,15 @@ export class FitArtifactReader implements ArtifactReader {
adapterKey: `fit:${String(image.index)}`,
formatMetadata: image.metadata,
} satisfies ArtifactEntry;
entries.set(entry.adapterKey, entry);
this.#entries.set(entry.adapterKey, entry);
}
await this.#source.read(signal);
this.#assertOpen(signal);
this.#entries = entries;
this.#metadata = structure.metadata;
if (structure.limitations.length > 0)
this.#coverage = {
status: "complete",
limitations: structure.limitations,
};
await this.#source.read(signal);
}
/** One streamed pass per image feeds every supported digest. */
@@ -1,6 +1,3 @@
import { artifactPreparationRecoveryCases } from "../../tests/fixtures/artifactReaderLifecycle.js";
import { ArtifactFileSource } from "./ArtifactFileSource.js";
import { openRegularFile } from "../filesystem/RegularFile.js";
import { writeFile } from "node:fs/promises";
import { join } from "node:path";
@@ -162,72 +159,6 @@ it("settles an active region stream when closed", async () => {
}
});
it("retries decoded-region preparation cancelled while reading", async () => {
const path = await writeImage("retry.hex", INTEL_HEX_I32_GAP);
const admitted = await openRegularFile(path, { symlinks: "reject" });
const reader = new HexRecordArtifactReader(path, "intel-hex", admitted);
const controller = new AbortController();
const reason = { stage: "decoded region read" };
const nativeRead = admitted.handle.read.bind(admitted.handle);
const read = vi.spyOn(admitted.handle, "read").mockImplementation((async (
...args: unknown[]
) => {
const result = await Reflect.apply(nativeRead, admitted.handle, args);
controller.abort(reason);
return result;
}) as typeof admitted.handle.read);
try {
await expect(
reader.entries(controller.signal)[Symbol.asyncIterator]().next(),
).rejects.toMatchObject({ reason: "cancelled", cause: reason });
read.mockRestore();
expect(
(await reader.entries()[Symbol.asyncIterator]().next()).value,
).toMatchObject({ kind: "file" });
} finally {
read.mockRestore();
await reader.close();
await admitted.handle.close();
}
});
it("does not publish decoded regions when final source verification fails", async () => {
const path = await writeImage("partial.hex", INTEL_HEX_I32_GAP);
const control = new HexRecordArtifactReader(path, "intel-hex");
const reader = new HexRecordArtifactReader(path, "intel-hex");
const expected = (await control.entries()[Symbol.asyncIterator]().next())
.value!;
const nativeRead = ArtifactFileSource.prototype.read;
const physical = Object.assign(
new Error("final source verification failed"),
{ code: "EIO" },
);
let calls = 0;
const read = vi
.spyOn(ArtifactFileSource.prototype, "read")
.mockImplementation(function (this: ArtifactFileSource, ...args) {
calls += 1;
if (calls === 2) return Promise.reject(physical);
return nativeRead.apply(this, args);
});
try {
await expect(reader.entries()[Symbol.asyncIterator]().next()).rejects.toBe(
physical,
);
await expect(reader.open(expected)).rejects.toMatchObject({
reason: "integrity",
});
read.mockRestore();
expect(
(await reader.entries()[Symbol.asyncIterator]().next()).value,
).toMatchObject({ path: expected.path });
} finally {
read.mockRestore();
await reader.close();
await control.close();
}
});
it("honors cancellation before admitting a file with the exact reason", async () => {
const reader = new HexRecordArtifactReader(
await writeImage("cancel.hex", INTEL_HEX_I32_GAP),
@@ -406,9 +337,3 @@ it("bounds copied region chunks and isolates cached bytes across opens", async (
await reader.close();
}
});
artifactPreparationRecoveryCases({
image: () => Buffer.from(INTEL_HEX_I32_GAP),
createReader: (path, admitted) =>
new HexRecordArtifactReader(path, "intel-hex", admitted),
});
-211
View File
@@ -1,211 +0,0 @@
import { writeFile } from "node:fs/promises";
import { join } from "node:path";
import { expect, it, vi } from "vitest";
import { ArtifactFileSource } from "./ArtifactFileSource.js";
import { assertArtifactReadActive } from "./ArtifactFileOperation.js";
import { NtfsMftArtifactReader } from "./NtfsMftArtifactReader.js";
import { MFT_DEFAULT_RECORD_BYTES } from "../domain/windows/ntfsMft.js";
import { createTestTempDirectory } from "../../tests/fixtures/temporaryDirectory.js";
import { scanCanonicalArtifactInventory } from "../../tests/fixtures/artifactInventory.js";
/** A 1 KiB FILE record whose update sequence array covers both sectors. */
const recordBytes = (options: {
readonly sequenceNumber?: number;
readonly flags?: number;
readonly index?: number;
}): Buffer => {
const record = Buffer.alloc(MFT_DEFAULT_RECORD_BYTES);
record.write("FILE", 0, "latin1");
const sequenceNumber = options.sequenceNumber ?? 1;
record.writeUInt16LE(48, 4);
record.writeUInt16LE(3, 6);
record.writeUInt16LE(sequenceNumber, 48);
record.writeUInt16LE(0xa5a5, 50);
record.writeUInt16LE(0xa5a5, 52);
record.writeUInt16LE(sequenceNumber, 16);
record.writeUInt16LE(1, 18);
record.writeUInt32LE(options.index ?? 0, 44);
record.writeUInt16LE(56, 20);
record.writeUInt16LE(options.flags ?? 0x0001, 22);
record.writeUInt32LE(64, 24);
record.writeUInt32LE(0xffff_ffff, 56);
record.writeUInt32LE(MFT_DEFAULT_RECORD_BYTES, 28);
record.writeUInt16LE(sequenceNumber, 510);
record.writeUInt16LE(sequenceNumber, 1022);
return record;
};
const writeMft = async (name: string, bytes: Buffer): Promise<string> => {
const directory = await createTestTempDirectory("rea-ntfs-mft-");
const path = join(directory, name);
await writeFile(path, bytes);
return path;
};
it("reports a $MFT stream's record headers as the root occurrence", async () => {
const inventory = await scanCanonicalArtifactInventory(
await writeMft(
"$MFT",
Buffer.concat([
recordBytes({}),
recordBytes({ flags: 0x0003, index: 1 }),
]),
),
);
expect(inventory.manifest.root_format).toBe("ntfs-mft");
expect(inventory.coverage.status).toBe("complete");
// A record stream declares no members, so the root is its whole inventory.
expect(inventory.occurrences.map((row) => row.logical_path)).toEqual(["."]);
expect(inventory.occurrences[0]).toMatchObject({
artifact_format: "ntfs-mft",
format_metadata: {
kind: "ntfs-mft",
record_bytes: MFT_DEFAULT_RECORD_BYTES,
record_count: 2,
in_use_record_count: 2,
directory_record_count: 1,
extension_record_count: 0,
fixup_mismatch_count: 0,
fixup_unverified_record_count: 0,
truncated_tail_bytes: 0,
},
});
});
it("names a failed fixup in the coverage limitations", async () => {
const record = recordBytes({ sequenceNumber: 4 });
// A trailer that is not the sequence number marks the sector as not cleanly written.
record.writeUInt16LE(0x1234, 510);
const inventory = await scanCanonicalArtifactInventory(
await writeMft("$MFT", record),
);
expect(inventory.manifest.root_format).toBe("ntfs-mft");
expect(inventory.coverage.status).toBe("unknown");
expect(inventory.limitations).toContainEqual(
expect.stringMatching(/sector trailer that is not the update sequence/u),
);
});
it("reports an unaligned used-byte count as incomplete coverage", async () => {
const record = recordBytes({});
record.writeUInt32LE(57, 24);
const inventory = await scanCanonicalArtifactInventory(
await writeMft("$MFT", record),
);
expect(inventory.coverage.status).toBe("unknown");
expect(inventory.occurrences[0]?.format_metadata).toMatchObject({
record_headers: [{ usedBytes: 57, extentValid: false }],
});
});
it("reports incomplete coverage when FILE extents contradict the record slots", async () => {
const usedTooLarge = recordBytes({});
usedTooLarge.writeUInt32LE(4096, 24);
const totalTooSmall = recordBytes({});
totalTooSmall.writeUInt32LE(512, 28);
const inventory = await scanCanonicalArtifactInventory(
await writeMft("$MFT", Buffer.concat([usedTooLarge, totalTooSmall])),
);
expect(inventory.coverage.status).toBe("unknown");
expect(inventory.limitations).toContainEqual(
"2 FILE record(s) declare inconsistent record extents, beginning at offset 0",
);
expect(inventory.occurrences[0]?.format_metadata).toMatchObject({
record_count: 2,
fixup_mismatch_count: 0,
fixup_unverified_record_count: 0,
});
});
it("retries after final source verification fails without publishing its metadata", async () => {
const reader = new NtfsMftArtifactReader(
await writeMft("$MFT", recordBytes({})),
);
const read = ArtifactFileSource.prototype.read;
const spy = vi
.spyOn(ArtifactFileSource.prototype, "read")
.mockImplementationOnce(function (this: ArtifactFileSource, signal) {
return read.call(this, signal);
})
.mockRejectedValueOnce(new Error("final source verification failed"));
try {
await expect(
reader.entries()[Symbol.asyncIterator]().next(),
).rejects.toThrow("final source verification failed");
expect(reader.formatMetadata()).toBeUndefined();
await expect(
reader.entries()[Symbol.asyncIterator]().next(),
).resolves.toMatchObject({
done: true,
});
expect(reader.formatMetadata()).toMatchObject({
kind: "ntfs-mft",
record_count: 1,
});
} finally {
spy.mockRestore();
await reader.close();
}
});
it("retries a shared preparation cancelled by another traversal", async () => {
const reader = new NtfsMftArtifactReader(
await writeMft("$MFT", recordBytes({})),
);
const controller = new AbortController();
let entered: () => void = () => {};
const started = new Promise<void>((resolve) => {
entered = resolve;
});
const read = ArtifactFileSource.prototype.read;
const spy = vi
.spyOn(ArtifactFileSource.prototype, "read")
.mockImplementationOnce(async function (this: ArtifactFileSource, signal) {
entered();
await new Promise<void>((resolve) => {
signal?.addEventListener("abort", () => resolve(), { once: true });
});
assertArtifactReadActive(signal);
return read.call(this, signal);
});
try {
const cancelled = reader
.entries(controller.signal)
[Symbol.asyncIterator]()
.next();
const cancelledResult = expect(cancelled).rejects.toMatchObject({
reason: "cancelled",
});
await started;
const other = reader.entries()[Symbol.asyncIterator]().next();
controller.abort("cancelled first traversal");
await cancelledResult;
await expect(other).resolves.toMatchObject({ done: true });
expect(reader.formatMetadata()).toMatchObject({ record_count: 1 });
} finally {
controller.abort();
spy.mockRestore();
await reader.close();
}
});
it("revalidates the source before reusing a completed header inventory", async () => {
const path = await writeMft("$MFT", recordBytes({}));
const reader = new NtfsMftArtifactReader(path);
try {
await reader.entries()[Symbol.asyncIterator]().next();
await writeFile(
path,
Buffer.concat([recordBytes({}), recordBytes({ index: 1 })]),
);
await expect(
reader.entries()[Symbol.asyncIterator]().next(),
).rejects.toMatchObject({ reason: "integrity" });
} finally {
await reader.close();
}
});
-174
View File
@@ -1,174 +0,0 @@
import type { Readable } from "node:stream";
import {
ntfsMftMetadataSchema,
type ArtifactFormatMetadata,
} from "../domain/artifactFormatMetadata.js";
import type { RegionDecodeFailure } from "../domain/firmware/regionDecoder.js";
import {
createMftDecoder,
type MftInventory,
} from "../domain/windows/ntfsMft.js";
import {
ArtifactReaderFailure,
type ArtifactEntry,
type ArtifactReader,
type ArtifactReaderTraversal,
type ArtifactTraversalOptions,
} from "./ArtifactReader.js";
import { ArtifactResourceScope } from "./ArtifactResourceScope.js";
import {
artifactReadCancelledBy,
assertArtifactReadActive,
} from "./ArtifactFileOperation.js";
import { ArtifactFileSource } from "./ArtifactFileSource.js";
import { readFileHandleChunks } from "../filesystem/readFileHandleChunks.js";
import type { StableRegularFileDescriptor } from "../filesystem/RegularFile.js";
/**
* An MFT stream is its own content: the FILE records describe the source
* volume, not members inside the stream, so header facts are attached to
* the root occurrence.
*/
export class NtfsMftArtifactReader implements ArtifactReader {
readonly format = "ntfs-mft" as const;
readonly #scope = new ArtifactResourceScope();
readonly #source: ArtifactFileSource;
#prepared:
| { signal: AbortSignal | undefined; promise: Promise<void> }
| undefined;
#metadata: ArtifactFormatMetadata | undefined;
#limitations: readonly string[] = [];
#closing = false;
constructor(path: string, admitted?: StableRegularFileDescriptor) {
this.#source = new ArtifactFileSource(path, "$MFT stream", admitted);
}
formatMetadata(): ArtifactFormatMetadata | undefined {
return this.#metadata === undefined
? undefined
: structuredClone(this.#metadata);
}
async prepareContainer(
expectedSha256: string,
signal?: AbortSignal,
): Promise<void> {
await this.#scope.run(async () => {
this.#assertOpen(signal);
await this.#source.bindSha256(expectedSha256, signal);
});
}
async *entries(
signal?: AbortSignal,
options?: ArtifactTraversalOptions,
): AsyncIterable<ArtifactEntry> {
await this.#scope.run(async () => {
this.#assertOpen(signal);
for (;;) {
const pending = (this.#prepared ??= {
signal,
promise: this.#prepare(signal),
});
try {
await pending.promise;
break;
} catch (cause: unknown) {
if (this.#prepared === pending) this.#prepared = undefined;
if (
pending.signal !== signal &&
signal?.aborted !== true &&
artifactReadCancelledBy(cause, pending.signal)
) {
this.#assertOpen(signal);
continue;
}
throw cause;
}
}
this.#assertOpen(signal);
await this.#source.read(signal);
});
// An MFT declares no members, so the root occurrence is the whole
// inventory; the coverage callback reports what its parse did not establish.
options?.onCoverage?.(coverageOf(this.#limitations));
}
async open(entry: ArtifactEntry, _signal?: AbortSignal): Promise<Readable> {
return this.#scope.run(async () => {
throw new ArtifactReaderFailure(
"unavailable",
`$MFT stream declares no member to open for ${entry.path}`,
);
});
}
async close(): Promise<void> {
this.#closing = true;
await this.#scope.close();
await this.#source.close();
this.#metadata = undefined;
}
provenance(): readonly [] {
return [];
}
#assertOpen(signal?: AbortSignal): void {
assertArtifactReadActive(signal);
if (this.#closing)
throw new ArtifactReaderFailure(
"unavailable",
"$MFT stream reader is closed",
);
}
async #prepare(signal?: AbortSignal): Promise<void> {
const image = await this.#decode(signal);
await this.#source.read(signal);
this.#metadata = ntfsMftMetadataSchema.parse(image.metadata);
this.#limitations = image.limitations;
}
async #decode(signal?: AbortSignal): Promise<MftInventory> {
const { handle, initial } = await this.#source.read(signal);
const decoder = createMftDecoder(initial.size);
if (!decoder.ok) throw this.#failure(decoder.error);
if (initial.size > 0)
for await (const chunk of readFileHandleChunks(handle, {
end: initial.size - 1,
signal,
cancellationFailure: (reason) =>
new ArtifactReaderFailure("cancelled", "$MFT stream read cancelled", {
cause: reason,
}),
}) as AsyncIterable<Buffer>) {
this.#assertOpen(signal);
const written = decoder.value.write(chunk);
if (!written.ok) throw this.#failure(written.error);
}
const image = decoder.value.finish();
if (!image.ok) throw this.#failure(image.error);
return image.value;
}
#failure(failure: RegionDecodeFailure): ArtifactReaderFailure {
return failure.kind === "format"
? new ArtifactReaderFailure(
"format",
`$MFT stream is malformed: ${failure.message}`,
)
: new ArtifactReaderFailure(
"limit",
`$MFT stream exceeds a fixed limit: ${failure.message}`,
failure.boundary === undefined ? {} : { boundary: failure.boundary },
);
}
}
const coverageOf = (limitations: readonly string[]): ArtifactReaderTraversal =>
limitations.length === 0
? { status: "complete", limitations: [] }
: { status: "unknown", limitations };
@@ -1,7 +1,4 @@
import {
artifactPreparationRecoveryCases,
artifactReaderLifecycleCases,
} from "../../tests/fixtures/artifactReaderLifecycle.js";
import { artifactReaderLifecycleCases } from "../../tests/fixtures/artifactReaderLifecycle.js";
import { writeFile } from "node:fs/promises";
import { join } from "node:path";
import { buffer } from "node:stream/consumers";
@@ -94,14 +91,6 @@ it("preserves cancellation during payload CRC admission", async () => {
}
})(),
).rejects.toMatchObject({ reason: "cancelled", cause: reason });
reads.mockRestore();
const recovered = await entries(reader);
expect(recovered).toHaveLength(1);
expect(
(await buffer(await reader.open(recovered[0]!))).equals(
Buffer.alloc(128 * 1024, 0x61),
),
).toBe(true);
} finally {
await reader.close();
reads.mockRestore();
@@ -216,8 +205,3 @@ it("reports impossible over-cap component extents as malformed", async () => {
await reader.close();
}
});
artifactPreparationRecoveryCases({
image: () => buildUImage({ payload: Buffer.from("payload") }),
createReader: (path, admitted) => new UImageArtifactReader(path, admitted),
});
+4 -28
View File
@@ -8,10 +8,7 @@ import type {
UImageMetadata,
} from "../domain/artifactFormatMetadata.js";
import { ArtifactFileSource } from "./ArtifactFileSource.js";
import {
artifactReadCancelledBy,
assertArtifactReadActive,
} from "./ArtifactFileOperation.js";
import { assertArtifactReadActive } from "./ArtifactFileOperation.js";
import {
ArtifactReaderFailure,
copyArtifactEntry,
@@ -50,9 +47,7 @@ export class UImageArtifactReader implements ArtifactReader {
readonly #entries = new Map<string, UImageEntry>();
readonly #streams = new Set<Readable>();
readonly #source: ArtifactFileSource;
#prepared:
| { signal: AbortSignal | undefined; promise: Promise<void> }
| undefined;
#prepared: Promise<void> | undefined;
#header: UImageMetadata | undefined;
#closing = false;
@@ -80,27 +75,8 @@ export class UImageArtifactReader implements ArtifactReader {
): AsyncIterable<ArtifactEntry> {
await this.#scope.run(async () => {
this.#assertOpen(signal);
for (;;) {
const pending = (this.#prepared ??= {
signal,
promise: this.#prepare(signal),
});
try {
await pending.promise;
break;
} catch (cause: unknown) {
if (this.#prepared === pending) this.#prepared = undefined;
if (
pending.signal !== signal &&
signal?.aborted !== true &&
artifactReadCancelledBy(cause, pending.signal)
) {
this.#assertOpen(signal);
continue;
}
throw cause;
}
}
this.#prepared ??= this.#prepare(signal);
await this.#prepared;
await this.#source.read(signal);
});
this.#assertOpen(signal);
@@ -1,5 +1,3 @@
import { artifactPreparationRecoveryCases } from "../../tests/fixtures/artifactReaderLifecycle.js";
import { ArtifactFileSource } from "./ArtifactFileSource.js";
import { truncate, writeFile } from "node:fs/promises";
import { join } from "node:path";
import { buffer, text } from "node:stream/consumers";
@@ -45,68 +43,6 @@ const repeatedBlock = (data: Buffer): Buffer => {
]);
};
it("retries UnityFs preparation cancelled during file reading", async () => {
const path = await fixture(buildUnityFs([{ path: "a", data: "x" }]));
const admitted = await openRegularFile(path, { symlinks: "reject" });
const reader = new UnityFsArtifactReader(path, admitted);
const controller = new AbortController();
const reason = { stage: "header read" };
const nativeRead = admitted.handle.read.bind(admitted.handle);
const read = vi.spyOn(admitted.handle, "read").mockImplementation((async (
...args: unknown[]
) => {
const result = await Reflect.apply(nativeRead, admitted.handle, args);
controller.abort(reason);
return result;
}) as typeof admitted.handle.read);
try {
await expect(
(async () => {
for await (const _ of reader.entries(controller.signal)) {
}
})(),
).rejects.toMatchObject({ reason: "cancelled", cause: reason });
read.mockRestore();
expect((await entries(reader)).map((entry) => entry.path)).toEqual(["a"]);
} finally {
read.mockRestore();
await reader.close();
await admitted.handle.close();
}
});
it("does not publish UnityFs entries when final source verification fails", async () => {
const path = await fixture(buildUnityFs([{ path: "a", data: "x" }]));
const control = new UnityFsArtifactReader(path);
const reader = new UnityFsArtifactReader(path);
const expected = (await entries(control))[0]!;
const nativeRead = ArtifactFileSource.prototype.read;
const physical = Object.assign(
new Error("final source verification failed"),
{ code: "EIO" },
);
let calls = 0;
const read = vi
.spyOn(ArtifactFileSource.prototype, "read")
.mockImplementation(function (this: ArtifactFileSource, ...args) {
calls += 1;
if (calls === 2) return Promise.reject(physical);
return nativeRead.apply(this, args);
});
try {
await expect(entries(reader)).rejects.toBe(physical);
await expect(reader.open(expected)).rejects.toMatchObject({
reason: "integrity",
});
read.mockRestore();
expect((await entries(reader)).map((entry) => entry.path)).toEqual(["a"]);
} finally {
read.mockRestore();
await reader.close();
await control.close();
}
});
describe("UnityFS allocation and malformed-input boundaries", () => {
it("rejects an oversized directory extending past the physical file", async () => {
const bytes = buildUnityFs([{ path: "data", data: "x" }]);
@@ -495,8 +431,3 @@ describe("UnityFS stream capacity and cache ownership", () => {
}
});
});
artifactPreparationRecoveryCases({
image: () => buildUnityFs([{ path: "a", data: "x" }]),
createReader: (path, admitted) => new UnityFsArtifactReader(path, admitted),
});
+24 -65
View File
@@ -12,10 +12,7 @@ import {
type ArtifactTraversalOptions,
} from "./ArtifactReader.js";
import { ArtifactResourceScope } from "./ArtifactResourceScope.js";
import {
artifactReadCancelledBy,
assertArtifactReadActive,
} from "./ArtifactFileOperation.js";
import { assertArtifactReadActive } from "./ArtifactFileOperation.js";
import { readArtifactFileBytes } from "./ArtifactFileRange.js";
import type {
ArtifactFormatMetadata,
@@ -55,12 +52,10 @@ interface UnityFsEntryState {
export class UnityFsArtifactReader implements ArtifactReader {
readonly format = "unityfs" as const;
readonly #scope = new ArtifactResourceScope();
#entries = new Map<string, UnityFsEntryState>();
readonly #entries = new Map<string, UnityFsEntryState>();
readonly #streams = new Map<Readable, AbortController>();
readonly #source: ArtifactFileSource;
#prepared:
| { signal: AbortSignal | undefined; promise: Promise<void> }
| undefined;
#prepared: Promise<void> | undefined;
#header: UnityFsMetadata | undefined;
#blocks: readonly UnityFsBlock[] = [];
#dataStart = 0;
@@ -93,27 +88,8 @@ export class UnityFsArtifactReader implements ArtifactReader {
): AsyncIterable<ArtifactEntry> {
await this.#scope.run(async () => {
this.#assertOpen(signal);
for (;;) {
const pending = (this.#prepared ??= {
signal,
promise: this.#prepare(signal),
});
try {
await pending.promise;
break;
} catch (cause: unknown) {
if (this.#prepared === pending) this.#prepared = undefined;
if (
pending.signal !== signal &&
signal?.aborted !== true &&
artifactReadCancelledBy(cause, pending.signal)
) {
this.#assertOpen(signal);
continue;
}
throw cause;
}
}
this.#prepared ??= this.#prepare(signal);
await this.#prepared;
await this.#source.read(signal);
});
this.#assertOpen(signal);
@@ -253,7 +229,7 @@ export class UnityFsArtifactReader implements ArtifactReader {
if (header === undefined)
throw unityFsFormatFailure("no complete UnityFS header");
const status = header.layout.status;
const metadata: UnityFsMetadata = {
this.#header = {
kind: "unityfs",
signature: "UnityFS",
format_version: header.formatVersion,
@@ -269,12 +245,7 @@ export class UnityFsArtifactReader implements ArtifactReader {
data_hash: null,
directory_status: status,
};
if (status !== "available") {
await this.#source.read(signal);
this.#assertOpen(signal);
this.#header = metadata;
return;
}
if (status !== "available") return;
const { dataHash, blocks, nodes, dataStart } = await readUnityFsBlocksInfo(
handle,
header,
@@ -288,55 +259,44 @@ export class UnityFsArtifactReader implements ArtifactReader {
if (streamBytes > BigInt(budget))
throw expansionLimit("declared virtual stream", streamBytes, budget);
}
const preparedMetadata: UnityFsMetadata = {
...metadata,
this.#header = {
...this.#header,
data_hash: dataHash,
block_count: blocks.length,
data_start: dataStart,
};
this.#dataStart = dataStart;
// Keep the source block count above, but do not revisit zero-output rows
// for every member. Parsed stream/compressed offsets remain authoritative.
const readableBlocks = blocks.filter((block) => block.uncompressedSize > 0);
const registered = await this.#registerNodes(
nodes,
readableBlocks,
budget,
signal,
);
await this.#source.read(signal);
this.#assertOpen(signal);
this.#header = preparedMetadata;
this.#dataStart = dataStart;
this.#blocks = readableBlocks;
this.#blocks = blocks.filter((block) => block.uncompressedSize > 0);
this.#remainingDecodeBytes = budget;
this.#entries = registered.entries;
this.#remainingReadOperations = registered.remainingReadOperations;
await this.#registerNodes(nodes, budget, signal);
await this.#source.read(signal);
}
async #registerNodes(
nodes: readonly UnityFsNode[],
blocks: readonly UnityFsBlock[],
budget: number,
signal?: AbortSignal,
): Promise<{
entries: Map<string, UnityFsEntryState>;
remainingReadOperations: number;
}> {
const entries = new Map<string, UnityFsEntryState>();
): Promise<void> {
let readableBytes = 0n;
for (let index = 0; index < nodes.length; index++) {
if (index % 256 === 0) await setImmediate();
this.#assertOpen(signal);
const node = nodes[index]!;
const firstBlock =
node.size === 0 ? -1 : blockEndingAfter(blocks, node.offset);
node.size === 0 ? -1 : blockEndingAfter(this.#blocks, node.offset);
const lastBlock =
node.size === 0
? -1
: blockEndingAfter(blocks, node.offset + node.size - 1);
: blockEndingAfter(this.#blocks, node.offset + node.size - 1);
if (node.size !== 0 && (firstBlock < 0 || lastBlock < firstBlock))
throw unityFsFormatFailure("member has no block range");
const unavailable = firstUnavailableBlock(blocks, firstBlock, lastBlock);
const unavailable = firstUnavailableBlock(
this.#blocks,
firstBlock,
lastBlock,
);
const kind =
(node.flags & 1) !== 0 ? ("directory" as const) : ("file" as const);
if (kind === "file" && unavailable === undefined) {
@@ -371,7 +331,7 @@ export class UnityFsArtifactReader implements ArtifactReader {
flags: node.flags,
},
} satisfies UnityFsEntry;
entries.set(adapterKey, {
this.#entries.set(adapterKey, {
entry,
firstBlock,
lastBlock,
@@ -380,15 +340,14 @@ export class UnityFsArtifactReader implements ArtifactReader {
}
// Four ordinary passes over admitted blocks, nodes and plaintext chunks,
// with a small-reader floor. Byte quotas alone cannot bound tiny-block I/O.
const remainingReadOperations =
this.#remainingReadOperations =
4 *
Math.max(
1024,
blocks.length +
this.#blocks.length +
nodes.length +
Math.ceil(Number(readableBytes) / CHUNK_BYTES),
);
return { entries, remainingReadOperations };
}
async *#memberChunks(
@@ -1,5 +1,3 @@
import { artifactPreparationRecoveryCases } from "../../tests/fixtures/artifactReaderLifecycle.js";
import { ArtifactFileSource } from "./ArtifactFileSource.js";
import { once } from "node:events";
import { writeFile } from "node:fs/promises";
import { join } from "node:path";
@@ -28,68 +26,6 @@ const entries = async (reader: WxapkgArtifactReader) => {
return result;
};
it("retries Wxapkg preparation cancelled during file reading", async () => {
const path = await fixture(buildWxapkg([{ path: "a", data: "x" }]));
const admitted = await openRegularFile(path, { symlinks: "reject" });
const reader = new WxapkgArtifactReader(path, admitted);
const controller = new AbortController();
const reason = { stage: "header read" };
const nativeRead = admitted.handle.read.bind(admitted.handle);
const read = vi.spyOn(admitted.handle, "read").mockImplementation((async (
...args: unknown[]
) => {
const result = await Reflect.apply(nativeRead, admitted.handle, args);
controller.abort(reason);
return result;
}) as typeof admitted.handle.read);
try {
await expect(
(async () => {
for await (const _ of reader.entries(controller.signal)) {
}
})(),
).rejects.toMatchObject({ reason: "cancelled", cause: reason });
read.mockRestore();
expect((await entries(reader)).map((entry) => entry.path)).toEqual(["a"]);
} finally {
read.mockRestore();
await reader.close();
await admitted.handle.close();
}
});
it("does not publish Wxapkg entries when final source verification fails", async () => {
const path = await fixture(buildWxapkg([{ path: "a", data: "x" }]));
const control = new WxapkgArtifactReader(path);
const reader = new WxapkgArtifactReader(path);
const expected = (await entries(control))[0]!;
const nativeRead = ArtifactFileSource.prototype.read;
const physical = Object.assign(
new Error("final source verification failed"),
{ code: "EIO" },
);
let calls = 0;
const read = vi
.spyOn(ArtifactFileSource.prototype, "read")
.mockImplementation(function (this: ArtifactFileSource, ...args) {
calls += 1;
if (calls === 2) return Promise.reject(physical);
return nativeRead.apply(this, args);
});
try {
await expect(entries(reader)).rejects.toBe(physical);
await expect(reader.open(expected)).rejects.toMatchObject({
reason: "integrity",
});
read.mockRestore();
expect((await entries(reader)).map((entry) => entry.path)).toEqual(["a"]);
} finally {
read.mockRestore();
await reader.close();
await control.close();
}
});
describe("wxapkg allocation and malformed-input boundaries", () => {
it("admits the encoded table budget before reading it", async () => {
const bytes = buildWxapkg([{ path: "a", data: "x" }]);
@@ -372,8 +308,3 @@ describe("wxapkg stream capacity and freshness", () => {
}
});
});
artifactPreparationRecoveryCases({
image: () => buildWxapkg([{ path: "a", data: "x" }]),
createReader: (path, admitted) => new WxapkgArtifactReader(path, admitted),
});
+10 -44
View File
@@ -12,10 +12,7 @@ import {
type ArtifactTraversalOptions,
} from "./ArtifactReader.js";
import { ArtifactResourceScope } from "./ArtifactResourceScope.js";
import {
artifactReadCancelledBy,
assertArtifactReadActive,
} from "./ArtifactFileOperation.js";
import { assertArtifactReadActive } from "./ArtifactFileOperation.js";
import { readArtifactFileBytes } from "./ArtifactFileRange.js";
import type {
ArtifactFormatMetadata,
@@ -51,12 +48,10 @@ interface WxapkgEntryState {
export class WxapkgArtifactReader implements ArtifactReader {
readonly format = "wxapkg" as const;
readonly #scope = new ArtifactResourceScope();
#entries = new Map<string, WxapkgEntryState>();
readonly #entries = new Map<string, WxapkgEntryState>();
readonly #streams = new Map<Readable, AbortController>();
readonly #source: ArtifactFileSource;
#prepared:
| { signal: AbortSignal | undefined; promise: Promise<void> }
| undefined;
#prepared: Promise<void> | undefined;
#header: WxapkgMetadata | undefined;
#remainingReadOperations = 0;
#admittedReadOperations = 0;
@@ -86,27 +81,8 @@ export class WxapkgArtifactReader implements ArtifactReader {
): AsyncIterable<ArtifactEntry> {
await this.#scope.run(async () => {
this.#assertOpen(signal);
for (;;) {
const pending = (this.#prepared ??= {
signal,
promise: this.#prepare(signal),
});
try {
await pending.promise;
break;
} catch (cause: unknown) {
if (this.#prepared === pending) this.#prepared = undefined;
if (
pending.signal !== signal &&
signal?.aborted !== true &&
artifactReadCancelledBy(cause, pending.signal)
) {
this.#assertOpen(signal);
continue;
}
throw cause;
}
}
this.#prepared ??= this.#prepare(signal);
await this.#prepared;
await this.#source.read(signal);
});
this.#assertOpen(signal);
@@ -225,7 +201,7 @@ export class WxapkgArtifactReader implements ArtifactReader {
const header = await readWxapkgHeader(handle, initial.size, signal);
if (header === undefined)
throw wxapkgFormatFailure("no complete wxapkg header");
const metadata: WxapkgMetadata = {
this.#header = {
kind: "wxapkg",
info: header.layout.status === "available" ? header.info : null,
index_length:
@@ -236,12 +212,7 @@ export class WxapkgArtifactReader implements ArtifactReader {
header.layout.status === "available" ? header.fileCount : null,
directory_status: header.layout.status,
};
if (header.layout.status !== "available") {
await this.#source.read(signal);
this.#assertOpen(signal);
this.#header = metadata;
return;
}
if (header.layout.status !== "available") return;
const { members } = await readWxapkgFileTable(
handle,
header,
@@ -249,7 +220,6 @@ export class WxapkgArtifactReader implements ArtifactReader {
signal,
);
const budget = artifactExpansionBudgetBytes(initial.size);
const entries = new Map<string, WxapkgEntryState>();
let declaredBytes = 0n;
for (let index = 0; index < members.length; index++) {
if (index % 256 === 0) await setImmediate();
@@ -264,7 +234,7 @@ export class WxapkgArtifactReader implements ArtifactReader {
{ boundary: "memory" },
);
const adapterKey = `wxapkg-entry:${String(index)}`;
entries.set(adapterKey, {
this.#entries.set(adapterKey, {
entry: {
path: member.path,
sourcePath: member.declaredPath,
@@ -290,18 +260,14 @@ export class WxapkgArtifactReader implements ArtifactReader {
});
}
// Header, table and per-chunk member reads; a small-reader floor applies.
const admittedReadOperations =
this.#admittedReadOperations =
4 *
Math.max(
1024,
members.length + Math.ceil(Number(declaredBytes) / CHUNK_BYTES),
);
this.#remainingReadOperations = this.#admittedReadOperations;
await this.#source.read(signal);
this.#assertOpen(signal);
this.#header = metadata;
this.#entries = entries;
this.#admittedReadOperations = admittedReadOperations;
this.#remainingReadOperations = admittedReadOperations;
}
async *#memberChunks(
+1 -2
View File
@@ -43,8 +43,7 @@ type FormatsWithReaders =
| "uf2"
| "dfuse"
| "esp-flash-image"
| "fit"
| "ntfs-mft";
| "fit";
const readerFormatsMatch: Equal<ArtifactReaderFormat, FormatsWithReaders> =
true;
-9
View File
@@ -40,7 +40,6 @@ import { NodeSeaArtifactReader } from "./NodeSeaArtifactReader.js";
import type { PyInstallerAdmission } from "./pyinstaller/PyInstallerAdmission.js";
import { PyInstallerArtifactReader } from "./PyInstallerArtifactReader.js";
import { Uf2ArtifactReader } from "./Uf2ArtifactReader.js";
import { NtfsMftArtifactReader } from "./NtfsMftArtifactReader.js";
import { UImageArtifactReader } from "./UImageArtifactReader.js";
import { UnityFsArtifactReader } from "./UnityFsArtifactReader.js";
import { WxapkgArtifactReader } from "./WxapkgArtifactReader.js";
@@ -466,14 +465,6 @@ const NON_ZIP_FORMATS = {
createReader: (path, options) =>
new FitArtifactReader(path, sourceOf(options)),
}),
"ntfs-mft": described("ntfs-mft", {
...plain,
retainClassifiedSource: true,
readerLimitation:
"MFT inventory reports whole-stream counts and a bounded sample of the first 32 decoded FILE record headers, with the omitted-header count. Attributes, filenames, data streams and the $UsnJrnl:$J journal are not decoded, so no path, file size or change history is established.",
createReader: (path, options) =>
new NtfsMftArtifactReader(path, sourceOf(options)),
}),
unknown: described("unknown", plain),
} satisfies Record<
Exclude<ArtifactFormatId, ZipPackageFormat>,
@@ -576,28 +576,6 @@ it("classifies nested UF2 by block magics and DfuSe by prefix and suffix", () =>
).toMatchObject({ format: "file" });
});
it("classifies a $MFT stream by its FILE record header, under any name", () => {
const mft = Buffer.alloc(2_048);
mft.write("FILE", 0, "latin1");
mft.writeUInt32LE(1024, 28);
for (const path of ["evidence/$MFT", "evidence/mft.raw", "evidence/records"])
expect(classifyArtifactBytesAtPath(path, mft)).toEqual({
kind: "executable",
format: "ntfs-mft",
});
});
it("leaves plain content beginning with FILE unclassified as MFT", () => {
const text = Buffer.from(
"FILE is a plain text prefix, not an NTFS record header",
);
expect(
classifyArtifactBytesAtPath("evidence/readme.txt", text),
).toMatchObject({
format: "file",
});
});
it("classifies a FIT by a /images node within the first 4 KiB", () => {
const images = [{ name: "kernel", data: Buffer.from("kernel") }];
const late = devicetreeFixture({
@@ -11,7 +11,6 @@ import { recognizeEspImage } from "../../domain/firmware/espImage.js";
import { hasUf2Block } from "../../domain/firmware/uf2.js";
import { isDfuSeFile } from "../../domain/firmware/dfuse.js";
import { hasFitImagesNode } from "../../domain/firmware/fitImage.js";
import { hasMftFileRecordHeader } from "../../domain/windows/ntfsMft.js";
import type { ArtifactClassificationEvidence } from "../ArtifactClassificationProbe.js";
import { asarHeaderAdmissionFailure } from "../AsarHeader.js";
import { hasDestinationPath } from "../ArtifactPaths.js";
@@ -623,7 +622,6 @@ export const classifyArtifactBytes = (
return "dfuse";
if (recognizeEspImage(prefix, fileSize) === "esp-image") return "esp-image";
if (hasFitImagesNode(prefix)) return "fit";
if (hasMftFileRecordHeader(prefix)) return "ntfs-mft";
if (hasBinaryPlistSignature(prefix)) return "plist";
if (hasXmlPlistSignature(prefix)) return "plist";
const fatFormat = classifyFatArtifact(evidence, fileSize);
-1
View File
@@ -86,7 +86,6 @@ const formatSchema = z.enum([
"dfuse",
"esp-flash-image",
"fit",
"ntfs-mft",
"plist",
"javascript",
"source-map",
-56
View File
@@ -1,5 +1,4 @@
import { z } from "zod";
import { MFT_INLINE_HEADER_LIMIT } from "./windows/ntfsMft.js";
import { jsonRecordSchema } from "./jsonRecord.js";
import { jsonValueSchema } from "./jsonValue.js";
@@ -300,56 +299,6 @@ export const hexRecordRegionMetadataSchema = z.strictObject({
last_line: offset,
});
const ntfsFileReferenceSchema = z.strictObject({
recordNumber: offset,
sequenceNumber: offset,
});
const ntfsRecordHeaderSchema = z.strictObject({
index: offset,
fileOffset: offset,
sequenceNumber: offset,
hardLinkCount: offset,
flags: z.strictObject({
raw: offset,
names: z.array(z.string()),
unknownMask: offset,
}),
inUse: z.boolean(),
directory: z.boolean(),
usedBytes: offset,
totalBytes: offset,
extentValid: z.boolean(),
baseRecordReference: ntfsFileReferenceSchema.nullable(),
declaredRecordNumber: offset.nullable(),
recordNumberMatchesSlot: z.boolean().nullable(),
logFileSequenceNumber: z.string().regex(/^(?:0|[1-9][0-9]*)$/u),
firstAttributeOffset: offset,
nextAttributeId: offset,
fixups: z.strictObject({
sequenceNumber: offset,
entries: offset,
verifiedSectors: offset,
coveredSectors: offset,
mismatchSector: offset.nullable(),
malformed: z.boolean(),
}),
});
/** NTFS $MFT record-header facts; attributes, names and the USN journal are later steps. */
export const ntfsMftMetadataSchema = z.strictObject({
kind: z.literal("ntfs-mft"),
record_bytes: offset,
record_count: offset,
in_use_record_count: offset,
directory_record_count: offset,
extension_record_count: offset,
fixup_mismatch_count: offset,
fixup_unverified_record_count: offset,
truncated_tail_bytes: offset,
record_headers: z.array(ntfsRecordHeaderSchema).max(MFT_INLINE_HEADER_LIMIT),
omitted_record_headers: offset,
});
/**
* U-Boot FIT root facts. Configuration references are declared names; a
* reference that names no image is reported as a limitation, not removed.
@@ -937,7 +886,6 @@ export const artifactFormatMetadataSchema = z.discriminatedUnion("kind", [
espImageSignatureBlockMetadataSchema,
uf2MetadataSchema,
uf2RegionMetadataSchema,
ntfsMftMetadataSchema,
dfuSeMetadataSchema,
dfuSeElementMetadataSchema,
espFlashImageMetadataSchema,
@@ -1088,10 +1036,6 @@ const declarationsByKind: {
notCompared:
"Its familyID and addresses are in its path, family_name is REA's label for the familyID, and its block counts and positions are layout; content comparison covers its bytes.",
},
"ntfs-mft": {
notCompared:
"Its fields come from the record headers in the file's own FILE records, which content comparison covers.",
},
dfuse: {
notCompared:
"Its fields come from the file's own prefix, target headers and suffix, which content comparison covers.",
-2
View File
@@ -103,7 +103,6 @@ export const ARTIFACT_FORMAT_IDS = [
"dfuse",
"esp-flash-image",
"fit",
"ntfs-mft",
"unknown",
] as const;
@@ -125,7 +124,6 @@ const artifactContentFormatSchema = artifactFormatSchema.extract([
"uf2",
"dfuse",
"fit",
"ntfs-mft",
"mach-o-universal",
"mach-o",
"elf",
+1 -2
View File
@@ -302,8 +302,7 @@ export type BinaryTarget =
| "esp-image"
| "uf2"
| "dfuse"
| "fit"
| "ntfs-mft";
| "fit";
})
| (BinaryTargetIdentity &
NonExecutableMetadata & {
-1
View File
@@ -52,7 +52,6 @@ export const evidenceSubjectFormatSchema = z.enum([
"dfuse",
"esp-flash-image",
"fit",
"ntfs-mft",
]);
/** Format-owned architecture declarations shared by Evidence and persisted executions. */
@@ -137,8 +137,7 @@ describe("child-process options detection", () => {
["negative-zero", `spawn("tool", [], { env: -0 })`, false],
["bigint-zero", `spawn("tool", [], { env: 0n })`, false],
["zero-float", `spawn("tool", [], { env: 0.0 })`, false],
["nan", `spawn("tool", [], { env: NaN })`, false],
["shadowed-nan", `((NaN) => spawn("tool", [], { env: NaN }))({})`, true],
["nan", `spawn("tool", [], { env: NaN })`, true],
["empty-string", `spawn("tool", [], { env: "" })`, false],
["false", `spawn("tool", [], { env: false })`, false],
["empty-object", `spawn("tool", [], { env: {} })`, true],
@@ -81,7 +81,7 @@ export const collectJavaScriptSemanticChildProcesses = (
!rejectsOptionsShape(candidate, context, state),
);
const childProcessSpawns = candidates.map((candidate) =>
immutableSpawn(candidate, context, state),
immutableSpawn(candidate, context),
);
return {
childProcessSpawns,
@@ -146,7 +146,6 @@ const collectSpawns = (
const immutableSpawn = (
candidate: SpawnCandidate,
context: ChildSpawnContext,
state: JavaScriptSemanticAnalysisState,
): JavaScriptSemanticChildProcessSpawn => {
const { options, argvCount } = spawnArguments(candidate, context);
const command = dataEffectLiteralString(candidate.node.arguments[0]);
@@ -162,7 +161,7 @@ const immutableSpawn = (
projectedResultBindings: candidate.projectedResultBindings,
command,
argvCount,
environmentSupplied: environmentSupplied(options, context, state),
environmentSupplied: environmentSupplied(options, context),
stdioMode: stdioMode(options, candidate.method, context),
resolution: command === null ? "partial" : "complete",
};
@@ -444,17 +443,11 @@ const spawnArguments = (
const environmentSupplied = (
options: t.Node | undefined,
context: ChildSpawnContext,
state: JavaScriptSemanticAnalysisState,
): boolean => {
const lookup = staticObjectProperty(options, "env", context.isNullish);
if (lookup.status !== "explicit") return false;
const value = unwrapJavaScriptExpression(lookup.value).node;
if (context.isNullish(value)) return false;
if (
t.isIdentifier(value, { name: "NaN" }) &&
isUnshadowedGlobal(value, state, "NaN")
)
return false;
return staticJavaScriptTruthiness(value) !== false;
};
-1
View File
@@ -83,7 +83,6 @@ export const liveBinaryTargetSummaryVariants = [
"uf2",
"dfuse",
"fit",
"ntfs-mft",
]),
architecture: z.null(),
}),
+14 -3
View File
@@ -157,6 +157,7 @@ const extractImports = (
ReferenceSourceRequireCall
>,
): void => {
let unresolvedRequireTarget = false;
traverseJavaScriptAst(ast.program, {
enter: (node) => {
if (
@@ -197,8 +198,14 @@ const extractImports = (
) {
const requireCall = requireCalls.get(node);
if (requireCall === undefined) return;
const specifier = exactStringModuleName(requireCall.arguments?.[0]);
if (specifier === undefined) return;
const target = requireCall.arguments?.[0];
if (target === undefined) return;
const literal = readExactJavaScriptLiteral(target);
if (!literal.found) {
unresolvedRequireTarget = true;
return;
}
if (typeof literal.value !== "string") return;
if (
requireCall.limitation !== undefined &&
!limitations.includes(requireCall.limitation)
@@ -207,7 +214,7 @@ const extractImports = (
const uncertainResolution = requireCall.resolutionBase === "unknown";
appendRelationship(relationships, limitations, {
fromPath: from_path,
to: specifier,
to: literal.value,
kind: "requires",
parseState: uncertainResolution ? "partial" : "parsed",
...(uncertainResolution ? { resolution: "unknown" } : {}),
@@ -221,6 +228,10 @@ const extractImports = (
}
},
});
if (unresolvedRequireTarget)
limitations.push(
`A CommonJS module target in ${from_path} could not be determined statically.`,
);
};
const parseWithBabel = (
-14
View File
@@ -1,4 +1,3 @@
import { posix } from "node:path";
import { err, ok, type Result } from "./result.js";
export interface ReferenceSourcePathLookup {
@@ -6,8 +5,6 @@ export interface ReferenceSourcePathLookup {
readonly has: (path: string) => boolean;
readonly resolve: (path: string) => string | undefined;
readonly resolveCandidates: (paths: readonly string[]) => string | undefined;
/** Resolve a path to the closest recorded parent identity. */
readonly resolveAncestor: (path: string) => string | undefined;
}
/**
@@ -35,17 +32,6 @@ export const createReferenceSourcePathLookup = (
resolveCandidates: (candidates) =>
candidates.find((path) => exact.has(path)) ??
candidates.map(equivalent).find((path) => path !== undefined),
resolveAncestor: (path) => {
for (
let candidate = path;
candidate !== "." && candidate !== "/";
candidate = posix.dirname(candidate)
) {
const found = resolve(candidate);
if (found !== undefined) return found;
}
return undefined;
},
};
};
+17 -16
View File
@@ -75,9 +75,14 @@ export const referenceSourceRequireCalls = (
!t.isNewExpression(node)
)
return;
if (callReceiverIdentifier(node.callee) === undefined) return;
let fact = semanticRequireLoaderAt(node, index.bindings, factoryBindings);
const callee = unwrapJavaScriptExpression(node.callee).node;
if (
!t.isCallExpression(callee) &&
!t.isOptionalCallExpression(callee) &&
!hasIdentifierReceiver(callee)
)
return;
let fact = semanticRequireLoaderAt(node, index.bindings, factoryBindings);
if (
fact === "create-require" ||
((t.isMemberExpression(callee) ||
@@ -153,12 +158,7 @@ export const referenceSourceRequireCalls = (
});
},
});
if (
candidates.some(
({ call }) =>
!t.isIdentifier(unwrapJavaScriptExpression(call.callee).node),
)
)
if (candidates.some(({ call }) => hasMemberCallee(call)))
completeSemanticSteps(index.indexMemberMutationFactsSteps());
return new Map(
candidates.map(
@@ -169,8 +169,7 @@ export const referenceSourceRequireCalls = (
arguments: callArguments,
...(limitation === undefined ? {} : { limitation }),
resolutionBase:
!t.isIdentifier(unwrapJavaScriptExpression(call.callee).node) &&
hasPriorMemberMutation(index, call)
hasMemberCallee(call) && hasPriorMemberMutation(index, call)
? "unknown"
: resolutionBase,
},
@@ -220,15 +219,17 @@ const hasPriorMemberMutation = (
);
};
const callReceiverIdentifier = (
callee: t.Node | null | undefined,
): t.Identifier | undefined => {
if (!t.isNode(callee)) return undefined;
let expression = unwrapJavaScriptExpression(callee).node;
const hasMemberCallee = (call: ReferenceSourceRequireInvocation): boolean => {
const callee = unwrapJavaScriptExpression(call.callee).node;
return t.isMemberExpression(callee) || t.isOptionalMemberExpression(callee);
};
const hasIdentifierReceiver = (callee: t.Node): boolean => {
let expression = callee;
while (
t.isMemberExpression(expression) ||
t.isOptionalMemberExpression(expression)
)
expression = unwrapJavaScriptExpression(expression.object).node;
return t.isIdentifier(expression) ? expression : undefined;
return t.isIdentifier(expression);
};
-478
View File
@@ -1,478 +0,0 @@
import { describe, expect, it } from "vitest";
import {
MFT_DEFAULT_RECORD_BYTES,
MFT_MAX_FILE_BYTES,
createMftDecoder,
parseMftImage,
} from "./ntfsMft.js";
interface RecordOptions {
readonly sequenceNumber?: number;
readonly hardLinkCount?: number;
readonly flags?: number;
readonly baseRecordReference?: bigint;
readonly recordBytes?: number;
/** Write a wrong trailer value into this sector, leaving the others correct. */
readonly corruptSector?: number;
/** Truncate the update sequence array to this many entries; 0 omits it. */
readonly usaEntries?: number;
readonly usaOffset?: number;
}
/** One FILE record with a zeroed body and fixups applied the way NTFS writes them. */
const recordOf = (options: RecordOptions = {}): Uint8Array => {
const recordBytes = options.recordBytes ?? MFT_DEFAULT_RECORD_BYTES;
const record = new Uint8Array(recordBytes);
const view = new DataView(record.buffer);
record.set(Buffer.from("FILE", "latin1"), 0);
const sequenceNumber = options.sequenceNumber ?? 1;
const sectors = recordBytes / 512;
const usaEntries = options.usaEntries ?? sectors + 1;
if (usaEntries > 0) {
const usaOffset = options.usaOffset ?? 48;
view.setUint16(4, usaOffset, true);
view.setUint16(6, usaEntries, true);
view.setUint16(usaOffset, sequenceNumber, true);
for (let sector = 0; sector < sectors; sector += 1) {
const trailer = sector * 512 + 510;
// A saved trailer is what the sector held before the fixup replaced it.
view.setUint16(usaOffset + 2 * (sector + 1), 0xa5a5, true);
view.setUint16(
trailer,
sector >= usaEntries - 1
? 0xa5a5
: sector === options.corruptSector
? 0x1234
: sequenceNumber,
true,
);
}
} else {
view.setUint16(4, 0, true);
view.setUint16(6, 0, true);
}
view.setUint16(16, sequenceNumber, true);
view.setUint16(18, options.hardLinkCount ?? 1, true);
const firstAttributeOffset =
Math.ceil((48 + Math.max(0, usaEntries) * 2) / 8) * 8;
view.setUint16(20, firstAttributeOffset, true);
view.setUint16(22, options.flags ?? 0x0001, true);
view.setUint32(24, firstAttributeOffset + 8, true);
view.setUint32(firstAttributeOffset, 0xffff_ffff, true);
view.setUint32(28, recordBytes, true);
view.setBigUint64(32, options.baseRecordReference ?? 0n, true);
view.setUint16(40, 0, true);
return record;
};
const streamOf = (...records: readonly Uint8Array[]): Uint8Array => {
const total = records.reduce((sum, record) => sum + record.length, 0);
const stream = new Uint8Array(total);
let offset = 0;
for (const record of records) {
stream.set(record, offset);
const view = new DataView(stream.buffer, offset, record.length);
if (view.getUint16(4, true) >= 48)
view.setUint32(44, offset / record.length, true);
offset += record.length;
}
return stream;
};
describe("MFT header facts", () => {
it("inventories a clean record's header and verified fixups", () => {
const parsed = parseMftImage(streamOf(recordOf({ sequenceNumber: 5 })));
expect(parsed.ok).toBe(true);
if (!parsed.ok) return;
expect(parsed.value.metadata).toMatchObject({
kind: "ntfs-mft",
record_bytes: MFT_DEFAULT_RECORD_BYTES,
record_count: 1,
in_use_record_count: 1,
directory_record_count: 0,
extension_record_count: 0,
fixup_mismatch_count: 0,
fixup_unverified_record_count: 0,
truncated_tail_bytes: 0,
});
expect(parsed.value.records[0]).toMatchObject({
index: 0,
fileOffset: 0,
sequenceNumber: 5,
inUse: true,
directory: false,
baseRecordReference: null,
firstAttributeOffset: 56,
fixups: {
sequenceNumber: 5,
verifiedSectors: 2,
coveredSectors: 2,
mismatchSector: null,
malformed: false,
},
});
expect(parsed.value.limitations).toEqual([]);
});
it("identifies unrecognized flags without losing their source bits", () => {
const parsed = parseMftImage(recordOf({ flags: 0x8001 }));
if (!parsed.ok) throw new Error(parsed.error.message);
expect(parsed.value.metadata.record_headers[0]?.flags).toEqual({
raw: 0x8001,
names: ["in-use"],
unknownMask: 0x8000,
});
});
it.each([0, 49, 64, 72])(
"reports an invalid first attribute offset %i",
(offset) => {
const record = recordOf();
new DataView(record.buffer).setUint16(20, offset, true);
const parsed = parseMftImage(record);
if (!parsed.ok) throw new Error(parsed.error.message);
expect(parsed.value.records[0]?.extentValid).toBe(false);
expect(parsed.value.limitations.join(" ")).toContain("inconsistent");
},
);
it("reports a declared record number that contradicts its stream position", () => {
const bytes = streamOf(recordOf(), recordOf());
new DataView(bytes.buffer).setUint32(1024 + 44, 0, true);
const parsed = parseMftImage(bytes);
if (!parsed.ok) throw new Error(parsed.error.message);
expect(parsed.value.records[1]).toMatchObject({
index: 1,
declaredRecordNumber: 0,
recordNumberMatchesSlot: false,
});
expect(parsed.value.limitations.join(" ")).toContain(
"disagrees with their stream slot",
);
});
it("keeps the inline header window separate from the parsed record count", () => {
const parsed = parseMftImage(
streamOf(...Array.from({ length: 33 }, () => recordOf())),
);
if (!parsed.ok) throw new Error(parsed.error.message);
expect(parsed.value.metadata).toMatchObject({
record_count: 33,
omitted_record_headers: 1,
});
expect(parsed.value.metadata.record_headers).toHaveLength(32);
expect(parsed.value.records).toHaveLength(33);
expect(parsed.value.limitations).toEqual([]);
});
it("preserves a log sequence number above the safe integer range", () => {
const record = recordOf();
new DataView(record.buffer).setBigUint64(8, 9007199254740993n, true);
const parsed = parseMftImage(record);
if (!parsed.ok) throw new Error(parsed.error.message);
expect(parsed.value.records[0]?.logFileSequenceNumber).toBe(
"9007199254740993",
);
});
it("verifies the older FILE header without reading its fixup words as a record number", () => {
const oldRecord = recordOf({ usaOffset: 42 });
const newRecord = recordOf();
new DataView(newRecord.buffer).setUint32(44, 1, true);
const parsed = parseMftImage(streamOf(oldRecord, newRecord));
if (!parsed.ok) throw new Error(parsed.error.message);
expect(parsed.value.records).toMatchObject([
{
declaredRecordNumber: null,
fixups: { malformed: false, verifiedSectors: 2 },
},
{
declaredRecordNumber: 1,
fixups: { malformed: false, verifiedSectors: 2 },
},
]);
expect(parsed.value.limitations).toEqual([]);
});
});
describe("MFT fixup coverage", () => {
it.each([0, 0xffff])(
"leaves reserved USA sequence %i unverified",
(sequenceNumber) => {
const parsed = parseMftImage(recordOf({ sequenceNumber }));
if (!parsed.ok) throw new Error(parsed.error.message);
expect(parsed.value.records[0]?.fixups).toMatchObject({
sequenceNumber,
verifiedSectors: 0,
malformed: true,
});
expect(parsed.value.metadata.fixup_unverified_record_count).toBe(1);
expect(parsed.value.limitations.join(" ")).toContain(
"invalid or incomplete",
);
},
);
it("keeps a record whose sector trailer is not the sequence number, and reports it", () => {
const parsed = parseMftImage(
streamOf(recordOf(), recordOf({ sequenceNumber: 9, corruptSector: 1 })),
);
expect(parsed.ok).toBe(true);
if (!parsed.ok) return;
expect(parsed.value.records[1]?.fixups).toMatchObject({
sequenceNumber: 9,
verifiedSectors: 1,
mismatchSector: 1,
});
expect(parsed.value.metadata.fixup_mismatch_count).toBe(1);
expect(parsed.value.limitations.join(" ")).toMatch(
/1 record\(s\) hold a sector trailer/u,
);
});
it("treats a missing update sequence array as unverified rather than clean", () => {
const parsed = parseMftImage(streamOf(recordOf({ usaEntries: 0 })));
expect(parsed.ok).toBe(true);
if (!parsed.ok) return;
expect(parsed.value.records[0]?.fixups).toMatchObject({
malformed: true,
mismatchSector: null,
coveredSectors: 0,
});
expect(parsed.value.metadata.fixup_mismatch_count).toBe(0);
expect(parsed.value.metadata.fixup_unverified_record_count).toBe(1);
expect(parsed.value.limitations.join(" ")).toMatch(
/invalid or incomplete update sequence array/u,
);
});
it.each([
[48, 4],
[49, 3],
[508, 3],
])(
"rejects a malformed update sequence array at %i with %i entries",
(offset, count) => {
const bytes = recordOf();
const view = new DataView(bytes.buffer);
view.setUint16(4, offset, true);
view.setUint16(6, count, true);
const parsed = parseMftImage(bytes);
if (!parsed.ok) throw new Error(parsed.error.message);
expect(parsed.value.records[0]?.fixups).toMatchObject({
malformed: true,
verifiedSectors: 0,
});
expect(parsed.value.metadata.fixup_unverified_record_count).toBe(1);
},
);
it("counts uncovered sectors independently of a mismatched trailer", () => {
const parsed = parseMftImage(
recordOf({ recordBytes: 4096, usaEntries: 3, corruptSector: 0 }),
);
if (!parsed.ok) throw new Error(parsed.error.message);
expect(parsed.value.metadata).toMatchObject({
fixup_mismatch_count: 1,
fixup_unverified_record_count: 1,
});
expect(parsed.value.limitations.join(" ")).toContain(
"invalid or incomplete",
);
});
it("reports an array that covers fewer sectors than the record has", () => {
// A 4 KiB record holds eight sectors; a three-entry array leaves five unverified.
const parsed = parseMftImage(
streamOf(recordOf({ recordBytes: 4096, usaEntries: 3 })),
);
expect(parsed.ok).toBe(true);
if (!parsed.ok) return;
expect(parsed.value.records[0]?.fixups).toMatchObject({
coveredSectors: 2,
verifiedSectors: 2,
mismatchSector: null,
malformed: false,
});
expect(parsed.value.metadata.fixup_unverified_record_count).toBe(1);
});
});
describe("MFT record slots", () => {
it("distinguishes extension records from base records", () => {
const parsed = parseMftImage(
streamOf(
recordOf(),
recordOf({
flags: 0x0005,
baseRecordReference: 0xabcd_fedc_ba98_7654n,
}),
),
);
expect(parsed.ok).toBe(true);
if (!parsed.ok) return;
expect(parsed.value.records[1]).toMatchObject({
baseRecordReference: {
recordNumber: 0xfedc_ba98_7654,
sequenceNumber: 0xabcd,
},
flags: { raw: 0x0005, names: ["in-use", "system"] },
});
expect(parsed.value.metadata.extension_record_count).toBe(1);
});
it("names directory records without losing the in-use flag", () => {
const parsed = parseMftImage(streamOf(recordOf({ flags: 0x0003 })));
expect(parsed.ok).toBe(true);
if (!parsed.ok) return;
expect(parsed.value.records[0]).toMatchObject({
directory: true,
inUse: true,
flags: { names: ["in-use", "directory"] },
});
expect(parsed.value.metadata.directory_record_count).toBe(1);
});
it("skips malformed and sparse slots without renumbering valid records", () => {
const stream = streamOf(
recordOf(),
recordOf(),
new Uint8Array(MFT_DEFAULT_RECORD_BYTES),
recordOf(),
);
stream.set(Buffer.from("BAAD", "latin1"), MFT_DEFAULT_RECORD_BYTES);
const parsed = parseMftImage(stream);
expect(parsed.ok).toBe(true);
if (!parsed.ok) return;
expect(parsed.value.metadata.record_count).toBe(2);
expect(
parsed.value.records.map(({ index, fileOffset }) => [index, fileOffset]),
).toEqual([
[0, 0],
[3, 3 * MFT_DEFAULT_RECORD_BYTES],
]);
expect(parsed.value.limitations).toEqual([
"record at 1024 does not begin with the FILE signature; 2 slot(s) without a FILE signature were found",
]);
});
it("keeps a trailing partial record out of the inventory", () => {
const stream = new Uint8Array(1024 + 300);
stream.set(recordOf(), 0);
const parsed = parseMftImage(stream);
expect(parsed.ok).toBe(true);
if (!parsed.ok) return;
expect(parsed.value.metadata.record_count).toBe(1);
expect(parsed.value.metadata.truncated_tail_bytes).toBe(300);
expect(parsed.value.limitations.join(" ")).toMatch(
/300 trailing byte\(s\)/u,
);
});
it("falls back to the default record size when record 0 declares none", () => {
const record = recordOf();
new DataView(record.buffer).setUint32(28, 0, true);
const parsed = parseMftImage(record);
expect(parsed.ok).toBe(true);
if (!parsed.ok) return;
expect(parsed.value.recordBytes).toBe(MFT_DEFAULT_RECORD_BYTES);
expect(parsed.value.limitations.join(" ")).toMatch(
/not a supported power of two/u,
);
});
it("reads a 4 KiB record at its declared size", () => {
const parsed = parseMftImage(streamOf(recordOf({ recordBytes: 4096 })));
expect(parsed.ok).toBe(true);
if (!parsed.ok) return;
expect(parsed.value.recordBytes).toBe(4096);
expect(parsed.value.metadata.record_count).toBe(1);
});
});
describe("parseMftImage refusals and limits", () => {
it("refuses a file shorter than one record header", () => {
const parsed = parseMftImage(new Uint8Array(32));
expect(parsed.ok).toBe(false);
if (parsed.ok) return;
expect(parsed.error.message).toMatch(
/shorter than one 42-byte record header/u,
);
});
it("refuses bytes that do not start with the FILE signature", () => {
const parsed = parseMftImage(Buffer.alloc(2048, 0x41));
expect(parsed.ok).toBe(false);
if (parsed.ok) return;
expect(parsed.error.message).toMatch(
/does not begin with the FILE signature/u,
);
});
it("refuses a file above the fixed ceiling", () => {
const decoder = createMftDecoder(MFT_MAX_FILE_BYTES + 1);
expect(decoder.ok).toBe(false);
if (decoder.ok) return;
expect(decoder.error).toMatchObject({
kind: "limit",
boundary: "file-size",
});
});
it("refuses streams that differ from the admitted byte count", () => {
const stream = recordOf();
const short = createMftDecoder(stream.length + 1);
if (!short.ok) throw new Error(short.error.message);
expect(short.value.write(stream).ok).toBe(true);
expect(short.value.finish()).toMatchObject({
ok: false,
error: {
kind: "format",
message: "stream ended before the admitted file length",
},
});
const long = createMftDecoder(stream.length - 1);
if (!long.ok) throw new Error(long.error.message);
expect(long.value.write(stream)).toMatchObject({
ok: false,
error: {
kind: "format",
message: "received more bytes than the admitted file length",
},
});
});
it("returns the same inventory without retaining all the decoded records", () => {
const bytes = streamOf(...Array.from({ length: 33 }, () => recordOf()));
const decoder = createMftDecoder(bytes.length);
const parsed = parseMftImage(bytes);
if (!decoder.ok || !parsed.ok) throw new Error("invalid fixture");
expect(decoder.value.write(bytes).ok).toBe(true);
const inventory = decoder.value.finish();
if (!inventory.ok) throw new Error(inventory.error.message);
expect(inventory.value).not.toHaveProperty("records");
expect(inventory.value.metadata).toEqual(parsed.value.metadata);
expect(inventory.value.limitations).toEqual(parsed.value.limitations);
});
it("decodes the same stream through the chunked decoder", () => {
const stream = streamOf(recordOf(), recordOf({ flags: 0x0003 }));
const decoder = createMftDecoder(stream.length);
expect(decoder.ok).toBe(true);
if (!decoder.ok) return;
// Feed one byte past a record boundary so a chunk never aligns with a record.
for (const chunk of [stream.subarray(0, 1025), stream.subarray(1025)])
expect(decoder.value.write(chunk).ok).toBe(true);
const image = decoder.value.finish();
expect(image.ok).toBe(true);
if (!image.ok) return;
expect(image.value.metadata.record_count).toBe(2);
expect(image.value.metadata.directory_record_count).toBe(1);
});
});
-512
View File
@@ -1,512 +0,0 @@
import { err, ok, type Result } from "../result.js";
import type {
RegionDecodeFailure,
RegionDecoder,
} from "../firmware/regionDecoder.js";
/** An NTFS FILE record begins with this signature. */
const MFT_FILE_RECORD_MAGIC = "FILE";
const MFT_FILE_RECORD_MAGIC_BYTES = 4;
/** The shortest header worth reading: a FILE record header ends at `nextAttributeId`. */
const NTFS_MFT_MIN_RECORD_BYTES = 42;
const hasMftFileRecordSignature = (prefix: Uint8Array): boolean =>
prefix.length >= MFT_FILE_RECORD_MAGIC_BYTES &&
text(prefix.subarray(0, MFT_FILE_RECORD_MAGIC_BYTES)) ===
MFT_FILE_RECORD_MAGIC;
/** A record's declared size is a power of two from 512 upward; 1024 is NTFS's default. */
export const MFT_DEFAULT_RECORD_BYTES = 1024;
const MFT_MIN_RECORD_BYTES = 512;
const MFT_MAX_RECORD_BYTES = 65536;
/** An update sequence array holds at least the sequence number and one fixup word. */
const USA_MIN_ENTRIES = 2;
/** NTFS fixes up one 512-byte sector at a time. */
const FIXUP_SECTOR_BYTES = 512;
/**
* Fixed whole-file ceiling. Records are inventoried from their headers only;
* attributes and their payloads are not decoded, so this is deliberately
* smaller than a volume's MFT can be.
*/
export const MFT_MAX_FILE_BYTES = 512 * 1024 * 1024;
/** Limit the inline header sample independently of full stream coverage. */
export const MFT_INLINE_HEADER_LIMIT = 32;
/** FILE record header fields; offsets are from the record start. */
const RECORD_HEADER = {
updateSequenceOffset: 4,
updateSequenceCount: 6,
logFileSequenceNumber: 8,
sequenceNumber: 16,
hardLinkCount: 18,
firstAttributeOffset: 20,
flags: 22,
usedBytes: 24,
totalBytes: 28,
baseRecordReference: 32,
nextAttributeId: 40,
recordNumber: 44,
} as const;
const RECORD_HEADER_WITH_NUMBER_BYTES = 48;
const FILE_RECORD_IN_USE = 0x0001;
const FILE_RECORD_IS_DIRECTORY = 0x0002;
const FILE_RECORD_NAMES = new Map<number, string>([
[FILE_RECORD_IN_USE, "in-use"],
[FILE_RECORD_IS_DIRECTORY, "directory"],
[0x0004, "system"],
[0x0008, "view-index"],
]);
export interface MftRecordFixups {
/** The update sequence number the array's first entry holds, or 0 when absent. */
readonly sequenceNumber: number;
readonly entries: number;
/** Sectors whose trailer held the sequence number before a mismatch. */
readonly verifiedSectors: number;
/** Sectors the array covers; fewer than the record's sectors leaves a tail unverified. */
readonly coveredSectors: number;
/** The first sector whose trailer did not hold the sequence number, if any. */
readonly mismatchSector: number | null;
/** The array itself is missing, misplaced or too short to cover one sector. */
readonly malformed: boolean;
}
export interface MftFileReference {
readonly recordNumber: number;
readonly sequenceNumber: number;
}
const decodeMftFileReference = (reference: bigint): MftFileReference => ({
recordNumber: Number(reference & 0xffff_ffff_ffffn),
sequenceNumber: Number(reference >> 48n),
});
export interface MftRecord {
/** Zero-based ordinal of this record in the stream. */
readonly index: number;
/** File offset where the record starts. */
readonly fileOffset: number;
readonly sequenceNumber: number;
/** Hard link count as recorded; a zero count marks a record with no names. */
readonly hardLinkCount: number;
readonly flags: {
readonly raw: number;
readonly names: readonly string[];
readonly unknownMask: number;
};
readonly inUse: boolean;
readonly directory: boolean;
/** Used and allocated byte counts as declared by the FILE record. */
readonly usedBytes: number;
readonly totalBytes: number;
readonly extentValid: boolean;
/** The base record this is an extension of, or `null` for a base record. */
readonly baseRecordReference: MftFileReference | null;
/** The record number the header declares; `null` when the header predates the field. */
readonly declaredRecordNumber: number | null;
readonly recordNumberMatchesSlot: boolean | null;
/** `$LogFile` sequence number as recorded; no log is read. */
readonly logFileSequenceNumber: string;
/** Declared offset of the first attribute; no attribute decoding follows yet. */
readonly firstAttributeOffset: number;
/** Declared id of the next attribute; no attribute decoding follows yet. */
readonly nextAttributeId: number;
readonly fixups: MftRecordFixups;
}
export interface MftInventory {
readonly metadata: MftMetadata;
/** The record size the stream declared, or the fixed default when it did not. */
readonly recordBytes: number;
readonly limitations: readonly string[];
}
export interface MftImage extends MftInventory {
readonly records: readonly MftRecord[];
}
export interface MftMetadata {
readonly kind: "ntfs-mft";
readonly record_bytes: number;
readonly record_count: number;
readonly in_use_record_count: number;
readonly directory_record_count: number;
readonly extension_record_count: number;
readonly fixup_mismatch_count: number;
readonly fixup_unverified_record_count: number;
readonly truncated_tail_bytes: number;
readonly record_headers: readonly MftRecord[];
readonly omitted_record_headers: number;
}
const text = (ascii: Uint8Array): string =>
Buffer.from(ascii).toString("latin1");
const failure = (
message: string,
boundary?: RegionDecodeFailure["boundary"],
): RegionDecodeFailure => ({
kind: "format",
message,
...(boundary === undefined ? {} : { boundary }),
});
const limit = (
message: string,
boundary?: RegionDecodeFailure["boundary"],
): RegionDecodeFailure => ({
kind: "limit",
message,
...(boundary === undefined ? {} : { boundary }),
});
const parseRecordFlags = (raw: number): MftRecord["flags"] => {
const names: string[] = [];
let unknownMask = raw;
for (const [bit, name] of FILE_RECORD_NAMES) {
if ((raw & bit) !== 0) names.push(name);
unknownMask &= ~bit;
}
return { raw, names, unknownMask };
};
const isRecordSize = (bytes: number): boolean =>
bytes >= MFT_MIN_RECORD_BYTES &&
bytes <= MFT_MAX_RECORD_BYTES &&
(bytes & (bytes - 1)) === 0;
const updateSequenceArrayFits = (
offset: number,
entries: number,
sectors: number,
): boolean =>
entries >= USA_MIN_ENTRIES &&
entries <= sectors + 1 &&
offset >= NTFS_MFT_MIN_RECORD_BYTES &&
offset % 2 === 0 &&
offset + entries * 2 <= FIXUP_SECTOR_BYTES - 2;
/** Recognize a FILE header without requiring its trailers or attributes to verify. */
export const hasMftFileRecordHeader = (prefix: Uint8Array): boolean => {
if (
prefix.length < NTFS_MFT_MIN_RECORD_BYTES ||
!hasMftFileRecordSignature(prefix)
)
return false;
const view = new DataView(
prefix.buffer,
prefix.byteOffset,
prefix.byteLength,
);
return (
isRecordSize(view.getUint32(RECORD_HEADER.totalBytes, true)) ||
updateSequenceArrayFits(
view.getUint16(RECORD_HEADER.updateSequenceOffset, true),
view.getUint16(RECORD_HEADER.updateSequenceCount, true),
MFT_DEFAULT_RECORD_BYTES / FIXUP_SECTOR_BYTES,
)
);
};
/**
* Parse FILE record headers and verify their update-sequence fixups.
*
* Only the record structure is established: attributes, their names, the
* filenames and the USN journal are left to later steps, so no path, timestamp
* or file identity beyond the record itself is claimed. A record whose fixup
* does not match keeps its header facts, because the stream still declares it,
* and the mismatch is reported on the record and counted in the metadata.
*/
export const parseMftImage = (
bytes: Uint8Array,
): Result<MftImage, RegionDecodeFailure> => {
const records: MftRecord[] = [];
const inventory = scanMft(bytes, records);
return inventory.ok ? ok({ ...inventory.value, records }) : inventory;
};
const scanMft = (
bytes: Uint8Array,
retainedRecords?: MftRecord[],
): Result<MftInventory, RegionDecodeFailure> => {
if (bytes.length < NTFS_MFT_MIN_RECORD_BYTES)
return err(
failure(
`file is ${String(bytes.length)} bytes, shorter than one ${String(NTFS_MFT_MIN_RECORD_BYTES)}-byte record header`,
),
);
if (!hasMftFileRecordSignature(bytes))
return err(failure("record 0 does not begin with the FILE signature"));
const limitations: string[] = [];
const header = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
const declaredRecordBytes = header.getUint32(RECORD_HEADER.totalBytes, true);
const recordBytes = isRecordSize(declaredRecordBytes)
? declaredRecordBytes
: MFT_DEFAULT_RECORD_BYTES;
if (!isRecordSize(declaredRecordBytes))
limitations.push(
`record 0 declares a record size of ${String(declaredRecordBytes)} bytes, which is not a supported power of two from ${String(MFT_MIN_RECORD_BYTES)} to ${String(MFT_MAX_RECORD_BYTES)}; records were read at the ${String(MFT_DEFAULT_RECORD_BYTES)}-byte default`,
);
const recordHeaders: MftRecord[] = [];
let recordCount = 0;
let inUseCount = 0;
let directoryCount = 0;
let extensionCount = 0;
let mismatchCount = 0;
let unverifiedCount = 0;
let invalidSlots = 0;
let invalidExtents = 0;
let inconsistentRecordNumbers = 0;
let firstInvalidExtentOffset: number | undefined;
let firstInvalidOffset: number | undefined;
for (
let offset = 0;
offset + recordBytes <= bytes.length;
offset += recordBytes
) {
if (
!hasMftFileRecordSignature(
bytes.subarray(offset, offset + MFT_FILE_RECORD_MAGIC_BYTES),
)
) {
invalidSlots += 1;
firstInvalidOffset ??= offset;
if (!isRecordSize(declaredRecordBytes)) break;
continue;
}
const record = readRecord(bytes, offset, offset / recordBytes, recordBytes);
retainedRecords?.push(record);
if (recordHeaders.length < MFT_INLINE_HEADER_LIMIT)
recordHeaders.push(record);
recordCount += 1;
if (record.inUse) inUseCount += 1;
if (record.directory) directoryCount += 1;
if (record.baseRecordReference !== null) extensionCount += 1;
if (record.fixups.mismatchSector !== null) mismatchCount += 1;
if (
record.fixups.malformed ||
record.fixups.coveredSectors < recordBytes / FIXUP_SECTOR_BYTES
)
unverifiedCount += 1;
if (record.recordNumberMatchesSlot === false)
inconsistentRecordNumbers += 1;
if (!record.extentValid) {
invalidExtents += 1;
firstInvalidExtentOffset ??= offset;
}
}
if (firstInvalidOffset !== undefined)
limitations.push(
`record at ${String(firstInvalidOffset)} does not begin with the FILE signature; ${String(invalidSlots)} slot(s) without a FILE signature were found`,
);
if (firstInvalidExtentOffset !== undefined)
limitations.push(
`${String(invalidExtents)} FILE record(s) declare inconsistent record extents, beginning at offset ${String(firstInvalidExtentOffset)}`,
);
if (inconsistentRecordNumbers !== 0)
limitations.push(
`${String(inconsistentRecordNumbers)} FILE record(s) declare a record number that disagrees with their stream slot`,
);
const truncatedTailBytes = bytes.length % recordBytes;
if (truncatedTailBytes !== 0)
limitations.push(
`${String(truncatedTailBytes)} trailing byte(s) do not fill a ${String(recordBytes)}-byte record and were not parsed`,
);
if (mismatchCount !== 0)
limitations.push(
`${String(mismatchCount)} record(s) hold a sector trailer that is not the update sequence number; their attributes were not read and no later step should trust them`,
);
if (unverifiedCount !== 0)
limitations.push(
`${String(unverifiedCount)} record(s) carry an invalid or incomplete update sequence array; their sector trailers were not fully verified`,
);
const metadata: MftMetadata = {
kind: "ntfs-mft",
record_bytes: recordBytes,
record_count: recordCount,
in_use_record_count: inUseCount,
directory_record_count: directoryCount,
extension_record_count: extensionCount,
fixup_mismatch_count: mismatchCount,
fixup_unverified_record_count: unverifiedCount,
truncated_tail_bytes: truncatedTailBytes,
record_headers: recordHeaders,
omitted_record_headers: recordCount - recordHeaders.length,
};
return ok({ metadata, recordBytes, limitations });
};
const readRecord = (
bytes: Uint8Array,
fileOffset: number,
index: number,
recordBytes: number,
): MftRecord => {
const record = bytes.subarray(fileOffset, fileOffset + recordBytes);
const view = new DataView(
record.buffer,
record.byteOffset,
record.byteLength,
);
const flags = view.getUint16(RECORD_HEADER.flags, true);
const baseRecordReference = view.getBigUint64(
RECORD_HEADER.baseRecordReference,
true,
);
const usaOffset = view.getUint16(RECORD_HEADER.updateSequenceOffset, true);
const fixups = verifyFixups(record, view, usaOffset);
const usedBytes = view.getUint32(RECORD_HEADER.usedBytes, true);
const totalBytes = view.getUint32(RECORD_HEADER.totalBytes, true);
const firstAttributeOffset = view.getUint16(
RECORD_HEADER.firstAttributeOffset,
true,
);
const declaredRecordNumber =
!fixups.malformed && usaOffset >= RECORD_HEADER_WITH_NUMBER_BYTES
? view.getUint32(RECORD_HEADER.recordNumber, true)
: null;
return {
index,
fileOffset,
sequenceNumber: view.getUint16(RECORD_HEADER.sequenceNumber, true),
hardLinkCount: view.getUint16(RECORD_HEADER.hardLinkCount, true),
flags: parseRecordFlags(flags),
inUse: (flags & FILE_RECORD_IN_USE) !== 0,
directory: (flags & FILE_RECORD_IS_DIRECTORY) !== 0,
usedBytes,
totalBytes,
extentValid:
totalBytes === recordBytes &&
usedBytes >= NTFS_MFT_MIN_RECORD_BYTES &&
usedBytes % 8 === 0 &&
usedBytes <= totalBytes &&
firstAttributeOffset % 8 === 0 &&
firstAttributeOffset >= NTFS_MFT_MIN_RECORD_BYTES &&
firstAttributeOffset + 4 <= usedBytes &&
(fixups.malformed ||
firstAttributeOffset >= usaOffset + fixups.entries * 2),
baseRecordReference:
baseRecordReference === 0n
? null
: decodeMftFileReference(baseRecordReference),
declaredRecordNumber,
recordNumberMatchesSlot:
declaredRecordNumber === null ? null : declaredRecordNumber === index,
logFileSequenceNumber: view
.getBigUint64(RECORD_HEADER.logFileSequenceNumber, true)
.toString(),
firstAttributeOffset,
nextAttributeId: view.getUint16(RECORD_HEADER.nextAttributeId, true),
fixups,
};
};
/**
* Each sector's last two bytes are replaced on disk by the update sequence
* number; the array's first entry is that number and the rest are the saved
* trailer words. A sector whose trailer is not the sequence number was not
* cleanly written, so the record's attributes cannot be trusted.
*/
const verifyFixups = (
record: Uint8Array,
view: DataView,
usaOffset: number,
): MftRecordFixups => {
const usaCount = view.getUint16(RECORD_HEADER.updateSequenceCount, true);
const sectors = Math.floor(record.length / FIXUP_SECTOR_BYTES);
// Entry 0 holds the sequence number, so sector n's saved trailer is entry n+1.
const coveredSectors = Math.max(0, usaCount - 1);
if (!updateSequenceArrayFits(usaOffset, usaCount, sectors))
return {
sequenceNumber: 0,
entries: usaCount,
verifiedSectors: 0,
coveredSectors: 0,
mismatchSector: null,
malformed: true,
};
const sequenceNumber = view.getUint16(usaOffset, true);
if (sequenceNumber === 0 || sequenceNumber === 0xffff)
return {
sequenceNumber,
entries: usaCount,
verifiedSectors: 0,
coveredSectors,
mismatchSector: null,
malformed: true,
};
for (let sector = 0; sector < coveredSectors; sector += 1) {
const trailerOffset = sector * FIXUP_SECTOR_BYTES + FIXUP_SECTOR_BYTES - 2;
if (view.getUint16(trailerOffset, true) !== sequenceNumber)
return {
sequenceNumber,
entries: usaCount,
verifiedSectors: sector,
coveredSectors,
mismatchSector: sector,
malformed: false,
};
}
return {
sequenceNumber,
entries: usaCount,
verifiedSectors: coveredSectors,
coveredSectors,
mismatchSector: null,
malformed: false,
};
};
/** Buffer the admitted input once while accepting arbitrary chunk boundaries. */
class StreamingMftDecoder implements RegionDecoder<MftInventory> {
readonly #bytes: Uint8Array;
#written = 0;
constructor(fileBytes: number) {
this.#bytes = new Uint8Array(fileBytes);
}
write(chunk: Uint8Array): Result<void, RegionDecodeFailure> {
if (chunk.length > this.#bytes.length - this.#written)
return err(failure("received more bytes than the admitted file length"));
this.#bytes.set(chunk, this.#written);
this.#written += chunk.length;
return ok(undefined);
}
finish(): Result<MftInventory, RegionDecodeFailure> {
if (this.#written !== this.#bytes.length)
return err(failure("stream ended before the admitted file length"));
return scanMft(this.#bytes);
}
}
export const createMftDecoder = (
fileBytes: number,
): Result<RegionDecoder<MftInventory>, RegionDecodeFailure> =>
Number.isSafeInteger(fileBytes) &&
fileBytes >= 0 &&
fileBytes <= MFT_MAX_FILE_BYTES
? ok(new StreamingMftDecoder(fileBytes))
: err(
limit(
`file is ${String(fileBytes)} bytes, exceeding the fixed ${String(MFT_MAX_FILE_BYTES)}-byte (512 MiB) ceiling`,
"file-size",
),
);
+20
View File
@@ -1,3 +1,4 @@
import { readFile } from "node:fs/promises";
import { describe, expect, it } from "vitest";
import type { JsonValue } from "../domain/jsonValue.js";
@@ -1119,6 +1120,25 @@ describe("Ghidra native instruction decoder context for ARM TMode and MIPS ISA_M
});
});
describe("Ghidra bridge script processor context contract", () => {
it("binds TMode to ARM and ISA_MODE to MIPS in ReaGhidraBridge.java", async () => {
const bridgePath = new URL(
"../../bridge/ghidra/ReaGhidraBridge.java",
import.meta.url,
);
const content = await readFile(bridgePath, "utf8");
expect(content).toContain(
'currentProgram.getLanguageID().getIdAsString().startsWith("ARM:")',
);
expect(content).toContain(
'currentProgram.getLanguageID().getIdAsString().startsWith("MIPS:")',
);
expect(content).toContain('"TMode"');
expect(content).toContain('"ISA_MODE"');
expect(content).toContain('"ghidra-program-context"');
});
});
describe("Ghidra dossier nested boundary composition", () => {
const identityLocations = [
"caller",
+1 -1
View File
@@ -43,7 +43,7 @@ const sectionBytes = (
section: NativeSection,
budget?: CudaMetadataBudget,
): Promise<Buffer> => {
if (!section.file_backed || section.compressed)
if (!section.file_backed || (section.flags & 0x800n) !== 0n)
rejectGpu("GPU metadata section has no supported uncompressed file range");
budget?.read(gpuCount(section.size));
return source.read(gpuCount(section.offset), gpuCount(section.size));
+4 -1
View File
@@ -194,7 +194,10 @@ export class GpuInventory {
continue;
}
if (!isGpuSection(section.name)) continue;
if (!section.file_backed || section.compressed) {
if (
!section.file_backed ||
(directory.format === "elf" && (section.flags & 0x800n) !== 0n)
) {
this.discovery = "partial";
const message = `GPU section ${section.index} has no supported uncompressed file range`;
this.#retain(message);
+5 -1
View File
@@ -340,7 +340,11 @@ export const inspectAndroidJniLibrary = async (
for (const section of directory.sections) {
if ((section.flags & 3n) !== 3n || section.size === 0n) continue;
active();
if (section.compressed || !section.file_backed || section.type === 8) {
if (
(section.flags & 0x800n) !== 0n ||
!section.file_backed ||
section.type === 8
) {
tablesStatus = "partial";
unknowns.add(
"Allocated writable sections without ordinary file bytes were not scanned; runtime-built tables remain unknown",
-55
View File
@@ -462,33 +462,6 @@ const findSection = (bytes: Buffer, name: string) => {
throw new Error(`fixture has no ${name}`);
};
it.each(["compressed-symbols", "compressed-names", "non-string-names"])(
"retains unwind ranges without interpreting %s as ordinary symbol storage",
(storage) => {
const bytes = elfUnwindFixture({ header: "match" });
expect(parse(bytes).ranges[0]?.symbol?.text).toBe("alpha");
const section = findSection(
bytes,
storage === "compressed-symbols" ? ".symtab" : ".strtab",
);
if (storage === "non-string-names")
bytes.writeUInt32LE(1, section.headerOffset + 4);
else bytes.writeBigUInt64LE(0x800n, section.headerOffset + 8);
const report = parse(bytes);
expect(report.ranges.map(({ start }) => start)).toEqual([
address(UNWIND_FN_A),
address(UNWIND_FN_B),
]);
expect(report.ranges.map(({ symbol }) => symbol)).toEqual([null, null]);
expect(report.coverage).toMatchObject({
status: "incomplete",
ranges_with_symbols: 0,
eh_frame_hdr_agreement: "agree",
});
},
);
it("keeps .eh_frame ranges when .debug_frame is SHT_NOBITS", () => {
const report = parse(
elfUnwindFixture({
@@ -618,18 +591,6 @@ it("caps symbol lookup across .symtab and .dynsym together", () => {
);
});
it("keeps the symbol scan allowance for later tables when an earlier table has no string table", () => {
const bytes = elfUnwindFixture({ header: "match", symbolCap: true });
const section = findSection(bytes, ".symtab");
bytes.writeUInt32LE(0, section.headerOffset + 40);
const report = parse(bytes);
expect(report.ranges[0]?.symbol?.text).toBe("alpha");
expect(report.coverage.status).toBe("incomplete");
expect(report.limitations).not.toContain(
"Symbol tables were capped at 65536 combined entries.",
);
});
it.each([".eh_frame", ".eh_frame_hdr"] as const)(
"reports coverage incomplete when a duplicate %s is ignored",
(duplicate) => {
@@ -704,22 +665,6 @@ it.each([
},
);
it("keeps the GOT unknown when compressed dynamic storage resembles decoded entries", () => {
const bytes = elfUnwindFixture({ header: "match", datarel: "dynamic" });
const section = findSection(bytes, ".dynamic");
// The payload still resembles ordinary DT_PLTGOT/DT_NULL entries, but the
// declaration makes it opaque compressed storage, not a decoded table.
bytes.writeBigUInt64LE(0x800n, section.headerOffset + 8);
const report = parse(bytes);
expect(report.ranges.map((range) => range.start)).not.toContain(
address(UNWIND_FN_A),
);
expect(report.coverage.status).toBe("incomplete");
expect(report.limitations).toContain(
"A DW_EH_PE_datarel .eh_frame value needs a GOT base that could not be determined; that value is unresolved.",
);
});
it("caps .eh_frame_hdr rows at the max_ranges budget and marks the comparison incomplete", () => {
const report = parse(elfUnwindFixture({ header: "duplicate-row" }), {
max_ranges: 2,
+14 -27
View File
@@ -34,7 +34,6 @@ const AUG_R = 0x52;
const AUG_S = 0x53;
const AUG_B = 0x42;
const SHT_SYMTAB = 2;
const SHT_STRTAB = 3;
const SHT_DYNAMIC = 6;
const SHT_NOBITS = 8;
const SHT_DYNSYM = 11;
@@ -50,6 +49,7 @@ const SHN_ABS = 0xfff1;
const SHN_LORESERVE = 0xff00;
const SHF_ALLOC = 2n;
const SHF_EXECINSTR = 4n;
const SHF_COMPRESSED = 0x800n;
const {
cfa: CFA_LIMITATION,
@@ -639,7 +639,7 @@ export const parseElfUnwindRanges = (
? ".eh_frame_hdr is declared but not file-backed."
: ".eh_frame_hdr extends past the end of the file and was not decoded.",
);
} else if (hdrSection.compressed) {
} else if ((hdrSection.flags & SHF_COMPRESSED) !== 0n) {
headerLocation = placement.location;
headerStatus = "incomplete";
fail(".eh_frame_hdr is SHF_COMPRESSED and was not decompressed.");
@@ -705,7 +705,7 @@ export const parseElfUnwindRanges = (
} else if (frameLocation === null) {
frameStatus = "incomplete";
fail(".eh_frame lies outside the file.");
} else if (ehSection.compressed) {
} else if ((ehSection.flags & SHF_COMPRESSED) !== 0n) {
frameStatus = "incomplete";
fail(".eh_frame is SHF_COMPRESSED and was not decompressed.");
} else {
@@ -1135,12 +1135,7 @@ const resolveGotBase = (
? undefined
: { offset: dynamicProgram.offset, fileSize: dynamicProgram.size }) ??
directory.sections
.filter(
(section) =>
section.type === SHT_DYNAMIC &&
section.file_backed &&
!section.compressed,
)
.filter((section) => section.type === SHT_DYNAMIC && section.file_backed)
.map((section) => ({
offset: section.offset,
fileSize: section.size,
@@ -1819,10 +1814,6 @@ const scanSymbols = (
let remaining = MAX_ELF_UNWIND_SYMBOLS;
for (const table of tables) {
assertArtifactReadActive(signal);
if (table.compressed) {
fail("A symbol table is SHF_COMPRESSED and was not decompressed.");
continue;
}
const width = table.entry_size === 0n ? standard : Number(table.entry_size);
if (width !== standard) {
fail(
@@ -1838,15 +1829,6 @@ const scanSymbols = (
const size = Number(table.size);
if (size % width !== 0)
fail("A symbol table size is not a multiple of its entry width.");
const strings = byIndex.get(table.link);
if (strings?.type !== SHT_STRTAB || !strings.file_backed) {
fail("A symbol table does not link to a file-backed SHT_STRTAB section.");
continue;
}
if (strings.compressed) {
fail("A symbol string table is SHF_COMPRESSED and was not decompressed.");
continue;
}
let count = Math.floor(size / width);
if (count > remaining) {
fail(
@@ -1855,10 +1837,15 @@ const scanSymbols = (
count = remaining;
}
remaining -= count;
const stringBytes = bytes.subarray(
Number(strings.offset),
Number(strings.offset + strings.size),
);
const strings = byIndex.get(table.link);
const stringBytes =
strings !== undefined && strings.file_backed
? bytes.subarray(
Number(strings.offset),
Number(strings.offset + strings.size),
)
: null;
if (stringBytes === null) fail("A symbol string table could not be read.");
// Observation per string-table offset; null when the name is unreadable.
const names = new Map<number, SymbolName | null>();
for (let index = 0; index < count; index += 1) {
@@ -1891,7 +1878,7 @@ const scanSymbols = (
if (owner !== undefined) address += owner.address;
}
// Only a symbol at a decoded FDE start can name a range.
if (!wanted.has(address)) continue;
if (stringBytes === null || !wanted.has(address)) continue;
let name = names.get(symbol.name_offset);
if (name === undefined) {
const observed = observeElfStringTableEntry(
+7 -177
View File
@@ -1,5 +1,4 @@
import { describe, expect, it, vi } from "vitest";
import { deflateSync } from "node:zlib";
import {
elfExportImage,
@@ -23,172 +22,6 @@ const parse = (bytes: Buffer) =>
parseNativeExports(bytes, artifact),
);
/** Encode real gABI compressed storage, retaining logical section links. */
const compressExportSection = (
bytes: Buffer,
sectionIndex: number,
versionStrings = false,
): Buffer => {
const wide = bytes[4] === 2;
const little = bytes[5] === 1;
const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
const word = (at: number) =>
Number(wide ? view.getBigUint64(at, little) : view.getUint32(at, little));
const table = word(wide ? 40 : 32);
const width = wide ? 64 : 40;
const section = table + sectionIndex * width;
const offset = word(section + (wide ? 24 : 16));
const size = word(section + (wide ? 32 : 20));
const header = Buffer.alloc(wide ? 24 : 12);
const chdr = new DataView(
header.buffer,
header.byteOffset,
header.byteLength,
);
chdr.setUint32(0, 1, little); // ELFCOMPRESS_ZLIB
if (wide) {
chdr.setBigUint64(8, BigInt(size), little);
chdr.setBigUint64(16, 1n, little);
} else {
chdr.setUint32(4, size, little);
chdr.setUint32(8, 1, little);
}
const storage = Buffer.concat([
header,
deflateSync(bytes.subarray(offset, offset + size)),
]);
const count = view.getUint16(wide ? 60 : 48, little);
const result = Buffer.concat([
bytes,
Buffer.alloc(versionStrings ? width : 0),
storage,
]);
const updated = new DataView(
result.buffer,
result.byteOffset,
result.byteLength,
);
const setWord = (at: number, value: number) =>
wide
? updated.setBigUint64(at, BigInt(value), little)
: updated.setUint32(at, value, little);
const target = versionStrings ? table + count * width : section;
if (versionStrings) {
bytes.copy(result, target, section, section + width);
updated.setUint16(wide ? 60 : 48, count + 1, little);
// Definitions and requirements use the copied string table; dynsym keeps
// its original uncompressed table and still supplies trustworthy names.
for (const index of [4, 5])
updated.setUint32(
table + index * width + (wide ? 40 : 24),
count,
little,
);
}
setWord(target + 8, 0x800); // SHF_COMPRESSED, without SHF_ALLOC
setWord(
target + (wide ? 24 : 16),
bytes.length + (versionStrings ? width : 0),
);
setWord(target + (wide ? 32 : 20), storage.length);
return result;
};
it.each(
([32, 64] as const).flatMap((bits) =>
(
[
"symbols",
"strings",
"indices",
"definitions",
"requirements",
"version-strings",
] as const
).map((kind) => ({ bits, kind })),
),
)(
"keeps compressed ELF$bits $kind distinct from logical table bytes",
({ bits, kind }) => {
for (const little of [false, true]) {
const bytes = elfExportImage({
bits,
little,
symbols: [
{ name: "defined", version: 2 },
{ name: "required", version: 3 },
],
versionDefinitions: ["lib.so", "DEFINED"],
versionRequirements: [{ index: 3, name: "REQUIRED" }],
});
expect(parse(bytes)).toMatchObject({
coverage: { status: "complete", exports: 2 },
});
const index = {
symbols: 1,
strings: 2,
indices: 3,
definitions: 4,
requirements: 5,
"version-strings": 2,
}[kind];
const result = parse(
compressExportSection(bytes, index, kind === "version-strings"),
);
if (kind === "symbols" || kind === "strings") {
expect(result).toMatchObject({
exports: [],
coverage: {
status: "unsupported",
examined_entries: 0,
reason: expect.stringContaining("SHF_COMPRESSED"),
},
});
} else {
expect(result).toMatchObject({
coverage: { status: "partial", exports: 2 },
exports: [
{
name: "defined",
version:
kind === "indices"
? "unknown"
: {
index: 2,
name:
kind === "definitions" || kind === "version-strings"
? null
: "DEFINED",
},
},
{
name: "required",
version:
kind === "indices"
? "unknown"
: {
index: 3,
name:
kind === "definitions" ||
kind === "requirements" ||
kind === "version-strings"
? null
: "REQUIRED",
},
},
],
});
expect(result.issues).toContainEqual(
expect.objectContaining({
code: "version_unresolved",
message: expect.stringContaining("SHF_COMPRESSED"),
}),
);
}
}
},
);
it.each(["requirements", "definitions"] as const)(
"bounds aliased GNU version %s work and keeps exports with unknown versions",
(kind) => {
@@ -873,12 +706,13 @@ describe("ELF version requirements", () => {
});
it("uses a definition name when a requirement shares its index", () => {
const bytes = elfExportImage({
symbols: [{ name: "ambiguous", version: 2 }],
versionDefinitions: ["libfixture.so", "DEFINITION"],
versionRequirements: [{ index: 2, name: "REQUIREMENT" }],
});
const result = parse(bytes);
const result = parse(
elfExportImage({
symbols: [{ name: "ambiguous", version: 2 }],
versionDefinitions: ["libfixture.so", "DEFINITION"],
versionRequirements: [{ index: 2, name: "REQUIREMENT" }],
}),
);
expect(result.exports).toMatchObject([
{
@@ -888,10 +722,6 @@ describe("ELF version requirements", () => {
]);
expect(result.coverage.status).toBe("complete");
expect(result.issues).toEqual([]);
expect(parse(compressExportSection(bytes, 4))).toMatchObject({
exports: [{ name: "ambiguous", version: { index: 2, name: null } }],
coverage: { status: "partial" },
});
});
it("retains exports and reports a truncated version-requirement chain", () => {
+1 -48
View File
@@ -294,22 +294,12 @@ const versionSectionData = (
table: { readonly offset: number; readonly size: number };
names: Buffer;
} => {
if (section.compressed)
throw new BinaryFormatFailure(
"unsupported",
`${label} are SHF_COMPRESSED and were not decompressed`,
);
const strings = sections[section.link];
if (strings?.type !== SHT_STRTAB)
throw new BinaryFormatFailure(
"malformed",
`${label} do not link to a string table`,
);
if (strings.compressed)
throw new BinaryFormatFailure(
"unsupported",
`${label} strings are SHF_COMPRESSED and were not decompressed`,
);
const stringRange = reader.fileRange(
strings.offset,
strings.size,
@@ -555,13 +545,7 @@ function* readVersionNames(
}
}
} catch (cause) {
// An unread compressed definition table may replace any requirement
// index. Its absence cannot justify falling back to requirement names.
if (
cause instanceof ElfVersionWorkFailure ||
(cause instanceof BinaryFormatFailure && cause.kind === "unsupported")
)
namesByIndex.clear();
if (cause instanceof ElfVersionWorkFailure) namesByIndex.clear();
issues.push({
code: "version_unresolved",
location: hex(verdef.offset),
@@ -621,32 +605,6 @@ function* parseElfExports(
};
const width = directory.bits === 64 ? 24 : 16;
const strings = sections[dynsym.link];
const compressed = dynsym.compressed
? dynsym
: strings?.type === SHT_STRTAB && strings.compressed
? strings
: null;
if (compressed !== null) {
const reason = `The ELF ${compressed === dynsym ? "dynamic symbol table" : "dynamic string table"} is SHF_COMPRESSED and was not decompressed; export identities remain unknown`;
return {
...result,
exports: [],
coverage: {
status: "unsupported",
reason,
examined_entries: 0,
exports: 0,
},
issues: [
{
code: "table_unavailable",
location: hex(compressed.offset),
message: reason,
},
],
limitations: ELF_LIMITATIONS,
};
}
if (
dynsym.entry_size !== BigInt(width) ||
dynsym.size % BigInt(width) !== 0n ||
@@ -689,11 +647,6 @@ function* parseElfExports(
let versions: number | null = null;
if (versym !== undefined) {
try {
if (versym.compressed)
throw new BinaryFormatFailure(
"unsupported",
"ELF symbol versions are SHF_COMPRESSED and were not decompressed",
);
if (versym.size !== BigInt(count * 2))
throw new BinaryFormatFailure(
"malformed",
@@ -1405,40 +1405,13 @@ it("sizes an empty DT_GNU_HASH table only from an agreeing SHT_DYNSYM", () => {
strings.readBigUInt64LE(elfSectionAt(strings, SHT_STRTAB) + 32) - 1n,
elfSectionAt(strings, SHT_STRTAB) + 32,
);
const compressedSymbols = Buffer.from(agreeing);
compressedSymbols.writeBigUInt64LE(0x800n, section + 8);
const compressedStrings = Buffer.from(agreeing);
compressedStrings.writeBigUInt64LE(
0x800n,
elfSectionAt(compressedStrings, SHT_STRTAB) + 8,
);
const wrongStringType = Buffer.from(agreeing);
wrongStringType.writeUInt32LE(
1,
elfSectionAt(wrongStringType, SHT_STRTAB) + 4,
);
for (const bytes of [
moved,
offset,
strings,
compressedSymbols,
compressedStrings,
wrongStringType,
])
for (const bytes of [moved, offset, strings])
expect(
findRow(readElfHardening(bytes).rows, "stack_protector"),
).toMatchObject({
state: "unknown",
note: expect.stringContaining("DT_GNU_HASH"),
});
// A loader-owned hash count needs no section fallback, so compressed section
// declarations cannot hide imports named by DT_SYMTAB and DT_STRTAB.
const hashed = elfHardeningFixture({ stackChk: true });
for (const type of [SHT_DYNSYM, SHT_STRTAB])
hashed.writeBigUInt64LE(0x800n, elfSectionAt(hashed, type) + 8);
expect(findRow(readElfHardening(hashed).rows, "stack_protector").state).toBe(
"enabled",
);
});
it("keeps an unresolved symbol count distinct from an absent table", () => {
+8 -10
View File
@@ -51,7 +51,6 @@ import {
const ET_EXEC = 2;
const ET_DYN = 3;
const SHT_SYMTAB = 2;
const SHT_STRTAB = 3;
const SHT_NOTE = 7;
const SHT_DYNSYM = 11;
const DT_NULL = 0n;
@@ -72,6 +71,7 @@ const DF_1_PIE = 0x08000000n;
const PF_X = 1;
const GNU_PROPERTY_AARCH64_FEATURE_1_AND = 0xc0000000;
const GNU_PROPERTY_X86_FEATURE_1_AND = 0xc0000002;
const SHF_COMPRESSED = 0x800n;
/** Tags hardening reads as a single value. Repeating tags such as DT_NEEDED stay. */
const SINGULAR_DYNAMIC_TAGS = new Set<bigint>([
DT_HASH,
@@ -175,6 +175,7 @@ export const readElfHardening = (
assertElfDeclaredRangesFromBuffer(bytes);
const { bits, little, machine, programs, sections } = directory;
const headerWidth = bits === 64 ? 56 : 32;
const symbolWidth = bits === 64 ? 24 : 16;
const dynamicWidth = bits === 64 ? 16 : 8;
const propertyAlign = bits === 64 ? 8 : 4;
const loads = programs.filter(isSpan).filter((program) => program.type === 1);
@@ -545,14 +546,10 @@ const agreeingSectionCount = (
if (dynsym === undefined) return null;
const strings = sections[dynsym.link];
if (
dynsym.compressed ||
strings === undefined ||
strings.compressed ||
strings.type !== SHT_STRTAB ||
dynsym.address !== symtab.value ||
strings.address !== dynamic.entries.get(DT_STRTAB)?.value ||
strings.size !== dynamic.entries.get(DT_STRSZ)?.value ||
strings.offset !== BigInt(dynamic.stringFileOffset ?? -1)
strings?.address !== dynamic.entries.get(DT_STRTAB)?.value ||
strings?.size !== dynamic.entries.get(DT_STRSZ)?.value ||
strings?.offset !== BigInt(dynamic.stringFileOffset ?? -1)
)
return null;
const entry = dynsym.entry_size === 0n ? BigInt(width) : dynsym.entry_size;
@@ -900,10 +897,11 @@ const readFeatureNotes = (
if (section.type !== SHT_NOTE) continue;
// An empty note section owns no bytes, so it has nothing to walk.
if (section.size === 0n) continue;
if (section.compressed)
const compressed = (section.flags & SHF_COMPRESSED) !== 0n;
if (compressed)
incomplete ??=
"A compressed SHT_NOTE section may hold GNU property notes, and its descriptors were not read.";
if (section.compressed) {
if (compressed) {
if (!section.file_backed) continue;
otherSpans.push({
offset: exact(section.offset, "SHT_NOTE offset"),
@@ -124,17 +124,11 @@ const stringAt = (
return { name: text, missing: false };
};
const stringTableBytes = (
const tableBytes = (
bytes: Buffer,
section: NativeSection | undefined,
): Buffer | null => {
if (
section === undefined ||
section.type !== 3 ||
!section.file_backed ||
section.compressed
)
return null;
if (section === undefined || !section.file_backed) return null;
const offset = Number(section.offset);
const size = Number(section.size);
if (offset < 0 || size < 0 || offset > bytes.length - size) return null;
@@ -150,19 +144,9 @@ const readTable = (
): {
readonly symbols: ElfInitializerSymbol[];
readonly truncated: boolean;
readonly incomplete: boolean;
} => {
const width = facts.bits === 64 ? 24 : 16;
const strings = stringTableBytes(facts.bytes, sections[section.link]);
const compressedStrings = sections[section.link]?.compressed === true;
if (compressedStrings)
notes.add(
"A symbol string table is compressed; its names were not decoded.",
);
else if (strings === null)
notes.add(
"A symbol table does not link a readable SHT_STRTAB section; its names were not decoded.",
);
const strings = tableBytes(facts.bytes, sections[section.link]);
const start = Number(section.offset);
const declared = section.size / BigInt(width);
const count = Number(declared < BigInt(budget) ? declared : BigInt(budget));
@@ -193,7 +177,7 @@ const readTable = (
type: parsed.value.type,
});
}
return { symbols, truncated, incomplete: nameMissing || strings === null };
return { symbols, truncated: truncated || nameMissing };
};
/** Load SHT_SYMTAB and SHT_DYNSYM once. Defined STT_FUNC names win ties. */
@@ -239,20 +223,10 @@ export const loadElfSymbolIndex = (
sections.forEach((section, index) => {
if (truncated) return;
if (section.type !== 2 && section.type !== 11) return;
if (section.compressed) {
notes.add("A symbol table is compressed; its symbols were not decoded.");
unreadable = true;
return;
}
const width = BigInt(facts.bits === 64 ? 24 : 16);
if (section.entry_size !== 0n && section.entry_size !== width) {
notes.add(
"A symbol table has an unexpected entry size; its symbols were not decoded.",
);
unreadable = true;
return;
}
if (!section.file_backed || section.size % width !== 0n) {
if (
!section.file_backed ||
section.size % BigInt(facts.bits === 64 ? 24 : 16) !== 0n
) {
notes.add(
"A symbol table is not stored in the file or its size is not a whole number of entries, so its names were not read.",
);
@@ -263,7 +237,6 @@ export const loadElfSymbolIndex = (
budget -= table.symbols.length;
tables.set(index, table.symbols);
truncated ||= table.truncated;
unreadable ||= table.incomplete;
for (const symbol of table.symbols) consider(symbol);
});
return {
@@ -467,11 +440,7 @@ export const sectionRelocations = async (
for (const [position, section] of input.sections.entries()) {
const encoding = applicable(input, section);
if (encoding === null) continue;
if (!section.file_backed || section.compressed) {
if (section.compressed)
input.notes.add(
"A relocation section is compressed; its records were not decoded.",
);
if (!section.file_backed) {
skip(section);
continue;
}
+3 -17
View File
@@ -269,7 +269,7 @@ const indexSections = (sections: readonly NativeSection[]): ElfSectionIndex => {
: left.index - right.index,
);
const backed = buildAddressRangeIndex(
ranges((section) => section.file_backed && !section.compressed),
ranges((section) => section.file_backed),
(left, right) => left.index - right.index,
);
return {
@@ -354,8 +354,7 @@ const findSections = (
const withinSection =
(section: NativeSection, width: number) =>
(address: bigint): number | null => {
if (!section.file_backed || section.compressed || address < section.address)
return null;
if (!section.file_backed || address < section.address) return null;
const delta = address - section.address;
if (delta + BigInt(width) > section.size) return null;
return Number(section.offset + delta);
@@ -375,13 +374,6 @@ const sectionSource = (
section.name.length <= MAX_NATIVE_INITIALIZER_NAME_BYTES
? section.name
: fallbackName;
if (section.compressed) {
note(
scan,
`${label} is compressed. Its initializer slots were not decoded.`,
);
return { kind, sentinel, detail: label, slots: [] };
}
if (!section.file_backed) {
note(scan, `${label} is not file-backed. Its slots were omitted.`);
return { kind, sentinel, detail: label, slots: [] };
@@ -679,7 +671,6 @@ const loadWrites = async (
};
const dynamicSymbols = (
scan: ElfScan,
sections: readonly NativeSection[],
tags: ReadonlyMap<bigint, DynTag>,
tables: ReadonlyMap<number, readonly ElfInitializerSymbol[]>,
@@ -692,11 +683,6 @@ const dynamicSymbols = (
(section.type === 11 || section.type === 2),
);
if (match >= 0) return tables.get(match);
note(
scan,
"DT_SYMTAB does not identify a decoded symbol table; dynamic relocation symbols are unresolved.",
);
return undefined;
}
const dynsym = sections.findIndex((section) => section.type === 11);
return dynsym < 0 ? undefined : tables.get(dynsym);
@@ -1056,7 +1042,7 @@ export const readElfInitializers = async (
loaded,
sectionRels.found,
sectionRels.clobbered,
dynamicSymbols(scan, directory.sections, tags, symbols.tables),
dynamicSymbols(directory.sections, tags, symbols.tables),
symbols,
loaded.trustRaw && sectionRels.complete,
directory.sections,
@@ -1,5 +1,4 @@
import { describe, expect, it, vi } from "vitest";
import { deflateSync } from "node:zlib";
import { readNativeInitializers } from "./NativeInitializersProvider.js";
/** Pass-through spies on the shared mapping module, so tests can count lookups made from other modules. */
@@ -321,106 +320,6 @@ describe("ELF slot semantics", () => {
});
});
describe("ELF section content admission", () => {
it("keeps symbol relocations unresolved for a conflicting symbol entry width", async () => {
const bytes = elfRelocatableCtors();
bytes.writeBigUInt64LE(16n, 0x500 + 4 * 64 + 56);
const report = await read(bytes);
expect(
report.entries.find((entry) => entry.kind === "init_array"),
).toMatchObject({
resolution: "unresolved",
value_va: null,
symbol: null,
});
expect(report.status).toBe("incomplete");
expect(report.limitations.join("\n")).toContain("entry size");
});
it.each([
{ section: 7, label: "initializer array" },
{ section: 4, label: "symbol table" },
{ section: 5, label: "symbol string table" },
{ section: 8, label: "relocation table" },
])("preserves uncertainty for a compressed $label", async ({ section }) => {
const base = elfRelocatableCtors();
const at = 0x500 + section * 64;
const offset = Number(base.readBigUInt64LE(at + 24));
const size = Number(base.readBigUInt64LE(at + 32));
const header = Buffer.alloc(24);
header.writeUInt32LE(1, 0);
header.writeBigUInt64LE(BigInt(size), 8);
header.writeBigUInt64LE(1n, 16);
const compressed = Buffer.concat([
header,
deflateSync(base.subarray(offset, offset + size)),
]);
const bytes = Buffer.concat([base, compressed]);
bytes.writeBigUInt64LE(base.readBigUInt64LE(at + 8) | 0x800n, at + 8);
bytes.writeBigUInt64LE(BigInt(base.length), at + 24);
bytes.writeBigUInt64LE(BigInt(compressed.length), at + 32);
const report = await read(bytes);
expect(report.status).toBe("incomplete");
expect(report.coverage.total_entries).toBeNull();
expect(report.limitations.join("\n")).toContain("compressed");
if (section === 7)
expect(report.entries.some((entry) => entry.kind === "init_array")).toBe(
false,
);
else {
const init = report.entries.find((entry) => entry.kind === "init_array");
expect(init).toBeDefined();
if (section === 5) expect(init?.symbol).toBeNull();
else
expect(init).toMatchObject({
resolution: "unresolved",
value_va: null,
});
}
// The uncompressed fixture remains the working comparison.
expect((await read(base)).status).toBe("parsed");
});
it("does not substitute an unrelated dynamic symbol table for DT_SYMTAB", async () => {
const bytes = elf64PieInitializers();
bytes.writeBigUInt64LE(0x777n, 0x300 + 14 * 16 + 8);
const report = await read(bytes);
expect(
report.entries.find((entry) => entry.slot_va === "0x210"),
).toMatchObject({
resolution: "unresolved",
symbol: null,
value_va: null,
});
expect(report.status).toBe("incomplete");
});
it.each(["compressed", "not a string table"])(
"retains later symbol tables when an earlier name source is %s",
async (source) => {
const bytes = elf64PieInitializers();
// The first (dynamic) symbol table links section 8; its bytes still contain imp_init.
if (source === "compressed")
bytes.writeBigUInt64LE(0x802n, 0x800 + 8 * 64 + 8);
else bytes.writeUInt32LE(1, 0x800 + 8 * 64 + 4);
const report = await read(bytes);
expect(report.status).toBe("incomplete");
expect(
report.entries.find((entry) => entry.kind === "init"),
).toMatchObject({
symbol: { name: "ctor_a" },
value_va: "0x100",
});
expect(
report.entries.find((entry) => entry.slot_va === "0x210"),
).toMatchObject({
resolution: "unresolved_import",
symbol: null,
});
},
);
});
describe("ELF incomplete inputs", () => {
it("reports an unsupported relocation machine as incomplete, not as raw zeros", async () => {
const bytes = elf64PieInitializers();
+2 -1
View File
@@ -68,7 +68,8 @@ export const readElfImage = (
}
for (const section of directory.sections) {
const range = declaredFileRange(bytes.length, section.offset, section.size);
const { file_backed, compressed } = section;
const file_backed = section.type !== 8 && range.file_backed;
const compressed = (section.flags & 0x800n) !== 0n;
const read_only =
section.type === 1 &&
file_backed &&
@@ -2,7 +2,6 @@ import { expect, it } from "vitest";
import { pe } from "../../domain/binaryTarget.fixture.js";
import { elfHardeningFixture } from "../hardening/BinaryHardening.fixture.js";
import { elfUnwindSectionlessFixture } from "../elf/ElfUnwindTables.fixture.js";
import { compressedElfNoteSectionFixture } from "../elf/ElfNotes.fixture.js";
import {
ELF_PT_GNU_EH_FRAME,
ELF_PT_GNU_RELRO,
@@ -95,35 +94,3 @@ it("retains a confirmed overlapping PE when the COFF record extends past EOF", (
}),
);
});
it.each([
[32, true],
[32, false],
[64, true],
[64, false],
] as const)(
"retains ELF%i little=%s compression independently of stored section coverage",
(bits, little) => {
const bytes = compressedElfNoteSectionFixture(bits, little);
const initial = readElfDirectoryFromBuffer(bytes).sections[1]!;
expect(initial).toMatchObject({ compressed: true, file_backed: true });
expect(readElfDirectoryFromBuffer(bytes).sections[2]).toMatchObject({
compressed: false,
file_backed: true,
});
// Compression metadata describes the encoding even if its stored range
// subsequently proves incomplete. It must not be inferred from readability.
const sizeAt = initial.header_offset + (bits === 64 ? 32 : 20);
if (bits === 64) {
if (little) bytes.writeBigUInt64LE(BigInt(bytes.length), sizeAt);
else bytes.writeBigUInt64BE(BigInt(bytes.length), sizeAt);
} else if (little) bytes.writeUInt32LE(bytes.length, sizeAt);
else bytes.writeUInt32BE(bytes.length, sizeAt);
expect(readElfDirectoryFromBuffer(bytes).sections[1]).toMatchObject({
compressed: true,
file_backed: false,
offset: initial.offset,
size: BigInt(bytes.length),
});
},
);
@@ -45,10 +45,7 @@ export interface NativeSection {
readonly entry_size: bigint;
/** `sh_addralign` for ELF. PE sections have no note alignment and use 1. */
readonly alignment: bigint;
/** Declared stored bytes are present; compressed bytes are not decoded content. */
readonly file_backed: boolean;
/** ELF SHF_COMPRESSED storage encoding; always false for PE sections. */
readonly compressed: boolean;
}
/** Program header types retained for load, dynamic, hardening, and unwind consumers. */
export const ELF_PT_NOTE = 4 as const;
@@ -311,7 +308,6 @@ export function* readElfSectionDirectory(
alignment: section.alignment,
file_backed:
type !== 8 && declaredFileRange(fileSize, offset, size).file_backed,
compressed: (section.flags & 0x800n) !== 0n,
});
}
}
@@ -414,7 +410,6 @@ export function* readPeSectionDirectory(
BigInt(row.offset),
BigInt(row.size),
).file_backed,
compressed: false,
});
}
return {
@@ -1,53 +0,0 @@
import {
ArtifactResourceScope,
type ArtifactResourceOwner,
} from "../../artifacts/ArtifactResourceScope.js";
/** Retain a cancelled response branch until its underlying cancellation settles. */
export class StateResponseCleanup {
readonly #scope = new ArtifactResourceScope();
constructor(private readonly changed: () => void = () => {}) {}
get cleanupResources(): readonly string[] {
return this.#scope.cleanupResources;
}
cancel(cancel: () => Promise<void>): void {
let state:
| { status: "pending" }
| { status: "done" }
| { status: "failed"; cause: unknown } = { status: "pending" };
const owner: ArtifactResourceOwner = {
kind: "lease",
resource: "state response body",
lease: {
release: async () => {
if (state.status === "pending")
throw new Error("State response cancellation is still pending");
if (state.status === "failed") throw state.cause;
},
},
};
this.#scope.retain(owner);
this.changed();
void Promise.resolve()
.then(cancel)
.then(
async () => {
state = { status: "done" };
await this.#scope.release(owner);
this.changed();
},
(cause: unknown) => {
state = { status: "failed", cause };
this.changed();
},
)
.catch(() => undefined);
}
close(): Promise<void> {
return this.#scope.close();
}
}
+3 -12
View File
@@ -1,4 +1,3 @@
import { StateResponseCleanup } from "./StateResponseCleanup.js";
import type { EnvironmentStateStore } from "../../application/environment/environmentStateStore.js";
import { ArtifactResourceScope } from "../../artifacts/ArtifactResourceScope.js";
import type { AnalysisInputError } from "../../domain/analysisErrorCore.js";
@@ -47,24 +46,17 @@ interface CallContext {
readonly intake: StateIntake;
readonly cancel: AbortSignal;
readonly files: ArtifactResourceScope;
readonly responses: StateResponseCleanup;
}
/** `data/add`: exactly one part, whose document replaces the whole state only when it is valid and the call was not cancelled. */
const addState = async (
{ store, intake, cancel, files, responses }: CallContext,
{ store, intake, cancel, files }: CallContext,
params: unknown,
): Promise<RpcOutcome> => {
const parts = isRecord(params) ? params.parts : undefined;
if (!Array.isArray(parts) || parts.length !== 1)
return { error: invalidParams("data/add takes exactly one part") };
const document = await partDocument(
parts[0],
intake,
cancel,
files,
responses,
);
const document = await partDocument(parts[0], intake, cancel, files);
if (!document.ok) return { error: document.error };
// A file read is not interruptible, so a cancellation during it is honoured here, before the state changes.
if (cancel.aborted) return { error: CANCELLED };
@@ -109,7 +101,6 @@ export const stateDataPlane = (
intake: StateIntake = DEFAULT_STATE_INTAKE,
stopped: AbortSignal = new AbortController().signal,
files: ArtifactResourceScope = new ArtifactResourceScope(),
responses: StateResponseCleanup = new StateResponseCleanup(),
): RpcMethods => {
let previous: Promise<unknown> = Promise.resolve();
return (method, params, signal) => {
@@ -120,7 +111,7 @@ export const stateDataPlane = (
const next = previous.then(() =>
cancel.aborted
? { error: CANCELLED }
: run({ store, intake, cancel, files, responses }, params),
: run({ store, intake, cancel, files }, params),
);
previous = next.catch(() => undefined);
return next;
+1 -43
View File
@@ -1,4 +1,3 @@
import { StateResponseCleanup } from "./StateResponseCleanup.js";
import type { HttpHostname } from "../../domain/httpHostname.js";
import { STATE_INTAKE_LIMIT_BYTES } from "../../application/environment/environmentStateDocument.js";
import {
@@ -478,8 +477,6 @@ export const serveEnvironment = async (
};
// A state file whose close failed stays owned here: later file reads retry it, and close reports it if it is still open.
const stateFiles = new ArtifactResourceScope();
let responseStateChanged = (): void => undefined;
const stateResponses = new StateResponseCleanup(() => responseStateChanged());
const served = await serveStreamableHttp({
host,
port,
@@ -502,7 +499,6 @@ export const serveEnvironment = async (
},
stopping.signal,
stateFiles,
stateResponses,
),
),
...(logger === undefined ? {} : { logger }),
@@ -516,14 +512,11 @@ export const serveEnvironment = async (
});
let httpClosePending = false;
let stateClosePending = false;
let responseClosePending = false;
let settledSuccessful = false;
let closing: Promise<void> | undefined;
const shutdownFailures = new Set<AnalysisError>();
const owner: EnvironmentShutdownOwner = {
get cleanupResources() {
const stateResources = stateFiles.cleanupResources;
const responseResources = stateResponses.cleanupResources;
return [
...(httpClosePending ? [served.url] : []),
...(stateClosePending
@@ -531,11 +524,6 @@ export const serveEnvironment = async (
? stateResources
: ["environment state-file scope"]
: []),
...(responseClosePending
? responseResources.length > 0
? responseResources
: ["state response cleanup"]
: []),
];
},
close: () => {
@@ -543,7 +531,7 @@ export const serveEnvironment = async (
const attempt = (async () => {
stop();
const failures: {
readonly stage: "http-server" | "state-files" | "state-responses";
readonly stage: "http-server" | "state-files";
readonly cause: unknown;
}[] = [];
try {
@@ -560,15 +548,6 @@ export const serveEnvironment = async (
stateClosePending = true;
failures.push({ stage: "state-files", cause });
}
try {
await stateResponses.close();
if (stateResponses.cleanupResources.length > 0)
throw new Error("State response cleanup remains pending");
responseClosePending = false;
} catch (cause: unknown) {
responseClosePending = true;
failures.push({ stage: "state-responses", cause });
}
if (failures.length > 0) {
retainedEnvironmentShutdowns.add(owner);
const failure = new AnalysisOutputError(
@@ -604,7 +583,6 @@ export const serveEnvironment = async (
retainedEnvironmentShutdowns.delete(owner);
for (const failure of shutdownFailures) failure.resolveCleanup();
shutdownFailures.clear();
settledSuccessful = true;
})();
closing = attempt;
void attempt.catch(() => {
@@ -613,26 +591,6 @@ export const serveEnvironment = async (
return attempt;
},
};
// A fetcher may hand back a response after an earlier stop completed. Keep
// that newly acquired body owned and make a later close observe its outcome.
responseStateChanged = () => {
if (!stopping.signal.aborted) return;
if (stateResponses.cleanupResources.length > 0) {
responseClosePending = true;
retainedEnvironmentShutdowns.add(owner);
if (settledSuccessful) {
settledSuccessful = false;
closing = undefined;
}
} else if (
shutdownFailures.size === 0 &&
!httpClosePending &&
!stateClosePending
) {
responseClosePending = false;
retainedEnvironmentShutdowns.delete(owner);
}
};
return ok({
...served,
card,
@@ -1037,122 +1037,7 @@ describe("serveEnvironment with a state", () => {
expect(missing.isError).toBe(true);
});
});
});
describe("state response cleanup", () => {
it("reports pending teed response cleanup and retries after its sibling closes", async () => {
let sourceCancelled = false;
const stream = new ReadableStream<Uint8Array>({
start(controller) {
controller.enqueue(new Uint8Array([1]));
},
cancel() {
sourceCancelled = true;
},
});
const [body, sibling] = stream.tee();
const environment = await serveState(
{ source, state },
{
fetchState: async () =>
new Response(body, {
headers: { "content-length": String(STATE_INTAKE_LIMIT_BYTES + 1) },
}),
},
);
let timer: ReturnType<typeof setTimeout> | undefined;
try {
const response = await fetch(
new URL(ENVIRONMENT_RPC_PATH, environment.url),
{
method: "POST",
headers: { "content-type": "application/json" },
body: JSON.stringify(addByUri(1)),
signal: AbortSignal.timeout(10_000),
},
);
await expect(response.json()).resolves.toMatchObject({
error: { code: -32602 },
});
const closing = Promise.race([
environment.close(),
new Promise<never>((_, reject) => {
timer = setTimeout(
() => reject(new Error("server close waited for tee sibling")),
10_000,
);
}),
]);
await expect(closing).rejects.toMatchObject({
_tag: "AnalysisOutputError",
cleanupIncomplete: true,
diagnostics: { cleanup_stages: ["state-responses"] },
});
expect(sourceCancelled).toBe(false);
await sibling.cancel();
expect(sourceCancelled).toBe(true);
await expect(environment.close()).resolves.toBeUndefined();
} finally {
if (timer !== undefined) clearTimeout(timer);
await Promise.allSettled([body.cancel(), sibling.cancel()]);
await environment.close();
}
});
it("retains a late response cleanup after an earlier successful stop", async () => {
let giveResponse!: (response: Response) => void;
const gate = new Promise<Response>((resolve) => {
giveResponse = resolve;
});
const held = heldFetcher(() => gate);
let release!: () => void;
let acknowledge!: () => void;
const cleanup = new Promise<void>((resolve) => {
release = resolve;
});
const entered = new Promise<void>((resolve) => {
acknowledge = resolve;
});
const environment = await serveState(
{ source, state },
{ fetchState: held.fetchState },
);
const answered = fetch(new URL(ENVIRONMENT_RPC_PATH, environment.url), {
method: "POST",
headers: { "content-type": "application/json" },
body: JSON.stringify(addByUri(1)),
signal: AbortSignal.timeout(10_000),
}).catch(() => undefined);
try {
await held.fetching;
await environment.close();
giveResponse(
new Response(
new ReadableStream<Uint8Array>({
cancel() {
acknowledge();
return cleanup;
},
}),
),
);
await entered;
await expect(environment.close()).rejects.toMatchObject({
_tag: "AnalysisOutputError",
cleanupIncomplete: true,
diagnostics: { cleanup_stages: ["state-responses"] },
});
release();
await expect(environment.close()).resolves.toBeUndefined();
} finally {
release();
await answered;
await environment.close();
}
});
});
describe("serveEnvironment state file cleanup", () => {
it("stops without waiting for a stalled download", async () => {
const held = heldFetcher(() => new Promise<Response>(() => undefined));
const environment = await serveState(
-110
View File
@@ -1,4 +1,3 @@
import { StateResponseCleanup } from "./StateResponseCleanup.js";
import { expect, it, vi } from "vitest";
import { ArtifactResourceScope } from "../../artifacts/ArtifactResourceScope.js";
@@ -12,17 +11,14 @@ import {
const inlineDocument = async (bytes: string) => {
const files = new ArtifactResourceScope();
const responses = new StateResponseCleanup();
try {
return await partDocument(
{ kind: "file", file: { bytes } },
DEFAULT_STATE_INTAKE,
new AbortController().signal,
files,
responses,
);
} finally {
await responses.close();
await files.close();
}
};
@@ -83,109 +79,3 @@ it("downloads with redirects refused and the 30 s timeout by default", async ()
fetched.mockRestore();
}
});
it.each(["http-status", "declared-overflow", "stream-overflow"] as const)(
"returns %s while a teed response's other branch remains open",
async (scenario) => {
const source = new ReadableStream<Uint8Array>({
start(controller) {
controller.enqueue(
new Uint8Array(
scenario === "stream-overflow" ? STATE_INTAKE_LIMIT_BYTES + 1 : 1,
),
);
},
});
const [body, sibling] = source.tee();
const response = new Response(body, {
status: scenario === "http-status" ? 503 : 200,
...(scenario === "declared-overflow"
? {
headers: { "content-length": String(STATE_INTAKE_LIMIT_BYTES + 1) },
}
: {}),
});
const files = new ArtifactResourceScope();
const responses = new StateResponseCleanup();
const operation = partDocument(
{ kind: "file", file: { uri: "https://state.example.test/state.json" } },
{ fetchState: async () => response, timeoutMs: 1_000 },
new AbortController().signal,
files,
responses,
);
let timer: ReturnType<typeof setTimeout> | undefined;
try {
const outcome = await Promise.race([
operation.then((result) => ({ kind: "result" as const, result })),
new Promise<{ kind: "deadline" }>((resolve) => {
timer = setTimeout(() => resolve({ kind: "deadline" }), 3_000);
}),
]);
expect(outcome.kind).toBe("result");
expect(responses.cleanupResources).toEqual(["state response body"]);
await expect(responses.close()).rejects.toMatchObject({
reason: "unavailable",
});
if (outcome.kind === "result") {
expect(outcome.result).toMatchObject(
scenario === "http-status"
? { ok: false, error: { data: { reason: "http-status" } } }
: {
ok: false,
error: {
message: expect.stringContaining(
String(STATE_INTAKE_LIMIT_BYTES),
),
},
},
);
}
} finally {
if (timer !== undefined) clearTimeout(timer);
await sibling.cancel();
await operation;
await responses.close();
await files.close();
}
},
);
it("retains the exact failure when response cancellation rejects", async () => {
const failure = new Error("controlled cancellation failure");
const body = new ReadableStream<Uint8Array>({
cancel() {
throw failure;
},
});
const files = new ArtifactResourceScope();
const responses = new StateResponseCleanup();
const result = await partDocument(
{ kind: "file", file: { uri: "https://state.example.test/state.json" } },
{
fetchState: async () => new Response(body, { status: 503 }),
timeoutMs: 1_000,
},
new AbortController().signal,
files,
responses,
);
expect(result).toMatchObject({
ok: false,
error: { data: { reason: "http-status" } },
});
await new Promise<void>((resolve) => setImmediate(resolve));
const rejected: unknown = await responses
.close()
.catch((cause: unknown) => cause);
expect(rejected).toMatchObject({
reason: "unavailable",
cleanup: { resources: ["state response body"] },
});
expect((rejected as Error).cause).toBeInstanceOf(AggregateError);
expect(((rejected as Error).cause as AggregateError).errors).toContain(
failure,
);
expect(responses.cleanupResources).toEqual(["state response body"]);
await files.close();
});
+7 -18
View File
@@ -1,4 +1,3 @@
import { StateResponseCleanup } from "./StateResponseCleanup.js";
import { fileURLToPath } from "node:url";
import { mediaTypeEssence } from "../../domain/analysisView/recordedMediaType.js";
@@ -229,7 +228,6 @@ const downloadBounds = (
const readCapped = async (
body: ReadableStream<Uint8Array> | null,
bounds: DownloadBounds,
responses: StateResponseCleanup,
): Promise<Result<Buffer, RpcError>> => {
if (body === null) return ok(Buffer.alloc(0));
const reader = body.getReader();
@@ -242,15 +240,13 @@ const readCapped = async (
if (done) return ok(Buffer.concat(chunks, total));
total += value.byteLength;
if (total > STATE_INTAKE_LIMIT_BYTES) {
// A tee's sibling can be caller-owned; cancelling this branch must not
// delay the refusal until that sibling finishes.
responses.cancel(() => reader.cancel());
await reader.cancel().catch(() => undefined);
return err(TOO_LARGE);
}
chunks.push(value);
}
} catch {
responses.cancel(() => reader.cancel());
void reader.cancel().catch(() => undefined);
return err(fetchFailed(bounds.failure()));
}
};
@@ -260,7 +256,6 @@ const downloadState = async (
url: URL,
intake: StateIntake,
cancel: AbortSignal,
responses: StateResponseCleanup,
): Promise<Result<Buffer, RpcError>> => {
const bounds = downloadBounds(intake.timeoutMs, cancel);
const fetching = Promise.resolve().then(() =>
@@ -273,9 +268,7 @@ const downloadState = async (
// A response that arrives after the download ended is released unread.
void fetching
.then(
(late) => {
if (late.body !== null) responses.cancel(() => late.body!.cancel());
},
(late) => late.body?.cancel(),
() => undefined,
)
.catch(() => undefined);
@@ -283,8 +276,7 @@ const downloadState = async (
}
const declared = Number(response.headers.get("content-length") ?? "0");
if (!response.ok || declared > STATE_INTAKE_LIMIT_BYTES) {
// Cancel this branch without waiting for a potentially independent tee sibling.
if (response.body !== null) responses.cancel(() => response.body!.cancel());
await response.body?.cancel().catch(() => undefined);
return err(
response.ok
? TOO_LARGE
@@ -294,7 +286,7 @@ const downloadState = async (
},
);
}
return readCapped(response.body, bounds, responses);
return readCapped(response.body, bounds);
};
const NETWORK_SHARE = invalidParams(
@@ -323,7 +315,6 @@ const fileBytes = async (
intake: StateIntake,
cancel: AbortSignal,
files: ArtifactResourceScope,
responses: StateResponseCleanup,
): Promise<Result<Uint8Array, RpcError>> => {
if (typeof file.bytes === "string" && file.uri === undefined) {
if (file.bytes.length > MAX_INLINE_BASE64_CHARACTERS) return err(TOO_LARGE);
@@ -342,8 +333,7 @@ const fileBytes = async (
invalidParams("a file part carries exactly one of bytes or a uri"),
);
const url = new URL(file.uri);
if (url.protocol === "https:")
return downloadState(url, intake, cancel, responses);
if (url.protocol === "https:") return downloadState(url, intake, cancel);
if (url.protocol !== "file:")
return err(invalidParams("a file uri is a file:// or https:// URL"));
const path = localFilePath(url);
@@ -356,7 +346,6 @@ export const partDocument = async (
intake: StateIntake,
cancel: AbortSignal,
files: ArtifactResourceScope,
responses: StateResponseCleanup,
): Promise<Result<unknown, RpcError>> => {
if (isRecord(part) && part.kind === "data" && isRecord(part.data))
return ok(part.data);
@@ -373,6 +362,6 @@ export const partDocument = async (
mediaTypeEssence(mimeType) !== "application/json")
)
return err(invalidParams("a file part's mimeType is application/json"));
const bytes = await fileBytes(part.file, intake, cancel, files, responses);
const bytes = await fileBytes(part.file, intake, cancel, files);
return bytes.ok ? parseDocumentBytes(bytes.value) : bytes;
};
@@ -15,13 +15,6 @@ const cases = [
0,
],
['spawn("control", [], {stdio:"inherit"})', "control", "inherit", 0],
['spawn("global-nan-env", [], {env:NaN})', "global-nan-env", "default", 0],
[
'((NaN) => spawn("shadowed-nan-env", [], {env:NaN}))({})',
"shadowed-nan-env",
"default",
0,
],
[
'spawn("spread-after", [], {stdio:"pipe",...input})',
"spread-after",
@@ -240,12 +233,6 @@ const exportAssignmentSource = `
export = argv;
`;
const environmentCommands = [
"named-argv",
"execfile-dynamic",
"shadowed-nan-env",
];
// Callback/options ambiguity cannot change exec's effective stdio.
const opaqueExecCallbackCommands = [
"eval-callback",
@@ -355,7 +342,7 @@ cliTest.for(["CLI", "stdio MCP"] as const)(
expect(child.properties.stdio_mode, String(command)).toBe(mode);
expect(child.properties.argv_count, String(command)).toBe(count);
expect(child.properties.environment_supplied, String(command)).toBe(
environmentCommands.includes(String(command)),
["named-argv", "execfile-dynamic"].includes(String(command)),
);
expect(child.identity.source_range).not.toBeNull();
const stdio = graph.relations.filter(
@@ -390,7 +377,9 @@ cliTest.for(["CLI", "stdio MCP"] as const)(
relation.relation === "supplies-env" &&
relation.target_node_id === child.node_id,
),
).toHaveLength(environmentCommands.includes(String(command)) ? 1 : 0);
).toHaveLength(
["named-argv", "execfile-dynamic"].includes(String(command)) ? 1 : 0,
);
expect(
graph.relations.filter(
(relation) =>
@@ -1,72 +1,81 @@
import { readFile, readdir } from "node:fs/promises";
import { execFile } from "node:child_process";
import { readFile, readdir, writeFile } from "node:fs/promises";
import { join } from "node:path";
import { expect } from "vitest";
import { promisify } from "node:util";
import { expect, it } from "vitest";
import { parse } from "yaml";
import { resolveSwiftcCompileTimeoutMs } from "../../../src/process/DarwinProcessRunTokenReader.js";
import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js";
import { processTest } from "../../support/process/processFixture.js";
const hostLabels: Partial<Record<NodeJS.Platform, string>> = {
linux: "linux",
darwin: "macOS",
win32: "Windows",
};
const hostLabel = hostLabels[process.platform];
const preparationBudget =
process.platform === "darwin" ? resolveSwiftcCompileTimeoutMs() : 0;
const execute = promisify(execFile);
processTest.skipIf(hostLabel === undefined)(
"enforces workflow platform labels before contacting Cutter",
{ timeout: Math.min(2_147_483_647, 2 * preparationBudget + 30_000) },
async ({ processes }) => {
const workflow = parse(
await readFile(".github/workflows/real-cutter.yml", "utf8"),
);
const labels = workflow.jobs["cutter-upstream"].strategy.matrix.include.map(
(entry: { platform: string }) => entry.platform,
);
expect(labels).toEqual(["linux", "macOS", "Windows"]);
const root = await createTestTempDirectory("rea-cutter-platform-");
// A private absent directory proves actual discovery without contacting a user's bridge.
const bridgeDirectory = join(root, "absent-bridge");
const check = async (selected: string | undefined) => {
const env: NodeJS.ProcessEnv = {
...process.env,
TMPDIR: root,
TMP: root,
TEMP: root,
REA_CUTTER_BRIDGE_DIR: bridgeDirectory,
};
if (selected === undefined) delete env.EXPECTED_PLATFORM;
else env.EXPECTED_PLATFORM = selected;
const result = await processes.run(
process.execPath,
["scripts/verify-real-cutter.mjs"],
{
env,
timeoutMs: Math.min(2_147_483_647, preparationBudget + 15_000),
},
);
expect(result.exitCode, result.stderr).toBe(1);
return result.stderr;
it("enforces workflow platform labels before contacting Cutter", async () => {
const workflow = parse(
await readFile(".github/workflows/real-cutter.yml", "utf8"),
);
const labels = workflow.jobs["cutter-upstream"].strategy.matrix.include.map(
(entry: { platform: string }) => entry.platform,
);
expect(labels).toEqual(["linux", "macOS", "Windows"]);
const root = await createTestTempDirectory("rea-cutter-platform-");
const preload = join(root, "preload.mjs");
await writeFile(
preload,
`
import childProcess from 'node:child_process';
import { syncBuiltinESMExports } from 'node:module';
Object.defineProperty(process, 'platform', { value: process.env.REA_PROBE_PLATFORM });
childProcess.execFile = () => { throw new Error('fixture discovery reached'); };
syncBuiltinESMExports();
`,
);
const check = async (selected: string | undefined, actual: string) => {
const env: NodeJS.ProcessEnv = {
...process.env,
TMPDIR: root,
TMP: root,
TEMP: root,
REA_PROBE_PLATFORM: actual,
};
for (const selected of [hostLabel, undefined]) {
const stderr = await check(selected);
expect(stderr).toContain("Open upstream Cutter with the REA plugin");
expect(stderr).toContain("no_live_bridge_found");
}
const other = labels.find((label: string) => label !== hostLabel);
expect(other).toBeDefined();
expect(await check(other)).toContain(
"Cutter runner did not match its declared platform",
if (selected === undefined) delete env.EXPECTED_PLATFORM;
else env.EXPECTED_PLATFORM = selected;
const result: unknown = await execute(
process.execPath,
["--import", preload, "scripts/verify-real-cutter.mjs"],
{ env, timeout: 10_000 },
).then(
() => {
throw new Error("Fixture discovery should stop the verifier");
},
(error: unknown) => error,
);
expect(await check("unknown")).toContain(
"Unrecognized Cutter runner platform",
if (
!(result instanceof Error) ||
!("stderr" in result) ||
typeof result.stderr !== "string"
)
throw new Error("Missing verifier diagnostics");
return result.stderr;
};
for (const [selected, actual] of [
["linux", "linux"],
["macOS", "darwin"],
["Windows", "win32"],
] as const)
expect(await check(selected, actual)).toContain(
"fixture discovery reached",
);
expect(
(await readdir(root)).filter((name) =>
name.startsWith("rea-cutter-smoke-"),
),
).toEqual([]);
},
);
expect(await check("linux", "darwin")).toContain(
"Cutter runner did not match its declared platform",
);
expect(await check("unknown", "linux")).toContain(
"Unrecognized Cutter runner platform",
);
expect(await check(undefined, "linux")).toContain(
"fixture discovery reached",
);
expect(
(await readdir(root)).filter((name) =>
name.startsWith("rea-cutter-smoke-"),
),
).toEqual([]);
});
@@ -783,7 +783,7 @@ it.each([
{ code: false, ghidra_bridge: true, runtime: false, package: false },
],
[
"tests/conformance/ghidra/ReaInstructionContextProbe.java",
"tests/conformance/ghidra/ReaArmContextProbe.java",
{ ghidra_bridge: true, runtime: false, package: false },
],
[
@@ -1,84 +0,0 @@
import { writeFile } from "node:fs/promises";
import { join } from "node:path";
import { expect } from "vitest";
import { createBinarySession } from "../../../src/composition/binary.js";
import { parseConfig } from "../../../src/config/parseConfig.js";
import { artifactInspectionResultSchema } from "../../../src/domain/artifactInspection.js";
import { parseEvidence } from "../../../src/domain/evidence.js";
import { silentLogger } from "../../../src/logger.js";
import { createServer } from "../../../src/server/createServer.js";
import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js";
import { mcpTest } from "../../support/mcp/mcpFixture.js";
import { serveCli } from "../cli/cliServeHarness.js";
const expectMftInventory = (value: unknown): void => {
const evidence = parseEvidence(value);
const inspection = artifactInspectionResultSchema.parse(
evidence.normalized_result,
);
const inventory = inspection.substeps[0]?.evidence.normalized_result;
expect(inventory?.manifest.root_format).toBe("ntfs-mft");
expect(inventory?.occurrences[0]?.format_metadata).toMatchObject({
kind: "ntfs-mft",
record_count: 1,
fixup_unverified_record_count: 0,
record_headers: [
{
index: 0,
fileOffset: 0,
sequenceNumber: 1,
declaredRecordNumber: null,
fixups: { verifiedSectors: 2 },
},
],
omitted_record_headers: 0,
});
};
mcpTest(
"admits an MFT capture by signature through CLI and MCP",
async ({ mcp, onTestFinished }) => {
const root = await createTestTempDirectory("rea-ntfs-mft-admission-");
const path = join(root, "capture.bin");
const record = Buffer.alloc(1024);
record.write("FILE");
record.writeUInt16LE(42, 4);
record.writeUInt16LE(3, 6);
record.writeUInt16LE(1, 16);
record.writeUInt16LE(56, 20);
record.writeUInt32LE(64, 24);
record.writeUInt32LE(0xffff_ffff, 56);
record.writeUInt32LE(1024, 28);
record.writeUInt16LE(5, 42);
record.writeUInt16LE(5, 510);
record.writeUInt16LE(5, 1022);
await writeFile(path, record);
const cli = await serveCli(["inspect-artifact", path, "--json"]);
expect(cli.exitCode, cli.stdout).toBe(0);
expectMftInventory(JSON.parse(cli.stdout));
const configured = parseConfig({ REA_ANALYSIS_PROVIDER: "auto" });
if (!configured.ok) throw configured.error;
const session = createBinarySession(configured.value, silentLogger, {});
onTestFinished(async () => {
await session.close();
});
const client = await mcp.connect(
createServer({ kind: "session", session }),
);
const opened = await client.callTool({
name: "open_binary",
arguments: { path },
});
expect(opened.isError, JSON.stringify(opened.content)).not.toBe(true);
const inspected = await client.callTool({
name: "inspect_artifact",
arguments: {},
});
expect(inspected.isError, JSON.stringify(inspected.content)).not.toBe(true);
expectMftInventory(inspected.structuredContent);
},
);
@@ -1,4 +1,4 @@
import { readFile, stat, writeFile } from "node:fs/promises";
import { readFile, writeFile } from "node:fs/promises";
import { join } from "node:path";
import { fileURLToPath } from "node:url";
import { expect } from "vitest";
@@ -175,11 +175,7 @@ registerHooks({ load(url, context, nextLoad) {
),
],
{
env: {
...process.env,
REA_AGENT_EVAL_ROOT: parent,
REA_AGENT_EVAL_KEEP_FIXTURES: "false",
},
env: { ...process.env, REA_AGENT_EVAL_ROOT: parent },
timeoutMs: 10_000,
},
);
@@ -194,13 +190,14 @@ registerHooks({ load(url, context, nextLoad) {
if (incomplete) {
expect(result.stderr).toContain("Artifact cleanup remains incomplete");
expect(result.stderr).toContain("fixture cleanup denied");
expect((await stat(run)).isDirectory()).toBe(true);
expect(await readFile(join(run, "codex/auth.json"), "utf8")).toBe(
"inert fixture",
);
} else {
await expect(stat(run)).rejects.toMatchObject({ code: "ENOENT" });
await expect(
readFile(join(run, "codex/auth.json")),
).rejects.toMatchObject({ code: "ENOENT" });
}
await expect(readFile(join(run, "codex/auth.json"))).rejects.toMatchObject({
code: "ENOENT",
});
},
);
@@ -277,11 +274,7 @@ registerHooks({ load(url, context, nextLoad) {
),
],
{
env: {
...process.env,
REA_AGENT_EVAL_ROOT: parent,
REA_AGENT_EVAL_KEEP_FIXTURES: "false",
},
env: { ...process.env, REA_AGENT_EVAL_ROOT: parent },
timeoutMs: 10_000,
},
);
@@ -305,14 +298,13 @@ registerHooks({ load(url, context, nextLoad) {
if (incomplete) {
expect(result.stderr).toContain("Artifact cleanup remains incomplete");
expect(result.stderr).toContain("darwin-fixture-reader");
expect((await stat(observed.run)).isDirectory()).toBe(true);
expect(
await readFile(join(observed.run, "codex/auth.json"), "utf8"),
).toBe("inert preparation fixture");
} else {
await expect(stat(observed.run)).rejects.toMatchObject({
code: "ENOENT",
});
await expect(
readFile(join(observed.run, "codex/auth.json")),
).rejects.toMatchObject({ code: "ENOENT" });
}
await expect(
readFile(join(observed.run, "codex/auth.json")),
).rejects.toMatchObject({ code: "ENOENT" });
},
);
@@ -1,57 +1,24 @@
import { execFile } from "node:child_process";
import {
copyFile,
cp,
mkdir,
readFile,
symlink,
writeFile,
} from "node:fs/promises";
import { join, resolve } from "node:path";
import { expect } from "vitest";
import { dirname, join, resolve } from "node:path";
import { promisify } from "node:util";
import { expect, it } from "vitest";
import { z } from "zod";
import { createApplicationMcpHarness } from "../../fixtures/applicationMcpHarness.js";
import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js";
import { processTest } from "../../support/process/processFixture.js";
import { resolveSwiftcCompileTimeoutMs } from "../../../src/process/DarwinProcessRunTokenReader.js";
const preparationBudget =
process.platform === "darwin" ? resolveSwiftcCompileTimeoutMs() : 0;
async function prepareController(controller: string, version: string) {
// The verifier loads its production-owned TypeScript source graph at runtime.
// Keep that graph in the isolated controller so imports cannot drift from a hand-maintained list.
await mkdir(join(controller, "scripts"), { recursive: true });
await copyFile(
"scripts/verify-release-artifact.mjs",
join(controller, "scripts/verify-release-artifact.mjs"),
);
await cp("scripts/lib", join(controller, "scripts/lib"), {
recursive: true,
});
await mkdir(join(controller, "bridge"), { recursive: true });
await cp("bridge/process", join(controller, "bridge/process"), {
recursive: true,
});
await cp("src", join(controller, "src"), {
recursive: true,
filter: (source) => !/\.(?:test|spec)\.[cm]?[jt]sx?$/u.test(source),
});
await writeFile(
join(controller, "package.json"),
JSON.stringify({ name: "rea-agents", version, type: "module" }),
);
await symlink(
resolve("node_modules"),
join(controller, "node_modules"),
process.platform === "win32" ? "junction" : "dir",
);
}
const execute = promisify(execFile);
// Release rehearsal runs on Linux; the npm process adapter is also exercised on macOS.
processTest.skipIf(process.platform === "win32")(
it.skipIf(process.platform === "win32")(
"fails without declared runtime, then rehearses an installed tarball and rejects catalog drift",
{ timeout: Math.min(2_147_483_647, 3 * preparationBudget + 210_000) },
async ({ processes }) => {
async () => {
const root = await createTestTempDirectory("rea-release-canary-");
const candidate = join(root, "candidate");
const controller = join(root, "controller");
@@ -109,46 +76,56 @@ if (process.argv[2] === "mcp") {
} else { process.exitCode = 1; }
`,
);
const pack = async () => {
const result = await processes.run(
"npm",
["pack", "--ignore-scripts", "--json"],
{ cwd: candidate, timeoutMs: 30_000 },
);
expect(result.exitCode, result.stderr).toBe(0);
return result;
};
const packed = await pack();
const packed = await execute(
"npm",
["pack", "--ignore-scripts", "--json"],
{
cwd: candidate,
},
);
const [{ filename }] = z
.tuple([z.object({ filename: z.string() })])
.parse(JSON.parse(packed.stdout));
const tarball = join(candidate, filename);
await prepareController(controller, version);
for (const file of [
"scripts/verify-release-artifact.mjs",
"scripts/lib/release-package-canary.mjs",
"scripts/lib/verify-package-core.mjs",
"scripts/lib/mcp-verifier-results.mjs",
"scripts/lib/verifier-run.mjs",
]) {
const target = join(controller, file);
await mkdir(dirname(target), { recursive: true });
await copyFile(file, target);
}
await writeFile(
join(controller, "package.json"),
JSON.stringify({ name: "rea-agents", version, type: "module" }),
);
await symlink(
resolve("node_modules"),
join(controller, "node_modules"),
process.platform === "win32" ? "junction" : "dir",
);
const invoke = (selected = tarball) =>
processes.run(
execute(
process.execPath,
[
join(controller, "scripts/verify-release-artifact.mjs"),
selected,
version,
],
{
cwd: controller,
timeoutMs: Math.min(2_147_483_647, preparationBudget + 30_000),
},
{ cwd: controller, maxBuffer: 16 * 1024 * 1024 },
);
const missing = await invoke();
expect(missing.exitCode).toBe(1);
expect(missing.stderr).toContain("ERR_MODULE_NOT_FOUND");
expect(missing.stderr).toMatch(/dist[/\\]catalogIdentity\.js/u);
await expect(invoke()).rejects.toMatchObject({
stderr: expect.stringContaining("ERR_MODULE_NOT_FOUND"),
});
await symlink(
resolve("dist"),
join(controller, "dist"),
process.platform === "win32" ? "junction" : "dir",
);
const successful = await invoke();
expect(successful.exitCode, successful.stderr).toBe(0);
const result = JSON.parse(successful.stdout);
const result = JSON.parse((await invoke()).stdout);
expect(result).toMatchObject({
package: "rea-agents",
version,
@@ -167,10 +144,12 @@ if (process.argv[2] === "mcp") {
join(candidate, "scripts/reply.json"),
JSON.stringify(snapshot),
);
await pack();
const drifted = await invoke();
expect(drifted.exitCode).toBe(1);
expect(drifted.stderr).toContain("invalid canonical tool catalog");
await execute("npm", ["pack", "--ignore-scripts", "--json"], {
cwd: candidate,
});
await expect(invoke()).rejects.toMatchObject({
stderr: expect.stringContaining("invalid canonical tool catalog"),
});
await writeFile(
join(candidate, "package.json"),
JSON.stringify({
@@ -180,12 +159,15 @@ if (process.argv[2] === "mcp") {
files: ["scripts"],
}),
);
const wrong = await pack();
const wrong = await execute("npm", ["pack", "--ignore-scripts", "--json"], {
cwd: candidate,
});
const [{ filename: wrongFilename }] = z
.tuple([z.object({ filename: z.string() })])
.parse(JSON.parse(wrong.stdout));
const mismatched = await invoke(join(candidate, wrongFilename));
expect(mismatched.exitCode).toBe(1);
expect(mismatched.stderr).toContain("Installed package identity");
await expect(invoke(join(candidate, wrongFilename))).rejects.toMatchObject({
stderr: expect.stringContaining("Installed package identity"),
});
},
30_000,
);
-6
View File
@@ -21,12 +21,6 @@ no-overwrite CLI delivery. It checks the target and project files for changes.
Clang/LLD/LLVM readelf/LLVM objdump; `REA_RISCV_CLANG`, `REA_RISCV_LLD`,
`REA_RISCV_READELF` and `REA_RISCV_OBJDUMP` select existing tool commands.
- `ReaInstructionContextProbe.java` inspects source-owned ARM, Thumb, MIPS32
and MIPS16 instructions through the production bridge in an ephemeral,
read-only Ghidra project. It checks exact decoder-context register names,
values and provenance; the ARM lane also injects an unavailable provider
value. The `ghidra-bridge-sources` CI job runs both processor families. The
input bytes are never executed.
- `ReaDecodeProbe.java` decodes source-owned instruction bytes from a
language's default context and checks each mnemonic, length and optional
complete operand. The `ghidra-bridge-sources` CI job runs it for `ARM:LE:32:Cortex`,
@@ -15,20 +15,16 @@ import java.lang.reflect.Proxy;
import java.math.BigInteger;
/** Real ARM/Thumb decoding; only the unavailable context value is injected. */
public class ReaInstructionContextProbe extends GhidraScript {
public class ReaArmContextProbe extends GhidraScript {
@Override
public void run() throws Exception {
String[] args = getScriptArgs();
require(args.length == 1 || (args.length == 2 && args[1].equals("mips")), "Expected the bridge directory and optional MIPS mode");
require(args.length == 1, "Expected the exact production bridge directory");
var source = GhidraScriptUtil.findScriptByName("ReaGhidraBridge.java");
require(source != null && new File(source.getAbsolutePath()).getCanonicalFile().equals(
new File(args[0], "ReaGhidraBridge.java").getCanonicalFile()), "Production bridge source");
GhidraScript bridge = GhidraScriptUtil.getProvider(source).getScriptInstance(source, errorWriter);
bridge.set(getState(), getControls());
if (args.length == 2) {
checkMips(bridge);
return;
}
require(currentProgram.getLanguageID().getIdAsString().equals("ARM:LE:32:v7"), "ARM fixture language");
var mode = currentProgram.getRegister("TMode");
require(mode != null, "Real TMode register");
@@ -44,8 +40,8 @@ public class ReaInstructionContextProbe extends GhidraScript {
space.set(null, arm.getAddressSpace());
Method inspect = bridge.getClass().getDeclaredMethod("inspectNativeInstruction", JsonObject.class, boolean.class);
inspect.setAccessible(true);
check(inspect, bridge, arm, "0", 4, "1eff2fe1", "TMode");
check(inspect, bridge, thumb, "1", 2, "7047", "TMode");
check(inspect, bridge, arm, "0", 4, "1eff2fe1");
check(inspect, bridge, thumb, "1", 2, "7047");
// getValue includes language defaults. Inject only its unavailable result,
// preserving the real instruction, register, language and all other APIs.
@@ -61,49 +57,26 @@ public class ReaInstructionContextProbe extends GhidraScript {
programField.setAccessible(true);
try {
programField.set(bridge, program);
check(inspect, bridge, arm, null, 4, "1eff2fe1", "TMode");
check(inspect, bridge, arm, null, 4, "1eff2fe1");
} finally {
programField.set(bridge, original);
}
println("REA_ARM_CONTEXT_OK arm=0 thumb=1 unavailable=null (injected provider value)");
}
private void checkMips(GhidraScript bridge) throws Exception {
require(currentProgram.getLanguageID().getIdAsString().equals("MIPS:LE:32:default"), "MIPS fixture language");
var mode = currentProgram.getRegister("ISA_MODE");
require(mode != null, "Real ISA_MODE register");
Address standard = currentProgram.getMemory().getMinAddress();
Address compressed = standard.add(4);
var context = currentProgram.getProgramContext();
context.setValue(mode, standard, standard.add(3), BigInteger.ZERO);
require(disassemble(standard), "MIPS32 instruction decoded");
// MIPS32 flow may have decoded following bytes in the default mode.
clearListing(compressed, compressed.add(3));
context.setValue(mode, compressed, compressed.add(1), BigInteger.ONE);
require(disassemble(compressed), "MIPS16 instruction decoded");
Field space = bridge.getClass().getDeclaredField("sessionDefaultAddressSpace");
space.setAccessible(true);
space.set(null, standard.getAddressSpace());
Method inspect = bridge.getClass().getDeclaredMethod("inspectNativeInstruction", JsonObject.class, boolean.class);
inspect.setAccessible(true);
check(inspect, bridge, standard, "0", 4, "03008224", "ISA_MODE");
check(inspect, bridge, compressed, "1", 2, "5a6a", "ISA_MODE");
println("REA_MIPS_CONTEXT_OK standard=0 mips16=1");
}
private static void check(Method inspect, Object bridge, Address address, String mode,
int length, String bytes, String register) throws Exception {
int length, String bytes) throws Exception {
JsonObject input = new JsonObject();
input.add("document", JsonNull.INSTANCE);
input.addProperty("address", "0x" + Long.toUnsignedString(address.getOffset(), 16));
JsonObject result = (JsonObject) inspect.invoke(bridge, input, false);
require(result.get("status").getAsString().equals("decoded"), "Decoded bridge response: " + result);
require(result.get("length").getAsInt() == length, "Instruction width");
require(result.get("status").getAsString().equals("decoded"), "Decoded bridge response");
require(result.get("length").getAsInt() == length, "ARM/Thumb instruction width");
require(result.get("bytes").getAsString().equals(bytes), "Original instruction bytes");
var contexts = result.getAsJsonArray("decoder_context");
require(contexts != null && contexts.size() == 1, "One retained mode context");
var value = contexts.get(0).getAsJsonObject();
require(value.get("register").getAsString().equals(register), "Exact context register name");
require(value.get("register").getAsString().equals("TMode"), "TMode register name");
require(value.get("source").getAsString().equals("ghidra-program-context"), "Context provenance");
require(mode == null ? value.get("value").isJsonNull()
: mode.equals(value.get("value").getAsString()), "Exact mode value " + mode);
-71
View File
@@ -252,74 +252,3 @@ export const artifactReaderLifecycleCases = (setup: LifecycleFixture): void => {
validationCases(setup);
});
};
/** An interrupted preparation belongs to its caller, not concurrent readers. */
export const artifactPreparationRecoveryCases = ({
image,
createReader,
}: LifecycleFixture): void => {
it.each(["cancelled", "physical failure"] as const)(
"preserves independent preparation after %s",
async (scenario) => {
const path = await fixture(image());
const admitted = await openRegularFile(path, { symlinks: "reject" });
const reader = createReader(path, admitted);
const controller = new AbortController();
const reason = { request: "first inventory" };
const physical = Object.assign(new Error("controlled read failure"), {
code: "EIO",
});
const reached = createDeferred<void>();
const release = createDeferred<void>();
const nativeRead = admitted.handle.read.bind(admitted.handle);
let blocked = false;
const read = vi.spyOn(admitted.handle, "read").mockImplementation((async (
...args: unknown[]
) => {
const result = await Reflect.apply(nativeRead, admitted.handle, args);
if (!blocked) {
blocked = true;
reached.resolve();
await release.promise;
if (scenario === "physical failure") throw physical;
}
return result;
}) as typeof admitted.handle.read);
const pending: Promise<unknown>[] = [];
try {
const first = (async () => {
for await (const _ of reader.entries(controller.signal)) {
}
})();
pending.push(first);
void first.catch(() => undefined);
await awaitAcknowledgement(reached.promise, first, "preparation read");
const peer = entries(reader);
pending.push(peer);
void peer.catch(() => undefined);
controller.abort(reason);
release.resolve();
if (scenario === "physical failure") {
await expect(first).rejects.toBe(physical);
await expect(peer).rejects.toBe(physical);
} else {
await expect(first).rejects.toMatchObject({
reason: "cancelled",
cause: reason,
});
const recovered = await peer;
expect(recovered.length).toBeGreaterThan(0);
expect(
(await buffer(await reader.open(recovered[0]!))).length,
).toBeGreaterThan(0);
}
} finally {
release.resolve();
await Promise.allSettled(pending);
read.mockRestore();
await reader.close();
await admitted.handle.close();
}
},
);
};