Compare commits

...
Author SHA1 Message Date
Cursor Agent 5fb8939a28 fix(javascript): keep Object.defineProperty identified at its own call
The call that escapes Object was treated as a prior replacement of
defineProperty, so a descriptor value never replaced the exported sender
and bundled IPC paths were dropped.
2026-11-11 02:35:53 +00:00
Cursor Agent f366d21446 chore: merge main into codex/audit-javascript-semantics 2026-11-11 02:26:35 +00:00
2885347cde test(ghidra): remove the retired processor-context suite (#9915)
* test(ghidra): keep the bridge context contract out of an empty suite

Source tests collect src/ghidra/GhidraFunctionValues.test.ts directly.
The empty describe failed that file with "No test found in suite" even
though every shard passed. Assert the bridge bindings at file level.

* test(ghidra): retire superseded source-text context assertions

Co-authored-by: gensokyo-dev <[email protected]>

---------

Co-authored-by: Cursor Agent <[email protected]>
Co-authored-by: N0zoM1z0 <[email protected]>
Co-authored-by: gensokyo-dev <[email protected]>
2026-11-10 21:20:07 -05:00
N0zoM1z0andgensokyo-dev a6a2527ade fix(artifacts): revalidate cached MFT observations
Co-authored-by: gensokyo-dev <[email protected]>
2026-11-11 10:03:35 +08:00
N0zoM1z0andgensokyo-dev f8fc572e4a fix(ntfs): report contradictory declared record identities
Co-authored-by: gensokyo-dev <[email protected]>
2026-11-11 10:03:35 +08:00
N0zoM1z0andgensokyo-dev ab9e5fa204 fix(ntfs): require space for the first attribute marker
Co-authored-by: gensokyo-dev <[email protected]>
2026-11-11 10:03:35 +08:00
N0zoM1z0andgensokyo-dev 8d45cb45c4 fix(ntfs): validate first attribute bounds in FILE records
Co-authored-by: gensokyo-dev <[email protected]>
2026-11-11 10:03:35 +08:00
N0zoM1z0andgensokyo-dev 7712e3185e fix(ntfs): retain incomplete coverage alongside fixup mismatches
Co-authored-by: gensokyo-dev <[email protected]>
2026-11-11 10:03:35 +08:00
N0zoM1z0andgensokyo-dev 466149169e fix(ntfs): validate aligned FILE record extents
Co-authored-by: gensokyo-dev <[email protected]>
2026-11-11 10:03:35 +08:00
N0zoM1z0andgensokyo-dev 1c2626edbd fix(ntfs): reject reserved update sequence values
Co-authored-by: gensokyo-dev <[email protected]>
2026-11-11 10:03:35 +08:00
N0zoM1z0andgensokyo-dev b16d18f198 test(artifacts): cover independently cancelled MFT traversals
Co-authored-by: gensokyo-dev <[email protected]>
2026-11-11 10:03:35 +08:00
N0zoM1z0andgensokyo-dev 14030e2f11 fix(ntfs): expose unrecognized FILE record flag bits
Co-authored-by: gensokyo-dev <[email protected]>
2026-11-11 10:03:35 +08:00
N0zoM1z0andgensokyo-dev 9505c73824 fix(artifacts): retain only the bounded MFT header sample
Co-authored-by: gensokyo-dev <[email protected]>
2026-11-11 10:03:35 +08:00
N0zoM1z0andgensokyo-dev 1a446cdade fix(ntfs): label the system flag independently of extensions
Co-authored-by: gensokyo-dev <[email protected]>
2026-11-11 10:03:35 +08:00
N0zoM1z0andgensokyo-dev 1ef7f02cae test(ntfs): group MFT header and fixup regressions
Co-authored-by: gensokyo-dev <[email protected]>
2026-11-11 10:03:35 +08:00
N0zoM1z0andgensokyo-dev 7b86474be7 test(artifacts): use the MFT traversal iterable contract
Co-authored-by: gensokyo-dev <[email protected]>
2026-11-11 10:03:35 +08:00
N0zoM1z0andgensokyo-dev 5d10337709 feat(artifacts): expose a bounded sample of MFT record headers
Co-authored-by: gensokyo-dev <[email protected]>
2026-11-11 10:03:35 +08:00
N0zoM1z0andgensokyo-dev 8eadb872f2 fix(ntfs): preserve full-width MFT log sequence numbers
Co-authored-by: gensokyo-dev <[email protected]>
2026-11-11 10:03:35 +08:00
N0zoM1z0andgensokyo-dev f170175ec8 fix(artifacts): retry interrupted MFT preparation
Co-authored-by: gensokyo-dev <[email protected]>
2026-11-11 10:03:35 +08:00
N0zoM1z0andgensokyo-dev fbe09f84df fix(artifacts): verify MFT source before publishing metadata
Co-authored-by: gensokyo-dev <[email protected]>
2026-11-11 10:03:35 +08:00
N0zoM1z0andgensokyo-dev ebb480ba22 fix(ntfs): buffer the admitted MFT byte range once
Co-authored-by: gensokyo-dev <[email protected]>
2026-11-11 10:03:35 +08:00
N0zoM1z0andgensokyo-dev da1f029c2f fix(artifacts): admit MFT captures through CLI and MCP
Co-authored-by: gensokyo-dev <[email protected]>
2026-11-11 10:03:35 +08:00
N0zoM1z0andgensokyo-dev 4db9c166ef fix(ntfs): report contradictory FILE record extents
Co-authored-by: gensokyo-dev <[email protected]>
2026-11-11 10:03:35 +08:00
N0zoM1z0andgensokyo-dev 29fd6a7d03 fix(ntfs): accept older FILE record headers
Co-authored-by: gensokyo-dev <[email protected]>
2026-11-11 10:03:35 +08:00
N0zoM1z0andgensokyo-dev d7338876d6 fix(ntfs): reject malformed MFT update sequence arrays
Co-authored-by: gensokyo-dev <[email protected]>
2026-11-11 10:03:35 +08:00
N0zoM1z0andgensokyo-dev 9df3f7174f fix(ntfs): keep record identity across invalid MFT slots
Co-authored-by: gensokyo-dev <[email protected]>
2026-11-11 10:03:35 +08:00
N0zoM1z0andgensokyo-dev 79e476204f fix(ntfs): preserve MFT reference sequence numbers
Co-authored-by: gensokyo-dev <[email protected]>
2026-11-11 10:03:35 +08:00
2SinceJuly 9c95218374 style(artifacts): apply the repository formatter to the MFT reader
The new NTFS $MFT files were committed unformatted, so the Static checks lane
failed on format:check. Formatting only; no behaviour change.

(cherry picked from commit dc6a6d40ff)
2026-11-11 10:03:35 +08:00
2SinceJuly d9d2641981 feat(artifacts): inventory NTFS $MFT record headers offline
An acquired $MFT stream had no reader, so an analyst holding one could only
treat it as an opaque file. Recognize the FILE signature of the first record
and inventory every record's header: sequence number, flags, used and total
bytes, base record reference and update-sequence array.

Each record's update-sequence fixups are verified against its sector
trailers. A mismatch keeps the header facts, is reported on the record and is
counted in the metadata, so a record that was not cleanly written is visible
rather than silently accepted. A missing or short array is reported as
unverified, a wrong signature stops the scan at its offset, and a trailing
partial record is reported as a byte count. Attributes, filenames and the
$UsnJrnl:$J journal are not decoded, so no path or change history is claimed.

Part of #9867

(cherry picked from commit ccb45f5712)
2026-11-11 10:03:35 +08:00
Cursor Agent 881aec416a fix(javascript): keep source accessors known after a copied value escapes
An unknown computed getter makes accessor lookup consider every property.
A spread copy already records a value escape, and that escape was treated
as an unknown getter on the source, so the next read escaped the whole
object. Accessor lookup still fails closed for unknown writes and calls.
2026-11-11 01:18:58 +00:00
Cursor Agent f8d17481e8 test(javascript): include the request link in endpoint traces
Response consumers now also link the originating request to the response, so an endpoint trace reports both that association and the response's json consumer.
2026-11-10 23:34:45 +00:00
morluto e2585aa7ce test(ghidra): qualify instruction mode with real provider 2026-11-10 17:58:04 -05:00
morluto 1d1c7f98ff test(ci): repair verifier fixtures and cleanup assertions 2026-11-10 17:58:04 -05:00
morluto b1f1b3eff3 fix(javascript): recognize inherited environment for NaN 2026-11-10 17:58:04 -05:00
morluto c1d0f2bb45 fix(environment): retain response cleanup through shutdown 2026-11-10 17:58:04 -05:00
morluto 04de5f3200 fix(artifacts): recover interrupted reader preparation 2026-11-10 17:58:04 -05:00
morluto 199b006cc3 fix(javascript): retain semantic graph evidence and coverage 2026-11-10 17:32:25 -05:00
morluto db1db1a965 fix(javascript): preserve bundler identities and helper bindings 2026-11-10 17:32:25 -05:00
morluto bf7f518e2d fix(javascript): preserve semantic facts across writes and implicit effects 2026-11-10 17:32:25 -05:00
95 changed files with 4874 additions and 871 deletions

No files matched your search

+14 -3
View File
@@ -25,7 +25,7 @@ jobs:
node-version-file: .nvmrc
- name: Validate the SDK pin against the provider's verified release
run: node scripts/verify-ghidra-bridge-sources.mjs --check-sdk-pin >> "$GITHUB_ENV"
# The bridge probe imports only source-owned ARM/Thumb bytes and never
# The bridge probe imports only source-owned ARM, Thumb and MIPS bytes and never
# executes target code. All SDK/project state stays in runner storage.
- name: Acquire checksum-verified Ghidra SDK in isolated runner storage
shell: bash
@@ -46,7 +46,7 @@ jobs:
java_home_variable="JAVA_HOME_${GHIDRA_JAVA_MAJOR}_X64"
test -n "${!java_home_variable}"
JAVA_HOME="${!java_home_variable}" npm run verify:ghidra:bridge-sources -- --verified-sdk
- name: Inspect real ARM and Thumb instruction context through the bridge
- name: Inspect real ARM, Thumb and MIPS instruction context through the bridge
shell: bash
run: |
java_home_variable="JAVA_HOME_${GHIDRA_JAVA_MAJOR}_X64"
@@ -58,10 +58,21 @@ jobs:
-import "${probe_root}/arm-thumb.bin" -loader BinaryLoader -processor ARM:LE:32:v7 \
-noanalysis -readOnly -deleteProject \
-scriptPath "${GITHUB_WORKSPACE}/tests/conformance/ghidra;${GITHUB_WORKSPACE}/bridge/ghidra" \
-postScript ReaArmContextProbe.java "${GITHUB_WORKSPACE}/bridge/ghidra" \
-postScript ReaInstructionContextProbe.java "${GITHUB_WORKSPACE}/bridge/ghidra" \
> "${probe_root}/output.log" 2>&1 || { cat "${probe_root}/output.log"; exit 1; }
cat "${probe_root}/output.log"
grep -F 'REA_ARM_CONTEXT_OK arm=0 thumb=1 unavailable=null (injected provider value)' "${probe_root}/output.log"
mkdir -p "${probe_root}/mips-project"
# MIPS32 addiu v0,a0,3; MIPS16 li v0,90. These bytes are never executed.
printf '\003\000\202\044\132\152\000\000' > "${probe_root}/mips.bin"
timeout 180s "${GHIDRA_INSTALL_DIR}/support/analyzeHeadless" "${probe_root}/mips-project" MipsContext \
-import "${probe_root}/mips.bin" -loader BinaryLoader -processor MIPS:LE:32:default \
-noanalysis -readOnly -deleteProject \
-scriptPath "${GITHUB_WORKSPACE}/tests/conformance/ghidra;${GITHUB_WORKSPACE}/bridge/ghidra" \
-postScript ReaInstructionContextProbe.java "${GITHUB_WORKSPACE}/bridge/ghidra" mips \
> "${probe_root}/mips-output.log" 2>&1 || { cat "${probe_root}/mips-output.log"; exit 1; }
cat "${probe_root}/mips-output.log"
grep -F 'REA_MIPS_CONTEXT_OK standard=0 mips16=1' "${probe_root}/mips-output.log"
# Each probe imports source-owned bytes at 0x1000, away from the Cortex-M
# vector symbols that ARMCortex.pspec defines at 0, and decodes them from
# the language's default context, so M-profile Thumb state must hold at
+1
View File
@@ -26,6 +26,7 @@ for new work, roadmap granularity and producer-owned contracts.
- [Native call source coordinates](native/call-source-locations/spec.md) ([plan](native/call-source-locations/plan.md), [tasks](native/call-source-locations/tasks.md), [verification](native/call-source-locations/verification.md))
- [Go static string-header literal recovery](native/go-string-literals/spec.md) ([plan](native/go-string-literals/plan.md), [tasks](native/go-string-literals/tasks.md))
- [Windows Shell Link shortcut inspection](native/windows-shortcut/spec.md) ([plan](native/windows-shortcut/plan.md), [tasks](native/windows-shortcut/tasks.md))
- [NTFS $MFT and USN Journal inspection](native/ntfs-mft/spec.md) ([plan](native/ntfs-mft/plan.md), [tasks](native/ntfs-mft/tasks.md))
- [Solidity terminal metadata inspection](solidity/cbor-metadata/spec.md) ([plan](solidity/cbor-metadata/plan.md), [tasks](solidity/cbor-metadata/tasks.md), [verification](solidity/cbor-metadata/verification.md))
- [Inspect a V8 code cache container](v8/code-cache-inspection/spec.md) ([plan](v8/code-cache-inspection/plan.md), [tasks](v8/code-cache-inspection/tasks.md))
- [Deno compile executable inventory and extraction](artifacts/deno-standalone/spec.md) ([plan](artifacts/deno-standalone/plan.md), [tasks](artifacts/deno-standalone/tasks.md))
+39
View File
@@ -0,0 +1,39 @@
# Implementation plan
## Boundary and contract
Follow the existing artifact readers. `src/domain/windows/ntfsMft.ts` owns FILE
record bytes and coverage; `src/domain/artifactFormatMetadata.ts` owns the
`ntfs-mft` result schema. `NtfsMftArtifactReader` reads the admitted file
through `ArtifactFileSource` and exposes its facts through `formatMetadata()`.
A record stream is its own content, so the root occurrence carries every fact
and no members are enumerated.
Recognition lives in `hasMftFileRecordSignature` and is applied in
`classifyArtifactBytes`, next to the other unambiguous magic probes.
## Incremental work and validation
1. Define the first-step contract, limits and observations; add parser fixtures.
2. Implement the FILE record header and update-sequence fixup parse with
explicit limitations.
3. Add the reader, format descriptor and byte classification.
4. Decode attribute records, then `$FILE_NAME` attributes so a record's parent
reference, names and timestamps become explicit; keep hard links separate.
5. Parse `$UsnJrnl:$J` records and correlate them to records by full file
reference, including sequence numbers, reporting unresolved references.
6. Document the workflow in a native Windows artifact guide.
## Effects and compatibility
The workflow reads one caller-selected local file. It adds no target execution,
injection, mutation, protection recovery, network access, volume mount or live
filesystem access, so no maintainer scope decision is required.
`inspect_artifact` routing for every existing format is unchanged, and the new
format id appears only for bytes whose first record carries the `FILE`
signature.
## References
- https://learn.microsoft.com/en-us/windows/win32/fileio/master-file-table
- https://github.com/libyal/libfsntfs/blob/main/documentation/New%20Technologies%20File%20System%20(NTFS).asciidoc
+51
View File
@@ -0,0 +1,51 @@
# NTFS $MFT and USN Journal inspection
This specification records an implementation increment and its original problem
statement. For current behavior and limitations, see the artifact inventory
guide.
## Problem and outcome
REA has no reader for acquired NTFS metadata artifacts. An analyst holding a
captured `$MFT` stream cannot ask which FILE records it contains, whether their
update-sequence fixups survived acquisition, or which records are directories,
extensions of another record, or no longer in use. Add offline, read-only
inventory of a selected `$MFT` file, delivered in steps so each one merges on
its own.
## Requirements
- Recognize a `$MFT` stream by a plausible first `FILE` record header under
any file name. CLI and MCP target admission route it to the artifact reader.
- Validate each FILE record header: the `FILE` signature, sequence number, flags,
used and total byte counts, base record reference, and the update-sequence
array's offset, count and array bounds.
- Verify the update-sequence fixups: every sector trailer the array covers must
hold the array's sequence number. A mismatch keeps the record's header facts
and is reported per record and in the metadata count.
- Report a missing, misplaced or short update-sequence array as unverified
rather than as a clean record, and a trailing partial record as a byte count.
- Report the first invalid record signature and the number of invalid slots.
Continue at the next slot when the stream declared a supported record size.
With a fallback record size, stop at the first invalid signature.
- Bound the whole file at a fixed 512 MiB ceiling and report the ceiling as a
`limit` failure with its boundary.
- Return the first 32 decoded record headers with their source offsets and the
number of omitted headers. Aggregate counts cover the full parsed stream.
- Parse only the records' headers and fixups. Attributes, filenames, data
streams and the `$UsnJrnl:$J` journal are not decoded in this step, so no path,
file size, file reference correlation or change history is claimed.
## Acceptance
Synthesized fixtures cover a clean record, a failed fixup, a missing update
sequence array, an array shorter than the record's sector count, extension and
directory records, a record whose signature is wrong mid-stream, a trailing
partial record, an undeclared record size, a 4 KiB record, a stream shorter than
one header, bytes without the `FILE` signature, and the file-size ceiling.
Inventory tests cover the root occurrence's metadata and the coverage status
when a fixup fails, and classification tests cover several file names for the
same bytes.
Not covered in this step: real `$MFT` files written by a Windows host, USN
journal records, attribute decoding, and any correlation between the two.
+11
View File
@@ -0,0 +1,11 @@
# Tasks
- [x] Define the first-step scope, contract and limits.
- [x] Parse FILE record headers and update-sequence fixups with synthesized
fixtures and explicit limitations.
- [x] Add the reader, format descriptor and byte classification.
- [ ] Decode attribute records, then `$FILE_NAME` attributes with parent
references, names, hard links and timestamps.
- [ ] Parse `$UsnJrnl:$J` records and correlate them to records by full file
reference.
- [ ] Verify against real `$MFT` and journal captures from a Windows host.
+1 -1
View File
@@ -636,7 +636,7 @@ const PATH_SCOPES = {
"src/ghidra/GhidraJavaProperties.ts",
"src/ghidra/GhidraReleasePolicy.ts",
"scripts/verify-ghidra-bridge-sources.mjs",
"tests/conformance/ghidra/ReaArmContextProbe.java",
"tests/conformance/ghidra/ReaInstructionContextProbe.java",
"tests/conformance/ghidra/ReaComputedCallCoverageProbe.java",
"tests/conformance/ghidra/ReaDecodeProbe.java",
".github/workflows/ghidra-bridge-sources.yml",
+14 -13
View File
@@ -50,7 +50,7 @@ transport.stderr?.on("data", (bytes) => {
const ajv = new Ajv2020({ strict: false, validateFormats: false });
const measurements = [];
const contract = toolContract("inspect_web_network_capture");
await withVerifierCleanup(async () => {
const report = await withVerifierCleanup(async () => {
try {
await client.connect(transport);
} catch (cause) {
@@ -196,7 +196,7 @@ await withVerifierCleanup(async () => {
});
}
await client.ping();
const report = {
return {
status: "passed",
profile: "aioquic-1.5.0-json-0.3",
measurements,
@@ -206,17 +206,6 @@ await withVerifierCleanup(async () => {
"No PCAP or keylog is read by this verifier; repeated STREAM coverage is not proof of retransmission or network loss.",
],
};
await writeFile(reportPath, JSON.stringify(report, null, 2) + "\n", {
flag: "wx",
mode: 0o600,
});
console.log(
JSON.stringify({
status: report.status,
cases: measurements.length,
report: reportPath,
}),
);
}, [
async () => {
await closeVerifierResources([
@@ -225,3 +214,15 @@ await withVerifierCleanup(async () => {
]);
},
]);
await writeFile(reportPath, JSON.stringify(report, null, 2) + "\n", {
flag: "wx",
mode: 0o600,
});
console.log(
JSON.stringify({
status: report.status,
cases: measurements.length,
report: reportPath,
}),
);
+3
View File
@@ -97,6 +97,7 @@ import {
isDfuSeFile,
} from "../domain/firmware/dfuse.js";
import { hasFitImagesNode } from "../domain/firmware/fitImage.js";
import { hasMftFileRecordHeader } from "../domain/windows/ntfsMft.js";
import {
locateEspFlashLayout,
type EspFlashSelection,
@@ -816,6 +817,7 @@ const detectArtifactFormat = async (
});
if (esp === "esp-image") return "esp-image";
if (hasFitImagesNode(magic)) return "fit";
if (hasMftFileRecordHeader(magic)) return "ntfs-mft";
// Binary plists, including keyed archives, often use other extensions.
if (hasBinaryPlistSignature(magic)) return "plist";
const named = namedArtifactFormat(lower);
@@ -899,6 +901,7 @@ const isArchiveFormat = (
"dfuse",
"esp-flash-image",
"fit",
"ntfs-mft",
].includes(format);
/** Whether the leading bytes begin any executable header family, valid or not. */
@@ -135,6 +135,7 @@ const TARGETS: TargetFixtures = {
espFlash: { selected: true, flashOffset: null, partitionTableOffset: null },
},
fit: { ...archive, format: "fit" },
"ntfs-mft": { ...archive, format: "ntfs-mft" },
plist: { ...artifact, format: "plist" },
javascript: { ...artifact, format: "javascript" },
"source-map": { ...artifact, format: "source-map" },
@@ -10,6 +10,7 @@ import {
addSemanticGraphUnknown,
} from "./JavaScriptSemanticGraphConstruction.js";
import {
inferredSemanticEvidenceAt,
unknownSemanticEvidence,
builtinSemanticNode,
builtinSemanticRelation,
@@ -39,6 +40,7 @@ export const projectSemanticEvents = (
source: eventEmitterNode(context, operation),
target: event,
relation,
evidence: inferredSemanticEvidenceAt(context.file, operation.location),
resolution:
operation.resolution === "complete" ? "resolved" : "candidate",
});
@@ -48,6 +50,7 @@ export const projectSemanticEvents = (
source: event,
target: listener,
relation,
evidence: inferredSemanticEvidenceAt(context.file, operation.location),
resolution:
operation.resolution === "complete" ? "resolved" : "candidate",
});
@@ -480,10 +480,8 @@ export const createJavaScriptSemanticGraphProjection = (
limit,
);
}
state.fileNodeBudget = Math.min(
fileNodeCeiling,
Math.max(0, nodeCeiling - state.nodes.size),
);
const remainingTreeBudget = Math.max(0, nodeCeiling - state.nodes.size);
state.fileNodeBudget = Math.min(fileNodeCeiling, remainingTreeBudget);
const module = retainNode(state, file, {
kind: "module",
roleKey: "module",
@@ -493,6 +491,14 @@ export const createJavaScriptSemanticGraphProjection = (
});
state.fileNodeBudget = null;
if (module !== null) state.roots.add(module.node_id);
else {
truncatedFiles += 1;
noteSemanticCeilingReached(
nodeCeilingsReached,
remainingTreeBudget,
fileNodeCeiling,
);
}
const evidence = {
...observedSemanticEvidence(file, null),
authority: "unknown" as const,
@@ -10,6 +10,7 @@ import {
addSemanticGraphUnknown,
} from "./JavaScriptSemanticGraphConstruction.js";
import {
inferredSemanticEvidenceAt,
builtinSemanticNode,
builtinSemanticRelation,
unknownSemanticEvidence,
@@ -135,7 +136,7 @@ export const projectSemanticRequests = (
if (request === undefined) continue;
if (operation.kind === "request")
projectRequestConstruction(context, operation, request);
else projectResponseConsumer(context, operation, request);
else projectResponseConsumer(context, operation, request, requestNodes);
if (operation.resolution !== "complete")
addDataUnknown({
context,
@@ -224,16 +225,31 @@ const projectResponseConsumer = (
context: SemanticFlowProjectionContext,
operation: JavaScriptSemanticRequestOperation,
response: JavaScriptSemanticGraphNode,
requestNodes: ReadonlyMap<string, JavaScriptSemanticGraphNode>,
): void => {
const authority = builtinSemanticRelation(
context.file,
operation,
operation.resolution === "complete" ? "resolved" : "candidate",
);
const evidence =
authority.evidence ??
inferredSemanticEvidenceAt(context.file, operation.location);
for (const requestId of operation.linkedRequestIds)
addSemanticGraphRelation(context.state, {
source: requestNodes.get(requestId),
target: response,
relation: "consumed-by",
...authority,
evidence,
properties: { method: operation.method },
});
addSemanticGraphRelation(context.state, {
source: response,
target: context.callSiteAt(operation.location),
relation: "consumed-by",
...builtinSemanticRelation(
context.file,
operation,
operation.resolution === "complete" ? "resolved" : "candidate",
),
...authority,
evidence,
properties: { method: operation.method },
});
};
@@ -14,7 +14,10 @@ import {
addSemanticGraphRelation,
addSemanticGraphUnknown,
} from "./JavaScriptSemanticGraphConstruction.js";
import { unknownSemanticEvidence } from "./JavaScriptSemanticGraphEvidence.js";
import {
inferredSemanticEvidenceAt,
unknownSemanticEvidence,
} from "./JavaScriptSemanticGraphEvidence.js";
import type { SemanticFlowProjectionContext } from "./JavaScriptSemanticGraphFlowProjection.js";
import { semanticPropertySlot } from "./JavaScriptSemanticGraphValueProjection.js";
@@ -47,6 +50,7 @@ export function* projectSemanticObjects(
...ends,
relation: OBJECT_RELATIONS[operation.kind],
resolution: resolved ? "resolved" : "candidate",
evidence: inferredSemanticEvidenceAt(context.file, operation.location),
});
if (!resolved)
addObjectUnknown(context, operation, target.slot ?? occurrence, target);
@@ -340,3 +340,37 @@ it("bounds reads within unmodeled nested operator operands", () => {
);
expect(trace.status).not.toBe("found");
});
it.each([1, 2])(
"reports unavailable-module node omission only when total capacity %s rejects it",
(totalNodes) => {
const projection = createJavaScriptSemanticGraphProjection({
totalNodes,
fileNodes: 10,
});
const retained = source("retained.js", "");
projection.projectFile(retained, analyzeJavaScriptSemantics(""));
projection.recordUnavailableFile(
source("unavailable.js"),
"Source unavailable",
null,
"incomplete-module",
);
const graph = projection.finish(digest, application);
expect(graph.nodes).toHaveLength(totalNodes);
expect(graph.unknowns).toHaveLength(12);
expect(graph.coverage).toMatchObject({
status: "partial",
truncated: totalNodes === 1,
omitted_nodes: null,
});
if (totalNodes === 1) {
expect(graph.coverage.limits).toContainEqual({
name: "max_semantic_nodes",
value: 1,
unit: "items",
});
expect(graph.limitations.join("\n")).toContain("max_semantic_nodes is 1");
} else expect(graph.coverage.limits).toEqual([]);
},
);
+1
View File
@@ -215,6 +215,7 @@ export interface ArtifactReader {
| "dfuse"
| "esp-flash-image"
| "fit"
| "ntfs-mft"
| "file";
entries(
signal?: AbortSignal,
+38 -9
View File
@@ -18,7 +18,10 @@ import {
type ArtifactTraversalOptions,
} from "./ArtifactReader.js";
import { ArtifactResourceScope } from "./ArtifactResourceScope.js";
import { assertArtifactReadActive } from "./ArtifactFileOperation.js";
import {
artifactReadCancelledBy,
assertArtifactReadActive,
} from "./ArtifactFileOperation.js";
import { ArtifactFileSource } from "./ArtifactFileSource.js";
import { readFileHandleChunks } from "../filesystem/readFileHandleChunks.js";
import type { StableRegularFileDescriptor } from "../filesystem/RegularFile.js";
@@ -64,12 +67,14 @@ export const regionFileName = (start: number, end: number): string =>
*/
export class DecodedRegionArtifactReader<Image> implements ArtifactReader {
readonly #scope = new ArtifactResourceScope();
readonly #entries = new Map<string, ArtifactEntry>();
readonly #regions = new Map<string, Uint8Array>();
#entries = new Map<string, ArtifactEntry>();
#regions = new Map<string, Uint8Array>();
readonly #streams = new Set<Readable>();
readonly #source: ArtifactFileSource;
readonly #decoding: RegionDecoding<Image>;
#prepared: Promise<void> | undefined;
#prepared:
| { signal: AbortSignal | undefined; promise: Promise<void> }
| undefined;
/** Whole-file facts only; the region map is the sole owner of region bytes. */
#described: Omit<DecodedRegions, "regions"> | undefined;
#closing = false;
@@ -110,8 +115,27 @@ export class DecodedRegionArtifactReader<Image> implements ArtifactReader {
): AsyncIterable<ArtifactEntry> {
await this.#scope.run(async () => {
this.#assertOpen(signal);
this.#prepared ??= this.#prepare(signal);
await this.#prepared;
for (;;) {
const pending = (this.#prepared ??= {
signal,
promise: this.#prepare(signal),
});
try {
await pending.promise;
break;
} catch (cause: unknown) {
if (this.#prepared === pending) this.#prepared = undefined;
if (
pending.signal !== signal &&
signal?.aborted !== true &&
artifactReadCancelledBy(cause, pending.signal)
) {
this.#assertOpen(signal);
continue;
}
throw cause;
}
}
await this.#source.read(signal);
});
const coverage = this.#described?.coverage;
@@ -230,6 +254,8 @@ export class DecodedRegionArtifactReader<Image> implements ArtifactReader {
async #prepare(signal?: AbortSignal): Promise<void> {
const described = this.#decoding.describe(await this.#decode(signal));
const entries = new Map<string, ArtifactEntry>();
const regions = new Map<string, Uint8Array>();
let index = 0;
for (const region of described.regions) {
if (index % 256 === 0) await setImmediate();
@@ -248,13 +274,16 @@ export class DecodedRegionArtifactReader<Image> implements ArtifactReader {
adapterKey: `${this.#decoding.keyPrefix}:${String(index)}`,
formatMetadata: region.formatMetadata,
} satisfies ArtifactEntry;
this.#entries.set(entry.adapterKey, entry);
this.#regions.set(entry.adapterKey, region.bytes);
entries.set(entry.adapterKey, entry);
regions.set(entry.adapterKey, region.bytes);
index += 1;
}
const { regions: _regions, ...facts } = described;
this.#described = facts;
await this.#source.read(signal);
this.#assertOpen(signal);
this.#entries = entries;
this.#regions = regions;
this.#described = facts;
}
async #decode(signal?: AbortSignal): Promise<Image> {
+40
View File
@@ -1,3 +1,4 @@
import { artifactPreparationRecoveryCases } from "../../tests/fixtures/artifactReaderLifecycle.js";
import { getEventListeners, once } from "node:events";
import { readFile, truncate, writeFile } from "node:fs/promises";
import { join } from "node:path";
@@ -404,6 +405,38 @@ it("keeps the cancellation reason before admission", async () => {
}
});
it("does not publish FIT entries when final source verification fails", async () => {
const path = await writeFit("partial-publication.itb", FIT_EMBEDDED);
const control = new FitArtifactReader(path);
const reader = new FitArtifactReader(path);
const expected = await firstEntry(control);
const nativeRead = ArtifactFileSource.prototype.read;
const physical = Object.assign(
new Error("final source verification failed"),
{ code: "EIO" },
);
let calls = 0;
const read = vi
.spyOn(ArtifactFileSource.prototype, "read")
.mockImplementation(function (this: ArtifactFileSource, ...args) {
calls += 1;
if (calls === 4) return Promise.reject(physical);
return nativeRead.apply(this, args);
});
try {
await expect(firstEntry(reader)).rejects.toBe(physical);
await expect(reader.open(expected)).rejects.toMatchObject({
reason: "integrity",
});
read.mockRestore();
expect((await firstEntry(reader)).path).toBe(expected.path);
} finally {
read.mockRestore();
await reader.close();
await control.close();
}
});
it("keeps the cancellation reason during hash verification", async () => {
const reader = new FitArtifactReader(
await writeFit("hashing.itb", LARGE_FIT),
@@ -430,6 +463,8 @@ it("keeps the cancellation reason during hash verification", async () => {
reader.entries(controller.signal)[Symbol.asyncIterator]().next(),
).rejects.toMatchObject({ reason: "cancelled", cause: reason });
expect(observer).toHaveBeenCalledWith("sha256");
observer.mockRestore();
expect((await firstEntry(reader)).kind).toBe("file");
} finally {
observer.mockRestore();
await reader.close();
@@ -569,3 +604,8 @@ it("preserves a final validation failure when cancellation overlaps it", async (
await reader.close();
}
});
artifactPreparationRecoveryCases({
image: () => FIT_EMBEDDED,
createReader: (path, admitted) => new FitArtifactReader(path, admitted),
});
+34 -7
View File
@@ -13,7 +13,10 @@ import {
type ArtifactTraversalOptions,
} from "./ArtifactReader.js";
import { ArtifactResourceScope } from "./ArtifactResourceScope.js";
import { assertArtifactReadActive } from "./ArtifactFileOperation.js";
import {
artifactReadCancelledBy,
assertArtifactReadActive,
} from "./ArtifactFileOperation.js";
import { ArtifactFileSource } from "./ArtifactFileSource.js";
import {
createFitDigest,
@@ -30,11 +33,13 @@ const FDT_HEADER_BYTES = 40;
export class FitArtifactReader implements ArtifactReader {
readonly format = "fit" as const;
readonly #scope = new ArtifactResourceScope();
readonly #entries = new Map<string, ArtifactEntry>();
#entries = new Map<string, ArtifactEntry>();
readonly #streams = new Set<Readable>();
readonly #path: string;
readonly #source: ArtifactFileSource;
#prepared: Promise<void> | undefined;
#prepared:
| { signal: AbortSignal | undefined; promise: Promise<void> }
| undefined;
#metadata: FitMetadata | undefined;
#coverage: ArtifactReaderTraversal | undefined;
#closing = false;
@@ -66,8 +71,27 @@ export class FitArtifactReader implements ArtifactReader {
): AsyncIterable<ArtifactEntry> {
await this.#scope.run(async () => {
this.#assertOpen(signal);
this.#prepared ??= this.#prepare(signal);
await this.#prepared;
for (;;) {
const pending = (this.#prepared ??= {
signal,
promise: this.#prepare(signal),
});
try {
await pending.promise;
break;
} catch (cause: unknown) {
if (this.#prepared === pending) this.#prepared = undefined;
if (
pending.signal !== signal &&
signal?.aborted !== true &&
artifactReadCancelledBy(cause, pending.signal)
) {
this.#assertOpen(signal);
continue;
}
throw cause;
}
}
await this.#source.read(signal);
});
if (this.#coverage !== undefined)
@@ -207,6 +231,7 @@ export class FitArtifactReader implements ArtifactReader {
this.#path,
initial.size,
);
const entries = new Map<string, ArtifactEntry>();
let index = 0;
for (const image of structure.images) {
if (index % 256 === 0) await setImmediate();
@@ -228,15 +253,17 @@ export class FitArtifactReader implements ArtifactReader {
adapterKey: `fit:${String(image.index)}`,
formatMetadata: image.metadata,
} satisfies ArtifactEntry;
this.#entries.set(entry.adapterKey, entry);
entries.set(entry.adapterKey, entry);
}
await this.#source.read(signal);
this.#assertOpen(signal);
this.#entries = entries;
this.#metadata = structure.metadata;
if (structure.limitations.length > 0)
this.#coverage = {
status: "complete",
limitations: structure.limitations,
};
await this.#source.read(signal);
}
/** One streamed pass per image feeds every supported digest. */
@@ -1,3 +1,6 @@
import { artifactPreparationRecoveryCases } from "../../tests/fixtures/artifactReaderLifecycle.js";
import { ArtifactFileSource } from "./ArtifactFileSource.js";
import { openRegularFile } from "../filesystem/RegularFile.js";
import { writeFile } from "node:fs/promises";
import { join } from "node:path";
@@ -159,6 +162,72 @@ it("settles an active region stream when closed", async () => {
}
});
it("retries decoded-region preparation cancelled while reading", async () => {
const path = await writeImage("retry.hex", INTEL_HEX_I32_GAP);
const admitted = await openRegularFile(path, { symlinks: "reject" });
const reader = new HexRecordArtifactReader(path, "intel-hex", admitted);
const controller = new AbortController();
const reason = { stage: "decoded region read" };
const nativeRead = admitted.handle.read.bind(admitted.handle);
const read = vi.spyOn(admitted.handle, "read").mockImplementation((async (
...args: unknown[]
) => {
const result = await Reflect.apply(nativeRead, admitted.handle, args);
controller.abort(reason);
return result;
}) as typeof admitted.handle.read);
try {
await expect(
reader.entries(controller.signal)[Symbol.asyncIterator]().next(),
).rejects.toMatchObject({ reason: "cancelled", cause: reason });
read.mockRestore();
expect(
(await reader.entries()[Symbol.asyncIterator]().next()).value,
).toMatchObject({ kind: "file" });
} finally {
read.mockRestore();
await reader.close();
await admitted.handle.close();
}
});
it("does not publish decoded regions when final source verification fails", async () => {
const path = await writeImage("partial.hex", INTEL_HEX_I32_GAP);
const control = new HexRecordArtifactReader(path, "intel-hex");
const reader = new HexRecordArtifactReader(path, "intel-hex");
const expected = (await control.entries()[Symbol.asyncIterator]().next())
.value!;
const nativeRead = ArtifactFileSource.prototype.read;
const physical = Object.assign(
new Error("final source verification failed"),
{ code: "EIO" },
);
let calls = 0;
const read = vi
.spyOn(ArtifactFileSource.prototype, "read")
.mockImplementation(function (this: ArtifactFileSource, ...args) {
calls += 1;
if (calls === 2) return Promise.reject(physical);
return nativeRead.apply(this, args);
});
try {
await expect(reader.entries()[Symbol.asyncIterator]().next()).rejects.toBe(
physical,
);
await expect(reader.open(expected)).rejects.toMatchObject({
reason: "integrity",
});
read.mockRestore();
expect(
(await reader.entries()[Symbol.asyncIterator]().next()).value,
).toMatchObject({ path: expected.path });
} finally {
read.mockRestore();
await reader.close();
await control.close();
}
});
it("honors cancellation before admitting a file with the exact reason", async () => {
const reader = new HexRecordArtifactReader(
await writeImage("cancel.hex", INTEL_HEX_I32_GAP),
@@ -337,3 +406,9 @@ it("bounds copied region chunks and isolates cached bytes across opens", async (
await reader.close();
}
});
artifactPreparationRecoveryCases({
image: () => Buffer.from(INTEL_HEX_I32_GAP),
createReader: (path, admitted) =>
new HexRecordArtifactReader(path, "intel-hex", admitted),
});
+211
View File
@@ -0,0 +1,211 @@
import { writeFile } from "node:fs/promises";
import { join } from "node:path";
import { expect, it, vi } from "vitest";
import { ArtifactFileSource } from "./ArtifactFileSource.js";
import { assertArtifactReadActive } from "./ArtifactFileOperation.js";
import { NtfsMftArtifactReader } from "./NtfsMftArtifactReader.js";
import { MFT_DEFAULT_RECORD_BYTES } from "../domain/windows/ntfsMft.js";
import { createTestTempDirectory } from "../../tests/fixtures/temporaryDirectory.js";
import { scanCanonicalArtifactInventory } from "../../tests/fixtures/artifactInventory.js";
/** A 1 KiB FILE record whose update sequence array covers both sectors. */
const recordBytes = (options: {
readonly sequenceNumber?: number;
readonly flags?: number;
readonly index?: number;
}): Buffer => {
const record = Buffer.alloc(MFT_DEFAULT_RECORD_BYTES);
record.write("FILE", 0, "latin1");
const sequenceNumber = options.sequenceNumber ?? 1;
record.writeUInt16LE(48, 4);
record.writeUInt16LE(3, 6);
record.writeUInt16LE(sequenceNumber, 48);
record.writeUInt16LE(0xa5a5, 50);
record.writeUInt16LE(0xa5a5, 52);
record.writeUInt16LE(sequenceNumber, 16);
record.writeUInt16LE(1, 18);
record.writeUInt32LE(options.index ?? 0, 44);
record.writeUInt16LE(56, 20);
record.writeUInt16LE(options.flags ?? 0x0001, 22);
record.writeUInt32LE(64, 24);
record.writeUInt32LE(0xffff_ffff, 56);
record.writeUInt32LE(MFT_DEFAULT_RECORD_BYTES, 28);
record.writeUInt16LE(sequenceNumber, 510);
record.writeUInt16LE(sequenceNumber, 1022);
return record;
};
const writeMft = async (name: string, bytes: Buffer): Promise<string> => {
const directory = await createTestTempDirectory("rea-ntfs-mft-");
const path = join(directory, name);
await writeFile(path, bytes);
return path;
};
it("reports a $MFT stream's record headers as the root occurrence", async () => {
const inventory = await scanCanonicalArtifactInventory(
await writeMft(
"$MFT",
Buffer.concat([
recordBytes({}),
recordBytes({ flags: 0x0003, index: 1 }),
]),
),
);
expect(inventory.manifest.root_format).toBe("ntfs-mft");
expect(inventory.coverage.status).toBe("complete");
// A record stream declares no members, so the root is its whole inventory.
expect(inventory.occurrences.map((row) => row.logical_path)).toEqual(["."]);
expect(inventory.occurrences[0]).toMatchObject({
artifact_format: "ntfs-mft",
format_metadata: {
kind: "ntfs-mft",
record_bytes: MFT_DEFAULT_RECORD_BYTES,
record_count: 2,
in_use_record_count: 2,
directory_record_count: 1,
extension_record_count: 0,
fixup_mismatch_count: 0,
fixup_unverified_record_count: 0,
truncated_tail_bytes: 0,
},
});
});
it("names a failed fixup in the coverage limitations", async () => {
const record = recordBytes({ sequenceNumber: 4 });
// A trailer that is not the sequence number marks the sector as not cleanly written.
record.writeUInt16LE(0x1234, 510);
const inventory = await scanCanonicalArtifactInventory(
await writeMft("$MFT", record),
);
expect(inventory.manifest.root_format).toBe("ntfs-mft");
expect(inventory.coverage.status).toBe("unknown");
expect(inventory.limitations).toContainEqual(
expect.stringMatching(/sector trailer that is not the update sequence/u),
);
});
it("reports an unaligned used-byte count as incomplete coverage", async () => {
const record = recordBytes({});
record.writeUInt32LE(57, 24);
const inventory = await scanCanonicalArtifactInventory(
await writeMft("$MFT", record),
);
expect(inventory.coverage.status).toBe("unknown");
expect(inventory.occurrences[0]?.format_metadata).toMatchObject({
record_headers: [{ usedBytes: 57, extentValid: false }],
});
});
it("reports incomplete coverage when FILE extents contradict the record slots", async () => {
const usedTooLarge = recordBytes({});
usedTooLarge.writeUInt32LE(4096, 24);
const totalTooSmall = recordBytes({});
totalTooSmall.writeUInt32LE(512, 28);
const inventory = await scanCanonicalArtifactInventory(
await writeMft("$MFT", Buffer.concat([usedTooLarge, totalTooSmall])),
);
expect(inventory.coverage.status).toBe("unknown");
expect(inventory.limitations).toContainEqual(
"2 FILE record(s) declare inconsistent record extents, beginning at offset 0",
);
expect(inventory.occurrences[0]?.format_metadata).toMatchObject({
record_count: 2,
fixup_mismatch_count: 0,
fixup_unverified_record_count: 0,
});
});
it("retries after final source verification fails without publishing its metadata", async () => {
const reader = new NtfsMftArtifactReader(
await writeMft("$MFT", recordBytes({})),
);
const read = ArtifactFileSource.prototype.read;
const spy = vi
.spyOn(ArtifactFileSource.prototype, "read")
.mockImplementationOnce(function (this: ArtifactFileSource, signal) {
return read.call(this, signal);
})
.mockRejectedValueOnce(new Error("final source verification failed"));
try {
await expect(
reader.entries()[Symbol.asyncIterator]().next(),
).rejects.toThrow("final source verification failed");
expect(reader.formatMetadata()).toBeUndefined();
await expect(
reader.entries()[Symbol.asyncIterator]().next(),
).resolves.toMatchObject({
done: true,
});
expect(reader.formatMetadata()).toMatchObject({
kind: "ntfs-mft",
record_count: 1,
});
} finally {
spy.mockRestore();
await reader.close();
}
});
it("retries a shared preparation cancelled by another traversal", async () => {
const reader = new NtfsMftArtifactReader(
await writeMft("$MFT", recordBytes({})),
);
const controller = new AbortController();
let entered: () => void = () => {};
const started = new Promise<void>((resolve) => {
entered = resolve;
});
const read = ArtifactFileSource.prototype.read;
const spy = vi
.spyOn(ArtifactFileSource.prototype, "read")
.mockImplementationOnce(async function (this: ArtifactFileSource, signal) {
entered();
await new Promise<void>((resolve) => {
signal?.addEventListener("abort", () => resolve(), { once: true });
});
assertArtifactReadActive(signal);
return read.call(this, signal);
});
try {
const cancelled = reader
.entries(controller.signal)
[Symbol.asyncIterator]()
.next();
const cancelledResult = expect(cancelled).rejects.toMatchObject({
reason: "cancelled",
});
await started;
const other = reader.entries()[Symbol.asyncIterator]().next();
controller.abort("cancelled first traversal");
await cancelledResult;
await expect(other).resolves.toMatchObject({ done: true });
expect(reader.formatMetadata()).toMatchObject({ record_count: 1 });
} finally {
controller.abort();
spy.mockRestore();
await reader.close();
}
});
it("revalidates the source before reusing a completed header inventory", async () => {
const path = await writeMft("$MFT", recordBytes({}));
const reader = new NtfsMftArtifactReader(path);
try {
await reader.entries()[Symbol.asyncIterator]().next();
await writeFile(
path,
Buffer.concat([recordBytes({}), recordBytes({ index: 1 })]),
);
await expect(
reader.entries()[Symbol.asyncIterator]().next(),
).rejects.toMatchObject({ reason: "integrity" });
} finally {
await reader.close();
}
});
+174
View File
@@ -0,0 +1,174 @@
import type { Readable } from "node:stream";
import {
ntfsMftMetadataSchema,
type ArtifactFormatMetadata,
} from "../domain/artifactFormatMetadata.js";
import type { RegionDecodeFailure } from "../domain/firmware/regionDecoder.js";
import {
createMftDecoder,
type MftInventory,
} from "../domain/windows/ntfsMft.js";
import {
ArtifactReaderFailure,
type ArtifactEntry,
type ArtifactReader,
type ArtifactReaderTraversal,
type ArtifactTraversalOptions,
} from "./ArtifactReader.js";
import { ArtifactResourceScope } from "./ArtifactResourceScope.js";
import {
artifactReadCancelledBy,
assertArtifactReadActive,
} from "./ArtifactFileOperation.js";
import { ArtifactFileSource } from "./ArtifactFileSource.js";
import { readFileHandleChunks } from "../filesystem/readFileHandleChunks.js";
import type { StableRegularFileDescriptor } from "../filesystem/RegularFile.js";
/**
* An MFT stream is its own content: the FILE records describe the source
* volume, not members inside the stream, so header facts are attached to
* the root occurrence.
*/
export class NtfsMftArtifactReader implements ArtifactReader {
readonly format = "ntfs-mft" as const;
readonly #scope = new ArtifactResourceScope();
readonly #source: ArtifactFileSource;
#prepared:
| { signal: AbortSignal | undefined; promise: Promise<void> }
| undefined;
#metadata: ArtifactFormatMetadata | undefined;
#limitations: readonly string[] = [];
#closing = false;
constructor(path: string, admitted?: StableRegularFileDescriptor) {
this.#source = new ArtifactFileSource(path, "$MFT stream", admitted);
}
formatMetadata(): ArtifactFormatMetadata | undefined {
return this.#metadata === undefined
? undefined
: structuredClone(this.#metadata);
}
async prepareContainer(
expectedSha256: string,
signal?: AbortSignal,
): Promise<void> {
await this.#scope.run(async () => {
this.#assertOpen(signal);
await this.#source.bindSha256(expectedSha256, signal);
});
}
async *entries(
signal?: AbortSignal,
options?: ArtifactTraversalOptions,
): AsyncIterable<ArtifactEntry> {
await this.#scope.run(async () => {
this.#assertOpen(signal);
for (;;) {
const pending = (this.#prepared ??= {
signal,
promise: this.#prepare(signal),
});
try {
await pending.promise;
break;
} catch (cause: unknown) {
if (this.#prepared === pending) this.#prepared = undefined;
if (
pending.signal !== signal &&
signal?.aborted !== true &&
artifactReadCancelledBy(cause, pending.signal)
) {
this.#assertOpen(signal);
continue;
}
throw cause;
}
}
this.#assertOpen(signal);
await this.#source.read(signal);
});
// An MFT declares no members, so the root occurrence is the whole
// inventory; the coverage callback reports what its parse did not establish.
options?.onCoverage?.(coverageOf(this.#limitations));
}
async open(entry: ArtifactEntry, _signal?: AbortSignal): Promise<Readable> {
return this.#scope.run(async () => {
throw new ArtifactReaderFailure(
"unavailable",
`$MFT stream declares no member to open for ${entry.path}`,
);
});
}
async close(): Promise<void> {
this.#closing = true;
await this.#scope.close();
await this.#source.close();
this.#metadata = undefined;
}
provenance(): readonly [] {
return [];
}
#assertOpen(signal?: AbortSignal): void {
assertArtifactReadActive(signal);
if (this.#closing)
throw new ArtifactReaderFailure(
"unavailable",
"$MFT stream reader is closed",
);
}
async #prepare(signal?: AbortSignal): Promise<void> {
const image = await this.#decode(signal);
await this.#source.read(signal);
this.#metadata = ntfsMftMetadataSchema.parse(image.metadata);
this.#limitations = image.limitations;
}
async #decode(signal?: AbortSignal): Promise<MftInventory> {
const { handle, initial } = await this.#source.read(signal);
const decoder = createMftDecoder(initial.size);
if (!decoder.ok) throw this.#failure(decoder.error);
if (initial.size > 0)
for await (const chunk of readFileHandleChunks(handle, {
end: initial.size - 1,
signal,
cancellationFailure: (reason) =>
new ArtifactReaderFailure("cancelled", "$MFT stream read cancelled", {
cause: reason,
}),
}) as AsyncIterable<Buffer>) {
this.#assertOpen(signal);
const written = decoder.value.write(chunk);
if (!written.ok) throw this.#failure(written.error);
}
const image = decoder.value.finish();
if (!image.ok) throw this.#failure(image.error);
return image.value;
}
#failure(failure: RegionDecodeFailure): ArtifactReaderFailure {
return failure.kind === "format"
? new ArtifactReaderFailure(
"format",
`$MFT stream is malformed: ${failure.message}`,
)
: new ArtifactReaderFailure(
"limit",
`$MFT stream exceeds a fixed limit: ${failure.message}`,
failure.boundary === undefined ? {} : { boundary: failure.boundary },
);
}
}
const coverageOf = (limitations: readonly string[]): ArtifactReaderTraversal =>
limitations.length === 0
? { status: "complete", limitations: [] }
: { status: "unknown", limitations };
@@ -1,4 +1,7 @@
import { artifactReaderLifecycleCases } from "../../tests/fixtures/artifactReaderLifecycle.js";
import {
artifactPreparationRecoveryCases,
artifactReaderLifecycleCases,
} from "../../tests/fixtures/artifactReaderLifecycle.js";
import { writeFile } from "node:fs/promises";
import { join } from "node:path";
import { buffer } from "node:stream/consumers";
@@ -91,6 +94,14 @@ it("preserves cancellation during payload CRC admission", async () => {
}
})(),
).rejects.toMatchObject({ reason: "cancelled", cause: reason });
reads.mockRestore();
const recovered = await entries(reader);
expect(recovered).toHaveLength(1);
expect(
(await buffer(await reader.open(recovered[0]!))).equals(
Buffer.alloc(128 * 1024, 0x61),
),
).toBe(true);
} finally {
await reader.close();
reads.mockRestore();
@@ -205,3 +216,8 @@ it("reports impossible over-cap component extents as malformed", async () => {
await reader.close();
}
});
artifactPreparationRecoveryCases({
image: () => buildUImage({ payload: Buffer.from("payload") }),
createReader: (path, admitted) => new UImageArtifactReader(path, admitted),
});
+28 -4
View File
@@ -8,7 +8,10 @@ import type {
UImageMetadata,
} from "../domain/artifactFormatMetadata.js";
import { ArtifactFileSource } from "./ArtifactFileSource.js";
import { assertArtifactReadActive } from "./ArtifactFileOperation.js";
import {
artifactReadCancelledBy,
assertArtifactReadActive,
} from "./ArtifactFileOperation.js";
import {
ArtifactReaderFailure,
copyArtifactEntry,
@@ -47,7 +50,9 @@ export class UImageArtifactReader implements ArtifactReader {
readonly #entries = new Map<string, UImageEntry>();
readonly #streams = new Set<Readable>();
readonly #source: ArtifactFileSource;
#prepared: Promise<void> | undefined;
#prepared:
| { signal: AbortSignal | undefined; promise: Promise<void> }
| undefined;
#header: UImageMetadata | undefined;
#closing = false;
@@ -75,8 +80,27 @@ export class UImageArtifactReader implements ArtifactReader {
): AsyncIterable<ArtifactEntry> {
await this.#scope.run(async () => {
this.#assertOpen(signal);
this.#prepared ??= this.#prepare(signal);
await this.#prepared;
for (;;) {
const pending = (this.#prepared ??= {
signal,
promise: this.#prepare(signal),
});
try {
await pending.promise;
break;
} catch (cause: unknown) {
if (this.#prepared === pending) this.#prepared = undefined;
if (
pending.signal !== signal &&
signal?.aborted !== true &&
artifactReadCancelledBy(cause, pending.signal)
) {
this.#assertOpen(signal);
continue;
}
throw cause;
}
}
await this.#source.read(signal);
});
this.#assertOpen(signal);
@@ -1,3 +1,5 @@
import { artifactPreparationRecoveryCases } from "../../tests/fixtures/artifactReaderLifecycle.js";
import { ArtifactFileSource } from "./ArtifactFileSource.js";
import { truncate, writeFile } from "node:fs/promises";
import { join } from "node:path";
import { buffer, text } from "node:stream/consumers";
@@ -43,6 +45,68 @@ const repeatedBlock = (data: Buffer): Buffer => {
]);
};
it("retries UnityFs preparation cancelled during file reading", async () => {
const path = await fixture(buildUnityFs([{ path: "a", data: "x" }]));
const admitted = await openRegularFile(path, { symlinks: "reject" });
const reader = new UnityFsArtifactReader(path, admitted);
const controller = new AbortController();
const reason = { stage: "header read" };
const nativeRead = admitted.handle.read.bind(admitted.handle);
const read = vi.spyOn(admitted.handle, "read").mockImplementation((async (
...args: unknown[]
) => {
const result = await Reflect.apply(nativeRead, admitted.handle, args);
controller.abort(reason);
return result;
}) as typeof admitted.handle.read);
try {
await expect(
(async () => {
for await (const _ of reader.entries(controller.signal)) {
}
})(),
).rejects.toMatchObject({ reason: "cancelled", cause: reason });
read.mockRestore();
expect((await entries(reader)).map((entry) => entry.path)).toEqual(["a"]);
} finally {
read.mockRestore();
await reader.close();
await admitted.handle.close();
}
});
it("does not publish UnityFs entries when final source verification fails", async () => {
const path = await fixture(buildUnityFs([{ path: "a", data: "x" }]));
const control = new UnityFsArtifactReader(path);
const reader = new UnityFsArtifactReader(path);
const expected = (await entries(control))[0]!;
const nativeRead = ArtifactFileSource.prototype.read;
const physical = Object.assign(
new Error("final source verification failed"),
{ code: "EIO" },
);
let calls = 0;
const read = vi
.spyOn(ArtifactFileSource.prototype, "read")
.mockImplementation(function (this: ArtifactFileSource, ...args) {
calls += 1;
if (calls === 2) return Promise.reject(physical);
return nativeRead.apply(this, args);
});
try {
await expect(entries(reader)).rejects.toBe(physical);
await expect(reader.open(expected)).rejects.toMatchObject({
reason: "integrity",
});
read.mockRestore();
expect((await entries(reader)).map((entry) => entry.path)).toEqual(["a"]);
} finally {
read.mockRestore();
await reader.close();
await control.close();
}
});
describe("UnityFS allocation and malformed-input boundaries", () => {
it("rejects an oversized directory extending past the physical file", async () => {
const bytes = buildUnityFs([{ path: "data", data: "x" }]);
@@ -431,3 +495,8 @@ describe("UnityFS stream capacity and cache ownership", () => {
}
});
});
artifactPreparationRecoveryCases({
image: () => buildUnityFs([{ path: "a", data: "x" }]),
createReader: (path, admitted) => new UnityFsArtifactReader(path, admitted),
});
+65 -24
View File
@@ -12,7 +12,10 @@ import {
type ArtifactTraversalOptions,
} from "./ArtifactReader.js";
import { ArtifactResourceScope } from "./ArtifactResourceScope.js";
import { assertArtifactReadActive } from "./ArtifactFileOperation.js";
import {
artifactReadCancelledBy,
assertArtifactReadActive,
} from "./ArtifactFileOperation.js";
import { readArtifactFileBytes } from "./ArtifactFileRange.js";
import type {
ArtifactFormatMetadata,
@@ -52,10 +55,12 @@ interface UnityFsEntryState {
export class UnityFsArtifactReader implements ArtifactReader {
readonly format = "unityfs" as const;
readonly #scope = new ArtifactResourceScope();
readonly #entries = new Map<string, UnityFsEntryState>();
#entries = new Map<string, UnityFsEntryState>();
readonly #streams = new Map<Readable, AbortController>();
readonly #source: ArtifactFileSource;
#prepared: Promise<void> | undefined;
#prepared:
| { signal: AbortSignal | undefined; promise: Promise<void> }
| undefined;
#header: UnityFsMetadata | undefined;
#blocks: readonly UnityFsBlock[] = [];
#dataStart = 0;
@@ -88,8 +93,27 @@ export class UnityFsArtifactReader implements ArtifactReader {
): AsyncIterable<ArtifactEntry> {
await this.#scope.run(async () => {
this.#assertOpen(signal);
this.#prepared ??= this.#prepare(signal);
await this.#prepared;
for (;;) {
const pending = (this.#prepared ??= {
signal,
promise: this.#prepare(signal),
});
try {
await pending.promise;
break;
} catch (cause: unknown) {
if (this.#prepared === pending) this.#prepared = undefined;
if (
pending.signal !== signal &&
signal?.aborted !== true &&
artifactReadCancelledBy(cause, pending.signal)
) {
this.#assertOpen(signal);
continue;
}
throw cause;
}
}
await this.#source.read(signal);
});
this.#assertOpen(signal);
@@ -229,7 +253,7 @@ export class UnityFsArtifactReader implements ArtifactReader {
if (header === undefined)
throw unityFsFormatFailure("no complete UnityFS header");
const status = header.layout.status;
this.#header = {
const metadata: UnityFsMetadata = {
kind: "unityfs",
signature: "UnityFS",
format_version: header.formatVersion,
@@ -245,7 +269,12 @@ export class UnityFsArtifactReader implements ArtifactReader {
data_hash: null,
directory_status: status,
};
if (status !== "available") return;
if (status !== "available") {
await this.#source.read(signal);
this.#assertOpen(signal);
this.#header = metadata;
return;
}
const { dataHash, blocks, nodes, dataStart } = await readUnityFsBlocksInfo(
handle,
header,
@@ -259,44 +288,55 @@ export class UnityFsArtifactReader implements ArtifactReader {
if (streamBytes > BigInt(budget))
throw expansionLimit("declared virtual stream", streamBytes, budget);
}
this.#header = {
...this.#header,
const preparedMetadata: UnityFsMetadata = {
...metadata,
data_hash: dataHash,
block_count: blocks.length,
data_start: dataStart,
};
this.#dataStart = dataStart;
// Keep the source block count above, but do not revisit zero-output rows
// for every member. Parsed stream/compressed offsets remain authoritative.
this.#blocks = blocks.filter((block) => block.uncompressedSize > 0);
this.#remainingDecodeBytes = budget;
await this.#registerNodes(nodes, budget, signal);
const readableBlocks = blocks.filter((block) => block.uncompressedSize > 0);
const registered = await this.#registerNodes(
nodes,
readableBlocks,
budget,
signal,
);
await this.#source.read(signal);
this.#assertOpen(signal);
this.#header = preparedMetadata;
this.#dataStart = dataStart;
this.#blocks = readableBlocks;
this.#remainingDecodeBytes = budget;
this.#entries = registered.entries;
this.#remainingReadOperations = registered.remainingReadOperations;
}
async #registerNodes(
nodes: readonly UnityFsNode[],
blocks: readonly UnityFsBlock[],
budget: number,
signal?: AbortSignal,
): Promise<void> {
): Promise<{
entries: Map<string, UnityFsEntryState>;
remainingReadOperations: number;
}> {
const entries = new Map<string, UnityFsEntryState>();
let readableBytes = 0n;
for (let index = 0; index < nodes.length; index++) {
if (index % 256 === 0) await setImmediate();
this.#assertOpen(signal);
const node = nodes[index]!;
const firstBlock =
node.size === 0 ? -1 : blockEndingAfter(this.#blocks, node.offset);
node.size === 0 ? -1 : blockEndingAfter(blocks, node.offset);
const lastBlock =
node.size === 0
? -1
: blockEndingAfter(this.#blocks, node.offset + node.size - 1);
: blockEndingAfter(blocks, node.offset + node.size - 1);
if (node.size !== 0 && (firstBlock < 0 || lastBlock < firstBlock))
throw unityFsFormatFailure("member has no block range");
const unavailable = firstUnavailableBlock(
this.#blocks,
firstBlock,
lastBlock,
);
const unavailable = firstUnavailableBlock(blocks, firstBlock, lastBlock);
const kind =
(node.flags & 1) !== 0 ? ("directory" as const) : ("file" as const);
if (kind === "file" && unavailable === undefined) {
@@ -331,7 +371,7 @@ export class UnityFsArtifactReader implements ArtifactReader {
flags: node.flags,
},
} satisfies UnityFsEntry;
this.#entries.set(adapterKey, {
entries.set(adapterKey, {
entry,
firstBlock,
lastBlock,
@@ -340,14 +380,15 @@ export class UnityFsArtifactReader implements ArtifactReader {
}
// Four ordinary passes over admitted blocks, nodes and plaintext chunks,
// with a small-reader floor. Byte quotas alone cannot bound tiny-block I/O.
this.#remainingReadOperations =
const remainingReadOperations =
4 *
Math.max(
1024,
this.#blocks.length +
blocks.length +
nodes.length +
Math.ceil(Number(readableBytes) / CHUNK_BYTES),
);
return { entries, remainingReadOperations };
}
async *#memberChunks(
@@ -1,3 +1,5 @@
import { artifactPreparationRecoveryCases } from "../../tests/fixtures/artifactReaderLifecycle.js";
import { ArtifactFileSource } from "./ArtifactFileSource.js";
import { once } from "node:events";
import { writeFile } from "node:fs/promises";
import { join } from "node:path";
@@ -26,6 +28,68 @@ const entries = async (reader: WxapkgArtifactReader) => {
return result;
};
it("retries Wxapkg preparation cancelled during file reading", async () => {
const path = await fixture(buildWxapkg([{ path: "a", data: "x" }]));
const admitted = await openRegularFile(path, { symlinks: "reject" });
const reader = new WxapkgArtifactReader(path, admitted);
const controller = new AbortController();
const reason = { stage: "header read" };
const nativeRead = admitted.handle.read.bind(admitted.handle);
const read = vi.spyOn(admitted.handle, "read").mockImplementation((async (
...args: unknown[]
) => {
const result = await Reflect.apply(nativeRead, admitted.handle, args);
controller.abort(reason);
return result;
}) as typeof admitted.handle.read);
try {
await expect(
(async () => {
for await (const _ of reader.entries(controller.signal)) {
}
})(),
).rejects.toMatchObject({ reason: "cancelled", cause: reason });
read.mockRestore();
expect((await entries(reader)).map((entry) => entry.path)).toEqual(["a"]);
} finally {
read.mockRestore();
await reader.close();
await admitted.handle.close();
}
});
it("does not publish Wxapkg entries when final source verification fails", async () => {
const path = await fixture(buildWxapkg([{ path: "a", data: "x" }]));
const control = new WxapkgArtifactReader(path);
const reader = new WxapkgArtifactReader(path);
const expected = (await entries(control))[0]!;
const nativeRead = ArtifactFileSource.prototype.read;
const physical = Object.assign(
new Error("final source verification failed"),
{ code: "EIO" },
);
let calls = 0;
const read = vi
.spyOn(ArtifactFileSource.prototype, "read")
.mockImplementation(function (this: ArtifactFileSource, ...args) {
calls += 1;
if (calls === 2) return Promise.reject(physical);
return nativeRead.apply(this, args);
});
try {
await expect(entries(reader)).rejects.toBe(physical);
await expect(reader.open(expected)).rejects.toMatchObject({
reason: "integrity",
});
read.mockRestore();
expect((await entries(reader)).map((entry) => entry.path)).toEqual(["a"]);
} finally {
read.mockRestore();
await reader.close();
await control.close();
}
});
describe("wxapkg allocation and malformed-input boundaries", () => {
it("admits the encoded table budget before reading it", async () => {
const bytes = buildWxapkg([{ path: "a", data: "x" }]);
@@ -308,3 +372,8 @@ describe("wxapkg stream capacity and freshness", () => {
}
});
});
artifactPreparationRecoveryCases({
image: () => buildWxapkg([{ path: "a", data: "x" }]),
createReader: (path, admitted) => new WxapkgArtifactReader(path, admitted),
});
+44 -10
View File
@@ -12,7 +12,10 @@ import {
type ArtifactTraversalOptions,
} from "./ArtifactReader.js";
import { ArtifactResourceScope } from "./ArtifactResourceScope.js";
import { assertArtifactReadActive } from "./ArtifactFileOperation.js";
import {
artifactReadCancelledBy,
assertArtifactReadActive,
} from "./ArtifactFileOperation.js";
import { readArtifactFileBytes } from "./ArtifactFileRange.js";
import type {
ArtifactFormatMetadata,
@@ -48,10 +51,12 @@ interface WxapkgEntryState {
export class WxapkgArtifactReader implements ArtifactReader {
readonly format = "wxapkg" as const;
readonly #scope = new ArtifactResourceScope();
readonly #entries = new Map<string, WxapkgEntryState>();
#entries = new Map<string, WxapkgEntryState>();
readonly #streams = new Map<Readable, AbortController>();
readonly #source: ArtifactFileSource;
#prepared: Promise<void> | undefined;
#prepared:
| { signal: AbortSignal | undefined; promise: Promise<void> }
| undefined;
#header: WxapkgMetadata | undefined;
#remainingReadOperations = 0;
#admittedReadOperations = 0;
@@ -81,8 +86,27 @@ export class WxapkgArtifactReader implements ArtifactReader {
): AsyncIterable<ArtifactEntry> {
await this.#scope.run(async () => {
this.#assertOpen(signal);
this.#prepared ??= this.#prepare(signal);
await this.#prepared;
for (;;) {
const pending = (this.#prepared ??= {
signal,
promise: this.#prepare(signal),
});
try {
await pending.promise;
break;
} catch (cause: unknown) {
if (this.#prepared === pending) this.#prepared = undefined;
if (
pending.signal !== signal &&
signal?.aborted !== true &&
artifactReadCancelledBy(cause, pending.signal)
) {
this.#assertOpen(signal);
continue;
}
throw cause;
}
}
await this.#source.read(signal);
});
this.#assertOpen(signal);
@@ -201,7 +225,7 @@ export class WxapkgArtifactReader implements ArtifactReader {
const header = await readWxapkgHeader(handle, initial.size, signal);
if (header === undefined)
throw wxapkgFormatFailure("no complete wxapkg header");
this.#header = {
const metadata: WxapkgMetadata = {
kind: "wxapkg",
info: header.layout.status === "available" ? header.info : null,
index_length:
@@ -212,7 +236,12 @@ export class WxapkgArtifactReader implements ArtifactReader {
header.layout.status === "available" ? header.fileCount : null,
directory_status: header.layout.status,
};
if (header.layout.status !== "available") return;
if (header.layout.status !== "available") {
await this.#source.read(signal);
this.#assertOpen(signal);
this.#header = metadata;
return;
}
const { members } = await readWxapkgFileTable(
handle,
header,
@@ -220,6 +249,7 @@ export class WxapkgArtifactReader implements ArtifactReader {
signal,
);
const budget = artifactExpansionBudgetBytes(initial.size);
const entries = new Map<string, WxapkgEntryState>();
let declaredBytes = 0n;
for (let index = 0; index < members.length; index++) {
if (index % 256 === 0) await setImmediate();
@@ -234,7 +264,7 @@ export class WxapkgArtifactReader implements ArtifactReader {
{ boundary: "memory" },
);
const adapterKey = `wxapkg-entry:${String(index)}`;
this.#entries.set(adapterKey, {
entries.set(adapterKey, {
entry: {
path: member.path,
sourcePath: member.declaredPath,
@@ -260,14 +290,18 @@ export class WxapkgArtifactReader implements ArtifactReader {
});
}
// Header, table and per-chunk member reads; a small-reader floor applies.
this.#admittedReadOperations =
const admittedReadOperations =
4 *
Math.max(
1024,
members.length + Math.ceil(Number(declaredBytes) / CHUNK_BYTES),
);
this.#remainingReadOperations = this.#admittedReadOperations;
await this.#source.read(signal);
this.#assertOpen(signal);
this.#header = metadata;
this.#entries = entries;
this.#admittedReadOperations = admittedReadOperations;
this.#remainingReadOperations = admittedReadOperations;
}
async *#memberChunks(
+2 -1
View File
@@ -43,7 +43,8 @@ type FormatsWithReaders =
| "uf2"
| "dfuse"
| "esp-flash-image"
| "fit";
| "fit"
| "ntfs-mft";
const readerFormatsMatch: Equal<ArtifactReaderFormat, FormatsWithReaders> =
true;
+9
View File
@@ -40,6 +40,7 @@ import { NodeSeaArtifactReader } from "./NodeSeaArtifactReader.js";
import type { PyInstallerAdmission } from "./pyinstaller/PyInstallerAdmission.js";
import { PyInstallerArtifactReader } from "./PyInstallerArtifactReader.js";
import { Uf2ArtifactReader } from "./Uf2ArtifactReader.js";
import { NtfsMftArtifactReader } from "./NtfsMftArtifactReader.js";
import { UImageArtifactReader } from "./UImageArtifactReader.js";
import { UnityFsArtifactReader } from "./UnityFsArtifactReader.js";
import { WxapkgArtifactReader } from "./WxapkgArtifactReader.js";
@@ -465,6 +466,14 @@ const NON_ZIP_FORMATS = {
createReader: (path, options) =>
new FitArtifactReader(path, sourceOf(options)),
}),
"ntfs-mft": described("ntfs-mft", {
...plain,
retainClassifiedSource: true,
readerLimitation:
"MFT inventory reports whole-stream counts and a bounded sample of the first 32 decoded FILE record headers, with the omitted-header count. Attributes, filenames, data streams and the $UsnJrnl:$J journal are not decoded, so no path, file size or change history is established.",
createReader: (path, options) =>
new NtfsMftArtifactReader(path, sourceOf(options)),
}),
unknown: described("unknown", plain),
} satisfies Record<
Exclude<ArtifactFormatId, ZipPackageFormat>,
@@ -576,6 +576,28 @@ it("classifies nested UF2 by block magics and DfuSe by prefix and suffix", () =>
).toMatchObject({ format: "file" });
});
it("classifies a $MFT stream by its FILE record header, under any name", () => {
const mft = Buffer.alloc(2_048);
mft.write("FILE", 0, "latin1");
mft.writeUInt32LE(1024, 28);
for (const path of ["evidence/$MFT", "evidence/mft.raw", "evidence/records"])
expect(classifyArtifactBytesAtPath(path, mft)).toEqual({
kind: "executable",
format: "ntfs-mft",
});
});
it("leaves plain content beginning with FILE unclassified as MFT", () => {
const text = Buffer.from(
"FILE is a plain text prefix, not an NTFS record header",
);
expect(
classifyArtifactBytesAtPath("evidence/readme.txt", text),
).toMatchObject({
format: "file",
});
});
it("classifies a FIT by a /images node within the first 4 KiB", () => {
const images = [{ name: "kernel", data: Buffer.from("kernel") }];
const late = devicetreeFixture({
@@ -11,6 +11,7 @@ import { recognizeEspImage } from "../../domain/firmware/espImage.js";
import { hasUf2Block } from "../../domain/firmware/uf2.js";
import { isDfuSeFile } from "../../domain/firmware/dfuse.js";
import { hasFitImagesNode } from "../../domain/firmware/fitImage.js";
import { hasMftFileRecordHeader } from "../../domain/windows/ntfsMft.js";
import type { ArtifactClassificationEvidence } from "../ArtifactClassificationProbe.js";
import { asarHeaderAdmissionFailure } from "../AsarHeader.js";
import { hasDestinationPath } from "../ArtifactPaths.js";
@@ -622,6 +623,7 @@ export const classifyArtifactBytes = (
return "dfuse";
if (recognizeEspImage(prefix, fileSize) === "esp-image") return "esp-image";
if (hasFitImagesNode(prefix)) return "fit";
if (hasMftFileRecordHeader(prefix)) return "ntfs-mft";
if (hasBinaryPlistSignature(prefix)) return "plist";
if (hasXmlPlistSignature(prefix)) return "plist";
const fatFormat = classifyFatArtifact(evidence, fileSize);
+1
View File
@@ -86,6 +86,7 @@ const formatSchema = z.enum([
"dfuse",
"esp-flash-image",
"fit",
"ntfs-mft",
"plist",
"javascript",
"source-map",
+56
View File
@@ -1,4 +1,5 @@
import { z } from "zod";
import { MFT_INLINE_HEADER_LIMIT } from "./windows/ntfsMft.js";
import { jsonRecordSchema } from "./jsonRecord.js";
import { jsonValueSchema } from "./jsonValue.js";
@@ -299,6 +300,56 @@ export const hexRecordRegionMetadataSchema = z.strictObject({
last_line: offset,
});
const ntfsFileReferenceSchema = z.strictObject({
recordNumber: offset,
sequenceNumber: offset,
});
const ntfsRecordHeaderSchema = z.strictObject({
index: offset,
fileOffset: offset,
sequenceNumber: offset,
hardLinkCount: offset,
flags: z.strictObject({
raw: offset,
names: z.array(z.string()),
unknownMask: offset,
}),
inUse: z.boolean(),
directory: z.boolean(),
usedBytes: offset,
totalBytes: offset,
extentValid: z.boolean(),
baseRecordReference: ntfsFileReferenceSchema.nullable(),
declaredRecordNumber: offset.nullable(),
recordNumberMatchesSlot: z.boolean().nullable(),
logFileSequenceNumber: z.string().regex(/^(?:0|[1-9][0-9]*)$/u),
firstAttributeOffset: offset,
nextAttributeId: offset,
fixups: z.strictObject({
sequenceNumber: offset,
entries: offset,
verifiedSectors: offset,
coveredSectors: offset,
mismatchSector: offset.nullable(),
malformed: z.boolean(),
}),
});
/** NTFS $MFT record-header facts; attributes, names and the USN journal are later steps. */
export const ntfsMftMetadataSchema = z.strictObject({
kind: z.literal("ntfs-mft"),
record_bytes: offset,
record_count: offset,
in_use_record_count: offset,
directory_record_count: offset,
extension_record_count: offset,
fixup_mismatch_count: offset,
fixup_unverified_record_count: offset,
truncated_tail_bytes: offset,
record_headers: z.array(ntfsRecordHeaderSchema).max(MFT_INLINE_HEADER_LIMIT),
omitted_record_headers: offset,
});
/**
* U-Boot FIT root facts. Configuration references are declared names; a
* reference that names no image is reported as a limitation, not removed.
@@ -886,6 +937,7 @@ export const artifactFormatMetadataSchema = z.discriminatedUnion("kind", [
espImageSignatureBlockMetadataSchema,
uf2MetadataSchema,
uf2RegionMetadataSchema,
ntfsMftMetadataSchema,
dfuSeMetadataSchema,
dfuSeElementMetadataSchema,
espFlashImageMetadataSchema,
@@ -1036,6 +1088,10 @@ const declarationsByKind: {
notCompared:
"Its familyID and addresses are in its path, family_name is REA's label for the familyID, and its block counts and positions are layout; content comparison covers its bytes.",
},
"ntfs-mft": {
notCompared:
"Its fields come from the record headers in the file's own FILE records, which content comparison covers.",
},
dfuse: {
notCompared:
"Its fields come from the file's own prefix, target headers and suffix, which content comparison covers.",
+2
View File
@@ -103,6 +103,7 @@ export const ARTIFACT_FORMAT_IDS = [
"dfuse",
"esp-flash-image",
"fit",
"ntfs-mft",
"unknown",
] as const;
@@ -124,6 +125,7 @@ const artifactContentFormatSchema = artifactFormatSchema.extract([
"uf2",
"dfuse",
"fit",
"ntfs-mft",
"mach-o-universal",
"mach-o",
"elf",
+2 -1
View File
@@ -302,7 +302,8 @@ export type BinaryTarget =
| "esp-image"
| "uf2"
| "dfuse"
| "fit";
| "fit"
| "ntfs-mft";
})
| (BinaryTargetIdentity &
NonExecutableMetadata & {
+1
View File
@@ -52,6 +52,7 @@ export const evidenceSubjectFormatSchema = z.enum([
"dfuse",
"esp-flash-image",
"fit",
"ntfs-mft",
]);
/** Format-owned architecture declarations shared by Evidence and persisted executions. */
@@ -137,7 +137,8 @@ describe("child-process options detection", () => {
["negative-zero", `spawn("tool", [], { env: -0 })`, false],
["bigint-zero", `spawn("tool", [], { env: 0n })`, false],
["zero-float", `spawn("tool", [], { env: 0.0 })`, false],
["nan", `spawn("tool", [], { env: NaN })`, true],
["nan", `spawn("tool", [], { env: NaN })`, false],
["shadowed-nan", `((NaN) => spawn("tool", [], { env: NaN }))({})`, true],
["empty-string", `spawn("tool", [], { env: "" })`, false],
["false", `spawn("tool", [], { env: false })`, false],
["empty-object", `spawn("tool", [], { env: {} })`, true],
@@ -650,6 +650,32 @@ it("connects exposed closures to IPC sends and text clipboard calls", () => {
expect(ipcPath?.steps.at(-1)).toBeDefined();
expect(ipcFinding?.location).toEqual(ipcPath?.steps.at(-1)?.location);
});
it("uses a defineProperty value that replaces an exported sender", () => {
const result = analyzeJavaScriptStaticSource(`(() => {
const modules = {
2(module) { module.exports = require("electron"); },
9(module,exports,load) {
const electron=load(2),helper=load(5);
electron.contextBridge.exposeInMainWorld("desktop", {copy:helper.sender("defined:copy")});
},
5(module,exports,load) {
const electron=load(2);
exports.sender=function(){return ()=>{};};
Object.defineProperty(exports,"sender",{value:function(channel){return (...args)=>send(channel,...args);}});
function send(channel,...args){electron.ipcRenderer.send(channel,...args);}
}
};
function load(id) { const m={exports:{}}; modules[id].call(m.exports,m,m.exports,load); return m.exports; }
load(9);
})();`);
expect(result.electron.call_paths).toContainEqual(
expect.objectContaining({
entry: "desktop.copy",
channel: "defined:copy",
terminal: "electron.ipcRenderer.send",
}),
);
});
it("resolves a helper factory listed after its bundle consumer", () => {
const result = analyzeJavaScriptStaticSource(`(() => {
const modules = {
@@ -1,7 +1,11 @@
import * as t from "@babel/types";
import type { javascriptBootstrapAt } from "./javascriptBootstrap.js";
import { semanticStaticPropertyKey } from "./javascriptAstValues.js";
import {
semanticStaticPropertyKey,
semanticWriteSiteTarget,
semanticWriteTargets,
} from "./javascriptAstValues.js";
import { semanticIdentifierRole } from "./javascriptSemanticProjection.js";
import {
resolveSemanticBindingState,
@@ -285,8 +289,13 @@ const moduleObjectContained = (
state: JavaScriptSemanticAnalysisState,
): boolean => {
let contained = true;
const writtenTargets = new Set<t.Node>();
traverseJavaScriptValueAst(loader.body, state.erasedSyntaxNodes, {
enter: (node, parent, readAncestors) => {
const target = semanticWriteSiteTarget(node);
if (target !== null)
for (const leaf of semanticWriteTargets(target))
writtenTargets.add(leaf);
if (!contained || !t.isIdentifier(node) || parent === null) return;
const ancestors = readAncestors();
if (semanticIdentifierRole(node, parent, () => ancestors) === null)
@@ -294,7 +303,7 @@ const moduleObjectContained = (
if (bindingOf(state, node) !== moduleId) return;
contained =
!ancestors.some((ancestor) => t.isFunction(ancestor)) &&
moduleUse(node, [parent, ancestors.at(-2)], dispatch);
moduleUse(node, parent, dispatch, writtenTargets.has(parent));
},
});
return contained;
@@ -302,21 +311,18 @@ const moduleObjectContained = (
const moduleUse = (
node: t.Identifier,
[parent, grandparent]: readonly [t.Node, t.Node | undefined],
parent: t.Node,
dispatch: t.CallExpression,
written: boolean,
): boolean => {
if (parent === dispatch) return dispatch.arguments.includes(node);
if (!t.isMemberExpression(parent) || parent.object !== node) return false;
const key = semanticStaticPropertyKey(parent.property, parent.computed);
const written =
(t.isAssignmentExpression(grandparent) && grandparent.left === parent) ||
t.isUpdateExpression(grandparent) ||
t.isUnaryExpression(grandparent, { operator: "delete" });
if (written) return key !== null && key !== "exports";
if (key !== "exports") return false;
// `module.exports` passed to the factory, or read after the dispatch ran.
return (
grandparent === dispatch ||
dispatch.arguments.some((argument) => argument === parent) ||
(parent.start ?? -1) > (dispatch.end ?? Number.POSITIVE_INFINITY)
);
};
@@ -10,9 +10,54 @@ import {
completeSemanticSteps,
traverseJavaScriptAst,
} from "./javascriptSemanticTraversal.js";
import { createGlobalReferenceReader } from "./javascriptSemanticAmbientWrites.js";
import {
createGlobalPathReader,
createGlobalReferenceReader,
} from "./javascriptSemanticAmbientWrites.js";
import { parseJavaScriptSource } from "./javascriptSourceParser.js";
it.each([
[
"await value; Object.prototype.then=null; await value;",
["unmodified", "null"],
],
[
"Object.prototype.then=function(resolve){resolve()}; await value;",
["function"],
],
["if(flag) Object.prototype.then=null; await value;", ["unknown"]],
["mutate(Object.prototype); await value;", ["unknown"]],
["await value; mutate(Object.prototype);", ["unmodified"]],
["Array.prototype.then=null; await value;", ["unmodified"]],
] as const)(
"reads intrinsic property facts at implicit await sites: %s",
(source, expected) => {
const parsed = parseJavaScriptSource(source, "app.mjs");
if (parsed === null) throw new Error("Expected implicit read fixture");
const state = completeSemanticSteps(
indexJavaScriptBindingsSteps(parsed.program),
);
const read = createGlobalPathReader(state, ["Object", "prototype", "then"]);
const statuses: string[] = [];
traverseJavaScriptAst(parsed.program, {
enter(node) {
if (!t.isAwaitExpression(node)) return;
const fact = read(node);
statuses.push(
fact.status === "replacement"
? t.isNullLiteral(fact.value)
? "null"
: t.isFunction(fact.value)
? "function"
: "other"
: fact.status,
);
},
});
expect(statuses).toEqual(expected);
},
);
it("keeps wrapper module aliases distinct from global-object module properties", () => {
const parsed = parseJavaScriptSource(
`
@@ -306,6 +351,11 @@ it.each([
"Object.defineProperty=()=>{};inspect(globalThis.Object.defineProperty);",
["replacement"],
],
[
"an escaped global owner",
"mutate(Object);inspect(Object.defineProperty);",
["unknown"],
],
[
"a namespaced global",
"inspect(globalThis.Object.defineProperty);globalThis.Object.defineProperty=()=>{};inspect(globalThis.Object.defineProperty);",
@@ -902,37 +902,92 @@ export const createGlobalReferenceReader = (
state: JavaScriptSemanticAnalysisState,
path: readonly [string, ...string[]],
): ((callee: t.Node) => GlobalReferenceRead) => {
const selected = ["global", ...path];
let generation = -1;
const byNamespace = new Map<
string | null,
typeof state.ambientWriteEffects
>();
const read = createGlobalPathRead(state, path);
return (callee) => {
const namespace = globalReferenceNamespace(callee, state, path);
if (namespace === undefined) return { status: "unrelated" };
return namespace === undefined
? { status: "unrelated" }
: read(callee, namespace);
};
};
/** Read an implicit intrinsic property at its actual source site, without inventing a callee. */
export const createGlobalPathReader = (
state: JavaScriptSemanticAnalysisState,
path: readonly [string, ...string[]],
): ((site: t.Node) => GlobalReferenceRead) => {
const read = createGlobalPathRead(state, path);
return (site) => read(site, null);
};
const createGlobalPathRead = (
state: JavaScriptSemanticAnalysisState,
path: readonly [string, ...string[]],
): ((site: t.Node, namespace: string | null) => GlobalReferenceRead) => {
const selected = ["global", ...path];
let writesGeneration = -1;
let escapesGeneration = -1;
const byNamespace = new Map<
string | null,
{
writes: typeof state.ambientWriteEffects;
escapes: typeof state.ambientEscapeEffects;
}
>();
return (site, namespace) => {
const covered = (paths: typeof state.ambientWrites): boolean =>
paths.covers(selected) ||
(namespace !== null && paths.covers(["global", namespace]));
const writesCovered = covered(state.ambientWrites);
const escapesCovered = covered(state.ambientEscapes);
if (!writesCovered && !escapesCovered) return { status: "unmodified" };
if (
!state.ambientWrites.covers(selected) &&
(namespace === null || !state.ambientWrites.covers(["global", namespace]))
)
return { status: "unmodified" };
if (generation !== state.ambientWriteEffects.length) {
generation = state.ambientWriteEffects.length;
writesGeneration !== state.ambientWriteEffects.length ||
escapesGeneration !== state.ambientEscapeEffects.length
) {
writesGeneration = state.ambientWriteEffects.length;
escapesGeneration = state.ambientEscapeEffects.length;
byNamespace.clear();
}
let effects = byNamespace.get(namespace);
if (effects === undefined) {
effects = state.ambientWriteEffects.filter(
(effect) =>
ambientEffectCovers(effect, selected) ||
(namespace !== null &&
ambientEffectCovers(effect, ["global", namespace])),
);
const matches = (
effect: (typeof state.ambientWriteEffects)[number],
): boolean =>
ambientEffectCovers(effect, selected) ||
(namespace !== null &&
ambientEffectCovers(effect, ["global", namespace]));
effects = {
writes: state.ambientWriteEffects.filter(matches),
escapes: state.ambientEscapeEffects.filter(matches),
};
byNamespace.set(namespace, effects);
}
if (effects.length === 0) return { status: "unknown" };
const active = effects.filter(
(effect) => semanticEffectTiming(callee, effect.node, state) !== "after",
// The call that escapes a receiver contains the callee being identified.
// That escape cannot have replaced the callee before the call selects it.
const escapePrecedesRead = effects.escapes.some((effect) => {
const escape = effect.node;
if (
escape !== undefined &&
escape !== site &&
typeof escape.start === "number" &&
typeof escape.end === "number" &&
typeof site.start === "number" &&
typeof site.end === "number" &&
site.start >= escape.start &&
site.end <= escape.end
)
return false;
return semanticEffectTiming(site, escape, state) !== "after";
});
if (
(escapesCovered && effects.escapes.length === 0) ||
escapePrecedesRead ||
(writesCovered && effects.writes.length === 0)
)
return { status: "unknown" };
const active = effects.writes.filter(
(effect) => semanticEffectTiming(site, effect.node, state) !== "after",
);
if (active.length === 0) return { status: "unmodified" };
if (
@@ -943,47 +998,57 @@ export const createGlobalReferenceReader = (
)
)
return { status: "unknown" };
let replacement: t.AssignmentExpression | undefined;
for (const effect of active) {
const write = effect.node;
const writeNamespace = t.isAssignmentExpression(write)
? globalReferenceNamespace(write.left, state, path)
: undefined;
if (
t.isAssignmentExpression(write, { operator: "=" }) &&
writeNamespace !== undefined &&
(writeNamespace === null ||
!state.ambientWrites.covers(["global", writeNamespace])) &&
semanticDefiniteEffectPrecedesReference(
write,
callee,
state.parentsByNode,
) &&
(replacement === undefined ||
semanticDefiniteEffectPrecedesReference(
replacement,
write,
state.parentsByNode,
))
)
replacement = write;
}
if (replacement === undefined) return { status: "unknown" };
const confirmed = replacement;
if (
active.some(
(effect) =>
effect.node !== confirmed &&
semanticEffectTiming(confirmed, effect.node, state) !== "before" &&
(effect.node === undefined ||
!semanticDefiniteEffectPrecedesReference(
effect.node,
confirmed,
state.parentsByNode,
)),
)
)
return { status: "unknown" };
return { status: "replacement", value: confirmed.right };
return globalPathReplacement(state, path, site, active);
};
};
/** Select a definite replacement only when every competing write is proven earlier. */
const globalPathReplacement = (
state: JavaScriptSemanticAnalysisState,
path: readonly [string, ...string[]],
site: t.Node,
effects: typeof state.ambientWriteEffects,
): GlobalReferenceRead => {
let replacement: t.AssignmentExpression | undefined;
for (const effect of effects) {
const write = effect.node;
const writeNamespace = t.isAssignmentExpression(write)
? globalReferenceNamespace(write.left, state, path)
: undefined;
if (
t.isAssignmentExpression(write, { operator: "=" }) &&
writeNamespace !== undefined &&
(writeNamespace === null ||
!state.ambientWrites.covers(["global", writeNamespace])) &&
semanticDefiniteEffectPrecedesReference(
write,
site,
state.parentsByNode,
) &&
(replacement === undefined ||
semanticDefiniteEffectPrecedesReference(
replacement,
write,
state.parentsByNode,
))
)
replacement = write;
}
if (replacement === undefined) return { status: "unknown" };
const confirmed = replacement;
if (
effects.some(
(effect) =>
effect.node !== confirmed &&
semanticEffectTiming(confirmed, effect.node, state) !== "before" &&
(effect.node === undefined ||
!semanticDefiniteEffectPrecedesReference(
effect.node,
confirmed,
state.parentsByNode,
)),
)
)
return { status: "unknown" };
return { status: "replacement", value: confirmed.right };
};
@@ -249,3 +249,20 @@ it("does not initialize a class self name during computed-key evaluation", () =>
expect(ir.callSites[0]).toBeDefined();
expect(ir.callSites[0]?.resolution).not.toBe("exact");
});
it.each(["__proto__", '"__proto__"', "['__proto__']"])(
"distinguishes prototype setters from own data keys in loader-write scoping: %s",
(key) => {
const ir = analyze(`
with ({require: local, [Symbol.unscopables]: {${key}: {require: true}}}) {
require = replacement;
}
const selected = require("node:fs");
`);
const selected = onlyBinding(ir, "selected");
if (key.startsWith("[")) {
// A computed key is an own data property, so it supplies no inherited exclusion.
expect(selected.provenance.status).toBe("module");
} else expect(selected.provenance.status).not.toBe("module");
},
);
@@ -1,7 +1,11 @@
import { describe, expect, it } from "vitest";
import { runInNewContext } from "node:vm";
import { analyzeJavaScriptSemantics } from "./javascriptSemanticAnalysis.js";
import { onlyCallable } from "./javascriptSemanticAnalysis.fixture.js";
import {
onlyBinding,
onlyCallable,
} from "./javascriptSemanticAnalysis.fixture.js";
const resultValue = (body: string) =>
onlyCallable(
@@ -9,6 +13,128 @@ const resultValue = (body: string) =>
"result",
).returnSites[0]?.value;
it.each([
[
"own getter",
"const value = { count: 1, get then() { this.count = 2; } };",
2,
],
[
"own method",
"const value = { count: 1, then(resolve) { this.count = 2; resolve(); } };",
2,
],
[
"function-valued slot",
"const value = { count: 1, then: function(resolve) { this.count = 2; resolve(); } };",
2,
],
[
"getter returning a method",
"const value = { count: 1, get then() { return function(resolve) { this.count = 2; resolve(); }; } };",
2,
],
[
"inherited getter",
"const proto = { get then() { this.count = 2; } }; const value = { __proto__: proto, count: 1 };",
2,
],
[
"inherited method",
"const proto = { then(resolve) { this.count = 2; resolve(); } }; const value = { __proto__: proto, count: 1 };",
2,
],
[
"class method",
"class Value { count = 1; then(resolve) { this.count = 2; resolve(); } } const value = new Value();",
2,
],
[
"stored method",
"const value = { count: 1 }; value.then = function(resolve) { this.count = 2; resolve(); };",
2,
],
[
"method stored on a copied shared child",
"const source = { child: { count: 1 } }; const box = { ...source }; source.child.then = function(resolve) { this.count = 2; resolve(); }; const value = box.child;",
2,
],
[
"copied method retained after source replacement",
"const source = { count: 1, then(resolve) { this.count = 2; resolve(); } }; const value = { ...source }; source.then = null;",
2,
],
["inert object", "const value = { count: 1 };", 1],
["noncallable then", "const value = { count: 1, then: null };", 1],
[
"noncallable getter result",
"const value = { count: 1, get then() { return 3; } };",
1,
],
[
"own noncallable shadow",
"const proto = { get then() { this.count = 2; } }; const value = { __proto__: proto, count: 1, then: null };",
1,
],
[
"method replaced before await",
"const value = { count: 1, then(resolve) { this.count = 2; resolve(); } }; value.then = null;",
1,
],
] as const)(
"accounts for thenable effects at await: %s",
async (_kind, declaration, expected) => {
const body = `${declaration} await value; return value.count;`;
expect(await runInNewContext(`(async () => { ${body} })()`)).toBe(expected);
const ir = analyzeJavaScriptSemantics(
`export async function result() { ${body} }`,
);
const value = onlyCallable(ir, "result").returnSites[0]?.value;
if (expected === 2) expect(value?.status).toBe("unknown");
else expect(value).toEqual({ status: "literal", value: 1 });
},
);
it("keeps a then slot captured before a later top-level source write", async () => {
const body = `
const source = { count: 1 };
const value = { ...source };
source.then = function(resolve) { this.count = 2; resolve(); };
await value;
const observed = value.count;
`;
expect(
await runInNewContext(`(async () => { ${body} return observed; })()`),
).toBe(1);
expect(
onlyBinding(analyzeJavaScriptSemantics(body), "observed").value,
).toEqual({ status: "literal", value: 1 });
});
it.each([
"const value = { count: 1, then: external };",
"const value = { count: 1, then: null }; value.then = external;",
])("keeps an externally supplied then callable possible: %s", async (slot) => {
const body = `${slot} await value; return value.count;`;
for (const external of [
null,
function (this: { count: number }, resolve: () => void) {
this.count = 2;
resolve();
},
]) {
expect(
await runInNewContext(`(async () => { ${body} })()`, { external }),
).toBe(external === null ? 1 : 2);
}
const ir = analyzeJavaScriptSemantics(
`export async function result(external) { ${body} }`,
);
expect(onlyCallable(ir, "result").returnSites[0]?.value.status).toBe(
"unknown",
);
});
it.each([
["field = shared;", "declare field: { mode: string };", "new Derived()"],
[
@@ -41,6 +167,83 @@ it.each([
},
);
it("retains then invocation effects through a getter-invoked async callable", async () => {
const body = `
const value = {
count: 1,
get start() { (async () => { await this; })(); return null; },
then(resolve) { this.count = 2; resolve(); },
};
await value.start;
return value.count;
`;
expect(await runInNewContext(`(async () => { ${body} })()`)).toBe(2);
const ir = analyzeJavaScriptSemantics(
`export async function result() { ${body} }`,
);
expect(onlyCallable(ir, "result").returnSites[0]?.value.status).toBe(
"unknown",
);
});
it.each([
[
"Object.prototype.then = function(resolve) { this.count = 2; resolve(); }; const value = { count: 1 }; await value; return value.count;",
2,
],
[
"Array.prototype.then = function(resolve) { this[0] = 2; resolve(); }; const value = [1]; await value; return value[0];",
2,
],
[
"Object.prototype.then = function(resolve) { this.count = 2; resolve(); }; const value = { count: 1, then: null }; await value; return value.count;",
1,
],
[
"Object.prototype.then = function(resolve) { this.count = 2; resolve(); }; const value = { __proto__: null, count: 1 }; await value; return value.count;",
1,
],
[
"Array.prototype.then = function(resolve) { this[0] = 2; resolve(); }; const value = [1]; value.then = null; await value; return value[0];",
1,
],
[
"Object.prototype.then = function(resolve) { this[0] = 2; resolve(); }; const value = [1]; await value; return value[0];",
2,
],
[
"Object.prototype.then = function(resolve) { this[0] = 2; resolve(); }; Array.prototype.then = null; const value = [1]; await value; return value[0];",
1,
],
[
"Object.prototype.then = null; const value = { count: 1 }; await value; return value.count;",
1,
],
[
"Object.defineProperty(Object.prototype, 'then', { get() { this.count = 2; } }); const value = { count: 1 }; await value; return value.count;",
2,
],
[
"Array.prototype.__proto__ = { then(resolve) { this[0] = 2; resolve(); } }; const value = [1]; await value; return value[0];",
2,
],
[
"Object.prototype.then = function(resolve) { this[0] = 2; resolve(); }; Array.prototype.__proto__ = null; const value = [1]; await value; return value[0];",
1,
],
] as const)(
"retains source-written prototype then invocation: %s",
async (body, expected) => {
expect(await runInNewContext(`(async () => { ${body} })()`)).toBe(expected);
const ir = analyzeJavaScriptSemantics(
`export async function result() { ${body} }`,
);
const value = onlyCallable(ir, "result").returnSites[0]?.value;
if (expected === 2) expect(value?.status).toBe("unknown");
else expect(value).toEqual({ status: "literal", value: 1 });
},
);
it.each(["field;", "field = { mode: 'other' };"])(
"preserves a real derived field's shadowing: %s",
(field) => {
@@ -269,3 +269,18 @@ describe("JavaScript semantic analysis: configuration value escapes", () => {
expect(configurationReads(body)).toEqual([]);
});
});
it.each([
["if (flag) child = spawn('tool');", "partial"],
["child = spawn('tool');", "complete"],
] as const)(
"keeps child-process incoming parameter uncertainty after %s",
(assignment, resolution) => {
const ir = analyzeJavaScriptSemantics(
`import { spawn } from 'node:child_process'; function stop(child, flag) { ${assignment} child.kill(); }`,
);
expect(ir.childProcessInteractions).toEqual([
expect.objectContaining({ kind: "signal", resolution }),
]);
},
);
@@ -28,6 +28,13 @@ const eventsFor = (call: string) =>
}));
it.each([
["a missing on listener", `e.on("x")`],
["a missing once listener", `e.once("x")`],
["a missing addListener listener", `e.addListener("x")`],
["a missing prependListener listener", `e.prependListener("x")`],
["a missing prependOnceListener listener", `e.prependOnceListener("x")`],
["a missing removeListener listener", `e.removeListener("x")`],
["a missing off listener", `e.off("x")`],
["a numeric listener", `e.on("x", 1)`],
["a string listener", `e.on("x", "fn")`],
["a template-literal listener", 'e.on("x", `fn`)'],
@@ -22,6 +22,54 @@ const releases = (source: string) =>
resolution,
}));
it.each([
["net", "connect", "-0"],
["net", "createConnection", "-0.0"],
["tls", "connect", "-0"],
["net", "connect", "typeof 1.5"],
["net", "connect", "~(-1.5)"],
["tls", "connect", "void 0, { port: 443 }"],
] as const)(
"retains %s.%s(%s) and its socket release when the unary result is valid",
(module, method, argument) => {
expect(
operations(
`import socket from "node:${module}"; const handle = socket.${method}(${argument}); handle.destroy();`,
).map(({ kind, method, resolution }) => ({ kind, method, resolution })),
).toEqual([
{ kind: "acquire", method, resolution: "complete" },
{ kind: "release", method: "destroy", resolution: "complete" },
]);
},
);
it.each([
["connect", "void 0"],
["createConnection", "void value"],
["connect", "undefined"],
["connect", "void 0, { port: 443 }"],
] as const)(
"emits no net.%s acquisition or release for a missing first argument: %s",
(method, argument) => {
expect(
operations(
`import net from 'node:net'; const handle = net.${method}(${argument}); handle.destroy();`,
),
).toEqual([]);
},
);
it("retains unknown net argument positions and shadowed undefined values", () => {
for (const body of [
"function use(undefined) { const handle=net.connect(undefined);handle.destroy(); }",
"const handle=net.connect(...args, void 0);handle.destroy();",
])
expect(operations(`import net from 'node:net';${body}`)).toEqual([
expect.objectContaining({ kind: "acquire", method: "connect" }),
expect.objectContaining({ kind: "release", method: "destroy" }),
]);
});
const commonJsPromise = `const { open } = require("node:fs/promises"); const h = open("input.txt"); h.close();`;
it("keeps conditional release facts when the loader is unknown", () => {
@@ -198,3 +246,17 @@ it("keeps an awaited handle release uncertain after an opaque receiver write", (
),
).toEqual([{ method: "close", linked: 1, resolution: "partial" }]);
});
it.each([
["if (flag) handle = createReadStream('input.txt');", "partial"],
["handle = createReadStream('input.txt');", "complete"],
] as const)(
"preserves incoming parameter uncertainty before release: %s",
(assignment, resolution) => {
expect(
releases(
`import { createReadStream } from 'node:fs'; function release(handle, flag) { ${assignment} handle.destroy(); }`,
),
).toEqual([{ method: "destroy", linked: 1, resolution }]);
},
);
@@ -16,7 +16,8 @@ import { analyzeJavaScriptSemantics } from "./javascriptSemanticAnalysis.js";
* net.connect("") -> THREW ERR_SOCKET_BAD_PORT
* net.connect("/tmp/x.sock") -> Socket created (Unix socket path)
* net.connect({ port: 80 }) -> Socket created
* net.connect(undefined) / {} / [] -> THREW ERR_MISSING_ARGS (undecided here)
* net.connect() / undefined / void 0 -> THREW ERR_MISSING_ARGS
* net.connect({}) / [] -> THREW ERR_MISSING_ARGS (undecided here)
*/
const socketAcquisitionsFor = (call: string) =>
analyzeJavaScriptSemantics(
@@ -34,6 +35,10 @@ it.each([
["a boolean port", `net.connect(true)`],
["a null port", `net.connect(null)`],
["an empty path", `net.connect("")`],
["an omitted argument", `net.connect()`],
["an explicit undefined port", `net.connect(undefined)`],
["a void port", `net.connect(void 0)`],
["an omitted createConnection argument", `net.createConnection()`],
["a tls port above the range", `tls.connect(65536)`],
["a tls negative port", `tls.connect(-1)`],
["a tls NaN port", `tls.connect(NaN)`],
@@ -49,8 +54,12 @@ it.each([
["the highest port", `net.connect(65535)`, "connect"],
["a Unix socket path", `net.connect("/tmp/x.sock")`, "connect"],
["an options object", `net.connect({ port: 80 })`, "connect"],
["an omitted argument", `net.connect()`, "connect"],
["an explicit undefined port", `net.connect(undefined)`, "connect"],
["a computed argument", `net.connect(p)`, "connect"],
[
"a TLS port supplied by later options",
`tls.connect(void 0, { port: 443 })`,
"connect",
],
["a tls port", `tls.connect(80)`, "connect"],
["a createConnection port", `net.createConnection(80)`, "createConnection"],
] as const)("keeps the socket acquisition for %s", (_name, call, method) => {
@@ -16,7 +16,6 @@ import {
resolveSemanticBindingState,
semanticResolutionBlocked,
type JavaScriptSemanticAnalysisState,
type JavaScriptSemanticBindingState,
} from "./javascriptSemanticState.js";
import { semanticBindingAtRead } from "./javascriptSemanticMutationInitializers.js";
import { traverseJavaScriptValueAst } from "./javascriptSemanticTraversal.js";
@@ -160,7 +159,10 @@ export const collectJavaScriptSemanticAsyncEffects = (
*
* A class expression passes Node's own `typeof` check, so it is kept.
*/
const rejectsEventListener = (listener: t.Node): boolean => {
export const rejectsSemanticEventListener = (
listener: t.Node | undefined,
): boolean => {
if (listener === undefined) return true;
if (t.isSpreadElement(listener)) return false;
const value = unwrapJavaScriptExpression(listener).node;
if (t.isFunctionExpression(value) || t.isArrowFunctionExpression(value))
@@ -204,9 +206,9 @@ const collectEvent = (
// provably the listener: `e.on(...["x", fn], 1)` registers `fn` and ignores the
// trailing `1`.
if (
EVENT_LISTENER_METHODS.includes(method) &&
!t.isSpreadElement(eventArgument) &&
listenerArgument !== undefined &&
rejectsEventListener(listenerArgument)
rejectsSemanticEventListener(listenerArgument)
)
return;
const emitter = emitterIdentity(member.object, state);
@@ -7,6 +7,25 @@ const analyze = (source: string) =>
`import { spawn } from 'node:child_process';\n${source}`,
);
it.each(
["on", "once", "addListener"].flatMap((method) =>
["", ", 1", ", null", ", {}", ", void 0"].map((listener) => ({
method,
listener,
})),
),
)(
"does not invent a child $method registration with rejected listener $listener",
({ method, listener }) => {
const ir = analyze(
`const child = spawn('worker'); child.${method}('exit'${listener});`,
);
expect(ir.childProcessSpawns).toHaveLength(1);
expect(ir.childProcessInteractions).toEqual([]);
expect(ir.eventOperations).toEqual([]);
},
);
it("keeps interactions before replacement resolved and later ownership uncertain", () => {
const ir = analyze(`
let child = spawn('old');
@@ -7,6 +7,7 @@ import type {
JavaScriptSemanticChildProcessSpawn,
} from "./javascriptSemanticIr.js";
import { createSemanticCallableIdStack } from "./javascriptSemanticCallableStack.js";
import { rejectsSemanticEventListener } from "./javascriptSemanticAsyncEffects.js";
import {
createSemanticExpressionShapeResolver,
optionsAfterOptionalArgv,
@@ -81,7 +82,7 @@ export const collectJavaScriptSemanticChildProcesses = (
!rejectsOptionsShape(candidate, context, state),
);
const childProcessSpawns = candidates.map((candidate) =>
immutableSpawn(candidate, context),
immutableSpawn(candidate, context, state),
);
return {
childProcessSpawns,
@@ -146,6 +147,7 @@ const collectSpawns = (
const immutableSpawn = (
candidate: SpawnCandidate,
context: ChildSpawnContext,
state: JavaScriptSemanticAnalysisState,
): JavaScriptSemanticChildProcessSpawn => {
const { options, argvCount } = spawnArguments(candidate, context);
const command = dataEffectLiteralString(candidate.node.arguments[0]);
@@ -161,7 +163,7 @@ const immutableSpawn = (
projectedResultBindings: candidate.projectedResultBindings,
command,
argvCount,
environmentSupplied: environmentSupplied(options, context),
environmentSupplied: environmentSupplied(options, context, state),
stdioMode: stdioMode(options, candidate.method, context),
resolution: command === null ? "partial" : "complete",
};
@@ -224,6 +226,11 @@ const childInteraction = (
const method = semanticStaticPropertyName(member.property, member.computed);
const classification = classifyInteraction(method, node.arguments[0]);
if (classification === null) return null;
if (
classification.kind === "listener" &&
rejectsSemanticEventListener(node.arguments[1])
)
return null;
const binding = objectBinding(member.object, state);
const { candidates: linked, complete } = resolveChildCandidates(
member.object,
@@ -443,11 +450,17 @@ const spawnArguments = (
const environmentSupplied = (
options: t.Node | undefined,
context: ChildSpawnContext,
state: JavaScriptSemanticAnalysisState,
): boolean => {
const lookup = staticObjectProperty(options, "env", context.isNullish);
if (lookup.status !== "explicit") return false;
const value = unwrapJavaScriptExpression(lookup.value).node;
if (context.isNullish(value)) return false;
if (
t.isIdentifier(value, { name: "NaN" }) &&
isUnshadowedGlobal(value, state, "NaN")
)
return false;
return staticJavaScriptTruthiness(value) !== false;
};
@@ -30,6 +30,8 @@ export interface SemanticPropertyReadSource
extends IterationSourceValue, SourceGuards {
readonly mutation: t.Node;
readonly lookup: "prototype" | "own";
/** The language protocol invokes a callable value read from this slot. */
readonly invoke?: true;
}
interface PatternSource extends IterationSourceValue {
@@ -337,6 +339,7 @@ const semanticSourceFacts = (
t.isAssignmentExpression(node) ||
t.isForOfStatement(node) ||
t.isSpreadElement(node) ||
t.isAwaitExpression(node) ||
t.isYieldExpression(node) ||
t.isCatchClause(node) ||
t.isFunction(node)
@@ -372,11 +375,15 @@ const semanticSourceFacts = (
mutation: node,
lookup: "own",
});
} else if (
t.isYieldExpression(node) &&
node.delegate &&
node.argument != null
)
} else if (t.isAwaitExpression(node))
propertyReads.push({
node: node.argument,
projection: ["then"],
mutation: node,
lookup: "prototype",
invoke: true,
});
else if (t.isYieldExpression(node) && node.delegate && node.argument != null)
add(sources, { node: node.argument, projection: [] }, node, {});
else if (
t.isCatchClause(node) &&
@@ -2,6 +2,7 @@ import * as t from "@babel/types";
import { semanticKnownSymbolKey } from "./javascriptSemanticPropertyKeys.js";
import {
createAmbientPathResolver,
createGlobalPathReader,
retainSemanticAmbientEffect,
} from "./javascriptSemanticAmbientWrites.js";
import { isIntrinsicMapLookup } from "./javascriptSemanticMapOperations.js";
@@ -37,6 +38,7 @@ import {
import { semanticContainer } from "./javascriptSemanticSlots.js";
import {
semanticProjectedSourceFactsAtRead,
semanticCapturedSelectionsAfterPropertyRead,
semanticShiftCapturedSelections,
semanticNormalizeCapturedSelections,
type SemanticCapturedSelection,
@@ -49,6 +51,7 @@ import {
semanticClassAccessorReferences,
semanticObjectPropertyReferences,
semanticObjectAccessorReferences,
semanticNativePrototypeChain,
type JavaScriptSemanticObjectReference,
} from "./javascriptSemanticObjectReferences.js";
import {
@@ -122,27 +125,55 @@ const arrayPrototypeIterationMayChange = (path: PropertyPath): boolean => {
/** Preserve uncertainty from property writes and references exposed to calls, yielding during traversal and fixpoint iteration. */
/** Accessor selection uses the same slots, prototype shadowing and array indices as aliases. */
const localAccessorMethods = (
/** Local methods use the same slots, prototype shadowing and array indices as aliases. */
const localPropertyMethods = (
root: t.Node,
path: PropertyPath,
accessor: "get" | "set",
accessor: "get" | "set" | "invoke",
state: JavaScriptSemanticAnalysisState,
keyMayExist: (
key: PropertyPath[number] | undefined,
kind: "get" | "set",
) => boolean,
lookup: "prototype" | "own" = "prototype",
): readonly (t.ObjectMethod | t.ClassMethod)[] | undefined => {
if (!keyMayExist(path.at(-1), accessor)) return [];
const methods = new Set<t.ObjectMethod | t.ClassMethod>();
const pending = [{ node: root, path }];
): readonly t.Function[] | undefined => {
if (accessor !== "invoke" && !keyMayExist(path.at(-1), accessor)) return [];
const methods = new Set<t.Function>();
const pending: (MutationSelectionContext & {
node: t.Node;
path: PropertyPath;
})[] = [{ node: root, path, memberReadAt: root }];
const examined = new WeakMap<t.Node, Set<string>>();
const nativeReaders = new Map<
string,
ReturnType<typeof createGlobalPathReader>
>();
const nativeRead = (
prototype: "Object" | "Array",
key: string,
at: t.Node,
) => {
const identity = JSON.stringify([prototype, key]);
let read = nativeReaders.get(identity);
if (read === undefined) {
read = createGlobalPathReader(state, [prototype, "prototype", key]);
nativeReaders.set(identity, read);
}
return read(at);
};
let selections = 0;
for (let cursor = 0; cursor < pending.length; cursor++) {
const current = pending[cursor]!;
const node = unwrapJavaScriptExpression(current.node).node;
const key = JSON.stringify(current.path);
const key = JSON.stringify([
current.path,
current.capturedSelections?.map(({ at, depth, ownEnumerable }) => [
at.start,
at.end,
depth,
ownEnumerable,
]),
]);
const paths = examined.get(node) ?? new Set<string>();
if (paths.has(key)) continue;
paths.add(key);
@@ -152,7 +183,10 @@ const localAccessorMethods = (
return undefined;
}
if (
(t.isObjectMethod(node) || t.isClassMethod(node)) &&
t.isFunction(node) &&
(accessor === "invoke" ||
t.isObjectMethod(node) ||
t.isClassMethod(node)) &&
current.path.length === 0
) {
methods.add(node);
@@ -160,15 +194,42 @@ const localAccessorMethods = (
}
if (t.isIdentifier(node)) {
const binding = resolveSemanticBindingState(state, node, node.name);
if (binding === undefined) continue;
const selected = semanticBindingAtRead(binding, node, state).binding;
for (const source of [
...selected.initializers,
...selected.referenceInitializers,
])
if (binding === undefined) {
if (accessor === "invoke") return undefined;
continue;
}
if (
accessor === "invoke" &&
!semanticBindingAtRead(binding, node, state).hasDefiniteInitializer
)
return undefined;
const selected = semanticProjectedSourceFactsAtRead(
binding,
node,
current.path,
state,
true,
current.memberReadAt,
current.capturedSelections,
false,
// A spread copy escapes the copied values. That must not make the
// source's own accessors unknown, or the next read escapes the
// receiver itself.
accessor === "invoke",
);
if (selected === undefined) return undefined;
for (const source of selected.sources)
pending.push({
node: source.node,
path: [...source.projection, ...current.path],
path: source.path,
memberReadAt: current.memberReadAt,
capturedSelections: semanticNormalizeCapturedSelections([
...(semanticShiftCapturedSelections(
current.capturedSelections,
source.path.length - current.path.length,
) ?? []),
...(source.capturedSelections ?? []),
]),
});
} else if (
t.isMemberExpression(node) ||
@@ -180,15 +241,57 @@ const localAccessorMethods = (
semanticStaticPropertyKey(node.property, node.computed),
...current.path,
],
memberReadAt: current.memberReadAt,
capturedSelections: semanticShiftCapturedSelections(
current.capturedSelections,
1,
),
});
} else if (
current.path.length > 0 &&
(t.isObjectExpression(node) || t.isArrayExpression(node))
) {
const [selected, ...remaining] = current.path;
if (
accessor === "invoke" &&
lookup === "prototype" &&
!current.capturedSelections?.some(
(capture) => capture.depth === 1 && capture.ownEnumerable,
)
) {
for (const prototype of semanticNativePrototypeChain(node, selected!)) {
if (selected === null || typeof selected === "object")
return undefined;
const at = current.memberReadAt ?? root;
const member = nativeRead(prototype, String(selected), at);
if (member.status === "unknown" || member.status === "unrelated")
return undefined;
if (member.status === "replacement") {
pending.push({
node: member.value,
path: remaining,
memberReadAt: at,
capturedSelections: semanticShiftCapturedSelections(
current.capturedSelections,
-1,
),
});
break;
}
// A changed native link selects its retained source rather than the
// default next prototype. Null naturally ends that lookup.
const link = nativeRead(prototype, "__proto__", at);
if (link.status === "unknown" || link.status === "unrelated")
return undefined;
if (link.status === "replacement") {
pending.push({ ...current, node: link.value });
break;
}
}
}
const references = t.isArrayExpression(node)
? semanticArrayPropertyReferences(node, [selected!])
: remaining.length === 0
: remaining.length === 0 && accessor !== "invoke"
? semanticObjectAccessorReferences(
node,
selected!,
@@ -197,11 +300,24 @@ const localAccessorMethods = (
lookup,
)
: semanticObjectPropertyReferences(node, [selected!], state);
for (const source of references)
for (const source of references) {
if (
source.inherited &&
current.capturedSelections?.some(
(capture) => capture.depth === 1 && capture.ownEnumerable,
)
)
continue;
pending.push({
node: source.node,
path: [...source.path, ...remaining],
memberReadAt: current.memberReadAt,
capturedSelections: semanticCapturedSelectionsAfterPropertyRead(
current.capturedSelections,
source,
),
});
}
} else if (
(t.isClass(node) || t.isNewExpression(node)) &&
current.path.length > 0
@@ -218,23 +334,39 @@ const localAccessorMethods = (
const references =
remaining.length > 0
? semanticClassPropertyReferences(selection, [selected!], state)
: lookup === "own"
? []
: semanticClassAccessorReferences(
selection,
selected!,
state,
accessor,
);
: accessor === "invoke"
? semanticClassPropertyReferences(selection, [selected!], state)
: lookup === "own"
? []
: semanticClassAccessorReferences(
selection,
selected!,
state,
accessor,
);
for (const source of references)
pending.push({
node: source.node,
path: [...source.path, ...remaining],
memberReadAt: current.memberReadAt,
capturedSelections: semanticCapturedSelectionsAfterPropertyRead(
current.capturedSelections,
source,
),
});
}
} else {
for (const source of branchReferenceValues(node, current.path))
pending.push(source);
const references = branchReferenceValues(node, current.path);
if (accessor === "invoke" && references.length === 0) {
const value = evaluateSemanticExpression(
node,
state,
current.memberReadAt,
);
if (!["literal", "union", "object", "array"].includes(value.status))
return undefined;
}
for (const source of references) pending.push({ ...current, ...source });
}
}
return [...methods];
@@ -247,6 +379,7 @@ type AccessorEffectOperand =
readonly node: t.Node;
readonly path: PropertyPath;
readonly lookup: "prototype" | "own";
readonly invoke?: true;
readonly guards?: SemanticPropertyReadSource;
}
| {
@@ -333,6 +466,7 @@ const accessorEffectOperands = (
node: source.node,
path: source.projection,
lookup: source.lookup,
...(source.invoke ? { invoke: true } : {}),
guards: source,
});
if (
@@ -383,13 +517,14 @@ export function* collectSemanticMemberMutationsSteps(
);
};
const returnedReferences = createSemanticReturnedReferences(state);
const getterEffects = (
const propertyReadEffects = (
root: t.Node,
path: PropertyPath,
lookup: "prototype" | "own" = "prototype",
invoke = false,
): readonly JavaScriptSemanticObjectReference[] => {
if (!accessorKeyMayExist(path.at(-1), "get")) return [];
const pending = [{ node: root, path, lookup }];
if (!invoke && !accessorKeyMayExist(path.at(-1), "get")) return [];
const pending = [{ node: root, path, lookup, invoke }];
const examined = new Set<string>();
const effects: JavaScriptSemanticObjectReference[] = [];
for (let cursor = 0; cursor < pending.length; cursor++) {
@@ -397,12 +532,14 @@ export function* collectSemanticMemberMutationsSteps(
node: selectedRoot,
path: selected,
lookup: selectedLookup,
invoke: selectedInvoke,
} = pending[cursor]!;
const identity = JSON.stringify([
selectedRoot.start,
selectedRoot.end,
selected,
selectedLookup,
selectedInvoke,
]);
if (examined.has(identity)) continue;
examined.add(identity);
@@ -410,7 +547,20 @@ export function* collectSemanticMemberMutationsSteps(
state.valueResourceLimits.add("alias-selections");
return [{ node: root, path: [] }];
}
const methods = localAccessorMethods(
const receiver = selected.slice(0, -1);
if (
selectedInvoke &&
(localPropertyMethods(
selectedRoot,
selected,
"invoke",
state,
accessorKeyMayExist,
selectedLookup,
)?.length ?? 1) > 0
)
effects.push({ node: selectedRoot, path: receiver });
const methods = localPropertyMethods(
selectedRoot,
selected,
"get",
@@ -418,12 +568,12 @@ export function* collectSemanticMemberMutationsSteps(
accessorKeyMayExist,
selectedLookup,
);
const receiver = selected.slice(0, -1);
if (methods === undefined) {
effects.push({ node: selectedRoot, path: receiver });
continue;
}
for (const method of methods) {
if (!t.isObjectMethod(method) && !t.isClassMethod(method)) continue;
const callables: t.Function[] = [method];
const visitedCallables = new Set<t.Function>();
for (let index = 0; index < callables.length; index++) {
@@ -441,7 +591,8 @@ export function* collectSemanticMemberMutationsSteps(
}
if (
operand.kind === "read" &&
(!accessorKeyMayExist(operand.path.at(-1), "get") ||
((!operand.invoke &&
!accessorKeyMayExist(operand.path.at(-1), "get")) ||
(operand.guards !== undefined &&
semanticReferenceSourcesInactive(operand.guards, state)))
)
@@ -465,6 +616,7 @@ export function* collectSemanticMemberMutationsSteps(
node: targetRoot,
path: [...target, ...operand.path],
lookup: operand.lookup,
invoke: operand.invoke === true,
});
}
}
@@ -1114,7 +1266,7 @@ export function* collectSemanticMemberMutationsSteps(
if (
origins.initializers.some(
(source) =>
(localAccessorMethods(
(localPropertyMethods(
source.node,
[...source.projection, ...selectedPath],
"set",
@@ -1596,7 +1748,7 @@ export function* collectSemanticMemberMutationsSteps(
source.kind === "reference-store" &&
source.target !== undefined &&
!state.escapingReferenceStores.has(source.target) &&
(localAccessorMethods(
(localPropertyMethods(
source.target.object,
[
semanticStaticPropertyKey(
@@ -1658,17 +1810,18 @@ export function* collectSemanticMemberMutationsSteps(
}
for (const source of semanticPropertyReadSources(node, parent, state)) {
if (semanticReferenceSourcesInactive(source, state)) continue;
for (const effect of getterEffects(
for (const effect of propertyReadEffects(
source.node,
source.projection,
source.lookup,
source.invoke === true,
))
markEscaped(effect.node, source.mutation, effect.path);
}
if (t.isMemberExpression(node) || t.isOptionalMemberExpression(node)) {
// The write owner classifies direct and destructuring stores once.
if (!writeOnlyMembers.has(node))
for (const effect of getterEffects(node.object, [
for (const effect of propertyReadEffects(node.object, [
semanticStaticPropertyKey(node.property, node.computed),
]))
markEscaped(effect.node, node, effect.path);
@@ -4,6 +4,7 @@ import { expect, it } from "vitest";
import { indexJavaScriptBindingsSteps } from "./javascriptSemanticAnalysis.js";
import {
semanticClassPropertyReferences,
semanticNativePrototypeChain,
semanticObjectPropertyReferences,
} from "./javascriptSemanticObjectReferences.js";
import { completeSemanticSteps } from "./javascriptSemanticTraversal.js";
@@ -36,6 +37,37 @@ const spelled = (
],
);
it.each([
["{}", "then", ["Object"]],
["{then:null}", "then", []],
["{then}", "then", []],
["{...{then:null}}", "then", []],
["{...unknown}", "then", ["Object"]],
["{__proto__:null}", "then", []],
["{__proto__:{}}", "then", []],
["{__proto__:0}", "then", ["Object"]],
["{__proto__:unknown}", "then", ["Object"]],
['{["__proto__"]:null}', "then", ["Object"]],
["[]", "then", ["Array", "Object"]],
["[]", "length", []],
["[value]", "0", []],
["[,value]", "0", ["Array", "Object"]],
["[value,...unknown]", "0", []],
["[...unknown,value]", "0", ["Array", "Object"]],
] as const)(
"retains possible native prototype lookup for %s[%s]",
(value, key, expected) => {
const parsed = parseJavaScriptSource(`const o=${value};`, "prototype.mjs");
const declaration = parsed?.program.body[0];
if (!t.isVariableDeclaration(declaration))
throw new Error("Expected declaration");
const node = declaration.declarations[0]?.init;
if (!t.isObjectExpression(node) && !t.isArrayExpression(node))
throw new Error("Expected literal");
expect(semanticNativePrototypeChain(node, key)).toEqual(expected);
},
);
it("resolves exact keys with source-order overrides, spreads and accessors", () => {
const fixture = objectAndState(
`const o = { a: 1, ...base, b: 2, a: 3, [dynamic]: 4, __proto__: proto, ...{ b: 5 }, get c() { return 6; }, c: 7 };`,
@@ -54,14 +86,13 @@ it("resolves exact keys with source-order overrides, spreads and accessors", ()
["4", "[]"],
["base", '["missing"]'],
]);
// A wildcard key still scans every property.
// A wildcard keeps possible slots, excluding data replaced by a known spread.
expect(spelled(fixture, null)).toEqual([
["7", "[]"],
["{ b: 5 }", '[{"excludedKeys":["c"]}]'],
["proto", "[null]"],
["4", "[]"],
["3", "[]"],
["2", "[]"],
["base", '[{"excludedKeys":["a","b","c"]}]'],
]);
// Numeric keys use the same exact spelling as string keys.
@@ -70,6 +101,112 @@ it("resolves exact keys with source-order overrides, spreads and accessors", ()
).toEqual([['"one"', "[]"]]);
});
it.each([1, "1"])(
"preserves inherited numeric property selection %j and own shadowing",
(key) => {
expect(
spelled(objectAndState("const o = { __proto__: prototype };"), key),
).toEqual([["prototype", JSON.stringify([String(key)])]]);
expect(
spelled(
objectAndState('const o = { __proto__: prototype, 1: "own" };'),
key,
),
).toEqual([['"own"', "[]"]]);
},
);
it.each([
[
"static",
"static method(){return 1;} method(){return 2;}",
"static method(){return 1;}",
],
[
"instance",
"static method(){return 1;} method(){return 2;}",
"method(){return 2;}",
],
[
"prototype",
"static method(){return 1;} method(){return 2;}",
"method(){return 2;}",
],
["static", "static method(){return 1;} static set method(value){}", null],
["static", "static method=child; static method(){return 1;}", "child"],
[
"static",
"static get method(){return child;} static method(){return 1;}",
"static method(){return 1;}",
],
[
"static",
"static method(){return 1;} static get method(){return child;}",
"child",
],
[
"static",
"static get method(){return child;} static method(){return 1;} static set method(value){}",
null,
],
] as const)(
"selects method-valued %s descriptors with runtime precedence: %s",
(kind, body, expected) => {
const source = `class Box { ${body} }`;
const parsed = parseJavaScriptSource(source, "methods.mjs");
if (parsed === null) throw new Error("Expected a parsed class");
const owner = parsed.program.body[0];
if (!t.isClassDeclaration(owner)) throw new Error("Expected a class");
const state = completeSemanticSteps(
indexJavaScriptBindingsSteps(parsed.program),
);
expect(
semanticClassPropertyReferences({ owner, kind }, ["method"], state).map(
({ node }) => source.slice(node.start ?? 0, node.end ?? 0),
),
).toEqual(expected === null ? [] : [expected]);
},
);
it.each(["method(){return 1;}", "...{method: child}"])(
"keeps an intervening object data definition between accessor pairs: %s",
(definition) => {
expect(
spelled(
objectAndState(
`const o = { get method(){return child;} , ${definition}, set method(value){} };`,
),
"method",
),
).toEqual([]);
},
);
it.each(["static", "instance"] as const)(
"retains inherited %s field ownership for enumerable copies",
(kind) => {
const source = `class Base { ${kind === "static" ? "static" : ""} method=child; } class Box extends Base {}`;
const parsed = parseJavaScriptSource(source, "fields.mjs");
if (parsed === null) throw new Error("Expected a parsed class");
const owner = parsed.program.body[1];
if (!t.isClassDeclaration(owner)) throw new Error("Expected derived class");
const state = completeSemanticSteps(
indexJavaScriptBindingsSteps(parsed.program),
);
const references = semanticClassPropertyReferences(
{ owner, kind, fieldsOnly: true },
["method"],
state,
);
expect(
references.map(({ node }) =>
source.slice(node.start ?? 0, node.end ?? 0),
),
).toEqual(["child"]);
expect(references[0]?.inherited).toBe(kind === "static" ? true : undefined);
},
);
it("preserves references through a small static getter permutation", () => {
const getters = Array.from(
{ length: 5 },
@@ -154,96 +154,73 @@ const classPropertyDescriptors = (
initializedBefore?: t.ClassProperty,
accessor: "get" | "set" = "get",
) => {
const isStatic = kind === "static";
const selectedAccessors: t.ClassMethod[] = [];
const fields: t.ClassProperty[] = [];
const indexedMembers: {
readonly member: t.ClassMethod | t.ClassProperty;
readonly index: number;
readonly key: string | null;
}[] = [];
const methods: t.ClassMethod[] = [];
const accessors: t.ClassMethod[] = [];
const definitions: (t.ClassMethod | t.ClassProperty)[] = [];
const before =
initializedBefore === undefined
? -1
: klass.body.body.indexOf(initializedBefore);
klass.body.body.forEach((member, index) => {
for (const [index, member] of klass.body.body.entries()) {
if (
(t.isClassMethod(member) || t.isClassProperty(member)) &&
propertyName !== null &&
member.computed &&
semanticKnownSymbolKey(member.key, state)
(!t.isClassMethod(member) &&
!(kind === "static" && isRuntimeJavaScriptClassProperty(member))) ||
member.static !== (kind === "static") ||
(before >= 0 &&
isRuntimeJavaScriptClassProperty(member) &&
index >= before) ||
(propertyName !== null &&
member.computed &&
semanticKnownSymbolKey(member.key, state))
)
return;
if (
before >= 0 &&
index >= before &&
isRuntimeJavaScriptClassProperty(member)
)
return;
if (
(t.isClassMethod(member) ||
(kind === "static" && isRuntimeJavaScriptClassProperty(member))) &&
member.static === isStatic
)
indexedMembers.push({
member,
index,
key: semanticStaticPropertyKey(member.key, member.computed),
});
});
const matchingMembers =
propertyName === null
? []
: indexedMembers.filter(({ key }) => key === propertyName);
const uncertainAccessors = indexedMembers.filter(
(entry): entry is typeof entry & { member: t.ClassMethod } => {
const { member, key } = entry;
return (
t.isClassMethod(member) &&
member.kind === accessor &&
(propertyName === null || (member.computed && key === null))
);
},
);
const getters = matchingMembers.filter(
(entry): entry is typeof entry & { member: t.ClassMethod } =>
t.isClassMethod(entry.member) && entry.member.kind === accessor,
);
const getter = getters.at(-1);
const getterIndex = getter?.index ?? -1;
const laterDataMember = matchingMembers.some(
({ member, index }) =>
t.isClassProperty(member) ||
(t.isClassMethod(member) &&
member.kind === "method" &&
index > getterIndex),
);
const seenFields = new Set<string>();
for (const { member, key } of [...indexedMembers].reverse()) {
if (!t.isClassProperty(member)) continue;
if (key !== null && seenFields.has(key)) continue;
if (key !== null) seenFields.add(key);
if (propertyName === null || key === null || key === propertyName)
fields.push(member);
continue;
definitions.push(member);
}
if (getters.length > 0 && !laterDataMember && getter !== undefined)
selectedAccessors.push(getter.member);
for (const { member, index } of uncertainAccessors)
// Methods/accessors are installed before static fields execute, even when a
// field appears earlier in source. Instance fields are owned separately.
const ordered = [
...definitions.filter((member) => t.isClassMethod(member)),
...definitions.filter((member) => t.isClassProperty(member)),
];
const data = new Set<string>();
const getterNames = new Set<string>();
const setterNames = new Set<string>();
let inherits = true;
for (const member of ordered.reverse()) {
const key = semanticStaticPropertyKey(member.key, member.computed);
if (propertyName !== null && key !== null && key !== propertyName) continue;
if (propertyName !== null && key === propertyName) inherits = false;
const descriptor =
t.isClassMethod(member) &&
(member.kind === "get" || member.kind === "set")
? member.kind
: "data";
const selected = key ?? propertyName;
if (
!matchingMembers.some(
({ member: matching, index: matchingIndex }) =>
(t.isClassProperty(matching) ||
(t.isClassMethod(matching) &&
(matching.kind === "method" || matching.kind === accessor))) &&
matchingIndex > index,
)
)
selectedAccessors.push(member);
return {
accessors: selectedAccessors,
fields,
inherits: matchingMembers.length === 0,
};
selected !== null &&
(data.has(selected) ||
(descriptor === "data"
? getterNames.has(selected) || setterNames.has(selected)
: (descriptor === "get" ? getterNames : setterNames).has(selected)))
) {
// A later accessor replaced this data descriptor, but the data definition
// still prevents an earlier accessor from pairing across it.
if (descriptor === "data" && key !== null) data.add(key);
continue;
}
if (t.isClassProperty(member)) fields.push(member);
else if (member.kind === "method") methods.push(member);
else if (member.kind === accessor) accessors.push(member);
if (key !== null)
(descriptor === "data"
? data
: descriptor === "get"
? getterNames
: setterNames
).add(key);
}
return { accessors, fields, methods, inherits };
};
export const semanticClassPropertyReferences = (
@@ -345,9 +322,21 @@ class ClassReferenceWalk {
? descriptors.fields
: [])
if (field.value != null)
this.#references.push({ node: field.value, path: this.#path });
this.#references.push({
node: field.value,
path: this.#path,
...(this.lookup.kind === "static" && klass !== this.lookup.owner
? { inherited: true }
: {}),
});
for (const getter of this.lookup.fieldsOnly ? [] : descriptors.accessors)
this.#getter(klass, getter);
if (this.context.accessor === undefined && !this.lookup.fieldsOnly)
for (const method of descriptors.methods) {
const name = semanticStaticPropertyKey(method.key, method.computed);
if (name !== null && this.#fields.has(name)) continue;
this.#references.push({ node: method, path: this.#path });
}
// Only a definite own descriptor can suppress possible superclass results.
if (descriptors.inherits && klass.superClass != null)
for (const base of semanticClassCandidates(klass.superClass, this.state))
@@ -559,6 +548,18 @@ const expandGetterResult = (
type ObjectLiteralMember = t.ObjectExpression["properties"][number];
/** Only this colon-form definition selects an object literal's prototype. */
const isObjectPrototypeSetter = (
property: ObjectLiteralMember,
name = t.isSpreadElement(property)
? null
: semanticStaticPropertyKey(property.key, property.computed),
): property is t.ObjectProperty =>
t.isObjectProperty(property) &&
!property.computed &&
!property.shorthand &&
name === "__proto__";
/** Property positions grouped by whether an exact key can reach them. */
interface ObjectPropertyIndex {
/** Spreads, computed keys and `__proto__` data entries reach every key. */
@@ -579,13 +580,7 @@ const objectPropertyIndex = (node: t.ObjectExpression): ObjectPropertyIndex => {
const name = t.isSpreadElement(property)
? null
: semanticStaticPropertyKey(property.key, property.computed);
if (
name === null ||
(t.isObjectProperty(property) &&
!property.computed &&
!property.shorthand &&
name === "__proto__")
) {
if (name === null || isObjectPrototypeSetter(property, name)) {
keyIndependent.push(index);
return;
}
@@ -671,7 +666,9 @@ class ObjectReferenceWalk {
readonly accessor?: "get" | "set",
readonly accessorLookup: "prototype" | "own" = "prototype",
) {
[this.#key, ...this.#remaining] = path;
const [key, ...remaining] = path;
this.#key = typeof key === "number" ? String(key) : key;
this.#remaining = remaining;
}
#spreadKey(): PropertyPath[number] {
@@ -691,38 +688,29 @@ class ObjectReferenceWalk {
#spread(property: t.SpreadElement): void {
const selected = this.#key ?? null;
if (typeof selected !== "object" && this.#overwritten.has(String(selected)))
return;
// Spread creates data properties; the source descriptor is not installed.
if (this.accessor === undefined)
if (
this.accessor === undefined &&
(typeof selected === "object" || !this.#overwritten.has(String(selected)))
)
this.#references.push({
node: property.argument,
path: [this.#spreadKey(), ...this.#remaining],
capturedSelectionAt: property.argument,
ownEnumerable: true,
});
if (
selected !== null &&
typeof selected !== "object" &&
t.isObjectExpression(property.argument) &&
objectExpressionDefinesKey(property.argument, String(selected))
) {
this.#overwritten.add(String(selected));
this.#data.add(String(selected));
}
if (t.isObjectExpression(property.argument))
for (const name of objectExpressionDefinedKeys(property.argument)) {
this.#overwritten.add(name);
this.#data.add(name);
}
}
#prototype(
property: t.ObjectMethod | t.ObjectProperty,
name: string | null,
): boolean {
if (
!t.isObjectProperty(property) ||
property.computed ||
property.shorthand ||
name !== "__proto__"
)
return false;
if (!isObjectPrototypeSetter(property, name)) return false;
const selected = this.#key ?? null;
if (this.accessor !== undefined && this.accessorLookup === "own")
return true;
@@ -731,7 +719,7 @@ class ObjectReferenceWalk {
(selected === null ||
typeof selected === "object" ||
(typeof selected === "string" &&
!objectExpressionDefinesKey(this.node, selected)))
!objectExpressionDefinedKeys(this.node).has(selected)))
)
this.#references.push({
node: property.value,
@@ -819,8 +807,10 @@ class ObjectReferenceWalk {
? this.#data.has(name) || this.#accessors.has(name)
: this.#data.has(name) ||
(kind === "get" ? this.#getters : this.#setters).has(name))
)
) {
if (kind === "data") this.#data.add(name);
return;
}
this.#read(property, name);
if (name !== null) this.#record(name, kind);
}
@@ -839,12 +829,12 @@ const objectPropertyReferences = (
): readonly JavaScriptSemanticObjectReference[] =>
new ObjectReferenceWalk(node, path, query).run();
const objectExpressionDefinesKey = (
const objectExpressionDefinedKeys = (
root: t.ObjectExpression,
key: string,
): boolean => {
): ReadonlySet<string> => {
const pending: t.ObjectExpression[] = [root];
const seen = new Set<t.ObjectExpression>();
const names = new Set<string>();
for (let cursor = 0; cursor < pending.length; cursor++) {
const object = pending[cursor];
if (object === undefined || seen.has(object)) continue;
@@ -856,19 +846,11 @@ const objectExpressionDefinesKey = (
continue;
}
const name = semanticStaticPropertyKey(property.key, property.computed);
if (
name === key &&
!(
t.isObjectProperty(property) &&
!property.computed &&
!property.shorthand &&
name === "__proto__"
)
)
return true;
if (name !== null && !isObjectPrototypeSetter(property, name))
names.add(name);
}
}
return false;
return names;
};
export const semanticArrayPropertyReferences = (
@@ -924,3 +906,44 @@ export const semanticArrayPropertyReferences = (
}
return references;
};
/** Possible native fallback after own slots; explicit prototype references are walked separately. */
export const semanticNativePrototypeChain = (
node: t.ObjectExpression | t.ArrayExpression,
key: PropertyPath[number],
): readonly ("Object" | "Array")[] => {
const exact =
key !== null && typeof key !== "object" ? String(key) : undefined;
if (t.isArrayExpression(node)) {
if (exact === "length") return [];
if (exact !== undefined) {
const sources = semanticArrayPropertyReferences(node, [exact]);
if (
sources.length > 0 &&
sources.every((source) => source.capturedSelectionAt === undefined)
)
return [];
}
return ["Array", "Object"];
}
if (exact !== undefined && objectExpressionDefinedKeys(node).has(exact))
return [];
for (const property of node.properties) {
if (!isObjectPrototypeSetter(property)) continue;
const prototype = unwrapJavaScriptExpression(property.value).node;
// Null removes the prototype. Objects select the explicitly retained
// chain, while primitive values are ignored and unknowns may be primitive.
if (
t.isNullLiteral(prototype) ||
t.isObjectExpression(prototype) ||
t.isArrayExpression(prototype) ||
t.isFunctionExpression(prototype) ||
t.isArrowFunctionExpression(prototype) ||
t.isClassExpression(prototype) ||
t.isRegExpLiteral(prototype) ||
t.isNewExpression(prototype)
)
return [];
}
return ["Object"];
};
@@ -190,6 +190,7 @@ export function semanticProjectedSourceFactsAtRead(
memberReadAt?: t.Node,
capturedSelections?: readonly SemanticCapturedSelection[],
retainEligibility?: boolean,
valueEscapesHideSources?: boolean,
):
| {
readonly sources: readonly (Omit<ProjectedSource, "path"> & {
@@ -219,6 +220,9 @@ const projectedSourceFactsAtRead = (
memberReadAt: t.Node = reference,
capturedSelections: readonly SemanticCapturedSelection[] = [],
retainEligibility = false,
// Accessor lookup reads source syntax. A prior value escape does not hide
// that syntax; unknown writes and invocations still fail closed.
valueEscapesHideSources = true,
):
| {
readonly sources: readonly ProjectedSource[];
@@ -365,6 +369,7 @@ const projectedSourceFactsAtRead = (
(replacement.assignedSources?.[0]?.target?.start ?? -1)));
if (
includeUnknown &&
valueEscapesHideSources &&
semanticSomeEffectPathCandidate(
binding.escapedPaths,
path,
@@ -2,6 +2,40 @@ import { expect, it } from "vitest";
import { analyzeJavaScriptStaticSource } from "./javascriptStaticAnalysis.js";
it.each([
"load('./dep.cjs')",
"load.call(null,'./dep.cjs')",
"load.apply(null,['./dep.cjs'])",
"Reflect.apply(load,null,['./dep.cjs'])",
])("retains a createRequire filename through %s", (call) => {
const result = analyzeJavaScriptStaticSource(
`import {createRequire} from 'node:module';const load=createRequire(new URL('./sub/entry.cjs',import.meta.url));${call};`,
);
expect(result.references.filter(({ kind }) => kind === "require")).toEqual([
expect.objectContaining({
specifier: "./dep.cjs",
resolution_base_path: "./sub/entry.cjs",
}),
]);
});
it.each([
"load.call(null,'./dep.cjs')",
"Reflect.apply(load,null,['./dep.cjs'])",
])("keeps mixed createRequire bases unknown through %s", (call) => {
const result = analyzeJavaScriptStaticSource(
`import {createRequire} from 'node:module';const load=createRequire(flag?import.meta.url:new URL('./sub/entry.cjs',import.meta.url));${call};`,
);
expect(result.references.filter(({ kind }) => kind === "require")).toEqual([
expect.objectContaining({ specifier: null }),
]);
expect(
result.references.every(
(reference) => !("resolution_base_path" in reference),
),
).toBe(true);
});
it.each([
['const h={require:module.require};h.require("./shared.cjs");', true],
[
@@ -30,6 +30,8 @@ import {
import {
semanticStaticPropertyKey,
semanticMemberChain,
semanticWriteSiteTarget,
semanticWriteTargets,
unwrapJavaScriptExpression,
} from "./javascriptAstValues.js";
import { semanticPropertyPathKeyMatches } from "./javascriptSemanticPropertyPaths.js";
@@ -71,30 +73,33 @@ export type {
const possibleCommonJsDelegateWriteStates =
new WeakSet<JavaScriptSemanticAnalysisState>();
/** Admission and indexing share the same direct, projected and iteration leaves. */
function* commonJsDelegateWriteTargets(
node: t.Node,
): Generator<t.MemberExpression | t.OptionalMemberExpression> {
const write = semanticWriteSiteTarget(node);
if (write === null) return;
for (const target of semanticWriteTargets(write)) {
if (!t.isMemberExpression(target) && !t.isOptionalMemberExpression(target))
continue;
const key = semanticStaticPropertyKey(target.property, target.computed);
if (key === "require" || (key === null && target.computed)) yield target;
}
}
/** Source eligibility only; the shared write owner still proves receiver identity. */
export const observePossibleCommonJsDelegateWrite = (
node: t.Node,
state: JavaScriptSemanticAnalysisState,
): void => {
if (possibleCommonJsDelegateWriteStates.has(state)) return;
const target = t.isAssignmentExpression(node)
? node.left
: t.isUpdateExpression(node) ||
t.isUnaryExpression(node, { operator: "delete" })
? node.argument
: undefined;
if (
semanticIterationSources(node, state.parentsByNode.get(node) ?? null).some(
(source) => source.kind === "reference-store",
)
)
possibleCommonJsDelegateWriteStates.add(state);
if (
(t.isMemberExpression(target) || t.isOptionalMemberExpression(target)) &&
["require", null].includes(
semanticStaticPropertyKey(target.property, target.computed),
)
)
if (!commonJsDelegateWriteTargets(node).next().done)
possibleCommonJsDelegateWriteStates.add(state);
else if (t.isCallExpression(node) || t.isOptionalCallExpression(node))
possibleCommonJsDelegateWriteStates.add(state);
@@ -886,6 +891,9 @@ export const semanticRequireCallAt = (
}
const interop = loaderInvocationInterop(state);
if (!t.isNewExpression(node)) {
// Invocation normalization retains the selected path identity. Keep its
// filename alongside that path rather than selecting the outer call again.
const sourceRelativePaths = new Map<readonly string[], string>();
const invocation = interop.intrinsicInvocation(node, (callee) => {
const loader = requireLoaderValueAt(
callee,
@@ -912,22 +920,21 @@ export const semanticRequireCallAt = (
state,
factoryBindingIds,
);
if (selected !== undefined)
return [
if (selected !== undefined) {
const path = [
"require-loader",
selected.loader,
selected.sourceRelative ? "relative" : "unknown",
...(selected.loaderAuthority === "candidate" ? ["candidate"] : []),
];
if (selected.sourceRelativePath !== undefined)
sourceRelativePaths.set(path, selected.sourceRelativePath);
return path;
}
return interop.intrinsicPath(callee);
});
if (invocation?.path[0] === "require-loader") {
const selected = selectedRequireSourcesAt(
node.callee,
node,
state,
factoryBindingIds,
);
const sourceRelativePath = sourceRelativePaths.get(invocation.path);
return {
operation: "load",
loader:
@@ -942,9 +949,9 @@ export const semanticRequireCallAt = (
: {}),
sourceRelative:
invocation.kind === "known" && invocation.path[2] === "relative",
...(selected?.sourceRelativePath === undefined
? {}
: { sourceRelativePath: selected.sourceRelativePath }),
...(invocation.kind === "known" && sourceRelativePath !== undefined
? { sourceRelativePath }
: {}),
};
}
}
@@ -1556,20 +1563,8 @@ const writesModuleRequire = (
};
traverseJavaScriptValueAst(state.program, state.erasedSyntaxNodes, {
enter: (node) => {
const target = t.isAssignmentExpression(node)
? node.left
: t.isUpdateExpression(node) ||
t.isUnaryExpression(node, { operator: "delete" })
? node.argument
: undefined;
if (
t.isMemberExpression(target) ||
t.isOptionalMemberExpression(target)
) {
const key = semanticStaticPropertyKey(target.property, target.computed);
if (key === "require" || (key === null && target.computed))
record(node, target.object);
}
for (const target of commonJsDelegateWriteTargets(node))
record(node, target.object);
if (t.isCallExpression(node) || t.isOptionalCallExpression(node)) {
const invocation = interop.intrinsicInvocation(node);
const owner =
@@ -1583,42 +1578,21 @@ const writesModuleRequire = (
const key = interop.staticString(args?.[1]);
if (key === "require" || key === undefined)
record(node, args?.[0] ?? t.nullLiteral());
} else if (owner === "Reflect" && method === "set") {
const key = interop.staticString(args?.[1]);
if (key === "require" || key === undefined)
record(node, args?.[3] ?? args?.[0] ?? t.nullLiteral());
} else if (owner === "Object" && method === "defineProperties") {
const descriptors = args?.[1];
if (
!t.isObjectExpression(descriptors) ||
descriptors.properties.some(
(property) =>
t.isSpreadElement(property) ||
(t.isObjectProperty(property) &&
semanticStaticPropertyKey(property.key, property.computed) ===
"require"),
)
)
if (objectMayProvideRequire(descriptors))
record(node, args?.[0] ?? t.nullLiteral());
} else if (owner === "Object" && method === "assign") {
const target = args?.[0];
if (target !== undefined) {
const sourceHasRequire = args
?.slice(1)
.some(
(arg) =>
t.isObjectExpression(arg) &&
arg.properties.some(
(property) =>
t.isObjectProperty(property) &&
semanticStaticPropertyKey(
property.key,
property.computed,
) === "require",
),
);
if (
sourceHasRequire ||
args?.slice(1).some((arg) => !t.isObjectExpression(arg))
)
record(node, target);
}
if (
target !== undefined &&
args?.slice(1).some(objectMayProvideRequire)
)
record(node, target);
}
}
},
@@ -1627,6 +1601,15 @@ const writesModuleRequire = (
return writes;
};
/** Methods, spreads and unknown computed keys can all supply an own delegate. */
const objectMayProvideRequire = (node: t.Node | undefined): boolean =>
!t.isObjectExpression(node) ||
node.properties.some((property) => {
if (t.isSpreadElement(property)) return true;
const key = semanticStaticPropertyKey(property.key, property.computed);
return key === "require" || key === null;
});
/** Whether the original global process object can still own effects at this source site. */
export const semanticOriginalGlobalProcessAt = (
node: t.Node,
@@ -78,7 +78,11 @@ export const collectJavaScriptSemanticResources = (
const acquisitions = collectAcquisitions(program, context).filter(
(candidate) =>
!socketPortInFirstArgument(candidate) ||
(!rejectsSocketPortLiteral(candidate.node, context) &&
(!rejectsSocketLiteralArguments(
candidate.node,
candidate.specifier,
context,
) &&
!rejectsTlsProtocolVersionLiteral(
candidate.node,
candidate.specifier,
@@ -447,6 +451,9 @@ const collectReleases = (
resolution:
!opaqueReceiver &&
linked.length === 1 &&
(binding.kind !== "parameter" ||
semanticBindingAtRead(binding, member.object, context.state)
.hasDefiniteInitializer) &&
holdsOnlyAcquisitions(binding, linked, context.state)
? "complete"
: "partial",
@@ -573,16 +580,25 @@ const resourceCandidateId = (candidate: ResourceCandidate): string =>
* readable non-port literal is decided. `""` is a path with no length, which
* Node also rejects.
*
* `undefined` and an empty object throw `ERR_MISSING_ARGS`, but those are a
* missing option rather than a bad port, so they stay undecided here.
* `net` also rejects an omitted or undefined port, before it returns a socket.
* TLS merges later options, so an undefined first argument alone cannot decide
* its admission. Object arguments and spread positions remain undecided here.
*/
const rejectsSocketPortLiteral = (
const rejectsSocketLiteralArguments = (
node: t.CallExpression | t.OptionalCallExpression,
specifier: string,
context: DataEffectTraversalContext,
): boolean => {
const port = node.arguments[0];
if (port === undefined || t.isSpreadElement(port)) return false;
if (port === undefined) return specifier === "net";
if (t.isSpreadElement(port)) return false;
const value = unwrapJavaScriptExpression(port).node;
if (
specifier === "net" &&
(t.isUnaryExpression(value, { operator: "void" }) ||
isUnmodifiedGlobalReference(value, context.state, "undefined"))
)
return true;
if (t.isStringLiteral(value)) return value.value.length === 0;
if (
t.isNullLiteral(value) ||
@@ -601,13 +617,14 @@ const rejectsSocketPortLiteral = (
isUnmodifiedGlobalReference(value, context.state, value.name)
)
return true;
// `-1` and `-1.5` are unary expressions over numeric literals: always out of
// range because a port is never negative.
// Numeric unary operators are judged by their result: `-0` is port zero,
// and `~(-1.5)` is zero too. `typeof` instead yields a nonempty IPC path.
if (t.isUnaryExpression(value)) {
const operand = unwrapJavaScriptExpression(value.argument).node;
if (value.operator === "!" || value.operator === "delete") return true;
const number = unaryOperandNumber(value, context, 0);
return (
t.isNumericLiteral(operand) &&
(value.operator === "-" || !Number.isInteger(operand.value))
number !== undefined &&
(!Number.isInteger(number) || number < 0 || number > 65535)
);
}
return false;
@@ -205,3 +205,21 @@ it("widens a cycle that rewrites the path head while increasing its depth", () =
returnedExpression(parsed.program, "target"),
);
});
it("retains both finite class-prototype callable selections", () => {
const { parsed, state } = indexed(`
class C { first(){return firstResult} second(){return secondResult} }
const selected=flag?C.prototype.first:C.prototype.second;
selected();
`);
const [read] = callReads(parsed.program, "selected");
if (read === undefined)
throw new Error("Missing selected prototype method call");
const result = createSemanticReturnedReferences(state)(read);
expect(result.precision).toBe("exact");
expect(
result.references
.map(({ node }) => (t.isIdentifier(node) ? node.name : null))
.sort(),
).toEqual(["firstResult", "secondResult"]);
});
@@ -24,8 +24,13 @@ import {
semanticKnownArrayIterationAt,
semanticSelectedReferenceSlot,
} from "./javascriptSemanticReferenceSources.js";
import { semanticArrayPropertyReferences } from "./javascriptSemanticObjectReferences.js";
import {
semanticArrayPropertyReferences,
semanticObjectPropertyReferences,
semanticClassPropertyReferences,
} from "./javascriptSemanticObjectReferences.js";
import {
semanticCapturedSelectionsAfterPropertyRead,
semanticNormalizeCapturedSelections,
semanticProjectedSourcesAtRead,
semanticShiftCapturedSelections,
@@ -62,6 +67,7 @@ interface Candidate {
| readonly SemanticCapturedSelection[]
| undefined;
readonly invocation?: JavaScriptSourceInvocationContext;
readonly classKind?: "instance" | "static";
}
export interface SemanticReturnedReferenceResult {
@@ -74,22 +80,6 @@ export interface SemanticReturnedReferenceResult {
readonly precision: "exact" | "widened";
}
interface ObjectTargetIndex {
readonly propertyByName: ReadonlyMap<
string,
t.ObjectMethod | t.ObjectProperty
>;
readonly trailingUnknownsByProperty: ReadonlyMap<
t.ObjectMethod | t.ObjectProperty,
ObjectTargetUnknown
>;
}
interface ObjectTargetUnknown {
readonly node: t.ObjectMethod | t.ObjectProperty | t.SpreadElement;
readonly previous: ObjectTargetUnknown | undefined;
}
/** Resolve local result references conservatively, without claiming call execution. */
export const createSemanticReturnedReferences = (
state: JavaScriptSemanticAnalysisState,
@@ -115,7 +105,6 @@ export const createSemanticReturnedReferences = (
}[];
}
>();
const objectTargets = new WeakMap<t.ObjectExpression, ObjectTargetIndex>();
return (callee, invocation) => {
const existing = results.get(callee);
if (
@@ -141,7 +130,7 @@ export const createSemanticReturnedReferences = (
// shared diamonds and reassignment cycles converge to the graph fixed point.
const projectionDepths = new WeakMap<
t.Node,
Map<string, number | undefined>
Map<string, { depth: number | undefined }>
>();
const expanded = new WeakMap<t.Node, Set<string>>();
let additionalContexts = 0;
@@ -190,6 +179,7 @@ export const createSemanticReturnedReferences = (
readAt?: t.Node,
capturedSelections?: readonly SemanticCapturedSelection[],
invocationContext?: JavaScriptSourceInvocationContext,
classKind?: Candidate["classKind"],
): void => {
const source = unwrapJavaScriptExpression(node).node;
if (contextsWidened) {
@@ -213,20 +203,25 @@ export const createSemanticReturnedReferences = (
readAt?.start ?? -1,
readAt?.end ?? -1,
invocationKey(),
classKind,
]);
const depths = projectionDepths.get(source) ?? new Map();
projectionDepths.set(source, depths);
if (!depths.has(readKey)) {
depths.set(readKey, path?.length);
} else {
const previous = depths.get(readKey);
if (previous === undefined || previous !== path?.length) {
if (previous !== undefined) result.precision = "widened";
depths.set(readKey, undefined);
path = undefined;
}
let context = depths.get(readKey);
if (context === undefined) {
context = { depth: path?.length };
depths.set(readKey, context);
} else if (
context.depth === undefined ||
context.depth !== path?.length
) {
if (context.depth !== undefined) result.precision = "widened";
context.depth = undefined;
path = undefined;
}
if (path !== undefined) path = selections.normalize(source, path);
// Selector joins share the transition's read/invocation/receiver context.
// Joining only by syntax would cross static and instance member names.
if (path !== undefined) path = selections.normalize(context, path);
capturedSelections =
path === undefined
? undefined
@@ -279,9 +274,10 @@ export const createSemanticReturnedReferences = (
readAt?.start ?? -1,
readAt?.end ?? -1,
null,
classKind,
]);
if (sourceGenericInvocation) result.unresolvedInvocation = true;
depths.set(readKey, undefined);
depths.set(readKey, { depth: undefined });
key = expansionKey();
if (paths.has(key)) return;
}
@@ -289,6 +285,7 @@ export const createSemanticReturnedReferences = (
expanded.set(source, paths);
work.push({
node: source,
...(classKind === undefined ? {} : { classKind }),
path,
...(readAt === undefined ? {} : { readAt }),
...(capturedSelections === undefined ? {} : { capturedSelections }),
@@ -382,6 +379,7 @@ export const createSemanticReturnedReferences = (
child.readAt,
child.capturedSelections,
child.invocation ?? current.invocation,
child.classKind ?? current.classKind,
);
}
continue;
@@ -505,9 +503,38 @@ export const createSemanticReturnedReferences = (
result.unresolvedInvocation = true;
}
}
} else if (t.isObjectExpression(node))
selectObjectTargets(node, current, pending, objectTargets);
else if (t.isArrayExpression(node)) {
} else if (t.isObjectExpression(node)) {
const [key, ...suffix] = current.path ?? [];
for (const choice of key === undefined
? [undefined]
: semanticAliasSelectionChoices(key))
for (const source of semanticObjectPropertyReferences(
node,
choice === undefined ? [] : [choice],
state,
)) {
if (source.widenedGetterResult) result.precision = "widened";
if (
source.inherited &&
current.capturedSelections?.some(
(capture) => capture.depth === 1 && capture.ownEnumerable,
)
)
continue;
pending.push({
...current,
node: source.node,
path:
current.path === undefined || source.widenedGetterResult
? undefined
: [...source.path, ...suffix],
capturedSelections: semanticCapturedSelectionsAfterPropertyRead(
current.capturedSelections,
source,
),
});
}
} else if (t.isArrayExpression(node)) {
const [key, ...path] = current.path ?? [];
if (current.path !== undefined && key !== undefined) {
for (const choice of semanticAliasSelectionChoices(key))
@@ -552,6 +579,7 @@ export const createSemanticReturnedReferences = (
child.readAt,
child.capturedSelections,
child.invocation ?? current.invocation,
child.classKind ?? current.classKind,
);
}
continue;
@@ -590,63 +618,83 @@ export const createSemanticReturnedReferences = (
});
});
} else if (t.isNewExpression(node))
pending.push({ ...current, node: node.callee });
pending.push({ ...current, node: node.callee, classKind: "instance" });
else if (t.isClass(node)) {
const [key, ...path] = current.path ?? [];
if (current.path !== undefined && key !== undefined) {
for (const choice of semanticAliasSelectionChoices(key)) {
const prototype =
current.classKind !== "instance" && choice === "prototype";
if (prototype && path.length === 0) continue;
const ownEnumerable = current.capturedSelections?.some(
(capture) =>
capture.depth === (prototype ? 2 : 1) && capture.ownEnumerable,
);
const member = prototype ? path[0] : choice;
if (member === undefined) continue;
for (const selected of semanticAliasSelectionChoices(member))
for (const source of semanticClassPropertyReferences(
{
owner: node,
kind: prototype
? "prototype"
: (current.classKind ?? "static"),
...(ownEnumerable ? { fieldsOnly: true as const } : {}),
},
[selected],
state,
)) {
if (source.inherited && ownEnumerable) continue;
if (source.widenedGetterResult) result.precision = "widened";
pending.push({
...current,
node: source.node,
path: source.widenedGetterResult
? undefined
: [...source.path, ...(prototype ? path.slice(1) : path)],
classKind: "static",
capturedSelections:
semanticCapturedSelectionsAfterPropertyRead(
prototype
? semanticShiftCapturedSelections(
current.capturedSelections,
-1,
)
: current.capturedSelections,
source,
),
});
}
}
for (const child of pending)
enqueue(
child.node,
child.path,
child.readAt,
child.capturedSelections,
child.invocation ?? current.invocation,
child.classKind ?? current.classKind,
);
continue;
}
// Unknown projections retain every possible callable descendant. At
// zero depth only construction contributes a source-local result.
for (const member of node.body.body) {
if (t.isClassMethod(member)) {
if (
((current.path === undefined || current.path.length === 0) &&
member.kind === "constructor") ||
current.path === undefined ||
(key !== undefined &&
semanticAliasSelectionChoices(key).some((choice) => {
const name = semanticStaticPropertyKey(
member.key,
member.computed,
);
return (
name === null ||
semanticPropertyPathKeyMatches(choice, name)
);
}))
)
pending.push({
...current,
node: member,
path: current.path === undefined ? undefined : path,
capturedSelections: semanticShiftCapturedSelections(
current.capturedSelections,
-1,
),
});
} else if (
t.isClassProperty(member) &&
member.value !== null &&
member.value !== undefined &&
(key !== undefined || current.path === undefined) &&
(current.path === undefined ||
(key !== undefined &&
semanticAliasSelectionChoices(key).some((choice) => {
const name = semanticStaticPropertyKey(
member.key,
member.computed,
);
return (
name === null ||
semanticPropertyPathKeyMatches(choice, name)
);
})))
if (
t.isClassMethod(member) &&
(current.path === undefined || member.kind === "constructor")
)
pending.push({
...current,
node: member.value,
path: current.path === undefined ? undefined : path,
capturedSelections: semanticShiftCapturedSelections(
current.capturedSelections,
-1,
),
node: member,
path: current.path === undefined ? undefined : [],
});
else if (
current.path === undefined &&
t.isClassProperty(member) &&
member.value != null
)
pending.push({ ...current, node: member.value, path: undefined });
}
if (node.superClass !== null && node.superClass !== undefined)
pending.push({ ...current, node: node.superClass });
@@ -660,6 +708,7 @@ export const createSemanticReturnedReferences = (
child.readAt,
child.capturedSelections,
child.invocation ?? current.invocation,
child.classKind ?? current.classKind,
);
}
}
@@ -699,139 +748,3 @@ export const createSemanticReturnedReferences = (
return result;
};
};
const selectObjectTargets = (
node: t.ObjectExpression,
item: Candidate,
pending: Candidate[],
indexes: WeakMap<t.ObjectExpression, ObjectTargetIndex>,
): void => {
const [key, ...path] = item.path ?? [];
const childPath = item.path === undefined ? undefined : path;
if (key !== null && typeof key === "object" && "alternatives" in key) {
for (const choice of semanticAliasSelectionChoices(key))
selectObjectTargets(
node,
{ ...item, path: [choice, ...path] },
pending,
indexes,
);
return;
}
if (key !== null && key !== undefined && typeof key !== "object") {
const index = objectTargetIndex(node, indexes);
const property = index.propertyByName.get(String(key));
if (property !== undefined) {
const unknowns: ObjectTargetUnknown[] = [];
for (
let unknown = index.trailingUnknownsByProperty.get(property);
unknown !== undefined;
unknown = unknown.previous
)
unknowns.push(unknown);
for (const { node: unknown } of unknowns.reverse())
pending.push({
...item,
node: t.isSpreadElement(unknown)
? unknown.argument
: t.isObjectProperty(unknown)
? unknown.value
: unknown,
...(t.isSpreadElement(unknown) ? {} : { path: childPath }),
capturedSelections: t.isSpreadElement(unknown)
? [
...(item.capturedSelections ?? []),
{ at: unknown.argument, depth: 1, ownEnumerable: true },
]
: (semanticShiftCapturedSelections(item.capturedSelections, -1) ??
[]),
});
pending.push({
...item,
node: t.isObjectProperty(property) ? property.value : property,
path: childPath,
capturedSelections:
semanticShiftCapturedSelections(item.capturedSelections, -1) ?? [],
});
return;
}
}
let replaced = false;
for (const property of [...node.properties].reverse()) {
if (replaced) break;
if (t.isSpreadElement(property)) {
pending.push({
...item,
node: property.argument,
capturedSelections: [
...(item.capturedSelections ?? []),
{ at: property.argument, depth: 1, ownEnumerable: true },
],
});
continue;
}
const name = semanticStaticPropertyKey(property.key, property.computed);
if (
key !== null &&
key !== undefined &&
name !== null &&
!semanticPropertyPathKeyMatches(key, name)
)
continue;
if (t.isObjectProperty(property))
pending.push({
...item,
node: property.value,
path: childPath,
capturedSelections:
semanticShiftCapturedSelections(item.capturedSelections, -1) ?? [],
});
else
pending.push({
...item,
node: property,
path: childPath,
capturedSelections:
semanticShiftCapturedSelections(item.capturedSelections, -1) ?? [],
});
if (
key !== null &&
key !== undefined &&
typeof key !== "object" &&
name === String(key)
)
replaced = true;
}
};
const objectTargetIndex = (
node: t.ObjectExpression,
indexes: WeakMap<t.ObjectExpression, ObjectTargetIndex>,
): ObjectTargetIndex => {
const existing = indexes.get(node);
if (existing !== undefined) return existing;
const propertyByName = new Map<string, t.ObjectMethod | t.ObjectProperty>();
const trailingUnknownsByProperty = new Map<
t.ObjectMethod | t.ObjectProperty,
ObjectTargetUnknown
>();
let trailingUnknown: ObjectTargetUnknown | undefined;
for (const property of [...node.properties].reverse()) {
if (t.isSpreadElement(property)) {
trailingUnknown = { node: property, previous: trailingUnknown };
continue;
}
const name = semanticStaticPropertyKey(property.key, property.computed);
if (name === null) {
trailingUnknown = { node: property, previous: trailingUnknown };
continue;
}
if (propertyByName.has(name)) continue;
propertyByName.set(name, property);
if (trailingUnknown !== undefined)
trailingUnknownsByProperty.set(property, trailingUnknown);
}
const created = { propertyByName, trailingUnknownsByProperty };
indexes.set(node, created);
return created;
};
@@ -155,3 +155,31 @@ it.each([
}
},
);
it("discloses getter-result widening when selecting returned callable targets", () => {
const parsed = parseJavaScriptSource(
"const box={get item(){return {make:()=>result}}};box.item.make();",
);
if (parsed === null) throw new Error("Expected getter fixture");
const steps = indexJavaScriptBindingsSteps(parsed.program, {
getter_expansions: 1,
});
let step = steps.next();
while (!step.done) step = steps.next();
let call: t.CallExpression | undefined;
t.traverseFast(parsed.program, (node) => {
if (t.isCallExpression(node)) call = node;
});
if (call === undefined)
throw new Error("Expected getter-produced callable invocation");
const result = createSemanticReturnedReferences(step.value)(
call.callee,
call,
);
expect(result.precision).toBe("widened");
expect(result.references).toContainEqual({
node: expect.objectContaining({ type: "Identifier", name: "result" }),
projection: [],
});
expect(step.value.returnedReferenceWidenings.has(call.callee)).toBe(true);
});
@@ -81,3 +81,29 @@ it.each([
);
},
);
it.each([
["const box={...{__proto__:{make:()=>module}}};box.make()", false],
["const box={...{get make(){return ()=>module}}};box.make()", true],
["class C{static make(){return module}};const box={...C};box.make()", false],
["class C{make(){return module}};const box={...new C()};box.make()", false],
[
"class Base{static make=()=>module};class C extends Base{};const box={...C};box.make()",
false,
],
["class C{static make=()=>module};const box={...C};box.make()", true],
["class C{make=()=>module};const box={...new C()};box.make()", true],
] as const)(
"copies only own enumerable callable values: %s",
(expression, expected) => {
const analysis = analyzeJavaScriptStaticSource(
`${expression}.require('./shared.cjs');`,
);
expect(analysis.parse_error_count).toBe(0);
expect(analysis.references).toEqual(
expected
? [expect.objectContaining({ kind: "require", specifier: null })]
: [],
);
},
);
@@ -201,3 +201,82 @@ it("refreshes cached returned targets after the shared member index settles", ()
}),
]);
});
it.each([
["const box={set make(value){return module}};box.make()", false],
["const box={get make(){return module}};box.make()", false],
["const box={get make(){return ()=>module}};box.make()", true],
[
"const box={get make(){return ()=>module},set make(value){}};box.make()",
true,
],
["const box={get make(){return ()=>module},make:()=>({})};box.make()", false],
["const box={__proto__:{make:()=>module}};box.make()", true],
["const box={__proto__:{make:()=>module},make:()=>({})};box.make()", false],
] as const)(
"uses property-read ownership before invoking a returned callable: %s",
(expression, expected) => {
const analysis = analyzeJavaScriptStaticSource(
`${expression}.require('./shared.cjs');`,
);
expect(analysis.parse_error_count).toBe(0);
expect(analysis.references).toEqual(
expected
? [expect.objectContaining({ kind: "require", specifier: null })]
: [],
);
},
);
it.each([
["class C{make(){return module}};C.make()", false],
["class C{static make(){return module}};new C().make()", false],
["class C{static make(){return module}};C.make()", true],
["class C{make(){return module}};new C().make()", true],
["class C{make(){return module}};C.prototype.make()", true],
["class C{get make(){return ()=>module}};new C().make()", true],
["class C{set make(value){return module}};new C().make()", false],
[
"class Base{static make(){return module}};class C extends Base{};C.make()",
true,
],
[
"class Base{make(){return module}};class C extends Base{make=()=>({})};new C().make()",
false,
],
] as const)(
"preserves the class property receiver before invoking its value: %s",
(expression, expected) => {
const analysis = analyzeJavaScriptStaticSource(
`${expression}.require('./shared.cjs');`,
);
expect(analysis.parse_error_count).toBe(0);
expect(analysis.references).toEqual(
expected
? [expect.objectContaining({ kind: "require", specifier: null })]
: [],
);
},
);
it.each([
["C.first:new C().second", false],
["C.second:new C().first", true],
] as const)(
"keeps static and instance selector alternatives correlated: %s",
(selection, expected) => {
const analysis = analyzeJavaScriptStaticSource(`
class C {
static first(){return {}} first(){return module}
static second(){return module} second(){return {}}
}
const selected=flag?${selection};selected().require('./shared.cjs');
`);
expect(analysis.parse_error_count).toBe(0);
expect(analysis.references).toEqual(
expected
? [expect.objectContaining({ kind: "require", specifier: null })]
: [],
);
},
);
@@ -535,7 +535,7 @@ const semanticWithPrototypeKind = (
t.isObjectProperty(property) &&
!property.computed &&
!property.shorthand &&
t.isIdentifier(property.key, { name: "__proto__" })
semanticStaticPropertyKey(property.key, false) === "__proto__"
)
return t.isNullLiteral(unwrapJavaScriptExpression(property.value).node)
? "null"
@@ -3,7 +3,58 @@ import { expect, it } from "vitest";
import { analyzeJavaScriptSemantics } from "./javascriptSemanticAnalysis.js";
import { topLevelBinding } from "./javascriptSemanticAnalysis.fixture.js";
import { analyzeJavaScriptStaticSource } from "./javascriptStaticAnalysis.js";
import { analyzeCommonJsStaticFixture } from "../../../tests/fixtures/javascriptCommonJsAnalysis.js";
import {
analyzeCommonJsStaticFixture,
analyzeCommonJsSemanticFixture,
} from "../../../tests/fixtures/javascriptCommonJsAnalysis.js";
it.each([
"Object.assign(module,{require(){return {}}});",
"Object.assign(module,{...replacement});",
"Object.defineProperties(module,{[key]:{value:()=>({})}});",
'Reflect.set(module,"require",()=>({}));',
'Reflect.set({},"require",()=>({}),module);',
"({x:module.require}={x:replacement});",
"[module.require]=[replacement];",
"for(module.require of [replacement]){}",
])("retains uncertainty after a possible delegate replacement: %s", (write) => {
const analysis = analyzeCommonJsStaticFixture(`${write}require("electron");`);
expect(analysis.references).not.toContainEqual(
expect.objectContaining({ kind: "require", specifier: "electron" }),
);
expect(analysis.references).toContainEqual(
expect.objectContaining({ kind: "require", specifier: null }),
);
});
it.each([
"Object.assign(module,{other(){}});",
"Object.defineProperties(module,{other:{value:()=>({})}});",
"Object.assign({},{require(){}});",
'Reflect.set(module,"other",()=>({}));',
'Reflect.set(module,"require",()=>({}),{});',
"({x:module.other}={x:replacement});",
"[module.other]=[replacement];",
"for(module.other of [replacement]){}",
])("preserves the host loader after an unrelated write: %s", (write) => {
const analysis = analyzeCommonJsStaticFixture(`${write}require("electron");`);
expect(analysis.references).toEqual([
expect.objectContaining({ kind: "require", specifier: "electron" }),
]);
});
it.each([
"({x:module.require}={x:replacement});",
"[module.require]=[replacement];",
"for(module.require of [replacement]){}",
])("preserves a builtin captured before its delegate write: %s", (write) => {
const analysis = analyzeCommonJsSemanticFixture(
`const fs=require('node:fs');${write}fs.readFileSync('kept');`,
);
expect(analysis.configurationOperations.map(({ key }) => key)).toEqual([
"kept",
]);
});
it.each([
'var require;require("./before.cjs");',
@@ -76,3 +76,44 @@ it("recognizes a bundler runtime through nested member names", () => {
modules: [expect.objectContaining({ exports: ["ok"] })],
});
});
it.each(["1 as const", "1 satisfies number"])(
"keeps numeric bundle identities through erased syntax: %s",
(entry) => {
const analysis = analyzeJavaScriptStaticSource(`
globalThis.webpackChunkApp.push([[(${entry})], {
1: function(module, exports, req) { req((2 as const)); req.e((3 satisfies number)); },
2: function() {}
}, function(req) { req((${entry})); }]);
`);
expect(analysis.parse_error_count).toBe(0);
expect(analysis.bundler_registrations).toMatchObject([
{
chunk_keys: ["1"],
unknown_chunk_keys: 0,
entry_module_keys: ["1"],
unknown_entry_module_keys: 0,
async_chunk_keys: ["3"],
unknown_async_chunk_keys: 0,
},
]);
expect(analysis.references).toContainEqual(
expect.objectContaining({
specifier: "2",
loader: "factory-loader",
module_key: "1",
}),
);
},
);
it.each(["0, {}", "[], modules"])(
"does not grant factory-loader authority to a rejected registration: %s",
(entry) => {
const analysis = analyzeJavaScriptStaticSource(`
globalThis.webpackChunkApp.push([${entry}, function(req) { req("electron"); }]);
`);
expect(analysis.bundler_registrations).toEqual([]);
expect(analysis.references).toEqual([]);
},
);
@@ -140,7 +140,7 @@ export function* analyzeParsedJavaScriptStaticSourceSteps(
accumulator,
vitePreloadDependencyTable: indexVitePreloadDependencyTable(
file.program,
bindings.erasedSyntaxNodes,
bindings,
),
openReceiverFacts: receivers.openReceivers,
electronBindings: electronIdentity.bindings,
@@ -48,12 +48,12 @@ export const inspectBundlerRegistration = (
if (runtime === undefined || !t.isArrayExpression(entry)) return;
const chunkIds = entry.elements[0];
const table = entry.elements[1];
if (!t.isArrayExpression(chunkIds) || !t.isObjectExpression(table)) return;
const runtimeValue = runtimeMetadata(
entry.elements[2],
bindings,
accumulator,
);
if (!t.isArrayExpression(chunkIds) || !t.isObjectExpression(table)) return;
const recovered = recoverBundlerModules(source, table, accumulator, {
runtime,
bindings,
@@ -43,6 +43,7 @@ import type {
import { VITE_OUTPUT_DEPENDENCY_PREFIX } from "./javascriptStaticAnalysisTypes.js";
import {
resolveSemanticBindingState,
semanticBindingLookupAt,
semanticStableInitializer,
type JavaScriptBindingIndex,
type JavaScriptSemanticAnalysisState,
@@ -411,7 +412,7 @@ const chunkLoadSpecifier = (
const vitePreloadDependencySpecifiers = (
node: JavaScriptCallLike,
name: string,
table: readonly (string | undefined)[] | null,
table: FindingContext["vitePreloadDependencyTable"],
bindings: JavaScriptSemanticAnalysisState,
): readonly (string | undefined)[] => {
if (
@@ -420,10 +421,18 @@ const vitePreloadDependencySpecifiers = (
!t.isArrayExpression(node.arguments[0])
)
return [];
const callee = unwrapJavaScriptExpression(node.callee).node;
const lookup = semanticBindingLookupAt(bindings, callee, name);
if (
(lookup.kind !== "binding" && lookup.kind !== "uncertain-binding") ||
semanticStableInitializer(lookup.binding, bindings)?.node !== table.helper
)
return [];
const indexes = node.arguments[0].elements;
// Explicit cache overrides change which table the helper reads. Retain the
// call slots as unknown unless every override is statically undefined.
if (
lookup.kind !== "binding" ||
!node.arguments
.slice(1)
.every((argument) => isExplicitDefaultArgument(argument, bindings))
@@ -433,7 +442,7 @@ const vitePreloadDependencySpecifiers = (
if (element === null || !t.isNumericLiteral(element)) return undefined;
const index = element.value;
return Number.isSafeInteger(index) && index >= 0
? viteDependencySpecifier(table[index])
? viteDependencySpecifier(table.dependencies[index])
: undefined;
});
};
@@ -473,8 +482,9 @@ type ViteMapDepsFunction =
/** Index the canonical Vite helper's cache initializer from parsed syntax. */
export const indexVitePreloadDependencyTable = (
program: t.Program,
erasedSyntaxNodes: WeakSet<t.Node>,
): readonly (string | undefined)[] | null => {
bindings: JavaScriptSemanticAnalysisState,
): FindingContext["vitePreloadDependencyTable"] => {
const { erasedSyntaxNodes } = bindings;
const helpers: ViteMapDepsFunction[] = [];
traverseJavaScriptValueAst(program, erasedSyntaxNodes, {
enter: (node) => {
@@ -497,7 +507,7 @@ export const indexVitePreloadDependencyTable = (
const mapIdentifier = t.isAssignmentPattern(mapParameter)
? mapParameter.left
: mapParameter;
if (!t.isIdentifier(mapIdentifier)) return [];
if (!t.isIdentifier(mapIdentifier)) return { helper, dependencies: [] };
const tables: t.ArrayExpression[] = [];
traverseJavaScriptValueAst(helper, erasedSyntaxNodes, {
skip: (node) => t.isFunction(node),
@@ -515,9 +525,9 @@ export const indexVitePreloadDependencyTable = (
tables.push(node.right);
},
});
if (tables.length !== 1) return [];
if (tables.length !== 1) return { helper, dependencies: [] };
const table = tables[0];
if (table === undefined) return [];
if (table === undefined) return { helper, dependencies: [] };
// A spread contributes an unknown number of entries, so every later slot
// has an unknown index; leave those out instead of shifting them.
const spread = table.elements.findIndex((element) =>
@@ -540,6 +550,14 @@ export const indexVitePreloadDependencyTable = (
semanticIdentifierRole(node, parent, readAncestors) === null
)
return;
const lookup = semanticBindingLookupAt(bindings, node, node.name);
if (
lookup.kind === "ambient" ||
lookup.kind === "captured" ||
(lookup.kind === "binding" &&
semanticStableInitializer(lookup.binding, bindings)?.node !== helper)
)
return;
if (
(t.isCallExpression(parent) || t.isOptionalCallExpression(parent)) &&
parent.callee === node
@@ -548,7 +566,12 @@ export const indexVitePreloadDependencyTable = (
escapedOrMutated = true;
},
});
return escapedOrMutated ? dependencies.map(() => undefined) : dependencies;
return {
helper,
dependencies: escapedOrMutated
? dependencies.map(() => undefined)
: dependencies,
};
};
const requestConstructorUrl = (
@@ -769,8 +769,11 @@ export const stringValue = (
/** Return a string or numeric argument literal. */
export const argumentValue = (
node: t.Node | null | undefined,
): string | undefined =>
t.isNumericLiteral(node) ? String(node.value) : stringValue(node);
): string | undefined => {
if (node === null || node === undefined) return undefined;
const value = unwrapJavaScriptExpression(node).node;
return t.isNumericLiteral(value) ? String(value.value) : stringValue(value);
};
/** Prefix a static argument value when one is available. */
export const prefixedArgument = (
@@ -133,7 +133,10 @@ export interface JavaScriptFindingContext {
readonly expressionShapes: JavaScriptExpressionShapeResolver;
readonly source: string;
readonly accumulator: JavaScriptAnalysisAccumulator;
readonly vitePreloadDependencyTable: readonly (string | undefined)[] | null;
readonly vitePreloadDependencyTable: {
readonly helper: t.Node;
readonly dependencies: readonly (string | undefined)[];
} | null;
readonly openReceiverFacts?: ReadonlyMap<number, JavaScriptOpenReceiverFact>;
/** Proven Electron export path segments by root offset. */
readonly electronBindings?: ReadonlyMap<number, readonly string[]>;
@@ -1,6 +1,7 @@
import { expect, it } from "vitest";
import { analyzeJavaScriptStaticSource } from "./javascriptStaticAnalysis.js";
import { analyzeCommonJsStaticFixture } from "../../../tests/fixtures/javascriptCommonJsAnalysis.js";
const windowPrefix =
"import {BrowserWindow} from 'electron'; import {fileURLToPath} from 'node:url';";
@@ -152,3 +153,52 @@ it("keeps a reused renderer receiver uncertain after its native method call", ()
"independent.html",
]);
});
it.each([
["untouched exports", "", 1],
[
"object-pattern exports write",
"({value: module.exports} = replacement);",
0,
],
["array-pattern exports write", "[module.exports] = replacement;", 0],
["loop-target exports write", "for (module.exports of replacement) {}", 0],
] as const)(
"requires intact bootstrap exports after %s",
(_name, write, expected) => {
const source = `(() => {
const table = {
external(module, exports, load) { module.exports = require('electron'); },
entry(module, exports, load) { new (load('external').BrowserWindow)(); }
};
function load(key) {
const module = {exports: {}};
table[key](module, module.exports, load);
${write}
return module.exports;
}
load('entry');
})();`;
const observed: string[] = [];
const windowClass = (identity: string) =>
class {
constructor() {
observed.push(identity);
}
};
const local = { BrowserWindow: windowClass("replacement") };
runInNewContext(source, {
replacement: Object.assign([local], { value: local }),
require: (specifier: string) => {
expect(specifier).toBe("electron");
return { BrowserWindow: windowClass("electron") };
},
});
expect(observed).toEqual([expected === 1 ? "electron" : "replacement"]);
expect(
analyzeCommonJsStaticFixture(source).electron.browser_windows,
).toHaveLength(expected);
},
);
import { runInNewContext } from "node:vm";
@@ -7,6 +7,7 @@ import {
traverseJavaScriptAst,
} from "./javascriptSemanticTraversal.js";
import { JavaScriptSymbolicBindings } from "./javascriptSymbolicBindings.js";
import { analyzeJavaScriptStaticSource } from "./javascriptStaticAnalysis.js";
import {
symbolicTruth,
symbolicVariants,
@@ -1076,3 +1077,94 @@ it("retains each external selection guard once across deep member reads", () =>
});
expect(resolver.truncated).toBe(false);
});
it.each([
[
"module property after replacement",
'module.exports={api:null};module.exports.api=require("electron");',
{ kind: "external", loaderAuthority: "candidate", path: ["electron"] },
],
[
"module property overwritten after replacement",
'module.exports={api:require("electron")};module.exports.api=null;',
{ kind: "literal", value: null },
],
[
"detached exports property",
'module.exports={api:null};exports.api=require("electron");',
{ kind: "literal", value: null },
],
[
"rebound exports parameter",
'module.exports={api:null};exports={api:require("electron")};',
{ kind: "literal", value: null },
],
[
"identity-preserving module write",
'module.exports=module.exports;exports.api=require("electron");',
{ kind: "external", loaderAuthority: "candidate", path: ["electron"] },
],
[
"identity-preserving original exports write",
'module.exports=exports;exports.api=require("electron");',
{ kind: "external", loaderAuthority: "candidate", path: ["electron"] },
],
[
"reconnected exports parameter",
'module.exports={api:null};exports=module.exports;exports.api=require("electron");',
{ kind: "external", loaderAuthority: "candidate", path: ["electron"] },
],
[
"rebound module parameter",
'module={exports:{}};module.exports.api=require("electron");',
{ kind: "unknown", reason: "ambiguous export writes" },
],
] as const)("retains namespace write order for %s", (_name, body, expected) => {
const value = inspectOne(`(() => {
const modules={
1(module,exports){${body}},
2(module,exports,load){inspect(load(1).api);}
};
function load(id){const m={exports:{}};modules[id](m,m.exports,load);return m.exports;}
load(2);
})();`);
expect(value).toEqual(expected);
});
it("lists members added to the public module object after its replacement", () => {
const { resolver, values } = inspectFixture(`(() => {
const modules={
1(module,exports){module.exports={first:1};module.exports.later=2;exports.detached=3;},
2(module,exports,load){inspect(load(1));}
};
function load(id){const m={exports:{}};modules[id](m,m.exports,load);return m.exports;}
load(2);
})();`);
expect(values).toHaveLength(1);
expect(resolver.members(values[0]!)).toEqual(["first", "later"]);
});
it("derives bridge paths from the latest public module member writes", () => {
const result = analyzeJavaScriptStaticSource(`(() => {
const modules={
1(module,exports){
const e=require("electron");
module.exports={action:()=>e.ipcRenderer.send("stale")};
module.exports.action=()=>e.ipcRenderer.send("current");
exports.detached=()=>e.ipcRenderer.send("detached");
},
2(module,exports,load){
const api=load(1);
require("electron").contextBridge.exposeInMainWorld("desktop",api);
}
};
function load(id){const m={exports:{}};modules[id](m,m.exports,load);return m.exports;}
load(2);
})();`);
expect(
result.electron.call_paths?.map(({ entry, channel }) => ({
entry,
channel,
})),
).toEqual([{ entry: "desktop.action", channel: "current" }]);
});
@@ -52,6 +52,9 @@ interface PropertyWrite {
readonly path: readonly string[];
readonly value: t.Node;
readonly site: t.Node;
readonly exportReceiver?: "module" | "exports";
readonly exportValueReceiver?: "module" | "exports";
readonly receiverUncertain?: true;
}
interface SymbolicReturnFlow extends SymbolicMetadata {
readonly values: readonly SymbolicValue[];
@@ -198,6 +201,12 @@ export class JavaScriptSymbolicBindings {
}
},
});
for (const writes of this.#writes.values())
writes.sort(
(left, right) =>
(left.site.end ?? Infinity) - (right.site.end ?? Infinity) ||
(right.site.start ?? 0) - (left.site.start ?? 0),
);
for (const table of this.#bootstrapTables)
this.#invalidateBootstrapTable(table.tableAliases, table.modules);
}
@@ -396,19 +405,19 @@ export class JavaScriptSymbolicBindings {
this.unresolved = true;
return symbolicUnknown("module export read boundary is unavailable");
}
const replacement = available.findLast(
(write) => write.path.length === 0,
const write = available.findLast(
(write) =>
write.path.length === 0 ||
(write.path.length === 1 && write.path[0] === key),
);
if (replacement)
if (write?.receiverUncertain)
return symbolicUnknown("uncertain export receiver");
if (write?.path.length === 0)
return this.property(
this.evaluate(replacement.value, value.environment),
this.evaluate(write.value, value.environment),
key,
read,
);
const writes = available.filter(
(w) => w.path.length === 1 && w.path[0] === key,
);
const write = writes.at(-1);
return write
? this.evaluate(write.value, value.environment)
: symbolicUnknown("unresolved module export");
@@ -509,19 +518,23 @@ export class JavaScriptSymbolicBindings {
if (value.kind === "namespace") {
const available = this.#namespaceWrites(value);
if (available === undefined) return [];
const replacement = available.findLast(
const replacementIndex = available.findLastIndex(
(write) => write.path.length === 0,
);
return replacement
? this.members(
this.evaluate(replacement.value, value.environment),
active,
)
: [
...new Set(
available.flatMap((w) => (w.path.length === 1 ? w.path : [])),
),
];
const replacement = available[replacementIndex];
return [
...new Set([
...(replacement === undefined
? []
: this.members(
this.evaluate(replacement.value, value.environment),
active,
)),
...available
.slice(replacementIndex + 1)
.flatMap((write) => (write.path.length === 1 ? write.path : [])),
]),
];
}
if (value.kind !== "object") return [];
return [
@@ -563,6 +576,7 @@ export class JavaScriptSymbolicBindings {
if (this.#mutated.has(value.bindingId)) return 1;
const available = this.#namespaceWrites(value);
if (available === undefined) return 1;
if (available.some((write) => write.receiverUncertain)) return 1;
const replacement = available.findLast(
(write) => write.path.length === 0,
);
@@ -1043,7 +1057,9 @@ export class JavaScriptSymbolicBindings {
if (boundary !== undefined && sourcePoint == null) return undefined;
if (completedAt !== undefined && completedAt.end == null) return undefined;
if (sourcePoint == null && completedAt === undefined)
return read === undefined ? writes : undefined;
return read === undefined
? this.#publicNamespaceWrites(writes)
: undefined;
// An owned snapshot cannot read beyond either its capture or this exit.
const point = Math.min(
sourcePoint ?? Infinity,
@@ -1069,17 +1085,84 @@ export class JavaScriptSymbolicBindings {
}
}
}
return writes.filter(
(write) =>
write.site.end != null &&
write.site.end <= point &&
(changed.size === 0 ||
(write.path.length > 0
? !changed.has(write.path[0]!)
: selectedKey !== undefined && !changed.has(selectedKey))),
return this.#publicNamespaceWrites(
writes.filter(
(write) =>
write.site.end != null &&
write.site.end <= point &&
(changed.size === 0 ||
(write.path.length > 0
? !changed.has(write.path[0]!)
: selectedKey !== undefined && !changed.has(selectedKey))),
),
);
}
/** Keep publication separate from writes through a detached or rebound alias. */
#publicNamespaceWrites(
writes: readonly PropertyWrite[],
): readonly PropertyWrite[] {
let relation: "connected" | "detached" | "uncertain" = "connected";
const published: PropertyWrite[] = [];
for (const write of writes) {
if (write.path.length === 0 && write.exportReceiver === "exports") {
if (write.exportValueReceiver === "module") relation = "connected";
else if (write.exportValueReceiver !== "exports")
relation = "uncertain";
continue;
}
if (write.path.length === 0 && write.exportReceiver === "module") {
if (
write.exportValueReceiver === "module" ||
(write.exportValueReceiver === "exports" && relation === "connected")
)
continue;
relation =
write.exportValueReceiver === "exports"
? "connected"
: relation === "uncertain"
? "uncertain"
: "detached";
}
if (write.exportReceiver === "exports") {
if (relation === "detached") continue;
if (relation === "uncertain") {
published.push({ ...write, receiverUncertain: true });
continue;
}
}
published.push(write);
}
return published;
}
#exportValueReceiver(
input: t.Node,
alias: string,
): "module" | "exports" | undefined {
const expression = unwrapJavaScriptExpression(input).node;
const node = t.isAssignmentExpression(expression, { operator: "=" })
? expression.left
: expression;
const root = t.isMemberExpression(node) ? node.object : node;
if (!t.isIdentifier(root)) return undefined;
const binding = resolveSemanticBindingState(this.state, root, root.name);
if (
binding === undefined ||
this.#exportAliases.get(binding.bindingId) !== alias
)
return undefined;
if (t.isIdentifier(node))
return this.#moduleBindings.has(binding.bindingId)
? undefined
: "exports";
return t.isMemberExpression(node) &&
this.#moduleBindings.has(binding.bindingId) &&
semanticStaticPropertyKey(node.property, node.computed) === "exports"
? "module"
: undefined;
}
#completeNamespace(
value: SymbolicNamespace,
point: t.Node,
@@ -1548,6 +1631,14 @@ export class JavaScriptSymbolicBindings {
const binding = resolveSemanticBindingState(this.state, root, root.name);
if (!binding) return;
const alias = this.#exportAliases.get(binding.bindingId);
if (
alias &&
this.#moduleBindings.has(binding.bindingId) &&
path.length === 0
) {
this.#mutated.add(alias);
return;
}
if (this.#moduleBindings.has(binding.bindingId) && path[0] !== "exports")
return;
const key = alias ?? binding.bindingId;
@@ -1577,7 +1668,25 @@ export class JavaScriptSymbolicBindings {
}
}
const writes = this.#writes.get(key) ?? [];
writes.push({ path: selected, value, site });
const exportValueReceiver =
alias !== undefined &&
selected.length === 0 &&
t.isAssignmentExpression(value, { operator: "=" })
? this.#exportValueReceiver(value.right, alias)
: undefined;
writes.push({
path: selected,
value,
site,
...(alias === undefined
? {}
: {
exportReceiver: this.#moduleBindings.has(binding.bindingId)
? ("module" as const)
: ("exports" as const),
}),
...(exportValueReceiver === undefined ? {} : { exportValueReceiver }),
});
this.#writes.set(key, writes);
}
}
@@ -1,6 +1,10 @@
import { describe, expect, it } from "vitest";
import { analyzeJavaScriptStaticSource } from "./javascriptStaticAnalysis.js";
import {
analyzeJavaScriptStaticSource,
analyzeParsedJavaScriptStaticSource,
} from "./javascriptStaticAnalysis.js";
import { parseJavaScriptSource } from "./javascriptSourceParser.js";
describe("Vite preload dependency recovery", () => {
it("decodes helper table values and preserves sparse and dynamic slots", () => {
@@ -143,3 +147,84 @@ describe("Vite preload dependency recovery", () => {
]);
});
});
it.each([
{
name: "canonical declaration",
prefix: "",
suffix: "__vite__mapDeps([0]);",
specifiers: ["./canonical.js"],
},
{
name: "captured outer binding",
prefix: "",
suffix: "function load(){return __vite__mapDeps([0]);}load();",
specifiers: ["./canonical.js"],
},
{
name: "shadowed parameter",
prefix: "",
suffix:
"function load(__vite__mapDeps){return __vite__mapDeps([0]);}load(()=>[]);",
specifiers: [],
},
{
name: "out-of-scope declaration",
prefix: "function setup(){",
suffix: "}__vite__mapDeps([0]);",
specifiers: [],
},
{
name: "unrelated shadowed cache mutation",
prefix: "",
suffix:
"function change(__vite__mapDeps){__vite__mapDeps.f=['./other.js'];}__vite__mapDeps([0]);",
specifiers: ["./canonical.js"],
},
{
name: "uncertain with receiver",
prefix: "",
suffix: "with(unknown){__vite__mapDeps([0]);}",
specifiers: [null],
},
{
name: "uncertain with mutation",
prefix: "",
suffix:
"with(unknown){__vite__mapDeps.f=['./other.js'];}__vite__mapDeps([0]);",
specifiers: [null],
},
])(
"uses the canonical helper's lexical identity for $name",
({ prefix, suffix, specifiers }) => {
const source = `${prefix}
const __vite__mapDeps = (indexes, map = __vite__mapDeps, dependencies =
(map.f || (map.f = ["./canonical.js"]))) => indexes.map((index) => dependencies[index]);
${suffix}
`;
const parsed = parseJavaScriptSource(source, "preload.cjs");
expect(parsed).not.toBeNull();
if (parsed === null) throw new Error("preload fixture did not parse");
const analysis = analyzeParsedJavaScriptStaticSource(source, parsed);
expect(analysis.parse_error_count).toBe(0);
expect(
analysis.references
.filter(({ kind }) => kind === "dynamic-import")
.map(({ specifier }) => specifier),
).toEqual(specifiers);
},
);
it("retains the identity of a function-declared preload helper", () => {
const analysis = analyzeJavaScriptStaticSource(`
function __vite__mapDeps(indexes, map = __vite__mapDeps, dependencies =
(map.f || (map.f = ["./canonical.js"]))) { return indexes.map((index) => dependencies[index]); }
__vite__mapDeps([0]);
`);
expect(analysis.parse_error_count).toBe(0);
expect(
analysis.references
.filter(({ kind }) => kind === "dynamic-import")
.map(({ specifier }) => specifier),
).toEqual(["./canonical.js"]);
});
+1
View File
@@ -83,6 +83,7 @@ export const liveBinaryTargetSummaryVariants = [
"uf2",
"dfuse",
"fit",
"ntfs-mft",
]),
architecture: z.null(),
}),
+478
View File
@@ -0,0 +1,478 @@
import { describe, expect, it } from "vitest";
import {
MFT_DEFAULT_RECORD_BYTES,
MFT_MAX_FILE_BYTES,
createMftDecoder,
parseMftImage,
} from "./ntfsMft.js";
interface RecordOptions {
readonly sequenceNumber?: number;
readonly hardLinkCount?: number;
readonly flags?: number;
readonly baseRecordReference?: bigint;
readonly recordBytes?: number;
/** Write a wrong trailer value into this sector, leaving the others correct. */
readonly corruptSector?: number;
/** Truncate the update sequence array to this many entries; 0 omits it. */
readonly usaEntries?: number;
readonly usaOffset?: number;
}
/** One FILE record with a zeroed body and fixups applied the way NTFS writes them. */
const recordOf = (options: RecordOptions = {}): Uint8Array => {
const recordBytes = options.recordBytes ?? MFT_DEFAULT_RECORD_BYTES;
const record = new Uint8Array(recordBytes);
const view = new DataView(record.buffer);
record.set(Buffer.from("FILE", "latin1"), 0);
const sequenceNumber = options.sequenceNumber ?? 1;
const sectors = recordBytes / 512;
const usaEntries = options.usaEntries ?? sectors + 1;
if (usaEntries > 0) {
const usaOffset = options.usaOffset ?? 48;
view.setUint16(4, usaOffset, true);
view.setUint16(6, usaEntries, true);
view.setUint16(usaOffset, sequenceNumber, true);
for (let sector = 0; sector < sectors; sector += 1) {
const trailer = sector * 512 + 510;
// A saved trailer is what the sector held before the fixup replaced it.
view.setUint16(usaOffset + 2 * (sector + 1), 0xa5a5, true);
view.setUint16(
trailer,
sector >= usaEntries - 1
? 0xa5a5
: sector === options.corruptSector
? 0x1234
: sequenceNumber,
true,
);
}
} else {
view.setUint16(4, 0, true);
view.setUint16(6, 0, true);
}
view.setUint16(16, sequenceNumber, true);
view.setUint16(18, options.hardLinkCount ?? 1, true);
const firstAttributeOffset =
Math.ceil((48 + Math.max(0, usaEntries) * 2) / 8) * 8;
view.setUint16(20, firstAttributeOffset, true);
view.setUint16(22, options.flags ?? 0x0001, true);
view.setUint32(24, firstAttributeOffset + 8, true);
view.setUint32(firstAttributeOffset, 0xffff_ffff, true);
view.setUint32(28, recordBytes, true);
view.setBigUint64(32, options.baseRecordReference ?? 0n, true);
view.setUint16(40, 0, true);
return record;
};
const streamOf = (...records: readonly Uint8Array[]): Uint8Array => {
const total = records.reduce((sum, record) => sum + record.length, 0);
const stream = new Uint8Array(total);
let offset = 0;
for (const record of records) {
stream.set(record, offset);
const view = new DataView(stream.buffer, offset, record.length);
if (view.getUint16(4, true) >= 48)
view.setUint32(44, offset / record.length, true);
offset += record.length;
}
return stream;
};
describe("MFT header facts", () => {
it("inventories a clean record's header and verified fixups", () => {
const parsed = parseMftImage(streamOf(recordOf({ sequenceNumber: 5 })));
expect(parsed.ok).toBe(true);
if (!parsed.ok) return;
expect(parsed.value.metadata).toMatchObject({
kind: "ntfs-mft",
record_bytes: MFT_DEFAULT_RECORD_BYTES,
record_count: 1,
in_use_record_count: 1,
directory_record_count: 0,
extension_record_count: 0,
fixup_mismatch_count: 0,
fixup_unverified_record_count: 0,
truncated_tail_bytes: 0,
});
expect(parsed.value.records[0]).toMatchObject({
index: 0,
fileOffset: 0,
sequenceNumber: 5,
inUse: true,
directory: false,
baseRecordReference: null,
firstAttributeOffset: 56,
fixups: {
sequenceNumber: 5,
verifiedSectors: 2,
coveredSectors: 2,
mismatchSector: null,
malformed: false,
},
});
expect(parsed.value.limitations).toEqual([]);
});
it("identifies unrecognized flags without losing their source bits", () => {
const parsed = parseMftImage(recordOf({ flags: 0x8001 }));
if (!parsed.ok) throw new Error(parsed.error.message);
expect(parsed.value.metadata.record_headers[0]?.flags).toEqual({
raw: 0x8001,
names: ["in-use"],
unknownMask: 0x8000,
});
});
it.each([0, 49, 64, 72])(
"reports an invalid first attribute offset %i",
(offset) => {
const record = recordOf();
new DataView(record.buffer).setUint16(20, offset, true);
const parsed = parseMftImage(record);
if (!parsed.ok) throw new Error(parsed.error.message);
expect(parsed.value.records[0]?.extentValid).toBe(false);
expect(parsed.value.limitations.join(" ")).toContain("inconsistent");
},
);
it("reports a declared record number that contradicts its stream position", () => {
const bytes = streamOf(recordOf(), recordOf());
new DataView(bytes.buffer).setUint32(1024 + 44, 0, true);
const parsed = parseMftImage(bytes);
if (!parsed.ok) throw new Error(parsed.error.message);
expect(parsed.value.records[1]).toMatchObject({
index: 1,
declaredRecordNumber: 0,
recordNumberMatchesSlot: false,
});
expect(parsed.value.limitations.join(" ")).toContain(
"disagrees with their stream slot",
);
});
it("keeps the inline header window separate from the parsed record count", () => {
const parsed = parseMftImage(
streamOf(...Array.from({ length: 33 }, () => recordOf())),
);
if (!parsed.ok) throw new Error(parsed.error.message);
expect(parsed.value.metadata).toMatchObject({
record_count: 33,
omitted_record_headers: 1,
});
expect(parsed.value.metadata.record_headers).toHaveLength(32);
expect(parsed.value.records).toHaveLength(33);
expect(parsed.value.limitations).toEqual([]);
});
it("preserves a log sequence number above the safe integer range", () => {
const record = recordOf();
new DataView(record.buffer).setBigUint64(8, 9007199254740993n, true);
const parsed = parseMftImage(record);
if (!parsed.ok) throw new Error(parsed.error.message);
expect(parsed.value.records[0]?.logFileSequenceNumber).toBe(
"9007199254740993",
);
});
it("verifies the older FILE header without reading its fixup words as a record number", () => {
const oldRecord = recordOf({ usaOffset: 42 });
const newRecord = recordOf();
new DataView(newRecord.buffer).setUint32(44, 1, true);
const parsed = parseMftImage(streamOf(oldRecord, newRecord));
if (!parsed.ok) throw new Error(parsed.error.message);
expect(parsed.value.records).toMatchObject([
{
declaredRecordNumber: null,
fixups: { malformed: false, verifiedSectors: 2 },
},
{
declaredRecordNumber: 1,
fixups: { malformed: false, verifiedSectors: 2 },
},
]);
expect(parsed.value.limitations).toEqual([]);
});
});
describe("MFT fixup coverage", () => {
it.each([0, 0xffff])(
"leaves reserved USA sequence %i unverified",
(sequenceNumber) => {
const parsed = parseMftImage(recordOf({ sequenceNumber }));
if (!parsed.ok) throw new Error(parsed.error.message);
expect(parsed.value.records[0]?.fixups).toMatchObject({
sequenceNumber,
verifiedSectors: 0,
malformed: true,
});
expect(parsed.value.metadata.fixup_unverified_record_count).toBe(1);
expect(parsed.value.limitations.join(" ")).toContain(
"invalid or incomplete",
);
},
);
it("keeps a record whose sector trailer is not the sequence number, and reports it", () => {
const parsed = parseMftImage(
streamOf(recordOf(), recordOf({ sequenceNumber: 9, corruptSector: 1 })),
);
expect(parsed.ok).toBe(true);
if (!parsed.ok) return;
expect(parsed.value.records[1]?.fixups).toMatchObject({
sequenceNumber: 9,
verifiedSectors: 1,
mismatchSector: 1,
});
expect(parsed.value.metadata.fixup_mismatch_count).toBe(1);
expect(parsed.value.limitations.join(" ")).toMatch(
/1 record\(s\) hold a sector trailer/u,
);
});
it("treats a missing update sequence array as unverified rather than clean", () => {
const parsed = parseMftImage(streamOf(recordOf({ usaEntries: 0 })));
expect(parsed.ok).toBe(true);
if (!parsed.ok) return;
expect(parsed.value.records[0]?.fixups).toMatchObject({
malformed: true,
mismatchSector: null,
coveredSectors: 0,
});
expect(parsed.value.metadata.fixup_mismatch_count).toBe(0);
expect(parsed.value.metadata.fixup_unverified_record_count).toBe(1);
expect(parsed.value.limitations.join(" ")).toMatch(
/invalid or incomplete update sequence array/u,
);
});
it.each([
[48, 4],
[49, 3],
[508, 3],
])(
"rejects a malformed update sequence array at %i with %i entries",
(offset, count) => {
const bytes = recordOf();
const view = new DataView(bytes.buffer);
view.setUint16(4, offset, true);
view.setUint16(6, count, true);
const parsed = parseMftImage(bytes);
if (!parsed.ok) throw new Error(parsed.error.message);
expect(parsed.value.records[0]?.fixups).toMatchObject({
malformed: true,
verifiedSectors: 0,
});
expect(parsed.value.metadata.fixup_unverified_record_count).toBe(1);
},
);
it("counts uncovered sectors independently of a mismatched trailer", () => {
const parsed = parseMftImage(
recordOf({ recordBytes: 4096, usaEntries: 3, corruptSector: 0 }),
);
if (!parsed.ok) throw new Error(parsed.error.message);
expect(parsed.value.metadata).toMatchObject({
fixup_mismatch_count: 1,
fixup_unverified_record_count: 1,
});
expect(parsed.value.limitations.join(" ")).toContain(
"invalid or incomplete",
);
});
it("reports an array that covers fewer sectors than the record has", () => {
// A 4 KiB record holds eight sectors; a three-entry array leaves five unverified.
const parsed = parseMftImage(
streamOf(recordOf({ recordBytes: 4096, usaEntries: 3 })),
);
expect(parsed.ok).toBe(true);
if (!parsed.ok) return;
expect(parsed.value.records[0]?.fixups).toMatchObject({
coveredSectors: 2,
verifiedSectors: 2,
mismatchSector: null,
malformed: false,
});
expect(parsed.value.metadata.fixup_unverified_record_count).toBe(1);
});
});
describe("MFT record slots", () => {
it("distinguishes extension records from base records", () => {
const parsed = parseMftImage(
streamOf(
recordOf(),
recordOf({
flags: 0x0005,
baseRecordReference: 0xabcd_fedc_ba98_7654n,
}),
),
);
expect(parsed.ok).toBe(true);
if (!parsed.ok) return;
expect(parsed.value.records[1]).toMatchObject({
baseRecordReference: {
recordNumber: 0xfedc_ba98_7654,
sequenceNumber: 0xabcd,
},
flags: { raw: 0x0005, names: ["in-use", "system"] },
});
expect(parsed.value.metadata.extension_record_count).toBe(1);
});
it("names directory records without losing the in-use flag", () => {
const parsed = parseMftImage(streamOf(recordOf({ flags: 0x0003 })));
expect(parsed.ok).toBe(true);
if (!parsed.ok) return;
expect(parsed.value.records[0]).toMatchObject({
directory: true,
inUse: true,
flags: { names: ["in-use", "directory"] },
});
expect(parsed.value.metadata.directory_record_count).toBe(1);
});
it("skips malformed and sparse slots without renumbering valid records", () => {
const stream = streamOf(
recordOf(),
recordOf(),
new Uint8Array(MFT_DEFAULT_RECORD_BYTES),
recordOf(),
);
stream.set(Buffer.from("BAAD", "latin1"), MFT_DEFAULT_RECORD_BYTES);
const parsed = parseMftImage(stream);
expect(parsed.ok).toBe(true);
if (!parsed.ok) return;
expect(parsed.value.metadata.record_count).toBe(2);
expect(
parsed.value.records.map(({ index, fileOffset }) => [index, fileOffset]),
).toEqual([
[0, 0],
[3, 3 * MFT_DEFAULT_RECORD_BYTES],
]);
expect(parsed.value.limitations).toEqual([
"record at 1024 does not begin with the FILE signature; 2 slot(s) without a FILE signature were found",
]);
});
it("keeps a trailing partial record out of the inventory", () => {
const stream = new Uint8Array(1024 + 300);
stream.set(recordOf(), 0);
const parsed = parseMftImage(stream);
expect(parsed.ok).toBe(true);
if (!parsed.ok) return;
expect(parsed.value.metadata.record_count).toBe(1);
expect(parsed.value.metadata.truncated_tail_bytes).toBe(300);
expect(parsed.value.limitations.join(" ")).toMatch(
/300 trailing byte\(s\)/u,
);
});
it("falls back to the default record size when record 0 declares none", () => {
const record = recordOf();
new DataView(record.buffer).setUint32(28, 0, true);
const parsed = parseMftImage(record);
expect(parsed.ok).toBe(true);
if (!parsed.ok) return;
expect(parsed.value.recordBytes).toBe(MFT_DEFAULT_RECORD_BYTES);
expect(parsed.value.limitations.join(" ")).toMatch(
/not a supported power of two/u,
);
});
it("reads a 4 KiB record at its declared size", () => {
const parsed = parseMftImage(streamOf(recordOf({ recordBytes: 4096 })));
expect(parsed.ok).toBe(true);
if (!parsed.ok) return;
expect(parsed.value.recordBytes).toBe(4096);
expect(parsed.value.metadata.record_count).toBe(1);
});
});
describe("parseMftImage refusals and limits", () => {
it("refuses a file shorter than one record header", () => {
const parsed = parseMftImage(new Uint8Array(32));
expect(parsed.ok).toBe(false);
if (parsed.ok) return;
expect(parsed.error.message).toMatch(
/shorter than one 42-byte record header/u,
);
});
it("refuses bytes that do not start with the FILE signature", () => {
const parsed = parseMftImage(Buffer.alloc(2048, 0x41));
expect(parsed.ok).toBe(false);
if (parsed.ok) return;
expect(parsed.error.message).toMatch(
/does not begin with the FILE signature/u,
);
});
it("refuses a file above the fixed ceiling", () => {
const decoder = createMftDecoder(MFT_MAX_FILE_BYTES + 1);
expect(decoder.ok).toBe(false);
if (decoder.ok) return;
expect(decoder.error).toMatchObject({
kind: "limit",
boundary: "file-size",
});
});
it("refuses streams that differ from the admitted byte count", () => {
const stream = recordOf();
const short = createMftDecoder(stream.length + 1);
if (!short.ok) throw new Error(short.error.message);
expect(short.value.write(stream).ok).toBe(true);
expect(short.value.finish()).toMatchObject({
ok: false,
error: {
kind: "format",
message: "stream ended before the admitted file length",
},
});
const long = createMftDecoder(stream.length - 1);
if (!long.ok) throw new Error(long.error.message);
expect(long.value.write(stream)).toMatchObject({
ok: false,
error: {
kind: "format",
message: "received more bytes than the admitted file length",
},
});
});
it("returns the same inventory without retaining all the decoded records", () => {
const bytes = streamOf(...Array.from({ length: 33 }, () => recordOf()));
const decoder = createMftDecoder(bytes.length);
const parsed = parseMftImage(bytes);
if (!decoder.ok || !parsed.ok) throw new Error("invalid fixture");
expect(decoder.value.write(bytes).ok).toBe(true);
const inventory = decoder.value.finish();
if (!inventory.ok) throw new Error(inventory.error.message);
expect(inventory.value).not.toHaveProperty("records");
expect(inventory.value.metadata).toEqual(parsed.value.metadata);
expect(inventory.value.limitations).toEqual(parsed.value.limitations);
});
it("decodes the same stream through the chunked decoder", () => {
const stream = streamOf(recordOf(), recordOf({ flags: 0x0003 }));
const decoder = createMftDecoder(stream.length);
expect(decoder.ok).toBe(true);
if (!decoder.ok) return;
// Feed one byte past a record boundary so a chunk never aligns with a record.
for (const chunk of [stream.subarray(0, 1025), stream.subarray(1025)])
expect(decoder.value.write(chunk).ok).toBe(true);
const image = decoder.value.finish();
expect(image.ok).toBe(true);
if (!image.ok) return;
expect(image.value.metadata.record_count).toBe(2);
expect(image.value.metadata.directory_record_count).toBe(1);
});
});
+512
View File
@@ -0,0 +1,512 @@
import { err, ok, type Result } from "../result.js";
import type {
RegionDecodeFailure,
RegionDecoder,
} from "../firmware/regionDecoder.js";
/** An NTFS FILE record begins with this signature. */
const MFT_FILE_RECORD_MAGIC = "FILE";
const MFT_FILE_RECORD_MAGIC_BYTES = 4;
/** The shortest header worth reading: a FILE record header ends at `nextAttributeId`. */
const NTFS_MFT_MIN_RECORD_BYTES = 42;
const hasMftFileRecordSignature = (prefix: Uint8Array): boolean =>
prefix.length >= MFT_FILE_RECORD_MAGIC_BYTES &&
text(prefix.subarray(0, MFT_FILE_RECORD_MAGIC_BYTES)) ===
MFT_FILE_RECORD_MAGIC;
/** A record's declared size is a power of two from 512 upward; 1024 is NTFS's default. */
export const MFT_DEFAULT_RECORD_BYTES = 1024;
const MFT_MIN_RECORD_BYTES = 512;
const MFT_MAX_RECORD_BYTES = 65536;
/** An update sequence array holds at least the sequence number and one fixup word. */
const USA_MIN_ENTRIES = 2;
/** NTFS fixes up one 512-byte sector at a time. */
const FIXUP_SECTOR_BYTES = 512;
/**
* Fixed whole-file ceiling. Records are inventoried from their headers only;
* attributes and their payloads are not decoded, so this is deliberately
* smaller than a volume's MFT can be.
*/
export const MFT_MAX_FILE_BYTES = 512 * 1024 * 1024;
/** Limit the inline header sample independently of full stream coverage. */
export const MFT_INLINE_HEADER_LIMIT = 32;
/** FILE record header fields; offsets are from the record start. */
const RECORD_HEADER = {
updateSequenceOffset: 4,
updateSequenceCount: 6,
logFileSequenceNumber: 8,
sequenceNumber: 16,
hardLinkCount: 18,
firstAttributeOffset: 20,
flags: 22,
usedBytes: 24,
totalBytes: 28,
baseRecordReference: 32,
nextAttributeId: 40,
recordNumber: 44,
} as const;
const RECORD_HEADER_WITH_NUMBER_BYTES = 48;
const FILE_RECORD_IN_USE = 0x0001;
const FILE_RECORD_IS_DIRECTORY = 0x0002;
const FILE_RECORD_NAMES = new Map<number, string>([
[FILE_RECORD_IN_USE, "in-use"],
[FILE_RECORD_IS_DIRECTORY, "directory"],
[0x0004, "system"],
[0x0008, "view-index"],
]);
export interface MftRecordFixups {
/** The update sequence number the array's first entry holds, or 0 when absent. */
readonly sequenceNumber: number;
readonly entries: number;
/** Sectors whose trailer held the sequence number before a mismatch. */
readonly verifiedSectors: number;
/** Sectors the array covers; fewer than the record's sectors leaves a tail unverified. */
readonly coveredSectors: number;
/** The first sector whose trailer did not hold the sequence number, if any. */
readonly mismatchSector: number | null;
/** The array itself is missing, misplaced or too short to cover one sector. */
readonly malformed: boolean;
}
export interface MftFileReference {
readonly recordNumber: number;
readonly sequenceNumber: number;
}
const decodeMftFileReference = (reference: bigint): MftFileReference => ({
recordNumber: Number(reference & 0xffff_ffff_ffffn),
sequenceNumber: Number(reference >> 48n),
});
export interface MftRecord {
/** Zero-based ordinal of this record in the stream. */
readonly index: number;
/** File offset where the record starts. */
readonly fileOffset: number;
readonly sequenceNumber: number;
/** Hard link count as recorded; a zero count marks a record with no names. */
readonly hardLinkCount: number;
readonly flags: {
readonly raw: number;
readonly names: readonly string[];
readonly unknownMask: number;
};
readonly inUse: boolean;
readonly directory: boolean;
/** Used and allocated byte counts as declared by the FILE record. */
readonly usedBytes: number;
readonly totalBytes: number;
readonly extentValid: boolean;
/** The base record this is an extension of, or `null` for a base record. */
readonly baseRecordReference: MftFileReference | null;
/** The record number the header declares; `null` when the header predates the field. */
readonly declaredRecordNumber: number | null;
readonly recordNumberMatchesSlot: boolean | null;
/** `$LogFile` sequence number as recorded; no log is read. */
readonly logFileSequenceNumber: string;
/** Declared offset of the first attribute; no attribute decoding follows yet. */
readonly firstAttributeOffset: number;
/** Declared id of the next attribute; no attribute decoding follows yet. */
readonly nextAttributeId: number;
readonly fixups: MftRecordFixups;
}
export interface MftInventory {
readonly metadata: MftMetadata;
/** The record size the stream declared, or the fixed default when it did not. */
readonly recordBytes: number;
readonly limitations: readonly string[];
}
export interface MftImage extends MftInventory {
readonly records: readonly MftRecord[];
}
export interface MftMetadata {
readonly kind: "ntfs-mft";
readonly record_bytes: number;
readonly record_count: number;
readonly in_use_record_count: number;
readonly directory_record_count: number;
readonly extension_record_count: number;
readonly fixup_mismatch_count: number;
readonly fixup_unverified_record_count: number;
readonly truncated_tail_bytes: number;
readonly record_headers: readonly MftRecord[];
readonly omitted_record_headers: number;
}
const text = (ascii: Uint8Array): string =>
Buffer.from(ascii).toString("latin1");
const failure = (
message: string,
boundary?: RegionDecodeFailure["boundary"],
): RegionDecodeFailure => ({
kind: "format",
message,
...(boundary === undefined ? {} : { boundary }),
});
const limit = (
message: string,
boundary?: RegionDecodeFailure["boundary"],
): RegionDecodeFailure => ({
kind: "limit",
message,
...(boundary === undefined ? {} : { boundary }),
});
const parseRecordFlags = (raw: number): MftRecord["flags"] => {
const names: string[] = [];
let unknownMask = raw;
for (const [bit, name] of FILE_RECORD_NAMES) {
if ((raw & bit) !== 0) names.push(name);
unknownMask &= ~bit;
}
return { raw, names, unknownMask };
};
const isRecordSize = (bytes: number): boolean =>
bytes >= MFT_MIN_RECORD_BYTES &&
bytes <= MFT_MAX_RECORD_BYTES &&
(bytes & (bytes - 1)) === 0;
const updateSequenceArrayFits = (
offset: number,
entries: number,
sectors: number,
): boolean =>
entries >= USA_MIN_ENTRIES &&
entries <= sectors + 1 &&
offset >= NTFS_MFT_MIN_RECORD_BYTES &&
offset % 2 === 0 &&
offset + entries * 2 <= FIXUP_SECTOR_BYTES - 2;
/** Recognize a FILE header without requiring its trailers or attributes to verify. */
export const hasMftFileRecordHeader = (prefix: Uint8Array): boolean => {
if (
prefix.length < NTFS_MFT_MIN_RECORD_BYTES ||
!hasMftFileRecordSignature(prefix)
)
return false;
const view = new DataView(
prefix.buffer,
prefix.byteOffset,
prefix.byteLength,
);
return (
isRecordSize(view.getUint32(RECORD_HEADER.totalBytes, true)) ||
updateSequenceArrayFits(
view.getUint16(RECORD_HEADER.updateSequenceOffset, true),
view.getUint16(RECORD_HEADER.updateSequenceCount, true),
MFT_DEFAULT_RECORD_BYTES / FIXUP_SECTOR_BYTES,
)
);
};
/**
* Parse FILE record headers and verify their update-sequence fixups.
*
* Only the record structure is established: attributes, their names, the
* filenames and the USN journal are left to later steps, so no path, timestamp
* or file identity beyond the record itself is claimed. A record whose fixup
* does not match keeps its header facts, because the stream still declares it,
* and the mismatch is reported on the record and counted in the metadata.
*/
export const parseMftImage = (
bytes: Uint8Array,
): Result<MftImage, RegionDecodeFailure> => {
const records: MftRecord[] = [];
const inventory = scanMft(bytes, records);
return inventory.ok ? ok({ ...inventory.value, records }) : inventory;
};
const scanMft = (
bytes: Uint8Array,
retainedRecords?: MftRecord[],
): Result<MftInventory, RegionDecodeFailure> => {
if (bytes.length < NTFS_MFT_MIN_RECORD_BYTES)
return err(
failure(
`file is ${String(bytes.length)} bytes, shorter than one ${String(NTFS_MFT_MIN_RECORD_BYTES)}-byte record header`,
),
);
if (!hasMftFileRecordSignature(bytes))
return err(failure("record 0 does not begin with the FILE signature"));
const limitations: string[] = [];
const header = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
const declaredRecordBytes = header.getUint32(RECORD_HEADER.totalBytes, true);
const recordBytes = isRecordSize(declaredRecordBytes)
? declaredRecordBytes
: MFT_DEFAULT_RECORD_BYTES;
if (!isRecordSize(declaredRecordBytes))
limitations.push(
`record 0 declares a record size of ${String(declaredRecordBytes)} bytes, which is not a supported power of two from ${String(MFT_MIN_RECORD_BYTES)} to ${String(MFT_MAX_RECORD_BYTES)}; records were read at the ${String(MFT_DEFAULT_RECORD_BYTES)}-byte default`,
);
const recordHeaders: MftRecord[] = [];
let recordCount = 0;
let inUseCount = 0;
let directoryCount = 0;
let extensionCount = 0;
let mismatchCount = 0;
let unverifiedCount = 0;
let invalidSlots = 0;
let invalidExtents = 0;
let inconsistentRecordNumbers = 0;
let firstInvalidExtentOffset: number | undefined;
let firstInvalidOffset: number | undefined;
for (
let offset = 0;
offset + recordBytes <= bytes.length;
offset += recordBytes
) {
if (
!hasMftFileRecordSignature(
bytes.subarray(offset, offset + MFT_FILE_RECORD_MAGIC_BYTES),
)
) {
invalidSlots += 1;
firstInvalidOffset ??= offset;
if (!isRecordSize(declaredRecordBytes)) break;
continue;
}
const record = readRecord(bytes, offset, offset / recordBytes, recordBytes);
retainedRecords?.push(record);
if (recordHeaders.length < MFT_INLINE_HEADER_LIMIT)
recordHeaders.push(record);
recordCount += 1;
if (record.inUse) inUseCount += 1;
if (record.directory) directoryCount += 1;
if (record.baseRecordReference !== null) extensionCount += 1;
if (record.fixups.mismatchSector !== null) mismatchCount += 1;
if (
record.fixups.malformed ||
record.fixups.coveredSectors < recordBytes / FIXUP_SECTOR_BYTES
)
unverifiedCount += 1;
if (record.recordNumberMatchesSlot === false)
inconsistentRecordNumbers += 1;
if (!record.extentValid) {
invalidExtents += 1;
firstInvalidExtentOffset ??= offset;
}
}
if (firstInvalidOffset !== undefined)
limitations.push(
`record at ${String(firstInvalidOffset)} does not begin with the FILE signature; ${String(invalidSlots)} slot(s) without a FILE signature were found`,
);
if (firstInvalidExtentOffset !== undefined)
limitations.push(
`${String(invalidExtents)} FILE record(s) declare inconsistent record extents, beginning at offset ${String(firstInvalidExtentOffset)}`,
);
if (inconsistentRecordNumbers !== 0)
limitations.push(
`${String(inconsistentRecordNumbers)} FILE record(s) declare a record number that disagrees with their stream slot`,
);
const truncatedTailBytes = bytes.length % recordBytes;
if (truncatedTailBytes !== 0)
limitations.push(
`${String(truncatedTailBytes)} trailing byte(s) do not fill a ${String(recordBytes)}-byte record and were not parsed`,
);
if (mismatchCount !== 0)
limitations.push(
`${String(mismatchCount)} record(s) hold a sector trailer that is not the update sequence number; their attributes were not read and no later step should trust them`,
);
if (unverifiedCount !== 0)
limitations.push(
`${String(unverifiedCount)} record(s) carry an invalid or incomplete update sequence array; their sector trailers were not fully verified`,
);
const metadata: MftMetadata = {
kind: "ntfs-mft",
record_bytes: recordBytes,
record_count: recordCount,
in_use_record_count: inUseCount,
directory_record_count: directoryCount,
extension_record_count: extensionCount,
fixup_mismatch_count: mismatchCount,
fixup_unverified_record_count: unverifiedCount,
truncated_tail_bytes: truncatedTailBytes,
record_headers: recordHeaders,
omitted_record_headers: recordCount - recordHeaders.length,
};
return ok({ metadata, recordBytes, limitations });
};
const readRecord = (
bytes: Uint8Array,
fileOffset: number,
index: number,
recordBytes: number,
): MftRecord => {
const record = bytes.subarray(fileOffset, fileOffset + recordBytes);
const view = new DataView(
record.buffer,
record.byteOffset,
record.byteLength,
);
const flags = view.getUint16(RECORD_HEADER.flags, true);
const baseRecordReference = view.getBigUint64(
RECORD_HEADER.baseRecordReference,
true,
);
const usaOffset = view.getUint16(RECORD_HEADER.updateSequenceOffset, true);
const fixups = verifyFixups(record, view, usaOffset);
const usedBytes = view.getUint32(RECORD_HEADER.usedBytes, true);
const totalBytes = view.getUint32(RECORD_HEADER.totalBytes, true);
const firstAttributeOffset = view.getUint16(
RECORD_HEADER.firstAttributeOffset,
true,
);
const declaredRecordNumber =
!fixups.malformed && usaOffset >= RECORD_HEADER_WITH_NUMBER_BYTES
? view.getUint32(RECORD_HEADER.recordNumber, true)
: null;
return {
index,
fileOffset,
sequenceNumber: view.getUint16(RECORD_HEADER.sequenceNumber, true),
hardLinkCount: view.getUint16(RECORD_HEADER.hardLinkCount, true),
flags: parseRecordFlags(flags),
inUse: (flags & FILE_RECORD_IN_USE) !== 0,
directory: (flags & FILE_RECORD_IS_DIRECTORY) !== 0,
usedBytes,
totalBytes,
extentValid:
totalBytes === recordBytes &&
usedBytes >= NTFS_MFT_MIN_RECORD_BYTES &&
usedBytes % 8 === 0 &&
usedBytes <= totalBytes &&
firstAttributeOffset % 8 === 0 &&
firstAttributeOffset >= NTFS_MFT_MIN_RECORD_BYTES &&
firstAttributeOffset + 4 <= usedBytes &&
(fixups.malformed ||
firstAttributeOffset >= usaOffset + fixups.entries * 2),
baseRecordReference:
baseRecordReference === 0n
? null
: decodeMftFileReference(baseRecordReference),
declaredRecordNumber,
recordNumberMatchesSlot:
declaredRecordNumber === null ? null : declaredRecordNumber === index,
logFileSequenceNumber: view
.getBigUint64(RECORD_HEADER.logFileSequenceNumber, true)
.toString(),
firstAttributeOffset,
nextAttributeId: view.getUint16(RECORD_HEADER.nextAttributeId, true),
fixups,
};
};
/**
* Each sector's last two bytes are replaced on disk by the update sequence
* number; the array's first entry is that number and the rest are the saved
* trailer words. A sector whose trailer is not the sequence number was not
* cleanly written, so the record's attributes cannot be trusted.
*/
const verifyFixups = (
record: Uint8Array,
view: DataView,
usaOffset: number,
): MftRecordFixups => {
const usaCount = view.getUint16(RECORD_HEADER.updateSequenceCount, true);
const sectors = Math.floor(record.length / FIXUP_SECTOR_BYTES);
// Entry 0 holds the sequence number, so sector n's saved trailer is entry n+1.
const coveredSectors = Math.max(0, usaCount - 1);
if (!updateSequenceArrayFits(usaOffset, usaCount, sectors))
return {
sequenceNumber: 0,
entries: usaCount,
verifiedSectors: 0,
coveredSectors: 0,
mismatchSector: null,
malformed: true,
};
const sequenceNumber = view.getUint16(usaOffset, true);
if (sequenceNumber === 0 || sequenceNumber === 0xffff)
return {
sequenceNumber,
entries: usaCount,
verifiedSectors: 0,
coveredSectors,
mismatchSector: null,
malformed: true,
};
for (let sector = 0; sector < coveredSectors; sector += 1) {
const trailerOffset = sector * FIXUP_SECTOR_BYTES + FIXUP_SECTOR_BYTES - 2;
if (view.getUint16(trailerOffset, true) !== sequenceNumber)
return {
sequenceNumber,
entries: usaCount,
verifiedSectors: sector,
coveredSectors,
mismatchSector: sector,
malformed: false,
};
}
return {
sequenceNumber,
entries: usaCount,
verifiedSectors: coveredSectors,
coveredSectors,
mismatchSector: null,
malformed: false,
};
};
/** Buffer the admitted input once while accepting arbitrary chunk boundaries. */
class StreamingMftDecoder implements RegionDecoder<MftInventory> {
readonly #bytes: Uint8Array;
#written = 0;
constructor(fileBytes: number) {
this.#bytes = new Uint8Array(fileBytes);
}
write(chunk: Uint8Array): Result<void, RegionDecodeFailure> {
if (chunk.length > this.#bytes.length - this.#written)
return err(failure("received more bytes than the admitted file length"));
this.#bytes.set(chunk, this.#written);
this.#written += chunk.length;
return ok(undefined);
}
finish(): Result<MftInventory, RegionDecodeFailure> {
if (this.#written !== this.#bytes.length)
return err(failure("stream ended before the admitted file length"));
return scanMft(this.#bytes);
}
}
export const createMftDecoder = (
fileBytes: number,
): Result<RegionDecoder<MftInventory>, RegionDecodeFailure> =>
Number.isSafeInteger(fileBytes) &&
fileBytes >= 0 &&
fileBytes <= MFT_MAX_FILE_BYTES
? ok(new StreamingMftDecoder(fileBytes))
: err(
limit(
`file is ${String(fileBytes)} bytes, exceeding the fixed ${String(MFT_MAX_FILE_BYTES)}-byte (512 MiB) ceiling`,
"file-size",
),
);
-20
View File
@@ -1,4 +1,3 @@
import { readFile } from "node:fs/promises";
import { describe, expect, it } from "vitest";
import type { JsonValue } from "../domain/jsonValue.js";
@@ -1120,25 +1119,6 @@ describe("Ghidra native instruction decoder context for ARM TMode and MIPS ISA_M
});
});
describe("Ghidra bridge script processor context contract", () => {
it("binds TMode to ARM and ISA_MODE to MIPS in ReaGhidraBridge.java", async () => {
const bridgePath = new URL(
"../../bridge/ghidra/ReaGhidraBridge.java",
import.meta.url,
);
const content = await readFile(bridgePath, "utf8");
expect(content).toContain(
'currentProgram.getLanguageID().getIdAsString().startsWith("ARM:")',
);
expect(content).toContain(
'currentProgram.getLanguageID().getIdAsString().startsWith("MIPS:")',
);
expect(content).toContain('"TMode"');
expect(content).toContain('"ISA_MODE"');
expect(content).toContain('"ghidra-program-context"');
});
});
describe("Ghidra dossier nested boundary composition", () => {
const identityLocations = [
"caller",
@@ -0,0 +1,53 @@
import {
ArtifactResourceScope,
type ArtifactResourceOwner,
} from "../../artifacts/ArtifactResourceScope.js";
/** Retain a cancelled response branch until its underlying cancellation settles. */
export class StateResponseCleanup {
readonly #scope = new ArtifactResourceScope();
constructor(private readonly changed: () => void = () => {}) {}
get cleanupResources(): readonly string[] {
return this.#scope.cleanupResources;
}
cancel(cancel: () => Promise<void>): void {
let state:
| { status: "pending" }
| { status: "done" }
| { status: "failed"; cause: unknown } = { status: "pending" };
const owner: ArtifactResourceOwner = {
kind: "lease",
resource: "state response body",
lease: {
release: async () => {
if (state.status === "pending")
throw new Error("State response cancellation is still pending");
if (state.status === "failed") throw state.cause;
},
},
};
this.#scope.retain(owner);
this.changed();
void Promise.resolve()
.then(cancel)
.then(
async () => {
state = { status: "done" };
await this.#scope.release(owner);
this.changed();
},
(cause: unknown) => {
state = { status: "failed", cause };
this.changed();
},
)
.catch(() => undefined);
}
close(): Promise<void> {
return this.#scope.close();
}
}
+12 -3
View File
@@ -1,3 +1,4 @@
import { StateResponseCleanup } from "./StateResponseCleanup.js";
import type { EnvironmentStateStore } from "../../application/environment/environmentStateStore.js";
import { ArtifactResourceScope } from "../../artifacts/ArtifactResourceScope.js";
import type { AnalysisInputError } from "../../domain/analysisErrorCore.js";
@@ -46,17 +47,24 @@ interface CallContext {
readonly intake: StateIntake;
readonly cancel: AbortSignal;
readonly files: ArtifactResourceScope;
readonly responses: StateResponseCleanup;
}
/** `data/add`: exactly one part, whose document replaces the whole state only when it is valid and the call was not cancelled. */
const addState = async (
{ store, intake, cancel, files }: CallContext,
{ store, intake, cancel, files, responses }: CallContext,
params: unknown,
): Promise<RpcOutcome> => {
const parts = isRecord(params) ? params.parts : undefined;
if (!Array.isArray(parts) || parts.length !== 1)
return { error: invalidParams("data/add takes exactly one part") };
const document = await partDocument(parts[0], intake, cancel, files);
const document = await partDocument(
parts[0],
intake,
cancel,
files,
responses,
);
if (!document.ok) return { error: document.error };
// A file read is not interruptible, so a cancellation during it is honoured here, before the state changes.
if (cancel.aborted) return { error: CANCELLED };
@@ -101,6 +109,7 @@ export const stateDataPlane = (
intake: StateIntake = DEFAULT_STATE_INTAKE,
stopped: AbortSignal = new AbortController().signal,
files: ArtifactResourceScope = new ArtifactResourceScope(),
responses: StateResponseCleanup = new StateResponseCleanup(),
): RpcMethods => {
let previous: Promise<unknown> = Promise.resolve();
return (method, params, signal) => {
@@ -111,7 +120,7 @@ export const stateDataPlane = (
const next = previous.then(() =>
cancel.aborted
? { error: CANCELLED }
: run({ store, intake, cancel, files }, params),
: run({ store, intake, cancel, files, responses }, params),
);
previous = next.catch(() => undefined);
return next;
+43 -1
View File
@@ -1,3 +1,4 @@
import { StateResponseCleanup } from "./StateResponseCleanup.js";
import type { HttpHostname } from "../../domain/httpHostname.js";
import { STATE_INTAKE_LIMIT_BYTES } from "../../application/environment/environmentStateDocument.js";
import {
@@ -477,6 +478,8 @@ export const serveEnvironment = async (
};
// A state file whose close failed stays owned here: later file reads retry it, and close reports it if it is still open.
const stateFiles = new ArtifactResourceScope();
let responseStateChanged = (): void => undefined;
const stateResponses = new StateResponseCleanup(() => responseStateChanged());
const served = await serveStreamableHttp({
host,
port,
@@ -499,6 +502,7 @@ export const serveEnvironment = async (
},
stopping.signal,
stateFiles,
stateResponses,
),
),
...(logger === undefined ? {} : { logger }),
@@ -512,11 +516,14 @@ export const serveEnvironment = async (
});
let httpClosePending = false;
let stateClosePending = false;
let responseClosePending = false;
let settledSuccessful = false;
let closing: Promise<void> | undefined;
const shutdownFailures = new Set<AnalysisError>();
const owner: EnvironmentShutdownOwner = {
get cleanupResources() {
const stateResources = stateFiles.cleanupResources;
const responseResources = stateResponses.cleanupResources;
return [
...(httpClosePending ? [served.url] : []),
...(stateClosePending
@@ -524,6 +531,11 @@ export const serveEnvironment = async (
? stateResources
: ["environment state-file scope"]
: []),
...(responseClosePending
? responseResources.length > 0
? responseResources
: ["state response cleanup"]
: []),
];
},
close: () => {
@@ -531,7 +543,7 @@ export const serveEnvironment = async (
const attempt = (async () => {
stop();
const failures: {
readonly stage: "http-server" | "state-files";
readonly stage: "http-server" | "state-files" | "state-responses";
readonly cause: unknown;
}[] = [];
try {
@@ -548,6 +560,15 @@ export const serveEnvironment = async (
stateClosePending = true;
failures.push({ stage: "state-files", cause });
}
try {
await stateResponses.close();
if (stateResponses.cleanupResources.length > 0)
throw new Error("State response cleanup remains pending");
responseClosePending = false;
} catch (cause: unknown) {
responseClosePending = true;
failures.push({ stage: "state-responses", cause });
}
if (failures.length > 0) {
retainedEnvironmentShutdowns.add(owner);
const failure = new AnalysisOutputError(
@@ -583,6 +604,7 @@ export const serveEnvironment = async (
retainedEnvironmentShutdowns.delete(owner);
for (const failure of shutdownFailures) failure.resolveCleanup();
shutdownFailures.clear();
settledSuccessful = true;
})();
closing = attempt;
void attempt.catch(() => {
@@ -591,6 +613,26 @@ export const serveEnvironment = async (
return attempt;
},
};
// A fetcher may hand back a response after an earlier stop completed. Keep
// that newly acquired body owned and make a later close observe its outcome.
responseStateChanged = () => {
if (!stopping.signal.aborted) return;
if (stateResponses.cleanupResources.length > 0) {
responseClosePending = true;
retainedEnvironmentShutdowns.add(owner);
if (settledSuccessful) {
settledSuccessful = false;
closing = undefined;
}
} else if (
shutdownFailures.size === 0 &&
!httpClosePending &&
!stateClosePending
) {
responseClosePending = false;
retainedEnvironmentShutdowns.delete(owner);
}
};
return ok({
...served,
card,
@@ -1037,7 +1037,122 @@ describe("serveEnvironment with a state", () => {
expect(missing.isError).toBe(true);
});
});
});
describe("state response cleanup", () => {
it("reports pending teed response cleanup and retries after its sibling closes", async () => {
let sourceCancelled = false;
const stream = new ReadableStream<Uint8Array>({
start(controller) {
controller.enqueue(new Uint8Array([1]));
},
cancel() {
sourceCancelled = true;
},
});
const [body, sibling] = stream.tee();
const environment = await serveState(
{ source, state },
{
fetchState: async () =>
new Response(body, {
headers: { "content-length": String(STATE_INTAKE_LIMIT_BYTES + 1) },
}),
},
);
let timer: ReturnType<typeof setTimeout> | undefined;
try {
const response = await fetch(
new URL(ENVIRONMENT_RPC_PATH, environment.url),
{
method: "POST",
headers: { "content-type": "application/json" },
body: JSON.stringify(addByUri(1)),
signal: AbortSignal.timeout(10_000),
},
);
await expect(response.json()).resolves.toMatchObject({
error: { code: -32602 },
});
const closing = Promise.race([
environment.close(),
new Promise<never>((_, reject) => {
timer = setTimeout(
() => reject(new Error("server close waited for tee sibling")),
10_000,
);
}),
]);
await expect(closing).rejects.toMatchObject({
_tag: "AnalysisOutputError",
cleanupIncomplete: true,
diagnostics: { cleanup_stages: ["state-responses"] },
});
expect(sourceCancelled).toBe(false);
await sibling.cancel();
expect(sourceCancelled).toBe(true);
await expect(environment.close()).resolves.toBeUndefined();
} finally {
if (timer !== undefined) clearTimeout(timer);
await Promise.allSettled([body.cancel(), sibling.cancel()]);
await environment.close();
}
});
it("retains a late response cleanup after an earlier successful stop", async () => {
let giveResponse!: (response: Response) => void;
const gate = new Promise<Response>((resolve) => {
giveResponse = resolve;
});
const held = heldFetcher(() => gate);
let release!: () => void;
let acknowledge!: () => void;
const cleanup = new Promise<void>((resolve) => {
release = resolve;
});
const entered = new Promise<void>((resolve) => {
acknowledge = resolve;
});
const environment = await serveState(
{ source, state },
{ fetchState: held.fetchState },
);
const answered = fetch(new URL(ENVIRONMENT_RPC_PATH, environment.url), {
method: "POST",
headers: { "content-type": "application/json" },
body: JSON.stringify(addByUri(1)),
signal: AbortSignal.timeout(10_000),
}).catch(() => undefined);
try {
await held.fetching;
await environment.close();
giveResponse(
new Response(
new ReadableStream<Uint8Array>({
cancel() {
acknowledge();
return cleanup;
},
}),
),
);
await entered;
await expect(environment.close()).rejects.toMatchObject({
_tag: "AnalysisOutputError",
cleanupIncomplete: true,
diagnostics: { cleanup_stages: ["state-responses"] },
});
release();
await expect(environment.close()).resolves.toBeUndefined();
} finally {
release();
await answered;
await environment.close();
}
});
});
describe("serveEnvironment state file cleanup", () => {
it("stops without waiting for a stalled download", async () => {
const held = heldFetcher(() => new Promise<Response>(() => undefined));
const environment = await serveState(
+110
View File
@@ -1,3 +1,4 @@
import { StateResponseCleanup } from "./StateResponseCleanup.js";
import { expect, it, vi } from "vitest";
import { ArtifactResourceScope } from "../../artifacts/ArtifactResourceScope.js";
@@ -11,14 +12,17 @@ import {
const inlineDocument = async (bytes: string) => {
const files = new ArtifactResourceScope();
const responses = new StateResponseCleanup();
try {
return await partDocument(
{ kind: "file", file: { bytes } },
DEFAULT_STATE_INTAKE,
new AbortController().signal,
files,
responses,
);
} finally {
await responses.close();
await files.close();
}
};
@@ -79,3 +83,109 @@ it("downloads with redirects refused and the 30 s timeout by default", async ()
fetched.mockRestore();
}
});
it.each(["http-status", "declared-overflow", "stream-overflow"] as const)(
"returns %s while a teed response's other branch remains open",
async (scenario) => {
const source = new ReadableStream<Uint8Array>({
start(controller) {
controller.enqueue(
new Uint8Array(
scenario === "stream-overflow" ? STATE_INTAKE_LIMIT_BYTES + 1 : 1,
),
);
},
});
const [body, sibling] = source.tee();
const response = new Response(body, {
status: scenario === "http-status" ? 503 : 200,
...(scenario === "declared-overflow"
? {
headers: { "content-length": String(STATE_INTAKE_LIMIT_BYTES + 1) },
}
: {}),
});
const files = new ArtifactResourceScope();
const responses = new StateResponseCleanup();
const operation = partDocument(
{ kind: "file", file: { uri: "https://state.example.test/state.json" } },
{ fetchState: async () => response, timeoutMs: 1_000 },
new AbortController().signal,
files,
responses,
);
let timer: ReturnType<typeof setTimeout> | undefined;
try {
const outcome = await Promise.race([
operation.then((result) => ({ kind: "result" as const, result })),
new Promise<{ kind: "deadline" }>((resolve) => {
timer = setTimeout(() => resolve({ kind: "deadline" }), 3_000);
}),
]);
expect(outcome.kind).toBe("result");
expect(responses.cleanupResources).toEqual(["state response body"]);
await expect(responses.close()).rejects.toMatchObject({
reason: "unavailable",
});
if (outcome.kind === "result") {
expect(outcome.result).toMatchObject(
scenario === "http-status"
? { ok: false, error: { data: { reason: "http-status" } } }
: {
ok: false,
error: {
message: expect.stringContaining(
String(STATE_INTAKE_LIMIT_BYTES),
),
},
},
);
}
} finally {
if (timer !== undefined) clearTimeout(timer);
await sibling.cancel();
await operation;
await responses.close();
await files.close();
}
},
);
it("retains the exact failure when response cancellation rejects", async () => {
const failure = new Error("controlled cancellation failure");
const body = new ReadableStream<Uint8Array>({
cancel() {
throw failure;
},
});
const files = new ArtifactResourceScope();
const responses = new StateResponseCleanup();
const result = await partDocument(
{ kind: "file", file: { uri: "https://state.example.test/state.json" } },
{
fetchState: async () => new Response(body, { status: 503 }),
timeoutMs: 1_000,
},
new AbortController().signal,
files,
responses,
);
expect(result).toMatchObject({
ok: false,
error: { data: { reason: "http-status" } },
});
await new Promise<void>((resolve) => setImmediate(resolve));
const rejected: unknown = await responses
.close()
.catch((cause: unknown) => cause);
expect(rejected).toMatchObject({
reason: "unavailable",
cleanup: { resources: ["state response body"] },
});
expect((rejected as Error).cause).toBeInstanceOf(AggregateError);
expect(((rejected as Error).cause as AggregateError).errors).toContain(
failure,
);
expect(responses.cleanupResources).toEqual(["state response body"]);
await files.close();
});
+18 -7
View File
@@ -1,3 +1,4 @@
import { StateResponseCleanup } from "./StateResponseCleanup.js";
import { fileURLToPath } from "node:url";
import { mediaTypeEssence } from "../../domain/analysisView/recordedMediaType.js";
@@ -228,6 +229,7 @@ const downloadBounds = (
const readCapped = async (
body: ReadableStream<Uint8Array> | null,
bounds: DownloadBounds,
responses: StateResponseCleanup,
): Promise<Result<Buffer, RpcError>> => {
if (body === null) return ok(Buffer.alloc(0));
const reader = body.getReader();
@@ -240,13 +242,15 @@ const readCapped = async (
if (done) return ok(Buffer.concat(chunks, total));
total += value.byteLength;
if (total > STATE_INTAKE_LIMIT_BYTES) {
await reader.cancel().catch(() => undefined);
// A tee's sibling can be caller-owned; cancelling this branch must not
// delay the refusal until that sibling finishes.
responses.cancel(() => reader.cancel());
return err(TOO_LARGE);
}
chunks.push(value);
}
} catch {
void reader.cancel().catch(() => undefined);
responses.cancel(() => reader.cancel());
return err(fetchFailed(bounds.failure()));
}
};
@@ -256,6 +260,7 @@ const downloadState = async (
url: URL,
intake: StateIntake,
cancel: AbortSignal,
responses: StateResponseCleanup,
): Promise<Result<Buffer, RpcError>> => {
const bounds = downloadBounds(intake.timeoutMs, cancel);
const fetching = Promise.resolve().then(() =>
@@ -268,7 +273,9 @@ const downloadState = async (
// A response that arrives after the download ended is released unread.
void fetching
.then(
(late) => late.body?.cancel(),
(late) => {
if (late.body !== null) responses.cancel(() => late.body!.cancel());
},
() => undefined,
)
.catch(() => undefined);
@@ -276,7 +283,8 @@ const downloadState = async (
}
const declared = Number(response.headers.get("content-length") ?? "0");
if (!response.ok || declared > STATE_INTAKE_LIMIT_BYTES) {
await response.body?.cancel().catch(() => undefined);
// Cancel this branch without waiting for a potentially independent tee sibling.
if (response.body !== null) responses.cancel(() => response.body!.cancel());
return err(
response.ok
? TOO_LARGE
@@ -286,7 +294,7 @@ const downloadState = async (
},
);
}
return readCapped(response.body, bounds);
return readCapped(response.body, bounds, responses);
};
const NETWORK_SHARE = invalidParams(
@@ -315,6 +323,7 @@ const fileBytes = async (
intake: StateIntake,
cancel: AbortSignal,
files: ArtifactResourceScope,
responses: StateResponseCleanup,
): Promise<Result<Uint8Array, RpcError>> => {
if (typeof file.bytes === "string" && file.uri === undefined) {
if (file.bytes.length > MAX_INLINE_BASE64_CHARACTERS) return err(TOO_LARGE);
@@ -333,7 +342,8 @@ const fileBytes = async (
invalidParams("a file part carries exactly one of bytes or a uri"),
);
const url = new URL(file.uri);
if (url.protocol === "https:") return downloadState(url, intake, cancel);
if (url.protocol === "https:")
return downloadState(url, intake, cancel, responses);
if (url.protocol !== "file:")
return err(invalidParams("a file uri is a file:// or https:// URL"));
const path = localFilePath(url);
@@ -346,6 +356,7 @@ export const partDocument = async (
intake: StateIntake,
cancel: AbortSignal,
files: ArtifactResourceScope,
responses: StateResponseCleanup,
): Promise<Result<unknown, RpcError>> => {
if (isRecord(part) && part.kind === "data" && isRecord(part.data))
return ok(part.data);
@@ -362,6 +373,6 @@ export const partDocument = async (
mediaTypeEssence(mimeType) !== "application/json")
)
return err(invalidParams("a file part's mimeType is application/json"));
const bytes = await fileBytes(part.file, intake, cancel, files);
const bytes = await fileBytes(part.file, intake, cancel, files, responses);
return bytes.ok ? parseDocumentBytes(bytes.value) : bytes;
};
@@ -923,12 +923,27 @@ it("traces all endpoint carriers and their response consumers through CLI and MC
"response",
]);
const response = carriers.find(({ kind }) => kind === "response");
if (response === undefined) throw new Error("Expected matching response");
const request = carriers.find(({ kind }) => kind === "request");
if (response === undefined || request === undefined)
throw new Error("Expected matching request and response");
const consumer = graph.relations.find(
({ source_node_id, relation }) =>
source_node_id === response.node_id && relation === "consumed-by",
);
expect(consumer).toMatchObject({ resolution: "resolved" });
const requestLink = graph.relations.find(
({ source_node_id, target_node_id, relation }) =>
source_node_id === request.node_id &&
target_node_id === response.node_id &&
relation === "consumed-by",
);
expect(consumer).toMatchObject({
resolution: "resolved",
properties: { method: "json" },
});
expect(requestLink).toMatchObject({
resolution: "resolved",
properties: { method: "json" },
});
const client = await connectedMcpClient();
const trace = async (seed: JavaScriptSemanticQueryInput["seed"]) => {
const input = {
@@ -980,13 +995,13 @@ it("traces all endpoint carriers and their response consumers through CLI and MC
"request",
"response",
]);
expect(selected.relations).toEqual([consumer]);
expect(selected.relations).toEqual([requestLink, consumer]);
expect(selected).toMatchObject({
status: "ambiguous",
summary: {
total_seed_matches: 2,
traversed_nodes: 3,
traversed_relations: 1,
traversed_relations: 2,
},
coverage: { status: "partial" },
});
@@ -15,6 +15,13 @@ const cases = [
0,
],
['spawn("control", [], {stdio:"inherit"})', "control", "inherit", 0],
['spawn("global-nan-env", [], {env:NaN})', "global-nan-env", "default", 0],
[
'((NaN) => spawn("shadowed-nan-env", [], {env:NaN}))({})',
"shadowed-nan-env",
"default",
0,
],
[
'spawn("spread-after", [], {stdio:"pipe",...input})',
"spread-after",
@@ -233,6 +240,12 @@ const exportAssignmentSource = `
export = argv;
`;
const environmentCommands = [
"named-argv",
"execfile-dynamic",
"shadowed-nan-env",
];
// Callback/options ambiguity cannot change exec's effective stdio.
const opaqueExecCallbackCommands = [
"eval-callback",
@@ -342,7 +355,7 @@ cliTest.for(["CLI", "stdio MCP"] as const)(
expect(child.properties.stdio_mode, String(command)).toBe(mode);
expect(child.properties.argv_count, String(command)).toBe(count);
expect(child.properties.environment_supplied, String(command)).toBe(
["named-argv", "execfile-dynamic"].includes(String(command)),
environmentCommands.includes(String(command)),
);
expect(child.identity.source_range).not.toBeNull();
const stdio = graph.relations.filter(
@@ -377,9 +390,7 @@ cliTest.for(["CLI", "stdio MCP"] as const)(
relation.relation === "supplies-env" &&
relation.target_node_id === child.node_id,
),
).toHaveLength(
["named-argv", "execfile-dynamic"].includes(String(command)) ? 1 : 0,
);
).toHaveLength(environmentCommands.includes(String(command)) ? 1 : 0);
expect(
graph.relations.filter(
(relation) =>
@@ -1,81 +1,72 @@
import { execFile } from "node:child_process";
import { readFile, readdir, writeFile } from "node:fs/promises";
import { readFile, readdir } from "node:fs/promises";
import { join } from "node:path";
import { promisify } from "node:util";
import { expect, it } from "vitest";
import { expect } from "vitest";
import { parse } from "yaml";
import { resolveSwiftcCompileTimeoutMs } from "../../../src/process/DarwinProcessRunTokenReader.js";
import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js";
import { processTest } from "../../support/process/processFixture.js";
const execute = promisify(execFile);
const hostLabels: Partial<Record<NodeJS.Platform, string>> = {
linux: "linux",
darwin: "macOS",
win32: "Windows",
};
const hostLabel = hostLabels[process.platform];
const preparationBudget =
process.platform === "darwin" ? resolveSwiftcCompileTimeoutMs() : 0;
it("enforces workflow platform labels before contacting Cutter", async () => {
const workflow = parse(
await readFile(".github/workflows/real-cutter.yml", "utf8"),
);
const labels = workflow.jobs["cutter-upstream"].strategy.matrix.include.map(
(entry: { platform: string }) => entry.platform,
);
expect(labels).toEqual(["linux", "macOS", "Windows"]);
const root = await createTestTempDirectory("rea-cutter-platform-");
const preload = join(root, "preload.mjs");
await writeFile(
preload,
`
import childProcess from 'node:child_process';
import { syncBuiltinESMExports } from 'node:module';
Object.defineProperty(process, 'platform', { value: process.env.REA_PROBE_PLATFORM });
childProcess.execFile = () => { throw new Error('fixture discovery reached'); };
syncBuiltinESMExports();
`,
);
const check = async (selected: string | undefined, actual: string) => {
const env: NodeJS.ProcessEnv = {
...process.env,
TMPDIR: root,
TMP: root,
TEMP: root,
REA_PROBE_PLATFORM: actual,
processTest.skipIf(hostLabel === undefined)(
"enforces workflow platform labels before contacting Cutter",
{ timeout: Math.min(2_147_483_647, 2 * preparationBudget + 30_000) },
async ({ processes }) => {
const workflow = parse(
await readFile(".github/workflows/real-cutter.yml", "utf8"),
);
const labels = workflow.jobs["cutter-upstream"].strategy.matrix.include.map(
(entry: { platform: string }) => entry.platform,
);
expect(labels).toEqual(["linux", "macOS", "Windows"]);
const root = await createTestTempDirectory("rea-cutter-platform-");
// A private absent directory proves actual discovery without contacting a user's bridge.
const bridgeDirectory = join(root, "absent-bridge");
const check = async (selected: string | undefined) => {
const env: NodeJS.ProcessEnv = {
...process.env,
TMPDIR: root,
TMP: root,
TEMP: root,
REA_CUTTER_BRIDGE_DIR: bridgeDirectory,
};
if (selected === undefined) delete env.EXPECTED_PLATFORM;
else env.EXPECTED_PLATFORM = selected;
const result = await processes.run(
process.execPath,
["scripts/verify-real-cutter.mjs"],
{
env,
timeoutMs: Math.min(2_147_483_647, preparationBudget + 15_000),
},
);
expect(result.exitCode, result.stderr).toBe(1);
return result.stderr;
};
if (selected === undefined) delete env.EXPECTED_PLATFORM;
else env.EXPECTED_PLATFORM = selected;
const result: unknown = await execute(
process.execPath,
["--import", preload, "scripts/verify-real-cutter.mjs"],
{ env, timeout: 10_000 },
).then(
() => {
throw new Error("Fixture discovery should stop the verifier");
},
(error: unknown) => error,
for (const selected of [hostLabel, undefined]) {
const stderr = await check(selected);
expect(stderr).toContain("Open upstream Cutter with the REA plugin");
expect(stderr).toContain("no_live_bridge_found");
}
const other = labels.find((label: string) => label !== hostLabel);
expect(other).toBeDefined();
expect(await check(other)).toContain(
"Cutter runner did not match its declared platform",
);
if (
!(result instanceof Error) ||
!("stderr" in result) ||
typeof result.stderr !== "string"
)
throw new Error("Missing verifier diagnostics");
return result.stderr;
};
for (const [selected, actual] of [
["linux", "linux"],
["macOS", "darwin"],
["Windows", "win32"],
] as const)
expect(await check(selected, actual)).toContain(
"fixture discovery reached",
expect(await check("unknown")).toContain(
"Unrecognized Cutter runner platform",
);
expect(await check("linux", "darwin")).toContain(
"Cutter runner did not match its declared platform",
);
expect(await check("unknown", "linux")).toContain(
"Unrecognized Cutter runner platform",
);
expect(await check(undefined, "linux")).toContain(
"fixture discovery reached",
);
expect(
(await readdir(root)).filter((name) =>
name.startsWith("rea-cutter-smoke-"),
),
).toEqual([]);
});
expect(
(await readdir(root)).filter((name) =>
name.startsWith("rea-cutter-smoke-"),
),
).toEqual([]);
},
);
@@ -783,7 +783,7 @@ it.each([
{ code: false, ghidra_bridge: true, runtime: false, package: false },
],
[
"tests/conformance/ghidra/ReaArmContextProbe.java",
"tests/conformance/ghidra/ReaInstructionContextProbe.java",
{ ghidra_bridge: true, runtime: false, package: false },
],
[
@@ -0,0 +1,84 @@
import { writeFile } from "node:fs/promises";
import { join } from "node:path";
import { expect } from "vitest";
import { createBinarySession } from "../../../src/composition/binary.js";
import { parseConfig } from "../../../src/config/parseConfig.js";
import { artifactInspectionResultSchema } from "../../../src/domain/artifactInspection.js";
import { parseEvidence } from "../../../src/domain/evidence.js";
import { silentLogger } from "../../../src/logger.js";
import { createServer } from "../../../src/server/createServer.js";
import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js";
import { mcpTest } from "../../support/mcp/mcpFixture.js";
import { serveCli } from "../cli/cliServeHarness.js";
const expectMftInventory = (value: unknown): void => {
const evidence = parseEvidence(value);
const inspection = artifactInspectionResultSchema.parse(
evidence.normalized_result,
);
const inventory = inspection.substeps[0]?.evidence.normalized_result;
expect(inventory?.manifest.root_format).toBe("ntfs-mft");
expect(inventory?.occurrences[0]?.format_metadata).toMatchObject({
kind: "ntfs-mft",
record_count: 1,
fixup_unverified_record_count: 0,
record_headers: [
{
index: 0,
fileOffset: 0,
sequenceNumber: 1,
declaredRecordNumber: null,
fixups: { verifiedSectors: 2 },
},
],
omitted_record_headers: 0,
});
};
mcpTest(
"admits an MFT capture by signature through CLI and MCP",
async ({ mcp, onTestFinished }) => {
const root = await createTestTempDirectory("rea-ntfs-mft-admission-");
const path = join(root, "capture.bin");
const record = Buffer.alloc(1024);
record.write("FILE");
record.writeUInt16LE(42, 4);
record.writeUInt16LE(3, 6);
record.writeUInt16LE(1, 16);
record.writeUInt16LE(56, 20);
record.writeUInt32LE(64, 24);
record.writeUInt32LE(0xffff_ffff, 56);
record.writeUInt32LE(1024, 28);
record.writeUInt16LE(5, 42);
record.writeUInt16LE(5, 510);
record.writeUInt16LE(5, 1022);
await writeFile(path, record);
const cli = await serveCli(["inspect-artifact", path, "--json"]);
expect(cli.exitCode, cli.stdout).toBe(0);
expectMftInventory(JSON.parse(cli.stdout));
const configured = parseConfig({ REA_ANALYSIS_PROVIDER: "auto" });
if (!configured.ok) throw configured.error;
const session = createBinarySession(configured.value, silentLogger, {});
onTestFinished(async () => {
await session.close();
});
const client = await mcp.connect(
createServer({ kind: "session", session }),
);
const opened = await client.callTool({
name: "open_binary",
arguments: { path },
});
expect(opened.isError, JSON.stringify(opened.content)).not.toBe(true);
const inspected = await client.callTool({
name: "inspect_artifact",
arguments: {},
});
expect(inspected.isError, JSON.stringify(inspected.content)).not.toBe(true);
expectMftInventory(inspected.structuredContent);
},
);
@@ -1,4 +1,4 @@
import { readFile, writeFile } from "node:fs/promises";
import { readFile, stat, writeFile } from "node:fs/promises";
import { join } from "node:path";
import { fileURLToPath } from "node:url";
import { expect } from "vitest";
@@ -175,7 +175,11 @@ registerHooks({ load(url, context, nextLoad) {
),
],
{
env: { ...process.env, REA_AGENT_EVAL_ROOT: parent },
env: {
...process.env,
REA_AGENT_EVAL_ROOT: parent,
REA_AGENT_EVAL_KEEP_FIXTURES: "false",
},
timeoutMs: 10_000,
},
);
@@ -190,14 +194,13 @@ registerHooks({ load(url, context, nextLoad) {
if (incomplete) {
expect(result.stderr).toContain("Artifact cleanup remains incomplete");
expect(result.stderr).toContain("fixture cleanup denied");
expect(await readFile(join(run, "codex/auth.json"), "utf8")).toBe(
"inert fixture",
);
expect((await stat(run)).isDirectory()).toBe(true);
} else {
await expect(
readFile(join(run, "codex/auth.json")),
).rejects.toMatchObject({ code: "ENOENT" });
await expect(stat(run)).rejects.toMatchObject({ code: "ENOENT" });
}
await expect(readFile(join(run, "codex/auth.json"))).rejects.toMatchObject({
code: "ENOENT",
});
},
);
@@ -274,7 +277,11 @@ registerHooks({ load(url, context, nextLoad) {
),
],
{
env: { ...process.env, REA_AGENT_EVAL_ROOT: parent },
env: {
...process.env,
REA_AGENT_EVAL_ROOT: parent,
REA_AGENT_EVAL_KEEP_FIXTURES: "false",
},
timeoutMs: 10_000,
},
);
@@ -298,13 +305,14 @@ registerHooks({ load(url, context, nextLoad) {
if (incomplete) {
expect(result.stderr).toContain("Artifact cleanup remains incomplete");
expect(result.stderr).toContain("darwin-fixture-reader");
expect(
await readFile(join(observed.run, "codex/auth.json"), "utf8"),
).toBe("inert preparation fixture");
expect((await stat(observed.run)).isDirectory()).toBe(true);
} else {
await expect(
readFile(join(observed.run, "codex/auth.json")),
).rejects.toMatchObject({ code: "ENOENT" });
await expect(stat(observed.run)).rejects.toMatchObject({
code: "ENOENT",
});
}
await expect(
readFile(join(observed.run, "codex/auth.json")),
).rejects.toMatchObject({ code: "ENOENT" });
},
);
@@ -1,24 +1,57 @@
import { execFile } from "node:child_process";
import {
copyFile,
cp,
mkdir,
readFile,
symlink,
writeFile,
} from "node:fs/promises";
import { dirname, join, resolve } from "node:path";
import { promisify } from "node:util";
import { expect, it } from "vitest";
import { join, resolve } from "node:path";
import { expect } from "vitest";
import { z } from "zod";
import { createApplicationMcpHarness } from "../../fixtures/applicationMcpHarness.js";
import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js";
import { processTest } from "../../support/process/processFixture.js";
import { resolveSwiftcCompileTimeoutMs } from "../../../src/process/DarwinProcessRunTokenReader.js";
const execute = promisify(execFile);
const preparationBudget =
process.platform === "darwin" ? resolveSwiftcCompileTimeoutMs() : 0;
async function prepareController(controller: string, version: string) {
// The verifier loads its production-owned TypeScript source graph at runtime.
// Keep that graph in the isolated controller so imports cannot drift from a hand-maintained list.
await mkdir(join(controller, "scripts"), { recursive: true });
await copyFile(
"scripts/verify-release-artifact.mjs",
join(controller, "scripts/verify-release-artifact.mjs"),
);
await cp("scripts/lib", join(controller, "scripts/lib"), {
recursive: true,
});
await mkdir(join(controller, "bridge"), { recursive: true });
await cp("bridge/process", join(controller, "bridge/process"), {
recursive: true,
});
await cp("src", join(controller, "src"), {
recursive: true,
filter: (source) => !/\.(?:test|spec)\.[cm]?[jt]sx?$/u.test(source),
});
await writeFile(
join(controller, "package.json"),
JSON.stringify({ name: "rea-agents", version, type: "module" }),
);
await symlink(
resolve("node_modules"),
join(controller, "node_modules"),
process.platform === "win32" ? "junction" : "dir",
);
}
// Release rehearsal runs on Linux; the npm process adapter is also exercised on macOS.
it.skipIf(process.platform === "win32")(
processTest.skipIf(process.platform === "win32")(
"fails without declared runtime, then rehearses an installed tarball and rejects catalog drift",
async () => {
{ timeout: Math.min(2_147_483_647, 3 * preparationBudget + 210_000) },
async ({ processes }) => {
const root = await createTestTempDirectory("rea-release-canary-");
const candidate = join(root, "candidate");
const controller = join(root, "controller");
@@ -76,56 +109,46 @@ if (process.argv[2] === "mcp") {
} else { process.exitCode = 1; }
`,
);
const packed = await execute(
"npm",
["pack", "--ignore-scripts", "--json"],
{
cwd: candidate,
},
);
const pack = async () => {
const result = await processes.run(
"npm",
["pack", "--ignore-scripts", "--json"],
{ cwd: candidate, timeoutMs: 30_000 },
);
expect(result.exitCode, result.stderr).toBe(0);
return result;
};
const packed = await pack();
const [{ filename }] = z
.tuple([z.object({ filename: z.string() })])
.parse(JSON.parse(packed.stdout));
const tarball = join(candidate, filename);
for (const file of [
"scripts/verify-release-artifact.mjs",
"scripts/lib/release-package-canary.mjs",
"scripts/lib/verify-package-core.mjs",
"scripts/lib/mcp-verifier-results.mjs",
"scripts/lib/verifier-run.mjs",
]) {
const target = join(controller, file);
await mkdir(dirname(target), { recursive: true });
await copyFile(file, target);
}
await writeFile(
join(controller, "package.json"),
JSON.stringify({ name: "rea-agents", version, type: "module" }),
);
await symlink(
resolve("node_modules"),
join(controller, "node_modules"),
process.platform === "win32" ? "junction" : "dir",
);
await prepareController(controller, version);
const invoke = (selected = tarball) =>
execute(
processes.run(
process.execPath,
[
join(controller, "scripts/verify-release-artifact.mjs"),
selected,
version,
],
{ cwd: controller, maxBuffer: 16 * 1024 * 1024 },
{
cwd: controller,
timeoutMs: Math.min(2_147_483_647, preparationBudget + 30_000),
},
);
await expect(invoke()).rejects.toMatchObject({
stderr: expect.stringContaining("ERR_MODULE_NOT_FOUND"),
});
const missing = await invoke();
expect(missing.exitCode).toBe(1);
expect(missing.stderr).toContain("ERR_MODULE_NOT_FOUND");
expect(missing.stderr).toMatch(/dist[/\\]catalogIdentity\.js/u);
await symlink(
resolve("dist"),
join(controller, "dist"),
process.platform === "win32" ? "junction" : "dir",
);
const result = JSON.parse((await invoke()).stdout);
const successful = await invoke();
expect(successful.exitCode, successful.stderr).toBe(0);
const result = JSON.parse(successful.stdout);
expect(result).toMatchObject({
package: "rea-agents",
version,
@@ -144,12 +167,10 @@ if (process.argv[2] === "mcp") {
join(candidate, "scripts/reply.json"),
JSON.stringify(snapshot),
);
await execute("npm", ["pack", "--ignore-scripts", "--json"], {
cwd: candidate,
});
await expect(invoke()).rejects.toMatchObject({
stderr: expect.stringContaining("invalid canonical tool catalog"),
});
await pack();
const drifted = await invoke();
expect(drifted.exitCode).toBe(1);
expect(drifted.stderr).toContain("invalid canonical tool catalog");
await writeFile(
join(candidate, "package.json"),
JSON.stringify({
@@ -159,15 +180,12 @@ if (process.argv[2] === "mcp") {
files: ["scripts"],
}),
);
const wrong = await execute("npm", ["pack", "--ignore-scripts", "--json"], {
cwd: candidate,
});
const wrong = await pack();
const [{ filename: wrongFilename }] = z
.tuple([z.object({ filename: z.string() })])
.parse(JSON.parse(wrong.stdout));
await expect(invoke(join(candidate, wrongFilename))).rejects.toMatchObject({
stderr: expect.stringContaining("Installed package identity"),
});
const mismatched = await invoke(join(candidate, wrongFilename));
expect(mismatched.exitCode).toBe(1);
expect(mismatched.stderr).toContain("Installed package identity");
},
30_000,
);
+6
View File
@@ -21,6 +21,12 @@ no-overwrite CLI delivery. It checks the target and project files for changes.
Clang/LLD/LLVM readelf/LLVM objdump; `REA_RISCV_CLANG`, `REA_RISCV_LLD`,
`REA_RISCV_READELF` and `REA_RISCV_OBJDUMP` select existing tool commands.
- `ReaInstructionContextProbe.java` inspects source-owned ARM, Thumb, MIPS32
and MIPS16 instructions through the production bridge in an ephemeral,
read-only Ghidra project. It checks exact decoder-context register names,
values and provenance; the ARM lane also injects an unavailable provider
value. The `ghidra-bridge-sources` CI job runs both processor families. The
input bytes are never executed.
- `ReaDecodeProbe.java` decodes source-owned instruction bytes from a
language's default context and checks each mnemonic, length and optional
complete operand. The `ghidra-bridge-sources` CI job runs it for `ARM:LE:32:Cortex`,
@@ -15,16 +15,20 @@ import java.lang.reflect.Proxy;
import java.math.BigInteger;
/** Real ARM/Thumb decoding; only the unavailable context value is injected. */
public class ReaArmContextProbe extends GhidraScript {
public class ReaInstructionContextProbe extends GhidraScript {
@Override
public void run() throws Exception {
String[] args = getScriptArgs();
require(args.length == 1, "Expected the exact production bridge directory");
require(args.length == 1 || (args.length == 2 && args[1].equals("mips")), "Expected the bridge directory and optional MIPS mode");
var source = GhidraScriptUtil.findScriptByName("ReaGhidraBridge.java");
require(source != null && new File(source.getAbsolutePath()).getCanonicalFile().equals(
new File(args[0], "ReaGhidraBridge.java").getCanonicalFile()), "Production bridge source");
GhidraScript bridge = GhidraScriptUtil.getProvider(source).getScriptInstance(source, errorWriter);
bridge.set(getState(), getControls());
if (args.length == 2) {
checkMips(bridge);
return;
}
require(currentProgram.getLanguageID().getIdAsString().equals("ARM:LE:32:v7"), "ARM fixture language");
var mode = currentProgram.getRegister("TMode");
require(mode != null, "Real TMode register");
@@ -40,8 +44,8 @@ public class ReaArmContextProbe extends GhidraScript {
space.set(null, arm.getAddressSpace());
Method inspect = bridge.getClass().getDeclaredMethod("inspectNativeInstruction", JsonObject.class, boolean.class);
inspect.setAccessible(true);
check(inspect, bridge, arm, "0", 4, "1eff2fe1");
check(inspect, bridge, thumb, "1", 2, "7047");
check(inspect, bridge, arm, "0", 4, "1eff2fe1", "TMode");
check(inspect, bridge, thumb, "1", 2, "7047", "TMode");
// getValue includes language defaults. Inject only its unavailable result,
// preserving the real instruction, register, language and all other APIs.
@@ -57,26 +61,49 @@ public class ReaArmContextProbe extends GhidraScript {
programField.setAccessible(true);
try {
programField.set(bridge, program);
check(inspect, bridge, arm, null, 4, "1eff2fe1");
check(inspect, bridge, arm, null, 4, "1eff2fe1", "TMode");
} finally {
programField.set(bridge, original);
}
println("REA_ARM_CONTEXT_OK arm=0 thumb=1 unavailable=null (injected provider value)");
}
private void checkMips(GhidraScript bridge) throws Exception {
require(currentProgram.getLanguageID().getIdAsString().equals("MIPS:LE:32:default"), "MIPS fixture language");
var mode = currentProgram.getRegister("ISA_MODE");
require(mode != null, "Real ISA_MODE register");
Address standard = currentProgram.getMemory().getMinAddress();
Address compressed = standard.add(4);
var context = currentProgram.getProgramContext();
context.setValue(mode, standard, standard.add(3), BigInteger.ZERO);
require(disassemble(standard), "MIPS32 instruction decoded");
// MIPS32 flow may have decoded following bytes in the default mode.
clearListing(compressed, compressed.add(3));
context.setValue(mode, compressed, compressed.add(1), BigInteger.ONE);
require(disassemble(compressed), "MIPS16 instruction decoded");
Field space = bridge.getClass().getDeclaredField("sessionDefaultAddressSpace");
space.setAccessible(true);
space.set(null, standard.getAddressSpace());
Method inspect = bridge.getClass().getDeclaredMethod("inspectNativeInstruction", JsonObject.class, boolean.class);
inspect.setAccessible(true);
check(inspect, bridge, standard, "0", 4, "03008224", "ISA_MODE");
check(inspect, bridge, compressed, "1", 2, "5a6a", "ISA_MODE");
println("REA_MIPS_CONTEXT_OK standard=0 mips16=1");
}
private static void check(Method inspect, Object bridge, Address address, String mode,
int length, String bytes) throws Exception {
int length, String bytes, String register) throws Exception {
JsonObject input = new JsonObject();
input.add("document", JsonNull.INSTANCE);
input.addProperty("address", "0x" + Long.toUnsignedString(address.getOffset(), 16));
JsonObject result = (JsonObject) inspect.invoke(bridge, input, false);
require(result.get("status").getAsString().equals("decoded"), "Decoded bridge response");
require(result.get("length").getAsInt() == length, "ARM/Thumb instruction width");
require(result.get("status").getAsString().equals("decoded"), "Decoded bridge response: " + result);
require(result.get("length").getAsInt() == length, "Instruction width");
require(result.get("bytes").getAsString().equals(bytes), "Original instruction bytes");
var contexts = result.getAsJsonArray("decoder_context");
require(contexts != null && contexts.size() == 1, "One retained mode context");
var value = contexts.get(0).getAsJsonObject();
require(value.get("register").getAsString().equals("TMode"), "TMode register name");
require(value.get("register").getAsString().equals(register), "Exact context register name");
require(value.get("source").getAsString().equals("ghidra-program-context"), "Context provenance");
require(mode == null ? value.get("value").isJsonNull()
: mode.equals(value.get("value").getAsString()), "Exact mode value " + mode);
+71
View File
@@ -252,3 +252,74 @@ export const artifactReaderLifecycleCases = (setup: LifecycleFixture): void => {
validationCases(setup);
});
};
/** An interrupted preparation belongs to its caller, not concurrent readers. */
export const artifactPreparationRecoveryCases = ({
image,
createReader,
}: LifecycleFixture): void => {
it.each(["cancelled", "physical failure"] as const)(
"preserves independent preparation after %s",
async (scenario) => {
const path = await fixture(image());
const admitted = await openRegularFile(path, { symlinks: "reject" });
const reader = createReader(path, admitted);
const controller = new AbortController();
const reason = { request: "first inventory" };
const physical = Object.assign(new Error("controlled read failure"), {
code: "EIO",
});
const reached = createDeferred<void>();
const release = createDeferred<void>();
const nativeRead = admitted.handle.read.bind(admitted.handle);
let blocked = false;
const read = vi.spyOn(admitted.handle, "read").mockImplementation((async (
...args: unknown[]
) => {
const result = await Reflect.apply(nativeRead, admitted.handle, args);
if (!blocked) {
blocked = true;
reached.resolve();
await release.promise;
if (scenario === "physical failure") throw physical;
}
return result;
}) as typeof admitted.handle.read);
const pending: Promise<unknown>[] = [];
try {
const first = (async () => {
for await (const _ of reader.entries(controller.signal)) {
}
})();
pending.push(first);
void first.catch(() => undefined);
await awaitAcknowledgement(reached.promise, first, "preparation read");
const peer = entries(reader);
pending.push(peer);
void peer.catch(() => undefined);
controller.abort(reason);
release.resolve();
if (scenario === "physical failure") {
await expect(first).rejects.toBe(physical);
await expect(peer).rejects.toBe(physical);
} else {
await expect(first).rejects.toMatchObject({
reason: "cancelled",
cause: reason,
});
const recovered = await peer;
expect(recovered.length).toBeGreaterThan(0);
expect(
(await buffer(await reader.open(recovered[0]!))).length,
).toBeGreaterThan(0);
}
} finally {
release.resolve();
await Promise.allSettled(pending);
read.mockRestore();
await reader.close();
await admitted.handle.close();
}
},
);
};