mirror of
https://github.com/morluto/rea.git
synced 2026-11-10 17:30:32 -10:00
Compare commits
38
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
8bb499789c | ||
|
|
2885347cde | ||
|
|
a6a2527ade | ||
|
|
f8fc572e4a | ||
|
|
ab9e5fa204 | ||
|
|
8d45cb45c4 | ||
|
|
7712e3185e | ||
|
|
466149169e | ||
|
|
1c2626edbd | ||
|
|
b16d18f198 | ||
|
|
14030e2f11 | ||
|
|
9505c73824 | ||
|
|
1a446cdade | ||
|
|
1ef7f02cae | ||
|
|
7b86474be7 | ||
|
|
5d10337709 | ||
|
|
8eadb872f2 | ||
|
|
f170175ec8 | ||
|
|
fbe09f84df | ||
|
|
ebb480ba22 | ||
|
|
da1f029c2f | ||
|
|
4db9c166ef | ||
|
|
29fd6a7d03 | ||
|
|
d7338876d6 | ||
|
|
9df3f7174f | ||
|
|
79e476204f | ||
|
|
9c95218374 | ||
|
|
d9d2641981 | ||
|
|
e2585aa7ce | ||
|
|
1d1c7f98ff | ||
|
|
b1f1b3eff3 | ||
|
|
c1d0f2bb45 | ||
|
|
04de5f3200 | ||
|
|
9a1777c7f5 | ||
|
|
11e69ce49b | ||
|
|
2fac2dc667 | ||
|
|
d9ee948b33 | ||
|
|
250220f21b |
No files matched your search
@@ -25,7 +25,7 @@ jobs:
|
||||
node-version-file: .nvmrc
|
||||
- name: Validate the SDK pin against the provider's verified release
|
||||
run: node scripts/verify-ghidra-bridge-sources.mjs --check-sdk-pin >> "$GITHUB_ENV"
|
||||
# The bridge probe imports only source-owned ARM/Thumb bytes and never
|
||||
# The bridge probe imports only source-owned ARM, Thumb and MIPS bytes and never
|
||||
# executes target code. All SDK/project state stays in runner storage.
|
||||
- name: Acquire checksum-verified Ghidra SDK in isolated runner storage
|
||||
shell: bash
|
||||
@@ -46,7 +46,7 @@ jobs:
|
||||
java_home_variable="JAVA_HOME_${GHIDRA_JAVA_MAJOR}_X64"
|
||||
test -n "${!java_home_variable}"
|
||||
JAVA_HOME="${!java_home_variable}" npm run verify:ghidra:bridge-sources -- --verified-sdk
|
||||
- name: Inspect real ARM and Thumb instruction context through the bridge
|
||||
- name: Inspect real ARM, Thumb and MIPS instruction context through the bridge
|
||||
shell: bash
|
||||
run: |
|
||||
java_home_variable="JAVA_HOME_${GHIDRA_JAVA_MAJOR}_X64"
|
||||
@@ -58,10 +58,21 @@ jobs:
|
||||
-import "${probe_root}/arm-thumb.bin" -loader BinaryLoader -processor ARM:LE:32:v7 \
|
||||
-noanalysis -readOnly -deleteProject \
|
||||
-scriptPath "${GITHUB_WORKSPACE}/tests/conformance/ghidra;${GITHUB_WORKSPACE}/bridge/ghidra" \
|
||||
-postScript ReaArmContextProbe.java "${GITHUB_WORKSPACE}/bridge/ghidra" \
|
||||
-postScript ReaInstructionContextProbe.java "${GITHUB_WORKSPACE}/bridge/ghidra" \
|
||||
> "${probe_root}/output.log" 2>&1 || { cat "${probe_root}/output.log"; exit 1; }
|
||||
cat "${probe_root}/output.log"
|
||||
grep -F 'REA_ARM_CONTEXT_OK arm=0 thumb=1 unavailable=null (injected provider value)' "${probe_root}/output.log"
|
||||
mkdir -p "${probe_root}/mips-project"
|
||||
# MIPS32 addiu v0,a0,3; MIPS16 li v0,90. These bytes are never executed.
|
||||
printf '\003\000\202\044\132\152\000\000' > "${probe_root}/mips.bin"
|
||||
timeout 180s "${GHIDRA_INSTALL_DIR}/support/analyzeHeadless" "${probe_root}/mips-project" MipsContext \
|
||||
-import "${probe_root}/mips.bin" -loader BinaryLoader -processor MIPS:LE:32:default \
|
||||
-noanalysis -readOnly -deleteProject \
|
||||
-scriptPath "${GITHUB_WORKSPACE}/tests/conformance/ghidra;${GITHUB_WORKSPACE}/bridge/ghidra" \
|
||||
-postScript ReaInstructionContextProbe.java "${GITHUB_WORKSPACE}/bridge/ghidra" mips \
|
||||
> "${probe_root}/mips-output.log" 2>&1 || { cat "${probe_root}/mips-output.log"; exit 1; }
|
||||
cat "${probe_root}/mips-output.log"
|
||||
grep -F 'REA_MIPS_CONTEXT_OK standard=0 mips16=1' "${probe_root}/mips-output.log"
|
||||
# Each probe imports source-owned bytes at 0x1000, away from the Cortex-M
|
||||
# vector symbols that ARMCortex.pspec defines at 0, and decodes them from
|
||||
# the language's default context, so M-profile Thumb state must hold at
|
||||
|
||||
@@ -185,8 +185,9 @@ package that installed it. A Debian package records its name, version,
|
||||
architecture and the apt sources that list the installed version, with the
|
||||
origin (`o=`) apt reports for each source in `source_origins`, such as `Ubuntu`
|
||||
for the Ubuntu archive and its mirrors or `LP-PPA-<name>` for a PPA; an empty
|
||||
source list means the version came from a local `.deb` or the package lists
|
||||
are missing, which `package_lists_present` tells apart. apt's lists folder,
|
||||
source list means apt reports no source for the installed version.
|
||||
`package_lists_present` reports whether the lists are present. It is `null`
|
||||
when REA cannot read the lists directory. apt's lists folder,
|
||||
and the trusted keys below, are where `apt-config` resolves
|
||||
`Dir::State::Lists`, `Dir::Etc::TrustedParts` and `Dir::Etc::Trusted` under
|
||||
`Dir`, so an `APT_CONFIG` file that moves them is followed: on a computer
|
||||
@@ -214,9 +215,9 @@ does not bring it. A program link that dpkg does not own counts as owned when
|
||||
dpkg owns the file it resolves to, as an alternatives link does. An
|
||||
`--executable` launch, or an entry no package owns, such as
|
||||
one in `~/.local/share/applications`, records the executable's package. When
|
||||
`dpkg-query` does not finish or exits with an error for the executable or the
|
||||
entry, the package is `unknown` and the limitations say which path, since a
|
||||
failed lookup is not proof that no package owns it. The sources, vendor
|
||||
`dpkg-query` fails for the executable or entry, or the selected path contains
|
||||
characters `dpkg-query -S` would interpret as a pattern, package ownership is
|
||||
`unknown` and the limitations name the path. The sources, vendor
|
||||
repositories and `--package-file` match below are the recorded package's; the
|
||||
library walk still reads the executable.
|
||||
|
||||
@@ -243,7 +244,9 @@ writes it on Ubuntu 24.04, has that key as its only candidate, as apt trusts
|
||||
no other for it: REA reads the field's continuation lines as deb822 writes
|
||||
them, a `.` line standing for an empty one, from the key block's `BEGIN` line
|
||||
to its last `END` line, and records the key's path as the entry's sources
|
||||
file. A key counts when gpgv reports a valid signature by it and no bad,
|
||||
file. A fingerprint restriction in `Signed-By` leaves `signing_key` null.
|
||||
The reason names the restriction that REA could not verify. A key counts
|
||||
when gpgv reports a valid signature by it and no bad,
|
||||
expired, revoked or unverifiable signature by it, whatever gpgv's exit status:
|
||||
a release also signed by a key the host lacks makes gpgv fail. Each candidate
|
||||
is copied, an armored key decoded to a binary keyring, into a private
|
||||
|
||||
@@ -336,6 +336,9 @@ A withheld file larger than 48 KiB is refused, since its base64 must fit one
|
||||
environment variable. `--withheld-copy <folder>` copies the withheld files,
|
||||
as captured, into a new or empty folder outside the bundle. The result lists
|
||||
them as `withheld_files`, each with its `setting`, `variable` and `handle`.
|
||||
REA resolves both destinations before writing either. A path that cannot be
|
||||
resolved, including a dangling link, is refused; the copy and bundle cannot
|
||||
contain one another, including through symbolic links.
|
||||
|
||||
`--consent <file>` ships the person's consents, a `consent.json` that
|
||||
`record_build_consent` records, as the bundle's own `consent.json`.
|
||||
|
||||
@@ -26,6 +26,7 @@ for new work, roadmap granularity and producer-owned contracts.
|
||||
- [Native call source coordinates](native/call-source-locations/spec.md) ([plan](native/call-source-locations/plan.md), [tasks](native/call-source-locations/tasks.md), [verification](native/call-source-locations/verification.md))
|
||||
- [Go static string-header literal recovery](native/go-string-literals/spec.md) ([plan](native/go-string-literals/plan.md), [tasks](native/go-string-literals/tasks.md))
|
||||
- [Windows Shell Link shortcut inspection](native/windows-shortcut/spec.md) ([plan](native/windows-shortcut/plan.md), [tasks](native/windows-shortcut/tasks.md))
|
||||
- [NTFS $MFT and USN Journal inspection](native/ntfs-mft/spec.md) ([plan](native/ntfs-mft/plan.md), [tasks](native/ntfs-mft/tasks.md))
|
||||
- [Solidity terminal metadata inspection](solidity/cbor-metadata/spec.md) ([plan](solidity/cbor-metadata/plan.md), [tasks](solidity/cbor-metadata/tasks.md), [verification](solidity/cbor-metadata/verification.md))
|
||||
- [Inspect a V8 code cache container](v8/code-cache-inspection/spec.md) ([plan](v8/code-cache-inspection/plan.md), [tasks](v8/code-cache-inspection/tasks.md))
|
||||
- [Deno compile executable inventory and extraction](artifacts/deno-standalone/spec.md) ([plan](artifacts/deno-standalone/plan.md), [tasks](artifacts/deno-standalone/tasks.md))
|
||||
|
||||
@@ -0,0 +1,39 @@
|
||||
# Implementation plan
|
||||
|
||||
## Boundary and contract
|
||||
|
||||
Follow the existing artifact readers. `src/domain/windows/ntfsMft.ts` owns FILE
|
||||
record bytes and coverage; `src/domain/artifactFormatMetadata.ts` owns the
|
||||
`ntfs-mft` result schema. `NtfsMftArtifactReader` reads the admitted file
|
||||
through `ArtifactFileSource` and exposes its facts through `formatMetadata()`.
|
||||
A record stream is its own content, so the root occurrence carries every fact
|
||||
and no members are enumerated.
|
||||
|
||||
Recognition lives in `hasMftFileRecordSignature` and is applied in
|
||||
`classifyArtifactBytes`, next to the other unambiguous magic probes.
|
||||
|
||||
## Incremental work and validation
|
||||
|
||||
1. Define the first-step contract, limits and observations; add parser fixtures.
|
||||
2. Implement the FILE record header and update-sequence fixup parse with
|
||||
explicit limitations.
|
||||
3. Add the reader, format descriptor and byte classification.
|
||||
4. Decode attribute records, then `$FILE_NAME` attributes so a record's parent
|
||||
reference, names and timestamps become explicit; keep hard links separate.
|
||||
5. Parse `$UsnJrnl:$J` records and correlate them to records by full file
|
||||
reference, including sequence numbers, reporting unresolved references.
|
||||
6. Document the workflow in a native Windows artifact guide.
|
||||
|
||||
## Effects and compatibility
|
||||
|
||||
The workflow reads one caller-selected local file. It adds no target execution,
|
||||
injection, mutation, protection recovery, network access, volume mount or live
|
||||
filesystem access, so no maintainer scope decision is required.
|
||||
`inspect_artifact` routing for every existing format is unchanged, and the new
|
||||
format id appears only for bytes whose first record carries the `FILE`
|
||||
signature.
|
||||
|
||||
## References
|
||||
|
||||
- https://learn.microsoft.com/en-us/windows/win32/fileio/master-file-table
|
||||
- https://github.com/libyal/libfsntfs/blob/main/documentation/New%20Technologies%20File%20System%20(NTFS).asciidoc
|
||||
@@ -0,0 +1,51 @@
|
||||
# NTFS $MFT and USN Journal inspection
|
||||
|
||||
This specification records an implementation increment and its original problem
|
||||
statement. For current behavior and limitations, see the artifact inventory
|
||||
guide.
|
||||
|
||||
## Problem and outcome
|
||||
|
||||
REA has no reader for acquired NTFS metadata artifacts. An analyst holding a
|
||||
captured `$MFT` stream cannot ask which FILE records it contains, whether their
|
||||
update-sequence fixups survived acquisition, or which records are directories,
|
||||
extensions of another record, or no longer in use. Add offline, read-only
|
||||
inventory of a selected `$MFT` file, delivered in steps so each one merges on
|
||||
its own.
|
||||
|
||||
## Requirements
|
||||
|
||||
- Recognize a `$MFT` stream by a plausible first `FILE` record header under
|
||||
any file name. CLI and MCP target admission route it to the artifact reader.
|
||||
- Validate each FILE record header: the `FILE` signature, sequence number, flags,
|
||||
used and total byte counts, base record reference, and the update-sequence
|
||||
array's offset, count and array bounds.
|
||||
- Verify the update-sequence fixups: every sector trailer the array covers must
|
||||
hold the array's sequence number. A mismatch keeps the record's header facts
|
||||
and is reported per record and in the metadata count.
|
||||
- Report a missing, misplaced or short update-sequence array as unverified
|
||||
rather than as a clean record, and a trailing partial record as a byte count.
|
||||
- Report the first invalid record signature and the number of invalid slots.
|
||||
Continue at the next slot when the stream declared a supported record size.
|
||||
With a fallback record size, stop at the first invalid signature.
|
||||
- Bound the whole file at a fixed 512 MiB ceiling and report the ceiling as a
|
||||
`limit` failure with its boundary.
|
||||
- Return the first 32 decoded record headers with their source offsets and the
|
||||
number of omitted headers. Aggregate counts cover the full parsed stream.
|
||||
- Parse only the records' headers and fixups. Attributes, filenames, data
|
||||
streams and the `$UsnJrnl:$J` journal are not decoded in this step, so no path,
|
||||
file size, file reference correlation or change history is claimed.
|
||||
|
||||
## Acceptance
|
||||
|
||||
Synthesized fixtures cover a clean record, a failed fixup, a missing update
|
||||
sequence array, an array shorter than the record's sector count, extension and
|
||||
directory records, a record whose signature is wrong mid-stream, a trailing
|
||||
partial record, an undeclared record size, a 4 KiB record, a stream shorter than
|
||||
one header, bytes without the `FILE` signature, and the file-size ceiling.
|
||||
Inventory tests cover the root occurrence's metadata and the coverage status
|
||||
when a fixup fails, and classification tests cover several file names for the
|
||||
same bytes.
|
||||
|
||||
Not covered in this step: real `$MFT` files written by a Windows host, USN
|
||||
journal records, attribute decoding, and any correlation between the two.
|
||||
@@ -0,0 +1,11 @@
|
||||
# Tasks
|
||||
|
||||
- [x] Define the first-step scope, contract and limits.
|
||||
- [x] Parse FILE record headers and update-sequence fixups with synthesized
|
||||
fixtures and explicit limitations.
|
||||
- [x] Add the reader, format descriptor and byte classification.
|
||||
- [ ] Decode attribute records, then `$FILE_NAME` attributes with parent
|
||||
references, names, hard links and timestamps.
|
||||
- [ ] Parse `$UsnJrnl:$J` records and correlate them to records by full file
|
||||
reference.
|
||||
- [ ] Verify against real `$MFT` and journal captures from a Windows host.
|
||||
@@ -636,7 +636,7 @@ const PATH_SCOPES = {
|
||||
"src/ghidra/GhidraJavaProperties.ts",
|
||||
"src/ghidra/GhidraReleasePolicy.ts",
|
||||
"scripts/verify-ghidra-bridge-sources.mjs",
|
||||
"tests/conformance/ghidra/ReaArmContextProbe.java",
|
||||
"tests/conformance/ghidra/ReaInstructionContextProbe.java",
|
||||
"tests/conformance/ghidra/ReaComputedCallCoverageProbe.java",
|
||||
"tests/conformance/ghidra/ReaDecodeProbe.java",
|
||||
".github/workflows/ghidra-bridge-sources.yml",
|
||||
|
||||
@@ -50,7 +50,7 @@ transport.stderr?.on("data", (bytes) => {
|
||||
const ajv = new Ajv2020({ strict: false, validateFormats: false });
|
||||
const measurements = [];
|
||||
const contract = toolContract("inspect_web_network_capture");
|
||||
await withVerifierCleanup(async () => {
|
||||
const report = await withVerifierCleanup(async () => {
|
||||
try {
|
||||
await client.connect(transport);
|
||||
} catch (cause) {
|
||||
@@ -196,7 +196,7 @@ await withVerifierCleanup(async () => {
|
||||
});
|
||||
}
|
||||
await client.ping();
|
||||
const report = {
|
||||
return {
|
||||
status: "passed",
|
||||
profile: "aioquic-1.5.0-json-0.3",
|
||||
measurements,
|
||||
@@ -206,17 +206,6 @@ await withVerifierCleanup(async () => {
|
||||
"No PCAP or keylog is read by this verifier; repeated STREAM coverage is not proof of retransmission or network loss.",
|
||||
],
|
||||
};
|
||||
await writeFile(reportPath, JSON.stringify(report, null, 2) + "\n", {
|
||||
flag: "wx",
|
||||
mode: 0o600,
|
||||
});
|
||||
console.log(
|
||||
JSON.stringify({
|
||||
status: report.status,
|
||||
cases: measurements.length,
|
||||
report: reportPath,
|
||||
}),
|
||||
);
|
||||
}, [
|
||||
async () => {
|
||||
await closeVerifierResources([
|
||||
@@ -225,3 +214,15 @@ await withVerifierCleanup(async () => {
|
||||
]);
|
||||
},
|
||||
]);
|
||||
|
||||
await writeFile(reportPath, JSON.stringify(report, null, 2) + "\n", {
|
||||
flag: "wx",
|
||||
mode: 0o600,
|
||||
});
|
||||
console.log(
|
||||
JSON.stringify({
|
||||
status: report.status,
|
||||
cases: measurements.length,
|
||||
report: reportPath,
|
||||
}),
|
||||
);
|
||||
@@ -97,6 +97,7 @@ import {
|
||||
isDfuSeFile,
|
||||
} from "../domain/firmware/dfuse.js";
|
||||
import { hasFitImagesNode } from "../domain/firmware/fitImage.js";
|
||||
import { hasMftFileRecordHeader } from "../domain/windows/ntfsMft.js";
|
||||
import {
|
||||
locateEspFlashLayout,
|
||||
type EspFlashSelection,
|
||||
@@ -816,6 +817,7 @@ const detectArtifactFormat = async (
|
||||
});
|
||||
if (esp === "esp-image") return "esp-image";
|
||||
if (hasFitImagesNode(magic)) return "fit";
|
||||
if (hasMftFileRecordHeader(magic)) return "ntfs-mft";
|
||||
// Binary plists, including keyed archives, often use other extensions.
|
||||
if (hasBinaryPlistSignature(magic)) return "plist";
|
||||
const named = namedArtifactFormat(lower);
|
||||
@@ -899,6 +901,7 @@ const isArchiveFormat = (
|
||||
"dfuse",
|
||||
"esp-flash-image",
|
||||
"fit",
|
||||
"ntfs-mft",
|
||||
].includes(format);
|
||||
|
||||
/** Whether the leading bytes begin any executable header family, valid or not. */
|
||||
|
||||
@@ -135,6 +135,7 @@ const TARGETS: TargetFixtures = {
|
||||
espFlash: { selected: true, flashOffset: null, partitionTableOffset: null },
|
||||
},
|
||||
fit: { ...archive, format: "fit" },
|
||||
"ntfs-mft": { ...archive, format: "ntfs-mft" },
|
||||
plist: { ...artifact, format: "plist" },
|
||||
javascript: { ...artifact, format: "javascript" },
|
||||
"source-map": { ...artifact, format: "source-map" },
|
||||
|
||||
@@ -14,17 +14,22 @@ import { join } from "node:path";
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
|
||||
import { createTestTempDirectory } from "../../../tests/fixtures/temporaryDirectory.js";
|
||||
import { AnalysisCleanupLease } from "../../artifacts/AnalysisCleanupLease.js";
|
||||
import {
|
||||
desktopApplicationCaptureSchema,
|
||||
desktopCaptureEventSchema,
|
||||
} from "../../domain/desktop/desktopCapture.js";
|
||||
import type { SessionRequest } from "../../desktop/DesktopApplicationSession.js";
|
||||
import * as desktopSession from "../../desktop/DesktopApplicationSession.js";
|
||||
import * as linuxInventory from "../../desktop/LinuxApplicationInventory.js";
|
||||
import { X11WindowObserver } from "../../desktop/X11WindowObserver.js";
|
||||
import { OwnedFileHandle } from "../../filesystem/OwnedFileHandle.js";
|
||||
import { CaptureFile } from "./DesktopCaptureOutput.js";
|
||||
import {
|
||||
captureDesktopApplication,
|
||||
type DesktopCaptureRequest,
|
||||
} from "./DesktopApplicationCapture.js";
|
||||
import { capture as captureFixture } from "./desktopCapture.fixture.js";
|
||||
|
||||
const sha256 = (text: string) =>
|
||||
createHash("sha256").update(text).digest("hex");
|
||||
@@ -170,6 +175,15 @@ describe("captureDesktopApplication refusals", () => {
|
||||
await captureDesktopApplication(request(output), host(environment)),
|
||||
),
|
||||
).toContain("is not empty");
|
||||
const file = join(root, "existing-file");
|
||||
await writeFile(file, "keep");
|
||||
expect(
|
||||
await captureDesktopApplication(request(file), host(environment)),
|
||||
).toMatchObject({
|
||||
code: "invalid_request",
|
||||
details: { issues: [{ path: ["output"], reason: "invalid_value" }] },
|
||||
});
|
||||
expect(await readFile(file, "utf8")).toBe("keep");
|
||||
});
|
||||
|
||||
it("refuses a window manager that does not publish its client list", async () => {
|
||||
@@ -226,6 +240,56 @@ describe("captureDesktopApplication refusals", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it.each([false, true])(
|
||||
"preserves unresolved session ownership when publication fails (unreleased: %s)",
|
||||
async (unreleased) => {
|
||||
const { root, home, environment } = await fixture();
|
||||
const output = join(root, "out");
|
||||
const captured = captureFixture();
|
||||
const lease = new AnalysisCleanupLease(async () => {}, ["run-pending"]);
|
||||
vi.spyOn(linuxInventory, "inspectLinuxApplication").mockResolvedValue({
|
||||
ok: true,
|
||||
value: {
|
||||
inventory: { ...captured.application, state_directories: [] },
|
||||
limitations: [],
|
||||
signingKeys: new Map(),
|
||||
},
|
||||
});
|
||||
vi.spyOn(X11WindowObserver.prototype, "sample").mockResolvedValue({
|
||||
state: "listed",
|
||||
windows: [],
|
||||
});
|
||||
vi.spyOn(desktopSession, "runDesktopSession").mockResolvedValue({
|
||||
session: {
|
||||
...captured.session,
|
||||
cleanup: { verified: !unreleased, reason: unreleased ? "busy" : null },
|
||||
},
|
||||
unreleased: unreleased ? { reason: "busy", leases: [lease] } : undefined,
|
||||
unreadableWindowSamples: 0,
|
||||
});
|
||||
vi.spyOn(CaptureFile.prototype, "publish").mockRejectedValue(
|
||||
new Error("publication denied"),
|
||||
);
|
||||
try {
|
||||
const result = await captureDesktopApplication(request(output), {
|
||||
...host(environment),
|
||||
callerDirectory: home,
|
||||
});
|
||||
expect(result).toMatchObject({
|
||||
code: unreleased ? "cleanup_incomplete" : "execution_failure",
|
||||
message: expect.stringContaining("publication denied"),
|
||||
...(unreleased
|
||||
? { details: { cleanup: "incomplete", resources: ["run-pending"] } }
|
||||
: {}),
|
||||
});
|
||||
expect(await readdir(output)).toEqual([]);
|
||||
} finally {
|
||||
vi.restoreAllMocks();
|
||||
await lease.release();
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
it.skipIf(process.platform !== "linux")(
|
||||
"returns the published evidence when cancellation arrives during publication",
|
||||
async () => {
|
||||
|
||||
@@ -19,11 +19,13 @@ import {
|
||||
import { createEvidence, type Evidence } from "../../domain/evidence.js";
|
||||
import { DesktopContentStore } from "../../desktop/DesktopContentStore.js";
|
||||
import {
|
||||
DesktopSessionCleanupError,
|
||||
runDesktopSession,
|
||||
runningApplicationWindows,
|
||||
type SessionRequest,
|
||||
} from "../../desktop/DesktopApplicationSession.js";
|
||||
import { inspectLinuxApplication } from "../../desktop/LinuxApplicationInventory.js";
|
||||
import { canonicalCallerPath } from "../../filesystem/canonicalCallerPath.js";
|
||||
import { X11WindowObserver } from "../../desktop/X11WindowObserver.js";
|
||||
import {
|
||||
discardFailedCapture,
|
||||
@@ -31,11 +33,7 @@ import {
|
||||
projectFailure,
|
||||
publishedCaptureCleanup,
|
||||
} from "./DesktopCaptureFailure.js";
|
||||
import {
|
||||
CaptureFile,
|
||||
canonicalPath,
|
||||
prepareOutputDirectory,
|
||||
} from "./DesktopCaptureOutput.js";
|
||||
import { CaptureFile, prepareOutputDirectory } from "./DesktopCaptureOutput.js";
|
||||
import {
|
||||
DESKTOP_OPERATION,
|
||||
DESKTOP_PROVIDER,
|
||||
@@ -232,7 +230,7 @@ const admit = async (
|
||||
throw outputRefusal(request.output, prepared.refusal);
|
||||
return {
|
||||
display,
|
||||
home: await canonicalPath(home),
|
||||
home: await canonicalCallerPath(home),
|
||||
output: prepared.path,
|
||||
inventory,
|
||||
limitations,
|
||||
@@ -244,7 +242,7 @@ const admit = async (
|
||||
|
||||
type Admitted = Awaited<ReturnType<typeof admit>>;
|
||||
|
||||
/** Record the baseline, run the session and publish `capture.json`. */
|
||||
/** Record the baseline, run the session and assemble its Evidence. */
|
||||
const recordCapture = async (
|
||||
request: DesktopCaptureRequest,
|
||||
host: DesktopCaptureHost,
|
||||
@@ -252,7 +250,6 @@ const recordCapture = async (
|
||||
readonly admitted: Admitted;
|
||||
readonly scope: CaptureScope;
|
||||
readonly store: DesktopContentStore;
|
||||
readonly captureFile: CaptureFile;
|
||||
},
|
||||
signal: AbortSignal | undefined,
|
||||
) => {
|
||||
@@ -341,8 +338,7 @@ const recordCapture = async (
|
||||
],
|
||||
};
|
||||
const evidence = captureEvidence(request, capture);
|
||||
const published = await context.captureFile.publish(evidence);
|
||||
return { evidence, published, session, unreleased };
|
||||
return { evidence, session, unreleased };
|
||||
};
|
||||
|
||||
const runCapture = async (
|
||||
@@ -360,22 +356,26 @@ const runCapture = async (
|
||||
);
|
||||
await store.prepare();
|
||||
const captureFile = new CaptureFile(output);
|
||||
let recorded: Awaited<ReturnType<typeof recordCapture>>;
|
||||
let recorded: Awaited<ReturnType<typeof recordCapture>> | undefined;
|
||||
let published: string;
|
||||
try {
|
||||
recorded = await recordCapture(
|
||||
request,
|
||||
host,
|
||||
{ admitted, scope, store, captureFile },
|
||||
{ admitted, scope, store },
|
||||
signal,
|
||||
);
|
||||
published = await captureFile.publish(recorded.evidence);
|
||||
} catch (cause: unknown) {
|
||||
// Nothing was published, so a failure or cancellation removes what was written.
|
||||
throw failedCaptureError(
|
||||
cause,
|
||||
recorded?.unreleased === undefined
|
||||
? cause
|
||||
: new DesktopSessionCleanupError(cause, recorded.unreleased),
|
||||
await discardFailedCapture(store, captureFile),
|
||||
);
|
||||
}
|
||||
const { evidence, published, session, unreleased } = recorded;
|
||||
const { evidence, session, unreleased } = recorded;
|
||||
const cleanup = await publishedCaptureCleanup(published, store, unreleased);
|
||||
if (cleanup !== undefined) throw cleanup;
|
||||
// A cancellation that arrives during publication must not read as success.
|
||||
|
||||
@@ -9,6 +9,11 @@ import { z } from "zod";
|
||||
|
||||
import { createTestTempDirectory } from "../../../tests/fixtures/temporaryDirectory.js";
|
||||
import { SafeOutputTree } from "../../artifacts/SafeOutputTree.js";
|
||||
import { ArtifactResourceScope } from "../../artifacts/ArtifactResourceScope.js";
|
||||
import type { AnalysisCleanupLease } from "../../artifacts/AnalysisCleanupLease.js";
|
||||
import { AnalysisError } from "../../domain/analysisErrorBase.js";
|
||||
import { OwnedFileHandle } from "../../filesystem/OwnedFileHandle.js";
|
||||
import { RetainedContent } from "./DesktopRetainedContent.js";
|
||||
import {
|
||||
VARICAD_FILE,
|
||||
VARICAD_URL,
|
||||
@@ -2217,6 +2222,114 @@ describe("exportDesktopApplication launch document beside its lock", () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe("exportDesktopApplication retained input finalization", () => {
|
||||
it("honors cancellation during retained input finalization before publication", async () => {
|
||||
const controller = new AbortController();
|
||||
const reason = new Error("cancel during seed finalization");
|
||||
let writing = false;
|
||||
let finalized = false;
|
||||
const write = unspied("write");
|
||||
vi.spyOn(SafeOutputTree.prototype, "write").mockImplementation(
|
||||
async function (this: SafeOutputTree, ...args) {
|
||||
const written = await write.apply(this, args);
|
||||
writing = true;
|
||||
return written;
|
||||
},
|
||||
);
|
||||
const release = RetainedContent.prototype.release;
|
||||
vi.spyOn(RetainedContent.prototype, "release").mockImplementation(
|
||||
async function (this: RetainedContent, resources) {
|
||||
const result = await release.call(this, resources);
|
||||
if (writing) {
|
||||
finalized = true;
|
||||
controller.abort(reason);
|
||||
}
|
||||
return result;
|
||||
},
|
||||
);
|
||||
const { result, directory } = await exportCapture(
|
||||
capture(),
|
||||
controller.signal,
|
||||
);
|
||||
expect(finalized).toBe(true);
|
||||
expect(result).toMatchObject({ _tag: "AnalysisCancelledError" });
|
||||
expect((result as Error).cause).toBe(reason);
|
||||
expect(await exists(directory)).toBe(false);
|
||||
});
|
||||
|
||||
it.each([false, true])(
|
||||
"finalizes retained seed inputs before publication and retains cleanup retry (rollback fails: %s)",
|
||||
async (rollbackFails) => {
|
||||
let writing = false;
|
||||
let reading = false;
|
||||
const blocked = new Set<OwnedFileHandle>();
|
||||
const leases: AnalysisCleanupLease[] = [];
|
||||
if (rollbackFails)
|
||||
vi.spyOn(
|
||||
SafeOutputTree.prototype,
|
||||
"closeHandles",
|
||||
).mockRejectedValueOnce(new Error("output close failed"));
|
||||
const write = unspied("write");
|
||||
vi.spyOn(SafeOutputTree.prototype, "write").mockImplementation(
|
||||
async function (this: SafeOutputTree, ...args) {
|
||||
const written = await write.apply(this, args);
|
||||
writing = true;
|
||||
return written;
|
||||
},
|
||||
);
|
||||
const read = RetainedContent.prototype.read;
|
||||
vi.spyOn(RetainedContent.prototype, "read").mockImplementation(
|
||||
async function (this: RetainedContent, ...args) {
|
||||
reading = writing;
|
||||
try {
|
||||
return await read.apply(this, args);
|
||||
} finally {
|
||||
reading = false;
|
||||
}
|
||||
},
|
||||
);
|
||||
const close = OwnedFileHandle.prototype.close;
|
||||
const closing = vi
|
||||
.spyOn(OwnedFileHandle.prototype, "close")
|
||||
.mockImplementation(function (this: OwnedFileHandle) {
|
||||
if (reading && blocked.size === 0) blocked.add(this);
|
||||
return blocked.has(this)
|
||||
? Promise.reject(new Error("retained seed close failed"))
|
||||
: close.call(this);
|
||||
});
|
||||
const release = ArtifactResourceScope.prototype.release;
|
||||
vi.spyOn(ArtifactResourceScope.prototype, "release").mockImplementation(
|
||||
async function (this: ArtifactResourceScope, owner) {
|
||||
const result = await release.call(this, owner);
|
||||
if (result.kind === "failed") leases.push(result.lease);
|
||||
return result;
|
||||
},
|
||||
);
|
||||
try {
|
||||
const { result, directory } = await exportCapture(capture());
|
||||
expect(blocked.size).toBe(1);
|
||||
expect(result).toBeInstanceOf(AnalysisError);
|
||||
if (!(result instanceof AnalysisError))
|
||||
throw new Error("expected cleanup failure");
|
||||
expect(result.cleanupIncomplete).toBe(true);
|
||||
expect(result.cleanupResources).toHaveLength(rollbackFails ? 2 : 1);
|
||||
if (rollbackFails) expect(result.cleanupResources).toContain(directory);
|
||||
expect(await exists(directory)).toBe(rollbackFails);
|
||||
expect(leases).toHaveLength(rollbackFails ? 2 : 1);
|
||||
closing.mockRestore();
|
||||
await leases[0]!.release();
|
||||
expect(result.cleanupIncomplete).toBe(rollbackFails);
|
||||
for (const lease of leases.slice(1)) await lease.release();
|
||||
expect(result.cleanupIncomplete).toBe(false);
|
||||
expect(await exists(directory)).toBe(false);
|
||||
} finally {
|
||||
closing.mockRestore();
|
||||
for (const lease of leases) await lease.release();
|
||||
}
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
describe("exportDesktopApplication bundle writes", () => {
|
||||
it.each(["cancelled", "out of space"])(
|
||||
"removes the unpublished bundle when its write fails (%s)",
|
||||
|
||||
@@ -403,19 +403,19 @@ const runExport = async (
|
||||
const text = bundleText(plan, admitted, dconf.holdsIdentity, held.consent);
|
||||
const { generated, readme } = text;
|
||||
let scrub = NO_SCRUB;
|
||||
const unclosed = await withRetainedContent(
|
||||
request.capture,
|
||||
(content) =>
|
||||
writeOutputTree(
|
||||
request.directory,
|
||||
async (tree) => {
|
||||
const unclosed = await writeOutputTree(
|
||||
copy.bundle,
|
||||
(tree) =>
|
||||
withRetainedContent(
|
||||
request.capture,
|
||||
async (content) => {
|
||||
const bundle = { generated, copies, identity, readme };
|
||||
scrub = await writeFiles(tree, content, bundle, signal);
|
||||
},
|
||||
{ operation: OPERATION, field: "directory" },
|
||||
signal,
|
||||
SEED_MAX_BYTES,
|
||||
),
|
||||
SEED_MAX_BYTES,
|
||||
{ operation: OPERATION, field: "directory" },
|
||||
signal,
|
||||
);
|
||||
const limitations = exportLimitations(capture, plan, {
|
||||
install: install.limitations,
|
||||
|
||||
@@ -1,10 +1,17 @@
|
||||
import { readFile } from "node:fs/promises";
|
||||
import { readFile, writeFile } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
import { DatabaseSync } from "node:sqlite";
|
||||
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
|
||||
import { AnalysisCancelledError } from "../../domain/analysisErrorCore.js";
|
||||
import {
|
||||
AnalysisError,
|
||||
primaryAnalysisError,
|
||||
} from "../../domain/analysisErrorBase.js";
|
||||
import { EvidenceFileError } from "../../domain/evidenceErrors.js";
|
||||
import { ArtifactResourceScope } from "../../artifacts/ArtifactResourceScope.js";
|
||||
import { OwnedFileHandle } from "../../filesystem/OwnedFileHandle.js";
|
||||
import type {
|
||||
DesktopApplicationCapture,
|
||||
DesktopCheckpoint,
|
||||
@@ -12,6 +19,7 @@ import type {
|
||||
import { desktopApplicationStateSchema } from "../../domain/desktop/desktopState.js";
|
||||
import {
|
||||
inspectDesktopApplicationState,
|
||||
readDesktopCapture,
|
||||
type DesktopStateReaders,
|
||||
} from "./DesktopApplicationState.js";
|
||||
import { draftDesktopApplicationTask } from "./DesktopTaskDraft.js";
|
||||
@@ -27,6 +35,49 @@ import {
|
||||
writeCaptureFolder,
|
||||
} from "./desktopCapture.fixture.js";
|
||||
|
||||
it("retains invalid capture JSON and the descriptor owner through failed final cleanup", async () => {
|
||||
const root = await createTestTempDirectory("rea-capture-read-cleanup-");
|
||||
await writeFile(join(root, "capture.json"), '{"broken":');
|
||||
await expect(readDesktopCapture(root, undefined)).rejects.toMatchObject({
|
||||
_tag: "EvidenceFileError",
|
||||
reason: "invalid-json",
|
||||
});
|
||||
const scopes = new Set<ArtifactResourceScope>();
|
||||
const originalClose = ArtifactResourceScope.prototype.close;
|
||||
const scopeClose = vi
|
||||
.spyOn(ArtifactResourceScope.prototype, "close")
|
||||
.mockImplementation(function (this: ArtifactResourceScope) {
|
||||
scopes.add(this);
|
||||
return originalClose.call(this);
|
||||
});
|
||||
const fileClose = vi
|
||||
.spyOn(OwnedFileHandle.prototype, "close")
|
||||
.mockRejectedValue(new Error("descriptor close remains unavailable"));
|
||||
let failure: AnalysisError | undefined;
|
||||
try {
|
||||
const reported = await readDesktopCapture(root, undefined).catch(
|
||||
(cause: unknown) => cause,
|
||||
);
|
||||
expect(reported).toBeInstanceOf(AnalysisError);
|
||||
if (!(reported instanceof AnalysisError)) throw reported;
|
||||
failure = reported;
|
||||
expect(primaryAnalysisError(failure)).toBeInstanceOf(EvidenceFileError);
|
||||
expect(primaryAnalysisError(failure)).toMatchObject({
|
||||
reason: "invalid-json",
|
||||
});
|
||||
expect(failure.cleanupIncomplete).toBe(true);
|
||||
expect(failure.cleanupResources).toEqual([join(root, "capture.json")]);
|
||||
} finally {
|
||||
fileClose.mockRestore();
|
||||
scopeClose.mockRestore();
|
||||
for (const scope of scopes) await scope.close();
|
||||
}
|
||||
expect(failure?.cleanupIncomplete).toBe(false);
|
||||
expect(primaryAnalysisError(failure!)).toMatchObject({
|
||||
reason: "invalid-json",
|
||||
});
|
||||
});
|
||||
|
||||
/** The fixture capture with extra files at the baseline and the second checkpoint. */
|
||||
const extended = (extra: {
|
||||
readonly baseline?: DesktopCheckpoint["retained"];
|
||||
|
||||
@@ -10,6 +10,7 @@ import { join } from "node:path";
|
||||
import { ArtifactResourceScope } from "../../artifacts/ArtifactResourceScope.js";
|
||||
import { AnalysisError } from "../../domain/analysisErrorBase.js";
|
||||
import { AnalysisInputError } from "../../domain/analysisErrorCore.js";
|
||||
import { analysisErrorWithCleanupFailure } from "../../domain/analysisErrorCleanup.js";
|
||||
import { analysisInputErrorFromIssues } from "../../domain/inputIssueProjection.js";
|
||||
import {
|
||||
desktopApplicationCaptureSchema,
|
||||
@@ -92,44 +93,69 @@ export const readDesktopCapture = async (
|
||||
signal: AbortSignal | undefined,
|
||||
): Promise<{ evidence: Evidence; capture: DesktopApplicationCapture }> => {
|
||||
const resources = new ArtifactResourceScope();
|
||||
const outcome = await resources
|
||||
.run(async () => {
|
||||
const loaded = await readJsonFile(
|
||||
join(folder, "capture.json"),
|
||||
signal,
|
||||
resources,
|
||||
OPERATION,
|
||||
);
|
||||
if (!loaded.ok) {
|
||||
if (loaded.error instanceof AnalysisError) throw loaded.error;
|
||||
throw inputError(
|
||||
`${join(folder, "capture.json")} could not be read as JSON.`,
|
||||
"Pass a folder written by capture-desktop-application.",
|
||||
);
|
||||
}
|
||||
const evidence = safeParseEvidence(loaded.value);
|
||||
if (
|
||||
!evidence.ok ||
|
||||
evidence.value.operation !== DESKTOP_OPERATION ||
|
||||
evidence.value.predicate_type !== "rea.desktop-application-capture" ||
|
||||
evidence.value.provider.id !== DESKTOP_PROVIDER.id ||
|
||||
evidence.value.provider.version !== DESKTOP_PROVIDER.version
|
||||
)
|
||||
throw inputError(
|
||||
"capture.json is not desktop application capture Evidence.",
|
||||
"Pass a folder written by capture-desktop-application.",
|
||||
);
|
||||
const capture = desktopApplicationCaptureSchema.safeParse(
|
||||
evidence.value.normalized_result,
|
||||
);
|
||||
if (!capture.success)
|
||||
throw inputError(
|
||||
`The capture result is malformed (${describeValidationFailure(capture.error)}).`,
|
||||
"Capture the application again.",
|
||||
);
|
||||
return { evidence: evidence.value, capture: capture.data };
|
||||
})
|
||||
.then(ok, (cause: unknown) => err(cause));
|
||||
try {
|
||||
const loaded = await readJsonFile(
|
||||
join(folder, "capture.json"),
|
||||
signal,
|
||||
resources,
|
||||
OPERATION,
|
||||
);
|
||||
if (!loaded.ok) {
|
||||
if (loaded.error instanceof AnalysisError) throw loaded.error;
|
||||
throw inputError(
|
||||
`${join(folder, "capture.json")} could not be read as JSON.`,
|
||||
"Pass a folder written by capture-desktop-application.",
|
||||
);
|
||||
}
|
||||
const evidence = safeParseEvidence(loaded.value);
|
||||
if (
|
||||
!evidence.ok ||
|
||||
evidence.value.operation !== DESKTOP_OPERATION ||
|
||||
evidence.value.predicate_type !== "rea.desktop-application-capture" ||
|
||||
evidence.value.provider.id !== DESKTOP_PROVIDER.id ||
|
||||
evidence.value.provider.version !== DESKTOP_PROVIDER.version
|
||||
)
|
||||
throw inputError(
|
||||
"capture.json is not desktop application capture Evidence.",
|
||||
"Pass a folder written by capture-desktop-application.",
|
||||
);
|
||||
const capture = desktopApplicationCaptureSchema.safeParse(
|
||||
evidence.value.normalized_result,
|
||||
);
|
||||
if (!capture.success)
|
||||
throw inputError(
|
||||
`The capture result is malformed (${describeValidationFailure(capture.error)}).`,
|
||||
"Capture the application again.",
|
||||
);
|
||||
return { evidence: evidence.value, capture: capture.data };
|
||||
} finally {
|
||||
await resources.close();
|
||||
} catch (cause: unknown) {
|
||||
const cleanup = desktopOperationFailure(
|
||||
cause,
|
||||
signal,
|
||||
OPERATION,
|
||||
"Desktop capture input",
|
||||
);
|
||||
if (outcome.ok)
|
||||
throw cleanup.retainPartialObservation(outcome.value.evidence);
|
||||
const primary = desktopOperationFailure(
|
||||
outcome.error,
|
||||
signal,
|
||||
OPERATION,
|
||||
"Desktop capture input",
|
||||
);
|
||||
throw analysisErrorWithCleanupFailure(
|
||||
primary,
|
||||
cleanup,
|
||||
OPERATION,
|
||||
).bindCleanupDependencies([primary, cleanup]);
|
||||
}
|
||||
if (!outcome.ok) throw outcome.error;
|
||||
return outcome.value;
|
||||
};
|
||||
|
||||
interface ViewContext {
|
||||
|
||||
@@ -46,6 +46,7 @@ import {
|
||||
WORK,
|
||||
} from "./DesktopBuildCommands.js";
|
||||
import {
|
||||
planOutputTree,
|
||||
writeOutputTree,
|
||||
writtenWithOpenHandles,
|
||||
} from "./DesktopOutputTree.js";
|
||||
@@ -183,7 +184,7 @@ const writeBundle = async (
|
||||
unchecked: boolean;
|
||||
}[] = [];
|
||||
const unclosed = await writeOutputTree(
|
||||
directory,
|
||||
await planOutputTree(directory, { operation: EXPORT, field: "directory" }),
|
||||
async (tree) => {
|
||||
fetched = await fetchHomeFolder(
|
||||
computer,
|
||||
|
||||
@@ -28,6 +28,7 @@ import {
|
||||
import { errorText } from "./DesktopBuildCommands.js";
|
||||
import { DESKTOP_PROVIDER } from "./DesktopCaptureRefusals.js";
|
||||
import {
|
||||
planOutputTree,
|
||||
writeOutputTree,
|
||||
writtenWithOpenHandles,
|
||||
} from "./DesktopOutputTree.js";
|
||||
@@ -153,7 +154,7 @@ const writeRecordFolder = async (
|
||||
> => {
|
||||
try {
|
||||
const unclosed = await writeOutputTree(
|
||||
directory,
|
||||
await planOutputTree(directory, { operation: CHECK, field: "directory" }),
|
||||
async (tree) => {
|
||||
for (const file of files)
|
||||
await tree.write(
|
||||
|
||||
@@ -13,6 +13,7 @@ import { DesktopSessionCleanupError } from "../../desktop/DesktopApplicationSess
|
||||
import { DesktopContentStore } from "../../desktop/DesktopContentStore.js";
|
||||
import { walkHomeState } from "../../desktop/HomeStateWalker.js";
|
||||
import { OwnedFileHandle } from "../../filesystem/OwnedFileHandle.js";
|
||||
import { captureDesktopApplication } from "./DesktopApplicationCapture.js";
|
||||
import { CaptureFile } from "./DesktopCaptureOutput.js";
|
||||
import {
|
||||
discardFailedCapture,
|
||||
@@ -236,6 +237,77 @@ describe("failedCaptureError", () => {
|
||||
});
|
||||
|
||||
describe("projectFailure", () => {
|
||||
it.each([
|
||||
{ oversized: false, category: "cancelled" },
|
||||
{ oversized: true, category: "execution_failure" },
|
||||
])(
|
||||
"preserves the physical read outcome beside final cancellation and cleanup (oversized: $oversized)",
|
||||
async ({ oversized, category }) => {
|
||||
const root = await createTestTempDirectory("desktop-inventory-cancel-");
|
||||
const entry = join(root, "notes.desktop");
|
||||
await writeFile(
|
||||
entry,
|
||||
oversized
|
||||
? "x".repeat(1024 * 1024 + 1)
|
||||
: "[Desktop Entry]\nType=Application\nName=Notes\nExec=notes\n",
|
||||
);
|
||||
const controller = new AbortController();
|
||||
const reason = new Error("cancelled while closing the desktop entry");
|
||||
let owner: OwnedFileHandle | undefined;
|
||||
const close = vi
|
||||
.spyOn(OwnedFileHandle.prototype, "close")
|
||||
.mockImplementation(function (this: OwnedFileHandle) {
|
||||
// oxlint-disable-next-line typescript/no-this-alias -- Release the exact denied descriptor after the public call.
|
||||
owner = this;
|
||||
controller.abort(reason);
|
||||
return Promise.reject(new Error("close denied"));
|
||||
});
|
||||
try {
|
||||
const result = await captureDesktopApplication(
|
||||
{
|
||||
desktopEntry: entry,
|
||||
executable: undefined,
|
||||
arguments: [],
|
||||
output: join(root, "capture"),
|
||||
statePaths: [],
|
||||
excludes: [],
|
||||
maxDurationSeconds: 1,
|
||||
maxEntries: 10,
|
||||
maxFileBytes: 1024,
|
||||
maxContentBytes: 1024,
|
||||
packageFile: undefined,
|
||||
},
|
||||
{
|
||||
platform: "linux",
|
||||
callerDirectory: root,
|
||||
environment: {
|
||||
HOME: root,
|
||||
DISPLAY: ":controlled",
|
||||
REA_XPROP_COMMAND: process.execPath,
|
||||
},
|
||||
nextRequest: () => undefined,
|
||||
report: async () => {
|
||||
throw new Error(
|
||||
"inventory failure must precede target execution",
|
||||
);
|
||||
},
|
||||
},
|
||||
controller.signal,
|
||||
);
|
||||
expect(owner).toBeDefined();
|
||||
expect(result).toMatchObject({
|
||||
error: "Desktop capture failed",
|
||||
code: "cleanup_incomplete",
|
||||
category,
|
||||
details: { resources: [entry] },
|
||||
});
|
||||
} finally {
|
||||
close.mockRestore();
|
||||
await owner?.close();
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
it("keeps the owner of a resource the inventory could not release", async () => {
|
||||
const resources = new ArtifactResourceScope();
|
||||
let attempts = 0;
|
||||
|
||||
@@ -14,10 +14,23 @@ import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { createTestTempDirectory } from "../../../tests/fixtures/temporaryDirectory.js";
|
||||
import type { Evidence } from "../../domain/evidence.js";
|
||||
import { OwnedFileHandle } from "../../filesystem/OwnedFileHandle.js";
|
||||
import { CaptureFile } from "./DesktopCaptureOutput.js";
|
||||
import { CaptureFile, prepareOutputDirectory } from "./DesktopCaptureOutput.js";
|
||||
|
||||
const evidence = { id: "evidence" } as unknown as Evidence;
|
||||
|
||||
it.each(["", "/child"])(
|
||||
"refuses an output path blocked by a regular file (%s)",
|
||||
async (suffix) => {
|
||||
const root = await createTestTempDirectory("desktop-output-file-");
|
||||
const file = join(root, "existing");
|
||||
await writeFile(file, "keep");
|
||||
await expect(prepareOutputDirectory(`${file}${suffix}`)).resolves.toEqual({
|
||||
refusal: "not_a_directory",
|
||||
});
|
||||
expect(await readFile(file, "utf8")).toBe("keep");
|
||||
},
|
||||
);
|
||||
|
||||
describe("CaptureFile", () => {
|
||||
it("publishes through its partial file and leaves nothing to discard", async () => {
|
||||
const output = await createTestTempDirectory("desktop-output-");
|
||||
|
||||
@@ -8,31 +8,12 @@ import {
|
||||
rename,
|
||||
rm,
|
||||
} from "node:fs/promises";
|
||||
import { basename, dirname, join, resolve } from "node:path";
|
||||
import { join } from "node:path";
|
||||
|
||||
import { ArtifactResourceScope } from "../../artifacts/ArtifactResourceScope.js";
|
||||
import type { Evidence } from "../../domain/evidence.js";
|
||||
import { OwnedFileHandle } from "../../filesystem/OwnedFileHandle.js";
|
||||
|
||||
/**
|
||||
* The physical spelling of a path that may not exist yet: the real path of
|
||||
* its longest existing ancestor followed by the rest. Walks, exclusions and
|
||||
* state directories compare in this spelling, so a symbolic link in the
|
||||
* caller's spelling cannot make one directory look like two.
|
||||
*/
|
||||
export const canonicalPath = async (path: string): Promise<string> => {
|
||||
const absolute = resolve(path);
|
||||
const rest: string[] = [];
|
||||
for (let current = absolute; ; current = dirname(current)) {
|
||||
try {
|
||||
return join(await realpath(current), ...rest.reverse());
|
||||
} catch {
|
||||
if (dirname(current) === current) return absolute;
|
||||
rest.push(basename(current));
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
/** Why the output folder cannot hold a private capture. */
|
||||
export type OutputRefusal = "not_a_directory" | "foreign_owner" | "not_empty";
|
||||
|
||||
@@ -43,7 +24,14 @@ export type OutputRefusal = "not_a_directory" | "foreign_owner" | "not_empty";
|
||||
export const prepareOutputDirectory = async (
|
||||
output: string,
|
||||
): Promise<{ readonly path: string } | { readonly refusal: OutputRefusal }> => {
|
||||
await mkdir(output, { recursive: true, mode: 0o700 });
|
||||
try {
|
||||
await mkdir(output, { recursive: true, mode: 0o700 });
|
||||
} catch (cause: unknown) {
|
||||
const code = (cause as NodeJS.ErrnoException).code;
|
||||
if (code === "EEXIST" || code === "ENOTDIR")
|
||||
return { refusal: "not_a_directory" };
|
||||
throw cause;
|
||||
}
|
||||
const stats = await lstat(output);
|
||||
if (!stats.isDirectory()) return { refusal: "not_a_directory" };
|
||||
const uid = process.getuid?.();
|
||||
|
||||
@@ -2,7 +2,10 @@ import { readdir } from "node:fs/promises";
|
||||
import { isAbsolute } from "node:path";
|
||||
|
||||
import { ArtifactReaderFailure } from "../../artifacts/ArtifactReader.js";
|
||||
import { bindArtifactFileCleanup } from "../../artifacts/ArtifactFileOperation.js";
|
||||
import {
|
||||
artifactFileCancellationCause,
|
||||
bindArtifactFileCleanup,
|
||||
} from "../../artifacts/ArtifactFileOperation.js";
|
||||
import { AnalysisError } from "../../domain/analysisErrorBase.js";
|
||||
import {
|
||||
AnalysisCancelledError,
|
||||
@@ -193,6 +196,8 @@ export const checkDesktopCaptureHost = async (
|
||||
);
|
||||
const existing = await readdir(request.output).catch((cause: unknown) => {
|
||||
if ((cause as NodeJS.ErrnoException).code === "ENOENT") return [];
|
||||
if ((cause as NodeJS.ErrnoException).code === "ENOTDIR")
|
||||
throw outputRefusal(request.output, "not_a_directory");
|
||||
throw cause;
|
||||
});
|
||||
if (existing.length > 0) throw outputRefusal(request.output, "not_empty");
|
||||
@@ -232,8 +237,15 @@ export const desktopOperationFailure = (
|
||||
}
|
||||
// A resource that could not be released stays reported, with its owner.
|
||||
if (cause instanceof ArtifactReaderFailure && cause.cleanup !== undefined) {
|
||||
const primary =
|
||||
cause.reason === "cancelled"
|
||||
? new AnalysisCancelledError(operation, {
|
||||
cause: artifactFileCancellationCause(cause),
|
||||
})
|
||||
: undefined;
|
||||
const error = new ProviderAdapterError(DESKTOP_PROVIDER.id, operation, {
|
||||
cause,
|
||||
...(primary === undefined ? {} : { cleanupPrimary: primary }),
|
||||
cleanup: cause.cleanup,
|
||||
diagnostics: { phase: operation },
|
||||
userMessage: `${label} could not release a resource it used: ${describeValidationFailure(cause)}`,
|
||||
|
||||
@@ -141,6 +141,36 @@ describe("captureScope", () => {
|
||||
);
|
||||
expect(scope.contentRoots).not.toContain(linked);
|
||||
});
|
||||
|
||||
it("follows a selected symlink before its parent segment for capture and exclusion", async () => {
|
||||
const root = await realpath(
|
||||
await createTestTempDirectory("desktop-scope-parent-"),
|
||||
);
|
||||
const home = join(root, "home");
|
||||
await mkdir(home);
|
||||
await mkdir(join(root, "physical", "child"), { recursive: true });
|
||||
await mkdir(join(root, "named"));
|
||||
await symlink(join(root, "physical", "child"), join(root, "named", "link"));
|
||||
await writeFile(join(root, "physical", "selected"), "selected");
|
||||
await writeFile(join(root, "named", "selected"), "decoy");
|
||||
const selected = `${root}/named/link/../selected`;
|
||||
const { scope, selectedPaths } = await captureScope(
|
||||
{ ...request([]), statePaths: [selected], excludes: [selected] },
|
||||
inventory(),
|
||||
home,
|
||||
join(root, "out"),
|
||||
);
|
||||
expect(selectedPaths).toEqual([
|
||||
{
|
||||
path: selected,
|
||||
physical: join(root, "physical", "selected"),
|
||||
kind: "file",
|
||||
},
|
||||
]);
|
||||
expect(scope.contentRoots).toContain(join(root, "physical", "selected"));
|
||||
expect(scope.excluded).toContain(join(root, "physical", "selected"));
|
||||
expect(scope.excluded).not.toContain(join(root, "named", "selected"));
|
||||
});
|
||||
});
|
||||
|
||||
describe("applicationWindowMatching", () => {
|
||||
|
||||
@@ -1,11 +1,12 @@
|
||||
import { lstat } from "node:fs/promises";
|
||||
import { basename, isAbsolute, resolve } from "node:path";
|
||||
import { basename, isAbsolute } from "node:path";
|
||||
|
||||
import type { DesktopApplicationInventory } from "../../domain/desktop/desktopCapture.js";
|
||||
import { isWithinAny } from "../../domain/desktop/desktopStateDiff.js";
|
||||
import { SQLITE_SIDECARS } from "../../domain/desktop/stateFormats.js";
|
||||
import type { WindowMatching } from "../../desktop/DesktopSessionObservations.js";
|
||||
import { canonicalPath } from "./DesktopCaptureOutput.js";
|
||||
import { absoluteCallerPath } from "../../filesystem/callerPath.js";
|
||||
import { canonicalCallerPath } from "../../filesystem/canonicalCallerPath.js";
|
||||
import type { CheckpointScope } from "./DesktopCheckpointRecorder.js";
|
||||
|
||||
const unique = (paths: readonly string[]) => [...new Set(paths)];
|
||||
@@ -14,7 +15,10 @@ const unique = (paths: readonly string[]) => [...new Set(paths)];
|
||||
const pathKind = async (path: string): Promise<"file" | "directory"> =>
|
||||
(await lstat(path).then(
|
||||
(stats) => stats.isDirectory(),
|
||||
() => false,
|
||||
(cause: unknown) => {
|
||||
if ((cause as NodeJS.ErrnoException).code === "ENOENT") return false;
|
||||
throw cause;
|
||||
},
|
||||
))
|
||||
? "directory"
|
||||
: "file";
|
||||
@@ -55,12 +59,12 @@ export const captureScope = async (
|
||||
output: string,
|
||||
): Promise<CaptureScope> => {
|
||||
const canonical = (paths: readonly string[]) =>
|
||||
Promise.all(paths.map((path) => canonicalPath(path)));
|
||||
Promise.all(paths.map((path) => canonicalCallerPath(path)));
|
||||
const named = (paths: readonly string[]) =>
|
||||
Promise.all(
|
||||
paths.map(async (path) => ({
|
||||
path: resolve(path),
|
||||
physical: await canonicalPath(path),
|
||||
path: absoluteCallerPath(path),
|
||||
physical: await canonicalCallerPath(path),
|
||||
})),
|
||||
);
|
||||
const stateDirectories = await canonical(
|
||||
|
||||
@@ -164,7 +164,7 @@ const evidence = (result: Evidence | Error): Evidence => {
|
||||
return result;
|
||||
};
|
||||
|
||||
it("reports the published bundle when the withheld copy destination is occupied", async () => {
|
||||
it("refuses an occupied withheld copy destination before writing the bundle", async () => {
|
||||
const root = await createTestTempDirectory("desktop-copy-refusal-");
|
||||
const occupied = join(root, "occupied");
|
||||
await mkdir(occupied);
|
||||
@@ -173,21 +173,31 @@ it("reports the published bundle when the withheld copy destination is occupied"
|
||||
withhold: { paths: [SETTINGS] },
|
||||
withheldCopy: occupied,
|
||||
});
|
||||
expect(result).toMatchObject({
|
||||
partialObservation: {
|
||||
operation: "export_desktop_application",
|
||||
normalized_result: { directory },
|
||||
},
|
||||
});
|
||||
expect(await exists(join(directory, "README.md"))).toBe(true);
|
||||
expect(result).toBeInstanceOf(AnalysisError);
|
||||
expect(await exists(directory)).toBe(false);
|
||||
expect(await readFile(join(occupied, "keep"), "utf8")).toBe("existing");
|
||||
const occupiedFile = join(root, "file");
|
||||
await writeFile(occupiedFile, "existing file");
|
||||
const file = await exportWithheld({
|
||||
withhold: { paths: [SETTINGS] },
|
||||
withheldCopy: occupiedFile,
|
||||
});
|
||||
expect(file.result).toBeInstanceOf(AnalysisError);
|
||||
expect(await exists(file.directory)).toBe(false);
|
||||
expect(await readFile(occupiedFile, "utf8")).toBe("existing file");
|
||||
});
|
||||
|
||||
it("retains the bundle cleanup owner when a later withheld copy fails", async () => {
|
||||
const root = await createTestTempDirectory("desktop-copy-close-");
|
||||
const occupied = join(root, "occupied");
|
||||
await mkdir(occupied);
|
||||
await writeFile(join(occupied, "keep"), "existing");
|
||||
const commit = SafeOutputTree.prototype.commit;
|
||||
vi.spyOn(SafeOutputTree.prototype, "commit").mockImplementationOnce(
|
||||
async function (this: SafeOutputTree, signal) {
|
||||
await commit.call(this, signal);
|
||||
await writeFile(join(occupied, "keep"), "existing");
|
||||
},
|
||||
);
|
||||
const original = SafeOutputTree.prototype.closeHandles;
|
||||
const release = ArtifactResourceScope.prototype.release;
|
||||
const failures: Extract<ArtifactCleanupAttempt, { kind: "failed" }>[] = [];
|
||||
@@ -562,7 +572,9 @@ describe("exportDesktopApplication of withheld paths and copies", () => {
|
||||
).toContain("49152");
|
||||
expect(await exists(directory)).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe("exportDesktopApplication withheld copy destinations", () => {
|
||||
it("refuses a withheld copy inside the bundle, also through a link, and writes neither", async () => {
|
||||
const root = await createTestTempDirectory("desktop-withheld-inside-");
|
||||
const directory = join(root, "bundle");
|
||||
@@ -582,6 +594,38 @@ describe("exportDesktopApplication of withheld paths and copies", () => {
|
||||
});
|
||||
expect(linked.result).toBeInstanceOf(Error);
|
||||
expect(await readdir(directory)).toEqual([]);
|
||||
await mkdir(join(root, "outside"));
|
||||
await symlink(directory, join(root, "outside", "link"));
|
||||
const parentTraversal = await exportWithheld({
|
||||
withhold: { paths: [SETTINGS] },
|
||||
directory,
|
||||
withheldCopy: `${root}/outside/link/../bundle/private`,
|
||||
});
|
||||
expect(parentTraversal.result).toBeInstanceOf(Error);
|
||||
expect(await readdir(directory)).toEqual([]);
|
||||
const unresolved = await exportWithheld({
|
||||
withhold: { paths: [SETTINGS] },
|
||||
directory: join(root, "new-bundle"),
|
||||
withheldCopy: `${root}/missing/../private`,
|
||||
});
|
||||
expect(unresolved.result).toBeInstanceOf(Error);
|
||||
expect(await exists(join(root, "missing"))).toBe(false);
|
||||
expect(await exists(join(root, "new-bundle"))).toBe(false);
|
||||
await symlink(join(root, "absent"), join(root, "dangling"));
|
||||
const dangling = await exportWithheld({
|
||||
withhold: { paths: [SETTINGS] },
|
||||
directory: join(root, "new-bundle"),
|
||||
withheldCopy: join(root, "dangling", "private"),
|
||||
});
|
||||
expect(dangling.result).toBeInstanceOf(AnalysisError);
|
||||
expect(await exists(join(root, "new-bundle"))).toBe(false);
|
||||
const leafLink = await exportWithheld({
|
||||
withhold: { paths: [SETTINGS] },
|
||||
directory: `${root}/alias//`,
|
||||
withheldCopy: join(root, "outside-copy"),
|
||||
});
|
||||
expect(leafLink.result).toBeInstanceOf(Error);
|
||||
expect(await readdir(directory)).toEqual([]);
|
||||
});
|
||||
|
||||
it("reports a written copy whose handles cannot be closed, with the export's Evidence", async () => {
|
||||
|
||||
@@ -1,9 +1,9 @@
|
||||
import { createHash } from "node:crypto";
|
||||
import { readFile, realpath } from "node:fs/promises";
|
||||
import { basename, dirname, join, resolve, sep } from "node:path";
|
||||
import { readFile } from "node:fs/promises";
|
||||
import { Readable } from "node:stream";
|
||||
|
||||
import { AnalysisInputError } from "../../domain/analysisErrorCore.js";
|
||||
import { isPathWithinRoot } from "../../domain/localPath.js";
|
||||
import {
|
||||
consentFileSchema,
|
||||
MAX_WITHHELD_BYTES,
|
||||
@@ -24,7 +24,12 @@ import {
|
||||
import { isWithinAny } from "../../domain/desktop/desktopStateDiff.js";
|
||||
import { safeParseUtf8Json } from "../../domain/safeJson.js";
|
||||
import type { SeedVersion } from "./DesktopExportFiles.js";
|
||||
import { writeOutputTree, type FailedRelease } from "./DesktopOutputTree.js";
|
||||
import {
|
||||
planOutputTree,
|
||||
writeOutputTree,
|
||||
type DesktopOutputPlan,
|
||||
type FailedRelease,
|
||||
} from "./DesktopOutputTree.js";
|
||||
import {
|
||||
withRetainedContent,
|
||||
type RetainedContent,
|
||||
@@ -262,7 +267,7 @@ const readWithheld = async (
|
||||
|
||||
/** Write the withheld files, by home path, into a new or empty folder outside the bundle; a failed release of a written copy is returned. */
|
||||
const writeWithheldCopy = (
|
||||
directory: string,
|
||||
directory: DesktopOutputPlan,
|
||||
files: ReadonlyMap<string, Buffer>,
|
||||
signal: AbortSignal | undefined,
|
||||
): Promise<FailedRelease | undefined> =>
|
||||
@@ -284,32 +289,18 @@ const writeWithheldCopy = (
|
||||
signal,
|
||||
);
|
||||
|
||||
/** A path with every link in its existing part resolved, so two spellings of one folder compare equal. */
|
||||
const resolvedPath = async (path: string): Promise<string> => {
|
||||
const rest: string[] = [];
|
||||
for (let current = resolve(path); ; current = dirname(current)) {
|
||||
try {
|
||||
return join(await realpath(current), ...rest);
|
||||
} catch {
|
||||
if (dirname(current) === current) return resolve(path);
|
||||
rest.unshift(basename(current));
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
const isWithin = (path: string, root: string) =>
|
||||
path === root || path.startsWith(root.endsWith(sep) ? root : `${root}${sep}`);
|
||||
|
||||
/** Refuse a copy folder that is the bundle's, inside it or holding it, also through a link, before either is written. */
|
||||
const refuseCopyInBundle = async (copy: string, bundle: string) => {
|
||||
const [copied, bundled] = await Promise.all([
|
||||
resolvedPath(copy),
|
||||
resolvedPath(bundle),
|
||||
]);
|
||||
if (isWithin(copied, bundled) || isWithin(bundled, copied))
|
||||
const refuseCopyInBundle = (
|
||||
copy: DesktopOutputPlan,
|
||||
bundle: DesktopOutputPlan,
|
||||
) => {
|
||||
if (
|
||||
isPathWithinRoot(bundle.physical, copy.physical) ||
|
||||
isPathWithinRoot(copy.physical, bundle.physical)
|
||||
)
|
||||
throw refused(
|
||||
"withheld_copy",
|
||||
`The withheld copy ${copy} lies in the bundle ${bundle}, or holds it, so the withheld files would ship with it.`,
|
||||
`The withheld copy ${copy.requested} lies in the bundle ${bundle.requested}, or holds it, so the withheld files would ship with it.`,
|
||||
"Name a copy folder outside the bundle's folder.",
|
||||
);
|
||||
};
|
||||
@@ -325,22 +316,35 @@ export const withheldCopy = async (
|
||||
captured: ReadonlyMap<string, SeedVersion>,
|
||||
signal: AbortSignal | undefined,
|
||||
): Promise<{
|
||||
readonly bundle: DesktopOutputPlan;
|
||||
readonly write: () => Promise<FailedRelease | undefined>;
|
||||
}> => {
|
||||
const directory = request.withheldCopy;
|
||||
const withheld = plan.withheld ?? [];
|
||||
if (directory !== undefined)
|
||||
await refuseCopyInBundle(directory, request.directory);
|
||||
if (withheld.length === 0) return { write: () => Promise.resolve(undefined) };
|
||||
const bundle = await planOutputTree(request.directory, {
|
||||
operation: OPERATION,
|
||||
field: "directory",
|
||||
});
|
||||
const copy =
|
||||
directory === undefined
|
||||
? undefined
|
||||
: await planOutputTree(directory, {
|
||||
operation: OPERATION,
|
||||
field: "withheld_copy",
|
||||
});
|
||||
if (copy !== undefined) refuseCopyInBundle(copy, bundle);
|
||||
if (withheld.length === 0)
|
||||
return { bundle, write: () => Promise.resolve(undefined) };
|
||||
const files = await withRetainedContent(
|
||||
request.capture,
|
||||
(content) => readWithheld(content, withheld, captured, signal),
|
||||
SEED_MAX_BYTES,
|
||||
);
|
||||
return {
|
||||
bundle,
|
||||
write: () =>
|
||||
directory === undefined
|
||||
copy === undefined
|
||||
? Promise.resolve(undefined)
|
||||
: writeWithheldCopy(directory, files, signal),
|
||||
: writeWithheldCopy(copy, files, signal),
|
||||
};
|
||||
};
|
||||
@@ -0,0 +1,30 @@
|
||||
import { readdir } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
import { expect, it } from "vitest";
|
||||
import { createTestTempDirectory } from "../../../tests/fixtures/temporaryDirectory.js";
|
||||
import { planOutputTree, writeOutputTree } from "./DesktopOutputTree.js";
|
||||
|
||||
it("does not create output parents or invoke the writer after cancellation", async () => {
|
||||
const root = await createTestTempDirectory("rea-output-cancel-");
|
||||
const directory = join(root, "new-parent", "output");
|
||||
const refusal = {
|
||||
operation: "export_desktop_application",
|
||||
field: "directory",
|
||||
};
|
||||
const plan = await planOutputTree(directory, refusal);
|
||||
const controller = new AbortController();
|
||||
const reason = new Error("cancelled before output acquisition");
|
||||
controller.abort(reason);
|
||||
let written = false;
|
||||
const failure = await writeOutputTree(
|
||||
plan,
|
||||
async () => {
|
||||
written = true;
|
||||
},
|
||||
refusal,
|
||||
controller.signal,
|
||||
).catch((cause: unknown) => cause);
|
||||
expect(written).toBe(false);
|
||||
expect(await readdir(root)).toEqual([]);
|
||||
expect(failure).toBe(reason);
|
||||
});
|
||||
@@ -1,13 +1,18 @@
|
||||
import { mkdir } from "node:fs/promises";
|
||||
import { dirname } from "node:path";
|
||||
import { lstat, mkdir, readdir } from "node:fs/promises";
|
||||
import { basename, dirname, isAbsolute } from "node:path";
|
||||
|
||||
import {
|
||||
ArtifactResourceScope,
|
||||
type ArtifactCleanupAttempt,
|
||||
} from "../../artifacts/ArtifactResourceScope.js";
|
||||
import { SafeOutputTree } from "../../artifacts/SafeOutputTree.js";
|
||||
import { outputDirectoryAdmissionRefusal } from "../../artifacts/SafeOutputTreeAdmission.js";
|
||||
import {
|
||||
outputDirectoryAdmissionRefusal,
|
||||
SafeOutputTreeDestinationRefusal,
|
||||
} from "../../artifacts/SafeOutputTreeAdmission.js";
|
||||
import { SafeOutputTreeCreationFailure } from "../../artifacts/SafeOutputTreeCreationFailure.js";
|
||||
import { canonicalCallerPath } from "../../filesystem/canonicalCallerPath.js";
|
||||
import { absoluteCallerPath } from "../../filesystem/callerPath.js";
|
||||
import type { Evidence } from "../../domain/evidence.js";
|
||||
import { ProviderAdapterError } from "../../domain/providerAdapterError.js";
|
||||
|
||||
@@ -17,6 +22,65 @@ export type FailedRelease = Extract<
|
||||
{ readonly kind: "failed" }
|
||||
>;
|
||||
|
||||
/** A caller-selected destination and the filesystem path admitted for writing. */
|
||||
export interface DesktopOutputPlan {
|
||||
readonly requested: string;
|
||||
readonly physical: string;
|
||||
}
|
||||
|
||||
/** Establish destination identity before either output tree is created. */
|
||||
export const planOutputTree = async (
|
||||
requested: string,
|
||||
refusal: { readonly operation: string; readonly field: string },
|
||||
): Promise<DesktopOutputPlan> => {
|
||||
try {
|
||||
if (!isAbsolute(requested))
|
||||
throw new SafeOutputTreeDestinationRefusal(
|
||||
"Output root must be absolute",
|
||||
);
|
||||
if (basename(requested) === ".." || basename(requested) === ".")
|
||||
throw new SafeOutputTreeDestinationRefusal("Invalid output directory");
|
||||
const physical = await canonicalCallerPath(requested);
|
||||
const absolute = absoluteCallerPath(requested);
|
||||
const selected =
|
||||
absolute.length > 1 ? absolute.replace(/\/$/u, "") : absolute;
|
||||
const existing = await lstat(selected).then(
|
||||
(value) => value,
|
||||
(cause: unknown) => {
|
||||
if ((cause as NodeJS.ErrnoException).code === "ENOENT")
|
||||
return undefined;
|
||||
throw cause;
|
||||
},
|
||||
);
|
||||
if (existing?.isSymbolicLink())
|
||||
throw new SafeOutputTreeDestinationRefusal(
|
||||
"Output root cannot be a symbolic link",
|
||||
);
|
||||
if (existing !== undefined && !existing.isDirectory())
|
||||
throw new SafeOutputTreeDestinationRefusal(
|
||||
"Output root must be a directory",
|
||||
);
|
||||
if (existing !== undefined && (await readdir(selected)).length > 0)
|
||||
throw new SafeOutputTreeDestinationRefusal(
|
||||
"Output directory is not empty",
|
||||
);
|
||||
return Object.freeze({ requested, physical });
|
||||
} catch (cause: unknown) {
|
||||
const error =
|
||||
cause instanceof SafeOutputTreeDestinationRefusal
|
||||
? cause
|
||||
: new SafeOutputTreeDestinationRefusal(
|
||||
"Output destination cannot be resolved",
|
||||
{ cause },
|
||||
);
|
||||
throw outputDirectoryAdmissionRefusal(
|
||||
refusal.operation,
|
||||
refusal.field,
|
||||
error,
|
||||
);
|
||||
}
|
||||
};
|
||||
|
||||
/**
|
||||
* Write a new or empty folder through the safe output tree, rolling an
|
||||
* unpublished tree back on any failure so no partial copy remains. A
|
||||
@@ -24,12 +88,13 @@ export type FailedRelease = Extract<
|
||||
* refused destination names the caller's `field` under `operation`.
|
||||
*/
|
||||
export const writeOutputTree = async (
|
||||
directory: string,
|
||||
plan: DesktopOutputPlan,
|
||||
write: (tree: SafeOutputTree) => Promise<void>,
|
||||
refusal: { readonly operation: string; readonly field: string },
|
||||
signal: AbortSignal | undefined,
|
||||
): Promise<FailedRelease | undefined> => {
|
||||
await mkdir(dirname(directory), { recursive: true, mode: 0o700 });
|
||||
signal?.throwIfAborted();
|
||||
await mkdir(dirname(plan.physical), { recursive: true, mode: 0o700 });
|
||||
const resources = new ArtifactResourceScope();
|
||||
let tree: SafeOutputTree | undefined;
|
||||
let outcome:
|
||||
@@ -37,7 +102,7 @@ export const writeOutputTree = async (
|
||||
| { readonly kind: "failed"; readonly cause: unknown };
|
||||
try {
|
||||
try {
|
||||
tree = await SafeOutputTree.create(directory, process.platform, {
|
||||
tree = await SafeOutputTree.create(plan.physical, process.platform, {
|
||||
acceptExistingEmpty: true,
|
||||
});
|
||||
} catch (cause: unknown) {
|
||||
@@ -53,7 +118,7 @@ export const writeOutputTree = async (
|
||||
if (tree !== undefined) {
|
||||
const released = await resources.release({
|
||||
kind: "output-tree",
|
||||
resource: directory,
|
||||
resource: plan.requested,
|
||||
tree,
|
||||
});
|
||||
if (released.kind === "failed") {
|
||||
|
||||
@@ -7,6 +7,8 @@ import {
|
||||
} from "../../domain/analysisErrorCore.js";
|
||||
import { createEvidence } from "../../domain/evidence.js";
|
||||
import { err, ok } from "../../domain/result.js";
|
||||
import type { SqliteDatabase } from "../../domain/sqlite/sqliteDatabase.js";
|
||||
import { diffStateViews } from "../../domain/desktop/stateChanges.js";
|
||||
import { readSqliteStateView } from "./DesktopSqliteState.js";
|
||||
|
||||
const OPERATION = "inspect_sqlite_database";
|
||||
@@ -34,7 +36,7 @@ const table = (name: string) => ({
|
||||
});
|
||||
|
||||
/** A SQLite report with two tables, and one row of the selected table. */
|
||||
const report = (selected: string | undefined) => ({
|
||||
const report = (selected: string | undefined): SqliteDatabase => ({
|
||||
artifact: { path: "/staged/state.db", sha256: "a".repeat(64), bytes: 4096 },
|
||||
wal: null,
|
||||
engine: { name: "SQLite", version: "3.50.2" },
|
||||
@@ -86,6 +88,79 @@ const service = (failure: (table: string) => AnalysisError | undefined) => ({
|
||||
});
|
||||
|
||||
describe("readSqliteStateView", () => {
|
||||
it("distinguishes special REAL values from text when comparing captured rows", async () => {
|
||||
const read = async (type: "real" | "text") =>
|
||||
readSqliteStateView(
|
||||
{
|
||||
inspect: async (input: unknown) => {
|
||||
const selected = (input as { table?: string }).table;
|
||||
const result: SqliteDatabase = report(selected);
|
||||
result.schema.tables = [table("notes")];
|
||||
if (result.rows !== null) {
|
||||
result.rows.values = (
|
||||
["Infinity", "-Infinity", "-0"] as const
|
||||
).map((value) => [
|
||||
type === "real"
|
||||
? { type: "real", value }
|
||||
: {
|
||||
type: "text",
|
||||
value,
|
||||
bytes_hex: Buffer.from(value).toString("hex"),
|
||||
encoding: "UTF-8",
|
||||
},
|
||||
]);
|
||||
result.rows.returned_rows = 3;
|
||||
}
|
||||
return ok(
|
||||
createEvidence(
|
||||
undefined,
|
||||
{ id: "sqlite", name: "SQLite", version: "1" },
|
||||
{
|
||||
operation: OPERATION,
|
||||
parameters: {},
|
||||
result,
|
||||
confidence: "observed",
|
||||
},
|
||||
),
|
||||
);
|
||||
},
|
||||
},
|
||||
"/staged/state.db",
|
||||
);
|
||||
const real = await read("real");
|
||||
const text = await read("text");
|
||||
expect(real).toMatchObject({
|
||||
ok: true,
|
||||
view: {
|
||||
tables: {
|
||||
notes: {
|
||||
rows: [
|
||||
[{ real: "Infinity" }],
|
||||
[{ real: "-Infinity" }],
|
||||
[{ real: "-0" }],
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
expect(text).toMatchObject({
|
||||
ok: true,
|
||||
view: {
|
||||
tables: {
|
||||
notes: {
|
||||
rows: [["Infinity"], ["-Infinity"], ["-0"]],
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
if (!real.ok || !text.ok) throw new Error("Expected readable SQLite views");
|
||||
expect(diffStateViews("sqlite", real.view, text.view)).toMatchObject({
|
||||
added_count: 3,
|
||||
removed_count: 3,
|
||||
complete: true,
|
||||
});
|
||||
});
|
||||
|
||||
it("keeps the tables that read and records the error of one that did not", async () => {
|
||||
const read = await readSqliteStateView(
|
||||
service((name) =>
|
||||
|
||||
@@ -27,7 +27,7 @@ type Cell = z.infer<typeof sqliteValueSchema>;
|
||||
/**
|
||||
* SQLite cells as JSON. Values that plain JSON would conflate keep a tagged
|
||||
* form: integers beyond the safe range, text whose bytes did not decode
|
||||
* cleanly, and blobs.
|
||||
* cleanly, special REAL values, and blobs.
|
||||
*/
|
||||
const cellValue = (cell: Cell): JsonValue => {
|
||||
switch (cell.type) {
|
||||
@@ -38,7 +38,7 @@ const cellValue = (cell: Cell): JsonValue => {
|
||||
return Number.isSafeInteger(value) ? value : { integer: cell.value };
|
||||
}
|
||||
case "real":
|
||||
return cell.value;
|
||||
return typeof cell.value === "string" ? { real: cell.value } : cell.value;
|
||||
case "text":
|
||||
return cell.value.includes("\uFFFD")
|
||||
? { text: cell.value, bytes_hex: cell.bytes_hex }
|
||||
|
||||
@@ -4,6 +4,7 @@ import { AnalysisCapabilityUnavailableError } from "../../domain/analysisErrorCo
|
||||
import { err, ok } from "../../domain/result.js";
|
||||
import { createAnalysisExecution } from "../AnalysisProvider.js";
|
||||
import { sqliteDatabaseSchema } from "../../domain/sqlite/sqliteDatabase.js";
|
||||
import { createAnalysisProfile } from "../../domain/analysisProfile.js";
|
||||
|
||||
const fixture = () => ({
|
||||
artifact: { path: "/selected.db", sha256: "a".repeat(64), bytes: 4096 },
|
||||
@@ -236,39 +237,59 @@ it.each(["path", "digest", "subject", "wal", "table"])(
|
||||
},
|
||||
);
|
||||
|
||||
it("preserves completed observed evidence when cancellation arrives after provider completion", async () => {
|
||||
const value = fixture();
|
||||
const service = new SqliteDatabaseService({
|
||||
inspect: () => Promise.resolve(ok(execution(value))),
|
||||
});
|
||||
expect(await service.inspect({ path: "/selected.db" })).toMatchObject({
|
||||
ok: true,
|
||||
value: {
|
||||
it.each([false, true])(
|
||||
"retains completed SQLite facts with late cancellation=%s",
|
||||
async (cancelled) => {
|
||||
const value = fixture();
|
||||
const observed = execution(value);
|
||||
const profile = createAnalysisProfile(
|
||||
{ ...observed.provider, version: value.engine.version },
|
||||
{
|
||||
schema: "inventory",
|
||||
},
|
||||
);
|
||||
const relationshipCoverage = {
|
||||
status: "unknown",
|
||||
reason: "Relationships were not inspected",
|
||||
} as const;
|
||||
const completed = {
|
||||
...observed,
|
||||
analysisProfile: profile,
|
||||
relationshipCoverage,
|
||||
};
|
||||
const controller = new AbortController();
|
||||
const reason = Object.freeze({
|
||||
requested: "cancel after database inspection",
|
||||
});
|
||||
const service = new SqliteDatabaseService({
|
||||
inspect: () => {
|
||||
if (cancelled) controller.abort(reason);
|
||||
return Promise.resolve(ok(completed));
|
||||
},
|
||||
});
|
||||
const result = await service.inspect(
|
||||
{ path: "/selected.db" },
|
||||
{ signal: controller.signal },
|
||||
);
|
||||
expect(result.ok).toBe(!cancelled);
|
||||
if (!result.ok) {
|
||||
expect(result.error._tag).toBe("AnalysisCancelledError");
|
||||
expect(result.error.cause).toBe(reason);
|
||||
}
|
||||
const evidence = result.ok ? result.value : result.error.partialObservation;
|
||||
expect(evidence).toMatchObject({
|
||||
operation: "inspect_sqlite_database",
|
||||
confidence: "observed",
|
||||
subject: {
|
||||
local_path: "/selected.db",
|
||||
digest: { sha256: "a".repeat(64) },
|
||||
},
|
||||
normalized_result: value,
|
||||
},
|
||||
});
|
||||
const controller = new AbortController();
|
||||
const cancelled = new SqliteDatabaseService({
|
||||
inspect: () => {
|
||||
controller.abort();
|
||||
return Promise.resolve(ok(execution(value)));
|
||||
},
|
||||
});
|
||||
expect(
|
||||
await cancelled.inspect(
|
||||
{ path: "/selected.db" },
|
||||
{ signal: controller.signal },
|
||||
),
|
||||
).toMatchObject({
|
||||
ok: true,
|
||||
value: { normalized_result: value },
|
||||
});
|
||||
});
|
||||
analysis_profile: profile,
|
||||
relationship_coverage: relationshipCoverage,
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
it("preserves the exact caller reason when inspection is already aborted", async () => {
|
||||
const reason = Object.freeze({ requested: "stop" });
|
||||
|
||||
@@ -3,11 +3,10 @@ import type { ExecutionOptions } from "../AnalysisProvider.js";
|
||||
import type { SqliteDatabasePort } from "./SqliteDatabasePort.js";
|
||||
import type { AnalysisError } from "../../domain/analysisErrorBase.js";
|
||||
import { AnalysisOutputError } from "../../domain/analysisErrorCore.js";
|
||||
import {
|
||||
createEvidence,
|
||||
type Evidence,
|
||||
type EvidenceSubjectTarget,
|
||||
} from "../../domain/evidence.js";
|
||||
import type { Evidence, EvidenceSubjectTarget } from "../../domain/evidence.js";
|
||||
import { createAnalysisExecutionEvidence } from "../../domain/analysisObservation.js";
|
||||
import { ownedJsonValueSchema } from "../../domain/jsonValue.js";
|
||||
import { completeProviderEvidence } from "../ProviderOperationExecution.js";
|
||||
import {
|
||||
inspectSqliteDatabaseInputSchema,
|
||||
sqliteDatabaseSchema,
|
||||
@@ -19,7 +18,7 @@ import {
|
||||
sqliteSchemaScope,
|
||||
sqliteInspectionParameters,
|
||||
} from "../../domain/sqlite/sqliteDatabase.js";
|
||||
import { err, ok, type Result } from "../../domain/result.js";
|
||||
import { err, type Result } from "../../domain/result.js";
|
||||
|
||||
const OPERATION = "inspect_sqlite_database";
|
||||
|
||||
@@ -102,18 +101,14 @@ export class SqliteDatabaseService {
|
||||
"SQLite adapter returned malformed data or changed the selected database identity/table/row limit/worker budget.",
|
||||
),
|
||||
);
|
||||
return ok(
|
||||
createEvidence(execution.subject, execution.provider, {
|
||||
operation: OPERATION,
|
||||
parameters: sqliteInspectionParameters(input.value),
|
||||
result: report.data,
|
||||
...(execution.rawResult === null
|
||||
? {}
|
||||
: { rawResult: execution.rawResult }),
|
||||
confidence: "observed",
|
||||
locations: execution.locations,
|
||||
limitations: execution.limitations,
|
||||
}),
|
||||
return completeProviderEvidence(
|
||||
createAnalysisExecutionEvidence(
|
||||
OPERATION,
|
||||
sqliteInspectionParameters(input.value),
|
||||
{ ...execution, result: ownedJsonValueSchema.parse(report.data) },
|
||||
"observed",
|
||||
),
|
||||
options?.signal,
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -215,6 +215,7 @@ export interface ArtifactReader {
|
||||
| "dfuse"
|
||||
| "esp-flash-image"
|
||||
| "fit"
|
||||
| "ntfs-mft"
|
||||
| "file";
|
||||
entries(
|
||||
signal?: AbortSignal,
|
||||
|
||||
@@ -18,7 +18,10 @@ import {
|
||||
type ArtifactTraversalOptions,
|
||||
} from "./ArtifactReader.js";
|
||||
import { ArtifactResourceScope } from "./ArtifactResourceScope.js";
|
||||
import { assertArtifactReadActive } from "./ArtifactFileOperation.js";
|
||||
import {
|
||||
artifactReadCancelledBy,
|
||||
assertArtifactReadActive,
|
||||
} from "./ArtifactFileOperation.js";
|
||||
import { ArtifactFileSource } from "./ArtifactFileSource.js";
|
||||
import { readFileHandleChunks } from "../filesystem/readFileHandleChunks.js";
|
||||
import type { StableRegularFileDescriptor } from "../filesystem/RegularFile.js";
|
||||
@@ -64,12 +67,14 @@ export const regionFileName = (start: number, end: number): string =>
|
||||
*/
|
||||
export class DecodedRegionArtifactReader<Image> implements ArtifactReader {
|
||||
readonly #scope = new ArtifactResourceScope();
|
||||
readonly #entries = new Map<string, ArtifactEntry>();
|
||||
readonly #regions = new Map<string, Uint8Array>();
|
||||
#entries = new Map<string, ArtifactEntry>();
|
||||
#regions = new Map<string, Uint8Array>();
|
||||
readonly #streams = new Set<Readable>();
|
||||
readonly #source: ArtifactFileSource;
|
||||
readonly #decoding: RegionDecoding<Image>;
|
||||
#prepared: Promise<void> | undefined;
|
||||
#prepared:
|
||||
| { signal: AbortSignal | undefined; promise: Promise<void> }
|
||||
| undefined;
|
||||
/** Whole-file facts only; the region map is the sole owner of region bytes. */
|
||||
#described: Omit<DecodedRegions, "regions"> | undefined;
|
||||
#closing = false;
|
||||
@@ -110,8 +115,27 @@ export class DecodedRegionArtifactReader<Image> implements ArtifactReader {
|
||||
): AsyncIterable<ArtifactEntry> {
|
||||
await this.#scope.run(async () => {
|
||||
this.#assertOpen(signal);
|
||||
this.#prepared ??= this.#prepare(signal);
|
||||
await this.#prepared;
|
||||
for (;;) {
|
||||
const pending = (this.#prepared ??= {
|
||||
signal,
|
||||
promise: this.#prepare(signal),
|
||||
});
|
||||
try {
|
||||
await pending.promise;
|
||||
break;
|
||||
} catch (cause: unknown) {
|
||||
if (this.#prepared === pending) this.#prepared = undefined;
|
||||
if (
|
||||
pending.signal !== signal &&
|
||||
signal?.aborted !== true &&
|
||||
artifactReadCancelledBy(cause, pending.signal)
|
||||
) {
|
||||
this.#assertOpen(signal);
|
||||
continue;
|
||||
}
|
||||
throw cause;
|
||||
}
|
||||
}
|
||||
await this.#source.read(signal);
|
||||
});
|
||||
const coverage = this.#described?.coverage;
|
||||
@@ -230,6 +254,8 @@ export class DecodedRegionArtifactReader<Image> implements ArtifactReader {
|
||||
|
||||
async #prepare(signal?: AbortSignal): Promise<void> {
|
||||
const described = this.#decoding.describe(await this.#decode(signal));
|
||||
const entries = new Map<string, ArtifactEntry>();
|
||||
const regions = new Map<string, Uint8Array>();
|
||||
let index = 0;
|
||||
for (const region of described.regions) {
|
||||
if (index % 256 === 0) await setImmediate();
|
||||
@@ -248,13 +274,16 @@ export class DecodedRegionArtifactReader<Image> implements ArtifactReader {
|
||||
adapterKey: `${this.#decoding.keyPrefix}:${String(index)}`,
|
||||
formatMetadata: region.formatMetadata,
|
||||
} satisfies ArtifactEntry;
|
||||
this.#entries.set(entry.adapterKey, entry);
|
||||
this.#regions.set(entry.adapterKey, region.bytes);
|
||||
entries.set(entry.adapterKey, entry);
|
||||
regions.set(entry.adapterKey, region.bytes);
|
||||
index += 1;
|
||||
}
|
||||
const { regions: _regions, ...facts } = described;
|
||||
this.#described = facts;
|
||||
await this.#source.read(signal);
|
||||
this.#assertOpen(signal);
|
||||
this.#entries = entries;
|
||||
this.#regions = regions;
|
||||
this.#described = facts;
|
||||
}
|
||||
|
||||
async #decode(signal?: AbortSignal): Promise<Image> {
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { artifactPreparationRecoveryCases } from "../../tests/fixtures/artifactReaderLifecycle.js";
|
||||
import { getEventListeners, once } from "node:events";
|
||||
import { readFile, truncate, writeFile } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
@@ -404,6 +405,38 @@ it("keeps the cancellation reason before admission", async () => {
|
||||
}
|
||||
});
|
||||
|
||||
it("does not publish FIT entries when final source verification fails", async () => {
|
||||
const path = await writeFit("partial-publication.itb", FIT_EMBEDDED);
|
||||
const control = new FitArtifactReader(path);
|
||||
const reader = new FitArtifactReader(path);
|
||||
const expected = await firstEntry(control);
|
||||
const nativeRead = ArtifactFileSource.prototype.read;
|
||||
const physical = Object.assign(
|
||||
new Error("final source verification failed"),
|
||||
{ code: "EIO" },
|
||||
);
|
||||
let calls = 0;
|
||||
const read = vi
|
||||
.spyOn(ArtifactFileSource.prototype, "read")
|
||||
.mockImplementation(function (this: ArtifactFileSource, ...args) {
|
||||
calls += 1;
|
||||
if (calls === 4) return Promise.reject(physical);
|
||||
return nativeRead.apply(this, args);
|
||||
});
|
||||
try {
|
||||
await expect(firstEntry(reader)).rejects.toBe(physical);
|
||||
await expect(reader.open(expected)).rejects.toMatchObject({
|
||||
reason: "integrity",
|
||||
});
|
||||
read.mockRestore();
|
||||
expect((await firstEntry(reader)).path).toBe(expected.path);
|
||||
} finally {
|
||||
read.mockRestore();
|
||||
await reader.close();
|
||||
await control.close();
|
||||
}
|
||||
});
|
||||
|
||||
it("keeps the cancellation reason during hash verification", async () => {
|
||||
const reader = new FitArtifactReader(
|
||||
await writeFit("hashing.itb", LARGE_FIT),
|
||||
@@ -430,6 +463,8 @@ it("keeps the cancellation reason during hash verification", async () => {
|
||||
reader.entries(controller.signal)[Symbol.asyncIterator]().next(),
|
||||
).rejects.toMatchObject({ reason: "cancelled", cause: reason });
|
||||
expect(observer).toHaveBeenCalledWith("sha256");
|
||||
observer.mockRestore();
|
||||
expect((await firstEntry(reader)).kind).toBe("file");
|
||||
} finally {
|
||||
observer.mockRestore();
|
||||
await reader.close();
|
||||
@@ -569,3 +604,8 @@ it("preserves a final validation failure when cancellation overlaps it", async (
|
||||
await reader.close();
|
||||
}
|
||||
});
|
||||
|
||||
artifactPreparationRecoveryCases({
|
||||
image: () => FIT_EMBEDDED,
|
||||
createReader: (path, admitted) => new FitArtifactReader(path, admitted),
|
||||
});
|
||||
@@ -13,7 +13,10 @@ import {
|
||||
type ArtifactTraversalOptions,
|
||||
} from "./ArtifactReader.js";
|
||||
import { ArtifactResourceScope } from "./ArtifactResourceScope.js";
|
||||
import { assertArtifactReadActive } from "./ArtifactFileOperation.js";
|
||||
import {
|
||||
artifactReadCancelledBy,
|
||||
assertArtifactReadActive,
|
||||
} from "./ArtifactFileOperation.js";
|
||||
import { ArtifactFileSource } from "./ArtifactFileSource.js";
|
||||
import {
|
||||
createFitDigest,
|
||||
@@ -30,11 +33,13 @@ const FDT_HEADER_BYTES = 40;
|
||||
export class FitArtifactReader implements ArtifactReader {
|
||||
readonly format = "fit" as const;
|
||||
readonly #scope = new ArtifactResourceScope();
|
||||
readonly #entries = new Map<string, ArtifactEntry>();
|
||||
#entries = new Map<string, ArtifactEntry>();
|
||||
readonly #streams = new Set<Readable>();
|
||||
readonly #path: string;
|
||||
readonly #source: ArtifactFileSource;
|
||||
#prepared: Promise<void> | undefined;
|
||||
#prepared:
|
||||
| { signal: AbortSignal | undefined; promise: Promise<void> }
|
||||
| undefined;
|
||||
#metadata: FitMetadata | undefined;
|
||||
#coverage: ArtifactReaderTraversal | undefined;
|
||||
#closing = false;
|
||||
@@ -66,8 +71,27 @@ export class FitArtifactReader implements ArtifactReader {
|
||||
): AsyncIterable<ArtifactEntry> {
|
||||
await this.#scope.run(async () => {
|
||||
this.#assertOpen(signal);
|
||||
this.#prepared ??= this.#prepare(signal);
|
||||
await this.#prepared;
|
||||
for (;;) {
|
||||
const pending = (this.#prepared ??= {
|
||||
signal,
|
||||
promise: this.#prepare(signal),
|
||||
});
|
||||
try {
|
||||
await pending.promise;
|
||||
break;
|
||||
} catch (cause: unknown) {
|
||||
if (this.#prepared === pending) this.#prepared = undefined;
|
||||
if (
|
||||
pending.signal !== signal &&
|
||||
signal?.aborted !== true &&
|
||||
artifactReadCancelledBy(cause, pending.signal)
|
||||
) {
|
||||
this.#assertOpen(signal);
|
||||
continue;
|
||||
}
|
||||
throw cause;
|
||||
}
|
||||
}
|
||||
await this.#source.read(signal);
|
||||
});
|
||||
if (this.#coverage !== undefined)
|
||||
@@ -207,6 +231,7 @@ export class FitArtifactReader implements ArtifactReader {
|
||||
this.#path,
|
||||
initial.size,
|
||||
);
|
||||
const entries = new Map<string, ArtifactEntry>();
|
||||
let index = 0;
|
||||
for (const image of structure.images) {
|
||||
if (index % 256 === 0) await setImmediate();
|
||||
@@ -228,15 +253,17 @@ export class FitArtifactReader implements ArtifactReader {
|
||||
adapterKey: `fit:${String(image.index)}`,
|
||||
formatMetadata: image.metadata,
|
||||
} satisfies ArtifactEntry;
|
||||
this.#entries.set(entry.adapterKey, entry);
|
||||
entries.set(entry.adapterKey, entry);
|
||||
}
|
||||
await this.#source.read(signal);
|
||||
this.#assertOpen(signal);
|
||||
this.#entries = entries;
|
||||
this.#metadata = structure.metadata;
|
||||
if (structure.limitations.length > 0)
|
||||
this.#coverage = {
|
||||
status: "complete",
|
||||
limitations: structure.limitations,
|
||||
};
|
||||
await this.#source.read(signal);
|
||||
}
|
||||
|
||||
/** One streamed pass per image feeds every supported digest. */
|
||||
|
||||
@@ -1,3 +1,6 @@
|
||||
import { artifactPreparationRecoveryCases } from "../../tests/fixtures/artifactReaderLifecycle.js";
|
||||
import { ArtifactFileSource } from "./ArtifactFileSource.js";
|
||||
import { openRegularFile } from "../filesystem/RegularFile.js";
|
||||
import { writeFile } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
|
||||
@@ -159,6 +162,72 @@ it("settles an active region stream when closed", async () => {
|
||||
}
|
||||
});
|
||||
|
||||
it("retries decoded-region preparation cancelled while reading", async () => {
|
||||
const path = await writeImage("retry.hex", INTEL_HEX_I32_GAP);
|
||||
const admitted = await openRegularFile(path, { symlinks: "reject" });
|
||||
const reader = new HexRecordArtifactReader(path, "intel-hex", admitted);
|
||||
const controller = new AbortController();
|
||||
const reason = { stage: "decoded region read" };
|
||||
const nativeRead = admitted.handle.read.bind(admitted.handle);
|
||||
const read = vi.spyOn(admitted.handle, "read").mockImplementation((async (
|
||||
...args: unknown[]
|
||||
) => {
|
||||
const result = await Reflect.apply(nativeRead, admitted.handle, args);
|
||||
controller.abort(reason);
|
||||
return result;
|
||||
}) as typeof admitted.handle.read);
|
||||
try {
|
||||
await expect(
|
||||
reader.entries(controller.signal)[Symbol.asyncIterator]().next(),
|
||||
).rejects.toMatchObject({ reason: "cancelled", cause: reason });
|
||||
read.mockRestore();
|
||||
expect(
|
||||
(await reader.entries()[Symbol.asyncIterator]().next()).value,
|
||||
).toMatchObject({ kind: "file" });
|
||||
} finally {
|
||||
read.mockRestore();
|
||||
await reader.close();
|
||||
await admitted.handle.close();
|
||||
}
|
||||
});
|
||||
|
||||
it("does not publish decoded regions when final source verification fails", async () => {
|
||||
const path = await writeImage("partial.hex", INTEL_HEX_I32_GAP);
|
||||
const control = new HexRecordArtifactReader(path, "intel-hex");
|
||||
const reader = new HexRecordArtifactReader(path, "intel-hex");
|
||||
const expected = (await control.entries()[Symbol.asyncIterator]().next())
|
||||
.value!;
|
||||
const nativeRead = ArtifactFileSource.prototype.read;
|
||||
const physical = Object.assign(
|
||||
new Error("final source verification failed"),
|
||||
{ code: "EIO" },
|
||||
);
|
||||
let calls = 0;
|
||||
const read = vi
|
||||
.spyOn(ArtifactFileSource.prototype, "read")
|
||||
.mockImplementation(function (this: ArtifactFileSource, ...args) {
|
||||
calls += 1;
|
||||
if (calls === 2) return Promise.reject(physical);
|
||||
return nativeRead.apply(this, args);
|
||||
});
|
||||
try {
|
||||
await expect(reader.entries()[Symbol.asyncIterator]().next()).rejects.toBe(
|
||||
physical,
|
||||
);
|
||||
await expect(reader.open(expected)).rejects.toMatchObject({
|
||||
reason: "integrity",
|
||||
});
|
||||
read.mockRestore();
|
||||
expect(
|
||||
(await reader.entries()[Symbol.asyncIterator]().next()).value,
|
||||
).toMatchObject({ path: expected.path });
|
||||
} finally {
|
||||
read.mockRestore();
|
||||
await reader.close();
|
||||
await control.close();
|
||||
}
|
||||
});
|
||||
|
||||
it("honors cancellation before admitting a file with the exact reason", async () => {
|
||||
const reader = new HexRecordArtifactReader(
|
||||
await writeImage("cancel.hex", INTEL_HEX_I32_GAP),
|
||||
@@ -337,3 +406,9 @@ it("bounds copied region chunks and isolates cached bytes across opens", async (
|
||||
await reader.close();
|
||||
}
|
||||
});
|
||||
|
||||
artifactPreparationRecoveryCases({
|
||||
image: () => Buffer.from(INTEL_HEX_I32_GAP),
|
||||
createReader: (path, admitted) =>
|
||||
new HexRecordArtifactReader(path, "intel-hex", admitted),
|
||||
});
|
||||
@@ -0,0 +1,211 @@
|
||||
import { writeFile } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
|
||||
import { expect, it, vi } from "vitest";
|
||||
|
||||
import { ArtifactFileSource } from "./ArtifactFileSource.js";
|
||||
import { assertArtifactReadActive } from "./ArtifactFileOperation.js";
|
||||
import { NtfsMftArtifactReader } from "./NtfsMftArtifactReader.js";
|
||||
|
||||
import { MFT_DEFAULT_RECORD_BYTES } from "../domain/windows/ntfsMft.js";
|
||||
import { createTestTempDirectory } from "../../tests/fixtures/temporaryDirectory.js";
|
||||
import { scanCanonicalArtifactInventory } from "../../tests/fixtures/artifactInventory.js";
|
||||
|
||||
/** A 1 KiB FILE record whose update sequence array covers both sectors. */
|
||||
const recordBytes = (options: {
|
||||
readonly sequenceNumber?: number;
|
||||
readonly flags?: number;
|
||||
readonly index?: number;
|
||||
}): Buffer => {
|
||||
const record = Buffer.alloc(MFT_DEFAULT_RECORD_BYTES);
|
||||
record.write("FILE", 0, "latin1");
|
||||
const sequenceNumber = options.sequenceNumber ?? 1;
|
||||
record.writeUInt16LE(48, 4);
|
||||
record.writeUInt16LE(3, 6);
|
||||
record.writeUInt16LE(sequenceNumber, 48);
|
||||
record.writeUInt16LE(0xa5a5, 50);
|
||||
record.writeUInt16LE(0xa5a5, 52);
|
||||
record.writeUInt16LE(sequenceNumber, 16);
|
||||
record.writeUInt16LE(1, 18);
|
||||
record.writeUInt32LE(options.index ?? 0, 44);
|
||||
record.writeUInt16LE(56, 20);
|
||||
record.writeUInt16LE(options.flags ?? 0x0001, 22);
|
||||
record.writeUInt32LE(64, 24);
|
||||
record.writeUInt32LE(0xffff_ffff, 56);
|
||||
record.writeUInt32LE(MFT_DEFAULT_RECORD_BYTES, 28);
|
||||
record.writeUInt16LE(sequenceNumber, 510);
|
||||
record.writeUInt16LE(sequenceNumber, 1022);
|
||||
return record;
|
||||
};
|
||||
|
||||
const writeMft = async (name: string, bytes: Buffer): Promise<string> => {
|
||||
const directory = await createTestTempDirectory("rea-ntfs-mft-");
|
||||
const path = join(directory, name);
|
||||
await writeFile(path, bytes);
|
||||
return path;
|
||||
};
|
||||
|
||||
it("reports a $MFT stream's record headers as the root occurrence", async () => {
|
||||
const inventory = await scanCanonicalArtifactInventory(
|
||||
await writeMft(
|
||||
"$MFT",
|
||||
Buffer.concat([
|
||||
recordBytes({}),
|
||||
recordBytes({ flags: 0x0003, index: 1 }),
|
||||
]),
|
||||
),
|
||||
);
|
||||
expect(inventory.manifest.root_format).toBe("ntfs-mft");
|
||||
expect(inventory.coverage.status).toBe("complete");
|
||||
// A record stream declares no members, so the root is its whole inventory.
|
||||
expect(inventory.occurrences.map((row) => row.logical_path)).toEqual(["."]);
|
||||
expect(inventory.occurrences[0]).toMatchObject({
|
||||
artifact_format: "ntfs-mft",
|
||||
format_metadata: {
|
||||
kind: "ntfs-mft",
|
||||
record_bytes: MFT_DEFAULT_RECORD_BYTES,
|
||||
record_count: 2,
|
||||
in_use_record_count: 2,
|
||||
directory_record_count: 1,
|
||||
extension_record_count: 0,
|
||||
fixup_mismatch_count: 0,
|
||||
fixup_unverified_record_count: 0,
|
||||
truncated_tail_bytes: 0,
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it("names a failed fixup in the coverage limitations", async () => {
|
||||
const record = recordBytes({ sequenceNumber: 4 });
|
||||
// A trailer that is not the sequence number marks the sector as not cleanly written.
|
||||
record.writeUInt16LE(0x1234, 510);
|
||||
const inventory = await scanCanonicalArtifactInventory(
|
||||
await writeMft("$MFT", record),
|
||||
);
|
||||
expect(inventory.manifest.root_format).toBe("ntfs-mft");
|
||||
expect(inventory.coverage.status).toBe("unknown");
|
||||
expect(inventory.limitations).toContainEqual(
|
||||
expect.stringMatching(/sector trailer that is not the update sequence/u),
|
||||
);
|
||||
});
|
||||
|
||||
it("reports an unaligned used-byte count as incomplete coverage", async () => {
|
||||
const record = recordBytes({});
|
||||
record.writeUInt32LE(57, 24);
|
||||
const inventory = await scanCanonicalArtifactInventory(
|
||||
await writeMft("$MFT", record),
|
||||
);
|
||||
expect(inventory.coverage.status).toBe("unknown");
|
||||
expect(inventory.occurrences[0]?.format_metadata).toMatchObject({
|
||||
record_headers: [{ usedBytes: 57, extentValid: false }],
|
||||
});
|
||||
});
|
||||
|
||||
it("reports incomplete coverage when FILE extents contradict the record slots", async () => {
|
||||
const usedTooLarge = recordBytes({});
|
||||
usedTooLarge.writeUInt32LE(4096, 24);
|
||||
const totalTooSmall = recordBytes({});
|
||||
totalTooSmall.writeUInt32LE(512, 28);
|
||||
|
||||
const inventory = await scanCanonicalArtifactInventory(
|
||||
await writeMft("$MFT", Buffer.concat([usedTooLarge, totalTooSmall])),
|
||||
);
|
||||
expect(inventory.coverage.status).toBe("unknown");
|
||||
expect(inventory.limitations).toContainEqual(
|
||||
"2 FILE record(s) declare inconsistent record extents, beginning at offset 0",
|
||||
);
|
||||
expect(inventory.occurrences[0]?.format_metadata).toMatchObject({
|
||||
record_count: 2,
|
||||
fixup_mismatch_count: 0,
|
||||
fixup_unverified_record_count: 0,
|
||||
});
|
||||
});
|
||||
|
||||
it("retries after final source verification fails without publishing its metadata", async () => {
|
||||
const reader = new NtfsMftArtifactReader(
|
||||
await writeMft("$MFT", recordBytes({})),
|
||||
);
|
||||
const read = ArtifactFileSource.prototype.read;
|
||||
const spy = vi
|
||||
.spyOn(ArtifactFileSource.prototype, "read")
|
||||
.mockImplementationOnce(function (this: ArtifactFileSource, signal) {
|
||||
return read.call(this, signal);
|
||||
})
|
||||
.mockRejectedValueOnce(new Error("final source verification failed"));
|
||||
try {
|
||||
await expect(
|
||||
reader.entries()[Symbol.asyncIterator]().next(),
|
||||
).rejects.toThrow("final source verification failed");
|
||||
expect(reader.formatMetadata()).toBeUndefined();
|
||||
await expect(
|
||||
reader.entries()[Symbol.asyncIterator]().next(),
|
||||
).resolves.toMatchObject({
|
||||
done: true,
|
||||
});
|
||||
expect(reader.formatMetadata()).toMatchObject({
|
||||
kind: "ntfs-mft",
|
||||
record_count: 1,
|
||||
});
|
||||
} finally {
|
||||
spy.mockRestore();
|
||||
await reader.close();
|
||||
}
|
||||
});
|
||||
|
||||
it("retries a shared preparation cancelled by another traversal", async () => {
|
||||
const reader = new NtfsMftArtifactReader(
|
||||
await writeMft("$MFT", recordBytes({})),
|
||||
);
|
||||
const controller = new AbortController();
|
||||
let entered: () => void = () => {};
|
||||
const started = new Promise<void>((resolve) => {
|
||||
entered = resolve;
|
||||
});
|
||||
const read = ArtifactFileSource.prototype.read;
|
||||
const spy = vi
|
||||
.spyOn(ArtifactFileSource.prototype, "read")
|
||||
.mockImplementationOnce(async function (this: ArtifactFileSource, signal) {
|
||||
entered();
|
||||
await new Promise<void>((resolve) => {
|
||||
signal?.addEventListener("abort", () => resolve(), { once: true });
|
||||
});
|
||||
assertArtifactReadActive(signal);
|
||||
return read.call(this, signal);
|
||||
});
|
||||
try {
|
||||
const cancelled = reader
|
||||
.entries(controller.signal)
|
||||
[Symbol.asyncIterator]()
|
||||
.next();
|
||||
const cancelledResult = expect(cancelled).rejects.toMatchObject({
|
||||
reason: "cancelled",
|
||||
});
|
||||
await started;
|
||||
const other = reader.entries()[Symbol.asyncIterator]().next();
|
||||
controller.abort("cancelled first traversal");
|
||||
await cancelledResult;
|
||||
await expect(other).resolves.toMatchObject({ done: true });
|
||||
expect(reader.formatMetadata()).toMatchObject({ record_count: 1 });
|
||||
} finally {
|
||||
controller.abort();
|
||||
spy.mockRestore();
|
||||
await reader.close();
|
||||
}
|
||||
});
|
||||
|
||||
it("revalidates the source before reusing a completed header inventory", async () => {
|
||||
const path = await writeMft("$MFT", recordBytes({}));
|
||||
const reader = new NtfsMftArtifactReader(path);
|
||||
try {
|
||||
await reader.entries()[Symbol.asyncIterator]().next();
|
||||
await writeFile(
|
||||
path,
|
||||
Buffer.concat([recordBytes({}), recordBytes({ index: 1 })]),
|
||||
);
|
||||
await expect(
|
||||
reader.entries()[Symbol.asyncIterator]().next(),
|
||||
).rejects.toMatchObject({ reason: "integrity" });
|
||||
} finally {
|
||||
await reader.close();
|
||||
}
|
||||
});
|
||||
@@ -0,0 +1,174 @@
|
||||
import type { Readable } from "node:stream";
|
||||
|
||||
import {
|
||||
ntfsMftMetadataSchema,
|
||||
type ArtifactFormatMetadata,
|
||||
} from "../domain/artifactFormatMetadata.js";
|
||||
import type { RegionDecodeFailure } from "../domain/firmware/regionDecoder.js";
|
||||
import {
|
||||
createMftDecoder,
|
||||
type MftInventory,
|
||||
} from "../domain/windows/ntfsMft.js";
|
||||
import {
|
||||
ArtifactReaderFailure,
|
||||
type ArtifactEntry,
|
||||
type ArtifactReader,
|
||||
type ArtifactReaderTraversal,
|
||||
type ArtifactTraversalOptions,
|
||||
} from "./ArtifactReader.js";
|
||||
import { ArtifactResourceScope } from "./ArtifactResourceScope.js";
|
||||
import {
|
||||
artifactReadCancelledBy,
|
||||
assertArtifactReadActive,
|
||||
} from "./ArtifactFileOperation.js";
|
||||
import { ArtifactFileSource } from "./ArtifactFileSource.js";
|
||||
import { readFileHandleChunks } from "../filesystem/readFileHandleChunks.js";
|
||||
import type { StableRegularFileDescriptor } from "../filesystem/RegularFile.js";
|
||||
|
||||
/**
|
||||
* An MFT stream is its own content: the FILE records describe the source
|
||||
* volume, not members inside the stream, so header facts are attached to
|
||||
* the root occurrence.
|
||||
*/
|
||||
export class NtfsMftArtifactReader implements ArtifactReader {
|
||||
readonly format = "ntfs-mft" as const;
|
||||
readonly #scope = new ArtifactResourceScope();
|
||||
readonly #source: ArtifactFileSource;
|
||||
#prepared:
|
||||
| { signal: AbortSignal | undefined; promise: Promise<void> }
|
||||
| undefined;
|
||||
#metadata: ArtifactFormatMetadata | undefined;
|
||||
#limitations: readonly string[] = [];
|
||||
#closing = false;
|
||||
|
||||
constructor(path: string, admitted?: StableRegularFileDescriptor) {
|
||||
this.#source = new ArtifactFileSource(path, "$MFT stream", admitted);
|
||||
}
|
||||
|
||||
formatMetadata(): ArtifactFormatMetadata | undefined {
|
||||
return this.#metadata === undefined
|
||||
? undefined
|
||||
: structuredClone(this.#metadata);
|
||||
}
|
||||
|
||||
async prepareContainer(
|
||||
expectedSha256: string,
|
||||
signal?: AbortSignal,
|
||||
): Promise<void> {
|
||||
await this.#scope.run(async () => {
|
||||
this.#assertOpen(signal);
|
||||
await this.#source.bindSha256(expectedSha256, signal);
|
||||
});
|
||||
}
|
||||
|
||||
async *entries(
|
||||
signal?: AbortSignal,
|
||||
options?: ArtifactTraversalOptions,
|
||||
): AsyncIterable<ArtifactEntry> {
|
||||
await this.#scope.run(async () => {
|
||||
this.#assertOpen(signal);
|
||||
for (;;) {
|
||||
const pending = (this.#prepared ??= {
|
||||
signal,
|
||||
promise: this.#prepare(signal),
|
||||
});
|
||||
try {
|
||||
await pending.promise;
|
||||
break;
|
||||
} catch (cause: unknown) {
|
||||
if (this.#prepared === pending) this.#prepared = undefined;
|
||||
if (
|
||||
pending.signal !== signal &&
|
||||
signal?.aborted !== true &&
|
||||
artifactReadCancelledBy(cause, pending.signal)
|
||||
) {
|
||||
this.#assertOpen(signal);
|
||||
continue;
|
||||
}
|
||||
throw cause;
|
||||
}
|
||||
}
|
||||
this.#assertOpen(signal);
|
||||
await this.#source.read(signal);
|
||||
});
|
||||
// An MFT declares no members, so the root occurrence is the whole
|
||||
// inventory; the coverage callback reports what its parse did not establish.
|
||||
options?.onCoverage?.(coverageOf(this.#limitations));
|
||||
}
|
||||
|
||||
async open(entry: ArtifactEntry, _signal?: AbortSignal): Promise<Readable> {
|
||||
return this.#scope.run(async () => {
|
||||
throw new ArtifactReaderFailure(
|
||||
"unavailable",
|
||||
`$MFT stream declares no member to open for ${entry.path}`,
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
async close(): Promise<void> {
|
||||
this.#closing = true;
|
||||
await this.#scope.close();
|
||||
await this.#source.close();
|
||||
this.#metadata = undefined;
|
||||
}
|
||||
|
||||
provenance(): readonly [] {
|
||||
return [];
|
||||
}
|
||||
|
||||
#assertOpen(signal?: AbortSignal): void {
|
||||
assertArtifactReadActive(signal);
|
||||
if (this.#closing)
|
||||
throw new ArtifactReaderFailure(
|
||||
"unavailable",
|
||||
"$MFT stream reader is closed",
|
||||
);
|
||||
}
|
||||
|
||||
async #prepare(signal?: AbortSignal): Promise<void> {
|
||||
const image = await this.#decode(signal);
|
||||
await this.#source.read(signal);
|
||||
this.#metadata = ntfsMftMetadataSchema.parse(image.metadata);
|
||||
this.#limitations = image.limitations;
|
||||
}
|
||||
|
||||
async #decode(signal?: AbortSignal): Promise<MftInventory> {
|
||||
const { handle, initial } = await this.#source.read(signal);
|
||||
const decoder = createMftDecoder(initial.size);
|
||||
if (!decoder.ok) throw this.#failure(decoder.error);
|
||||
if (initial.size > 0)
|
||||
for await (const chunk of readFileHandleChunks(handle, {
|
||||
end: initial.size - 1,
|
||||
signal,
|
||||
cancellationFailure: (reason) =>
|
||||
new ArtifactReaderFailure("cancelled", "$MFT stream read cancelled", {
|
||||
cause: reason,
|
||||
}),
|
||||
}) as AsyncIterable<Buffer>) {
|
||||
this.#assertOpen(signal);
|
||||
const written = decoder.value.write(chunk);
|
||||
if (!written.ok) throw this.#failure(written.error);
|
||||
}
|
||||
const image = decoder.value.finish();
|
||||
if (!image.ok) throw this.#failure(image.error);
|
||||
return image.value;
|
||||
}
|
||||
|
||||
#failure(failure: RegionDecodeFailure): ArtifactReaderFailure {
|
||||
return failure.kind === "format"
|
||||
? new ArtifactReaderFailure(
|
||||
"format",
|
||||
`$MFT stream is malformed: ${failure.message}`,
|
||||
)
|
||||
: new ArtifactReaderFailure(
|
||||
"limit",
|
||||
`$MFT stream exceeds a fixed limit: ${failure.message}`,
|
||||
failure.boundary === undefined ? {} : { boundary: failure.boundary },
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
const coverageOf = (limitations: readonly string[]): ArtifactReaderTraversal =>
|
||||
limitations.length === 0
|
||||
? { status: "complete", limitations: [] }
|
||||
: { status: "unknown", limitations };
|
||||
@@ -1,4 +1,7 @@
|
||||
import { artifactReaderLifecycleCases } from "../../tests/fixtures/artifactReaderLifecycle.js";
|
||||
import {
|
||||
artifactPreparationRecoveryCases,
|
||||
artifactReaderLifecycleCases,
|
||||
} from "../../tests/fixtures/artifactReaderLifecycle.js";
|
||||
import { writeFile } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
import { buffer } from "node:stream/consumers";
|
||||
@@ -91,6 +94,14 @@ it("preserves cancellation during payload CRC admission", async () => {
|
||||
}
|
||||
})(),
|
||||
).rejects.toMatchObject({ reason: "cancelled", cause: reason });
|
||||
reads.mockRestore();
|
||||
const recovered = await entries(reader);
|
||||
expect(recovered).toHaveLength(1);
|
||||
expect(
|
||||
(await buffer(await reader.open(recovered[0]!))).equals(
|
||||
Buffer.alloc(128 * 1024, 0x61),
|
||||
),
|
||||
).toBe(true);
|
||||
} finally {
|
||||
await reader.close();
|
||||
reads.mockRestore();
|
||||
@@ -205,3 +216,8 @@ it("reports impossible over-cap component extents as malformed", async () => {
|
||||
await reader.close();
|
||||
}
|
||||
});
|
||||
|
||||
artifactPreparationRecoveryCases({
|
||||
image: () => buildUImage({ payload: Buffer.from("payload") }),
|
||||
createReader: (path, admitted) => new UImageArtifactReader(path, admitted),
|
||||
});
|
||||
@@ -8,7 +8,10 @@ import type {
|
||||
UImageMetadata,
|
||||
} from "../domain/artifactFormatMetadata.js";
|
||||
import { ArtifactFileSource } from "./ArtifactFileSource.js";
|
||||
import { assertArtifactReadActive } from "./ArtifactFileOperation.js";
|
||||
import {
|
||||
artifactReadCancelledBy,
|
||||
assertArtifactReadActive,
|
||||
} from "./ArtifactFileOperation.js";
|
||||
import {
|
||||
ArtifactReaderFailure,
|
||||
copyArtifactEntry,
|
||||
@@ -47,7 +50,9 @@ export class UImageArtifactReader implements ArtifactReader {
|
||||
readonly #entries = new Map<string, UImageEntry>();
|
||||
readonly #streams = new Set<Readable>();
|
||||
readonly #source: ArtifactFileSource;
|
||||
#prepared: Promise<void> | undefined;
|
||||
#prepared:
|
||||
| { signal: AbortSignal | undefined; promise: Promise<void> }
|
||||
| undefined;
|
||||
#header: UImageMetadata | undefined;
|
||||
#closing = false;
|
||||
|
||||
@@ -75,8 +80,27 @@ export class UImageArtifactReader implements ArtifactReader {
|
||||
): AsyncIterable<ArtifactEntry> {
|
||||
await this.#scope.run(async () => {
|
||||
this.#assertOpen(signal);
|
||||
this.#prepared ??= this.#prepare(signal);
|
||||
await this.#prepared;
|
||||
for (;;) {
|
||||
const pending = (this.#prepared ??= {
|
||||
signal,
|
||||
promise: this.#prepare(signal),
|
||||
});
|
||||
try {
|
||||
await pending.promise;
|
||||
break;
|
||||
} catch (cause: unknown) {
|
||||
if (this.#prepared === pending) this.#prepared = undefined;
|
||||
if (
|
||||
pending.signal !== signal &&
|
||||
signal?.aborted !== true &&
|
||||
artifactReadCancelledBy(cause, pending.signal)
|
||||
) {
|
||||
this.#assertOpen(signal);
|
||||
continue;
|
||||
}
|
||||
throw cause;
|
||||
}
|
||||
}
|
||||
await this.#source.read(signal);
|
||||
});
|
||||
this.#assertOpen(signal);
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
import { artifactPreparationRecoveryCases } from "../../tests/fixtures/artifactReaderLifecycle.js";
|
||||
import { ArtifactFileSource } from "./ArtifactFileSource.js";
|
||||
import { truncate, writeFile } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
import { buffer, text } from "node:stream/consumers";
|
||||
@@ -43,6 +45,68 @@ const repeatedBlock = (data: Buffer): Buffer => {
|
||||
]);
|
||||
};
|
||||
|
||||
it("retries UnityFs preparation cancelled during file reading", async () => {
|
||||
const path = await fixture(buildUnityFs([{ path: "a", data: "x" }]));
|
||||
const admitted = await openRegularFile(path, { symlinks: "reject" });
|
||||
const reader = new UnityFsArtifactReader(path, admitted);
|
||||
const controller = new AbortController();
|
||||
const reason = { stage: "header read" };
|
||||
const nativeRead = admitted.handle.read.bind(admitted.handle);
|
||||
const read = vi.spyOn(admitted.handle, "read").mockImplementation((async (
|
||||
...args: unknown[]
|
||||
) => {
|
||||
const result = await Reflect.apply(nativeRead, admitted.handle, args);
|
||||
controller.abort(reason);
|
||||
return result;
|
||||
}) as typeof admitted.handle.read);
|
||||
try {
|
||||
await expect(
|
||||
(async () => {
|
||||
for await (const _ of reader.entries(controller.signal)) {
|
||||
}
|
||||
})(),
|
||||
).rejects.toMatchObject({ reason: "cancelled", cause: reason });
|
||||
read.mockRestore();
|
||||
expect((await entries(reader)).map((entry) => entry.path)).toEqual(["a"]);
|
||||
} finally {
|
||||
read.mockRestore();
|
||||
await reader.close();
|
||||
await admitted.handle.close();
|
||||
}
|
||||
});
|
||||
|
||||
it("does not publish UnityFs entries when final source verification fails", async () => {
|
||||
const path = await fixture(buildUnityFs([{ path: "a", data: "x" }]));
|
||||
const control = new UnityFsArtifactReader(path);
|
||||
const reader = new UnityFsArtifactReader(path);
|
||||
const expected = (await entries(control))[0]!;
|
||||
const nativeRead = ArtifactFileSource.prototype.read;
|
||||
const physical = Object.assign(
|
||||
new Error("final source verification failed"),
|
||||
{ code: "EIO" },
|
||||
);
|
||||
let calls = 0;
|
||||
const read = vi
|
||||
.spyOn(ArtifactFileSource.prototype, "read")
|
||||
.mockImplementation(function (this: ArtifactFileSource, ...args) {
|
||||
calls += 1;
|
||||
if (calls === 2) return Promise.reject(physical);
|
||||
return nativeRead.apply(this, args);
|
||||
});
|
||||
try {
|
||||
await expect(entries(reader)).rejects.toBe(physical);
|
||||
await expect(reader.open(expected)).rejects.toMatchObject({
|
||||
reason: "integrity",
|
||||
});
|
||||
read.mockRestore();
|
||||
expect((await entries(reader)).map((entry) => entry.path)).toEqual(["a"]);
|
||||
} finally {
|
||||
read.mockRestore();
|
||||
await reader.close();
|
||||
await control.close();
|
||||
}
|
||||
});
|
||||
|
||||
describe("UnityFS allocation and malformed-input boundaries", () => {
|
||||
it("rejects an oversized directory extending past the physical file", async () => {
|
||||
const bytes = buildUnityFs([{ path: "data", data: "x" }]);
|
||||
@@ -431,3 +495,8 @@ describe("UnityFS stream capacity and cache ownership", () => {
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
artifactPreparationRecoveryCases({
|
||||
image: () => buildUnityFs([{ path: "a", data: "x" }]),
|
||||
createReader: (path, admitted) => new UnityFsArtifactReader(path, admitted),
|
||||
});
|
||||
@@ -12,7 +12,10 @@ import {
|
||||
type ArtifactTraversalOptions,
|
||||
} from "./ArtifactReader.js";
|
||||
import { ArtifactResourceScope } from "./ArtifactResourceScope.js";
|
||||
import { assertArtifactReadActive } from "./ArtifactFileOperation.js";
|
||||
import {
|
||||
artifactReadCancelledBy,
|
||||
assertArtifactReadActive,
|
||||
} from "./ArtifactFileOperation.js";
|
||||
import { readArtifactFileBytes } from "./ArtifactFileRange.js";
|
||||
import type {
|
||||
ArtifactFormatMetadata,
|
||||
@@ -52,10 +55,12 @@ interface UnityFsEntryState {
|
||||
export class UnityFsArtifactReader implements ArtifactReader {
|
||||
readonly format = "unityfs" as const;
|
||||
readonly #scope = new ArtifactResourceScope();
|
||||
readonly #entries = new Map<string, UnityFsEntryState>();
|
||||
#entries = new Map<string, UnityFsEntryState>();
|
||||
readonly #streams = new Map<Readable, AbortController>();
|
||||
readonly #source: ArtifactFileSource;
|
||||
#prepared: Promise<void> | undefined;
|
||||
#prepared:
|
||||
| { signal: AbortSignal | undefined; promise: Promise<void> }
|
||||
| undefined;
|
||||
#header: UnityFsMetadata | undefined;
|
||||
#blocks: readonly UnityFsBlock[] = [];
|
||||
#dataStart = 0;
|
||||
@@ -88,8 +93,27 @@ export class UnityFsArtifactReader implements ArtifactReader {
|
||||
): AsyncIterable<ArtifactEntry> {
|
||||
await this.#scope.run(async () => {
|
||||
this.#assertOpen(signal);
|
||||
this.#prepared ??= this.#prepare(signal);
|
||||
await this.#prepared;
|
||||
for (;;) {
|
||||
const pending = (this.#prepared ??= {
|
||||
signal,
|
||||
promise: this.#prepare(signal),
|
||||
});
|
||||
try {
|
||||
await pending.promise;
|
||||
break;
|
||||
} catch (cause: unknown) {
|
||||
if (this.#prepared === pending) this.#prepared = undefined;
|
||||
if (
|
||||
pending.signal !== signal &&
|
||||
signal?.aborted !== true &&
|
||||
artifactReadCancelledBy(cause, pending.signal)
|
||||
) {
|
||||
this.#assertOpen(signal);
|
||||
continue;
|
||||
}
|
||||
throw cause;
|
||||
}
|
||||
}
|
||||
await this.#source.read(signal);
|
||||
});
|
||||
this.#assertOpen(signal);
|
||||
@@ -229,7 +253,7 @@ export class UnityFsArtifactReader implements ArtifactReader {
|
||||
if (header === undefined)
|
||||
throw unityFsFormatFailure("no complete UnityFS header");
|
||||
const status = header.layout.status;
|
||||
this.#header = {
|
||||
const metadata: UnityFsMetadata = {
|
||||
kind: "unityfs",
|
||||
signature: "UnityFS",
|
||||
format_version: header.formatVersion,
|
||||
@@ -245,7 +269,12 @@ export class UnityFsArtifactReader implements ArtifactReader {
|
||||
data_hash: null,
|
||||
directory_status: status,
|
||||
};
|
||||
if (status !== "available") return;
|
||||
if (status !== "available") {
|
||||
await this.#source.read(signal);
|
||||
this.#assertOpen(signal);
|
||||
this.#header = metadata;
|
||||
return;
|
||||
}
|
||||
const { dataHash, blocks, nodes, dataStart } = await readUnityFsBlocksInfo(
|
||||
handle,
|
||||
header,
|
||||
@@ -259,44 +288,55 @@ export class UnityFsArtifactReader implements ArtifactReader {
|
||||
if (streamBytes > BigInt(budget))
|
||||
throw expansionLimit("declared virtual stream", streamBytes, budget);
|
||||
}
|
||||
this.#header = {
|
||||
...this.#header,
|
||||
const preparedMetadata: UnityFsMetadata = {
|
||||
...metadata,
|
||||
data_hash: dataHash,
|
||||
block_count: blocks.length,
|
||||
data_start: dataStart,
|
||||
};
|
||||
this.#dataStart = dataStart;
|
||||
// Keep the source block count above, but do not revisit zero-output rows
|
||||
// for every member. Parsed stream/compressed offsets remain authoritative.
|
||||
this.#blocks = blocks.filter((block) => block.uncompressedSize > 0);
|
||||
this.#remainingDecodeBytes = budget;
|
||||
await this.#registerNodes(nodes, budget, signal);
|
||||
const readableBlocks = blocks.filter((block) => block.uncompressedSize > 0);
|
||||
const registered = await this.#registerNodes(
|
||||
nodes,
|
||||
readableBlocks,
|
||||
budget,
|
||||
signal,
|
||||
);
|
||||
await this.#source.read(signal);
|
||||
this.#assertOpen(signal);
|
||||
this.#header = preparedMetadata;
|
||||
this.#dataStart = dataStart;
|
||||
this.#blocks = readableBlocks;
|
||||
this.#remainingDecodeBytes = budget;
|
||||
this.#entries = registered.entries;
|
||||
this.#remainingReadOperations = registered.remainingReadOperations;
|
||||
}
|
||||
|
||||
async #registerNodes(
|
||||
nodes: readonly UnityFsNode[],
|
||||
blocks: readonly UnityFsBlock[],
|
||||
budget: number,
|
||||
signal?: AbortSignal,
|
||||
): Promise<void> {
|
||||
): Promise<{
|
||||
entries: Map<string, UnityFsEntryState>;
|
||||
remainingReadOperations: number;
|
||||
}> {
|
||||
const entries = new Map<string, UnityFsEntryState>();
|
||||
let readableBytes = 0n;
|
||||
for (let index = 0; index < nodes.length; index++) {
|
||||
if (index % 256 === 0) await setImmediate();
|
||||
this.#assertOpen(signal);
|
||||
const node = nodes[index]!;
|
||||
const firstBlock =
|
||||
node.size === 0 ? -1 : blockEndingAfter(this.#blocks, node.offset);
|
||||
node.size === 0 ? -1 : blockEndingAfter(blocks, node.offset);
|
||||
const lastBlock =
|
||||
node.size === 0
|
||||
? -1
|
||||
: blockEndingAfter(this.#blocks, node.offset + node.size - 1);
|
||||
: blockEndingAfter(blocks, node.offset + node.size - 1);
|
||||
if (node.size !== 0 && (firstBlock < 0 || lastBlock < firstBlock))
|
||||
throw unityFsFormatFailure("member has no block range");
|
||||
const unavailable = firstUnavailableBlock(
|
||||
this.#blocks,
|
||||
firstBlock,
|
||||
lastBlock,
|
||||
);
|
||||
const unavailable = firstUnavailableBlock(blocks, firstBlock, lastBlock);
|
||||
const kind =
|
||||
(node.flags & 1) !== 0 ? ("directory" as const) : ("file" as const);
|
||||
if (kind === "file" && unavailable === undefined) {
|
||||
@@ -331,7 +371,7 @@ export class UnityFsArtifactReader implements ArtifactReader {
|
||||
flags: node.flags,
|
||||
},
|
||||
} satisfies UnityFsEntry;
|
||||
this.#entries.set(adapterKey, {
|
||||
entries.set(adapterKey, {
|
||||
entry,
|
||||
firstBlock,
|
||||
lastBlock,
|
||||
@@ -340,14 +380,15 @@ export class UnityFsArtifactReader implements ArtifactReader {
|
||||
}
|
||||
// Four ordinary passes over admitted blocks, nodes and plaintext chunks,
|
||||
// with a small-reader floor. Byte quotas alone cannot bound tiny-block I/O.
|
||||
this.#remainingReadOperations =
|
||||
const remainingReadOperations =
|
||||
4 *
|
||||
Math.max(
|
||||
1024,
|
||||
this.#blocks.length +
|
||||
blocks.length +
|
||||
nodes.length +
|
||||
Math.ceil(Number(readableBytes) / CHUNK_BYTES),
|
||||
);
|
||||
return { entries, remainingReadOperations };
|
||||
}
|
||||
|
||||
async *#memberChunks(
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
import { artifactPreparationRecoveryCases } from "../../tests/fixtures/artifactReaderLifecycle.js";
|
||||
import { ArtifactFileSource } from "./ArtifactFileSource.js";
|
||||
import { once } from "node:events";
|
||||
import { writeFile } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
@@ -26,6 +28,68 @@ const entries = async (reader: WxapkgArtifactReader) => {
|
||||
return result;
|
||||
};
|
||||
|
||||
it("retries Wxapkg preparation cancelled during file reading", async () => {
|
||||
const path = await fixture(buildWxapkg([{ path: "a", data: "x" }]));
|
||||
const admitted = await openRegularFile(path, { symlinks: "reject" });
|
||||
const reader = new WxapkgArtifactReader(path, admitted);
|
||||
const controller = new AbortController();
|
||||
const reason = { stage: "header read" };
|
||||
const nativeRead = admitted.handle.read.bind(admitted.handle);
|
||||
const read = vi.spyOn(admitted.handle, "read").mockImplementation((async (
|
||||
...args: unknown[]
|
||||
) => {
|
||||
const result = await Reflect.apply(nativeRead, admitted.handle, args);
|
||||
controller.abort(reason);
|
||||
return result;
|
||||
}) as typeof admitted.handle.read);
|
||||
try {
|
||||
await expect(
|
||||
(async () => {
|
||||
for await (const _ of reader.entries(controller.signal)) {
|
||||
}
|
||||
})(),
|
||||
).rejects.toMatchObject({ reason: "cancelled", cause: reason });
|
||||
read.mockRestore();
|
||||
expect((await entries(reader)).map((entry) => entry.path)).toEqual(["a"]);
|
||||
} finally {
|
||||
read.mockRestore();
|
||||
await reader.close();
|
||||
await admitted.handle.close();
|
||||
}
|
||||
});
|
||||
|
||||
it("does not publish Wxapkg entries when final source verification fails", async () => {
|
||||
const path = await fixture(buildWxapkg([{ path: "a", data: "x" }]));
|
||||
const control = new WxapkgArtifactReader(path);
|
||||
const reader = new WxapkgArtifactReader(path);
|
||||
const expected = (await entries(control))[0]!;
|
||||
const nativeRead = ArtifactFileSource.prototype.read;
|
||||
const physical = Object.assign(
|
||||
new Error("final source verification failed"),
|
||||
{ code: "EIO" },
|
||||
);
|
||||
let calls = 0;
|
||||
const read = vi
|
||||
.spyOn(ArtifactFileSource.prototype, "read")
|
||||
.mockImplementation(function (this: ArtifactFileSource, ...args) {
|
||||
calls += 1;
|
||||
if (calls === 2) return Promise.reject(physical);
|
||||
return nativeRead.apply(this, args);
|
||||
});
|
||||
try {
|
||||
await expect(entries(reader)).rejects.toBe(physical);
|
||||
await expect(reader.open(expected)).rejects.toMatchObject({
|
||||
reason: "integrity",
|
||||
});
|
||||
read.mockRestore();
|
||||
expect((await entries(reader)).map((entry) => entry.path)).toEqual(["a"]);
|
||||
} finally {
|
||||
read.mockRestore();
|
||||
await reader.close();
|
||||
await control.close();
|
||||
}
|
||||
});
|
||||
|
||||
describe("wxapkg allocation and malformed-input boundaries", () => {
|
||||
it("admits the encoded table budget before reading it", async () => {
|
||||
const bytes = buildWxapkg([{ path: "a", data: "x" }]);
|
||||
@@ -308,3 +372,8 @@ describe("wxapkg stream capacity and freshness", () => {
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
artifactPreparationRecoveryCases({
|
||||
image: () => buildWxapkg([{ path: "a", data: "x" }]),
|
||||
createReader: (path, admitted) => new WxapkgArtifactReader(path, admitted),
|
||||
});
|
||||
@@ -12,7 +12,10 @@ import {
|
||||
type ArtifactTraversalOptions,
|
||||
} from "./ArtifactReader.js";
|
||||
import { ArtifactResourceScope } from "./ArtifactResourceScope.js";
|
||||
import { assertArtifactReadActive } from "./ArtifactFileOperation.js";
|
||||
import {
|
||||
artifactReadCancelledBy,
|
||||
assertArtifactReadActive,
|
||||
} from "./ArtifactFileOperation.js";
|
||||
import { readArtifactFileBytes } from "./ArtifactFileRange.js";
|
||||
import type {
|
||||
ArtifactFormatMetadata,
|
||||
@@ -48,10 +51,12 @@ interface WxapkgEntryState {
|
||||
export class WxapkgArtifactReader implements ArtifactReader {
|
||||
readonly format = "wxapkg" as const;
|
||||
readonly #scope = new ArtifactResourceScope();
|
||||
readonly #entries = new Map<string, WxapkgEntryState>();
|
||||
#entries = new Map<string, WxapkgEntryState>();
|
||||
readonly #streams = new Map<Readable, AbortController>();
|
||||
readonly #source: ArtifactFileSource;
|
||||
#prepared: Promise<void> | undefined;
|
||||
#prepared:
|
||||
| { signal: AbortSignal | undefined; promise: Promise<void> }
|
||||
| undefined;
|
||||
#header: WxapkgMetadata | undefined;
|
||||
#remainingReadOperations = 0;
|
||||
#admittedReadOperations = 0;
|
||||
@@ -81,8 +86,27 @@ export class WxapkgArtifactReader implements ArtifactReader {
|
||||
): AsyncIterable<ArtifactEntry> {
|
||||
await this.#scope.run(async () => {
|
||||
this.#assertOpen(signal);
|
||||
this.#prepared ??= this.#prepare(signal);
|
||||
await this.#prepared;
|
||||
for (;;) {
|
||||
const pending = (this.#prepared ??= {
|
||||
signal,
|
||||
promise: this.#prepare(signal),
|
||||
});
|
||||
try {
|
||||
await pending.promise;
|
||||
break;
|
||||
} catch (cause: unknown) {
|
||||
if (this.#prepared === pending) this.#prepared = undefined;
|
||||
if (
|
||||
pending.signal !== signal &&
|
||||
signal?.aborted !== true &&
|
||||
artifactReadCancelledBy(cause, pending.signal)
|
||||
) {
|
||||
this.#assertOpen(signal);
|
||||
continue;
|
||||
}
|
||||
throw cause;
|
||||
}
|
||||
}
|
||||
await this.#source.read(signal);
|
||||
});
|
||||
this.#assertOpen(signal);
|
||||
@@ -201,7 +225,7 @@ export class WxapkgArtifactReader implements ArtifactReader {
|
||||
const header = await readWxapkgHeader(handle, initial.size, signal);
|
||||
if (header === undefined)
|
||||
throw wxapkgFormatFailure("no complete wxapkg header");
|
||||
this.#header = {
|
||||
const metadata: WxapkgMetadata = {
|
||||
kind: "wxapkg",
|
||||
info: header.layout.status === "available" ? header.info : null,
|
||||
index_length:
|
||||
@@ -212,7 +236,12 @@ export class WxapkgArtifactReader implements ArtifactReader {
|
||||
header.layout.status === "available" ? header.fileCount : null,
|
||||
directory_status: header.layout.status,
|
||||
};
|
||||
if (header.layout.status !== "available") return;
|
||||
if (header.layout.status !== "available") {
|
||||
await this.#source.read(signal);
|
||||
this.#assertOpen(signal);
|
||||
this.#header = metadata;
|
||||
return;
|
||||
}
|
||||
const { members } = await readWxapkgFileTable(
|
||||
handle,
|
||||
header,
|
||||
@@ -220,6 +249,7 @@ export class WxapkgArtifactReader implements ArtifactReader {
|
||||
signal,
|
||||
);
|
||||
const budget = artifactExpansionBudgetBytes(initial.size);
|
||||
const entries = new Map<string, WxapkgEntryState>();
|
||||
let declaredBytes = 0n;
|
||||
for (let index = 0; index < members.length; index++) {
|
||||
if (index % 256 === 0) await setImmediate();
|
||||
@@ -234,7 +264,7 @@ export class WxapkgArtifactReader implements ArtifactReader {
|
||||
{ boundary: "memory" },
|
||||
);
|
||||
const adapterKey = `wxapkg-entry:${String(index)}`;
|
||||
this.#entries.set(adapterKey, {
|
||||
entries.set(adapterKey, {
|
||||
entry: {
|
||||
path: member.path,
|
||||
sourcePath: member.declaredPath,
|
||||
@@ -260,14 +290,18 @@ export class WxapkgArtifactReader implements ArtifactReader {
|
||||
});
|
||||
}
|
||||
// Header, table and per-chunk member reads; a small-reader floor applies.
|
||||
this.#admittedReadOperations =
|
||||
const admittedReadOperations =
|
||||
4 *
|
||||
Math.max(
|
||||
1024,
|
||||
members.length + Math.ceil(Number(declaredBytes) / CHUNK_BYTES),
|
||||
);
|
||||
this.#remainingReadOperations = this.#admittedReadOperations;
|
||||
await this.#source.read(signal);
|
||||
this.#assertOpen(signal);
|
||||
this.#header = metadata;
|
||||
this.#entries = entries;
|
||||
this.#admittedReadOperations = admittedReadOperations;
|
||||
this.#remainingReadOperations = admittedReadOperations;
|
||||
}
|
||||
|
||||
async *#memberChunks(
|
||||
|
||||
@@ -43,7 +43,8 @@ type FormatsWithReaders =
|
||||
| "uf2"
|
||||
| "dfuse"
|
||||
| "esp-flash-image"
|
||||
| "fit";
|
||||
| "fit"
|
||||
| "ntfs-mft";
|
||||
|
||||
const readerFormatsMatch: Equal<ArtifactReaderFormat, FormatsWithReaders> =
|
||||
true;
|
||||
|
||||
@@ -40,6 +40,7 @@ import { NodeSeaArtifactReader } from "./NodeSeaArtifactReader.js";
|
||||
import type { PyInstallerAdmission } from "./pyinstaller/PyInstallerAdmission.js";
|
||||
import { PyInstallerArtifactReader } from "./PyInstallerArtifactReader.js";
|
||||
import { Uf2ArtifactReader } from "./Uf2ArtifactReader.js";
|
||||
import { NtfsMftArtifactReader } from "./NtfsMftArtifactReader.js";
|
||||
import { UImageArtifactReader } from "./UImageArtifactReader.js";
|
||||
import { UnityFsArtifactReader } from "./UnityFsArtifactReader.js";
|
||||
import { WxapkgArtifactReader } from "./WxapkgArtifactReader.js";
|
||||
@@ -465,6 +466,14 @@ const NON_ZIP_FORMATS = {
|
||||
createReader: (path, options) =>
|
||||
new FitArtifactReader(path, sourceOf(options)),
|
||||
}),
|
||||
"ntfs-mft": described("ntfs-mft", {
|
||||
...plain,
|
||||
retainClassifiedSource: true,
|
||||
readerLimitation:
|
||||
"MFT inventory reports whole-stream counts and a bounded sample of the first 32 decoded FILE record headers, with the omitted-header count. Attributes, filenames, data streams and the $UsnJrnl:$J journal are not decoded, so no path, file size or change history is established.",
|
||||
createReader: (path, options) =>
|
||||
new NtfsMftArtifactReader(path, sourceOf(options)),
|
||||
}),
|
||||
unknown: described("unknown", plain),
|
||||
} satisfies Record<
|
||||
Exclude<ArtifactFormatId, ZipPackageFormat>,
|
||||
|
||||
@@ -576,6 +576,28 @@ it("classifies nested UF2 by block magics and DfuSe by prefix and suffix", () =>
|
||||
).toMatchObject({ format: "file" });
|
||||
});
|
||||
|
||||
it("classifies a $MFT stream by its FILE record header, under any name", () => {
|
||||
const mft = Buffer.alloc(2_048);
|
||||
mft.write("FILE", 0, "latin1");
|
||||
mft.writeUInt32LE(1024, 28);
|
||||
for (const path of ["evidence/$MFT", "evidence/mft.raw", "evidence/records"])
|
||||
expect(classifyArtifactBytesAtPath(path, mft)).toEqual({
|
||||
kind: "executable",
|
||||
format: "ntfs-mft",
|
||||
});
|
||||
});
|
||||
|
||||
it("leaves plain content beginning with FILE unclassified as MFT", () => {
|
||||
const text = Buffer.from(
|
||||
"FILE is a plain text prefix, not an NTFS record header",
|
||||
);
|
||||
expect(
|
||||
classifyArtifactBytesAtPath("evidence/readme.txt", text),
|
||||
).toMatchObject({
|
||||
format: "file",
|
||||
});
|
||||
});
|
||||
|
||||
it("classifies a FIT by a /images node within the first 4 KiB", () => {
|
||||
const images = [{ name: "kernel", data: Buffer.from("kernel") }];
|
||||
const late = devicetreeFixture({
|
||||
|
||||
@@ -11,6 +11,7 @@ import { recognizeEspImage } from "../../domain/firmware/espImage.js";
|
||||
import { hasUf2Block } from "../../domain/firmware/uf2.js";
|
||||
import { isDfuSeFile } from "../../domain/firmware/dfuse.js";
|
||||
import { hasFitImagesNode } from "../../domain/firmware/fitImage.js";
|
||||
import { hasMftFileRecordHeader } from "../../domain/windows/ntfsMft.js";
|
||||
import type { ArtifactClassificationEvidence } from "../ArtifactClassificationProbe.js";
|
||||
import { asarHeaderAdmissionFailure } from "../AsarHeader.js";
|
||||
import { hasDestinationPath } from "../ArtifactPaths.js";
|
||||
@@ -622,6 +623,7 @@ export const classifyArtifactBytes = (
|
||||
return "dfuse";
|
||||
if (recognizeEspImage(prefix, fileSize) === "esp-image") return "esp-image";
|
||||
if (hasFitImagesNode(prefix)) return "fit";
|
||||
if (hasMftFileRecordHeader(prefix)) return "ntfs-mft";
|
||||
if (hasBinaryPlistSignature(prefix)) return "plist";
|
||||
if (hasXmlPlistSignature(prefix)) return "plist";
|
||||
const fatFormat = classifyFatArtifact(evidence, fileSize);
|
||||
|
||||
@@ -0,0 +1,158 @@
|
||||
import { createHash } from "node:crypto";
|
||||
|
||||
import { expect, it, vi } from "vitest";
|
||||
|
||||
import { StructuredContentMcpClient } from "../providers/StructuredContentMcpClient.js";
|
||||
import { AgentEnvComputer } from "./AgentEnvComputer.js";
|
||||
import type { ComputerHelper } from "./AgentEnvComputerHelper.js";
|
||||
|
||||
it.each(["http", "invalid_json", "readiness"] as const)(
|
||||
"redacts the complete control token in %s failure diagnostics",
|
||||
async (failure) => {
|
||||
const token = "fake-control-token-keep-private-123456789";
|
||||
const status = failure === "http" ? 403 : 200;
|
||||
const ordinary =
|
||||
failure === "http" ? "x".repeat(500) : "ordinary target text";
|
||||
const body =
|
||||
failure === "readiness"
|
||||
? JSON.stringify({
|
||||
ready: false,
|
||||
checks: [
|
||||
{
|
||||
name: "app",
|
||||
ok: false,
|
||||
final: true,
|
||||
detail: `${token} ${ordinary}`,
|
||||
},
|
||||
],
|
||||
})
|
||||
: failure === "http"
|
||||
? `${ordinary}${token}`
|
||||
: `${token} ${ordinary}`;
|
||||
const connect = vi
|
||||
.spyOn(StructuredContentMcpClient.prototype, "connect")
|
||||
.mockResolvedValue(undefined);
|
||||
const close = vi
|
||||
.spyOn(StructuredContentMcpClient.prototype, "close")
|
||||
.mockResolvedValue(undefined);
|
||||
const fetch = vi
|
||||
.spyOn(globalThis, "fetch")
|
||||
.mockResolvedValue(new Response(body, { status }));
|
||||
try {
|
||||
const error = await AgentEnvComputer.connect(
|
||||
{
|
||||
handshake: async () => ({
|
||||
sandbox_id: "controlled-sandbox",
|
||||
base_url: "http://127.0.0.1:1",
|
||||
mcp_url: "http://127.0.0.1:1/mcp",
|
||||
control_token: token,
|
||||
}),
|
||||
} as unknown as ComputerHelper,
|
||||
new AbortController().signal,
|
||||
).catch((cause: unknown) => cause);
|
||||
expect(error).toBeInstanceOf(Error);
|
||||
if (!(error instanceof Error)) throw error;
|
||||
expect(error.message).not.toContain(token.slice(0, 10));
|
||||
expect(error.message).toContain("[control tok");
|
||||
expect(error.message).toContain(ordinary.slice(0, 100));
|
||||
if (failure !== "readiness") {
|
||||
expect(error.cause).toBeInstanceOf(Error);
|
||||
expect(String(error.cause)).not.toContain(token.slice(0, 10));
|
||||
}
|
||||
} finally {
|
||||
fetch.mockRestore();
|
||||
close.mockRestore();
|
||||
connect.mockRestore();
|
||||
}
|
||||
},
|
||||
);
|
||||
it.each(["canonical", "missing", "rpc_error"] as const)(
|
||||
"validates recording replies and redacts their errors (%s)",
|
||||
async (answer) => {
|
||||
const present = answer === "canonical";
|
||||
const token = "fake-recording-control-token-123456789";
|
||||
const path = "rec-0123/0000.mp4";
|
||||
const homePath = `.agentenv-recordings/${path}`;
|
||||
const bytes = Buffer.from("controlled recording");
|
||||
const connect = vi
|
||||
.spyOn(StructuredContentMcpClient.prototype, "connect")
|
||||
.mockResolvedValue(undefined);
|
||||
const close = vi
|
||||
.spyOn(StructuredContentMcpClient.prototype, "close")
|
||||
.mockResolvedValue(undefined);
|
||||
const call = vi
|
||||
.spyOn(StructuredContentMcpClient.prototype, "callTool")
|
||||
.mockResolvedValue({ content: [] });
|
||||
const fetch = vi
|
||||
.spyOn(globalThis, "fetch")
|
||||
.mockResolvedValueOnce(Response.json({ ready: true }))
|
||||
.mockResolvedValueOnce(
|
||||
Response.json(
|
||||
answer === "rpc_error"
|
||||
? { error: { message: `${token} ordinary target text` } }
|
||||
: {
|
||||
result: {
|
||||
parts: [
|
||||
{
|
||||
kind: "file",
|
||||
file: { bytes: bytes.toString("base64") },
|
||||
metadata: {
|
||||
"urn:unrelated": {
|
||||
path: present ? "wrong-path" : homePath,
|
||||
},
|
||||
...(present
|
||||
? {
|
||||
"urn:agentenv:data-objects/v1": {
|
||||
path: homePath,
|
||||
},
|
||||
}
|
||||
: {}),
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
),
|
||||
);
|
||||
try {
|
||||
const { computer } = await AgentEnvComputer.connect(
|
||||
{
|
||||
handshake: async () => ({
|
||||
sandbox_id: "controlled-sandbox",
|
||||
base_url: "http://127.0.0.1:1",
|
||||
mcp_url: "http://127.0.0.1:1/mcp",
|
||||
control_token: token,
|
||||
}),
|
||||
release: async () => ({ verified: true, teardown: "agentenv" }),
|
||||
} as unknown as ComputerHelper,
|
||||
new AbortController().signal,
|
||||
);
|
||||
const read = computer.recordingSegments({
|
||||
recording_id: "rec-0123",
|
||||
segments: [
|
||||
{
|
||||
index: 0,
|
||||
path,
|
||||
bytes: bytes.length,
|
||||
sha256: createHash("sha256").update(bytes).digest("hex"),
|
||||
},
|
||||
],
|
||||
});
|
||||
if (present) await expect(read).resolves.toEqual([{ path, bytes }]);
|
||||
else if (answer === "rpc_error") {
|
||||
const failure = await read.catch((cause: unknown) => cause);
|
||||
expect(failure).toBeInstanceOf(Error);
|
||||
if (!(failure instanceof Error)) throw failure;
|
||||
expect(failure.message).toContain("ordinary target text");
|
||||
expect(failure.message).toContain("[control token]");
|
||||
expect(failure.message).not.toContain(token);
|
||||
} else await expect(read).rejects.toThrow("missing or differs");
|
||||
await computer.close();
|
||||
} finally {
|
||||
fetch.mockRestore();
|
||||
call.mockRestore();
|
||||
close.mockRestore();
|
||||
connect.mockRestore();
|
||||
}
|
||||
},
|
||||
);
|
||||
@@ -14,6 +14,7 @@ import type {
|
||||
ComputerRelease,
|
||||
} from "./AgentEnvComputerHelper.js";
|
||||
import {
|
||||
DATA_OBJECTS,
|
||||
dataGetSchema,
|
||||
readHomeFiles,
|
||||
writeHomeFiles,
|
||||
@@ -498,10 +499,10 @@ export class AgentEnvComputer {
|
||||
);
|
||||
if ("error" in answer)
|
||||
throw new Error(
|
||||
`data/get of recording segment ${segment.path} failed: ${answer.error.message}`,
|
||||
`data/get of recording segment ${segment.path} failed: ${this.redacted(answer.error.message)}`,
|
||||
);
|
||||
const encoded = answer.result.parts.find(
|
||||
(part) => Object.values(part.metadata ?? {})[0]?.path === path,
|
||||
(part) => part.metadata?.[DATA_OBJECTS]?.path === path,
|
||||
)?.file?.bytes;
|
||||
const bytes =
|
||||
encoded === undefined ? undefined : Buffer.from(encoded, "base64");
|
||||
@@ -653,9 +654,11 @@ export class AgentEnvComputer {
|
||||
const failed = waiting.find((check) => check.final === true);
|
||||
if (failed !== undefined || Date.now() > deadline)
|
||||
throw new ComputerStartFailure(
|
||||
failed === undefined
|
||||
? `The computer was not ready within ${String(READY_TIMEOUT_MS / 1000)} s; waiting on ${waiting.map((check) => check.name).join(", ")}.`
|
||||
: `The computer's ${failed.name} check failed: ${failed.detail ?? "no detail"}.`,
|
||||
this.redacted(
|
||||
failed === undefined
|
||||
? `The computer was not ready within ${String(READY_TIMEOUT_MS / 1000)} s; waiting on ${waiting.map((check) => check.name).join(", ")}.`
|
||||
: `The computer's ${failed.name} check failed: ${failed.detail ?? "no detail"}.`,
|
||||
),
|
||||
"Read the helper's output; `agent-env run <bundle> --task smoke --no-emulate` reproduces the deploy.",
|
||||
{ reason: "not_ready" },
|
||||
);
|
||||
@@ -702,8 +705,15 @@ export class AgentEnvComputer {
|
||||
const text = await readBounded(response, limit.bytes, limit.what);
|
||||
if (response.status < 200 || response.status > 299)
|
||||
throw new Error(
|
||||
`The computer answered ${path} with HTTP ${String(response.status)}: ${this.redacted(text.slice(0, 512))}`,
|
||||
`The computer answered ${path} with HTTP ${String(response.status)}: ${this.redacted(text).slice(0, 512)}`,
|
||||
);
|
||||
return JSON.parse(text) as unknown;
|
||||
try {
|
||||
return JSON.parse(text) as unknown;
|
||||
} catch {
|
||||
// Parser diagnostics can quote a token fragment that exact redaction cannot match.
|
||||
throw new Error(
|
||||
`The computer answered ${path} with invalid JSON: ${this.redacted(text).slice(0, 512)}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,54 @@
|
||||
import { chmod, lstat, mkdir } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
|
||||
import { expect, it } from "vitest";
|
||||
|
||||
import { createTestTempDirectory } from "../../tests/fixtures/temporaryDirectory.js";
|
||||
import { desktopApplicationInventorySchema } from "../domain/desktop/desktopCapture.js";
|
||||
import { stateDirectories } from "./ApplicationStateDirectories.js";
|
||||
|
||||
it.skipIf(process.platform === "win32" || process.getuid?.() === 0)(
|
||||
"preserves unknown state-path existence when permission denies the probe",
|
||||
async () => {
|
||||
const home = await createTestTempDirectory("state-path-existence-");
|
||||
const sealed = join(home, "sealed");
|
||||
const denied = join(sealed, "notes");
|
||||
const present = join(home, ".notes");
|
||||
const absent = join(home, ".local/share/notes");
|
||||
await mkdir(denied, { recursive: true });
|
||||
await mkdir(present);
|
||||
await chmod(sealed, 0o000);
|
||||
try {
|
||||
await expect(lstat(denied)).rejects.toMatchObject({ code: "EACCES" });
|
||||
const directories = await stateDirectories(
|
||||
["notes"],
|
||||
{
|
||||
home,
|
||||
environment: { XDG_CONFIG_HOME: sealed },
|
||||
},
|
||||
[],
|
||||
);
|
||||
const facts = new Map(directories.map((entry) => [entry.path, entry]));
|
||||
expect(facts.get(present)).toEqual({
|
||||
path: present,
|
||||
exists_at_start: true,
|
||||
});
|
||||
expect(facts.get(absent)).toEqual({
|
||||
path: absent,
|
||||
exists_at_start: false,
|
||||
});
|
||||
expect(facts.get(denied)).toMatchObject({
|
||||
path: denied,
|
||||
exists_at_start: null,
|
||||
existence_error: expect.stringContaining("EACCES"),
|
||||
});
|
||||
expect(
|
||||
desktopApplicationInventorySchema.shape.state_directories.parse(
|
||||
directories,
|
||||
),
|
||||
).toEqual(directories);
|
||||
} finally {
|
||||
await chmod(sealed, 0o700);
|
||||
}
|
||||
},
|
||||
);
|
||||
@@ -102,9 +102,21 @@ export const stateDirectories = async (
|
||||
...extra,
|
||||
]);
|
||||
return Promise.all(
|
||||
[...paths].map(async (path) => ({
|
||||
path,
|
||||
exists_at_start: (await lstat(path).catch(() => undefined)) !== undefined,
|
||||
})),
|
||||
[...paths].map(async (path) => {
|
||||
try {
|
||||
await lstat(path);
|
||||
return { path, exists_at_start: true };
|
||||
} catch (cause: unknown) {
|
||||
const code = (cause as NodeJS.ErrnoException).code;
|
||||
return code === "ENOENT" || code === "ENOTDIR"
|
||||
? { path, exists_at_start: false }
|
||||
: {
|
||||
path,
|
||||
exists_at_start: null,
|
||||
existence_error:
|
||||
cause instanceof Error ? cause.message : String(cause),
|
||||
};
|
||||
}
|
||||
}),
|
||||
);
|
||||
};
|
||||
@@ -1,5 +1,5 @@
|
||||
import { existsSync } from "node:fs";
|
||||
import { writeFile } from "node:fs/promises";
|
||||
import { mkdir, writeFile } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
|
||||
import { describe, expect, it } from "vitest";
|
||||
@@ -7,9 +7,89 @@ import { describe, expect, it } from "vitest";
|
||||
import { createTestTempDirectory } from "../../tests/fixtures/temporaryDirectory.js";
|
||||
import {
|
||||
APT_LOCATIONS,
|
||||
inspectAptRepositories,
|
||||
resolveAptLocations,
|
||||
} from "./AptRepositoryInventory.js";
|
||||
import { runHostQuery, type QueryRunner } from "./HostQuery.js";
|
||||
import {
|
||||
GPGV_A_ONLY,
|
||||
VENDOR_A,
|
||||
VENDOR_B,
|
||||
} from "../domain/desktop/aptKeys.fixture.js";
|
||||
|
||||
it.each(["list", "sources"] as const)(
|
||||
"does not broaden %s Signed-By fingerprint restrictions to every key in a keyring",
|
||||
async (format) => {
|
||||
const root = await createTestTempDirectory("apt-signed-by-");
|
||||
const sources = join(root, `vendor.${format}`);
|
||||
const key = join(root, "vendor.gpg");
|
||||
const trustedParts = join(root, "trusted.gpg.d");
|
||||
await mkdir(trustedParts);
|
||||
await writeFile(key, VENDOR_A.key);
|
||||
await writeFile(
|
||||
join(root, "vendor.example_repo_InRelease"),
|
||||
"controlled release",
|
||||
);
|
||||
const source = "https://vendor.example/repo stable/main amd64 Packages";
|
||||
const targets = [
|
||||
`Description: ${source}`,
|
||||
`Sourcesentry: ${sources}:1`,
|
||||
"Repo-URI: https://vendor.example/repo/",
|
||||
"Release: stable",
|
||||
"Component: main",
|
||||
"Base-URI: https://vendor.example/repo/",
|
||||
`Filename: ${join(root, "Packages")}`,
|
||||
].join("\n");
|
||||
let verifications = 0;
|
||||
const runQuery: QueryRunner = async (command) => {
|
||||
if (command === "apt-get") return { stdout: targets, exitCode: 0 };
|
||||
if (command !== "gpgv") throw new Error(`Unexpected query ${command}`);
|
||||
verifications += 1;
|
||||
return { stdout: GPGV_A_ONLY, exitCode: 1 };
|
||||
};
|
||||
const inspect = async (signedBy: string) => {
|
||||
await writeFile(
|
||||
sources,
|
||||
format === "list"
|
||||
? `deb [signed-by=${signedBy.replaceAll(" ", ",")}] https://vendor.example/repo stable main\n`
|
||||
: `Types: deb\nURIs: https://vendor.example/repo\nSuites: stable\nComponents: main\nSigned-By: ${signedBy}\n`,
|
||||
);
|
||||
return inspectAptRepositories(
|
||||
{ sources: [source], origins: ["Vendor"] },
|
||||
{
|
||||
runQuery,
|
||||
apt: { lists: root, trustedParts, trusted: key },
|
||||
},
|
||||
undefined,
|
||||
);
|
||||
};
|
||||
const pathOnly = await inspect(key);
|
||||
expect(pathOnly.repositories[0]?.signing_key).toMatchObject({
|
||||
path: key,
|
||||
fingerprint: VENDOR_A.fingerprint,
|
||||
});
|
||||
expect(pathOnly.keys.size).toBe(1);
|
||||
for (const selector of [VENDOR_B.fingerprint, `${VENDOR_B.fingerprint}!`]) {
|
||||
for (const signedBy of [selector, `${key} ${selector}`]) {
|
||||
const restricted = await inspect(signedBy);
|
||||
expect(restricted.repositories[0]).toMatchObject({
|
||||
source,
|
||||
sources_file: sources,
|
||||
uri: "https://vendor.example/repo",
|
||||
suite: "stable",
|
||||
components: ["main"],
|
||||
signing_key: null,
|
||||
signing_key_reason: expect.stringContaining(selector),
|
||||
});
|
||||
expect(restricted.repositories[0]?.signing_key_reason).toContain(
|
||||
"not supported",
|
||||
);
|
||||
expect(restricted.keys.size).toBe(0);
|
||||
}
|
||||
}
|
||||
expect(verifications).toBe(1);
|
||||
},
|
||||
);
|
||||
|
||||
describe.skipIf(
|
||||
process.platform !== "linux" || !existsSync("/usr/bin/apt-config"),
|
||||
|
||||
@@ -138,8 +138,7 @@ const candidateKeys = async (
|
||||
entry: AptSourceEntry,
|
||||
apt: AptLocations,
|
||||
): Promise<readonly string[]> => {
|
||||
const signedBy = entry.signedBy.filter((value) => value.startsWith("/"));
|
||||
if (signedBy.length > 0) return signedBy;
|
||||
if (entry.signedBy.length > 0) return entry.signedBy;
|
||||
return [
|
||||
...(await fileNames(apt.trustedParts))
|
||||
.filter((name) => name.endsWith(".gpg") || name.endsWith(".asc"))
|
||||
@@ -199,6 +198,11 @@ const keyCheck = async (
|
||||
return {
|
||||
reason: `no sources entry at ${target.sourcesFile}:${String(target.position)} configures it`,
|
||||
};
|
||||
const selectors = entry.signedBy.filter((value) => !value.startsWith("/"));
|
||||
if (selectors.length > 0)
|
||||
return {
|
||||
reason: `Signed-By selectors ${selectors.map((value) => JSON.stringify(value)).join(", ")} are not supported; REA cannot verify this repository's fingerprint restrictions, so no signing key is retained`,
|
||||
};
|
||||
const release = await signedRelease(target);
|
||||
if (release === undefined)
|
||||
return {
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import { z } from "zod";
|
||||
|
||||
/** Where a data-plane file part names its path in the home. */
|
||||
const DATA_OBJECTS = "urn:agentenv:data-objects/v1";
|
||||
export const DATA_OBJECTS = "urn:agentenv:data-objects/v1";
|
||||
/** A `data/get` answer: file parts, each naming its home path, or an error. */
|
||||
export const dataGetSchema = z.union([
|
||||
z.object({
|
||||
@@ -74,7 +74,7 @@ export const readHomeFiles = async (
|
||||
? []
|
||||
: [
|
||||
{
|
||||
path: Object.values(part.metadata ?? {})[0]?.path,
|
||||
path: part.metadata?.[DATA_OBJECTS]?.path,
|
||||
bytes: Buffer.from(encoded, "base64"),
|
||||
},
|
||||
];
|
||||
|
||||
@@ -1045,3 +1045,52 @@ describe.skipIf(process.platform === "win32")(
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
it.skipIf(process.platform === "win32" || process.getuid?.() === 0)(
|
||||
"keeps apt list presence unknown when its directory cannot be read",
|
||||
async () => {
|
||||
const { root, inspect } = await writerFixture();
|
||||
const lists = join(root, "lists");
|
||||
await chmod(lists, 0o000);
|
||||
try {
|
||||
const { installed, limitations } = await inspect({
|
||||
executable: "libreoffice",
|
||||
});
|
||||
expect(installed.package_lists_present).toBeNull();
|
||||
expect(limitations.join("\n")).toContain("could not be read");
|
||||
expect(limitations.join("\n")).toContain("EACCES");
|
||||
} finally {
|
||||
await chmod(lists, 0o755);
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
it.skipIf(process.platform === "win32")(
|
||||
"keeps package ownership unknown for a literal path dpkg would treat as a pattern",
|
||||
async () => {
|
||||
const { root, runQuery, inspect } = await writerFixture();
|
||||
const executable = join(root, "bin/libreoffice[1]");
|
||||
await writeFile(executable, "#!/bin/sh\n");
|
||||
await chmod(executable, 0o755);
|
||||
const result = await inspect(
|
||||
{ executable },
|
||||
{
|
||||
run: async (command, argv, signal) => {
|
||||
if (
|
||||
command === "dpkg-query" &&
|
||||
argv[0] === "-S" &&
|
||||
argv.includes(executable)
|
||||
)
|
||||
// A glob search can name another file without establishing who owns the literal brackets.
|
||||
return {
|
||||
stdout: `other-package: ${executable.replace("[1]", "1")}`,
|
||||
exitCode: 0,
|
||||
};
|
||||
return runQuery(command, argv, signal);
|
||||
},
|
||||
},
|
||||
);
|
||||
expect(result.installed).toMatchObject({ kind: "unknown", name: null });
|
||||
expect(result.limitations.join("\n")).toContain("pattern characters");
|
||||
},
|
||||
);
|
||||
@@ -1,4 +1,5 @@
|
||||
import type { DesktopLibraryPolicy } from "../domain/desktop/libraryPolicy.js";
|
||||
import { describeValidationFailure } from "../domain/validationFailure.js";
|
||||
import { readdir } from "node:fs/promises";
|
||||
|
||||
import {
|
||||
@@ -203,6 +204,11 @@ const dpkgOwner = async (
|
||||
{ host, signal }: InventoryContext,
|
||||
): Promise<Ownership> => {
|
||||
for (const path of ownershipCandidates(paths)) {
|
||||
if (/[*?[\\]/u.test(path))
|
||||
return {
|
||||
kind: "unknown",
|
||||
reason: `dpkg-query cannot safely query ${path} containing pattern characters`,
|
||||
};
|
||||
const outcome = await host.runQuery("dpkg-query", ["-S", path], signal);
|
||||
if (outcome === "missing")
|
||||
return { kind: "unknown", reason: "dpkg-query is not installed" };
|
||||
@@ -283,10 +289,17 @@ export const debianPackage = async (
|
||||
await host.runQuery("apt-cache", ["policy", name], signal),
|
||||
);
|
||||
const apt = await resolveAptLocations(host.runQuery, signal, host.apt);
|
||||
const listsPresent = (
|
||||
await readdir(apt.lists).catch(() => [] as string[])
|
||||
).some((list) => /_Packages(?:\.[a-z0-9]+)?$/u.test(list));
|
||||
if (!listsPresent)
|
||||
const listsPresent = await readdir(apt.lists).then(
|
||||
(lists) => lists.some((list) => /_Packages(?:\.[a-z0-9]+)?$/u.test(list)),
|
||||
(cause: unknown) => {
|
||||
if ((cause as NodeJS.ErrnoException).code === "ENOENT") return false;
|
||||
limitations.push(
|
||||
`The apt package lists directory ${apt.lists} could not be read, so whether package lists are present is unknown: ${describeValidationFailure(cause)}`,
|
||||
);
|
||||
return null;
|
||||
},
|
||||
);
|
||||
if (listsPresent === false)
|
||||
limitations.push(
|
||||
"No apt package lists are present, so whether the installed version comes from a package archive is unknown.",
|
||||
);
|
||||
|
||||
@@ -143,6 +143,50 @@ describe.skipIf(process.platform !== "linux")(
|
||||
describe.skipIf(process.platform === "win32")(
|
||||
"runDesktopSession stderr",
|
||||
() => {
|
||||
it.each([
|
||||
{ fails: false, cancelAt: "cleanup" },
|
||||
{ fails: true, cancelAt: "cleanup" },
|
||||
{ fails: false, cancelAt: "exit" },
|
||||
])(
|
||||
"preserves the session outcome when cancellation arrives during $cancelAt (checkpoint fails: $fails)",
|
||||
async ({ fails, cancelAt }) => {
|
||||
const root = await createTestTempDirectory(
|
||||
"desktop-session-final-cancel-",
|
||||
);
|
||||
const controller = new AbortController();
|
||||
const reason = new Error("cancelled during cleanup");
|
||||
const failure = new Error("stop checkpoint failed");
|
||||
const terminate = vi.mocked(terminateTokenOwnedProcesses);
|
||||
terminate.mockImplementationOnce(async (...args) => {
|
||||
if (cancelAt === "cleanup") controller.abort(reason);
|
||||
const original = await vi.importActual<
|
||||
typeof import("../process/TokenOwnedTermination.js")
|
||||
>("../process/TokenOwnedTermination.js");
|
||||
return original.terminateTokenOwnedProcesses(...args);
|
||||
});
|
||||
const checkpoints: string[] = [];
|
||||
const result = await runDesktopSession({
|
||||
...shellPlan(root, "exit 0"),
|
||||
signal: controller.signal,
|
||||
nextRequest: () => ({ kind: "stop" }),
|
||||
checkpoint: async (name) => {
|
||||
checkpoints.push(name);
|
||||
if (cancelAt === "exit" && name === "exit")
|
||||
controller.abort(reason);
|
||||
if (fails) throw failure;
|
||||
},
|
||||
}).catch((cause: unknown) => cause);
|
||||
if (fails) expect(result).toBe(failure);
|
||||
else
|
||||
expect(result).toMatchObject({
|
||||
session: { end_reason: "cancelled", cleanup: { verified: true } },
|
||||
});
|
||||
expect(checkpoints).toEqual(
|
||||
cancelAt === "exit" ? ["stop", "exit"] : ["stop"],
|
||||
);
|
||||
},
|
||||
);
|
||||
|
||||
it("keeps the end of an app's stderr when it writes more than the session keeps", async () => {
|
||||
const root = await createTestTempDirectory("desktop-session-stderr-");
|
||||
const cause =
|
||||
|
||||
@@ -364,10 +364,13 @@ export const runDesktopSession = async (
|
||||
"stop",
|
||||
);
|
||||
cleanup = await releaseOnce();
|
||||
signal.throwIfAborted();
|
||||
await checkpointNow(plan, observations, "exit", "exit");
|
||||
signal.throwIfAborted();
|
||||
} catch (cause: unknown) {
|
||||
const cancelled = signal.aborted;
|
||||
cleanup = await releaseOnce();
|
||||
if (!signal.aborted) {
|
||||
if (!cancelled) {
|
||||
const left = unreleased(spawned, supervisor, cleanup);
|
||||
if (left === undefined) throw cause;
|
||||
throw new DesktopSessionCleanupError(cause, left);
|
||||
|
||||
@@ -2,12 +2,14 @@ import { createHash } from "node:crypto";
|
||||
import { mkdir, readFile, readdir, rm, writeFile } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
|
||||
import { createTestTempDirectory } from "../../tests/fixtures/temporaryDirectory.js";
|
||||
import type { DesktopPathState } from "../domain/desktop/desktopStateDiff.js";
|
||||
import { DesktopContentStore } from "./DesktopContentStore.js";
|
||||
import { walkHomeState } from "./HomeStateWalker.js";
|
||||
import { OwnedFileHandle } from "../filesystem/OwnedFileHandle.js";
|
||||
import { RegularFileCleanupFailure } from "../filesystem/OwnedRegularFile.js";
|
||||
|
||||
const sha256 = (text: string) =>
|
||||
createHash("sha256").update(text).digest("hex");
|
||||
@@ -33,6 +35,63 @@ const store = async (root: string, maxFileBytes: number, maxBytes: number) => {
|
||||
return created;
|
||||
};
|
||||
|
||||
it("DesktopContentStore leaves an occupied partial filename unowned", async () => {
|
||||
const root = await createTestTempDirectory("desktop-content-occupied-");
|
||||
const content = await store(root, 1024, 1024);
|
||||
const bytes = Buffer.from("new content");
|
||||
const partial = join(content.directory, `${sha256("new content")}.partial`);
|
||||
await writeFile(partial, "foreign content");
|
||||
await expect(content.retainBytes(bytes)).rejects.toMatchObject({
|
||||
code: "EEXIST",
|
||||
});
|
||||
await expect(content.discard()).rejects.toThrow();
|
||||
expect(await readFile(partial, "utf8")).toBe("foreign content");
|
||||
});
|
||||
|
||||
it("DesktopContentStore retains every unpublished file until discard", async () => {
|
||||
const root = await createTestTempDirectory("desktop-content-unpublished-");
|
||||
const content = await store(root, 1024, 1024);
|
||||
const digests = [sha256("first"), sha256("second")];
|
||||
for (const [index, text] of ["first", "second"].entries()) {
|
||||
await mkdir(join(content.directory, digests[index]!));
|
||||
await expect(content.retainBytes(Buffer.from(text))).rejects.toThrow();
|
||||
}
|
||||
await expect(content.discard()).rejects.toThrow();
|
||||
expect((await readdir(content.directory)).sort()).toEqual(digests.sort());
|
||||
});
|
||||
|
||||
it("DesktopContentStore retains a failed output close until cleanup retry", async () => {
|
||||
const root = await createTestTempDirectory("desktop-content-close-");
|
||||
const content = await store(root, 1024, 1024);
|
||||
const closeFailure = new Error("output descriptor close failed");
|
||||
const close = vi
|
||||
.spyOn(OwnedFileHandle.prototype, "close")
|
||||
.mockRejectedValue(closeFailure);
|
||||
try {
|
||||
const failure = await content
|
||||
.retainBytes(Buffer.from("pending"))
|
||||
.catch((cause: unknown) => cause);
|
||||
expect(failure).toBeInstanceOf(RegularFileCleanupFailure);
|
||||
if (!(failure instanceof RegularFileCleanupFailure))
|
||||
throw new Error("Expected retained output close failure");
|
||||
await expect(content.close()).rejects.toMatchObject({
|
||||
cleanup: {
|
||||
resources: [join(content.directory, `${sha256("pending")}.partial`)],
|
||||
},
|
||||
});
|
||||
close.mockRestore();
|
||||
await content.close();
|
||||
await expect(failure.owner.handle.stat()).rejects.toMatchObject({
|
||||
code: "EBADF",
|
||||
});
|
||||
await content.discard();
|
||||
expect(await readdir(join(root, "capture"))).toEqual([]);
|
||||
} finally {
|
||||
close.mockRestore();
|
||||
await content.close();
|
||||
}
|
||||
});
|
||||
|
||||
describe("DesktopContentStore", () => {
|
||||
it("stores each distinct content once and charges the budget once", async () => {
|
||||
const root = await createTestTempDirectory("desktop-content-");
|
||||
|
||||
@@ -1,13 +1,14 @@
|
||||
import { createHash } from "node:crypto";
|
||||
import { mkdir, rename, rm, rmdir, writeFile } from "node:fs/promises";
|
||||
import { mkdir, open, rename, rm, rmdir } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
|
||||
import { ArtifactResourceScope } from "../artifacts/ArtifactResourceScope.js";
|
||||
import type { DesktopApplicationCapture } from "../domain/desktop/desktopCapture.js";
|
||||
import type { DesktopPathState } from "../domain/desktop/desktopStateDiff.js";
|
||||
import type { OwnedFileHandle } from "../filesystem/OwnedFileHandle.js";
|
||||
import { OwnedFileHandle } from "../filesystem/OwnedFileHandle.js";
|
||||
import {
|
||||
RegularFileCleanupFailure,
|
||||
withOwnedFileHandle,
|
||||
withOwnedRegularFile,
|
||||
} from "../filesystem/OwnedRegularFile.js";
|
||||
import {
|
||||
@@ -45,8 +46,8 @@ export class DesktopContentStore {
|
||||
readonly #omissions: Omission[] = [];
|
||||
/** Handles whose close failed; kept owned so `close` can retry them. */
|
||||
readonly #unclosed = new ArtifactResourceScope();
|
||||
/** The partial file being written, removed by `discard` if it was left. */
|
||||
#writing: string | undefined;
|
||||
/** Every partial file this store created but has not published or removed. */
|
||||
readonly #unpublished = new Set<string>();
|
||||
#remainingBytes: number;
|
||||
|
||||
constructor(
|
||||
@@ -94,8 +95,10 @@ export class DesktopContentStore {
|
||||
* when the directory holds anything else or a removal fails.
|
||||
*/
|
||||
async discard(): Promise<void> {
|
||||
if (this.#writing !== undefined) await rm(this.#writing, { force: true });
|
||||
this.#writing = undefined;
|
||||
for (const path of this.#unpublished) {
|
||||
await rm(path, { force: true });
|
||||
this.#unpublished.delete(path);
|
||||
}
|
||||
for (const sha256 of this.#stored.keys()) {
|
||||
await rm(join(this.#directory, sha256), { force: true });
|
||||
this.#stored.delete(sha256);
|
||||
@@ -139,10 +142,24 @@ export class DesktopContentStore {
|
||||
return { kind: "omitted", reason: "budget_exhausted" };
|
||||
const target = join(this.#directory, sha256);
|
||||
const partial = `${target}.partial`;
|
||||
this.#writing = partial;
|
||||
await writeFile(partial, bytes, { mode: 0o600, flag: "wx" });
|
||||
const owner = new OwnedFileHandle(await open(partial, "wx", 0o600));
|
||||
this.#unpublished.add(partial);
|
||||
await withOwnedFileHandle(
|
||||
partial,
|
||||
owner,
|
||||
{
|
||||
retainFailedClose: (handle) => {
|
||||
this.#unclosed.retain({
|
||||
kind: "file-handle",
|
||||
resource: partial,
|
||||
handle,
|
||||
});
|
||||
},
|
||||
},
|
||||
(file) => file.writeFile(bytes),
|
||||
);
|
||||
await rename(partial, target);
|
||||
this.#writing = undefined;
|
||||
this.#unpublished.delete(partial);
|
||||
this.#stored.set(sha256, size);
|
||||
this.#storedSizes.add(size);
|
||||
this.#remainingBytes -= size;
|
||||
|
||||
@@ -85,6 +85,49 @@ describe.skipIf(!hasGpgv)("withGpgvKeyCheck with gpgv", () => {
|
||||
});
|
||||
|
||||
describe("withGpgvKeyCheck cleanup", () => {
|
||||
it.each(["completed", "failed"])(
|
||||
"observes cancellation during final keyring cleanup after %s work",
|
||||
async (completion) => {
|
||||
const cancel = new AbortController();
|
||||
const reason = new Error("cancelled during keyring cleanup");
|
||||
const primary = new Error("verification failed");
|
||||
const create = PrivateRuntimeRoot.create.bind(PrivateRuntimeRoot);
|
||||
let path: string | undefined;
|
||||
vi.spyOn(PrivateRuntimeRoot, "create").mockImplementationOnce(
|
||||
async (options) => {
|
||||
const root = await create(options);
|
||||
path = root.path;
|
||||
const close = root.close.bind(root);
|
||||
vi.spyOn(root, "close").mockImplementationOnce(async () => {
|
||||
cancel.abort(reason);
|
||||
await close();
|
||||
});
|
||||
return root;
|
||||
},
|
||||
);
|
||||
const { release, paths } = await files({ "a.gpg": VENDOR_A.key });
|
||||
const result = withGpgvKeyCheck(
|
||||
async () => ({ stdout: "", exitCode: 2 }),
|
||||
cancel.signal,
|
||||
async (check) => {
|
||||
const verified = await check.firstVerifying(
|
||||
[paths["a.gpg"] ?? ""],
|
||||
[release],
|
||||
);
|
||||
if (completion === "failed") throw primary;
|
||||
return verified;
|
||||
},
|
||||
);
|
||||
await expect(result).rejects.toBe(
|
||||
completion === "failed" ? primary : reason,
|
||||
);
|
||||
expect(path).toBeDefined();
|
||||
await expect(access(path ?? "")).rejects.toMatchObject({
|
||||
code: "ENOENT",
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
it("keeps the private keyring folder's owner when its removal fails", async () => {
|
||||
const create = PrivateRuntimeRoot.create.bind(PrivateRuntimeRoot);
|
||||
let owned: PrivateRuntimeRoot | undefined;
|
||||
|
||||
@@ -174,5 +174,6 @@ export const withGpgvKeyCheck = async <Value>(
|
||||
throw resources.cleanupFailure(outcome, [removed]);
|
||||
}
|
||||
if (outcome.kind === "failed") throw outcome.cause;
|
||||
signal?.throwIfAborted();
|
||||
return outcome.value;
|
||||
};
|
||||
@@ -1,18 +1,54 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import { AnalysisResourceConstraintError } from "../domain/analysisErrorCore.js";
|
||||
import { readHomeFiles } from "./ComputerHomeFiles.js";
|
||||
import {
|
||||
AgentEnvComputer,
|
||||
GUEST_FILE_ANSWER_COPIES,
|
||||
GUEST_FILE_ANSWER_SOURCE,
|
||||
guestFileAnswerBytes,
|
||||
guestFileAnswerCeiling,
|
||||
guestFileAnswerRefusal,
|
||||
guestFileAnswerWorkingBytes,
|
||||
retainGuestFiles,
|
||||
} from "./AgentEnvComputer.js";
|
||||
|
||||
describe("guest file answers", () => {
|
||||
it.each([true, false])(
|
||||
"uses only the named data-object metadata for file identity (present: %s)",
|
||||
async (present) => {
|
||||
const files = await readHomeFiles(
|
||||
async () => ({
|
||||
result: {
|
||||
parts: [
|
||||
{
|
||||
kind: "file",
|
||||
file: { bytes: Buffer.from("source bytes").toString("base64") },
|
||||
metadata: {
|
||||
"urn:unrelated": { path: "other.txt" },
|
||||
...(present
|
||||
? {
|
||||
"urn:agentenv:data-objects/v1": {
|
||||
path: "selected.txt",
|
||||
},
|
||||
}
|
||||
: {}),
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
}),
|
||||
(text) => text,
|
||||
["selected.txt"],
|
||||
);
|
||||
expect(files).toEqual([
|
||||
{
|
||||
path: present ? "selected.txt" : undefined,
|
||||
bytes: Buffer.from("source bytes"),
|
||||
},
|
||||
]);
|
||||
},
|
||||
);
|
||||
|
||||
it("counts base64, the envelope, and the copies held while the answer is parsed", () => {
|
||||
expect(guestFileAnswerBytes(3, 1)).toBeGreaterThan(3);
|
||||
expect(guestFileAnswerBytes(3, 2)).toBeGreaterThan(
|
||||
|
||||
@@ -54,7 +54,7 @@ it.skipIf(process.platform === "win32")(
|
||||
const bin = join(root, "bin");
|
||||
const lib = join(root, "lib");
|
||||
await Promise.all([mkdir(bin), mkdir(lib)]);
|
||||
const paths = Array.from({ length: 102 }, (_, index) =>
|
||||
const paths = Array.from({ length: 201 }, (_, index) =>
|
||||
join(lib, `lib${index}.so`),
|
||||
);
|
||||
const executable = join(bin, "app");
|
||||
@@ -77,7 +77,7 @@ it.skipIf(process.platform === "win32")(
|
||||
expect(batches).toBe(2);
|
||||
expect(result.library_packages).toEqual(["owner"]);
|
||||
expect(result.unpackaged_libraries).toContain(paths[1]);
|
||||
expect(result.unpackaged_libraries).not.toContain(paths[101]);
|
||||
expect(result.unpackaged_libraries).not.toContain(paths[200]);
|
||||
expect(result.library_packages_reason).toContain("dpkg-query");
|
||||
},
|
||||
);
|
||||
|
||||
@@ -1,10 +1,20 @@
|
||||
import { chmod, mkdir, stat, symlink, writeFile } from "node:fs/promises";
|
||||
import {
|
||||
chmod,
|
||||
mkdir,
|
||||
realpath,
|
||||
stat,
|
||||
symlink,
|
||||
writeFile,
|
||||
} from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
|
||||
import { createTestTempDirectory } from "../../tests/fixtures/temporaryDirectory.js";
|
||||
import { inspectLinuxApplication } from "./LinuxApplicationInventory.js";
|
||||
import { OwnedFileHandle } from "../filesystem/OwnedFileHandle.js";
|
||||
import { ArtifactFileCleanupFailure } from "../artifacts/ArtifactFileOperation.js";
|
||||
import { desktopOperationFailure } from "../application/desktop/DesktopCaptureRefusals.js";
|
||||
|
||||
const fixture = async () => {
|
||||
const root = await createTestTempDirectory("desktop-inventory-");
|
||||
@@ -37,6 +47,58 @@ const fixture = async () => {
|
||||
};
|
||||
};
|
||||
|
||||
describe.skipIf(process.platform === "win32")(
|
||||
"explicit executable traversal",
|
||||
() => {
|
||||
it("refuses a directory symlink as an executable while accepting a file symlink", async () => {
|
||||
const { home, bin, environment } = await fixture();
|
||||
const directoryAlias = join(bin, "directory-alias");
|
||||
const fileAlias = join(bin, "file-alias");
|
||||
await symlink(home, directoryAlias);
|
||||
await symlink(join(bin, "notes-app"), fileAlias);
|
||||
const inspect = (executable: string) =>
|
||||
inspectLinuxApplication({
|
||||
desktopEntry: undefined,
|
||||
executable,
|
||||
arguments: [],
|
||||
environment,
|
||||
home,
|
||||
callerDirectory: home,
|
||||
});
|
||||
expect(await inspect(fileAlias)).toMatchObject({ ok: true });
|
||||
expect(await inspect(directoryAlias)).toEqual({
|
||||
ok: false,
|
||||
error: { reason: "executable_not_found", program: directoryAlias },
|
||||
});
|
||||
});
|
||||
|
||||
it("preserves symlink and parent traversal for the selected executable", async () => {
|
||||
const { home, environment } = await fixture();
|
||||
const physical = join(home, "physical");
|
||||
const named = join(home, "named");
|
||||
await mkdir(join(physical, "child"), { recursive: true });
|
||||
await mkdir(named);
|
||||
await symlink(join(physical, "child"), join(named, "link"));
|
||||
await writeFile(join(physical, "app"), "#!/bin/sh\n", { mode: 0o755 });
|
||||
await writeFile(join(named, "app"), "#!/bin/sh\n", { mode: 0o755 });
|
||||
const selected = `${named}/link/../app`;
|
||||
const result = await inspectLinuxApplication({
|
||||
desktopEntry: undefined,
|
||||
executable: selected,
|
||||
arguments: [],
|
||||
environment,
|
||||
home,
|
||||
callerDirectory: home,
|
||||
});
|
||||
if (!result.ok) throw new Error(result.error.reason);
|
||||
expect(result.value.inventory.argv[0]).toBe(selected);
|
||||
expect(result.value.inventory.executable.resolved_path).toBe(
|
||||
await realpath(join(physical, "app")),
|
||||
);
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
describe.skipIf(process.platform === "win32")("inspectLinuxApplication", () => {
|
||||
it("resolves a desktop entry id into a launch command and state directories", async () => {
|
||||
const { home, bin, environment } = await fixture();
|
||||
@@ -726,3 +788,129 @@ describe.skipIf(process.platform === "win32")(
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
it.skipIf(process.platform === "win32").each([
|
||||
{ stage: "desktop entry", closeAt: 1, oversized: false, cancel: false },
|
||||
{
|
||||
stage: "desktop entry capacity failure",
|
||||
closeAt: 1,
|
||||
oversized: true,
|
||||
cancel: false,
|
||||
},
|
||||
{ stage: "executable header", closeAt: 2, oversized: false, cancel: false },
|
||||
{
|
||||
stage: "optional Electron probe",
|
||||
closeAt: 3,
|
||||
oversized: false,
|
||||
cancel: false,
|
||||
},
|
||||
{
|
||||
stage: "desktop entry final cancellation",
|
||||
closeAt: 1,
|
||||
oversized: false,
|
||||
cancel: true,
|
||||
},
|
||||
{
|
||||
stage: "capacity failure before final cancellation",
|
||||
closeAt: 1,
|
||||
oversized: true,
|
||||
cancel: true,
|
||||
},
|
||||
])(
|
||||
"retains and retries a failed close during $stage",
|
||||
async ({ closeAt, oversized, cancel }) => {
|
||||
const { home, bin, environment } = await fixture();
|
||||
const entry = join(
|
||||
environment.XDG_DATA_DIRS,
|
||||
"applications/org.example.notes.desktop",
|
||||
);
|
||||
if (oversized) await writeFile(entry, "x".repeat(1024 * 1024 + 1));
|
||||
const controller = new AbortController();
|
||||
const reason = new Error("cancelled during close");
|
||||
const denied = new Error("descriptor close denied");
|
||||
const originalClose = OwnedFileHandle.prototype.close;
|
||||
let closes = 0;
|
||||
let retained: OwnedFileHandle | undefined;
|
||||
const close = vi
|
||||
.spyOn(OwnedFileHandle.prototype, "close")
|
||||
.mockImplementation(function (this: OwnedFileHandle) {
|
||||
closes += 1;
|
||||
if (closes === closeAt || this === retained) {
|
||||
// oxlint-disable-next-line typescript/no-this-alias -- Retain the exact failed descriptor owner for retry.
|
||||
retained = this;
|
||||
if (cancel) controller.abort(reason);
|
||||
return Promise.reject(denied);
|
||||
}
|
||||
return originalClose.call(this);
|
||||
});
|
||||
try {
|
||||
const failure = await inspectLinuxApplication(
|
||||
{
|
||||
desktopEntry: "org.example.notes",
|
||||
executable: undefined,
|
||||
arguments: [],
|
||||
environment,
|
||||
home,
|
||||
callerDirectory: home,
|
||||
},
|
||||
controller.signal,
|
||||
{
|
||||
runQuery: async () => "missing",
|
||||
apt: {
|
||||
trusted: join(home, "trusted"),
|
||||
trustedParts: join(home, "trusted.d"),
|
||||
lists: join(home, "lists"),
|
||||
},
|
||||
},
|
||||
).catch((cause: unknown) => cause);
|
||||
expect(retained).toBeDefined();
|
||||
if (!(failure instanceof ArtifactFileCleanupFailure))
|
||||
throw new Error("expected transferable descriptor cleanup", {
|
||||
cause: failure,
|
||||
});
|
||||
const expectedPath =
|
||||
closeAt === 1 ? entry : await realpath(join(bin, "notes-app"));
|
||||
expect(failure.cleanup?.resources).toEqual([expectedPath]);
|
||||
expect(failure.fileCleanupFailure.owner).toBe(retained);
|
||||
expect(failure.fileCleanupFailure.cleanupCause).toBe(denied);
|
||||
const outcome = failure.fileCleanupFailure.outcome;
|
||||
if (oversized) {
|
||||
expect(outcome).toMatchObject({
|
||||
kind: "failed",
|
||||
cause: {
|
||||
maximumBytes: 1024 * 1024,
|
||||
observedBytes: BigInt(1024 * 1024 + 1),
|
||||
},
|
||||
});
|
||||
} else if (closeAt === 3) {
|
||||
expect(outcome).toMatchObject({
|
||||
kind: "failed",
|
||||
cause: { kind: "unsupported" },
|
||||
});
|
||||
} else {
|
||||
expect(outcome.kind).toBe("completed");
|
||||
}
|
||||
if (cancel && !oversized)
|
||||
expect(failure.cancellation?.reason).toBe(reason);
|
||||
else expect(failure.reason).not.toBe("cancelled");
|
||||
const projected = desktopOperationFailure(
|
||||
failure,
|
||||
controller.signal,
|
||||
"capture_desktop_application",
|
||||
"Desktop capture",
|
||||
);
|
||||
expect(projected.cleanupIncomplete).toBe(true);
|
||||
expect(projected.cleanupResources).toEqual([expectedPath]);
|
||||
await expect(failure.cleanupLease.release()).rejects.toBe(denied);
|
||||
expect(projected.cleanupIncomplete).toBe(true);
|
||||
close.mockRestore();
|
||||
await failure.cleanupLease.release();
|
||||
expect(projected.cleanupIncomplete).toBe(false);
|
||||
expect(projected.cleanupResources).toEqual([]);
|
||||
expect(retained?.handle.fd).toBe(-1);
|
||||
} finally {
|
||||
close.mockRestore();
|
||||
await retained?.close();
|
||||
}
|
||||
},
|
||||
);
|
||||
@@ -3,7 +3,7 @@ import {
|
||||
type DesktopLibraryPolicyInput,
|
||||
} from "../domain/desktop/libraryPolicy.js";
|
||||
import { lstat, realpath, stat } from "node:fs/promises";
|
||||
import { basename, dirname, isAbsolute, join, resolve } from "node:path";
|
||||
import { basename, dirname, isAbsolute, join } from "node:path";
|
||||
|
||||
import type { DesktopApplicationInventory } from "../domain/desktop/desktopCapture.js";
|
||||
import {
|
||||
@@ -19,7 +19,12 @@ import {
|
||||
} from "../domain/desktop/linuxPackages.js";
|
||||
import { err, ok, type Result } from "../domain/result.js";
|
||||
import { readElectronFuseImage } from "../electron/ElectronFuseReader.js";
|
||||
import { withOwnedRegularFile } from "../filesystem/OwnedRegularFile.js";
|
||||
import { ArtifactReaderFailure } from "../artifacts/ArtifactReader.js";
|
||||
import {
|
||||
RegularFileCleanupFailure,
|
||||
withOwnedRegularFile,
|
||||
} from "../filesystem/OwnedRegularFile.js";
|
||||
import { absoluteCallerPath } from "../filesystem/callerPath.js";
|
||||
import {
|
||||
RegularFileCapacityError,
|
||||
readStableFileBytes,
|
||||
@@ -33,7 +38,7 @@ import {
|
||||
type DebianPackageHost,
|
||||
type InventoryContext,
|
||||
} from "./DebianPackageInventory.js";
|
||||
import { isRefusedRead } from "./HostFileRead.js";
|
||||
import { hostCloseFailure, isRefusedRead } from "./HostFileRead.js";
|
||||
import { runHostQuery } from "./HostQuery.js";
|
||||
import { resolveProgram } from "./ProgramLookup.js";
|
||||
|
||||
@@ -230,6 +235,8 @@ const readDesktopEntry = async (
|
||||
)
|
||||
).toString("utf8");
|
||||
} catch (cause: unknown) {
|
||||
if (cause instanceof RegularFileCleanupFailure)
|
||||
throw hostCloseFailure(cause);
|
||||
signal?.throwIfAborted();
|
||||
if (!isRefusedRead(cause)) throw cause;
|
||||
return err({
|
||||
@@ -267,6 +274,8 @@ const readHeader = async (
|
||||
(await readStableFilePrefix(handle, path, 16, options)).bytes,
|
||||
);
|
||||
} catch (cause: unknown) {
|
||||
if (cause instanceof RegularFileCleanupFailure)
|
||||
throw hostCloseFailure(cause);
|
||||
signal?.throwIfAborted();
|
||||
if (!isRefusedRead(cause)) throw cause;
|
||||
return new Uint8Array();
|
||||
@@ -328,7 +337,9 @@ const electronMarkers = async (
|
||||
);
|
||||
if (image.slices.some((slice) => slice.sentinel_count > 0))
|
||||
markers.push("fuse-sentinel");
|
||||
} catch {
|
||||
} catch (cause: unknown) {
|
||||
if (cause instanceof ArtifactReaderFailure && cause.cleanup !== undefined)
|
||||
throw cause;
|
||||
signal?.throwIfAborted();
|
||||
// Scripts and other non-binary launchers carry no fuse wire.
|
||||
}
|
||||
@@ -358,7 +369,7 @@ const launchCommand = async (
|
||||
return ok({
|
||||
argv: [
|
||||
executable.includes("/")
|
||||
? resolve(request.callerDirectory, executable)
|
||||
? absoluteCallerPath(executable, { cwd: request.callerDirectory })
|
||||
: executable,
|
||||
...request.arguments,
|
||||
],
|
||||
|
||||
@@ -1,11 +1,11 @@
|
||||
import { constants } from "node:fs";
|
||||
import { access, lstat } from "node:fs/promises";
|
||||
import { access, stat } from "node:fs/promises";
|
||||
import { delimiter, isAbsolute } from "node:path";
|
||||
|
||||
const isExecutable = async (path: string): Promise<boolean> => {
|
||||
try {
|
||||
await access(path, constants.X_OK);
|
||||
return (await lstat(path)).isDirectory() === false;
|
||||
return (await stat(path)).isFile();
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
|
||||
@@ -86,6 +86,7 @@ const formatSchema = z.enum([
|
||||
"dfuse",
|
||||
"esp-flash-image",
|
||||
"fit",
|
||||
"ntfs-mft",
|
||||
"plist",
|
||||
"javascript",
|
||||
"source-map",
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { z } from "zod";
|
||||
import { MFT_INLINE_HEADER_LIMIT } from "./windows/ntfsMft.js";
|
||||
|
||||
import { jsonRecordSchema } from "./jsonRecord.js";
|
||||
import { jsonValueSchema } from "./jsonValue.js";
|
||||
@@ -299,6 +300,56 @@ export const hexRecordRegionMetadataSchema = z.strictObject({
|
||||
last_line: offset,
|
||||
});
|
||||
|
||||
const ntfsFileReferenceSchema = z.strictObject({
|
||||
recordNumber: offset,
|
||||
sequenceNumber: offset,
|
||||
});
|
||||
const ntfsRecordHeaderSchema = z.strictObject({
|
||||
index: offset,
|
||||
fileOffset: offset,
|
||||
sequenceNumber: offset,
|
||||
hardLinkCount: offset,
|
||||
flags: z.strictObject({
|
||||
raw: offset,
|
||||
names: z.array(z.string()),
|
||||
unknownMask: offset,
|
||||
}),
|
||||
inUse: z.boolean(),
|
||||
directory: z.boolean(),
|
||||
usedBytes: offset,
|
||||
totalBytes: offset,
|
||||
extentValid: z.boolean(),
|
||||
baseRecordReference: ntfsFileReferenceSchema.nullable(),
|
||||
declaredRecordNumber: offset.nullable(),
|
||||
recordNumberMatchesSlot: z.boolean().nullable(),
|
||||
logFileSequenceNumber: z.string().regex(/^(?:0|[1-9][0-9]*)$/u),
|
||||
firstAttributeOffset: offset,
|
||||
nextAttributeId: offset,
|
||||
fixups: z.strictObject({
|
||||
sequenceNumber: offset,
|
||||
entries: offset,
|
||||
verifiedSectors: offset,
|
||||
coveredSectors: offset,
|
||||
mismatchSector: offset.nullable(),
|
||||
malformed: z.boolean(),
|
||||
}),
|
||||
});
|
||||
|
||||
/** NTFS $MFT record-header facts; attributes, names and the USN journal are later steps. */
|
||||
export const ntfsMftMetadataSchema = z.strictObject({
|
||||
kind: z.literal("ntfs-mft"),
|
||||
record_bytes: offset,
|
||||
record_count: offset,
|
||||
in_use_record_count: offset,
|
||||
directory_record_count: offset,
|
||||
extension_record_count: offset,
|
||||
fixup_mismatch_count: offset,
|
||||
fixup_unverified_record_count: offset,
|
||||
truncated_tail_bytes: offset,
|
||||
record_headers: z.array(ntfsRecordHeaderSchema).max(MFT_INLINE_HEADER_LIMIT),
|
||||
omitted_record_headers: offset,
|
||||
});
|
||||
|
||||
/**
|
||||
* U-Boot FIT root facts. Configuration references are declared names; a
|
||||
* reference that names no image is reported as a limitation, not removed.
|
||||
@@ -886,6 +937,7 @@ export const artifactFormatMetadataSchema = z.discriminatedUnion("kind", [
|
||||
espImageSignatureBlockMetadataSchema,
|
||||
uf2MetadataSchema,
|
||||
uf2RegionMetadataSchema,
|
||||
ntfsMftMetadataSchema,
|
||||
dfuSeMetadataSchema,
|
||||
dfuSeElementMetadataSchema,
|
||||
espFlashImageMetadataSchema,
|
||||
@@ -1036,6 +1088,10 @@ const declarationsByKind: {
|
||||
notCompared:
|
||||
"Its familyID and addresses are in its path, family_name is REA's label for the familyID, and its block counts and positions are layout; content comparison covers its bytes.",
|
||||
},
|
||||
"ntfs-mft": {
|
||||
notCompared:
|
||||
"Its fields come from the record headers in the file's own FILE records, which content comparison covers.",
|
||||
},
|
||||
dfuse: {
|
||||
notCompared:
|
||||
"Its fields come from the file's own prefix, target headers and suffix, which content comparison covers.",
|
||||
|
||||
@@ -103,6 +103,7 @@ export const ARTIFACT_FORMAT_IDS = [
|
||||
"dfuse",
|
||||
"esp-flash-image",
|
||||
"fit",
|
||||
"ntfs-mft",
|
||||
"unknown",
|
||||
] as const;
|
||||
|
||||
@@ -124,6 +125,7 @@ const artifactContentFormatSchema = artifactFormatSchema.extract([
|
||||
"uf2",
|
||||
"dfuse",
|
||||
"fit",
|
||||
"ntfs-mft",
|
||||
"mach-o-universal",
|
||||
"mach-o",
|
||||
"elf",
|
||||
|
||||
@@ -302,7 +302,8 @@ export type BinaryTarget =
|
||||
| "esp-image"
|
||||
| "uf2"
|
||||
| "dfuse"
|
||||
| "fit";
|
||||
| "fit"
|
||||
| "ntfs-mft";
|
||||
})
|
||||
| (BinaryTargetIdentity &
|
||||
NonExecutableMetadata & {
|
||||
|
||||
@@ -218,9 +218,9 @@ def _cell(value, encoding: str):
|
||||
return _sqlite_text(data, encoding) if value[0] == "t" else {"blob_hex": data.hex()}
|
||||
if isinstance(value, int) and abs(value) > MAX_SAFE_INTEGER:
|
||||
return {"integer": str(value)}
|
||||
# JSON has no infinities or negative zero; REA's SQLite reader spells them as strings.
|
||||
# Keep JSON's missing REAL values distinct from literal TEXT spellings.
|
||||
if isinstance(value, float) and (math.isinf(value) or (value == 0 and math.copysign(1, value) < 0)):
|
||||
return "Infinity" if value == math.inf else "-Infinity" if value == -math.inf else "-0"
|
||||
return {"real": "Infinity" if value == math.inf else "-Infinity" if value == -math.inf else "-0"}
|
||||
return value
|
||||
|
||||
|
||||
|
||||
@@ -118,7 +118,9 @@ describe("withheld files", () => {
|
||||
expect(gapped(4, { truncated: true })).toMatchObject({ ok: true });
|
||||
expect(gapped(1, { unconfirmed_changes: 1 })).toMatchObject({ ok: true });
|
||||
});
|
||||
});
|
||||
|
||||
describe("withheld file settings", () => {
|
||||
it("names each withheld file's setting, environment variable and handle", () => {
|
||||
expect(withheldSettings("typora", [".config/Typora/mAxvdiPNmi"])).toEqual([
|
||||
{
|
||||
@@ -140,8 +142,24 @@ describe("withheld files", () => {
|
||||
expect(other?.variable).not.toBe(again?.variable);
|
||||
});
|
||||
|
||||
it("names settings Python takes as arguments, distinct whatever the files' order", () => {
|
||||
const files = [".pass", "1license", "file_1license", "class"];
|
||||
it("names settings Python and Pydantic accept, distinct whatever the files' order", () => {
|
||||
const files = [
|
||||
".pass",
|
||||
"1license",
|
||||
"file_1license",
|
||||
"class",
|
||||
"model_config",
|
||||
"model_post_init",
|
||||
"model_dump",
|
||||
"model_dump_json",
|
||||
"model_validate",
|
||||
"model_validate_json",
|
||||
"model_validate_strings",
|
||||
"file_model_config",
|
||||
"model_id",
|
||||
"model_copy",
|
||||
"model_fields",
|
||||
];
|
||||
for (const order of [files, [...files].reverse()]) {
|
||||
const settings = withheldSettings("an-app", order).map(
|
||||
({ setting }) => setting,
|
||||
@@ -150,7 +168,19 @@ describe("withheld files", () => {
|
||||
for (const setting of settings) {
|
||||
expect(setting).toMatch(/^[a-z]\w*$/u);
|
||||
expect(["pass", "class"]).not.toContain(setting);
|
||||
expect([
|
||||
"model_config",
|
||||
"model_post_init",
|
||||
"model_dump",
|
||||
"model_dump_json",
|
||||
"model_validate",
|
||||
"model_validate_json",
|
||||
"model_validate_strings",
|
||||
]).not.toContain(setting);
|
||||
}
|
||||
expect(settings[order.indexOf("model_id")]).toBe("model_id");
|
||||
expect(settings[order.indexOf("model_copy")]).toBe("model_copy");
|
||||
expect(settings[order.indexOf("model_fields")]).toBe("model_fields");
|
||||
}
|
||||
});
|
||||
|
||||
|
||||
@@ -169,10 +169,23 @@ const PYTHON_KEYWORDS = new Set(
|
||||
),
|
||||
);
|
||||
|
||||
/** Pydantic >=2.12: configuration, its instance initialization hook and protected inherited methods. */
|
||||
const PYDANTIC_RESERVED_FIELDS = new Set([
|
||||
"model_config",
|
||||
"model_post_init",
|
||||
"model_dump",
|
||||
"model_dump_json",
|
||||
"model_validate",
|
||||
"model_validate_json",
|
||||
"model_validate_strings",
|
||||
]);
|
||||
|
||||
/** A setting Python takes as an argument's name, distinct from those `settings` holds, which it joins. */
|
||||
const distinctSetting = (named: string, settings: Set<string>): string => {
|
||||
const valid =
|
||||
/^[a-z]/u.test(named) && !PYTHON_KEYWORDS.has(named)
|
||||
/^[a-z]/u.test(named) &&
|
||||
!PYTHON_KEYWORDS.has(named) &&
|
||||
!PYDANTIC_RESERVED_FIELDS.has(named)
|
||||
? named
|
||||
: `file_${named}`;
|
||||
let setting = valid;
|
||||
|
||||
@@ -5,10 +5,25 @@ import {
|
||||
checkpointTaken,
|
||||
desktopApplicationInventorySchema,
|
||||
desktopCaptureEventSchema,
|
||||
desktopFileFactsSchema,
|
||||
type DesktopCheckpoint,
|
||||
type DesktopStateChange,
|
||||
} from "./desktopCapture.js";
|
||||
|
||||
it.each(["-315619200000000000", "0", "1"])(
|
||||
"retains filesystem modification timestamp %s without changing its spelling",
|
||||
(mtime_ns) => {
|
||||
const value = {
|
||||
type: "file",
|
||||
size: 3,
|
||||
mode: 0o644,
|
||||
mtime_ns,
|
||||
sha256: null,
|
||||
};
|
||||
expect(desktopFileFactsSchema.parse(value)).toEqual(value);
|
||||
},
|
||||
);
|
||||
|
||||
describe("checkpointNameRefusal", () => {
|
||||
it("accepts an omitted name and a new one", () => {
|
||||
expect(checkpointNameRefusal(undefined, new Set(["saved"]))).toBe(
|
||||
|
||||
@@ -11,7 +11,7 @@ export const desktopFileFactsSchema = z.strictObject({
|
||||
type: z.enum(["file", "directory", "symlink", "other"]),
|
||||
size: z.int().min(0),
|
||||
mode: z.int().min(0),
|
||||
mtime_ns: z.string().regex(/^\d+$/u),
|
||||
mtime_ns: z.string().regex(/^-?\d+$/u),
|
||||
sha256: sha256.nullable(),
|
||||
});
|
||||
export type DesktopFileFacts = z.infer<typeof desktopFileFactsSchema>;
|
||||
@@ -271,7 +271,10 @@ export const desktopApplicationInventorySchema = z.strictObject({
|
||||
state_directories: z.array(
|
||||
z.strictObject({
|
||||
path: z.string().min(1),
|
||||
exists_at_start: z.boolean(),
|
||||
/** Null when the filesystem probe could not establish presence or absence. */
|
||||
exists_at_start: z.boolean().nullable(),
|
||||
/** The failed existence probe's reason; absent for confirmed presence or absence. */
|
||||
existence_error: z.string().optional(),
|
||||
}),
|
||||
),
|
||||
});
|
||||
|
||||
@@ -22,7 +22,6 @@ const VALUE_ESCAPES: Readonly<Record<string, string>> = {
|
||||
t: "\t",
|
||||
r: "\r",
|
||||
"\\": "\\",
|
||||
";": "\\;",
|
||||
};
|
||||
|
||||
/** Undo the escapes the desktop entry format allows in string values. */
|
||||
@@ -69,13 +68,25 @@ export const escapeDesktopEntryValue = (value: string): string => {
|
||||
return escaped;
|
||||
};
|
||||
|
||||
const listValue = (value: string | undefined): readonly string[] =>
|
||||
value === undefined
|
||||
? []
|
||||
: value
|
||||
.split(/(?<!\\);/u)
|
||||
.map((part) => part.replaceAll("\\;", ";"))
|
||||
.filter((part) => part.length > 0);
|
||||
/** Decode list separators and escapes together, before either loses its meaning. */
|
||||
const listValue = (value: string | undefined): readonly string[] => {
|
||||
if (value === undefined) return [];
|
||||
const entries: string[] = [];
|
||||
let entry = "";
|
||||
for (let index = 0; index < value.length; index += 1) {
|
||||
const character = value.charAt(index);
|
||||
const next = value.charAt(index + 1);
|
||||
if (character === "\\" && next.length > 0) {
|
||||
entry += next === ";" ? ";" : (VALUE_ESCAPES[next] ?? `\\${next}`);
|
||||
index += 1;
|
||||
} else if (character === ";") {
|
||||
entries.push(entry);
|
||||
entry = "";
|
||||
} else entry += character;
|
||||
}
|
||||
if (entry.length > 0) entries.push(entry);
|
||||
return entries;
|
||||
};
|
||||
|
||||
/** Read the main group of a freedesktop desktop entry, ignoring localized keys. */
|
||||
export const parseDesktopEntry = (
|
||||
@@ -101,13 +112,14 @@ export const parseDesktopEntry = (
|
||||
// Keep the existing enum/boolean token treatment before value unescaping.
|
||||
values.set(
|
||||
key,
|
||||
unescapeValue(
|
||||
key === "Type" || key === "Terminal" ? value.trimEnd() : value,
|
||||
),
|
||||
key === "Type" || key === "Terminal" ? value.trimEnd() : value,
|
||||
);
|
||||
}
|
||||
if (!found) return err("missing_desktop_entry_group");
|
||||
const field = (key: string) => values.get(key) ?? null;
|
||||
const field = (key: string) => {
|
||||
const value = values.get(key);
|
||||
return value === undefined ? null : unescapeValue(value);
|
||||
};
|
||||
return ok({
|
||||
type: field("Type"),
|
||||
name: field("Name"),
|
||||
@@ -118,7 +130,7 @@ export const parseDesktopEntry = (
|
||||
startup_wm_class: field("StartupWMClass"),
|
||||
categories: listValue(values.get("Categories")),
|
||||
mime_types: listValue(values.get("MimeType")),
|
||||
terminal: values.get("Terminal") === "true",
|
||||
terminal: field("Terminal") === "true",
|
||||
});
|
||||
};
|
||||
|
||||
|
||||
@@ -82,3 +82,16 @@ it("does not broaden Type or Terminal recognition by trimming decoded escapes",
|
||||
if (!parsed.ok) throw new Error(parsed.error);
|
||||
expect(parsed.value).toMatchObject({ type: "Application ", terminal: false });
|
||||
});
|
||||
|
||||
it.each([
|
||||
[String.raw`First\\;Second;`, ["First\\", "Second"]],
|
||||
[String.raw`First\;Second;`, ["First;Second"]],
|
||||
[String.raw`First\\\;Second;`, ["First\\;Second"]],
|
||||
["First;;", ["First", ""]],
|
||||
["First;", ["First"]],
|
||||
["", []],
|
||||
])("decodes string-list escapes and terminators once: %s", (wire, expected) => {
|
||||
const parsed = parseDesktopEntry(`[Desktop Entry]\nCategories=${wire}\n`);
|
||||
if (!parsed.ok) throw new Error(parsed.error);
|
||||
expect(parsed.value.categories).toEqual(expected);
|
||||
});
|
||||
@@ -2,6 +2,7 @@ import { describe, expect, it } from "vitest";
|
||||
|
||||
import {
|
||||
isAppImageHeader,
|
||||
ownershipCandidates,
|
||||
packageFromPath,
|
||||
parseAptPolicyOrigins,
|
||||
parseAptPolicyPackageSources,
|
||||
@@ -11,6 +12,42 @@ import {
|
||||
} from "./linuxPackages.js";
|
||||
|
||||
describe("dpkg output", () => {
|
||||
it("adds merged-/usr ownership candidates only for merged system directories", () => {
|
||||
const paths = [
|
||||
"/usr/bin/editor",
|
||||
"/sbin/helper",
|
||||
"/usr/lib/x86_64-linux-gnu/libexample.so",
|
||||
"/lib64/ld-linux-x86-64.so.2",
|
||||
"/usr/lib32/libexample.so",
|
||||
"/libx32/libexample.so",
|
||||
"/usr/libo32/libexample.so",
|
||||
"/opt/vendor/app",
|
||||
"/home/ada/bin/app",
|
||||
"/usr/share/applications/app.desktop",
|
||||
"/usr/local/bin/app",
|
||||
];
|
||||
expect(ownershipCandidates(paths)).toEqual([
|
||||
"/usr/bin/editor",
|
||||
"/bin/editor",
|
||||
"/sbin/helper",
|
||||
"/usr/sbin/helper",
|
||||
"/usr/lib/x86_64-linux-gnu/libexample.so",
|
||||
"/lib/x86_64-linux-gnu/libexample.so",
|
||||
"/lib64/ld-linux-x86-64.so.2",
|
||||
"/usr/lib64/ld-linux-x86-64.so.2",
|
||||
"/usr/lib32/libexample.so",
|
||||
"/lib32/libexample.so",
|
||||
"/libx32/libexample.so",
|
||||
"/usr/libx32/libexample.so",
|
||||
"/usr/libo32/libexample.so",
|
||||
"/libo32/libexample.so",
|
||||
"/opt/vendor/app",
|
||||
"/home/ada/bin/app",
|
||||
"/usr/share/applications/app.desktop",
|
||||
"/usr/local/bin/app",
|
||||
]);
|
||||
});
|
||||
|
||||
it("reads every owner of a path and skips diversions", () => {
|
||||
expect(
|
||||
parseDpkgOwners(
|
||||
|
||||
@@ -26,14 +26,18 @@ export const parseDpkgOwners = (
|
||||
return [...owners];
|
||||
};
|
||||
|
||||
/** `/bin/x` and `/usr/bin/x` name the same file on merged-/usr systems, and dpkg may know either. */
|
||||
/** Only merged system directories have alternate `/usr` spellings; application, home and shared-data paths retain their own identity. */
|
||||
export const ownershipCandidates = (
|
||||
paths: readonly string[],
|
||||
): readonly string[] => [
|
||||
...new Set(
|
||||
paths.flatMap((path) =>
|
||||
path.startsWith("/usr/") ? [path, path.slice(4)] : [path, `/usr${path}`],
|
||||
),
|
||||
paths.flatMap((path) => {
|
||||
if (!/^\/(?:usr\/)?(?:bin|sbin|lib(?:32|64|o32|x32)?)\//u.test(path))
|
||||
return [path];
|
||||
return path.startsWith("/usr/")
|
||||
? [path, path.slice(4)]
|
||||
: [path, `/usr${path}`];
|
||||
}),
|
||||
),
|
||||
];
|
||||
|
||||
|
||||
@@ -52,6 +52,7 @@ export const evidenceSubjectFormatSchema = z.enum([
|
||||
"dfuse",
|
||||
"esp-flash-image",
|
||||
"fit",
|
||||
"ntfs-mft",
|
||||
]);
|
||||
|
||||
/** Format-owned architecture declarations shared by Evidence and persisted executions. */
|
||||
|
||||
@@ -137,7 +137,8 @@ describe("child-process options detection", () => {
|
||||
["negative-zero", `spawn("tool", [], { env: -0 })`, false],
|
||||
["bigint-zero", `spawn("tool", [], { env: 0n })`, false],
|
||||
["zero-float", `spawn("tool", [], { env: 0.0 })`, false],
|
||||
["nan", `spawn("tool", [], { env: NaN })`, true],
|
||||
["nan", `spawn("tool", [], { env: NaN })`, false],
|
||||
["shadowed-nan", `((NaN) => spawn("tool", [], { env: NaN }))({})`, true],
|
||||
["empty-string", `spawn("tool", [], { env: "" })`, false],
|
||||
["false", `spawn("tool", [], { env: false })`, false],
|
||||
["empty-object", `spawn("tool", [], { env: {} })`, true],
|
||||
|
||||
@@ -81,7 +81,7 @@ export const collectJavaScriptSemanticChildProcesses = (
|
||||
!rejectsOptionsShape(candidate, context, state),
|
||||
);
|
||||
const childProcessSpawns = candidates.map((candidate) =>
|
||||
immutableSpawn(candidate, context),
|
||||
immutableSpawn(candidate, context, state),
|
||||
);
|
||||
return {
|
||||
childProcessSpawns,
|
||||
@@ -146,6 +146,7 @@ const collectSpawns = (
|
||||
const immutableSpawn = (
|
||||
candidate: SpawnCandidate,
|
||||
context: ChildSpawnContext,
|
||||
state: JavaScriptSemanticAnalysisState,
|
||||
): JavaScriptSemanticChildProcessSpawn => {
|
||||
const { options, argvCount } = spawnArguments(candidate, context);
|
||||
const command = dataEffectLiteralString(candidate.node.arguments[0]);
|
||||
@@ -161,7 +162,7 @@ const immutableSpawn = (
|
||||
projectedResultBindings: candidate.projectedResultBindings,
|
||||
command,
|
||||
argvCount,
|
||||
environmentSupplied: environmentSupplied(options, context),
|
||||
environmentSupplied: environmentSupplied(options, context, state),
|
||||
stdioMode: stdioMode(options, candidate.method, context),
|
||||
resolution: command === null ? "partial" : "complete",
|
||||
};
|
||||
@@ -443,11 +444,17 @@ const spawnArguments = (
|
||||
const environmentSupplied = (
|
||||
options: t.Node | undefined,
|
||||
context: ChildSpawnContext,
|
||||
state: JavaScriptSemanticAnalysisState,
|
||||
): boolean => {
|
||||
const lookup = staticObjectProperty(options, "env", context.isNullish);
|
||||
if (lookup.status !== "explicit") return false;
|
||||
const value = unwrapJavaScriptExpression(lookup.value).node;
|
||||
if (context.isNullish(value)) return false;
|
||||
if (
|
||||
t.isIdentifier(value, { name: "NaN" }) &&
|
||||
isUnshadowedGlobal(value, state, "NaN")
|
||||
)
|
||||
return false;
|
||||
return staticJavaScriptTruthiness(value) !== false;
|
||||
};
|
||||
|
||||
|
||||
@@ -83,6 +83,7 @@ export const liveBinaryTargetSummaryVariants = [
|
||||
"uf2",
|
||||
"dfuse",
|
||||
"fit",
|
||||
"ntfs-mft",
|
||||
]),
|
||||
architecture: z.null(),
|
||||
}),
|
||||
|
||||
@@ -0,0 +1,478 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import {
|
||||
MFT_DEFAULT_RECORD_BYTES,
|
||||
MFT_MAX_FILE_BYTES,
|
||||
createMftDecoder,
|
||||
parseMftImage,
|
||||
} from "./ntfsMft.js";
|
||||
|
||||
interface RecordOptions {
|
||||
readonly sequenceNumber?: number;
|
||||
readonly hardLinkCount?: number;
|
||||
readonly flags?: number;
|
||||
readonly baseRecordReference?: bigint;
|
||||
readonly recordBytes?: number;
|
||||
/** Write a wrong trailer value into this sector, leaving the others correct. */
|
||||
readonly corruptSector?: number;
|
||||
/** Truncate the update sequence array to this many entries; 0 omits it. */
|
||||
readonly usaEntries?: number;
|
||||
readonly usaOffset?: number;
|
||||
}
|
||||
|
||||
/** One FILE record with a zeroed body and fixups applied the way NTFS writes them. */
|
||||
const recordOf = (options: RecordOptions = {}): Uint8Array => {
|
||||
const recordBytes = options.recordBytes ?? MFT_DEFAULT_RECORD_BYTES;
|
||||
const record = new Uint8Array(recordBytes);
|
||||
const view = new DataView(record.buffer);
|
||||
record.set(Buffer.from("FILE", "latin1"), 0);
|
||||
const sequenceNumber = options.sequenceNumber ?? 1;
|
||||
const sectors = recordBytes / 512;
|
||||
|
||||
const usaEntries = options.usaEntries ?? sectors + 1;
|
||||
if (usaEntries > 0) {
|
||||
const usaOffset = options.usaOffset ?? 48;
|
||||
view.setUint16(4, usaOffset, true);
|
||||
view.setUint16(6, usaEntries, true);
|
||||
view.setUint16(usaOffset, sequenceNumber, true);
|
||||
for (let sector = 0; sector < sectors; sector += 1) {
|
||||
const trailer = sector * 512 + 510;
|
||||
// A saved trailer is what the sector held before the fixup replaced it.
|
||||
view.setUint16(usaOffset + 2 * (sector + 1), 0xa5a5, true);
|
||||
view.setUint16(
|
||||
trailer,
|
||||
sector >= usaEntries - 1
|
||||
? 0xa5a5
|
||||
: sector === options.corruptSector
|
||||
? 0x1234
|
||||
: sequenceNumber,
|
||||
true,
|
||||
);
|
||||
}
|
||||
} else {
|
||||
view.setUint16(4, 0, true);
|
||||
view.setUint16(6, 0, true);
|
||||
}
|
||||
|
||||
view.setUint16(16, sequenceNumber, true);
|
||||
view.setUint16(18, options.hardLinkCount ?? 1, true);
|
||||
const firstAttributeOffset =
|
||||
Math.ceil((48 + Math.max(0, usaEntries) * 2) / 8) * 8;
|
||||
view.setUint16(20, firstAttributeOffset, true);
|
||||
view.setUint16(22, options.flags ?? 0x0001, true);
|
||||
view.setUint32(24, firstAttributeOffset + 8, true);
|
||||
view.setUint32(firstAttributeOffset, 0xffff_ffff, true);
|
||||
view.setUint32(28, recordBytes, true);
|
||||
view.setBigUint64(32, options.baseRecordReference ?? 0n, true);
|
||||
view.setUint16(40, 0, true);
|
||||
return record;
|
||||
};
|
||||
|
||||
const streamOf = (...records: readonly Uint8Array[]): Uint8Array => {
|
||||
const total = records.reduce((sum, record) => sum + record.length, 0);
|
||||
const stream = new Uint8Array(total);
|
||||
let offset = 0;
|
||||
for (const record of records) {
|
||||
stream.set(record, offset);
|
||||
const view = new DataView(stream.buffer, offset, record.length);
|
||||
if (view.getUint16(4, true) >= 48)
|
||||
view.setUint32(44, offset / record.length, true);
|
||||
offset += record.length;
|
||||
}
|
||||
return stream;
|
||||
};
|
||||
|
||||
describe("MFT header facts", () => {
|
||||
it("inventories a clean record's header and verified fixups", () => {
|
||||
const parsed = parseMftImage(streamOf(recordOf({ sequenceNumber: 5 })));
|
||||
expect(parsed.ok).toBe(true);
|
||||
if (!parsed.ok) return;
|
||||
expect(parsed.value.metadata).toMatchObject({
|
||||
kind: "ntfs-mft",
|
||||
record_bytes: MFT_DEFAULT_RECORD_BYTES,
|
||||
record_count: 1,
|
||||
in_use_record_count: 1,
|
||||
directory_record_count: 0,
|
||||
extension_record_count: 0,
|
||||
fixup_mismatch_count: 0,
|
||||
fixup_unverified_record_count: 0,
|
||||
truncated_tail_bytes: 0,
|
||||
});
|
||||
expect(parsed.value.records[0]).toMatchObject({
|
||||
index: 0,
|
||||
fileOffset: 0,
|
||||
sequenceNumber: 5,
|
||||
inUse: true,
|
||||
directory: false,
|
||||
baseRecordReference: null,
|
||||
firstAttributeOffset: 56,
|
||||
fixups: {
|
||||
sequenceNumber: 5,
|
||||
verifiedSectors: 2,
|
||||
coveredSectors: 2,
|
||||
mismatchSector: null,
|
||||
malformed: false,
|
||||
},
|
||||
});
|
||||
expect(parsed.value.limitations).toEqual([]);
|
||||
});
|
||||
|
||||
it("identifies unrecognized flags without losing their source bits", () => {
|
||||
const parsed = parseMftImage(recordOf({ flags: 0x8001 }));
|
||||
if (!parsed.ok) throw new Error(parsed.error.message);
|
||||
expect(parsed.value.metadata.record_headers[0]?.flags).toEqual({
|
||||
raw: 0x8001,
|
||||
names: ["in-use"],
|
||||
unknownMask: 0x8000,
|
||||
});
|
||||
});
|
||||
|
||||
it.each([0, 49, 64, 72])(
|
||||
"reports an invalid first attribute offset %i",
|
||||
(offset) => {
|
||||
const record = recordOf();
|
||||
new DataView(record.buffer).setUint16(20, offset, true);
|
||||
const parsed = parseMftImage(record);
|
||||
if (!parsed.ok) throw new Error(parsed.error.message);
|
||||
expect(parsed.value.records[0]?.extentValid).toBe(false);
|
||||
expect(parsed.value.limitations.join(" ")).toContain("inconsistent");
|
||||
},
|
||||
);
|
||||
|
||||
it("reports a declared record number that contradicts its stream position", () => {
|
||||
const bytes = streamOf(recordOf(), recordOf());
|
||||
new DataView(bytes.buffer).setUint32(1024 + 44, 0, true);
|
||||
const parsed = parseMftImage(bytes);
|
||||
if (!parsed.ok) throw new Error(parsed.error.message);
|
||||
expect(parsed.value.records[1]).toMatchObject({
|
||||
index: 1,
|
||||
declaredRecordNumber: 0,
|
||||
recordNumberMatchesSlot: false,
|
||||
});
|
||||
expect(parsed.value.limitations.join(" ")).toContain(
|
||||
"disagrees with their stream slot",
|
||||
);
|
||||
});
|
||||
|
||||
it("keeps the inline header window separate from the parsed record count", () => {
|
||||
const parsed = parseMftImage(
|
||||
streamOf(...Array.from({ length: 33 }, () => recordOf())),
|
||||
);
|
||||
if (!parsed.ok) throw new Error(parsed.error.message);
|
||||
expect(parsed.value.metadata).toMatchObject({
|
||||
record_count: 33,
|
||||
omitted_record_headers: 1,
|
||||
});
|
||||
expect(parsed.value.metadata.record_headers).toHaveLength(32);
|
||||
expect(parsed.value.records).toHaveLength(33);
|
||||
expect(parsed.value.limitations).toEqual([]);
|
||||
});
|
||||
|
||||
it("preserves a log sequence number above the safe integer range", () => {
|
||||
const record = recordOf();
|
||||
new DataView(record.buffer).setBigUint64(8, 9007199254740993n, true);
|
||||
const parsed = parseMftImage(record);
|
||||
if (!parsed.ok) throw new Error(parsed.error.message);
|
||||
expect(parsed.value.records[0]?.logFileSequenceNumber).toBe(
|
||||
"9007199254740993",
|
||||
);
|
||||
});
|
||||
|
||||
it("verifies the older FILE header without reading its fixup words as a record number", () => {
|
||||
const oldRecord = recordOf({ usaOffset: 42 });
|
||||
const newRecord = recordOf();
|
||||
new DataView(newRecord.buffer).setUint32(44, 1, true);
|
||||
|
||||
const parsed = parseMftImage(streamOf(oldRecord, newRecord));
|
||||
if (!parsed.ok) throw new Error(parsed.error.message);
|
||||
expect(parsed.value.records).toMatchObject([
|
||||
{
|
||||
declaredRecordNumber: null,
|
||||
fixups: { malformed: false, verifiedSectors: 2 },
|
||||
},
|
||||
{
|
||||
declaredRecordNumber: 1,
|
||||
fixups: { malformed: false, verifiedSectors: 2 },
|
||||
},
|
||||
]);
|
||||
expect(parsed.value.limitations).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("MFT fixup coverage", () => {
|
||||
it.each([0, 0xffff])(
|
||||
"leaves reserved USA sequence %i unverified",
|
||||
(sequenceNumber) => {
|
||||
const parsed = parseMftImage(recordOf({ sequenceNumber }));
|
||||
if (!parsed.ok) throw new Error(parsed.error.message);
|
||||
expect(parsed.value.records[0]?.fixups).toMatchObject({
|
||||
sequenceNumber,
|
||||
verifiedSectors: 0,
|
||||
malformed: true,
|
||||
});
|
||||
expect(parsed.value.metadata.fixup_unverified_record_count).toBe(1);
|
||||
expect(parsed.value.limitations.join(" ")).toContain(
|
||||
"invalid or incomplete",
|
||||
);
|
||||
},
|
||||
);
|
||||
|
||||
it("keeps a record whose sector trailer is not the sequence number, and reports it", () => {
|
||||
const parsed = parseMftImage(
|
||||
streamOf(recordOf(), recordOf({ sequenceNumber: 9, corruptSector: 1 })),
|
||||
);
|
||||
expect(parsed.ok).toBe(true);
|
||||
if (!parsed.ok) return;
|
||||
expect(parsed.value.records[1]?.fixups).toMatchObject({
|
||||
sequenceNumber: 9,
|
||||
verifiedSectors: 1,
|
||||
mismatchSector: 1,
|
||||
});
|
||||
expect(parsed.value.metadata.fixup_mismatch_count).toBe(1);
|
||||
expect(parsed.value.limitations.join(" ")).toMatch(
|
||||
/1 record\(s\) hold a sector trailer/u,
|
||||
);
|
||||
});
|
||||
|
||||
it("treats a missing update sequence array as unverified rather than clean", () => {
|
||||
const parsed = parseMftImage(streamOf(recordOf({ usaEntries: 0 })));
|
||||
expect(parsed.ok).toBe(true);
|
||||
if (!parsed.ok) return;
|
||||
expect(parsed.value.records[0]?.fixups).toMatchObject({
|
||||
malformed: true,
|
||||
mismatchSector: null,
|
||||
coveredSectors: 0,
|
||||
});
|
||||
expect(parsed.value.metadata.fixup_mismatch_count).toBe(0);
|
||||
expect(parsed.value.metadata.fixup_unverified_record_count).toBe(1);
|
||||
expect(parsed.value.limitations.join(" ")).toMatch(
|
||||
/invalid or incomplete update sequence array/u,
|
||||
);
|
||||
});
|
||||
|
||||
it.each([
|
||||
[48, 4],
|
||||
[49, 3],
|
||||
[508, 3],
|
||||
])(
|
||||
"rejects a malformed update sequence array at %i with %i entries",
|
||||
(offset, count) => {
|
||||
const bytes = recordOf();
|
||||
const view = new DataView(bytes.buffer);
|
||||
view.setUint16(4, offset, true);
|
||||
view.setUint16(6, count, true);
|
||||
|
||||
const parsed = parseMftImage(bytes);
|
||||
if (!parsed.ok) throw new Error(parsed.error.message);
|
||||
expect(parsed.value.records[0]?.fixups).toMatchObject({
|
||||
malformed: true,
|
||||
verifiedSectors: 0,
|
||||
});
|
||||
expect(parsed.value.metadata.fixup_unverified_record_count).toBe(1);
|
||||
},
|
||||
);
|
||||
|
||||
it("counts uncovered sectors independently of a mismatched trailer", () => {
|
||||
const parsed = parseMftImage(
|
||||
recordOf({ recordBytes: 4096, usaEntries: 3, corruptSector: 0 }),
|
||||
);
|
||||
if (!parsed.ok) throw new Error(parsed.error.message);
|
||||
expect(parsed.value.metadata).toMatchObject({
|
||||
fixup_mismatch_count: 1,
|
||||
fixup_unverified_record_count: 1,
|
||||
});
|
||||
expect(parsed.value.limitations.join(" ")).toContain(
|
||||
"invalid or incomplete",
|
||||
);
|
||||
});
|
||||
|
||||
it("reports an array that covers fewer sectors than the record has", () => {
|
||||
// A 4 KiB record holds eight sectors; a three-entry array leaves five unverified.
|
||||
const parsed = parseMftImage(
|
||||
streamOf(recordOf({ recordBytes: 4096, usaEntries: 3 })),
|
||||
);
|
||||
expect(parsed.ok).toBe(true);
|
||||
if (!parsed.ok) return;
|
||||
expect(parsed.value.records[0]?.fixups).toMatchObject({
|
||||
coveredSectors: 2,
|
||||
verifiedSectors: 2,
|
||||
mismatchSector: null,
|
||||
malformed: false,
|
||||
});
|
||||
expect(parsed.value.metadata.fixup_unverified_record_count).toBe(1);
|
||||
});
|
||||
});
|
||||
|
||||
describe("MFT record slots", () => {
|
||||
it("distinguishes extension records from base records", () => {
|
||||
const parsed = parseMftImage(
|
||||
streamOf(
|
||||
recordOf(),
|
||||
recordOf({
|
||||
flags: 0x0005,
|
||||
baseRecordReference: 0xabcd_fedc_ba98_7654n,
|
||||
}),
|
||||
),
|
||||
);
|
||||
expect(parsed.ok).toBe(true);
|
||||
if (!parsed.ok) return;
|
||||
expect(parsed.value.records[1]).toMatchObject({
|
||||
baseRecordReference: {
|
||||
recordNumber: 0xfedc_ba98_7654,
|
||||
sequenceNumber: 0xabcd,
|
||||
},
|
||||
flags: { raw: 0x0005, names: ["in-use", "system"] },
|
||||
});
|
||||
expect(parsed.value.metadata.extension_record_count).toBe(1);
|
||||
});
|
||||
|
||||
it("names directory records without losing the in-use flag", () => {
|
||||
const parsed = parseMftImage(streamOf(recordOf({ flags: 0x0003 })));
|
||||
expect(parsed.ok).toBe(true);
|
||||
if (!parsed.ok) return;
|
||||
expect(parsed.value.records[0]).toMatchObject({
|
||||
directory: true,
|
||||
inUse: true,
|
||||
flags: { names: ["in-use", "directory"] },
|
||||
});
|
||||
expect(parsed.value.metadata.directory_record_count).toBe(1);
|
||||
});
|
||||
|
||||
it("skips malformed and sparse slots without renumbering valid records", () => {
|
||||
const stream = streamOf(
|
||||
recordOf(),
|
||||
recordOf(),
|
||||
new Uint8Array(MFT_DEFAULT_RECORD_BYTES),
|
||||
recordOf(),
|
||||
);
|
||||
stream.set(Buffer.from("BAAD", "latin1"), MFT_DEFAULT_RECORD_BYTES);
|
||||
const parsed = parseMftImage(stream);
|
||||
expect(parsed.ok).toBe(true);
|
||||
if (!parsed.ok) return;
|
||||
expect(parsed.value.metadata.record_count).toBe(2);
|
||||
expect(
|
||||
parsed.value.records.map(({ index, fileOffset }) => [index, fileOffset]),
|
||||
).toEqual([
|
||||
[0, 0],
|
||||
[3, 3 * MFT_DEFAULT_RECORD_BYTES],
|
||||
]);
|
||||
expect(parsed.value.limitations).toEqual([
|
||||
"record at 1024 does not begin with the FILE signature; 2 slot(s) without a FILE signature were found",
|
||||
]);
|
||||
});
|
||||
|
||||
it("keeps a trailing partial record out of the inventory", () => {
|
||||
const stream = new Uint8Array(1024 + 300);
|
||||
stream.set(recordOf(), 0);
|
||||
const parsed = parseMftImage(stream);
|
||||
expect(parsed.ok).toBe(true);
|
||||
if (!parsed.ok) return;
|
||||
expect(parsed.value.metadata.record_count).toBe(1);
|
||||
expect(parsed.value.metadata.truncated_tail_bytes).toBe(300);
|
||||
expect(parsed.value.limitations.join(" ")).toMatch(
|
||||
/300 trailing byte\(s\)/u,
|
||||
);
|
||||
});
|
||||
|
||||
it("falls back to the default record size when record 0 declares none", () => {
|
||||
const record = recordOf();
|
||||
new DataView(record.buffer).setUint32(28, 0, true);
|
||||
const parsed = parseMftImage(record);
|
||||
expect(parsed.ok).toBe(true);
|
||||
if (!parsed.ok) return;
|
||||
expect(parsed.value.recordBytes).toBe(MFT_DEFAULT_RECORD_BYTES);
|
||||
expect(parsed.value.limitations.join(" ")).toMatch(
|
||||
/not a supported power of two/u,
|
||||
);
|
||||
});
|
||||
|
||||
it("reads a 4 KiB record at its declared size", () => {
|
||||
const parsed = parseMftImage(streamOf(recordOf({ recordBytes: 4096 })));
|
||||
expect(parsed.ok).toBe(true);
|
||||
if (!parsed.ok) return;
|
||||
expect(parsed.value.recordBytes).toBe(4096);
|
||||
expect(parsed.value.metadata.record_count).toBe(1);
|
||||
});
|
||||
});
|
||||
|
||||
describe("parseMftImage refusals and limits", () => {
|
||||
it("refuses a file shorter than one record header", () => {
|
||||
const parsed = parseMftImage(new Uint8Array(32));
|
||||
expect(parsed.ok).toBe(false);
|
||||
if (parsed.ok) return;
|
||||
expect(parsed.error.message).toMatch(
|
||||
/shorter than one 42-byte record header/u,
|
||||
);
|
||||
});
|
||||
|
||||
it("refuses bytes that do not start with the FILE signature", () => {
|
||||
const parsed = parseMftImage(Buffer.alloc(2048, 0x41));
|
||||
expect(parsed.ok).toBe(false);
|
||||
if (parsed.ok) return;
|
||||
expect(parsed.error.message).toMatch(
|
||||
/does not begin with the FILE signature/u,
|
||||
);
|
||||
});
|
||||
|
||||
it("refuses a file above the fixed ceiling", () => {
|
||||
const decoder = createMftDecoder(MFT_MAX_FILE_BYTES + 1);
|
||||
expect(decoder.ok).toBe(false);
|
||||
if (decoder.ok) return;
|
||||
expect(decoder.error).toMatchObject({
|
||||
kind: "limit",
|
||||
boundary: "file-size",
|
||||
});
|
||||
});
|
||||
|
||||
it("refuses streams that differ from the admitted byte count", () => {
|
||||
const stream = recordOf();
|
||||
const short = createMftDecoder(stream.length + 1);
|
||||
if (!short.ok) throw new Error(short.error.message);
|
||||
expect(short.value.write(stream).ok).toBe(true);
|
||||
expect(short.value.finish()).toMatchObject({
|
||||
ok: false,
|
||||
error: {
|
||||
kind: "format",
|
||||
message: "stream ended before the admitted file length",
|
||||
},
|
||||
});
|
||||
|
||||
const long = createMftDecoder(stream.length - 1);
|
||||
if (!long.ok) throw new Error(long.error.message);
|
||||
expect(long.value.write(stream)).toMatchObject({
|
||||
ok: false,
|
||||
error: {
|
||||
kind: "format",
|
||||
message: "received more bytes than the admitted file length",
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it("returns the same inventory without retaining all the decoded records", () => {
|
||||
const bytes = streamOf(...Array.from({ length: 33 }, () => recordOf()));
|
||||
const decoder = createMftDecoder(bytes.length);
|
||||
const parsed = parseMftImage(bytes);
|
||||
if (!decoder.ok || !parsed.ok) throw new Error("invalid fixture");
|
||||
expect(decoder.value.write(bytes).ok).toBe(true);
|
||||
const inventory = decoder.value.finish();
|
||||
if (!inventory.ok) throw new Error(inventory.error.message);
|
||||
expect(inventory.value).not.toHaveProperty("records");
|
||||
expect(inventory.value.metadata).toEqual(parsed.value.metadata);
|
||||
expect(inventory.value.limitations).toEqual(parsed.value.limitations);
|
||||
});
|
||||
|
||||
it("decodes the same stream through the chunked decoder", () => {
|
||||
const stream = streamOf(recordOf(), recordOf({ flags: 0x0003 }));
|
||||
const decoder = createMftDecoder(stream.length);
|
||||
expect(decoder.ok).toBe(true);
|
||||
if (!decoder.ok) return;
|
||||
// Feed one byte past a record boundary so a chunk never aligns with a record.
|
||||
for (const chunk of [stream.subarray(0, 1025), stream.subarray(1025)])
|
||||
expect(decoder.value.write(chunk).ok).toBe(true);
|
||||
const image = decoder.value.finish();
|
||||
expect(image.ok).toBe(true);
|
||||
if (!image.ok) return;
|
||||
expect(image.value.metadata.record_count).toBe(2);
|
||||
expect(image.value.metadata.directory_record_count).toBe(1);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,512 @@
|
||||
import { err, ok, type Result } from "../result.js";
|
||||
import type {
|
||||
RegionDecodeFailure,
|
||||
RegionDecoder,
|
||||
} from "../firmware/regionDecoder.js";
|
||||
|
||||
/** An NTFS FILE record begins with this signature. */
|
||||
const MFT_FILE_RECORD_MAGIC = "FILE";
|
||||
const MFT_FILE_RECORD_MAGIC_BYTES = 4;
|
||||
/** The shortest header worth reading: a FILE record header ends at `nextAttributeId`. */
|
||||
const NTFS_MFT_MIN_RECORD_BYTES = 42;
|
||||
|
||||
const hasMftFileRecordSignature = (prefix: Uint8Array): boolean =>
|
||||
prefix.length >= MFT_FILE_RECORD_MAGIC_BYTES &&
|
||||
text(prefix.subarray(0, MFT_FILE_RECORD_MAGIC_BYTES)) ===
|
||||
MFT_FILE_RECORD_MAGIC;
|
||||
|
||||
/** A record's declared size is a power of two from 512 upward; 1024 is NTFS's default. */
|
||||
export const MFT_DEFAULT_RECORD_BYTES = 1024;
|
||||
const MFT_MIN_RECORD_BYTES = 512;
|
||||
const MFT_MAX_RECORD_BYTES = 65536;
|
||||
/** An update sequence array holds at least the sequence number and one fixup word. */
|
||||
const USA_MIN_ENTRIES = 2;
|
||||
/** NTFS fixes up one 512-byte sector at a time. */
|
||||
const FIXUP_SECTOR_BYTES = 512;
|
||||
/**
|
||||
* Fixed whole-file ceiling. Records are inventoried from their headers only;
|
||||
* attributes and their payloads are not decoded, so this is deliberately
|
||||
* smaller than a volume's MFT can be.
|
||||
*/
|
||||
export const MFT_MAX_FILE_BYTES = 512 * 1024 * 1024;
|
||||
/** Limit the inline header sample independently of full stream coverage. */
|
||||
export const MFT_INLINE_HEADER_LIMIT = 32;
|
||||
|
||||
/** FILE record header fields; offsets are from the record start. */
|
||||
const RECORD_HEADER = {
|
||||
updateSequenceOffset: 4,
|
||||
updateSequenceCount: 6,
|
||||
logFileSequenceNumber: 8,
|
||||
sequenceNumber: 16,
|
||||
hardLinkCount: 18,
|
||||
firstAttributeOffset: 20,
|
||||
flags: 22,
|
||||
usedBytes: 24,
|
||||
totalBytes: 28,
|
||||
baseRecordReference: 32,
|
||||
nextAttributeId: 40,
|
||||
recordNumber: 44,
|
||||
} as const;
|
||||
const RECORD_HEADER_WITH_NUMBER_BYTES = 48;
|
||||
|
||||
const FILE_RECORD_IN_USE = 0x0001;
|
||||
const FILE_RECORD_IS_DIRECTORY = 0x0002;
|
||||
const FILE_RECORD_NAMES = new Map<number, string>([
|
||||
[FILE_RECORD_IN_USE, "in-use"],
|
||||
[FILE_RECORD_IS_DIRECTORY, "directory"],
|
||||
[0x0004, "system"],
|
||||
[0x0008, "view-index"],
|
||||
]);
|
||||
|
||||
export interface MftRecordFixups {
|
||||
/** The update sequence number the array's first entry holds, or 0 when absent. */
|
||||
readonly sequenceNumber: number;
|
||||
readonly entries: number;
|
||||
/** Sectors whose trailer held the sequence number before a mismatch. */
|
||||
readonly verifiedSectors: number;
|
||||
/** Sectors the array covers; fewer than the record's sectors leaves a tail unverified. */
|
||||
readonly coveredSectors: number;
|
||||
/** The first sector whose trailer did not hold the sequence number, if any. */
|
||||
readonly mismatchSector: number | null;
|
||||
/** The array itself is missing, misplaced or too short to cover one sector. */
|
||||
readonly malformed: boolean;
|
||||
}
|
||||
|
||||
export interface MftFileReference {
|
||||
readonly recordNumber: number;
|
||||
readonly sequenceNumber: number;
|
||||
}
|
||||
|
||||
const decodeMftFileReference = (reference: bigint): MftFileReference => ({
|
||||
recordNumber: Number(reference & 0xffff_ffff_ffffn),
|
||||
sequenceNumber: Number(reference >> 48n),
|
||||
});
|
||||
|
||||
export interface MftRecord {
|
||||
/** Zero-based ordinal of this record in the stream. */
|
||||
readonly index: number;
|
||||
/** File offset where the record starts. */
|
||||
readonly fileOffset: number;
|
||||
readonly sequenceNumber: number;
|
||||
/** Hard link count as recorded; a zero count marks a record with no names. */
|
||||
readonly hardLinkCount: number;
|
||||
readonly flags: {
|
||||
readonly raw: number;
|
||||
readonly names: readonly string[];
|
||||
readonly unknownMask: number;
|
||||
};
|
||||
readonly inUse: boolean;
|
||||
readonly directory: boolean;
|
||||
/** Used and allocated byte counts as declared by the FILE record. */
|
||||
readonly usedBytes: number;
|
||||
readonly totalBytes: number;
|
||||
readonly extentValid: boolean;
|
||||
/** The base record this is an extension of, or `null` for a base record. */
|
||||
readonly baseRecordReference: MftFileReference | null;
|
||||
/** The record number the header declares; `null` when the header predates the field. */
|
||||
readonly declaredRecordNumber: number | null;
|
||||
readonly recordNumberMatchesSlot: boolean | null;
|
||||
/** `$LogFile` sequence number as recorded; no log is read. */
|
||||
readonly logFileSequenceNumber: string;
|
||||
/** Declared offset of the first attribute; no attribute decoding follows yet. */
|
||||
readonly firstAttributeOffset: number;
|
||||
/** Declared id of the next attribute; no attribute decoding follows yet. */
|
||||
readonly nextAttributeId: number;
|
||||
readonly fixups: MftRecordFixups;
|
||||
}
|
||||
|
||||
export interface MftInventory {
|
||||
readonly metadata: MftMetadata;
|
||||
/** The record size the stream declared, or the fixed default when it did not. */
|
||||
readonly recordBytes: number;
|
||||
readonly limitations: readonly string[];
|
||||
}
|
||||
|
||||
export interface MftImage extends MftInventory {
|
||||
readonly records: readonly MftRecord[];
|
||||
}
|
||||
|
||||
export interface MftMetadata {
|
||||
readonly kind: "ntfs-mft";
|
||||
readonly record_bytes: number;
|
||||
readonly record_count: number;
|
||||
readonly in_use_record_count: number;
|
||||
readonly directory_record_count: number;
|
||||
readonly extension_record_count: number;
|
||||
readonly fixup_mismatch_count: number;
|
||||
readonly fixup_unverified_record_count: number;
|
||||
readonly truncated_tail_bytes: number;
|
||||
readonly record_headers: readonly MftRecord[];
|
||||
readonly omitted_record_headers: number;
|
||||
}
|
||||
|
||||
const text = (ascii: Uint8Array): string =>
|
||||
Buffer.from(ascii).toString("latin1");
|
||||
|
||||
const failure = (
|
||||
message: string,
|
||||
boundary?: RegionDecodeFailure["boundary"],
|
||||
): RegionDecodeFailure => ({
|
||||
kind: "format",
|
||||
message,
|
||||
...(boundary === undefined ? {} : { boundary }),
|
||||
});
|
||||
|
||||
const limit = (
|
||||
message: string,
|
||||
boundary?: RegionDecodeFailure["boundary"],
|
||||
): RegionDecodeFailure => ({
|
||||
kind: "limit",
|
||||
message,
|
||||
...(boundary === undefined ? {} : { boundary }),
|
||||
});
|
||||
|
||||
const parseRecordFlags = (raw: number): MftRecord["flags"] => {
|
||||
const names: string[] = [];
|
||||
let unknownMask = raw;
|
||||
for (const [bit, name] of FILE_RECORD_NAMES) {
|
||||
if ((raw & bit) !== 0) names.push(name);
|
||||
unknownMask &= ~bit;
|
||||
}
|
||||
return { raw, names, unknownMask };
|
||||
};
|
||||
|
||||
const isRecordSize = (bytes: number): boolean =>
|
||||
bytes >= MFT_MIN_RECORD_BYTES &&
|
||||
bytes <= MFT_MAX_RECORD_BYTES &&
|
||||
(bytes & (bytes - 1)) === 0;
|
||||
|
||||
const updateSequenceArrayFits = (
|
||||
offset: number,
|
||||
entries: number,
|
||||
sectors: number,
|
||||
): boolean =>
|
||||
entries >= USA_MIN_ENTRIES &&
|
||||
entries <= sectors + 1 &&
|
||||
offset >= NTFS_MFT_MIN_RECORD_BYTES &&
|
||||
offset % 2 === 0 &&
|
||||
offset + entries * 2 <= FIXUP_SECTOR_BYTES - 2;
|
||||
|
||||
/** Recognize a FILE header without requiring its trailers or attributes to verify. */
|
||||
export const hasMftFileRecordHeader = (prefix: Uint8Array): boolean => {
|
||||
if (
|
||||
prefix.length < NTFS_MFT_MIN_RECORD_BYTES ||
|
||||
!hasMftFileRecordSignature(prefix)
|
||||
)
|
||||
return false;
|
||||
const view = new DataView(
|
||||
prefix.buffer,
|
||||
prefix.byteOffset,
|
||||
prefix.byteLength,
|
||||
);
|
||||
return (
|
||||
isRecordSize(view.getUint32(RECORD_HEADER.totalBytes, true)) ||
|
||||
updateSequenceArrayFits(
|
||||
view.getUint16(RECORD_HEADER.updateSequenceOffset, true),
|
||||
view.getUint16(RECORD_HEADER.updateSequenceCount, true),
|
||||
MFT_DEFAULT_RECORD_BYTES / FIXUP_SECTOR_BYTES,
|
||||
)
|
||||
);
|
||||
};
|
||||
|
||||
/**
|
||||
* Parse FILE record headers and verify their update-sequence fixups.
|
||||
*
|
||||
* Only the record structure is established: attributes, their names, the
|
||||
* filenames and the USN journal are left to later steps, so no path, timestamp
|
||||
* or file identity beyond the record itself is claimed. A record whose fixup
|
||||
* does not match keeps its header facts, because the stream still declares it,
|
||||
* and the mismatch is reported on the record and counted in the metadata.
|
||||
*/
|
||||
export const parseMftImage = (
|
||||
bytes: Uint8Array,
|
||||
): Result<MftImage, RegionDecodeFailure> => {
|
||||
const records: MftRecord[] = [];
|
||||
const inventory = scanMft(bytes, records);
|
||||
return inventory.ok ? ok({ ...inventory.value, records }) : inventory;
|
||||
};
|
||||
|
||||
const scanMft = (
|
||||
bytes: Uint8Array,
|
||||
retainedRecords?: MftRecord[],
|
||||
): Result<MftInventory, RegionDecodeFailure> => {
|
||||
if (bytes.length < NTFS_MFT_MIN_RECORD_BYTES)
|
||||
return err(
|
||||
failure(
|
||||
`file is ${String(bytes.length)} bytes, shorter than one ${String(NTFS_MFT_MIN_RECORD_BYTES)}-byte record header`,
|
||||
),
|
||||
);
|
||||
if (!hasMftFileRecordSignature(bytes))
|
||||
return err(failure("record 0 does not begin with the FILE signature"));
|
||||
|
||||
const limitations: string[] = [];
|
||||
const header = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
|
||||
const declaredRecordBytes = header.getUint32(RECORD_HEADER.totalBytes, true);
|
||||
const recordBytes = isRecordSize(declaredRecordBytes)
|
||||
? declaredRecordBytes
|
||||
: MFT_DEFAULT_RECORD_BYTES;
|
||||
if (!isRecordSize(declaredRecordBytes))
|
||||
limitations.push(
|
||||
`record 0 declares a record size of ${String(declaredRecordBytes)} bytes, which is not a supported power of two from ${String(MFT_MIN_RECORD_BYTES)} to ${String(MFT_MAX_RECORD_BYTES)}; records were read at the ${String(MFT_DEFAULT_RECORD_BYTES)}-byte default`,
|
||||
);
|
||||
|
||||
const recordHeaders: MftRecord[] = [];
|
||||
let recordCount = 0;
|
||||
let inUseCount = 0;
|
||||
let directoryCount = 0;
|
||||
let extensionCount = 0;
|
||||
let mismatchCount = 0;
|
||||
let unverifiedCount = 0;
|
||||
let invalidSlots = 0;
|
||||
let invalidExtents = 0;
|
||||
let inconsistentRecordNumbers = 0;
|
||||
let firstInvalidExtentOffset: number | undefined;
|
||||
let firstInvalidOffset: number | undefined;
|
||||
for (
|
||||
let offset = 0;
|
||||
offset + recordBytes <= bytes.length;
|
||||
offset += recordBytes
|
||||
) {
|
||||
if (
|
||||
!hasMftFileRecordSignature(
|
||||
bytes.subarray(offset, offset + MFT_FILE_RECORD_MAGIC_BYTES),
|
||||
)
|
||||
) {
|
||||
invalidSlots += 1;
|
||||
firstInvalidOffset ??= offset;
|
||||
if (!isRecordSize(declaredRecordBytes)) break;
|
||||
continue;
|
||||
}
|
||||
const record = readRecord(bytes, offset, offset / recordBytes, recordBytes);
|
||||
retainedRecords?.push(record);
|
||||
if (recordHeaders.length < MFT_INLINE_HEADER_LIMIT)
|
||||
recordHeaders.push(record);
|
||||
recordCount += 1;
|
||||
if (record.inUse) inUseCount += 1;
|
||||
if (record.directory) directoryCount += 1;
|
||||
if (record.baseRecordReference !== null) extensionCount += 1;
|
||||
if (record.fixups.mismatchSector !== null) mismatchCount += 1;
|
||||
if (
|
||||
record.fixups.malformed ||
|
||||
record.fixups.coveredSectors < recordBytes / FIXUP_SECTOR_BYTES
|
||||
)
|
||||
unverifiedCount += 1;
|
||||
if (record.recordNumberMatchesSlot === false)
|
||||
inconsistentRecordNumbers += 1;
|
||||
if (!record.extentValid) {
|
||||
invalidExtents += 1;
|
||||
firstInvalidExtentOffset ??= offset;
|
||||
}
|
||||
}
|
||||
if (firstInvalidOffset !== undefined)
|
||||
limitations.push(
|
||||
`record at ${String(firstInvalidOffset)} does not begin with the FILE signature; ${String(invalidSlots)} slot(s) without a FILE signature were found`,
|
||||
);
|
||||
|
||||
if (firstInvalidExtentOffset !== undefined)
|
||||
limitations.push(
|
||||
`${String(invalidExtents)} FILE record(s) declare inconsistent record extents, beginning at offset ${String(firstInvalidExtentOffset)}`,
|
||||
);
|
||||
|
||||
if (inconsistentRecordNumbers !== 0)
|
||||
limitations.push(
|
||||
`${String(inconsistentRecordNumbers)} FILE record(s) declare a record number that disagrees with their stream slot`,
|
||||
);
|
||||
|
||||
const truncatedTailBytes = bytes.length % recordBytes;
|
||||
if (truncatedTailBytes !== 0)
|
||||
limitations.push(
|
||||
`${String(truncatedTailBytes)} trailing byte(s) do not fill a ${String(recordBytes)}-byte record and were not parsed`,
|
||||
);
|
||||
|
||||
if (mismatchCount !== 0)
|
||||
limitations.push(
|
||||
`${String(mismatchCount)} record(s) hold a sector trailer that is not the update sequence number; their attributes were not read and no later step should trust them`,
|
||||
);
|
||||
if (unverifiedCount !== 0)
|
||||
limitations.push(
|
||||
`${String(unverifiedCount)} record(s) carry an invalid or incomplete update sequence array; their sector trailers were not fully verified`,
|
||||
);
|
||||
|
||||
const metadata: MftMetadata = {
|
||||
kind: "ntfs-mft",
|
||||
record_bytes: recordBytes,
|
||||
record_count: recordCount,
|
||||
in_use_record_count: inUseCount,
|
||||
directory_record_count: directoryCount,
|
||||
extension_record_count: extensionCount,
|
||||
fixup_mismatch_count: mismatchCount,
|
||||
fixup_unverified_record_count: unverifiedCount,
|
||||
truncated_tail_bytes: truncatedTailBytes,
|
||||
record_headers: recordHeaders,
|
||||
omitted_record_headers: recordCount - recordHeaders.length,
|
||||
};
|
||||
|
||||
return ok({ metadata, recordBytes, limitations });
|
||||
};
|
||||
|
||||
const readRecord = (
|
||||
bytes: Uint8Array,
|
||||
fileOffset: number,
|
||||
index: number,
|
||||
recordBytes: number,
|
||||
): MftRecord => {
|
||||
const record = bytes.subarray(fileOffset, fileOffset + recordBytes);
|
||||
const view = new DataView(
|
||||
record.buffer,
|
||||
record.byteOffset,
|
||||
record.byteLength,
|
||||
);
|
||||
|
||||
const flags = view.getUint16(RECORD_HEADER.flags, true);
|
||||
const baseRecordReference = view.getBigUint64(
|
||||
RECORD_HEADER.baseRecordReference,
|
||||
true,
|
||||
);
|
||||
|
||||
const usaOffset = view.getUint16(RECORD_HEADER.updateSequenceOffset, true);
|
||||
const fixups = verifyFixups(record, view, usaOffset);
|
||||
const usedBytes = view.getUint32(RECORD_HEADER.usedBytes, true);
|
||||
const totalBytes = view.getUint32(RECORD_HEADER.totalBytes, true);
|
||||
const firstAttributeOffset = view.getUint16(
|
||||
RECORD_HEADER.firstAttributeOffset,
|
||||
true,
|
||||
);
|
||||
|
||||
const declaredRecordNumber =
|
||||
!fixups.malformed && usaOffset >= RECORD_HEADER_WITH_NUMBER_BYTES
|
||||
? view.getUint32(RECORD_HEADER.recordNumber, true)
|
||||
: null;
|
||||
|
||||
return {
|
||||
index,
|
||||
fileOffset,
|
||||
sequenceNumber: view.getUint16(RECORD_HEADER.sequenceNumber, true),
|
||||
hardLinkCount: view.getUint16(RECORD_HEADER.hardLinkCount, true),
|
||||
flags: parseRecordFlags(flags),
|
||||
inUse: (flags & FILE_RECORD_IN_USE) !== 0,
|
||||
directory: (flags & FILE_RECORD_IS_DIRECTORY) !== 0,
|
||||
usedBytes,
|
||||
totalBytes,
|
||||
extentValid:
|
||||
totalBytes === recordBytes &&
|
||||
usedBytes >= NTFS_MFT_MIN_RECORD_BYTES &&
|
||||
usedBytes % 8 === 0 &&
|
||||
usedBytes <= totalBytes &&
|
||||
firstAttributeOffset % 8 === 0 &&
|
||||
firstAttributeOffset >= NTFS_MFT_MIN_RECORD_BYTES &&
|
||||
firstAttributeOffset + 4 <= usedBytes &&
|
||||
(fixups.malformed ||
|
||||
firstAttributeOffset >= usaOffset + fixups.entries * 2),
|
||||
baseRecordReference:
|
||||
baseRecordReference === 0n
|
||||
? null
|
||||
: decodeMftFileReference(baseRecordReference),
|
||||
declaredRecordNumber,
|
||||
recordNumberMatchesSlot:
|
||||
declaredRecordNumber === null ? null : declaredRecordNumber === index,
|
||||
logFileSequenceNumber: view
|
||||
.getBigUint64(RECORD_HEADER.logFileSequenceNumber, true)
|
||||
.toString(),
|
||||
firstAttributeOffset,
|
||||
nextAttributeId: view.getUint16(RECORD_HEADER.nextAttributeId, true),
|
||||
fixups,
|
||||
};
|
||||
};
|
||||
|
||||
/**
|
||||
* Each sector's last two bytes are replaced on disk by the update sequence
|
||||
* number; the array's first entry is that number and the rest are the saved
|
||||
* trailer words. A sector whose trailer is not the sequence number was not
|
||||
* cleanly written, so the record's attributes cannot be trusted.
|
||||
*/
|
||||
const verifyFixups = (
|
||||
record: Uint8Array,
|
||||
view: DataView,
|
||||
usaOffset: number,
|
||||
): MftRecordFixups => {
|
||||
const usaCount = view.getUint16(RECORD_HEADER.updateSequenceCount, true);
|
||||
const sectors = Math.floor(record.length / FIXUP_SECTOR_BYTES);
|
||||
// Entry 0 holds the sequence number, so sector n's saved trailer is entry n+1.
|
||||
const coveredSectors = Math.max(0, usaCount - 1);
|
||||
|
||||
if (!updateSequenceArrayFits(usaOffset, usaCount, sectors))
|
||||
return {
|
||||
sequenceNumber: 0,
|
||||
entries: usaCount,
|
||||
verifiedSectors: 0,
|
||||
coveredSectors: 0,
|
||||
mismatchSector: null,
|
||||
malformed: true,
|
||||
};
|
||||
|
||||
const sequenceNumber = view.getUint16(usaOffset, true);
|
||||
if (sequenceNumber === 0 || sequenceNumber === 0xffff)
|
||||
return {
|
||||
sequenceNumber,
|
||||
entries: usaCount,
|
||||
verifiedSectors: 0,
|
||||
coveredSectors,
|
||||
mismatchSector: null,
|
||||
malformed: true,
|
||||
};
|
||||
for (let sector = 0; sector < coveredSectors; sector += 1) {
|
||||
const trailerOffset = sector * FIXUP_SECTOR_BYTES + FIXUP_SECTOR_BYTES - 2;
|
||||
if (view.getUint16(trailerOffset, true) !== sequenceNumber)
|
||||
return {
|
||||
sequenceNumber,
|
||||
entries: usaCount,
|
||||
verifiedSectors: sector,
|
||||
coveredSectors,
|
||||
mismatchSector: sector,
|
||||
malformed: false,
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
sequenceNumber,
|
||||
entries: usaCount,
|
||||
verifiedSectors: coveredSectors,
|
||||
coveredSectors,
|
||||
mismatchSector: null,
|
||||
malformed: false,
|
||||
};
|
||||
};
|
||||
|
||||
/** Buffer the admitted input once while accepting arbitrary chunk boundaries. */
|
||||
class StreamingMftDecoder implements RegionDecoder<MftInventory> {
|
||||
readonly #bytes: Uint8Array;
|
||||
#written = 0;
|
||||
|
||||
constructor(fileBytes: number) {
|
||||
this.#bytes = new Uint8Array(fileBytes);
|
||||
}
|
||||
|
||||
write(chunk: Uint8Array): Result<void, RegionDecodeFailure> {
|
||||
if (chunk.length > this.#bytes.length - this.#written)
|
||||
return err(failure("received more bytes than the admitted file length"));
|
||||
this.#bytes.set(chunk, this.#written);
|
||||
this.#written += chunk.length;
|
||||
return ok(undefined);
|
||||
}
|
||||
|
||||
finish(): Result<MftInventory, RegionDecodeFailure> {
|
||||
if (this.#written !== this.#bytes.length)
|
||||
return err(failure("stream ended before the admitted file length"));
|
||||
return scanMft(this.#bytes);
|
||||
}
|
||||
}
|
||||
|
||||
export const createMftDecoder = (
|
||||
fileBytes: number,
|
||||
): Result<RegionDecoder<MftInventory>, RegionDecodeFailure> =>
|
||||
Number.isSafeInteger(fileBytes) &&
|
||||
fileBytes >= 0 &&
|
||||
fileBytes <= MFT_MAX_FILE_BYTES
|
||||
? ok(new StreamingMftDecoder(fileBytes))
|
||||
: err(
|
||||
limit(
|
||||
`file is ${String(fileBytes)} bytes, exceeding the fixed ${String(MFT_MAX_FILE_BYTES)}-byte (512 MiB) ceiling`,
|
||||
"file-size",
|
||||
),
|
||||
);
|
||||
@@ -0,0 +1,39 @@
|
||||
import { lstat, realpath, stat } from "node:fs/promises";
|
||||
import { basename, dirname, join } from "node:path";
|
||||
import { absoluteCallerPath } from "./callerPath.js";
|
||||
|
||||
/** Resolve existing path components without changing the caller's traversal. */
|
||||
export const canonicalCallerPath = async (path: string): Promise<string> => {
|
||||
const remaining: string[] = [];
|
||||
let current = absoluteCallerPath(path);
|
||||
for (;;) {
|
||||
try {
|
||||
const found = await stat(current);
|
||||
if (remaining.length > 0 && !found.isDirectory())
|
||||
throw Object.assign(
|
||||
new Error(`Path parent is not a directory: ${current}`),
|
||||
{ code: "ENOTDIR" },
|
||||
);
|
||||
return join(await realpath(current), ...remaining);
|
||||
} catch (cause: unknown) {
|
||||
if ((cause as NodeJS.ErrnoException).code !== "ENOENT") throw cause;
|
||||
// A dangling symlink is not an absent destination: its target cannot be
|
||||
// established. A missing component followed by .. is likewise unknown.
|
||||
const existing = await lstat(current).then(
|
||||
() => true,
|
||||
(error: unknown) => {
|
||||
if ((error as NodeJS.ErrnoException).code === "ENOENT") return false;
|
||||
throw error;
|
||||
},
|
||||
);
|
||||
if (
|
||||
existing ||
|
||||
basename(current) === ".." ||
|
||||
dirname(current) === current
|
||||
)
|
||||
throw cause;
|
||||
remaining.unshift(basename(current));
|
||||
current = dirname(current);
|
||||
}
|
||||
}
|
||||
};
|
||||
@@ -1,4 +1,3 @@
|
||||
import { readFile } from "node:fs/promises";
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import type { JsonValue } from "../domain/jsonValue.js";
|
||||
@@ -1120,25 +1119,6 @@ describe("Ghidra native instruction decoder context for ARM TMode and MIPS ISA_M
|
||||
});
|
||||
});
|
||||
|
||||
describe("Ghidra bridge script processor context contract", () => {
|
||||
it("binds TMode to ARM and ISA_MODE to MIPS in ReaGhidraBridge.java", async () => {
|
||||
const bridgePath = new URL(
|
||||
"../../bridge/ghidra/ReaGhidraBridge.java",
|
||||
import.meta.url,
|
||||
);
|
||||
const content = await readFile(bridgePath, "utf8");
|
||||
expect(content).toContain(
|
||||
'currentProgram.getLanguageID().getIdAsString().startsWith("ARM:")',
|
||||
);
|
||||
expect(content).toContain(
|
||||
'currentProgram.getLanguageID().getIdAsString().startsWith("MIPS:")',
|
||||
);
|
||||
expect(content).toContain('"TMode"');
|
||||
expect(content).toContain('"ISA_MODE"');
|
||||
expect(content).toContain('"ghidra-program-context"');
|
||||
});
|
||||
});
|
||||
|
||||
describe("Ghidra dossier nested boundary composition", () => {
|
||||
const identityLocations = [
|
||||
"caller",
|
||||
|
||||
@@ -0,0 +1,53 @@
|
||||
import {
|
||||
ArtifactResourceScope,
|
||||
type ArtifactResourceOwner,
|
||||
} from "../../artifacts/ArtifactResourceScope.js";
|
||||
|
||||
/** Retain a cancelled response branch until its underlying cancellation settles. */
|
||||
export class StateResponseCleanup {
|
||||
readonly #scope = new ArtifactResourceScope();
|
||||
|
||||
constructor(private readonly changed: () => void = () => {}) {}
|
||||
|
||||
get cleanupResources(): readonly string[] {
|
||||
return this.#scope.cleanupResources;
|
||||
}
|
||||
|
||||
cancel(cancel: () => Promise<void>): void {
|
||||
let state:
|
||||
| { status: "pending" }
|
||||
| { status: "done" }
|
||||
| { status: "failed"; cause: unknown } = { status: "pending" };
|
||||
const owner: ArtifactResourceOwner = {
|
||||
kind: "lease",
|
||||
resource: "state response body",
|
||||
lease: {
|
||||
release: async () => {
|
||||
if (state.status === "pending")
|
||||
throw new Error("State response cancellation is still pending");
|
||||
if (state.status === "failed") throw state.cause;
|
||||
},
|
||||
},
|
||||
};
|
||||
this.#scope.retain(owner);
|
||||
this.changed();
|
||||
void Promise.resolve()
|
||||
.then(cancel)
|
||||
.then(
|
||||
async () => {
|
||||
state = { status: "done" };
|
||||
await this.#scope.release(owner);
|
||||
this.changed();
|
||||
},
|
||||
(cause: unknown) => {
|
||||
state = { status: "failed", cause };
|
||||
this.changed();
|
||||
},
|
||||
)
|
||||
.catch(() => undefined);
|
||||
}
|
||||
|
||||
close(): Promise<void> {
|
||||
return this.#scope.close();
|
||||
}
|
||||
}
|
||||
@@ -1,3 +1,4 @@
|
||||
import { StateResponseCleanup } from "./StateResponseCleanup.js";
|
||||
import type { EnvironmentStateStore } from "../../application/environment/environmentStateStore.js";
|
||||
import { ArtifactResourceScope } from "../../artifacts/ArtifactResourceScope.js";
|
||||
import type { AnalysisInputError } from "../../domain/analysisErrorCore.js";
|
||||
@@ -46,17 +47,24 @@ interface CallContext {
|
||||
readonly intake: StateIntake;
|
||||
readonly cancel: AbortSignal;
|
||||
readonly files: ArtifactResourceScope;
|
||||
readonly responses: StateResponseCleanup;
|
||||
}
|
||||
|
||||
/** `data/add`: exactly one part, whose document replaces the whole state only when it is valid and the call was not cancelled. */
|
||||
const addState = async (
|
||||
{ store, intake, cancel, files }: CallContext,
|
||||
{ store, intake, cancel, files, responses }: CallContext,
|
||||
params: unknown,
|
||||
): Promise<RpcOutcome> => {
|
||||
const parts = isRecord(params) ? params.parts : undefined;
|
||||
if (!Array.isArray(parts) || parts.length !== 1)
|
||||
return { error: invalidParams("data/add takes exactly one part") };
|
||||
const document = await partDocument(parts[0], intake, cancel, files);
|
||||
const document = await partDocument(
|
||||
parts[0],
|
||||
intake,
|
||||
cancel,
|
||||
files,
|
||||
responses,
|
||||
);
|
||||
if (!document.ok) return { error: document.error };
|
||||
// A file read is not interruptible, so a cancellation during it is honoured here, before the state changes.
|
||||
if (cancel.aborted) return { error: CANCELLED };
|
||||
@@ -101,6 +109,7 @@ export const stateDataPlane = (
|
||||
intake: StateIntake = DEFAULT_STATE_INTAKE,
|
||||
stopped: AbortSignal = new AbortController().signal,
|
||||
files: ArtifactResourceScope = new ArtifactResourceScope(),
|
||||
responses: StateResponseCleanup = new StateResponseCleanup(),
|
||||
): RpcMethods => {
|
||||
let previous: Promise<unknown> = Promise.resolve();
|
||||
return (method, params, signal) => {
|
||||
@@ -111,7 +120,7 @@ export const stateDataPlane = (
|
||||
const next = previous.then(() =>
|
||||
cancel.aborted
|
||||
? { error: CANCELLED }
|
||||
: run({ store, intake, cancel, files }, params),
|
||||
: run({ store, intake, cancel, files, responses }, params),
|
||||
);
|
||||
previous = next.catch(() => undefined);
|
||||
return next;
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { StateResponseCleanup } from "./StateResponseCleanup.js";
|
||||
import type { HttpHostname } from "../../domain/httpHostname.js";
|
||||
import { STATE_INTAKE_LIMIT_BYTES } from "../../application/environment/environmentStateDocument.js";
|
||||
import {
|
||||
@@ -477,6 +478,8 @@ export const serveEnvironment = async (
|
||||
};
|
||||
// A state file whose close failed stays owned here: later file reads retry it, and close reports it if it is still open.
|
||||
const stateFiles = new ArtifactResourceScope();
|
||||
let responseStateChanged = (): void => undefined;
|
||||
const stateResponses = new StateResponseCleanup(() => responseStateChanged());
|
||||
const served = await serveStreamableHttp({
|
||||
host,
|
||||
port,
|
||||
@@ -499,6 +502,7 @@ export const serveEnvironment = async (
|
||||
},
|
||||
stopping.signal,
|
||||
stateFiles,
|
||||
stateResponses,
|
||||
),
|
||||
),
|
||||
...(logger === undefined ? {} : { logger }),
|
||||
@@ -512,11 +516,14 @@ export const serveEnvironment = async (
|
||||
});
|
||||
let httpClosePending = false;
|
||||
let stateClosePending = false;
|
||||
let responseClosePending = false;
|
||||
let settledSuccessful = false;
|
||||
let closing: Promise<void> | undefined;
|
||||
const shutdownFailures = new Set<AnalysisError>();
|
||||
const owner: EnvironmentShutdownOwner = {
|
||||
get cleanupResources() {
|
||||
const stateResources = stateFiles.cleanupResources;
|
||||
const responseResources = stateResponses.cleanupResources;
|
||||
return [
|
||||
...(httpClosePending ? [served.url] : []),
|
||||
...(stateClosePending
|
||||
@@ -524,6 +531,11 @@ export const serveEnvironment = async (
|
||||
? stateResources
|
||||
: ["environment state-file scope"]
|
||||
: []),
|
||||
...(responseClosePending
|
||||
? responseResources.length > 0
|
||||
? responseResources
|
||||
: ["state response cleanup"]
|
||||
: []),
|
||||
];
|
||||
},
|
||||
close: () => {
|
||||
@@ -531,7 +543,7 @@ export const serveEnvironment = async (
|
||||
const attempt = (async () => {
|
||||
stop();
|
||||
const failures: {
|
||||
readonly stage: "http-server" | "state-files";
|
||||
readonly stage: "http-server" | "state-files" | "state-responses";
|
||||
readonly cause: unknown;
|
||||
}[] = [];
|
||||
try {
|
||||
@@ -548,6 +560,15 @@ export const serveEnvironment = async (
|
||||
stateClosePending = true;
|
||||
failures.push({ stage: "state-files", cause });
|
||||
}
|
||||
try {
|
||||
await stateResponses.close();
|
||||
if (stateResponses.cleanupResources.length > 0)
|
||||
throw new Error("State response cleanup remains pending");
|
||||
responseClosePending = false;
|
||||
} catch (cause: unknown) {
|
||||
responseClosePending = true;
|
||||
failures.push({ stage: "state-responses", cause });
|
||||
}
|
||||
if (failures.length > 0) {
|
||||
retainedEnvironmentShutdowns.add(owner);
|
||||
const failure = new AnalysisOutputError(
|
||||
@@ -583,6 +604,7 @@ export const serveEnvironment = async (
|
||||
retainedEnvironmentShutdowns.delete(owner);
|
||||
for (const failure of shutdownFailures) failure.resolveCleanup();
|
||||
shutdownFailures.clear();
|
||||
settledSuccessful = true;
|
||||
})();
|
||||
closing = attempt;
|
||||
void attempt.catch(() => {
|
||||
@@ -591,6 +613,26 @@ export const serveEnvironment = async (
|
||||
return attempt;
|
||||
},
|
||||
};
|
||||
// A fetcher may hand back a response after an earlier stop completed. Keep
|
||||
// that newly acquired body owned and make a later close observe its outcome.
|
||||
responseStateChanged = () => {
|
||||
if (!stopping.signal.aborted) return;
|
||||
if (stateResponses.cleanupResources.length > 0) {
|
||||
responseClosePending = true;
|
||||
retainedEnvironmentShutdowns.add(owner);
|
||||
if (settledSuccessful) {
|
||||
settledSuccessful = false;
|
||||
closing = undefined;
|
||||
}
|
||||
} else if (
|
||||
shutdownFailures.size === 0 &&
|
||||
!httpClosePending &&
|
||||
!stateClosePending
|
||||
) {
|
||||
responseClosePending = false;
|
||||
retainedEnvironmentShutdowns.delete(owner);
|
||||
}
|
||||
};
|
||||
return ok({
|
||||
...served,
|
||||
card,
|
||||
|
||||
@@ -1037,7 +1037,122 @@ describe("serveEnvironment with a state", () => {
|
||||
expect(missing.isError).toBe(true);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("state response cleanup", () => {
|
||||
it("reports pending teed response cleanup and retries after its sibling closes", async () => {
|
||||
let sourceCancelled = false;
|
||||
const stream = new ReadableStream<Uint8Array>({
|
||||
start(controller) {
|
||||
controller.enqueue(new Uint8Array([1]));
|
||||
},
|
||||
cancel() {
|
||||
sourceCancelled = true;
|
||||
},
|
||||
});
|
||||
const [body, sibling] = stream.tee();
|
||||
const environment = await serveState(
|
||||
{ source, state },
|
||||
{
|
||||
fetchState: async () =>
|
||||
new Response(body, {
|
||||
headers: { "content-length": String(STATE_INTAKE_LIMIT_BYTES + 1) },
|
||||
}),
|
||||
},
|
||||
);
|
||||
let timer: ReturnType<typeof setTimeout> | undefined;
|
||||
try {
|
||||
const response = await fetch(
|
||||
new URL(ENVIRONMENT_RPC_PATH, environment.url),
|
||||
{
|
||||
method: "POST",
|
||||
headers: { "content-type": "application/json" },
|
||||
body: JSON.stringify(addByUri(1)),
|
||||
signal: AbortSignal.timeout(10_000),
|
||||
},
|
||||
);
|
||||
await expect(response.json()).resolves.toMatchObject({
|
||||
error: { code: -32602 },
|
||||
});
|
||||
const closing = Promise.race([
|
||||
environment.close(),
|
||||
new Promise<never>((_, reject) => {
|
||||
timer = setTimeout(
|
||||
() => reject(new Error("server close waited for tee sibling")),
|
||||
10_000,
|
||||
);
|
||||
}),
|
||||
]);
|
||||
await expect(closing).rejects.toMatchObject({
|
||||
_tag: "AnalysisOutputError",
|
||||
cleanupIncomplete: true,
|
||||
diagnostics: { cleanup_stages: ["state-responses"] },
|
||||
});
|
||||
expect(sourceCancelled).toBe(false);
|
||||
await sibling.cancel();
|
||||
expect(sourceCancelled).toBe(true);
|
||||
await expect(environment.close()).resolves.toBeUndefined();
|
||||
} finally {
|
||||
if (timer !== undefined) clearTimeout(timer);
|
||||
await Promise.allSettled([body.cancel(), sibling.cancel()]);
|
||||
await environment.close();
|
||||
}
|
||||
});
|
||||
|
||||
it("retains a late response cleanup after an earlier successful stop", async () => {
|
||||
let giveResponse!: (response: Response) => void;
|
||||
const gate = new Promise<Response>((resolve) => {
|
||||
giveResponse = resolve;
|
||||
});
|
||||
const held = heldFetcher(() => gate);
|
||||
let release!: () => void;
|
||||
let acknowledge!: () => void;
|
||||
const cleanup = new Promise<void>((resolve) => {
|
||||
release = resolve;
|
||||
});
|
||||
const entered = new Promise<void>((resolve) => {
|
||||
acknowledge = resolve;
|
||||
});
|
||||
const environment = await serveState(
|
||||
{ source, state },
|
||||
{ fetchState: held.fetchState },
|
||||
);
|
||||
const answered = fetch(new URL(ENVIRONMENT_RPC_PATH, environment.url), {
|
||||
method: "POST",
|
||||
headers: { "content-type": "application/json" },
|
||||
body: JSON.stringify(addByUri(1)),
|
||||
signal: AbortSignal.timeout(10_000),
|
||||
}).catch(() => undefined);
|
||||
try {
|
||||
await held.fetching;
|
||||
await environment.close();
|
||||
giveResponse(
|
||||
new Response(
|
||||
new ReadableStream<Uint8Array>({
|
||||
cancel() {
|
||||
acknowledge();
|
||||
return cleanup;
|
||||
},
|
||||
}),
|
||||
),
|
||||
);
|
||||
await entered;
|
||||
await expect(environment.close()).rejects.toMatchObject({
|
||||
_tag: "AnalysisOutputError",
|
||||
cleanupIncomplete: true,
|
||||
diagnostics: { cleanup_stages: ["state-responses"] },
|
||||
});
|
||||
release();
|
||||
await expect(environment.close()).resolves.toBeUndefined();
|
||||
} finally {
|
||||
release();
|
||||
await answered;
|
||||
await environment.close();
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe("serveEnvironment state file cleanup", () => {
|
||||
it("stops without waiting for a stalled download", async () => {
|
||||
const held = heldFetcher(() => new Promise<Response>(() => undefined));
|
||||
const environment = await serveState(
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { StateResponseCleanup } from "./StateResponseCleanup.js";
|
||||
import { expect, it, vi } from "vitest";
|
||||
|
||||
import { ArtifactResourceScope } from "../../artifacts/ArtifactResourceScope.js";
|
||||
@@ -11,14 +12,17 @@ import {
|
||||
|
||||
const inlineDocument = async (bytes: string) => {
|
||||
const files = new ArtifactResourceScope();
|
||||
const responses = new StateResponseCleanup();
|
||||
try {
|
||||
return await partDocument(
|
||||
{ kind: "file", file: { bytes } },
|
||||
DEFAULT_STATE_INTAKE,
|
||||
new AbortController().signal,
|
||||
files,
|
||||
responses,
|
||||
);
|
||||
} finally {
|
||||
await responses.close();
|
||||
await files.close();
|
||||
}
|
||||
};
|
||||
@@ -79,3 +83,109 @@ it("downloads with redirects refused and the 30 s timeout by default", async ()
|
||||
fetched.mockRestore();
|
||||
}
|
||||
});
|
||||
|
||||
it.each(["http-status", "declared-overflow", "stream-overflow"] as const)(
|
||||
"returns %s while a teed response's other branch remains open",
|
||||
async (scenario) => {
|
||||
const source = new ReadableStream<Uint8Array>({
|
||||
start(controller) {
|
||||
controller.enqueue(
|
||||
new Uint8Array(
|
||||
scenario === "stream-overflow" ? STATE_INTAKE_LIMIT_BYTES + 1 : 1,
|
||||
),
|
||||
);
|
||||
},
|
||||
});
|
||||
const [body, sibling] = source.tee();
|
||||
const response = new Response(body, {
|
||||
status: scenario === "http-status" ? 503 : 200,
|
||||
...(scenario === "declared-overflow"
|
||||
? {
|
||||
headers: { "content-length": String(STATE_INTAKE_LIMIT_BYTES + 1) },
|
||||
}
|
||||
: {}),
|
||||
});
|
||||
const files = new ArtifactResourceScope();
|
||||
const responses = new StateResponseCleanup();
|
||||
const operation = partDocument(
|
||||
{ kind: "file", file: { uri: "https://state.example.test/state.json" } },
|
||||
{ fetchState: async () => response, timeoutMs: 1_000 },
|
||||
new AbortController().signal,
|
||||
files,
|
||||
responses,
|
||||
);
|
||||
let timer: ReturnType<typeof setTimeout> | undefined;
|
||||
try {
|
||||
const outcome = await Promise.race([
|
||||
operation.then((result) => ({ kind: "result" as const, result })),
|
||||
new Promise<{ kind: "deadline" }>((resolve) => {
|
||||
timer = setTimeout(() => resolve({ kind: "deadline" }), 3_000);
|
||||
}),
|
||||
]);
|
||||
expect(outcome.kind).toBe("result");
|
||||
expect(responses.cleanupResources).toEqual(["state response body"]);
|
||||
await expect(responses.close()).rejects.toMatchObject({
|
||||
reason: "unavailable",
|
||||
});
|
||||
if (outcome.kind === "result") {
|
||||
expect(outcome.result).toMatchObject(
|
||||
scenario === "http-status"
|
||||
? { ok: false, error: { data: { reason: "http-status" } } }
|
||||
: {
|
||||
ok: false,
|
||||
error: {
|
||||
message: expect.stringContaining(
|
||||
String(STATE_INTAKE_LIMIT_BYTES),
|
||||
),
|
||||
},
|
||||
},
|
||||
);
|
||||
}
|
||||
} finally {
|
||||
if (timer !== undefined) clearTimeout(timer);
|
||||
await sibling.cancel();
|
||||
await operation;
|
||||
await responses.close();
|
||||
await files.close();
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
it("retains the exact failure when response cancellation rejects", async () => {
|
||||
const failure = new Error("controlled cancellation failure");
|
||||
const body = new ReadableStream<Uint8Array>({
|
||||
cancel() {
|
||||
throw failure;
|
||||
},
|
||||
});
|
||||
const files = new ArtifactResourceScope();
|
||||
const responses = new StateResponseCleanup();
|
||||
const result = await partDocument(
|
||||
{ kind: "file", file: { uri: "https://state.example.test/state.json" } },
|
||||
{
|
||||
fetchState: async () => new Response(body, { status: 503 }),
|
||||
timeoutMs: 1_000,
|
||||
},
|
||||
new AbortController().signal,
|
||||
files,
|
||||
responses,
|
||||
);
|
||||
expect(result).toMatchObject({
|
||||
ok: false,
|
||||
error: { data: { reason: "http-status" } },
|
||||
});
|
||||
await new Promise<void>((resolve) => setImmediate(resolve));
|
||||
const rejected: unknown = await responses
|
||||
.close()
|
||||
.catch((cause: unknown) => cause);
|
||||
expect(rejected).toMatchObject({
|
||||
reason: "unavailable",
|
||||
cleanup: { resources: ["state response body"] },
|
||||
});
|
||||
expect((rejected as Error).cause).toBeInstanceOf(AggregateError);
|
||||
expect(((rejected as Error).cause as AggregateError).errors).toContain(
|
||||
failure,
|
||||
);
|
||||
expect(responses.cleanupResources).toEqual(["state response body"]);
|
||||
await files.close();
|
||||
});
|
||||
@@ -1,3 +1,4 @@
|
||||
import { StateResponseCleanup } from "./StateResponseCleanup.js";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
import { mediaTypeEssence } from "../../domain/analysisView/recordedMediaType.js";
|
||||
@@ -228,6 +229,7 @@ const downloadBounds = (
|
||||
const readCapped = async (
|
||||
body: ReadableStream<Uint8Array> | null,
|
||||
bounds: DownloadBounds,
|
||||
responses: StateResponseCleanup,
|
||||
): Promise<Result<Buffer, RpcError>> => {
|
||||
if (body === null) return ok(Buffer.alloc(0));
|
||||
const reader = body.getReader();
|
||||
@@ -240,13 +242,15 @@ const readCapped = async (
|
||||
if (done) return ok(Buffer.concat(chunks, total));
|
||||
total += value.byteLength;
|
||||
if (total > STATE_INTAKE_LIMIT_BYTES) {
|
||||
await reader.cancel().catch(() => undefined);
|
||||
// A tee's sibling can be caller-owned; cancelling this branch must not
|
||||
// delay the refusal until that sibling finishes.
|
||||
responses.cancel(() => reader.cancel());
|
||||
return err(TOO_LARGE);
|
||||
}
|
||||
chunks.push(value);
|
||||
}
|
||||
} catch {
|
||||
void reader.cancel().catch(() => undefined);
|
||||
responses.cancel(() => reader.cancel());
|
||||
return err(fetchFailed(bounds.failure()));
|
||||
}
|
||||
};
|
||||
@@ -256,6 +260,7 @@ const downloadState = async (
|
||||
url: URL,
|
||||
intake: StateIntake,
|
||||
cancel: AbortSignal,
|
||||
responses: StateResponseCleanup,
|
||||
): Promise<Result<Buffer, RpcError>> => {
|
||||
const bounds = downloadBounds(intake.timeoutMs, cancel);
|
||||
const fetching = Promise.resolve().then(() =>
|
||||
@@ -268,7 +273,9 @@ const downloadState = async (
|
||||
// A response that arrives after the download ended is released unread.
|
||||
void fetching
|
||||
.then(
|
||||
(late) => late.body?.cancel(),
|
||||
(late) => {
|
||||
if (late.body !== null) responses.cancel(() => late.body!.cancel());
|
||||
},
|
||||
() => undefined,
|
||||
)
|
||||
.catch(() => undefined);
|
||||
@@ -276,7 +283,8 @@ const downloadState = async (
|
||||
}
|
||||
const declared = Number(response.headers.get("content-length") ?? "0");
|
||||
if (!response.ok || declared > STATE_INTAKE_LIMIT_BYTES) {
|
||||
await response.body?.cancel().catch(() => undefined);
|
||||
// Cancel this branch without waiting for a potentially independent tee sibling.
|
||||
if (response.body !== null) responses.cancel(() => response.body!.cancel());
|
||||
return err(
|
||||
response.ok
|
||||
? TOO_LARGE
|
||||
@@ -286,7 +294,7 @@ const downloadState = async (
|
||||
},
|
||||
);
|
||||
}
|
||||
return readCapped(response.body, bounds);
|
||||
return readCapped(response.body, bounds, responses);
|
||||
};
|
||||
|
||||
const NETWORK_SHARE = invalidParams(
|
||||
@@ -315,6 +323,7 @@ const fileBytes = async (
|
||||
intake: StateIntake,
|
||||
cancel: AbortSignal,
|
||||
files: ArtifactResourceScope,
|
||||
responses: StateResponseCleanup,
|
||||
): Promise<Result<Uint8Array, RpcError>> => {
|
||||
if (typeof file.bytes === "string" && file.uri === undefined) {
|
||||
if (file.bytes.length > MAX_INLINE_BASE64_CHARACTERS) return err(TOO_LARGE);
|
||||
@@ -333,7 +342,8 @@ const fileBytes = async (
|
||||
invalidParams("a file part carries exactly one of bytes or a uri"),
|
||||
);
|
||||
const url = new URL(file.uri);
|
||||
if (url.protocol === "https:") return downloadState(url, intake, cancel);
|
||||
if (url.protocol === "https:")
|
||||
return downloadState(url, intake, cancel, responses);
|
||||
if (url.protocol !== "file:")
|
||||
return err(invalidParams("a file uri is a file:// or https:// URL"));
|
||||
const path = localFilePath(url);
|
||||
@@ -346,6 +356,7 @@ export const partDocument = async (
|
||||
intake: StateIntake,
|
||||
cancel: AbortSignal,
|
||||
files: ArtifactResourceScope,
|
||||
responses: StateResponseCleanup,
|
||||
): Promise<Result<unknown, RpcError>> => {
|
||||
if (isRecord(part) && part.kind === "data" && isRecord(part.data))
|
||||
return ok(part.data);
|
||||
@@ -362,6 +373,6 @@ export const partDocument = async (
|
||||
mediaTypeEssence(mimeType) !== "application/json")
|
||||
)
|
||||
return err(invalidParams("a file part's mimeType is application/json"));
|
||||
const bytes = await fileBytes(part.file, intake, cancel, files);
|
||||
const bytes = await fileBytes(part.file, intake, cancel, files, responses);
|
||||
return bytes.ok ? parseDocumentBytes(bytes.value) : bytes;
|
||||
};
|
||||
@@ -15,6 +15,13 @@ const cases = [
|
||||
0,
|
||||
],
|
||||
['spawn("control", [], {stdio:"inherit"})', "control", "inherit", 0],
|
||||
['spawn("global-nan-env", [], {env:NaN})', "global-nan-env", "default", 0],
|
||||
[
|
||||
'((NaN) => spawn("shadowed-nan-env", [], {env:NaN}))({})',
|
||||
"shadowed-nan-env",
|
||||
"default",
|
||||
0,
|
||||
],
|
||||
[
|
||||
'spawn("spread-after", [], {stdio:"pipe",...input})',
|
||||
"spread-after",
|
||||
@@ -233,6 +240,12 @@ const exportAssignmentSource = `
|
||||
export = argv;
|
||||
`;
|
||||
|
||||
const environmentCommands = [
|
||||
"named-argv",
|
||||
"execfile-dynamic",
|
||||
"shadowed-nan-env",
|
||||
];
|
||||
|
||||
// Callback/options ambiguity cannot change exec's effective stdio.
|
||||
const opaqueExecCallbackCommands = [
|
||||
"eval-callback",
|
||||
@@ -342,7 +355,7 @@ cliTest.for(["CLI", "stdio MCP"] as const)(
|
||||
expect(child.properties.stdio_mode, String(command)).toBe(mode);
|
||||
expect(child.properties.argv_count, String(command)).toBe(count);
|
||||
expect(child.properties.environment_supplied, String(command)).toBe(
|
||||
["named-argv", "execfile-dynamic"].includes(String(command)),
|
||||
environmentCommands.includes(String(command)),
|
||||
);
|
||||
expect(child.identity.source_range).not.toBeNull();
|
||||
const stdio = graph.relations.filter(
|
||||
@@ -377,9 +390,7 @@ cliTest.for(["CLI", "stdio MCP"] as const)(
|
||||
relation.relation === "supplies-env" &&
|
||||
relation.target_node_id === child.node_id,
|
||||
),
|
||||
).toHaveLength(
|
||||
["named-argv", "execfile-dynamic"].includes(String(command)) ? 1 : 0,
|
||||
);
|
||||
).toHaveLength(environmentCommands.includes(String(command)) ? 1 : 0);
|
||||
expect(
|
||||
graph.relations.filter(
|
||||
(relation) =>
|
||||
|
||||
@@ -1,81 +1,72 @@
|
||||
import { execFile } from "node:child_process";
|
||||
import { readFile, readdir, writeFile } from "node:fs/promises";
|
||||
import { readFile, readdir } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
import { promisify } from "node:util";
|
||||
import { expect, it } from "vitest";
|
||||
import { expect } from "vitest";
|
||||
import { parse } from "yaml";
|
||||
import { resolveSwiftcCompileTimeoutMs } from "../../../src/process/DarwinProcessRunTokenReader.js";
|
||||
import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js";
|
||||
import { processTest } from "../../support/process/processFixture.js";
|
||||
|
||||
const execute = promisify(execFile);
|
||||
const hostLabels: Partial<Record<NodeJS.Platform, string>> = {
|
||||
linux: "linux",
|
||||
darwin: "macOS",
|
||||
win32: "Windows",
|
||||
};
|
||||
const hostLabel = hostLabels[process.platform];
|
||||
const preparationBudget =
|
||||
process.platform === "darwin" ? resolveSwiftcCompileTimeoutMs() : 0;
|
||||
|
||||
it("enforces workflow platform labels before contacting Cutter", async () => {
|
||||
const workflow = parse(
|
||||
await readFile(".github/workflows/real-cutter.yml", "utf8"),
|
||||
);
|
||||
const labels = workflow.jobs["cutter-upstream"].strategy.matrix.include.map(
|
||||
(entry: { platform: string }) => entry.platform,
|
||||
);
|
||||
expect(labels).toEqual(["linux", "macOS", "Windows"]);
|
||||
const root = await createTestTempDirectory("rea-cutter-platform-");
|
||||
const preload = join(root, "preload.mjs");
|
||||
await writeFile(
|
||||
preload,
|
||||
`
|
||||
import childProcess from 'node:child_process';
|
||||
import { syncBuiltinESMExports } from 'node:module';
|
||||
Object.defineProperty(process, 'platform', { value: process.env.REA_PROBE_PLATFORM });
|
||||
childProcess.execFile = () => { throw new Error('fixture discovery reached'); };
|
||||
syncBuiltinESMExports();
|
||||
`,
|
||||
);
|
||||
const check = async (selected: string | undefined, actual: string) => {
|
||||
const env: NodeJS.ProcessEnv = {
|
||||
...process.env,
|
||||
TMPDIR: root,
|
||||
TMP: root,
|
||||
TEMP: root,
|
||||
REA_PROBE_PLATFORM: actual,
|
||||
processTest.skipIf(hostLabel === undefined)(
|
||||
"enforces workflow platform labels before contacting Cutter",
|
||||
{ timeout: Math.min(2_147_483_647, 2 * preparationBudget + 30_000) },
|
||||
async ({ processes }) => {
|
||||
const workflow = parse(
|
||||
await readFile(".github/workflows/real-cutter.yml", "utf8"),
|
||||
);
|
||||
const labels = workflow.jobs["cutter-upstream"].strategy.matrix.include.map(
|
||||
(entry: { platform: string }) => entry.platform,
|
||||
);
|
||||
expect(labels).toEqual(["linux", "macOS", "Windows"]);
|
||||
const root = await createTestTempDirectory("rea-cutter-platform-");
|
||||
// A private absent directory proves actual discovery without contacting a user's bridge.
|
||||
const bridgeDirectory = join(root, "absent-bridge");
|
||||
const check = async (selected: string | undefined) => {
|
||||
const env: NodeJS.ProcessEnv = {
|
||||
...process.env,
|
||||
TMPDIR: root,
|
||||
TMP: root,
|
||||
TEMP: root,
|
||||
REA_CUTTER_BRIDGE_DIR: bridgeDirectory,
|
||||
};
|
||||
if (selected === undefined) delete env.EXPECTED_PLATFORM;
|
||||
else env.EXPECTED_PLATFORM = selected;
|
||||
const result = await processes.run(
|
||||
process.execPath,
|
||||
["scripts/verify-real-cutter.mjs"],
|
||||
{
|
||||
env,
|
||||
timeoutMs: Math.min(2_147_483_647, preparationBudget + 15_000),
|
||||
},
|
||||
);
|
||||
expect(result.exitCode, result.stderr).toBe(1);
|
||||
return result.stderr;
|
||||
};
|
||||
if (selected === undefined) delete env.EXPECTED_PLATFORM;
|
||||
else env.EXPECTED_PLATFORM = selected;
|
||||
const result: unknown = await execute(
|
||||
process.execPath,
|
||||
["--import", preload, "scripts/verify-real-cutter.mjs"],
|
||||
{ env, timeout: 10_000 },
|
||||
).then(
|
||||
() => {
|
||||
throw new Error("Fixture discovery should stop the verifier");
|
||||
},
|
||||
(error: unknown) => error,
|
||||
for (const selected of [hostLabel, undefined]) {
|
||||
const stderr = await check(selected);
|
||||
expect(stderr).toContain("Open upstream Cutter with the REA plugin");
|
||||
expect(stderr).toContain("no_live_bridge_found");
|
||||
}
|
||||
const other = labels.find((label: string) => label !== hostLabel);
|
||||
expect(other).toBeDefined();
|
||||
expect(await check(other)).toContain(
|
||||
"Cutter runner did not match its declared platform",
|
||||
);
|
||||
if (
|
||||
!(result instanceof Error) ||
|
||||
!("stderr" in result) ||
|
||||
typeof result.stderr !== "string"
|
||||
)
|
||||
throw new Error("Missing verifier diagnostics");
|
||||
return result.stderr;
|
||||
};
|
||||
for (const [selected, actual] of [
|
||||
["linux", "linux"],
|
||||
["macOS", "darwin"],
|
||||
["Windows", "win32"],
|
||||
] as const)
|
||||
expect(await check(selected, actual)).toContain(
|
||||
"fixture discovery reached",
|
||||
expect(await check("unknown")).toContain(
|
||||
"Unrecognized Cutter runner platform",
|
||||
);
|
||||
expect(await check("linux", "darwin")).toContain(
|
||||
"Cutter runner did not match its declared platform",
|
||||
);
|
||||
expect(await check("unknown", "linux")).toContain(
|
||||
"Unrecognized Cutter runner platform",
|
||||
);
|
||||
expect(await check(undefined, "linux")).toContain(
|
||||
"fixture discovery reached",
|
||||
);
|
||||
expect(
|
||||
(await readdir(root)).filter((name) =>
|
||||
name.startsWith("rea-cutter-smoke-"),
|
||||
),
|
||||
).toEqual([]);
|
||||
});
|
||||
expect(
|
||||
(await readdir(root)).filter((name) =>
|
||||
name.startsWith("rea-cutter-smoke-"),
|
||||
),
|
||||
).toEqual([]);
|
||||
},
|
||||
);
|
||||
@@ -783,7 +783,7 @@ it.each([
|
||||
{ code: false, ghidra_bridge: true, runtime: false, package: false },
|
||||
],
|
||||
[
|
||||
"tests/conformance/ghidra/ReaArmContextProbe.java",
|
||||
"tests/conformance/ghidra/ReaInstructionContextProbe.java",
|
||||
{ ghidra_bridge: true, runtime: false, package: false },
|
||||
],
|
||||
[
|
||||
|
||||
Loaded 100 of 107 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user