mirror of
https://github.com/morluto/rea.git
synced 2026-11-10 17:30:32 -10:00
Compare commits
4
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
452ed97031 | ||
|
|
2885347cde | ||
|
|
c837575512 | ||
|
|
cdb51058f6 |
No files matched your search
@@ -32,11 +32,9 @@ its own.
|
||||
`limit` failure with its boundary.
|
||||
- Return the first 32 decoded record headers with their source offsets and the
|
||||
number of omitted headers. Aggregate counts cover the full parsed stream.
|
||||
- Restore verified sector trailers in a per-record scratch buffer before
|
||||
scanning attributes. The domain decoder retains resident `$FILE_NAME`
|
||||
names, namespaces, parent references and source offsets. Report incomplete
|
||||
coverage when the walk cannot finish or an attribute list may refer to
|
||||
extensions. The inventory does not yet expose the individual names.
|
||||
- Parse only the records' headers and fixups. Attributes, filenames, data
|
||||
streams and the `$UsnJrnl:$J` journal are not decoded in this step, so no path,
|
||||
file size, file reference correlation or change history is claimed.
|
||||
|
||||
## Acceptance
|
||||
|
||||
@@ -49,20 +47,5 @@ Inventory tests cover the root occurrence's metadata and the coverage status
|
||||
when a fixup fails, and classification tests cover several file names for the
|
||||
same bytes.
|
||||
|
||||
The paired acquisition workflow is available through
|
||||
`rea inspect-ntfs-journal ./mft ./journal --maxRows 50 --json` and MCP
|
||||
`inspect_ntfs_journal` with `mft_path` and `journal_path`. Select an acquired
|
||||
logical MFT stream and the journal data stream from the same volume. Each
|
||||
file has a 64 MiB working-set admission limit. Follow `nextFileOffset` as
|
||||
`start_file_offset` on MCP or `--startFileOffset` on CLI and compare the two
|
||||
input digests between pages.
|
||||
|
||||
USN V2 results preserve the full record reference, sequence number,
|
||||
source offset, FILETIME, name and raw flags. Older and newer record layouts
|
||||
are reported as unsupported. Padding, malformed records and truncation
|
||||
appear as coverage rows. Record order follows the acquisition, and the
|
||||
results do not establish which journal history is absent due to wrap or a
|
||||
partial acquisition.
|
||||
|
||||
The CLI and MCP workflow tests use synthesized acquisitions. Verification
|
||||
with real `$MFT` and journal files from a Windows host remains pending.
|
||||
Not covered in this step: real `$MFT` files written by a Windows host, USN
|
||||
journal records, attribute decoding, and any correlation between the two.
|
||||
@@ -6,8 +6,6 @@
|
||||
- [x] Add the reader, format descriptor and byte classification.
|
||||
- [ ] Decode attribute records, then `$FILE_NAME` attributes with parent
|
||||
references, names, hard links and timestamps.
|
||||
- [x] Parse `$UsnJrnl:$J` V2 records and correlate them by full file reference.
|
||||
- [x] Return a bounded source-order page through CLI and MCP, preserving both
|
||||
input digests and explicit coverage rows.
|
||||
- [ ] Decode supported non-V2 journal layouts and surface acquisition gaps.
|
||||
- [ ] Parse `$UsnJrnl:$J` records and correlate them to records by full file
|
||||
reference.
|
||||
- [ ] Verify against real `$MFT` and journal captures from a Windows host.
|
||||
@@ -1,152 +0,0 @@
|
||||
import { createHash } from "node:crypto";
|
||||
import { lstat, open, realpath, writeFile } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
|
||||
import { expect, it, vi } from "vitest";
|
||||
|
||||
import { ArtifactResourceScope } from "../../artifacts/ArtifactResourceScope.js";
|
||||
import * as stableArtifact from "../../artifacts/readStableArtifact.js";
|
||||
import { projectAnalysisError } from "../../domain/analysisErrorProjection.js";
|
||||
import { inspectNtfsJournalEvidence } from "./NtfsJournalService.js";
|
||||
import { parseEvidence } from "../../domain/evidence.js";
|
||||
import { createTestTempDirectory } from "../../../tests/fixtures/temporaryDirectory.js";
|
||||
|
||||
it.each([
|
||||
["EACCES", "access_denied"],
|
||||
["EIO", "artifact_operation_failed"],
|
||||
])(
|
||||
"preserves the acquisition failure category for %s",
|
||||
async (systemCode, expectedCode) => {
|
||||
const resources = new ArtifactResourceScope();
|
||||
const spy = vi
|
||||
.spyOn(stableArtifact, "readStableArtifact")
|
||||
.mockRejectedValueOnce(
|
||||
Object.assign(new Error("acquisition read failed"), {
|
||||
code: systemCode,
|
||||
}),
|
||||
);
|
||||
try {
|
||||
const result = await inspectNtfsJournalEvidence(
|
||||
{
|
||||
mft_path: "/fixture/mft",
|
||||
journal_path: "/fixture/journal",
|
||||
},
|
||||
resources,
|
||||
);
|
||||
expect(result.ok).toBe(false);
|
||||
if (result.ok) throw new Error("expected acquisition failure");
|
||||
expect(projectAnalysisError(result.error).code).toBe(expectedCode);
|
||||
} finally {
|
||||
spy.mockRestore();
|
||||
await resources.close();
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
it("preserves the completed file identity when MFT interpretation fails", async () => {
|
||||
const resources = new ArtifactResourceScope();
|
||||
const bytes = Buffer.from("not an MFT");
|
||||
const sha256 = "a".repeat(64);
|
||||
const spy = vi
|
||||
.spyOn(stableArtifact, "readStableArtifact")
|
||||
.mockResolvedValueOnce({ bytes, sha256 });
|
||||
try {
|
||||
const result = await inspectNtfsJournalEvidence(
|
||||
{
|
||||
mft_path: "/fixture/mft",
|
||||
journal_path: "/fixture/journal",
|
||||
},
|
||||
resources,
|
||||
);
|
||||
if (result.ok) throw new Error("expected format failure");
|
||||
expect(result.error.partialObservation).toEqual({
|
||||
kind: "file-snapshot-hash",
|
||||
path: "/fixture/mft",
|
||||
sha256,
|
||||
bytes: bytes.length,
|
||||
});
|
||||
expect(spy).toHaveBeenCalledTimes(1);
|
||||
} finally {
|
||||
spy.mockRestore();
|
||||
await resources.close();
|
||||
}
|
||||
});
|
||||
|
||||
it("retains both acquisitions through a failed journal close and cleanup retry", async () => {
|
||||
const directory = await createTestTempDirectory("rea-ntfs-partial-");
|
||||
const mftPath = join(directory, "mft");
|
||||
const journalPath = join(directory, "journal");
|
||||
const mft = Buffer.alloc(1024);
|
||||
mft.write("FILE");
|
||||
mft.writeUInt16LE(48, 4);
|
||||
mft.writeUInt16LE(3, 6);
|
||||
mft.writeUInt16LE(56, 20);
|
||||
mft.writeUInt32LE(64, 24);
|
||||
mft.writeUInt32LE(1024, 28);
|
||||
mft.writeUInt32LE(0xffffffff, 56);
|
||||
for (const offset of [48, 510, 1022]) mft.writeUInt16LE(1, offset);
|
||||
const journal = Buffer.from("acquired journal bytes");
|
||||
await writeFile(mftPath, mft);
|
||||
await writeFile(journalPath, journal);
|
||||
const resources = new ArtifactResourceScope();
|
||||
const read = stableArtifact.readStableArtifact;
|
||||
let restoreClose: (() => void) | undefined;
|
||||
const spy = vi
|
||||
.spyOn(stableArtifact, "readStableArtifact")
|
||||
.mockImplementation((path, maximumBytes, signal, fileSystem, scope) =>
|
||||
read(
|
||||
path,
|
||||
maximumBytes,
|
||||
signal,
|
||||
path === journalPath
|
||||
? {
|
||||
lstat: (selected) => lstat(selected, { bigint: true }),
|
||||
realpath,
|
||||
open: async (selected, flags) => {
|
||||
const handle = await open(selected, flags);
|
||||
const close = vi
|
||||
.spyOn(handle, "close")
|
||||
.mockRejectedValueOnce(new Error("close failed"));
|
||||
restoreClose = () => close.mockRestore();
|
||||
return handle;
|
||||
},
|
||||
}
|
||||
: fileSystem,
|
||||
scope,
|
||||
),
|
||||
);
|
||||
try {
|
||||
const result = await inspectNtfsJournalEvidence(
|
||||
{
|
||||
mft_path: mftPath,
|
||||
journal_path: journalPath,
|
||||
},
|
||||
resources,
|
||||
);
|
||||
if (result.ok) throw new Error("expected journal cleanup failure");
|
||||
expect(result.error.cleanupIncomplete).toBe(true);
|
||||
const partial = parseEvidence(result.error.partialObservation);
|
||||
expect(partial.normalized_result).toMatchObject({
|
||||
kind: "ntfs-journal-partial",
|
||||
mft: {
|
||||
sha256: createHash("sha256").update(mft).digest("hex"),
|
||||
metadata: { record_count: 1 },
|
||||
},
|
||||
journal_observation: {
|
||||
kind: "file-snapshot-hash",
|
||||
path: journalPath,
|
||||
sha256: createHash("sha256").update(journal).digest("hex"),
|
||||
bytes: journal.length,
|
||||
},
|
||||
});
|
||||
await resources.close();
|
||||
expect(result.error.cleanupIncomplete).toBe(false);
|
||||
expect(
|
||||
projectAnalysisError(result.error).details?.partial_observation,
|
||||
).toEqual(partial);
|
||||
} finally {
|
||||
spy.mockRestore();
|
||||
await resources.close();
|
||||
restoreClose?.();
|
||||
}
|
||||
});
|
||||
@@ -1,259 +0,0 @@
|
||||
import { ArtifactReaderFailure } from "../../artifacts/ArtifactReader.js";
|
||||
import type { ArtifactResourceScope } from "../../artifacts/ArtifactResourceScope.js";
|
||||
import {
|
||||
artifactFileCancellationCause,
|
||||
artifactFilePrimaryFacts,
|
||||
} from "../../artifacts/ArtifactFileOperation.js";
|
||||
import { readStableArtifact } from "../../artifacts/readStableArtifact.js";
|
||||
import {
|
||||
projectStableArtifactFailure,
|
||||
stableArtifactInputRecovery,
|
||||
} from "../../artifacts/StableArtifactFailure.js";
|
||||
import {
|
||||
primaryAnalysisError,
|
||||
type AnalysisError,
|
||||
} from "../../domain/analysisErrorBase.js";
|
||||
import {
|
||||
AnalysisAccessDeniedError,
|
||||
AnalysisArtifactChangedError,
|
||||
AnalysisCancelledError,
|
||||
AnalysisInputError,
|
||||
AnalysisResourceConstraintError,
|
||||
} from "../../domain/analysisErrorCore.js";
|
||||
import { ArtifactOperationError } from "../../domain/artifactOperationError.js";
|
||||
import { isSelectedRegularFileError } from "../../filesystem/RegularFile.js";
|
||||
import { createEvidence, type Evidence } from "../../domain/evidence.js";
|
||||
import { err, ok, type Result } from "../../domain/result.js";
|
||||
import {
|
||||
NTFS_JOURNAL_FILE_BYTES,
|
||||
NTFS_JOURNAL_OPERATION as operation,
|
||||
NTFS_JOURNAL_PROVIDER,
|
||||
ntfsJournalInputSchema,
|
||||
ntfsJournalResultSchema,
|
||||
} from "../../domain/windows/ntfsJournalQuery.js";
|
||||
import { composeNtfsJournalTimeline } from "../../domain/windows/ntfsJournalTimeline.js";
|
||||
import { parseMftImage } from "../../domain/windows/ntfsMft.js";
|
||||
import { parseAnalysisInput } from "../AnalysisInput.js";
|
||||
import { completeProviderEvidence } from "../ProviderOperationExecution.js";
|
||||
|
||||
type PathParameter = "mft_path" | "journal_path";
|
||||
const inputFailure = (
|
||||
parameter: PathParameter,
|
||||
message: string,
|
||||
cause?: unknown,
|
||||
) =>
|
||||
new AnalysisInputError(
|
||||
operation,
|
||||
{
|
||||
cause,
|
||||
...(cause instanceof ArtifactReaderFailure
|
||||
? stableArtifactInputRecovery(cause)
|
||||
: {}),
|
||||
},
|
||||
[{ path: [parameter], reason: "invalid_format", message }],
|
||||
);
|
||||
|
||||
const readFailure = (
|
||||
failure: unknown,
|
||||
path: string,
|
||||
parameter: PathParameter,
|
||||
): AnalysisError => {
|
||||
const { cause } = artifactFilePrimaryFacts(failure);
|
||||
if (cause instanceof ArtifactReaderFailure) {
|
||||
if (cause.reason === "cancelled")
|
||||
return new AnalysisCancelledError(operation, {
|
||||
cause: artifactFileCancellationCause(cause),
|
||||
});
|
||||
if (cause.reason === "integrity")
|
||||
return new AnalysisArtifactChangedError(operation, path, cause.message, {
|
||||
cause,
|
||||
});
|
||||
if (cause.reason === "limit")
|
||||
return new AnalysisResourceConstraintError(
|
||||
operation,
|
||||
"file-size",
|
||||
cause.message,
|
||||
{
|
||||
maximum_file_bytes: NTFS_JOURNAL_FILE_BYTES,
|
||||
},
|
||||
{ cause, retryable: false },
|
||||
);
|
||||
}
|
||||
if (
|
||||
cause instanceof ArtifactReaderFailure &&
|
||||
(cause.reason === "path" || cause.reason === "format")
|
||||
)
|
||||
return inputFailure(parameter, cause.message, cause);
|
||||
if (cause instanceof Error && "code" in cause) {
|
||||
if (cause.code === "EACCES" || cause.code === "EPERM")
|
||||
return new AnalysisAccessDeniedError(operation, path, cause.code, {
|
||||
cause,
|
||||
});
|
||||
if (isSelectedRegularFileError(cause))
|
||||
return inputFailure(
|
||||
parameter,
|
||||
`Could not read the selected acquisition at ${path}`,
|
||||
cause,
|
||||
);
|
||||
}
|
||||
return new ArtifactOperationError(
|
||||
operation,
|
||||
"io",
|
||||
undefined,
|
||||
"NTFS acquisition read failed.",
|
||||
{ cause },
|
||||
);
|
||||
};
|
||||
|
||||
const readAcquisition = async (
|
||||
path: string,
|
||||
parameter: PathParameter,
|
||||
resources: ArtifactResourceScope,
|
||||
signal?: AbortSignal,
|
||||
) => {
|
||||
try {
|
||||
return ok(
|
||||
await resources.run(() =>
|
||||
readStableArtifact(
|
||||
path,
|
||||
NTFS_JOURNAL_FILE_BYTES,
|
||||
signal,
|
||||
undefined,
|
||||
resources,
|
||||
),
|
||||
),
|
||||
);
|
||||
} catch (cause) {
|
||||
return err(
|
||||
projectStableArtifactFailure(
|
||||
cause,
|
||||
readFailure(cause, path, parameter),
|
||||
path,
|
||||
NTFS_JOURNAL_PROVIDER.id,
|
||||
operation,
|
||||
),
|
||||
);
|
||||
}
|
||||
};
|
||||
|
||||
export const inspectNtfsJournalEvidence = async (
|
||||
rawInput: unknown,
|
||||
resources: ArtifactResourceScope,
|
||||
signal?: AbortSignal,
|
||||
): Promise<Result<Evidence, AnalysisError>> => {
|
||||
const admitted = parseAnalysisInput(
|
||||
operation,
|
||||
rawInput,
|
||||
ntfsJournalInputSchema,
|
||||
signal,
|
||||
);
|
||||
if (!admitted.ok) return admitted;
|
||||
const input = admitted.value;
|
||||
const mft = await readAcquisition(
|
||||
input.mft_path,
|
||||
"mft_path",
|
||||
resources,
|
||||
signal,
|
||||
);
|
||||
if (!mft.ok) return mft;
|
||||
const parsed = parseMftImage(mft.value.bytes);
|
||||
if (!parsed.ok)
|
||||
return err(
|
||||
inputFailure("mft_path", parsed.error.message).retainPartialObservation({
|
||||
kind: "file-snapshot-hash",
|
||||
path: input.mft_path,
|
||||
sha256: mft.value.sha256,
|
||||
bytes: mft.value.bytes.length,
|
||||
}),
|
||||
);
|
||||
|
||||
const journal = await readAcquisition(
|
||||
input.journal_path,
|
||||
"journal_path",
|
||||
resources,
|
||||
signal,
|
||||
);
|
||||
if (!journal.ok) {
|
||||
const failure = journal.error;
|
||||
const prior =
|
||||
failure.partialObservation ??
|
||||
primaryAnalysisError(failure).partialObservation;
|
||||
const observation = createEvidence(
|
||||
{ path: input.mft_path, format: "ntfs-mft", sha256: mft.value.sha256 },
|
||||
NTFS_JOURNAL_PROVIDER,
|
||||
{
|
||||
operation,
|
||||
predicateType: "rea.ntfs-journal-partial",
|
||||
parameters: input,
|
||||
result: {
|
||||
kind: "ntfs-journal-partial",
|
||||
mft: {
|
||||
path: input.mft_path,
|
||||
sha256: mft.value.sha256,
|
||||
bytes: mft.value.bytes.length,
|
||||
metadata: parsed.value.metadata,
|
||||
},
|
||||
journal_observation: prior ?? null,
|
||||
},
|
||||
confidence: "derived",
|
||||
limitations: [
|
||||
...parsed.value.limitations,
|
||||
"Journal acquisition did not complete.",
|
||||
],
|
||||
locations: [{ kind: "artifact-path", path: input.mft_path }],
|
||||
},
|
||||
);
|
||||
let current: AnalysisError | undefined = failure;
|
||||
while (current !== undefined) {
|
||||
if (
|
||||
current.partialObservation === prior ||
|
||||
current.partialObservation === undefined
|
||||
)
|
||||
current.partialObservation = observation;
|
||||
current = current.cleanupPrimary;
|
||||
}
|
||||
return journal;
|
||||
}
|
||||
const result = ntfsJournalResultSchema.parse({
|
||||
mft: {
|
||||
path: input.mft_path,
|
||||
bytes: mft.value.bytes.length,
|
||||
sha256: mft.value.sha256,
|
||||
},
|
||||
journal: {
|
||||
path: input.journal_path,
|
||||
bytes: journal.value.bytes.length,
|
||||
sha256: journal.value.sha256,
|
||||
},
|
||||
...composeNtfsJournalTimeline(parsed.value, journal.value.bytes, {
|
||||
startFileOffset: input.start_file_offset,
|
||||
maxRows: input.max_rows,
|
||||
}),
|
||||
});
|
||||
return completeProviderEvidence(
|
||||
createEvidence(
|
||||
{
|
||||
path: input.journal_path,
|
||||
format: "file",
|
||||
sha256: journal.value.sha256,
|
||||
},
|
||||
NTFS_JOURNAL_PROVIDER,
|
||||
{
|
||||
operation,
|
||||
predicateType: "rea.ntfs-journal",
|
||||
parameters: input,
|
||||
result,
|
||||
confidence: "derived",
|
||||
limitations: [
|
||||
...result.mftLimitations,
|
||||
"Journal bytes alone do not establish the acquisition's historical completeness.",
|
||||
],
|
||||
locations: [
|
||||
{ kind: "artifact-path", path: input.mft_path },
|
||||
{ kind: "artifact-path", path: input.journal_path },
|
||||
],
|
||||
},
|
||||
),
|
||||
signal,
|
||||
);
|
||||
};
|
||||
@@ -16,7 +16,6 @@ const recordBytes = (options: {
|
||||
readonly sequenceNumber?: number;
|
||||
readonly flags?: number;
|
||||
readonly index?: number;
|
||||
readonly name?: string;
|
||||
}): Buffer => {
|
||||
const record = Buffer.alloc(MFT_DEFAULT_RECORD_BYTES);
|
||||
record.write("FILE", 0, "latin1");
|
||||
@@ -31,22 +30,8 @@ const recordBytes = (options: {
|
||||
record.writeUInt32LE(options.index ?? 0, 44);
|
||||
record.writeUInt16LE(56, 20);
|
||||
record.writeUInt16LE(options.flags ?? 0x0001, 22);
|
||||
const attribute = 56;
|
||||
let end = attribute;
|
||||
if (options.name !== undefined) {
|
||||
const valueBytes = 66 + Buffer.byteLength(options.name, "utf16le");
|
||||
const length = Math.ceil((24 + valueBytes) / 8) * 8;
|
||||
record.writeUInt32LE(0x30, attribute);
|
||||
record.writeUInt32LE(length, attribute + 4);
|
||||
record.writeUInt32LE(valueBytes, attribute + 16);
|
||||
record.writeUInt16LE(24, attribute + 20);
|
||||
const value = attribute + 24;
|
||||
record.writeUInt8(options.name.length, value + 64);
|
||||
record.write(options.name, value + 66, "utf16le");
|
||||
end += length;
|
||||
}
|
||||
record.writeUInt32LE(end + 8, 24);
|
||||
record.writeUInt32LE(0xffff_ffff, end);
|
||||
record.writeUInt32LE(64, 24);
|
||||
record.writeUInt32LE(0xffff_ffff, 56);
|
||||
record.writeUInt32LE(MFT_DEFAULT_RECORD_BYTES, 28);
|
||||
record.writeUInt16LE(sequenceNumber, 510);
|
||||
record.writeUInt16LE(sequenceNumber, 1022);
|
||||
@@ -65,7 +50,7 @@ it("reports a $MFT stream's record headers as the root occurrence", async () =>
|
||||
await writeMft(
|
||||
"$MFT",
|
||||
Buffer.concat([
|
||||
recordBytes({ name: "sample" }),
|
||||
recordBytes({}),
|
||||
recordBytes({ flags: 0x0003, index: 1 }),
|
||||
]),
|
||||
),
|
||||
@@ -85,8 +70,6 @@ it("reports a $MFT stream's record headers as the root occurrence", async () =>
|
||||
extension_record_count: 0,
|
||||
fixup_mismatch_count: 0,
|
||||
fixup_unverified_record_count: 0,
|
||||
file_name_count: 1,
|
||||
file_name_incomplete_record_count: 0,
|
||||
truncated_tail_bytes: 0,
|
||||
},
|
||||
});
|
||||
|
||||
@@ -20,6 +20,11 @@ it("reads a Python 3.9 reference-flagged PYZ table", () => {
|
||||
]);
|
||||
});
|
||||
|
||||
it("reads marshal one-byte string tags as Latin-1", () => {
|
||||
for (const hex of ["6101000000e9", "4101000000e9", "7a01e9", "5a01e9"])
|
||||
expect(readMarshalSubset(bytes(hex))).toEqual({ ok: true, value: "é" });
|
||||
});
|
||||
|
||||
it("decodes CPython marshal Unicode with surrogatepass", () => {
|
||||
expect(
|
||||
readMarshalSubset(
|
||||
|
||||
@@ -168,14 +168,8 @@ const readAsciiBytes = (
|
||||
): Result<string, PyInstallerFailure> => {
|
||||
const bytes = readBytes(cursor, length);
|
||||
if (!bytes.ok) return bytes;
|
||||
if (bytes.value.some((byte) => byte > 0x7f))
|
||||
return err(
|
||||
pyInstallerFailure(
|
||||
"format",
|
||||
"PYZ marshal ASCII string contains a non-ASCII byte",
|
||||
),
|
||||
);
|
||||
return ok(bytes.value.toString("ascii"));
|
||||
// These tags use CPython's one-byte Unicode constructor, not an ASCII check.
|
||||
return ok(bytes.value.toString("latin1"));
|
||||
};
|
||||
|
||||
const readUtf8 = (
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
import { registerPlaywrightTraceCommand } from "./playwrightTraceCommand.js";
|
||||
import { registerApiContractCommand } from "./apiContractCommand.js";
|
||||
import { registerNtfsJournalCommand } from "./ntfsJournalCommand.js";
|
||||
import { Cli, z } from "incur";
|
||||
|
||||
import { CLI_COMMANDS } from "../cliCommandNames.js";
|
||||
@@ -78,7 +77,6 @@ export const registerApplicationCommands = (
|
||||
registerSourceToBundleCommand(cli, logger);
|
||||
registerStaticObservedEndpointsCommand(cli, logger);
|
||||
registerApiContractCommand(cli, logger);
|
||||
registerNtfsJournalCommand(cli, logger);
|
||||
registerPlaywrightTraceCommand(cli, logger);
|
||||
registerEvidenceJsonCommand({
|
||||
cli,
|
||||
|
||||
@@ -1,79 +0,0 @@
|
||||
import { z } from "incur";
|
||||
import type { Logger } from "pino";
|
||||
|
||||
import { ArtifactResourceScope } from "../artifacts/ArtifactResourceScope.js";
|
||||
import { projectArtifactCloseFailure } from "../artifacts/StableArtifactFailure.js";
|
||||
import { CLI_COMMANDS } from "../cliCommandNames.js";
|
||||
import { logCliCommand } from "../cliLogging.js";
|
||||
import { localPathStringSchema } from "../domain/localPath.js";
|
||||
import {
|
||||
NTFS_JOURNAL_OPERATION as operation,
|
||||
NTFS_JOURNAL_PROVIDER,
|
||||
ntfsJournalInputSchema,
|
||||
} from "../domain/windows/ntfsJournalQuery.js";
|
||||
import { absoluteCallerPath } from "../filesystem/callerPath.js";
|
||||
import { deliverCliAnalysisEvidence } from "./analysisEvidenceDelivery.js";
|
||||
import { withCommandCancellation } from "./commandCancellation.js";
|
||||
import { cliDecimalField } from "./options.js";
|
||||
import type { CliInstance } from "./types.js";
|
||||
|
||||
export const registerNtfsJournalCommand = (
|
||||
cli: CliInstance,
|
||||
logger: Logger,
|
||||
): void => {
|
||||
cli.command(CLI_COMMANDS.inspectNtfsJournal, {
|
||||
description: "Inspect paired offline NTFS MFT and USN journal acquisitions",
|
||||
args: z.object({
|
||||
mftPath: localPathStringSchema.describe(
|
||||
"Acquired NTFS Master File Table stream",
|
||||
),
|
||||
journalPath: localPathStringSchema.describe(
|
||||
"Acquired NTFS change journal data stream",
|
||||
),
|
||||
}),
|
||||
options: z.object({
|
||||
startFileOffset: cliDecimalField(
|
||||
ntfsJournalInputSchema.shape.start_file_offset,
|
||||
),
|
||||
maxRows: cliDecimalField(ntfsJournalInputSchema.shape.max_rows),
|
||||
}),
|
||||
run: ({ args, options }) =>
|
||||
withCommandCancellation((signal) =>
|
||||
logCliCommand(logger, CLI_COMMANDS.inspectNtfsJournal, async () => {
|
||||
const resources = new ArtifactResourceScope();
|
||||
const { inspectNtfsJournalEvidence } =
|
||||
await import("../application/ntfs/NtfsJournalService.js");
|
||||
return deliverCliAnalysisEvidence({
|
||||
callerInputPath: args.journalPath,
|
||||
callerInputPaths: { mft_path: args.mftPath },
|
||||
run: () =>
|
||||
inspectNtfsJournalEvidence(
|
||||
{
|
||||
mft_path: absoluteCallerPath(args.mftPath),
|
||||
journal_path: absoluteCallerPath(args.journalPath),
|
||||
start_file_offset: options.startFileOffset,
|
||||
max_rows: options.maxRows,
|
||||
},
|
||||
resources,
|
||||
signal,
|
||||
),
|
||||
service: {
|
||||
close: () =>
|
||||
resources.close().catch((cause: unknown) => {
|
||||
throw projectArtifactCloseFailure(
|
||||
cause,
|
||||
NTFS_JOURNAL_PROVIDER.id,
|
||||
operation,
|
||||
);
|
||||
}),
|
||||
},
|
||||
providerId: NTFS_JOURNAL_PROVIDER.id,
|
||||
operation,
|
||||
signal,
|
||||
detail: "complete",
|
||||
evidenceOutput: undefined,
|
||||
});
|
||||
}),
|
||||
),
|
||||
});
|
||||
};
|
||||
@@ -70,7 +70,6 @@ export const CLI_COMMANDS = Object.freeze({
|
||||
inspectIl2cppMetadata: "inspect-il2cpp-metadata",
|
||||
inspectModelArtifact: "inspect-model-artifact",
|
||||
inspectApiContract: "inspect-api-contract",
|
||||
inspectNtfsJournal: "inspect-ntfs-journal",
|
||||
inspectPlaywrightTrace: "inspect-playwright-trace",
|
||||
inspectGpuCodeObjects: "inspect-gpu-code-objects",
|
||||
inspectRustBinary: "inspect-rust-binary",
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
import { PLAYWRIGHT_TRACE_TOOL_CONTRACTS } from "./playwrightTraceToolContracts.js";
|
||||
import { API_CONTRACT_TOOL_CONTRACTS } from "./apiContractToolContracts.js";
|
||||
import { NTFS_JOURNAL_TOOL_CONTRACTS } from "./ntfsJournalToolContracts.js";
|
||||
import { applicationInventoryProjectionInputSchema } from "../domain/mobileApplicationGraphProjection.js";
|
||||
import { z } from "zod";
|
||||
import {
|
||||
@@ -96,7 +95,6 @@ const retained = (evidence_id: string) => ({
|
||||
/** Provider-neutral graph workflow contracts shared by MCP and CLI adapters. */
|
||||
export const APPLICATION_TOOL_CONTRACTS = [
|
||||
...API_CONTRACT_TOOL_CONTRACTS,
|
||||
...NTFS_JOURNAL_TOOL_CONTRACTS,
|
||||
...PLAYWRIGHT_TRACE_TOOL_CONTRACTS,
|
||||
{
|
||||
name: "search_guidance",
|
||||
|
||||
@@ -1,30 +0,0 @@
|
||||
import {
|
||||
NTFS_JOURNAL_OPERATION,
|
||||
ntfsJournalInputSchema,
|
||||
ntfsJournalResultSchema,
|
||||
} from "../domain/windows/ntfsJournalQuery.js";
|
||||
import type { ToolContract } from "./toolContractTypes.js";
|
||||
import { toolContractMetadata } from "./toolEffects.js";
|
||||
import { evidenceResultOf } from "./toolOutputSchemaPrimitives.js";
|
||||
|
||||
export const NTFS_JOURNAL_TOOL_CONTRACTS = [
|
||||
{
|
||||
name: NTFS_JOURNAL_OPERATION,
|
||||
...toolContractMetadata(NTFS_JOURNAL_OPERATION),
|
||||
kind: "application",
|
||||
description:
|
||||
"Correlate USN V2 records from a local acquired NTFS journal with a caller-paired MFT from the same volume. Reads at most 64 MiB per file. Returns source-order events, explicit coverage rows, both SHA-256 identities, and nextFileOffset for another page using the same inputs. Full file references include sequence numbers. Other journal versions and unverifiable references remain explicit. Reads only the selected acquisition files.",
|
||||
inputSchema: ntfsJournalInputSchema,
|
||||
outputSchema: evidenceResultOf(ntfsJournalResultSchema),
|
||||
examples: [
|
||||
{
|
||||
title: "Read a page of a paired NTFS acquisition",
|
||||
input: {
|
||||
mft_path: "/artifacts/$MFT",
|
||||
journal_path: "/artifacts/UsnJrnl-J",
|
||||
max_rows: 50,
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
] as const satisfies readonly ToolContract[];
|
||||
@@ -159,7 +159,6 @@ const TOOL_EFFECT_AUDIT: Readonly<Record<string, EffectAudit>> = {
|
||||
inspect_il2cpp_metadata: evidence,
|
||||
inspect_model_artifact: evidence,
|
||||
inspect_api_contract: evidence,
|
||||
inspect_ntfs_journal: evidence,
|
||||
inspect_playwright_trace: evidence,
|
||||
inspect_gpu_code_objects: evidence,
|
||||
inspect_rust_binary: evidence,
|
||||
|
||||
@@ -335,7 +335,7 @@ const ntfsRecordHeaderSchema = z.strictObject({
|
||||
}),
|
||||
});
|
||||
|
||||
/** NTFS $MFT record-header and resident file-name coverage facts. */
|
||||
/** NTFS $MFT record-header facts; attributes, names and the USN journal are later steps. */
|
||||
export const ntfsMftMetadataSchema = z.strictObject({
|
||||
kind: z.literal("ntfs-mft"),
|
||||
record_bytes: offset,
|
||||
@@ -348,8 +348,6 @@ export const ntfsMftMetadataSchema = z.strictObject({
|
||||
truncated_tail_bytes: offset,
|
||||
record_headers: z.array(ntfsRecordHeaderSchema).max(MFT_INLINE_HEADER_LIMIT),
|
||||
omitted_record_headers: offset,
|
||||
file_name_count: offset.exactOptional(),
|
||||
file_name_incomplete_record_count: offset.exactOptional(),
|
||||
});
|
||||
|
||||
/**
|
||||
@@ -1092,7 +1090,7 @@ const declarationsByKind: {
|
||||
},
|
||||
"ntfs-mft": {
|
||||
notCompared:
|
||||
"Its fields come from the file's own FILE record headers and resident file-name attributes, which content comparison covers.",
|
||||
"Its fields come from the record headers in the file's own FILE records, which content comparison covers.",
|
||||
},
|
||||
dfuse: {
|
||||
notCompared:
|
||||
|
||||
@@ -38,7 +38,6 @@ export class ArtifactOperationError extends AnalysisError {
|
||||
| "inspect_elf_notes"
|
||||
| "inspect_elf_unwind_ranges"
|
||||
| "inspect_devicetree"
|
||||
| "inspect_ntfs_journal"
|
||||
| "inspect_api_contract"
|
||||
| "inspect_playwright_trace"
|
||||
| "inspect_kernel_module"
|
||||
|
||||
@@ -1,98 +0,0 @@
|
||||
import { expect, it } from "vitest";
|
||||
|
||||
import { createUsnMftCorrelator } from "./ntfsJournalCorrelation.js";
|
||||
|
||||
const baseRecord = {
|
||||
index: 7,
|
||||
sequenceNumber: 2,
|
||||
fileOffset: 7168,
|
||||
totalBytes: 1024,
|
||||
extentValid: true,
|
||||
baseRecordReference: null,
|
||||
declaredRecordNumber: 7,
|
||||
recordNumberMatchesSlot: true,
|
||||
fixups: {
|
||||
sequenceNumber: 5,
|
||||
entries: 3,
|
||||
verifiedSectors: 2,
|
||||
coveredSectors: 2,
|
||||
mismatchSector: null,
|
||||
malformed: false,
|
||||
},
|
||||
};
|
||||
|
||||
it("keeps a reused record number unresolved while resolving a full parent reference", () => {
|
||||
const correlate = createUsnMftCorrelator([baseRecord]);
|
||||
expect(
|
||||
correlate({
|
||||
fileReference: 0x0001_0000_0000_0007n,
|
||||
parentReference: 0x0002_0000_0000_0007n,
|
||||
}),
|
||||
).toEqual({
|
||||
file: { kind: "unresolved" },
|
||||
parent: {
|
||||
kind: "matched",
|
||||
record: { index: 7, sequenceNumber: 2, fileOffset: 7168 },
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it("uses the MFT decoder's record identity verdict", () => {
|
||||
const correlate = createUsnMftCorrelator([
|
||||
{
|
||||
...baseRecord,
|
||||
declaredRecordNumber: 0,
|
||||
recordNumberMatchesSlot: false,
|
||||
},
|
||||
]);
|
||||
expect(
|
||||
correlate({
|
||||
fileReference: 0x0002_0000_0000_0007n,
|
||||
parentReference: 0n,
|
||||
}).file,
|
||||
).toEqual({ kind: "unresolved" });
|
||||
});
|
||||
|
||||
it("leaves unused zero-sequence records unresolved", () => {
|
||||
const correlate = createUsnMftCorrelator([
|
||||
{ ...baseRecord, sequenceNumber: 0 },
|
||||
]);
|
||||
expect(correlate({ fileReference: 7n, parentReference: 7n })).toEqual({
|
||||
file: { kind: "unresolved" },
|
||||
parent: { kind: "unresolved" },
|
||||
});
|
||||
});
|
||||
|
||||
it("keeps duplicates ambiguous and skips extensions and unverified records", () => {
|
||||
const correlate = createUsnMftCorrelator([
|
||||
baseRecord,
|
||||
{ ...baseRecord, fileOffset: 8192 },
|
||||
{
|
||||
...baseRecord,
|
||||
index: 9,
|
||||
declaredRecordNumber: 9,
|
||||
baseRecordReference: { recordNumber: 7, sequenceNumber: 2 },
|
||||
},
|
||||
{
|
||||
...baseRecord,
|
||||
index: 9,
|
||||
declaredRecordNumber: 9,
|
||||
fixups: { ...baseRecord.fixups, verifiedSectors: 1, mismatchSector: 1 },
|
||||
},
|
||||
]);
|
||||
const correlateContradictory = createUsnMftCorrelator([
|
||||
{ ...baseRecord, extentValid: false },
|
||||
]);
|
||||
expect(
|
||||
correlateContradictory({
|
||||
fileReference: 0x0002_0000_0000_0007n,
|
||||
parentReference: 0n,
|
||||
}).file,
|
||||
).toEqual({ kind: "unresolved" });
|
||||
expect(
|
||||
correlate({
|
||||
fileReference: 0x0002_0000_0000_0007n,
|
||||
parentReference: 0x0002_0000_0000_0009n,
|
||||
}),
|
||||
).toEqual({ file: { kind: "ambiguous" }, parent: { kind: "unresolved" } });
|
||||
});
|
||||
@@ -1,72 +0,0 @@
|
||||
import { FIXUP_SECTOR_BYTES, type MftRecord } from "./ntfsMft.js";
|
||||
import type { UsnRecordV2 } from "./ntfsUsnJournal.js";
|
||||
|
||||
type RecordIdentity = Pick<
|
||||
MftRecord,
|
||||
"index" | "sequenceNumber" | "fileOffset"
|
||||
>;
|
||||
|
||||
type CorrelatableRecord = RecordIdentity &
|
||||
Pick<
|
||||
MftRecord,
|
||||
| "baseRecordReference"
|
||||
| "declaredRecordNumber"
|
||||
| "recordNumberMatchesSlot"
|
||||
| "fixups"
|
||||
| "totalBytes"
|
||||
| "extentValid"
|
||||
>;
|
||||
type JournalReferences = Pick<UsnRecordV2, "fileReference" | "parentReference">;
|
||||
|
||||
export type MftJournalMatch =
|
||||
| { readonly kind: "matched"; readonly record: RecordIdentity }
|
||||
| { readonly kind: "unresolved" | "ambiguous" };
|
||||
|
||||
export interface UsnMftMatches {
|
||||
readonly file: MftJournalMatch;
|
||||
readonly parent: MftJournalMatch;
|
||||
}
|
||||
|
||||
/** Build one lookup for a selected MFT, then reuse it for each journal entry. */
|
||||
export const createUsnMftCorrelator = (
|
||||
records: Iterable<CorrelatableRecord>,
|
||||
): ((entry: JournalReferences) => UsnMftMatches) => {
|
||||
const byReference = new Map<bigint, RecordIdentity | null>();
|
||||
for (const record of records) {
|
||||
// The stream ordinal is the segment number in an acquisition of the
|
||||
// logical MFT stream. Extension records do not identify separate files.
|
||||
if (
|
||||
record.baseRecordReference !== null ||
|
||||
record.sequenceNumber === 0 ||
|
||||
!record.extentValid ||
|
||||
record.fixups.verifiedSectors * FIXUP_SECTOR_BYTES !== record.totalBytes
|
||||
)
|
||||
continue;
|
||||
if (record.recordNumberMatchesSlot === false) continue;
|
||||
|
||||
const reference =
|
||||
(BigInt(record.sequenceNumber) << 48n) | BigInt(record.index);
|
||||
byReference.set(
|
||||
reference,
|
||||
byReference.has(reference)
|
||||
? null
|
||||
: {
|
||||
index: record.index,
|
||||
sequenceNumber: record.sequenceNumber,
|
||||
fileOffset: record.fileOffset,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
const lookup = (reference: bigint): MftJournalMatch => {
|
||||
const record = byReference.get(reference);
|
||||
if (record === undefined) return { kind: "unresolved" };
|
||||
if (record === null) return { kind: "ambiguous" };
|
||||
return { kind: "matched", record };
|
||||
};
|
||||
|
||||
return (entry) => ({
|
||||
file: lookup(entry.fileReference),
|
||||
parent: lookup(entry.parentReference),
|
||||
});
|
||||
};
|
||||
@@ -1,91 +0,0 @@
|
||||
import { z } from "zod";
|
||||
|
||||
import { digestSchema } from "../digests.js";
|
||||
import { absoluteLocalPathStringSchema } from "../localPath.js";
|
||||
|
||||
export const NTFS_JOURNAL_OPERATION = "inspect_ntfs_journal" as const;
|
||||
export const NTFS_JOURNAL_PROVIDER = {
|
||||
id: "rea-ntfs-journal",
|
||||
name: "REA offline NTFS change journal",
|
||||
version: "1",
|
||||
} as const;
|
||||
|
||||
/** Working-set allowance for each in-memory acquisition, independent of page size. */
|
||||
export const NTFS_JOURNAL_FILE_BYTES = 64 * 1024 * 1024;
|
||||
const offset = z.int().nonnegative();
|
||||
export const ntfsJournalRowLimitSchema = z.int().min(1).max(200).default(50);
|
||||
export const ntfsJournalInputSchema = z.strictObject({
|
||||
mft_path: absoluteLocalPathStringSchema,
|
||||
journal_path: absoluteLocalPathStringSchema,
|
||||
start_file_offset: offset
|
||||
.default(0)
|
||||
.describe(
|
||||
"Begin decoding exactly at this source offset, normally the prior nextFileOffset for unchanged input digests. Earlier bytes are not examined.",
|
||||
),
|
||||
max_rows: ntfsJournalRowLimitSchema,
|
||||
});
|
||||
|
||||
const sourceSchema = z.strictObject({
|
||||
path: absoluteLocalPathStringSchema,
|
||||
bytes: offset,
|
||||
sha256: digestSchema,
|
||||
});
|
||||
const matchSchema = z.discriminatedUnion("kind", [
|
||||
z.strictObject({
|
||||
kind: z.literal("matched"),
|
||||
record: z.strictObject({
|
||||
index: offset,
|
||||
sequenceNumber: offset,
|
||||
fileOffset: offset,
|
||||
}),
|
||||
}),
|
||||
z.strictObject({ kind: z.enum(["unresolved", "ambiguous"]) }),
|
||||
]);
|
||||
const issueSchema = z.discriminatedUnion("kind", [
|
||||
z.strictObject({
|
||||
kind: z.literal("padding"),
|
||||
fileOffset: offset,
|
||||
bytes: offset,
|
||||
}),
|
||||
z.strictObject({
|
||||
kind: z.enum(["malformed", "truncated"]),
|
||||
fileOffset: offset,
|
||||
}),
|
||||
z.strictObject({
|
||||
kind: z.literal("unsupported"),
|
||||
fileOffset: offset,
|
||||
major: offset,
|
||||
minor: offset,
|
||||
}),
|
||||
]);
|
||||
const unsignedDecimal = z.string().regex(/^(?:0|[1-9][0-9]*)$/u);
|
||||
const signedDecimal = z.string().regex(/^(?:0|-?[1-9][0-9]*)$/u);
|
||||
|
||||
export const ntfsJournalResultSchema = z.strictObject({
|
||||
mft: sourceSchema,
|
||||
journal: sourceSchema,
|
||||
rows: z.array(
|
||||
z.discriminatedUnion("kind", [
|
||||
z.strictObject({
|
||||
kind: z.literal("event"),
|
||||
event: z.strictObject({
|
||||
fileOffset: offset,
|
||||
recordBytes: offset,
|
||||
fileReference: unsignedDecimal,
|
||||
parentReference: unsignedDecimal,
|
||||
usn: signedDecimal,
|
||||
timestampFileTime: unsignedDecimal,
|
||||
reasonFlags: offset,
|
||||
sourceFlags: offset,
|
||||
securityId: offset,
|
||||
fileAttributes: offset,
|
||||
name: z.string(),
|
||||
mft: z.strictObject({ file: matchSchema, parent: matchSchema }),
|
||||
}),
|
||||
}),
|
||||
z.strictObject({ kind: z.literal("coverage"), issue: issueSchema }),
|
||||
]),
|
||||
),
|
||||
nextFileOffset: offset.nullable(),
|
||||
mftLimitations: z.array(z.string()),
|
||||
});
|
||||
@@ -1,71 +0,0 @@
|
||||
import { expect, it } from "vitest";
|
||||
|
||||
import { composeNtfsJournalTimeline } from "./ntfsJournalTimeline.js";
|
||||
import { parseMftImage } from "./ntfsMft.js";
|
||||
|
||||
it("keeps exact USN values and coverage rows on bounded source-order pages", () => {
|
||||
const record = Buffer.alloc(1024);
|
||||
record.write("FILE");
|
||||
record.writeUInt16LE(48, 4);
|
||||
record.writeUInt16LE(3, 6);
|
||||
record.writeUInt16LE(1, 48);
|
||||
record.writeUInt16LE(1, 510);
|
||||
record.writeUInt16LE(1, 1022);
|
||||
record.writeUInt16LE(2, 16);
|
||||
record.writeUInt16LE(56, 20);
|
||||
record.writeUInt32LE(64, 24);
|
||||
record.writeUInt32LE(1024, 28);
|
||||
record.writeUInt32LE(0xffff_ffff, 56);
|
||||
const mft = parseMftImage(record);
|
||||
if (!mft.ok) throw new Error(mft.error.message);
|
||||
|
||||
const journal = Buffer.alloc(72);
|
||||
journal.writeUInt32LE(64, 8);
|
||||
journal.writeUInt16LE(2, 12);
|
||||
journal.writeBigUInt64LE(0x0002_0000_0000_0000n, 16);
|
||||
journal.writeBigInt64LE(9007199254740993n, 32);
|
||||
journal.writeBigUInt64LE(133000000000000001n, 40);
|
||||
journal.writeUInt16LE(60, 66);
|
||||
|
||||
const first = composeNtfsJournalTimeline(mft.value, journal, {
|
||||
startFileOffset: 0,
|
||||
maxRows: 1,
|
||||
});
|
||||
expect(first).toEqual({
|
||||
rows: [
|
||||
{ kind: "coverage", issue: { kind: "padding", fileOffset: 0, bytes: 8 } },
|
||||
],
|
||||
nextFileOffset: 8,
|
||||
mftLimitations: [],
|
||||
});
|
||||
|
||||
const second = composeNtfsJournalTimeline(mft.value, journal, {
|
||||
startFileOffset: first.nextFileOffset ?? 0,
|
||||
maxRows: 1,
|
||||
});
|
||||
expect(second).toMatchObject({
|
||||
rows: [
|
||||
{
|
||||
kind: "event",
|
||||
event: {
|
||||
fileOffset: 8,
|
||||
usn: "9007199254740993",
|
||||
timestampFileTime: "133000000000000001",
|
||||
mft: {
|
||||
file: { kind: "matched", record: { index: 0, sequenceNumber: 2 } },
|
||||
parent: { kind: "unresolved" },
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
nextFileOffset: null,
|
||||
});
|
||||
const changedPrefix = Buffer.from(journal);
|
||||
changedPrefix.writeUInt32LE(1, 0);
|
||||
expect(
|
||||
composeNtfsJournalTimeline(mft.value, changedPrefix, {
|
||||
startFileOffset: 8,
|
||||
maxRows: 1,
|
||||
}),
|
||||
).toEqual(second);
|
||||
});
|
||||
@@ -1,72 +0,0 @@
|
||||
import type { MftImage } from "./ntfsMft.js";
|
||||
import {
|
||||
createUsnMftCorrelator,
|
||||
type UsnMftMatches,
|
||||
} from "./ntfsJournalCorrelation.js";
|
||||
import {
|
||||
readUsnJournal,
|
||||
type UsnRecordIssue,
|
||||
type UsnRecordV2,
|
||||
} from "./ntfsUsnJournal.js";
|
||||
|
||||
type JournalEvent = Omit<
|
||||
UsnRecordV2,
|
||||
"fileReference" | "parentReference" | "usn" | "timestampFileTime"
|
||||
> & {
|
||||
readonly fileReference: string;
|
||||
readonly parentReference: string;
|
||||
readonly usn: string;
|
||||
readonly timestampFileTime: string;
|
||||
readonly mft: UsnMftMatches;
|
||||
};
|
||||
|
||||
type TimelineRow =
|
||||
| { readonly kind: "event"; readonly event: JournalEvent }
|
||||
| { readonly kind: "coverage"; readonly issue: UsnRecordIssue };
|
||||
|
||||
export interface NtfsJournalTimeline {
|
||||
readonly rows: readonly TimelineRow[];
|
||||
/** First source offset left out of the delivery window. */
|
||||
readonly nextFileOffset: number | null;
|
||||
readonly mftLimitations: readonly string[];
|
||||
}
|
||||
|
||||
/** Compose a bounded source-order page for an explicitly paired acquisition. */
|
||||
export const composeNtfsJournalTimeline = (
|
||||
mft: MftImage,
|
||||
journal: Buffer,
|
||||
window: { readonly startFileOffset: number; readonly maxRows: number },
|
||||
): NtfsJournalTimeline => {
|
||||
const rows: TimelineRow[] = [];
|
||||
const correlate = createUsnMftCorrelator(mft.records);
|
||||
let nextFileOffset: number | null = null;
|
||||
|
||||
for (const result of readUsnJournal(journal, window.startFileOffset)) {
|
||||
const fileOffset = result.ok
|
||||
? result.value.fileOffset
|
||||
: result.error.fileOffset;
|
||||
if (rows.length >= window.maxRows) {
|
||||
nextFileOffset = fileOffset;
|
||||
break;
|
||||
}
|
||||
if (!result.ok) {
|
||||
rows.push({ kind: "coverage", issue: result.error });
|
||||
continue;
|
||||
}
|
||||
|
||||
const entry = result.value;
|
||||
rows.push({
|
||||
kind: "event",
|
||||
event: {
|
||||
...entry,
|
||||
fileReference: entry.fileReference.toString(),
|
||||
parentReference: entry.parentReference.toString(),
|
||||
usn: entry.usn.toString(),
|
||||
timestampFileTime: entry.timestampFileTime.toString(),
|
||||
mft: correlate(entry),
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
return { rows, nextFileOffset, mftLimitations: mft.limitations };
|
||||
};
|
||||
@@ -97,8 +97,6 @@ describe("MFT header facts", () => {
|
||||
fixup_mismatch_count: 0,
|
||||
fixup_unverified_record_count: 0,
|
||||
truncated_tail_bytes: 0,
|
||||
file_name_count: 0,
|
||||
file_name_incomplete_record_count: 0,
|
||||
});
|
||||
expect(parsed.value.records[0]).toMatchObject({
|
||||
index: 0,
|
||||
@@ -218,40 +216,6 @@ describe("MFT fixup coverage", () => {
|
||||
);
|
||||
},
|
||||
);
|
||||
it("decodes a FILE_NAME across a restored sector trailer without changing the source bytes", () => {
|
||||
const bytes = recordOf();
|
||||
const view = new DataView(bytes.buffer);
|
||||
const attribute = 416;
|
||||
const value = attribute + 24;
|
||||
view.setUint16(20, attribute, true);
|
||||
view.setUint32(24, 528, true);
|
||||
view.setUint32(attribute, 0x30, true);
|
||||
view.setUint32(attribute + 4, 104, true);
|
||||
view.setUint32(attribute + 16, 76, true);
|
||||
view.setUint16(attribute + 20, 24, true);
|
||||
view.setBigUint64(value, 0x1234_0000_0000_0005n, true);
|
||||
bytes.set([5, 1], value + 64);
|
||||
bytes.set(Buffer.from("names", "utf16le"), value + 66);
|
||||
view.setUint16(50, view.getUint16(510, true), true);
|
||||
view.setUint16(510, view.getUint16(48, true), true);
|
||||
view.setUint32(520, 0xffff_ffff, true);
|
||||
const original = bytes.slice();
|
||||
|
||||
const parsed = parseMftImage(bytes);
|
||||
if (!parsed.ok) throw new Error(parsed.error.message);
|
||||
expect(parsed.value.records[0]).toMatchObject({
|
||||
fileNameCoverage: "complete",
|
||||
fileNames: [
|
||||
{
|
||||
fileOffset: value,
|
||||
parent: { recordNumber: 5, sequenceNumber: 0x1234 },
|
||||
namespace: 1,
|
||||
name: "names",
|
||||
},
|
||||
],
|
||||
});
|
||||
expect(bytes).toEqual(original);
|
||||
});
|
||||
|
||||
it("keeps a record whose sector trailer is not the sequence number, and reports it", () => {
|
||||
const parsed = parseMftImage(
|
||||
|
||||
+24
-135
@@ -22,7 +22,7 @@ const MFT_MAX_RECORD_BYTES = 65536;
|
||||
/** An update sequence array holds at least the sequence number and one fixup word. */
|
||||
const USA_MIN_ENTRIES = 2;
|
||||
/** NTFS fixes up one 512-byte sector at a time. */
|
||||
export const FIXUP_SECTOR_BYTES = 512;
|
||||
const FIXUP_SECTOR_BYTES = 512;
|
||||
/**
|
||||
* Fixed whole-file ceiling. Records are inventoried from their headers only;
|
||||
* attributes and their payloads are not decoded, so this is deliberately
|
||||
@@ -82,13 +82,6 @@ const decodeMftFileReference = (reference: bigint): MftFileReference => ({
|
||||
sequenceNumber: Number(reference >> 48n),
|
||||
});
|
||||
|
||||
export interface MftFileName {
|
||||
readonly fileOffset: number;
|
||||
readonly parent: MftFileReference;
|
||||
readonly namespace: number;
|
||||
readonly name: string;
|
||||
}
|
||||
|
||||
export interface MftRecord {
|
||||
/** Zero-based ordinal of this record in the stream. */
|
||||
readonly index: number;
|
||||
@@ -115,14 +108,11 @@ export interface MftRecord {
|
||||
readonly recordNumberMatchesSlot: boolean | null;
|
||||
/** `$LogFile` sequence number as recorded; no log is read. */
|
||||
readonly logFileSequenceNumber: string;
|
||||
/** Declared offset of the first attribute. */
|
||||
/** Declared offset of the first attribute; no attribute decoding follows yet. */
|
||||
readonly firstAttributeOffset: number;
|
||||
/** Declared id of the next attribute; no attribute decoding follows yet. */
|
||||
readonly nextAttributeId: number;
|
||||
readonly fixups: MftRecordFixups;
|
||||
readonly fileNames: readonly MftFileName[];
|
||||
/** An incomplete scan can still retain names from the valid prefix. */
|
||||
readonly fileNameCoverage: "complete" | "incomplete";
|
||||
}
|
||||
|
||||
export interface MftInventory {
|
||||
@@ -146,13 +136,8 @@ export interface MftMetadata {
|
||||
readonly fixup_mismatch_count: number;
|
||||
readonly fixup_unverified_record_count: number;
|
||||
readonly truncated_tail_bytes: number;
|
||||
readonly record_headers: readonly Omit<
|
||||
MftRecord,
|
||||
"fileNames" | "fileNameCoverage"
|
||||
>[];
|
||||
readonly record_headers: readonly MftRecord[];
|
||||
readonly omitted_record_headers: number;
|
||||
readonly file_name_count: number;
|
||||
readonly file_name_incomplete_record_count: number;
|
||||
}
|
||||
|
||||
const text = (ascii: Uint8Array): string =>
|
||||
@@ -225,8 +210,13 @@ export const hasMftFileRecordHeader = (prefix: Uint8Array): boolean => {
|
||||
};
|
||||
|
||||
/**
|
||||
* Parse FILE record headers and resident file names. A record with failed
|
||||
* fixups keeps its header facts; its attributes are left unread.
|
||||
* Parse FILE record headers and verify their update-sequence fixups.
|
||||
*
|
||||
* Only the record structure is established: attributes, their names, the
|
||||
* filenames and the USN journal are left to later steps, so no path, timestamp
|
||||
* or file identity beyond the record itself is claimed. A record whose fixup
|
||||
* does not match keeps its header facts, because the stream still declares it,
|
||||
* and the mismatch is reported on the record and counted in the metadata.
|
||||
*/
|
||||
export const parseMftImage = (
|
||||
bytes: Uint8Array,
|
||||
@@ -267,9 +257,6 @@ const scanMft = (
|
||||
let extensionCount = 0;
|
||||
let mismatchCount = 0;
|
||||
let unverifiedCount = 0;
|
||||
const restored = Buffer.allocUnsafe(recordBytes);
|
||||
let incompleteNames = 0;
|
||||
let fileNameCount = 0;
|
||||
let invalidSlots = 0;
|
||||
let invalidExtents = 0;
|
||||
let inconsistentRecordNumbers = 0;
|
||||
@@ -290,9 +277,8 @@ const scanMft = (
|
||||
if (!isRecordSize(declaredRecordBytes)) break;
|
||||
continue;
|
||||
}
|
||||
const record = readRecord(bytes, offset, restored);
|
||||
const record = readRecord(bytes, offset, offset / recordBytes, recordBytes);
|
||||
retainedRecords?.push(record);
|
||||
fileNameCount += record.fileNames.length;
|
||||
if (recordHeaders.length < MFT_INLINE_HEADER_LIMIT)
|
||||
recordHeaders.push(record);
|
||||
recordCount += 1;
|
||||
@@ -307,7 +293,6 @@ const scanMft = (
|
||||
unverifiedCount += 1;
|
||||
if (record.recordNumberMatchesSlot === false)
|
||||
inconsistentRecordNumbers += 1;
|
||||
if (record.fileNameCoverage === "incomplete") incompleteNames += 1;
|
||||
if (!record.extentValid) {
|
||||
invalidExtents += 1;
|
||||
firstInvalidExtentOffset ??= offset;
|
||||
@@ -343,11 +328,6 @@ const scanMft = (
|
||||
`${String(unverifiedCount)} record(s) carry an invalid or incomplete update sequence array; their sector trailers were not fully verified`,
|
||||
);
|
||||
|
||||
if (incompleteNames > 0)
|
||||
limitations.push(
|
||||
`File name coverage is incomplete for ${String(incompleteNames)} record(s)`,
|
||||
);
|
||||
|
||||
const metadata: MftMetadata = {
|
||||
kind: "ntfs-mft",
|
||||
record_bytes: recordBytes,
|
||||
@@ -358,13 +338,8 @@ const scanMft = (
|
||||
fixup_mismatch_count: mismatchCount,
|
||||
fixup_unverified_record_count: unverifiedCount,
|
||||
truncated_tail_bytes: truncatedTailBytes,
|
||||
record_headers: recordHeaders.map(
|
||||
({ fileNames: _fileNames, fileNameCoverage: _coverage, ...header }) =>
|
||||
header,
|
||||
),
|
||||
record_headers: recordHeaders,
|
||||
omitted_record_headers: recordCount - recordHeaders.length,
|
||||
file_name_count: fileNameCount,
|
||||
file_name_incomplete_record_count: incompleteNames,
|
||||
};
|
||||
|
||||
return ok({ metadata, recordBytes, limitations });
|
||||
@@ -373,10 +348,9 @@ const scanMft = (
|
||||
const readRecord = (
|
||||
bytes: Uint8Array,
|
||||
fileOffset: number,
|
||||
restored: Buffer,
|
||||
index: number,
|
||||
recordBytes: number,
|
||||
): MftRecord => {
|
||||
const recordBytes = restored.length;
|
||||
const index = fileOffset / recordBytes;
|
||||
const record = bytes.subarray(fileOffset, fileOffset + recordBytes);
|
||||
const view = new DataView(
|
||||
record.buffer,
|
||||
@@ -398,29 +372,6 @@ const readRecord = (
|
||||
RECORD_HEADER.firstAttributeOffset,
|
||||
true,
|
||||
);
|
||||
const extentValid =
|
||||
totalBytes === recordBytes &&
|
||||
usedBytes >= NTFS_MFT_MIN_RECORD_BYTES &&
|
||||
usedBytes % 8 === 0 &&
|
||||
usedBytes <= totalBytes &&
|
||||
firstAttributeOffset % 8 === 0 &&
|
||||
firstAttributeOffset >= NTFS_MFT_MIN_RECORD_BYTES &&
|
||||
firstAttributeOffset + 4 <= usedBytes &&
|
||||
(fixups.malformed ||
|
||||
firstAttributeOffset >= usaOffset + fixups.entries * 2);
|
||||
const fileNames: MftFileName[] = [];
|
||||
const completeNames =
|
||||
fixups.verifiedSectors === recordBytes / FIXUP_SECTOR_BYTES &&
|
||||
extentValid &&
|
||||
readFileNames(
|
||||
record,
|
||||
view,
|
||||
restored,
|
||||
firstAttributeOffset,
|
||||
usedBytes,
|
||||
fileOffset,
|
||||
fileNames,
|
||||
);
|
||||
|
||||
const declaredRecordNumber =
|
||||
!fixups.malformed && usaOffset >= RECORD_HEADER_WITH_NUMBER_BYTES
|
||||
@@ -437,7 +388,16 @@ const readRecord = (
|
||||
directory: (flags & FILE_RECORD_IS_DIRECTORY) !== 0,
|
||||
usedBytes,
|
||||
totalBytes,
|
||||
extentValid,
|
||||
extentValid:
|
||||
totalBytes === recordBytes &&
|
||||
usedBytes >= NTFS_MFT_MIN_RECORD_BYTES &&
|
||||
usedBytes % 8 === 0 &&
|
||||
usedBytes <= totalBytes &&
|
||||
firstAttributeOffset % 8 === 0 &&
|
||||
firstAttributeOffset >= NTFS_MFT_MIN_RECORD_BYTES &&
|
||||
firstAttributeOffset + 4 <= usedBytes &&
|
||||
(fixups.malformed ||
|
||||
firstAttributeOffset >= usaOffset + fixups.entries * 2),
|
||||
baseRecordReference:
|
||||
baseRecordReference === 0n
|
||||
? null
|
||||
@@ -451,80 +411,9 @@ const readRecord = (
|
||||
firstAttributeOffset,
|
||||
nextAttributeId: view.getUint16(RECORD_HEADER.nextAttributeId, true),
|
||||
fixups,
|
||||
fileNames,
|
||||
fileNameCoverage: completeNames ? "complete" : "incomplete",
|
||||
};
|
||||
};
|
||||
|
||||
const ATTRIBUTE_END = 0xffff_ffff;
|
||||
const ATTRIBUTE_LIST = 0x20;
|
||||
const ATTRIBUTE_FILE_NAME = 0x30;
|
||||
const RESIDENT_HEADER_BYTES = 24;
|
||||
const FILE_NAME_HEADER_BYTES = 66;
|
||||
|
||||
const readFileNames = (
|
||||
record: Uint8Array,
|
||||
raw: DataView,
|
||||
restored: Buffer,
|
||||
firstAttributeOffset: number,
|
||||
usedBytes: number,
|
||||
fileOffset: number,
|
||||
names: MftFileName[],
|
||||
): boolean => {
|
||||
restored.set(record.subarray(0, usedBytes));
|
||||
const usaOffset = raw.getUint16(RECORD_HEADER.updateSequenceOffset, true);
|
||||
for (
|
||||
let end = FIXUP_SECTOR_BYTES;
|
||||
end <= usedBytes + 1;
|
||||
end += FIXUP_SECTOR_BYTES
|
||||
)
|
||||
restored.writeUInt16LE(
|
||||
raw.getUint16(usaOffset + 2 * (end / FIXUP_SECTOR_BYTES), true),
|
||||
end - 2,
|
||||
);
|
||||
|
||||
let hasAttributeList = false;
|
||||
for (let offset = firstAttributeOffset; offset + 4 <= usedBytes;) {
|
||||
const type = restored.readUInt32LE(offset);
|
||||
if (type === ATTRIBUTE_END) return !hasAttributeList;
|
||||
if (offset + RESIDENT_HEADER_BYTES > usedBytes) return false;
|
||||
const length = restored.readUInt32LE(offset + 4);
|
||||
const form = restored.readUInt8(offset + 8);
|
||||
const headerBytes = form === 0 ? RESIDENT_HEADER_BYTES : 64;
|
||||
if (
|
||||
form > 1 ||
|
||||
length < headerBytes ||
|
||||
length % 8 !== 0 ||
|
||||
offset + length > usedBytes
|
||||
)
|
||||
return false;
|
||||
if (type === ATTRIBUTE_LIST) hasAttributeList = true;
|
||||
if (type === ATTRIBUTE_FILE_NAME) {
|
||||
if (form !== 0) return false;
|
||||
const valueLength = restored.readUInt32LE(offset + 16);
|
||||
const valueOffset = restored.readUInt16LE(offset + 20);
|
||||
if (
|
||||
valueOffset < headerBytes ||
|
||||
valueLength < FILE_NAME_HEADER_BYTES ||
|
||||
valueOffset + valueLength > length
|
||||
)
|
||||
return false;
|
||||
const value = offset + valueOffset;
|
||||
const nameBytes = restored.readUInt8(value + 64) * 2;
|
||||
if (FILE_NAME_HEADER_BYTES + nameBytes > valueLength) return false;
|
||||
const nameStart = value + FILE_NAME_HEADER_BYTES;
|
||||
names.push({
|
||||
fileOffset: fileOffset + value,
|
||||
parent: decodeMftFileReference(restored.readBigUInt64LE(value)),
|
||||
namespace: restored.readUInt8(value + 65),
|
||||
name: restored.toString("utf16le", nameStart, nameStart + nameBytes),
|
||||
});
|
||||
}
|
||||
offset += length;
|
||||
}
|
||||
return false;
|
||||
};
|
||||
|
||||
/**
|
||||
* Each sector's last two bytes are replaced on disk by the update sequence
|
||||
* number; the array's first entry is that number and the rest are the saved
|
||||
|
||||
@@ -1,67 +0,0 @@
|
||||
import { expect, it } from "vitest";
|
||||
import { readUsnJournal } from "./ntfsUsnJournal.js";
|
||||
|
||||
const journalRecord = (): Buffer => {
|
||||
const bytes = Buffer.alloc(72);
|
||||
bytes.writeUInt32LE(bytes.length);
|
||||
bytes.writeUInt16LE(2, 4);
|
||||
bytes.writeBigUInt64LE(0xabcd_1234_5678_9abcn, 8);
|
||||
bytes.writeBigUInt64LE(0x1234_0000_0000_0005n, 16);
|
||||
bytes.writeBigInt64LE(9007199254740993n, 24);
|
||||
bytes.writeBigUInt64LE(133000000000000001n, 32);
|
||||
bytes.writeUInt32LE(0x80001000, 40);
|
||||
bytes.writeUInt32LE(4, 44);
|
||||
bytes.writeUInt16LE(8, 56);
|
||||
bytes.writeUInt16LE(62, 58);
|
||||
bytes.write("a😃b", 62, "utf16le");
|
||||
return bytes;
|
||||
};
|
||||
|
||||
it("preserves full references, flags, offsets and UTF-16 names after sparse padding", () => {
|
||||
const results = [
|
||||
...readUsnJournal(Buffer.concat([Buffer.alloc(16), journalRecord()])),
|
||||
];
|
||||
expect(results).toEqual([
|
||||
{ ok: false, error: { kind: "padding", fileOffset: 0, bytes: 16 } },
|
||||
{
|
||||
ok: true,
|
||||
value: {
|
||||
fileOffset: 16,
|
||||
recordBytes: 72,
|
||||
fileReference: 0xabcd_1234_5678_9abcn,
|
||||
parentReference: 0x1234_0000_0000_0005n,
|
||||
usn: 9007199254740993n,
|
||||
timestampFileTime: 133000000000000001n,
|
||||
reasonFlags: 0x80001000,
|
||||
sourceFlags: 4,
|
||||
securityId: 0,
|
||||
fileAttributes: 0,
|
||||
name: "a😃b",
|
||||
},
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it("uses a validated record length to skip unsupported and malformed records", () => {
|
||||
const unsupported = journalRecord();
|
||||
unsupported.writeUInt16LE(4, 4);
|
||||
const malformed = journalRecord();
|
||||
malformed.writeUInt16LE(12, 56);
|
||||
const results = [
|
||||
...readUsnJournal(Buffer.concat([unsupported, malformed, journalRecord()])),
|
||||
];
|
||||
expect(results).toMatchObject([
|
||||
{
|
||||
ok: false,
|
||||
error: { kind: "unsupported", fileOffset: 0, major: 4, minor: 0 },
|
||||
},
|
||||
{ ok: false, error: { kind: "malformed", fileOffset: 72 } },
|
||||
{ ok: true, value: { fileOffset: 144 } },
|
||||
]);
|
||||
});
|
||||
|
||||
it("stops at a truncated record", () => {
|
||||
expect([...readUsnJournal(journalRecord().subarray(0, 64))]).toEqual([
|
||||
{ ok: false, error: { kind: "truncated", fileOffset: 0 } },
|
||||
]);
|
||||
});
|
||||
@@ -1,107 +0,0 @@
|
||||
import { err, ok, type Result } from "../result.js";
|
||||
|
||||
const COMMON_HEADER_BYTES = 8;
|
||||
const V2_HEADER_BYTES = 60;
|
||||
|
||||
export interface UsnRecordV2 {
|
||||
readonly fileOffset: number;
|
||||
readonly recordBytes: number;
|
||||
readonly fileReference: bigint;
|
||||
readonly parentReference: bigint;
|
||||
readonly usn: bigint;
|
||||
readonly timestampFileTime: bigint;
|
||||
readonly reasonFlags: number;
|
||||
readonly sourceFlags: number;
|
||||
readonly securityId: number;
|
||||
readonly fileAttributes: number;
|
||||
readonly name: string;
|
||||
}
|
||||
|
||||
export type UsnRecordIssue =
|
||||
| {
|
||||
readonly kind: "padding";
|
||||
readonly fileOffset: number;
|
||||
readonly bytes: number;
|
||||
}
|
||||
| { readonly kind: "malformed" | "truncated"; readonly fileOffset: number }
|
||||
| {
|
||||
readonly kind: "unsupported";
|
||||
readonly fileOffset: number;
|
||||
readonly major: number;
|
||||
readonly minor: number;
|
||||
};
|
||||
|
||||
/** Walk acquired journal bytes in source order without accumulating records. */
|
||||
export function* readUsnJournal(
|
||||
bytes: Buffer,
|
||||
startFileOffset = 0,
|
||||
): Generator<Result<UsnRecordV2, UsnRecordIssue>> {
|
||||
let fileOffset = startFileOffset;
|
||||
while (fileOffset + COMMON_HEADER_BYTES <= bytes.length) {
|
||||
const length = bytes.readUInt32LE(fileOffset);
|
||||
if (length === 0 && bytes.readUInt32LE(fileOffset + 4) === 0) {
|
||||
const start = fileOffset;
|
||||
do {
|
||||
fileOffset += COMMON_HEADER_BYTES;
|
||||
} while (
|
||||
fileOffset + COMMON_HEADER_BYTES <= bytes.length &&
|
||||
bytes.readBigUInt64LE(fileOffset) === 0n
|
||||
);
|
||||
yield err({
|
||||
kind: "padding",
|
||||
fileOffset: start,
|
||||
bytes: fileOffset - start,
|
||||
});
|
||||
continue;
|
||||
}
|
||||
if (length < COMMON_HEADER_BYTES || length % 8 !== 0) {
|
||||
yield err({ kind: "malformed", fileOffset });
|
||||
return;
|
||||
}
|
||||
if (length > bytes.length - fileOffset) {
|
||||
yield err({ kind: "truncated", fileOffset });
|
||||
return;
|
||||
}
|
||||
|
||||
const major = bytes.readUInt16LE(fileOffset + 4);
|
||||
const minor = bytes.readUInt16LE(fileOffset + 6);
|
||||
if (major !== 2 || minor !== 0) {
|
||||
yield err({ kind: "unsupported", fileOffset, major, minor });
|
||||
} else {
|
||||
yield readV2(bytes.subarray(fileOffset, fileOffset + length), fileOffset);
|
||||
}
|
||||
fileOffset += length;
|
||||
}
|
||||
if (fileOffset < bytes.length) yield err({ kind: "truncated", fileOffset });
|
||||
}
|
||||
|
||||
const readV2 = (
|
||||
record: Buffer,
|
||||
fileOffset: number,
|
||||
): Result<UsnRecordV2, UsnRecordIssue> => {
|
||||
if (record.length < V2_HEADER_BYTES)
|
||||
return err({ kind: "malformed", fileOffset });
|
||||
const nameBytes = record.readUInt16LE(56);
|
||||
const nameOffset = record.readUInt16LE(58);
|
||||
if (
|
||||
nameOffset < V2_HEADER_BYTES ||
|
||||
nameOffset % 2 !== 0 ||
|
||||
nameBytes % 2 !== 0 ||
|
||||
nameOffset + nameBytes > record.length
|
||||
)
|
||||
return err({ kind: "malformed", fileOffset });
|
||||
|
||||
return ok({
|
||||
fileOffset,
|
||||
recordBytes: record.length,
|
||||
fileReference: record.readBigUInt64LE(8),
|
||||
parentReference: record.readBigUInt64LE(16),
|
||||
usn: record.readBigInt64LE(24),
|
||||
timestampFileTime: record.readBigUInt64LE(32),
|
||||
reasonFlags: record.readUInt32LE(40),
|
||||
sourceFlags: record.readUInt32LE(44),
|
||||
securityId: record.readUInt32LE(48),
|
||||
fileAttributes: record.readUInt32LE(52),
|
||||
name: record.toString("utf16le", nameOffset, nameOffset + nameBytes),
|
||||
});
|
||||
};
|
||||
@@ -1119,8 +1119,6 @@ describe("Ghidra native instruction decoder context for ARM TMode and MIPS ISA_M
|
||||
});
|
||||
});
|
||||
|
||||
describe("Ghidra bridge script processor context contract", () => {});
|
||||
|
||||
describe("Ghidra dossier nested boundary composition", () => {
|
||||
const identityLocations = [
|
||||
"caller",
|
||||
|
||||
@@ -407,6 +407,30 @@ it("charges long digits to the string budget and keeps large previews bounded",
|
||||
);
|
||||
});
|
||||
|
||||
it("reads marshal one-byte string tags as Latin-1 with Unicode identity", () => {
|
||||
for (const tag of [0x61, 0x41, 0x7a, 0x5a]) {
|
||||
const encoded = Buffer.concat([
|
||||
Buffer.from([tag]),
|
||||
tag === 0x61 || tag === 0x41 ? u32(1) : Buffer.from([1]),
|
||||
Buffer.from([0xe9]),
|
||||
]);
|
||||
const sameUnicode = marshalUnicode(Buffer.from("é", "utf8"));
|
||||
const set = Buffer.concat([
|
||||
Buffer.from([0x3c]),
|
||||
u32(2),
|
||||
encoded,
|
||||
sameUnicode,
|
||||
]);
|
||||
const report = root(parse(withConsts([encoded, set])));
|
||||
expect(report.consts[0]).toMatchObject({
|
||||
kind: "str",
|
||||
value: "é",
|
||||
bytes: 1,
|
||||
});
|
||||
expect(report.consts[1]).toEqual({ kind: "set", length: 1 });
|
||||
}
|
||||
});
|
||||
|
||||
it("decodes marshal strings with CPython surrogatepass rules", () => {
|
||||
const report = parse(
|
||||
withConsts([marshalUnicode(Buffer.from("efbbbf73eda080efbbbf", "hex"))]),
|
||||
@@ -1534,6 +1558,36 @@ it("hashes a long TYPE_REF tuple chain without recursing", () => {
|
||||
expect(pythonPycSchema.safeParse(report).success).toBe(true);
|
||||
});
|
||||
|
||||
it("hashes shallow TYPE_REF code chains without using the JavaScript stack", () => {
|
||||
const links = 3000;
|
||||
const codes = Array.from({ length: links }, (_, index) =>
|
||||
marshalCode("py311", {
|
||||
flagged: true,
|
||||
name: "f",
|
||||
consts: index === 0 ? [Buffer.from([0x4e])] : [marshalRef(index - 1)],
|
||||
}),
|
||||
);
|
||||
const bytes = Buffer.concat([
|
||||
pythonPycHeader(3531),
|
||||
marshalCode("py311", {
|
||||
constsObject: Buffer.concat([
|
||||
Buffer.from([0x28]),
|
||||
u32(links + 1),
|
||||
...codes,
|
||||
Buffer.from([0x3c]),
|
||||
u32(1),
|
||||
marshalRef(links - 1),
|
||||
]),
|
||||
}),
|
||||
]);
|
||||
const report = parseWith(bytes, { max_objects: 65_536 });
|
||||
expect(root(report).consts.at(-1)).toEqual({ kind: "set", length: 1 });
|
||||
expect(report.coverage).toMatchObject({
|
||||
status: "complete",
|
||||
code_objects: links + 1,
|
||||
});
|
||||
});
|
||||
|
||||
it("rejects code CPython treats as corrupt instead of complete coverage", () => {
|
||||
const file = (
|
||||
layout: "py37" | "py38" | "py311",
|
||||
|
||||
@@ -332,7 +332,7 @@ class MarshalReader {
|
||||
readonly #codeKeys = new WeakMap<CodeNode, string>();
|
||||
readonly #codeIdentities = new WeakMap<CodeNode, number>();
|
||||
readonly #codeEquality = new WeakMap<CodeNode, CodeEquality>();
|
||||
/** Python hashability per code object. False while that check is in progress. */
|
||||
/** Completed Python hashability checks per code object. */
|
||||
readonly #codeHashable = new WeakMap<CodeNode, boolean>();
|
||||
/** Tuples whose items are still being read; CPython cannot hash them yet. */
|
||||
readonly #readingTuples = new WeakSet<MarshalValue>();
|
||||
@@ -436,13 +436,19 @@ class MarshalReader {
|
||||
};
|
||||
case TYPE_INTERNED:
|
||||
case TYPE_UNICODE:
|
||||
return this.utf8(this.nonNegativeInt32("Marshal string length"), false);
|
||||
return this.string(
|
||||
this.nonNegativeInt32("Marshal string length"),
|
||||
"utf8",
|
||||
);
|
||||
case TYPE_ASCII:
|
||||
case TYPE_ASCII_INTERNED:
|
||||
return this.utf8(this.nonNegativeInt32("Marshal string length"), true);
|
||||
return this.string(
|
||||
this.nonNegativeInt32("Marshal string length"),
|
||||
"latin1",
|
||||
);
|
||||
case TYPE_SHORT_ASCII:
|
||||
case TYPE_SHORT_ASCII_INTERNED:
|
||||
return this.utf8(this.u8(), true);
|
||||
return this.string(this.u8(), "latin1");
|
||||
case TYPE_TUPLE:
|
||||
return this.sequenceValue(
|
||||
"tuple",
|
||||
@@ -1309,13 +1315,70 @@ class MarshalReader {
|
||||
#isCodeHashable(node: CodeNode): boolean {
|
||||
const known = this.#codeHashable.get(node);
|
||||
if (known !== undefined) return known;
|
||||
this.#codeHashable.set(node, false);
|
||||
const material =
|
||||
this.#codeEquality.get(node) ??
|
||||
malformed("Code object is missing equality material.");
|
||||
const hashable = this.#equalityKey(material.constsTuple, "hash") !== null;
|
||||
this.#codeHashable.set(node, hashable);
|
||||
return hashable;
|
||||
const frames: { value: MarshalValue; finish: boolean }[] = [
|
||||
{ value: { kind: "code", value: node }, finish: false },
|
||||
];
|
||||
const gray = new Set<MarshalValue | CodeNode>();
|
||||
const complete = new Set<MarshalValue>();
|
||||
const reject = (): false => {
|
||||
this.#codeHashable.set(node, false);
|
||||
return false;
|
||||
};
|
||||
while (frames.length > 0) {
|
||||
this.#takeKeyStep();
|
||||
const frame = frames.pop()!;
|
||||
const value = frame.value;
|
||||
const identity = value.kind === "code" ? value.value : value;
|
||||
if (frame.finish) {
|
||||
gray.delete(identity);
|
||||
if (value.kind === "code") {
|
||||
const material =
|
||||
this.#codeEquality.get(value.value) ??
|
||||
malformed("Code object is missing equality material.");
|
||||
// Child code checks are now complete, so the existing tuple key
|
||||
// walker preserves its equality and expansion budgets without recursion.
|
||||
const hashable =
|
||||
this.#equalityKey(material.constsTuple, "hash") !== null;
|
||||
this.#codeHashable.set(value.value, hashable);
|
||||
if (!hashable) return reject();
|
||||
} else complete.add(value);
|
||||
continue;
|
||||
}
|
||||
if (value.kind === "code") {
|
||||
const cached = this.#codeHashable.get(value.value);
|
||||
if (cached === false) return reject();
|
||||
if (cached === true) continue;
|
||||
} else {
|
||||
const cached = this.#keyCache.get(value);
|
||||
if (cached === null) return reject();
|
||||
if (cached !== undefined || complete.has(value)) continue;
|
||||
}
|
||||
if (
|
||||
gray.has(identity) ||
|
||||
this.#readingTuples.has(value) ||
|
||||
value.kind === "list" ||
|
||||
value.kind === "set" ||
|
||||
value.kind === "dict" ||
|
||||
value.kind === "null"
|
||||
)
|
||||
return reject();
|
||||
const children =
|
||||
value.kind === "code"
|
||||
? [
|
||||
(
|
||||
this.#codeEquality.get(value.value) ??
|
||||
malformed("Code object is missing equality material.")
|
||||
).constsTuple,
|
||||
]
|
||||
: value.kind === "tuple" || value.kind === "frozenset"
|
||||
? value.items
|
||||
: [];
|
||||
gray.add(identity);
|
||||
frames.push({ value, finish: true });
|
||||
for (let index = children.length - 1; index >= 0; index -= 1)
|
||||
frames.push({ value: children[index]!, finish: false });
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
#codeIdentity(node: CodeNode): number {
|
||||
@@ -1492,17 +1555,20 @@ class MarshalReader {
|
||||
);
|
||||
}
|
||||
|
||||
utf8(length: number, ascii: boolean): MarshalValue {
|
||||
string(length: number, encoding: "latin1" | "utf8"): MarshalValue {
|
||||
const bytes = this.blob(length);
|
||||
if (ascii) {
|
||||
if (bytes.some((byte) => byte > 0x7f))
|
||||
malformed("Marshal ASCII string contains a non-ASCII byte.");
|
||||
if (encoding === "latin1") {
|
||||
// CPython reads the ASCII-tagged forms as one-byte Unicode, even when
|
||||
// high bytes appear. Identity must agree with the UTF-8 string form.
|
||||
const value = bytes.toString("latin1");
|
||||
return {
|
||||
kind: "str",
|
||||
value: bytes.toString("latin1"),
|
||||
value,
|
||||
previewEnd: Math.min(length, PYTHON_PYC_CONST_PREVIEW_LIMIT),
|
||||
bytes: length,
|
||||
raw: bytes,
|
||||
raw: bytes.some((byte) => byte > 0x7f)
|
||||
? Buffer.from(value, "utf8")
|
||||
: bytes,
|
||||
};
|
||||
}
|
||||
const decoded = decodeMarshalUtf8(bytes, PYTHON_PYC_CONST_PREVIEW_LIMIT);
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
import { registerPlaywrightTraceTool } from "./registerApplicationTools/playwrightTrace.js";
|
||||
import { registerApiContractTool } from "./registerApplicationTools/apiContract.js";
|
||||
import { registerNtfsJournalTool } from "./registerApplicationTools/ntfsJournal.js";
|
||||
import type { EvidenceMcpServer } from "./EvidenceMcpServer.js";
|
||||
|
||||
import { registerTraceFeatureTool } from "./registerApplicationTools/traceFeature.js";
|
||||
@@ -29,7 +28,6 @@ export const registerApplicationTools = (
|
||||
registerCompareApplicationVersionsTool(server, options);
|
||||
registerCompareEndpointsTool(server, options);
|
||||
registerApiContractTool(server, options);
|
||||
registerNtfsJournalTool(server, options);
|
||||
registerPlaywrightTraceTool(server, options);
|
||||
registerCompareSourceToBundleTool(server, options);
|
||||
registerCompareJavaScriptExportShapesTool(server, options);
|
||||
|
||||
@@ -1,54 +0,0 @@
|
||||
import { inspectNtfsJournalEvidence } from "../../application/ntfs/NtfsJournalService.js";
|
||||
import { toolContract } from "../../contracts/toolContracts.js";
|
||||
import type { EvidenceMcpServer } from "../EvidenceMcpServer.js";
|
||||
import { runAdmittedToolOperation } from "../admittedToolOperation.js";
|
||||
import { toolRegistrationOptions } from "../toolRegistrationOptions.js";
|
||||
import { logToolExecution } from "../toolLogging.js";
|
||||
import { deliverApplicationEvidence } from "./helpers.js";
|
||||
import type { ApplicationToolRegistration } from "./types.js";
|
||||
|
||||
const contract = toolContract("inspect_ntfs_journal");
|
||||
|
||||
export const registerNtfsJournalTool = (
|
||||
server: EvidenceMcpServer,
|
||||
options: ApplicationToolRegistration,
|
||||
): void => {
|
||||
server.registerTool(
|
||||
contract.name,
|
||||
toolRegistrationOptions(contract),
|
||||
async (input, context) =>
|
||||
runAdmittedToolOperation(
|
||||
server,
|
||||
options.withAdmittedAnalysis,
|
||||
contract.name,
|
||||
context.mcpReq.signal,
|
||||
async () => {
|
||||
const signal = context.mcpReq.signal;
|
||||
const result = await logToolExecution(
|
||||
options.logger,
|
||||
contract.name,
|
||||
() =>
|
||||
inspectNtfsJournalEvidence(
|
||||
input,
|
||||
options.artifactResources,
|
||||
signal,
|
||||
),
|
||||
);
|
||||
if (!result.ok)
|
||||
return server.delivery.toPartialErrorToolResult(
|
||||
result.error,
|
||||
contract.returnsTargetContent,
|
||||
options.recordEvidence,
|
||||
);
|
||||
return deliverApplicationEvidence(
|
||||
server,
|
||||
options,
|
||||
contract,
|
||||
{ evidence: result.value, sources: [] },
|
||||
undefined,
|
||||
signal,
|
||||
);
|
||||
},
|
||||
),
|
||||
);
|
||||
};
|
||||
@@ -1,113 +0,0 @@
|
||||
import { writeFile } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
|
||||
import { expect } from "vitest";
|
||||
|
||||
import { createBinarySession } from "../../../src/composition/binary.js";
|
||||
import { parseConfig } from "../../../src/config/parseConfig.js";
|
||||
import { parseEvidence } from "../../../src/domain/evidence.js";
|
||||
import { ntfsJournalResultSchema } from "../../../src/domain/windows/ntfsJournalQuery.js";
|
||||
import { silentLogger } from "../../../src/logger.js";
|
||||
import { createServer } from "../../../src/server/createServer.js";
|
||||
import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js";
|
||||
import { mcpTest } from "../../support/mcp/mcpFixture.js";
|
||||
import { parseMcpToolError } from "../../fixtures/mcpToolError.js";
|
||||
import { serveCli } from "../cli/cliServeHarness.js";
|
||||
|
||||
mcpTest(
|
||||
"CLI and MCP inspect the same paired NTFS acquisition",
|
||||
async ({ mcp, onTestFinished }) => {
|
||||
const directory = await createTestTempDirectory("rea-ntfs-journal-");
|
||||
const mftPath = join(directory, "records");
|
||||
const journalPath = join(directory, "journal");
|
||||
const record = Buffer.alloc(1024);
|
||||
record.write("FILE");
|
||||
record.writeUInt16LE(48, 4);
|
||||
record.writeUInt16LE(3, 6);
|
||||
record.writeUInt16LE(1, 48);
|
||||
record.writeUInt16LE(1, 510);
|
||||
record.writeUInt16LE(1, 1022);
|
||||
record.writeUInt16LE(2, 16);
|
||||
record.writeUInt16LE(56, 20);
|
||||
record.writeUInt32LE(64, 24);
|
||||
record.writeUInt32LE(1024, 28);
|
||||
record.writeUInt32LE(0xffff_ffff, 56);
|
||||
const journal = Buffer.alloc(64);
|
||||
journal.writeUInt32LE(64);
|
||||
journal.writeUInt16LE(2, 4);
|
||||
journal.writeBigUInt64LE(0x0002_0000_0000_0000n, 8);
|
||||
journal.writeUInt16LE(60, 58);
|
||||
await writeFile(mftPath, record);
|
||||
await writeFile(journalPath, journal);
|
||||
|
||||
const cli = await serveCli([
|
||||
"inspect-ntfs-journal",
|
||||
mftPath,
|
||||
journalPath,
|
||||
"--json",
|
||||
]);
|
||||
expect(cli.exitCode, cli.stdout).toBe(0);
|
||||
const fromCli = parseEvidence(JSON.parse(cli.stdout));
|
||||
const expected = ntfsJournalResultSchema.parse(fromCli.normalized_result);
|
||||
expect(expected).toMatchObject({
|
||||
mft: { path: mftPath, bytes: 1024 },
|
||||
journal: { path: journalPath, bytes: 64 },
|
||||
rows: [
|
||||
{
|
||||
kind: "event",
|
||||
event: {
|
||||
mft: { file: { kind: "matched", record: { sequenceNumber: 2 } } },
|
||||
},
|
||||
},
|
||||
],
|
||||
nextFileOffset: null,
|
||||
});
|
||||
|
||||
const config = parseConfig({});
|
||||
if (!config.ok) throw config.error;
|
||||
const session = createBinarySession(config.value, silentLogger, {});
|
||||
onTestFinished(async () => {
|
||||
await session.close();
|
||||
});
|
||||
const client = await mcp.connect(
|
||||
createServer({ kind: "session", session }),
|
||||
);
|
||||
const reply = await client.callTool({
|
||||
name: "inspect_ntfs_journal",
|
||||
arguments: { mft_path: mftPath, journal_path: journalPath },
|
||||
});
|
||||
expect(reply.isError, JSON.stringify(reply.content)).not.toBe(true);
|
||||
expect(parseEvidence(reply.structuredContent).normalized_result).toEqual(
|
||||
expected,
|
||||
);
|
||||
|
||||
const refused = await client.callTool({
|
||||
name: "inspect_ntfs_journal",
|
||||
arguments: {
|
||||
mft_path: mftPath,
|
||||
journal_path: join(directory, "missing"),
|
||||
},
|
||||
});
|
||||
expect(refused.isError).toBe(true);
|
||||
const { error } = parseMcpToolError(refused);
|
||||
const reference = error.details?.partial_observation;
|
||||
if (
|
||||
typeof reference !== "object" ||
|
||||
reference === null ||
|
||||
!("evidence_id" in reference) ||
|
||||
typeof reference.evidence_id !== "string"
|
||||
)
|
||||
throw new Error("Missing retained MFT observation");
|
||||
const partial = parseEvidence(session.evidenceById(reference.evidence_id));
|
||||
expect(partial.normalized_result).toMatchObject({
|
||||
kind: "ntfs-journal-partial",
|
||||
mft: {
|
||||
path: mftPath,
|
||||
sha256: expected.mft.sha256,
|
||||
bytes: 1024,
|
||||
metadata: { record_count: 1, record_headers: [{ sequenceNumber: 2 }] },
|
||||
},
|
||||
journal_observation: null,
|
||||
});
|
||||
},
|
||||
);
|
||||
Reference in new issue
Block a user