mirror of
https://github.com/morluto/rea.git
synced 2026-11-10 17:30:32 -10:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
e240782c72 | ||
|
|
2885347cde | ||
|
|
a6a2527ade | ||
|
|
f8fc572e4a | ||
|
|
ab9e5fa204 | ||
|
|
8d45cb45c4 | ||
|
|
7712e3185e | ||
|
|
466149169e | ||
|
|
1c2626edbd | ||
|
|
b16d18f198 | ||
|
|
14030e2f11 | ||
|
|
9505c73824 | ||
|
|
1a446cdade | ||
|
|
1ef7f02cae | ||
|
|
7b86474be7 | ||
|
|
5d10337709 | ||
|
|
8eadb872f2 | ||
|
|
f170175ec8 | ||
|
|
fbe09f84df | ||
|
|
ebb480ba22 | ||
|
|
da1f029c2f | ||
|
|
4db9c166ef | ||
|
|
29fd6a7d03 | ||
|
|
d7338876d6 | ||
|
|
9df3f7174f | ||
|
|
79e476204f | ||
|
|
9c95218374 | ||
|
|
d9d2641981 | ||
|
|
e2585aa7ce | ||
|
|
1d1c7f98ff | ||
|
|
b1f1b3eff3 | ||
|
|
c1d0f2bb45 | ||
|
|
04de5f3200 | ||
|
|
006a16acff |
No files matched your search
@@ -25,7 +25,7 @@ jobs:
|
||||
node-version-file: .nvmrc
|
||||
- name: Validate the SDK pin against the provider's verified release
|
||||
run: node scripts/verify-ghidra-bridge-sources.mjs --check-sdk-pin >> "$GITHUB_ENV"
|
||||
# The bridge probe imports only source-owned ARM/Thumb bytes and never
|
||||
# The bridge probe imports only source-owned ARM, Thumb and MIPS bytes and never
|
||||
# executes target code. All SDK/project state stays in runner storage.
|
||||
- name: Acquire checksum-verified Ghidra SDK in isolated runner storage
|
||||
shell: bash
|
||||
@@ -46,7 +46,7 @@ jobs:
|
||||
java_home_variable="JAVA_HOME_${GHIDRA_JAVA_MAJOR}_X64"
|
||||
test -n "${!java_home_variable}"
|
||||
JAVA_HOME="${!java_home_variable}" npm run verify:ghidra:bridge-sources -- --verified-sdk
|
||||
- name: Inspect real ARM and Thumb instruction context through the bridge
|
||||
- name: Inspect real ARM, Thumb and MIPS instruction context through the bridge
|
||||
shell: bash
|
||||
run: |
|
||||
java_home_variable="JAVA_HOME_${GHIDRA_JAVA_MAJOR}_X64"
|
||||
@@ -58,10 +58,21 @@ jobs:
|
||||
-import "${probe_root}/arm-thumb.bin" -loader BinaryLoader -processor ARM:LE:32:v7 \
|
||||
-noanalysis -readOnly -deleteProject \
|
||||
-scriptPath "${GITHUB_WORKSPACE}/tests/conformance/ghidra;${GITHUB_WORKSPACE}/bridge/ghidra" \
|
||||
-postScript ReaArmContextProbe.java "${GITHUB_WORKSPACE}/bridge/ghidra" \
|
||||
-postScript ReaInstructionContextProbe.java "${GITHUB_WORKSPACE}/bridge/ghidra" \
|
||||
> "${probe_root}/output.log" 2>&1 || { cat "${probe_root}/output.log"; exit 1; }
|
||||
cat "${probe_root}/output.log"
|
||||
grep -F 'REA_ARM_CONTEXT_OK arm=0 thumb=1 unavailable=null (injected provider value)' "${probe_root}/output.log"
|
||||
mkdir -p "${probe_root}/mips-project"
|
||||
# MIPS32 addiu v0,a0,3; MIPS16 li v0,90. These bytes are never executed.
|
||||
printf '\003\000\202\044\132\152\000\000' > "${probe_root}/mips.bin"
|
||||
timeout 180s "${GHIDRA_INSTALL_DIR}/support/analyzeHeadless" "${probe_root}/mips-project" MipsContext \
|
||||
-import "${probe_root}/mips.bin" -loader BinaryLoader -processor MIPS:LE:32:default \
|
||||
-noanalysis -readOnly -deleteProject \
|
||||
-scriptPath "${GITHUB_WORKSPACE}/tests/conformance/ghidra;${GITHUB_WORKSPACE}/bridge/ghidra" \
|
||||
-postScript ReaInstructionContextProbe.java "${GITHUB_WORKSPACE}/bridge/ghidra" mips \
|
||||
> "${probe_root}/mips-output.log" 2>&1 || { cat "${probe_root}/mips-output.log"; exit 1; }
|
||||
cat "${probe_root}/mips-output.log"
|
||||
grep -F 'REA_MIPS_CONTEXT_OK standard=0 mips16=1' "${probe_root}/mips-output.log"
|
||||
# Each probe imports source-owned bytes at 0x1000, away from the Cortex-M
|
||||
# vector symbols that ARMCortex.pspec defines at 0, and decodes them from
|
||||
# the language's default context, so M-profile Thumb state must hold at
|
||||
|
||||
@@ -26,6 +26,7 @@ for new work, roadmap granularity and producer-owned contracts.
|
||||
- [Native call source coordinates](native/call-source-locations/spec.md) ([plan](native/call-source-locations/plan.md), [tasks](native/call-source-locations/tasks.md), [verification](native/call-source-locations/verification.md))
|
||||
- [Go static string-header literal recovery](native/go-string-literals/spec.md) ([plan](native/go-string-literals/plan.md), [tasks](native/go-string-literals/tasks.md))
|
||||
- [Windows Shell Link shortcut inspection](native/windows-shortcut/spec.md) ([plan](native/windows-shortcut/plan.md), [tasks](native/windows-shortcut/tasks.md))
|
||||
- [NTFS $MFT and USN Journal inspection](native/ntfs-mft/spec.md) ([plan](native/ntfs-mft/plan.md), [tasks](native/ntfs-mft/tasks.md))
|
||||
- [Solidity terminal metadata inspection](solidity/cbor-metadata/spec.md) ([plan](solidity/cbor-metadata/plan.md), [tasks](solidity/cbor-metadata/tasks.md), [verification](solidity/cbor-metadata/verification.md))
|
||||
- [Inspect a V8 code cache container](v8/code-cache-inspection/spec.md) ([plan](v8/code-cache-inspection/plan.md), [tasks](v8/code-cache-inspection/tasks.md))
|
||||
- [Deno compile executable inventory and extraction](artifacts/deno-standalone/spec.md) ([plan](artifacts/deno-standalone/plan.md), [tasks](artifacts/deno-standalone/tasks.md))
|
||||
|
||||
@@ -0,0 +1,39 @@
|
||||
# Implementation plan
|
||||
|
||||
## Boundary and contract
|
||||
|
||||
Follow the existing artifact readers. `src/domain/windows/ntfsMft.ts` owns FILE
|
||||
record bytes and coverage; `src/domain/artifactFormatMetadata.ts` owns the
|
||||
`ntfs-mft` result schema. `NtfsMftArtifactReader` reads the admitted file
|
||||
through `ArtifactFileSource` and exposes its facts through `formatMetadata()`.
|
||||
A record stream is its own content, so the root occurrence carries every fact
|
||||
and no members are enumerated.
|
||||
|
||||
Recognition lives in `hasMftFileRecordSignature` and is applied in
|
||||
`classifyArtifactBytes`, next to the other unambiguous magic probes.
|
||||
|
||||
## Incremental work and validation
|
||||
|
||||
1. Define the first-step contract, limits and observations; add parser fixtures.
|
||||
2. Implement the FILE record header and update-sequence fixup parse with
|
||||
explicit limitations.
|
||||
3. Add the reader, format descriptor and byte classification.
|
||||
4. Decode attribute records, then `$FILE_NAME` attributes so a record's parent
|
||||
reference, names and timestamps become explicit; keep hard links separate.
|
||||
5. Parse `$UsnJrnl:$J` records and correlate them to records by full file
|
||||
reference, including sequence numbers, reporting unresolved references.
|
||||
6. Document the workflow in a native Windows artifact guide.
|
||||
|
||||
## Effects and compatibility
|
||||
|
||||
The workflow reads one caller-selected local file. It adds no target execution,
|
||||
injection, mutation, protection recovery, network access, volume mount or live
|
||||
filesystem access, so no maintainer scope decision is required.
|
||||
`inspect_artifact` routing for every existing format is unchanged, and the new
|
||||
format id appears only for bytes whose first record carries the `FILE`
|
||||
signature.
|
||||
|
||||
## References
|
||||
|
||||
- https://learn.microsoft.com/en-us/windows/win32/fileio/master-file-table
|
||||
- https://github.com/libyal/libfsntfs/blob/main/documentation/New%20Technologies%20File%20System%20(NTFS).asciidoc
|
||||
@@ -0,0 +1,51 @@
|
||||
# NTFS $MFT and USN Journal inspection
|
||||
|
||||
This specification records an implementation increment and its original problem
|
||||
statement. For current behavior and limitations, see the artifact inventory
|
||||
guide.
|
||||
|
||||
## Problem and outcome
|
||||
|
||||
REA has no reader for acquired NTFS metadata artifacts. An analyst holding a
|
||||
captured `$MFT` stream cannot ask which FILE records it contains, whether their
|
||||
update-sequence fixups survived acquisition, or which records are directories,
|
||||
extensions of another record, or no longer in use. Add offline, read-only
|
||||
inventory of a selected `$MFT` file, delivered in steps so each one merges on
|
||||
its own.
|
||||
|
||||
## Requirements
|
||||
|
||||
- Recognize a `$MFT` stream by a plausible first `FILE` record header under
|
||||
any file name. CLI and MCP target admission route it to the artifact reader.
|
||||
- Validate each FILE record header: the `FILE` signature, sequence number, flags,
|
||||
used and total byte counts, base record reference, and the update-sequence
|
||||
array's offset, count and array bounds.
|
||||
- Verify the update-sequence fixups: every sector trailer the array covers must
|
||||
hold the array's sequence number. A mismatch keeps the record's header facts
|
||||
and is reported per record and in the metadata count.
|
||||
- Report a missing, misplaced or short update-sequence array as unverified
|
||||
rather than as a clean record, and a trailing partial record as a byte count.
|
||||
- Report the first invalid record signature and the number of invalid slots.
|
||||
Continue at the next slot when the stream declared a supported record size.
|
||||
With a fallback record size, stop at the first invalid signature.
|
||||
- Bound the whole file at a fixed 512 MiB ceiling and report the ceiling as a
|
||||
`limit` failure with its boundary.
|
||||
- Return the first 32 decoded record headers with their source offsets and the
|
||||
number of omitted headers. Aggregate counts cover the full parsed stream.
|
||||
- Parse only the records' headers and fixups. Attributes, filenames, data
|
||||
streams and the `$UsnJrnl:$J` journal are not decoded in this step, so no path,
|
||||
file size, file reference correlation or change history is claimed.
|
||||
|
||||
## Acceptance
|
||||
|
||||
Synthesized fixtures cover a clean record, a failed fixup, a missing update
|
||||
sequence array, an array shorter than the record's sector count, extension and
|
||||
directory records, a record whose signature is wrong mid-stream, a trailing
|
||||
partial record, an undeclared record size, a 4 KiB record, a stream shorter than
|
||||
one header, bytes without the `FILE` signature, and the file-size ceiling.
|
||||
Inventory tests cover the root occurrence's metadata and the coverage status
|
||||
when a fixup fails, and classification tests cover several file names for the
|
||||
same bytes.
|
||||
|
||||
Not covered in this step: real `$MFT` files written by a Windows host, USN
|
||||
journal records, attribute decoding, and any correlation between the two.
|
||||
@@ -0,0 +1,11 @@
|
||||
# Tasks
|
||||
|
||||
- [x] Define the first-step scope, contract and limits.
|
||||
- [x] Parse FILE record headers and update-sequence fixups with synthesized
|
||||
fixtures and explicit limitations.
|
||||
- [x] Add the reader, format descriptor and byte classification.
|
||||
- [ ] Decode attribute records, then `$FILE_NAME` attributes with parent
|
||||
references, names, hard links and timestamps.
|
||||
- [ ] Parse `$UsnJrnl:$J` records and correlate them to records by full file
|
||||
reference.
|
||||
- [ ] Verify against real `$MFT` and journal captures from a Windows host.
|
||||
@@ -636,7 +636,7 @@ const PATH_SCOPES = {
|
||||
"src/ghidra/GhidraJavaProperties.ts",
|
||||
"src/ghidra/GhidraReleasePolicy.ts",
|
||||
"scripts/verify-ghidra-bridge-sources.mjs",
|
||||
"tests/conformance/ghidra/ReaArmContextProbe.java",
|
||||
"tests/conformance/ghidra/ReaInstructionContextProbe.java",
|
||||
"tests/conformance/ghidra/ReaComputedCallCoverageProbe.java",
|
||||
"tests/conformance/ghidra/ReaDecodeProbe.java",
|
||||
".github/workflows/ghidra-bridge-sources.yml",
|
||||
|
||||
@@ -50,7 +50,7 @@ transport.stderr?.on("data", (bytes) => {
|
||||
const ajv = new Ajv2020({ strict: false, validateFormats: false });
|
||||
const measurements = [];
|
||||
const contract = toolContract("inspect_web_network_capture");
|
||||
await withVerifierCleanup(async () => {
|
||||
const report = await withVerifierCleanup(async () => {
|
||||
try {
|
||||
await client.connect(transport);
|
||||
} catch (cause) {
|
||||
@@ -196,7 +196,7 @@ await withVerifierCleanup(async () => {
|
||||
});
|
||||
}
|
||||
await client.ping();
|
||||
const report = {
|
||||
return {
|
||||
status: "passed",
|
||||
profile: "aioquic-1.5.0-json-0.3",
|
||||
measurements,
|
||||
@@ -206,17 +206,6 @@ await withVerifierCleanup(async () => {
|
||||
"No PCAP or keylog is read by this verifier; repeated STREAM coverage is not proof of retransmission or network loss.",
|
||||
],
|
||||
};
|
||||
await writeFile(reportPath, JSON.stringify(report, null, 2) + "\n", {
|
||||
flag: "wx",
|
||||
mode: 0o600,
|
||||
});
|
||||
console.log(
|
||||
JSON.stringify({
|
||||
status: report.status,
|
||||
cases: measurements.length,
|
||||
report: reportPath,
|
||||
}),
|
||||
);
|
||||
}, [
|
||||
async () => {
|
||||
await closeVerifierResources([
|
||||
@@ -225,3 +214,15 @@ await withVerifierCleanup(async () => {
|
||||
]);
|
||||
},
|
||||
]);
|
||||
|
||||
await writeFile(reportPath, JSON.stringify(report, null, 2) + "\n", {
|
||||
flag: "wx",
|
||||
mode: 0o600,
|
||||
});
|
||||
console.log(
|
||||
JSON.stringify({
|
||||
status: report.status,
|
||||
cases: measurements.length,
|
||||
report: reportPath,
|
||||
}),
|
||||
);
|
||||
@@ -97,6 +97,7 @@ import {
|
||||
isDfuSeFile,
|
||||
} from "../domain/firmware/dfuse.js";
|
||||
import { hasFitImagesNode } from "../domain/firmware/fitImage.js";
|
||||
import { hasMftFileRecordHeader } from "../domain/windows/ntfsMft.js";
|
||||
import {
|
||||
locateEspFlashLayout,
|
||||
type EspFlashSelection,
|
||||
@@ -816,6 +817,7 @@ const detectArtifactFormat = async (
|
||||
});
|
||||
if (esp === "esp-image") return "esp-image";
|
||||
if (hasFitImagesNode(magic)) return "fit";
|
||||
if (hasMftFileRecordHeader(magic)) return "ntfs-mft";
|
||||
// Binary plists, including keyed archives, often use other extensions.
|
||||
if (hasBinaryPlistSignature(magic)) return "plist";
|
||||
const named = namedArtifactFormat(lower);
|
||||
@@ -899,6 +901,7 @@ const isArchiveFormat = (
|
||||
"dfuse",
|
||||
"esp-flash-image",
|
||||
"fit",
|
||||
"ntfs-mft",
|
||||
].includes(format);
|
||||
|
||||
/** Whether the leading bytes begin any executable header family, valid or not. */
|
||||
|
||||
@@ -135,6 +135,7 @@ const TARGETS: TargetFixtures = {
|
||||
espFlash: { selected: true, flashOffset: null, partitionTableOffset: null },
|
||||
},
|
||||
fit: { ...archive, format: "fit" },
|
||||
"ntfs-mft": { ...archive, format: "ntfs-mft" },
|
||||
plist: { ...artifact, format: "plist" },
|
||||
javascript: { ...artifact, format: "javascript" },
|
||||
"source-map": { ...artifact, format: "source-map" },
|
||||
|
||||
@@ -25,6 +25,7 @@ import {
|
||||
import { pageBound } from "../../domain/analysisView/environmentStatePaging.js";
|
||||
import { replayReason } from "../../domain/analysisView/environmentStateVariants.js";
|
||||
import { isAbsence } from "../../domain/analysisView/environmentStateWriteChecks.js";
|
||||
import { updatedEntity } from "../../domain/analysisView/environmentStateWriteApply.js";
|
||||
import {
|
||||
evaluateQuery,
|
||||
renderQueryList,
|
||||
@@ -291,16 +292,6 @@ export const softDeleteFlag = (
|
||||
return undefined;
|
||||
};
|
||||
|
||||
/** Fields as an application that soft-deletes shows them once deleted: with the type's deleted flag set, when it has one. */
|
||||
const deletedFields = (
|
||||
serving: StateServing,
|
||||
type: string,
|
||||
fields: JsonObject,
|
||||
): JsonObject => {
|
||||
const flag = softDeleteFlag(serving, type);
|
||||
return flag === undefined ? fields : { ...fields, [flag]: true };
|
||||
};
|
||||
|
||||
/** Fields as a delete's answer shows them: with the type's deleted flag as its recorded answer `shown` gave it, if it did. */
|
||||
export const deleteAnswerFields = (
|
||||
serving: StateServing,
|
||||
@@ -326,9 +317,11 @@ const deletedEntity = (
|
||||
call,
|
||||
(collection, id) => store.deleted(type, collection, id),
|
||||
);
|
||||
return entity === undefined
|
||||
? undefined
|
||||
: { ...entity, fields: deletedFields(serving, type, entity.fields) };
|
||||
if (entity === undefined) return undefined;
|
||||
const flag = softDeleteFlag(serving, type);
|
||||
return flag === undefined
|
||||
? entity
|
||||
: updatedEntity(entity, { ...entity.fields, [flag]: true });
|
||||
};
|
||||
|
||||
/** The answer for an entity the state does not hold: the tool's latest recorded 400, 404 or 410, else a tool error. */
|
||||
|
||||
@@ -33,6 +33,44 @@ const objectOf = (body: JsonValue | undefined): Record<string, JsonValue> => {
|
||||
return body;
|
||||
};
|
||||
|
||||
describe("served path-derived fields", () => {
|
||||
it.each([false, true])(
|
||||
"preserves repeated parameter occurrences (ambiguous: %s)",
|
||||
(ambiguous) => {
|
||||
const project = ambiguous ? 1 : 3;
|
||||
const path = `/api/orgs/1/projects/${project}/tasks`;
|
||||
const first = { id: 1, name: "Old", orgId: 1, projectId: project };
|
||||
const made = { id: 2, name: "Recorded", orgId: 1, projectId: project };
|
||||
const tasks = servedFrom([
|
||||
["GET", path, [first]],
|
||||
["GET", `${path}/1`, first],
|
||||
["GET", "/api/orgs/2/projects/4/tasks", []],
|
||||
["POST", path, made, 201, { name: "Recorded" }],
|
||||
["GET", `${path}/2`, made],
|
||||
["GET", path, [first, made]],
|
||||
]);
|
||||
const tool = "post_orgs_projects_tasks_by_id_id_2";
|
||||
const args = { id: "2", id_2: "4", body: { name: "Elsewhere" } };
|
||||
if (ambiguous) {
|
||||
// Equal recorded values establish no choice between the path positions.
|
||||
const before = tasks.snapshot();
|
||||
expect(tasks.call(tool, args).ok).toBe(false);
|
||||
expect(tasks.snapshot()).toBe(before);
|
||||
expect(
|
||||
tasks.answer(tool, { id: "1", id_2: "1", body: { name: "Here" } })
|
||||
.body,
|
||||
).toMatchObject({ name: "Here", orgId: 1, projectId: 1 });
|
||||
} else {
|
||||
expect(tasks.answer(tool, args).body).toMatchObject({
|
||||
name: "Elsewhere",
|
||||
orgId: 2,
|
||||
projectId: 4,
|
||||
});
|
||||
}
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
describe("served values an application derives from others", () => {
|
||||
it("stamps a created tag's clocks with trailing fraction zeros dropped, as the capture spelled them", () => {
|
||||
const tags = servedFrom(TAG_ROWS);
|
||||
|
||||
@@ -181,3 +181,47 @@ describe("served writes modelled on later reads", () => {
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
it.each([
|
||||
["item", SOFT_DELETED_ROWS, "delete_contacts_by_id", { id: "1" }],
|
||||
["bulk", STALE_BULK_DELETE_ROWS, "delete_contacts", { body: [{ id: 1 }] }],
|
||||
] as const)(
|
||||
"invalidates the deletion flag override after a served %s delete",
|
||||
(_kind, rows, tool, args) => {
|
||||
const contacts = servedFrom(rows);
|
||||
const snapshot = contacts.store.snapshot();
|
||||
expect(
|
||||
contacts.store.replace({
|
||||
rea_state: {
|
||||
...snapshot.rea_state,
|
||||
entities: snapshot.rea_state.entities.map((entity) =>
|
||||
entity.id === "1"
|
||||
? {
|
||||
...entity,
|
||||
view_fields: [
|
||||
{
|
||||
tool: "get_contacts_by_id",
|
||||
fields: { deleted: false, name: "Item display" },
|
||||
},
|
||||
],
|
||||
}
|
||||
: entity,
|
||||
),
|
||||
},
|
||||
}).ok,
|
||||
).toBe(true);
|
||||
expect(
|
||||
contacts.answer("get_contacts_by_id", { id: "1" }).body,
|
||||
).toMatchObject({
|
||||
deleted: false,
|
||||
});
|
||||
contacts.answer(tool, args);
|
||||
expect(
|
||||
contacts.answer("get_contacts_by_id", { id: "1" }).body,
|
||||
).toMatchObject({
|
||||
id: 1,
|
||||
name: "Item display",
|
||||
deleted: true,
|
||||
});
|
||||
},
|
||||
);
|
||||
@@ -32,6 +32,7 @@ import {
|
||||
} from "../../domain/analysisView/environmentStateWriteLinks.js";
|
||||
import {
|
||||
pathValuesOf,
|
||||
pathParameterKeys,
|
||||
type GeneratedRule,
|
||||
type StateWritePlan,
|
||||
type WriteCall,
|
||||
@@ -106,14 +107,12 @@ export const pathValues = (
|
||||
args: ReplayArguments,
|
||||
): Map<string, string> => {
|
||||
const names = environmentToolArguments(tool).path;
|
||||
const template = tool.path_template
|
||||
.slice(tool.path_template.indexOf("/") + 1)
|
||||
.split("/");
|
||||
const keys = pathParameterKeys(tool.path_template);
|
||||
const values = new Map<string, string>();
|
||||
for (const param of tool.params) {
|
||||
const value = ownMember(args, names.get(param.name) ?? param.name);
|
||||
const segment = template[param.index] ?? "";
|
||||
if (typeof value === "string") values.set(segment.slice(1, -1), value);
|
||||
const key = keys.get(param.index);
|
||||
if (typeof value === "string" && key !== undefined) values.set(key, value);
|
||||
}
|
||||
return values;
|
||||
};
|
||||
|
||||
@@ -11,6 +11,8 @@ import {
|
||||
type Book,
|
||||
} from "../../../tests/fixtures/environmentStateWrites.js";
|
||||
import { servedCatalog } from "./environmentState.js";
|
||||
import { environmentToolAnswer } from "./environmentToolAnswer.js";
|
||||
import { projectAnalysisError } from "../../domain/analysisErrorProjection.js";
|
||||
|
||||
const RENAMED = renamedBook(FIRST_BOOK, "Renamed Book", 3);
|
||||
|
||||
@@ -72,6 +74,33 @@ const toolOf = (rows: readonly TimedCaptureRow[], name: string) => {
|
||||
};
|
||||
|
||||
describe("served write queries", () => {
|
||||
it.each([
|
||||
"?notify=quiet&unmodeled=yes",
|
||||
"?notify=quiet¬ify=quiet",
|
||||
"?notify=quiet&unmodeled=yes&unmodeled=no",
|
||||
])("refuses unsupported observed query %s before changing state", (query) => {
|
||||
const { served } = toolOf(PINNED_ROWS, "patch_books_by_id");
|
||||
const tool = served.source.manifest.tools.find(
|
||||
(candidate) => candidate.name === "patch_books_by_id",
|
||||
);
|
||||
if (tool === undefined) throw new Error("Expected patch tool");
|
||||
const before = served.snapshot();
|
||||
const pending = served.store.pendingWrite();
|
||||
const respond = environmentToolAnswer(served.source, served.store);
|
||||
const result = respond(
|
||||
tool,
|
||||
{ id: "2", notify: "quiet", body: { name: "Changed" } },
|
||||
query,
|
||||
);
|
||||
expect(result.ok).toBe(false);
|
||||
if (result.ok) throw new Error("Expected pinned-query refusal");
|
||||
expect(projectAnalysisError(result.error).message).toMatch(
|
||||
/query|repeated/u,
|
||||
);
|
||||
expect(served.snapshot()).toBe(before);
|
||||
expect(served.store.pendingWrite()).toEqual(pending);
|
||||
});
|
||||
|
||||
it("advertises the recorded query as optional so omission reaches its default", () => {
|
||||
const { served, tool } = toolOf(PINNED_ROWS, "patch_books_by_id");
|
||||
expect(tool.inputSchema.properties["notify"]).toMatchObject({
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import { pinnedQuery } from "../../domain/analysisView/environmentStateWriteStamps.js";
|
||||
import type { AnalysisError } from "../../domain/analysisErrorBase.js";
|
||||
import { iterateUrlQueryParameters } from "../../domain/recordedTransportUrl.js";
|
||||
import { err, ok, type Result } from "../../domain/result.js";
|
||||
import { refused, type StateServing } from "./environmentStateAnswer.js";
|
||||
import { callQuery } from "./environmentStateCallLines.js";
|
||||
@@ -15,7 +16,16 @@ export const pinnedRequest = (
|
||||
request: WriteRequest,
|
||||
): Result<WriteRequest, AnalysisError> => {
|
||||
const { tool, plan, args } = request;
|
||||
if (hasRepeatedQueryArgument(tool, args))
|
||||
const supplied = callQuery(tool, args);
|
||||
const observed = new Map<string, string>();
|
||||
let repeated = hasRepeatedQueryArgument(tool, args);
|
||||
for (const [name, value] of iterateUrlQueryParameters(
|
||||
request.observedQuery ?? "",
|
||||
)) {
|
||||
if (observed.has(name)) repeated = true;
|
||||
observed.set(name, value);
|
||||
}
|
||||
if (repeated)
|
||||
return err(
|
||||
refused(
|
||||
tool,
|
||||
@@ -24,7 +34,9 @@ export const pinnedRequest = (
|
||||
);
|
||||
const query = pinnedQuery(
|
||||
serving.source.exchanges.get(plan.exemplar_ordinal ?? -1)?.query,
|
||||
callQuery(tool, args),
|
||||
// Comparison supplies the raw query as well as translated known arguments.
|
||||
// Keep unmodeled parameters visible without replacing those translations.
|
||||
new Map([...observed, ...supplied]),
|
||||
);
|
||||
if (!query.ok) return err(refused(tool, `Tool ${tool.name} ${query.error}`));
|
||||
const names = environmentToolArguments(tool).query;
|
||||
|
||||
@@ -215,6 +215,7 @@ export interface ArtifactReader {
|
||||
| "dfuse"
|
||||
| "esp-flash-image"
|
||||
| "fit"
|
||||
| "ntfs-mft"
|
||||
| "file";
|
||||
entries(
|
||||
signal?: AbortSignal,
|
||||
|
||||
@@ -18,7 +18,10 @@ import {
|
||||
type ArtifactTraversalOptions,
|
||||
} from "./ArtifactReader.js";
|
||||
import { ArtifactResourceScope } from "./ArtifactResourceScope.js";
|
||||
import { assertArtifactReadActive } from "./ArtifactFileOperation.js";
|
||||
import {
|
||||
artifactReadCancelledBy,
|
||||
assertArtifactReadActive,
|
||||
} from "./ArtifactFileOperation.js";
|
||||
import { ArtifactFileSource } from "./ArtifactFileSource.js";
|
||||
import { readFileHandleChunks } from "../filesystem/readFileHandleChunks.js";
|
||||
import type { StableRegularFileDescriptor } from "../filesystem/RegularFile.js";
|
||||
@@ -64,12 +67,14 @@ export const regionFileName = (start: number, end: number): string =>
|
||||
*/
|
||||
export class DecodedRegionArtifactReader<Image> implements ArtifactReader {
|
||||
readonly #scope = new ArtifactResourceScope();
|
||||
readonly #entries = new Map<string, ArtifactEntry>();
|
||||
readonly #regions = new Map<string, Uint8Array>();
|
||||
#entries = new Map<string, ArtifactEntry>();
|
||||
#regions = new Map<string, Uint8Array>();
|
||||
readonly #streams = new Set<Readable>();
|
||||
readonly #source: ArtifactFileSource;
|
||||
readonly #decoding: RegionDecoding<Image>;
|
||||
#prepared: Promise<void> | undefined;
|
||||
#prepared:
|
||||
| { signal: AbortSignal | undefined; promise: Promise<void> }
|
||||
| undefined;
|
||||
/** Whole-file facts only; the region map is the sole owner of region bytes. */
|
||||
#described: Omit<DecodedRegions, "regions"> | undefined;
|
||||
#closing = false;
|
||||
@@ -110,8 +115,27 @@ export class DecodedRegionArtifactReader<Image> implements ArtifactReader {
|
||||
): AsyncIterable<ArtifactEntry> {
|
||||
await this.#scope.run(async () => {
|
||||
this.#assertOpen(signal);
|
||||
this.#prepared ??= this.#prepare(signal);
|
||||
await this.#prepared;
|
||||
for (;;) {
|
||||
const pending = (this.#prepared ??= {
|
||||
signal,
|
||||
promise: this.#prepare(signal),
|
||||
});
|
||||
try {
|
||||
await pending.promise;
|
||||
break;
|
||||
} catch (cause: unknown) {
|
||||
if (this.#prepared === pending) this.#prepared = undefined;
|
||||
if (
|
||||
pending.signal !== signal &&
|
||||
signal?.aborted !== true &&
|
||||
artifactReadCancelledBy(cause, pending.signal)
|
||||
) {
|
||||
this.#assertOpen(signal);
|
||||
continue;
|
||||
}
|
||||
throw cause;
|
||||
}
|
||||
}
|
||||
await this.#source.read(signal);
|
||||
});
|
||||
const coverage = this.#described?.coverage;
|
||||
@@ -230,6 +254,8 @@ export class DecodedRegionArtifactReader<Image> implements ArtifactReader {
|
||||
|
||||
async #prepare(signal?: AbortSignal): Promise<void> {
|
||||
const described = this.#decoding.describe(await this.#decode(signal));
|
||||
const entries = new Map<string, ArtifactEntry>();
|
||||
const regions = new Map<string, Uint8Array>();
|
||||
let index = 0;
|
||||
for (const region of described.regions) {
|
||||
if (index % 256 === 0) await setImmediate();
|
||||
@@ -248,13 +274,16 @@ export class DecodedRegionArtifactReader<Image> implements ArtifactReader {
|
||||
adapterKey: `${this.#decoding.keyPrefix}:${String(index)}`,
|
||||
formatMetadata: region.formatMetadata,
|
||||
} satisfies ArtifactEntry;
|
||||
this.#entries.set(entry.adapterKey, entry);
|
||||
this.#regions.set(entry.adapterKey, region.bytes);
|
||||
entries.set(entry.adapterKey, entry);
|
||||
regions.set(entry.adapterKey, region.bytes);
|
||||
index += 1;
|
||||
}
|
||||
const { regions: _regions, ...facts } = described;
|
||||
this.#described = facts;
|
||||
await this.#source.read(signal);
|
||||
this.#assertOpen(signal);
|
||||
this.#entries = entries;
|
||||
this.#regions = regions;
|
||||
this.#described = facts;
|
||||
}
|
||||
|
||||
async #decode(signal?: AbortSignal): Promise<Image> {
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { artifactPreparationRecoveryCases } from "../../tests/fixtures/artifactReaderLifecycle.js";
|
||||
import { getEventListeners, once } from "node:events";
|
||||
import { readFile, truncate, writeFile } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
@@ -404,6 +405,38 @@ it("keeps the cancellation reason before admission", async () => {
|
||||
}
|
||||
});
|
||||
|
||||
it("does not publish FIT entries when final source verification fails", async () => {
|
||||
const path = await writeFit("partial-publication.itb", FIT_EMBEDDED);
|
||||
const control = new FitArtifactReader(path);
|
||||
const reader = new FitArtifactReader(path);
|
||||
const expected = await firstEntry(control);
|
||||
const nativeRead = ArtifactFileSource.prototype.read;
|
||||
const physical = Object.assign(
|
||||
new Error("final source verification failed"),
|
||||
{ code: "EIO" },
|
||||
);
|
||||
let calls = 0;
|
||||
const read = vi
|
||||
.spyOn(ArtifactFileSource.prototype, "read")
|
||||
.mockImplementation(function (this: ArtifactFileSource, ...args) {
|
||||
calls += 1;
|
||||
if (calls === 4) return Promise.reject(physical);
|
||||
return nativeRead.apply(this, args);
|
||||
});
|
||||
try {
|
||||
await expect(firstEntry(reader)).rejects.toBe(physical);
|
||||
await expect(reader.open(expected)).rejects.toMatchObject({
|
||||
reason: "integrity",
|
||||
});
|
||||
read.mockRestore();
|
||||
expect((await firstEntry(reader)).path).toBe(expected.path);
|
||||
} finally {
|
||||
read.mockRestore();
|
||||
await reader.close();
|
||||
await control.close();
|
||||
}
|
||||
});
|
||||
|
||||
it("keeps the cancellation reason during hash verification", async () => {
|
||||
const reader = new FitArtifactReader(
|
||||
await writeFit("hashing.itb", LARGE_FIT),
|
||||
@@ -430,6 +463,8 @@ it("keeps the cancellation reason during hash verification", async () => {
|
||||
reader.entries(controller.signal)[Symbol.asyncIterator]().next(),
|
||||
).rejects.toMatchObject({ reason: "cancelled", cause: reason });
|
||||
expect(observer).toHaveBeenCalledWith("sha256");
|
||||
observer.mockRestore();
|
||||
expect((await firstEntry(reader)).kind).toBe("file");
|
||||
} finally {
|
||||
observer.mockRestore();
|
||||
await reader.close();
|
||||
@@ -569,3 +604,8 @@ it("preserves a final validation failure when cancellation overlaps it", async (
|
||||
await reader.close();
|
||||
}
|
||||
});
|
||||
|
||||
artifactPreparationRecoveryCases({
|
||||
image: () => FIT_EMBEDDED,
|
||||
createReader: (path, admitted) => new FitArtifactReader(path, admitted),
|
||||
});
|
||||
@@ -13,7 +13,10 @@ import {
|
||||
type ArtifactTraversalOptions,
|
||||
} from "./ArtifactReader.js";
|
||||
import { ArtifactResourceScope } from "./ArtifactResourceScope.js";
|
||||
import { assertArtifactReadActive } from "./ArtifactFileOperation.js";
|
||||
import {
|
||||
artifactReadCancelledBy,
|
||||
assertArtifactReadActive,
|
||||
} from "./ArtifactFileOperation.js";
|
||||
import { ArtifactFileSource } from "./ArtifactFileSource.js";
|
||||
import {
|
||||
createFitDigest,
|
||||
@@ -30,11 +33,13 @@ const FDT_HEADER_BYTES = 40;
|
||||
export class FitArtifactReader implements ArtifactReader {
|
||||
readonly format = "fit" as const;
|
||||
readonly #scope = new ArtifactResourceScope();
|
||||
readonly #entries = new Map<string, ArtifactEntry>();
|
||||
#entries = new Map<string, ArtifactEntry>();
|
||||
readonly #streams = new Set<Readable>();
|
||||
readonly #path: string;
|
||||
readonly #source: ArtifactFileSource;
|
||||
#prepared: Promise<void> | undefined;
|
||||
#prepared:
|
||||
| { signal: AbortSignal | undefined; promise: Promise<void> }
|
||||
| undefined;
|
||||
#metadata: FitMetadata | undefined;
|
||||
#coverage: ArtifactReaderTraversal | undefined;
|
||||
#closing = false;
|
||||
@@ -66,8 +71,27 @@ export class FitArtifactReader implements ArtifactReader {
|
||||
): AsyncIterable<ArtifactEntry> {
|
||||
await this.#scope.run(async () => {
|
||||
this.#assertOpen(signal);
|
||||
this.#prepared ??= this.#prepare(signal);
|
||||
await this.#prepared;
|
||||
for (;;) {
|
||||
const pending = (this.#prepared ??= {
|
||||
signal,
|
||||
promise: this.#prepare(signal),
|
||||
});
|
||||
try {
|
||||
await pending.promise;
|
||||
break;
|
||||
} catch (cause: unknown) {
|
||||
if (this.#prepared === pending) this.#prepared = undefined;
|
||||
if (
|
||||
pending.signal !== signal &&
|
||||
signal?.aborted !== true &&
|
||||
artifactReadCancelledBy(cause, pending.signal)
|
||||
) {
|
||||
this.#assertOpen(signal);
|
||||
continue;
|
||||
}
|
||||
throw cause;
|
||||
}
|
||||
}
|
||||
await this.#source.read(signal);
|
||||
});
|
||||
if (this.#coverage !== undefined)
|
||||
@@ -207,6 +231,7 @@ export class FitArtifactReader implements ArtifactReader {
|
||||
this.#path,
|
||||
initial.size,
|
||||
);
|
||||
const entries = new Map<string, ArtifactEntry>();
|
||||
let index = 0;
|
||||
for (const image of structure.images) {
|
||||
if (index % 256 === 0) await setImmediate();
|
||||
@@ -228,15 +253,17 @@ export class FitArtifactReader implements ArtifactReader {
|
||||
adapterKey: `fit:${String(image.index)}`,
|
||||
formatMetadata: image.metadata,
|
||||
} satisfies ArtifactEntry;
|
||||
this.#entries.set(entry.adapterKey, entry);
|
||||
entries.set(entry.adapterKey, entry);
|
||||
}
|
||||
await this.#source.read(signal);
|
||||
this.#assertOpen(signal);
|
||||
this.#entries = entries;
|
||||
this.#metadata = structure.metadata;
|
||||
if (structure.limitations.length > 0)
|
||||
this.#coverage = {
|
||||
status: "complete",
|
||||
limitations: structure.limitations,
|
||||
};
|
||||
await this.#source.read(signal);
|
||||
}
|
||||
|
||||
/** One streamed pass per image feeds every supported digest. */
|
||||
|
||||
@@ -1,3 +1,6 @@
|
||||
import { artifactPreparationRecoveryCases } from "../../tests/fixtures/artifactReaderLifecycle.js";
|
||||
import { ArtifactFileSource } from "./ArtifactFileSource.js";
|
||||
import { openRegularFile } from "../filesystem/RegularFile.js";
|
||||
import { writeFile } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
|
||||
@@ -159,6 +162,72 @@ it("settles an active region stream when closed", async () => {
|
||||
}
|
||||
});
|
||||
|
||||
it("retries decoded-region preparation cancelled while reading", async () => {
|
||||
const path = await writeImage("retry.hex", INTEL_HEX_I32_GAP);
|
||||
const admitted = await openRegularFile(path, { symlinks: "reject" });
|
||||
const reader = new HexRecordArtifactReader(path, "intel-hex", admitted);
|
||||
const controller = new AbortController();
|
||||
const reason = { stage: "decoded region read" };
|
||||
const nativeRead = admitted.handle.read.bind(admitted.handle);
|
||||
const read = vi.spyOn(admitted.handle, "read").mockImplementation((async (
|
||||
...args: unknown[]
|
||||
) => {
|
||||
const result = await Reflect.apply(nativeRead, admitted.handle, args);
|
||||
controller.abort(reason);
|
||||
return result;
|
||||
}) as typeof admitted.handle.read);
|
||||
try {
|
||||
await expect(
|
||||
reader.entries(controller.signal)[Symbol.asyncIterator]().next(),
|
||||
).rejects.toMatchObject({ reason: "cancelled", cause: reason });
|
||||
read.mockRestore();
|
||||
expect(
|
||||
(await reader.entries()[Symbol.asyncIterator]().next()).value,
|
||||
).toMatchObject({ kind: "file" });
|
||||
} finally {
|
||||
read.mockRestore();
|
||||
await reader.close();
|
||||
await admitted.handle.close();
|
||||
}
|
||||
});
|
||||
|
||||
it("does not publish decoded regions when final source verification fails", async () => {
|
||||
const path = await writeImage("partial.hex", INTEL_HEX_I32_GAP);
|
||||
const control = new HexRecordArtifactReader(path, "intel-hex");
|
||||
const reader = new HexRecordArtifactReader(path, "intel-hex");
|
||||
const expected = (await control.entries()[Symbol.asyncIterator]().next())
|
||||
.value!;
|
||||
const nativeRead = ArtifactFileSource.prototype.read;
|
||||
const physical = Object.assign(
|
||||
new Error("final source verification failed"),
|
||||
{ code: "EIO" },
|
||||
);
|
||||
let calls = 0;
|
||||
const read = vi
|
||||
.spyOn(ArtifactFileSource.prototype, "read")
|
||||
.mockImplementation(function (this: ArtifactFileSource, ...args) {
|
||||
calls += 1;
|
||||
if (calls === 2) return Promise.reject(physical);
|
||||
return nativeRead.apply(this, args);
|
||||
});
|
||||
try {
|
||||
await expect(reader.entries()[Symbol.asyncIterator]().next()).rejects.toBe(
|
||||
physical,
|
||||
);
|
||||
await expect(reader.open(expected)).rejects.toMatchObject({
|
||||
reason: "integrity",
|
||||
});
|
||||
read.mockRestore();
|
||||
expect(
|
||||
(await reader.entries()[Symbol.asyncIterator]().next()).value,
|
||||
).toMatchObject({ path: expected.path });
|
||||
} finally {
|
||||
read.mockRestore();
|
||||
await reader.close();
|
||||
await control.close();
|
||||
}
|
||||
});
|
||||
|
||||
it("honors cancellation before admitting a file with the exact reason", async () => {
|
||||
const reader = new HexRecordArtifactReader(
|
||||
await writeImage("cancel.hex", INTEL_HEX_I32_GAP),
|
||||
@@ -337,3 +406,9 @@ it("bounds copied region chunks and isolates cached bytes across opens", async (
|
||||
await reader.close();
|
||||
}
|
||||
});
|
||||
|
||||
artifactPreparationRecoveryCases({
|
||||
image: () => Buffer.from(INTEL_HEX_I32_GAP),
|
||||
createReader: (path, admitted) =>
|
||||
new HexRecordArtifactReader(path, "intel-hex", admitted),
|
||||
});
|
||||
@@ -0,0 +1,211 @@
|
||||
import { writeFile } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
|
||||
import { expect, it, vi } from "vitest";
|
||||
|
||||
import { ArtifactFileSource } from "./ArtifactFileSource.js";
|
||||
import { assertArtifactReadActive } from "./ArtifactFileOperation.js";
|
||||
import { NtfsMftArtifactReader } from "./NtfsMftArtifactReader.js";
|
||||
|
||||
import { MFT_DEFAULT_RECORD_BYTES } from "../domain/windows/ntfsMft.js";
|
||||
import { createTestTempDirectory } from "../../tests/fixtures/temporaryDirectory.js";
|
||||
import { scanCanonicalArtifactInventory } from "../../tests/fixtures/artifactInventory.js";
|
||||
|
||||
/** A 1 KiB FILE record whose update sequence array covers both sectors. */
|
||||
const recordBytes = (options: {
|
||||
readonly sequenceNumber?: number;
|
||||
readonly flags?: number;
|
||||
readonly index?: number;
|
||||
}): Buffer => {
|
||||
const record = Buffer.alloc(MFT_DEFAULT_RECORD_BYTES);
|
||||
record.write("FILE", 0, "latin1");
|
||||
const sequenceNumber = options.sequenceNumber ?? 1;
|
||||
record.writeUInt16LE(48, 4);
|
||||
record.writeUInt16LE(3, 6);
|
||||
record.writeUInt16LE(sequenceNumber, 48);
|
||||
record.writeUInt16LE(0xa5a5, 50);
|
||||
record.writeUInt16LE(0xa5a5, 52);
|
||||
record.writeUInt16LE(sequenceNumber, 16);
|
||||
record.writeUInt16LE(1, 18);
|
||||
record.writeUInt32LE(options.index ?? 0, 44);
|
||||
record.writeUInt16LE(56, 20);
|
||||
record.writeUInt16LE(options.flags ?? 0x0001, 22);
|
||||
record.writeUInt32LE(64, 24);
|
||||
record.writeUInt32LE(0xffff_ffff, 56);
|
||||
record.writeUInt32LE(MFT_DEFAULT_RECORD_BYTES, 28);
|
||||
record.writeUInt16LE(sequenceNumber, 510);
|
||||
record.writeUInt16LE(sequenceNumber, 1022);
|
||||
return record;
|
||||
};
|
||||
|
||||
const writeMft = async (name: string, bytes: Buffer): Promise<string> => {
|
||||
const directory = await createTestTempDirectory("rea-ntfs-mft-");
|
||||
const path = join(directory, name);
|
||||
await writeFile(path, bytes);
|
||||
return path;
|
||||
};
|
||||
|
||||
it("reports a $MFT stream's record headers as the root occurrence", async () => {
|
||||
const inventory = await scanCanonicalArtifactInventory(
|
||||
await writeMft(
|
||||
"$MFT",
|
||||
Buffer.concat([
|
||||
recordBytes({}),
|
||||
recordBytes({ flags: 0x0003, index: 1 }),
|
||||
]),
|
||||
),
|
||||
);
|
||||
expect(inventory.manifest.root_format).toBe("ntfs-mft");
|
||||
expect(inventory.coverage.status).toBe("complete");
|
||||
// A record stream declares no members, so the root is its whole inventory.
|
||||
expect(inventory.occurrences.map((row) => row.logical_path)).toEqual(["."]);
|
||||
expect(inventory.occurrences[0]).toMatchObject({
|
||||
artifact_format: "ntfs-mft",
|
||||
format_metadata: {
|
||||
kind: "ntfs-mft",
|
||||
record_bytes: MFT_DEFAULT_RECORD_BYTES,
|
||||
record_count: 2,
|
||||
in_use_record_count: 2,
|
||||
directory_record_count: 1,
|
||||
extension_record_count: 0,
|
||||
fixup_mismatch_count: 0,
|
||||
fixup_unverified_record_count: 0,
|
||||
truncated_tail_bytes: 0,
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it("names a failed fixup in the coverage limitations", async () => {
|
||||
const record = recordBytes({ sequenceNumber: 4 });
|
||||
// A trailer that is not the sequence number marks the sector as not cleanly written.
|
||||
record.writeUInt16LE(0x1234, 510);
|
||||
const inventory = await scanCanonicalArtifactInventory(
|
||||
await writeMft("$MFT", record),
|
||||
);
|
||||
expect(inventory.manifest.root_format).toBe("ntfs-mft");
|
||||
expect(inventory.coverage.status).toBe("unknown");
|
||||
expect(inventory.limitations).toContainEqual(
|
||||
expect.stringMatching(/sector trailer that is not the update sequence/u),
|
||||
);
|
||||
});
|
||||
|
||||
it("reports an unaligned used-byte count as incomplete coverage", async () => {
|
||||
const record = recordBytes({});
|
||||
record.writeUInt32LE(57, 24);
|
||||
const inventory = await scanCanonicalArtifactInventory(
|
||||
await writeMft("$MFT", record),
|
||||
);
|
||||
expect(inventory.coverage.status).toBe("unknown");
|
||||
expect(inventory.occurrences[0]?.format_metadata).toMatchObject({
|
||||
record_headers: [{ usedBytes: 57, extentValid: false }],
|
||||
});
|
||||
});
|
||||
|
||||
it("reports incomplete coverage when FILE extents contradict the record slots", async () => {
|
||||
const usedTooLarge = recordBytes({});
|
||||
usedTooLarge.writeUInt32LE(4096, 24);
|
||||
const totalTooSmall = recordBytes({});
|
||||
totalTooSmall.writeUInt32LE(512, 28);
|
||||
|
||||
const inventory = await scanCanonicalArtifactInventory(
|
||||
await writeMft("$MFT", Buffer.concat([usedTooLarge, totalTooSmall])),
|
||||
);
|
||||
expect(inventory.coverage.status).toBe("unknown");
|
||||
expect(inventory.limitations).toContainEqual(
|
||||
"2 FILE record(s) declare inconsistent record extents, beginning at offset 0",
|
||||
);
|
||||
expect(inventory.occurrences[0]?.format_metadata).toMatchObject({
|
||||
record_count: 2,
|
||||
fixup_mismatch_count: 0,
|
||||
fixup_unverified_record_count: 0,
|
||||
});
|
||||
});
|
||||
|
||||
it("retries after final source verification fails without publishing its metadata", async () => {
|
||||
const reader = new NtfsMftArtifactReader(
|
||||
await writeMft("$MFT", recordBytes({})),
|
||||
);
|
||||
const read = ArtifactFileSource.prototype.read;
|
||||
const spy = vi
|
||||
.spyOn(ArtifactFileSource.prototype, "read")
|
||||
.mockImplementationOnce(function (this: ArtifactFileSource, signal) {
|
||||
return read.call(this, signal);
|
||||
})
|
||||
.mockRejectedValueOnce(new Error("final source verification failed"));
|
||||
try {
|
||||
await expect(
|
||||
reader.entries()[Symbol.asyncIterator]().next(),
|
||||
).rejects.toThrow("final source verification failed");
|
||||
expect(reader.formatMetadata()).toBeUndefined();
|
||||
await expect(
|
||||
reader.entries()[Symbol.asyncIterator]().next(),
|
||||
).resolves.toMatchObject({
|
||||
done: true,
|
||||
});
|
||||
expect(reader.formatMetadata()).toMatchObject({
|
||||
kind: "ntfs-mft",
|
||||
record_count: 1,
|
||||
});
|
||||
} finally {
|
||||
spy.mockRestore();
|
||||
await reader.close();
|
||||
}
|
||||
});
|
||||
|
||||
it("retries a shared preparation cancelled by another traversal", async () => {
|
||||
const reader = new NtfsMftArtifactReader(
|
||||
await writeMft("$MFT", recordBytes({})),
|
||||
);
|
||||
const controller = new AbortController();
|
||||
let entered: () => void = () => {};
|
||||
const started = new Promise<void>((resolve) => {
|
||||
entered = resolve;
|
||||
});
|
||||
const read = ArtifactFileSource.prototype.read;
|
||||
const spy = vi
|
||||
.spyOn(ArtifactFileSource.prototype, "read")
|
||||
.mockImplementationOnce(async function (this: ArtifactFileSource, signal) {
|
||||
entered();
|
||||
await new Promise<void>((resolve) => {
|
||||
signal?.addEventListener("abort", () => resolve(), { once: true });
|
||||
});
|
||||
assertArtifactReadActive(signal);
|
||||
return read.call(this, signal);
|
||||
});
|
||||
try {
|
||||
const cancelled = reader
|
||||
.entries(controller.signal)
|
||||
[Symbol.asyncIterator]()
|
||||
.next();
|
||||
const cancelledResult = expect(cancelled).rejects.toMatchObject({
|
||||
reason: "cancelled",
|
||||
});
|
||||
await started;
|
||||
const other = reader.entries()[Symbol.asyncIterator]().next();
|
||||
controller.abort("cancelled first traversal");
|
||||
await cancelledResult;
|
||||
await expect(other).resolves.toMatchObject({ done: true });
|
||||
expect(reader.formatMetadata()).toMatchObject({ record_count: 1 });
|
||||
} finally {
|
||||
controller.abort();
|
||||
spy.mockRestore();
|
||||
await reader.close();
|
||||
}
|
||||
});
|
||||
|
||||
it("revalidates the source before reusing a completed header inventory", async () => {
|
||||
const path = await writeMft("$MFT", recordBytes({}));
|
||||
const reader = new NtfsMftArtifactReader(path);
|
||||
try {
|
||||
await reader.entries()[Symbol.asyncIterator]().next();
|
||||
await writeFile(
|
||||
path,
|
||||
Buffer.concat([recordBytes({}), recordBytes({ index: 1 })]),
|
||||
);
|
||||
await expect(
|
||||
reader.entries()[Symbol.asyncIterator]().next(),
|
||||
).rejects.toMatchObject({ reason: "integrity" });
|
||||
} finally {
|
||||
await reader.close();
|
||||
}
|
||||
});
|
||||
@@ -0,0 +1,174 @@
|
||||
import type { Readable } from "node:stream";
|
||||
|
||||
import {
|
||||
ntfsMftMetadataSchema,
|
||||
type ArtifactFormatMetadata,
|
||||
} from "../domain/artifactFormatMetadata.js";
|
||||
import type { RegionDecodeFailure } from "../domain/firmware/regionDecoder.js";
|
||||
import {
|
||||
createMftDecoder,
|
||||
type MftInventory,
|
||||
} from "../domain/windows/ntfsMft.js";
|
||||
import {
|
||||
ArtifactReaderFailure,
|
||||
type ArtifactEntry,
|
||||
type ArtifactReader,
|
||||
type ArtifactReaderTraversal,
|
||||
type ArtifactTraversalOptions,
|
||||
} from "./ArtifactReader.js";
|
||||
import { ArtifactResourceScope } from "./ArtifactResourceScope.js";
|
||||
import {
|
||||
artifactReadCancelledBy,
|
||||
assertArtifactReadActive,
|
||||
} from "./ArtifactFileOperation.js";
|
||||
import { ArtifactFileSource } from "./ArtifactFileSource.js";
|
||||
import { readFileHandleChunks } from "../filesystem/readFileHandleChunks.js";
|
||||
import type { StableRegularFileDescriptor } from "../filesystem/RegularFile.js";
|
||||
|
||||
/**
|
||||
* An MFT stream is its own content: the FILE records describe the source
|
||||
* volume, not members inside the stream, so header facts are attached to
|
||||
* the root occurrence.
|
||||
*/
|
||||
export class NtfsMftArtifactReader implements ArtifactReader {
|
||||
readonly format = "ntfs-mft" as const;
|
||||
readonly #scope = new ArtifactResourceScope();
|
||||
readonly #source: ArtifactFileSource;
|
||||
#prepared:
|
||||
| { signal: AbortSignal | undefined; promise: Promise<void> }
|
||||
| undefined;
|
||||
#metadata: ArtifactFormatMetadata | undefined;
|
||||
#limitations: readonly string[] = [];
|
||||
#closing = false;
|
||||
|
||||
constructor(path: string, admitted?: StableRegularFileDescriptor) {
|
||||
this.#source = new ArtifactFileSource(path, "$MFT stream", admitted);
|
||||
}
|
||||
|
||||
formatMetadata(): ArtifactFormatMetadata | undefined {
|
||||
return this.#metadata === undefined
|
||||
? undefined
|
||||
: structuredClone(this.#metadata);
|
||||
}
|
||||
|
||||
async prepareContainer(
|
||||
expectedSha256: string,
|
||||
signal?: AbortSignal,
|
||||
): Promise<void> {
|
||||
await this.#scope.run(async () => {
|
||||
this.#assertOpen(signal);
|
||||
await this.#source.bindSha256(expectedSha256, signal);
|
||||
});
|
||||
}
|
||||
|
||||
async *entries(
|
||||
signal?: AbortSignal,
|
||||
options?: ArtifactTraversalOptions,
|
||||
): AsyncIterable<ArtifactEntry> {
|
||||
await this.#scope.run(async () => {
|
||||
this.#assertOpen(signal);
|
||||
for (;;) {
|
||||
const pending = (this.#prepared ??= {
|
||||
signal,
|
||||
promise: this.#prepare(signal),
|
||||
});
|
||||
try {
|
||||
await pending.promise;
|
||||
break;
|
||||
} catch (cause: unknown) {
|
||||
if (this.#prepared === pending) this.#prepared = undefined;
|
||||
if (
|
||||
pending.signal !== signal &&
|
||||
signal?.aborted !== true &&
|
||||
artifactReadCancelledBy(cause, pending.signal)
|
||||
) {
|
||||
this.#assertOpen(signal);
|
||||
continue;
|
||||
}
|
||||
throw cause;
|
||||
}
|
||||
}
|
||||
this.#assertOpen(signal);
|
||||
await this.#source.read(signal);
|
||||
});
|
||||
// An MFT declares no members, so the root occurrence is the whole
|
||||
// inventory; the coverage callback reports what its parse did not establish.
|
||||
options?.onCoverage?.(coverageOf(this.#limitations));
|
||||
}
|
||||
|
||||
async open(entry: ArtifactEntry, _signal?: AbortSignal): Promise<Readable> {
|
||||
return this.#scope.run(async () => {
|
||||
throw new ArtifactReaderFailure(
|
||||
"unavailable",
|
||||
`$MFT stream declares no member to open for ${entry.path}`,
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
async close(): Promise<void> {
|
||||
this.#closing = true;
|
||||
await this.#scope.close();
|
||||
await this.#source.close();
|
||||
this.#metadata = undefined;
|
||||
}
|
||||
|
||||
provenance(): readonly [] {
|
||||
return [];
|
||||
}
|
||||
|
||||
#assertOpen(signal?: AbortSignal): void {
|
||||
assertArtifactReadActive(signal);
|
||||
if (this.#closing)
|
||||
throw new ArtifactReaderFailure(
|
||||
"unavailable",
|
||||
"$MFT stream reader is closed",
|
||||
);
|
||||
}
|
||||
|
||||
async #prepare(signal?: AbortSignal): Promise<void> {
|
||||
const image = await this.#decode(signal);
|
||||
await this.#source.read(signal);
|
||||
this.#metadata = ntfsMftMetadataSchema.parse(image.metadata);
|
||||
this.#limitations = image.limitations;
|
||||
}
|
||||
|
||||
async #decode(signal?: AbortSignal): Promise<MftInventory> {
|
||||
const { handle, initial } = await this.#source.read(signal);
|
||||
const decoder = createMftDecoder(initial.size);
|
||||
if (!decoder.ok) throw this.#failure(decoder.error);
|
||||
if (initial.size > 0)
|
||||
for await (const chunk of readFileHandleChunks(handle, {
|
||||
end: initial.size - 1,
|
||||
signal,
|
||||
cancellationFailure: (reason) =>
|
||||
new ArtifactReaderFailure("cancelled", "$MFT stream read cancelled", {
|
||||
cause: reason,
|
||||
}),
|
||||
}) as AsyncIterable<Buffer>) {
|
||||
this.#assertOpen(signal);
|
||||
const written = decoder.value.write(chunk);
|
||||
if (!written.ok) throw this.#failure(written.error);
|
||||
}
|
||||
const image = decoder.value.finish();
|
||||
if (!image.ok) throw this.#failure(image.error);
|
||||
return image.value;
|
||||
}
|
||||
|
||||
#failure(failure: RegionDecodeFailure): ArtifactReaderFailure {
|
||||
return failure.kind === "format"
|
||||
? new ArtifactReaderFailure(
|
||||
"format",
|
||||
`$MFT stream is malformed: ${failure.message}`,
|
||||
)
|
||||
: new ArtifactReaderFailure(
|
||||
"limit",
|
||||
`$MFT stream exceeds a fixed limit: ${failure.message}`,
|
||||
failure.boundary === undefined ? {} : { boundary: failure.boundary },
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
const coverageOf = (limitations: readonly string[]): ArtifactReaderTraversal =>
|
||||
limitations.length === 0
|
||||
? { status: "complete", limitations: [] }
|
||||
: { status: "unknown", limitations };
|
||||
@@ -1,4 +1,7 @@
|
||||
import { artifactReaderLifecycleCases } from "../../tests/fixtures/artifactReaderLifecycle.js";
|
||||
import {
|
||||
artifactPreparationRecoveryCases,
|
||||
artifactReaderLifecycleCases,
|
||||
} from "../../tests/fixtures/artifactReaderLifecycle.js";
|
||||
import { writeFile } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
import { buffer } from "node:stream/consumers";
|
||||
@@ -91,6 +94,14 @@ it("preserves cancellation during payload CRC admission", async () => {
|
||||
}
|
||||
})(),
|
||||
).rejects.toMatchObject({ reason: "cancelled", cause: reason });
|
||||
reads.mockRestore();
|
||||
const recovered = await entries(reader);
|
||||
expect(recovered).toHaveLength(1);
|
||||
expect(
|
||||
(await buffer(await reader.open(recovered[0]!))).equals(
|
||||
Buffer.alloc(128 * 1024, 0x61),
|
||||
),
|
||||
).toBe(true);
|
||||
} finally {
|
||||
await reader.close();
|
||||
reads.mockRestore();
|
||||
@@ -205,3 +216,8 @@ it("reports impossible over-cap component extents as malformed", async () => {
|
||||
await reader.close();
|
||||
}
|
||||
});
|
||||
|
||||
artifactPreparationRecoveryCases({
|
||||
image: () => buildUImage({ payload: Buffer.from("payload") }),
|
||||
createReader: (path, admitted) => new UImageArtifactReader(path, admitted),
|
||||
});
|
||||
@@ -8,7 +8,10 @@ import type {
|
||||
UImageMetadata,
|
||||
} from "../domain/artifactFormatMetadata.js";
|
||||
import { ArtifactFileSource } from "./ArtifactFileSource.js";
|
||||
import { assertArtifactReadActive } from "./ArtifactFileOperation.js";
|
||||
import {
|
||||
artifactReadCancelledBy,
|
||||
assertArtifactReadActive,
|
||||
} from "./ArtifactFileOperation.js";
|
||||
import {
|
||||
ArtifactReaderFailure,
|
||||
copyArtifactEntry,
|
||||
@@ -47,7 +50,9 @@ export class UImageArtifactReader implements ArtifactReader {
|
||||
readonly #entries = new Map<string, UImageEntry>();
|
||||
readonly #streams = new Set<Readable>();
|
||||
readonly #source: ArtifactFileSource;
|
||||
#prepared: Promise<void> | undefined;
|
||||
#prepared:
|
||||
| { signal: AbortSignal | undefined; promise: Promise<void> }
|
||||
| undefined;
|
||||
#header: UImageMetadata | undefined;
|
||||
#closing = false;
|
||||
|
||||
@@ -75,8 +80,27 @@ export class UImageArtifactReader implements ArtifactReader {
|
||||
): AsyncIterable<ArtifactEntry> {
|
||||
await this.#scope.run(async () => {
|
||||
this.#assertOpen(signal);
|
||||
this.#prepared ??= this.#prepare(signal);
|
||||
await this.#prepared;
|
||||
for (;;) {
|
||||
const pending = (this.#prepared ??= {
|
||||
signal,
|
||||
promise: this.#prepare(signal),
|
||||
});
|
||||
try {
|
||||
await pending.promise;
|
||||
break;
|
||||
} catch (cause: unknown) {
|
||||
if (this.#prepared === pending) this.#prepared = undefined;
|
||||
if (
|
||||
pending.signal !== signal &&
|
||||
signal?.aborted !== true &&
|
||||
artifactReadCancelledBy(cause, pending.signal)
|
||||
) {
|
||||
this.#assertOpen(signal);
|
||||
continue;
|
||||
}
|
||||
throw cause;
|
||||
}
|
||||
}
|
||||
await this.#source.read(signal);
|
||||
});
|
||||
this.#assertOpen(signal);
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
import { artifactPreparationRecoveryCases } from "../../tests/fixtures/artifactReaderLifecycle.js";
|
||||
import { ArtifactFileSource } from "./ArtifactFileSource.js";
|
||||
import { truncate, writeFile } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
import { buffer, text } from "node:stream/consumers";
|
||||
@@ -43,6 +45,68 @@ const repeatedBlock = (data: Buffer): Buffer => {
|
||||
]);
|
||||
};
|
||||
|
||||
it("retries UnityFs preparation cancelled during file reading", async () => {
|
||||
const path = await fixture(buildUnityFs([{ path: "a", data: "x" }]));
|
||||
const admitted = await openRegularFile(path, { symlinks: "reject" });
|
||||
const reader = new UnityFsArtifactReader(path, admitted);
|
||||
const controller = new AbortController();
|
||||
const reason = { stage: "header read" };
|
||||
const nativeRead = admitted.handle.read.bind(admitted.handle);
|
||||
const read = vi.spyOn(admitted.handle, "read").mockImplementation((async (
|
||||
...args: unknown[]
|
||||
) => {
|
||||
const result = await Reflect.apply(nativeRead, admitted.handle, args);
|
||||
controller.abort(reason);
|
||||
return result;
|
||||
}) as typeof admitted.handle.read);
|
||||
try {
|
||||
await expect(
|
||||
(async () => {
|
||||
for await (const _ of reader.entries(controller.signal)) {
|
||||
}
|
||||
})(),
|
||||
).rejects.toMatchObject({ reason: "cancelled", cause: reason });
|
||||
read.mockRestore();
|
||||
expect((await entries(reader)).map((entry) => entry.path)).toEqual(["a"]);
|
||||
} finally {
|
||||
read.mockRestore();
|
||||
await reader.close();
|
||||
await admitted.handle.close();
|
||||
}
|
||||
});
|
||||
|
||||
it("does not publish UnityFs entries when final source verification fails", async () => {
|
||||
const path = await fixture(buildUnityFs([{ path: "a", data: "x" }]));
|
||||
const control = new UnityFsArtifactReader(path);
|
||||
const reader = new UnityFsArtifactReader(path);
|
||||
const expected = (await entries(control))[0]!;
|
||||
const nativeRead = ArtifactFileSource.prototype.read;
|
||||
const physical = Object.assign(
|
||||
new Error("final source verification failed"),
|
||||
{ code: "EIO" },
|
||||
);
|
||||
let calls = 0;
|
||||
const read = vi
|
||||
.spyOn(ArtifactFileSource.prototype, "read")
|
||||
.mockImplementation(function (this: ArtifactFileSource, ...args) {
|
||||
calls += 1;
|
||||
if (calls === 2) return Promise.reject(physical);
|
||||
return nativeRead.apply(this, args);
|
||||
});
|
||||
try {
|
||||
await expect(entries(reader)).rejects.toBe(physical);
|
||||
await expect(reader.open(expected)).rejects.toMatchObject({
|
||||
reason: "integrity",
|
||||
});
|
||||
read.mockRestore();
|
||||
expect((await entries(reader)).map((entry) => entry.path)).toEqual(["a"]);
|
||||
} finally {
|
||||
read.mockRestore();
|
||||
await reader.close();
|
||||
await control.close();
|
||||
}
|
||||
});
|
||||
|
||||
describe("UnityFS allocation and malformed-input boundaries", () => {
|
||||
it("rejects an oversized directory extending past the physical file", async () => {
|
||||
const bytes = buildUnityFs([{ path: "data", data: "x" }]);
|
||||
@@ -431,3 +495,8 @@ describe("UnityFS stream capacity and cache ownership", () => {
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
artifactPreparationRecoveryCases({
|
||||
image: () => buildUnityFs([{ path: "a", data: "x" }]),
|
||||
createReader: (path, admitted) => new UnityFsArtifactReader(path, admitted),
|
||||
});
|
||||
@@ -12,7 +12,10 @@ import {
|
||||
type ArtifactTraversalOptions,
|
||||
} from "./ArtifactReader.js";
|
||||
import { ArtifactResourceScope } from "./ArtifactResourceScope.js";
|
||||
import { assertArtifactReadActive } from "./ArtifactFileOperation.js";
|
||||
import {
|
||||
artifactReadCancelledBy,
|
||||
assertArtifactReadActive,
|
||||
} from "./ArtifactFileOperation.js";
|
||||
import { readArtifactFileBytes } from "./ArtifactFileRange.js";
|
||||
import type {
|
||||
ArtifactFormatMetadata,
|
||||
@@ -52,10 +55,12 @@ interface UnityFsEntryState {
|
||||
export class UnityFsArtifactReader implements ArtifactReader {
|
||||
readonly format = "unityfs" as const;
|
||||
readonly #scope = new ArtifactResourceScope();
|
||||
readonly #entries = new Map<string, UnityFsEntryState>();
|
||||
#entries = new Map<string, UnityFsEntryState>();
|
||||
readonly #streams = new Map<Readable, AbortController>();
|
||||
readonly #source: ArtifactFileSource;
|
||||
#prepared: Promise<void> | undefined;
|
||||
#prepared:
|
||||
| { signal: AbortSignal | undefined; promise: Promise<void> }
|
||||
| undefined;
|
||||
#header: UnityFsMetadata | undefined;
|
||||
#blocks: readonly UnityFsBlock[] = [];
|
||||
#dataStart = 0;
|
||||
@@ -88,8 +93,27 @@ export class UnityFsArtifactReader implements ArtifactReader {
|
||||
): AsyncIterable<ArtifactEntry> {
|
||||
await this.#scope.run(async () => {
|
||||
this.#assertOpen(signal);
|
||||
this.#prepared ??= this.#prepare(signal);
|
||||
await this.#prepared;
|
||||
for (;;) {
|
||||
const pending = (this.#prepared ??= {
|
||||
signal,
|
||||
promise: this.#prepare(signal),
|
||||
});
|
||||
try {
|
||||
await pending.promise;
|
||||
break;
|
||||
} catch (cause: unknown) {
|
||||
if (this.#prepared === pending) this.#prepared = undefined;
|
||||
if (
|
||||
pending.signal !== signal &&
|
||||
signal?.aborted !== true &&
|
||||
artifactReadCancelledBy(cause, pending.signal)
|
||||
) {
|
||||
this.#assertOpen(signal);
|
||||
continue;
|
||||
}
|
||||
throw cause;
|
||||
}
|
||||
}
|
||||
await this.#source.read(signal);
|
||||
});
|
||||
this.#assertOpen(signal);
|
||||
@@ -229,7 +253,7 @@ export class UnityFsArtifactReader implements ArtifactReader {
|
||||
if (header === undefined)
|
||||
throw unityFsFormatFailure("no complete UnityFS header");
|
||||
const status = header.layout.status;
|
||||
this.#header = {
|
||||
const metadata: UnityFsMetadata = {
|
||||
kind: "unityfs",
|
||||
signature: "UnityFS",
|
||||
format_version: header.formatVersion,
|
||||
@@ -245,7 +269,12 @@ export class UnityFsArtifactReader implements ArtifactReader {
|
||||
data_hash: null,
|
||||
directory_status: status,
|
||||
};
|
||||
if (status !== "available") return;
|
||||
if (status !== "available") {
|
||||
await this.#source.read(signal);
|
||||
this.#assertOpen(signal);
|
||||
this.#header = metadata;
|
||||
return;
|
||||
}
|
||||
const { dataHash, blocks, nodes, dataStart } = await readUnityFsBlocksInfo(
|
||||
handle,
|
||||
header,
|
||||
@@ -259,44 +288,55 @@ export class UnityFsArtifactReader implements ArtifactReader {
|
||||
if (streamBytes > BigInt(budget))
|
||||
throw expansionLimit("declared virtual stream", streamBytes, budget);
|
||||
}
|
||||
this.#header = {
|
||||
...this.#header,
|
||||
const preparedMetadata: UnityFsMetadata = {
|
||||
...metadata,
|
||||
data_hash: dataHash,
|
||||
block_count: blocks.length,
|
||||
data_start: dataStart,
|
||||
};
|
||||
this.#dataStart = dataStart;
|
||||
// Keep the source block count above, but do not revisit zero-output rows
|
||||
// for every member. Parsed stream/compressed offsets remain authoritative.
|
||||
this.#blocks = blocks.filter((block) => block.uncompressedSize > 0);
|
||||
this.#remainingDecodeBytes = budget;
|
||||
await this.#registerNodes(nodes, budget, signal);
|
||||
const readableBlocks = blocks.filter((block) => block.uncompressedSize > 0);
|
||||
const registered = await this.#registerNodes(
|
||||
nodes,
|
||||
readableBlocks,
|
||||
budget,
|
||||
signal,
|
||||
);
|
||||
await this.#source.read(signal);
|
||||
this.#assertOpen(signal);
|
||||
this.#header = preparedMetadata;
|
||||
this.#dataStart = dataStart;
|
||||
this.#blocks = readableBlocks;
|
||||
this.#remainingDecodeBytes = budget;
|
||||
this.#entries = registered.entries;
|
||||
this.#remainingReadOperations = registered.remainingReadOperations;
|
||||
}
|
||||
|
||||
async #registerNodes(
|
||||
nodes: readonly UnityFsNode[],
|
||||
blocks: readonly UnityFsBlock[],
|
||||
budget: number,
|
||||
signal?: AbortSignal,
|
||||
): Promise<void> {
|
||||
): Promise<{
|
||||
entries: Map<string, UnityFsEntryState>;
|
||||
remainingReadOperations: number;
|
||||
}> {
|
||||
const entries = new Map<string, UnityFsEntryState>();
|
||||
let readableBytes = 0n;
|
||||
for (let index = 0; index < nodes.length; index++) {
|
||||
if (index % 256 === 0) await setImmediate();
|
||||
this.#assertOpen(signal);
|
||||
const node = nodes[index]!;
|
||||
const firstBlock =
|
||||
node.size === 0 ? -1 : blockEndingAfter(this.#blocks, node.offset);
|
||||
node.size === 0 ? -1 : blockEndingAfter(blocks, node.offset);
|
||||
const lastBlock =
|
||||
node.size === 0
|
||||
? -1
|
||||
: blockEndingAfter(this.#blocks, node.offset + node.size - 1);
|
||||
: blockEndingAfter(blocks, node.offset + node.size - 1);
|
||||
if (node.size !== 0 && (firstBlock < 0 || lastBlock < firstBlock))
|
||||
throw unityFsFormatFailure("member has no block range");
|
||||
const unavailable = firstUnavailableBlock(
|
||||
this.#blocks,
|
||||
firstBlock,
|
||||
lastBlock,
|
||||
);
|
||||
const unavailable = firstUnavailableBlock(blocks, firstBlock, lastBlock);
|
||||
const kind =
|
||||
(node.flags & 1) !== 0 ? ("directory" as const) : ("file" as const);
|
||||
if (kind === "file" && unavailable === undefined) {
|
||||
@@ -331,7 +371,7 @@ export class UnityFsArtifactReader implements ArtifactReader {
|
||||
flags: node.flags,
|
||||
},
|
||||
} satisfies UnityFsEntry;
|
||||
this.#entries.set(adapterKey, {
|
||||
entries.set(adapterKey, {
|
||||
entry,
|
||||
firstBlock,
|
||||
lastBlock,
|
||||
@@ -340,14 +380,15 @@ export class UnityFsArtifactReader implements ArtifactReader {
|
||||
}
|
||||
// Four ordinary passes over admitted blocks, nodes and plaintext chunks,
|
||||
// with a small-reader floor. Byte quotas alone cannot bound tiny-block I/O.
|
||||
this.#remainingReadOperations =
|
||||
const remainingReadOperations =
|
||||
4 *
|
||||
Math.max(
|
||||
1024,
|
||||
this.#blocks.length +
|
||||
blocks.length +
|
||||
nodes.length +
|
||||
Math.ceil(Number(readableBytes) / CHUNK_BYTES),
|
||||
);
|
||||
return { entries, remainingReadOperations };
|
||||
}
|
||||
|
||||
async *#memberChunks(
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
import { artifactPreparationRecoveryCases } from "../../tests/fixtures/artifactReaderLifecycle.js";
|
||||
import { ArtifactFileSource } from "./ArtifactFileSource.js";
|
||||
import { once } from "node:events";
|
||||
import { writeFile } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
@@ -26,6 +28,68 @@ const entries = async (reader: WxapkgArtifactReader) => {
|
||||
return result;
|
||||
};
|
||||
|
||||
it("retries Wxapkg preparation cancelled during file reading", async () => {
|
||||
const path = await fixture(buildWxapkg([{ path: "a", data: "x" }]));
|
||||
const admitted = await openRegularFile(path, { symlinks: "reject" });
|
||||
const reader = new WxapkgArtifactReader(path, admitted);
|
||||
const controller = new AbortController();
|
||||
const reason = { stage: "header read" };
|
||||
const nativeRead = admitted.handle.read.bind(admitted.handle);
|
||||
const read = vi.spyOn(admitted.handle, "read").mockImplementation((async (
|
||||
...args: unknown[]
|
||||
) => {
|
||||
const result = await Reflect.apply(nativeRead, admitted.handle, args);
|
||||
controller.abort(reason);
|
||||
return result;
|
||||
}) as typeof admitted.handle.read);
|
||||
try {
|
||||
await expect(
|
||||
(async () => {
|
||||
for await (const _ of reader.entries(controller.signal)) {
|
||||
}
|
||||
})(),
|
||||
).rejects.toMatchObject({ reason: "cancelled", cause: reason });
|
||||
read.mockRestore();
|
||||
expect((await entries(reader)).map((entry) => entry.path)).toEqual(["a"]);
|
||||
} finally {
|
||||
read.mockRestore();
|
||||
await reader.close();
|
||||
await admitted.handle.close();
|
||||
}
|
||||
});
|
||||
|
||||
it("does not publish Wxapkg entries when final source verification fails", async () => {
|
||||
const path = await fixture(buildWxapkg([{ path: "a", data: "x" }]));
|
||||
const control = new WxapkgArtifactReader(path);
|
||||
const reader = new WxapkgArtifactReader(path);
|
||||
const expected = (await entries(control))[0]!;
|
||||
const nativeRead = ArtifactFileSource.prototype.read;
|
||||
const physical = Object.assign(
|
||||
new Error("final source verification failed"),
|
||||
{ code: "EIO" },
|
||||
);
|
||||
let calls = 0;
|
||||
const read = vi
|
||||
.spyOn(ArtifactFileSource.prototype, "read")
|
||||
.mockImplementation(function (this: ArtifactFileSource, ...args) {
|
||||
calls += 1;
|
||||
if (calls === 2) return Promise.reject(physical);
|
||||
return nativeRead.apply(this, args);
|
||||
});
|
||||
try {
|
||||
await expect(entries(reader)).rejects.toBe(physical);
|
||||
await expect(reader.open(expected)).rejects.toMatchObject({
|
||||
reason: "integrity",
|
||||
});
|
||||
read.mockRestore();
|
||||
expect((await entries(reader)).map((entry) => entry.path)).toEqual(["a"]);
|
||||
} finally {
|
||||
read.mockRestore();
|
||||
await reader.close();
|
||||
await control.close();
|
||||
}
|
||||
});
|
||||
|
||||
describe("wxapkg allocation and malformed-input boundaries", () => {
|
||||
it("admits the encoded table budget before reading it", async () => {
|
||||
const bytes = buildWxapkg([{ path: "a", data: "x" }]);
|
||||
@@ -308,3 +372,8 @@ describe("wxapkg stream capacity and freshness", () => {
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
artifactPreparationRecoveryCases({
|
||||
image: () => buildWxapkg([{ path: "a", data: "x" }]),
|
||||
createReader: (path, admitted) => new WxapkgArtifactReader(path, admitted),
|
||||
});
|
||||
@@ -12,7 +12,10 @@ import {
|
||||
type ArtifactTraversalOptions,
|
||||
} from "./ArtifactReader.js";
|
||||
import { ArtifactResourceScope } from "./ArtifactResourceScope.js";
|
||||
import { assertArtifactReadActive } from "./ArtifactFileOperation.js";
|
||||
import {
|
||||
artifactReadCancelledBy,
|
||||
assertArtifactReadActive,
|
||||
} from "./ArtifactFileOperation.js";
|
||||
import { readArtifactFileBytes } from "./ArtifactFileRange.js";
|
||||
import type {
|
||||
ArtifactFormatMetadata,
|
||||
@@ -48,10 +51,12 @@ interface WxapkgEntryState {
|
||||
export class WxapkgArtifactReader implements ArtifactReader {
|
||||
readonly format = "wxapkg" as const;
|
||||
readonly #scope = new ArtifactResourceScope();
|
||||
readonly #entries = new Map<string, WxapkgEntryState>();
|
||||
#entries = new Map<string, WxapkgEntryState>();
|
||||
readonly #streams = new Map<Readable, AbortController>();
|
||||
readonly #source: ArtifactFileSource;
|
||||
#prepared: Promise<void> | undefined;
|
||||
#prepared:
|
||||
| { signal: AbortSignal | undefined; promise: Promise<void> }
|
||||
| undefined;
|
||||
#header: WxapkgMetadata | undefined;
|
||||
#remainingReadOperations = 0;
|
||||
#admittedReadOperations = 0;
|
||||
@@ -81,8 +86,27 @@ export class WxapkgArtifactReader implements ArtifactReader {
|
||||
): AsyncIterable<ArtifactEntry> {
|
||||
await this.#scope.run(async () => {
|
||||
this.#assertOpen(signal);
|
||||
this.#prepared ??= this.#prepare(signal);
|
||||
await this.#prepared;
|
||||
for (;;) {
|
||||
const pending = (this.#prepared ??= {
|
||||
signal,
|
||||
promise: this.#prepare(signal),
|
||||
});
|
||||
try {
|
||||
await pending.promise;
|
||||
break;
|
||||
} catch (cause: unknown) {
|
||||
if (this.#prepared === pending) this.#prepared = undefined;
|
||||
if (
|
||||
pending.signal !== signal &&
|
||||
signal?.aborted !== true &&
|
||||
artifactReadCancelledBy(cause, pending.signal)
|
||||
) {
|
||||
this.#assertOpen(signal);
|
||||
continue;
|
||||
}
|
||||
throw cause;
|
||||
}
|
||||
}
|
||||
await this.#source.read(signal);
|
||||
});
|
||||
this.#assertOpen(signal);
|
||||
@@ -201,7 +225,7 @@ export class WxapkgArtifactReader implements ArtifactReader {
|
||||
const header = await readWxapkgHeader(handle, initial.size, signal);
|
||||
if (header === undefined)
|
||||
throw wxapkgFormatFailure("no complete wxapkg header");
|
||||
this.#header = {
|
||||
const metadata: WxapkgMetadata = {
|
||||
kind: "wxapkg",
|
||||
info: header.layout.status === "available" ? header.info : null,
|
||||
index_length:
|
||||
@@ -212,7 +236,12 @@ export class WxapkgArtifactReader implements ArtifactReader {
|
||||
header.layout.status === "available" ? header.fileCount : null,
|
||||
directory_status: header.layout.status,
|
||||
};
|
||||
if (header.layout.status !== "available") return;
|
||||
if (header.layout.status !== "available") {
|
||||
await this.#source.read(signal);
|
||||
this.#assertOpen(signal);
|
||||
this.#header = metadata;
|
||||
return;
|
||||
}
|
||||
const { members } = await readWxapkgFileTable(
|
||||
handle,
|
||||
header,
|
||||
@@ -220,6 +249,7 @@ export class WxapkgArtifactReader implements ArtifactReader {
|
||||
signal,
|
||||
);
|
||||
const budget = artifactExpansionBudgetBytes(initial.size);
|
||||
const entries = new Map<string, WxapkgEntryState>();
|
||||
let declaredBytes = 0n;
|
||||
for (let index = 0; index < members.length; index++) {
|
||||
if (index % 256 === 0) await setImmediate();
|
||||
@@ -234,7 +264,7 @@ export class WxapkgArtifactReader implements ArtifactReader {
|
||||
{ boundary: "memory" },
|
||||
);
|
||||
const adapterKey = `wxapkg-entry:${String(index)}`;
|
||||
this.#entries.set(adapterKey, {
|
||||
entries.set(adapterKey, {
|
||||
entry: {
|
||||
path: member.path,
|
||||
sourcePath: member.declaredPath,
|
||||
@@ -260,14 +290,18 @@ export class WxapkgArtifactReader implements ArtifactReader {
|
||||
});
|
||||
}
|
||||
// Header, table and per-chunk member reads; a small-reader floor applies.
|
||||
this.#admittedReadOperations =
|
||||
const admittedReadOperations =
|
||||
4 *
|
||||
Math.max(
|
||||
1024,
|
||||
members.length + Math.ceil(Number(declaredBytes) / CHUNK_BYTES),
|
||||
);
|
||||
this.#remainingReadOperations = this.#admittedReadOperations;
|
||||
await this.#source.read(signal);
|
||||
this.#assertOpen(signal);
|
||||
this.#header = metadata;
|
||||
this.#entries = entries;
|
||||
this.#admittedReadOperations = admittedReadOperations;
|
||||
this.#remainingReadOperations = admittedReadOperations;
|
||||
}
|
||||
|
||||
async *#memberChunks(
|
||||
|
||||
@@ -43,7 +43,8 @@ type FormatsWithReaders =
|
||||
| "uf2"
|
||||
| "dfuse"
|
||||
| "esp-flash-image"
|
||||
| "fit";
|
||||
| "fit"
|
||||
| "ntfs-mft";
|
||||
|
||||
const readerFormatsMatch: Equal<ArtifactReaderFormat, FormatsWithReaders> =
|
||||
true;
|
||||
|
||||
@@ -40,6 +40,7 @@ import { NodeSeaArtifactReader } from "./NodeSeaArtifactReader.js";
|
||||
import type { PyInstallerAdmission } from "./pyinstaller/PyInstallerAdmission.js";
|
||||
import { PyInstallerArtifactReader } from "./PyInstallerArtifactReader.js";
|
||||
import { Uf2ArtifactReader } from "./Uf2ArtifactReader.js";
|
||||
import { NtfsMftArtifactReader } from "./NtfsMftArtifactReader.js";
|
||||
import { UImageArtifactReader } from "./UImageArtifactReader.js";
|
||||
import { UnityFsArtifactReader } from "./UnityFsArtifactReader.js";
|
||||
import { WxapkgArtifactReader } from "./WxapkgArtifactReader.js";
|
||||
@@ -465,6 +466,14 @@ const NON_ZIP_FORMATS = {
|
||||
createReader: (path, options) =>
|
||||
new FitArtifactReader(path, sourceOf(options)),
|
||||
}),
|
||||
"ntfs-mft": described("ntfs-mft", {
|
||||
...plain,
|
||||
retainClassifiedSource: true,
|
||||
readerLimitation:
|
||||
"MFT inventory reports whole-stream counts and a bounded sample of the first 32 decoded FILE record headers, with the omitted-header count. Attributes, filenames, data streams and the $UsnJrnl:$J journal are not decoded, so no path, file size or change history is established.",
|
||||
createReader: (path, options) =>
|
||||
new NtfsMftArtifactReader(path, sourceOf(options)),
|
||||
}),
|
||||
unknown: described("unknown", plain),
|
||||
} satisfies Record<
|
||||
Exclude<ArtifactFormatId, ZipPackageFormat>,
|
||||
|
||||
@@ -576,6 +576,28 @@ it("classifies nested UF2 by block magics and DfuSe by prefix and suffix", () =>
|
||||
).toMatchObject({ format: "file" });
|
||||
});
|
||||
|
||||
it("classifies a $MFT stream by its FILE record header, under any name", () => {
|
||||
const mft = Buffer.alloc(2_048);
|
||||
mft.write("FILE", 0, "latin1");
|
||||
mft.writeUInt32LE(1024, 28);
|
||||
for (const path of ["evidence/$MFT", "evidence/mft.raw", "evidence/records"])
|
||||
expect(classifyArtifactBytesAtPath(path, mft)).toEqual({
|
||||
kind: "executable",
|
||||
format: "ntfs-mft",
|
||||
});
|
||||
});
|
||||
|
||||
it("leaves plain content beginning with FILE unclassified as MFT", () => {
|
||||
const text = Buffer.from(
|
||||
"FILE is a plain text prefix, not an NTFS record header",
|
||||
);
|
||||
expect(
|
||||
classifyArtifactBytesAtPath("evidence/readme.txt", text),
|
||||
).toMatchObject({
|
||||
format: "file",
|
||||
});
|
||||
});
|
||||
|
||||
it("classifies a FIT by a /images node within the first 4 KiB", () => {
|
||||
const images = [{ name: "kernel", data: Buffer.from("kernel") }];
|
||||
const late = devicetreeFixture({
|
||||
|
||||
@@ -11,6 +11,7 @@ import { recognizeEspImage } from "../../domain/firmware/espImage.js";
|
||||
import { hasUf2Block } from "../../domain/firmware/uf2.js";
|
||||
import { isDfuSeFile } from "../../domain/firmware/dfuse.js";
|
||||
import { hasFitImagesNode } from "../../domain/firmware/fitImage.js";
|
||||
import { hasMftFileRecordHeader } from "../../domain/windows/ntfsMft.js";
|
||||
import type { ArtifactClassificationEvidence } from "../ArtifactClassificationProbe.js";
|
||||
import { asarHeaderAdmissionFailure } from "../AsarHeader.js";
|
||||
import { hasDestinationPath } from "../ArtifactPaths.js";
|
||||
@@ -622,6 +623,7 @@ export const classifyArtifactBytes = (
|
||||
return "dfuse";
|
||||
if (recognizeEspImage(prefix, fileSize) === "esp-image") return "esp-image";
|
||||
if (hasFitImagesNode(prefix)) return "fit";
|
||||
if (hasMftFileRecordHeader(prefix)) return "ntfs-mft";
|
||||
if (hasBinaryPlistSignature(prefix)) return "plist";
|
||||
if (hasXmlPlistSignature(prefix)) return "plist";
|
||||
const fatFormat = classifyFatArtifact(evidence, fileSize);
|
||||
|
||||
@@ -86,6 +86,7 @@ const formatSchema = z.enum([
|
||||
"dfuse",
|
||||
"esp-flash-image",
|
||||
"fit",
|
||||
"ntfs-mft",
|
||||
"plist",
|
||||
"javascript",
|
||||
"source-map",
|
||||
|
||||
@@ -120,3 +120,48 @@ it("charges the group key before building it so an oversized request method fail
|
||||
});
|
||||
expect(finished(accumulator).tools).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("counts only admitted scenario transactions and refreshes their snapshot limitations", () => {
|
||||
const accumulator = new EnvironmentManifestAccumulator(32 * 1024);
|
||||
expect(finished(accumulator).origin).toBe("");
|
||||
expect(accumulator.add(sample(0, "/api/items"), "absent").ok).toBe(true);
|
||||
const before = finished(accumulator);
|
||||
expect(before.limitations.join(" ")).not.toContain("no primary origin");
|
||||
expect(
|
||||
accumulator.add(
|
||||
{
|
||||
...sample(1, "/api/items", `{"name":"${"x".repeat(2_000)}"}`),
|
||||
limitations: ["rejected-only limitation"],
|
||||
},
|
||||
"redacted",
|
||||
).ok,
|
||||
).toBe(false);
|
||||
expect(finished(accumulator)).toEqual(before);
|
||||
expect(
|
||||
accumulator.add({
|
||||
...sample(1, "/api/items"),
|
||||
limitations: ["uncommitted limitation", "x".repeat(100_000)],
|
||||
}).ok,
|
||||
).toBe(false);
|
||||
expect(finished(accumulator)).toEqual(before);
|
||||
expect(
|
||||
accumulator.add(
|
||||
{
|
||||
...sample(2, "/api/items", '{"name":"n"}'),
|
||||
limitations: ["accepted record limitation"],
|
||||
},
|
||||
"redacted",
|
||||
).ok,
|
||||
).toBe(true);
|
||||
const after = finished(accumulator);
|
||||
expect(after.limitations).toContain("accepted record limitation");
|
||||
expect(after.records.map((record) => record.ordinal)).toEqual([0, 2]);
|
||||
expect(
|
||||
after.limitations.filter((limitation) =>
|
||||
limitation.includes("scenario transactions with a usable request line"),
|
||||
),
|
||||
).toEqual([
|
||||
"1 of 2 scenario transactions with a usable request line retained a response body with declared secret values replaced by redaction markers; tools citing them replay the redacted bytes.",
|
||||
"1 of 2 scenario transactions with a usable request line retained no response body; tools citing them replay only the recorded status and content type, either of which may be absent.",
|
||||
]);
|
||||
});
|
||||
@@ -215,9 +215,11 @@ export class EnvironmentManifestAccumulator extends WebNetworkProjectionAccumula
|
||||
record: WebNetworkCaptureRecord,
|
||||
scenarioResponseBody?: ScenarioResponseBodyRetention,
|
||||
): Result<void, AnalysisError> {
|
||||
const limitations = this.addLimitations(record.limitations);
|
||||
if (!limitations.ok) return limitations;
|
||||
const rollbackAdmission = this.retainedCheckpoint();
|
||||
let committed = false;
|
||||
try {
|
||||
const limitations = this.prepareLimitations(record.limitations);
|
||||
if (!limitations.ok) return limitations;
|
||||
const line = readRecordedRequestLine(
|
||||
record.reported,
|
||||
this.includeQueryParameterNames,
|
||||
@@ -226,16 +228,14 @@ export class EnvironmentManifestAccumulator extends WebNetworkProjectionAccumula
|
||||
if (line === null) return this.overBudget();
|
||||
const releaseGrouping = this.temporaryCheckpoint();
|
||||
if (line.kind === "excluded") {
|
||||
this.#unrepresented += 1;
|
||||
releaseGrouping();
|
||||
return this.onRecordedRequestLine(record, line);
|
||||
}
|
||||
// Coverage uses the same admitted request line as grouping, including origin and raw path.
|
||||
if (scenarioResponseBody !== undefined) {
|
||||
this.#scenarioTransactions += 1;
|
||||
if (scenarioResponseBody === "absent") this.#scenarioWithoutBody += 1;
|
||||
if (scenarioResponseBody === "redacted")
|
||||
this.#scenarioRedactedBodies += 1;
|
||||
const prepared = this.prepareRecordedRequestLine(record, line);
|
||||
if (!prepared.ok) return prepared;
|
||||
prepared.value();
|
||||
limitations.value();
|
||||
this.#unrepresented += 1;
|
||||
committed = true;
|
||||
return ok(undefined);
|
||||
}
|
||||
const route = scanRouteTemplate(line.path, this.scratch);
|
||||
if (route === null) return this.overBudget();
|
||||
@@ -266,7 +266,7 @@ export class EnvironmentManifestAccumulator extends WebNetworkProjectionAccumula
|
||||
? this.#bodyShape(record.reported)
|
||||
: ok(null);
|
||||
if (!bodyShape.ok) return bodyShape;
|
||||
const grouped = this.#append(
|
||||
const grouped = this.#prepareAppend(
|
||||
record.ordinal,
|
||||
line,
|
||||
route,
|
||||
@@ -277,18 +277,33 @@ export class EnvironmentManifestAccumulator extends WebNetworkProjectionAccumula
|
||||
if (!grouped.ok) return grouped;
|
||||
// Keep the parsed line's admission while releasing grouping-only scratch.
|
||||
releaseGrouping();
|
||||
return this.onRecordedRequestLine(record, line);
|
||||
const prepared = this.prepareRecordedRequestLine(record, line);
|
||||
if (!prepared.ok) return prepared;
|
||||
// Both owners admitted their facts; neither commit can reject the record.
|
||||
grouped.value();
|
||||
prepared.value();
|
||||
limitations.value();
|
||||
// Coverage uses the same admitted request line as grouping, including origin and raw path.
|
||||
if (scenarioResponseBody !== undefined) {
|
||||
this.#scenarioTransactions += 1;
|
||||
if (scenarioResponseBody === "absent") this.#scenarioWithoutBody += 1;
|
||||
if (scenarioResponseBody === "redacted")
|
||||
this.#scenarioRedactedBodies += 1;
|
||||
}
|
||||
committed = true;
|
||||
return ok(undefined);
|
||||
} finally {
|
||||
if (!committed) rollbackAdmission();
|
||||
this.releaseTemporary();
|
||||
}
|
||||
}
|
||||
|
||||
/** Consume the owner's parsed request line after successful grouping, including exclusions. */
|
||||
protected onRecordedRequestLine(
|
||||
/** Admit derived facts from the owner's parsed line, then commit only after all owners admit it. */
|
||||
protected prepareRecordedRequestLine(
|
||||
_record: WebNetworkCaptureRecord,
|
||||
_line: RecordedRequestLine,
|
||||
): Result<void, AnalysisError> {
|
||||
return ok(undefined);
|
||||
): Result<() => void, AnalysisError> {
|
||||
return ok(() => {});
|
||||
}
|
||||
|
||||
/** Read the top-level key types of the group's first recorded JSON-typed request body; nested values are never walked. */
|
||||
@@ -316,14 +331,14 @@ export class EnvironmentManifestAccumulator extends WebNetworkProjectionAccumula
|
||||
return ok({ type: "object", properties: Object.fromEntries(entries) });
|
||||
}
|
||||
|
||||
#append(
|
||||
#prepareAppend(
|
||||
ordinal: number,
|
||||
line: RequestLine,
|
||||
route: RouteTemplate,
|
||||
queryNames: ReadonlySet<string> | undefined,
|
||||
bucket: { readonly key: string; readonly group: GroupState | undefined },
|
||||
exchange: { readonly shape: BodyShape | null; readonly empty: boolean },
|
||||
): Result<void, AnalysisError> {
|
||||
): Result<() => void, AnalysisError> {
|
||||
const bodyShape = exchange.shape;
|
||||
const mime = mediaTypeEssence(line.rawMime);
|
||||
let group = bucket.group;
|
||||
@@ -366,48 +381,49 @@ export class EnvironmentManifestAccumulator extends WebNetworkProjectionAccumula
|
||||
bytes += ENTRY_BYTES + textBytes(name.length);
|
||||
// Reserve the whole mutation while route scratch is still live.
|
||||
if (!this.retain(bytes)) return this.overBudget();
|
||||
const position = this.#records.length;
|
||||
this.#records.push({
|
||||
ordinal,
|
||||
method: line.method,
|
||||
path: line.path,
|
||||
status: line.status,
|
||||
contentType: mime,
|
||||
});
|
||||
const jsonLike = isJsonLike(mime);
|
||||
const stats = origin ?? { records: 0, jsonLike: 0 };
|
||||
stats.records += 1;
|
||||
if (jsonLike) stats.jsonLike += 1;
|
||||
this.#origins.set(line.origin, stats);
|
||||
if (group === undefined) {
|
||||
group = {
|
||||
return ok(() => {
|
||||
const position = this.#records.length;
|
||||
this.#records.push({
|
||||
ordinal,
|
||||
method: line.method,
|
||||
origin: line.origin,
|
||||
api_surface_template: route.template,
|
||||
parts: routeParts(route),
|
||||
positions: [],
|
||||
statuses: new Set(),
|
||||
contentTypes: new Set(),
|
||||
queryNames: new Set(),
|
||||
bodyShape,
|
||||
apiPath: isApiPath(route.template),
|
||||
jsonLike: false,
|
||||
bodiless: BODILESS_WRITE_METHODS.has(line.method),
|
||||
};
|
||||
this.#groups.set(bucket.key, group);
|
||||
} else {
|
||||
for (const segment of route.segments) {
|
||||
const part = group.parts[segment.segment_index];
|
||||
if (part?.kind === "param") part.values.add(segment.value);
|
||||
path: line.path,
|
||||
status: line.status,
|
||||
contentType: mime,
|
||||
});
|
||||
const jsonLike = isJsonLike(mime);
|
||||
const stats = origin ?? { records: 0, jsonLike: 0 };
|
||||
stats.records += 1;
|
||||
if (jsonLike) stats.jsonLike += 1;
|
||||
this.#origins.set(line.origin, stats);
|
||||
if (group === undefined) {
|
||||
group = {
|
||||
method: line.method,
|
||||
origin: line.origin,
|
||||
api_surface_template: route.template,
|
||||
parts: routeParts(route),
|
||||
positions: [],
|
||||
statuses: new Set(),
|
||||
contentTypes: new Set(),
|
||||
queryNames: new Set(),
|
||||
bodyShape,
|
||||
apiPath: isApiPath(route.template),
|
||||
jsonLike: false,
|
||||
bodiless: BODILESS_WRITE_METHODS.has(line.method),
|
||||
};
|
||||
this.#groups.set(bucket.key, group);
|
||||
} else {
|
||||
for (const segment of route.segments) {
|
||||
const part = group.parts[segment.segment_index];
|
||||
if (part?.kind === "param") part.values.add(segment.value);
|
||||
}
|
||||
}
|
||||
}
|
||||
group.positions.push(position);
|
||||
group.jsonLike ||= jsonLike;
|
||||
group.bodiless &&= exchange.empty;
|
||||
if (line.status !== null) group.statuses.add(line.status);
|
||||
if (mime !== null) group.contentTypes.add(mime);
|
||||
for (const name of queryNames ?? []) group.queryNames.add(name);
|
||||
return ok(undefined);
|
||||
group.positions.push(position);
|
||||
group.jsonLike ||= jsonLike;
|
||||
group.bodiless &&= exchange.empty;
|
||||
if (line.status !== null) group.statuses.add(line.status);
|
||||
if (mime !== null) group.contentTypes.add(mime);
|
||||
for (const name of queryNames ?? []) group.queryNames.add(name);
|
||||
});
|
||||
}
|
||||
|
||||
#tools(groups: readonly ToolGroup[]): Tool[] {
|
||||
@@ -513,7 +529,8 @@ export class EnvironmentManifestAccumulator extends WebNetworkProjectionAccumula
|
||||
compare(left.method, right.method) ||
|
||||
compare(left.reason, right.reason),
|
||||
);
|
||||
const limitations = this.addLimitations([
|
||||
// Derived limitations describe this snapshot, not an earlier finish.
|
||||
const limitations = this.snapshotLimitations([
|
||||
...(this.reportsBodyShapes &&
|
||||
tools.some((tool) => tool.body_shape !== null)
|
||||
? [
|
||||
@@ -521,16 +538,6 @@ export class EnvironmentManifestAccumulator extends WebNetworkProjectionAccumula
|
||||
]
|
||||
: []),
|
||||
...this.#segmentLimitLimitations(toolGroups),
|
||||
...(this.#scenarioWithoutBody > 0
|
||||
? [
|
||||
`${String(this.#scenarioWithoutBody)} of ${String(this.#scenarioTransactions)} scenario transactions with a usable request line retained no response body; tools citing them replay only the recorded status and content type, either of which may be absent.`,
|
||||
]
|
||||
: []),
|
||||
...(this.#scenarioRedactedBodies > 0
|
||||
? [
|
||||
`${String(this.#scenarioRedactedBodies)} of ${String(this.#scenarioTransactions)} scenario transactions with a usable request line retained a response body with declared secret values replaced by redaction markers; tools citing them replay the redacted bytes.`,
|
||||
]
|
||||
: []),
|
||||
...(this.#unrepresented > 0
|
||||
? [
|
||||
`${String(this.#unrepresented)} retained records without a usable request line are not represented in the manifest.`,
|
||||
@@ -541,6 +548,16 @@ export class EnvironmentManifestAccumulator extends WebNetworkProjectionAccumula
|
||||
"No retained record had a parseable origin; the manifest names no primary origin.",
|
||||
]
|
||||
: []),
|
||||
...(this.#scenarioWithoutBody > 0
|
||||
? [
|
||||
`${String(this.#scenarioWithoutBody)} of ${String(this.#scenarioTransactions)} scenario transactions with a usable request line retained no response body; tools citing them replay only the recorded status and content type, either of which may be absent.`,
|
||||
]
|
||||
: []),
|
||||
...(this.#scenarioRedactedBodies > 0
|
||||
? [
|
||||
`${String(this.#scenarioRedactedBodies)} of ${String(this.#scenarioTransactions)} scenario transactions with a usable request line retained a response body with declared secret values replaced by redaction markers; tools citing them replay the redacted bytes.`,
|
||||
]
|
||||
: []),
|
||||
]);
|
||||
if (!limitations.ok) return limitations;
|
||||
return ok({
|
||||
@@ -567,7 +584,7 @@ export class EnvironmentManifestAccumulator extends WebNetworkProjectionAccumula
|
||||
tools.filter((tool) => tool.params.length > 0).length +
|
||||
excluded.filter((exclusion) => exclusion.templated).length,
|
||||
coverage: completeWithinViewCoverage(tools.length, tools.length),
|
||||
limitations: this.sortedLimitations(),
|
||||
limitations: limitations.value,
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -3,6 +3,7 @@ import type { AnalysisError } from "../analysisErrorBase.js";
|
||||
import { parseExactJson } from "../exactJsonNumber.js";
|
||||
import { MAX_JSON_DEPTH, type JsonValue } from "../jsonValue.js";
|
||||
import {
|
||||
addOffsetDateTimeMilliseconds,
|
||||
compareOffsetDateTimeInstants,
|
||||
readRecordedStart,
|
||||
} from "../offsetDateTime.js";
|
||||
@@ -29,6 +30,7 @@ import {
|
||||
base64PrefixChars,
|
||||
readRecordedResponseBody,
|
||||
readRecordedResponseDeclaration,
|
||||
recordedResponseStatus,
|
||||
recordedResponseBodyIsEmpty,
|
||||
type RecordedResponseBody,
|
||||
} from "./recordedResponseBody.js";
|
||||
@@ -45,6 +47,9 @@ import {
|
||||
* room under the JSON depth limit to be sealed and served.
|
||||
*/
|
||||
const STATE_NESTING_DEPTH = 16;
|
||||
// Number.MIN_VALUE milliseconds needs this many decimal places in seconds;
|
||||
// this also covers the whole-second digits of every finite number duration.
|
||||
const DURATION_DIGITS = 3 - Math.floor(Math.log10(Number.MIN_VALUE));
|
||||
|
||||
/** A recorded response body as state JSON: a JSON-like media type, complete UTF-8 text within the state body limit, exact numbers; otherwise undefined. */
|
||||
export const stateBodyJson = (
|
||||
@@ -69,14 +74,14 @@ export const stateBodyJson = (
|
||||
|
||||
const startSchema = z.object({ startedDateTime: z.string() });
|
||||
const durationSchema = z.object({ time: z.number().nonnegative() });
|
||||
const redactedWriteSchema = z.object({
|
||||
const unlocatedWriteSchema = z.object({
|
||||
request: z.object({ method: z.string() }),
|
||||
response: z.object({ status: z.number() }).optional(),
|
||||
});
|
||||
interface RecordEntry {
|
||||
readonly facts: StateRecordFacts;
|
||||
/** Null for a write whose URL was redacted: its target is unknown, so it bounds every type. */
|
||||
/** Null for a write without a usable URL: its unknown target bounds every type. */
|
||||
readonly origin: string | null;
|
||||
readonly targetExclusion?: "url-redacted" | "url-unparseable";
|
||||
readonly contentType: string;
|
||||
readonly startInterpretation:
|
||||
| NonNullable<ReturnType<typeof readRecordedStart>>["interpretation"]
|
||||
@@ -119,10 +124,10 @@ export class EnvironmentStateAccumulator extends EnvironmentManifestAccumulator
|
||||
return false;
|
||||
}
|
||||
|
||||
protected override onRecordedRequestLine(
|
||||
protected override prepareRecordedRequestLine(
|
||||
record: WebNetworkCaptureRecord,
|
||||
line: RecordedRequestLine,
|
||||
): Result<void, AnalysisError> {
|
||||
): Result<() => void, AnalysisError> {
|
||||
const started = startSchema.safeParse(record.reported);
|
||||
const duration = durationSchema.safeParse(record.reported);
|
||||
// Admit normalization and fraction parsing before materializing their strings.
|
||||
@@ -148,28 +153,52 @@ export class EnvironmentStateAccumulator extends EnvironmentManifestAccumulator
|
||||
startMs === undefined || !duration.success
|
||||
? undefined
|
||||
: startMs + duration.data.time;
|
||||
// Exact completion keeps arbitrary source precision. Reserve digit-array and
|
||||
// string expansion before the shared decimal addition allocates them.
|
||||
if (
|
||||
startInstant !== undefined &&
|
||||
duration.success &&
|
||||
duration.data.time !== 0 &&
|
||||
!this.scratch(
|
||||
RECORD_SCRATCH_BYTES +
|
||||
textBytes(Math.max(startInstant.fraction.length, DURATION_DIGITS)) *
|
||||
3,
|
||||
)
|
||||
)
|
||||
return this.overBudget();
|
||||
const endInstant =
|
||||
startInstant === undefined || !duration.success
|
||||
? undefined
|
||||
: addOffsetDateTimeMilliseconds(startInstant, duration.data.time);
|
||||
const timing: StateRecordFacts["timing"] = {
|
||||
start:
|
||||
startInstant === undefined || startMs === undefined
|
||||
? undefined
|
||||
: { instant: startInstant, ms: startMs },
|
||||
end: endInstant,
|
||||
endMs: end !== undefined && Number.isFinite(end) ? end : undefined,
|
||||
};
|
||||
// A fraction substring may retain its normalized timestamp's backing store,
|
||||
// including trailing zeroes removed from the exact comparison value.
|
||||
const instantBytes =
|
||||
started.success && startInstant !== undefined
|
||||
(started.success && startInstant !== undefined
|
||||
? ENTRY_BYTES + textBytes(started.data.startedDateTime.length + 6)
|
||||
: 0;
|
||||
: 0) +
|
||||
(endInstant !== undefined && endInstant !== startInstant
|
||||
? ENTRY_BYTES + textBytes(endInstant.fraction.length)
|
||||
: 0);
|
||||
if (line.kind === "excluded")
|
||||
return line.reason === "url-redacted"
|
||||
? this.#redactedWrite(
|
||||
return line.reason === "url-redacted" ||
|
||||
line.reason === "url-unparseable" ||
|
||||
line.reason === "request-unavailable"
|
||||
? this.#unlocatedWrite(
|
||||
record,
|
||||
line.reason === "url-redacted" ? "url-redacted" : "url-unparseable",
|
||||
timing,
|
||||
instantBytes,
|
||||
reading?.interpretation,
|
||||
)
|
||||
: ok(undefined);
|
||||
: ok(() => undefined);
|
||||
// The entry keeps the line's strings and a reference to the already admitted record.
|
||||
if (
|
||||
!this.retain(
|
||||
@@ -185,56 +214,62 @@ export class EnvironmentStateAccumulator extends EnvironmentManifestAccumulator
|
||||
)
|
||||
)
|
||||
return this.overBudget();
|
||||
this.#entries.set(record.ordinal, {
|
||||
facts: {
|
||||
ordinal: record.ordinal,
|
||||
method: line.method,
|
||||
path: line.path,
|
||||
query: line.query === "" ? undefined : line.query,
|
||||
status: line.status,
|
||||
timing,
|
||||
},
|
||||
origin: line.origin,
|
||||
contentType: line.rawMime,
|
||||
startInterpretation: reading?.interpretation,
|
||||
record,
|
||||
return ok(() => {
|
||||
this.#entries.set(record.ordinal, {
|
||||
facts: {
|
||||
ordinal: record.ordinal,
|
||||
method: line.method,
|
||||
path: line.path,
|
||||
query: line.query === "" ? undefined : line.query,
|
||||
status: line.status,
|
||||
timing,
|
||||
},
|
||||
origin: line.origin,
|
||||
contentType: line.rawMime,
|
||||
startInterpretation: reading?.interpretation,
|
||||
record,
|
||||
});
|
||||
});
|
||||
return ok(undefined);
|
||||
}
|
||||
|
||||
/** Keep a successful write whose URL was redacted, with an empty path whose root relates to every type. */
|
||||
#redactedWrite(
|
||||
/** Prepare a successful write with unknown target, whose empty root relates to every type. */
|
||||
#unlocatedWrite(
|
||||
record: WebNetworkCaptureRecord,
|
||||
reason: "url-redacted" | "url-unparseable",
|
||||
timing: StateRecordFacts["timing"],
|
||||
instantBytes: number,
|
||||
startInterpretation: RecordEntry["startInterpretation"],
|
||||
): Result<void, AnalysisError> {
|
||||
const parsed = redactedWriteSchema.safeParse(record.reported);
|
||||
if (!parsed.success) return ok(undefined);
|
||||
const status = parsed.data.response?.status ?? 0;
|
||||
): Result<() => void, AnalysisError> {
|
||||
const parsed = unlocatedWriteSchema.safeParse(record.reported);
|
||||
if (!parsed.success) return ok(() => undefined);
|
||||
const status = recordedResponseStatus(
|
||||
readRecordedResponseDeclaration(record.reported),
|
||||
);
|
||||
const facts: StateRecordFacts = {
|
||||
ordinal: record.ordinal,
|
||||
method: parsed.data.request.method,
|
||||
path: "",
|
||||
query: undefined,
|
||||
status: status > 0 ? status : null,
|
||||
status,
|
||||
timing,
|
||||
};
|
||||
if (!isStateWrite(facts)) return ok(undefined);
|
||||
if (!isStateWrite(facts)) return ok(() => undefined);
|
||||
if (
|
||||
!this.retain(
|
||||
ENTRY_BYTES * 8 + textBytes(facts.method.length) + instantBytes,
|
||||
)
|
||||
)
|
||||
return this.overBudget();
|
||||
this.#entries.set(record.ordinal, {
|
||||
facts,
|
||||
origin: null,
|
||||
contentType: "",
|
||||
startInterpretation,
|
||||
record,
|
||||
return ok(() => {
|
||||
this.#entries.set(record.ordinal, {
|
||||
facts,
|
||||
origin: null,
|
||||
targetExclusion: reason,
|
||||
contentType: "",
|
||||
startInterpretation,
|
||||
record,
|
||||
});
|
||||
});
|
||||
return ok(undefined);
|
||||
}
|
||||
|
||||
/** Parse one retained response body as state JSON, charging its decoding and tree as temporary work. */
|
||||
@@ -330,7 +365,10 @@ export class EnvironmentStateAccumulator extends EnvironmentManifestAccumulator
|
||||
entry.origin === manifest.value.origin || entry.origin === null,
|
||||
);
|
||||
const redactedWrites = entries.filter(
|
||||
(entry) => entry.origin === null,
|
||||
(entry) => entry.targetExclusion === "url-redacted",
|
||||
).length;
|
||||
const unparseableWrites = entries.filter(
|
||||
(entry) => entry.targetExclusion === "url-unparseable",
|
||||
).length;
|
||||
const timeline: StateTimeline =
|
||||
entries.length > 0 &&
|
||||
@@ -361,7 +399,8 @@ export class EnvironmentStateAccumulator extends EnvironmentManifestAccumulator
|
||||
},
|
||||
}).run();
|
||||
if (state === null) return this.overBudget();
|
||||
const limitations = this.addLimitations([
|
||||
const limitations = this.snapshotLimitations([
|
||||
...manifest.value.limitations,
|
||||
...(entries.some((entry) => entry.startInterpretation === "native-date")
|
||||
? [
|
||||
"Some recorded start spellings use native Date parsing at millisecond precision; spellings without an explicit zone depend on the host timezone.",
|
||||
@@ -372,6 +411,11 @@ export class EnvironmentStateAccumulator extends EnvironmentManifestAccumulator
|
||||
`${String(redactedWrites)} of the recorded writes had a redacted URL; each ends the initial state of every type, since its target is unknown.`,
|
||||
]
|
||||
: []),
|
||||
...(unparseableWrites > 0
|
||||
? [
|
||||
`${String(unparseableWrites)} of the recorded writes had an unusable URL; each ends the initial state of every type, since its target is unknown.`,
|
||||
]
|
||||
: []),
|
||||
...(timeline === "record-order"
|
||||
? [
|
||||
"Some records carry no start time, so the state's timeline is record order and overlapping requests are not detected.",
|
||||
@@ -397,7 +441,7 @@ export class EnvironmentStateAccumulator extends EnvironmentManifestAccumulator
|
||||
...state,
|
||||
unknowns: UNKNOWNS.map((unknown) => ({ ...unknown })),
|
||||
coverage: manifest.value.coverage,
|
||||
limitations: this.sortedLimitations(),
|
||||
limitations: limitations.value,
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -1418,7 +1418,7 @@ describe("encoded parent spellings", () => {
|
||||
path: "/api/orgs/uid/acme1/books",
|
||||
query: undefined,
|
||||
status: 200,
|
||||
timing: { start: undefined, endMs: undefined },
|
||||
timing: { start: undefined, end: undefined, endMs: undefined },
|
||||
};
|
||||
const lines = checkedLines({
|
||||
tools: [tool],
|
||||
@@ -1621,7 +1621,7 @@ describe("resolved encoded item observations", () => {
|
||||
path: "/api/orgs/acme1/books/%61",
|
||||
query: undefined,
|
||||
status: 204,
|
||||
timing: { start: undefined, endMs: undefined },
|
||||
timing: { start: undefined, end: undefined, endMs: undefined },
|
||||
};
|
||||
const target = itemTarget(state, "/api/orgs/{org_id}/books", write);
|
||||
const itemView = {
|
||||
@@ -1642,7 +1642,7 @@ describe("resolved encoded item observations", () => {
|
||||
path: "/api/orgs/acme1/books/%61",
|
||||
query: undefined,
|
||||
status: 200,
|
||||
timing: { start: undefined, endMs: undefined },
|
||||
timing: { start: undefined, end: undefined, endMs: undefined },
|
||||
},
|
||||
];
|
||||
const search: ObservationSearch = {
|
||||
@@ -2308,23 +2308,53 @@ describe("reads and calls on an entity an unmodelled write left unknown", () =>
|
||||
{},
|
||||
];
|
||||
|
||||
it("keeps the entity unknown through a failed read that is no absence, for a later read to check", () => {
|
||||
const rows: TimedCaptureRow[] = [
|
||||
["GET", "/api/books", [named("A"), other]],
|
||||
["GET", "/api/books/1", named("A")],
|
||||
["GET", "/api/books/2", other],
|
||||
["PATCH", "/api/books/1", named("Z"), 200, { name: "Z" }],
|
||||
ARCHIVE,
|
||||
// The application was briefly unavailable, so this answer shows nothing of the book.
|
||||
["GET", "/api/books/1", { error: "unavailable" }, 503],
|
||||
["GET", "/api/books/1", named("Z")],
|
||||
];
|
||||
expect(gated(rows, "patch_books_by_id")).toMatchObject({
|
||||
served: true,
|
||||
checked_record_ordinals: [6],
|
||||
contradicted_by: null,
|
||||
});
|
||||
});
|
||||
it.each([
|
||||
{ label: "scalar", read: "unavailable", served: false },
|
||||
{ label: "array", read: [], served: false },
|
||||
{ label: "object", read: named("A"), served: true },
|
||||
])(
|
||||
"requires a usable item baseline after an unmodelled write ($label)",
|
||||
({ read, served }) => {
|
||||
const rows: TimedCaptureRow[] = [
|
||||
["GET", "/api/books", [named("A"), other]],
|
||||
["GET", "/api/books/1", named("A")],
|
||||
["GET", "/api/books/2", other],
|
||||
ARCHIVE,
|
||||
["GET", "/api/books/1", read],
|
||||
["PATCH", "/api/books/1", named("Z"), 200, { name: "Z" }],
|
||||
["GET", "/api/books/1", named("Z")],
|
||||
];
|
||||
expect(gated(rows, "patch_books_by_id")).toMatchObject({
|
||||
served,
|
||||
checked_record_ordinals: served ? [6] : [],
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
it.each([
|
||||
{ status: 503, body: { error: "unavailable" } },
|
||||
{ status: 200, body: "unavailable" },
|
||||
{ status: 200, body: [] },
|
||||
])(
|
||||
"keeps pending effects through an unusable item read ($status, $body)",
|
||||
({ status, body }) => {
|
||||
const rows: TimedCaptureRow[] = [
|
||||
["GET", "/api/books", [named("A"), other]],
|
||||
["GET", "/api/books/1", named("A")],
|
||||
["GET", "/api/books/2", other],
|
||||
["PATCH", "/api/books/1", named("Z"), 200, { name: "Z" }],
|
||||
ARCHIVE,
|
||||
// Neither failure nor a successful non-object response shows the book.
|
||||
["GET", "/api/books/1", body, status],
|
||||
["GET", "/api/books/1", named("Z")],
|
||||
];
|
||||
expect(gated(rows, "patch_books_by_id")).toMatchObject({
|
||||
served: true,
|
||||
checked_record_ordinals: [6],
|
||||
contradicted_by: null,
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
it("confirms no update of a volatile field by a read that shows the entity again, as no other read does", () => {
|
||||
const seen = (id: number, count: number) => ({
|
||||
@@ -2468,7 +2498,7 @@ describe("entities an unmodelled write made", () => {
|
||||
path: "/api/books/3",
|
||||
query: undefined,
|
||||
status: 200,
|
||||
timing: { start: undefined, endMs: undefined },
|
||||
timing: { start: undefined, end: undefined, endMs: undefined },
|
||||
};
|
||||
expect(
|
||||
unseen.adopt(view as unknown as StateItemView, { record, index: 1 }),
|
||||
@@ -2493,7 +2523,7 @@ describe("entities an unmodelled write made", () => {
|
||||
path: "/api/books/3",
|
||||
query: undefined,
|
||||
status: 200,
|
||||
timing: { start: undefined, endMs: undefined },
|
||||
timing: { start: undefined, end: undefined, endMs: undefined },
|
||||
},
|
||||
index,
|
||||
});
|
||||
@@ -3201,7 +3231,7 @@ it("learns omitted fields using the raw ID resolved by the item owner", () => {
|
||||
path: "/api/books/%63",
|
||||
query: undefined,
|
||||
status: 200,
|
||||
timing: { start: undefined, endMs: undefined },
|
||||
timing: { start: undefined, end: undefined, endMs: undefined },
|
||||
};
|
||||
const target = itemTarget(state, view.type, record);
|
||||
expect(target.id).toBe("%63");
|
||||
|
||||
@@ -555,8 +555,6 @@ export class EnvironmentStateWriteSimulation {
|
||||
): void {
|
||||
const target = entityTarget(entity.type, entity.collection, entity.id);
|
||||
const writes = this.#touched.get(target) ?? new Map();
|
||||
this.#taint.known(entity.type, entity.collection, entity.id);
|
||||
this.#touched.delete(target);
|
||||
if (
|
||||
isAbsence(record.status) ||
|
||||
this.#input.io.withBody(record.ordinal, (body) =>
|
||||
@@ -567,12 +565,16 @@ export class EnvironmentStateWriteSimulation {
|
||||
),
|
||||
)
|
||||
) {
|
||||
this.#taint.known(entity.type, entity.collection, entity.id);
|
||||
this.#touched.delete(target);
|
||||
this.#removeEntity(entity);
|
||||
return;
|
||||
}
|
||||
const object = this.#checkFields(view, record, entity, writes);
|
||||
if (object !== undefined)
|
||||
this.#state.put(updatedEntity(entity, { ...entity.fields, ...object }));
|
||||
if (object === undefined) return;
|
||||
this.#taint.known(entity.type, entity.collection, entity.id);
|
||||
this.#touched.delete(target);
|
||||
this.#state.put(updatedEntity(entity, { ...entity.fields, ...object }));
|
||||
}
|
||||
|
||||
/** Whether an item read shows an entity an unmodelled write made, or one left unknown, again rather than checks it. */
|
||||
|
||||
@@ -30,9 +30,9 @@ it("bounds query parser expansion before publishing or merging observed names",
|
||||
const accumulator = new WebNetworkApiSurfaceAccumulator(64 * 1024, true);
|
||||
add(accumulator, record(0, "/products?kept=1"));
|
||||
const before = accumulator.page(0, 1);
|
||||
expect(
|
||||
accumulator.add(record(1, `/products?${"x=1&".repeat(200)}`)),
|
||||
).toMatchObject({
|
||||
const rejected = record(1, `/products?${"x=1&".repeat(200)}`);
|
||||
rejected.limitations = ["Uncommitted rejected record caveat"];
|
||||
expect(accumulator.add(rejected)).toMatchObject({
|
||||
ok: false,
|
||||
error: { _tag: "AnalysisResourceConstraintError" },
|
||||
});
|
||||
|
||||
@@ -92,8 +92,10 @@ export class WebNetworkApiSurfaceAccumulator extends WebNetworkProjectionAccumul
|
||||
}
|
||||
|
||||
add(record: WebNetworkCaptureRecord): Result<void, AnalysisError> {
|
||||
const limitations = this.addLimitations(record.limitations);
|
||||
const rollbackAdmission = this.retainedCheckpoint();
|
||||
const limitations = this.prepareLimitations(record.limitations);
|
||||
if (!limitations.ok) return limitations;
|
||||
let committed = false;
|
||||
try {
|
||||
const line = readRecordedRequestLine(
|
||||
record.reported,
|
||||
@@ -101,14 +103,23 @@ export class WebNetworkApiSurfaceAccumulator extends WebNetworkProjectionAccumul
|
||||
this.scratch,
|
||||
);
|
||||
if (line === null) return this.overBudget();
|
||||
let appended: Result<void, AnalysisError>;
|
||||
if (line.kind === "excluded")
|
||||
return this.#exclude(record.ordinal, line.reason);
|
||||
const route = scanRouteTemplate(line.path, this.scratch);
|
||||
if (route === null) return this.overBudget();
|
||||
const queryNames = readQueryParameterNames(line.query, this.scratch);
|
||||
if (queryNames === null) return this.overBudget();
|
||||
return this.#append(record.ordinal, line, route, queryNames);
|
||||
appended = this.#exclude(record.ordinal, line.reason);
|
||||
else {
|
||||
const route = scanRouteTemplate(line.path, this.scratch);
|
||||
if (route === null) return this.overBudget();
|
||||
const queryNames = readQueryParameterNames(line.query, this.scratch);
|
||||
if (queryNames === null) return this.overBudget();
|
||||
appended = this.#append(record.ordinal, line, route, queryNames);
|
||||
}
|
||||
if (appended.ok) {
|
||||
limitations.value();
|
||||
committed = true;
|
||||
}
|
||||
return appended;
|
||||
} finally {
|
||||
if (!committed) rollbackAdmission();
|
||||
this.releaseTemporary();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -105,15 +105,54 @@ export abstract class WebNetworkProjectionAccumulator {
|
||||
}
|
||||
|
||||
addLimitations(limitations: readonly string[]): Result<void, AnalysisError> {
|
||||
for (const limitation of limitations) {
|
||||
if (this.#limitations.has(limitation)) continue;
|
||||
if (!this.retain(ENTRY_BYTES + textBytes(limitation.length)))
|
||||
return this.overBudget();
|
||||
this.#limitations.add(limitation);
|
||||
}
|
||||
const prepared = this.prepareLimitations(limitations);
|
||||
if (!prepared.ok) return prepared;
|
||||
prepared.value();
|
||||
return ok(undefined);
|
||||
}
|
||||
|
||||
/** Admit record qualifications without publishing them before the record commits. */
|
||||
protected prepareLimitations(
|
||||
limitations: readonly string[],
|
||||
): Result<() => void, AnalysisError> {
|
||||
const rollbackAdmission = this.retainedCheckpoint();
|
||||
const pending = new Set<string>();
|
||||
for (const limitation of limitations) {
|
||||
if (this.#limitations.has(limitation) || pending.has(limitation))
|
||||
continue;
|
||||
if (!this.retain(ENTRY_BYTES + textBytes(limitation.length))) {
|
||||
rollbackAdmission();
|
||||
return this.overBudget();
|
||||
}
|
||||
pending.add(limitation);
|
||||
}
|
||||
return ok(() => {
|
||||
for (const limitation of pending) this.#limitations.add(limitation);
|
||||
});
|
||||
}
|
||||
|
||||
/** Derive budgeted qualifications for this snapshot without retaining old snapshots. */
|
||||
protected snapshotLimitations(
|
||||
derived: readonly string[],
|
||||
): Result<string[], AnalysisError> {
|
||||
const release = this.temporaryCheckpoint();
|
||||
if (
|
||||
!this.scratch(
|
||||
derived.reduce(
|
||||
(bytes, limitation) =>
|
||||
bytes + ENTRY_BYTES + textBytes(limitation.length),
|
||||
0,
|
||||
),
|
||||
)
|
||||
)
|
||||
return this.overBudget();
|
||||
try {
|
||||
return ok([...new Set([...this.#limitations, ...derived])].sort(compare));
|
||||
} finally {
|
||||
release();
|
||||
}
|
||||
}
|
||||
|
||||
protected sortedLimitations(): string[] {
|
||||
return [...this.#limitations].sort(compare);
|
||||
}
|
||||
|
||||
@@ -67,6 +67,19 @@ const nativeParent = (
|
||||
artifact: { path: "/fixtures/example.exe", sha256: "a".repeat(64) },
|
||||
});
|
||||
|
||||
it("returns input errors for malformed parent Evidence identifiers before projection", () => {
|
||||
for (const evidenceId of [`xx_${"a".repeat(64)}`, "ev_bad"])
|
||||
expect(
|
||||
projectAnalysisView(
|
||||
{ ...nativeParent(), evidenceId },
|
||||
{ kind: "summary" },
|
||||
),
|
||||
).toMatchObject({ ok: false, error: { _tag: "AnalysisInputError" } });
|
||||
expect(projectAnalysisView(nativeParent(), { kind: "summary" }).ok).toBe(
|
||||
true,
|
||||
);
|
||||
});
|
||||
|
||||
it("projects native procedure and bounded assembly and high-pcode with exact parent identity", () => {
|
||||
const parent = nativeParent();
|
||||
const procedure = projectAnalysisView(parent, {
|
||||
|
||||
@@ -53,7 +53,7 @@ import {
|
||||
analysisViewInputError,
|
||||
} from "./analysisViewErrors.js";
|
||||
import { digestCanonicalValue } from "../canonicalDigest.js";
|
||||
import { digestSchema } from "../digests.js";
|
||||
import { prefixedDigestSchema } from "../digests.js";
|
||||
import { err, ok, type Result } from "../result.js";
|
||||
import type { AnalysisError } from "../analysisErrorBase.js";
|
||||
import { safeParseOwnedBinaryLayout } from "../native/binaryLayout.js";
|
||||
@@ -269,7 +269,7 @@ export const sealAnalysisView = (
|
||||
view_digest: digestCanonicalValue(projection, "Analysis view"),
|
||||
});
|
||||
|
||||
const parentDigest = (evidenceId: string): string => evidenceId.slice(3);
|
||||
const parentEvidenceIdSchema = prefixedDigestSchema("ev");
|
||||
|
||||
const parseJavaScriptParent = (
|
||||
input: unknown,
|
||||
@@ -473,8 +473,7 @@ export const projectAnalysisView = (
|
||||
view: AnalysisViewRequest,
|
||||
accumulators: AnalysisViewAccumulators = {},
|
||||
): Result<AnalysisViewResult, AnalysisError> => {
|
||||
const digest = parentDigest(parent.evidenceId);
|
||||
if (!digestSchema.safeParse(digest).success)
|
||||
if (!parentEvidenceIdSchema.safeParse(parent.evidenceId).success)
|
||||
return err(
|
||||
analysisViewInputError([
|
||||
{
|
||||
|
||||
@@ -3,6 +3,7 @@ import {
|
||||
checkItemPlan,
|
||||
type StateGateContext,
|
||||
} from "./environmentStateGate.js";
|
||||
import { locateItem, locateItems } from "./environmentStateRules.js";
|
||||
import type {
|
||||
JsonObject,
|
||||
StateEntity,
|
||||
@@ -21,7 +22,7 @@ const check = (
|
||||
path: `/api/books/${ordinal + 1}`,
|
||||
query: undefined,
|
||||
status: 200,
|
||||
timing: { start: undefined, endMs: undefined },
|
||||
timing: { start: undefined, end: undefined, endMs: undefined },
|
||||
}));
|
||||
const bodies: JsonObject[] = clocks.map(([, stamp], index) => ({
|
||||
id: index + 1,
|
||||
@@ -116,3 +117,37 @@ describe("clock spelling facts at the read gate", () => {
|
||||
expect(large.bytes).toBe(small.bytes);
|
||||
});
|
||||
});
|
||||
|
||||
describe("item response identity selection", () => {
|
||||
it("uses the known identity field before choosing an envelope tier", () => {
|
||||
const body = { request_id: "42", data: { id: "42", name: "Book" } };
|
||||
expect(locateItem(body, "42", "id")).toEqual({
|
||||
pointer: "/data",
|
||||
keys: ["id"],
|
||||
});
|
||||
expect(locateItem(body, "42")).toEqual({
|
||||
pointer: "",
|
||||
keys: ["request_id"],
|
||||
});
|
||||
expect(locateItem(body, "42", "uid")).toBeUndefined();
|
||||
});
|
||||
|
||||
it("retains ambiguity between objects carrying the known identity", () => {
|
||||
const body = { request_id: "42", left: { id: "42" }, right: { id: "42" } };
|
||||
expect(locateItem(body, "42", "id")).toBeUndefined();
|
||||
expect(locateItems(body, "42", "id")).toEqual([
|
||||
{ pointer: "/left", keys: ["id"] },
|
||||
{ pointer: "/right", keys: ["id"] },
|
||||
]);
|
||||
expect(locateItems(body, "42", "uid")).toEqual([]);
|
||||
});
|
||||
|
||||
it("admits an explicitly known custom identity in a sole array element", () => {
|
||||
const body = { data: [{ bookNumber: "42", name: "Book" }] };
|
||||
expect(locateItem(body, "42", "bookNumber")).toEqual({
|
||||
pointer: "/data/0",
|
||||
keys: ["bookNumber"],
|
||||
});
|
||||
expect(locateItem(body, "42")).toBeUndefined();
|
||||
});
|
||||
});
|
||||
@@ -108,3 +108,75 @@ it("takes no nested member for a counter unless it equals the number listed", ()
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
it.each([{ nextCursor: "cursor-2" }, { has_more: true }, { nextPage: 2 }])(
|
||||
"keeps membership recorded when paging metadata says more items follow: %j",
|
||||
(pageInfo) => {
|
||||
const state = environmentStateViewOf(
|
||||
timedCaptureEvidence([
|
||||
["GET", "/api/records", { list: SEEDED, pageInfo }],
|
||||
...SEEDED.map((item): TimedCaptureRow => [
|
||||
"GET",
|
||||
`/api/records/${String(item["Id"])}`,
|
||||
item,
|
||||
]),
|
||||
]),
|
||||
);
|
||||
expect(
|
||||
state.views.find((view) => view.tool === "get_records"),
|
||||
).toMatchObject({
|
||||
shape: "list",
|
||||
membership: "recorded",
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
it("keeps completed paging envelopes whole and does not read entity fields as continuation metadata", () => {
|
||||
const items = SEEDED.map((item): JsonObject => ({
|
||||
...item,
|
||||
nextCursor: "entity-value",
|
||||
}));
|
||||
const state = environmentStateViewOf(
|
||||
timedCaptureEvidence([
|
||||
[
|
||||
"GET",
|
||||
"/api/records",
|
||||
{ list: items, pageInfo: { nextCursor: null, has_more: false } },
|
||||
],
|
||||
...items.map((item): TimedCaptureRow => [
|
||||
"GET",
|
||||
`/api/records/${String(item["Id"])}`,
|
||||
item,
|
||||
]),
|
||||
]),
|
||||
);
|
||||
expect(state.views.find((view) => view.tool === "get_records")).toMatchObject(
|
||||
{
|
||||
shape: "list",
|
||||
membership: "all",
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
it("keeps a direction-only recording on its recorded members instead of learning an unused direction", () => {
|
||||
const state = environmentStateViewOf(
|
||||
timedCaptureEvidence([
|
||||
["GET", "/api/records", SEEDED],
|
||||
["GET", "/api/records?direction=desc", SEEDED],
|
||||
...SEEDED.map((item): TimedCaptureRow => [
|
||||
"GET",
|
||||
`/api/records/${String(item["Id"])}`,
|
||||
item,
|
||||
]),
|
||||
]),
|
||||
);
|
||||
const view = state.views.find(
|
||||
(candidate) => candidate.tool === "get_records",
|
||||
);
|
||||
expect(view).toMatchObject({
|
||||
shape: "list",
|
||||
membership: "recorded",
|
||||
checked_record_ordinals: [0, 1],
|
||||
});
|
||||
expect(view).not.toHaveProperty("query.parameters.direction");
|
||||
});
|
||||
@@ -72,7 +72,7 @@ export const listFacts = (
|
||||
counted.some(
|
||||
([, value]) => recordedItemCount(value) !== items.length,
|
||||
) ||
|
||||
Object.keys(parent).some((name) => continues(name, parent[name]))
|
||||
hasContinuation(parent, plan.pointer)
|
||||
)
|
||||
return;
|
||||
const ids = items.map((item) => scalarText(ownMember(item, field)) ?? "");
|
||||
@@ -150,3 +150,15 @@ const continues = (name: string, value: JsonValue | undefined): boolean =>
|
||||
value !== null &&
|
||||
value !== false &&
|
||||
value !== "";
|
||||
|
||||
/** Continuation facts in the envelope or its sibling paging objects, never inside the list payload. */
|
||||
const hasContinuation = (parent: JsonObject, listPointer: string): boolean => {
|
||||
const root = pointerTokens(listPointer)[0];
|
||||
return Object.entries(parent).some(
|
||||
([name, value]) =>
|
||||
continues(name, value) ||
|
||||
(name !== root &&
|
||||
isObject(value) &&
|
||||
Object.entries(value).some(([inner, flag]) => continues(inner, flag))),
|
||||
);
|
||||
};
|
||||
@@ -376,6 +376,16 @@ describe("queries the state can always answer", () => {
|
||||
});
|
||||
|
||||
describe("parameters that ask for an order", () => {
|
||||
it("cannot apply a direction without a sort field, even when the recorded order happens to match", () => {
|
||||
expect(explainLine(line("?direction=desc", [1, 2, 3, 4]))).toBeUndefined();
|
||||
const parameters = { direction: { role: "direction" } } as const;
|
||||
expect(ids("?direction=asc", parameters)).toBeUndefined();
|
||||
expect(ids("?direction=desc", parameters)).toBeUndefined();
|
||||
expect(
|
||||
ids("?sort=id&direction=desc", { ...parameters, sort: { role: "sort" } }),
|
||||
).toEqual(["4", "3", "2", "1"]);
|
||||
});
|
||||
|
||||
it("reads a sort field in any spelling as a sort, never as a parameter without effect", () => {
|
||||
expect(explainLine(line("?_sort=id&_order=DESC", [4, 3, 2, 1]))).toEqual({
|
||||
_sort: { role: "sort" },
|
||||
|
||||
@@ -206,7 +206,11 @@ const readCall = (
|
||||
call.page = { index: number, first: parameter.first };
|
||||
else return undefined;
|
||||
}
|
||||
return call;
|
||||
// A direction cannot identify the field being ordered. Leaving it unused
|
||||
// would turn a recorded direction into permission to answer its opposite.
|
||||
return call.direction !== undefined && call.sort === undefined
|
||||
? undefined
|
||||
: call;
|
||||
};
|
||||
|
||||
/** Entities in the order a sort value names, ties in state order, and those whose place state order decides: all without a sort, else those tied with a neighbour; undefined when it names no field. */
|
||||
|
||||
@@ -51,6 +51,12 @@ const readOnlyOf = (
|
||||
),
|
||||
ms: call.start ?? index,
|
||||
},
|
||||
end:
|
||||
call.end === null
|
||||
? undefined
|
||||
: parseOffsetDateTimeInstant(
|
||||
new Date(call.end ?? index).toISOString(),
|
||||
),
|
||||
endMs: call.end === null ? undefined : (call.end ?? index),
|
||||
},
|
||||
}));
|
||||
@@ -505,6 +511,7 @@ describe("read-only decisions over many calls", () => {
|
||||
),
|
||||
ms: index,
|
||||
},
|
||||
end: parseOffsetDateTimeInstant(new Date(index).toISOString()),
|
||||
endMs: index,
|
||||
},
|
||||
};
|
||||
@@ -573,6 +580,7 @@ describe("read-only decisions over a long run of calls", () => {
|
||||
),
|
||||
ms: index,
|
||||
},
|
||||
end: parseOffsetDateTimeInstant(new Date(index).toISOString()),
|
||||
endMs: index,
|
||||
},
|
||||
};
|
||||
@@ -641,6 +649,7 @@ describe("read-only decisions over many lines", () => {
|
||||
),
|
||||
ms: index,
|
||||
},
|
||||
end: parseOffsetDateTimeInstant(new Date(index).toISOString()),
|
||||
endMs: index,
|
||||
},
|
||||
};
|
||||
|
||||
@@ -24,6 +24,10 @@ const record = (
|
||||
instant: parseOffsetDateTimeInstant(new Date(start).toISOString()),
|
||||
ms: start,
|
||||
},
|
||||
end:
|
||||
endMs === undefined
|
||||
? undefined
|
||||
: parseOffsetDateTimeInstant(new Date(endMs).toISOString()),
|
||||
endMs,
|
||||
},
|
||||
});
|
||||
|
||||
@@ -31,6 +31,8 @@ export interface StateRecordFacts {
|
||||
readonly start:
|
||||
| { readonly instant: OffsetDateTimeInstant; readonly ms: number }
|
||||
| undefined;
|
||||
/** Exact completion, independent of the millisecond chronology policy. */
|
||||
readonly end: OffsetDateTimeInstant | undefined;
|
||||
readonly endMs: number | undefined;
|
||||
};
|
||||
}
|
||||
@@ -630,19 +632,22 @@ export const replaceAt = (
|
||||
const matchingKeys = (object: JsonObject, value: string): string[] =>
|
||||
Object.keys(object).filter((key) => scalarText(object[key]) === value);
|
||||
|
||||
/** The entity object an item response carries with the keys repeating its path value: at the root or in one wrapper member, else as the element of a one-element array there. */
|
||||
export const locateItem = (
|
||||
/** Candidate entity objects at the first matching envelope tier. A known identity field constrains admission before choosing a tier; multiple candidates remain ambiguous. */
|
||||
export const locateItems = (
|
||||
body: JsonValue,
|
||||
value: string,
|
||||
): { readonly pointer: string; readonly keys: string[] } | undefined => {
|
||||
idField?: string,
|
||||
): readonly { readonly pointer: string; readonly keys: string[] }[] => {
|
||||
const located = (
|
||||
object: JsonValue | undefined,
|
||||
pointer: string,
|
||||
named = false,
|
||||
) => {
|
||||
const keys = isObject(object)
|
||||
? matchingKeys(object, value).filter(
|
||||
(key) => !named || IDENTITY_NAMES.includes(key),
|
||||
? matchingKeys(object, value).filter((key) =>
|
||||
idField !== undefined
|
||||
? key === idField
|
||||
: !named || IDENTITY_NAMES.includes(key),
|
||||
)
|
||||
: [];
|
||||
return keys.length > 0 ? [{ pointer, keys }] : [];
|
||||
@@ -658,8 +663,17 @@ export const locateItem = (
|
||||
located(soleElement(item), `${pointerTo(name)}/0`, true),
|
||||
),
|
||||
];
|
||||
const chosen = tiers.find((tier) => tier.length > 0);
|
||||
return chosen?.length === 1 ? chosen[0] : undefined;
|
||||
return tiers.find((tier) => tier.length > 0) ?? [];
|
||||
};
|
||||
|
||||
/** The unique entity object at the first matching envelope tier; absent or ambiguous otherwise. */
|
||||
export const locateItem = (
|
||||
body: JsonValue,
|
||||
value: string,
|
||||
idField?: string,
|
||||
): { readonly pointer: string; readonly keys: string[] } | undefined => {
|
||||
const candidates = locateItems(body, value, idField);
|
||||
return candidates.length === 1 ? candidates[0] : undefined;
|
||||
};
|
||||
|
||||
/** Whether every item carries `name` as a distinct string or number. */
|
||||
|
||||
@@ -12,7 +12,7 @@ import {
|
||||
scenarioCaptureFixture,
|
||||
} from "../../../tests/fixtures/browserScenarioCapture.js";
|
||||
import type { Evidence } from "../evidence.js";
|
||||
import type { JsonValue } from "../jsonValue.js";
|
||||
import { ownedJsonObjectSchema, type JsonValue } from "../jsonValue.js";
|
||||
import { webNetworkCaptureSchema } from "../webNetworkCapture.js";
|
||||
import {
|
||||
analysisViewResultSchema,
|
||||
@@ -842,7 +842,7 @@ it("leaves every line checked rather than compare more lines than the variant pa
|
||||
path: "/api/items",
|
||||
query: `?view=${String(ordinal)}`,
|
||||
status: 200,
|
||||
timing: { start: undefined, endMs: undefined },
|
||||
timing: { start: undefined, end: undefined, endMs: undefined },
|
||||
}));
|
||||
const shape = {
|
||||
entities: new Map(
|
||||
@@ -869,7 +869,7 @@ it("tells apart member names that hold the separator, so a line spelling them ot
|
||||
path: "/api/items/1",
|
||||
query,
|
||||
status: 200,
|
||||
timing: { start: undefined, endMs: undefined },
|
||||
timing: { start: undefined, end: undefined, endMs: undefined },
|
||||
});
|
||||
const observed = (ordinal: number, object: JsonObject) => ({
|
||||
position: ordinal,
|
||||
@@ -909,7 +909,7 @@ it("leaves every line checked when comparing many mostly empty lines would excee
|
||||
method: "GET",
|
||||
path: "/api/items",
|
||||
status: 200,
|
||||
timing: { start: undefined, endMs: undefined },
|
||||
timing: { start: undefined, end: undefined, endMs: undefined },
|
||||
};
|
||||
return Object.defineProperty(record, "query", {
|
||||
enumerable: true,
|
||||
@@ -950,7 +950,7 @@ const expandedItemLines = (items: number) => {
|
||||
path: `/api/items/${String(ordinal % items)}`,
|
||||
query: ordinal < items ? undefined : "?expand=shelf",
|
||||
status: 200,
|
||||
timing: { start: undefined, endMs: undefined },
|
||||
timing: { start: undefined, end: undefined, endMs: undefined },
|
||||
}),
|
||||
);
|
||||
const shapes = new Map(
|
||||
@@ -1144,6 +1144,7 @@ it("finds each span's earliest write without scanning the writes before it", ()
|
||||
return start;
|
||||
},
|
||||
},
|
||||
end: offsetDateTimeInstantFromEpochMilliseconds(end),
|
||||
get endMs() {
|
||||
reads += 1;
|
||||
return end;
|
||||
@@ -2020,3 +2021,168 @@ it.each([
|
||||
if (missingDuration) expect(state.write_clock).toBeNull();
|
||||
},
|
||||
);
|
||||
|
||||
it.each(["not a URL", "", 42, undefined])(
|
||||
"bounds initial state at a successful write whose URL is unavailable: %j",
|
||||
(url) => {
|
||||
const source = captureOf([
|
||||
["GET", "/api/items/1", { id: 1, title: "initial" }],
|
||||
["PUT", "/api/items/1", { id: 1 }],
|
||||
["GET", "/api/items/1", { id: 1, title: "after" }],
|
||||
]);
|
||||
const capture = webNetworkCaptureSchema.parse(source.normalizedResult);
|
||||
const reported = ownedJsonObjectSchema.parse(capture.records[1]!.reported);
|
||||
const request = ownedJsonObjectSchema.parse(reported.request);
|
||||
if (url === undefined) delete request.url;
|
||||
else request.url = url;
|
||||
const before = structuredClone(capture);
|
||||
const state = stateOf(parentOf(analysisViewCaptureEvidence(capture)));
|
||||
expect(state.types).toMatchObject([
|
||||
{ init_before_ordinal: 1, volatile_fields: [] },
|
||||
]);
|
||||
expect(state.entities).toMatchObject([
|
||||
{ fields: { title: "initial" }, record_ordinals: [0] },
|
||||
]);
|
||||
expect(state.limitations).toContainEqual(
|
||||
expect.stringMatching(/1.*writes.*unusable URL.*target is unknown/iu),
|
||||
);
|
||||
expect(
|
||||
state.limitations.some((text) => text.includes("redacted URL")),
|
||||
).toBe(false);
|
||||
expect(capture).toEqual(before);
|
||||
},
|
||||
);
|
||||
|
||||
it("commits neither a manifest line nor state facts when inherited state admission fails", () => {
|
||||
const records = webNetworkCaptureSchema.parse(
|
||||
environmentManifestCaptureFixture([
|
||||
{
|
||||
startedDateTime: "2026-10-20T09:00:00Z",
|
||||
time: 0,
|
||||
request: { method: "GET", url: "https://manifest.test/api/health" },
|
||||
response: {
|
||||
status: 200,
|
||||
content: { mimeType: "application/json", text: '{"healthy":true}' },
|
||||
},
|
||||
},
|
||||
{
|
||||
startedDateTime: "x".repeat(5000),
|
||||
time: 0,
|
||||
request: { method: "GET", url: "https://manifest.test/api/items" },
|
||||
response: {
|
||||
status: 200,
|
||||
content: { mimeType: "application/json", text: "[]" },
|
||||
},
|
||||
},
|
||||
]),
|
||||
).records;
|
||||
const accumulator = new EnvironmentStateAccumulator(32 * 1024);
|
||||
expect(accumulator.add(records[0]!).ok).toBe(true);
|
||||
expect(accumulator.add(records[1]!).ok).toBe(false);
|
||||
const admitted = accumulator.finish();
|
||||
if (!admitted.ok) throw admitted.error;
|
||||
expect(admitted.value.records.map(({ ordinal }) => ordinal)).toEqual([0]);
|
||||
const retry = structuredClone(records[1]!);
|
||||
if (
|
||||
retry.reported === null ||
|
||||
typeof retry.reported !== "object" ||
|
||||
Array.isArray(retry.reported)
|
||||
)
|
||||
throw new Error("Expected fixture object");
|
||||
retry.reported.startedDateTime = "2026-10-20T09:00:01Z";
|
||||
expect(accumulator.add(retry).ok).toBe(true);
|
||||
const state = accumulator.finishState();
|
||||
if (!state.ok) throw state.error;
|
||||
expect(state.value.documents.map(({ path }) => path)).toEqual([
|
||||
"/api/health",
|
||||
"/api/items",
|
||||
]);
|
||||
});
|
||||
|
||||
it.each([
|
||||
{ method: "GET", status: 200 },
|
||||
{ method: "PUT", status: 500 },
|
||||
])(
|
||||
"does not infer a successful state write from an excluded $method/$status record",
|
||||
({ method, status }) => {
|
||||
const source = captureOf([
|
||||
["GET", "/api/items/1", { id: 1, title: "initial" }],
|
||||
[method, "/api/items/1", {}, status],
|
||||
["GET", "/api/items/1", { id: 1, title: "after" }],
|
||||
]);
|
||||
const capture = webNetworkCaptureSchema.parse(source.normalizedResult);
|
||||
const reported = ownedJsonObjectSchema.parse(capture.records[1]!.reported);
|
||||
ownedJsonObjectSchema.parse(reported.request).url = "not a URL";
|
||||
const state = stateOf(parentOf(analysisViewCaptureEvidence(capture)));
|
||||
expect(state.types).toMatchObject([
|
||||
{ init_before_ordinal: null, volatile_fields: ["title"] },
|
||||
]);
|
||||
expect(state.entities).toMatchObject([{ fields: { title: "after" } }]);
|
||||
expect(
|
||||
state.limitations.some((text) =>
|
||||
text.includes("writes had an unusable URL"),
|
||||
),
|
||||
).toBe(false);
|
||||
},
|
||||
);
|
||||
|
||||
it.each([{}, { status: "unknown" }, { status: null }])(
|
||||
"keeps a possible unlocated write when its response status is unknown: %j",
|
||||
(response) => {
|
||||
const source = captureOf([
|
||||
["GET", "/api/items/1", { id: 1, title: "initial" }],
|
||||
["PUT", "/api/items/1", {}],
|
||||
["GET", "/api/items/1", { id: 1, title: "after" }],
|
||||
]);
|
||||
const capture = webNetworkCaptureSchema.parse(source.normalizedResult);
|
||||
const reported = ownedJsonObjectSchema.parse(capture.records[1]!.reported);
|
||||
ownedJsonObjectSchema.parse(reported.request).url = "not a URL";
|
||||
reported.response = ownedJsonObjectSchema.parse(response);
|
||||
const state = stateOf(parentOf(analysisViewCaptureEvidence(capture)));
|
||||
expect(state.types).toMatchObject([
|
||||
{ init_before_ordinal: 1, volatile_fields: [] },
|
||||
]);
|
||||
expect(state.entities).toMatchObject([
|
||||
{ fields: { title: "initial" }, record_ordinals: [0] },
|
||||
]);
|
||||
},
|
||||
);
|
||||
|
||||
it.each([true, false])(
|
||||
"derives timeline limitations from the current state snapshot (timed: %s)",
|
||||
(timed) => {
|
||||
const accumulator = new EnvironmentStateAccumulator();
|
||||
const empty = accumulator.finishState();
|
||||
if (!empty.ok) throw empty.error;
|
||||
expect(empty.value.timeline).toBe("record-order");
|
||||
const record = webNetworkCaptureSchema.parse(
|
||||
environmentManifestCaptureFixture([
|
||||
{
|
||||
...(timed ? { startedDateTime: "2026-10-20T09:00:00Z" } : {}),
|
||||
time: 0,
|
||||
request: { method: "GET", url: "https://snapshot.test/api/items" },
|
||||
response: {
|
||||
status: 200,
|
||||
content: { mimeType: "application/json", text: "[]" },
|
||||
},
|
||||
},
|
||||
]),
|
||||
).records[0]!;
|
||||
expect(accumulator.add(record).ok).toBe(true);
|
||||
const current = accumulator.finishState();
|
||||
if (!current.ok) throw current.error;
|
||||
expect(current.value.timeline).toBe(
|
||||
timed ? "recorded-start-time" : "record-order",
|
||||
);
|
||||
expect(
|
||||
current.value.limitations.some((text) =>
|
||||
text.includes("Some records carry no start time"),
|
||||
),
|
||||
).toBe(!timed);
|
||||
expect(
|
||||
current.value.limitations.some((text) =>
|
||||
text.includes("No retained record had a parseable origin"),
|
||||
),
|
||||
).toBe(false);
|
||||
},
|
||||
);
|
||||
@@ -195,6 +195,34 @@ describe("item actions", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("locates an action's entity by its known id field before response metadata", () => {
|
||||
const response = { request_id: 1, data: ARCHIVED };
|
||||
expect(
|
||||
planOf(
|
||||
[
|
||||
[
|
||||
"GET",
|
||||
"/api/books",
|
||||
[shelved(FIRST_BOOK, false), shelved(SECOND_BOOK, false)],
|
||||
],
|
||||
["GET", "/api/books/1", { data: shelved(FIRST_BOOK, false) }],
|
||||
["GET", "/api/books/2", { data: shelved(SECOND_BOOK, false) }],
|
||||
["POST", "/api/books/1/archive", response],
|
||||
["GET", "/api/books/1", { data: ARCHIVED }],
|
||||
["POST", "/api/books/1/archive", response],
|
||||
["GET", "/api/books/1", { data: ARCHIVED }],
|
||||
],
|
||||
"post_books_archive_by_id",
|
||||
),
|
||||
).toMatchObject({
|
||||
effect: "update",
|
||||
served: true,
|
||||
address: { by: "action", segment: "archive" },
|
||||
response: { shape: "entity", pointer: "/data" },
|
||||
rules: [{ field: "archived", rule: "set", value: true }],
|
||||
});
|
||||
});
|
||||
|
||||
it("serves no action seen changing its field only once, which may toggle it", () => {
|
||||
expect(
|
||||
planOf(
|
||||
|
||||
@@ -18,6 +18,7 @@ import { entityTarget, succeeded } from "./environmentStateWriteChecks.js";
|
||||
import type { WriteIo } from "./environmentStateWriteEffects.js";
|
||||
import {
|
||||
pathValuesOf,
|
||||
pathParameterKeys,
|
||||
type BodyAddress,
|
||||
type StateWritePlan,
|
||||
type WriteAddress,
|
||||
@@ -95,19 +96,20 @@ export const itemAction = (
|
||||
)
|
||||
return undefined;
|
||||
const views = facts.itemViews(target);
|
||||
const itemKey = pathParameterKeys(tool.path_template).get(
|
||||
segments.length - 2,
|
||||
)!;
|
||||
const showsEntity = (record: StateRecordFacts): boolean => {
|
||||
const id = pathValuesOf(tool.path_template, record.path)?.get(
|
||||
item.slice(1, -1),
|
||||
);
|
||||
const id = pathValuesOf(tool.path_template, record.path)?.get(itemKey);
|
||||
if (id === undefined || record.status === 201) return false;
|
||||
if (facts.emptyResponse(record.ordinal)) return true;
|
||||
return facts.withBody(record.ordinal, (body) => {
|
||||
if (body === undefined || body === null) return true;
|
||||
const located = locateItem(body, id);
|
||||
const located = locateItem(body, id, idField);
|
||||
const object =
|
||||
located === undefined ? undefined : at(body, located.pointer);
|
||||
return (
|
||||
located?.keys.includes(idField) === true &&
|
||||
located !== undefined &&
|
||||
isObject(object) &&
|
||||
views.some(
|
||||
(view) =>
|
||||
|
||||
@@ -152,7 +152,7 @@ export class FailedReadBlame {
|
||||
const response =
|
||||
call === undefined
|
||||
? undefined
|
||||
: this.#responseObject(records[call]!, item.entity.id);
|
||||
: this.#responseObject(records[call]!, item.entity);
|
||||
if (response !== undefined) responses.set(name, response);
|
||||
}
|
||||
return fieldBlame(
|
||||
@@ -169,10 +169,13 @@ export class FailedReadBlame {
|
||||
/** The entity a write's response shows, located by its id. */
|
||||
#responseObject(
|
||||
record: StateRecordFacts,
|
||||
id: string,
|
||||
entity: StateEntity,
|
||||
): JsonObject | undefined {
|
||||
return this.#facts.io.withBody(record.ordinal, (body) => {
|
||||
const located = body === undefined ? undefined : locateItem(body, id);
|
||||
const located =
|
||||
body === undefined
|
||||
? undefined
|
||||
: locateItem(body, entity.id, this.#facts.idField(entity.type));
|
||||
const object =
|
||||
located === undefined ? undefined : at(body!, located.pointer);
|
||||
return isObject(object) ? object : undefined;
|
||||
|
||||
@@ -119,6 +119,22 @@ it(
|
||||
);
|
||||
|
||||
describe("coincidences no member derives a created field from", () => {
|
||||
it.each([
|
||||
["before", (value: string) => `𐐀${value}`],
|
||||
["after", (value: string) => `${value}𐐀`],
|
||||
])(
|
||||
"keeps a member accepted when an astral letter joins its text %s",
|
||||
(_, label) => {
|
||||
const value = "harbour-notes";
|
||||
const body = create(
|
||||
{ id: 9, name: value, label: label(value) },
|
||||
{ name: value },
|
||||
);
|
||||
expect(body.accepted).toEqual(["name"]);
|
||||
expect(body.fixed).toEqual([]);
|
||||
},
|
||||
);
|
||||
|
||||
it("keeps a reference accepted when a constant repeats another reference of the named entity", () => {
|
||||
const channel = "1bdxw3qoyj8zjyyhdjbdobdf4r";
|
||||
const user = "c10c054bc2a4478bb78e0210549376b1";
|
||||
|
||||
@@ -8,7 +8,12 @@ import {
|
||||
} from "./environmentStateRules.js";
|
||||
import { GENERATED } from "./environmentStateWriteGenerated.js";
|
||||
import type { FieldRule } from "./environmentStateWriteRules.js";
|
||||
import { delimitedOffsets, readClock } from "./environmentStateWriteValues.js";
|
||||
import {
|
||||
delimitedOffsets,
|
||||
joinsAfter,
|
||||
joinsBefore,
|
||||
readClock,
|
||||
} from "./environmentStateWriteValues.js";
|
||||
|
||||
/** Shorter body strings are too common to show that a field copied one. */
|
||||
export const COPY_MIN_LENGTH = 8;
|
||||
@@ -52,12 +57,6 @@ export const storedMembers = (
|
||||
}),
|
||||
);
|
||||
|
||||
/** A letter or digit, which joins a token to the text around it. */
|
||||
const WORD = /[\p{L}\p{N}]/u;
|
||||
|
||||
const joins = (character: string | undefined): boolean =>
|
||||
character !== undefined && WORD.test(character);
|
||||
|
||||
/**
|
||||
* The texts that hold each token delimited, read in one pass per text: from each start after a
|
||||
* delimiter, each length a token has that a delimiter ends, rather than a search per token and text.
|
||||
@@ -72,10 +71,10 @@ const holdersOf = (
|
||||
const holders = new Map<string, Set<string>>();
|
||||
for (const [field, text] of texts)
|
||||
for (let at = 0; at < text.length; at += 1) {
|
||||
if (joins(text[at - 1])) continue;
|
||||
if (joinsBefore(text, at)) continue;
|
||||
for (const length of lengths) {
|
||||
if (at + length > text.length) break;
|
||||
if (joins(text[at + length])) continue;
|
||||
if (joinsAfter(text, at + length)) continue;
|
||||
const token = text.slice(at, at + length);
|
||||
if (tokens.has(token))
|
||||
holders.set(token, (holders.get(token) ?? new Set()).add(field));
|
||||
|
||||
@@ -18,6 +18,7 @@ const write = (path: string, ordinal: number): StateRecordFacts => ({
|
||||
instant: offsetDateTimeInstantFromEpochMilliseconds(ordinal),
|
||||
ms: ordinal,
|
||||
},
|
||||
end: offsetDateTimeInstantFromEpochMilliseconds(ordinal),
|
||||
endMs: ordinal,
|
||||
},
|
||||
});
|
||||
|
||||
@@ -0,0 +1,118 @@
|
||||
import { expect, it } from "vitest";
|
||||
|
||||
import {
|
||||
timedCaptureEvidence,
|
||||
type TimedCaptureRow,
|
||||
} from "../../../tests/fixtures/analysisView.js";
|
||||
import {
|
||||
FIRST_BOOK,
|
||||
environmentStateViewOf,
|
||||
} from "../../../tests/fixtures/environmentStateWrites.js";
|
||||
|
||||
it.each([
|
||||
{ fraction: "123456790", served: true, label: "one nanosecond later" },
|
||||
{ fraction: "123456788", served: false, label: "one nanosecond earlier" },
|
||||
{ fraction: "123456789", served: false, label: "the same instant" },
|
||||
{ fraction: "124456789", served: true, label: "one millisecond later" },
|
||||
])("checks an update clock at $label", ({ fraction, served }) => {
|
||||
const first = {
|
||||
...FIRST_BOOK,
|
||||
name: "Second title",
|
||||
updated_at: "2026-10-17T18:00:03.123456789Z",
|
||||
};
|
||||
const second = {
|
||||
...first,
|
||||
name: "Third title",
|
||||
updated_at: `2026-10-17T18:00:03.${fraction}Z`,
|
||||
};
|
||||
const state = environmentStateViewOf(
|
||||
timedCaptureEvidence([
|
||||
["GET", "/api/books", [FIRST_BOOK]],
|
||||
["GET", "/api/books/1", FIRST_BOOK],
|
||||
["PATCH", "/api/books/1", first, 200, { name: first.name }],
|
||||
["GET", "/api/books/1", first],
|
||||
["PATCH", "/api/books/1", second, 200, { name: second.name }],
|
||||
["GET", "/api/books/1", second],
|
||||
]),
|
||||
);
|
||||
expect(
|
||||
state.writes.find((plan) => plan.tool === "patch_books_by_id"),
|
||||
).toMatchObject({
|
||||
served,
|
||||
reason: served ? null : "unexplained-change",
|
||||
checked_record_ordinals: served ? [3, 5] : [3],
|
||||
rules: expect.arrayContaining([
|
||||
{
|
||||
field: "updated_at",
|
||||
rule: "clock",
|
||||
format: { kind: "iso", separator: "T", fraction: 9, zone: "Z" },
|
||||
},
|
||||
]),
|
||||
});
|
||||
});
|
||||
|
||||
it.each([false, true])(
|
||||
"learns only after an unknown entity is observed again (update while unknown: %s)",
|
||||
(updateWhileUnknown) => {
|
||||
const initial = { ...FIRST_BOOK, revision: 1 };
|
||||
const observed = { ...initial, name: "Second title", revision: 2 };
|
||||
const final = { ...observed, name: "Third title" };
|
||||
const unknownCall: TimedCaptureRow[] = updateWhileUnknown
|
||||
? [["PATCH", "/api/books/1", observed, 200, { name: observed.name }]]
|
||||
: [];
|
||||
const state = environmentStateViewOf(
|
||||
timedCaptureEvidence([
|
||||
["GET", "/api/books", [initial]],
|
||||
["GET", "/api/books/1", initial],
|
||||
["POST", "/api/books/1/import", { queued: true }, 302, {}],
|
||||
...unknownCall,
|
||||
["GET", "/api/books/1", observed],
|
||||
["PATCH", "/api/books/1", final, 200, { name: final.name }],
|
||||
["GET", "/api/books/1", final],
|
||||
]),
|
||||
);
|
||||
expect(
|
||||
state.writes.find((plan) => plan.tool === "patch_books_by_id"),
|
||||
).toMatchObject({
|
||||
served: true,
|
||||
reason: null,
|
||||
exemplar_ordinal: updateWhileUnknown ? 5 : 4,
|
||||
checked_record_ordinals: [updateWhileUnknown ? 6 : 5],
|
||||
rules: [{ field: "name", rule: "echo" }],
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
it.each([
|
||||
{ requestId: 1, ambiguous: false, served: true },
|
||||
{ requestId: 2, ambiguous: false, served: true },
|
||||
{ requestId: 1, ambiguous: true, served: false },
|
||||
])(
|
||||
"uses the known identity field for response observation ($requestId, ambiguous: $ambiguous)",
|
||||
({ requestId, ambiguous, served }) => {
|
||||
const changed = { ...FIRST_BOOK, name: "Second title" };
|
||||
const response = {
|
||||
request_id: requestId,
|
||||
data: changed,
|
||||
...(ambiguous ? { other: changed } : {}),
|
||||
};
|
||||
const state = environmentStateViewOf(
|
||||
timedCaptureEvidence([
|
||||
["GET", "/api/books", [FIRST_BOOK]],
|
||||
["GET", "/api/books/1", FIRST_BOOK],
|
||||
["PATCH", "/api/books/1", response, 200, { name: changed.name }],
|
||||
["GET", "/api/books/1", changed],
|
||||
]),
|
||||
);
|
||||
const plan = state.writes.find(
|
||||
(candidate) => candidate.tool === "patch_books_by_id",
|
||||
);
|
||||
expect(plan?.served).toBe(served);
|
||||
if (served)
|
||||
expect(plan).toMatchObject({
|
||||
observation: { ordinal: 2, pointer: "/data" },
|
||||
response: { ordinal: 2, shape: "entity", pointer: "/data" },
|
||||
checked_record_ordinals: [3],
|
||||
});
|
||||
},
|
||||
);
|
||||
@@ -28,6 +28,7 @@ import {
|
||||
type SimulatedState,
|
||||
} from "./environmentStateWriteChecks.js";
|
||||
import {
|
||||
compareValues,
|
||||
explainUpdate,
|
||||
responseShapeOf,
|
||||
} from "./environmentStateWriteExplain.js";
|
||||
@@ -367,11 +368,12 @@ const updatedObservation = (
|
||||
step.record.ordinal,
|
||||
(body): Observed | undefined => {
|
||||
const located =
|
||||
body === undefined ? undefined : locateItem(body, entity.id);
|
||||
body === undefined
|
||||
? undefined
|
||||
: locateItem(body, entity.id, entity.idField);
|
||||
const object =
|
||||
located === undefined ? undefined : at(body!, located.pointer);
|
||||
return located?.keys.includes(entity.idField) === true &&
|
||||
isObject(object)
|
||||
return located !== undefined && isObject(object)
|
||||
? {
|
||||
ordinal: step.record.ordinal,
|
||||
pointer: located.pointer,
|
||||
@@ -657,6 +659,9 @@ export const updateEntity = (
|
||||
return UNKNOWN;
|
||||
}
|
||||
const { targets, target, recorded } = outcomes(context, entity, after.object);
|
||||
// An unknown baseline cannot establish this call's changes. Keep its outcome
|
||||
// without creating or refining rules from values another write may have changed.
|
||||
if (context.unknown(type, collection, id)) return recorded();
|
||||
// Only confirmed absence makes a bodyless action; malformed or unretained content stays unknown.
|
||||
const body =
|
||||
"body" in named
|
||||
@@ -670,7 +675,6 @@ export const updateEntity = (
|
||||
fail(draft, step, "body-unusable");
|
||||
return recorded();
|
||||
}
|
||||
const unknown = context.unknown(type, collection, id);
|
||||
const call = { ...callOf(step, body), body };
|
||||
const fitted = fitUpdateClocks(
|
||||
draft,
|
||||
@@ -704,7 +708,7 @@ export const updateEntity = (
|
||||
const explained = { ...fitted.explained, rules: later };
|
||||
for (const member of keptOrders(call.body, after.object))
|
||||
draft.kept.add(member);
|
||||
if (explained.unexplained.length > 0 && !unknown) {
|
||||
if (explained.unexplained.length > 0) {
|
||||
refuse(draft, step, "unexplained-change");
|
||||
return recorded();
|
||||
}
|
||||
@@ -713,16 +717,15 @@ export const updateEntity = (
|
||||
searchFrom(context, step),
|
||||
itemReads(context, { type, idField, collection, id }),
|
||||
)?.object;
|
||||
// Each call must agree with the plan; one on an unknown entity cannot be checked, so it adds nothing to the plan.
|
||||
// Each call with a known baseline must agree with the plan.
|
||||
if (
|
||||
built(draft) &&
|
||||
(unknown ||
|
||||
contradictsPlan(
|
||||
{ ...draft, plan: { ...draft.plan, rules } },
|
||||
entity.fields,
|
||||
after.object,
|
||||
explained,
|
||||
) ||
|
||||
(contradictsPlan(
|
||||
{ ...draft, plan: { ...draft.plan, rules } },
|
||||
entity.fields,
|
||||
after.object,
|
||||
explained,
|
||||
) ||
|
||||
// A member calls left out is cleared by every one or by none.
|
||||
// A reference this call or an earlier one showed counts its member as the field's, whichever came first.
|
||||
defaultsDisagree(explained.body.defaults, { earlier: draft.updates }, [
|
||||
@@ -735,7 +738,7 @@ export const updateEntity = (
|
||||
[...rules, ...explained.rules],
|
||||
))
|
||||
) {
|
||||
if (!unknown) refuse(draft, step, "unexplained-change");
|
||||
refuse(draft, step, "unexplained-change");
|
||||
return recorded();
|
||||
}
|
||||
draft.plan.rules = rules;
|
||||
@@ -778,9 +781,10 @@ export const updateEntity = (
|
||||
after: after.object,
|
||||
kept,
|
||||
});
|
||||
if (!written.ok || (disagrees && !unknown))
|
||||
if (!written.ok || disagrees) {
|
||||
refuse(draft, step, "unexplained-change");
|
||||
if (!written.ok || disagrees || unknown) return recorded();
|
||||
return recorded();
|
||||
}
|
||||
rememberCall(
|
||||
context.io.retain,
|
||||
draft,
|
||||
@@ -893,7 +897,7 @@ const contradictsGeneratedClock = (
|
||||
candidate.field === rule.field && candidate.rule === "clock",
|
||||
) ||
|
||||
next === undefined ||
|
||||
next.ms <= previous.ms
|
||||
compareValues(after[rule.field], ownMember(before, rule.field))! <= 0
|
||||
);
|
||||
});
|
||||
|
||||
|
||||
@@ -13,6 +13,7 @@ import {
|
||||
type CreateEvidence,
|
||||
} from "./environmentStateWriteExplain.js";
|
||||
import type { UpdateEvidence } from "./environmentStateWriteChanges.js";
|
||||
import { writtenResponse } from "./environmentStateWriteApply.js";
|
||||
import {
|
||||
callRenewal,
|
||||
unexplainedRenewals,
|
||||
@@ -504,6 +505,35 @@ it.each<[string, string[]]>([
|
||||
|
||||
type Shaped = [WriteResponse["shape"], string | null];
|
||||
|
||||
it.each([false, true])(
|
||||
"keeps response entity selection distinct from repeated envelope metadata, ambiguous=%s",
|
||||
(ambiguous) => {
|
||||
const recorded = {
|
||||
request_id: "42",
|
||||
data: { id: "42", title: "Captured title" },
|
||||
...(ambiguous ? { other: { id: "42", title: "Another entity" } } : {}),
|
||||
};
|
||||
const shape = responseShapeOf(3, recorded, false, {
|
||||
idField: "id",
|
||||
id: "42",
|
||||
});
|
||||
expect(
|
||||
writtenResponse(shape, recorded, {
|
||||
id: "43",
|
||||
fields: { id: "43", title: "Authored title" },
|
||||
}),
|
||||
).toEqual({
|
||||
ok: true,
|
||||
value: ambiguous
|
||||
? recorded
|
||||
: {
|
||||
request_id: "42",
|
||||
data: { id: "43", title: "Authored title" },
|
||||
},
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
it.each<[string, JsonValue | undefined, boolean, Shaped]>([
|
||||
["no body", undefined, true, ["empty", null]],
|
||||
["an empty response with a body", { id: "42" }, true, ["empty", null]],
|
||||
|
||||
@@ -4,7 +4,7 @@ import { compareUnicodeCodePoints as compare } from "../unicodeCodePointOrder.js
|
||||
import {
|
||||
ownMember,
|
||||
isObject,
|
||||
locateItem,
|
||||
locateItems,
|
||||
pointerTo,
|
||||
same,
|
||||
scalarText,
|
||||
@@ -174,7 +174,7 @@ const requestRule = (
|
||||
index: BodyIndex,
|
||||
field: string,
|
||||
value: JsonValue,
|
||||
): FieldRule | undefined => {
|
||||
): FieldRule | "ambiguous-path" | undefined => {
|
||||
const { body, pathValues } = evidence.call;
|
||||
if (Object.hasOwn(body, field) && same(body[field], value))
|
||||
return { field, rule: "echo" };
|
||||
@@ -185,9 +185,11 @@ const requestRule = (
|
||||
if (from !== undefined) return { field, rule: "copy", from };
|
||||
}
|
||||
const text = scalarText(value);
|
||||
for (const [name, pathValue] of pathValues)
|
||||
if (text !== undefined && text === pathValue)
|
||||
return { field, rule: "path", from: name };
|
||||
const paths = [...pathValues].filter(
|
||||
([, value]) => text !== undefined && text === value,
|
||||
);
|
||||
if (paths.length > 1) return "ambiguous-path";
|
||||
if (paths.length === 1) return { field, rule: "path", from: paths[0]![0] };
|
||||
const slugged = typeof value === "string" && value.length >= 3;
|
||||
const source = slugged ? index.slugs.get(value) : undefined;
|
||||
return source === undefined ? undefined : slugRule(field, source);
|
||||
@@ -357,8 +359,14 @@ export const explainCreate = (
|
||||
const index = indexBody(evidence.call.body);
|
||||
// The member each slug the held entities' `&` spellings refused was slugged from.
|
||||
const refusedSlugs = new Map<string, string>();
|
||||
let ambiguousPath = false;
|
||||
for (const [field, value] of Object.entries(evidence.entity)) {
|
||||
const requested = requestRule(evidence, index, field, value);
|
||||
if (requested === "ambiguous-path") {
|
||||
ambiguousPath = true;
|
||||
unexplained.push(field);
|
||||
continue;
|
||||
}
|
||||
const held = heldSlugRule(requested, evidence.call.body, evidence.type);
|
||||
if (requested?.rule === "slug" && held === undefined)
|
||||
refusedSlugs.set(field, requested.from);
|
||||
@@ -468,12 +476,14 @@ export const explainCreate = (
|
||||
rules,
|
||||
body: blank.length === 0 ? shaped : { ...shaped, blank },
|
||||
// A template rebuilds only its generated value, so other text it keeps ties it like a constant.
|
||||
tied: [
|
||||
...[...constants, ...templates].map((field) =>
|
||||
ownMember(evidence.entity, field)!,
|
||||
),
|
||||
...literals,
|
||||
].some((text) => tokens.some((token) => containsDelimited(text, token))),
|
||||
tied:
|
||||
ambiguousPath ||
|
||||
[
|
||||
...[...constants, ...templates].map((field) =>
|
||||
ownMember(evidence.entity, field)!,
|
||||
),
|
||||
...literals,
|
||||
].some((text) => tokens.some((token) => containsDelimited(text, token))),
|
||||
};
|
||||
};
|
||||
|
||||
@@ -646,8 +656,11 @@ export const responseShapeOf = (
|
||||
): WriteResponse => {
|
||||
if (empty) return { ordinal, shape: "empty", pointer: null };
|
||||
if (body === undefined) return { ordinal, shape: "recorded", pointer: null };
|
||||
const located = locateItem(body, entity.id);
|
||||
if (located?.keys.includes(entity.idField) === true)
|
||||
const candidates = locateItems(body, entity.id, entity.idField);
|
||||
if (candidates.length > 1)
|
||||
return { ordinal, shape: "recorded", pointer: null };
|
||||
const located = candidates[0];
|
||||
if (located !== undefined)
|
||||
return { ordinal, shape: "entity", pointer: located.pointer };
|
||||
if (scalarText(body) === entity.id)
|
||||
return { ordinal, shape: "identity", pointer: "" };
|
||||
|
||||
@@ -22,6 +22,7 @@ const record = (
|
||||
instant: parseOffsetDateTimeInstant(new Date(start).toISOString()),
|
||||
ms: start,
|
||||
},
|
||||
end: parseOffsetDateTimeInstant(new Date(end).toISOString()),
|
||||
endMs: end,
|
||||
},
|
||||
});
|
||||
@@ -68,7 +69,7 @@ describe("short-id creation limits", () => {
|
||||
const prior = record(0, "GET", "/api/projects", [0, 5]);
|
||||
const unknown = {
|
||||
...prior,
|
||||
timing: { ...prior.timing, endMs: undefined },
|
||||
timing: { ...prior.timing, end: undefined, endMs: undefined },
|
||||
};
|
||||
const post = record(1, "POST", "/api/imports", [10, 11]);
|
||||
const later = record(2, "GET", "/api/projects/7", [20, 21]);
|
||||
|
||||
@@ -23,6 +23,7 @@ import {
|
||||
import { ENTRY_BYTES, textBytes } from "./webNetworkRouteTemplates.js";
|
||||
import {
|
||||
pathValuesOf,
|
||||
pathParameterKeys,
|
||||
type FieldRule,
|
||||
type WriteEffect,
|
||||
} from "./environmentStateWriteRules.js";
|
||||
@@ -414,13 +415,9 @@ export const anyLinkedSource = (
|
||||
plan.collection !== null &&
|
||||
normalizedPath(plan.collection) === plan.target &&
|
||||
!segmentsOf(plan.target).some(isParam) &&
|
||||
segmentsOf(template)
|
||||
.filter(isParam)
|
||||
.every((segment) =>
|
||||
plan.rules.some(
|
||||
(rule) => rule.rule === "path" && rule.from === segment.slice(1, -1),
|
||||
),
|
||||
);
|
||||
[...pathParameterKeys(template).values()].every((name) =>
|
||||
plan.rules.some((rule) => rule.rule === "path" && rule.from === name),
|
||||
);
|
||||
|
||||
/**
|
||||
* Whether a linked create's call sends the path values its exemplar did, each as recorded or in the
|
||||
|
||||
@@ -407,6 +407,23 @@ export interface WriteCall {
|
||||
readonly body: JsonObject | undefined;
|
||||
}
|
||||
|
||||
/** Parameter lookup keys preserve repeated placeholder occurrences by segment position. */
|
||||
export const pathParameterKeys = (template: string): Map<number, string> => {
|
||||
const parameters = segmentsOf(template).flatMap((segment, index) =>
|
||||
isParam(segment) ? [[index, segment.slice(1, -1)] as const] : [],
|
||||
);
|
||||
const repeated =
|
||||
new Set(parameters.map(([, name]) => name)).size !== parameters.length;
|
||||
// If any name repeats, encode every key so a literal name cannot collide with
|
||||
// an indexed key. Unique-name templates retain their established lookup keys.
|
||||
return new Map(
|
||||
parameters.map(([index, name]) => [
|
||||
index,
|
||||
repeated ? JSON.stringify([index, name]) : name,
|
||||
]),
|
||||
);
|
||||
};
|
||||
|
||||
/** The value each template parameter takes in a concrete path, decoded; undefined when the path does not instantiate it. */
|
||||
export const pathValuesOf = (
|
||||
template: string,
|
||||
@@ -416,13 +433,14 @@ export const pathValuesOf = (
|
||||
const concrete = segmentsOf(path);
|
||||
if (pattern.length !== concrete.length) return undefined;
|
||||
const values = new Map<string, string>();
|
||||
const keys = pathParameterKeys(template);
|
||||
for (const [index, segment] of pattern.entries()) {
|
||||
const value = concrete[index]!;
|
||||
if (!isParam(segment)) {
|
||||
if (segment !== value) return undefined;
|
||||
continue;
|
||||
}
|
||||
values.set(segment.slice(1, -1), decodedSegment(value));
|
||||
values.set(keys.get(index)!, decodedSegment(value));
|
||||
}
|
||||
return values;
|
||||
};
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import type { JsonValue } from "../jsonValue.js";
|
||||
import { jsonObjectSchema, type JsonValue } from "../jsonValue.js";
|
||||
import { webNetworkCaptureSchema } from "../webNetworkCapture.js";
|
||||
import { EnvironmentStateAccumulator } from "./EnvironmentStateAccumulator.js";
|
||||
import {
|
||||
@@ -86,6 +86,43 @@ it("retains learned clock rules but withholds their serving when the global cloc
|
||||
).toEqual(plan);
|
||||
});
|
||||
|
||||
it.each([
|
||||
["000000000", 0, "2026-10-17T18:00:12.000Z"],
|
||||
["000001", 0, "2026-10-17T18:00:13.000Z"],
|
||||
["000000001", 0, "2026-10-17T18:00:13.000Z"],
|
||||
["123000000", 0, "2026-10-17T18:00:13.000Z"],
|
||||
["999999", 0.001, "2026-10-17T18:00:13.000Z"],
|
||||
["999999", 0.001001, "2026-10-17T18:00:14.000Z"],
|
||||
])(
|
||||
"starts the served clock after a final captured fractional instant .%s plus %sms",
|
||||
(fraction, duration, expected) => {
|
||||
const evidence = timedCaptureEvidence([
|
||||
...BOOK_ROWS,
|
||||
["GET", "/api/health", { ready: true }, 200, undefined, 0],
|
||||
]);
|
||||
const capture = webNetworkCaptureSchema.parse(evidence.normalized_result);
|
||||
const state = environmentStateViewOf({
|
||||
...evidence,
|
||||
normalized_result: {
|
||||
...capture,
|
||||
records: capture.records.map((record, index) =>
|
||||
index === capture.records.length - 1
|
||||
? {
|
||||
...record,
|
||||
reported: {
|
||||
...jsonObjectSchema.parse(record.reported),
|
||||
startedDateTime: `2026-10-17T18:00:12.${fraction}Z`,
|
||||
time: duration,
|
||||
},
|
||||
}
|
||||
: record,
|
||||
),
|
||||
},
|
||||
});
|
||||
expect(state.write_clock).toBe(expected);
|
||||
},
|
||||
);
|
||||
|
||||
it.each(["delete", "mismatched-query"])(
|
||||
"does not parse unused request content for %s under a bounded state allowance",
|
||||
(kind) => {
|
||||
|
||||
@@ -2,6 +2,10 @@ import type { ExtractedState } from "./environmentStateRules.js";
|
||||
import { EnvironmentStateWriteSimulation } from "./EnvironmentStateWriteSimulation.js";
|
||||
import type { WriteSimulationInput } from "./environmentStateWriteEffects.js";
|
||||
import { usesWriteClock } from "./environmentStateWriteRules.js";
|
||||
import {
|
||||
compareOffsetDateTimeInstants,
|
||||
type OffsetDateTimeInstant,
|
||||
} from "../offsetDateTime.js";
|
||||
|
||||
/** The latest end a record can have and still round up to a whole second a date can hold. */
|
||||
const LATEST_END_MS = 8_640_000_000_000_000 - 1000;
|
||||
@@ -13,17 +17,23 @@ const clockStart = (
|
||||
// The clock is global: any unknown completion may extend beyond every known end.
|
||||
if (records.some((record) => record.timing.endMs === undefined)) return null;
|
||||
// A record timed past the range of a date carries no usable time, so the clock starts from the others.
|
||||
const end = records.reduce(
|
||||
(latest, record) =>
|
||||
record.timing.endMs !== undefined &&
|
||||
Math.abs(record.timing.endMs) <= LATEST_END_MS
|
||||
? Math.max(latest, record.timing.endMs)
|
||||
: latest,
|
||||
Number.NEGATIVE_INFINITY,
|
||||
);
|
||||
return Number.isFinite(end)
|
||||
? new Date(Math.ceil(end / 1000) * 1000).toISOString()
|
||||
: null;
|
||||
let latest: OffsetDateTimeInstant | undefined;
|
||||
for (const record of records) {
|
||||
if (
|
||||
record.timing.endMs === undefined ||
|
||||
Math.abs(record.timing.endMs) > LATEST_END_MS
|
||||
)
|
||||
continue;
|
||||
const end = record.timing.end;
|
||||
if (end === undefined) return null;
|
||||
if (latest === undefined || compareOffsetDateTimeInstants(end, latest) > 0)
|
||||
latest = end;
|
||||
}
|
||||
return latest === undefined
|
||||
? null
|
||||
: new Date(
|
||||
(latest.epochSecond + (latest.fraction === "" ? 0 : 1)) * 1000,
|
||||
).toISOString();
|
||||
};
|
||||
|
||||
/**
|
||||
|
||||
@@ -44,6 +44,14 @@ export type RecordedResponseDeclaration = ReturnType<
|
||||
typeof readRecordedResponseDeclaration
|
||||
>;
|
||||
|
||||
/** HAR's zero status sentinel and nonnumeric declarations do not prove a response status. */
|
||||
export const recordedResponseStatus = (
|
||||
declaration: RecordedResponseDeclaration,
|
||||
): number | null => {
|
||||
const status = declaration?.status;
|
||||
return typeof status === "number" && status !== 0 ? status : null;
|
||||
};
|
||||
|
||||
/** Strict, and keeping a leading byte-order mark, so callers see the recorded text unchanged. */
|
||||
const strictUtf8 = (): TextDecoder =>
|
||||
new TextDecoder("utf-8", { fatal: true, ignoreBOM: true });
|
||||
@@ -83,6 +91,28 @@ export const readRecordedResponseBody = (
|
||||
): RecordedResponseBody => {
|
||||
const content = declaration?.content;
|
||||
if (content == null) return { bodyText: null };
|
||||
const bodyCoordinates = [
|
||||
"/response/content/text",
|
||||
"/response/content/encoding",
|
||||
];
|
||||
const within = (pointer: string, ancestor: string): boolean =>
|
||||
pointer.startsWith(ancestor) &&
|
||||
(pointer.length === ancestor.length || pointer[ancestor.length] === "/");
|
||||
// Explicit value exclusions also govern imported retained facts. Keep their
|
||||
// source text in complete Evidence, but never promote it into replay/state.
|
||||
if (
|
||||
record.redactions.some(
|
||||
(redaction) =>
|
||||
!("scope" in redaction) &&
|
||||
bodyCoordinates.some((pointer) => within(pointer, redaction.pointer)),
|
||||
) ||
|
||||
record.binary_fields.some(
|
||||
(field) =>
|
||||
field.state === "redacted" &&
|
||||
bodyCoordinates.some((pointer) => within(pointer, field.pointer)),
|
||||
)
|
||||
)
|
||||
return { bodyText: null };
|
||||
const { text, encoding } = content;
|
||||
if (typeof text !== "string") return { bodyText: null };
|
||||
if (encoding === undefined) {
|
||||
|
||||
@@ -3,7 +3,11 @@ import { createHash } from "node:crypto";
|
||||
import { createEvidence } from "../evidence.js";
|
||||
import { freezeJsonSnapshot } from "../immutableJson.js";
|
||||
import { webNetworkCaptureSchema } from "../webNetworkCapture.js";
|
||||
import { projectAnalysisView } from "./analysisView.js";
|
||||
import {
|
||||
projectAnalysisView,
|
||||
analysisViewRequestSchema,
|
||||
} from "./analysisView.js";
|
||||
import { timedCaptureEvidence } from "../../../tests/fixtures/analysisView.js";
|
||||
|
||||
const capture = () => {
|
||||
const body = Buffer.from("original body");
|
||||
@@ -800,18 +804,6 @@ it.each([
|
||||
"bytes that are not its text",
|
||||
{ binaryFields: [retainedBytes("/request/path", "/b")] },
|
||||
],
|
||||
[
|
||||
"a byte count that is not its text",
|
||||
{ binaryFields: [{ ...retainedBytes("/request/path", "/a"), bytes: 3 }] },
|
||||
],
|
||||
[
|
||||
"a digest that is not its text",
|
||||
{
|
||||
binaryFields: [
|
||||
{ ...retainedBytes("/request/path", "/a"), sha256: "0".repeat(64) },
|
||||
],
|
||||
},
|
||||
],
|
||||
] as const)(
|
||||
"keeps a mitmproxy path with %s out of the preview",
|
||||
(_label, facets) => {
|
||||
@@ -1040,3 +1032,111 @@ it.each(["scheme", "host", "port", "authority"])(
|
||||
expect(JSON.stringify(reported)).not.toContain("password");
|
||||
},
|
||||
);
|
||||
|
||||
it.each(["base64", "bytes", "sha256", "duplicate"] as const)(
|
||||
"refuses inconsistent retained binary %s at parent admission",
|
||||
(fault) => {
|
||||
const { parent, report } = capture();
|
||||
const record = report.records[0]!;
|
||||
const binary = record.binary_fields[0]!;
|
||||
if (binary.state !== "retained")
|
||||
throw new Error("Expected retained source bytes");
|
||||
const changed = {
|
||||
...binary,
|
||||
...(fault === "base64"
|
||||
? { content_base64: "!!!!" }
|
||||
: fault === "bytes"
|
||||
? { bytes: binary.bytes + 1 }
|
||||
: { sha256: "0".repeat(64) }),
|
||||
};
|
||||
const invalid = {
|
||||
...report,
|
||||
records: [
|
||||
{
|
||||
...record,
|
||||
binary_fields: fault === "duplicate" ? [binary, binary] : [changed],
|
||||
},
|
||||
],
|
||||
};
|
||||
const projected = projectAnalysisView(
|
||||
{ ...parent, normalizedResult: invalid },
|
||||
{ kind: "capture", detail: "preview", offset: 0, limit: 1 },
|
||||
);
|
||||
expect(projected).toMatchObject({
|
||||
ok: false,
|
||||
error: { _tag: "AnalysisInputError" },
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
it.each([
|
||||
"/response/content/text",
|
||||
"/response/content",
|
||||
"/response",
|
||||
"",
|
||||
"binary-redaction",
|
||||
"control",
|
||||
])(
|
||||
"respects retained response-body authority without altering raw facts: %s",
|
||||
(marker) => {
|
||||
const seed = timedCaptureEvidence([
|
||||
["GET", "/api/items", [{ id: 1, name: "withheld-value" }]],
|
||||
]);
|
||||
const report = webNetworkCaptureSchema.parse(seed.normalized_result);
|
||||
const record = report.records[0]!;
|
||||
if (marker === "binary-redaction")
|
||||
record.binary_fields.push({
|
||||
pointer: "/response/content/text",
|
||||
representation: "har-base64-content",
|
||||
state: "redacted",
|
||||
content_base64: null,
|
||||
bytes: null,
|
||||
sha256: null,
|
||||
});
|
||||
else if (marker !== "control")
|
||||
record.redactions.push({
|
||||
pointer: marker,
|
||||
reason: "explicit-sensitive-value",
|
||||
});
|
||||
const evidence = createEvidence(
|
||||
{
|
||||
path: report.artifact.path,
|
||||
sha256: report.artifact.sha256!,
|
||||
format: "file",
|
||||
},
|
||||
report.decoder,
|
||||
{
|
||||
operation: seed.operation,
|
||||
parameters: seed.parameters,
|
||||
result: report,
|
||||
},
|
||||
);
|
||||
const parent = {
|
||||
predicateType: evidence.predicate_type,
|
||||
evidenceId: evidence.evidence_id,
|
||||
operation: evidence.operation,
|
||||
normalizedResult: evidence.normalized_result,
|
||||
subject: evidence.subject,
|
||||
parameters: evidence.parameters,
|
||||
limitations: evidence.limitations,
|
||||
};
|
||||
const state = projectAnalysisView(
|
||||
parent,
|
||||
analysisViewRequestSchema.parse({ kind: "environment-state" }),
|
||||
);
|
||||
if (!state.ok || state.value.kind !== "environment-state")
|
||||
throw new Error("Expected retained state projection");
|
||||
expect(
|
||||
JSON.stringify(state.value.entities).includes("withheld-value"),
|
||||
).toBe(marker === "control");
|
||||
const complete = projectAnalysisView(parent, {
|
||||
kind: "capture",
|
||||
detail: "complete",
|
||||
offset: 0,
|
||||
limit: 1,
|
||||
});
|
||||
expect(
|
||||
complete.ok && JSON.stringify(complete.value).includes("withheld-value"),
|
||||
).toBe(true);
|
||||
},
|
||||
);
|
||||
@@ -1,5 +1,8 @@
|
||||
import { z } from "zod";
|
||||
import { readRecordedResponseDeclaration } from "./recordedResponseBody.js";
|
||||
import {
|
||||
readRecordedResponseDeclaration,
|
||||
recordedResponseStatus,
|
||||
} from "./recordedResponseBody.js";
|
||||
import {
|
||||
parseRecordedTransportUrl,
|
||||
parseUrlQueryParameterNames,
|
||||
@@ -148,7 +151,6 @@ export const readRecordedRequestLine = (
|
||||
return excluded("method-unavailable");
|
||||
if (parsed.path === null) return excluded("url-unparseable");
|
||||
const response = readRecordedResponseDeclaration(reported);
|
||||
const status = response?.status;
|
||||
const mime = response?.content?.mimeType;
|
||||
const rawMime = typeof mime === "string" ? mime : "";
|
||||
if (!temporary(textBytes(rawMime.length))) return null;
|
||||
@@ -158,7 +160,7 @@ export const readRecordedRequestLine = (
|
||||
origin: parsed.origin,
|
||||
path: parsed.path,
|
||||
query: parsed.query,
|
||||
status: typeof status === "number" && status !== 0 ? status : null,
|
||||
status: recordedResponseStatus(response),
|
||||
rawMime,
|
||||
};
|
||||
};
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { z } from "zod";
|
||||
import { MFT_INLINE_HEADER_LIMIT } from "./windows/ntfsMft.js";
|
||||
|
||||
import { jsonRecordSchema } from "./jsonRecord.js";
|
||||
import { jsonValueSchema } from "./jsonValue.js";
|
||||
@@ -299,6 +300,56 @@ export const hexRecordRegionMetadataSchema = z.strictObject({
|
||||
last_line: offset,
|
||||
});
|
||||
|
||||
const ntfsFileReferenceSchema = z.strictObject({
|
||||
recordNumber: offset,
|
||||
sequenceNumber: offset,
|
||||
});
|
||||
const ntfsRecordHeaderSchema = z.strictObject({
|
||||
index: offset,
|
||||
fileOffset: offset,
|
||||
sequenceNumber: offset,
|
||||
hardLinkCount: offset,
|
||||
flags: z.strictObject({
|
||||
raw: offset,
|
||||
names: z.array(z.string()),
|
||||
unknownMask: offset,
|
||||
}),
|
||||
inUse: z.boolean(),
|
||||
directory: z.boolean(),
|
||||
usedBytes: offset,
|
||||
totalBytes: offset,
|
||||
extentValid: z.boolean(),
|
||||
baseRecordReference: ntfsFileReferenceSchema.nullable(),
|
||||
declaredRecordNumber: offset.nullable(),
|
||||
recordNumberMatchesSlot: z.boolean().nullable(),
|
||||
logFileSequenceNumber: z.string().regex(/^(?:0|[1-9][0-9]*)$/u),
|
||||
firstAttributeOffset: offset,
|
||||
nextAttributeId: offset,
|
||||
fixups: z.strictObject({
|
||||
sequenceNumber: offset,
|
||||
entries: offset,
|
||||
verifiedSectors: offset,
|
||||
coveredSectors: offset,
|
||||
mismatchSector: offset.nullable(),
|
||||
malformed: z.boolean(),
|
||||
}),
|
||||
});
|
||||
|
||||
/** NTFS $MFT record-header facts; attributes, names and the USN journal are later steps. */
|
||||
export const ntfsMftMetadataSchema = z.strictObject({
|
||||
kind: z.literal("ntfs-mft"),
|
||||
record_bytes: offset,
|
||||
record_count: offset,
|
||||
in_use_record_count: offset,
|
||||
directory_record_count: offset,
|
||||
extension_record_count: offset,
|
||||
fixup_mismatch_count: offset,
|
||||
fixup_unverified_record_count: offset,
|
||||
truncated_tail_bytes: offset,
|
||||
record_headers: z.array(ntfsRecordHeaderSchema).max(MFT_INLINE_HEADER_LIMIT),
|
||||
omitted_record_headers: offset,
|
||||
});
|
||||
|
||||
/**
|
||||
* U-Boot FIT root facts. Configuration references are declared names; a
|
||||
* reference that names no image is reported as a limitation, not removed.
|
||||
@@ -886,6 +937,7 @@ export const artifactFormatMetadataSchema = z.discriminatedUnion("kind", [
|
||||
espImageSignatureBlockMetadataSchema,
|
||||
uf2MetadataSchema,
|
||||
uf2RegionMetadataSchema,
|
||||
ntfsMftMetadataSchema,
|
||||
dfuSeMetadataSchema,
|
||||
dfuSeElementMetadataSchema,
|
||||
espFlashImageMetadataSchema,
|
||||
@@ -1036,6 +1088,10 @@ const declarationsByKind: {
|
||||
notCompared:
|
||||
"Its familyID and addresses are in its path, family_name is REA's label for the familyID, and its block counts and positions are layout; content comparison covers its bytes.",
|
||||
},
|
||||
"ntfs-mft": {
|
||||
notCompared:
|
||||
"Its fields come from the record headers in the file's own FILE records, which content comparison covers.",
|
||||
},
|
||||
dfuse: {
|
||||
notCompared:
|
||||
"Its fields come from the file's own prefix, target headers and suffix, which content comparison covers.",
|
||||
|
||||
@@ -103,6 +103,7 @@ export const ARTIFACT_FORMAT_IDS = [
|
||||
"dfuse",
|
||||
"esp-flash-image",
|
||||
"fit",
|
||||
"ntfs-mft",
|
||||
"unknown",
|
||||
] as const;
|
||||
|
||||
@@ -124,6 +125,7 @@ const artifactContentFormatSchema = artifactFormatSchema.extract([
|
||||
"uf2",
|
||||
"dfuse",
|
||||
"fit",
|
||||
"ntfs-mft",
|
||||
"mach-o-universal",
|
||||
"mach-o",
|
||||
"elf",
|
||||
|
||||
@@ -302,7 +302,8 @@ export type BinaryTarget =
|
||||
| "esp-image"
|
||||
| "uf2"
|
||||
| "dfuse"
|
||||
| "fit";
|
||||
| "fit"
|
||||
| "ntfs-mft";
|
||||
})
|
||||
| (BinaryTargetIdentity &
|
||||
NonExecutableMetadata & {
|
||||
|
||||
@@ -52,6 +52,7 @@ export const evidenceSubjectFormatSchema = z.enum([
|
||||
"dfuse",
|
||||
"esp-flash-image",
|
||||
"fit",
|
||||
"ntfs-mft",
|
||||
]);
|
||||
|
||||
/** Format-owned architecture declarations shared by Evidence and persisted executions. */
|
||||
|
||||
@@ -137,7 +137,8 @@ describe("child-process options detection", () => {
|
||||
["negative-zero", `spawn("tool", [], { env: -0 })`, false],
|
||||
["bigint-zero", `spawn("tool", [], { env: 0n })`, false],
|
||||
["zero-float", `spawn("tool", [], { env: 0.0 })`, false],
|
||||
["nan", `spawn("tool", [], { env: NaN })`, true],
|
||||
["nan", `spawn("tool", [], { env: NaN })`, false],
|
||||
["shadowed-nan", `((NaN) => spawn("tool", [], { env: NaN }))({})`, true],
|
||||
["empty-string", `spawn("tool", [], { env: "" })`, false],
|
||||
["false", `spawn("tool", [], { env: false })`, false],
|
||||
["empty-object", `spawn("tool", [], { env: {} })`, true],
|
||||
|
||||
@@ -81,7 +81,7 @@ export const collectJavaScriptSemanticChildProcesses = (
|
||||
!rejectsOptionsShape(candidate, context, state),
|
||||
);
|
||||
const childProcessSpawns = candidates.map((candidate) =>
|
||||
immutableSpawn(candidate, context),
|
||||
immutableSpawn(candidate, context, state),
|
||||
);
|
||||
return {
|
||||
childProcessSpawns,
|
||||
@@ -146,6 +146,7 @@ const collectSpawns = (
|
||||
const immutableSpawn = (
|
||||
candidate: SpawnCandidate,
|
||||
context: ChildSpawnContext,
|
||||
state: JavaScriptSemanticAnalysisState,
|
||||
): JavaScriptSemanticChildProcessSpawn => {
|
||||
const { options, argvCount } = spawnArguments(candidate, context);
|
||||
const command = dataEffectLiteralString(candidate.node.arguments[0]);
|
||||
@@ -161,7 +162,7 @@ const immutableSpawn = (
|
||||
projectedResultBindings: candidate.projectedResultBindings,
|
||||
command,
|
||||
argvCount,
|
||||
environmentSupplied: environmentSupplied(options, context),
|
||||
environmentSupplied: environmentSupplied(options, context, state),
|
||||
stdioMode: stdioMode(options, candidate.method, context),
|
||||
resolution: command === null ? "partial" : "complete",
|
||||
};
|
||||
@@ -443,11 +444,17 @@ const spawnArguments = (
|
||||
const environmentSupplied = (
|
||||
options: t.Node | undefined,
|
||||
context: ChildSpawnContext,
|
||||
state: JavaScriptSemanticAnalysisState,
|
||||
): boolean => {
|
||||
const lookup = staticObjectProperty(options, "env", context.isNullish);
|
||||
if (lookup.status !== "explicit") return false;
|
||||
const value = unwrapJavaScriptExpression(lookup.value).node;
|
||||
if (context.isNullish(value)) return false;
|
||||
if (
|
||||
t.isIdentifier(value, { name: "NaN" }) &&
|
||||
isUnshadowedGlobal(value, state, "NaN")
|
||||
)
|
||||
return false;
|
||||
return staticJavaScriptTruthiness(value) !== false;
|
||||
};
|
||||
|
||||
|
||||
@@ -83,6 +83,7 @@ export const liveBinaryTargetSummaryVariants = [
|
||||
"uf2",
|
||||
"dfuse",
|
||||
"fit",
|
||||
"ntfs-mft",
|
||||
]),
|
||||
architecture: z.null(),
|
||||
}),
|
||||
|
||||
@@ -2,6 +2,7 @@ import { describe, expect, it } from "vitest";
|
||||
import { z } from "zod";
|
||||
|
||||
import {
|
||||
addOffsetDateTimeMilliseconds,
|
||||
inspectRecordedDateTime,
|
||||
offsetDateTimeSchema,
|
||||
readRecordedDateTime,
|
||||
@@ -22,6 +23,34 @@ it.each([
|
||||
expect(inspectRecordedDateTime(text).kind).toBe(kind);
|
||||
});
|
||||
|
||||
it.each([
|
||||
["000001", 0, 10, "000001"],
|
||||
["999999", 0.001, 11, ""],
|
||||
["999999", 0.001001, 11, "000000001"],
|
||||
["1234567890123456789", 1000.001, 11, "1234577890123456789"],
|
||||
["", 1e-7, 10, "0000000001"],
|
||||
["", 999.9999999999999, 10, "9999999999999999"],
|
||||
])(
|
||||
"adds duration %s + %sms without losing decimal precision",
|
||||
(fraction, duration, epochSecond, expectedFraction) => {
|
||||
expect(
|
||||
addOffsetDateTimeMilliseconds({ epochSecond: 10, fraction }, duration),
|
||||
).toEqual({ epochSecond, fraction: expectedFraction });
|
||||
},
|
||||
);
|
||||
|
||||
it.each([-1, Number.NaN, Number.POSITIVE_INFINITY, Number.MAX_VALUE])(
|
||||
"keeps unsupported recorded duration %s unknown",
|
||||
(duration) => {
|
||||
expect(
|
||||
addOffsetDateTimeMilliseconds(
|
||||
{ epochSecond: 10, fraction: "" },
|
||||
duration,
|
||||
),
|
||||
).toBeUndefined();
|
||||
},
|
||||
);
|
||||
|
||||
const ACCEPTED: readonly (readonly [string, string])[] = [
|
||||
["a UTC timestamp", "2026-10-09T04:00:00Z"],
|
||||
["a positive offset", "2026-10-09T04:00:00+02:00"],
|
||||
|
||||
@@ -146,12 +146,14 @@ export const offsetDateTimeInstantFromEpochMilliseconds = (
|
||||
const millisecondsWithinSecond = epochMilliseconds - epochSecond * 1000;
|
||||
return {
|
||||
epochSecond,
|
||||
fraction: fractionalSecondFromMilliseconds(millisecondsWithinSecond),
|
||||
fraction: secondsFromMilliseconds(millisecondsWithinSecond).fraction,
|
||||
};
|
||||
};
|
||||
|
||||
const fractionalSecondFromMilliseconds = (milliseconds: number): string => {
|
||||
if (milliseconds === 0) return "";
|
||||
const secondsFromMilliseconds = (
|
||||
milliseconds: number,
|
||||
): { readonly seconds: number; readonly fraction: string } => {
|
||||
if (milliseconds === 0) return { seconds: 0, fraction: "" };
|
||||
const [coefficient = "", exponentText] = String(milliseconds)
|
||||
.toLowerCase()
|
||||
.split("e");
|
||||
@@ -166,9 +168,43 @@ const fractionalSecondFromMilliseconds = (milliseconds: number): string => {
|
||||
: decimalPosition < digits.length
|
||||
? `${digits.slice(0, decimalPosition)}.${digits.slice(decimalPosition)}`
|
||||
: `${digits}${"0".repeat(decimalPosition - digits.length)}`;
|
||||
return (
|
||||
fraction.includes(".") ? fraction.slice(fraction.indexOf(".") + 1) : ""
|
||||
).replace(/0+$/u, "");
|
||||
const separator = fraction.indexOf(".");
|
||||
return {
|
||||
seconds: Number(separator < 0 ? fraction : fraction.slice(0, separator)),
|
||||
fraction: (separator < 0 ? "" : fraction.slice(separator + 1)).replace(
|
||||
/0+$/u,
|
||||
"",
|
||||
),
|
||||
};
|
||||
};
|
||||
|
||||
/** Add a recorded nonnegative decimal millisecond duration without rounding away either fraction. */
|
||||
export const addOffsetDateTimeMilliseconds = (
|
||||
start: OffsetDateTimeInstant,
|
||||
milliseconds: number,
|
||||
): OffsetDateTimeInstant | undefined => {
|
||||
if (!Number.isFinite(milliseconds) || milliseconds < 0) return undefined;
|
||||
if (milliseconds === 0) return start;
|
||||
const duration = secondsFromMilliseconds(milliseconds);
|
||||
const wholeSeconds = start.epochSecond + duration.seconds;
|
||||
if (!Number.isSafeInteger(wholeSeconds)) return undefined;
|
||||
const length = Math.max(start.fraction.length, duration.fraction.length);
|
||||
const digits: string[] = [];
|
||||
let carry = 0;
|
||||
for (let index = length - 1; index >= 0; index--) {
|
||||
const sum =
|
||||
(start.fraction.charCodeAt(index) || 48) -
|
||||
48 +
|
||||
(duration.fraction.charCodeAt(index) || 48) -
|
||||
48 +
|
||||
carry;
|
||||
digits.push(String(sum % 10));
|
||||
carry = Math.floor(sum / 10);
|
||||
}
|
||||
const epochSecond = wholeSeconds + carry;
|
||||
return Number.isSafeInteger(epochSecond)
|
||||
? { epochSecond, fraction: digits.reverse().join("").replace(/0+$/u, "") }
|
||||
: undefined;
|
||||
};
|
||||
|
||||
/** Compare instants exactly while treating absent fraction digits as zeroes. */
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { createHash } from "node:crypto";
|
||||
import { expect, it } from "vitest";
|
||||
import {
|
||||
freezeJsonSnapshot,
|
||||
@@ -116,3 +117,107 @@ it("retains schema normalization when an owned source contains a hidden reported
|
||||
Object.hasOwn(parsed.data.container.reported as object, "hidden"),
|
||||
).toBe(false);
|
||||
});
|
||||
|
||||
it.each([
|
||||
{
|
||||
name: "malformed base64",
|
||||
content_base64: "!!!!",
|
||||
bytes: 0,
|
||||
sha256: "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
|
||||
},
|
||||
{
|
||||
name: "noncanonical pad bits",
|
||||
content_base64: "AR==",
|
||||
bytes: 1,
|
||||
sha256: createHash("sha256")
|
||||
.update(Buffer.from([1]))
|
||||
.digest("hex"),
|
||||
},
|
||||
{
|
||||
name: "mismatched byte count",
|
||||
content_base64: "YQ==",
|
||||
bytes: 2,
|
||||
sha256: "ca978112ca1bbdcafac231b39a23dc4da786eff8147c4e72b9807785afee48bb",
|
||||
},
|
||||
{
|
||||
name: "mismatched digest",
|
||||
content_base64: "YQ==",
|
||||
bytes: 1,
|
||||
sha256: "0".repeat(64),
|
||||
},
|
||||
])(
|
||||
"refuses $name in retained binary facts before projection",
|
||||
({ name: _name, ...binary }) => {
|
||||
const report = capture();
|
||||
const record = {
|
||||
...report.records[0]!,
|
||||
binary_fields: [
|
||||
{
|
||||
pointer: "/response/content/text",
|
||||
representation: "har-base64-content",
|
||||
state: "retained",
|
||||
...binary,
|
||||
},
|
||||
],
|
||||
};
|
||||
const parsed = webNetworkCaptureSchema.safeParse({
|
||||
...report,
|
||||
records: [record],
|
||||
});
|
||||
expect(parsed.success).toBe(false);
|
||||
},
|
||||
);
|
||||
|
||||
it.each([
|
||||
{ name: "empty", bytes: Buffer.alloc(0) },
|
||||
{ name: "Unicode", bytes: Buffer.from("雪�") },
|
||||
{ name: "multiple windows", bytes: Buffer.alloc(100_001, 0xff) },
|
||||
])(
|
||||
"retains $name canonical binary identity across bounded digest windows",
|
||||
({ bytes }) => {
|
||||
const report = capture();
|
||||
const source = {
|
||||
...report,
|
||||
records: [
|
||||
{
|
||||
...report.records[0]!,
|
||||
binary_fields: [
|
||||
{
|
||||
pointer: "/body",
|
||||
representation: "producer-bytes",
|
||||
state: "retained",
|
||||
content_base64: bytes.toString("base64"),
|
||||
bytes: bytes.length,
|
||||
sha256: createHash("sha256").update(bytes).digest("hex"),
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
};
|
||||
const parsed = webNetworkCaptureSchema.parse(source);
|
||||
expect(parsed).toEqual(source);
|
||||
},
|
||||
);
|
||||
|
||||
it("refuses multiple binary declarations for one source coordinate", () => {
|
||||
const report = capture();
|
||||
const binary = (text: string) => ({
|
||||
pointer: "/response/content/text",
|
||||
representation: "har-base64-content",
|
||||
state: "retained",
|
||||
content_base64: Buffer.from(text).toString("base64"),
|
||||
bytes: Buffer.byteLength(text),
|
||||
sha256: createHash("sha256").update(text).digest("hex"),
|
||||
});
|
||||
expect(
|
||||
webNetworkCaptureSchema.safeParse({
|
||||
...report,
|
||||
records: [
|
||||
{
|
||||
...report.records[0]!,
|
||||
binary_fields: [binary("first"), binary("second")],
|
||||
},
|
||||
],
|
||||
}).success,
|
||||
).toBe(false);
|
||||
});
|
||||
@@ -1,3 +1,5 @@
|
||||
import { createHash } from "node:crypto";
|
||||
import { canonicalBase64Schema } from "./stringPatterns.js";
|
||||
import { z } from "zod";
|
||||
import {
|
||||
browserDeadlineIncrease,
|
||||
@@ -215,7 +217,35 @@ export const webNetworkCaptureBinaryDescriptorSchema = z.discriminatedUnion(
|
||||
[retainedBinaryDescriptorSchema, redactedBinaryDescriptorSchema],
|
||||
);
|
||||
const binarySchema = z.discriminatedUnion("state", [
|
||||
retainedBinaryDescriptorSchema.extend({ content_base64: z.string() }),
|
||||
retainedBinaryDescriptorSchema
|
||||
.extend({ content_base64: canonicalBase64Schema })
|
||||
.superRefine((field, context) => {
|
||||
const encoded = field.content_base64;
|
||||
const decodedBytes = Buffer.byteLength(encoded, "base64");
|
||||
if (field.bytes !== decodedBytes) {
|
||||
context.addIssue({
|
||||
code: "custom",
|
||||
path: ["bytes"],
|
||||
message:
|
||||
"Retained binary byte count differs from its canonical content.",
|
||||
});
|
||||
return;
|
||||
}
|
||||
// Quartet-aligned windows authenticate every retained byte without
|
||||
// allocating a second whole body while the capture remains live.
|
||||
const hash = createHash("sha256");
|
||||
for (let offset = 0; offset < encoded.length; offset += 65536)
|
||||
hash.update(
|
||||
Buffer.from(encoded.slice(offset, offset + 65536), "base64"),
|
||||
);
|
||||
if (hash.digest("hex") !== field.sha256)
|
||||
context.addIssue({
|
||||
code: "custom",
|
||||
path: ["sha256"],
|
||||
message:
|
||||
"Retained binary SHA-256 differs from its canonical content.",
|
||||
});
|
||||
}),
|
||||
redactedBinaryDescriptorSchema.extend({ content_base64: z.null() }),
|
||||
]);
|
||||
const numberSchema = z.strictObject({
|
||||
@@ -244,7 +274,21 @@ export const webNetworkCaptureRecordSchema = z.strictObject({
|
||||
}),
|
||||
]),
|
||||
reported: jsonValueSchema,
|
||||
binary_fields: z.array(binarySchema),
|
||||
binary_fields: z.array(binarySchema).superRefine((fields, context) => {
|
||||
const pointers = new Set<string>();
|
||||
for (const [index, field] of fields.entries()) {
|
||||
if (pointers.has(field.pointer)) {
|
||||
context.addIssue({
|
||||
code: "custom",
|
||||
path: [index, "pointer"],
|
||||
message:
|
||||
"Each retained source coordinate can declare binary facts only once.",
|
||||
});
|
||||
return;
|
||||
}
|
||||
pointers.add(field.pointer);
|
||||
}
|
||||
}),
|
||||
numeric_literals: z.array(numberSchema),
|
||||
redactions: z.array(redactionSchema),
|
||||
limitations: z.array(z.string()),
|
||||
|
||||
@@ -0,0 +1,478 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import {
|
||||
MFT_DEFAULT_RECORD_BYTES,
|
||||
MFT_MAX_FILE_BYTES,
|
||||
createMftDecoder,
|
||||
parseMftImage,
|
||||
} from "./ntfsMft.js";
|
||||
|
||||
interface RecordOptions {
|
||||
readonly sequenceNumber?: number;
|
||||
readonly hardLinkCount?: number;
|
||||
readonly flags?: number;
|
||||
readonly baseRecordReference?: bigint;
|
||||
readonly recordBytes?: number;
|
||||
/** Write a wrong trailer value into this sector, leaving the others correct. */
|
||||
readonly corruptSector?: number;
|
||||
/** Truncate the update sequence array to this many entries; 0 omits it. */
|
||||
readonly usaEntries?: number;
|
||||
readonly usaOffset?: number;
|
||||
}
|
||||
|
||||
/** One FILE record with a zeroed body and fixups applied the way NTFS writes them. */
|
||||
const recordOf = (options: RecordOptions = {}): Uint8Array => {
|
||||
const recordBytes = options.recordBytes ?? MFT_DEFAULT_RECORD_BYTES;
|
||||
const record = new Uint8Array(recordBytes);
|
||||
const view = new DataView(record.buffer);
|
||||
record.set(Buffer.from("FILE", "latin1"), 0);
|
||||
const sequenceNumber = options.sequenceNumber ?? 1;
|
||||
const sectors = recordBytes / 512;
|
||||
|
||||
const usaEntries = options.usaEntries ?? sectors + 1;
|
||||
if (usaEntries > 0) {
|
||||
const usaOffset = options.usaOffset ?? 48;
|
||||
view.setUint16(4, usaOffset, true);
|
||||
view.setUint16(6, usaEntries, true);
|
||||
view.setUint16(usaOffset, sequenceNumber, true);
|
||||
for (let sector = 0; sector < sectors; sector += 1) {
|
||||
const trailer = sector * 512 + 510;
|
||||
// A saved trailer is what the sector held before the fixup replaced it.
|
||||
view.setUint16(usaOffset + 2 * (sector + 1), 0xa5a5, true);
|
||||
view.setUint16(
|
||||
trailer,
|
||||
sector >= usaEntries - 1
|
||||
? 0xa5a5
|
||||
: sector === options.corruptSector
|
||||
? 0x1234
|
||||
: sequenceNumber,
|
||||
true,
|
||||
);
|
||||
}
|
||||
} else {
|
||||
view.setUint16(4, 0, true);
|
||||
view.setUint16(6, 0, true);
|
||||
}
|
||||
|
||||
view.setUint16(16, sequenceNumber, true);
|
||||
view.setUint16(18, options.hardLinkCount ?? 1, true);
|
||||
const firstAttributeOffset =
|
||||
Math.ceil((48 + Math.max(0, usaEntries) * 2) / 8) * 8;
|
||||
view.setUint16(20, firstAttributeOffset, true);
|
||||
view.setUint16(22, options.flags ?? 0x0001, true);
|
||||
view.setUint32(24, firstAttributeOffset + 8, true);
|
||||
view.setUint32(firstAttributeOffset, 0xffff_ffff, true);
|
||||
view.setUint32(28, recordBytes, true);
|
||||
view.setBigUint64(32, options.baseRecordReference ?? 0n, true);
|
||||
view.setUint16(40, 0, true);
|
||||
return record;
|
||||
};
|
||||
|
||||
const streamOf = (...records: readonly Uint8Array[]): Uint8Array => {
|
||||
const total = records.reduce((sum, record) => sum + record.length, 0);
|
||||
const stream = new Uint8Array(total);
|
||||
let offset = 0;
|
||||
for (const record of records) {
|
||||
stream.set(record, offset);
|
||||
const view = new DataView(stream.buffer, offset, record.length);
|
||||
if (view.getUint16(4, true) >= 48)
|
||||
view.setUint32(44, offset / record.length, true);
|
||||
offset += record.length;
|
||||
}
|
||||
return stream;
|
||||
};
|
||||
|
||||
describe("MFT header facts", () => {
|
||||
it("inventories a clean record's header and verified fixups", () => {
|
||||
const parsed = parseMftImage(streamOf(recordOf({ sequenceNumber: 5 })));
|
||||
expect(parsed.ok).toBe(true);
|
||||
if (!parsed.ok) return;
|
||||
expect(parsed.value.metadata).toMatchObject({
|
||||
kind: "ntfs-mft",
|
||||
record_bytes: MFT_DEFAULT_RECORD_BYTES,
|
||||
record_count: 1,
|
||||
in_use_record_count: 1,
|
||||
directory_record_count: 0,
|
||||
extension_record_count: 0,
|
||||
fixup_mismatch_count: 0,
|
||||
fixup_unverified_record_count: 0,
|
||||
truncated_tail_bytes: 0,
|
||||
});
|
||||
expect(parsed.value.records[0]).toMatchObject({
|
||||
index: 0,
|
||||
fileOffset: 0,
|
||||
sequenceNumber: 5,
|
||||
inUse: true,
|
||||
directory: false,
|
||||
baseRecordReference: null,
|
||||
firstAttributeOffset: 56,
|
||||
fixups: {
|
||||
sequenceNumber: 5,
|
||||
verifiedSectors: 2,
|
||||
coveredSectors: 2,
|
||||
mismatchSector: null,
|
||||
malformed: false,
|
||||
},
|
||||
});
|
||||
expect(parsed.value.limitations).toEqual([]);
|
||||
});
|
||||
|
||||
it("identifies unrecognized flags without losing their source bits", () => {
|
||||
const parsed = parseMftImage(recordOf({ flags: 0x8001 }));
|
||||
if (!parsed.ok) throw new Error(parsed.error.message);
|
||||
expect(parsed.value.metadata.record_headers[0]?.flags).toEqual({
|
||||
raw: 0x8001,
|
||||
names: ["in-use"],
|
||||
unknownMask: 0x8000,
|
||||
});
|
||||
});
|
||||
|
||||
it.each([0, 49, 64, 72])(
|
||||
"reports an invalid first attribute offset %i",
|
||||
(offset) => {
|
||||
const record = recordOf();
|
||||
new DataView(record.buffer).setUint16(20, offset, true);
|
||||
const parsed = parseMftImage(record);
|
||||
if (!parsed.ok) throw new Error(parsed.error.message);
|
||||
expect(parsed.value.records[0]?.extentValid).toBe(false);
|
||||
expect(parsed.value.limitations.join(" ")).toContain("inconsistent");
|
||||
},
|
||||
);
|
||||
|
||||
it("reports a declared record number that contradicts its stream position", () => {
|
||||
const bytes = streamOf(recordOf(), recordOf());
|
||||
new DataView(bytes.buffer).setUint32(1024 + 44, 0, true);
|
||||
const parsed = parseMftImage(bytes);
|
||||
if (!parsed.ok) throw new Error(parsed.error.message);
|
||||
expect(parsed.value.records[1]).toMatchObject({
|
||||
index: 1,
|
||||
declaredRecordNumber: 0,
|
||||
recordNumberMatchesSlot: false,
|
||||
});
|
||||
expect(parsed.value.limitations.join(" ")).toContain(
|
||||
"disagrees with their stream slot",
|
||||
);
|
||||
});
|
||||
|
||||
it("keeps the inline header window separate from the parsed record count", () => {
|
||||
const parsed = parseMftImage(
|
||||
streamOf(...Array.from({ length: 33 }, () => recordOf())),
|
||||
);
|
||||
if (!parsed.ok) throw new Error(parsed.error.message);
|
||||
expect(parsed.value.metadata).toMatchObject({
|
||||
record_count: 33,
|
||||
omitted_record_headers: 1,
|
||||
});
|
||||
expect(parsed.value.metadata.record_headers).toHaveLength(32);
|
||||
expect(parsed.value.records).toHaveLength(33);
|
||||
expect(parsed.value.limitations).toEqual([]);
|
||||
});
|
||||
|
||||
it("preserves a log sequence number above the safe integer range", () => {
|
||||
const record = recordOf();
|
||||
new DataView(record.buffer).setBigUint64(8, 9007199254740993n, true);
|
||||
const parsed = parseMftImage(record);
|
||||
if (!parsed.ok) throw new Error(parsed.error.message);
|
||||
expect(parsed.value.records[0]?.logFileSequenceNumber).toBe(
|
||||
"9007199254740993",
|
||||
);
|
||||
});
|
||||
|
||||
it("verifies the older FILE header without reading its fixup words as a record number", () => {
|
||||
const oldRecord = recordOf({ usaOffset: 42 });
|
||||
const newRecord = recordOf();
|
||||
new DataView(newRecord.buffer).setUint32(44, 1, true);
|
||||
|
||||
const parsed = parseMftImage(streamOf(oldRecord, newRecord));
|
||||
if (!parsed.ok) throw new Error(parsed.error.message);
|
||||
expect(parsed.value.records).toMatchObject([
|
||||
{
|
||||
declaredRecordNumber: null,
|
||||
fixups: { malformed: false, verifiedSectors: 2 },
|
||||
},
|
||||
{
|
||||
declaredRecordNumber: 1,
|
||||
fixups: { malformed: false, verifiedSectors: 2 },
|
||||
},
|
||||
]);
|
||||
expect(parsed.value.limitations).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("MFT fixup coverage", () => {
|
||||
it.each([0, 0xffff])(
|
||||
"leaves reserved USA sequence %i unverified",
|
||||
(sequenceNumber) => {
|
||||
const parsed = parseMftImage(recordOf({ sequenceNumber }));
|
||||
if (!parsed.ok) throw new Error(parsed.error.message);
|
||||
expect(parsed.value.records[0]?.fixups).toMatchObject({
|
||||
sequenceNumber,
|
||||
verifiedSectors: 0,
|
||||
malformed: true,
|
||||
});
|
||||
expect(parsed.value.metadata.fixup_unverified_record_count).toBe(1);
|
||||
expect(parsed.value.limitations.join(" ")).toContain(
|
||||
"invalid or incomplete",
|
||||
);
|
||||
},
|
||||
);
|
||||
|
||||
it("keeps a record whose sector trailer is not the sequence number, and reports it", () => {
|
||||
const parsed = parseMftImage(
|
||||
streamOf(recordOf(), recordOf({ sequenceNumber: 9, corruptSector: 1 })),
|
||||
);
|
||||
expect(parsed.ok).toBe(true);
|
||||
if (!parsed.ok) return;
|
||||
expect(parsed.value.records[1]?.fixups).toMatchObject({
|
||||
sequenceNumber: 9,
|
||||
verifiedSectors: 1,
|
||||
mismatchSector: 1,
|
||||
});
|
||||
expect(parsed.value.metadata.fixup_mismatch_count).toBe(1);
|
||||
expect(parsed.value.limitations.join(" ")).toMatch(
|
||||
/1 record\(s\) hold a sector trailer/u,
|
||||
);
|
||||
});
|
||||
|
||||
it("treats a missing update sequence array as unverified rather than clean", () => {
|
||||
const parsed = parseMftImage(streamOf(recordOf({ usaEntries: 0 })));
|
||||
expect(parsed.ok).toBe(true);
|
||||
if (!parsed.ok) return;
|
||||
expect(parsed.value.records[0]?.fixups).toMatchObject({
|
||||
malformed: true,
|
||||
mismatchSector: null,
|
||||
coveredSectors: 0,
|
||||
});
|
||||
expect(parsed.value.metadata.fixup_mismatch_count).toBe(0);
|
||||
expect(parsed.value.metadata.fixup_unverified_record_count).toBe(1);
|
||||
expect(parsed.value.limitations.join(" ")).toMatch(
|
||||
/invalid or incomplete update sequence array/u,
|
||||
);
|
||||
});
|
||||
|
||||
it.each([
|
||||
[48, 4],
|
||||
[49, 3],
|
||||
[508, 3],
|
||||
])(
|
||||
"rejects a malformed update sequence array at %i with %i entries",
|
||||
(offset, count) => {
|
||||
const bytes = recordOf();
|
||||
const view = new DataView(bytes.buffer);
|
||||
view.setUint16(4, offset, true);
|
||||
view.setUint16(6, count, true);
|
||||
|
||||
const parsed = parseMftImage(bytes);
|
||||
if (!parsed.ok) throw new Error(parsed.error.message);
|
||||
expect(parsed.value.records[0]?.fixups).toMatchObject({
|
||||
malformed: true,
|
||||
verifiedSectors: 0,
|
||||
});
|
||||
expect(parsed.value.metadata.fixup_unverified_record_count).toBe(1);
|
||||
},
|
||||
);
|
||||
|
||||
it("counts uncovered sectors independently of a mismatched trailer", () => {
|
||||
const parsed = parseMftImage(
|
||||
recordOf({ recordBytes: 4096, usaEntries: 3, corruptSector: 0 }),
|
||||
);
|
||||
if (!parsed.ok) throw new Error(parsed.error.message);
|
||||
expect(parsed.value.metadata).toMatchObject({
|
||||
fixup_mismatch_count: 1,
|
||||
fixup_unverified_record_count: 1,
|
||||
});
|
||||
expect(parsed.value.limitations.join(" ")).toContain(
|
||||
"invalid or incomplete",
|
||||
);
|
||||
});
|
||||
|
||||
it("reports an array that covers fewer sectors than the record has", () => {
|
||||
// A 4 KiB record holds eight sectors; a three-entry array leaves five unverified.
|
||||
const parsed = parseMftImage(
|
||||
streamOf(recordOf({ recordBytes: 4096, usaEntries: 3 })),
|
||||
);
|
||||
expect(parsed.ok).toBe(true);
|
||||
if (!parsed.ok) return;
|
||||
expect(parsed.value.records[0]?.fixups).toMatchObject({
|
||||
coveredSectors: 2,
|
||||
verifiedSectors: 2,
|
||||
mismatchSector: null,
|
||||
malformed: false,
|
||||
});
|
||||
expect(parsed.value.metadata.fixup_unverified_record_count).toBe(1);
|
||||
});
|
||||
});
|
||||
|
||||
describe("MFT record slots", () => {
|
||||
it("distinguishes extension records from base records", () => {
|
||||
const parsed = parseMftImage(
|
||||
streamOf(
|
||||
recordOf(),
|
||||
recordOf({
|
||||
flags: 0x0005,
|
||||
baseRecordReference: 0xabcd_fedc_ba98_7654n,
|
||||
}),
|
||||
),
|
||||
);
|
||||
expect(parsed.ok).toBe(true);
|
||||
if (!parsed.ok) return;
|
||||
expect(parsed.value.records[1]).toMatchObject({
|
||||
baseRecordReference: {
|
||||
recordNumber: 0xfedc_ba98_7654,
|
||||
sequenceNumber: 0xabcd,
|
||||
},
|
||||
flags: { raw: 0x0005, names: ["in-use", "system"] },
|
||||
});
|
||||
expect(parsed.value.metadata.extension_record_count).toBe(1);
|
||||
});
|
||||
|
||||
it("names directory records without losing the in-use flag", () => {
|
||||
const parsed = parseMftImage(streamOf(recordOf({ flags: 0x0003 })));
|
||||
expect(parsed.ok).toBe(true);
|
||||
if (!parsed.ok) return;
|
||||
expect(parsed.value.records[0]).toMatchObject({
|
||||
directory: true,
|
||||
inUse: true,
|
||||
flags: { names: ["in-use", "directory"] },
|
||||
});
|
||||
expect(parsed.value.metadata.directory_record_count).toBe(1);
|
||||
});
|
||||
|
||||
it("skips malformed and sparse slots without renumbering valid records", () => {
|
||||
const stream = streamOf(
|
||||
recordOf(),
|
||||
recordOf(),
|
||||
new Uint8Array(MFT_DEFAULT_RECORD_BYTES),
|
||||
recordOf(),
|
||||
);
|
||||
stream.set(Buffer.from("BAAD", "latin1"), MFT_DEFAULT_RECORD_BYTES);
|
||||
const parsed = parseMftImage(stream);
|
||||
expect(parsed.ok).toBe(true);
|
||||
if (!parsed.ok) return;
|
||||
expect(parsed.value.metadata.record_count).toBe(2);
|
||||
expect(
|
||||
parsed.value.records.map(({ index, fileOffset }) => [index, fileOffset]),
|
||||
).toEqual([
|
||||
[0, 0],
|
||||
[3, 3 * MFT_DEFAULT_RECORD_BYTES],
|
||||
]);
|
||||
expect(parsed.value.limitations).toEqual([
|
||||
"record at 1024 does not begin with the FILE signature; 2 slot(s) without a FILE signature were found",
|
||||
]);
|
||||
});
|
||||
|
||||
it("keeps a trailing partial record out of the inventory", () => {
|
||||
const stream = new Uint8Array(1024 + 300);
|
||||
stream.set(recordOf(), 0);
|
||||
const parsed = parseMftImage(stream);
|
||||
expect(parsed.ok).toBe(true);
|
||||
if (!parsed.ok) return;
|
||||
expect(parsed.value.metadata.record_count).toBe(1);
|
||||
expect(parsed.value.metadata.truncated_tail_bytes).toBe(300);
|
||||
expect(parsed.value.limitations.join(" ")).toMatch(
|
||||
/300 trailing byte\(s\)/u,
|
||||
);
|
||||
});
|
||||
|
||||
it("falls back to the default record size when record 0 declares none", () => {
|
||||
const record = recordOf();
|
||||
new DataView(record.buffer).setUint32(28, 0, true);
|
||||
const parsed = parseMftImage(record);
|
||||
expect(parsed.ok).toBe(true);
|
||||
if (!parsed.ok) return;
|
||||
expect(parsed.value.recordBytes).toBe(MFT_DEFAULT_RECORD_BYTES);
|
||||
expect(parsed.value.limitations.join(" ")).toMatch(
|
||||
/not a supported power of two/u,
|
||||
);
|
||||
});
|
||||
|
||||
it("reads a 4 KiB record at its declared size", () => {
|
||||
const parsed = parseMftImage(streamOf(recordOf({ recordBytes: 4096 })));
|
||||
expect(parsed.ok).toBe(true);
|
||||
if (!parsed.ok) return;
|
||||
expect(parsed.value.recordBytes).toBe(4096);
|
||||
expect(parsed.value.metadata.record_count).toBe(1);
|
||||
});
|
||||
});
|
||||
|
||||
describe("parseMftImage refusals and limits", () => {
|
||||
it("refuses a file shorter than one record header", () => {
|
||||
const parsed = parseMftImage(new Uint8Array(32));
|
||||
expect(parsed.ok).toBe(false);
|
||||
if (parsed.ok) return;
|
||||
expect(parsed.error.message).toMatch(
|
||||
/shorter than one 42-byte record header/u,
|
||||
);
|
||||
});
|
||||
|
||||
it("refuses bytes that do not start with the FILE signature", () => {
|
||||
const parsed = parseMftImage(Buffer.alloc(2048, 0x41));
|
||||
expect(parsed.ok).toBe(false);
|
||||
if (parsed.ok) return;
|
||||
expect(parsed.error.message).toMatch(
|
||||
/does not begin with the FILE signature/u,
|
||||
);
|
||||
});
|
||||
|
||||
it("refuses a file above the fixed ceiling", () => {
|
||||
const decoder = createMftDecoder(MFT_MAX_FILE_BYTES + 1);
|
||||
expect(decoder.ok).toBe(false);
|
||||
if (decoder.ok) return;
|
||||
expect(decoder.error).toMatchObject({
|
||||
kind: "limit",
|
||||
boundary: "file-size",
|
||||
});
|
||||
});
|
||||
|
||||
it("refuses streams that differ from the admitted byte count", () => {
|
||||
const stream = recordOf();
|
||||
const short = createMftDecoder(stream.length + 1);
|
||||
if (!short.ok) throw new Error(short.error.message);
|
||||
expect(short.value.write(stream).ok).toBe(true);
|
||||
expect(short.value.finish()).toMatchObject({
|
||||
ok: false,
|
||||
error: {
|
||||
kind: "format",
|
||||
message: "stream ended before the admitted file length",
|
||||
},
|
||||
});
|
||||
|
||||
const long = createMftDecoder(stream.length - 1);
|
||||
if (!long.ok) throw new Error(long.error.message);
|
||||
expect(long.value.write(stream)).toMatchObject({
|
||||
ok: false,
|
||||
error: {
|
||||
kind: "format",
|
||||
message: "received more bytes than the admitted file length",
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it("returns the same inventory without retaining all the decoded records", () => {
|
||||
const bytes = streamOf(...Array.from({ length: 33 }, () => recordOf()));
|
||||
const decoder = createMftDecoder(bytes.length);
|
||||
const parsed = parseMftImage(bytes);
|
||||
if (!decoder.ok || !parsed.ok) throw new Error("invalid fixture");
|
||||
expect(decoder.value.write(bytes).ok).toBe(true);
|
||||
const inventory = decoder.value.finish();
|
||||
if (!inventory.ok) throw new Error(inventory.error.message);
|
||||
expect(inventory.value).not.toHaveProperty("records");
|
||||
expect(inventory.value.metadata).toEqual(parsed.value.metadata);
|
||||
expect(inventory.value.limitations).toEqual(parsed.value.limitations);
|
||||
});
|
||||
|
||||
it("decodes the same stream through the chunked decoder", () => {
|
||||
const stream = streamOf(recordOf(), recordOf({ flags: 0x0003 }));
|
||||
const decoder = createMftDecoder(stream.length);
|
||||
expect(decoder.ok).toBe(true);
|
||||
if (!decoder.ok) return;
|
||||
// Feed one byte past a record boundary so a chunk never aligns with a record.
|
||||
for (const chunk of [stream.subarray(0, 1025), stream.subarray(1025)])
|
||||
expect(decoder.value.write(chunk).ok).toBe(true);
|
||||
const image = decoder.value.finish();
|
||||
expect(image.ok).toBe(true);
|
||||
if (!image.ok) return;
|
||||
expect(image.value.metadata.record_count).toBe(2);
|
||||
expect(image.value.metadata.directory_record_count).toBe(1);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,512 @@
|
||||
import { err, ok, type Result } from "../result.js";
|
||||
import type {
|
||||
RegionDecodeFailure,
|
||||
RegionDecoder,
|
||||
} from "../firmware/regionDecoder.js";
|
||||
|
||||
/** An NTFS FILE record begins with this signature. */
|
||||
const MFT_FILE_RECORD_MAGIC = "FILE";
|
||||
const MFT_FILE_RECORD_MAGIC_BYTES = 4;
|
||||
/** The shortest header worth reading: a FILE record header ends at `nextAttributeId`. */
|
||||
const NTFS_MFT_MIN_RECORD_BYTES = 42;
|
||||
|
||||
const hasMftFileRecordSignature = (prefix: Uint8Array): boolean =>
|
||||
prefix.length >= MFT_FILE_RECORD_MAGIC_BYTES &&
|
||||
text(prefix.subarray(0, MFT_FILE_RECORD_MAGIC_BYTES)) ===
|
||||
MFT_FILE_RECORD_MAGIC;
|
||||
|
||||
/** A record's declared size is a power of two from 512 upward; 1024 is NTFS's default. */
|
||||
export const MFT_DEFAULT_RECORD_BYTES = 1024;
|
||||
const MFT_MIN_RECORD_BYTES = 512;
|
||||
const MFT_MAX_RECORD_BYTES = 65536;
|
||||
/** An update sequence array holds at least the sequence number and one fixup word. */
|
||||
const USA_MIN_ENTRIES = 2;
|
||||
/** NTFS fixes up one 512-byte sector at a time. */
|
||||
const FIXUP_SECTOR_BYTES = 512;
|
||||
/**
|
||||
* Fixed whole-file ceiling. Records are inventoried from their headers only;
|
||||
* attributes and their payloads are not decoded, so this is deliberately
|
||||
* smaller than a volume's MFT can be.
|
||||
*/
|
||||
export const MFT_MAX_FILE_BYTES = 512 * 1024 * 1024;
|
||||
/** Limit the inline header sample independently of full stream coverage. */
|
||||
export const MFT_INLINE_HEADER_LIMIT = 32;
|
||||
|
||||
/** FILE record header fields; offsets are from the record start. */
|
||||
const RECORD_HEADER = {
|
||||
updateSequenceOffset: 4,
|
||||
updateSequenceCount: 6,
|
||||
logFileSequenceNumber: 8,
|
||||
sequenceNumber: 16,
|
||||
hardLinkCount: 18,
|
||||
firstAttributeOffset: 20,
|
||||
flags: 22,
|
||||
usedBytes: 24,
|
||||
totalBytes: 28,
|
||||
baseRecordReference: 32,
|
||||
nextAttributeId: 40,
|
||||
recordNumber: 44,
|
||||
} as const;
|
||||
const RECORD_HEADER_WITH_NUMBER_BYTES = 48;
|
||||
|
||||
const FILE_RECORD_IN_USE = 0x0001;
|
||||
const FILE_RECORD_IS_DIRECTORY = 0x0002;
|
||||
const FILE_RECORD_NAMES = new Map<number, string>([
|
||||
[FILE_RECORD_IN_USE, "in-use"],
|
||||
[FILE_RECORD_IS_DIRECTORY, "directory"],
|
||||
[0x0004, "system"],
|
||||
[0x0008, "view-index"],
|
||||
]);
|
||||
|
||||
export interface MftRecordFixups {
|
||||
/** The update sequence number the array's first entry holds, or 0 when absent. */
|
||||
readonly sequenceNumber: number;
|
||||
readonly entries: number;
|
||||
/** Sectors whose trailer held the sequence number before a mismatch. */
|
||||
readonly verifiedSectors: number;
|
||||
/** Sectors the array covers; fewer than the record's sectors leaves a tail unverified. */
|
||||
readonly coveredSectors: number;
|
||||
/** The first sector whose trailer did not hold the sequence number, if any. */
|
||||
readonly mismatchSector: number | null;
|
||||
/** The array itself is missing, misplaced or too short to cover one sector. */
|
||||
readonly malformed: boolean;
|
||||
}
|
||||
|
||||
export interface MftFileReference {
|
||||
readonly recordNumber: number;
|
||||
readonly sequenceNumber: number;
|
||||
}
|
||||
|
||||
const decodeMftFileReference = (reference: bigint): MftFileReference => ({
|
||||
recordNumber: Number(reference & 0xffff_ffff_ffffn),
|
||||
sequenceNumber: Number(reference >> 48n),
|
||||
});
|
||||
|
||||
export interface MftRecord {
|
||||
/** Zero-based ordinal of this record in the stream. */
|
||||
readonly index: number;
|
||||
/** File offset where the record starts. */
|
||||
readonly fileOffset: number;
|
||||
readonly sequenceNumber: number;
|
||||
/** Hard link count as recorded; a zero count marks a record with no names. */
|
||||
readonly hardLinkCount: number;
|
||||
readonly flags: {
|
||||
readonly raw: number;
|
||||
readonly names: readonly string[];
|
||||
readonly unknownMask: number;
|
||||
};
|
||||
readonly inUse: boolean;
|
||||
readonly directory: boolean;
|
||||
/** Used and allocated byte counts as declared by the FILE record. */
|
||||
readonly usedBytes: number;
|
||||
readonly totalBytes: number;
|
||||
readonly extentValid: boolean;
|
||||
/** The base record this is an extension of, or `null` for a base record. */
|
||||
readonly baseRecordReference: MftFileReference | null;
|
||||
/** The record number the header declares; `null` when the header predates the field. */
|
||||
readonly declaredRecordNumber: number | null;
|
||||
readonly recordNumberMatchesSlot: boolean | null;
|
||||
/** `$LogFile` sequence number as recorded; no log is read. */
|
||||
readonly logFileSequenceNumber: string;
|
||||
/** Declared offset of the first attribute; no attribute decoding follows yet. */
|
||||
readonly firstAttributeOffset: number;
|
||||
/** Declared id of the next attribute; no attribute decoding follows yet. */
|
||||
readonly nextAttributeId: number;
|
||||
readonly fixups: MftRecordFixups;
|
||||
}
|
||||
|
||||
export interface MftInventory {
|
||||
readonly metadata: MftMetadata;
|
||||
/** The record size the stream declared, or the fixed default when it did not. */
|
||||
readonly recordBytes: number;
|
||||
readonly limitations: readonly string[];
|
||||
}
|
||||
|
||||
export interface MftImage extends MftInventory {
|
||||
readonly records: readonly MftRecord[];
|
||||
}
|
||||
|
||||
export interface MftMetadata {
|
||||
readonly kind: "ntfs-mft";
|
||||
readonly record_bytes: number;
|
||||
readonly record_count: number;
|
||||
readonly in_use_record_count: number;
|
||||
readonly directory_record_count: number;
|
||||
readonly extension_record_count: number;
|
||||
readonly fixup_mismatch_count: number;
|
||||
readonly fixup_unverified_record_count: number;
|
||||
readonly truncated_tail_bytes: number;
|
||||
readonly record_headers: readonly MftRecord[];
|
||||
readonly omitted_record_headers: number;
|
||||
}
|
||||
|
||||
const text = (ascii: Uint8Array): string =>
|
||||
Buffer.from(ascii).toString("latin1");
|
||||
|
||||
const failure = (
|
||||
message: string,
|
||||
boundary?: RegionDecodeFailure["boundary"],
|
||||
): RegionDecodeFailure => ({
|
||||
kind: "format",
|
||||
message,
|
||||
...(boundary === undefined ? {} : { boundary }),
|
||||
});
|
||||
|
||||
const limit = (
|
||||
message: string,
|
||||
boundary?: RegionDecodeFailure["boundary"],
|
||||
): RegionDecodeFailure => ({
|
||||
kind: "limit",
|
||||
message,
|
||||
...(boundary === undefined ? {} : { boundary }),
|
||||
});
|
||||
|
||||
const parseRecordFlags = (raw: number): MftRecord["flags"] => {
|
||||
const names: string[] = [];
|
||||
let unknownMask = raw;
|
||||
for (const [bit, name] of FILE_RECORD_NAMES) {
|
||||
if ((raw & bit) !== 0) names.push(name);
|
||||
unknownMask &= ~bit;
|
||||
}
|
||||
return { raw, names, unknownMask };
|
||||
};
|
||||
|
||||
const isRecordSize = (bytes: number): boolean =>
|
||||
bytes >= MFT_MIN_RECORD_BYTES &&
|
||||
bytes <= MFT_MAX_RECORD_BYTES &&
|
||||
(bytes & (bytes - 1)) === 0;
|
||||
|
||||
const updateSequenceArrayFits = (
|
||||
offset: number,
|
||||
entries: number,
|
||||
sectors: number,
|
||||
): boolean =>
|
||||
entries >= USA_MIN_ENTRIES &&
|
||||
entries <= sectors + 1 &&
|
||||
offset >= NTFS_MFT_MIN_RECORD_BYTES &&
|
||||
offset % 2 === 0 &&
|
||||
offset + entries * 2 <= FIXUP_SECTOR_BYTES - 2;
|
||||
|
||||
/** Recognize a FILE header without requiring its trailers or attributes to verify. */
|
||||
export const hasMftFileRecordHeader = (prefix: Uint8Array): boolean => {
|
||||
if (
|
||||
prefix.length < NTFS_MFT_MIN_RECORD_BYTES ||
|
||||
!hasMftFileRecordSignature(prefix)
|
||||
)
|
||||
return false;
|
||||
const view = new DataView(
|
||||
prefix.buffer,
|
||||
prefix.byteOffset,
|
||||
prefix.byteLength,
|
||||
);
|
||||
return (
|
||||
isRecordSize(view.getUint32(RECORD_HEADER.totalBytes, true)) ||
|
||||
updateSequenceArrayFits(
|
||||
view.getUint16(RECORD_HEADER.updateSequenceOffset, true),
|
||||
view.getUint16(RECORD_HEADER.updateSequenceCount, true),
|
||||
MFT_DEFAULT_RECORD_BYTES / FIXUP_SECTOR_BYTES,
|
||||
)
|
||||
);
|
||||
};
|
||||
|
||||
/**
|
||||
* Parse FILE record headers and verify their update-sequence fixups.
|
||||
*
|
||||
* Only the record structure is established: attributes, their names, the
|
||||
* filenames and the USN journal are left to later steps, so no path, timestamp
|
||||
* or file identity beyond the record itself is claimed. A record whose fixup
|
||||
* does not match keeps its header facts, because the stream still declares it,
|
||||
* and the mismatch is reported on the record and counted in the metadata.
|
||||
*/
|
||||
export const parseMftImage = (
|
||||
bytes: Uint8Array,
|
||||
): Result<MftImage, RegionDecodeFailure> => {
|
||||
const records: MftRecord[] = [];
|
||||
const inventory = scanMft(bytes, records);
|
||||
return inventory.ok ? ok({ ...inventory.value, records }) : inventory;
|
||||
};
|
||||
|
||||
const scanMft = (
|
||||
bytes: Uint8Array,
|
||||
retainedRecords?: MftRecord[],
|
||||
): Result<MftInventory, RegionDecodeFailure> => {
|
||||
if (bytes.length < NTFS_MFT_MIN_RECORD_BYTES)
|
||||
return err(
|
||||
failure(
|
||||
`file is ${String(bytes.length)} bytes, shorter than one ${String(NTFS_MFT_MIN_RECORD_BYTES)}-byte record header`,
|
||||
),
|
||||
);
|
||||
if (!hasMftFileRecordSignature(bytes))
|
||||
return err(failure("record 0 does not begin with the FILE signature"));
|
||||
|
||||
const limitations: string[] = [];
|
||||
const header = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
|
||||
const declaredRecordBytes = header.getUint32(RECORD_HEADER.totalBytes, true);
|
||||
const recordBytes = isRecordSize(declaredRecordBytes)
|
||||
? declaredRecordBytes
|
||||
: MFT_DEFAULT_RECORD_BYTES;
|
||||
if (!isRecordSize(declaredRecordBytes))
|
||||
limitations.push(
|
||||
`record 0 declares a record size of ${String(declaredRecordBytes)} bytes, which is not a supported power of two from ${String(MFT_MIN_RECORD_BYTES)} to ${String(MFT_MAX_RECORD_BYTES)}; records were read at the ${String(MFT_DEFAULT_RECORD_BYTES)}-byte default`,
|
||||
);
|
||||
|
||||
const recordHeaders: MftRecord[] = [];
|
||||
let recordCount = 0;
|
||||
let inUseCount = 0;
|
||||
let directoryCount = 0;
|
||||
let extensionCount = 0;
|
||||
let mismatchCount = 0;
|
||||
let unverifiedCount = 0;
|
||||
let invalidSlots = 0;
|
||||
let invalidExtents = 0;
|
||||
let inconsistentRecordNumbers = 0;
|
||||
let firstInvalidExtentOffset: number | undefined;
|
||||
let firstInvalidOffset: number | undefined;
|
||||
for (
|
||||
let offset = 0;
|
||||
offset + recordBytes <= bytes.length;
|
||||
offset += recordBytes
|
||||
) {
|
||||
if (
|
||||
!hasMftFileRecordSignature(
|
||||
bytes.subarray(offset, offset + MFT_FILE_RECORD_MAGIC_BYTES),
|
||||
)
|
||||
) {
|
||||
invalidSlots += 1;
|
||||
firstInvalidOffset ??= offset;
|
||||
if (!isRecordSize(declaredRecordBytes)) break;
|
||||
continue;
|
||||
}
|
||||
const record = readRecord(bytes, offset, offset / recordBytes, recordBytes);
|
||||
retainedRecords?.push(record);
|
||||
if (recordHeaders.length < MFT_INLINE_HEADER_LIMIT)
|
||||
recordHeaders.push(record);
|
||||
recordCount += 1;
|
||||
if (record.inUse) inUseCount += 1;
|
||||
if (record.directory) directoryCount += 1;
|
||||
if (record.baseRecordReference !== null) extensionCount += 1;
|
||||
if (record.fixups.mismatchSector !== null) mismatchCount += 1;
|
||||
if (
|
||||
record.fixups.malformed ||
|
||||
record.fixups.coveredSectors < recordBytes / FIXUP_SECTOR_BYTES
|
||||
)
|
||||
unverifiedCount += 1;
|
||||
if (record.recordNumberMatchesSlot === false)
|
||||
inconsistentRecordNumbers += 1;
|
||||
if (!record.extentValid) {
|
||||
invalidExtents += 1;
|
||||
firstInvalidExtentOffset ??= offset;
|
||||
}
|
||||
}
|
||||
if (firstInvalidOffset !== undefined)
|
||||
limitations.push(
|
||||
`record at ${String(firstInvalidOffset)} does not begin with the FILE signature; ${String(invalidSlots)} slot(s) without a FILE signature were found`,
|
||||
);
|
||||
|
||||
if (firstInvalidExtentOffset !== undefined)
|
||||
limitations.push(
|
||||
`${String(invalidExtents)} FILE record(s) declare inconsistent record extents, beginning at offset ${String(firstInvalidExtentOffset)}`,
|
||||
);
|
||||
|
||||
if (inconsistentRecordNumbers !== 0)
|
||||
limitations.push(
|
||||
`${String(inconsistentRecordNumbers)} FILE record(s) declare a record number that disagrees with their stream slot`,
|
||||
);
|
||||
|
||||
const truncatedTailBytes = bytes.length % recordBytes;
|
||||
if (truncatedTailBytes !== 0)
|
||||
limitations.push(
|
||||
`${String(truncatedTailBytes)} trailing byte(s) do not fill a ${String(recordBytes)}-byte record and were not parsed`,
|
||||
);
|
||||
|
||||
if (mismatchCount !== 0)
|
||||
limitations.push(
|
||||
`${String(mismatchCount)} record(s) hold a sector trailer that is not the update sequence number; their attributes were not read and no later step should trust them`,
|
||||
);
|
||||
if (unverifiedCount !== 0)
|
||||
limitations.push(
|
||||
`${String(unverifiedCount)} record(s) carry an invalid or incomplete update sequence array; their sector trailers were not fully verified`,
|
||||
);
|
||||
|
||||
const metadata: MftMetadata = {
|
||||
kind: "ntfs-mft",
|
||||
record_bytes: recordBytes,
|
||||
record_count: recordCount,
|
||||
in_use_record_count: inUseCount,
|
||||
directory_record_count: directoryCount,
|
||||
extension_record_count: extensionCount,
|
||||
fixup_mismatch_count: mismatchCount,
|
||||
fixup_unverified_record_count: unverifiedCount,
|
||||
truncated_tail_bytes: truncatedTailBytes,
|
||||
record_headers: recordHeaders,
|
||||
omitted_record_headers: recordCount - recordHeaders.length,
|
||||
};
|
||||
|
||||
return ok({ metadata, recordBytes, limitations });
|
||||
};
|
||||
|
||||
const readRecord = (
|
||||
bytes: Uint8Array,
|
||||
fileOffset: number,
|
||||
index: number,
|
||||
recordBytes: number,
|
||||
): MftRecord => {
|
||||
const record = bytes.subarray(fileOffset, fileOffset + recordBytes);
|
||||
const view = new DataView(
|
||||
record.buffer,
|
||||
record.byteOffset,
|
||||
record.byteLength,
|
||||
);
|
||||
|
||||
const flags = view.getUint16(RECORD_HEADER.flags, true);
|
||||
const baseRecordReference = view.getBigUint64(
|
||||
RECORD_HEADER.baseRecordReference,
|
||||
true,
|
||||
);
|
||||
|
||||
const usaOffset = view.getUint16(RECORD_HEADER.updateSequenceOffset, true);
|
||||
const fixups = verifyFixups(record, view, usaOffset);
|
||||
const usedBytes = view.getUint32(RECORD_HEADER.usedBytes, true);
|
||||
const totalBytes = view.getUint32(RECORD_HEADER.totalBytes, true);
|
||||
const firstAttributeOffset = view.getUint16(
|
||||
RECORD_HEADER.firstAttributeOffset,
|
||||
true,
|
||||
);
|
||||
|
||||
const declaredRecordNumber =
|
||||
!fixups.malformed && usaOffset >= RECORD_HEADER_WITH_NUMBER_BYTES
|
||||
? view.getUint32(RECORD_HEADER.recordNumber, true)
|
||||
: null;
|
||||
|
||||
return {
|
||||
index,
|
||||
fileOffset,
|
||||
sequenceNumber: view.getUint16(RECORD_HEADER.sequenceNumber, true),
|
||||
hardLinkCount: view.getUint16(RECORD_HEADER.hardLinkCount, true),
|
||||
flags: parseRecordFlags(flags),
|
||||
inUse: (flags & FILE_RECORD_IN_USE) !== 0,
|
||||
directory: (flags & FILE_RECORD_IS_DIRECTORY) !== 0,
|
||||
usedBytes,
|
||||
totalBytes,
|
||||
extentValid:
|
||||
totalBytes === recordBytes &&
|
||||
usedBytes >= NTFS_MFT_MIN_RECORD_BYTES &&
|
||||
usedBytes % 8 === 0 &&
|
||||
usedBytes <= totalBytes &&
|
||||
firstAttributeOffset % 8 === 0 &&
|
||||
firstAttributeOffset >= NTFS_MFT_MIN_RECORD_BYTES &&
|
||||
firstAttributeOffset + 4 <= usedBytes &&
|
||||
(fixups.malformed ||
|
||||
firstAttributeOffset >= usaOffset + fixups.entries * 2),
|
||||
baseRecordReference:
|
||||
baseRecordReference === 0n
|
||||
? null
|
||||
: decodeMftFileReference(baseRecordReference),
|
||||
declaredRecordNumber,
|
||||
recordNumberMatchesSlot:
|
||||
declaredRecordNumber === null ? null : declaredRecordNumber === index,
|
||||
logFileSequenceNumber: view
|
||||
.getBigUint64(RECORD_HEADER.logFileSequenceNumber, true)
|
||||
.toString(),
|
||||
firstAttributeOffset,
|
||||
nextAttributeId: view.getUint16(RECORD_HEADER.nextAttributeId, true),
|
||||
fixups,
|
||||
};
|
||||
};
|
||||
|
||||
/**
|
||||
* Each sector's last two bytes are replaced on disk by the update sequence
|
||||
* number; the array's first entry is that number and the rest are the saved
|
||||
* trailer words. A sector whose trailer is not the sequence number was not
|
||||
* cleanly written, so the record's attributes cannot be trusted.
|
||||
*/
|
||||
const verifyFixups = (
|
||||
record: Uint8Array,
|
||||
view: DataView,
|
||||
usaOffset: number,
|
||||
): MftRecordFixups => {
|
||||
const usaCount = view.getUint16(RECORD_HEADER.updateSequenceCount, true);
|
||||
const sectors = Math.floor(record.length / FIXUP_SECTOR_BYTES);
|
||||
// Entry 0 holds the sequence number, so sector n's saved trailer is entry n+1.
|
||||
const coveredSectors = Math.max(0, usaCount - 1);
|
||||
|
||||
if (!updateSequenceArrayFits(usaOffset, usaCount, sectors))
|
||||
return {
|
||||
sequenceNumber: 0,
|
||||
entries: usaCount,
|
||||
verifiedSectors: 0,
|
||||
coveredSectors: 0,
|
||||
mismatchSector: null,
|
||||
malformed: true,
|
||||
};
|
||||
|
||||
const sequenceNumber = view.getUint16(usaOffset, true);
|
||||
if (sequenceNumber === 0 || sequenceNumber === 0xffff)
|
||||
return {
|
||||
sequenceNumber,
|
||||
entries: usaCount,
|
||||
verifiedSectors: 0,
|
||||
coveredSectors,
|
||||
mismatchSector: null,
|
||||
malformed: true,
|
||||
};
|
||||
for (let sector = 0; sector < coveredSectors; sector += 1) {
|
||||
const trailerOffset = sector * FIXUP_SECTOR_BYTES + FIXUP_SECTOR_BYTES - 2;
|
||||
if (view.getUint16(trailerOffset, true) !== sequenceNumber)
|
||||
return {
|
||||
sequenceNumber,
|
||||
entries: usaCount,
|
||||
verifiedSectors: sector,
|
||||
coveredSectors,
|
||||
mismatchSector: sector,
|
||||
malformed: false,
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
sequenceNumber,
|
||||
entries: usaCount,
|
||||
verifiedSectors: coveredSectors,
|
||||
coveredSectors,
|
||||
mismatchSector: null,
|
||||
malformed: false,
|
||||
};
|
||||
};
|
||||
|
||||
/** Buffer the admitted input once while accepting arbitrary chunk boundaries. */
|
||||
class StreamingMftDecoder implements RegionDecoder<MftInventory> {
|
||||
readonly #bytes: Uint8Array;
|
||||
#written = 0;
|
||||
|
||||
constructor(fileBytes: number) {
|
||||
this.#bytes = new Uint8Array(fileBytes);
|
||||
}
|
||||
|
||||
write(chunk: Uint8Array): Result<void, RegionDecodeFailure> {
|
||||
if (chunk.length > this.#bytes.length - this.#written)
|
||||
return err(failure("received more bytes than the admitted file length"));
|
||||
this.#bytes.set(chunk, this.#written);
|
||||
this.#written += chunk.length;
|
||||
return ok(undefined);
|
||||
}
|
||||
|
||||
finish(): Result<MftInventory, RegionDecodeFailure> {
|
||||
if (this.#written !== this.#bytes.length)
|
||||
return err(failure("stream ended before the admitted file length"));
|
||||
return scanMft(this.#bytes);
|
||||
}
|
||||
}
|
||||
|
||||
export const createMftDecoder = (
|
||||
fileBytes: number,
|
||||
): Result<RegionDecoder<MftInventory>, RegionDecodeFailure> =>
|
||||
Number.isSafeInteger(fileBytes) &&
|
||||
fileBytes >= 0 &&
|
||||
fileBytes <= MFT_MAX_FILE_BYTES
|
||||
? ok(new StreamingMftDecoder(fileBytes))
|
||||
: err(
|
||||
limit(
|
||||
`file is ${String(fileBytes)} bytes, exceeding the fixed ${String(MFT_MAX_FILE_BYTES)}-byte (512 MiB) ceiling`,
|
||||
"file-size",
|
||||
),
|
||||
);
|
||||
@@ -1,4 +1,3 @@
|
||||
import { readFile } from "node:fs/promises";
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import type { JsonValue } from "../domain/jsonValue.js";
|
||||
@@ -1120,25 +1119,6 @@ describe("Ghidra native instruction decoder context for ARM TMode and MIPS ISA_M
|
||||
});
|
||||
});
|
||||
|
||||
describe("Ghidra bridge script processor context contract", () => {
|
||||
it("binds TMode to ARM and ISA_MODE to MIPS in ReaGhidraBridge.java", async () => {
|
||||
const bridgePath = new URL(
|
||||
"../../bridge/ghidra/ReaGhidraBridge.java",
|
||||
import.meta.url,
|
||||
);
|
||||
const content = await readFile(bridgePath, "utf8");
|
||||
expect(content).toContain(
|
||||
'currentProgram.getLanguageID().getIdAsString().startsWith("ARM:")',
|
||||
);
|
||||
expect(content).toContain(
|
||||
'currentProgram.getLanguageID().getIdAsString().startsWith("MIPS:")',
|
||||
);
|
||||
expect(content).toContain('"TMode"');
|
||||
expect(content).toContain('"ISA_MODE"');
|
||||
expect(content).toContain('"ghidra-program-context"');
|
||||
});
|
||||
});
|
||||
|
||||
describe("Ghidra dossier nested boundary composition", () => {
|
||||
const identityLocations = [
|
||||
"caller",
|
||||
|
||||
@@ -0,0 +1,53 @@
|
||||
import {
|
||||
ArtifactResourceScope,
|
||||
type ArtifactResourceOwner,
|
||||
} from "../../artifacts/ArtifactResourceScope.js";
|
||||
|
||||
/** Retain a cancelled response branch until its underlying cancellation settles. */
|
||||
export class StateResponseCleanup {
|
||||
readonly #scope = new ArtifactResourceScope();
|
||||
|
||||
constructor(private readonly changed: () => void = () => {}) {}
|
||||
|
||||
get cleanupResources(): readonly string[] {
|
||||
return this.#scope.cleanupResources;
|
||||
}
|
||||
|
||||
cancel(cancel: () => Promise<void>): void {
|
||||
let state:
|
||||
| { status: "pending" }
|
||||
| { status: "done" }
|
||||
| { status: "failed"; cause: unknown } = { status: "pending" };
|
||||
const owner: ArtifactResourceOwner = {
|
||||
kind: "lease",
|
||||
resource: "state response body",
|
||||
lease: {
|
||||
release: async () => {
|
||||
if (state.status === "pending")
|
||||
throw new Error("State response cancellation is still pending");
|
||||
if (state.status === "failed") throw state.cause;
|
||||
},
|
||||
},
|
||||
};
|
||||
this.#scope.retain(owner);
|
||||
this.changed();
|
||||
void Promise.resolve()
|
||||
.then(cancel)
|
||||
.then(
|
||||
async () => {
|
||||
state = { status: "done" };
|
||||
await this.#scope.release(owner);
|
||||
this.changed();
|
||||
},
|
||||
(cause: unknown) => {
|
||||
state = { status: "failed", cause };
|
||||
this.changed();
|
||||
},
|
||||
)
|
||||
.catch(() => undefined);
|
||||
}
|
||||
|
||||
close(): Promise<void> {
|
||||
return this.#scope.close();
|
||||
}
|
||||
}
|
||||
@@ -1,3 +1,4 @@
|
||||
import { StateResponseCleanup } from "./StateResponseCleanup.js";
|
||||
import type { EnvironmentStateStore } from "../../application/environment/environmentStateStore.js";
|
||||
import { ArtifactResourceScope } from "../../artifacts/ArtifactResourceScope.js";
|
||||
import type { AnalysisInputError } from "../../domain/analysisErrorCore.js";
|
||||
@@ -46,17 +47,24 @@ interface CallContext {
|
||||
readonly intake: StateIntake;
|
||||
readonly cancel: AbortSignal;
|
||||
readonly files: ArtifactResourceScope;
|
||||
readonly responses: StateResponseCleanup;
|
||||
}
|
||||
|
||||
/** `data/add`: exactly one part, whose document replaces the whole state only when it is valid and the call was not cancelled. */
|
||||
const addState = async (
|
||||
{ store, intake, cancel, files }: CallContext,
|
||||
{ store, intake, cancel, files, responses }: CallContext,
|
||||
params: unknown,
|
||||
): Promise<RpcOutcome> => {
|
||||
const parts = isRecord(params) ? params.parts : undefined;
|
||||
if (!Array.isArray(parts) || parts.length !== 1)
|
||||
return { error: invalidParams("data/add takes exactly one part") };
|
||||
const document = await partDocument(parts[0], intake, cancel, files);
|
||||
const document = await partDocument(
|
||||
parts[0],
|
||||
intake,
|
||||
cancel,
|
||||
files,
|
||||
responses,
|
||||
);
|
||||
if (!document.ok) return { error: document.error };
|
||||
// A file read is not interruptible, so a cancellation during it is honoured here, before the state changes.
|
||||
if (cancel.aborted) return { error: CANCELLED };
|
||||
@@ -101,6 +109,7 @@ export const stateDataPlane = (
|
||||
intake: StateIntake = DEFAULT_STATE_INTAKE,
|
||||
stopped: AbortSignal = new AbortController().signal,
|
||||
files: ArtifactResourceScope = new ArtifactResourceScope(),
|
||||
responses: StateResponseCleanup = new StateResponseCleanup(),
|
||||
): RpcMethods => {
|
||||
let previous: Promise<unknown> = Promise.resolve();
|
||||
return (method, params, signal) => {
|
||||
@@ -111,7 +120,7 @@ export const stateDataPlane = (
|
||||
const next = previous.then(() =>
|
||||
cancel.aborted
|
||||
? { error: CANCELLED }
|
||||
: run({ store, intake, cancel, files }, params),
|
||||
: run({ store, intake, cancel, files, responses }, params),
|
||||
);
|
||||
previous = next.catch(() => undefined);
|
||||
return next;
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { StateResponseCleanup } from "./StateResponseCleanup.js";
|
||||
import type { HttpHostname } from "../../domain/httpHostname.js";
|
||||
import { STATE_INTAKE_LIMIT_BYTES } from "../../application/environment/environmentStateDocument.js";
|
||||
import {
|
||||
@@ -477,6 +478,8 @@ export const serveEnvironment = async (
|
||||
};
|
||||
// A state file whose close failed stays owned here: later file reads retry it, and close reports it if it is still open.
|
||||
const stateFiles = new ArtifactResourceScope();
|
||||
let responseStateChanged = (): void => undefined;
|
||||
const stateResponses = new StateResponseCleanup(() => responseStateChanged());
|
||||
const served = await serveStreamableHttp({
|
||||
host,
|
||||
port,
|
||||
@@ -499,6 +502,7 @@ export const serveEnvironment = async (
|
||||
},
|
||||
stopping.signal,
|
||||
stateFiles,
|
||||
stateResponses,
|
||||
),
|
||||
),
|
||||
...(logger === undefined ? {} : { logger }),
|
||||
@@ -512,11 +516,14 @@ export const serveEnvironment = async (
|
||||
});
|
||||
let httpClosePending = false;
|
||||
let stateClosePending = false;
|
||||
let responseClosePending = false;
|
||||
let settledSuccessful = false;
|
||||
let closing: Promise<void> | undefined;
|
||||
const shutdownFailures = new Set<AnalysisError>();
|
||||
const owner: EnvironmentShutdownOwner = {
|
||||
get cleanupResources() {
|
||||
const stateResources = stateFiles.cleanupResources;
|
||||
const responseResources = stateResponses.cleanupResources;
|
||||
return [
|
||||
...(httpClosePending ? [served.url] : []),
|
||||
...(stateClosePending
|
||||
@@ -524,6 +531,11 @@ export const serveEnvironment = async (
|
||||
? stateResources
|
||||
: ["environment state-file scope"]
|
||||
: []),
|
||||
...(responseClosePending
|
||||
? responseResources.length > 0
|
||||
? responseResources
|
||||
: ["state response cleanup"]
|
||||
: []),
|
||||
];
|
||||
},
|
||||
close: () => {
|
||||
@@ -531,7 +543,7 @@ export const serveEnvironment = async (
|
||||
const attempt = (async () => {
|
||||
stop();
|
||||
const failures: {
|
||||
readonly stage: "http-server" | "state-files";
|
||||
readonly stage: "http-server" | "state-files" | "state-responses";
|
||||
readonly cause: unknown;
|
||||
}[] = [];
|
||||
try {
|
||||
@@ -548,6 +560,15 @@ export const serveEnvironment = async (
|
||||
stateClosePending = true;
|
||||
failures.push({ stage: "state-files", cause });
|
||||
}
|
||||
try {
|
||||
await stateResponses.close();
|
||||
if (stateResponses.cleanupResources.length > 0)
|
||||
throw new Error("State response cleanup remains pending");
|
||||
responseClosePending = false;
|
||||
} catch (cause: unknown) {
|
||||
responseClosePending = true;
|
||||
failures.push({ stage: "state-responses", cause });
|
||||
}
|
||||
if (failures.length > 0) {
|
||||
retainedEnvironmentShutdowns.add(owner);
|
||||
const failure = new AnalysisOutputError(
|
||||
@@ -583,6 +604,7 @@ export const serveEnvironment = async (
|
||||
retainedEnvironmentShutdowns.delete(owner);
|
||||
for (const failure of shutdownFailures) failure.resolveCleanup();
|
||||
shutdownFailures.clear();
|
||||
settledSuccessful = true;
|
||||
})();
|
||||
closing = attempt;
|
||||
void attempt.catch(() => {
|
||||
@@ -591,6 +613,26 @@ export const serveEnvironment = async (
|
||||
return attempt;
|
||||
},
|
||||
};
|
||||
// A fetcher may hand back a response after an earlier stop completed. Keep
|
||||
// that newly acquired body owned and make a later close observe its outcome.
|
||||
responseStateChanged = () => {
|
||||
if (!stopping.signal.aborted) return;
|
||||
if (stateResponses.cleanupResources.length > 0) {
|
||||
responseClosePending = true;
|
||||
retainedEnvironmentShutdowns.add(owner);
|
||||
if (settledSuccessful) {
|
||||
settledSuccessful = false;
|
||||
closing = undefined;
|
||||
}
|
||||
} else if (
|
||||
shutdownFailures.size === 0 &&
|
||||
!httpClosePending &&
|
||||
!stateClosePending
|
||||
) {
|
||||
responseClosePending = false;
|
||||
retainedEnvironmentShutdowns.delete(owner);
|
||||
}
|
||||
};
|
||||
return ok({
|
||||
...served,
|
||||
card,
|
||||
|
||||
@@ -1037,7 +1037,122 @@ describe("serveEnvironment with a state", () => {
|
||||
expect(missing.isError).toBe(true);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("state response cleanup", () => {
|
||||
it("reports pending teed response cleanup and retries after its sibling closes", async () => {
|
||||
let sourceCancelled = false;
|
||||
const stream = new ReadableStream<Uint8Array>({
|
||||
start(controller) {
|
||||
controller.enqueue(new Uint8Array([1]));
|
||||
},
|
||||
cancel() {
|
||||
sourceCancelled = true;
|
||||
},
|
||||
});
|
||||
const [body, sibling] = stream.tee();
|
||||
const environment = await serveState(
|
||||
{ source, state },
|
||||
{
|
||||
fetchState: async () =>
|
||||
new Response(body, {
|
||||
headers: { "content-length": String(STATE_INTAKE_LIMIT_BYTES + 1) },
|
||||
}),
|
||||
},
|
||||
);
|
||||
let timer: ReturnType<typeof setTimeout> | undefined;
|
||||
try {
|
||||
const response = await fetch(
|
||||
new URL(ENVIRONMENT_RPC_PATH, environment.url),
|
||||
{
|
||||
method: "POST",
|
||||
headers: { "content-type": "application/json" },
|
||||
body: JSON.stringify(addByUri(1)),
|
||||
signal: AbortSignal.timeout(10_000),
|
||||
},
|
||||
);
|
||||
await expect(response.json()).resolves.toMatchObject({
|
||||
error: { code: -32602 },
|
||||
});
|
||||
const closing = Promise.race([
|
||||
environment.close(),
|
||||
new Promise<never>((_, reject) => {
|
||||
timer = setTimeout(
|
||||
() => reject(new Error("server close waited for tee sibling")),
|
||||
10_000,
|
||||
);
|
||||
}),
|
||||
]);
|
||||
await expect(closing).rejects.toMatchObject({
|
||||
_tag: "AnalysisOutputError",
|
||||
cleanupIncomplete: true,
|
||||
diagnostics: { cleanup_stages: ["state-responses"] },
|
||||
});
|
||||
expect(sourceCancelled).toBe(false);
|
||||
await sibling.cancel();
|
||||
expect(sourceCancelled).toBe(true);
|
||||
await expect(environment.close()).resolves.toBeUndefined();
|
||||
} finally {
|
||||
if (timer !== undefined) clearTimeout(timer);
|
||||
await Promise.allSettled([body.cancel(), sibling.cancel()]);
|
||||
await environment.close();
|
||||
}
|
||||
});
|
||||
|
||||
it("retains a late response cleanup after an earlier successful stop", async () => {
|
||||
let giveResponse!: (response: Response) => void;
|
||||
const gate = new Promise<Response>((resolve) => {
|
||||
giveResponse = resolve;
|
||||
});
|
||||
const held = heldFetcher(() => gate);
|
||||
let release!: () => void;
|
||||
let acknowledge!: () => void;
|
||||
const cleanup = new Promise<void>((resolve) => {
|
||||
release = resolve;
|
||||
});
|
||||
const entered = new Promise<void>((resolve) => {
|
||||
acknowledge = resolve;
|
||||
});
|
||||
const environment = await serveState(
|
||||
{ source, state },
|
||||
{ fetchState: held.fetchState },
|
||||
);
|
||||
const answered = fetch(new URL(ENVIRONMENT_RPC_PATH, environment.url), {
|
||||
method: "POST",
|
||||
headers: { "content-type": "application/json" },
|
||||
body: JSON.stringify(addByUri(1)),
|
||||
signal: AbortSignal.timeout(10_000),
|
||||
}).catch(() => undefined);
|
||||
try {
|
||||
await held.fetching;
|
||||
await environment.close();
|
||||
giveResponse(
|
||||
new Response(
|
||||
new ReadableStream<Uint8Array>({
|
||||
cancel() {
|
||||
acknowledge();
|
||||
return cleanup;
|
||||
},
|
||||
}),
|
||||
),
|
||||
);
|
||||
await entered;
|
||||
await expect(environment.close()).rejects.toMatchObject({
|
||||
_tag: "AnalysisOutputError",
|
||||
cleanupIncomplete: true,
|
||||
diagnostics: { cleanup_stages: ["state-responses"] },
|
||||
});
|
||||
release();
|
||||
await expect(environment.close()).resolves.toBeUndefined();
|
||||
} finally {
|
||||
release();
|
||||
await answered;
|
||||
await environment.close();
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe("serveEnvironment state file cleanup", () => {
|
||||
it("stops without waiting for a stalled download", async () => {
|
||||
const held = heldFetcher(() => new Promise<Response>(() => undefined));
|
||||
const environment = await serveState(
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { StateResponseCleanup } from "./StateResponseCleanup.js";
|
||||
import { expect, it, vi } from "vitest";
|
||||
|
||||
import { ArtifactResourceScope } from "../../artifacts/ArtifactResourceScope.js";
|
||||
@@ -11,14 +12,17 @@ import {
|
||||
|
||||
const inlineDocument = async (bytes: string) => {
|
||||
const files = new ArtifactResourceScope();
|
||||
const responses = new StateResponseCleanup();
|
||||
try {
|
||||
return await partDocument(
|
||||
{ kind: "file", file: { bytes } },
|
||||
DEFAULT_STATE_INTAKE,
|
||||
new AbortController().signal,
|
||||
files,
|
||||
responses,
|
||||
);
|
||||
} finally {
|
||||
await responses.close();
|
||||
await files.close();
|
||||
}
|
||||
};
|
||||
@@ -79,3 +83,109 @@ it("downloads with redirects refused and the 30 s timeout by default", async ()
|
||||
fetched.mockRestore();
|
||||
}
|
||||
});
|
||||
|
||||
it.each(["http-status", "declared-overflow", "stream-overflow"] as const)(
|
||||
"returns %s while a teed response's other branch remains open",
|
||||
async (scenario) => {
|
||||
const source = new ReadableStream<Uint8Array>({
|
||||
start(controller) {
|
||||
controller.enqueue(
|
||||
new Uint8Array(
|
||||
scenario === "stream-overflow" ? STATE_INTAKE_LIMIT_BYTES + 1 : 1,
|
||||
),
|
||||
);
|
||||
},
|
||||
});
|
||||
const [body, sibling] = source.tee();
|
||||
const response = new Response(body, {
|
||||
status: scenario === "http-status" ? 503 : 200,
|
||||
...(scenario === "declared-overflow"
|
||||
? {
|
||||
headers: { "content-length": String(STATE_INTAKE_LIMIT_BYTES + 1) },
|
||||
}
|
||||
: {}),
|
||||
});
|
||||
const files = new ArtifactResourceScope();
|
||||
const responses = new StateResponseCleanup();
|
||||
const operation = partDocument(
|
||||
{ kind: "file", file: { uri: "https://state.example.test/state.json" } },
|
||||
{ fetchState: async () => response, timeoutMs: 1_000 },
|
||||
new AbortController().signal,
|
||||
files,
|
||||
responses,
|
||||
);
|
||||
let timer: ReturnType<typeof setTimeout> | undefined;
|
||||
try {
|
||||
const outcome = await Promise.race([
|
||||
operation.then((result) => ({ kind: "result" as const, result })),
|
||||
new Promise<{ kind: "deadline" }>((resolve) => {
|
||||
timer = setTimeout(() => resolve({ kind: "deadline" }), 3_000);
|
||||
}),
|
||||
]);
|
||||
expect(outcome.kind).toBe("result");
|
||||
expect(responses.cleanupResources).toEqual(["state response body"]);
|
||||
await expect(responses.close()).rejects.toMatchObject({
|
||||
reason: "unavailable",
|
||||
});
|
||||
if (outcome.kind === "result") {
|
||||
expect(outcome.result).toMatchObject(
|
||||
scenario === "http-status"
|
||||
? { ok: false, error: { data: { reason: "http-status" } } }
|
||||
: {
|
||||
ok: false,
|
||||
error: {
|
||||
message: expect.stringContaining(
|
||||
String(STATE_INTAKE_LIMIT_BYTES),
|
||||
),
|
||||
},
|
||||
},
|
||||
);
|
||||
}
|
||||
} finally {
|
||||
if (timer !== undefined) clearTimeout(timer);
|
||||
await sibling.cancel();
|
||||
await operation;
|
||||
await responses.close();
|
||||
await files.close();
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
it("retains the exact failure when response cancellation rejects", async () => {
|
||||
const failure = new Error("controlled cancellation failure");
|
||||
const body = new ReadableStream<Uint8Array>({
|
||||
cancel() {
|
||||
throw failure;
|
||||
},
|
||||
});
|
||||
const files = new ArtifactResourceScope();
|
||||
const responses = new StateResponseCleanup();
|
||||
const result = await partDocument(
|
||||
{ kind: "file", file: { uri: "https://state.example.test/state.json" } },
|
||||
{
|
||||
fetchState: async () => new Response(body, { status: 503 }),
|
||||
timeoutMs: 1_000,
|
||||
},
|
||||
new AbortController().signal,
|
||||
files,
|
||||
responses,
|
||||
);
|
||||
expect(result).toMatchObject({
|
||||
ok: false,
|
||||
error: { data: { reason: "http-status" } },
|
||||
});
|
||||
await new Promise<void>((resolve) => setImmediate(resolve));
|
||||
const rejected: unknown = await responses
|
||||
.close()
|
||||
.catch((cause: unknown) => cause);
|
||||
expect(rejected).toMatchObject({
|
||||
reason: "unavailable",
|
||||
cleanup: { resources: ["state response body"] },
|
||||
});
|
||||
expect((rejected as Error).cause).toBeInstanceOf(AggregateError);
|
||||
expect(((rejected as Error).cause as AggregateError).errors).toContain(
|
||||
failure,
|
||||
);
|
||||
expect(responses.cleanupResources).toEqual(["state response body"]);
|
||||
await files.close();
|
||||
});
|
||||
@@ -1,3 +1,4 @@
|
||||
import { StateResponseCleanup } from "./StateResponseCleanup.js";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
import { mediaTypeEssence } from "../../domain/analysisView/recordedMediaType.js";
|
||||
@@ -228,6 +229,7 @@ const downloadBounds = (
|
||||
const readCapped = async (
|
||||
body: ReadableStream<Uint8Array> | null,
|
||||
bounds: DownloadBounds,
|
||||
responses: StateResponseCleanup,
|
||||
): Promise<Result<Buffer, RpcError>> => {
|
||||
if (body === null) return ok(Buffer.alloc(0));
|
||||
const reader = body.getReader();
|
||||
@@ -240,13 +242,15 @@ const readCapped = async (
|
||||
if (done) return ok(Buffer.concat(chunks, total));
|
||||
total += value.byteLength;
|
||||
if (total > STATE_INTAKE_LIMIT_BYTES) {
|
||||
await reader.cancel().catch(() => undefined);
|
||||
// A tee's sibling can be caller-owned; cancelling this branch must not
|
||||
// delay the refusal until that sibling finishes.
|
||||
responses.cancel(() => reader.cancel());
|
||||
return err(TOO_LARGE);
|
||||
}
|
||||
chunks.push(value);
|
||||
}
|
||||
} catch {
|
||||
void reader.cancel().catch(() => undefined);
|
||||
responses.cancel(() => reader.cancel());
|
||||
return err(fetchFailed(bounds.failure()));
|
||||
}
|
||||
};
|
||||
@@ -256,6 +260,7 @@ const downloadState = async (
|
||||
url: URL,
|
||||
intake: StateIntake,
|
||||
cancel: AbortSignal,
|
||||
responses: StateResponseCleanup,
|
||||
): Promise<Result<Buffer, RpcError>> => {
|
||||
const bounds = downloadBounds(intake.timeoutMs, cancel);
|
||||
const fetching = Promise.resolve().then(() =>
|
||||
@@ -268,7 +273,9 @@ const downloadState = async (
|
||||
// A response that arrives after the download ended is released unread.
|
||||
void fetching
|
||||
.then(
|
||||
(late) => late.body?.cancel(),
|
||||
(late) => {
|
||||
if (late.body !== null) responses.cancel(() => late.body!.cancel());
|
||||
},
|
||||
() => undefined,
|
||||
)
|
||||
.catch(() => undefined);
|
||||
@@ -276,7 +283,8 @@ const downloadState = async (
|
||||
}
|
||||
const declared = Number(response.headers.get("content-length") ?? "0");
|
||||
if (!response.ok || declared > STATE_INTAKE_LIMIT_BYTES) {
|
||||
await response.body?.cancel().catch(() => undefined);
|
||||
// Cancel this branch without waiting for a potentially independent tee sibling.
|
||||
if (response.body !== null) responses.cancel(() => response.body!.cancel());
|
||||
return err(
|
||||
response.ok
|
||||
? TOO_LARGE
|
||||
@@ -286,7 +294,7 @@ const downloadState = async (
|
||||
},
|
||||
);
|
||||
}
|
||||
return readCapped(response.body, bounds);
|
||||
return readCapped(response.body, bounds, responses);
|
||||
};
|
||||
|
||||
const NETWORK_SHARE = invalidParams(
|
||||
@@ -315,6 +323,7 @@ const fileBytes = async (
|
||||
intake: StateIntake,
|
||||
cancel: AbortSignal,
|
||||
files: ArtifactResourceScope,
|
||||
responses: StateResponseCleanup,
|
||||
): Promise<Result<Uint8Array, RpcError>> => {
|
||||
if (typeof file.bytes === "string" && file.uri === undefined) {
|
||||
if (file.bytes.length > MAX_INLINE_BASE64_CHARACTERS) return err(TOO_LARGE);
|
||||
@@ -333,7 +342,8 @@ const fileBytes = async (
|
||||
invalidParams("a file part carries exactly one of bytes or a uri"),
|
||||
);
|
||||
const url = new URL(file.uri);
|
||||
if (url.protocol === "https:") return downloadState(url, intake, cancel);
|
||||
if (url.protocol === "https:")
|
||||
return downloadState(url, intake, cancel, responses);
|
||||
if (url.protocol !== "file:")
|
||||
return err(invalidParams("a file uri is a file:// or https:// URL"));
|
||||
const path = localFilePath(url);
|
||||
@@ -346,6 +356,7 @@ export const partDocument = async (
|
||||
intake: StateIntake,
|
||||
cancel: AbortSignal,
|
||||
files: ArtifactResourceScope,
|
||||
responses: StateResponseCleanup,
|
||||
): Promise<Result<unknown, RpcError>> => {
|
||||
if (isRecord(part) && part.kind === "data" && isRecord(part.data))
|
||||
return ok(part.data);
|
||||
@@ -362,6 +373,6 @@ export const partDocument = async (
|
||||
mediaTypeEssence(mimeType) !== "application/json")
|
||||
)
|
||||
return err(invalidParams("a file part's mimeType is application/json"));
|
||||
const bytes = await fileBytes(part.file, intake, cancel, files);
|
||||
const bytes = await fileBytes(part.file, intake, cancel, files, responses);
|
||||
return bytes.ok ? parseDocumentBytes(bytes.value) : bytes;
|
||||
};
|
||||
@@ -15,6 +15,13 @@ const cases = [
|
||||
0,
|
||||
],
|
||||
['spawn("control", [], {stdio:"inherit"})', "control", "inherit", 0],
|
||||
['spawn("global-nan-env", [], {env:NaN})', "global-nan-env", "default", 0],
|
||||
[
|
||||
'((NaN) => spawn("shadowed-nan-env", [], {env:NaN}))({})',
|
||||
"shadowed-nan-env",
|
||||
"default",
|
||||
0,
|
||||
],
|
||||
[
|
||||
'spawn("spread-after", [], {stdio:"pipe",...input})',
|
||||
"spread-after",
|
||||
@@ -233,6 +240,12 @@ const exportAssignmentSource = `
|
||||
export = argv;
|
||||
`;
|
||||
|
||||
const environmentCommands = [
|
||||
"named-argv",
|
||||
"execfile-dynamic",
|
||||
"shadowed-nan-env",
|
||||
];
|
||||
|
||||
// Callback/options ambiguity cannot change exec's effective stdio.
|
||||
const opaqueExecCallbackCommands = [
|
||||
"eval-callback",
|
||||
@@ -342,7 +355,7 @@ cliTest.for(["CLI", "stdio MCP"] as const)(
|
||||
expect(child.properties.stdio_mode, String(command)).toBe(mode);
|
||||
expect(child.properties.argv_count, String(command)).toBe(count);
|
||||
expect(child.properties.environment_supplied, String(command)).toBe(
|
||||
["named-argv", "execfile-dynamic"].includes(String(command)),
|
||||
environmentCommands.includes(String(command)),
|
||||
);
|
||||
expect(child.identity.source_range).not.toBeNull();
|
||||
const stdio = graph.relations.filter(
|
||||
@@ -377,9 +390,7 @@ cliTest.for(["CLI", "stdio MCP"] as const)(
|
||||
relation.relation === "supplies-env" &&
|
||||
relation.target_node_id === child.node_id,
|
||||
),
|
||||
).toHaveLength(
|
||||
["named-argv", "execfile-dynamic"].includes(String(command)) ? 1 : 0,
|
||||
);
|
||||
).toHaveLength(environmentCommands.includes(String(command)) ? 1 : 0);
|
||||
expect(
|
||||
graph.relations.filter(
|
||||
(relation) =>
|
||||
|
||||
@@ -1,81 +1,72 @@
|
||||
import { execFile } from "node:child_process";
|
||||
import { readFile, readdir, writeFile } from "node:fs/promises";
|
||||
import { readFile, readdir } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
import { promisify } from "node:util";
|
||||
import { expect, it } from "vitest";
|
||||
import { expect } from "vitest";
|
||||
import { parse } from "yaml";
|
||||
import { resolveSwiftcCompileTimeoutMs } from "../../../src/process/DarwinProcessRunTokenReader.js";
|
||||
import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js";
|
||||
import { processTest } from "../../support/process/processFixture.js";
|
||||
|
||||
const execute = promisify(execFile);
|
||||
const hostLabels: Partial<Record<NodeJS.Platform, string>> = {
|
||||
linux: "linux",
|
||||
darwin: "macOS",
|
||||
win32: "Windows",
|
||||
};
|
||||
const hostLabel = hostLabels[process.platform];
|
||||
const preparationBudget =
|
||||
process.platform === "darwin" ? resolveSwiftcCompileTimeoutMs() : 0;
|
||||
|
||||
it("enforces workflow platform labels before contacting Cutter", async () => {
|
||||
const workflow = parse(
|
||||
await readFile(".github/workflows/real-cutter.yml", "utf8"),
|
||||
);
|
||||
const labels = workflow.jobs["cutter-upstream"].strategy.matrix.include.map(
|
||||
(entry: { platform: string }) => entry.platform,
|
||||
);
|
||||
expect(labels).toEqual(["linux", "macOS", "Windows"]);
|
||||
const root = await createTestTempDirectory("rea-cutter-platform-");
|
||||
const preload = join(root, "preload.mjs");
|
||||
await writeFile(
|
||||
preload,
|
||||
`
|
||||
import childProcess from 'node:child_process';
|
||||
import { syncBuiltinESMExports } from 'node:module';
|
||||
Object.defineProperty(process, 'platform', { value: process.env.REA_PROBE_PLATFORM });
|
||||
childProcess.execFile = () => { throw new Error('fixture discovery reached'); };
|
||||
syncBuiltinESMExports();
|
||||
`,
|
||||
);
|
||||
const check = async (selected: string | undefined, actual: string) => {
|
||||
const env: NodeJS.ProcessEnv = {
|
||||
...process.env,
|
||||
TMPDIR: root,
|
||||
TMP: root,
|
||||
TEMP: root,
|
||||
REA_PROBE_PLATFORM: actual,
|
||||
processTest.skipIf(hostLabel === undefined)(
|
||||
"enforces workflow platform labels before contacting Cutter",
|
||||
{ timeout: Math.min(2_147_483_647, 2 * preparationBudget + 30_000) },
|
||||
async ({ processes }) => {
|
||||
const workflow = parse(
|
||||
await readFile(".github/workflows/real-cutter.yml", "utf8"),
|
||||
);
|
||||
const labels = workflow.jobs["cutter-upstream"].strategy.matrix.include.map(
|
||||
(entry: { platform: string }) => entry.platform,
|
||||
);
|
||||
expect(labels).toEqual(["linux", "macOS", "Windows"]);
|
||||
const root = await createTestTempDirectory("rea-cutter-platform-");
|
||||
// A private absent directory proves actual discovery without contacting a user's bridge.
|
||||
const bridgeDirectory = join(root, "absent-bridge");
|
||||
const check = async (selected: string | undefined) => {
|
||||
const env: NodeJS.ProcessEnv = {
|
||||
...process.env,
|
||||
TMPDIR: root,
|
||||
TMP: root,
|
||||
TEMP: root,
|
||||
REA_CUTTER_BRIDGE_DIR: bridgeDirectory,
|
||||
};
|
||||
if (selected === undefined) delete env.EXPECTED_PLATFORM;
|
||||
else env.EXPECTED_PLATFORM = selected;
|
||||
const result = await processes.run(
|
||||
process.execPath,
|
||||
["scripts/verify-real-cutter.mjs"],
|
||||
{
|
||||
env,
|
||||
timeoutMs: Math.min(2_147_483_647, preparationBudget + 15_000),
|
||||
},
|
||||
);
|
||||
expect(result.exitCode, result.stderr).toBe(1);
|
||||
return result.stderr;
|
||||
};
|
||||
if (selected === undefined) delete env.EXPECTED_PLATFORM;
|
||||
else env.EXPECTED_PLATFORM = selected;
|
||||
const result: unknown = await execute(
|
||||
process.execPath,
|
||||
["--import", preload, "scripts/verify-real-cutter.mjs"],
|
||||
{ env, timeout: 10_000 },
|
||||
).then(
|
||||
() => {
|
||||
throw new Error("Fixture discovery should stop the verifier");
|
||||
},
|
||||
(error: unknown) => error,
|
||||
for (const selected of [hostLabel, undefined]) {
|
||||
const stderr = await check(selected);
|
||||
expect(stderr).toContain("Open upstream Cutter with the REA plugin");
|
||||
expect(stderr).toContain("no_live_bridge_found");
|
||||
}
|
||||
const other = labels.find((label: string) => label !== hostLabel);
|
||||
expect(other).toBeDefined();
|
||||
expect(await check(other)).toContain(
|
||||
"Cutter runner did not match its declared platform",
|
||||
);
|
||||
if (
|
||||
!(result instanceof Error) ||
|
||||
!("stderr" in result) ||
|
||||
typeof result.stderr !== "string"
|
||||
)
|
||||
throw new Error("Missing verifier diagnostics");
|
||||
return result.stderr;
|
||||
};
|
||||
for (const [selected, actual] of [
|
||||
["linux", "linux"],
|
||||
["macOS", "darwin"],
|
||||
["Windows", "win32"],
|
||||
] as const)
|
||||
expect(await check(selected, actual)).toContain(
|
||||
"fixture discovery reached",
|
||||
expect(await check("unknown")).toContain(
|
||||
"Unrecognized Cutter runner platform",
|
||||
);
|
||||
expect(await check("linux", "darwin")).toContain(
|
||||
"Cutter runner did not match its declared platform",
|
||||
);
|
||||
expect(await check("unknown", "linux")).toContain(
|
||||
"Unrecognized Cutter runner platform",
|
||||
);
|
||||
expect(await check(undefined, "linux")).toContain(
|
||||
"fixture discovery reached",
|
||||
);
|
||||
expect(
|
||||
(await readdir(root)).filter((name) =>
|
||||
name.startsWith("rea-cutter-smoke-"),
|
||||
),
|
||||
).toEqual([]);
|
||||
});
|
||||
expect(
|
||||
(await readdir(root)).filter((name) =>
|
||||
name.startsWith("rea-cutter-smoke-"),
|
||||
),
|
||||
).toEqual([]);
|
||||
},
|
||||
);
|
||||
@@ -783,7 +783,7 @@ it.each([
|
||||
{ code: false, ghidra_bridge: true, runtime: false, package: false },
|
||||
],
|
||||
[
|
||||
"tests/conformance/ghidra/ReaArmContextProbe.java",
|
||||
"tests/conformance/ghidra/ReaInstructionContextProbe.java",
|
||||
{ ghidra_bridge: true, runtime: false, package: false },
|
||||
],
|
||||
[
|
||||
|
||||
@@ -0,0 +1,84 @@
|
||||
import { writeFile } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
|
||||
import { expect } from "vitest";
|
||||
|
||||
import { createBinarySession } from "../../../src/composition/binary.js";
|
||||
import { parseConfig } from "../../../src/config/parseConfig.js";
|
||||
import { artifactInspectionResultSchema } from "../../../src/domain/artifactInspection.js";
|
||||
import { parseEvidence } from "../../../src/domain/evidence.js";
|
||||
import { silentLogger } from "../../../src/logger.js";
|
||||
import { createServer } from "../../../src/server/createServer.js";
|
||||
import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js";
|
||||
import { mcpTest } from "../../support/mcp/mcpFixture.js";
|
||||
import { serveCli } from "../cli/cliServeHarness.js";
|
||||
|
||||
const expectMftInventory = (value: unknown): void => {
|
||||
const evidence = parseEvidence(value);
|
||||
const inspection = artifactInspectionResultSchema.parse(
|
||||
evidence.normalized_result,
|
||||
);
|
||||
const inventory = inspection.substeps[0]?.evidence.normalized_result;
|
||||
expect(inventory?.manifest.root_format).toBe("ntfs-mft");
|
||||
expect(inventory?.occurrences[0]?.format_metadata).toMatchObject({
|
||||
kind: "ntfs-mft",
|
||||
record_count: 1,
|
||||
fixup_unverified_record_count: 0,
|
||||
record_headers: [
|
||||
{
|
||||
index: 0,
|
||||
fileOffset: 0,
|
||||
sequenceNumber: 1,
|
||||
declaredRecordNumber: null,
|
||||
fixups: { verifiedSectors: 2 },
|
||||
},
|
||||
],
|
||||
omitted_record_headers: 0,
|
||||
});
|
||||
};
|
||||
|
||||
mcpTest(
|
||||
"admits an MFT capture by signature through CLI and MCP",
|
||||
async ({ mcp, onTestFinished }) => {
|
||||
const root = await createTestTempDirectory("rea-ntfs-mft-admission-");
|
||||
const path = join(root, "capture.bin");
|
||||
const record = Buffer.alloc(1024);
|
||||
record.write("FILE");
|
||||
record.writeUInt16LE(42, 4);
|
||||
record.writeUInt16LE(3, 6);
|
||||
record.writeUInt16LE(1, 16);
|
||||
record.writeUInt16LE(56, 20);
|
||||
record.writeUInt32LE(64, 24);
|
||||
record.writeUInt32LE(0xffff_ffff, 56);
|
||||
record.writeUInt32LE(1024, 28);
|
||||
record.writeUInt16LE(5, 42);
|
||||
record.writeUInt16LE(5, 510);
|
||||
record.writeUInt16LE(5, 1022);
|
||||
await writeFile(path, record);
|
||||
|
||||
const cli = await serveCli(["inspect-artifact", path, "--json"]);
|
||||
expect(cli.exitCode, cli.stdout).toBe(0);
|
||||
expectMftInventory(JSON.parse(cli.stdout));
|
||||
|
||||
const configured = parseConfig({ REA_ANALYSIS_PROVIDER: "auto" });
|
||||
if (!configured.ok) throw configured.error;
|
||||
const session = createBinarySession(configured.value, silentLogger, {});
|
||||
onTestFinished(async () => {
|
||||
await session.close();
|
||||
});
|
||||
const client = await mcp.connect(
|
||||
createServer({ kind: "session", session }),
|
||||
);
|
||||
const opened = await client.callTool({
|
||||
name: "open_binary",
|
||||
arguments: { path },
|
||||
});
|
||||
expect(opened.isError, JSON.stringify(opened.content)).not.toBe(true);
|
||||
const inspected = await client.callTool({
|
||||
name: "inspect_artifact",
|
||||
arguments: {},
|
||||
});
|
||||
expect(inspected.isError, JSON.stringify(inspected.content)).not.toBe(true);
|
||||
expectMftInventory(inspected.structuredContent);
|
||||
},
|
||||
);
|
||||
@@ -1,4 +1,4 @@
|
||||
import { readFile, writeFile } from "node:fs/promises";
|
||||
import { readFile, stat, writeFile } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { expect } from "vitest";
|
||||
@@ -175,7 +175,11 @@ registerHooks({ load(url, context, nextLoad) {
|
||||
),
|
||||
],
|
||||
{
|
||||
env: { ...process.env, REA_AGENT_EVAL_ROOT: parent },
|
||||
env: {
|
||||
...process.env,
|
||||
REA_AGENT_EVAL_ROOT: parent,
|
||||
REA_AGENT_EVAL_KEEP_FIXTURES: "false",
|
||||
},
|
||||
timeoutMs: 10_000,
|
||||
},
|
||||
);
|
||||
@@ -190,14 +194,13 @@ registerHooks({ load(url, context, nextLoad) {
|
||||
if (incomplete) {
|
||||
expect(result.stderr).toContain("Artifact cleanup remains incomplete");
|
||||
expect(result.stderr).toContain("fixture cleanup denied");
|
||||
expect(await readFile(join(run, "codex/auth.json"), "utf8")).toBe(
|
||||
"inert fixture",
|
||||
);
|
||||
expect((await stat(run)).isDirectory()).toBe(true);
|
||||
} else {
|
||||
await expect(
|
||||
readFile(join(run, "codex/auth.json")),
|
||||
).rejects.toMatchObject({ code: "ENOENT" });
|
||||
await expect(stat(run)).rejects.toMatchObject({ code: "ENOENT" });
|
||||
}
|
||||
await expect(readFile(join(run, "codex/auth.json"))).rejects.toMatchObject({
|
||||
code: "ENOENT",
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
@@ -274,7 +277,11 @@ registerHooks({ load(url, context, nextLoad) {
|
||||
),
|
||||
],
|
||||
{
|
||||
env: { ...process.env, REA_AGENT_EVAL_ROOT: parent },
|
||||
env: {
|
||||
...process.env,
|
||||
REA_AGENT_EVAL_ROOT: parent,
|
||||
REA_AGENT_EVAL_KEEP_FIXTURES: "false",
|
||||
},
|
||||
timeoutMs: 10_000,
|
||||
},
|
||||
);
|
||||
@@ -298,13 +305,14 @@ registerHooks({ load(url, context, nextLoad) {
|
||||
if (incomplete) {
|
||||
expect(result.stderr).toContain("Artifact cleanup remains incomplete");
|
||||
expect(result.stderr).toContain("darwin-fixture-reader");
|
||||
expect(
|
||||
await readFile(join(observed.run, "codex/auth.json"), "utf8"),
|
||||
).toBe("inert preparation fixture");
|
||||
expect((await stat(observed.run)).isDirectory()).toBe(true);
|
||||
} else {
|
||||
await expect(
|
||||
readFile(join(observed.run, "codex/auth.json")),
|
||||
).rejects.toMatchObject({ code: "ENOENT" });
|
||||
await expect(stat(observed.run)).rejects.toMatchObject({
|
||||
code: "ENOENT",
|
||||
});
|
||||
}
|
||||
await expect(
|
||||
readFile(join(observed.run, "codex/auth.json")),
|
||||
).rejects.toMatchObject({ code: "ENOENT" });
|
||||
},
|
||||
);
|
||||
@@ -1,24 +1,57 @@
|
||||
import { execFile } from "node:child_process";
|
||||
import {
|
||||
copyFile,
|
||||
cp,
|
||||
mkdir,
|
||||
readFile,
|
||||
symlink,
|
||||
writeFile,
|
||||
} from "node:fs/promises";
|
||||
import { dirname, join, resolve } from "node:path";
|
||||
import { promisify } from "node:util";
|
||||
import { expect, it } from "vitest";
|
||||
import { join, resolve } from "node:path";
|
||||
import { expect } from "vitest";
|
||||
import { z } from "zod";
|
||||
import { createApplicationMcpHarness } from "../../fixtures/applicationMcpHarness.js";
|
||||
import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js";
|
||||
import { processTest } from "../../support/process/processFixture.js";
|
||||
import { resolveSwiftcCompileTimeoutMs } from "../../../src/process/DarwinProcessRunTokenReader.js";
|
||||
|
||||
const execute = promisify(execFile);
|
||||
const preparationBudget =
|
||||
process.platform === "darwin" ? resolveSwiftcCompileTimeoutMs() : 0;
|
||||
|
||||
async function prepareController(controller: string, version: string) {
|
||||
// The verifier loads its production-owned TypeScript source graph at runtime.
|
||||
// Keep that graph in the isolated controller so imports cannot drift from a hand-maintained list.
|
||||
await mkdir(join(controller, "scripts"), { recursive: true });
|
||||
await copyFile(
|
||||
"scripts/verify-release-artifact.mjs",
|
||||
join(controller, "scripts/verify-release-artifact.mjs"),
|
||||
);
|
||||
await cp("scripts/lib", join(controller, "scripts/lib"), {
|
||||
recursive: true,
|
||||
});
|
||||
await mkdir(join(controller, "bridge"), { recursive: true });
|
||||
await cp("bridge/process", join(controller, "bridge/process"), {
|
||||
recursive: true,
|
||||
});
|
||||
await cp("src", join(controller, "src"), {
|
||||
recursive: true,
|
||||
filter: (source) => !/\.(?:test|spec)\.[cm]?[jt]sx?$/u.test(source),
|
||||
});
|
||||
await writeFile(
|
||||
join(controller, "package.json"),
|
||||
JSON.stringify({ name: "rea-agents", version, type: "module" }),
|
||||
);
|
||||
await symlink(
|
||||
resolve("node_modules"),
|
||||
join(controller, "node_modules"),
|
||||
process.platform === "win32" ? "junction" : "dir",
|
||||
);
|
||||
}
|
||||
|
||||
// Release rehearsal runs on Linux; the npm process adapter is also exercised on macOS.
|
||||
it.skipIf(process.platform === "win32")(
|
||||
processTest.skipIf(process.platform === "win32")(
|
||||
"fails without declared runtime, then rehearses an installed tarball and rejects catalog drift",
|
||||
async () => {
|
||||
{ timeout: Math.min(2_147_483_647, 3 * preparationBudget + 210_000) },
|
||||
async ({ processes }) => {
|
||||
const root = await createTestTempDirectory("rea-release-canary-");
|
||||
const candidate = join(root, "candidate");
|
||||
const controller = join(root, "controller");
|
||||
@@ -76,56 +109,46 @@ if (process.argv[2] === "mcp") {
|
||||
} else { process.exitCode = 1; }
|
||||
`,
|
||||
);
|
||||
const packed = await execute(
|
||||
"npm",
|
||||
["pack", "--ignore-scripts", "--json"],
|
||||
{
|
||||
cwd: candidate,
|
||||
},
|
||||
);
|
||||
const pack = async () => {
|
||||
const result = await processes.run(
|
||||
"npm",
|
||||
["pack", "--ignore-scripts", "--json"],
|
||||
{ cwd: candidate, timeoutMs: 30_000 },
|
||||
);
|
||||
expect(result.exitCode, result.stderr).toBe(0);
|
||||
return result;
|
||||
};
|
||||
const packed = await pack();
|
||||
const [{ filename }] = z
|
||||
.tuple([z.object({ filename: z.string() })])
|
||||
.parse(JSON.parse(packed.stdout));
|
||||
const tarball = join(candidate, filename);
|
||||
for (const file of [
|
||||
"scripts/verify-release-artifact.mjs",
|
||||
"scripts/lib/release-package-canary.mjs",
|
||||
"scripts/lib/verify-package-core.mjs",
|
||||
"scripts/lib/mcp-verifier-results.mjs",
|
||||
"scripts/lib/verifier-run.mjs",
|
||||
]) {
|
||||
const target = join(controller, file);
|
||||
await mkdir(dirname(target), { recursive: true });
|
||||
await copyFile(file, target);
|
||||
}
|
||||
await writeFile(
|
||||
join(controller, "package.json"),
|
||||
JSON.stringify({ name: "rea-agents", version, type: "module" }),
|
||||
);
|
||||
await symlink(
|
||||
resolve("node_modules"),
|
||||
join(controller, "node_modules"),
|
||||
process.platform === "win32" ? "junction" : "dir",
|
||||
);
|
||||
await prepareController(controller, version);
|
||||
const invoke = (selected = tarball) =>
|
||||
execute(
|
||||
processes.run(
|
||||
process.execPath,
|
||||
[
|
||||
join(controller, "scripts/verify-release-artifact.mjs"),
|
||||
selected,
|
||||
version,
|
||||
],
|
||||
{ cwd: controller, maxBuffer: 16 * 1024 * 1024 },
|
||||
{
|
||||
cwd: controller,
|
||||
timeoutMs: Math.min(2_147_483_647, preparationBudget + 30_000),
|
||||
},
|
||||
);
|
||||
await expect(invoke()).rejects.toMatchObject({
|
||||
stderr: expect.stringContaining("ERR_MODULE_NOT_FOUND"),
|
||||
});
|
||||
const missing = await invoke();
|
||||
expect(missing.exitCode).toBe(1);
|
||||
expect(missing.stderr).toContain("ERR_MODULE_NOT_FOUND");
|
||||
expect(missing.stderr).toMatch(/dist[/\\]catalogIdentity\.js/u);
|
||||
await symlink(
|
||||
resolve("dist"),
|
||||
join(controller, "dist"),
|
||||
process.platform === "win32" ? "junction" : "dir",
|
||||
);
|
||||
const result = JSON.parse((await invoke()).stdout);
|
||||
const successful = await invoke();
|
||||
expect(successful.exitCode, successful.stderr).toBe(0);
|
||||
const result = JSON.parse(successful.stdout);
|
||||
expect(result).toMatchObject({
|
||||
package: "rea-agents",
|
||||
version,
|
||||
@@ -144,12 +167,10 @@ if (process.argv[2] === "mcp") {
|
||||
join(candidate, "scripts/reply.json"),
|
||||
JSON.stringify(snapshot),
|
||||
);
|
||||
await execute("npm", ["pack", "--ignore-scripts", "--json"], {
|
||||
cwd: candidate,
|
||||
});
|
||||
await expect(invoke()).rejects.toMatchObject({
|
||||
stderr: expect.stringContaining("invalid canonical tool catalog"),
|
||||
});
|
||||
await pack();
|
||||
const drifted = await invoke();
|
||||
expect(drifted.exitCode).toBe(1);
|
||||
expect(drifted.stderr).toContain("invalid canonical tool catalog");
|
||||
await writeFile(
|
||||
join(candidate, "package.json"),
|
||||
JSON.stringify({
|
||||
@@ -159,15 +180,12 @@ if (process.argv[2] === "mcp") {
|
||||
files: ["scripts"],
|
||||
}),
|
||||
);
|
||||
const wrong = await execute("npm", ["pack", "--ignore-scripts", "--json"], {
|
||||
cwd: candidate,
|
||||
});
|
||||
const wrong = await pack();
|
||||
const [{ filename: wrongFilename }] = z
|
||||
.tuple([z.object({ filename: z.string() })])
|
||||
.parse(JSON.parse(wrong.stdout));
|
||||
await expect(invoke(join(candidate, wrongFilename))).rejects.toMatchObject({
|
||||
stderr: expect.stringContaining("Installed package identity"),
|
||||
});
|
||||
const mismatched = await invoke(join(candidate, wrongFilename));
|
||||
expect(mismatched.exitCode).toBe(1);
|
||||
expect(mismatched.stderr).toContain("Installed package identity");
|
||||
},
|
||||
30_000,
|
||||
);
|
||||
@@ -21,6 +21,12 @@ no-overwrite CLI delivery. It checks the target and project files for changes.
|
||||
Clang/LLD/LLVM readelf/LLVM objdump; `REA_RISCV_CLANG`, `REA_RISCV_LLD`,
|
||||
`REA_RISCV_READELF` and `REA_RISCV_OBJDUMP` select existing tool commands.
|
||||
|
||||
- `ReaInstructionContextProbe.java` inspects source-owned ARM, Thumb, MIPS32
|
||||
and MIPS16 instructions through the production bridge in an ephemeral,
|
||||
read-only Ghidra project. It checks exact decoder-context register names,
|
||||
values and provenance; the ARM lane also injects an unavailable provider
|
||||
value. The `ghidra-bridge-sources` CI job runs both processor families. The
|
||||
input bytes are never executed.
|
||||
- `ReaDecodeProbe.java` decodes source-owned instruction bytes from a
|
||||
language's default context and checks each mnemonic, length and optional
|
||||
complete operand. The `ghidra-bridge-sources` CI job runs it for `ARM:LE:32:Cortex`,
|
||||
|
||||
+36
-9
@@ -15,16 +15,20 @@ import java.lang.reflect.Proxy;
|
||||
import java.math.BigInteger;
|
||||
|
||||
/** Real ARM/Thumb decoding; only the unavailable context value is injected. */
|
||||
public class ReaArmContextProbe extends GhidraScript {
|
||||
public class ReaInstructionContextProbe extends GhidraScript {
|
||||
@Override
|
||||
public void run() throws Exception {
|
||||
String[] args = getScriptArgs();
|
||||
require(args.length == 1, "Expected the exact production bridge directory");
|
||||
require(args.length == 1 || (args.length == 2 && args[1].equals("mips")), "Expected the bridge directory and optional MIPS mode");
|
||||
var source = GhidraScriptUtil.findScriptByName("ReaGhidraBridge.java");
|
||||
require(source != null && new File(source.getAbsolutePath()).getCanonicalFile().equals(
|
||||
new File(args[0], "ReaGhidraBridge.java").getCanonicalFile()), "Production bridge source");
|
||||
GhidraScript bridge = GhidraScriptUtil.getProvider(source).getScriptInstance(source, errorWriter);
|
||||
bridge.set(getState(), getControls());
|
||||
if (args.length == 2) {
|
||||
checkMips(bridge);
|
||||
return;
|
||||
}
|
||||
require(currentProgram.getLanguageID().getIdAsString().equals("ARM:LE:32:v7"), "ARM fixture language");
|
||||
var mode = currentProgram.getRegister("TMode");
|
||||
require(mode != null, "Real TMode register");
|
||||
@@ -40,8 +44,8 @@ public class ReaArmContextProbe extends GhidraScript {
|
||||
space.set(null, arm.getAddressSpace());
|
||||
Method inspect = bridge.getClass().getDeclaredMethod("inspectNativeInstruction", JsonObject.class, boolean.class);
|
||||
inspect.setAccessible(true);
|
||||
check(inspect, bridge, arm, "0", 4, "1eff2fe1");
|
||||
check(inspect, bridge, thumb, "1", 2, "7047");
|
||||
check(inspect, bridge, arm, "0", 4, "1eff2fe1", "TMode");
|
||||
check(inspect, bridge, thumb, "1", 2, "7047", "TMode");
|
||||
|
||||
// getValue includes language defaults. Inject only its unavailable result,
|
||||
// preserving the real instruction, register, language and all other APIs.
|
||||
@@ -57,26 +61,49 @@ public class ReaArmContextProbe extends GhidraScript {
|
||||
programField.setAccessible(true);
|
||||
try {
|
||||
programField.set(bridge, program);
|
||||
check(inspect, bridge, arm, null, 4, "1eff2fe1");
|
||||
check(inspect, bridge, arm, null, 4, "1eff2fe1", "TMode");
|
||||
} finally {
|
||||
programField.set(bridge, original);
|
||||
}
|
||||
println("REA_ARM_CONTEXT_OK arm=0 thumb=1 unavailable=null (injected provider value)");
|
||||
}
|
||||
|
||||
private void checkMips(GhidraScript bridge) throws Exception {
|
||||
require(currentProgram.getLanguageID().getIdAsString().equals("MIPS:LE:32:default"), "MIPS fixture language");
|
||||
var mode = currentProgram.getRegister("ISA_MODE");
|
||||
require(mode != null, "Real ISA_MODE register");
|
||||
Address standard = currentProgram.getMemory().getMinAddress();
|
||||
Address compressed = standard.add(4);
|
||||
var context = currentProgram.getProgramContext();
|
||||
context.setValue(mode, standard, standard.add(3), BigInteger.ZERO);
|
||||
require(disassemble(standard), "MIPS32 instruction decoded");
|
||||
// MIPS32 flow may have decoded following bytes in the default mode.
|
||||
clearListing(compressed, compressed.add(3));
|
||||
context.setValue(mode, compressed, compressed.add(1), BigInteger.ONE);
|
||||
require(disassemble(compressed), "MIPS16 instruction decoded");
|
||||
Field space = bridge.getClass().getDeclaredField("sessionDefaultAddressSpace");
|
||||
space.setAccessible(true);
|
||||
space.set(null, standard.getAddressSpace());
|
||||
Method inspect = bridge.getClass().getDeclaredMethod("inspectNativeInstruction", JsonObject.class, boolean.class);
|
||||
inspect.setAccessible(true);
|
||||
check(inspect, bridge, standard, "0", 4, "03008224", "ISA_MODE");
|
||||
check(inspect, bridge, compressed, "1", 2, "5a6a", "ISA_MODE");
|
||||
println("REA_MIPS_CONTEXT_OK standard=0 mips16=1");
|
||||
}
|
||||
|
||||
private static void check(Method inspect, Object bridge, Address address, String mode,
|
||||
int length, String bytes) throws Exception {
|
||||
int length, String bytes, String register) throws Exception {
|
||||
JsonObject input = new JsonObject();
|
||||
input.add("document", JsonNull.INSTANCE);
|
||||
input.addProperty("address", "0x" + Long.toUnsignedString(address.getOffset(), 16));
|
||||
JsonObject result = (JsonObject) inspect.invoke(bridge, input, false);
|
||||
require(result.get("status").getAsString().equals("decoded"), "Decoded bridge response");
|
||||
require(result.get("length").getAsInt() == length, "ARM/Thumb instruction width");
|
||||
require(result.get("status").getAsString().equals("decoded"), "Decoded bridge response: " + result);
|
||||
require(result.get("length").getAsInt() == length, "Instruction width");
|
||||
require(result.get("bytes").getAsString().equals(bytes), "Original instruction bytes");
|
||||
var contexts = result.getAsJsonArray("decoder_context");
|
||||
require(contexts != null && contexts.size() == 1, "One retained mode context");
|
||||
var value = contexts.get(0).getAsJsonObject();
|
||||
require(value.get("register").getAsString().equals("TMode"), "TMode register name");
|
||||
require(value.get("register").getAsString().equals(register), "Exact context register name");
|
||||
require(value.get("source").getAsString().equals("ghidra-program-context"), "Context provenance");
|
||||
require(mode == null ? value.get("value").isJsonNull()
|
||||
: mode.equals(value.get("value").getAsString()), "Exact mode value " + mode);
|
||||
+71
@@ -252,3 +252,74 @@ export const artifactReaderLifecycleCases = (setup: LifecycleFixture): void => {
|
||||
validationCases(setup);
|
||||
});
|
||||
};
|
||||
|
||||
/** An interrupted preparation belongs to its caller, not concurrent readers. */
|
||||
export const artifactPreparationRecoveryCases = ({
|
||||
image,
|
||||
createReader,
|
||||
}: LifecycleFixture): void => {
|
||||
it.each(["cancelled", "physical failure"] as const)(
|
||||
"preserves independent preparation after %s",
|
||||
async (scenario) => {
|
||||
const path = await fixture(image());
|
||||
const admitted = await openRegularFile(path, { symlinks: "reject" });
|
||||
const reader = createReader(path, admitted);
|
||||
const controller = new AbortController();
|
||||
const reason = { request: "first inventory" };
|
||||
const physical = Object.assign(new Error("controlled read failure"), {
|
||||
code: "EIO",
|
||||
});
|
||||
const reached = createDeferred<void>();
|
||||
const release = createDeferred<void>();
|
||||
const nativeRead = admitted.handle.read.bind(admitted.handle);
|
||||
let blocked = false;
|
||||
const read = vi.spyOn(admitted.handle, "read").mockImplementation((async (
|
||||
...args: unknown[]
|
||||
) => {
|
||||
const result = await Reflect.apply(nativeRead, admitted.handle, args);
|
||||
if (!blocked) {
|
||||
blocked = true;
|
||||
reached.resolve();
|
||||
await release.promise;
|
||||
if (scenario === "physical failure") throw physical;
|
||||
}
|
||||
return result;
|
||||
}) as typeof admitted.handle.read);
|
||||
const pending: Promise<unknown>[] = [];
|
||||
try {
|
||||
const first = (async () => {
|
||||
for await (const _ of reader.entries(controller.signal)) {
|
||||
}
|
||||
})();
|
||||
pending.push(first);
|
||||
void first.catch(() => undefined);
|
||||
await awaitAcknowledgement(reached.promise, first, "preparation read");
|
||||
const peer = entries(reader);
|
||||
pending.push(peer);
|
||||
void peer.catch(() => undefined);
|
||||
controller.abort(reason);
|
||||
release.resolve();
|
||||
if (scenario === "physical failure") {
|
||||
await expect(first).rejects.toBe(physical);
|
||||
await expect(peer).rejects.toBe(physical);
|
||||
} else {
|
||||
await expect(first).rejects.toMatchObject({
|
||||
reason: "cancelled",
|
||||
cause: reason,
|
||||
});
|
||||
const recovered = await peer;
|
||||
expect(recovered.length).toBeGreaterThan(0);
|
||||
expect(
|
||||
(await buffer(await reader.open(recovered[0]!))).length,
|
||||
).toBeGreaterThan(0);
|
||||
}
|
||||
} finally {
|
||||
release.resolve();
|
||||
await Promise.allSettled(pending);
|
||||
read.mockRestore();
|
||||
await reader.close();
|
||||
await admitted.handle.close();
|
||||
}
|
||||
},
|
||||
);
|
||||
};
|
||||
Reference in new issue
Block a user