mirror of
https://github.com/morluto/rea.git
synced 2026-11-10 17:30:32 -10:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
6d9618b467 | ||
|
|
0e7a140007 | ||
|
|
9935794a5a | ||
|
|
3ebd9292d0 | ||
|
|
2885347cde | ||
|
|
a6a2527ade | ||
|
|
f8fc572e4a | ||
|
|
ab9e5fa204 | ||
|
|
8d45cb45c4 | ||
|
|
7712e3185e | ||
|
|
466149169e | ||
|
|
1c2626edbd | ||
|
|
b16d18f198 | ||
|
|
14030e2f11 | ||
|
|
9505c73824 | ||
|
|
1a446cdade | ||
|
|
1ef7f02cae | ||
|
|
7b86474be7 | ||
|
|
5d10337709 | ||
|
|
8eadb872f2 | ||
|
|
f170175ec8 | ||
|
|
fbe09f84df | ||
|
|
ebb480ba22 | ||
|
|
da1f029c2f | ||
|
|
4db9c166ef | ||
|
|
29fd6a7d03 | ||
|
|
d7338876d6 | ||
|
|
9df3f7174f | ||
|
|
79e476204f | ||
|
|
9c95218374 | ||
|
|
d9d2641981 | ||
|
|
58bd9a6f02 | ||
|
|
02d8fd878d | ||
|
|
7816142cd1 | ||
|
|
b29909dbb8 | ||
|
|
fd9e91d11b | ||
|
|
6fedd53ebb | ||
|
|
5d79a3c395 | ||
|
|
0e1389858e | ||
|
|
6d58a5fbd4 | ||
|
|
944e203f2f | ||
|
|
d957d00db4 | ||
|
|
70f91ac602 | ||
|
|
f3dfe55eca | ||
|
|
e603982ddd | ||
|
|
e4801e357a | ||
|
|
8a7fbfdd21 | ||
|
|
bb598a23cf | ||
|
|
659fb1c7e6 | ||
|
|
42f42c1ffc | ||
|
|
e2585aa7ce | ||
|
|
1d1c7f98ff | ||
|
|
b1f1b3eff3 | ||
|
|
c1d0f2bb45 | ||
|
|
04de5f3200 | ||
|
|
58b4b17fd3 | ||
|
|
3bc85783b9 | ||
|
|
972d652483 | ||
|
|
f2b7c3bb92 | ||
|
|
f4f927c94e | ||
|
|
11839a2ef8 |
No files matched your search
@@ -25,7 +25,7 @@ jobs:
|
||||
node-version-file: .nvmrc
|
||||
- name: Validate the SDK pin against the provider's verified release
|
||||
run: node scripts/verify-ghidra-bridge-sources.mjs --check-sdk-pin >> "$GITHUB_ENV"
|
||||
# The bridge probe imports only source-owned ARM/Thumb bytes and never
|
||||
# The bridge probe imports only source-owned ARM, Thumb and MIPS bytes and never
|
||||
# executes target code. All SDK/project state stays in runner storage.
|
||||
- name: Acquire checksum-verified Ghidra SDK in isolated runner storage
|
||||
shell: bash
|
||||
@@ -46,7 +46,7 @@ jobs:
|
||||
java_home_variable="JAVA_HOME_${GHIDRA_JAVA_MAJOR}_X64"
|
||||
test -n "${!java_home_variable}"
|
||||
JAVA_HOME="${!java_home_variable}" npm run verify:ghidra:bridge-sources -- --verified-sdk
|
||||
- name: Inspect real ARM and Thumb instruction context through the bridge
|
||||
- name: Inspect real ARM, Thumb and MIPS instruction context through the bridge
|
||||
shell: bash
|
||||
run: |
|
||||
java_home_variable="JAVA_HOME_${GHIDRA_JAVA_MAJOR}_X64"
|
||||
@@ -58,10 +58,21 @@ jobs:
|
||||
-import "${probe_root}/arm-thumb.bin" -loader BinaryLoader -processor ARM:LE:32:v7 \
|
||||
-noanalysis -readOnly -deleteProject \
|
||||
-scriptPath "${GITHUB_WORKSPACE}/tests/conformance/ghidra;${GITHUB_WORKSPACE}/bridge/ghidra" \
|
||||
-postScript ReaArmContextProbe.java "${GITHUB_WORKSPACE}/bridge/ghidra" \
|
||||
-postScript ReaInstructionContextProbe.java "${GITHUB_WORKSPACE}/bridge/ghidra" \
|
||||
> "${probe_root}/output.log" 2>&1 || { cat "${probe_root}/output.log"; exit 1; }
|
||||
cat "${probe_root}/output.log"
|
||||
grep -F 'REA_ARM_CONTEXT_OK arm=0 thumb=1 unavailable=null (injected provider value)' "${probe_root}/output.log"
|
||||
mkdir -p "${probe_root}/mips-project"
|
||||
# MIPS32 addiu v0,a0,3; MIPS16 li v0,90. These bytes are never executed.
|
||||
printf '\003\000\202\044\132\152\000\000' > "${probe_root}/mips.bin"
|
||||
timeout 180s "${GHIDRA_INSTALL_DIR}/support/analyzeHeadless" "${probe_root}/mips-project" MipsContext \
|
||||
-import "${probe_root}/mips.bin" -loader BinaryLoader -processor MIPS:LE:32:default \
|
||||
-noanalysis -readOnly -deleteProject \
|
||||
-scriptPath "${GITHUB_WORKSPACE}/tests/conformance/ghidra;${GITHUB_WORKSPACE}/bridge/ghidra" \
|
||||
-postScript ReaInstructionContextProbe.java "${GITHUB_WORKSPACE}/bridge/ghidra" mips \
|
||||
> "${probe_root}/mips-output.log" 2>&1 || { cat "${probe_root}/mips-output.log"; exit 1; }
|
||||
cat "${probe_root}/mips-output.log"
|
||||
grep -F 'REA_MIPS_CONTEXT_OK standard=0 mips16=1' "${probe_root}/mips-output.log"
|
||||
# Each probe imports source-owned bytes at 0x1000, away from the Cortex-M
|
||||
# vector symbols that ARMCortex.pspec defines at 0, and decodes them from
|
||||
# the language's default context, so M-profile Thumb state must hold at
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
# Changelog
|
||||
|
||||
The next release changes analysis-setting output (#9885). `update_analysis_settings`, `inspect_analysis_settings`, and limit errors no longer include `rea_restart_required`, `restart_required`, `restartRequired`, or `effect`. Read `update_mode` (`runtime`, `restart`, or `parameter`) and `pending_restart`. `runtime` means the setter can change the active value. `restart` means this process keeps the value it started with. `parameter` means this call supplied the value, so `update_analysis_settings` cannot change it; artifact reader limits, web-network capture budgets, and an explicit `analysis_timeout_ms`, `max_heap_mb`, or `command_timeout_ms` use it. There is no compatibility alias. Release Please should copy this migration into that release's breaking-changes section.
|
||||
|
||||
## [6.8.0](https://github.com/morluto/rea/compare/rea-agents-6.7.0...rea-agents-6.8.0) (2026-10-25)
|
||||
|
||||
### Highlights
|
||||
|
||||
@@ -358,6 +358,11 @@ export const documentationSections = [
|
||||
source: "design/limits-and-recovery.md",
|
||||
route: "/design/limits-and-recovery",
|
||||
},
|
||||
{
|
||||
text: "REA limit inventory",
|
||||
source: "design/rea-limit-inventory.md",
|
||||
route: "/design/rea-limit-inventory",
|
||||
},
|
||||
{
|
||||
text: "MCP tool design",
|
||||
source: "development/tool-design.md",
|
||||
|
||||
@@ -111,6 +111,7 @@ product pages. The VitePress documentation source is this `docs/` tree.
|
||||
## Development and verification
|
||||
|
||||
- [Settings and recovery](design/limits-and-recovery.md)
|
||||
- [REA limit inventory](design/rea-limit-inventory.md)
|
||||
- [Flutter budget calibration](verification/flutter-budget-calibration.md)
|
||||
- [Source-map budget calibration](verification/source-map-budget-calibration.md)
|
||||
- [Source-location delivery qualification](verification/source-location-delivery.md)
|
||||
|
||||
@@ -5,7 +5,9 @@ who owns each value, when an update takes effect, how providers enforce it, and
|
||||
how to extend the system. The public tool contract is in
|
||||
[MCP contracts](../reference/mcp-contracts.md#editable-analysis-settings); recorded runtime
|
||||
checks and their limitations are in the
|
||||
[verification report](../verification/runtime-analysis-settings.md).
|
||||
[verification report](../verification/runtime-analysis-settings.md). The
|
||||
[limit inventory](rea-limit-inventory.md) lists every numeric `REA_` limit, which
|
||||
of them are runtime settings, and which stay restart-only.
|
||||
|
||||
## Problem and design boundary
|
||||
|
||||
@@ -255,8 +257,8 @@ are runtime-wide, not scoped to a target or an individual connection.
|
||||
### Failure and retry
|
||||
|
||||
An ASAR admission error supplies `details.limit` with the exact setting,
|
||||
effective value, units, bounds, scope, application point, restart requirement
|
||||
and, when known, required bytes and value source. The agent can immediately call:
|
||||
effective value, units, bounds, scope, application point, `update_mode`,
|
||||
`pending_restart` and, when known, required bytes and value source. The agent can immediately call:
|
||||
|
||||
```json
|
||||
{
|
||||
@@ -273,13 +275,27 @@ that argument; raising a runtime default cannot override it.
|
||||
|
||||
The resource owner constructs `details.limit` from the registry plus the budget
|
||||
it actually enforced. Metadata includes `agent_editable: true`,
|
||||
`scope: "rea-runtime"`, and `rea_restart_required: false`. Optional
|
||||
`scope: "rea-runtime"`, `update_mode`, and `pending_restart`. `update_mode`
|
||||
is `runtime` when `update_analysis_settings` can change the active value,
|
||||
`restart` when this process keeps the value it started with, and `parameter`
|
||||
when this call supplied the value. An explicit `analysis_timeout_ms`,
|
||||
`max_heap_mb`, or `command_timeout_ms`, plus artifact reader limits and
|
||||
web-network capture budgets, use `parameter`. `pending_restart` is false when
|
||||
that reported value is the one this process is using. Optional
|
||||
`required_value`, `source`, and `parameter` fields are supplied only where known
|
||||
or applicable. Do not infer a source because a value happens to equal a default.
|
||||
Error copying, attribution, redaction, and delivery must preserve these facts.
|
||||
Human-readable remediation explains the same failure; clients should not need
|
||||
to parse that sentence to identify an editable setting.
|
||||
|
||||
This output is breaking. Results and errors no longer carry the previous
|
||||
restart boolean or a single effect field, and there is no compatibility alias.
|
||||
Read `update_mode` to see whether the setter can change the active value, and
|
||||
`pending_restart` to see whether a configured value is waiting. A configuration
|
||||
error for a Windows ChangeTime setting carries those two fields on
|
||||
`details.settings`, not only in the constraint sentence. The removed field names
|
||||
are listed in the changelog.
|
||||
|
||||
`scope` and `applies_at` describe a continuing caller, such as an MCP
|
||||
connection. A one-shot invocation, such as a CLI command, has released its
|
||||
provider session and exited its runtime before its caller reads the failure,
|
||||
@@ -288,8 +304,10 @@ so its adapter projects with an explicit `"invocation"` caller lifetime.
|
||||
`scope: "rea-invocation"`, `applies_at: "next_invocation"` and the launch
|
||||
`environment` seed when one exists. `agent_editable: true` still holds, because
|
||||
the caller chooses that environment value or per-call `parameter` for the new
|
||||
invocation; `rea_restart_required: false` means no running REA process needs a
|
||||
restart. A producer keeps guidance that holds only while its caller retains the
|
||||
invocation; `update_mode: "runtime"` and `pending_restart: false` mean the
|
||||
reported value is active in this process. `update_mode: "restart"` means
|
||||
`update_analysis_settings` does not change it. `update_mode: "parameter"` means
|
||||
this call supplied the value. A producer keeps guidance that holds only while its caller retains the
|
||||
session, such as "the Ghidra session remains usable" or a same-session runtime
|
||||
update, in `continuingCallerGuidance`; the shared presentation adds it only for
|
||||
a continuing caller. The producer's own message and remediation must stay true
|
||||
@@ -499,7 +517,10 @@ own `expanded_bytes` controls its independently enforced bundle expansion.
|
||||
|
||||
Failures preserve `details.reader_limit` with the exact parameter, effective
|
||||
value, units, supported range, required value when known, next-call scope and
|
||||
`can_increase`. When `can_increase` is true, raise that field and retry the same
|
||||
`can_increase`. `update_mode` is `parameter`: the caller selects the value on
|
||||
the next call, and `update_analysis_settings` does not accept it.
|
||||
`pending_restart` is false. Web-network capture budgets use the same mode.
|
||||
When `can_increase` is true, raise that field and retry the same
|
||||
complete artifact. At the supported maximum, or when the required representation
|
||||
exceeds it, the failure is not retryable by increasing that budget; inspect the
|
||||
declared dimensions and input validity, or explicitly choose a suitable reader.
|
||||
|
||||
@@ -0,0 +1,229 @@
|
||||
# REA limit inventory
|
||||
|
||||
Audit for [#9881](https://github.com/morluto/rea/issues/9881). Counts below are
|
||||
from this revision of `src`. A numeric `REA_` limit is a name under `src` (tests
|
||||
excluded) that carries a byte, time, count, heap, budget, deadline, or maximum.
|
||||
`scripts/verify/rea-numeric-limits.mjs` fails the static checks when one of those
|
||||
names is neither an environment seed of
|
||||
[`ANALYSIS_SETTINGS`](https://github.com/morluto/rea/blob/main/src/domain/analysisSettingDefinitions.ts) nor an
|
||||
entry in [`numericLimitRegistry.ts`](https://github.com/morluto/rea/blob/main/src/domain/numericLimitRegistry.ts)
|
||||
with a reason.
|
||||
|
||||
`update_mode` is `runtime` when `update_analysis_settings` can change the active
|
||||
value, `restart` when this process keeps the value it started with, and
|
||||
`parameter` when this call supplied the value.
|
||||
`pending_restart` is true only when a configured value is not the one this
|
||||
process is using. A rejected Windows ChangeTime launch value is not active, so
|
||||
that failure reports `pending_restart: true`. Every other value reported here
|
||||
is the one in force, so `pending_restart` is false. Prose that only says to
|
||||
restart REA is marked `prose`. `mcp_server_restart_required` is a
|
||||
server-identity alignment state in `src/serverIdentity.ts`, not a setting field.
|
||||
|
||||
Three numeric limits are absent from this tree and are being moved into the
|
||||
runtime system on their own PRs. This audit does not change them:
|
||||
|
||||
| Name | Tracking |
|
||||
| -------------------------------------------- | -------- |
|
||||
| `REA_NTFS_MFT_MAX_FILE_BYTES` | #9868 |
|
||||
| `REA_REGF_HIVE_MAX_FILE_BYTES` | #9865 |
|
||||
| `REA_WEB_BUNDLE_ENDPOINT_LITERAL_CANDIDATES` | #9836 |
|
||||
|
||||
When those names land in `src`, the static check requires them to be analysis
|
||||
settings. They are not on the restart-only list.
|
||||
|
||||
## Counts
|
||||
|
||||
| Category | Count |
|
||||
| --------------------------------------------------------------------- | ---------------------------------------------------------------------------: |
|
||||
| Runtime analysis settings | 17 (14 have an `REA_` seed; 3 are per-call parameters only) |
|
||||
| Per-call numeric `REA_` limits still read from the launch environment | 11 |
|
||||
| Restart-only numeric `REA_` limits | 15 |
|
||||
| Numeric `REA_` names discovered in `src` | 40 |
|
||||
| Non-numeric `REA_` settings kept restart-only | 53 caller settings, plus process-protocol names that are not caller settings |
|
||||
| Hardcoded ceilings left for follow-up | grouped below; none converted in the inventory change |
|
||||
|
||||
## Runtime analysis settings
|
||||
|
||||
Already adjustable with `update_analysis_settings`. Each call reads the snapshot
|
||||
accepted for that operation. Launch source is `default` or `setting`; a runtime
|
||||
override is `source: "runtime"`. The structured field is `update_mode: "runtime"` and `pending_restart: false`.
|
||||
`applies_at` is the finer point
|
||||
(`next_operation`, `next_worker`, or `next_startup`).
|
||||
|
||||
| Setting | Environment seed | Owner | Default | Floor | Maximum | Read |
|
||||
| -------------------------------------------- | ------------------------------------------------ | ------------------------------------------------------------- | ----------------------------------------- | ---------------: | -------------------------------------: | --------------------- |
|
||||
| `asar.max_header_bytes` | `REA_ASAR_MAX_HEADER_BYTES` | `src/domain/analysisResourceLimits.ts` | 16777216 | 1 | 268435456 | per operation |
|
||||
| `asar.max_listing_bytes` | `REA_ASAR_MAX_LISTING_BYTES` | `src/domain/analysisResourceLimits.ts` | 67108864 | 1 | `Number.MAX_SAFE_INTEGER` | per operation |
|
||||
| `ghidra.value_flow_max_operations` | `REA_GHIDRA_VALUE_FLOW_MAX_OPERATIONS` | `src/domain/analysisResourceLimits.ts` | 3000 | 1 | 2147483647 | per operation |
|
||||
| `ghidra.value_flow_max_inputs_per_operation` | `REA_GHIDRA_VALUE_FLOW_MAX_INPUTS_PER_OPERATION` | `src/domain/analysisResourceLimits.ts` | 64 | 1 | 2147483647 | per operation |
|
||||
| `ghidra.value_flow_max_edges` | `REA_GHIDRA_VALUE_FLOW_MAX_EDGES` | `src/domain/analysisResourceLimits.ts` | 12000 | 1 | 2147483647 | per operation |
|
||||
| `ghidra.decompile_max_payload_mib` | `REA_GHIDRA_DECOMPILE_MAX_PAYLOAD_MIB` | `src/domain/analysisResourceLimits.ts` | unset (`null`, unbounded payload policy) | 1 | 2047 | per operation |
|
||||
| `ghidra.startup_timeout_ms` | `REA_GHIDRA_STARTUP_TIMEOUT_MS` | `src/domain/analysisResourceLimits.ts` | 330000 | 1 | 2147483647 | next provider startup |
|
||||
| `hopper.startup_timeout_ms` | `REA_HOPPER_STARTUP_TIMEOUT_MS` | `src/domain/analysisResourceLimits.ts` | 120000 | 1 | 2147483647 | next provider startup |
|
||||
| `ghidra.decompile_timeout_seconds` | `REA_GHIDRA_DECOMPILE_TIMEOUT_SECONDS` | `src/domain/analysisResourceLimits.ts` | unset (`null`, decompile stays unbounded) | 1 | `MAX_GHIDRA_DECOMPILE_TIMEOUT_SECONDS` | per operation |
|
||||
| `jadx.operation_timeout_ms` | `REA_JADX_OPERATION_TIMEOUT_MS` | `src/domain/analysisResourceLimits.ts` | 120000 | 1 | 2147483647 | per operation |
|
||||
| `regex.matching_deadline_ms` | `REA_REGEX_MATCHING_DEADLINE_MS` | `src/domain/analysisResourceLimits.ts` | 5000 | 1 | 2147483647 | per operation |
|
||||
| `regex.worker_heap_request_mib` | `REA_REGEX_WORKER_HEAP_REQUEST_MIB` | `src/domain/analysisResourceLimits.ts` | 128 | 7 | 16384 | next worker |
|
||||
| `capture.decoder_old_generation_mib` | `REA_CAPTURE_DECODER_OLD_GENERATION_MIB` | `src/domain/analysisResourceLimits.ts` | 192 | semi-space floor | 16384 | next worker |
|
||||
| `javascript.analysis_timeout_ms` | none (`analysis_timeout_ms`) | `src/domain/javascript/javascriptAnalysisResourceControls.ts` | owner default | 1 | 2147483647 | per operation |
|
||||
| `javascript.max_heap_mb` | none (`max_heap_mb`) | `src/domain/javascript/javascriptAnalysisResourceControls.ts` | owner default | owner floor | owner maximum | next worker |
|
||||
| `wasm.command_timeout_ms` | none (`command_timeout_ms`) | `src/domain/wasm/wasmArtifact.ts` | owner default | 1 | owner maximum | per operation |
|
||||
| `native.matching_representation_bytes` | `REA_NATIVE_MATCHING_REPRESENTATION_BYTES` | `src/domain/analysisResourceLimits.ts` | 100663296 | 1 | 1073741824 | per operation |
|
||||
|
||||
An out-of-range `update_analysis_settings` value is rejected by the setting's
|
||||
schema. Invalid launch seeds that the global environment parser accepts as
|
||||
provider-local stay inspectable with `source: "invalid"`.
|
||||
|
||||
## Per-call limits still outside the runtime system
|
||||
|
||||
These are checked when the work runs. They keep the `REA_` variable, default,
|
||||
floor, and maximum, and the migration reads the accepted snapshot instead of a
|
||||
value copied at process start. Until that lands they are `policy: "per-call"` in
|
||||
the registry so the static check still requires a reason.
|
||||
|
||||
| Environment | Owner | Default | Floor | Maximum | Read today | Restart field |
|
||||
| ------------------------------------------ | ------------------------------------------- | ----------------------: | ----: | ---------: | ------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------- |
|
||||
| `REA_BRIDGE_MAX_REQUEST_BYTES` | `src/config/bridgeMaxRequestBytes.ts` | 16777216 | 7464 | 536870887 | each bridge request, from the launch value | prose |
|
||||
| `REA_DEBUGGER_COMMAND_DEADLINE_MS` | `src/domain/analysisResourceLimits.ts` | 120000 | 1 | 2147483647 | copied when the debugger manager is constructed; applied to each command | `update_mode: "restart"` and `pending_restart: false` |
|
||||
| `REA_DEBUGGER_MAX_COMMAND_OUTPUT_BYTES` | `src/domain/analysisResourceLimits.ts` | 16777216 | 1 | 89303718 | copied when the debugger manager is constructed; applied to each command | `update_mode: "restart"` and `pending_restart: false` |
|
||||
| `REA_HOPPER_CALLER_SCAN_BUDGET_MS` | `src/domain/analysisResourceLimits.ts` | 2000 | 1 | 2147483647 | each caller-scan operation, from config resolved at launch | prose |
|
||||
| `REA_JEB_MCP_TIMEOUT_MS` | `src/jeb/JebMcpConnection.ts` | 60000 (MCP SDK default) | 1 | 2147483647 | captured when the JEB connection is created; applied to discovery and each tools/call | prose |
|
||||
| `REA_KUNA_PROJECT_TIMEOUT_MS` | `src/kuna/KunaConfiguration.ts` | 1800000 | 1 | 2147483647 | project export and whole-program scans, from the launch configuration | prose |
|
||||
| `REA_KUNA_TIMEOUT_MS` | `src/kuna/KunaConfiguration.ts` | 120000 | 1 | 2147483647 | each function decompile, from the launch configuration | prose |
|
||||
| `REA_PWNTOOLS_MAX_INPUT_BYTES` | `src/native/pwntools/PwntoolsInputLimit.ts` | 33554432 | 1 | 1073741824 | each pwntools decode, unless the call passes `producer_limits.input_bytes` | prose |
|
||||
| `REA_WINDOWS_CHANGE_TIME_DRIFT_MS` | `src/config/windowsChangeTimeSettle.ts` | 2000 | 1 | 60000 | each ChangeTime observation | `update_mode: "restart"` and `pending_restart: true` on an invalid launch value |
|
||||
| `REA_WINDOWS_CHANGE_TIME_SETTLE_POLL_MS` | `src/config/windowsChangeTimeSettle.ts` | 25 | 1 | 1000 | each ChangeTime observation | `update_mode: "restart"` and `pending_restart: true` on an invalid launch value |
|
||||
| `REA_WINDOWS_CHANGE_TIME_SETTLE_MAX_POLLS` | `src/config/windowsChangeTimeSettle.ts` | 40 | 1 | 1000 | each ChangeTime observation | `update_mode: "restart"` and `pending_restart: true` on an invalid launch value |
|
||||
|
||||
## Restart-only numeric limits
|
||||
|
||||
These stay out of `update_analysis_settings`.
|
||||
|
||||
| Environment | Owner | Default | Floor | Maximum | Read | Why restart-only |
|
||||
| ----------------------------------- | -------------------------------------- | ------------------: | -------: | ------------------------: | ------------------------------------------- | ----------------------------------------------------------------------------------------------------------- |
|
||||
| `REA_DEBUGGER_MAX_PENDING_COMMANDS` | `src/domain/analysisResourceLimits.ts` | 16 | 1 | 256 | debugger manager construction | Queue capacity of the manager created at startup. `update_mode: "restart"` and `pending_restart: false`. |
|
||||
| `REA_DEBUGGER_MAX_SESSIONS` | `src/domain/analysisResourceLimits.ts` | 8 | 1 | 64 | debugger manager construction | Session-pool size of the manager created at startup. `update_mode: "restart"` and `pending_restart: false`. |
|
||||
| `REA_HOPPER_DEMO_DIALOG_TIMEOUT_MS` | `src/domain/hopperStartupFailure.ts` | 10000 | 1 | 2147483647 | Linux private-display adapter start | The helper reads the process environment. Prose restart. |
|
||||
| `REA_JADX_ACTIVE_PROCESSOR_COUNT` | `src/android/JadxConfiguration.ts` | unset (JVM default) | 1 | 2147483647 | JADX JVM launch | `-XX:ActiveProcessorCount` is fixed for that JVM. Prose on invalid values. |
|
||||
| `REA_JADX_HEAP_MIB` | `src/android/JadxConfiguration.ts` | unset (JVM default) | 1 | `Number.MAX_SAFE_INTEGER` | JADX JVM launch | `-Xmx` is fixed for that JVM. Prose on invalid values. |
|
||||
| `REA_KUNA_CACHE_MAX_BYTES` | `src/kuna/KunaConfiguration.ts` | 4294967296 | 1048576 | `Number.MAX_SAFE_INTEGER` | Kuna cache construction | The cache is sized once for the provider. Prose restart. |
|
||||
| `REA_KUNA_JOBS` | `src/kuna/KunaConfiguration.ts` | 1 | 1 | 32 | Kuna run-slot construction | Worker-pool width for the provider. Prose restart. |
|
||||
| `REA_MCP_DOCTOR_DEADLINE_MS` | `src/config/mcpStartupBudgets.ts` | 30000 | 1 | 2147483647 | MCP server startup | Startup phase of the MCP server itself. Prose. |
|
||||
| `REA_MCP_FIRST_CATALOG_BUDGET_MS` | `src/config/mcpStartupBudgets.ts` | 20000 | 1 | 2147483647 | MCP server startup | Startup phase of the MCP server itself. Prose. |
|
||||
| `REA_MCP_INITIALIZE_BUDGET_MS` | `src/config/mcpStartupBudgets.ts` | 15000 | 1 | 2147483647 | MCP server startup | Startup phase of the MCP server itself. Prose. |
|
||||
| `REA_MCP_MAX_INPUT_ELEMENTS` | `src/config/mcpInputBudget.ts` | 1024 | 1 | `Number.MAX_SAFE_INTEGER` | MCP server creation | Argument admission for the server process. Prose restart. |
|
||||
| `REA_MCP_MAX_RESPONSE_BYTES` | `src/config/mcpResponseBudget.ts` | 10485760 when unset | 10485760 | `Number.MAX_SAFE_INTEGER` | MCP server creation | Stdio receive buffer and the paired client buffer. Prose restart. |
|
||||
| `REA_MCP_TOOLS_LIST_BUDGET_MS` | `src/config/mcpStartupBudgets.ts` | 5000 | 1 | 2147483647 | MCP server startup | Startup phase of the MCP server itself. Prose. |
|
||||
| `REA_RECONNECT_TIMEOUT_SECONDS` | `src/hopper/BridgeLauncher.ts` | 30 | 30 | 30 | written into the Hopper bridge helper | Not a caller setting. It is `HOPPER_SHUTDOWN_TIMEOUT_MS / 1000`. |
|
||||
| `REA_SWIFTC_COMPILE_TIMEOUT_MS` | `src/config/providerStartupTimeout.ts` | 120000 | 1 | 2147483647 | Darwin ownership-helper reader construction | Captured at process start for that helper compile. Prose restart. |
|
||||
|
||||
## Restart-only non-numeric settings
|
||||
|
||||
No floor or maximum. They are read from the launch environment (startup). None of them set `effect`. Recovery text tells the caller to change the launch
|
||||
environment, which takes effect on the next process. They stay
|
||||
restart-only because they choose a command, a binary, a filesystem root, the
|
||||
browser sandbox, secret patterns, the provider, or the shape of the MCP server
|
||||
or logger.
|
||||
|
||||
| Name | Owner | Why |
|
||||
| ------------------------------------ | -------------------------------------------------- | ------------------------------------------- |
|
||||
| `REA_ADB_PATH` | `src/adb/AdbConfiguration.ts` | Caller-selected executable path. |
|
||||
| `REA_AGENT_ENV_COMMAND` | `src/application/DesktopBuildDoctor.ts` | Caller-selected command. |
|
||||
| `REA_ANALYSIS_PROVIDER` | `src/config/environment.ts` | Provider choice for the process. |
|
||||
| `REA_APKTOOL_COMMAND` | `src/apktool/ApktoolConfiguration.ts` | Caller-selected command. |
|
||||
| `REA_BINWALK_COMMAND` | `src/application/SetupRegistrationEnvironment.ts` | Caller-selected command. |
|
||||
| `REA_BROWSER_EXECUTABLE` | `src/application/CapabilityInventory.ts` | Caller-selected browser binary. |
|
||||
| `REA_BROWSER_NO_SANDBOX` | `src/application/SetupRegistrationEnvironment.ts` | Browser sandbox policy. |
|
||||
| `REA_CAPINFOS_COMMAND` | `src/pcap/PcapConfiguration.ts` | Caller-selected command. |
|
||||
| `REA_CUTTER_BRIDGE_DIR` | `src/application/SetupRegistrationEnvironment.ts` | Filesystem root for the Cutter bridge. |
|
||||
| `REA_DARWIN_MODULE_CACHE_PATH` | `src/application/SetupRegistrationEnvironment.ts` | Filesystem path for the Swift module cache. |
|
||||
| `REA_DARWIN_SWIFTC_PATH` | `src/application/SetupRegistrationEnvironment.ts` | Caller-selected compiler path. |
|
||||
| `REA_DCONF_COMMAND` | `src/desktop/DconfDatabaseReader.ts` | Caller-selected command. |
|
||||
| `REA_DOCKER_COMMAND` | `src/application/DesktopBuildDoctor.ts` | Caller-selected command. |
|
||||
| `REA_ELECTRON_CAPTURE_ARGV` | `src/browser/PlaywrightElectronActiveProvider.ts` | Caller-selected process arguments. |
|
||||
| `REA_EVM_PRLIMIT_COMMAND` | `src/application/SetupRegistrationEnvironment.ts` | Caller-selected command. |
|
||||
| `REA_FIRMWARE_PRLIMIT_COMMAND` | `src/application/SetupRegistrationEnvironment.ts` | Caller-selected command. |
|
||||
| `REA_GDB_COMMAND` | `src/application/SetupRegistrationEnvironment.ts` | Caller-selected debugger binary. |
|
||||
| `REA_GHIDRA_CHECKPOINT_DIR` | `src/config/environment.ts` | Filesystem root. |
|
||||
| `REA_GHIDRA_COMPILER_SPEC_ID` | `src/config/environment.ts` | Provider analysis choice. |
|
||||
| `REA_GHIDRA_DISABLED_ANALYZERS` | `src/config/environment.ts` | Provider analysis choice. |
|
||||
| `REA_GHIDRA_LANGUAGE_ID` | `src/config/environment.ts` | Provider analysis choice. |
|
||||
| `REA_GHIDRA_NATIVEAOT_JAR` | `src/config/environment.ts` | Caller-selected binary path. |
|
||||
| `REA_GHIDRA_PROGRAM_NAME` | `src/config/environment.ts` | Provider project selection. |
|
||||
| `REA_GHIDRA_PROJECT_PATH` | `src/config/environment.ts` | Filesystem root. |
|
||||
| `REA_GHIDRA_SEED_FILE` | `src/config/environment.ts` | Filesystem path. |
|
||||
| `REA_IDA_MCP_CONFIG` | `src/config/environment.ts` | Filesystem path of the IDA MCP config. |
|
||||
| `REA_ILSPY_CMD_PATH` | `src/config/environment.ts` | Caller-selected executable path. |
|
||||
| `REA_JADX_MCP_JAR` | `src/android/JadxConfiguration.ts` | Caller-selected binary path. |
|
||||
| `REA_JAVASCRIPT_PRLIMIT_COMMAND` | `src/application/SetupRegistrationEnvironment.ts` | Caller-selected command. |
|
||||
| `REA_JEB_MCP_URL` | `src/config/environment.ts` | Provider endpoint. |
|
||||
| `REA_KUNA_CACHE_DIR` | `src/kuna/KunaConfiguration.ts` | Filesystem root. |
|
||||
| `REA_KUNA_COMMAND` | `src/kuna/KunaConfiguration.ts` | Caller-selected executable path. |
|
||||
| `REA_LOG_LEVEL` | `src/config/environment.ts` | Logger configuration for the process. |
|
||||
| `REA_MANAGED_LINUX_HOPPER_LAUNCHER` | `src/application/HopperLauncherDiscovery.ts` | Caller-selected launcher path. |
|
||||
| `REA_MCP_INPUT_SCHEMA_PROFILE` | `src/config/environment.ts` | MCP server schema shape. |
|
||||
| `REA_MCP_OUTPUT_SCHEMA_PROFILE` | `src/config/environment.ts` | MCP server schema shape. |
|
||||
| `REA_MCP_TOOL_CATALOG_PROFILE` | `src/config/environment.ts` | MCP server catalog shape. |
|
||||
| `REA_MCP_TOOL_FAMILIES` | `src/config/environment.ts` | Which tools the MCP server registers. |
|
||||
| `REA_MITMDUMP_COMMAND` | `src/application/SetupRegistrationEnvironment.ts` | Caller-selected command. |
|
||||
| `REA_OBJDUMP_COMMAND` | `src/application/SetupRegistrationEnvironment.ts` | Caller-selected command. |
|
||||
| `REA_PWNDBG_GDB` | `src/native/pwndbg/PwndbgCoreContext.ts` | Caller-selected debugger binary. |
|
||||
| `REA_PWNDBG_GDBINIT` | `src/native/pwndbg/PwndbgCoreContext.ts` | Caller-selected file path. |
|
||||
| `REA_PWNDBG_VENV_PATH` | `src/native/pwndbg/PwndbgCoreContext.ts` | Filesystem root. |
|
||||
| `REA_PWNTOOLS_PYTHON` | `src/native/pwntools/PwntoolsDecoder.ts` | Caller-selected interpreter path. |
|
||||
| `REA_REFERENCE_SECRET_PATTERNS_JSON` | `src/config/environment.ts` | Secret-pattern policy. |
|
||||
| `REA_RIZIN_COMMAND` | `src/rizin/RizinDebugSessionManager.ts` | Caller-selected debugger binary. |
|
||||
| `REA_SIGROK_COMMAND` | `src/signals/SigrokRecordingProvider.ts` | Caller-selected command. |
|
||||
| `REA_SIGROK_PRLIMIT_COMMAND` | `src/signals/SigrokRecordingProvider.ts` | Caller-selected command. |
|
||||
| `REA_TSHARK_COMMAND` | `src/pcap/PcapConfiguration.ts` | Caller-selected command. |
|
||||
| `REA_TSHARK_PRLIMIT_COMMAND` | `src/pcap/PcapNativeBound.ts` | Caller-selected command. |
|
||||
| `REA_UNBLOB_COMMAND` | `src/application/SetupRegistrationEnvironment.ts` | Caller-selected command. |
|
||||
| `REA_WABT_BIN_DIRECTORY` | `src/wasm/WabtArtifactProvider.ts` | Filesystem root of the WABT tools. |
|
||||
| `REA_WAKARU_COMMAND` | `src/application/SetupRegistrationEnvironment.ts` | Caller-selected command. |
|
||||
| `REA_WORKFLOW_PROVIDER` | `src/application/DirectAnalysis.ts` | Provider choice. |
|
||||
| `REA_XPROP_COMMAND` | `src/application/desktop/DesktopCaptureFailure.ts` | Caller-selected command. |
|
||||
|
||||
Names that look like `REA_` but are process protocol, not caller settings:
|
||||
`REA_BUILD_COMMIT`, `REA_BUILD_COMPUTER_REA_ARCHIVE`, `REA_INSTALL_SCRIPT`,
|
||||
`REA_OWNS_PROCESS_LIFETIME`, `REA_PAYLOAD`, `REA_PROCESS_RUN_ID`, `REA_RUN_ID`,
|
||||
`REA_SOCKET`, `REA_STATE_FORMAT`, `REA_TARGET_PATH`, `REA_TOKEN`, bridge markers
|
||||
such as `REA_COMMAND_COMPLETE_`, and the `ERR_REA_WINDOWS_NATIVE` code.
|
||||
`GHIDRA_INSTALL_DIR`, `JAVA_HOME`, `HOPPER_LAUNCHER_PATH`, `HOPPER_TARGET_PATH`,
|
||||
`HOPPER_TARGET_KIND`, and `HOPPER_LOADER_ARGS_JSON` are launch configuration
|
||||
without the `REA_` prefix. Paths and the provider choice among them stay
|
||||
restart-only for the same reason as the table above.
|
||||
|
||||
## Hardcoded ceilings left for follow-up
|
||||
|
||||
These are checked per call and tell the caller this release has no runtime
|
||||
override. They are not `REA_` settings, so the numeric-limit check does not
|
||||
list them. Each family has its own owner. Turning one into a setting needs that
|
||||
owner's default, floor, maximum, and an error that carries the setting id,
|
||||
value, source, and range. Format and protocol ceilings that the
|
||||
[limits guide](limits-and-recovery.md) already calls fixed stay fixed. In
|
||||
particular, JSON nesting stops at `MAX_JSON_DEPTH` (1000) and the debugger
|
||||
control-reply budget stays separate from
|
||||
`REA_DEBUGGER_MAX_COMMAND_OUTPUT_BYTES`.
|
||||
|
||||
| Family | Examples | Follow-up |
|
||||
| ------------------------------------------------ | ------------------------------------------------------------------------------------------------------------ | ------------------------------------------------------------------------------------------- |
|
||||
| Native structural input ceilings | PE resources, imports, load config, certificates, exports, initializers, binary hardening, Windows shortcuts | One setting per ceiling, or a shared input budget only where the owners match. |
|
||||
| Native plist | `MAX_NATIVE_PLIST_BYTES`, decode bytes, nesting depth | Separate from JSON depth. |
|
||||
| SQLite inspection | concurrent inspections, input bytes, diagnostic bytes | Pool width stays restart-only if it sizes the worker set; byte budgets can become settings. |
|
||||
| Producer command allowances | fixed command deadline and retained output where the text says there is no runtime override | Distinct from per-call `producer_limits`. |
|
||||
| Hopper caller-scan retention and page allowances | `HOPPER_CALLER_SCAN_RETENTION_BYTES`, `HOPPER_CALLER_PAGE_ALLOWANCE_BYTES` | Explicitly not raised by the scan time budget. |
|
||||
| Rizin frame protocol | `MAX_FRAME_BYTES`, queued frames | Protocol bounds, not caller budgets. |
|
||||
| Debugger detach and control replies | `DETACH_TIMEOUT_MS`, `DEBUGGER_CONTROL_REPLY_BYTES` | Ownership deadlines, not the shared command deadline. |
|
||||
| Web network capture nesting | `max_nesting_depth` already has a per-call parameter | The fixed depth maximum is the follow-up, not the parameter. |
|
||||
|
||||
## Static check
|
||||
|
||||
`npm run lint` runs `scripts/verify/rea-numeric-limits.mjs` after
|
||||
dependencies are installed. `scripts/verify/architecture-guards.mjs` runs the
|
||||
same check when `typescript6` is already installed; the pre-install CI step
|
||||
cannot. The check scans `src` excluding tests,
|
||||
keeps names that are numeric limits, and requires each name to be an
|
||||
`environment` on an analysis setting or a registry entry with a reason.
|
||||
`policy: "per-call"` is temporary. The migration that moves the limit deletes
|
||||
the entry and rejects any new `per-call` policy so the durable exceptions are
|
||||
only `restart-only`.
|
||||
@@ -26,6 +26,7 @@ for new work, roadmap granularity and producer-owned contracts.
|
||||
- [Native call source coordinates](native/call-source-locations/spec.md) ([plan](native/call-source-locations/plan.md), [tasks](native/call-source-locations/tasks.md), [verification](native/call-source-locations/verification.md))
|
||||
- [Go static string-header literal recovery](native/go-string-literals/spec.md) ([plan](native/go-string-literals/plan.md), [tasks](native/go-string-literals/tasks.md))
|
||||
- [Windows Shell Link shortcut inspection](native/windows-shortcut/spec.md) ([plan](native/windows-shortcut/plan.md), [tasks](native/windows-shortcut/tasks.md))
|
||||
- [NTFS $MFT and USN Journal inspection](native/ntfs-mft/spec.md) ([plan](native/ntfs-mft/plan.md), [tasks](native/ntfs-mft/tasks.md))
|
||||
- [Solidity terminal metadata inspection](solidity/cbor-metadata/spec.md) ([plan](solidity/cbor-metadata/plan.md), [tasks](solidity/cbor-metadata/tasks.md), [verification](solidity/cbor-metadata/verification.md))
|
||||
- [Inspect a V8 code cache container](v8/code-cache-inspection/spec.md) ([plan](v8/code-cache-inspection/plan.md), [tasks](v8/code-cache-inspection/tasks.md))
|
||||
- [Deno compile executable inventory and extraction](artifacts/deno-standalone/spec.md) ([plan](artifacts/deno-standalone/plan.md), [tasks](artifacts/deno-standalone/tasks.md))
|
||||
|
||||
@@ -0,0 +1,39 @@
|
||||
# Implementation plan
|
||||
|
||||
## Boundary and contract
|
||||
|
||||
Follow the existing artifact readers. `src/domain/windows/ntfsMft.ts` owns FILE
|
||||
record bytes and coverage; `src/domain/artifactFormatMetadata.ts` owns the
|
||||
`ntfs-mft` result schema. `NtfsMftArtifactReader` reads the admitted file
|
||||
through `ArtifactFileSource` and exposes its facts through `formatMetadata()`.
|
||||
A record stream is its own content, so the root occurrence carries every fact
|
||||
and no members are enumerated.
|
||||
|
||||
Recognition lives in `hasMftFileRecordSignature` and is applied in
|
||||
`classifyArtifactBytes`, next to the other unambiguous magic probes.
|
||||
|
||||
## Incremental work and validation
|
||||
|
||||
1. Define the first-step contract, limits and observations; add parser fixtures.
|
||||
2. Implement the FILE record header and update-sequence fixup parse with
|
||||
explicit limitations.
|
||||
3. Add the reader, format descriptor and byte classification.
|
||||
4. Decode attribute records, then `$FILE_NAME` attributes so a record's parent
|
||||
reference, names and timestamps become explicit; keep hard links separate.
|
||||
5. Parse `$UsnJrnl:$J` records and correlate them to records by full file
|
||||
reference, including sequence numbers, reporting unresolved references.
|
||||
6. Document the workflow in a native Windows artifact guide.
|
||||
|
||||
## Effects and compatibility
|
||||
|
||||
The workflow reads one caller-selected local file. It adds no target execution,
|
||||
injection, mutation, protection recovery, network access, volume mount or live
|
||||
filesystem access, so no maintainer scope decision is required.
|
||||
`inspect_artifact` routing for every existing format is unchanged, and the new
|
||||
format id appears only for bytes whose first record carries the `FILE`
|
||||
signature.
|
||||
|
||||
## References
|
||||
|
||||
- https://learn.microsoft.com/en-us/windows/win32/fileio/master-file-table
|
||||
- https://github.com/libyal/libfsntfs/blob/main/documentation/New%20Technologies%20File%20System%20(NTFS).asciidoc
|
||||
@@ -0,0 +1,51 @@
|
||||
# NTFS $MFT and USN Journal inspection
|
||||
|
||||
This specification records an implementation increment and its original problem
|
||||
statement. For current behavior and limitations, see the artifact inventory
|
||||
guide.
|
||||
|
||||
## Problem and outcome
|
||||
|
||||
REA has no reader for acquired NTFS metadata artifacts. An analyst holding a
|
||||
captured `$MFT` stream cannot ask which FILE records it contains, whether their
|
||||
update-sequence fixups survived acquisition, or which records are directories,
|
||||
extensions of another record, or no longer in use. Add offline, read-only
|
||||
inventory of a selected `$MFT` file, delivered in steps so each one merges on
|
||||
its own.
|
||||
|
||||
## Requirements
|
||||
|
||||
- Recognize a `$MFT` stream by a plausible first `FILE` record header under
|
||||
any file name. CLI and MCP target admission route it to the artifact reader.
|
||||
- Validate each FILE record header: the `FILE` signature, sequence number, flags,
|
||||
used and total byte counts, base record reference, and the update-sequence
|
||||
array's offset, count and array bounds.
|
||||
- Verify the update-sequence fixups: every sector trailer the array covers must
|
||||
hold the array's sequence number. A mismatch keeps the record's header facts
|
||||
and is reported per record and in the metadata count.
|
||||
- Report a missing, misplaced or short update-sequence array as unverified
|
||||
rather than as a clean record, and a trailing partial record as a byte count.
|
||||
- Report the first invalid record signature and the number of invalid slots.
|
||||
Continue at the next slot when the stream declared a supported record size.
|
||||
With a fallback record size, stop at the first invalid signature.
|
||||
- Bound the whole file at a fixed 512 MiB ceiling and report the ceiling as a
|
||||
`limit` failure with its boundary.
|
||||
- Return the first 32 decoded record headers with their source offsets and the
|
||||
number of omitted headers. Aggregate counts cover the full parsed stream.
|
||||
- Parse only the records' headers and fixups. Attributes, filenames, data
|
||||
streams and the `$UsnJrnl:$J` journal are not decoded in this step, so no path,
|
||||
file size, file reference correlation or change history is claimed.
|
||||
|
||||
## Acceptance
|
||||
|
||||
Synthesized fixtures cover a clean record, a failed fixup, a missing update
|
||||
sequence array, an array shorter than the record's sector count, extension and
|
||||
directory records, a record whose signature is wrong mid-stream, a trailing
|
||||
partial record, an undeclared record size, a 4 KiB record, a stream shorter than
|
||||
one header, bytes without the `FILE` signature, and the file-size ceiling.
|
||||
Inventory tests cover the root occurrence's metadata and the coverage status
|
||||
when a fixup fails, and classification tests cover several file names for the
|
||||
same bytes.
|
||||
|
||||
Not covered in this step: real `$MFT` files written by a Windows host, USN
|
||||
journal records, attribute decoding, and any correlation between the two.
|
||||
@@ -0,0 +1,11 @@
|
||||
# Tasks
|
||||
|
||||
- [x] Define the first-step scope, contract and limits.
|
||||
- [x] Parse FILE record headers and update-sequence fixups with synthesized
|
||||
fixtures and explicit limitations.
|
||||
- [x] Add the reader, format descriptor and byte classification.
|
||||
- [ ] Decode attribute records, then `$FILE_NAME` attributes with parent
|
||||
references, names, hard links and timestamps.
|
||||
- [ ] Parse `$UsnJrnl:$J` records and correlate them to records by full file
|
||||
reference.
|
||||
- [ ] Verify against real `$MFT` and journal captures from a Windows host.
|
||||
+1
-1
@@ -148,7 +148,7 @@
|
||||
"build:termux:run": "npm run compile:termux && npm run artifacts:skills",
|
||||
"format": "oxfmt .",
|
||||
"format:check": "oxfmt --check .",
|
||||
"lint": "node scripts/ci/static-check-report.mjs lint && npm run verify:module-boundaries",
|
||||
"lint": "node scripts/ci/static-check-report.mjs lint && npm run verify:module-boundaries && node scripts/verify/rea-numeric-limits.mjs",
|
||||
"lint:details": "oxlint && npm run verify:module-boundaries",
|
||||
"lint:fix": "oxlint --fix && npm run verify:module-boundaries",
|
||||
"knip": "knip --cache",
|
||||
|
||||
@@ -636,7 +636,7 @@ const PATH_SCOPES = {
|
||||
"src/ghidra/GhidraJavaProperties.ts",
|
||||
"src/ghidra/GhidraReleasePolicy.ts",
|
||||
"scripts/verify-ghidra-bridge-sources.mjs",
|
||||
"tests/conformance/ghidra/ReaArmContextProbe.java",
|
||||
"tests/conformance/ghidra/ReaInstructionContextProbe.java",
|
||||
"tests/conformance/ghidra/ReaComputedCallCoverageProbe.java",
|
||||
"tests/conformance/ghidra/ReaDecodeProbe.java",
|
||||
".github/workflows/ghidra-bridge-sources.yml",
|
||||
|
||||
@@ -50,7 +50,7 @@ transport.stderr?.on("data", (bytes) => {
|
||||
const ajv = new Ajv2020({ strict: false, validateFormats: false });
|
||||
const measurements = [];
|
||||
const contract = toolContract("inspect_web_network_capture");
|
||||
await withVerifierCleanup(async () => {
|
||||
const report = await withVerifierCleanup(async () => {
|
||||
try {
|
||||
await client.connect(transport);
|
||||
} catch (cause) {
|
||||
@@ -196,7 +196,7 @@ await withVerifierCleanup(async () => {
|
||||
});
|
||||
}
|
||||
await client.ping();
|
||||
const report = {
|
||||
return {
|
||||
status: "passed",
|
||||
profile: "aioquic-1.5.0-json-0.3",
|
||||
measurements,
|
||||
@@ -206,17 +206,6 @@ await withVerifierCleanup(async () => {
|
||||
"No PCAP or keylog is read by this verifier; repeated STREAM coverage is not proof of retransmission or network loss.",
|
||||
],
|
||||
};
|
||||
await writeFile(reportPath, JSON.stringify(report, null, 2) + "\n", {
|
||||
flag: "wx",
|
||||
mode: 0o600,
|
||||
});
|
||||
console.log(
|
||||
JSON.stringify({
|
||||
status: report.status,
|
||||
cases: measurements.length,
|
||||
report: reportPath,
|
||||
}),
|
||||
);
|
||||
}, [
|
||||
async () => {
|
||||
await closeVerifierResources([
|
||||
@@ -225,3 +214,15 @@ await withVerifierCleanup(async () => {
|
||||
]);
|
||||
},
|
||||
]);
|
||||
|
||||
await writeFile(reportPath, JSON.stringify(report, null, 2) + "\n", {
|
||||
flag: "wx",
|
||||
mode: 0o600,
|
||||
});
|
||||
console.log(
|
||||
JSON.stringify({
|
||||
status: report.status,
|
||||
cases: measurements.length,
|
||||
report: reportPath,
|
||||
}),
|
||||
);
|
||||
@@ -1,3 +1,5 @@
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { existsSync } from "node:fs";
|
||||
import { readFile, readdir } from "node:fs/promises";
|
||||
import { join, relative, sep } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
@@ -193,4 +195,23 @@ if (failures.length > 0) {
|
||||
console.error(`Architecture guards failed:\n\n${failures.join("\n\n")}\n`);
|
||||
process.exit(1);
|
||||
}
|
||||
// CI runs this file before `npm ci`. The numeric-limit parser needs
|
||||
// typescript6, so that check waits until dependencies exist. `npm run lint`
|
||||
// runs it after install.
|
||||
const typescript6 = new URL(
|
||||
"../../node_modules/typescript6/package.json",
|
||||
import.meta.url,
|
||||
);
|
||||
if (!existsSync(typescript6)) {
|
||||
console.log(
|
||||
"Numeric REA_ limit check waits until dependencies are installed.",
|
||||
);
|
||||
} else {
|
||||
const numericLimits = spawnSync(
|
||||
process.execPath,
|
||||
[fileURLToPath(new URL("./rea-numeric-limits.mjs", import.meta.url))],
|
||||
{ stdio: "inherit" },
|
||||
);
|
||||
if (numericLimits.status !== 0) process.exit(numericLimits.status ?? 1);
|
||||
}
|
||||
console.log(`Architecture guards passed across ${files.length} source files.`);
|
||||
@@ -0,0 +1,23 @@
|
||||
/** Relative path of the registry. Its text is a declaration, not a producer. */
|
||||
export const NUMERIC_LIMIT_DECLARATION: "src/domain/numericLimitRegistry.ts";
|
||||
|
||||
/** Limits that must become analysis settings and cannot be registry exceptions. */
|
||||
export const REQUIRED_RUNTIME_LIMITS: readonly [
|
||||
"REA_NTFS_MFT_MAX_FILE_BYTES",
|
||||
"REA_REGF_HIVE_MAX_FILE_BYTES",
|
||||
"REA_WEB_BUNDLE_ENDPOINT_LITERAL_CANDIDATES",
|
||||
];
|
||||
|
||||
/** True for a numeric `REA_` limit, including the documented endpoint-candidate count. */
|
||||
export function isNumericLimit(name: string): boolean;
|
||||
|
||||
/** Producer paths for numeric limits. Registry text, comments, and allowlists are omitted. */
|
||||
export function discoverNumericLimitUses(
|
||||
files: readonly { readonly path: string; readonly source: string }[],
|
||||
): Map<string, string>;
|
||||
|
||||
/** Why a tracked limit is not an analysis setting, when it is present. */
|
||||
export function requiredRuntimeLimitFailure(
|
||||
name: string,
|
||||
presence: { readonly inRuntime: boolean; readonly inRegistry: boolean },
|
||||
): string | undefined;
|
||||
@@ -0,0 +1,399 @@
|
||||
import { readFile, readdir } from "node:fs/promises";
|
||||
import { join, relative, resolve, sep } from "node:path";
|
||||
import { fileURLToPath, pathToFileURL } from "node:url";
|
||||
import ts from "typescript6";
|
||||
|
||||
// Every numeric REA_ limit under src must be an analysis setting or a
|
||||
// restart-only exception with a reason and an update_mode. A per-call limit
|
||||
// belongs in ANALYSIS_SETTINGS. The three limits moving on #9868, #9865, and
|
||||
// #9836 are not in this tree; once their names appear here they must join
|
||||
// ANALYSIS_SETTINGS. REA_WEB_BUNDLE_ENDPOINT_LITERAL_CANDIDATES is numeric
|
||||
// even though its name has no byte or time suffix.
|
||||
|
||||
const root = fileURLToPath(new URL("../../", import.meta.url));
|
||||
const definitionsPath = join(root, "src/domain/analysisSettingDefinitions.ts");
|
||||
const registryPath = join(root, "src/domain/numericLimitRegistry.ts");
|
||||
|
||||
const listFiles = async (dir) => {
|
||||
const output = [];
|
||||
for (const entry of await readdir(dir, { withFileTypes: true })) {
|
||||
const path = join(dir, entry.name);
|
||||
if (entry.isDirectory()) {
|
||||
if (entry.name === "node_modules" || entry.name === "dist") continue;
|
||||
output.push(...(await listFiles(path)));
|
||||
} else if (
|
||||
entry.name.endsWith(".ts") &&
|
||||
!entry.name.endsWith(".test.ts") &&
|
||||
!entry.name.endsWith(".fixture.ts")
|
||||
)
|
||||
output.push(path);
|
||||
}
|
||||
return output;
|
||||
};
|
||||
|
||||
/**
|
||||
* Count-shaped limits whose names do not use a byte, time, or maximum suffix.
|
||||
* `REA_WEB_BUNDLE_ENDPOINT_LITERAL_CANDIDATES` is the runtime setting on #9836.
|
||||
*/
|
||||
const DOCUMENTED_COUNT_LIMITS = new Set([
|
||||
"REA_WEB_BUNDLE_ENDPOINT_LITERAL_CANDIDATES",
|
||||
]);
|
||||
|
||||
/** Registry text is a declaration, not a producer that uses the limit. */
|
||||
export const NUMERIC_LIMIT_DECLARATION = "src/domain/numericLimitRegistry.ts";
|
||||
|
||||
/**
|
||||
* These limits are moving onto runtime analysis settings on their own PRs.
|
||||
* A producer or a registry entry cannot satisfy that requirement.
|
||||
*/
|
||||
export const REQUIRED_RUNTIME_LIMITS = [
|
||||
"REA_NTFS_MFT_MAX_FILE_BYTES",
|
||||
"REA_REGF_HIVE_MAX_FILE_BYTES",
|
||||
"REA_WEB_BUNDLE_ENDPOINT_LITERAL_CANDIDATES",
|
||||
];
|
||||
|
||||
const unwrapExpression = (node) => {
|
||||
let current = node;
|
||||
while (
|
||||
ts.isAsExpression(current) ||
|
||||
ts.isSatisfiesExpression(current) ||
|
||||
ts.isParenthesizedExpression(current) ||
|
||||
ts.isTypeAssertionExpression(current) ||
|
||||
ts.isNonNullExpression(current)
|
||||
)
|
||||
current = current.expression;
|
||||
return current;
|
||||
};
|
||||
|
||||
const literalText = (node) =>
|
||||
ts.isStringLiteral(node) || ts.isNoSubstitutionTemplateLiteral(node)
|
||||
? node.text
|
||||
: undefined;
|
||||
|
||||
/** Same-file const strings and `setting` fields, so `environment[name]` still resolves. */
|
||||
const stringBindings = (sourceFile) => {
|
||||
const bindings = new Map();
|
||||
const visit = (node) => {
|
||||
if (
|
||||
ts.isVariableDeclaration(node) &&
|
||||
ts.isIdentifier(node.name) &&
|
||||
node.initializer !== undefined
|
||||
) {
|
||||
const initializer = unwrapExpression(node.initializer);
|
||||
const literal = literalText(initializer);
|
||||
if (literal !== undefined) bindings.set(node.name.text, literal);
|
||||
if (ts.isObjectLiteralExpression(initializer)) {
|
||||
for (const property of initializer.properties) {
|
||||
if (
|
||||
!ts.isPropertyAssignment(property) ||
|
||||
(!ts.isIdentifier(property.name) &&
|
||||
!ts.isStringLiteral(property.name))
|
||||
)
|
||||
continue;
|
||||
const value = literalText(unwrapExpression(property.initializer));
|
||||
if (value !== undefined)
|
||||
bindings.set(`${node.name.text}.${property.name.text}`, value);
|
||||
}
|
||||
}
|
||||
}
|
||||
ts.forEachChild(node, visit);
|
||||
};
|
||||
visit(sourceFile);
|
||||
return bindings;
|
||||
};
|
||||
|
||||
const boundLimitName = (node, bindings) => {
|
||||
const expression = unwrapExpression(node);
|
||||
const literal = literalText(expression);
|
||||
if (literal !== undefined) return literal;
|
||||
if (ts.isIdentifier(expression)) return bindings.get(expression.text);
|
||||
if (
|
||||
ts.isPropertyAccessExpression(expression) &&
|
||||
ts.isIdentifier(expression.expression) &&
|
||||
(ts.isIdentifier(expression.name) || ts.isStringLiteral(expression.name))
|
||||
)
|
||||
return bindings.get(
|
||||
`${expression.expression.text}.${expression.name.text}`,
|
||||
);
|
||||
return undefined;
|
||||
};
|
||||
|
||||
export const isNumericLimit = (name) =>
|
||||
DOCUMENTED_COUNT_LIMITS.has(name) ||
|
||||
/(?:_BYTES|_MS|_MIB|_SECONDS|_POLLS|_JOBS|_ELEMENTS|_COUNT|_SESSIONS|_COMMANDS)$/u.test(
|
||||
name,
|
||||
) ||
|
||||
/_MAX_/u.test(name) ||
|
||||
/_(?:TIMEOUT|DEADLINE|BUDGET)(?:_|$)/u.test(name) ||
|
||||
/_HEAP(?:_|$)/u.test(name);
|
||||
|
||||
const limitName = (text) =>
|
||||
text !== undefined && /^REA_[A-Z0-9_]+$/u.test(text) && isNumericLimit(text)
|
||||
? text
|
||||
: undefined;
|
||||
|
||||
const remember = (discovered, path, name) => {
|
||||
const limit = limitName(name);
|
||||
if (limit === undefined || discovered.has(limit)) return;
|
||||
discovered.set(limit, path);
|
||||
};
|
||||
|
||||
/**
|
||||
* Producers are syntax, not text. A read is a property or element access,
|
||||
* including a const string used as the element, a string argument to a call,
|
||||
* an object-binding key (rename and default included), an `in` check, or an
|
||||
* object key that a spread can copy. Comments and array literals are not reads.
|
||||
*/
|
||||
export const discoverNumericLimitUses = (files) => {
|
||||
const discovered = new Map();
|
||||
for (const file of files) {
|
||||
const relativePath = file.path.split(sep).join("/");
|
||||
if (relativePath === NUMERIC_LIMIT_DECLARATION) continue;
|
||||
const sourceFile = ts.createSourceFile(
|
||||
relativePath,
|
||||
file.source,
|
||||
ts.ScriptTarget.Latest,
|
||||
true,
|
||||
ts.ScriptKind.TS,
|
||||
);
|
||||
const bindings = stringBindings(sourceFile);
|
||||
const visit = (node) => {
|
||||
if (
|
||||
ts.isPropertyAccessExpression(node) ||
|
||||
ts.isPropertyAccessChain?.(node)
|
||||
)
|
||||
remember(discovered, relativePath, node.name.text);
|
||||
if (ts.isElementAccessExpression(node))
|
||||
remember(
|
||||
discovered,
|
||||
relativePath,
|
||||
boundLimitName(node.argumentExpression, bindings),
|
||||
);
|
||||
if (ts.isCallExpression(node) || ts.isNewExpression(node)) {
|
||||
for (const argument of node.arguments ?? [])
|
||||
remember(
|
||||
discovered,
|
||||
relativePath,
|
||||
boundLimitName(argument, bindings),
|
||||
);
|
||||
}
|
||||
if (
|
||||
ts.isBinaryExpression(node) &&
|
||||
node.operatorToken.kind === ts.SyntaxKind.InKeyword
|
||||
)
|
||||
remember(discovered, relativePath, boundLimitName(node.left, bindings));
|
||||
if (
|
||||
ts.isBindingElement(node) &&
|
||||
node.dotDotDotToken === undefined &&
|
||||
ts.isObjectBindingPattern(node.parent)
|
||||
) {
|
||||
const key = node.propertyName ?? node.name;
|
||||
if (
|
||||
ts.isIdentifier(key) ||
|
||||
ts.isStringLiteral(key) ||
|
||||
ts.isNumericLiteral(key)
|
||||
)
|
||||
remember(discovered, relativePath, key.text);
|
||||
}
|
||||
if (
|
||||
(ts.isPropertyAssignment(node) ||
|
||||
ts.isShorthandPropertyAssignment(node)) &&
|
||||
(ts.isIdentifier(node.name) ||
|
||||
ts.isStringLiteral(node.name) ||
|
||||
ts.isNumericLiteral(node.name))
|
||||
)
|
||||
remember(discovered, relativePath, node.name.text);
|
||||
if (ts.isVariableDeclaration(node) && node.initializer !== undefined)
|
||||
remember(
|
||||
discovered,
|
||||
relativePath,
|
||||
literalText(unwrapExpression(node.initializer)),
|
||||
);
|
||||
if (ts.isPropertyAssignment(node))
|
||||
remember(
|
||||
discovered,
|
||||
relativePath,
|
||||
literalText(unwrapExpression(node.initializer)),
|
||||
);
|
||||
if (
|
||||
ts.isTemplateExpression(node) ||
|
||||
ts.isNoSubstitutionTemplateLiteral(node)
|
||||
) {
|
||||
const chunks = ts.isNoSubstitutionTemplateLiteral(node)
|
||||
? [node.text]
|
||||
: [
|
||||
node.head.text,
|
||||
...node.templateSpans.map((span) => span.literal.text),
|
||||
];
|
||||
for (const chunk of chunks)
|
||||
for (const token of chunk.match(/REA_[A-Z0-9_]+/gu) ?? [])
|
||||
remember(discovered, relativePath, token);
|
||||
}
|
||||
ts.forEachChild(node, visit);
|
||||
};
|
||||
visit(sourceFile);
|
||||
}
|
||||
return discovered;
|
||||
};
|
||||
|
||||
/** Tracked limits join ANALYSIS_SETTINGS. The exception registry cannot classify them. */
|
||||
export const requiredRuntimeLimitFailure = (
|
||||
name,
|
||||
{ inRuntime, inRegistry },
|
||||
) => {
|
||||
if (!REQUIRED_RUNTIME_LIMITS.includes(name)) return undefined;
|
||||
if (inRuntime && !inRegistry) return undefined;
|
||||
return `${name} must join ANALYSIS_SETTINGS; numericLimitRegistry cannot classify it`;
|
||||
};
|
||||
|
||||
const namesIn = (source, pattern) =>
|
||||
new Set([...source.matchAll(pattern)].map((match) => match[1]));
|
||||
|
||||
const direct =
|
||||
process.argv[1] !== undefined &&
|
||||
import.meta.url === pathToFileURL(resolve(process.argv[1])).href;
|
||||
|
||||
if (direct) {
|
||||
const failures = [];
|
||||
const definitions = await readFile(definitionsPath, "utf8");
|
||||
const registry = await readFile(registryPath, "utf8");
|
||||
const runtime = namesIn(definitions, /environment:\s*"(REA_[A-Z0-9_]+)"/gu);
|
||||
const declaredNames = [
|
||||
...registry.matchAll(/name:\s*"(REA_[A-Z0-9_]+)"/gu),
|
||||
].map((match) => match[1]);
|
||||
const entries = [
|
||||
...registry.matchAll(
|
||||
/name:\s*"(REA_[A-Z0-9_]+)"\s*,\s*update_mode:\s*"(runtime|restart)"\s*,\s*policy:\s*"(restart-only|per-call)"\s*,\s*read:\s*"(startup|per-call)"\s*,\s*reason:\s*"((?:\\.|[^"\\])*)"/gu,
|
||||
),
|
||||
];
|
||||
const listed = new Map();
|
||||
for (const match of entries) {
|
||||
const [, name, updateMode, policy, read, reason] = match;
|
||||
if (listed.has(name)) failures.push(`${name} is listed twice`);
|
||||
listed.set(name, { updateMode, policy, read, reason });
|
||||
if (reason.trim().length < 40)
|
||||
failures.push(`${name} needs a reason that states why`);
|
||||
if (updateMode !== "restart")
|
||||
failures.push(
|
||||
`${name} is outside ANALYSIS_SETTINGS, so its update_mode must be restart`,
|
||||
);
|
||||
if (runtime.has(name))
|
||||
failures.push(
|
||||
`${name} is both a runtime setting and on the exception list`,
|
||||
);
|
||||
if (policy === "per-call" && read !== "per-call")
|
||||
failures.push(`${name} is per-call policy but is not read per call`);
|
||||
if (policy === "restart-only" && read !== "startup")
|
||||
failures.push(`${name} is restart-only but is not read at startup`);
|
||||
}
|
||||
for (const name of declaredNames) {
|
||||
if (!listed.has(name))
|
||||
failures.push(`${name} is registered without update_mode`);
|
||||
}
|
||||
const settingBodies = [
|
||||
...definitions.matchAll(/\n {2}"[^"]+": \{([\s\S]*?)\n {2}\},/gu),
|
||||
];
|
||||
if (settingBodies.length === 0)
|
||||
failures.push("ANALYSIS_SETTINGS did not parse into setting bodies");
|
||||
for (const [, body] of settingBodies) {
|
||||
if (!/updateMode:\s*"(?:runtime|restart)"/u.test(body))
|
||||
failures.push("an analysis setting is missing updateMode");
|
||||
}
|
||||
|
||||
const discovered = discoverNumericLimitUses(
|
||||
await Promise.all(
|
||||
(await listFiles(join(root, "src"))).map(async (path) => ({
|
||||
path: relative(root, path),
|
||||
source: await readFile(path, "utf8"),
|
||||
})),
|
||||
),
|
||||
);
|
||||
|
||||
for (const [name, file] of [...discovered].sort(([left], [right]) =>
|
||||
left < right ? -1 : left > right ? 1 : 0,
|
||||
)) {
|
||||
if (!runtime.has(name) && !listed.has(name))
|
||||
failures.push(
|
||||
`${name} in ${file} is neither an analysis setting nor docs/design/rea-limit-inventory.md via src/domain/numericLimitRegistry.ts`,
|
||||
);
|
||||
}
|
||||
for (const name of listed.keys()) {
|
||||
if (!discovered.has(name))
|
||||
failures.push(`${name} is registered but no longer appears in src`);
|
||||
}
|
||||
const present = new Set([...discovered.keys(), ...listed.keys(), ...runtime]);
|
||||
for (const name of REQUIRED_RUNTIME_LIMITS) {
|
||||
if (!present.has(name)) continue;
|
||||
const failure = requiredRuntimeLimitFailure(name, {
|
||||
inRuntime: runtime.has(name),
|
||||
inRegistry: listed.has(name),
|
||||
});
|
||||
if (failure !== undefined) failures.push(failure);
|
||||
}
|
||||
|
||||
const parameterOwners = [
|
||||
"src/artifacts/ArtifactReaderLimits.ts",
|
||||
"src/domain/artifactReaderLimits.ts",
|
||||
"src/domain/webNetworkCapture.ts",
|
||||
];
|
||||
for (const relativePath of parameterOwners) {
|
||||
const source = await readFile(join(root, relativePath), "utf8");
|
||||
if (/update_mode:\s*"runtime"/u.test(source))
|
||||
failures.push(
|
||||
`${relativePath} is a per-call parameter, so update_mode cannot be runtime`,
|
||||
);
|
||||
if (!/update_mode:\s*"parameter"/u.test(source))
|
||||
failures.push(`${relativePath} must report update_mode parameter`);
|
||||
}
|
||||
|
||||
const forbidden =
|
||||
/(?<![A-Za-z0-9_])(?:rea_restart_required|restartRequired|restart_required)(?![A-Za-z0-9_])|effect:\s*"(?:immediate|next_process)"|SettingChangeEffect/gu;
|
||||
// Docs and verification receipts keep the field names they recorded.
|
||||
// Source, tests, skills, and scripts cannot reintroduce those names.
|
||||
const scanRoots = ["src", "tests", ".agents", "scripts"].map((dir) =>
|
||||
join(root, dir),
|
||||
);
|
||||
const skipDir = new Set([
|
||||
"node_modules",
|
||||
"dist",
|
||||
".turbo",
|
||||
".cache",
|
||||
"rea-numeric-limits.mjs",
|
||||
]);
|
||||
const listText = async (dir) => {
|
||||
const output = [];
|
||||
let entries;
|
||||
try {
|
||||
entries = await readdir(dir, { withFileTypes: true });
|
||||
} catch {
|
||||
return output;
|
||||
}
|
||||
for (const entry of entries) {
|
||||
if (skipDir.has(entry.name)) continue;
|
||||
const path = join(dir, entry.name);
|
||||
if (entry.isDirectory()) output.push(...(await listText(path)));
|
||||
else if (/\.(?:ts|tsx|mjs|js|md|json|py|yml|yaml)$/u.test(entry.name))
|
||||
output.push(path);
|
||||
}
|
||||
return output;
|
||||
};
|
||||
for (const dir of scanRoots) {
|
||||
for (const path of await listText(dir)) {
|
||||
if (path.endsWith(`${sep}rea-numeric-limits.mjs`)) continue;
|
||||
const source = await readFile(path, "utf8");
|
||||
const hit = source.match(forbidden);
|
||||
if (hit)
|
||||
failures.push(
|
||||
`${relative(root, path).split(sep).join("/")} still uses ${[...new Set(hit)].join(", ")}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
if (failures.length > 0) {
|
||||
console.error(`Numeric REA_ limit check failed:\n${failures.join("\n")}`);
|
||||
process.exit(1);
|
||||
}
|
||||
console.log(
|
||||
`Numeric REA_ limits: ${runtime.size} runtime, ${[...listed.values()].filter((entry) => entry.policy === "restart-only").length} restart-only, ${[...listed.values()].filter((entry) => entry.policy === "per-call").length} per-call pending migration, ${discovered.size} discovered.`,
|
||||
);
|
||||
}
|
||||
@@ -36,7 +36,8 @@ it("names REA_JADX_OPERATION_TIMEOUT_MS when the operation deadline elapses", ()
|
||||
applies_at: "next_operation",
|
||||
agent_editable: true,
|
||||
scope: "rea-runtime",
|
||||
rea_restart_required: false,
|
||||
update_mode: "runtime",
|
||||
pending_restart: false,
|
||||
},
|
||||
});
|
||||
expect(projected).toMatchObject({
|
||||
|
||||
@@ -134,7 +134,7 @@ export class AnalysisSettings {
|
||||
};
|
||||
}),
|
||||
active_work_changed: false as const,
|
||||
rea_restart_required: false as const,
|
||||
pending_restart: false as const,
|
||||
});
|
||||
}
|
||||
#describe(setting: AnalysisSettingId) {
|
||||
@@ -156,6 +156,8 @@ export class AnalysisSettings {
|
||||
minimum: definition.minimum,
|
||||
maximum: definition.maximum,
|
||||
applies_at: definition.appliesAt,
|
||||
update_mode: definition.updateMode,
|
||||
pending_restart: false as const,
|
||||
provider: definition.provider,
|
||||
operations: [...definition.operations],
|
||||
...(definition.environment === undefined
|
||||
|
||||
@@ -97,6 +97,7 @@ import {
|
||||
isDfuSeFile,
|
||||
} from "../domain/firmware/dfuse.js";
|
||||
import { hasFitImagesNode } from "../domain/firmware/fitImage.js";
|
||||
import { hasMftFileRecordHeader } from "../domain/windows/ntfsMft.js";
|
||||
import {
|
||||
locateEspFlashLayout,
|
||||
type EspFlashSelection,
|
||||
@@ -816,6 +817,7 @@ const detectArtifactFormat = async (
|
||||
});
|
||||
if (esp === "esp-image") return "esp-image";
|
||||
if (hasFitImagesNode(magic)) return "fit";
|
||||
if (hasMftFileRecordHeader(magic)) return "ntfs-mft";
|
||||
// Binary plists, including keyed archives, often use other extensions.
|
||||
if (hasBinaryPlistSignature(magic)) return "plist";
|
||||
const named = namedArtifactFormat(lower);
|
||||
@@ -899,6 +901,7 @@ const isArchiveFormat = (
|
||||
"dfuse",
|
||||
"esp-flash-image",
|
||||
"fit",
|
||||
"ntfs-mft",
|
||||
].includes(format);
|
||||
|
||||
/** Whether the leading bytes begin any executable header family, valid or not. */
|
||||
|
||||
@@ -3,10 +3,7 @@ import { createHash } from "node:crypto";
|
||||
import { expect, it } from "vitest";
|
||||
|
||||
import { parseReferenceSourceEntries } from "./ReferenceSourceImportEntries.js";
|
||||
import {
|
||||
historicalSourceInventoryHasPartialFacts,
|
||||
historicalSourceLanguages,
|
||||
} from "../domain/referenceSourceGraph.js";
|
||||
import { historicalSourceLanguages } from "../domain/referenceSourceGraph.js";
|
||||
import type { ReferenceSourceRead } from "../reference/ReferenceSourceReaderTypes.js";
|
||||
|
||||
it("projects an unsupported non-regular entry as unknown kind instead of file", () => {
|
||||
@@ -101,100 +98,6 @@ it("keeps source entries when a module has an empty specifier", () => {
|
||||
expect(parsed.limitations[0]).toMatch(/not recorded.*relationship/iu);
|
||||
});
|
||||
|
||||
it("resolves immediately invoked loaders from their proven source bases", () => {
|
||||
const files = [
|
||||
[
|
||||
"main.cjs",
|
||||
'(() => require)()("./dep.cjs"); (() => () => {})()("./local.cjs");',
|
||||
],
|
||||
[
|
||||
"main.mjs",
|
||||
[
|
||||
'import {createRequire} from "node:module";',
|
||||
'createRequire(new URL("./sub/base.cjs", import.meta.url))("./dep.cjs");',
|
||||
'createRequire(new URL("./sub/base.cjs", import.meta.url)).call(null, "./member.cjs");',
|
||||
].join("\n"),
|
||||
],
|
||||
["dep.cjs", ""],
|
||||
["sub/dep.cjs", ""],
|
||||
["member.cjs", ""],
|
||||
["sub/member.cjs", ""],
|
||||
] as const;
|
||||
const parsed = parseReferenceSourceEntries(
|
||||
{
|
||||
root: "/selected/source",
|
||||
entries: files.map(([path, content]) => ({
|
||||
status: "read",
|
||||
kind: "file",
|
||||
path,
|
||||
bytes: Buffer.from(content),
|
||||
size: Buffer.byteLength(content),
|
||||
})),
|
||||
bytesRead: files.reduce(
|
||||
(sum, [, content]) => sum + Buffer.byteLength(content),
|
||||
0,
|
||||
),
|
||||
limitations: [],
|
||||
},
|
||||
[],
|
||||
);
|
||||
expect(
|
||||
parsed.relationships.filter(({ kind }) => kind === "requires"),
|
||||
).toEqual([
|
||||
{
|
||||
from_path: "main.cjs",
|
||||
to: "dep.cjs",
|
||||
kind: "requires",
|
||||
resolution: "internal",
|
||||
parse_state: "parsed",
|
||||
},
|
||||
{
|
||||
from_path: "main.mjs",
|
||||
to: "sub/dep.cjs",
|
||||
kind: "requires",
|
||||
resolution: "internal",
|
||||
parse_state: "parsed",
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it.each([
|
||||
["function load(name) { require(name); }", true],
|
||||
["function load(require, name) { require(name); }", false],
|
||||
["require(7); require();", false],
|
||||
] as const)(
|
||||
"reports unresolved CommonJS target coverage for %s",
|
||||
(source, partial) => {
|
||||
const bytes = Buffer.from(source);
|
||||
const parsed = parseReferenceSourceEntries(
|
||||
{
|
||||
root: "/selected/source",
|
||||
entries: [
|
||||
{
|
||||
status: "read",
|
||||
kind: "file",
|
||||
path: "main.cjs",
|
||||
bytes,
|
||||
size: bytes.length,
|
||||
},
|
||||
],
|
||||
bytesRead: bytes.length,
|
||||
limitations: [],
|
||||
},
|
||||
[],
|
||||
);
|
||||
expect(parsed.relationships).toEqual([]);
|
||||
expect(
|
||||
historicalSourceInventoryHasPartialFacts({
|
||||
...parsed,
|
||||
parse_failures: parsed.parseFailures,
|
||||
exclusions: [],
|
||||
}),
|
||||
).toBe(partial);
|
||||
expect(parsed.entries[0]?.limitations).toEqual(parsed.limitations);
|
||||
},
|
||||
);
|
||||
|
||||
it("retains Kotlin source identity and language across the inventory projection", () => {
|
||||
const files = [
|
||||
[
|
||||
|
||||
@@ -135,6 +135,7 @@ const TARGETS: TargetFixtures = {
|
||||
espFlash: { selected: true, flashOffset: null, partitionTableOffset: null },
|
||||
},
|
||||
fit: { ...archive, format: "fit" },
|
||||
"ntfs-mft": { ...archive, format: "ntfs-mft" },
|
||||
plist: { ...artifact, format: "plist" },
|
||||
javascript: { ...artifact, format: "javascript" },
|
||||
"source-map": { ...artifact, format: "source-map" },
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import { discoverJavaScriptPreloadPaths } from "./JavaScriptPreloadDiscovery.js";
|
||||
import { artifactNameForClassification } from "../../domain/artifactNameSelection.js";
|
||||
import type { AnalysisSettingValues } from "../../domain/analysisSettings.js";
|
||||
import { selectedAnalysisSetting } from "../../domain/analysisSettingDefinitions.js";
|
||||
import { absoluteCallerPath } from "../../filesystem/callerPath.js";
|
||||
import {
|
||||
DEFAULT_ASAR_HEADER_ADMISSION,
|
||||
@@ -181,18 +182,24 @@ export const reconstructJavaScriptArtifact = async (
|
||||
? {}
|
||||
: { fileNodes: input.max_semantic_nodes_per_file }),
|
||||
});
|
||||
const heap = selectedAnalysisSetting(
|
||||
input.max_heap_mb,
|
||||
overrides?.["javascript.max_heap_mb"],
|
||||
JAVASCRIPT_ANALYSIS_RESOURCE_DEFAULTS.heapMb,
|
||||
);
|
||||
const timeout = selectedAnalysisSetting(
|
||||
input.analysis_timeout_ms,
|
||||
overrides?.["javascript.analysis_timeout_ms"],
|
||||
JAVASCRIPT_ANALYSIS_RESOURCE_DEFAULTS.timeoutMs,
|
||||
);
|
||||
const sourceLimits = {
|
||||
...(input.semantic_precision === undefined
|
||||
? {}
|
||||
: { semanticPrecision: input.semantic_precision }),
|
||||
heapMb:
|
||||
input.max_heap_mb ??
|
||||
overrides?.["javascript.max_heap_mb"] ??
|
||||
JAVASCRIPT_ANALYSIS_RESOURCE_DEFAULTS.heapMb,
|
||||
timeoutMs:
|
||||
input.analysis_timeout_ms ??
|
||||
overrides?.["javascript.analysis_timeout_ms"] ??
|
||||
JAVASCRIPT_ANALYSIS_RESOURCE_DEFAULTS.timeoutMs,
|
||||
heapMb: heap.value,
|
||||
heapSource: heap.source,
|
||||
timeoutMs: timeout.value,
|
||||
timeoutSource: timeout.source,
|
||||
};
|
||||
const preloads = await discoverJavaScriptPreloadPaths(
|
||||
files,
|
||||
|
||||
@@ -15,6 +15,8 @@ export type WasmArtifactExecution = Omit<AnalysisExecution, "subject"> & {
|
||||
/** Effective operation budget, separate from caller-authored Evidence parameters. */
|
||||
export interface WasmArtifactExecutionOptions extends ExecutionOptions {
|
||||
readonly commandTimeoutMs?: number;
|
||||
/** Selection owner for `commandTimeoutMs`. Absent when the caller did not record it. */
|
||||
readonly commandTimeoutSource?: "default" | "runtime" | "request" | undefined;
|
||||
}
|
||||
|
||||
/** Replaceable offline WASM inspection engine, independent of target execution and network clients. */
|
||||
|
||||
@@ -240,6 +240,7 @@ it("gives cancellation precedence over malformed raw input", async () => {
|
||||
it("uses runtime defaults only for omitted budgets, preserving explicit input and reset", async () => {
|
||||
const settings = new AnalysisSettings({});
|
||||
const accepted: number[] = [];
|
||||
const sources: (string | undefined)[] = [];
|
||||
const authored: (number | undefined)[] = [];
|
||||
const service = new WasmArtifactService(
|
||||
{
|
||||
@@ -248,6 +249,7 @@ it("uses runtime defaults only for omitted budgets, preserving explicit input an
|
||||
const value = wasmArtifactFixture();
|
||||
value.tool_profile.timeout_ms = options!.commandTimeoutMs!;
|
||||
accepted.push(options!.commandTimeoutMs!);
|
||||
sources.push(options!.commandTimeoutSource);
|
||||
authored.push(input.command_timeout_ms);
|
||||
return ok(wasmArtifactExecution(value));
|
||||
},
|
||||
@@ -275,6 +277,7 @@ it("uses runtime defaults only for omitted budgets, preserving explicit input an
|
||||
});
|
||||
expect((await settings.run(() => service.inspect({ path }))).ok).toBe(true);
|
||||
expect(accepted).toEqual([123000, 42000, 30000]);
|
||||
expect(sources).toEqual(["runtime", "request", "default"]);
|
||||
expect(authored).toEqual([undefined, 42000, undefined]);
|
||||
});
|
||||
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import type { AnalysisSettingsReader } from "../../domain/analysisSettings.js";
|
||||
import { selectedAnalysisSetting } from "../../domain/analysisSettingDefinitions.js";
|
||||
import { constants as bufferConstants } from "node:buffer";
|
||||
import type { ExecutionOptions } from "../AnalysisProvider.js";
|
||||
import {
|
||||
@@ -78,13 +79,15 @@ export class WasmArtifactService {
|
||||
),
|
||||
);
|
||||
}
|
||||
const timeout =
|
||||
input.value.command_timeout_ms ??
|
||||
this.settings?.().overrides["wasm.command_timeout_ms"] ??
|
||||
DEFAULT_WASM_COMMAND_TIMEOUT_MS;
|
||||
const timeout = selectedAnalysisSetting(
|
||||
input.value.command_timeout_ms,
|
||||
this.settings?.().overrides["wasm.command_timeout_ms"],
|
||||
DEFAULT_WASM_COMMAND_TIMEOUT_MS,
|
||||
);
|
||||
const inspected = await this.provider.inspect(input.value, {
|
||||
...options,
|
||||
commandTimeoutMs: timeout,
|
||||
commandTimeoutMs: timeout.value,
|
||||
commandTimeoutSource: timeout.source,
|
||||
});
|
||||
if (!inspected.ok) return inspected;
|
||||
const execution = inspected.value;
|
||||
@@ -108,7 +111,7 @@ export class WasmArtifactService {
|
||||
!report.success ||
|
||||
report.data.artifact.path !== input.value.path ||
|
||||
execution.subject.path !== input.value.path ||
|
||||
report.data.tool_profile.timeout_ms !== timeout ||
|
||||
report.data.tool_profile.timeout_ms !== timeout.value ||
|
||||
report.data.tool_profile.output_budget_bytes !==
|
||||
wasmResourceLimits(input.value).outputBytes ||
|
||||
!wasmArtifactMatchesSubject(report.data.artifact, execution.subject) ||
|
||||
|
||||
@@ -54,7 +54,8 @@ it("preserves adjustable reader admission facts and gives cleanup priority", ()
|
||||
required_value: 2,
|
||||
unit: "count",
|
||||
applies_at: "next_operation",
|
||||
rea_restart_required: false,
|
||||
update_mode: "parameter",
|
||||
pending_restart: false,
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
@@ -215,6 +215,7 @@ export interface ArtifactReader {
|
||||
| "dfuse"
|
||||
| "esp-flash-image"
|
||||
| "fit"
|
||||
| "ntfs-mft"
|
||||
| "file";
|
||||
entries(
|
||||
signal?: AbortSignal,
|
||||
|
||||
@@ -40,7 +40,8 @@ export const artifactReaderLimitFailure = (
|
||||
applies_at: "next_operation",
|
||||
agent_editable: true,
|
||||
can_increase: canIncrease,
|
||||
rea_restart_required: false,
|
||||
update_mode: "parameter" as const,
|
||||
pending_restart: false as const,
|
||||
},
|
||||
});
|
||||
};
|
||||
@@ -18,7 +18,10 @@ import {
|
||||
type ArtifactTraversalOptions,
|
||||
} from "./ArtifactReader.js";
|
||||
import { ArtifactResourceScope } from "./ArtifactResourceScope.js";
|
||||
import { assertArtifactReadActive } from "./ArtifactFileOperation.js";
|
||||
import {
|
||||
artifactReadCancelledBy,
|
||||
assertArtifactReadActive,
|
||||
} from "./ArtifactFileOperation.js";
|
||||
import { ArtifactFileSource } from "./ArtifactFileSource.js";
|
||||
import { readFileHandleChunks } from "../filesystem/readFileHandleChunks.js";
|
||||
import type { StableRegularFileDescriptor } from "../filesystem/RegularFile.js";
|
||||
@@ -64,12 +67,14 @@ export const regionFileName = (start: number, end: number): string =>
|
||||
*/
|
||||
export class DecodedRegionArtifactReader<Image> implements ArtifactReader {
|
||||
readonly #scope = new ArtifactResourceScope();
|
||||
readonly #entries = new Map<string, ArtifactEntry>();
|
||||
readonly #regions = new Map<string, Uint8Array>();
|
||||
#entries = new Map<string, ArtifactEntry>();
|
||||
#regions = new Map<string, Uint8Array>();
|
||||
readonly #streams = new Set<Readable>();
|
||||
readonly #source: ArtifactFileSource;
|
||||
readonly #decoding: RegionDecoding<Image>;
|
||||
#prepared: Promise<void> | undefined;
|
||||
#prepared:
|
||||
| { signal: AbortSignal | undefined; promise: Promise<void> }
|
||||
| undefined;
|
||||
/** Whole-file facts only; the region map is the sole owner of region bytes. */
|
||||
#described: Omit<DecodedRegions, "regions"> | undefined;
|
||||
#closing = false;
|
||||
@@ -110,8 +115,27 @@ export class DecodedRegionArtifactReader<Image> implements ArtifactReader {
|
||||
): AsyncIterable<ArtifactEntry> {
|
||||
await this.#scope.run(async () => {
|
||||
this.#assertOpen(signal);
|
||||
this.#prepared ??= this.#prepare(signal);
|
||||
await this.#prepared;
|
||||
for (;;) {
|
||||
const pending = (this.#prepared ??= {
|
||||
signal,
|
||||
promise: this.#prepare(signal),
|
||||
});
|
||||
try {
|
||||
await pending.promise;
|
||||
break;
|
||||
} catch (cause: unknown) {
|
||||
if (this.#prepared === pending) this.#prepared = undefined;
|
||||
if (
|
||||
pending.signal !== signal &&
|
||||
signal?.aborted !== true &&
|
||||
artifactReadCancelledBy(cause, pending.signal)
|
||||
) {
|
||||
this.#assertOpen(signal);
|
||||
continue;
|
||||
}
|
||||
throw cause;
|
||||
}
|
||||
}
|
||||
await this.#source.read(signal);
|
||||
});
|
||||
const coverage = this.#described?.coverage;
|
||||
@@ -230,6 +254,8 @@ export class DecodedRegionArtifactReader<Image> implements ArtifactReader {
|
||||
|
||||
async #prepare(signal?: AbortSignal): Promise<void> {
|
||||
const described = this.#decoding.describe(await this.#decode(signal));
|
||||
const entries = new Map<string, ArtifactEntry>();
|
||||
const regions = new Map<string, Uint8Array>();
|
||||
let index = 0;
|
||||
for (const region of described.regions) {
|
||||
if (index % 256 === 0) await setImmediate();
|
||||
@@ -248,13 +274,16 @@ export class DecodedRegionArtifactReader<Image> implements ArtifactReader {
|
||||
adapterKey: `${this.#decoding.keyPrefix}:${String(index)}`,
|
||||
formatMetadata: region.formatMetadata,
|
||||
} satisfies ArtifactEntry;
|
||||
this.#entries.set(entry.adapterKey, entry);
|
||||
this.#regions.set(entry.adapterKey, region.bytes);
|
||||
entries.set(entry.adapterKey, entry);
|
||||
regions.set(entry.adapterKey, region.bytes);
|
||||
index += 1;
|
||||
}
|
||||
const { regions: _regions, ...facts } = described;
|
||||
this.#described = facts;
|
||||
await this.#source.read(signal);
|
||||
this.#assertOpen(signal);
|
||||
this.#entries = entries;
|
||||
this.#regions = regions;
|
||||
this.#described = facts;
|
||||
}
|
||||
|
||||
async #decode(signal?: AbortSignal): Promise<Image> {
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { artifactPreparationRecoveryCases } from "../../tests/fixtures/artifactReaderLifecycle.js";
|
||||
import { getEventListeners, once } from "node:events";
|
||||
import { readFile, truncate, writeFile } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
@@ -404,6 +405,38 @@ it("keeps the cancellation reason before admission", async () => {
|
||||
}
|
||||
});
|
||||
|
||||
it("does not publish FIT entries when final source verification fails", async () => {
|
||||
const path = await writeFit("partial-publication.itb", FIT_EMBEDDED);
|
||||
const control = new FitArtifactReader(path);
|
||||
const reader = new FitArtifactReader(path);
|
||||
const expected = await firstEntry(control);
|
||||
const nativeRead = ArtifactFileSource.prototype.read;
|
||||
const physical = Object.assign(
|
||||
new Error("final source verification failed"),
|
||||
{ code: "EIO" },
|
||||
);
|
||||
let calls = 0;
|
||||
const read = vi
|
||||
.spyOn(ArtifactFileSource.prototype, "read")
|
||||
.mockImplementation(function (this: ArtifactFileSource, ...args) {
|
||||
calls += 1;
|
||||
if (calls === 4) return Promise.reject(physical);
|
||||
return nativeRead.apply(this, args);
|
||||
});
|
||||
try {
|
||||
await expect(firstEntry(reader)).rejects.toBe(physical);
|
||||
await expect(reader.open(expected)).rejects.toMatchObject({
|
||||
reason: "integrity",
|
||||
});
|
||||
read.mockRestore();
|
||||
expect((await firstEntry(reader)).path).toBe(expected.path);
|
||||
} finally {
|
||||
read.mockRestore();
|
||||
await reader.close();
|
||||
await control.close();
|
||||
}
|
||||
});
|
||||
|
||||
it("keeps the cancellation reason during hash verification", async () => {
|
||||
const reader = new FitArtifactReader(
|
||||
await writeFit("hashing.itb", LARGE_FIT),
|
||||
@@ -430,6 +463,8 @@ it("keeps the cancellation reason during hash verification", async () => {
|
||||
reader.entries(controller.signal)[Symbol.asyncIterator]().next(),
|
||||
).rejects.toMatchObject({ reason: "cancelled", cause: reason });
|
||||
expect(observer).toHaveBeenCalledWith("sha256");
|
||||
observer.mockRestore();
|
||||
expect((await firstEntry(reader)).kind).toBe("file");
|
||||
} finally {
|
||||
observer.mockRestore();
|
||||
await reader.close();
|
||||
@@ -569,3 +604,8 @@ it("preserves a final validation failure when cancellation overlaps it", async (
|
||||
await reader.close();
|
||||
}
|
||||
});
|
||||
|
||||
artifactPreparationRecoveryCases({
|
||||
image: () => FIT_EMBEDDED,
|
||||
createReader: (path, admitted) => new FitArtifactReader(path, admitted),
|
||||
});
|
||||
@@ -13,7 +13,10 @@ import {
|
||||
type ArtifactTraversalOptions,
|
||||
} from "./ArtifactReader.js";
|
||||
import { ArtifactResourceScope } from "./ArtifactResourceScope.js";
|
||||
import { assertArtifactReadActive } from "./ArtifactFileOperation.js";
|
||||
import {
|
||||
artifactReadCancelledBy,
|
||||
assertArtifactReadActive,
|
||||
} from "./ArtifactFileOperation.js";
|
||||
import { ArtifactFileSource } from "./ArtifactFileSource.js";
|
||||
import {
|
||||
createFitDigest,
|
||||
@@ -30,11 +33,13 @@ const FDT_HEADER_BYTES = 40;
|
||||
export class FitArtifactReader implements ArtifactReader {
|
||||
readonly format = "fit" as const;
|
||||
readonly #scope = new ArtifactResourceScope();
|
||||
readonly #entries = new Map<string, ArtifactEntry>();
|
||||
#entries = new Map<string, ArtifactEntry>();
|
||||
readonly #streams = new Set<Readable>();
|
||||
readonly #path: string;
|
||||
readonly #source: ArtifactFileSource;
|
||||
#prepared: Promise<void> | undefined;
|
||||
#prepared:
|
||||
| { signal: AbortSignal | undefined; promise: Promise<void> }
|
||||
| undefined;
|
||||
#metadata: FitMetadata | undefined;
|
||||
#coverage: ArtifactReaderTraversal | undefined;
|
||||
#closing = false;
|
||||
@@ -66,8 +71,27 @@ export class FitArtifactReader implements ArtifactReader {
|
||||
): AsyncIterable<ArtifactEntry> {
|
||||
await this.#scope.run(async () => {
|
||||
this.#assertOpen(signal);
|
||||
this.#prepared ??= this.#prepare(signal);
|
||||
await this.#prepared;
|
||||
for (;;) {
|
||||
const pending = (this.#prepared ??= {
|
||||
signal,
|
||||
promise: this.#prepare(signal),
|
||||
});
|
||||
try {
|
||||
await pending.promise;
|
||||
break;
|
||||
} catch (cause: unknown) {
|
||||
if (this.#prepared === pending) this.#prepared = undefined;
|
||||
if (
|
||||
pending.signal !== signal &&
|
||||
signal?.aborted !== true &&
|
||||
artifactReadCancelledBy(cause, pending.signal)
|
||||
) {
|
||||
this.#assertOpen(signal);
|
||||
continue;
|
||||
}
|
||||
throw cause;
|
||||
}
|
||||
}
|
||||
await this.#source.read(signal);
|
||||
});
|
||||
if (this.#coverage !== undefined)
|
||||
@@ -207,6 +231,7 @@ export class FitArtifactReader implements ArtifactReader {
|
||||
this.#path,
|
||||
initial.size,
|
||||
);
|
||||
const entries = new Map<string, ArtifactEntry>();
|
||||
let index = 0;
|
||||
for (const image of structure.images) {
|
||||
if (index % 256 === 0) await setImmediate();
|
||||
@@ -228,15 +253,17 @@ export class FitArtifactReader implements ArtifactReader {
|
||||
adapterKey: `fit:${String(image.index)}`,
|
||||
formatMetadata: image.metadata,
|
||||
} satisfies ArtifactEntry;
|
||||
this.#entries.set(entry.adapterKey, entry);
|
||||
entries.set(entry.adapterKey, entry);
|
||||
}
|
||||
await this.#source.read(signal);
|
||||
this.#assertOpen(signal);
|
||||
this.#entries = entries;
|
||||
this.#metadata = structure.metadata;
|
||||
if (structure.limitations.length > 0)
|
||||
this.#coverage = {
|
||||
status: "complete",
|
||||
limitations: structure.limitations,
|
||||
};
|
||||
await this.#source.read(signal);
|
||||
}
|
||||
|
||||
/** One streamed pass per image feeds every supported digest. */
|
||||
|
||||
@@ -1,3 +1,6 @@
|
||||
import { artifactPreparationRecoveryCases } from "../../tests/fixtures/artifactReaderLifecycle.js";
|
||||
import { ArtifactFileSource } from "./ArtifactFileSource.js";
|
||||
import { openRegularFile } from "../filesystem/RegularFile.js";
|
||||
import { writeFile } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
|
||||
@@ -159,6 +162,72 @@ it("settles an active region stream when closed", async () => {
|
||||
}
|
||||
});
|
||||
|
||||
it("retries decoded-region preparation cancelled while reading", async () => {
|
||||
const path = await writeImage("retry.hex", INTEL_HEX_I32_GAP);
|
||||
const admitted = await openRegularFile(path, { symlinks: "reject" });
|
||||
const reader = new HexRecordArtifactReader(path, "intel-hex", admitted);
|
||||
const controller = new AbortController();
|
||||
const reason = { stage: "decoded region read" };
|
||||
const nativeRead = admitted.handle.read.bind(admitted.handle);
|
||||
const read = vi.spyOn(admitted.handle, "read").mockImplementation((async (
|
||||
...args: unknown[]
|
||||
) => {
|
||||
const result = await Reflect.apply(nativeRead, admitted.handle, args);
|
||||
controller.abort(reason);
|
||||
return result;
|
||||
}) as typeof admitted.handle.read);
|
||||
try {
|
||||
await expect(
|
||||
reader.entries(controller.signal)[Symbol.asyncIterator]().next(),
|
||||
).rejects.toMatchObject({ reason: "cancelled", cause: reason });
|
||||
read.mockRestore();
|
||||
expect(
|
||||
(await reader.entries()[Symbol.asyncIterator]().next()).value,
|
||||
).toMatchObject({ kind: "file" });
|
||||
} finally {
|
||||
read.mockRestore();
|
||||
await reader.close();
|
||||
await admitted.handle.close();
|
||||
}
|
||||
});
|
||||
|
||||
it("does not publish decoded regions when final source verification fails", async () => {
|
||||
const path = await writeImage("partial.hex", INTEL_HEX_I32_GAP);
|
||||
const control = new HexRecordArtifactReader(path, "intel-hex");
|
||||
const reader = new HexRecordArtifactReader(path, "intel-hex");
|
||||
const expected = (await control.entries()[Symbol.asyncIterator]().next())
|
||||
.value!;
|
||||
const nativeRead = ArtifactFileSource.prototype.read;
|
||||
const physical = Object.assign(
|
||||
new Error("final source verification failed"),
|
||||
{ code: "EIO" },
|
||||
);
|
||||
let calls = 0;
|
||||
const read = vi
|
||||
.spyOn(ArtifactFileSource.prototype, "read")
|
||||
.mockImplementation(function (this: ArtifactFileSource, ...args) {
|
||||
calls += 1;
|
||||
if (calls === 2) return Promise.reject(physical);
|
||||
return nativeRead.apply(this, args);
|
||||
});
|
||||
try {
|
||||
await expect(reader.entries()[Symbol.asyncIterator]().next()).rejects.toBe(
|
||||
physical,
|
||||
);
|
||||
await expect(reader.open(expected)).rejects.toMatchObject({
|
||||
reason: "integrity",
|
||||
});
|
||||
read.mockRestore();
|
||||
expect(
|
||||
(await reader.entries()[Symbol.asyncIterator]().next()).value,
|
||||
).toMatchObject({ path: expected.path });
|
||||
} finally {
|
||||
read.mockRestore();
|
||||
await reader.close();
|
||||
await control.close();
|
||||
}
|
||||
});
|
||||
|
||||
it("honors cancellation before admitting a file with the exact reason", async () => {
|
||||
const reader = new HexRecordArtifactReader(
|
||||
await writeImage("cancel.hex", INTEL_HEX_I32_GAP),
|
||||
@@ -337,3 +406,9 @@ it("bounds copied region chunks and isolates cached bytes across opens", async (
|
||||
await reader.close();
|
||||
}
|
||||
});
|
||||
|
||||
artifactPreparationRecoveryCases({
|
||||
image: () => Buffer.from(INTEL_HEX_I32_GAP),
|
||||
createReader: (path, admitted) =>
|
||||
new HexRecordArtifactReader(path, "intel-hex", admitted),
|
||||
});
|
||||
@@ -0,0 +1,211 @@
|
||||
import { writeFile } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
|
||||
import { expect, it, vi } from "vitest";
|
||||
|
||||
import { ArtifactFileSource } from "./ArtifactFileSource.js";
|
||||
import { assertArtifactReadActive } from "./ArtifactFileOperation.js";
|
||||
import { NtfsMftArtifactReader } from "./NtfsMftArtifactReader.js";
|
||||
|
||||
import { MFT_DEFAULT_RECORD_BYTES } from "../domain/windows/ntfsMft.js";
|
||||
import { createTestTempDirectory } from "../../tests/fixtures/temporaryDirectory.js";
|
||||
import { scanCanonicalArtifactInventory } from "../../tests/fixtures/artifactInventory.js";
|
||||
|
||||
/** A 1 KiB FILE record whose update sequence array covers both sectors. */
|
||||
const recordBytes = (options: {
|
||||
readonly sequenceNumber?: number;
|
||||
readonly flags?: number;
|
||||
readonly index?: number;
|
||||
}): Buffer => {
|
||||
const record = Buffer.alloc(MFT_DEFAULT_RECORD_BYTES);
|
||||
record.write("FILE", 0, "latin1");
|
||||
const sequenceNumber = options.sequenceNumber ?? 1;
|
||||
record.writeUInt16LE(48, 4);
|
||||
record.writeUInt16LE(3, 6);
|
||||
record.writeUInt16LE(sequenceNumber, 48);
|
||||
record.writeUInt16LE(0xa5a5, 50);
|
||||
record.writeUInt16LE(0xa5a5, 52);
|
||||
record.writeUInt16LE(sequenceNumber, 16);
|
||||
record.writeUInt16LE(1, 18);
|
||||
record.writeUInt32LE(options.index ?? 0, 44);
|
||||
record.writeUInt16LE(56, 20);
|
||||
record.writeUInt16LE(options.flags ?? 0x0001, 22);
|
||||
record.writeUInt32LE(64, 24);
|
||||
record.writeUInt32LE(0xffff_ffff, 56);
|
||||
record.writeUInt32LE(MFT_DEFAULT_RECORD_BYTES, 28);
|
||||
record.writeUInt16LE(sequenceNumber, 510);
|
||||
record.writeUInt16LE(sequenceNumber, 1022);
|
||||
return record;
|
||||
};
|
||||
|
||||
const writeMft = async (name: string, bytes: Buffer): Promise<string> => {
|
||||
const directory = await createTestTempDirectory("rea-ntfs-mft-");
|
||||
const path = join(directory, name);
|
||||
await writeFile(path, bytes);
|
||||
return path;
|
||||
};
|
||||
|
||||
it("reports a $MFT stream's record headers as the root occurrence", async () => {
|
||||
const inventory = await scanCanonicalArtifactInventory(
|
||||
await writeMft(
|
||||
"$MFT",
|
||||
Buffer.concat([
|
||||
recordBytes({}),
|
||||
recordBytes({ flags: 0x0003, index: 1 }),
|
||||
]),
|
||||
),
|
||||
);
|
||||
expect(inventory.manifest.root_format).toBe("ntfs-mft");
|
||||
expect(inventory.coverage.status).toBe("complete");
|
||||
// A record stream declares no members, so the root is its whole inventory.
|
||||
expect(inventory.occurrences.map((row) => row.logical_path)).toEqual(["."]);
|
||||
expect(inventory.occurrences[0]).toMatchObject({
|
||||
artifact_format: "ntfs-mft",
|
||||
format_metadata: {
|
||||
kind: "ntfs-mft",
|
||||
record_bytes: MFT_DEFAULT_RECORD_BYTES,
|
||||
record_count: 2,
|
||||
in_use_record_count: 2,
|
||||
directory_record_count: 1,
|
||||
extension_record_count: 0,
|
||||
fixup_mismatch_count: 0,
|
||||
fixup_unverified_record_count: 0,
|
||||
truncated_tail_bytes: 0,
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it("names a failed fixup in the coverage limitations", async () => {
|
||||
const record = recordBytes({ sequenceNumber: 4 });
|
||||
// A trailer that is not the sequence number marks the sector as not cleanly written.
|
||||
record.writeUInt16LE(0x1234, 510);
|
||||
const inventory = await scanCanonicalArtifactInventory(
|
||||
await writeMft("$MFT", record),
|
||||
);
|
||||
expect(inventory.manifest.root_format).toBe("ntfs-mft");
|
||||
expect(inventory.coverage.status).toBe("unknown");
|
||||
expect(inventory.limitations).toContainEqual(
|
||||
expect.stringMatching(/sector trailer that is not the update sequence/u),
|
||||
);
|
||||
});
|
||||
|
||||
it("reports an unaligned used-byte count as incomplete coverage", async () => {
|
||||
const record = recordBytes({});
|
||||
record.writeUInt32LE(57, 24);
|
||||
const inventory = await scanCanonicalArtifactInventory(
|
||||
await writeMft("$MFT", record),
|
||||
);
|
||||
expect(inventory.coverage.status).toBe("unknown");
|
||||
expect(inventory.occurrences[0]?.format_metadata).toMatchObject({
|
||||
record_headers: [{ usedBytes: 57, extentValid: false }],
|
||||
});
|
||||
});
|
||||
|
||||
it("reports incomplete coverage when FILE extents contradict the record slots", async () => {
|
||||
const usedTooLarge = recordBytes({});
|
||||
usedTooLarge.writeUInt32LE(4096, 24);
|
||||
const totalTooSmall = recordBytes({});
|
||||
totalTooSmall.writeUInt32LE(512, 28);
|
||||
|
||||
const inventory = await scanCanonicalArtifactInventory(
|
||||
await writeMft("$MFT", Buffer.concat([usedTooLarge, totalTooSmall])),
|
||||
);
|
||||
expect(inventory.coverage.status).toBe("unknown");
|
||||
expect(inventory.limitations).toContainEqual(
|
||||
"2 FILE record(s) declare inconsistent record extents, beginning at offset 0",
|
||||
);
|
||||
expect(inventory.occurrences[0]?.format_metadata).toMatchObject({
|
||||
record_count: 2,
|
||||
fixup_mismatch_count: 0,
|
||||
fixup_unverified_record_count: 0,
|
||||
});
|
||||
});
|
||||
|
||||
it("retries after final source verification fails without publishing its metadata", async () => {
|
||||
const reader = new NtfsMftArtifactReader(
|
||||
await writeMft("$MFT", recordBytes({})),
|
||||
);
|
||||
const read = ArtifactFileSource.prototype.read;
|
||||
const spy = vi
|
||||
.spyOn(ArtifactFileSource.prototype, "read")
|
||||
.mockImplementationOnce(function (this: ArtifactFileSource, signal) {
|
||||
return read.call(this, signal);
|
||||
})
|
||||
.mockRejectedValueOnce(new Error("final source verification failed"));
|
||||
try {
|
||||
await expect(
|
||||
reader.entries()[Symbol.asyncIterator]().next(),
|
||||
).rejects.toThrow("final source verification failed");
|
||||
expect(reader.formatMetadata()).toBeUndefined();
|
||||
await expect(
|
||||
reader.entries()[Symbol.asyncIterator]().next(),
|
||||
).resolves.toMatchObject({
|
||||
done: true,
|
||||
});
|
||||
expect(reader.formatMetadata()).toMatchObject({
|
||||
kind: "ntfs-mft",
|
||||
record_count: 1,
|
||||
});
|
||||
} finally {
|
||||
spy.mockRestore();
|
||||
await reader.close();
|
||||
}
|
||||
});
|
||||
|
||||
it("retries a shared preparation cancelled by another traversal", async () => {
|
||||
const reader = new NtfsMftArtifactReader(
|
||||
await writeMft("$MFT", recordBytes({})),
|
||||
);
|
||||
const controller = new AbortController();
|
||||
let entered: () => void = () => {};
|
||||
const started = new Promise<void>((resolve) => {
|
||||
entered = resolve;
|
||||
});
|
||||
const read = ArtifactFileSource.prototype.read;
|
||||
const spy = vi
|
||||
.spyOn(ArtifactFileSource.prototype, "read")
|
||||
.mockImplementationOnce(async function (this: ArtifactFileSource, signal) {
|
||||
entered();
|
||||
await new Promise<void>((resolve) => {
|
||||
signal?.addEventListener("abort", () => resolve(), { once: true });
|
||||
});
|
||||
assertArtifactReadActive(signal);
|
||||
return read.call(this, signal);
|
||||
});
|
||||
try {
|
||||
const cancelled = reader
|
||||
.entries(controller.signal)
|
||||
[Symbol.asyncIterator]()
|
||||
.next();
|
||||
const cancelledResult = expect(cancelled).rejects.toMatchObject({
|
||||
reason: "cancelled",
|
||||
});
|
||||
await started;
|
||||
const other = reader.entries()[Symbol.asyncIterator]().next();
|
||||
controller.abort("cancelled first traversal");
|
||||
await cancelledResult;
|
||||
await expect(other).resolves.toMatchObject({ done: true });
|
||||
expect(reader.formatMetadata()).toMatchObject({ record_count: 1 });
|
||||
} finally {
|
||||
controller.abort();
|
||||
spy.mockRestore();
|
||||
await reader.close();
|
||||
}
|
||||
});
|
||||
|
||||
it("revalidates the source before reusing a completed header inventory", async () => {
|
||||
const path = await writeMft("$MFT", recordBytes({}));
|
||||
const reader = new NtfsMftArtifactReader(path);
|
||||
try {
|
||||
await reader.entries()[Symbol.asyncIterator]().next();
|
||||
await writeFile(
|
||||
path,
|
||||
Buffer.concat([recordBytes({}), recordBytes({ index: 1 })]),
|
||||
);
|
||||
await expect(
|
||||
reader.entries()[Symbol.asyncIterator]().next(),
|
||||
).rejects.toMatchObject({ reason: "integrity" });
|
||||
} finally {
|
||||
await reader.close();
|
||||
}
|
||||
});
|
||||
@@ -0,0 +1,174 @@
|
||||
import type { Readable } from "node:stream";
|
||||
|
||||
import {
|
||||
ntfsMftMetadataSchema,
|
||||
type ArtifactFormatMetadata,
|
||||
} from "../domain/artifactFormatMetadata.js";
|
||||
import type { RegionDecodeFailure } from "../domain/firmware/regionDecoder.js";
|
||||
import {
|
||||
createMftDecoder,
|
||||
type MftInventory,
|
||||
} from "../domain/windows/ntfsMft.js";
|
||||
import {
|
||||
ArtifactReaderFailure,
|
||||
type ArtifactEntry,
|
||||
type ArtifactReader,
|
||||
type ArtifactReaderTraversal,
|
||||
type ArtifactTraversalOptions,
|
||||
} from "./ArtifactReader.js";
|
||||
import { ArtifactResourceScope } from "./ArtifactResourceScope.js";
|
||||
import {
|
||||
artifactReadCancelledBy,
|
||||
assertArtifactReadActive,
|
||||
} from "./ArtifactFileOperation.js";
|
||||
import { ArtifactFileSource } from "./ArtifactFileSource.js";
|
||||
import { readFileHandleChunks } from "../filesystem/readFileHandleChunks.js";
|
||||
import type { StableRegularFileDescriptor } from "../filesystem/RegularFile.js";
|
||||
|
||||
/**
|
||||
* An MFT stream is its own content: the FILE records describe the source
|
||||
* volume, not members inside the stream, so header facts are attached to
|
||||
* the root occurrence.
|
||||
*/
|
||||
export class NtfsMftArtifactReader implements ArtifactReader {
|
||||
readonly format = "ntfs-mft" as const;
|
||||
readonly #scope = new ArtifactResourceScope();
|
||||
readonly #source: ArtifactFileSource;
|
||||
#prepared:
|
||||
| { signal: AbortSignal | undefined; promise: Promise<void> }
|
||||
| undefined;
|
||||
#metadata: ArtifactFormatMetadata | undefined;
|
||||
#limitations: readonly string[] = [];
|
||||
#closing = false;
|
||||
|
||||
constructor(path: string, admitted?: StableRegularFileDescriptor) {
|
||||
this.#source = new ArtifactFileSource(path, "$MFT stream", admitted);
|
||||
}
|
||||
|
||||
formatMetadata(): ArtifactFormatMetadata | undefined {
|
||||
return this.#metadata === undefined
|
||||
? undefined
|
||||
: structuredClone(this.#metadata);
|
||||
}
|
||||
|
||||
async prepareContainer(
|
||||
expectedSha256: string,
|
||||
signal?: AbortSignal,
|
||||
): Promise<void> {
|
||||
await this.#scope.run(async () => {
|
||||
this.#assertOpen(signal);
|
||||
await this.#source.bindSha256(expectedSha256, signal);
|
||||
});
|
||||
}
|
||||
|
||||
async *entries(
|
||||
signal?: AbortSignal,
|
||||
options?: ArtifactTraversalOptions,
|
||||
): AsyncIterable<ArtifactEntry> {
|
||||
await this.#scope.run(async () => {
|
||||
this.#assertOpen(signal);
|
||||
for (;;) {
|
||||
const pending = (this.#prepared ??= {
|
||||
signal,
|
||||
promise: this.#prepare(signal),
|
||||
});
|
||||
try {
|
||||
await pending.promise;
|
||||
break;
|
||||
} catch (cause: unknown) {
|
||||
if (this.#prepared === pending) this.#prepared = undefined;
|
||||
if (
|
||||
pending.signal !== signal &&
|
||||
signal?.aborted !== true &&
|
||||
artifactReadCancelledBy(cause, pending.signal)
|
||||
) {
|
||||
this.#assertOpen(signal);
|
||||
continue;
|
||||
}
|
||||
throw cause;
|
||||
}
|
||||
}
|
||||
this.#assertOpen(signal);
|
||||
await this.#source.read(signal);
|
||||
});
|
||||
// An MFT declares no members, so the root occurrence is the whole
|
||||
// inventory; the coverage callback reports what its parse did not establish.
|
||||
options?.onCoverage?.(coverageOf(this.#limitations));
|
||||
}
|
||||
|
||||
async open(entry: ArtifactEntry, _signal?: AbortSignal): Promise<Readable> {
|
||||
return this.#scope.run(async () => {
|
||||
throw new ArtifactReaderFailure(
|
||||
"unavailable",
|
||||
`$MFT stream declares no member to open for ${entry.path}`,
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
async close(): Promise<void> {
|
||||
this.#closing = true;
|
||||
await this.#scope.close();
|
||||
await this.#source.close();
|
||||
this.#metadata = undefined;
|
||||
}
|
||||
|
||||
provenance(): readonly [] {
|
||||
return [];
|
||||
}
|
||||
|
||||
#assertOpen(signal?: AbortSignal): void {
|
||||
assertArtifactReadActive(signal);
|
||||
if (this.#closing)
|
||||
throw new ArtifactReaderFailure(
|
||||
"unavailable",
|
||||
"$MFT stream reader is closed",
|
||||
);
|
||||
}
|
||||
|
||||
async #prepare(signal?: AbortSignal): Promise<void> {
|
||||
const image = await this.#decode(signal);
|
||||
await this.#source.read(signal);
|
||||
this.#metadata = ntfsMftMetadataSchema.parse(image.metadata);
|
||||
this.#limitations = image.limitations;
|
||||
}
|
||||
|
||||
async #decode(signal?: AbortSignal): Promise<MftInventory> {
|
||||
const { handle, initial } = await this.#source.read(signal);
|
||||
const decoder = createMftDecoder(initial.size);
|
||||
if (!decoder.ok) throw this.#failure(decoder.error);
|
||||
if (initial.size > 0)
|
||||
for await (const chunk of readFileHandleChunks(handle, {
|
||||
end: initial.size - 1,
|
||||
signal,
|
||||
cancellationFailure: (reason) =>
|
||||
new ArtifactReaderFailure("cancelled", "$MFT stream read cancelled", {
|
||||
cause: reason,
|
||||
}),
|
||||
}) as AsyncIterable<Buffer>) {
|
||||
this.#assertOpen(signal);
|
||||
const written = decoder.value.write(chunk);
|
||||
if (!written.ok) throw this.#failure(written.error);
|
||||
}
|
||||
const image = decoder.value.finish();
|
||||
if (!image.ok) throw this.#failure(image.error);
|
||||
return image.value;
|
||||
}
|
||||
|
||||
#failure(failure: RegionDecodeFailure): ArtifactReaderFailure {
|
||||
return failure.kind === "format"
|
||||
? new ArtifactReaderFailure(
|
||||
"format",
|
||||
`$MFT stream is malformed: ${failure.message}`,
|
||||
)
|
||||
: new ArtifactReaderFailure(
|
||||
"limit",
|
||||
`$MFT stream exceeds a fixed limit: ${failure.message}`,
|
||||
failure.boundary === undefined ? {} : { boundary: failure.boundary },
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
const coverageOf = (limitations: readonly string[]): ArtifactReaderTraversal =>
|
||||
limitations.length === 0
|
||||
? { status: "complete", limitations: [] }
|
||||
: { status: "unknown", limitations };
|
||||
@@ -1,4 +1,7 @@
|
||||
import { artifactReaderLifecycleCases } from "../../tests/fixtures/artifactReaderLifecycle.js";
|
||||
import {
|
||||
artifactPreparationRecoveryCases,
|
||||
artifactReaderLifecycleCases,
|
||||
} from "../../tests/fixtures/artifactReaderLifecycle.js";
|
||||
import { writeFile } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
import { buffer } from "node:stream/consumers";
|
||||
@@ -91,6 +94,14 @@ it("preserves cancellation during payload CRC admission", async () => {
|
||||
}
|
||||
})(),
|
||||
).rejects.toMatchObject({ reason: "cancelled", cause: reason });
|
||||
reads.mockRestore();
|
||||
const recovered = await entries(reader);
|
||||
expect(recovered).toHaveLength(1);
|
||||
expect(
|
||||
(await buffer(await reader.open(recovered[0]!))).equals(
|
||||
Buffer.alloc(128 * 1024, 0x61),
|
||||
),
|
||||
).toBe(true);
|
||||
} finally {
|
||||
await reader.close();
|
||||
reads.mockRestore();
|
||||
@@ -205,3 +216,8 @@ it("reports impossible over-cap component extents as malformed", async () => {
|
||||
await reader.close();
|
||||
}
|
||||
});
|
||||
|
||||
artifactPreparationRecoveryCases({
|
||||
image: () => buildUImage({ payload: Buffer.from("payload") }),
|
||||
createReader: (path, admitted) => new UImageArtifactReader(path, admitted),
|
||||
});
|
||||
@@ -8,7 +8,10 @@ import type {
|
||||
UImageMetadata,
|
||||
} from "../domain/artifactFormatMetadata.js";
|
||||
import { ArtifactFileSource } from "./ArtifactFileSource.js";
|
||||
import { assertArtifactReadActive } from "./ArtifactFileOperation.js";
|
||||
import {
|
||||
artifactReadCancelledBy,
|
||||
assertArtifactReadActive,
|
||||
} from "./ArtifactFileOperation.js";
|
||||
import {
|
||||
ArtifactReaderFailure,
|
||||
copyArtifactEntry,
|
||||
@@ -47,7 +50,9 @@ export class UImageArtifactReader implements ArtifactReader {
|
||||
readonly #entries = new Map<string, UImageEntry>();
|
||||
readonly #streams = new Set<Readable>();
|
||||
readonly #source: ArtifactFileSource;
|
||||
#prepared: Promise<void> | undefined;
|
||||
#prepared:
|
||||
| { signal: AbortSignal | undefined; promise: Promise<void> }
|
||||
| undefined;
|
||||
#header: UImageMetadata | undefined;
|
||||
#closing = false;
|
||||
|
||||
@@ -75,8 +80,27 @@ export class UImageArtifactReader implements ArtifactReader {
|
||||
): AsyncIterable<ArtifactEntry> {
|
||||
await this.#scope.run(async () => {
|
||||
this.#assertOpen(signal);
|
||||
this.#prepared ??= this.#prepare(signal);
|
||||
await this.#prepared;
|
||||
for (;;) {
|
||||
const pending = (this.#prepared ??= {
|
||||
signal,
|
||||
promise: this.#prepare(signal),
|
||||
});
|
||||
try {
|
||||
await pending.promise;
|
||||
break;
|
||||
} catch (cause: unknown) {
|
||||
if (this.#prepared === pending) this.#prepared = undefined;
|
||||
if (
|
||||
pending.signal !== signal &&
|
||||
signal?.aborted !== true &&
|
||||
artifactReadCancelledBy(cause, pending.signal)
|
||||
) {
|
||||
this.#assertOpen(signal);
|
||||
continue;
|
||||
}
|
||||
throw cause;
|
||||
}
|
||||
}
|
||||
await this.#source.read(signal);
|
||||
});
|
||||
this.#assertOpen(signal);
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
import { artifactPreparationRecoveryCases } from "../../tests/fixtures/artifactReaderLifecycle.js";
|
||||
import { ArtifactFileSource } from "./ArtifactFileSource.js";
|
||||
import { truncate, writeFile } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
import { buffer, text } from "node:stream/consumers";
|
||||
@@ -43,6 +45,68 @@ const repeatedBlock = (data: Buffer): Buffer => {
|
||||
]);
|
||||
};
|
||||
|
||||
it("retries UnityFs preparation cancelled during file reading", async () => {
|
||||
const path = await fixture(buildUnityFs([{ path: "a", data: "x" }]));
|
||||
const admitted = await openRegularFile(path, { symlinks: "reject" });
|
||||
const reader = new UnityFsArtifactReader(path, admitted);
|
||||
const controller = new AbortController();
|
||||
const reason = { stage: "header read" };
|
||||
const nativeRead = admitted.handle.read.bind(admitted.handle);
|
||||
const read = vi.spyOn(admitted.handle, "read").mockImplementation((async (
|
||||
...args: unknown[]
|
||||
) => {
|
||||
const result = await Reflect.apply(nativeRead, admitted.handle, args);
|
||||
controller.abort(reason);
|
||||
return result;
|
||||
}) as typeof admitted.handle.read);
|
||||
try {
|
||||
await expect(
|
||||
(async () => {
|
||||
for await (const _ of reader.entries(controller.signal)) {
|
||||
}
|
||||
})(),
|
||||
).rejects.toMatchObject({ reason: "cancelled", cause: reason });
|
||||
read.mockRestore();
|
||||
expect((await entries(reader)).map((entry) => entry.path)).toEqual(["a"]);
|
||||
} finally {
|
||||
read.mockRestore();
|
||||
await reader.close();
|
||||
await admitted.handle.close();
|
||||
}
|
||||
});
|
||||
|
||||
it("does not publish UnityFs entries when final source verification fails", async () => {
|
||||
const path = await fixture(buildUnityFs([{ path: "a", data: "x" }]));
|
||||
const control = new UnityFsArtifactReader(path);
|
||||
const reader = new UnityFsArtifactReader(path);
|
||||
const expected = (await entries(control))[0]!;
|
||||
const nativeRead = ArtifactFileSource.prototype.read;
|
||||
const physical = Object.assign(
|
||||
new Error("final source verification failed"),
|
||||
{ code: "EIO" },
|
||||
);
|
||||
let calls = 0;
|
||||
const read = vi
|
||||
.spyOn(ArtifactFileSource.prototype, "read")
|
||||
.mockImplementation(function (this: ArtifactFileSource, ...args) {
|
||||
calls += 1;
|
||||
if (calls === 2) return Promise.reject(physical);
|
||||
return nativeRead.apply(this, args);
|
||||
});
|
||||
try {
|
||||
await expect(entries(reader)).rejects.toBe(physical);
|
||||
await expect(reader.open(expected)).rejects.toMatchObject({
|
||||
reason: "integrity",
|
||||
});
|
||||
read.mockRestore();
|
||||
expect((await entries(reader)).map((entry) => entry.path)).toEqual(["a"]);
|
||||
} finally {
|
||||
read.mockRestore();
|
||||
await reader.close();
|
||||
await control.close();
|
||||
}
|
||||
});
|
||||
|
||||
describe("UnityFS allocation and malformed-input boundaries", () => {
|
||||
it("rejects an oversized directory extending past the physical file", async () => {
|
||||
const bytes = buildUnityFs([{ path: "data", data: "x" }]);
|
||||
@@ -431,3 +495,8 @@ describe("UnityFS stream capacity and cache ownership", () => {
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
artifactPreparationRecoveryCases({
|
||||
image: () => buildUnityFs([{ path: "a", data: "x" }]),
|
||||
createReader: (path, admitted) => new UnityFsArtifactReader(path, admitted),
|
||||
});
|
||||
@@ -12,7 +12,10 @@ import {
|
||||
type ArtifactTraversalOptions,
|
||||
} from "./ArtifactReader.js";
|
||||
import { ArtifactResourceScope } from "./ArtifactResourceScope.js";
|
||||
import { assertArtifactReadActive } from "./ArtifactFileOperation.js";
|
||||
import {
|
||||
artifactReadCancelledBy,
|
||||
assertArtifactReadActive,
|
||||
} from "./ArtifactFileOperation.js";
|
||||
import { readArtifactFileBytes } from "./ArtifactFileRange.js";
|
||||
import type {
|
||||
ArtifactFormatMetadata,
|
||||
@@ -52,10 +55,12 @@ interface UnityFsEntryState {
|
||||
export class UnityFsArtifactReader implements ArtifactReader {
|
||||
readonly format = "unityfs" as const;
|
||||
readonly #scope = new ArtifactResourceScope();
|
||||
readonly #entries = new Map<string, UnityFsEntryState>();
|
||||
#entries = new Map<string, UnityFsEntryState>();
|
||||
readonly #streams = new Map<Readable, AbortController>();
|
||||
readonly #source: ArtifactFileSource;
|
||||
#prepared: Promise<void> | undefined;
|
||||
#prepared:
|
||||
| { signal: AbortSignal | undefined; promise: Promise<void> }
|
||||
| undefined;
|
||||
#header: UnityFsMetadata | undefined;
|
||||
#blocks: readonly UnityFsBlock[] = [];
|
||||
#dataStart = 0;
|
||||
@@ -88,8 +93,27 @@ export class UnityFsArtifactReader implements ArtifactReader {
|
||||
): AsyncIterable<ArtifactEntry> {
|
||||
await this.#scope.run(async () => {
|
||||
this.#assertOpen(signal);
|
||||
this.#prepared ??= this.#prepare(signal);
|
||||
await this.#prepared;
|
||||
for (;;) {
|
||||
const pending = (this.#prepared ??= {
|
||||
signal,
|
||||
promise: this.#prepare(signal),
|
||||
});
|
||||
try {
|
||||
await pending.promise;
|
||||
break;
|
||||
} catch (cause: unknown) {
|
||||
if (this.#prepared === pending) this.#prepared = undefined;
|
||||
if (
|
||||
pending.signal !== signal &&
|
||||
signal?.aborted !== true &&
|
||||
artifactReadCancelledBy(cause, pending.signal)
|
||||
) {
|
||||
this.#assertOpen(signal);
|
||||
continue;
|
||||
}
|
||||
throw cause;
|
||||
}
|
||||
}
|
||||
await this.#source.read(signal);
|
||||
});
|
||||
this.#assertOpen(signal);
|
||||
@@ -229,7 +253,7 @@ export class UnityFsArtifactReader implements ArtifactReader {
|
||||
if (header === undefined)
|
||||
throw unityFsFormatFailure("no complete UnityFS header");
|
||||
const status = header.layout.status;
|
||||
this.#header = {
|
||||
const metadata: UnityFsMetadata = {
|
||||
kind: "unityfs",
|
||||
signature: "UnityFS",
|
||||
format_version: header.formatVersion,
|
||||
@@ -245,7 +269,12 @@ export class UnityFsArtifactReader implements ArtifactReader {
|
||||
data_hash: null,
|
||||
directory_status: status,
|
||||
};
|
||||
if (status !== "available") return;
|
||||
if (status !== "available") {
|
||||
await this.#source.read(signal);
|
||||
this.#assertOpen(signal);
|
||||
this.#header = metadata;
|
||||
return;
|
||||
}
|
||||
const { dataHash, blocks, nodes, dataStart } = await readUnityFsBlocksInfo(
|
||||
handle,
|
||||
header,
|
||||
@@ -259,44 +288,55 @@ export class UnityFsArtifactReader implements ArtifactReader {
|
||||
if (streamBytes > BigInt(budget))
|
||||
throw expansionLimit("declared virtual stream", streamBytes, budget);
|
||||
}
|
||||
this.#header = {
|
||||
...this.#header,
|
||||
const preparedMetadata: UnityFsMetadata = {
|
||||
...metadata,
|
||||
data_hash: dataHash,
|
||||
block_count: blocks.length,
|
||||
data_start: dataStart,
|
||||
};
|
||||
this.#dataStart = dataStart;
|
||||
// Keep the source block count above, but do not revisit zero-output rows
|
||||
// for every member. Parsed stream/compressed offsets remain authoritative.
|
||||
this.#blocks = blocks.filter((block) => block.uncompressedSize > 0);
|
||||
this.#remainingDecodeBytes = budget;
|
||||
await this.#registerNodes(nodes, budget, signal);
|
||||
const readableBlocks = blocks.filter((block) => block.uncompressedSize > 0);
|
||||
const registered = await this.#registerNodes(
|
||||
nodes,
|
||||
readableBlocks,
|
||||
budget,
|
||||
signal,
|
||||
);
|
||||
await this.#source.read(signal);
|
||||
this.#assertOpen(signal);
|
||||
this.#header = preparedMetadata;
|
||||
this.#dataStart = dataStart;
|
||||
this.#blocks = readableBlocks;
|
||||
this.#remainingDecodeBytes = budget;
|
||||
this.#entries = registered.entries;
|
||||
this.#remainingReadOperations = registered.remainingReadOperations;
|
||||
}
|
||||
|
||||
async #registerNodes(
|
||||
nodes: readonly UnityFsNode[],
|
||||
blocks: readonly UnityFsBlock[],
|
||||
budget: number,
|
||||
signal?: AbortSignal,
|
||||
): Promise<void> {
|
||||
): Promise<{
|
||||
entries: Map<string, UnityFsEntryState>;
|
||||
remainingReadOperations: number;
|
||||
}> {
|
||||
const entries = new Map<string, UnityFsEntryState>();
|
||||
let readableBytes = 0n;
|
||||
for (let index = 0; index < nodes.length; index++) {
|
||||
if (index % 256 === 0) await setImmediate();
|
||||
this.#assertOpen(signal);
|
||||
const node = nodes[index]!;
|
||||
const firstBlock =
|
||||
node.size === 0 ? -1 : blockEndingAfter(this.#blocks, node.offset);
|
||||
node.size === 0 ? -1 : blockEndingAfter(blocks, node.offset);
|
||||
const lastBlock =
|
||||
node.size === 0
|
||||
? -1
|
||||
: blockEndingAfter(this.#blocks, node.offset + node.size - 1);
|
||||
: blockEndingAfter(blocks, node.offset + node.size - 1);
|
||||
if (node.size !== 0 && (firstBlock < 0 || lastBlock < firstBlock))
|
||||
throw unityFsFormatFailure("member has no block range");
|
||||
const unavailable = firstUnavailableBlock(
|
||||
this.#blocks,
|
||||
firstBlock,
|
||||
lastBlock,
|
||||
);
|
||||
const unavailable = firstUnavailableBlock(blocks, firstBlock, lastBlock);
|
||||
const kind =
|
||||
(node.flags & 1) !== 0 ? ("directory" as const) : ("file" as const);
|
||||
if (kind === "file" && unavailable === undefined) {
|
||||
@@ -331,7 +371,7 @@ export class UnityFsArtifactReader implements ArtifactReader {
|
||||
flags: node.flags,
|
||||
},
|
||||
} satisfies UnityFsEntry;
|
||||
this.#entries.set(adapterKey, {
|
||||
entries.set(adapterKey, {
|
||||
entry,
|
||||
firstBlock,
|
||||
lastBlock,
|
||||
@@ -340,14 +380,15 @@ export class UnityFsArtifactReader implements ArtifactReader {
|
||||
}
|
||||
// Four ordinary passes over admitted blocks, nodes and plaintext chunks,
|
||||
// with a small-reader floor. Byte quotas alone cannot bound tiny-block I/O.
|
||||
this.#remainingReadOperations =
|
||||
const remainingReadOperations =
|
||||
4 *
|
||||
Math.max(
|
||||
1024,
|
||||
this.#blocks.length +
|
||||
blocks.length +
|
||||
nodes.length +
|
||||
Math.ceil(Number(readableBytes) / CHUNK_BYTES),
|
||||
);
|
||||
return { entries, remainingReadOperations };
|
||||
}
|
||||
|
||||
async *#memberChunks(
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
import { artifactPreparationRecoveryCases } from "../../tests/fixtures/artifactReaderLifecycle.js";
|
||||
import { ArtifactFileSource } from "./ArtifactFileSource.js";
|
||||
import { once } from "node:events";
|
||||
import { writeFile } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
@@ -26,6 +28,68 @@ const entries = async (reader: WxapkgArtifactReader) => {
|
||||
return result;
|
||||
};
|
||||
|
||||
it("retries Wxapkg preparation cancelled during file reading", async () => {
|
||||
const path = await fixture(buildWxapkg([{ path: "a", data: "x" }]));
|
||||
const admitted = await openRegularFile(path, { symlinks: "reject" });
|
||||
const reader = new WxapkgArtifactReader(path, admitted);
|
||||
const controller = new AbortController();
|
||||
const reason = { stage: "header read" };
|
||||
const nativeRead = admitted.handle.read.bind(admitted.handle);
|
||||
const read = vi.spyOn(admitted.handle, "read").mockImplementation((async (
|
||||
...args: unknown[]
|
||||
) => {
|
||||
const result = await Reflect.apply(nativeRead, admitted.handle, args);
|
||||
controller.abort(reason);
|
||||
return result;
|
||||
}) as typeof admitted.handle.read);
|
||||
try {
|
||||
await expect(
|
||||
(async () => {
|
||||
for await (const _ of reader.entries(controller.signal)) {
|
||||
}
|
||||
})(),
|
||||
).rejects.toMatchObject({ reason: "cancelled", cause: reason });
|
||||
read.mockRestore();
|
||||
expect((await entries(reader)).map((entry) => entry.path)).toEqual(["a"]);
|
||||
} finally {
|
||||
read.mockRestore();
|
||||
await reader.close();
|
||||
await admitted.handle.close();
|
||||
}
|
||||
});
|
||||
|
||||
it("does not publish Wxapkg entries when final source verification fails", async () => {
|
||||
const path = await fixture(buildWxapkg([{ path: "a", data: "x" }]));
|
||||
const control = new WxapkgArtifactReader(path);
|
||||
const reader = new WxapkgArtifactReader(path);
|
||||
const expected = (await entries(control))[0]!;
|
||||
const nativeRead = ArtifactFileSource.prototype.read;
|
||||
const physical = Object.assign(
|
||||
new Error("final source verification failed"),
|
||||
{ code: "EIO" },
|
||||
);
|
||||
let calls = 0;
|
||||
const read = vi
|
||||
.spyOn(ArtifactFileSource.prototype, "read")
|
||||
.mockImplementation(function (this: ArtifactFileSource, ...args) {
|
||||
calls += 1;
|
||||
if (calls === 2) return Promise.reject(physical);
|
||||
return nativeRead.apply(this, args);
|
||||
});
|
||||
try {
|
||||
await expect(entries(reader)).rejects.toBe(physical);
|
||||
await expect(reader.open(expected)).rejects.toMatchObject({
|
||||
reason: "integrity",
|
||||
});
|
||||
read.mockRestore();
|
||||
expect((await entries(reader)).map((entry) => entry.path)).toEqual(["a"]);
|
||||
} finally {
|
||||
read.mockRestore();
|
||||
await reader.close();
|
||||
await control.close();
|
||||
}
|
||||
});
|
||||
|
||||
describe("wxapkg allocation and malformed-input boundaries", () => {
|
||||
it("admits the encoded table budget before reading it", async () => {
|
||||
const bytes = buildWxapkg([{ path: "a", data: "x" }]);
|
||||
@@ -308,3 +372,8 @@ describe("wxapkg stream capacity and freshness", () => {
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
artifactPreparationRecoveryCases({
|
||||
image: () => buildWxapkg([{ path: "a", data: "x" }]),
|
||||
createReader: (path, admitted) => new WxapkgArtifactReader(path, admitted),
|
||||
});
|
||||
@@ -12,7 +12,10 @@ import {
|
||||
type ArtifactTraversalOptions,
|
||||
} from "./ArtifactReader.js";
|
||||
import { ArtifactResourceScope } from "./ArtifactResourceScope.js";
|
||||
import { assertArtifactReadActive } from "./ArtifactFileOperation.js";
|
||||
import {
|
||||
artifactReadCancelledBy,
|
||||
assertArtifactReadActive,
|
||||
} from "./ArtifactFileOperation.js";
|
||||
import { readArtifactFileBytes } from "./ArtifactFileRange.js";
|
||||
import type {
|
||||
ArtifactFormatMetadata,
|
||||
@@ -48,10 +51,12 @@ interface WxapkgEntryState {
|
||||
export class WxapkgArtifactReader implements ArtifactReader {
|
||||
readonly format = "wxapkg" as const;
|
||||
readonly #scope = new ArtifactResourceScope();
|
||||
readonly #entries = new Map<string, WxapkgEntryState>();
|
||||
#entries = new Map<string, WxapkgEntryState>();
|
||||
readonly #streams = new Map<Readable, AbortController>();
|
||||
readonly #source: ArtifactFileSource;
|
||||
#prepared: Promise<void> | undefined;
|
||||
#prepared:
|
||||
| { signal: AbortSignal | undefined; promise: Promise<void> }
|
||||
| undefined;
|
||||
#header: WxapkgMetadata | undefined;
|
||||
#remainingReadOperations = 0;
|
||||
#admittedReadOperations = 0;
|
||||
@@ -81,8 +86,27 @@ export class WxapkgArtifactReader implements ArtifactReader {
|
||||
): AsyncIterable<ArtifactEntry> {
|
||||
await this.#scope.run(async () => {
|
||||
this.#assertOpen(signal);
|
||||
this.#prepared ??= this.#prepare(signal);
|
||||
await this.#prepared;
|
||||
for (;;) {
|
||||
const pending = (this.#prepared ??= {
|
||||
signal,
|
||||
promise: this.#prepare(signal),
|
||||
});
|
||||
try {
|
||||
await pending.promise;
|
||||
break;
|
||||
} catch (cause: unknown) {
|
||||
if (this.#prepared === pending) this.#prepared = undefined;
|
||||
if (
|
||||
pending.signal !== signal &&
|
||||
signal?.aborted !== true &&
|
||||
artifactReadCancelledBy(cause, pending.signal)
|
||||
) {
|
||||
this.#assertOpen(signal);
|
||||
continue;
|
||||
}
|
||||
throw cause;
|
||||
}
|
||||
}
|
||||
await this.#source.read(signal);
|
||||
});
|
||||
this.#assertOpen(signal);
|
||||
@@ -201,7 +225,7 @@ export class WxapkgArtifactReader implements ArtifactReader {
|
||||
const header = await readWxapkgHeader(handle, initial.size, signal);
|
||||
if (header === undefined)
|
||||
throw wxapkgFormatFailure("no complete wxapkg header");
|
||||
this.#header = {
|
||||
const metadata: WxapkgMetadata = {
|
||||
kind: "wxapkg",
|
||||
info: header.layout.status === "available" ? header.info : null,
|
||||
index_length:
|
||||
@@ -212,7 +236,12 @@ export class WxapkgArtifactReader implements ArtifactReader {
|
||||
header.layout.status === "available" ? header.fileCount : null,
|
||||
directory_status: header.layout.status,
|
||||
};
|
||||
if (header.layout.status !== "available") return;
|
||||
if (header.layout.status !== "available") {
|
||||
await this.#source.read(signal);
|
||||
this.#assertOpen(signal);
|
||||
this.#header = metadata;
|
||||
return;
|
||||
}
|
||||
const { members } = await readWxapkgFileTable(
|
||||
handle,
|
||||
header,
|
||||
@@ -220,6 +249,7 @@ export class WxapkgArtifactReader implements ArtifactReader {
|
||||
signal,
|
||||
);
|
||||
const budget = artifactExpansionBudgetBytes(initial.size);
|
||||
const entries = new Map<string, WxapkgEntryState>();
|
||||
let declaredBytes = 0n;
|
||||
for (let index = 0; index < members.length; index++) {
|
||||
if (index % 256 === 0) await setImmediate();
|
||||
@@ -234,7 +264,7 @@ export class WxapkgArtifactReader implements ArtifactReader {
|
||||
{ boundary: "memory" },
|
||||
);
|
||||
const adapterKey = `wxapkg-entry:${String(index)}`;
|
||||
this.#entries.set(adapterKey, {
|
||||
entries.set(adapterKey, {
|
||||
entry: {
|
||||
path: member.path,
|
||||
sourcePath: member.declaredPath,
|
||||
@@ -260,14 +290,18 @@ export class WxapkgArtifactReader implements ArtifactReader {
|
||||
});
|
||||
}
|
||||
// Header, table and per-chunk member reads; a small-reader floor applies.
|
||||
this.#admittedReadOperations =
|
||||
const admittedReadOperations =
|
||||
4 *
|
||||
Math.max(
|
||||
1024,
|
||||
members.length + Math.ceil(Number(declaredBytes) / CHUNK_BYTES),
|
||||
);
|
||||
this.#remainingReadOperations = this.#admittedReadOperations;
|
||||
await this.#source.read(signal);
|
||||
this.#assertOpen(signal);
|
||||
this.#header = metadata;
|
||||
this.#entries = entries;
|
||||
this.#admittedReadOperations = admittedReadOperations;
|
||||
this.#remainingReadOperations = admittedReadOperations;
|
||||
}
|
||||
|
||||
async *#memberChunks(
|
||||
|
||||
@@ -43,7 +43,8 @@ type FormatsWithReaders =
|
||||
| "uf2"
|
||||
| "dfuse"
|
||||
| "esp-flash-image"
|
||||
| "fit";
|
||||
| "fit"
|
||||
| "ntfs-mft";
|
||||
|
||||
const readerFormatsMatch: Equal<ArtifactReaderFormat, FormatsWithReaders> =
|
||||
true;
|
||||
|
||||
@@ -40,6 +40,7 @@ import { NodeSeaArtifactReader } from "./NodeSeaArtifactReader.js";
|
||||
import type { PyInstallerAdmission } from "./pyinstaller/PyInstallerAdmission.js";
|
||||
import { PyInstallerArtifactReader } from "./PyInstallerArtifactReader.js";
|
||||
import { Uf2ArtifactReader } from "./Uf2ArtifactReader.js";
|
||||
import { NtfsMftArtifactReader } from "./NtfsMftArtifactReader.js";
|
||||
import { UImageArtifactReader } from "./UImageArtifactReader.js";
|
||||
import { UnityFsArtifactReader } from "./UnityFsArtifactReader.js";
|
||||
import { WxapkgArtifactReader } from "./WxapkgArtifactReader.js";
|
||||
@@ -465,6 +466,14 @@ const NON_ZIP_FORMATS = {
|
||||
createReader: (path, options) =>
|
||||
new FitArtifactReader(path, sourceOf(options)),
|
||||
}),
|
||||
"ntfs-mft": described("ntfs-mft", {
|
||||
...plain,
|
||||
retainClassifiedSource: true,
|
||||
readerLimitation:
|
||||
"MFT inventory reports whole-stream counts and a bounded sample of the first 32 decoded FILE record headers, with the omitted-header count. Attributes, filenames, data streams and the $UsnJrnl:$J journal are not decoded, so no path, file size or change history is established.",
|
||||
createReader: (path, options) =>
|
||||
new NtfsMftArtifactReader(path, sourceOf(options)),
|
||||
}),
|
||||
unknown: described("unknown", plain),
|
||||
} satisfies Record<
|
||||
Exclude<ArtifactFormatId, ZipPackageFormat>,
|
||||
|
||||
@@ -576,6 +576,28 @@ it("classifies nested UF2 by block magics and DfuSe by prefix and suffix", () =>
|
||||
).toMatchObject({ format: "file" });
|
||||
});
|
||||
|
||||
it("classifies a $MFT stream by its FILE record header, under any name", () => {
|
||||
const mft = Buffer.alloc(2_048);
|
||||
mft.write("FILE", 0, "latin1");
|
||||
mft.writeUInt32LE(1024, 28);
|
||||
for (const path of ["evidence/$MFT", "evidence/mft.raw", "evidence/records"])
|
||||
expect(classifyArtifactBytesAtPath(path, mft)).toEqual({
|
||||
kind: "executable",
|
||||
format: "ntfs-mft",
|
||||
});
|
||||
});
|
||||
|
||||
it("leaves plain content beginning with FILE unclassified as MFT", () => {
|
||||
const text = Buffer.from(
|
||||
"FILE is a plain text prefix, not an NTFS record header",
|
||||
);
|
||||
expect(
|
||||
classifyArtifactBytesAtPath("evidence/readme.txt", text),
|
||||
).toMatchObject({
|
||||
format: "file",
|
||||
});
|
||||
});
|
||||
|
||||
it("classifies a FIT by a /images node within the first 4 KiB", () => {
|
||||
const images = [{ name: "kernel", data: Buffer.from("kernel") }];
|
||||
const late = devicetreeFixture({
|
||||
|
||||
@@ -11,6 +11,7 @@ import { recognizeEspImage } from "../../domain/firmware/espImage.js";
|
||||
import { hasUf2Block } from "../../domain/firmware/uf2.js";
|
||||
import { isDfuSeFile } from "../../domain/firmware/dfuse.js";
|
||||
import { hasFitImagesNode } from "../../domain/firmware/fitImage.js";
|
||||
import { hasMftFileRecordHeader } from "../../domain/windows/ntfsMft.js";
|
||||
import type { ArtifactClassificationEvidence } from "../ArtifactClassificationProbe.js";
|
||||
import { asarHeaderAdmissionFailure } from "../AsarHeader.js";
|
||||
import { hasDestinationPath } from "../ArtifactPaths.js";
|
||||
@@ -622,6 +623,7 @@ export const classifyArtifactBytes = (
|
||||
return "dfuse";
|
||||
if (recognizeEspImage(prefix, fileSize) === "esp-image") return "esp-image";
|
||||
if (hasFitImagesNode(prefix)) return "fit";
|
||||
if (hasMftFileRecordHeader(prefix)) return "ntfs-mft";
|
||||
if (hasBinaryPlistSignature(prefix)) return "plist";
|
||||
if (hasXmlPlistSignature(prefix)) return "plist";
|
||||
const fatFormat = classifyFatArtifact(evidence, fileSize);
|
||||
|
||||
@@ -89,7 +89,8 @@ it.each(["max_input_bytes", "max_reply_bytes", "max_nesting_depth"] as const)(
|
||||
parameter: control,
|
||||
effective_value: selected,
|
||||
source: "request",
|
||||
rea_restart_required: false,
|
||||
update_mode: "parameter",
|
||||
pending_restart: false,
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
@@ -103,11 +103,10 @@ it("uses a valid REA_CAPTURE_DECODER_OLD_GENERATION_MIB override", async () => {
|
||||
/floor 8 MiB.*maximum 16384 MiB/iu,
|
||||
);
|
||||
expect(projected.remediation.action).toContain("update_analysis_settings");
|
||||
expect(projected.remediation.action).toMatch(/restart REA/iu);
|
||||
expect(projected.remediation.action).toContain("launch seed");
|
||||
expect(projected.remediation.action).toContain(
|
||||
"capture.decoder_old_generation_mib",
|
||||
);
|
||||
expect(projected.remediation.action).toMatch(/without a REA restart/iu);
|
||||
expect(projected.remediation.action).toMatch(
|
||||
/removes inherited NODE_OPTIONS/iu,
|
||||
);
|
||||
@@ -125,7 +124,8 @@ it("uses a valid REA_CAPTURE_DECODER_OLD_GENERATION_MIB override", async () => {
|
||||
unit: "MiB",
|
||||
applies_at: "next_worker",
|
||||
agent_editable: true,
|
||||
rea_restart_required: false,
|
||||
update_mode: "runtime",
|
||||
pending_restart: false,
|
||||
},
|
||||
reported_limits: {
|
||||
old_generation_heap_mib: 256,
|
||||
|
||||
@@ -71,7 +71,7 @@ export const resolveCaptureDecoderHeap = (
|
||||
export const captureDecoderHeapRecovery = (heap: CaptureDecoderHeap): string =>
|
||||
[
|
||||
`The effective ${CAPTURE_DECODER_OLD_GENERATION_SETTING} value is ${String(heap.oldGenerationMiB)} MiB (source ${heap.source}; default ${String(DEFAULT_CAPTURE_DECODER_OLD_GENERATION_MIB)} MiB, floor ${String(CAPTURE_DECODER_SEMI_SPACE_MIB)} MiB, maximum ${String(MAXIMUM_CAPTURE_DECODER_OLD_GENERATION_MIB)} MiB).`,
|
||||
`Set ${CAPTURE_DECODER_OLD_GENERATION_SETTING} in the environment that starts REA to a decimal integer from ${String(CAPTURE_DECODER_SEMI_SPACE_MIB)} to ${String(MAXIMUM_CAPTURE_DECODER_OLD_GENERATION_MIB)}, then restart REA. In a running MCP server, use update_analysis_settings with setting capture.decoder_old_generation_mib and action set instead; the new value applies to new work without a REA restart.`,
|
||||
`capture.decoder_old_generation_mib can be changed at runtime with update_analysis_settings. The launch seed is ${CAPTURE_DECODER_OLD_GENERATION_SETTING}, a decimal integer from ${String(CAPTURE_DECODER_SEMI_SPACE_MIB)} to ${String(MAXIMUM_CAPTURE_DECODER_OLD_GENERATION_MIB)}.`,
|
||||
`On Node.js the decoder passes --max-old-space-size=${String(heap.oldGenerationMiB)} and removes inherited NODE_OPTIONS, so additional host memory and NODE_OPTIONS do not raise this heap. The ${String(CAPTURE_DECODER_SEMI_SPACE_MIB)} MiB semi-space stays fixed.`,
|
||||
"On Bun the V8 flags are ignored. Windows limits the decoder's committed memory to this value plus a 512 MiB reserve. Other Bun hosts enforce no decoder heap limit.",
|
||||
].join(" ");
|
||||
@@ -171,7 +171,8 @@ it("names the session setting on a Rizin capacity refusal", async () => {
|
||||
minimum: MINIMUM_DEBUGGER_MAX_SESSIONS,
|
||||
maximum: MAXIMUM_DEBUGGER_MAX_SESSIONS,
|
||||
unit: "sessions",
|
||||
restart_required: true,
|
||||
update_mode: "restart",
|
||||
pending_restart: false,
|
||||
max_sessions: 1,
|
||||
active_or_reserved_sessions: 1,
|
||||
},
|
||||
|
||||
@@ -6,7 +6,10 @@ import { z } from "zod";
|
||||
import { mcpInputBudgetSchema } from "./mcpInputBudget.js";
|
||||
import { isAbsolute } from "node:path";
|
||||
|
||||
import { ConfigurationError } from "../domain/configurationErrors.js";
|
||||
import {
|
||||
ConfigurationError,
|
||||
type ConfigurationSettingIssue,
|
||||
} from "../domain/configurationErrors.js";
|
||||
import { err, ok, type Result } from "../domain/result.js";
|
||||
import { analysisProviderSelectorSchema } from "../domain/providerSelection.js";
|
||||
import {
|
||||
@@ -165,6 +168,31 @@ const environmentSchema = z.object({
|
||||
REA_REFERENCE_SECRET_PATTERNS_JSON: z.string().default("[]"),
|
||||
});
|
||||
|
||||
/** Copy producer-owned update facts from a Zod issue without inventing them. */
|
||||
const configurationSettingFromIssue = (issue: {
|
||||
readonly path: readonly PropertyKey[];
|
||||
readonly message: string;
|
||||
readonly code: string;
|
||||
readonly params?: Readonly<Record<string, unknown>> | undefined;
|
||||
}): ConfigurationSettingIssue => {
|
||||
const setting = String(issue.path[0] ?? "environment");
|
||||
const constraint = issue.message;
|
||||
if (issue.code !== "custom" || issue.params === undefined)
|
||||
return { setting, constraint };
|
||||
const updateMode = issue.params.update_mode;
|
||||
const pendingRestart = issue.params.pending_restart;
|
||||
return {
|
||||
setting,
|
||||
constraint,
|
||||
...(updateMode === "runtime" || updateMode === "restart"
|
||||
? { update_mode: updateMode }
|
||||
: {}),
|
||||
...(typeof pendingRestart === "boolean"
|
||||
? { pending_restart: pendingRestart }
|
||||
: {}),
|
||||
};
|
||||
};
|
||||
|
||||
/** Parse the JEB-only timeout at its isolated provider-composition boundary. */
|
||||
export const parseJebMcpTimeoutMs = (
|
||||
value: string | undefined,
|
||||
@@ -229,10 +257,10 @@ export const parseEnvironmentValues = (
|
||||
const parsed = environmentSchema.safeParse(literals);
|
||||
if (!parsed.success) {
|
||||
// Zod messages name the constraint, never the rejected value.
|
||||
const settings = parsed.error.issues.map((issue) => ({
|
||||
setting: String(issue.path[0] ?? "environment"),
|
||||
constraint: issue.message,
|
||||
}));
|
||||
// A producer may attach update_mode and pending_restart on a custom issue.
|
||||
const settings = parsed.error.issues.map((issue) =>
|
||||
configurationSettingFromIssue(issue),
|
||||
);
|
||||
return err(
|
||||
new ConfigurationError("Invalid REA environment configuration", {
|
||||
cause: parsed.error,
|
||||
|
||||
@@ -49,7 +49,8 @@ it("keeps the 5000 ms deadline when the setting is unset", () => {
|
||||
applies_at: "next_operation",
|
||||
agent_editable: true,
|
||||
scope: "rea-runtime",
|
||||
rea_restart_required: false,
|
||||
update_mode: "runtime",
|
||||
pending_restart: false,
|
||||
},
|
||||
reported_limits: {
|
||||
matching_deadline_ms: DEFAULT_REGEX_MATCHING_DEADLINE.deadlineMs,
|
||||
@@ -65,8 +66,7 @@ it("keeps the 5000 ms deadline when the setting is unset", () => {
|
||||
expect(projected.remediation.action).toContain("source default");
|
||||
expect(projected.remediation.action).toContain("update_analysis_settings");
|
||||
expect(projected.remediation.action).toContain("regex.matching_deadline_ms");
|
||||
expect(projected.remediation.action).toMatch(/restart REA/i);
|
||||
expect(projected.remediation.action).toMatch(/without a REA restart/i);
|
||||
expect(projected.remediation.action).toContain("launch seed");
|
||||
expect(projected.remediation.action).toMatch(
|
||||
/heap does not raise this deadline/i,
|
||||
);
|
||||
|
||||
@@ -70,6 +70,6 @@ export const regexMatchingDeadlineRecovery = (
|
||||
): string =>
|
||||
[
|
||||
`The effective ${REGEX_MATCHING_DEADLINE_SETTING} value is ${String(deadline.deadlineMs)} ms (source ${deadline.source}; default ${String(DEFAULT_REGEX_MATCHING_DEADLINE_MS)} ms, floor 1 ms, maximum ${String(MAXIMUM_REGEX_MATCHING_DEADLINE_MS)} ms).`,
|
||||
`Set ${REGEX_MATCHING_DEADLINE_SETTING} in the environment that starts REA to a positive decimal integer from 1 to ${String(MAXIMUM_REGEX_MATCHING_DEADLINE_MS)}, then restart REA. In a running MCP server, use update_analysis_settings with setting regex.matching_deadline_ms and action set instead; the new value applies to new work without a REA restart.`,
|
||||
`regex.matching_deadline_ms can be changed at runtime with update_analysis_settings. The launch seed is ${REGEX_MATCHING_DEADLINE_SETTING}, a positive decimal integer from 1 to ${String(MAXIMUM_REGEX_MATCHING_DEADLINE_MS)}.`,
|
||||
"The worker old-generation heap does not raise this deadline.",
|
||||
].join(" ");
|
||||
@@ -56,7 +56,8 @@ it("keeps the 128 MiB request when the setting is unset", () => {
|
||||
applies_at: "next_worker",
|
||||
agent_editable: true,
|
||||
scope: "rea-runtime",
|
||||
rea_restart_required: false,
|
||||
update_mode: "runtime",
|
||||
pending_restart: false,
|
||||
},
|
||||
reported_limits: {
|
||||
requested_worker_heap_mib: DEFAULT_REGEX_WORKER_HEAP_REQUEST.heapMiB,
|
||||
@@ -76,8 +77,7 @@ it("keeps the 128 MiB request when the setting is unset", () => {
|
||||
);
|
||||
expect(projected.remediation.action).toContain("--max-old-space-size");
|
||||
expect(projected.remediation.action).toContain("NODE_OPTIONS");
|
||||
expect(projected.remediation.action).toMatch(/restart REA/i);
|
||||
expect(projected.remediation.action).toMatch(/without a REA restart/i);
|
||||
expect(projected.remediation.action).toContain("launch seed");
|
||||
expect(projected.remediation.action).toMatch(/overrides this request/i);
|
||||
});
|
||||
|
||||
|
||||
@@ -72,6 +72,6 @@ export const regexWorkerHeapRequestRecovery = (
|
||||
): string =>
|
||||
[
|
||||
`The effective ${REGEX_WORKER_HEAP_REQUEST_SETTING} value is ${String(request.heapMiB)} MiB (source ${request.source}; default ${String(DEFAULT_REGEX_WORKER_HEAP_REQUEST_MIB)} MiB, floor ${String(MINIMUM_REGEX_WORKER_HEAP_REQUEST_MIB)} MiB, maximum ${String(MAXIMUM_REGEX_WORKER_HEAP_REQUEST_MIB)} MiB).`,
|
||||
`Set ${REGEX_WORKER_HEAP_REQUEST_SETTING} in the environment that starts REA to a decimal integer from ${String(MINIMUM_REGEX_WORKER_HEAP_REQUEST_MIB)} to ${String(MAXIMUM_REGEX_WORKER_HEAP_REQUEST_MIB)}, then restart REA. In a running MCP server, use update_analysis_settings with setting regex.worker_heap_request_mib and action set instead; the new value applies to new work without a REA restart.`,
|
||||
`regex.worker_heap_request_mib can be changed at runtime with update_analysis_settings. The launch seed is ${REGEX_WORKER_HEAP_REQUEST_SETTING}, a decimal integer from ${String(MINIMUM_REGEX_WORKER_HEAP_REQUEST_MIB)} to ${String(MAXIMUM_REGEX_WORKER_HEAP_REQUEST_MIB)}.`,
|
||||
"A positive Node --max-old-space-size, including NODE_OPTIONS, overrides this request and sets the effective heap. That flag is not limited by this setting's floor or maximum. Raising host memory alone does not raise the request.",
|
||||
].join(" ");
|
||||
@@ -27,7 +27,7 @@ const requiredFacts = (
|
||||
expect(message).toContain(String(fact.defaultValue));
|
||||
expect(message).toContain(String(fact.floor));
|
||||
expect(message).toContain(String(fact.maximum));
|
||||
expect(message).toContain("restart REA");
|
||||
expect(message).toContain("positive decimal integer");
|
||||
};
|
||||
|
||||
it("keeps today's settle defaults when the settings are unset", () => {
|
||||
@@ -98,17 +98,21 @@ it.each([
|
||||
const message = resolved.error.settings[0]?.constraint;
|
||||
if (message === undefined) throw new Error("Missing configuration guidance");
|
||||
requiredFacts(message, { ...bound, currentValue: value, source: "setting" });
|
||||
expect(resolved.error.settings).toEqual([
|
||||
{ setting: bound.setting, constraint: message },
|
||||
]);
|
||||
const facts = {
|
||||
setting: bound.setting,
|
||||
constraint: message,
|
||||
update_mode: "restart" as const,
|
||||
pending_restart: true as const,
|
||||
};
|
||||
expect(resolved.error.settings).toEqual([facts]);
|
||||
expect(projectAnalysisError(resolved.error)).toMatchObject({
|
||||
code: "configuration_invalid",
|
||||
details: { settings: [{ setting: bound.setting, constraint: message }] },
|
||||
details: { settings: [facts] },
|
||||
});
|
||||
const startup = parseEnvironment({ [bound.setting]: value });
|
||||
if (startup.ok) throw new Error(`Startup accepted ${bound.setting}=${value}`);
|
||||
expect(startup.error.settings).toContainEqual({
|
||||
setting: bound.setting,
|
||||
constraint: message,
|
||||
expect(startup.error.settings).toContainEqual(facts);
|
||||
expect(projectAnalysisError(startup.error).details).toMatchObject({
|
||||
settings: [facts],
|
||||
});
|
||||
});
|
||||
@@ -55,7 +55,9 @@ export interface WindowsChangeTimeSettingFailure {
|
||||
readonly floor: number;
|
||||
readonly maximum: number;
|
||||
readonly unit: "ms" | "polls";
|
||||
readonly restartRequired: true;
|
||||
readonly update_mode: "restart";
|
||||
/** The rejected environment value is not the value this process is using. */
|
||||
readonly pending_restart: true;
|
||||
}
|
||||
|
||||
/** Effective settle budget and whether each part came from the default or a setting. */
|
||||
@@ -79,10 +81,11 @@ const windowsChangeTimeSettingFailure = (
|
||||
floor: bound.floor,
|
||||
maximum: bound.maximum,
|
||||
unit: bound.unit,
|
||||
restartRequired: true,
|
||||
update_mode: "restart",
|
||||
pending_restart: true,
|
||||
});
|
||||
|
||||
/** Name the rejected value, its source, the allowed range, and the required restart. */
|
||||
/** Name the rejected value, its source, and the allowed range. */
|
||||
const windowsChangeTimeSettingMessage = (
|
||||
failure: WindowsChangeTimeSettingFailure,
|
||||
): string =>
|
||||
@@ -100,11 +103,14 @@ export const windowsChangeTimeSettingSchema = (bound: WindowsChangeTimeBound) =>
|
||||
.string()
|
||||
.superRefine((value, context) => {
|
||||
if (inRange(bound, value)) return;
|
||||
const failure = windowsChangeTimeSettingFailure(bound, value);
|
||||
context.addIssue({
|
||||
code: "custom",
|
||||
message: windowsChangeTimeSettingMessage(
|
||||
windowsChangeTimeSettingFailure(bound, value),
|
||||
),
|
||||
message: windowsChangeTimeSettingMessage(failure),
|
||||
params: {
|
||||
update_mode: failure.update_mode,
|
||||
pending_restart: failure.pending_restart,
|
||||
},
|
||||
});
|
||||
})
|
||||
.transform(Number);
|
||||
@@ -160,6 +166,8 @@ export const resolveWindowsChangeTimeSettle = (
|
||||
settings: failures.map((failure) => ({
|
||||
setting: failure.setting,
|
||||
constraint: windowsChangeTimeSettingMessage(failure),
|
||||
update_mode: failure.update_mode,
|
||||
pending_restart: failure.pending_restart,
|
||||
})),
|
||||
},
|
||||
),
|
||||
|
||||
@@ -48,6 +48,8 @@ const descriptor = z.strictObject({
|
||||
minimum: z.number(),
|
||||
maximum: z.number(),
|
||||
applies_at: z.enum(["next_operation", "next_worker", "next_startup"]),
|
||||
update_mode: z.enum(["runtime", "restart"]),
|
||||
pending_restart: z.boolean(),
|
||||
provider: z.string(),
|
||||
operations: z.array(z.string()),
|
||||
environment: z.string().optional(),
|
||||
@@ -83,7 +85,7 @@ export const updateAnalysisSettingsResultSchema =
|
||||
}),
|
||||
),
|
||||
active_work_changed: z.literal(false),
|
||||
rea_restart_required: z.literal(false),
|
||||
pending_restart: z.literal(false),
|
||||
});
|
||||
export const ANALYSIS_SETTINGS_TOOL_CONTRACTS = [
|
||||
{
|
||||
|
||||
@@ -1512,7 +1512,8 @@ describe("analysis error projection: caller lifetime", () => {
|
||||
minimum: limit.minimum,
|
||||
maximum: limit.maximum,
|
||||
agent_editable: true,
|
||||
rea_restart_required: false,
|
||||
update_mode: "runtime",
|
||||
pending_restart: false,
|
||||
applies_at: "next_invocation",
|
||||
scope: "rea-invocation",
|
||||
environment: "REA_REGEX_MATCHING_DEADLINE_MS",
|
||||
@@ -1529,11 +1530,25 @@ describe("analysis error projection: caller lifetime", () => {
|
||||
expect.objectContaining({
|
||||
setting: "wasm.command_timeout_ms",
|
||||
source: "request",
|
||||
update_mode: "parameter",
|
||||
parameter: "command_timeout_ms",
|
||||
applies_at: "next_invocation",
|
||||
scope: "rea-invocation",
|
||||
}),
|
||||
);
|
||||
expect(
|
||||
analysisSettingLimit("wasm.command_timeout_ms", 100, "runtime"),
|
||||
).toMatchObject({ source: "runtime", update_mode: "runtime" });
|
||||
expect(
|
||||
analysisSettingLimit("javascript.max_heap_mb", 128, "request"),
|
||||
).toMatchObject({
|
||||
source: "request",
|
||||
update_mode: "parameter",
|
||||
parameter: "max_heap_mb",
|
||||
});
|
||||
expect(
|
||||
analysisSettingLimit("javascript.analysis_timeout_ms", 1, "default"),
|
||||
).toMatchObject({ source: "default", update_mode: "runtime" });
|
||||
expect(
|
||||
projectAnalysisError(error, "invocation").details?.limit,
|
||||
).not.toHaveProperty("environment");
|
||||
|
||||
@@ -1044,9 +1044,15 @@ const lifecycleErrorDetails = (
|
||||
}
|
||||
if (error instanceof ConfigurationError && error.settings.length > 0)
|
||||
return {
|
||||
settings: error.settings.map(({ setting, constraint }) => ({
|
||||
setting,
|
||||
constraint,
|
||||
settings: error.settings.map((issue) => ({
|
||||
setting: issue.setting,
|
||||
constraint: issue.constraint,
|
||||
...(issue.update_mode === undefined
|
||||
? {}
|
||||
: { update_mode: issue.update_mode }),
|
||||
...(issue.pending_restart === undefined
|
||||
? {}
|
||||
: { pending_restart: issue.pending_restart }),
|
||||
})),
|
||||
};
|
||||
if (error instanceof BinaryTargetError)
|
||||
|
||||
@@ -46,6 +46,7 @@ interface SettingDefinition {
|
||||
readonly maximum: number;
|
||||
readonly unit: "bytes" | "ms" | "seconds" | "MiB" | "count";
|
||||
readonly appliesAt: "next_operation" | "next_worker" | "next_startup";
|
||||
readonly updateMode: "runtime" | "restart";
|
||||
readonly provider: string;
|
||||
readonly relatedProviders?: readonly string[];
|
||||
readonly operations: readonly string[];
|
||||
@@ -60,6 +61,7 @@ const timer = {
|
||||
/** Metadata aggregates existing owner definitions; no environment mutation or engine discovery. */
|
||||
export const ANALYSIS_SETTINGS = {
|
||||
"ghidra.decompile_max_payload_mib": {
|
||||
updateMode: "runtime",
|
||||
environment: "REA_GHIDRA_DECOMPILE_MAX_PAYLOAD_MIB",
|
||||
defaultValue: null,
|
||||
minimum: 1,
|
||||
@@ -77,6 +79,7 @@ export const ANALYSIS_SETTINGS = {
|
||||
],
|
||||
},
|
||||
"asar.max_listing_bytes": {
|
||||
updateMode: "runtime",
|
||||
environment: "REA_ASAR_MAX_LISTING_BYTES",
|
||||
defaultValue: DEFAULT_ASAR_MAX_LISTING_BYTES,
|
||||
minimum: 1,
|
||||
@@ -100,6 +103,7 @@ export const ANALYSIS_SETTINGS = {
|
||||
],
|
||||
},
|
||||
"ghidra.value_flow_max_operations": {
|
||||
updateMode: "runtime",
|
||||
environment: "REA_GHIDRA_VALUE_FLOW_MAX_OPERATIONS",
|
||||
defaultValue: DEFAULT_GHIDRA_VALUE_FLOW_MAX_OPERATIONS,
|
||||
minimum: 1,
|
||||
@@ -114,6 +118,7 @@ export const ANALYSIS_SETTINGS = {
|
||||
],
|
||||
},
|
||||
"ghidra.value_flow_max_inputs_per_operation": {
|
||||
updateMode: "runtime",
|
||||
environment: "REA_GHIDRA_VALUE_FLOW_MAX_INPUTS_PER_OPERATION",
|
||||
defaultValue: DEFAULT_GHIDRA_VALUE_FLOW_MAX_INPUTS_PER_OPERATION,
|
||||
minimum: 1,
|
||||
@@ -128,6 +133,7 @@ export const ANALYSIS_SETTINGS = {
|
||||
],
|
||||
},
|
||||
"ghidra.value_flow_max_edges": {
|
||||
updateMode: "runtime",
|
||||
environment: "REA_GHIDRA_VALUE_FLOW_MAX_EDGES",
|
||||
defaultValue: DEFAULT_GHIDRA_VALUE_FLOW_MAX_EDGES,
|
||||
minimum: 1,
|
||||
@@ -142,6 +148,7 @@ export const ANALYSIS_SETTINGS = {
|
||||
],
|
||||
},
|
||||
"asar.max_header_bytes": {
|
||||
updateMode: "runtime",
|
||||
environment: "REA_ASAR_MAX_HEADER_BYTES",
|
||||
defaultValue: DEFAULT_ASAR_MAX_HEADER_BYTES,
|
||||
minimum: 1,
|
||||
@@ -165,6 +172,7 @@ export const ANALYSIS_SETTINGS = {
|
||||
],
|
||||
},
|
||||
"ghidra.startup_timeout_ms": {
|
||||
updateMode: "runtime",
|
||||
...timer,
|
||||
environment: "REA_GHIDRA_STARTUP_TIMEOUT_MS",
|
||||
defaultValue: DEFAULT_GHIDRA_STARTUP_TIMEOUT_MS,
|
||||
@@ -185,6 +193,7 @@ export const ANALYSIS_SETTINGS = {
|
||||
],
|
||||
},
|
||||
"hopper.startup_timeout_ms": {
|
||||
updateMode: "runtime",
|
||||
...timer,
|
||||
environment: "REA_HOPPER_STARTUP_TIMEOUT_MS",
|
||||
defaultValue: DEFAULT_HOPPER_STARTUP_TIMEOUT_MS,
|
||||
@@ -205,6 +214,7 @@ export const ANALYSIS_SETTINGS = {
|
||||
],
|
||||
},
|
||||
"ghidra.decompile_timeout_seconds": {
|
||||
updateMode: "runtime",
|
||||
environment: "REA_GHIDRA_DECOMPILE_TIMEOUT_SECONDS",
|
||||
defaultValue: null,
|
||||
minimum: 1,
|
||||
@@ -220,6 +230,7 @@ export const ANALYSIS_SETTINGS = {
|
||||
],
|
||||
},
|
||||
"jadx.operation_timeout_ms": {
|
||||
updateMode: "runtime",
|
||||
...timer,
|
||||
environment: "REA_JADX_OPERATION_TIMEOUT_MS",
|
||||
defaultValue: DEFAULT_JADX_OPERATION_TIMEOUT_MS,
|
||||
@@ -233,6 +244,7 @@ export const ANALYSIS_SETTINGS = {
|
||||
],
|
||||
},
|
||||
"regex.matching_deadline_ms": {
|
||||
updateMode: "runtime",
|
||||
...timer,
|
||||
environment: "REA_REGEX_MATCHING_DEADLINE_MS",
|
||||
defaultValue: DEFAULT_REGEX_MATCHING_DEADLINE_MS,
|
||||
@@ -242,6 +254,7 @@ export const ANALYSIS_SETTINGS = {
|
||||
operations: ["search_strings", "search_procedures"],
|
||||
},
|
||||
"regex.worker_heap_request_mib": {
|
||||
updateMode: "runtime",
|
||||
environment: "REA_REGEX_WORKER_HEAP_REQUEST_MIB",
|
||||
defaultValue: DEFAULT_REGEX_WORKER_HEAP_REQUEST_MIB,
|
||||
minimum: MINIMUM_REGEX_WORKER_HEAP_REQUEST_MIB,
|
||||
@@ -253,6 +266,7 @@ export const ANALYSIS_SETTINGS = {
|
||||
operations: ["search_strings", "search_procedures"],
|
||||
},
|
||||
"capture.decoder_old_generation_mib": {
|
||||
updateMode: "runtime",
|
||||
environment: "REA_CAPTURE_DECODER_OLD_GENERATION_MIB",
|
||||
defaultValue: DEFAULT_CAPTURE_DECODER_OLD_GENERATION_MIB,
|
||||
minimum: CAPTURE_DECODER_SEMI_SPACE_MIB,
|
||||
@@ -263,6 +277,7 @@ export const ANALYSIS_SETTINGS = {
|
||||
operations: ["inspect_web_network_capture"],
|
||||
},
|
||||
"javascript.analysis_timeout_ms": {
|
||||
updateMode: "runtime",
|
||||
...timer,
|
||||
defaultValue: JAVASCRIPT_ANALYSIS_RESOURCE_DEFAULTS.timeoutMs,
|
||||
provider: "rea-javascript-application",
|
||||
@@ -270,6 +285,7 @@ export const ANALYSIS_SETTINGS = {
|
||||
parameter: "analysis_timeout_ms",
|
||||
},
|
||||
"javascript.max_heap_mb": {
|
||||
updateMode: "runtime",
|
||||
defaultValue: JAVASCRIPT_ANALYSIS_RESOURCE_DEFAULTS.heapMb,
|
||||
minimum: JAVASCRIPT_WORKER_HEAP_MIB_MINIMUM,
|
||||
maximum: JAVASCRIPT_WORKER_HEAP_MIB_MAXIMUM,
|
||||
@@ -280,6 +296,7 @@ export const ANALYSIS_SETTINGS = {
|
||||
parameter: "max_heap_mb",
|
||||
},
|
||||
"wasm.command_timeout_ms": {
|
||||
updateMode: "runtime",
|
||||
...timer,
|
||||
defaultValue: DEFAULT_WASM_COMMAND_TIMEOUT_MS,
|
||||
maximum: MAX_WASM_COMMAND_TIMEOUT_MS,
|
||||
@@ -288,6 +305,7 @@ export const ANALYSIS_SETTINGS = {
|
||||
parameter: "command_timeout_ms",
|
||||
},
|
||||
"native.matching_representation_bytes": {
|
||||
updateMode: "runtime",
|
||||
environment: "REA_NATIVE_MATCHING_REPRESENTATION_BYTES",
|
||||
defaultValue: DEFAULT_NATIVE_MATCHING_REPRESENTATION_BYTES,
|
||||
minimum: 1,
|
||||
@@ -317,6 +335,23 @@ export const analysisSettingValueSchema = (id: AnalysisSettingId) => {
|
||||
? number.nullable()
|
||||
: number;
|
||||
};
|
||||
/**
|
||||
* Caller argument, then the accepted runtime override, then the built-in
|
||||
* default. The third value is that default, not a launch-environment seed.
|
||||
*/
|
||||
export const selectedAnalysisSetting = (
|
||||
request: number | undefined,
|
||||
override: number | undefined,
|
||||
fallback: number,
|
||||
): {
|
||||
readonly value: number;
|
||||
readonly source: "default" | "runtime" | "request";
|
||||
} => {
|
||||
if (request !== undefined) return { value: request, source: "request" };
|
||||
if (override !== undefined) return { value: override, source: "runtime" };
|
||||
return { value: fallback, source: "default" };
|
||||
};
|
||||
|
||||
/** Project one owner's accepted budget, including the explicit per-call precedence. */
|
||||
export const analysisSettingLimit = (
|
||||
setting: AnalysisSettingId,
|
||||
@@ -334,7 +369,8 @@ export const analysisSettingLimit = (
|
||||
applies_at: definition.appliesAt,
|
||||
agent_editable: true,
|
||||
scope: "rea-runtime",
|
||||
rea_restart_required: false,
|
||||
update_mode: source === "request" ? "parameter" : definition.updateMode,
|
||||
pending_restart: false,
|
||||
...(definition.parameter === undefined
|
||||
? {}
|
||||
: { parameter: definition.parameter }),
|
||||
|
||||
@@ -26,6 +26,14 @@ export interface AnalysisSettingsSnapshot {
|
||||
/** Injected by the runtime owner; reads the accepted operation's immutable snapshot. */
|
||||
export type AnalysisSettingsReader = () => AnalysisSettingsSnapshot;
|
||||
|
||||
/**
|
||||
* Whether `update_analysis_settings` can change the active value.
|
||||
* `runtime` applies inside this process. `restart` stays at the value this
|
||||
* process already started with. A limit fact uses `parameter` when this call
|
||||
* supplied the value; the setting definition itself stays `runtime`.
|
||||
*/
|
||||
export type AnalysisSettingUpdateMode = "runtime" | "restart";
|
||||
|
||||
/** Limit facts supplied by the failing owner, never inferred from error prose. */
|
||||
export interface AnalysisSettingLimit {
|
||||
readonly setting: AnalysisSettingId;
|
||||
@@ -37,7 +45,12 @@ export interface AnalysisSettingLimit {
|
||||
readonly applies_at: "next_operation" | "next_worker" | "next_startup";
|
||||
readonly agent_editable: true;
|
||||
readonly scope: "rea-runtime";
|
||||
readonly rea_restart_required: false;
|
||||
readonly update_mode: AnalysisSettingUpdateMode | "parameter";
|
||||
/**
|
||||
* True only when a configured value is not the one this process is using.
|
||||
* The reported effective value is active, so limit errors leave this false.
|
||||
*/
|
||||
readonly pending_restart: false;
|
||||
readonly required_value?: number;
|
||||
readonly parameter?: string;
|
||||
}
|
||||
|
||||
@@ -86,6 +86,7 @@ const formatSchema = z.enum([
|
||||
"dfuse",
|
||||
"esp-flash-image",
|
||||
"fit",
|
||||
"ntfs-mft",
|
||||
"plist",
|
||||
"javascript",
|
||||
"source-map",
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { z } from "zod";
|
||||
import { MFT_INLINE_HEADER_LIMIT } from "./windows/ntfsMft.js";
|
||||
|
||||
import { jsonRecordSchema } from "./jsonRecord.js";
|
||||
import { jsonValueSchema } from "./jsonValue.js";
|
||||
@@ -299,6 +300,56 @@ export const hexRecordRegionMetadataSchema = z.strictObject({
|
||||
last_line: offset,
|
||||
});
|
||||
|
||||
const ntfsFileReferenceSchema = z.strictObject({
|
||||
recordNumber: offset,
|
||||
sequenceNumber: offset,
|
||||
});
|
||||
const ntfsRecordHeaderSchema = z.strictObject({
|
||||
index: offset,
|
||||
fileOffset: offset,
|
||||
sequenceNumber: offset,
|
||||
hardLinkCount: offset,
|
||||
flags: z.strictObject({
|
||||
raw: offset,
|
||||
names: z.array(z.string()),
|
||||
unknownMask: offset,
|
||||
}),
|
||||
inUse: z.boolean(),
|
||||
directory: z.boolean(),
|
||||
usedBytes: offset,
|
||||
totalBytes: offset,
|
||||
extentValid: z.boolean(),
|
||||
baseRecordReference: ntfsFileReferenceSchema.nullable(),
|
||||
declaredRecordNumber: offset.nullable(),
|
||||
recordNumberMatchesSlot: z.boolean().nullable(),
|
||||
logFileSequenceNumber: z.string().regex(/^(?:0|[1-9][0-9]*)$/u),
|
||||
firstAttributeOffset: offset,
|
||||
nextAttributeId: offset,
|
||||
fixups: z.strictObject({
|
||||
sequenceNumber: offset,
|
||||
entries: offset,
|
||||
verifiedSectors: offset,
|
||||
coveredSectors: offset,
|
||||
mismatchSector: offset.nullable(),
|
||||
malformed: z.boolean(),
|
||||
}),
|
||||
});
|
||||
|
||||
/** NTFS $MFT record-header facts; attributes, names and the USN journal are later steps. */
|
||||
export const ntfsMftMetadataSchema = z.strictObject({
|
||||
kind: z.literal("ntfs-mft"),
|
||||
record_bytes: offset,
|
||||
record_count: offset,
|
||||
in_use_record_count: offset,
|
||||
directory_record_count: offset,
|
||||
extension_record_count: offset,
|
||||
fixup_mismatch_count: offset,
|
||||
fixup_unverified_record_count: offset,
|
||||
truncated_tail_bytes: offset,
|
||||
record_headers: z.array(ntfsRecordHeaderSchema).max(MFT_INLINE_HEADER_LIMIT),
|
||||
omitted_record_headers: offset,
|
||||
});
|
||||
|
||||
/**
|
||||
* U-Boot FIT root facts. Configuration references are declared names; a
|
||||
* reference that names no image is reported as a limitation, not removed.
|
||||
@@ -886,6 +937,7 @@ export const artifactFormatMetadataSchema = z.discriminatedUnion("kind", [
|
||||
espImageSignatureBlockMetadataSchema,
|
||||
uf2MetadataSchema,
|
||||
uf2RegionMetadataSchema,
|
||||
ntfsMftMetadataSchema,
|
||||
dfuSeMetadataSchema,
|
||||
dfuSeElementMetadataSchema,
|
||||
espFlashImageMetadataSchema,
|
||||
@@ -1036,6 +1088,10 @@ const declarationsByKind: {
|
||||
notCompared:
|
||||
"Its familyID and addresses are in its path, family_name is REA's label for the familyID, and its block counts and positions are layout; content comparison covers its bytes.",
|
||||
},
|
||||
"ntfs-mft": {
|
||||
notCompared:
|
||||
"Its fields come from the record headers in the file's own FILE records, which content comparison covers.",
|
||||
},
|
||||
dfuse: {
|
||||
notCompared:
|
||||
"Its fields come from the file's own prefix, target headers and suffix, which content comparison covers.",
|
||||
|
||||
@@ -103,6 +103,7 @@ export const ARTIFACT_FORMAT_IDS = [
|
||||
"dfuse",
|
||||
"esp-flash-image",
|
||||
"fit",
|
||||
"ntfs-mft",
|
||||
"unknown",
|
||||
] as const;
|
||||
|
||||
@@ -124,6 +125,7 @@ const artifactContentFormatSchema = artifactFormatSchema.extract([
|
||||
"uf2",
|
||||
"dfuse",
|
||||
"fit",
|
||||
"ntfs-mft",
|
||||
"mach-o-universal",
|
||||
"mach-o",
|
||||
"elf",
|
||||
|
||||
@@ -191,7 +191,8 @@ export interface ArtifactReaderLimitFact {
|
||||
readonly applies_at: "next_operation";
|
||||
readonly agent_editable: true;
|
||||
readonly can_increase: boolean;
|
||||
readonly rea_restart_required: false;
|
||||
readonly update_mode: "parameter";
|
||||
readonly pending_restart: false;
|
||||
}
|
||||
|
||||
/** Runtime representation ceilings differ from aggregate plaintext-work budgets. */
|
||||
|
||||
@@ -302,7 +302,8 @@ export type BinaryTarget =
|
||||
| "esp-image"
|
||||
| "uf2"
|
||||
| "dfuse"
|
||||
| "fit";
|
||||
| "fit"
|
||||
| "ntfs-mft";
|
||||
})
|
||||
| (BinaryTargetIdentity &
|
||||
NonExecutableMetadata & {
|
||||
|
||||
@@ -7,6 +7,10 @@ import {
|
||||
export interface ConfigurationSettingIssue {
|
||||
readonly setting: string;
|
||||
readonly constraint: string;
|
||||
/** Whether `update_analysis_settings` can change this setting, when the producer owns that fact. */
|
||||
readonly update_mode?: "runtime" | "restart";
|
||||
/** True when the rejected value is configured but not active in this process. */
|
||||
readonly pending_restart?: boolean;
|
||||
}
|
||||
|
||||
/** Rejected settings retained for diagnostics; their values are not. */
|
||||
@@ -24,9 +28,16 @@ export class ConfigurationError extends AnalysisError {
|
||||
constructor(message: string, options?: ConfigurationErrorOptions) {
|
||||
super(message, options);
|
||||
this.remediationAction = options?.remediationAction;
|
||||
this.settings = (options?.settings ?? []).map(
|
||||
({ setting, constraint }) => ({ setting, constraint }),
|
||||
);
|
||||
this.settings = (options?.settings ?? []).map((issue) => ({
|
||||
setting: issue.setting,
|
||||
constraint: issue.constraint,
|
||||
...(issue.update_mode === undefined
|
||||
? {}
|
||||
: { update_mode: issue.update_mode }),
|
||||
...(issue.pending_restart === undefined
|
||||
? {}
|
||||
: { pending_restart: issue.pending_restart }),
|
||||
}));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -52,6 +52,7 @@ export const evidenceSubjectFormatSchema = z.enum([
|
||||
"dfuse",
|
||||
"esp-flash-image",
|
||||
"fit",
|
||||
"ntfs-mft",
|
||||
]);
|
||||
|
||||
/** Format-owned architecture declarations shared by Evidence and persisted executions. */
|
||||
|
||||
@@ -137,7 +137,8 @@ describe("child-process options detection", () => {
|
||||
["negative-zero", `spawn("tool", [], { env: -0 })`, false],
|
||||
["bigint-zero", `spawn("tool", [], { env: 0n })`, false],
|
||||
["zero-float", `spawn("tool", [], { env: 0.0 })`, false],
|
||||
["nan", `spawn("tool", [], { env: NaN })`, true],
|
||||
["nan", `spawn("tool", [], { env: NaN })`, false],
|
||||
["shadowed-nan", `((NaN) => spawn("tool", [], { env: NaN }))({})`, true],
|
||||
["empty-string", `spawn("tool", [], { env: "" })`, false],
|
||||
["false", `spawn("tool", [], { env: false })`, false],
|
||||
["empty-object", `spawn("tool", [], { env: {} })`, true],
|
||||
|
||||
@@ -81,7 +81,7 @@ export const collectJavaScriptSemanticChildProcesses = (
|
||||
!rejectsOptionsShape(candidate, context, state),
|
||||
);
|
||||
const childProcessSpawns = candidates.map((candidate) =>
|
||||
immutableSpawn(candidate, context),
|
||||
immutableSpawn(candidate, context, state),
|
||||
);
|
||||
return {
|
||||
childProcessSpawns,
|
||||
@@ -146,6 +146,7 @@ const collectSpawns = (
|
||||
const immutableSpawn = (
|
||||
candidate: SpawnCandidate,
|
||||
context: ChildSpawnContext,
|
||||
state: JavaScriptSemanticAnalysisState,
|
||||
): JavaScriptSemanticChildProcessSpawn => {
|
||||
const { options, argvCount } = spawnArguments(candidate, context);
|
||||
const command = dataEffectLiteralString(candidate.node.arguments[0]);
|
||||
@@ -161,7 +162,7 @@ const immutableSpawn = (
|
||||
projectedResultBindings: candidate.projectedResultBindings,
|
||||
command,
|
||||
argvCount,
|
||||
environmentSupplied: environmentSupplied(options, context),
|
||||
environmentSupplied: environmentSupplied(options, context, state),
|
||||
stdioMode: stdioMode(options, candidate.method, context),
|
||||
resolution: command === null ? "partial" : "complete",
|
||||
};
|
||||
@@ -443,11 +444,17 @@ const spawnArguments = (
|
||||
const environmentSupplied = (
|
||||
options: t.Node | undefined,
|
||||
context: ChildSpawnContext,
|
||||
state: JavaScriptSemanticAnalysisState,
|
||||
): boolean => {
|
||||
const lookup = staticObjectProperty(options, "env", context.isNullish);
|
||||
if (lookup.status !== "explicit") return false;
|
||||
const value = unwrapJavaScriptExpression(lookup.value).node;
|
||||
if (context.isNullish(value)) return false;
|
||||
if (
|
||||
t.isIdentifier(value, { name: "NaN" }) &&
|
||||
isUnshadowedGlobal(value, state, "NaN")
|
||||
)
|
||||
return false;
|
||||
return staticJavaScriptTruthiness(value) !== false;
|
||||
};
|
||||
|
||||
|
||||
@@ -41,7 +41,21 @@ export interface JavaScriptSourceAnalysisFailure {
|
||||
export interface JavaScriptSourceAnalysisLimits {
|
||||
readonly semanticPrecision?: JavaScriptSemanticPrecisionInput;
|
||||
readonly heapMb: number;
|
||||
/** Where `heapMb` was selected. Absent when the caller did not record it. */
|
||||
readonly heapSource?:
|
||||
| "default"
|
||||
| "setting"
|
||||
| "runtime"
|
||||
| "request"
|
||||
| undefined;
|
||||
readonly timeoutMs: number;
|
||||
/** Where `timeoutMs` was selected. Absent when the caller did not record it. */
|
||||
readonly timeoutSource?:
|
||||
| "default"
|
||||
| "setting"
|
||||
| "runtime"
|
||||
| "request"
|
||||
| undefined;
|
||||
}
|
||||
|
||||
/** Sequential source-analysis authority with independently verified cleanup. */
|
||||
|
||||
@@ -83,6 +83,7 @@ export const liveBinaryTargetSummaryVariants = [
|
||||
"uf2",
|
||||
"dfuse",
|
||||
"fit",
|
||||
"ntfs-mft",
|
||||
]),
|
||||
architecture: z.null(),
|
||||
}),
|
||||
|
||||
@@ -0,0 +1,37 @@
|
||||
import { readFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { expect, it } from "vitest";
|
||||
import { ANALYSIS_SETTINGS } from "./analysisSettingDefinitions.js";
|
||||
import { NUMERIC_LIMIT_REGISTRY } from "./numericLimitRegistry.js";
|
||||
|
||||
const inventory = readFileSync(
|
||||
join(import.meta.dirname, "../../docs/design/rea-limit-inventory.md"),
|
||||
"utf8",
|
||||
);
|
||||
|
||||
it("gives every numeric exception a reason and a matching inventory row", () => {
|
||||
const names = NUMERIC_LIMIT_REGISTRY.map((entry) => entry.name);
|
||||
expect(new Set(names).size).toBe(names.length);
|
||||
const runtimeEnvironments = new Set<string>(
|
||||
Object.values(ANALYSIS_SETTINGS).flatMap((definition) =>
|
||||
"environment" in definition && definition.environment !== undefined
|
||||
? [definition.environment]
|
||||
: [],
|
||||
),
|
||||
);
|
||||
for (const entry of NUMERIC_LIMIT_REGISTRY) {
|
||||
expect(entry.update_mode).toBe("restart");
|
||||
expect(entry.reason.trim().length).toBeGreaterThanOrEqual(40);
|
||||
expect(entry.read).toBe(
|
||||
entry.policy === "per-call" ? "per-call" : "startup",
|
||||
);
|
||||
expect(runtimeEnvironments.has(entry.name)).toBe(false);
|
||||
expect(inventory).toContain(`\`${entry.name}\``);
|
||||
}
|
||||
expect(
|
||||
NUMERIC_LIMIT_REGISTRY.filter((entry) => entry.policy === "restart-only"),
|
||||
).toHaveLength(15);
|
||||
expect(
|
||||
NUMERIC_LIMIT_REGISTRY.filter((entry) => entry.policy === "per-call"),
|
||||
).toHaveLength(11);
|
||||
});
|
||||
@@ -0,0 +1,230 @@
|
||||
/**
|
||||
* Numeric `REA_` limits that are not runtime analysis settings.
|
||||
*
|
||||
* The static check in `scripts/verify/rea-numeric-limits.mjs` fails when a
|
||||
* numeric `REA_` limit in `src` is absent from both this list and
|
||||
* `ANALYSIS_SETTINGS`. `restart-only` is the durable exception. `per-call`
|
||||
* names a limit still read from the launch environment on each use; the
|
||||
* runtime migration deletes that entry instead of leaving it here.
|
||||
*
|
||||
* Narrative bounds, owners, and the non-numeric settings live in
|
||||
* `docs/design/rea-limit-inventory.md`.
|
||||
*/
|
||||
export interface NumericLimitRegistryEntry {
|
||||
readonly name: string;
|
||||
readonly update_mode: "restart";
|
||||
readonly policy: "restart-only" | "per-call";
|
||||
readonly read: "startup" | "per-call";
|
||||
readonly reason: string;
|
||||
}
|
||||
|
||||
export const NUMERIC_LIMIT_REGISTRY = [
|
||||
{
|
||||
name: "REA_BRIDGE_MAX_REQUEST_BYTES",
|
||||
update_mode: "restart",
|
||||
policy: "per-call",
|
||||
read: "per-call",
|
||||
reason:
|
||||
"The Hopper, Ghidra, and Cutter clients measure each request against this ceiling. The value is still taken from the launch environment, so a running process does not see a new value until the runtime migration removes this entry.",
|
||||
},
|
||||
{
|
||||
name: "REA_DEBUGGER_COMMAND_DEADLINE_MS",
|
||||
update_mode: "restart",
|
||||
policy: "per-call",
|
||||
read: "per-call",
|
||||
reason:
|
||||
"GDB and Rizin apply this deadline to each debugger command. The session manager currently copies it at construction from the launch environment; the runtime migration reads the accepted operation snapshot instead.",
|
||||
},
|
||||
{
|
||||
name: "REA_DEBUGGER_MAX_COMMAND_OUTPUT_BYTES",
|
||||
update_mode: "restart",
|
||||
policy: "per-call",
|
||||
read: "per-call",
|
||||
reason:
|
||||
"Each debugger command retains output up to this ceiling. The session manager currently copies it at construction from the launch environment; the runtime migration reads the accepted operation snapshot instead.",
|
||||
},
|
||||
{
|
||||
name: "REA_DEBUGGER_MAX_PENDING_COMMANDS",
|
||||
update_mode: "restart",
|
||||
policy: "restart-only",
|
||||
read: "startup",
|
||||
reason:
|
||||
"Caps the pending-command queue created with the debugger session manager. It is pool capacity for that manager, not a budget checked independently on each command.",
|
||||
},
|
||||
{
|
||||
name: "REA_DEBUGGER_MAX_SESSIONS",
|
||||
update_mode: "restart",
|
||||
policy: "restart-only",
|
||||
read: "startup",
|
||||
reason:
|
||||
"Caps how many debugger sessions the session manager created at startup will admit. Changing the pool size of a manager that already exists requires a new REA process.",
|
||||
},
|
||||
{
|
||||
name: "REA_HOPPER_CALLER_SCAN_BUDGET_MS",
|
||||
update_mode: "restart",
|
||||
policy: "per-call",
|
||||
read: "per-call",
|
||||
reason:
|
||||
"Forward caller reconciliation reads this budget for each scan. It is still resolved from the launch environment; the runtime migration reads the accepted operation snapshot.",
|
||||
},
|
||||
{
|
||||
name: "REA_HOPPER_DEMO_DIALOG_TIMEOUT_MS",
|
||||
update_mode: "restart",
|
||||
policy: "restart-only",
|
||||
read: "startup",
|
||||
reason:
|
||||
"The Linux private-display adapter reads this watch from the process environment when that helper starts. A running adapter does not observe a value stored only in the REA process.",
|
||||
},
|
||||
{
|
||||
name: "REA_JADX_ACTIVE_PROCESSOR_COUNT",
|
||||
update_mode: "restart",
|
||||
policy: "restart-only",
|
||||
read: "startup",
|
||||
reason:
|
||||
"Passed to the JVM as -XX:ActiveProcessorCount when the JADX worker process starts. A running JVM does not adopt a new processor count.",
|
||||
},
|
||||
{
|
||||
name: "REA_JADX_HEAP_MIB",
|
||||
update_mode: "restart",
|
||||
policy: "restart-only",
|
||||
read: "startup",
|
||||
reason:
|
||||
"Passed to the JVM as -Xmx when the JADX worker process starts. A running JVM does not adopt a new heap size.",
|
||||
},
|
||||
{
|
||||
name: "REA_JEB_MCP_TIMEOUT_MS",
|
||||
update_mode: "restart",
|
||||
policy: "per-call",
|
||||
read: "per-call",
|
||||
reason:
|
||||
"Bounds each JEB tools/call and catalog discovery. The connection currently captures the launch value when it is created; the runtime migration reads the accepted snapshot for each request.",
|
||||
},
|
||||
{
|
||||
name: "REA_KUNA_CACHE_MAX_BYTES",
|
||||
update_mode: "restart",
|
||||
policy: "restart-only",
|
||||
read: "startup",
|
||||
reason:
|
||||
"Sizes the Kuna project cache when that cache is constructed for the provider. An existing cache does not resize from a later value.",
|
||||
},
|
||||
{
|
||||
name: "REA_KUNA_JOBS",
|
||||
update_mode: "restart",
|
||||
policy: "restart-only",
|
||||
read: "startup",
|
||||
reason:
|
||||
"Selects the Kuna worker-pool width when the provider builds its run slots. The pool is not resized for a process that is already running.",
|
||||
},
|
||||
{
|
||||
name: "REA_KUNA_PROJECT_TIMEOUT_MS",
|
||||
update_mode: "restart",
|
||||
policy: "per-call",
|
||||
read: "per-call",
|
||||
reason:
|
||||
"Bounds a Kuna project export and the whole-program scans that use that deadline. The configuration object still comes from the launch environment; the runtime migration reads the accepted snapshot at each use.",
|
||||
},
|
||||
{
|
||||
name: "REA_KUNA_TIMEOUT_MS",
|
||||
update_mode: "restart",
|
||||
policy: "per-call",
|
||||
read: "per-call",
|
||||
reason:
|
||||
"Bounds one Kuna function decompile. The configuration object still comes from the launch environment; the runtime migration reads the accepted snapshot at each use.",
|
||||
},
|
||||
{
|
||||
name: "REA_MCP_DOCTOR_DEADLINE_MS",
|
||||
update_mode: "restart",
|
||||
policy: "restart-only",
|
||||
read: "startup",
|
||||
reason:
|
||||
"Bounds the MCP doctor phase while the server is starting. It shapes server startup, not a later analysis call.",
|
||||
},
|
||||
{
|
||||
name: "REA_MCP_FIRST_CATALOG_BUDGET_MS",
|
||||
update_mode: "restart",
|
||||
policy: "restart-only",
|
||||
read: "startup",
|
||||
reason:
|
||||
"Bounds the first MCP catalog phase while the server is starting. It shapes server startup, not a later analysis call.",
|
||||
},
|
||||
{
|
||||
name: "REA_MCP_INITIALIZE_BUDGET_MS",
|
||||
update_mode: "restart",
|
||||
policy: "restart-only",
|
||||
read: "startup",
|
||||
reason:
|
||||
"Bounds the MCP initialize phase while the server is starting. It shapes server startup, not a later analysis call.",
|
||||
},
|
||||
{
|
||||
name: "REA_MCP_MAX_INPUT_ELEMENTS",
|
||||
update_mode: "restart",
|
||||
policy: "restart-only",
|
||||
read: "startup",
|
||||
reason:
|
||||
"Sizes argument admission for the MCP server created at process start. A running server does not rebuild that parser for a new allowance.",
|
||||
},
|
||||
{
|
||||
name: "REA_MCP_MAX_RESPONSE_BYTES",
|
||||
update_mode: "restart",
|
||||
policy: "restart-only",
|
||||
read: "startup",
|
||||
reason:
|
||||
"Sizes the MCP stdio receive buffer when the server process starts. A running transport and the paired client buffer do not adopt a new ceiling.",
|
||||
},
|
||||
{
|
||||
name: "REA_MCP_TOOLS_LIST_BUDGET_MS",
|
||||
update_mode: "restart",
|
||||
policy: "restart-only",
|
||||
read: "startup",
|
||||
reason:
|
||||
"Bounds the MCP tools/list phase while the server is starting. It shapes server startup, not a later analysis call.",
|
||||
},
|
||||
{
|
||||
name: "REA_PWNTOOLS_MAX_INPUT_BYTES",
|
||||
update_mode: "restart",
|
||||
policy: "per-call",
|
||||
read: "per-call",
|
||||
reason:
|
||||
"Default artifact-size admission for a pwntools decode. A per-call producer_limits.input_bytes override already exists; the launch default is still read from the environment and the runtime migration moves that default into the snapshot.",
|
||||
},
|
||||
{
|
||||
name: "REA_RECONNECT_TIMEOUT_SECONDS",
|
||||
update_mode: "restart",
|
||||
policy: "restart-only",
|
||||
read: "startup",
|
||||
reason:
|
||||
"Fixed 30-second value written into the Hopper bridge helper from HOPPER_SHUTDOWN_TIMEOUT_MS. It is not a caller-selected launch setting.",
|
||||
},
|
||||
{
|
||||
name: "REA_SWIFTC_COMPILE_TIMEOUT_MS",
|
||||
update_mode: "restart",
|
||||
policy: "restart-only",
|
||||
read: "startup",
|
||||
reason:
|
||||
"Captured when the Darwin ownership-helper reader is constructed at process start. That helper compile uses the captured deadline, so a later value needs a new REA process.",
|
||||
},
|
||||
{
|
||||
name: "REA_WINDOWS_CHANGE_TIME_DRIFT_MS",
|
||||
update_mode: "restart",
|
||||
policy: "per-call",
|
||||
read: "per-call",
|
||||
reason:
|
||||
"Each Windows ChangeTime observation compares the file against this drift window. The observation still parses the launch environment; the runtime migration reads the accepted operation snapshot.",
|
||||
},
|
||||
{
|
||||
name: "REA_WINDOWS_CHANGE_TIME_SETTLE_MAX_POLLS",
|
||||
update_mode: "restart",
|
||||
policy: "per-call",
|
||||
read: "per-call",
|
||||
reason:
|
||||
"Each Windows ChangeTime settle loop stops after this many polls. The observation still parses the launch environment; the runtime migration reads the accepted operation snapshot.",
|
||||
},
|
||||
{
|
||||
name: "REA_WINDOWS_CHANGE_TIME_SETTLE_POLL_MS",
|
||||
update_mode: "restart",
|
||||
policy: "per-call",
|
||||
read: "per-call",
|
||||
reason:
|
||||
"Each Windows ChangeTime settle loop waits this long between reads. The observation still parses the launch environment; the runtime migration reads the accepted operation snapshot.",
|
||||
},
|
||||
] as const satisfies readonly NumericLimitRegistryEntry[];
|
||||
@@ -157,7 +157,6 @@ const extractImports = (
|
||||
ReferenceSourceRequireCall
|
||||
>,
|
||||
): void => {
|
||||
let unresolvedRequireTarget = false;
|
||||
traverseJavaScriptAst(ast.program, {
|
||||
enter: (node) => {
|
||||
if (
|
||||
@@ -198,14 +197,8 @@ const extractImports = (
|
||||
) {
|
||||
const requireCall = requireCalls.get(node);
|
||||
if (requireCall === undefined) return;
|
||||
const target = requireCall.arguments?.[0];
|
||||
if (target === undefined) return;
|
||||
const literal = readExactJavaScriptLiteral(target);
|
||||
if (!literal.found) {
|
||||
unresolvedRequireTarget = true;
|
||||
return;
|
||||
}
|
||||
if (typeof literal.value !== "string") return;
|
||||
const specifier = exactStringModuleName(requireCall.arguments?.[0]);
|
||||
if (specifier === undefined) return;
|
||||
if (
|
||||
requireCall.limitation !== undefined &&
|
||||
!limitations.includes(requireCall.limitation)
|
||||
@@ -214,7 +207,7 @@ const extractImports = (
|
||||
const uncertainResolution = requireCall.resolutionBase === "unknown";
|
||||
appendRelationship(relationships, limitations, {
|
||||
fromPath: from_path,
|
||||
to: literal.value,
|
||||
to: specifier,
|
||||
kind: "requires",
|
||||
parseState: uncertainResolution ? "partial" : "parsed",
|
||||
...(uncertainResolution ? { resolution: "unknown" } : {}),
|
||||
@@ -228,10 +221,6 @@ const extractImports = (
|
||||
}
|
||||
},
|
||||
});
|
||||
if (unresolvedRequireTarget)
|
||||
limitations.push(
|
||||
`A CommonJS module target in ${from_path} could not be determined statically.`,
|
||||
);
|
||||
};
|
||||
|
||||
const parseWithBabel = (
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { posix } from "node:path";
|
||||
import { err, ok, type Result } from "./result.js";
|
||||
|
||||
export interface ReferenceSourcePathLookup {
|
||||
@@ -5,6 +6,8 @@ export interface ReferenceSourcePathLookup {
|
||||
readonly has: (path: string) => boolean;
|
||||
readonly resolve: (path: string) => string | undefined;
|
||||
readonly resolveCandidates: (paths: readonly string[]) => string | undefined;
|
||||
/** Resolve a path to the closest recorded parent identity. */
|
||||
readonly resolveAncestor: (path: string) => string | undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -32,6 +35,17 @@ export const createReferenceSourcePathLookup = (
|
||||
resolveCandidates: (candidates) =>
|
||||
candidates.find((path) => exact.has(path)) ??
|
||||
candidates.map(equivalent).find((path) => path !== undefined),
|
||||
resolveAncestor: (path) => {
|
||||
for (
|
||||
let candidate = path;
|
||||
candidate !== "." && candidate !== "/";
|
||||
candidate = posix.dirname(candidate)
|
||||
) {
|
||||
const found = resolve(candidate);
|
||||
if (found !== undefined) return found;
|
||||
}
|
||||
return undefined;
|
||||
},
|
||||
};
|
||||
};
|
||||
|
||||
|
||||
@@ -75,14 +75,9 @@ export const referenceSourceRequireCalls = (
|
||||
!t.isNewExpression(node)
|
||||
)
|
||||
return;
|
||||
const callee = unwrapJavaScriptExpression(node.callee).node;
|
||||
if (
|
||||
!t.isCallExpression(callee) &&
|
||||
!t.isOptionalCallExpression(callee) &&
|
||||
!hasIdentifierReceiver(callee)
|
||||
)
|
||||
return;
|
||||
if (callReceiverIdentifier(node.callee) === undefined) return;
|
||||
let fact = semanticRequireLoaderAt(node, index.bindings, factoryBindings);
|
||||
const callee = unwrapJavaScriptExpression(node.callee).node;
|
||||
if (
|
||||
fact === "create-require" ||
|
||||
((t.isMemberExpression(callee) ||
|
||||
@@ -158,7 +153,12 @@ export const referenceSourceRequireCalls = (
|
||||
});
|
||||
},
|
||||
});
|
||||
if (candidates.some(({ call }) => hasMemberCallee(call)))
|
||||
if (
|
||||
candidates.some(
|
||||
({ call }) =>
|
||||
!t.isIdentifier(unwrapJavaScriptExpression(call.callee).node),
|
||||
)
|
||||
)
|
||||
completeSemanticSteps(index.indexMemberMutationFactsSteps());
|
||||
return new Map(
|
||||
candidates.map(
|
||||
@@ -169,7 +169,8 @@ export const referenceSourceRequireCalls = (
|
||||
arguments: callArguments,
|
||||
...(limitation === undefined ? {} : { limitation }),
|
||||
resolutionBase:
|
||||
hasMemberCallee(call) && hasPriorMemberMutation(index, call)
|
||||
!t.isIdentifier(unwrapJavaScriptExpression(call.callee).node) &&
|
||||
hasPriorMemberMutation(index, call)
|
||||
? "unknown"
|
||||
: resolutionBase,
|
||||
},
|
||||
@@ -219,17 +220,15 @@ const hasPriorMemberMutation = (
|
||||
);
|
||||
};
|
||||
|
||||
const hasMemberCallee = (call: ReferenceSourceRequireInvocation): boolean => {
|
||||
const callee = unwrapJavaScriptExpression(call.callee).node;
|
||||
return t.isMemberExpression(callee) || t.isOptionalMemberExpression(callee);
|
||||
};
|
||||
|
||||
const hasIdentifierReceiver = (callee: t.Node): boolean => {
|
||||
let expression = callee;
|
||||
const callReceiverIdentifier = (
|
||||
callee: t.Node | null | undefined,
|
||||
): t.Identifier | undefined => {
|
||||
if (!t.isNode(callee)) return undefined;
|
||||
let expression = unwrapJavaScriptExpression(callee).node;
|
||||
while (
|
||||
t.isMemberExpression(expression) ||
|
||||
t.isOptionalMemberExpression(expression)
|
||||
)
|
||||
expression = unwrapJavaScriptExpression(expression.object).node;
|
||||
return t.isIdentifier(expression);
|
||||
return t.isIdentifier(expression) ? expression : undefined;
|
||||
};
|
||||
@@ -79,7 +79,8 @@ export const webNetworkCaptureBudgetFacts = (
|
||||
? MAX_WEB_NETWORK_CAPTURE_NESTING_DEPTH
|
||||
: Number.MAX_SAFE_INTEGER,
|
||||
applies_at: "next_operation",
|
||||
rea_restart_required: false,
|
||||
update_mode: "parameter" as const,
|
||||
pending_restart: false as const,
|
||||
};
|
||||
};
|
||||
|
||||
|
||||
@@ -0,0 +1,478 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import {
|
||||
MFT_DEFAULT_RECORD_BYTES,
|
||||
MFT_MAX_FILE_BYTES,
|
||||
createMftDecoder,
|
||||
parseMftImage,
|
||||
} from "./ntfsMft.js";
|
||||
|
||||
interface RecordOptions {
|
||||
readonly sequenceNumber?: number;
|
||||
readonly hardLinkCount?: number;
|
||||
readonly flags?: number;
|
||||
readonly baseRecordReference?: bigint;
|
||||
readonly recordBytes?: number;
|
||||
/** Write a wrong trailer value into this sector, leaving the others correct. */
|
||||
readonly corruptSector?: number;
|
||||
/** Truncate the update sequence array to this many entries; 0 omits it. */
|
||||
readonly usaEntries?: number;
|
||||
readonly usaOffset?: number;
|
||||
}
|
||||
|
||||
/** One FILE record with a zeroed body and fixups applied the way NTFS writes them. */
|
||||
const recordOf = (options: RecordOptions = {}): Uint8Array => {
|
||||
const recordBytes = options.recordBytes ?? MFT_DEFAULT_RECORD_BYTES;
|
||||
const record = new Uint8Array(recordBytes);
|
||||
const view = new DataView(record.buffer);
|
||||
record.set(Buffer.from("FILE", "latin1"), 0);
|
||||
const sequenceNumber = options.sequenceNumber ?? 1;
|
||||
const sectors = recordBytes / 512;
|
||||
|
||||
const usaEntries = options.usaEntries ?? sectors + 1;
|
||||
if (usaEntries > 0) {
|
||||
const usaOffset = options.usaOffset ?? 48;
|
||||
view.setUint16(4, usaOffset, true);
|
||||
view.setUint16(6, usaEntries, true);
|
||||
view.setUint16(usaOffset, sequenceNumber, true);
|
||||
for (let sector = 0; sector < sectors; sector += 1) {
|
||||
const trailer = sector * 512 + 510;
|
||||
// A saved trailer is what the sector held before the fixup replaced it.
|
||||
view.setUint16(usaOffset + 2 * (sector + 1), 0xa5a5, true);
|
||||
view.setUint16(
|
||||
trailer,
|
||||
sector >= usaEntries - 1
|
||||
? 0xa5a5
|
||||
: sector === options.corruptSector
|
||||
? 0x1234
|
||||
: sequenceNumber,
|
||||
true,
|
||||
);
|
||||
}
|
||||
} else {
|
||||
view.setUint16(4, 0, true);
|
||||
view.setUint16(6, 0, true);
|
||||
}
|
||||
|
||||
view.setUint16(16, sequenceNumber, true);
|
||||
view.setUint16(18, options.hardLinkCount ?? 1, true);
|
||||
const firstAttributeOffset =
|
||||
Math.ceil((48 + Math.max(0, usaEntries) * 2) / 8) * 8;
|
||||
view.setUint16(20, firstAttributeOffset, true);
|
||||
view.setUint16(22, options.flags ?? 0x0001, true);
|
||||
view.setUint32(24, firstAttributeOffset + 8, true);
|
||||
view.setUint32(firstAttributeOffset, 0xffff_ffff, true);
|
||||
view.setUint32(28, recordBytes, true);
|
||||
view.setBigUint64(32, options.baseRecordReference ?? 0n, true);
|
||||
view.setUint16(40, 0, true);
|
||||
return record;
|
||||
};
|
||||
|
||||
const streamOf = (...records: readonly Uint8Array[]): Uint8Array => {
|
||||
const total = records.reduce((sum, record) => sum + record.length, 0);
|
||||
const stream = new Uint8Array(total);
|
||||
let offset = 0;
|
||||
for (const record of records) {
|
||||
stream.set(record, offset);
|
||||
const view = new DataView(stream.buffer, offset, record.length);
|
||||
if (view.getUint16(4, true) >= 48)
|
||||
view.setUint32(44, offset / record.length, true);
|
||||
offset += record.length;
|
||||
}
|
||||
return stream;
|
||||
};
|
||||
|
||||
describe("MFT header facts", () => {
|
||||
it("inventories a clean record's header and verified fixups", () => {
|
||||
const parsed = parseMftImage(streamOf(recordOf({ sequenceNumber: 5 })));
|
||||
expect(parsed.ok).toBe(true);
|
||||
if (!parsed.ok) return;
|
||||
expect(parsed.value.metadata).toMatchObject({
|
||||
kind: "ntfs-mft",
|
||||
record_bytes: MFT_DEFAULT_RECORD_BYTES,
|
||||
record_count: 1,
|
||||
in_use_record_count: 1,
|
||||
directory_record_count: 0,
|
||||
extension_record_count: 0,
|
||||
fixup_mismatch_count: 0,
|
||||
fixup_unverified_record_count: 0,
|
||||
truncated_tail_bytes: 0,
|
||||
});
|
||||
expect(parsed.value.records[0]).toMatchObject({
|
||||
index: 0,
|
||||
fileOffset: 0,
|
||||
sequenceNumber: 5,
|
||||
inUse: true,
|
||||
directory: false,
|
||||
baseRecordReference: null,
|
||||
firstAttributeOffset: 56,
|
||||
fixups: {
|
||||
sequenceNumber: 5,
|
||||
verifiedSectors: 2,
|
||||
coveredSectors: 2,
|
||||
mismatchSector: null,
|
||||
malformed: false,
|
||||
},
|
||||
});
|
||||
expect(parsed.value.limitations).toEqual([]);
|
||||
});
|
||||
|
||||
it("identifies unrecognized flags without losing their source bits", () => {
|
||||
const parsed = parseMftImage(recordOf({ flags: 0x8001 }));
|
||||
if (!parsed.ok) throw new Error(parsed.error.message);
|
||||
expect(parsed.value.metadata.record_headers[0]?.flags).toEqual({
|
||||
raw: 0x8001,
|
||||
names: ["in-use"],
|
||||
unknownMask: 0x8000,
|
||||
});
|
||||
});
|
||||
|
||||
it.each([0, 49, 64, 72])(
|
||||
"reports an invalid first attribute offset %i",
|
||||
(offset) => {
|
||||
const record = recordOf();
|
||||
new DataView(record.buffer).setUint16(20, offset, true);
|
||||
const parsed = parseMftImage(record);
|
||||
if (!parsed.ok) throw new Error(parsed.error.message);
|
||||
expect(parsed.value.records[0]?.extentValid).toBe(false);
|
||||
expect(parsed.value.limitations.join(" ")).toContain("inconsistent");
|
||||
},
|
||||
);
|
||||
|
||||
it("reports a declared record number that contradicts its stream position", () => {
|
||||
const bytes = streamOf(recordOf(), recordOf());
|
||||
new DataView(bytes.buffer).setUint32(1024 + 44, 0, true);
|
||||
const parsed = parseMftImage(bytes);
|
||||
if (!parsed.ok) throw new Error(parsed.error.message);
|
||||
expect(parsed.value.records[1]).toMatchObject({
|
||||
index: 1,
|
||||
declaredRecordNumber: 0,
|
||||
recordNumberMatchesSlot: false,
|
||||
});
|
||||
expect(parsed.value.limitations.join(" ")).toContain(
|
||||
"disagrees with their stream slot",
|
||||
);
|
||||
});
|
||||
|
||||
it("keeps the inline header window separate from the parsed record count", () => {
|
||||
const parsed = parseMftImage(
|
||||
streamOf(...Array.from({ length: 33 }, () => recordOf())),
|
||||
);
|
||||
if (!parsed.ok) throw new Error(parsed.error.message);
|
||||
expect(parsed.value.metadata).toMatchObject({
|
||||
record_count: 33,
|
||||
omitted_record_headers: 1,
|
||||
});
|
||||
expect(parsed.value.metadata.record_headers).toHaveLength(32);
|
||||
expect(parsed.value.records).toHaveLength(33);
|
||||
expect(parsed.value.limitations).toEqual([]);
|
||||
});
|
||||
|
||||
it("preserves a log sequence number above the safe integer range", () => {
|
||||
const record = recordOf();
|
||||
new DataView(record.buffer).setBigUint64(8, 9007199254740993n, true);
|
||||
const parsed = parseMftImage(record);
|
||||
if (!parsed.ok) throw new Error(parsed.error.message);
|
||||
expect(parsed.value.records[0]?.logFileSequenceNumber).toBe(
|
||||
"9007199254740993",
|
||||
);
|
||||
});
|
||||
|
||||
it("verifies the older FILE header without reading its fixup words as a record number", () => {
|
||||
const oldRecord = recordOf({ usaOffset: 42 });
|
||||
const newRecord = recordOf();
|
||||
new DataView(newRecord.buffer).setUint32(44, 1, true);
|
||||
|
||||
const parsed = parseMftImage(streamOf(oldRecord, newRecord));
|
||||
if (!parsed.ok) throw new Error(parsed.error.message);
|
||||
expect(parsed.value.records).toMatchObject([
|
||||
{
|
||||
declaredRecordNumber: null,
|
||||
fixups: { malformed: false, verifiedSectors: 2 },
|
||||
},
|
||||
{
|
||||
declaredRecordNumber: 1,
|
||||
fixups: { malformed: false, verifiedSectors: 2 },
|
||||
},
|
||||
]);
|
||||
expect(parsed.value.limitations).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("MFT fixup coverage", () => {
|
||||
it.each([0, 0xffff])(
|
||||
"leaves reserved USA sequence %i unverified",
|
||||
(sequenceNumber) => {
|
||||
const parsed = parseMftImage(recordOf({ sequenceNumber }));
|
||||
if (!parsed.ok) throw new Error(parsed.error.message);
|
||||
expect(parsed.value.records[0]?.fixups).toMatchObject({
|
||||
sequenceNumber,
|
||||
verifiedSectors: 0,
|
||||
malformed: true,
|
||||
});
|
||||
expect(parsed.value.metadata.fixup_unverified_record_count).toBe(1);
|
||||
expect(parsed.value.limitations.join(" ")).toContain(
|
||||
"invalid or incomplete",
|
||||
);
|
||||
},
|
||||
);
|
||||
|
||||
it("keeps a record whose sector trailer is not the sequence number, and reports it", () => {
|
||||
const parsed = parseMftImage(
|
||||
streamOf(recordOf(), recordOf({ sequenceNumber: 9, corruptSector: 1 })),
|
||||
);
|
||||
expect(parsed.ok).toBe(true);
|
||||
if (!parsed.ok) return;
|
||||
expect(parsed.value.records[1]?.fixups).toMatchObject({
|
||||
sequenceNumber: 9,
|
||||
verifiedSectors: 1,
|
||||
mismatchSector: 1,
|
||||
});
|
||||
expect(parsed.value.metadata.fixup_mismatch_count).toBe(1);
|
||||
expect(parsed.value.limitations.join(" ")).toMatch(
|
||||
/1 record\(s\) hold a sector trailer/u,
|
||||
);
|
||||
});
|
||||
|
||||
it("treats a missing update sequence array as unverified rather than clean", () => {
|
||||
const parsed = parseMftImage(streamOf(recordOf({ usaEntries: 0 })));
|
||||
expect(parsed.ok).toBe(true);
|
||||
if (!parsed.ok) return;
|
||||
expect(parsed.value.records[0]?.fixups).toMatchObject({
|
||||
malformed: true,
|
||||
mismatchSector: null,
|
||||
coveredSectors: 0,
|
||||
});
|
||||
expect(parsed.value.metadata.fixup_mismatch_count).toBe(0);
|
||||
expect(parsed.value.metadata.fixup_unverified_record_count).toBe(1);
|
||||
expect(parsed.value.limitations.join(" ")).toMatch(
|
||||
/invalid or incomplete update sequence array/u,
|
||||
);
|
||||
});
|
||||
|
||||
it.each([
|
||||
[48, 4],
|
||||
[49, 3],
|
||||
[508, 3],
|
||||
])(
|
||||
"rejects a malformed update sequence array at %i with %i entries",
|
||||
(offset, count) => {
|
||||
const bytes = recordOf();
|
||||
const view = new DataView(bytes.buffer);
|
||||
view.setUint16(4, offset, true);
|
||||
view.setUint16(6, count, true);
|
||||
|
||||
const parsed = parseMftImage(bytes);
|
||||
if (!parsed.ok) throw new Error(parsed.error.message);
|
||||
expect(parsed.value.records[0]?.fixups).toMatchObject({
|
||||
malformed: true,
|
||||
verifiedSectors: 0,
|
||||
});
|
||||
expect(parsed.value.metadata.fixup_unverified_record_count).toBe(1);
|
||||
},
|
||||
);
|
||||
|
||||
it("counts uncovered sectors independently of a mismatched trailer", () => {
|
||||
const parsed = parseMftImage(
|
||||
recordOf({ recordBytes: 4096, usaEntries: 3, corruptSector: 0 }),
|
||||
);
|
||||
if (!parsed.ok) throw new Error(parsed.error.message);
|
||||
expect(parsed.value.metadata).toMatchObject({
|
||||
fixup_mismatch_count: 1,
|
||||
fixup_unverified_record_count: 1,
|
||||
});
|
||||
expect(parsed.value.limitations.join(" ")).toContain(
|
||||
"invalid or incomplete",
|
||||
);
|
||||
});
|
||||
|
||||
it("reports an array that covers fewer sectors than the record has", () => {
|
||||
// A 4 KiB record holds eight sectors; a three-entry array leaves five unverified.
|
||||
const parsed = parseMftImage(
|
||||
streamOf(recordOf({ recordBytes: 4096, usaEntries: 3 })),
|
||||
);
|
||||
expect(parsed.ok).toBe(true);
|
||||
if (!parsed.ok) return;
|
||||
expect(parsed.value.records[0]?.fixups).toMatchObject({
|
||||
coveredSectors: 2,
|
||||
verifiedSectors: 2,
|
||||
mismatchSector: null,
|
||||
malformed: false,
|
||||
});
|
||||
expect(parsed.value.metadata.fixup_unverified_record_count).toBe(1);
|
||||
});
|
||||
});
|
||||
|
||||
describe("MFT record slots", () => {
|
||||
it("distinguishes extension records from base records", () => {
|
||||
const parsed = parseMftImage(
|
||||
streamOf(
|
||||
recordOf(),
|
||||
recordOf({
|
||||
flags: 0x0005,
|
||||
baseRecordReference: 0xabcd_fedc_ba98_7654n,
|
||||
}),
|
||||
),
|
||||
);
|
||||
expect(parsed.ok).toBe(true);
|
||||
if (!parsed.ok) return;
|
||||
expect(parsed.value.records[1]).toMatchObject({
|
||||
baseRecordReference: {
|
||||
recordNumber: 0xfedc_ba98_7654,
|
||||
sequenceNumber: 0xabcd,
|
||||
},
|
||||
flags: { raw: 0x0005, names: ["in-use", "system"] },
|
||||
});
|
||||
expect(parsed.value.metadata.extension_record_count).toBe(1);
|
||||
});
|
||||
|
||||
it("names directory records without losing the in-use flag", () => {
|
||||
const parsed = parseMftImage(streamOf(recordOf({ flags: 0x0003 })));
|
||||
expect(parsed.ok).toBe(true);
|
||||
if (!parsed.ok) return;
|
||||
expect(parsed.value.records[0]).toMatchObject({
|
||||
directory: true,
|
||||
inUse: true,
|
||||
flags: { names: ["in-use", "directory"] },
|
||||
});
|
||||
expect(parsed.value.metadata.directory_record_count).toBe(1);
|
||||
});
|
||||
|
||||
it("skips malformed and sparse slots without renumbering valid records", () => {
|
||||
const stream = streamOf(
|
||||
recordOf(),
|
||||
recordOf(),
|
||||
new Uint8Array(MFT_DEFAULT_RECORD_BYTES),
|
||||
recordOf(),
|
||||
);
|
||||
stream.set(Buffer.from("BAAD", "latin1"), MFT_DEFAULT_RECORD_BYTES);
|
||||
const parsed = parseMftImage(stream);
|
||||
expect(parsed.ok).toBe(true);
|
||||
if (!parsed.ok) return;
|
||||
expect(parsed.value.metadata.record_count).toBe(2);
|
||||
expect(
|
||||
parsed.value.records.map(({ index, fileOffset }) => [index, fileOffset]),
|
||||
).toEqual([
|
||||
[0, 0],
|
||||
[3, 3 * MFT_DEFAULT_RECORD_BYTES],
|
||||
]);
|
||||
expect(parsed.value.limitations).toEqual([
|
||||
"record at 1024 does not begin with the FILE signature; 2 slot(s) without a FILE signature were found",
|
||||
]);
|
||||
});
|
||||
|
||||
it("keeps a trailing partial record out of the inventory", () => {
|
||||
const stream = new Uint8Array(1024 + 300);
|
||||
stream.set(recordOf(), 0);
|
||||
const parsed = parseMftImage(stream);
|
||||
expect(parsed.ok).toBe(true);
|
||||
if (!parsed.ok) return;
|
||||
expect(parsed.value.metadata.record_count).toBe(1);
|
||||
expect(parsed.value.metadata.truncated_tail_bytes).toBe(300);
|
||||
expect(parsed.value.limitations.join(" ")).toMatch(
|
||||
/300 trailing byte\(s\)/u,
|
||||
);
|
||||
});
|
||||
|
||||
it("falls back to the default record size when record 0 declares none", () => {
|
||||
const record = recordOf();
|
||||
new DataView(record.buffer).setUint32(28, 0, true);
|
||||
const parsed = parseMftImage(record);
|
||||
expect(parsed.ok).toBe(true);
|
||||
if (!parsed.ok) return;
|
||||
expect(parsed.value.recordBytes).toBe(MFT_DEFAULT_RECORD_BYTES);
|
||||
expect(parsed.value.limitations.join(" ")).toMatch(
|
||||
/not a supported power of two/u,
|
||||
);
|
||||
});
|
||||
|
||||
it("reads a 4 KiB record at its declared size", () => {
|
||||
const parsed = parseMftImage(streamOf(recordOf({ recordBytes: 4096 })));
|
||||
expect(parsed.ok).toBe(true);
|
||||
if (!parsed.ok) return;
|
||||
expect(parsed.value.recordBytes).toBe(4096);
|
||||
expect(parsed.value.metadata.record_count).toBe(1);
|
||||
});
|
||||
});
|
||||
|
||||
describe("parseMftImage refusals and limits", () => {
|
||||
it("refuses a file shorter than one record header", () => {
|
||||
const parsed = parseMftImage(new Uint8Array(32));
|
||||
expect(parsed.ok).toBe(false);
|
||||
if (parsed.ok) return;
|
||||
expect(parsed.error.message).toMatch(
|
||||
/shorter than one 42-byte record header/u,
|
||||
);
|
||||
});
|
||||
|
||||
it("refuses bytes that do not start with the FILE signature", () => {
|
||||
const parsed = parseMftImage(Buffer.alloc(2048, 0x41));
|
||||
expect(parsed.ok).toBe(false);
|
||||
if (parsed.ok) return;
|
||||
expect(parsed.error.message).toMatch(
|
||||
/does not begin with the FILE signature/u,
|
||||
);
|
||||
});
|
||||
|
||||
it("refuses a file above the fixed ceiling", () => {
|
||||
const decoder = createMftDecoder(MFT_MAX_FILE_BYTES + 1);
|
||||
expect(decoder.ok).toBe(false);
|
||||
if (decoder.ok) return;
|
||||
expect(decoder.error).toMatchObject({
|
||||
kind: "limit",
|
||||
boundary: "file-size",
|
||||
});
|
||||
});
|
||||
|
||||
it("refuses streams that differ from the admitted byte count", () => {
|
||||
const stream = recordOf();
|
||||
const short = createMftDecoder(stream.length + 1);
|
||||
if (!short.ok) throw new Error(short.error.message);
|
||||
expect(short.value.write(stream).ok).toBe(true);
|
||||
expect(short.value.finish()).toMatchObject({
|
||||
ok: false,
|
||||
error: {
|
||||
kind: "format",
|
||||
message: "stream ended before the admitted file length",
|
||||
},
|
||||
});
|
||||
|
||||
const long = createMftDecoder(stream.length - 1);
|
||||
if (!long.ok) throw new Error(long.error.message);
|
||||
expect(long.value.write(stream)).toMatchObject({
|
||||
ok: false,
|
||||
error: {
|
||||
kind: "format",
|
||||
message: "received more bytes than the admitted file length",
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it("returns the same inventory without retaining all the decoded records", () => {
|
||||
const bytes = streamOf(...Array.from({ length: 33 }, () => recordOf()));
|
||||
const decoder = createMftDecoder(bytes.length);
|
||||
const parsed = parseMftImage(bytes);
|
||||
if (!decoder.ok || !parsed.ok) throw new Error("invalid fixture");
|
||||
expect(decoder.value.write(bytes).ok).toBe(true);
|
||||
const inventory = decoder.value.finish();
|
||||
if (!inventory.ok) throw new Error(inventory.error.message);
|
||||
expect(inventory.value).not.toHaveProperty("records");
|
||||
expect(inventory.value.metadata).toEqual(parsed.value.metadata);
|
||||
expect(inventory.value.limitations).toEqual(parsed.value.limitations);
|
||||
});
|
||||
|
||||
it("decodes the same stream through the chunked decoder", () => {
|
||||
const stream = streamOf(recordOf(), recordOf({ flags: 0x0003 }));
|
||||
const decoder = createMftDecoder(stream.length);
|
||||
expect(decoder.ok).toBe(true);
|
||||
if (!decoder.ok) return;
|
||||
// Feed one byte past a record boundary so a chunk never aligns with a record.
|
||||
for (const chunk of [stream.subarray(0, 1025), stream.subarray(1025)])
|
||||
expect(decoder.value.write(chunk).ok).toBe(true);
|
||||
const image = decoder.value.finish();
|
||||
expect(image.ok).toBe(true);
|
||||
if (!image.ok) return;
|
||||
expect(image.value.metadata.record_count).toBe(2);
|
||||
expect(image.value.metadata.directory_record_count).toBe(1);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,512 @@
|
||||
import { err, ok, type Result } from "../result.js";
|
||||
import type {
|
||||
RegionDecodeFailure,
|
||||
RegionDecoder,
|
||||
} from "../firmware/regionDecoder.js";
|
||||
|
||||
/** An NTFS FILE record begins with this signature. */
|
||||
const MFT_FILE_RECORD_MAGIC = "FILE";
|
||||
const MFT_FILE_RECORD_MAGIC_BYTES = 4;
|
||||
/** The shortest header worth reading: a FILE record header ends at `nextAttributeId`. */
|
||||
const NTFS_MFT_MIN_RECORD_BYTES = 42;
|
||||
|
||||
const hasMftFileRecordSignature = (prefix: Uint8Array): boolean =>
|
||||
prefix.length >= MFT_FILE_RECORD_MAGIC_BYTES &&
|
||||
text(prefix.subarray(0, MFT_FILE_RECORD_MAGIC_BYTES)) ===
|
||||
MFT_FILE_RECORD_MAGIC;
|
||||
|
||||
/** A record's declared size is a power of two from 512 upward; 1024 is NTFS's default. */
|
||||
export const MFT_DEFAULT_RECORD_BYTES = 1024;
|
||||
const MFT_MIN_RECORD_BYTES = 512;
|
||||
const MFT_MAX_RECORD_BYTES = 65536;
|
||||
/** An update sequence array holds at least the sequence number and one fixup word. */
|
||||
const USA_MIN_ENTRIES = 2;
|
||||
/** NTFS fixes up one 512-byte sector at a time. */
|
||||
const FIXUP_SECTOR_BYTES = 512;
|
||||
/**
|
||||
* Fixed whole-file ceiling. Records are inventoried from their headers only;
|
||||
* attributes and their payloads are not decoded, so this is deliberately
|
||||
* smaller than a volume's MFT can be.
|
||||
*/
|
||||
export const MFT_MAX_FILE_BYTES = 512 * 1024 * 1024;
|
||||
/** Limit the inline header sample independently of full stream coverage. */
|
||||
export const MFT_INLINE_HEADER_LIMIT = 32;
|
||||
|
||||
/** FILE record header fields; offsets are from the record start. */
|
||||
const RECORD_HEADER = {
|
||||
updateSequenceOffset: 4,
|
||||
updateSequenceCount: 6,
|
||||
logFileSequenceNumber: 8,
|
||||
sequenceNumber: 16,
|
||||
hardLinkCount: 18,
|
||||
firstAttributeOffset: 20,
|
||||
flags: 22,
|
||||
usedBytes: 24,
|
||||
totalBytes: 28,
|
||||
baseRecordReference: 32,
|
||||
nextAttributeId: 40,
|
||||
recordNumber: 44,
|
||||
} as const;
|
||||
const RECORD_HEADER_WITH_NUMBER_BYTES = 48;
|
||||
|
||||
const FILE_RECORD_IN_USE = 0x0001;
|
||||
const FILE_RECORD_IS_DIRECTORY = 0x0002;
|
||||
const FILE_RECORD_NAMES = new Map<number, string>([
|
||||
[FILE_RECORD_IN_USE, "in-use"],
|
||||
[FILE_RECORD_IS_DIRECTORY, "directory"],
|
||||
[0x0004, "system"],
|
||||
[0x0008, "view-index"],
|
||||
]);
|
||||
|
||||
export interface MftRecordFixups {
|
||||
/** The update sequence number the array's first entry holds, or 0 when absent. */
|
||||
readonly sequenceNumber: number;
|
||||
readonly entries: number;
|
||||
/** Sectors whose trailer held the sequence number before a mismatch. */
|
||||
readonly verifiedSectors: number;
|
||||
/** Sectors the array covers; fewer than the record's sectors leaves a tail unverified. */
|
||||
readonly coveredSectors: number;
|
||||
/** The first sector whose trailer did not hold the sequence number, if any. */
|
||||
readonly mismatchSector: number | null;
|
||||
/** The array itself is missing, misplaced or too short to cover one sector. */
|
||||
readonly malformed: boolean;
|
||||
}
|
||||
|
||||
export interface MftFileReference {
|
||||
readonly recordNumber: number;
|
||||
readonly sequenceNumber: number;
|
||||
}
|
||||
|
||||
const decodeMftFileReference = (reference: bigint): MftFileReference => ({
|
||||
recordNumber: Number(reference & 0xffff_ffff_ffffn),
|
||||
sequenceNumber: Number(reference >> 48n),
|
||||
});
|
||||
|
||||
export interface MftRecord {
|
||||
/** Zero-based ordinal of this record in the stream. */
|
||||
readonly index: number;
|
||||
/** File offset where the record starts. */
|
||||
readonly fileOffset: number;
|
||||
readonly sequenceNumber: number;
|
||||
/** Hard link count as recorded; a zero count marks a record with no names. */
|
||||
readonly hardLinkCount: number;
|
||||
readonly flags: {
|
||||
readonly raw: number;
|
||||
readonly names: readonly string[];
|
||||
readonly unknownMask: number;
|
||||
};
|
||||
readonly inUse: boolean;
|
||||
readonly directory: boolean;
|
||||
/** Used and allocated byte counts as declared by the FILE record. */
|
||||
readonly usedBytes: number;
|
||||
readonly totalBytes: number;
|
||||
readonly extentValid: boolean;
|
||||
/** The base record this is an extension of, or `null` for a base record. */
|
||||
readonly baseRecordReference: MftFileReference | null;
|
||||
/** The record number the header declares; `null` when the header predates the field. */
|
||||
readonly declaredRecordNumber: number | null;
|
||||
readonly recordNumberMatchesSlot: boolean | null;
|
||||
/** `$LogFile` sequence number as recorded; no log is read. */
|
||||
readonly logFileSequenceNumber: string;
|
||||
/** Declared offset of the first attribute; no attribute decoding follows yet. */
|
||||
readonly firstAttributeOffset: number;
|
||||
/** Declared id of the next attribute; no attribute decoding follows yet. */
|
||||
readonly nextAttributeId: number;
|
||||
readonly fixups: MftRecordFixups;
|
||||
}
|
||||
|
||||
export interface MftInventory {
|
||||
readonly metadata: MftMetadata;
|
||||
/** The record size the stream declared, or the fixed default when it did not. */
|
||||
readonly recordBytes: number;
|
||||
readonly limitations: readonly string[];
|
||||
}
|
||||
|
||||
export interface MftImage extends MftInventory {
|
||||
readonly records: readonly MftRecord[];
|
||||
}
|
||||
|
||||
export interface MftMetadata {
|
||||
readonly kind: "ntfs-mft";
|
||||
readonly record_bytes: number;
|
||||
readonly record_count: number;
|
||||
readonly in_use_record_count: number;
|
||||
readonly directory_record_count: number;
|
||||
readonly extension_record_count: number;
|
||||
readonly fixup_mismatch_count: number;
|
||||
readonly fixup_unverified_record_count: number;
|
||||
readonly truncated_tail_bytes: number;
|
||||
readonly record_headers: readonly MftRecord[];
|
||||
readonly omitted_record_headers: number;
|
||||
}
|
||||
|
||||
const text = (ascii: Uint8Array): string =>
|
||||
Buffer.from(ascii).toString("latin1");
|
||||
|
||||
const failure = (
|
||||
message: string,
|
||||
boundary?: RegionDecodeFailure["boundary"],
|
||||
): RegionDecodeFailure => ({
|
||||
kind: "format",
|
||||
message,
|
||||
...(boundary === undefined ? {} : { boundary }),
|
||||
});
|
||||
|
||||
const limit = (
|
||||
message: string,
|
||||
boundary?: RegionDecodeFailure["boundary"],
|
||||
): RegionDecodeFailure => ({
|
||||
kind: "limit",
|
||||
message,
|
||||
...(boundary === undefined ? {} : { boundary }),
|
||||
});
|
||||
|
||||
const parseRecordFlags = (raw: number): MftRecord["flags"] => {
|
||||
const names: string[] = [];
|
||||
let unknownMask = raw;
|
||||
for (const [bit, name] of FILE_RECORD_NAMES) {
|
||||
if ((raw & bit) !== 0) names.push(name);
|
||||
unknownMask &= ~bit;
|
||||
}
|
||||
return { raw, names, unknownMask };
|
||||
};
|
||||
|
||||
const isRecordSize = (bytes: number): boolean =>
|
||||
bytes >= MFT_MIN_RECORD_BYTES &&
|
||||
bytes <= MFT_MAX_RECORD_BYTES &&
|
||||
(bytes & (bytes - 1)) === 0;
|
||||
|
||||
const updateSequenceArrayFits = (
|
||||
offset: number,
|
||||
entries: number,
|
||||
sectors: number,
|
||||
): boolean =>
|
||||
entries >= USA_MIN_ENTRIES &&
|
||||
entries <= sectors + 1 &&
|
||||
offset >= NTFS_MFT_MIN_RECORD_BYTES &&
|
||||
offset % 2 === 0 &&
|
||||
offset + entries * 2 <= FIXUP_SECTOR_BYTES - 2;
|
||||
|
||||
/** Recognize a FILE header without requiring its trailers or attributes to verify. */
|
||||
export const hasMftFileRecordHeader = (prefix: Uint8Array): boolean => {
|
||||
if (
|
||||
prefix.length < NTFS_MFT_MIN_RECORD_BYTES ||
|
||||
!hasMftFileRecordSignature(prefix)
|
||||
)
|
||||
return false;
|
||||
const view = new DataView(
|
||||
prefix.buffer,
|
||||
prefix.byteOffset,
|
||||
prefix.byteLength,
|
||||
);
|
||||
return (
|
||||
isRecordSize(view.getUint32(RECORD_HEADER.totalBytes, true)) ||
|
||||
updateSequenceArrayFits(
|
||||
view.getUint16(RECORD_HEADER.updateSequenceOffset, true),
|
||||
view.getUint16(RECORD_HEADER.updateSequenceCount, true),
|
||||
MFT_DEFAULT_RECORD_BYTES / FIXUP_SECTOR_BYTES,
|
||||
)
|
||||
);
|
||||
};
|
||||
|
||||
/**
|
||||
* Parse FILE record headers and verify their update-sequence fixups.
|
||||
*
|
||||
* Only the record structure is established: attributes, their names, the
|
||||
* filenames and the USN journal are left to later steps, so no path, timestamp
|
||||
* or file identity beyond the record itself is claimed. A record whose fixup
|
||||
* does not match keeps its header facts, because the stream still declares it,
|
||||
* and the mismatch is reported on the record and counted in the metadata.
|
||||
*/
|
||||
export const parseMftImage = (
|
||||
bytes: Uint8Array,
|
||||
): Result<MftImage, RegionDecodeFailure> => {
|
||||
const records: MftRecord[] = [];
|
||||
const inventory = scanMft(bytes, records);
|
||||
return inventory.ok ? ok({ ...inventory.value, records }) : inventory;
|
||||
};
|
||||
|
||||
const scanMft = (
|
||||
bytes: Uint8Array,
|
||||
retainedRecords?: MftRecord[],
|
||||
): Result<MftInventory, RegionDecodeFailure> => {
|
||||
if (bytes.length < NTFS_MFT_MIN_RECORD_BYTES)
|
||||
return err(
|
||||
failure(
|
||||
`file is ${String(bytes.length)} bytes, shorter than one ${String(NTFS_MFT_MIN_RECORD_BYTES)}-byte record header`,
|
||||
),
|
||||
);
|
||||
if (!hasMftFileRecordSignature(bytes))
|
||||
return err(failure("record 0 does not begin with the FILE signature"));
|
||||
|
||||
const limitations: string[] = [];
|
||||
const header = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
|
||||
const declaredRecordBytes = header.getUint32(RECORD_HEADER.totalBytes, true);
|
||||
const recordBytes = isRecordSize(declaredRecordBytes)
|
||||
? declaredRecordBytes
|
||||
: MFT_DEFAULT_RECORD_BYTES;
|
||||
if (!isRecordSize(declaredRecordBytes))
|
||||
limitations.push(
|
||||
`record 0 declares a record size of ${String(declaredRecordBytes)} bytes, which is not a supported power of two from ${String(MFT_MIN_RECORD_BYTES)} to ${String(MFT_MAX_RECORD_BYTES)}; records were read at the ${String(MFT_DEFAULT_RECORD_BYTES)}-byte default`,
|
||||
);
|
||||
|
||||
const recordHeaders: MftRecord[] = [];
|
||||
let recordCount = 0;
|
||||
let inUseCount = 0;
|
||||
let directoryCount = 0;
|
||||
let extensionCount = 0;
|
||||
let mismatchCount = 0;
|
||||
let unverifiedCount = 0;
|
||||
let invalidSlots = 0;
|
||||
let invalidExtents = 0;
|
||||
let inconsistentRecordNumbers = 0;
|
||||
let firstInvalidExtentOffset: number | undefined;
|
||||
let firstInvalidOffset: number | undefined;
|
||||
for (
|
||||
let offset = 0;
|
||||
offset + recordBytes <= bytes.length;
|
||||
offset += recordBytes
|
||||
) {
|
||||
if (
|
||||
!hasMftFileRecordSignature(
|
||||
bytes.subarray(offset, offset + MFT_FILE_RECORD_MAGIC_BYTES),
|
||||
)
|
||||
) {
|
||||
invalidSlots += 1;
|
||||
firstInvalidOffset ??= offset;
|
||||
if (!isRecordSize(declaredRecordBytes)) break;
|
||||
continue;
|
||||
}
|
||||
const record = readRecord(bytes, offset, offset / recordBytes, recordBytes);
|
||||
retainedRecords?.push(record);
|
||||
if (recordHeaders.length < MFT_INLINE_HEADER_LIMIT)
|
||||
recordHeaders.push(record);
|
||||
recordCount += 1;
|
||||
if (record.inUse) inUseCount += 1;
|
||||
if (record.directory) directoryCount += 1;
|
||||
if (record.baseRecordReference !== null) extensionCount += 1;
|
||||
if (record.fixups.mismatchSector !== null) mismatchCount += 1;
|
||||
if (
|
||||
record.fixups.malformed ||
|
||||
record.fixups.coveredSectors < recordBytes / FIXUP_SECTOR_BYTES
|
||||
)
|
||||
unverifiedCount += 1;
|
||||
if (record.recordNumberMatchesSlot === false)
|
||||
inconsistentRecordNumbers += 1;
|
||||
if (!record.extentValid) {
|
||||
invalidExtents += 1;
|
||||
firstInvalidExtentOffset ??= offset;
|
||||
}
|
||||
}
|
||||
if (firstInvalidOffset !== undefined)
|
||||
limitations.push(
|
||||
`record at ${String(firstInvalidOffset)} does not begin with the FILE signature; ${String(invalidSlots)} slot(s) without a FILE signature were found`,
|
||||
);
|
||||
|
||||
if (firstInvalidExtentOffset !== undefined)
|
||||
limitations.push(
|
||||
`${String(invalidExtents)} FILE record(s) declare inconsistent record extents, beginning at offset ${String(firstInvalidExtentOffset)}`,
|
||||
);
|
||||
|
||||
if (inconsistentRecordNumbers !== 0)
|
||||
limitations.push(
|
||||
`${String(inconsistentRecordNumbers)} FILE record(s) declare a record number that disagrees with their stream slot`,
|
||||
);
|
||||
|
||||
const truncatedTailBytes = bytes.length % recordBytes;
|
||||
if (truncatedTailBytes !== 0)
|
||||
limitations.push(
|
||||
`${String(truncatedTailBytes)} trailing byte(s) do not fill a ${String(recordBytes)}-byte record and were not parsed`,
|
||||
);
|
||||
|
||||
if (mismatchCount !== 0)
|
||||
limitations.push(
|
||||
`${String(mismatchCount)} record(s) hold a sector trailer that is not the update sequence number; their attributes were not read and no later step should trust them`,
|
||||
);
|
||||
if (unverifiedCount !== 0)
|
||||
limitations.push(
|
||||
`${String(unverifiedCount)} record(s) carry an invalid or incomplete update sequence array; their sector trailers were not fully verified`,
|
||||
);
|
||||
|
||||
const metadata: MftMetadata = {
|
||||
kind: "ntfs-mft",
|
||||
record_bytes: recordBytes,
|
||||
record_count: recordCount,
|
||||
in_use_record_count: inUseCount,
|
||||
directory_record_count: directoryCount,
|
||||
extension_record_count: extensionCount,
|
||||
fixup_mismatch_count: mismatchCount,
|
||||
fixup_unverified_record_count: unverifiedCount,
|
||||
truncated_tail_bytes: truncatedTailBytes,
|
||||
record_headers: recordHeaders,
|
||||
omitted_record_headers: recordCount - recordHeaders.length,
|
||||
};
|
||||
|
||||
return ok({ metadata, recordBytes, limitations });
|
||||
};
|
||||
|
||||
const readRecord = (
|
||||
bytes: Uint8Array,
|
||||
fileOffset: number,
|
||||
index: number,
|
||||
recordBytes: number,
|
||||
): MftRecord => {
|
||||
const record = bytes.subarray(fileOffset, fileOffset + recordBytes);
|
||||
const view = new DataView(
|
||||
record.buffer,
|
||||
record.byteOffset,
|
||||
record.byteLength,
|
||||
);
|
||||
|
||||
const flags = view.getUint16(RECORD_HEADER.flags, true);
|
||||
const baseRecordReference = view.getBigUint64(
|
||||
RECORD_HEADER.baseRecordReference,
|
||||
true,
|
||||
);
|
||||
|
||||
const usaOffset = view.getUint16(RECORD_HEADER.updateSequenceOffset, true);
|
||||
const fixups = verifyFixups(record, view, usaOffset);
|
||||
const usedBytes = view.getUint32(RECORD_HEADER.usedBytes, true);
|
||||
const totalBytes = view.getUint32(RECORD_HEADER.totalBytes, true);
|
||||
const firstAttributeOffset = view.getUint16(
|
||||
RECORD_HEADER.firstAttributeOffset,
|
||||
true,
|
||||
);
|
||||
|
||||
const declaredRecordNumber =
|
||||
!fixups.malformed && usaOffset >= RECORD_HEADER_WITH_NUMBER_BYTES
|
||||
? view.getUint32(RECORD_HEADER.recordNumber, true)
|
||||
: null;
|
||||
|
||||
return {
|
||||
index,
|
||||
fileOffset,
|
||||
sequenceNumber: view.getUint16(RECORD_HEADER.sequenceNumber, true),
|
||||
hardLinkCount: view.getUint16(RECORD_HEADER.hardLinkCount, true),
|
||||
flags: parseRecordFlags(flags),
|
||||
inUse: (flags & FILE_RECORD_IN_USE) !== 0,
|
||||
directory: (flags & FILE_RECORD_IS_DIRECTORY) !== 0,
|
||||
usedBytes,
|
||||
totalBytes,
|
||||
extentValid:
|
||||
totalBytes === recordBytes &&
|
||||
usedBytes >= NTFS_MFT_MIN_RECORD_BYTES &&
|
||||
usedBytes % 8 === 0 &&
|
||||
usedBytes <= totalBytes &&
|
||||
firstAttributeOffset % 8 === 0 &&
|
||||
firstAttributeOffset >= NTFS_MFT_MIN_RECORD_BYTES &&
|
||||
firstAttributeOffset + 4 <= usedBytes &&
|
||||
(fixups.malformed ||
|
||||
firstAttributeOffset >= usaOffset + fixups.entries * 2),
|
||||
baseRecordReference:
|
||||
baseRecordReference === 0n
|
||||
? null
|
||||
: decodeMftFileReference(baseRecordReference),
|
||||
declaredRecordNumber,
|
||||
recordNumberMatchesSlot:
|
||||
declaredRecordNumber === null ? null : declaredRecordNumber === index,
|
||||
logFileSequenceNumber: view
|
||||
.getBigUint64(RECORD_HEADER.logFileSequenceNumber, true)
|
||||
.toString(),
|
||||
firstAttributeOffset,
|
||||
nextAttributeId: view.getUint16(RECORD_HEADER.nextAttributeId, true),
|
||||
fixups,
|
||||
};
|
||||
};
|
||||
|
||||
/**
|
||||
* Each sector's last two bytes are replaced on disk by the update sequence
|
||||
* number; the array's first entry is that number and the rest are the saved
|
||||
* trailer words. A sector whose trailer is not the sequence number was not
|
||||
* cleanly written, so the record's attributes cannot be trusted.
|
||||
*/
|
||||
const verifyFixups = (
|
||||
record: Uint8Array,
|
||||
view: DataView,
|
||||
usaOffset: number,
|
||||
): MftRecordFixups => {
|
||||
const usaCount = view.getUint16(RECORD_HEADER.updateSequenceCount, true);
|
||||
const sectors = Math.floor(record.length / FIXUP_SECTOR_BYTES);
|
||||
// Entry 0 holds the sequence number, so sector n's saved trailer is entry n+1.
|
||||
const coveredSectors = Math.max(0, usaCount - 1);
|
||||
|
||||
if (!updateSequenceArrayFits(usaOffset, usaCount, sectors))
|
||||
return {
|
||||
sequenceNumber: 0,
|
||||
entries: usaCount,
|
||||
verifiedSectors: 0,
|
||||
coveredSectors: 0,
|
||||
mismatchSector: null,
|
||||
malformed: true,
|
||||
};
|
||||
|
||||
const sequenceNumber = view.getUint16(usaOffset, true);
|
||||
if (sequenceNumber === 0 || sequenceNumber === 0xffff)
|
||||
return {
|
||||
sequenceNumber,
|
||||
entries: usaCount,
|
||||
verifiedSectors: 0,
|
||||
coveredSectors,
|
||||
mismatchSector: null,
|
||||
malformed: true,
|
||||
};
|
||||
for (let sector = 0; sector < coveredSectors; sector += 1) {
|
||||
const trailerOffset = sector * FIXUP_SECTOR_BYTES + FIXUP_SECTOR_BYTES - 2;
|
||||
if (view.getUint16(trailerOffset, true) !== sequenceNumber)
|
||||
return {
|
||||
sequenceNumber,
|
||||
entries: usaCount,
|
||||
verifiedSectors: sector,
|
||||
coveredSectors,
|
||||
mismatchSector: sector,
|
||||
malformed: false,
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
sequenceNumber,
|
||||
entries: usaCount,
|
||||
verifiedSectors: coveredSectors,
|
||||
coveredSectors,
|
||||
mismatchSector: null,
|
||||
malformed: false,
|
||||
};
|
||||
};
|
||||
|
||||
/** Buffer the admitted input once while accepting arbitrary chunk boundaries. */
|
||||
class StreamingMftDecoder implements RegionDecoder<MftInventory> {
|
||||
readonly #bytes: Uint8Array;
|
||||
#written = 0;
|
||||
|
||||
constructor(fileBytes: number) {
|
||||
this.#bytes = new Uint8Array(fileBytes);
|
||||
}
|
||||
|
||||
write(chunk: Uint8Array): Result<void, RegionDecodeFailure> {
|
||||
if (chunk.length > this.#bytes.length - this.#written)
|
||||
return err(failure("received more bytes than the admitted file length"));
|
||||
this.#bytes.set(chunk, this.#written);
|
||||
this.#written += chunk.length;
|
||||
return ok(undefined);
|
||||
}
|
||||
|
||||
finish(): Result<MftInventory, RegionDecodeFailure> {
|
||||
if (this.#written !== this.#bytes.length)
|
||||
return err(failure("stream ended before the admitted file length"));
|
||||
return scanMft(this.#bytes);
|
||||
}
|
||||
}
|
||||
|
||||
export const createMftDecoder = (
|
||||
fileBytes: number,
|
||||
): Result<RegionDecoder<MftInventory>, RegionDecodeFailure> =>
|
||||
Number.isSafeInteger(fileBytes) &&
|
||||
fileBytes >= 0 &&
|
||||
fileBytes <= MFT_MAX_FILE_BYTES
|
||||
? ok(new StreamingMftDecoder(fileBytes))
|
||||
: err(
|
||||
limit(
|
||||
`file is ${String(fileBytes)} bytes, exceeding the fixed ${String(MFT_MAX_FILE_BYTES)}-byte (512 MiB) ceiling`,
|
||||
"file-size",
|
||||
),
|
||||
);
|
||||
@@ -1120,23 +1120,23 @@ describe("Ghidra native instruction decoder context for ARM TMode and MIPS ISA_M
|
||||
});
|
||||
});
|
||||
|
||||
describe("Ghidra bridge script processor context contract", () => {
|
||||
it("binds TMode to ARM and ISA_MODE to MIPS in ReaGhidraBridge.java", async () => {
|
||||
const bridgePath = new URL(
|
||||
"../../bridge/ghidra/ReaGhidraBridge.java",
|
||||
import.meta.url,
|
||||
);
|
||||
const content = await readFile(bridgePath, "utf8");
|
||||
expect(content).toContain(
|
||||
'currentProgram.getLanguageID().getIdAsString().startsWith("ARM:")',
|
||||
);
|
||||
expect(content).toContain(
|
||||
'currentProgram.getLanguageID().getIdAsString().startsWith("MIPS:")',
|
||||
);
|
||||
expect(content).toContain('"TMode"');
|
||||
expect(content).toContain('"ISA_MODE"');
|
||||
expect(content).toContain('"ghidra-program-context"');
|
||||
});
|
||||
// Kept on the file rather than in its own suite. The full source run collected
|
||||
// that suite with no tests and failed the file before this assertion could run.
|
||||
it("binds TMode to ARM and ISA_MODE to MIPS in the Ghidra bridge script", async () => {
|
||||
const bridgePath = new URL(
|
||||
"../../bridge/ghidra/ReaGhidraBridge.java",
|
||||
import.meta.url,
|
||||
);
|
||||
const content = await readFile(bridgePath, "utf8");
|
||||
expect(content).toContain(
|
||||
'currentProgram.getLanguageID().getIdAsString().startsWith("ARM:")',
|
||||
);
|
||||
expect(content).toContain(
|
||||
'currentProgram.getLanguageID().getIdAsString().startsWith("MIPS:")',
|
||||
);
|
||||
expect(content).toContain('"TMode"');
|
||||
expect(content).toContain('"ISA_MODE"');
|
||||
expect(content).toContain('"ghidra-program-context"');
|
||||
});
|
||||
|
||||
describe("Ghidra dossier nested boundary composition", () => {
|
||||
|
||||
@@ -1868,7 +1868,8 @@ describe("Ghidra decompiler failure projection", () => {
|
||||
applies_at: "next_operation",
|
||||
agent_editable: true,
|
||||
scope: "rea-runtime",
|
||||
rea_restart_required: false,
|
||||
update_mode: "runtime",
|
||||
pending_restart: false,
|
||||
},
|
||||
},
|
||||
});
|
||||
@@ -1983,7 +1984,8 @@ describe("Ghidra decompile deadline for a finished invocation", () => {
|
||||
minimum: 1,
|
||||
maximum: 2_147_483,
|
||||
agent_editable: true,
|
||||
rea_restart_required: false,
|
||||
update_mode: "runtime",
|
||||
pending_restart: false,
|
||||
};
|
||||
expect(invocation.details?.limit).toEqual({
|
||||
...enforced,
|
||||
@@ -4365,7 +4367,8 @@ describe("Ghidra startup deadline for a finished invocation", () => {
|
||||
setting: "ghidra.startup_timeout_ms",
|
||||
effective_value: 250,
|
||||
agent_editable: true,
|
||||
rea_restart_required: false,
|
||||
update_mode: "runtime",
|
||||
pending_restart: false,
|
||||
};
|
||||
const invocationLimit = {
|
||||
...startupLimit,
|
||||
|
||||
@@ -187,7 +187,8 @@ it.each(["initialize", "transport"])(
|
||||
applies_at: "next_operation",
|
||||
agent_editable: true,
|
||||
scope: "rea-runtime",
|
||||
rea_restart_required: false,
|
||||
update_mode: "runtime",
|
||||
pending_restart: false,
|
||||
},
|
||||
diagnostics: { producer_fact: "retained" },
|
||||
});
|
||||
|
||||
@@ -948,7 +948,7 @@ export class JavaScriptAnalysisWorker {
|
||||
limit: analysisSettingLimit(
|
||||
"javascript.max_heap_mb",
|
||||
this.limits.heapMb,
|
||||
undefined,
|
||||
this.limits.heapSource,
|
||||
),
|
||||
remediationAction: javaScriptWorkerHeapRemediation(this.limits.heapMb),
|
||||
},
|
||||
@@ -1147,7 +1147,7 @@ export class JavaScriptAnalysisWorker {
|
||||
limit: analysisSettingLimit(
|
||||
"javascript.analysis_timeout_ms",
|
||||
this.limits.timeoutMs,
|
||||
undefined,
|
||||
this.limits.timeoutSource,
|
||||
),
|
||||
capturedOutput: this.#captured(),
|
||||
userMessage: `JavaScript source analysis reached its ${String(this.limits.timeoutMs)}ms per-source server/provider deadline (analysis_timeout_ms), not the MCP client deadline.`,
|
||||
@@ -1182,7 +1182,7 @@ export class JavaScriptAnalysisWorker {
|
||||
limit: analysisSettingLimit(
|
||||
"javascript.max_heap_mb",
|
||||
this.limits.heapMb,
|
||||
undefined,
|
||||
this.limits.heapSource,
|
||||
),
|
||||
}
|
||||
: {}),
|
||||
|
||||
@@ -22,7 +22,8 @@ export const debuggerAdmissionDiagnostics = (
|
||||
readonly minimum: number;
|
||||
readonly maximum: number;
|
||||
readonly unit: DebuggerAdmissionLimit["unit"];
|
||||
readonly restart_required: true;
|
||||
readonly update_mode: "restart";
|
||||
readonly pending_restart: false;
|
||||
} => ({
|
||||
setting: admission.setting,
|
||||
effective_value: admission.value,
|
||||
@@ -31,7 +32,8 @@ export const debuggerAdmissionDiagnostics = (
|
||||
minimum: admission.minimum,
|
||||
maximum: admission.maximum,
|
||||
unit: admission.unit,
|
||||
restart_required: true,
|
||||
update_mode: "restart",
|
||||
pending_restart: false,
|
||||
});
|
||||
|
||||
/** Name the setting, the effective value, its source, the range, and a restart. */
|
||||
|
||||
@@ -0,0 +1,53 @@
|
||||
import {
|
||||
ArtifactResourceScope,
|
||||
type ArtifactResourceOwner,
|
||||
} from "../../artifacts/ArtifactResourceScope.js";
|
||||
|
||||
/** Retain a cancelled response branch until its underlying cancellation settles. */
|
||||
export class StateResponseCleanup {
|
||||
readonly #scope = new ArtifactResourceScope();
|
||||
|
||||
constructor(private readonly changed: () => void = () => {}) {}
|
||||
|
||||
get cleanupResources(): readonly string[] {
|
||||
return this.#scope.cleanupResources;
|
||||
}
|
||||
|
||||
cancel(cancel: () => Promise<void>): void {
|
||||
let state:
|
||||
| { status: "pending" }
|
||||
| { status: "done" }
|
||||
| { status: "failed"; cause: unknown } = { status: "pending" };
|
||||
const owner: ArtifactResourceOwner = {
|
||||
kind: "lease",
|
||||
resource: "state response body",
|
||||
lease: {
|
||||
release: async () => {
|
||||
if (state.status === "pending")
|
||||
throw new Error("State response cancellation is still pending");
|
||||
if (state.status === "failed") throw state.cause;
|
||||
},
|
||||
},
|
||||
};
|
||||
this.#scope.retain(owner);
|
||||
this.changed();
|
||||
void Promise.resolve()
|
||||
.then(cancel)
|
||||
.then(
|
||||
async () => {
|
||||
state = { status: "done" };
|
||||
await this.#scope.release(owner);
|
||||
this.changed();
|
||||
},
|
||||
(cause: unknown) => {
|
||||
state = { status: "failed", cause };
|
||||
this.changed();
|
||||
},
|
||||
)
|
||||
.catch(() => undefined);
|
||||
}
|
||||
|
||||
close(): Promise<void> {
|
||||
return this.#scope.close();
|
||||
}
|
||||
}
|
||||
@@ -1,3 +1,4 @@
|
||||
import { StateResponseCleanup } from "./StateResponseCleanup.js";
|
||||
import type { EnvironmentStateStore } from "../../application/environment/environmentStateStore.js";
|
||||
import { ArtifactResourceScope } from "../../artifacts/ArtifactResourceScope.js";
|
||||
import type { AnalysisInputError } from "../../domain/analysisErrorCore.js";
|
||||
@@ -46,17 +47,24 @@ interface CallContext {
|
||||
readonly intake: StateIntake;
|
||||
readonly cancel: AbortSignal;
|
||||
readonly files: ArtifactResourceScope;
|
||||
readonly responses: StateResponseCleanup;
|
||||
}
|
||||
|
||||
/** `data/add`: exactly one part, whose document replaces the whole state only when it is valid and the call was not cancelled. */
|
||||
const addState = async (
|
||||
{ store, intake, cancel, files }: CallContext,
|
||||
{ store, intake, cancel, files, responses }: CallContext,
|
||||
params: unknown,
|
||||
): Promise<RpcOutcome> => {
|
||||
const parts = isRecord(params) ? params.parts : undefined;
|
||||
if (!Array.isArray(parts) || parts.length !== 1)
|
||||
return { error: invalidParams("data/add takes exactly one part") };
|
||||
const document = await partDocument(parts[0], intake, cancel, files);
|
||||
const document = await partDocument(
|
||||
parts[0],
|
||||
intake,
|
||||
cancel,
|
||||
files,
|
||||
responses,
|
||||
);
|
||||
if (!document.ok) return { error: document.error };
|
||||
// A file read is not interruptible, so a cancellation during it is honoured here, before the state changes.
|
||||
if (cancel.aborted) return { error: CANCELLED };
|
||||
@@ -101,6 +109,7 @@ export const stateDataPlane = (
|
||||
intake: StateIntake = DEFAULT_STATE_INTAKE,
|
||||
stopped: AbortSignal = new AbortController().signal,
|
||||
files: ArtifactResourceScope = new ArtifactResourceScope(),
|
||||
responses: StateResponseCleanup = new StateResponseCleanup(),
|
||||
): RpcMethods => {
|
||||
let previous: Promise<unknown> = Promise.resolve();
|
||||
return (method, params, signal) => {
|
||||
@@ -111,7 +120,7 @@ export const stateDataPlane = (
|
||||
const next = previous.then(() =>
|
||||
cancel.aborted
|
||||
? { error: CANCELLED }
|
||||
: run({ store, intake, cancel, files }, params),
|
||||
: run({ store, intake, cancel, files, responses }, params),
|
||||
);
|
||||
previous = next.catch(() => undefined);
|
||||
return next;
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { StateResponseCleanup } from "./StateResponseCleanup.js";
|
||||
import type { HttpHostname } from "../../domain/httpHostname.js";
|
||||
import { STATE_INTAKE_LIMIT_BYTES } from "../../application/environment/environmentStateDocument.js";
|
||||
import {
|
||||
@@ -477,6 +478,8 @@ export const serveEnvironment = async (
|
||||
};
|
||||
// A state file whose close failed stays owned here: later file reads retry it, and close reports it if it is still open.
|
||||
const stateFiles = new ArtifactResourceScope();
|
||||
let responseStateChanged = (): void => undefined;
|
||||
const stateResponses = new StateResponseCleanup(() => responseStateChanged());
|
||||
const served = await serveStreamableHttp({
|
||||
host,
|
||||
port,
|
||||
@@ -499,6 +502,7 @@ export const serveEnvironment = async (
|
||||
},
|
||||
stopping.signal,
|
||||
stateFiles,
|
||||
stateResponses,
|
||||
),
|
||||
),
|
||||
...(logger === undefined ? {} : { logger }),
|
||||
@@ -512,11 +516,14 @@ export const serveEnvironment = async (
|
||||
});
|
||||
let httpClosePending = false;
|
||||
let stateClosePending = false;
|
||||
let responseClosePending = false;
|
||||
let settledSuccessful = false;
|
||||
let closing: Promise<void> | undefined;
|
||||
const shutdownFailures = new Set<AnalysisError>();
|
||||
const owner: EnvironmentShutdownOwner = {
|
||||
get cleanupResources() {
|
||||
const stateResources = stateFiles.cleanupResources;
|
||||
const responseResources = stateResponses.cleanupResources;
|
||||
return [
|
||||
...(httpClosePending ? [served.url] : []),
|
||||
...(stateClosePending
|
||||
@@ -524,6 +531,11 @@ export const serveEnvironment = async (
|
||||
? stateResources
|
||||
: ["environment state-file scope"]
|
||||
: []),
|
||||
...(responseClosePending
|
||||
? responseResources.length > 0
|
||||
? responseResources
|
||||
: ["state response cleanup"]
|
||||
: []),
|
||||
];
|
||||
},
|
||||
close: () => {
|
||||
@@ -531,7 +543,7 @@ export const serveEnvironment = async (
|
||||
const attempt = (async () => {
|
||||
stop();
|
||||
const failures: {
|
||||
readonly stage: "http-server" | "state-files";
|
||||
readonly stage: "http-server" | "state-files" | "state-responses";
|
||||
readonly cause: unknown;
|
||||
}[] = [];
|
||||
try {
|
||||
@@ -548,6 +560,15 @@ export const serveEnvironment = async (
|
||||
stateClosePending = true;
|
||||
failures.push({ stage: "state-files", cause });
|
||||
}
|
||||
try {
|
||||
await stateResponses.close();
|
||||
if (stateResponses.cleanupResources.length > 0)
|
||||
throw new Error("State response cleanup remains pending");
|
||||
responseClosePending = false;
|
||||
} catch (cause: unknown) {
|
||||
responseClosePending = true;
|
||||
failures.push({ stage: "state-responses", cause });
|
||||
}
|
||||
if (failures.length > 0) {
|
||||
retainedEnvironmentShutdowns.add(owner);
|
||||
const failure = new AnalysisOutputError(
|
||||
@@ -583,6 +604,7 @@ export const serveEnvironment = async (
|
||||
retainedEnvironmentShutdowns.delete(owner);
|
||||
for (const failure of shutdownFailures) failure.resolveCleanup();
|
||||
shutdownFailures.clear();
|
||||
settledSuccessful = true;
|
||||
})();
|
||||
closing = attempt;
|
||||
void attempt.catch(() => {
|
||||
@@ -591,6 +613,26 @@ export const serveEnvironment = async (
|
||||
return attempt;
|
||||
},
|
||||
};
|
||||
// A fetcher may hand back a response after an earlier stop completed. Keep
|
||||
// that newly acquired body owned and make a later close observe its outcome.
|
||||
responseStateChanged = () => {
|
||||
if (!stopping.signal.aborted) return;
|
||||
if (stateResponses.cleanupResources.length > 0) {
|
||||
responseClosePending = true;
|
||||
retainedEnvironmentShutdowns.add(owner);
|
||||
if (settledSuccessful) {
|
||||
settledSuccessful = false;
|
||||
closing = undefined;
|
||||
}
|
||||
} else if (
|
||||
shutdownFailures.size === 0 &&
|
||||
!httpClosePending &&
|
||||
!stateClosePending
|
||||
) {
|
||||
responseClosePending = false;
|
||||
retainedEnvironmentShutdowns.delete(owner);
|
||||
}
|
||||
};
|
||||
return ok({
|
||||
...served,
|
||||
card,
|
||||
|
||||
@@ -1037,7 +1037,122 @@ describe("serveEnvironment with a state", () => {
|
||||
expect(missing.isError).toBe(true);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("state response cleanup", () => {
|
||||
it("reports pending teed response cleanup and retries after its sibling closes", async () => {
|
||||
let sourceCancelled = false;
|
||||
const stream = new ReadableStream<Uint8Array>({
|
||||
start(controller) {
|
||||
controller.enqueue(new Uint8Array([1]));
|
||||
},
|
||||
cancel() {
|
||||
sourceCancelled = true;
|
||||
},
|
||||
});
|
||||
const [body, sibling] = stream.tee();
|
||||
const environment = await serveState(
|
||||
{ source, state },
|
||||
{
|
||||
fetchState: async () =>
|
||||
new Response(body, {
|
||||
headers: { "content-length": String(STATE_INTAKE_LIMIT_BYTES + 1) },
|
||||
}),
|
||||
},
|
||||
);
|
||||
let timer: ReturnType<typeof setTimeout> | undefined;
|
||||
try {
|
||||
const response = await fetch(
|
||||
new URL(ENVIRONMENT_RPC_PATH, environment.url),
|
||||
{
|
||||
method: "POST",
|
||||
headers: { "content-type": "application/json" },
|
||||
body: JSON.stringify(addByUri(1)),
|
||||
signal: AbortSignal.timeout(10_000),
|
||||
},
|
||||
);
|
||||
await expect(response.json()).resolves.toMatchObject({
|
||||
error: { code: -32602 },
|
||||
});
|
||||
const closing = Promise.race([
|
||||
environment.close(),
|
||||
new Promise<never>((_, reject) => {
|
||||
timer = setTimeout(
|
||||
() => reject(new Error("server close waited for tee sibling")),
|
||||
10_000,
|
||||
);
|
||||
}),
|
||||
]);
|
||||
await expect(closing).rejects.toMatchObject({
|
||||
_tag: "AnalysisOutputError",
|
||||
cleanupIncomplete: true,
|
||||
diagnostics: { cleanup_stages: ["state-responses"] },
|
||||
});
|
||||
expect(sourceCancelled).toBe(false);
|
||||
await sibling.cancel();
|
||||
expect(sourceCancelled).toBe(true);
|
||||
await expect(environment.close()).resolves.toBeUndefined();
|
||||
} finally {
|
||||
if (timer !== undefined) clearTimeout(timer);
|
||||
await Promise.allSettled([body.cancel(), sibling.cancel()]);
|
||||
await environment.close();
|
||||
}
|
||||
});
|
||||
|
||||
it("retains a late response cleanup after an earlier successful stop", async () => {
|
||||
let giveResponse!: (response: Response) => void;
|
||||
const gate = new Promise<Response>((resolve) => {
|
||||
giveResponse = resolve;
|
||||
});
|
||||
const held = heldFetcher(() => gate);
|
||||
let release!: () => void;
|
||||
let acknowledge!: () => void;
|
||||
const cleanup = new Promise<void>((resolve) => {
|
||||
release = resolve;
|
||||
});
|
||||
const entered = new Promise<void>((resolve) => {
|
||||
acknowledge = resolve;
|
||||
});
|
||||
const environment = await serveState(
|
||||
{ source, state },
|
||||
{ fetchState: held.fetchState },
|
||||
);
|
||||
const answered = fetch(new URL(ENVIRONMENT_RPC_PATH, environment.url), {
|
||||
method: "POST",
|
||||
headers: { "content-type": "application/json" },
|
||||
body: JSON.stringify(addByUri(1)),
|
||||
signal: AbortSignal.timeout(10_000),
|
||||
}).catch(() => undefined);
|
||||
try {
|
||||
await held.fetching;
|
||||
await environment.close();
|
||||
giveResponse(
|
||||
new Response(
|
||||
new ReadableStream<Uint8Array>({
|
||||
cancel() {
|
||||
acknowledge();
|
||||
return cleanup;
|
||||
},
|
||||
}),
|
||||
),
|
||||
);
|
||||
await entered;
|
||||
await expect(environment.close()).rejects.toMatchObject({
|
||||
_tag: "AnalysisOutputError",
|
||||
cleanupIncomplete: true,
|
||||
diagnostics: { cleanup_stages: ["state-responses"] },
|
||||
});
|
||||
release();
|
||||
await expect(environment.close()).resolves.toBeUndefined();
|
||||
} finally {
|
||||
release();
|
||||
await answered;
|
||||
await environment.close();
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe("serveEnvironment state file cleanup", () => {
|
||||
it("stops without waiting for a stalled download", async () => {
|
||||
const held = heldFetcher(() => new Promise<Response>(() => undefined));
|
||||
const environment = await serveState(
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { StateResponseCleanup } from "./StateResponseCleanup.js";
|
||||
import { expect, it, vi } from "vitest";
|
||||
|
||||
import { ArtifactResourceScope } from "../../artifacts/ArtifactResourceScope.js";
|
||||
@@ -11,14 +12,17 @@ import {
|
||||
|
||||
const inlineDocument = async (bytes: string) => {
|
||||
const files = new ArtifactResourceScope();
|
||||
const responses = new StateResponseCleanup();
|
||||
try {
|
||||
return await partDocument(
|
||||
{ kind: "file", file: { bytes } },
|
||||
DEFAULT_STATE_INTAKE,
|
||||
new AbortController().signal,
|
||||
files,
|
||||
responses,
|
||||
);
|
||||
} finally {
|
||||
await responses.close();
|
||||
await files.close();
|
||||
}
|
||||
};
|
||||
@@ -79,3 +83,109 @@ it("downloads with redirects refused and the 30 s timeout by default", async ()
|
||||
fetched.mockRestore();
|
||||
}
|
||||
});
|
||||
|
||||
it.each(["http-status", "declared-overflow", "stream-overflow"] as const)(
|
||||
"returns %s while a teed response's other branch remains open",
|
||||
async (scenario) => {
|
||||
const source = new ReadableStream<Uint8Array>({
|
||||
start(controller) {
|
||||
controller.enqueue(
|
||||
new Uint8Array(
|
||||
scenario === "stream-overflow" ? STATE_INTAKE_LIMIT_BYTES + 1 : 1,
|
||||
),
|
||||
);
|
||||
},
|
||||
});
|
||||
const [body, sibling] = source.tee();
|
||||
const response = new Response(body, {
|
||||
status: scenario === "http-status" ? 503 : 200,
|
||||
...(scenario === "declared-overflow"
|
||||
? {
|
||||
headers: { "content-length": String(STATE_INTAKE_LIMIT_BYTES + 1) },
|
||||
}
|
||||
: {}),
|
||||
});
|
||||
const files = new ArtifactResourceScope();
|
||||
const responses = new StateResponseCleanup();
|
||||
const operation = partDocument(
|
||||
{ kind: "file", file: { uri: "https://state.example.test/state.json" } },
|
||||
{ fetchState: async () => response, timeoutMs: 1_000 },
|
||||
new AbortController().signal,
|
||||
files,
|
||||
responses,
|
||||
);
|
||||
let timer: ReturnType<typeof setTimeout> | undefined;
|
||||
try {
|
||||
const outcome = await Promise.race([
|
||||
operation.then((result) => ({ kind: "result" as const, result })),
|
||||
new Promise<{ kind: "deadline" }>((resolve) => {
|
||||
timer = setTimeout(() => resolve({ kind: "deadline" }), 3_000);
|
||||
}),
|
||||
]);
|
||||
expect(outcome.kind).toBe("result");
|
||||
expect(responses.cleanupResources).toEqual(["state response body"]);
|
||||
await expect(responses.close()).rejects.toMatchObject({
|
||||
reason: "unavailable",
|
||||
});
|
||||
if (outcome.kind === "result") {
|
||||
expect(outcome.result).toMatchObject(
|
||||
scenario === "http-status"
|
||||
? { ok: false, error: { data: { reason: "http-status" } } }
|
||||
: {
|
||||
ok: false,
|
||||
error: {
|
||||
message: expect.stringContaining(
|
||||
String(STATE_INTAKE_LIMIT_BYTES),
|
||||
),
|
||||
},
|
||||
},
|
||||
);
|
||||
}
|
||||
} finally {
|
||||
if (timer !== undefined) clearTimeout(timer);
|
||||
await sibling.cancel();
|
||||
await operation;
|
||||
await responses.close();
|
||||
await files.close();
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
it("retains the exact failure when response cancellation rejects", async () => {
|
||||
const failure = new Error("controlled cancellation failure");
|
||||
const body = new ReadableStream<Uint8Array>({
|
||||
cancel() {
|
||||
throw failure;
|
||||
},
|
||||
});
|
||||
const files = new ArtifactResourceScope();
|
||||
const responses = new StateResponseCleanup();
|
||||
const result = await partDocument(
|
||||
{ kind: "file", file: { uri: "https://state.example.test/state.json" } },
|
||||
{
|
||||
fetchState: async () => new Response(body, { status: 503 }),
|
||||
timeoutMs: 1_000,
|
||||
},
|
||||
new AbortController().signal,
|
||||
files,
|
||||
responses,
|
||||
);
|
||||
expect(result).toMatchObject({
|
||||
ok: false,
|
||||
error: { data: { reason: "http-status" } },
|
||||
});
|
||||
await new Promise<void>((resolve) => setImmediate(resolve));
|
||||
const rejected: unknown = await responses
|
||||
.close()
|
||||
.catch((cause: unknown) => cause);
|
||||
expect(rejected).toMatchObject({
|
||||
reason: "unavailable",
|
||||
cleanup: { resources: ["state response body"] },
|
||||
});
|
||||
expect((rejected as Error).cause).toBeInstanceOf(AggregateError);
|
||||
expect(((rejected as Error).cause as AggregateError).errors).toContain(
|
||||
failure,
|
||||
);
|
||||
expect(responses.cleanupResources).toEqual(["state response body"]);
|
||||
await files.close();
|
||||
});
|
||||
@@ -1,3 +1,4 @@
|
||||
import { StateResponseCleanup } from "./StateResponseCleanup.js";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
import { mediaTypeEssence } from "../../domain/analysisView/recordedMediaType.js";
|
||||
@@ -228,6 +229,7 @@ const downloadBounds = (
|
||||
const readCapped = async (
|
||||
body: ReadableStream<Uint8Array> | null,
|
||||
bounds: DownloadBounds,
|
||||
responses: StateResponseCleanup,
|
||||
): Promise<Result<Buffer, RpcError>> => {
|
||||
if (body === null) return ok(Buffer.alloc(0));
|
||||
const reader = body.getReader();
|
||||
@@ -240,13 +242,15 @@ const readCapped = async (
|
||||
if (done) return ok(Buffer.concat(chunks, total));
|
||||
total += value.byteLength;
|
||||
if (total > STATE_INTAKE_LIMIT_BYTES) {
|
||||
await reader.cancel().catch(() => undefined);
|
||||
// A tee's sibling can be caller-owned; cancelling this branch must not
|
||||
// delay the refusal until that sibling finishes.
|
||||
responses.cancel(() => reader.cancel());
|
||||
return err(TOO_LARGE);
|
||||
}
|
||||
chunks.push(value);
|
||||
}
|
||||
} catch {
|
||||
void reader.cancel().catch(() => undefined);
|
||||
responses.cancel(() => reader.cancel());
|
||||
return err(fetchFailed(bounds.failure()));
|
||||
}
|
||||
};
|
||||
@@ -256,6 +260,7 @@ const downloadState = async (
|
||||
url: URL,
|
||||
intake: StateIntake,
|
||||
cancel: AbortSignal,
|
||||
responses: StateResponseCleanup,
|
||||
): Promise<Result<Buffer, RpcError>> => {
|
||||
const bounds = downloadBounds(intake.timeoutMs, cancel);
|
||||
const fetching = Promise.resolve().then(() =>
|
||||
@@ -268,7 +273,9 @@ const downloadState = async (
|
||||
// A response that arrives after the download ended is released unread.
|
||||
void fetching
|
||||
.then(
|
||||
(late) => late.body?.cancel(),
|
||||
(late) => {
|
||||
if (late.body !== null) responses.cancel(() => late.body!.cancel());
|
||||
},
|
||||
() => undefined,
|
||||
)
|
||||
.catch(() => undefined);
|
||||
@@ -276,7 +283,8 @@ const downloadState = async (
|
||||
}
|
||||
const declared = Number(response.headers.get("content-length") ?? "0");
|
||||
if (!response.ok || declared > STATE_INTAKE_LIMIT_BYTES) {
|
||||
await response.body?.cancel().catch(() => undefined);
|
||||
// Cancel this branch without waiting for a potentially independent tee sibling.
|
||||
if (response.body !== null) responses.cancel(() => response.body!.cancel());
|
||||
return err(
|
||||
response.ok
|
||||
? TOO_LARGE
|
||||
@@ -286,7 +294,7 @@ const downloadState = async (
|
||||
},
|
||||
);
|
||||
}
|
||||
return readCapped(response.body, bounds);
|
||||
return readCapped(response.body, bounds, responses);
|
||||
};
|
||||
|
||||
const NETWORK_SHARE = invalidParams(
|
||||
@@ -315,6 +323,7 @@ const fileBytes = async (
|
||||
intake: StateIntake,
|
||||
cancel: AbortSignal,
|
||||
files: ArtifactResourceScope,
|
||||
responses: StateResponseCleanup,
|
||||
): Promise<Result<Uint8Array, RpcError>> => {
|
||||
if (typeof file.bytes === "string" && file.uri === undefined) {
|
||||
if (file.bytes.length > MAX_INLINE_BASE64_CHARACTERS) return err(TOO_LARGE);
|
||||
@@ -333,7 +342,8 @@ const fileBytes = async (
|
||||
invalidParams("a file part carries exactly one of bytes or a uri"),
|
||||
);
|
||||
const url = new URL(file.uri);
|
||||
if (url.protocol === "https:") return downloadState(url, intake, cancel);
|
||||
if (url.protocol === "https:")
|
||||
return downloadState(url, intake, cancel, responses);
|
||||
if (url.protocol !== "file:")
|
||||
return err(invalidParams("a file uri is a file:// or https:// URL"));
|
||||
const path = localFilePath(url);
|
||||
@@ -346,6 +356,7 @@ export const partDocument = async (
|
||||
intake: StateIntake,
|
||||
cancel: AbortSignal,
|
||||
files: ArtifactResourceScope,
|
||||
responses: StateResponseCleanup,
|
||||
): Promise<Result<unknown, RpcError>> => {
|
||||
if (isRecord(part) && part.kind === "data" && isRecord(part.data))
|
||||
return ok(part.data);
|
||||
@@ -362,6 +373,6 @@ export const partDocument = async (
|
||||
mediaTypeEssence(mimeType) !== "application/json")
|
||||
)
|
||||
return err(invalidParams("a file part's mimeType is application/json"));
|
||||
const bytes = await fileBytes(part.file, intake, cancel, files);
|
||||
const bytes = await fileBytes(part.file, intake, cancel, files, responses);
|
||||
return bytes.ok ? parseDocumentBytes(bytes.value) : bytes;
|
||||
};
|
||||
@@ -1066,7 +1066,15 @@ it("enforces a selected command deadline and completes a corrected retry after c
|
||||
if (timed.ok) throw new Error("Selected 20 ms deadline was not enforced");
|
||||
expect(projectAnalysisError(timed.error)).toMatchObject({
|
||||
code: "provider_timeout",
|
||||
details: { timeout_ms: 20 },
|
||||
details: {
|
||||
timeout_ms: 20,
|
||||
limit: {
|
||||
setting: "wasm.command_timeout_ms",
|
||||
effective_value: 20,
|
||||
source: "request",
|
||||
update_mode: "parameter",
|
||||
},
|
||||
},
|
||||
remediation: { action: expect.stringContaining("command_timeout_ms") },
|
||||
});
|
||||
expect(timed.error.cleanupIncomplete).toBe(false);
|
||||
|
||||
@@ -229,6 +229,10 @@ export class WabtArtifactProvider implements WasmArtifactPort {
|
||||
input.command_timeout_ms ??
|
||||
options?.commandTimeoutMs ??
|
||||
DEFAULT_WASM_COMMAND_TIMEOUT_MS;
|
||||
const commandTimeoutSource =
|
||||
input.command_timeout_ms !== undefined
|
||||
? "request"
|
||||
: options?.commandTimeoutSource;
|
||||
try {
|
||||
if (options?.signal?.aborted)
|
||||
throw new AnalysisCancelledError(OPERATION, {
|
||||
@@ -695,6 +699,9 @@ export class WabtArtifactProvider implements WasmArtifactPort {
|
||||
phase === "artifact" ? artifactInput.path : input.path,
|
||||
{
|
||||
commandTimeoutMs,
|
||||
...(commandTimeoutSource === undefined
|
||||
? {}
|
||||
: { commandTimeoutSource }),
|
||||
limits,
|
||||
inputField: artifactInput.field,
|
||||
observedOutput: capturedOutput,
|
||||
|
||||
@@ -52,7 +52,8 @@ it.each(["configuration", "producer"] as const)(
|
||||
limit: {
|
||||
setting: "wasm.command_timeout_ms",
|
||||
effective_value: 1234,
|
||||
rea_restart_required: false,
|
||||
update_mode: "runtime",
|
||||
pending_restart: false,
|
||||
},
|
||||
},
|
||||
recovery: { kind: "inspect_timeout_state", timeout_ms: 1234 },
|
||||
@@ -115,11 +116,30 @@ it.each([1234, 2_147_483_647])(
|
||||
parameter: "command_timeout_ms",
|
||||
applies_at: "next_operation",
|
||||
agent_editable: true,
|
||||
rea_restart_required: false,
|
||||
update_mode: "runtime",
|
||||
pending_restart: false,
|
||||
},
|
||||
},
|
||||
recovery: { kind: "inspect_timeout_state", timeout_ms: commandTimeoutMs },
|
||||
});
|
||||
expect(
|
||||
projectAnalysisError(
|
||||
wabtFailure(
|
||||
new OwnedCommandFailure("timeout", "deadline"),
|
||||
"producer",
|
||||
"/module.wasm",
|
||||
{ commandTimeoutMs, commandTimeoutSource: "request" },
|
||||
),
|
||||
),
|
||||
).toMatchObject({
|
||||
details: {
|
||||
limit: {
|
||||
source: "request",
|
||||
update_mode: "parameter",
|
||||
effective_value: commandTimeoutMs,
|
||||
},
|
||||
},
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
|
||||
@@ -63,6 +63,12 @@ export type WabtVersionProbe = {
|
||||
|
||||
type WabtFailureContext = {
|
||||
readonly commandTimeoutMs: number;
|
||||
readonly commandTimeoutSource?:
|
||||
| "default"
|
||||
| "setting"
|
||||
| "runtime"
|
||||
| "request"
|
||||
| undefined;
|
||||
readonly limits?: WasmResourceLimits;
|
||||
readonly versionProbe?: WabtVersionProbe;
|
||||
readonly inputField?: readonly (string | number)[];
|
||||
@@ -182,13 +188,17 @@ const classifyWabtFailure = (
|
||||
// Classify once before cleanup takes precedence, so the primary remains actionable.
|
||||
const timeoutMs =
|
||||
execution?.commandTimeoutMs ?? DEFAULT_WASM_COMMAND_TIMEOUT_MS;
|
||||
const timeoutSource =
|
||||
execution?.commandTimeoutMs === undefined
|
||||
? "default"
|
||||
: execution.commandTimeoutSource;
|
||||
const timeout =
|
||||
cause.reason === "timeout"
|
||||
? new AnalysisTimeoutError("inspect_wasm_artifact", timeoutMs, {
|
||||
limit: analysisSettingLimit(
|
||||
"wasm.command_timeout_ms",
|
||||
timeoutMs,
|
||||
undefined,
|
||||
timeoutSource,
|
||||
),
|
||||
...options,
|
||||
// The cleanup wrapper retains output once; its nested primary needs only recovery facts.
|
||||
|
||||
@@ -148,7 +148,7 @@ it("advertises exact contracts and recovers a full large-index archive over one
|
||||
expect(updated.structuredContent).toMatchObject({
|
||||
revision: 1,
|
||||
active_work_changed: false,
|
||||
rea_restart_required: false,
|
||||
pending_restart: false,
|
||||
});
|
||||
const recovered = await invoke("inspect_artifact", {});
|
||||
expect(recovered.isError).not.toBe(true);
|
||||
|
||||
@@ -15,6 +15,13 @@ const cases = [
|
||||
0,
|
||||
],
|
||||
['spawn("control", [], {stdio:"inherit"})', "control", "inherit", 0],
|
||||
['spawn("global-nan-env", [], {env:NaN})', "global-nan-env", "default", 0],
|
||||
[
|
||||
'((NaN) => spawn("shadowed-nan-env", [], {env:NaN}))({})',
|
||||
"shadowed-nan-env",
|
||||
"default",
|
||||
0,
|
||||
],
|
||||
[
|
||||
'spawn("spread-after", [], {stdio:"pipe",...input})',
|
||||
"spread-after",
|
||||
@@ -233,6 +240,12 @@ const exportAssignmentSource = `
|
||||
export = argv;
|
||||
`;
|
||||
|
||||
const environmentCommands = [
|
||||
"named-argv",
|
||||
"execfile-dynamic",
|
||||
"shadowed-nan-env",
|
||||
];
|
||||
|
||||
// Callback/options ambiguity cannot change exec's effective stdio.
|
||||
const opaqueExecCallbackCommands = [
|
||||
"eval-callback",
|
||||
@@ -342,7 +355,7 @@ cliTest.for(["CLI", "stdio MCP"] as const)(
|
||||
expect(child.properties.stdio_mode, String(command)).toBe(mode);
|
||||
expect(child.properties.argv_count, String(command)).toBe(count);
|
||||
expect(child.properties.environment_supplied, String(command)).toBe(
|
||||
["named-argv", "execfile-dynamic"].includes(String(command)),
|
||||
environmentCommands.includes(String(command)),
|
||||
);
|
||||
expect(child.identity.source_range).not.toBeNull();
|
||||
const stdio = graph.relations.filter(
|
||||
@@ -377,9 +390,7 @@ cliTest.for(["CLI", "stdio MCP"] as const)(
|
||||
relation.relation === "supplies-env" &&
|
||||
relation.target_node_id === child.node_id,
|
||||
),
|
||||
).toHaveLength(
|
||||
["named-argv", "execfile-dynamic"].includes(String(command)) ? 1 : 0,
|
||||
);
|
||||
).toHaveLength(environmentCommands.includes(String(command)) ? 1 : 0);
|
||||
expect(
|
||||
graph.relations.filter(
|
||||
(relation) =>
|
||||
|
||||
@@ -186,7 +186,8 @@ for (const reader of readers)
|
||||
effective_value: reader.value,
|
||||
applies_at: "next_operation",
|
||||
agent_editable: true,
|
||||
rea_restart_required: false,
|
||||
update_mode: "parameter",
|
||||
pending_restart: false,
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
@@ -87,7 +87,8 @@ it.skipIf(process.versions.bun !== undefined && jobLimit === null)(
|
||||
setting: "capture.decoder_old_generation_mib",
|
||||
effective_value: testHeap.oldGenerationMiB,
|
||||
source: testHeap.source,
|
||||
rea_restart_required: false,
|
||||
update_mode: "runtime",
|
||||
pending_restart: false,
|
||||
},
|
||||
reported_limits:
|
||||
jobLimit === null
|
||||
@@ -147,7 +148,8 @@ it.skipIf(process.versions.bun !== undefined && jobLimit === null)(
|
||||
setting: "capture.decoder_old_generation_mib",
|
||||
effective_value: testHeap.oldGenerationMiB,
|
||||
source: testHeap.source,
|
||||
rea_restart_required: false,
|
||||
update_mode: "runtime",
|
||||
pending_restart: false,
|
||||
},
|
||||
},
|
||||
remediation: {
|
||||
|
||||
@@ -1,81 +1,72 @@
|
||||
import { execFile } from "node:child_process";
|
||||
import { readFile, readdir, writeFile } from "node:fs/promises";
|
||||
import { readFile, readdir } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
import { promisify } from "node:util";
|
||||
import { expect, it } from "vitest";
|
||||
import { expect } from "vitest";
|
||||
import { parse } from "yaml";
|
||||
import { resolveSwiftcCompileTimeoutMs } from "../../../src/process/DarwinProcessRunTokenReader.js";
|
||||
import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js";
|
||||
import { processTest } from "../../support/process/processFixture.js";
|
||||
|
||||
const execute = promisify(execFile);
|
||||
const hostLabels: Partial<Record<NodeJS.Platform, string>> = {
|
||||
linux: "linux",
|
||||
darwin: "macOS",
|
||||
win32: "Windows",
|
||||
};
|
||||
const hostLabel = hostLabels[process.platform];
|
||||
const preparationBudget =
|
||||
process.platform === "darwin" ? resolveSwiftcCompileTimeoutMs() : 0;
|
||||
|
||||
it("enforces workflow platform labels before contacting Cutter", async () => {
|
||||
const workflow = parse(
|
||||
await readFile(".github/workflows/real-cutter.yml", "utf8"),
|
||||
);
|
||||
const labels = workflow.jobs["cutter-upstream"].strategy.matrix.include.map(
|
||||
(entry: { platform: string }) => entry.platform,
|
||||
);
|
||||
expect(labels).toEqual(["linux", "macOS", "Windows"]);
|
||||
const root = await createTestTempDirectory("rea-cutter-platform-");
|
||||
const preload = join(root, "preload.mjs");
|
||||
await writeFile(
|
||||
preload,
|
||||
`
|
||||
import childProcess from 'node:child_process';
|
||||
import { syncBuiltinESMExports } from 'node:module';
|
||||
Object.defineProperty(process, 'platform', { value: process.env.REA_PROBE_PLATFORM });
|
||||
childProcess.execFile = () => { throw new Error('fixture discovery reached'); };
|
||||
syncBuiltinESMExports();
|
||||
`,
|
||||
);
|
||||
const check = async (selected: string | undefined, actual: string) => {
|
||||
const env: NodeJS.ProcessEnv = {
|
||||
...process.env,
|
||||
TMPDIR: root,
|
||||
TMP: root,
|
||||
TEMP: root,
|
||||
REA_PROBE_PLATFORM: actual,
|
||||
processTest.skipIf(hostLabel === undefined)(
|
||||
"enforces workflow platform labels before contacting Cutter",
|
||||
{ timeout: Math.min(2_147_483_647, 2 * preparationBudget + 30_000) },
|
||||
async ({ processes }) => {
|
||||
const workflow = parse(
|
||||
await readFile(".github/workflows/real-cutter.yml", "utf8"),
|
||||
);
|
||||
const labels = workflow.jobs["cutter-upstream"].strategy.matrix.include.map(
|
||||
(entry: { platform: string }) => entry.platform,
|
||||
);
|
||||
expect(labels).toEqual(["linux", "macOS", "Windows"]);
|
||||
const root = await createTestTempDirectory("rea-cutter-platform-");
|
||||
// A private absent directory proves actual discovery without contacting a user's bridge.
|
||||
const bridgeDirectory = join(root, "absent-bridge");
|
||||
const check = async (selected: string | undefined) => {
|
||||
const env: NodeJS.ProcessEnv = {
|
||||
...process.env,
|
||||
TMPDIR: root,
|
||||
TMP: root,
|
||||
TEMP: root,
|
||||
REA_CUTTER_BRIDGE_DIR: bridgeDirectory,
|
||||
};
|
||||
if (selected === undefined) delete env.EXPECTED_PLATFORM;
|
||||
else env.EXPECTED_PLATFORM = selected;
|
||||
const result = await processes.run(
|
||||
process.execPath,
|
||||
["scripts/verify-real-cutter.mjs"],
|
||||
{
|
||||
env,
|
||||
timeoutMs: Math.min(2_147_483_647, preparationBudget + 15_000),
|
||||
},
|
||||
);
|
||||
expect(result.exitCode, result.stderr).toBe(1);
|
||||
return result.stderr;
|
||||
};
|
||||
if (selected === undefined) delete env.EXPECTED_PLATFORM;
|
||||
else env.EXPECTED_PLATFORM = selected;
|
||||
const result: unknown = await execute(
|
||||
process.execPath,
|
||||
["--import", preload, "scripts/verify-real-cutter.mjs"],
|
||||
{ env, timeout: 10_000 },
|
||||
).then(
|
||||
() => {
|
||||
throw new Error("Fixture discovery should stop the verifier");
|
||||
},
|
||||
(error: unknown) => error,
|
||||
for (const selected of [hostLabel, undefined]) {
|
||||
const stderr = await check(selected);
|
||||
expect(stderr).toContain("Open upstream Cutter with the REA plugin");
|
||||
expect(stderr).toContain("no_live_bridge_found");
|
||||
}
|
||||
const other = labels.find((label: string) => label !== hostLabel);
|
||||
expect(other).toBeDefined();
|
||||
expect(await check(other)).toContain(
|
||||
"Cutter runner did not match its declared platform",
|
||||
);
|
||||
if (
|
||||
!(result instanceof Error) ||
|
||||
!("stderr" in result) ||
|
||||
typeof result.stderr !== "string"
|
||||
)
|
||||
throw new Error("Missing verifier diagnostics");
|
||||
return result.stderr;
|
||||
};
|
||||
for (const [selected, actual] of [
|
||||
["linux", "linux"],
|
||||
["macOS", "darwin"],
|
||||
["Windows", "win32"],
|
||||
] as const)
|
||||
expect(await check(selected, actual)).toContain(
|
||||
"fixture discovery reached",
|
||||
expect(await check("unknown")).toContain(
|
||||
"Unrecognized Cutter runner platform",
|
||||
);
|
||||
expect(await check("linux", "darwin")).toContain(
|
||||
"Cutter runner did not match its declared platform",
|
||||
);
|
||||
expect(await check("unknown", "linux")).toContain(
|
||||
"Unrecognized Cutter runner platform",
|
||||
);
|
||||
expect(await check(undefined, "linux")).toContain(
|
||||
"fixture discovery reached",
|
||||
);
|
||||
expect(
|
||||
(await readdir(root)).filter((name) =>
|
||||
name.startsWith("rea-cutter-smoke-"),
|
||||
),
|
||||
).toEqual([]);
|
||||
});
|
||||
expect(
|
||||
(await readdir(root)).filter((name) =>
|
||||
name.startsWith("rea-cutter-smoke-"),
|
||||
),
|
||||
).toEqual([]);
|
||||
},
|
||||
);
|
||||
@@ -216,7 +216,8 @@ it("names the default header admission budget and its setting when source select
|
||||
applies_at: "next_operation",
|
||||
agent_editable: true,
|
||||
scope: "rea-runtime",
|
||||
rea_restart_required: false,
|
||||
update_mode: "runtime",
|
||||
pending_restart: false,
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
Loaded 100 of 114 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user