mirror of
https://github.com/morluto/rea.git
synced 2026-10-12 02:19:24 +00:00
chore(review): integrate main for returned-target index verification
This commit is contained in:
@@ -31,7 +31,7 @@ When tools are absent or registration is stale:
|
||||
`npx -y rea-agents@latest doctor --client codex --json`. Substitute the current
|
||||
supported client: `claude_code`, `claude_desktop`, `codex`, `cursor`,
|
||||
`gemini_cli`, `windsurf`, `devin`, `opencode`, `antigravity`, `copilot_cli`,
|
||||
`commandcode`, `vscode`, `grok_build`, `omp`, or `grok_bot`. `grok_bot` has no local
|
||||
`commandcode`, `qwen_code`, `vscode`, `grok_build`, `omp`, `pi`, `hermes`, or `grok_bot`. `grok_bot` has no local
|
||||
file registration: doctor reports the Grok Bot chat step, and setup does not
|
||||
write `mcp.json`. If the client is unknown, use `doctor --json` and
|
||||
inspect its registration results before choosing a setup scope.
|
||||
|
||||
+73
-7
@@ -175,9 +175,26 @@ Setup and doctor follow Qwen Code's tilde and working-directory-relative home
|
||||
overrides. Qwen Code also discovers personal skills in `~/.agents/skills`,
|
||||
where setup installs the bundled workflow.
|
||||
|
||||
Gemini CLI uses `$GEMINI_CLI_HOME/.gemini/settings.json` when that home override
|
||||
is nonempty, including its shared personal skills at
|
||||
`$GEMINI_CLI_HOME/.agents/skills`. Setup and doctor check MCP allow/exclude
|
||||
policies from system defaults, user settings, trusted workspace settings, and
|
||||
system overrides. Workspace trust follows `GEMINI_CLI_TRUST_WORKSPACE`,
|
||||
`GEMINI_RESTRICTED_MODE`, the effective `security.folderTrust.enabled` setting,
|
||||
and the selected `trustedFolders.json` (including
|
||||
`GEMINI_CLI_TRUSTED_FOLDERS_PATH`). Settings in an untrusted workspace do not
|
||||
participate. A workspace or system `mcpServers.rea` replaces the entire user
|
||||
entry; setup stops if the winning definition conflicts with its planned entry.
|
||||
Review those files manually: setup never edits administrator/workspace settings
|
||||
or grants workspace trust. Trust supplied only by a running IDE is outside this
|
||||
standalone CLI check.
|
||||
|
||||
Codex treats an unset or empty `CODEX_HOME` as `~/.codex`. Devin honors
|
||||
`XDG_CONFIG_HOME` for its `devin/mcp_config.json` on non-Windows systems.
|
||||
|
||||
For OMP, setup writes a `type: "stdio"` entry to the user-level
|
||||
`~/.omp/agent/mcp.json`. It follows `PI_CONFIG_DIR`, an absolute
|
||||
`PI_CODING_AGENT_DIR`, and the profile selected by `OMP_PROFILE` or
|
||||
`~/.omp/agent/mcp.json`. It follows `PI_CONFIG_DIR` and
|
||||
`PI_CODING_AGENT_DIR` (relative to the setup working directory), and the profile selected by `OMP_PROFILE` or
|
||||
`PI_PROFILE` (`~/.omp/profiles/<name>/agent/mcp.json`). Setup also removes
|
||||
`rea` from that file's `disabledServers` list, which would otherwise hide the
|
||||
registration. Doctor treats an `enabled: false` entry as active when
|
||||
@@ -191,17 +208,65 @@ Windows, `~/.hermes` elsewhere — including any `HERMES_DATA_DIR_SUFFIX`. Herme
|
||||
connects an entry unless `enabled` is false, so setup states `enabled: true`.
|
||||
|
||||
Hermes personal skills install under the active Hermes home's `skills` directory.
|
||||
Setup, doctor and uninstall follow the root's `active_profile` selection, as
|
||||
plain Hermes CLI commands do. An explicit `HERMES_HOME` pointing to
|
||||
`profiles/<name>` selects that named profile directly. An invalid, missing or
|
||||
deleted sticky profile is reported instead of writing into the default home;
|
||||
repair the profile or run `hermes profile use default` before retrying.
|
||||
REA follows `HERMES_HOME` whitespace trimming, environment-variable expansion,
|
||||
and home expansion before resolving that profile. Setup previews the resolved
|
||||
configuration and skill destinations; doctor and uninstall use those same paths.
|
||||
|
||||
### DeepSeek Harness
|
||||
|
||||
DeepSeek Harness uses a Cordis YAML patch sequence rather than a `mcpServers`
|
||||
object. REA's `setup --client` does not edit this format. For Harness
|
||||
`0.2.0-rc.2`, append the following entry to `$DSH_HOME/cordis.patch.yml`
|
||||
(`~/.dsh/cordis.patch.yml` when `DSH_HOME` is unset). Preserve existing patches
|
||||
and replace the executable paths with absolute paths from your installation:
|
||||
|
||||
```yaml
|
||||
- insert:
|
||||
- id: rea-mcp
|
||||
name: "@deepseek-ai/dsh-mcp-client"
|
||||
config:
|
||||
serverName: rea
|
||||
transport: stdio
|
||||
command: /absolute/path/to/node
|
||||
args: ["/absolute/path/to/rea/scripts/rea.mjs", mcp]
|
||||
cwd: !!js process.cwd()
|
||||
```
|
||||
|
||||
For a global npm installation, the launcher is under
|
||||
`$(npm root -g)/rea-agents/scripts/rea.mjs`. Check the Node executable with
|
||||
`command -v node`. Register `serverName: rea` only once in each active profile,
|
||||
then restart Harness. It exposes tools as `mcp__rea__<tool_name>`.
|
||||
Run `rea setup --skill --dry-run`, review the destination, then
|
||||
`rea setup --skill --yes` to install the workflow in `~/.agents/skills`.
|
||||
Harness discovers that directory, or `$DSH_AGENTS_HOME/skills` when configured;
|
||||
its native `skill` tool can load `reverse-engineer-anything`.
|
||||
|
||||
REA's optional [native client verification](testing.md#end-to-end-integration-and-golden-evidence)
|
||||
checks headless chat, skill loading, all tool input schemas and actual analysis
|
||||
using the installed Harness and a local model fixture. This establishes the
|
||||
Harness integration, with live DeepSeek API compatibility remaining unverified.
|
||||
See the [Harness MCP guide](https://github.com/deepseek-ai/deepseek-harness/blob/639ed015397290b3745d163aafe02ffee4aa3f84/packages/mcp/mcp-client/README.md)
|
||||
for its native configuration contract.
|
||||
|
||||
### Other client settings
|
||||
|
||||
For OpenCode, setup writes the V1 `mcp.rea` entry, which OpenCode V1 and V2
|
||||
both load. If the configuration already uses OpenCode V2's native
|
||||
`mcp.servers` table, setup registers REA there instead and replaces any earlier
|
||||
`mcp.rea` entry from REA.
|
||||
`mcp.rea` entry from REA. Setup honors `OPENCODE_CONFIG_DIR` and
|
||||
`OPENCODE_CONFIG`, preferring the later-loaded JSONC file when both JSON and
|
||||
JSONC exist. The ordinary XDG global configuration still participates when a
|
||||
custom directory is selected. Doctor checks the merged user registration; uninstall removes
|
||||
REA-owned entries from every applicable user file so an older entry cannot
|
||||
become active again. Malformed applicable files block setup before changes.
|
||||
|
||||
Grok Build loads `[mcp_servers.rea]` from `$GROK_HOME/config.toml`, or from
|
||||
`~/.grok/config.toml` when `GROK_HOME` is unset. Setup edits that server
|
||||
`~/.grok/config.toml` when `GROK_HOME` is unset or empty. Setup edits that server
|
||||
table, `[mcp_servers.rea.env]`, and a root `disabled_mcp_servers` entry that
|
||||
names `rea`. It sets `startup_timeout_sec = 30` and leaves every other name
|
||||
in that list. The shared skill installed under `~/.agents/skills` is already
|
||||
@@ -241,9 +306,10 @@ save verified paths for an existing Ghidra installation.
|
||||
|
||||
The bundled skill is installed where each selected client discovers personal
|
||||
skills: Claude Code uses `~/.claude/skills` (or
|
||||
`$CLAUDE_CONFIG_DIR/skills` when configured), while other supported clients,
|
||||
including Qwen Code, use the shared `~/.agents/skills` directory. A mixed selection
|
||||
plans the relevant paths. Selecting the skill without a client uses the shared
|
||||
`$CLAUDE_CONFIG_DIR/skills` when configured). Gemini CLI uses `.agents/skills`
|
||||
under its selected home; Antigravity 2.0 uses `~/.gemini/config/skills`.
|
||||
Other integrations, including Qwen Code, use `~/.agents/skills`.
|
||||
A mixed selection plans each distinct destination. Selecting the skill without a client uses the shared
|
||||
directory. Setup leaves existing skill copies in other locations untouched.
|
||||
|
||||
`doctor --skill --json` verifies the selected copies against the bundled
|
||||
|
||||
@@ -36,6 +36,49 @@ after SDK conversion and check advertised validation and actual calls.
|
||||
Individual model APIs can impose additional nesting limits; complete producer
|
||||
captures can exceed ten structural levels.
|
||||
|
||||
### Compact input schema profile
|
||||
|
||||
Some function-calling providers reject a request while any single tool's
|
||||
`parameters` schema exceeds a literal serialized size cap; Moonshot clients
|
||||
such as kimi-cli fail every request while the default catalog is attached
|
||||
([#1484](https://github.com/morluto/rea/issues/1484)). Setting the server's
|
||||
`REA_MCP_INPUT_SCHEMA_PROFILE` environment variable to `compact` renders every
|
||||
advertised input schema within a 13,312-byte budget, measured to sit below
|
||||
Moonshot's observed cap (community measurements: a 13.7 KB schema is accepted
|
||||
while a ~15 KB one is rejected). The setting is read at startup; restart the
|
||||
server to apply changes, as with `REA_MCP_MAX_RESPONSE_BYTES`. The default
|
||||
value `full` keeps the complete advertisement unchanged.
|
||||
|
||||
The compact profile renders the same tool set, output schemas, annotations,
|
||||
and canonical server-side validation. It changes only the advertised input
|
||||
JSON Schema:
|
||||
|
||||
- Annotation prose leaves the advertised form: nested property descriptions,
|
||||
literal examples, defaults, and titles are dropped. The root keeps its
|
||||
description because it states the accepted input groups.
|
||||
- Schema-local references are inlined within the budget. Kimi Code expands
|
||||
references before sending provider requests, so sharing definitions cannot
|
||||
establish the provider-facing size. Expansion stops at the budget before
|
||||
allocating an oversized presentation. Unconstrained JSON fields advertise
|
||||
all JSON value types explicitly so Kimi cannot infer string for Evidence
|
||||
objects or arrays.
|
||||
- The few schemas whose validation structure alone exceeds the budget
|
||||
(`compare_web_captures`, `build_reconstruction_obligation_ledger`,
|
||||
`evaluate_reconstruction_coverage`, `project_managed_application_graph`,
|
||||
`compare_application_versions`, `capture_browser_scenario`) are advertised
|
||||
in a reduced form: property names, reference-resolved types (including union
|
||||
alternatives), first-level guidance, and the root's required-field constraints,
|
||||
with nested validation detail elided. The
|
||||
canonical schema still rejects malformed calls server-side, so callers
|
||||
following the reduced advertisement receive the canonical typed error
|
||||
instead of silent acceptance.
|
||||
|
||||
Boundary tests pin every advertised compact schema to the budget, keep them
|
||||
valid JSON Schema that still accepts every canonical example, and hold the
|
||||
set of six reduced presentations as contracts evolve. Client verification
|
||||
also checks provider-facing sizes and canonical examples after conversion by
|
||||
the actual Kimi Code CLI; local captures do not establish Moonshot acceptance.
|
||||
|
||||
Self-contained output schemas advertise a content-bound `$id`, including their
|
||||
declared dialect. SDK validators can reuse compiled schemas across complete
|
||||
catalog refreshes and equivalent tool outputs; changing the schema changes its
|
||||
@@ -101,6 +144,12 @@ if retention is unavailable or fails, the diagnostic reports that reason.
|
||||
REA accepts ordinary `tools/call` progress tokens. Updates are monotonic,
|
||||
rate-bounded to at most one intermediate update per 100 ms, and always allow a
|
||||
terminal update. Unknown totals are omitted; REA does not fabricate percentages.
|
||||
`capture_process_scenario` uses that boundary only when the client supplies a
|
||||
progress token. Notifications set `progress` to the count of collected terminal
|
||||
frames, process samples, and interaction events, omit `total`, and put the
|
||||
lifecycle phase, elapsed time, and final cleanup status in `message`. A client
|
||||
that does not send a progress token receives no progress notifications. The
|
||||
tool result is unchanged either way.
|
||||
Provider calls receive the request cancellation signal. Artifact traversal,
|
||||
hashing, version comparisons, Hopper requests, and process capture
|
||||
check the same signal. Cancellation is distinct from timeout. A cleanup failure
|
||||
|
||||
@@ -51,6 +51,34 @@ format is TOON. The file contains the complete capture Evidence record.
|
||||
Choose an output file distinct from the scenario input: shell redirection opens
|
||||
and truncates the output before REA reads the scenario.
|
||||
|
||||
While the capture runs, REA writes live status to stderr as one JSON object per
|
||||
line. Stdout stays the final Evidence document:
|
||||
|
||||
```json
|
||||
{
|
||||
"rea_progress": {
|
||||
"phase": "running",
|
||||
"completed": 3,
|
||||
"total": null,
|
||||
"message": "elapsed_ms=1200 frames=1 samples=2 interactions=0",
|
||||
"sequence": 4
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
`phase` is the observed lifecycle stage: `prepare`, `running`, `settling`, or
|
||||
`cleanup`. `completed` counts collected terminal frames, process samples, and
|
||||
interaction events. `total` stays null; REA does not invent a unit total or a
|
||||
percentage. `message` reports elapsed time and those counts. The final line uses
|
||||
`phase` `cleanup`, sets `terminal` to true, and adds `disposition` (`exited`,
|
||||
`timeout`, `idle_timeout`, `cancelled`, or `failed`) plus the owned-process,
|
||||
renderer, and temporary-root cleanup states. Updates use the shared progress
|
||||
boundary and are limited to one intermediate line per 100 ms; the terminal line
|
||||
is always emitted. If a receiver is slow, intermediate observations are merged
|
||||
into the latest status, and stale live observations are discarded before final
|
||||
cleanup status. Progress never copies child output. Child output remains in the
|
||||
capture Evidence on stdout.
|
||||
|
||||
For example:
|
||||
|
||||
```json
|
||||
@@ -119,6 +147,14 @@ selected normalization. Original admitted PTY chunks are available through
|
||||
comparison contract, so a capture whose lines were padded to full width does not
|
||||
compare as the same contract.
|
||||
|
||||
Custom `normalization.patterns` replace literal text with the caller's literal
|
||||
replacement, including dollar sequences such as `$&` and `$$`. An empty pattern
|
||||
also matches the first and last string positions. Captures with custom patterns
|
||||
commit `pattern_normalization_version: "literal-replacements-v1"`. Legacy captures
|
||||
remain readable. Cross-version comparison is rejected; capture both sides with
|
||||
the same replacement semantics. Captures without custom patterns keep their
|
||||
existing comparison contract.
|
||||
|
||||
Port normalization recognizes explicit `port`, `tcp_port`, `udp_port`, and
|
||||
`listen` fields, URL authorities, IP endpoints, and `localhost` endpoints, with
|
||||
ports from 0 through 65535. A port may end a sentence, as in `port: 8080.`;
|
||||
|
||||
@@ -88,6 +88,51 @@ entrypoint and include its verifier/fixture files in their cache inputs.
|
||||
|
||||
## End-to-end, integration and golden evidence
|
||||
|
||||
The optional `verify:qwen-client`, `verify:pi-client`, and `verify:hermes-client`
|
||||
lanes require an installed native client. Select its executable with
|
||||
`REA_VERIFY_QWEN_COMMAND`, `REA_VERIFY_PI_COMMAND`, or `REA_VERIFY_HERMES_COMMAND`.
|
||||
They configure disposable profiles through the public REA CLI, preserve caller
|
||||
settings and backups, and exercise real stdio MCP calls and skill loading. The
|
||||
`call` mode checks complete catalog discovery and actual JavaScript Evidence;
|
||||
pass `-- chat` to check plain chat with REA enabled. Qwen also reads the full
|
||||
result that its client offloads, and Pi exercises default codemode execution.
|
||||
Use `REA_VERIFY_RUNTIME_ROOT` to select a production-only installed REA package.
|
||||
Receipts retain client versions, result digests, host coverage and owned-process
|
||||
cleanup. These POSIX lanes use a deterministic loopback model, so they do not
|
||||
prove live model-provider or native Windows compatibility. Qwen and Pi use
|
||||
caller-configured skill directories to isolate the fixture account from the OS
|
||||
home; this does not prove their default home-directory discovery.
|
||||
Set `REA_VERIFY_HERMES_STICKY_PROFILE=1` to exercise native Hermes selection of
|
||||
a named sticky profile with `HERMES_HOME` still pointing to its root.
|
||||
|
||||
`verify:gemini-client` requires an installed Gemini CLI (verified with
|
||||
`@google/[email protected]`); select it with `REA_VERIFY_GEMINI_COMMAND`.
|
||||
The optional POSIX lane uses native `GEMINI_CLI_HOME` discovery in an isolated
|
||||
Git project, checks setup plans, backups and idempotence, activates the installed
|
||||
personal skill, validates all forwarded input JSON Schemas against their declared
|
||||
dialect, forwards the complete REA catalog and checks full JavaScript
|
||||
Evidence for a Unicode path. Use `-- chat` for ordinary chat with REA enabled,
|
||||
or `REA_VERIFY_RUNTIME_ROOT` for a production-only installed REA package.
|
||||
The local fixture exercises the real Gemini API adapter, including its
|
||||
`untrusted_context` tool-result envelope. Token counts are synthetic; live
|
||||
Google API and native Windows compatibility remain unverified. The lane disables
|
||||
the client's memory-based relaunch to preserve the caller's Node heap budget.
|
||||
|
||||
`verify:deepseek-client` requires an installed DeepSeek Harness (`dsh`;
|
||||
verified with `@deepseek-ai/[email protected]`) and Git. Set
|
||||
`REA_VERIFY_DEEPSEEK_COMMAND` to select its executable. This optional POSIX lane
|
||||
installs the skill through `rea setup --skill`, writes the native Cordis MCP
|
||||
patch into an isolated `DSH_HOME`, and uses `DSH_AGENTS_HOME` for native personal
|
||||
skill discovery. A separate Git root prevents inherited project skills from
|
||||
masking a missing installation. The default `call` mode checks skill loading,
|
||||
complete catalog discovery, the actual forwarded input schemas and regexes,
|
||||
and full JavaScript Evidence for a Unicode path. Use `-- chat` for ordinary
|
||||
chat with REA enabled, or `REA_VERIFY_RUNTIME_ROOT` for an installed package.
|
||||
The fixture uses Harness's custom OpenAI adapter and the one-million-token
|
||||
context capacity declared by its default DeepSeek Flash model; it does not
|
||||
establish live DeepSeek API or native Windows coverage. Receipts include
|
||||
request artifacts, result digests and owned-process lineage.
|
||||
|
||||
Full E2E tests invoke the production command dispatcher and real providers,
|
||||
without fake launchers, runners or responses. `verify:keyed-archive` writes an
|
||||
actual Foundation binary archive, runs the CLI and a separate stdio MCP
|
||||
|
||||
@@ -179,6 +179,11 @@
|
||||
"verify:readiness": "npm run build:cached && npm run verify:fixtures && node scripts/verify-reconstruction-readiness.mjs",
|
||||
"verify:package": "npm run build:cached && node scripts/verify-package.mjs",
|
||||
"verify:package:windows": "npm run build:cached && node scripts/verify-windows-package.mjs",
|
||||
"verify:qwen-client": "npm run build:cached && node scripts/verify-qwen-client.mjs",
|
||||
"verify:gemini-client": "npm run build:cached && node scripts/verify-gemini-client.mjs",
|
||||
"verify:pi-client": "npm run build:cached && node scripts/verify-native-agent-client.mjs pi",
|
||||
"verify:hermes-client": "npm run build:cached && node scripts/verify-native-agent-client.mjs hermes",
|
||||
"verify:deepseek-client": "npm run build:cached && node scripts/verify-deepseek-client.mjs",
|
||||
"probe:windows-capabilities": "npm run build:cached && node scripts/windows-capabilities.mjs",
|
||||
"verify:published": "node scripts/verify-published-package.mjs",
|
||||
"config:print": "node scripts/print-mcp-config.mjs",
|
||||
|
||||
@@ -40,7 +40,11 @@ export async function verifyGhidraBoundaries(
|
||||
ajv.validateSchema(tool[key]),
|
||||
`${tool.name} ${key}: ${JSON.stringify(ajv.errors)}`,
|
||||
);
|
||||
validators.set(tool.name, ajv.compile(tool.outputSchema));
|
||||
validators.set(
|
||||
tool.name,
|
||||
(tool.outputSchema?.$id && ajv.getSchema(tool.outputSchema.$id)) ||
|
||||
ajv.compile(tool.outputSchema),
|
||||
);
|
||||
}
|
||||
let successfulCalls = 0;
|
||||
let rejectedCalls = 0;
|
||||
|
||||
@@ -42,7 +42,11 @@ export async function verifyHopperBoundaryContracts(
|
||||
ajv.validateSchema(tool.outputSchema),
|
||||
`${tool.name} output schema`,
|
||||
);
|
||||
validators.set(tool.name, ajv.compile(tool.outputSchema));
|
||||
validators.set(
|
||||
tool.name,
|
||||
(tool.outputSchema?.$id && ajv.getSchema(tool.outputSchema.$id)) ||
|
||||
ajv.compile(tool.outputSchema),
|
||||
);
|
||||
}
|
||||
let successfulCalls = 0;
|
||||
let rejectedCalls = 0;
|
||||
|
||||
@@ -0,0 +1,346 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { createHash } from "node:crypto";
|
||||
import { mkdir, mkdtemp, readFile, writeFile } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { dirname, join } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { Ajv2020 } from "ajv/dist/2020.js";
|
||||
import {
|
||||
TOOL_CONTRACTS,
|
||||
toolContract,
|
||||
} from "../dist/contracts/toolContracts.js";
|
||||
import { exec } from "./lib/verify-package-core.mjs";
|
||||
import { createVerifierRun, completeVerifierRun } from "./lib/verifier-run.mjs";
|
||||
import { createOpenAiModelFixture } from "./verify/clients/model-fixture.mjs";
|
||||
|
||||
const mode = process.argv[2] ?? "call";
|
||||
assert(["chat", "call"].includes(mode), "Select chat or call.");
|
||||
assert(process.platform !== "win32", "Native Windows coverage is unverified.");
|
||||
const repo = fileURLToPath(new URL("..", import.meta.url));
|
||||
const runtimeRoot = process.env.REA_VERIFY_RUNTIME_ROOT ?? repo;
|
||||
const command = process.env.REA_VERIFY_DEEPSEEK_COMMAND ?? "dsh";
|
||||
const lab = await mkdtemp(join(tmpdir(), "rea-deepseek-client-"));
|
||||
const account = join(lab, "account");
|
||||
const profile = join(lab, "selected profile");
|
||||
const workspace = join(lab, "workspace");
|
||||
const target = join(workspace, "fixture α测试");
|
||||
const skillPath = join(
|
||||
account,
|
||||
".agents",
|
||||
"skills",
|
||||
"reverse-engineer-anything",
|
||||
"SKILL.md",
|
||||
);
|
||||
const cli = join(runtimeRoot, "scripts", "rea.mjs");
|
||||
const verifier = createVerifierRun();
|
||||
await Promise.all([
|
||||
mkdir(account),
|
||||
mkdir(profile),
|
||||
mkdir(target, { recursive: true }),
|
||||
]);
|
||||
const environment = {
|
||||
PATH: process.env.PATH ?? "",
|
||||
USERPROFILE: account,
|
||||
DSH_HOME: profile,
|
||||
DSH_AGENTS_HOME: join(account, ".agents"),
|
||||
OPENAI_API_KEY: "local-fixture-key",
|
||||
REA_PROCESS_RUN_ID: verifier.run_id,
|
||||
};
|
||||
// A temporary directory under the checkout must not inherit its project skills or AGENTS.md.
|
||||
await exec(
|
||||
"git",
|
||||
["-c", "init.defaultBranch=main", "init", "--quiet", workspace],
|
||||
{
|
||||
env: environment,
|
||||
timeout: 30_000,
|
||||
},
|
||||
);
|
||||
await writeFile(
|
||||
join(target, "app.js"),
|
||||
'export function result() { return { clientCompatibility: "REA_CLIENT_FIXTURE", count: 7 }; }\n',
|
||||
);
|
||||
const clientVersion = (
|
||||
await exec(command, ["--version"], {
|
||||
env: environment,
|
||||
cwd: workspace,
|
||||
timeout: 30_000,
|
||||
})
|
||||
).stdout.trim();
|
||||
for (const args of [
|
||||
["setup", "--skill", "--dry-run"],
|
||||
["setup", "--skill", "--yes"],
|
||||
]) {
|
||||
const result = await exec(process.execPath, [cli, ...args, "--json"], {
|
||||
env: environment,
|
||||
cwd: workspace,
|
||||
timeout: 60_000,
|
||||
});
|
||||
const parsed = JSON.parse(result.stdout);
|
||||
assert.equal(parsed.status, args.includes("--dry-run") ? "planned" : "ready");
|
||||
if (args.includes("--dry-run")) {
|
||||
assert.equal(parsed.plannedActions.length, 1);
|
||||
assert.equal(parsed.plannedActions[0].kind, "install_skill");
|
||||
assert.equal(parsed.plannedActions[0].target, dirname(skillPath));
|
||||
}
|
||||
await writeFile(
|
||||
join(lab, args.includes("--dry-run") ? "setup-plan.json" : "setup.json"),
|
||||
result.stdout,
|
||||
);
|
||||
}
|
||||
assert(
|
||||
(await readFile(skillPath, "utf8")).includes(
|
||||
"name: reverse-engineer-anything",
|
||||
),
|
||||
);
|
||||
|
||||
let catalog;
|
||||
let patternCount = 0;
|
||||
let skillRequested = false;
|
||||
let skillLoaded = false;
|
||||
let evidenceSeen = false;
|
||||
let finalSeen = false;
|
||||
let artifactDigest;
|
||||
let mainRequests = 0;
|
||||
const ajv = new Ajv2020({ strict: false, validateFormats: false });
|
||||
const patterns = new Set();
|
||||
const validatePatterns = (value) => {
|
||||
if (!value || typeof value !== "object") return;
|
||||
for (const [key, child] of Object.entries(value)) {
|
||||
if (key === "pattern" && typeof child === "string") {
|
||||
// Check the representation the native client actually forwards to its model adapter.
|
||||
for (const flags of ["", "u", "v"]) new RegExp(child, flags);
|
||||
patterns.add(child);
|
||||
patternCount++;
|
||||
} else validatePatterns(child);
|
||||
}
|
||||
};
|
||||
const extractEvidence = (value) => {
|
||||
if (typeof value === "string") {
|
||||
try {
|
||||
return extractEvidence(JSON.parse(value));
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
if (value && typeof value === "object") {
|
||||
if (
|
||||
typeof value.evidence_id === "string" &&
|
||||
value.normalized_result?.graph?.nodes
|
||||
)
|
||||
return value;
|
||||
for (const child of Object.values(value)) {
|
||||
const found = extractEvidence(child);
|
||||
if (found) return found;
|
||||
}
|
||||
}
|
||||
};
|
||||
const fixture = await createOpenAiModelFixture({
|
||||
directory: lab,
|
||||
prefix: `deepseek-${mode}`,
|
||||
onRequest: async ({ body, tools, requestIndex }) => {
|
||||
// Harness generates a session title in a separate, tool-free model request.
|
||||
if (tools.length === 0)
|
||||
return { role: "assistant", content: "REA validation" };
|
||||
assert(
|
||||
++mainRequests <= 4,
|
||||
"Harness did not complete the bounded fixture workflow.",
|
||||
);
|
||||
const rea = tools.filter((tool) => tool.name.startsWith("mcp__rea__"));
|
||||
const observedCatalog = rea
|
||||
.map((tool) => tool.name.slice("mcp__rea__".length))
|
||||
.sort();
|
||||
assert.deepEqual(
|
||||
observedCatalog,
|
||||
TOOL_CONTRACTS.map((tool) => tool.name).sort(),
|
||||
);
|
||||
if (catalog === undefined) {
|
||||
catalog = observedCatalog;
|
||||
for (const tool of rea) {
|
||||
assert(
|
||||
ajv.validateSchema(tool.parameters),
|
||||
`${tool.name}: ${ajv.errorsText(ajv.errors)}`,
|
||||
);
|
||||
validatePatterns(tool.parameters);
|
||||
}
|
||||
assert(
|
||||
patternCount > 0,
|
||||
"The advertised regex constraints must reach the model.",
|
||||
);
|
||||
}
|
||||
const toolMessages = (body.messages ?? []).filter(
|
||||
(message) => message.role === "tool",
|
||||
);
|
||||
const serialized = JSON.stringify(toolMessages);
|
||||
skillLoaded ||=
|
||||
serialized.includes(
|
||||
'<skill_content name=\\"reverse-engineer-anything\\">',
|
||||
) && serialized.includes(dirname(skillPath));
|
||||
for (const message of toolMessages) {
|
||||
const found = extractEvidence(message.content);
|
||||
if (found === undefined) continue;
|
||||
const evidence = toolContract(
|
||||
"analyze_javascript_application",
|
||||
).outputSchema.parse(found);
|
||||
assert.equal(evidence.subject.local_path, target);
|
||||
const shape = evidence.normalized_result.graph.nodes
|
||||
.flatMap((node) => node.observations)
|
||||
.map((observation) => observation.properties)
|
||||
.find(
|
||||
(properties) =>
|
||||
properties.semantic_role === "export-return-shapes" &&
|
||||
properties.exported_name === "result",
|
||||
);
|
||||
assert(shape?.return_shape_coverage?.projection_complete);
|
||||
const fields = shape.static_return_shapes.flatMap(
|
||||
(entry) => entry.fields,
|
||||
);
|
||||
assert(
|
||||
fields.some(
|
||||
(field) =>
|
||||
field.path === "/clientCompatibility" &&
|
||||
field.state === "literal" &&
|
||||
field.value === "REA_CLIENT_FIXTURE",
|
||||
),
|
||||
);
|
||||
assert(
|
||||
fields.some(
|
||||
(field) =>
|
||||
field.path === "/count" &&
|
||||
field.state === "literal" &&
|
||||
field.value === 7,
|
||||
),
|
||||
);
|
||||
const text = JSON.stringify(evidence);
|
||||
await writeFile(join(lab, "evidence.json"), text);
|
||||
artifactDigest = createHash("sha256").update(text).digest("hex");
|
||||
evidenceSeen = true;
|
||||
}
|
||||
if (mode === "chat" || evidenceSeen) {
|
||||
finalSeen = true;
|
||||
return { role: "assistant", content: "REA_CLIENT_COMPATIBILITY_OK" };
|
||||
}
|
||||
const call = (name, args) => ({
|
||||
role: "assistant",
|
||||
content: null,
|
||||
tool_calls: [
|
||||
{
|
||||
id: `call_rea_${requestIndex}`,
|
||||
type: "function",
|
||||
function: { name, arguments: JSON.stringify(args) },
|
||||
},
|
||||
],
|
||||
});
|
||||
if (!skillRequested) {
|
||||
assert(tools.some((tool) => tool.name === "skill"));
|
||||
skillRequested = true;
|
||||
return call("skill", { name: "reverse-engineer-anything" });
|
||||
}
|
||||
assert(
|
||||
skillLoaded,
|
||||
"Harness must load the installed skill before analysis.",
|
||||
);
|
||||
return call("mcp__rea__analyze_javascript_application", {
|
||||
input_path: target,
|
||||
format: "directory",
|
||||
});
|
||||
},
|
||||
});
|
||||
let failure;
|
||||
try {
|
||||
// Use the client's native Cordis patch DSL; generic `rea setup --client` does not edit it.
|
||||
await writeFile(
|
||||
join(profile, "cordis.patch.yml"),
|
||||
`
|
||||
- id: llm-pi-ai
|
||||
config:
|
||||
providers:
|
||||
rea-fixture:
|
||||
apiKeyEnv: OPENAI_API_KEY
|
||||
api: openai-completions
|
||||
baseURL: ${fixture.baseUrl}
|
||||
defaultMaxTokens: 4096
|
||||
retryPolicy:
|
||||
mode: normal
|
||||
maxRetries: 0
|
||||
models:
|
||||
- id: rea-client-fixture
|
||||
contextWindow: 1000000
|
||||
- id: agent-default-model
|
||||
config:
|
||||
provider: rea-fixture
|
||||
model: rea-client-fixture
|
||||
- insert:
|
||||
- id: rea-mcp
|
||||
name: '@deepseek-ai/dsh-mcp-client'
|
||||
config:
|
||||
serverName: rea
|
||||
transport: stdio
|
||||
command: ${JSON.stringify(process.execPath)}
|
||||
args: ${JSON.stringify([cli, "mcp"])}
|
||||
cwd: !!js process.cwd()
|
||||
env:
|
||||
REA_PROCESS_RUN_ID: ${JSON.stringify(verifier.run_id)}
|
||||
`,
|
||||
);
|
||||
const execution = exec(
|
||||
command,
|
||||
[
|
||||
"--profile",
|
||||
"headless",
|
||||
"--json",
|
||||
mode === "chat" ? "Say hello briefly." : `Use REA to analyze ${target}.`,
|
||||
],
|
||||
{
|
||||
env: environment,
|
||||
cwd: workspace,
|
||||
timeout: 120_000,
|
||||
maxBuffer: 10 * 1024 * 1024,
|
||||
},
|
||||
);
|
||||
execution.child.stdin.end();
|
||||
const result = await execution;
|
||||
await writeFile(join(lab, "client.stdout"), result.stdout);
|
||||
await writeFile(join(lab, "client.stderr"), result.stderr);
|
||||
assert.equal(fixture.failure, undefined);
|
||||
assert(finalSeen && result.stdout.includes("REA_CLIENT_COMPATIBILITY_OK"));
|
||||
if (mode === "call") assert(evidenceSeen && skillLoaded);
|
||||
} catch (cause) {
|
||||
failure = cause;
|
||||
if (typeof cause?.stdout === "string")
|
||||
await writeFile(join(lab, "client.stdout"), cause.stdout);
|
||||
if (typeof cause?.stderr === "string")
|
||||
await writeFile(join(lab, "client.stderr"), cause.stderr);
|
||||
} finally {
|
||||
await fixture.close();
|
||||
const receipt = {
|
||||
client: "deepseek-harness",
|
||||
clientVersion,
|
||||
mode,
|
||||
passed: failure === undefined,
|
||||
artifacts: lab,
|
||||
runtimeRoot,
|
||||
host: {
|
||||
platform: process.platform,
|
||||
architecture: process.arch,
|
||||
node: process.versions.node,
|
||||
},
|
||||
modelFixture:
|
||||
"loopback custom OpenAI-compatible adapter; no live DeepSeek provider claim",
|
||||
modelContextWindow: 1000000,
|
||||
skillDiscovery:
|
||||
"native DSH_AGENTS_HOME personal directory; isolated project root",
|
||||
catalogSize: catalog?.length,
|
||||
patternCount,
|
||||
uniquePatterns: patterns.size,
|
||||
skillLoaded,
|
||||
evidenceSeen,
|
||||
artifactDigest,
|
||||
finalSeen,
|
||||
requests: fixture.requests,
|
||||
fixtureFailure: fixture.failure,
|
||||
verifier: await completeVerifierRun(verifier),
|
||||
};
|
||||
await writeFile(join(lab, "receipt.json"), JSON.stringify(receipt, null, 2));
|
||||
console.log(JSON.stringify(receipt));
|
||||
}
|
||||
if (failure !== undefined) throw failure;
|
||||
@@ -0,0 +1,378 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { createHash } from "node:crypto";
|
||||
import { mkdir, mkdtemp, readFile, writeFile } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import Ajv2020 from "ajv/dist/2020.js";
|
||||
import {
|
||||
TOOL_CONTRACTS,
|
||||
toolContract,
|
||||
} from "../dist/contracts/toolContracts.js";
|
||||
import { exec } from "./lib/verify-package-core.mjs";
|
||||
import { createVerifierRun, completeVerifierRun } from "./lib/verifier-run.mjs";
|
||||
import { createGeminiModelFixture } from "./verify/clients/gemini-model-fixture.mjs";
|
||||
|
||||
assert(
|
||||
process.platform !== "win32",
|
||||
"Native Gemini verifier requires POSIX; Windows is unverified",
|
||||
);
|
||||
const repo = fileURLToPath(new URL("..", import.meta.url));
|
||||
const runtimeRoot = process.env.REA_VERIFY_RUNTIME_ROOT ?? repo;
|
||||
const command = process.env.REA_VERIFY_GEMINI_COMMAND ?? "gemini";
|
||||
const mode = process.argv[2] ?? "call";
|
||||
assert(
|
||||
["chat", "call"].includes(mode),
|
||||
"Usage: verify-gemini-client.mjs [chat|call]",
|
||||
);
|
||||
const verifier = createVerifierRun();
|
||||
const lab = await mkdtemp(join(tmpdir(), "rea-gemini-client-"));
|
||||
const account = join(lab, "account");
|
||||
const profile = join(lab, "selected profile");
|
||||
const workspace = join(lab, "workspace");
|
||||
const configPath = join(profile, ".gemini", "settings.json");
|
||||
const skillPath = join(
|
||||
profile,
|
||||
".agents",
|
||||
"skills",
|
||||
"reverse-engineer-anything",
|
||||
"SKILL.md",
|
||||
);
|
||||
await Promise.all([
|
||||
mkdir(account),
|
||||
mkdir(workspace),
|
||||
mkdir(join(profile, ".gemini"), { recursive: true }),
|
||||
]);
|
||||
await exec("git", ["init", "--quiet", workspace], { timeout: 30_000 });
|
||||
await writeFile(join(lab, "system-settings.json"), "{}\n");
|
||||
await writeFile(join(lab, "system-defaults.json"), "{}\n");
|
||||
const environment = {
|
||||
PATH: process.env.PATH ?? "",
|
||||
USERPROFILE: account,
|
||||
GEMINI_CLI_HOME: profile,
|
||||
GEMINI_CLI_SYSTEM_SETTINGS_PATH: join(lab, "system-settings.json"),
|
||||
GEMINI_CLI_SYSTEM_DEFAULTS_PATH: join(lab, "system-defaults.json"),
|
||||
// Preserve the caller's bounded Node heap instead of the launcher's host-memory auto-sizing.
|
||||
GEMINI_CLI_NO_RELAUNCH: "1",
|
||||
GEMINI_CLI_TRUST_WORKSPACE: "true",
|
||||
REA_PROCESS_RUN_ID: verifier.run_id,
|
||||
};
|
||||
let clientVersion;
|
||||
try {
|
||||
clientVersion = (
|
||||
await exec(command, ["--version"], { env: environment, timeout: 30_000 })
|
||||
).stdout.trim();
|
||||
} catch (cause) {
|
||||
throw new Error(
|
||||
`Gemini preflight failed for ${command}; install Gemini CLI or set REA_VERIFY_GEMINI_COMMAND. ${String(cause)}`,
|
||||
);
|
||||
}
|
||||
const original = JSON.stringify(
|
||||
{
|
||||
security: {
|
||||
auth: { selectedType: "gemini-api-key" },
|
||||
folderTrust: { enabled: false },
|
||||
},
|
||||
telemetry: { enabled: false },
|
||||
advanced: { autoConfigureMemory: false },
|
||||
model: { name: "gemini-2.5-flash", maxSessionTurns: 8 },
|
||||
mcpServers: { other: { command: "unrelated-server" } },
|
||||
},
|
||||
null,
|
||||
2,
|
||||
);
|
||||
await writeFile(configPath, original);
|
||||
for (const args of [
|
||||
["setup", "--client", "gemini_cli", "--dry-run"],
|
||||
["setup", "--client", "gemini_cli", "--yes"],
|
||||
["doctor", "--client", "gemini_cli", "--skill"],
|
||||
]) {
|
||||
const result = await exec(
|
||||
process.execPath,
|
||||
[join(runtimeRoot, "scripts/rea.mjs"), ...args, "--json"],
|
||||
{ env: environment, cwd: workspace, timeout: 60_000 },
|
||||
);
|
||||
const parsed = JSON.parse(result.stdout);
|
||||
if (args[0] === "doctor") assert.equal(parsed.healthy, true);
|
||||
else {
|
||||
assert.equal(
|
||||
parsed.status,
|
||||
args.includes("--dry-run") ? "planned" : "ready",
|
||||
);
|
||||
if (args.includes("--dry-run")) {
|
||||
assert(parsed.plannedActions.some((a) => a.target === configPath));
|
||||
assert(
|
||||
parsed.plannedActions.some((a) =>
|
||||
a.target.startsWith(join(profile, ".agents", "skills") + "/"),
|
||||
),
|
||||
);
|
||||
assert(
|
||||
parsed.plannedActions.every((a) => a.target.startsWith(profile + "/")),
|
||||
);
|
||||
}
|
||||
}
|
||||
await writeFile(
|
||||
join(lab, `${args[0]}${args.includes("--dry-run") ? "-plan" : ""}.json`),
|
||||
result.stdout,
|
||||
);
|
||||
}
|
||||
assert.equal(await readFile(`${configPath}.rea.backup`, "utf8"), original);
|
||||
const configured = await readFile(configPath, "utf8");
|
||||
assert.deepEqual(JSON.parse(configured).mcpServers.other, {
|
||||
command: "unrelated-server",
|
||||
});
|
||||
const repeated = await exec(
|
||||
process.execPath,
|
||||
[
|
||||
join(runtimeRoot, "scripts/rea.mjs"),
|
||||
"setup",
|
||||
"--client",
|
||||
"gemini_cli",
|
||||
"--yes",
|
||||
"--json",
|
||||
],
|
||||
{ env: environment, cwd: workspace, timeout: 60_000 },
|
||||
);
|
||||
assert.deepEqual(JSON.parse(repeated.stdout).appliedActions, []);
|
||||
assert.equal(await readFile(configPath, "utf8"), configured);
|
||||
assert.equal(await readFile(`${configPath}.rea.backup`, "utf8"), original);
|
||||
assert(
|
||||
(await readFile(skillPath, "utf8")).includes(
|
||||
"name: reverse-engineer-anything",
|
||||
),
|
||||
);
|
||||
const target = join(lab, "fixture α测试");
|
||||
await mkdir(target);
|
||||
await writeFile(
|
||||
join(target, "app.js"),
|
||||
'export function result() { return { clientCompatibility: "REA_CLIENT_FIXTURE", count: 7 }; }\n',
|
||||
);
|
||||
let evidenceSeen = false;
|
||||
let skillLoaded = false;
|
||||
let skillRequested = false;
|
||||
let finalSeen = false;
|
||||
let catalogSize = 0;
|
||||
let schemasChecked = 0;
|
||||
let artifactDigest;
|
||||
const schemaValidator = new Ajv2020({ strict: false, validateFormats: false });
|
||||
const extractEvidence = (value) => {
|
||||
if (typeof value === "string") {
|
||||
if (
|
||||
value.startsWith("<untrusted_context>\n") &&
|
||||
value.endsWith("\n</untrusted_context>")
|
||||
) {
|
||||
return extractEvidence(
|
||||
value.slice(
|
||||
"<untrusted_context>\n".length,
|
||||
-"\n</untrusted_context>".length,
|
||||
),
|
||||
);
|
||||
}
|
||||
try {
|
||||
return extractEvidence(JSON.parse(value));
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
if (value && typeof value === "object") {
|
||||
if (
|
||||
typeof value.evidence_id === "string" &&
|
||||
value.normalized_result?.graph?.nodes?.length > 0
|
||||
)
|
||||
return value;
|
||||
for (const nested of Object.values(value)) {
|
||||
const evidence = extractEvidence(nested);
|
||||
if (evidence) return evidence;
|
||||
}
|
||||
}
|
||||
return undefined;
|
||||
};
|
||||
const fixture = await createGeminiModelFixture({
|
||||
directory: lab,
|
||||
onRequest: async ({ body, tools }) => {
|
||||
const analysis = tools.find((t) =>
|
||||
t.name.endsWith("analyze_javascript_application"),
|
||||
);
|
||||
assert(analysis, "Gemini must expose the REA analysis tool");
|
||||
const prefix = analysis.name.slice(
|
||||
0,
|
||||
-"analyze_javascript_application".length,
|
||||
);
|
||||
const expected = TOOL_CONTRACTS.map((t) => prefix + t.name).sort();
|
||||
assert.deepEqual(
|
||||
tools
|
||||
.filter((t) => expected.includes(t.name))
|
||||
.map((t) => t.name)
|
||||
.sort(),
|
||||
expected,
|
||||
"Gemini must forward the complete REA tool catalog",
|
||||
);
|
||||
catalogSize = expected.length;
|
||||
if (schemasChecked === 0) {
|
||||
for (const tool of tools.filter((t) => expected.includes(t.name))) {
|
||||
assert(
|
||||
tool.parametersJsonSchema,
|
||||
`${tool.name}: missing advertised JSON Schema`,
|
||||
);
|
||||
assert(
|
||||
schemaValidator.validateSchema(tool.parametersJsonSchema),
|
||||
`${tool.name}: invalid advertised JSON Schema: ${JSON.stringify(schemaValidator.errors)}`,
|
||||
);
|
||||
schemasChecked++;
|
||||
}
|
||||
}
|
||||
const responses = body.contents
|
||||
.flatMap((c) => c.parts ?? [])
|
||||
.filter((p) => p.functionResponse)
|
||||
.map((p) => p.functionResponse);
|
||||
skillLoaded ||= responses.some(
|
||||
(r) =>
|
||||
r.name === "activate_skill" &&
|
||||
r.response?.output?.includes(
|
||||
`<activated_skill name="reverse-engineer-anything">`,
|
||||
) &&
|
||||
r.response.output.includes(
|
||||
join(profile, ".agents", "skills", "reverse-engineer-anything"),
|
||||
),
|
||||
);
|
||||
const observed = extractEvidence(responses);
|
||||
if (observed) {
|
||||
const evidence = toolContract(
|
||||
"analyze_javascript_application",
|
||||
).outputSchema.parse(observed);
|
||||
assert.equal(evidence.subject.local_path, target);
|
||||
const shape = evidence.normalized_result.graph.nodes
|
||||
.flatMap((n) => n.observations)
|
||||
.map((o) => o.properties)
|
||||
.find(
|
||||
(p) =>
|
||||
p.semantic_role === "export-return-shapes" &&
|
||||
p.exported_name === "result",
|
||||
);
|
||||
assert(shape?.return_shape_coverage?.projection_complete);
|
||||
const fields = shape.static_return_shapes.flatMap((s) => s.fields);
|
||||
assert(
|
||||
fields.some(
|
||||
(f) =>
|
||||
f.path === "/clientCompatibility" &&
|
||||
f.state === "literal" &&
|
||||
f.value === "REA_CLIENT_FIXTURE",
|
||||
),
|
||||
);
|
||||
assert(
|
||||
fields.some(
|
||||
(f) => f.path === "/count" && f.state === "literal" && f.value === 7,
|
||||
),
|
||||
);
|
||||
const bytes = JSON.stringify(evidence);
|
||||
artifactDigest = createHash("sha256").update(bytes).digest("hex");
|
||||
await writeFile(join(lab, "gemini-evidence.json"), bytes);
|
||||
evidenceSeen = true;
|
||||
}
|
||||
if (mode === "chat" || evidenceSeen) {
|
||||
finalSeen = true;
|
||||
return [{ text: "REA_CLIENT_COMPATIBILITY_OK" }];
|
||||
}
|
||||
if (!skillRequested) {
|
||||
const skill = tools.find((t) => t.name === "activate_skill");
|
||||
assert(skill, "Gemini must expose its native skill activation tool");
|
||||
skillRequested = true;
|
||||
return [
|
||||
{
|
||||
functionCall: {
|
||||
name: skill.name,
|
||||
args: { name: "reverse-engineer-anything" },
|
||||
},
|
||||
},
|
||||
];
|
||||
}
|
||||
assert(skillLoaded, "Gemini must load the installed personal REA skill");
|
||||
return [
|
||||
{
|
||||
functionCall: {
|
||||
name: analysis.name,
|
||||
args: { input_path: target, format: "directory" },
|
||||
},
|
||||
},
|
||||
];
|
||||
},
|
||||
});
|
||||
let failure;
|
||||
try {
|
||||
const response = await exec(
|
||||
command,
|
||||
[
|
||||
"--model",
|
||||
"gemini-2.5-flash",
|
||||
"--approval-mode",
|
||||
mode === "chat" ? "default" : "yolo",
|
||||
"--allowed-mcp-server-names",
|
||||
"rea",
|
||||
"--output-format",
|
||||
"stream-json",
|
||||
"-p",
|
||||
mode === "chat"
|
||||
? "Say hello briefly."
|
||||
: `Use REA to analyze the JavaScript directory ${target}. Report the result.`,
|
||||
],
|
||||
{
|
||||
env: {
|
||||
...environment,
|
||||
GEMINI_API_KEY: "local-fixture-key",
|
||||
GOOGLE_GEMINI_BASE_URL: fixture.baseUrl,
|
||||
},
|
||||
cwd: workspace,
|
||||
timeout: 120_000,
|
||||
maxBuffer: 10 * 1024 * 1024,
|
||||
},
|
||||
);
|
||||
await writeFile(join(lab, `${mode}.stdout`), response.stdout);
|
||||
await writeFile(join(lab, `${mode}.stderr`), response.stderr);
|
||||
const events = response.stdout
|
||||
.split("\n")
|
||||
.filter(Boolean)
|
||||
.map((line) => JSON.parse(line));
|
||||
assert.equal(events.findLast((e) => e.type === "result")?.status, "success");
|
||||
assert.equal(fixture.failure, undefined);
|
||||
assert(finalSeen && response.stdout.includes("REA_CLIENT_COMPATIBILITY_OK"));
|
||||
if (mode === "call") assert(evidenceSeen && skillLoaded);
|
||||
} catch (cause) {
|
||||
failure = cause;
|
||||
if (typeof cause?.stdout === "string")
|
||||
await writeFile(join(lab, `${mode}.stdout`), cause.stdout);
|
||||
if (typeof cause?.stderr === "string")
|
||||
await writeFile(join(lab, `${mode}.stderr`), cause.stderr);
|
||||
} finally {
|
||||
await fixture.close();
|
||||
const lineage = await completeVerifierRun(verifier);
|
||||
const receipt = {
|
||||
client: "Gemini CLI",
|
||||
clientVersion,
|
||||
mode,
|
||||
passed: failure === undefined,
|
||||
artifacts: lab,
|
||||
runtimeRoot,
|
||||
host: {
|
||||
platform: process.platform,
|
||||
architecture: process.arch,
|
||||
node: process.versions.node,
|
||||
},
|
||||
modelFixture:
|
||||
"loopback native Gemini API adapter; token counts synthetic; no live Google API claim",
|
||||
skillDiscovery:
|
||||
"native GEMINI_CLI_HOME personal shared directory; isolated Git project",
|
||||
catalogSize,
|
||||
schemasChecked,
|
||||
evidenceSeen,
|
||||
skillLoaded,
|
||||
artifactDigest,
|
||||
finalSeen,
|
||||
requests: fixture.requests,
|
||||
probes: fixture.probes,
|
||||
fixtureFailure: fixture.failure,
|
||||
verifier: lineage,
|
||||
};
|
||||
await writeFile(join(lab, "receipt.json"), JSON.stringify(receipt, null, 2));
|
||||
console.log(JSON.stringify(receipt));
|
||||
}
|
||||
if (failure !== undefined) throw failure;
|
||||
@@ -0,0 +1,467 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { createHash } from "node:crypto";
|
||||
import { mkdir, mkdtemp, readFile, writeFile } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import {
|
||||
TOOL_CONTRACTS,
|
||||
toolContract,
|
||||
} from "../dist/contracts/toolContracts.js";
|
||||
import { exec } from "./lib/verify-package-core.mjs";
|
||||
import { createVerifierRun, completeVerifierRun } from "./lib/verifier-run.mjs";
|
||||
import { createOpenAiModelFixture } from "./verify/clients/model-fixture.mjs";
|
||||
|
||||
const client = process.argv[2];
|
||||
const mode = process.argv[3] ?? "call";
|
||||
assert(["pi", "hermes"].includes(client), "Select pi or hermes.");
|
||||
assert(["chat", "call"].includes(mode), "Select chat or call.");
|
||||
assert(
|
||||
process.platform !== "win32",
|
||||
"This real-client lane requires POSIX; native Windows is unverified.",
|
||||
);
|
||||
const repo = fileURLToPath(new URL("..", import.meta.url));
|
||||
const runtimeRoot = process.env.REA_VERIFY_RUNTIME_ROOT ?? repo;
|
||||
const command =
|
||||
process.env[`REA_VERIFY_${client.toUpperCase()}_COMMAND`] ?? client;
|
||||
const lab = await mkdtemp(join(tmpdir(), `rea-${client}-client-`));
|
||||
const account = join(lab, "account");
|
||||
const profile = join(lab, "selected profile");
|
||||
const stickyHermes =
|
||||
client === "hermes" && process.env.REA_VERIFY_HERMES_STICKY_PROFILE === "1";
|
||||
const selectedProfile = stickyHermes
|
||||
? join(profile, "profiles", "coder")
|
||||
: profile;
|
||||
const workspace = join(lab, "workspace");
|
||||
const target = join(workspace, "fixture α测试");
|
||||
const artifactPath = join(workspace, "evidence.json");
|
||||
const skillPath = join(
|
||||
client === "pi" ? join(account, ".agents") : selectedProfile,
|
||||
"skills",
|
||||
"reverse-engineer-anything",
|
||||
"SKILL.md",
|
||||
);
|
||||
const verifier = createVerifierRun();
|
||||
await Promise.all([
|
||||
mkdir(account),
|
||||
mkdir(profile, { recursive: true }),
|
||||
...(stickyHermes ? [mkdir(selectedProfile, { recursive: true })] : []),
|
||||
mkdir(target, { recursive: true }),
|
||||
]);
|
||||
await writeFile(
|
||||
join(target, "app.js"),
|
||||
'export function result() { return { clientCompatibility: "REA_CLIENT_FIXTURE", count: 7 }; }\n',
|
||||
);
|
||||
const environment = {
|
||||
PATH: process.env.PATH ?? "",
|
||||
USERPROFILE: account,
|
||||
[client === "pi" ? "PI_CODING_AGENT_DIR" : "HERMES_HOME"]: profile,
|
||||
PI_OFFLINE: "1",
|
||||
REA_PROCESS_RUN_ID: verifier.run_id,
|
||||
};
|
||||
const configPath = join(
|
||||
selectedProfile,
|
||||
client === "pi" ? "mcp.json" : "config.yaml",
|
||||
);
|
||||
if (stickyHermes) await writeFile(join(profile, "active_profile"), "coder\n");
|
||||
const original =
|
||||
client === "pi"
|
||||
? JSON.stringify({
|
||||
mcpServers: { other: { command: "unrelated-server", enabled: false } },
|
||||
})
|
||||
: "# Keep caller settings.\nmcp_servers:\n other:\n command: unrelated-server\n enabled: false\n";
|
||||
await writeFile(configPath, original);
|
||||
if (client === "pi") {
|
||||
// The CLI fixture account and native os.homedir() differ; use Pi's caller-owned skill setting.
|
||||
await writeFile(
|
||||
join(profile, "settings.json"),
|
||||
JSON.stringify({ skills: [join(account, ".agents", "skills")] }),
|
||||
);
|
||||
}
|
||||
const clientVersion = (
|
||||
await exec(command, ["--version"], { env: environment, timeout: 30_000 })
|
||||
).stdout.trim();
|
||||
for (const args of [
|
||||
["setup", "--client", client, "--dry-run"],
|
||||
["setup", "--client", client, "--yes"],
|
||||
["doctor", "--client", client, "--skill"],
|
||||
]) {
|
||||
const result = await exec(
|
||||
process.execPath,
|
||||
[join(runtimeRoot, "scripts/rea.mjs"), ...args, "--json"],
|
||||
{ env: environment, cwd: workspace, timeout: 60_000 },
|
||||
);
|
||||
const parsed = JSON.parse(result.stdout);
|
||||
if (args[0] === "doctor") assert.equal(parsed.healthy, true);
|
||||
else
|
||||
assert.equal(
|
||||
parsed.status,
|
||||
args.includes("--dry-run") ? "planned" : "ready",
|
||||
);
|
||||
await writeFile(
|
||||
join(lab, `${args[0]}${args.includes("--dry-run") ? "-plan" : ""}.json`),
|
||||
result.stdout,
|
||||
);
|
||||
}
|
||||
assert.equal(await readFile(`${configPath}.rea.backup`, "utf8"), original);
|
||||
const installedConfiguration = await readFile(configPath, "utf8");
|
||||
assert(installedConfiguration.includes("unrelated-server"));
|
||||
if (client === "hermes")
|
||||
assert(installedConfiguration.includes("# Keep caller settings."));
|
||||
const repeated = await exec(
|
||||
process.execPath,
|
||||
[
|
||||
join(runtimeRoot, "scripts/rea.mjs"),
|
||||
"setup",
|
||||
"--client",
|
||||
client,
|
||||
"--yes",
|
||||
"--json",
|
||||
],
|
||||
{ env: environment, cwd: workspace, timeout: 60_000 },
|
||||
);
|
||||
assert.deepEqual(JSON.parse(repeated.stdout).appliedActions, []);
|
||||
assert.equal(await readFile(configPath, "utf8"), installedConfiguration);
|
||||
assert.equal(await readFile(`${configPath}.rea.backup`, "utf8"), original);
|
||||
assert(
|
||||
(await readFile(skillPath, "utf8")).includes(
|
||||
"name: reverse-engineer-anything",
|
||||
),
|
||||
);
|
||||
let evidenceSeen = false;
|
||||
let skillLoaded = false;
|
||||
let skillRequested = false;
|
||||
let callRequested = false;
|
||||
let searchRequested = false;
|
||||
let describeRequested = false;
|
||||
let finalSeen = false;
|
||||
let catalog;
|
||||
let artifactDigest;
|
||||
const extractEvidence = (value) => {
|
||||
if (typeof value === "string") {
|
||||
if (
|
||||
value.startsWith("<untrusted_tool_result ") &&
|
||||
value.endsWith("</untrusted_tool_result>")
|
||||
) {
|
||||
// Hermes wraps external MCP content with its own instruction banner.
|
||||
// Decode only that observed envelope, retaining the original request artifact.
|
||||
const start = value.indexOf("\n\n");
|
||||
if (start >= 0)
|
||||
return extractEvidence(
|
||||
value
|
||||
.slice(start + 2, value.lastIndexOf("</untrusted_tool_result>"))
|
||||
.trim(),
|
||||
);
|
||||
}
|
||||
try {
|
||||
return extractEvidence(JSON.parse(value));
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
if (value && typeof value === "object") {
|
||||
if (
|
||||
typeof value.evidence_id === "string" &&
|
||||
value.normalized_result?.graph?.nodes
|
||||
)
|
||||
return value;
|
||||
for (const child of Object.values(value)) {
|
||||
const found = extractEvidence(child);
|
||||
if (found) return found;
|
||||
}
|
||||
}
|
||||
};
|
||||
const validateEvidence = async (value) => {
|
||||
const evidence = toolContract(
|
||||
"analyze_javascript_application",
|
||||
).outputSchema.parse(value);
|
||||
assert.equal(evidence.subject.local_path, target);
|
||||
const shape = evidence.normalized_result.graph.nodes
|
||||
.flatMap((n) => n.observations)
|
||||
.map((o) => o.properties)
|
||||
.find(
|
||||
(p) =>
|
||||
p.semantic_role === "export-return-shapes" &&
|
||||
p.exported_name === "result",
|
||||
);
|
||||
assert(shape?.return_shape_coverage?.projection_complete);
|
||||
const fields = shape.static_return_shapes.flatMap((s) => s.fields);
|
||||
assert(
|
||||
fields.some(
|
||||
(f) =>
|
||||
f.path === "/clientCompatibility" &&
|
||||
f.state === "literal" &&
|
||||
f.value === "REA_CLIENT_FIXTURE",
|
||||
),
|
||||
);
|
||||
assert(
|
||||
fields.some(
|
||||
(f) => f.path === "/count" && f.state === "literal" && f.value === 7,
|
||||
),
|
||||
);
|
||||
const serialized = JSON.stringify(evidence);
|
||||
await writeFile(join(lab, "evidence.json"), serialized);
|
||||
artifactDigest = createHash("sha256").update(serialized).digest("hex");
|
||||
evidenceSeen = true;
|
||||
};
|
||||
const fixture = await createOpenAiModelFixture({
|
||||
directory: lab,
|
||||
prefix: `${client}-${mode}`,
|
||||
onRequest: async ({ body, tools, requestIndex }) => {
|
||||
assert(
|
||||
requestIndex <= 6,
|
||||
"Client did not complete the bounded fixture workflow.",
|
||||
);
|
||||
const messages = body.messages ?? [];
|
||||
const toolMessages = messages.filter((m) => m.role === "tool");
|
||||
const serializedTools = JSON.stringify(toolMessages);
|
||||
const call = (name, args) => ({
|
||||
role: "assistant",
|
||||
content: null,
|
||||
tool_calls: [
|
||||
{
|
||||
id: `call_rea_${requestIndex}`,
|
||||
type: "function",
|
||||
function: { name, arguments: JSON.stringify(args) },
|
||||
},
|
||||
],
|
||||
});
|
||||
if (
|
||||
client === "pi" &&
|
||||
callRequested &&
|
||||
serializedTools.includes("REA_CLIENT_FIXTURE")
|
||||
) {
|
||||
const fact = toolMessages
|
||||
.flatMap((m) => {
|
||||
const text =
|
||||
typeof m.content === "string"
|
||||
? m.content
|
||||
: JSON.stringify(m.content);
|
||||
const marker = text.indexOf('{"REA_CLIENT_FIXTURE":true');
|
||||
if (marker < 0) return [];
|
||||
const end = text.indexOf("}\n", marker);
|
||||
try {
|
||||
return [
|
||||
JSON.parse(text.slice(marker, end < 0 ? undefined : end + 1)),
|
||||
];
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
})
|
||||
.find((v) => Array.isArray(v.catalog));
|
||||
assert(
|
||||
fact,
|
||||
"Pi codemode must return its actual catalog and skill-read receipt.",
|
||||
);
|
||||
catalog = fact.catalog;
|
||||
skillLoaded = fact.skill_loaded === true;
|
||||
await validateEvidence(JSON.parse(await readFile(artifactPath, "utf8")));
|
||||
} else if (client === "hermes") {
|
||||
const search = tools.find((t) => t.name === "tool_search");
|
||||
const exposed = tools
|
||||
.filter((t) => t.name.startsWith("mcp__rea__"))
|
||||
.map((t) => t.name.slice(10));
|
||||
const deferred = [
|
||||
...(search?.description ?? "").matchAll(/^- mcp__rea__(\w+):/gmu),
|
||||
].map((m) => m[1]);
|
||||
const native = [...new Set([...exposed, ...deferred])];
|
||||
// Hermes advertises MCP prompt utilities beside server tools.
|
||||
if (native.length)
|
||||
catalog = native.filter(
|
||||
(name) => !["get_prompt", "list_prompts"].includes(name),
|
||||
);
|
||||
skillLoaded ||=
|
||||
serializedTools.includes(skillPath) &&
|
||||
serializedTools.includes("reverse-engineer-anything");
|
||||
for (const message of toolMessages) {
|
||||
const evidence = extractEvidence(message.content);
|
||||
if (evidence) await validateEvidence(evidence);
|
||||
}
|
||||
}
|
||||
if (mode === "chat" || evidenceSeen) {
|
||||
finalSeen = true;
|
||||
return { role: "assistant", content: "REA_CLIENT_COMPATIBILITY_OK" };
|
||||
}
|
||||
if (client === "pi") {
|
||||
const tool = tools.find((t) => t.name === "codemode");
|
||||
assert(
|
||||
tool,
|
||||
`Native Pi codemode unavailable: ${tools.map((t) => t.name).join(", ")}`,
|
||||
);
|
||||
assert(
|
||||
JSON.stringify(messages).includes(skillPath),
|
||||
"Pi must advertise the installed REA skill.",
|
||||
);
|
||||
callRequested = true;
|
||||
return call(tool.name, {
|
||||
code: `await describeNamespace("rea"); const skill = await tools.read({path:${JSON.stringify(skillPath)}}); const result = await tools.mcp__rea__analyze_javascript_application({input_path:${JSON.stringify(target)},format:"directory"}); if(result.isError) throw new Error(JSON.stringify(result)); const evidence = result.structuredContent ?? JSON.parse(result.content.find(c=>c.type==="text").text); await tools.write({path:${JSON.stringify(artifactPath)},content:JSON.stringify(evidence)}); text({REA_CLIENT_FIXTURE:true,skill_loaded:typeof skill==="string"&&skill.includes("name: reverse-engineer-anything"),catalog:ALL_TOOLS.filter(t=>t.name.startsWith("mcp__rea__")).map(t=>t.name.slice(10))});`,
|
||||
});
|
||||
}
|
||||
if (!skillRequested) {
|
||||
const tool = tools.find((t) => t.name === "skill_view");
|
||||
assert(tool, "Hermes skill_view must be available.");
|
||||
skillRequested = true;
|
||||
return call(tool.name, { name: "reverse-engineer-anything" });
|
||||
}
|
||||
if (!searchRequested) {
|
||||
const tool = tools.find((t) => t.name === "tool_search");
|
||||
assert(tool, "Hermes native deferred tool_search must be available.");
|
||||
searchRequested = true;
|
||||
return call(tool.name, { queries: ["analyze_javascript_application"] });
|
||||
}
|
||||
assert(
|
||||
serializedTools.includes("mcp__rea__analyze_javascript_application"),
|
||||
"Hermes must discover the analysis tool.",
|
||||
);
|
||||
if (!describeRequested) {
|
||||
const tool = tools.find((t) => t.name === "tool_describe");
|
||||
assert(tool, "Hermes must load the actual parameter schema.");
|
||||
describeRequested = true;
|
||||
return call(tool.name, {
|
||||
names: ["mcp__rea__analyze_javascript_application"],
|
||||
});
|
||||
}
|
||||
const tool = tools.find((t) => t.name === "tool_call");
|
||||
assert(tool, "Hermes native deferred tool_call must be available.");
|
||||
callRequested = true;
|
||||
return call(tool.name, {
|
||||
calls: [
|
||||
{
|
||||
name: "mcp__rea__analyze_javascript_application",
|
||||
arguments: { input_path: target, format: "directory" },
|
||||
},
|
||||
],
|
||||
});
|
||||
},
|
||||
});
|
||||
if (client === "pi") {
|
||||
await writeFile(
|
||||
join(profile, "models.json"),
|
||||
JSON.stringify({
|
||||
providers: {
|
||||
"rea-fixture": {
|
||||
baseUrl: fixture.baseUrl,
|
||||
api: "openai-completions",
|
||||
apiKey: "local-fixture-key",
|
||||
models: [
|
||||
{
|
||||
id: "rea-client-fixture",
|
||||
reasoning: false,
|
||||
contextWindow: 200000,
|
||||
maxTokens: 4096,
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
}),
|
||||
);
|
||||
} else {
|
||||
const configured = await readFile(configPath, "utf8");
|
||||
await writeFile(
|
||||
configPath,
|
||||
configured +
|
||||
`\nmodel:\n default: rea-client-fixture\n provider: custom\n base_url: ${fixture.baseUrl}\nagent:\n max_turns: 6\n`,
|
||||
);
|
||||
}
|
||||
const args =
|
||||
client === "pi"
|
||||
? [
|
||||
"--offline",
|
||||
"--provider",
|
||||
"rea-fixture",
|
||||
"--model",
|
||||
"rea-client-fixture",
|
||||
"--no-session",
|
||||
"--no-context-files",
|
||||
"--mode",
|
||||
"json",
|
||||
"-p",
|
||||
mode === "chat"
|
||||
? "Say hello briefly."
|
||||
: `Use REA to analyze ${target}.`,
|
||||
]
|
||||
: [
|
||||
"--oneshot",
|
||||
mode === "chat"
|
||||
? "Say hello briefly."
|
||||
: `Use REA to analyze ${target}.`,
|
||||
"--provider",
|
||||
"custom",
|
||||
"--model",
|
||||
"rea-client-fixture",
|
||||
"--reasoning",
|
||||
"none",
|
||||
];
|
||||
let failure;
|
||||
try {
|
||||
const execution = exec(command, args, {
|
||||
env: {
|
||||
...environment,
|
||||
OPENAI_API_KEY: "local-fixture-key",
|
||||
OPENAI_BASE_URL: fixture.baseUrl,
|
||||
},
|
||||
cwd: workspace,
|
||||
timeout: 120_000,
|
||||
maxBuffer: 10 * 1024 * 1024,
|
||||
});
|
||||
// Pi consumes piped input before its first prompt; an unused open pipe is not EOF.
|
||||
execution.child.stdin.end();
|
||||
const result = await execution;
|
||||
await writeFile(join(lab, "client.stdout"), result.stdout);
|
||||
await writeFile(join(lab, "client.stderr"), result.stderr);
|
||||
assert.equal(fixture.failure, undefined);
|
||||
assert(finalSeen);
|
||||
assert(result.stdout.includes("REA_CLIENT_COMPATIBILITY_OK"));
|
||||
if (mode === "call") {
|
||||
assert(evidenceSeen);
|
||||
assert(
|
||||
skillLoaded,
|
||||
"Native client must actually read the installed skill.",
|
||||
);
|
||||
assert.deepEqual(
|
||||
catalog?.sort(),
|
||||
TOOL_CONTRACTS.map((t) => t.name).sort(),
|
||||
"The complete REA catalog must survive native client discovery.",
|
||||
);
|
||||
}
|
||||
} catch (cause) {
|
||||
failure = cause;
|
||||
if (typeof cause?.stdout === "string")
|
||||
await writeFile(join(lab, "client.stdout"), cause.stdout);
|
||||
if (typeof cause?.stderr === "string")
|
||||
await writeFile(join(lab, "client.stderr"), cause.stderr);
|
||||
} finally {
|
||||
await fixture.close();
|
||||
const receipt = {
|
||||
client,
|
||||
clientVersion,
|
||||
mode,
|
||||
passed: failure === undefined,
|
||||
artifacts: lab,
|
||||
runtimeRoot,
|
||||
host: {
|
||||
platform: process.platform,
|
||||
architecture: process.arch,
|
||||
node: process.versions.node,
|
||||
},
|
||||
modelFixture:
|
||||
"loopback deterministic OpenAI-compatible endpoint; no live provider claim",
|
||||
skillDiscovery:
|
||||
client === "pi"
|
||||
? "caller-configured isolated shared directory; default OS home discovery unverified"
|
||||
: "native HERMES_HOME personal directory",
|
||||
stickyHermes,
|
||||
catalogSize: catalog?.length,
|
||||
evidenceSeen,
|
||||
skillLoaded,
|
||||
artifactDigest,
|
||||
finalSeen,
|
||||
requests: fixture.requests,
|
||||
probes: fixture.probes,
|
||||
fixtureFailure: fixture.failure,
|
||||
verifier: await completeVerifierRun(verifier),
|
||||
};
|
||||
await writeFile(join(lab, "receipt.json"), JSON.stringify(receipt, null, 2));
|
||||
console.log(JSON.stringify(receipt));
|
||||
}
|
||||
if (failure !== undefined) throw failure;
|
||||
@@ -87,6 +87,10 @@ export async function verifyPackageEnvironment({
|
||||
PI_PROFILE: undefined,
|
||||
QWEN_HOME: undefined,
|
||||
OPENCODE_CONFIG: undefined,
|
||||
OPENCODE_CONFIG_DIR: undefined,
|
||||
GEMINI_CLI_HOME: undefined,
|
||||
GEMINI_CLI_SYSTEM_SETTINGS_PATH: undefined,
|
||||
GEMINI_CLI_SYSTEM_DEFAULTS_PATH: undefined,
|
||||
XDG_CONFIG_HOME: join(home, ".config"),
|
||||
PATH: `${fakeBin}:${process.env.PATH ?? ""}`,
|
||||
REA_ANALYSIS_PROVIDER: "auto",
|
||||
|
||||
@@ -0,0 +1,403 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { createOpenAiModelFixture } from "./verify/clients/model-fixture.mjs";
|
||||
import {
|
||||
mkdir,
|
||||
writeFile,
|
||||
readFile,
|
||||
mkdtemp,
|
||||
realpath,
|
||||
} from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { createHash } from "node:crypto";
|
||||
import { exec } from "./lib/verify-package-core.mjs";
|
||||
import { createVerifierRun, completeVerifierRun } from "./lib/verifier-run.mjs";
|
||||
import {
|
||||
TOOL_CONTRACTS,
|
||||
toolContract,
|
||||
} from "../dist/contracts/toolContracts.js";
|
||||
|
||||
assert(
|
||||
process.platform !== "win32",
|
||||
"The Qwen real-client verifier currently requires a POSIX host; native Windows is unverified.",
|
||||
);
|
||||
const repo = fileURLToPath(new URL("..", import.meta.url));
|
||||
const runtimeRoot = process.env.REA_VERIFY_RUNTIME_ROOT ?? repo;
|
||||
const lab = await mkdtemp(join(tmpdir(), "rea-qwen-client-"));
|
||||
const client = "qwen";
|
||||
const mode = process.argv[2] ?? "call";
|
||||
assert(
|
||||
["chat", "call"].includes(mode),
|
||||
"Usage: node scripts/verify-qwen-client.mjs [chat|call]",
|
||||
);
|
||||
const command = process.env.REA_VERIFY_QWEN_COMMAND ?? "qwen";
|
||||
const verifier = createVerifierRun();
|
||||
const profile = join(lab, "selected profile");
|
||||
const account = join(lab, "account");
|
||||
const runtime = join(profile, "runtime");
|
||||
const workspace = join(lab, "workspace");
|
||||
const sharedSkills = join(account, ".agents", "skills");
|
||||
await Promise.all([
|
||||
mkdir(profile, { recursive: true }),
|
||||
mkdir(account),
|
||||
mkdir(workspace),
|
||||
]);
|
||||
const baseEnvironment = {
|
||||
PATH: process.env.PATH ?? "",
|
||||
USERPROFILE: account,
|
||||
QWEN_HOME: profile,
|
||||
QWEN_RUNTIME_DIR: runtime,
|
||||
REA_PROCESS_RUN_ID: verifier.run_id,
|
||||
};
|
||||
let clientVersion;
|
||||
try {
|
||||
clientVersion = (
|
||||
await exec(command, ["--version"], {
|
||||
env: baseEnvironment,
|
||||
timeout: 30_000,
|
||||
})
|
||||
).stdout.trim();
|
||||
} catch (cause) {
|
||||
throw new Error(
|
||||
`Qwen client preflight failed for ${command}; install Qwen Code or set REA_VERIFY_QWEN_COMMAND. ${String(cause)}`,
|
||||
);
|
||||
}
|
||||
const original = JSON.stringify(
|
||||
{
|
||||
security: {
|
||||
auth: { selectedType: "openai" },
|
||||
folderTrust: { enabled: false },
|
||||
},
|
||||
telemetry: { enabled: false },
|
||||
model: { name: "rea-client-fixture" },
|
||||
// Qwen uses os.homedir(), while REA's isolated CLI account uses USERPROFILE.
|
||||
// This caller-owned setting lets the real client load the REA-installed bundle.
|
||||
skills: { directories: [sharedSkills] },
|
||||
mcpServers: { other: { command: "unrelated-server", disabled: true } },
|
||||
},
|
||||
null,
|
||||
2,
|
||||
);
|
||||
await writeFile(join(profile, "settings.json"), original);
|
||||
for (const step of [
|
||||
["setup", "--client", "qwen_code", "--dry-run"],
|
||||
["setup", "--client", "qwen_code", "--yes"],
|
||||
["doctor", "--client", "qwen_code", "--skill"],
|
||||
]) {
|
||||
const response = await exec(
|
||||
process.execPath,
|
||||
[join(runtimeRoot, "scripts/rea.mjs"), ...step, "--json"],
|
||||
{ env: baseEnvironment, cwd: repo, timeout: 60_000 },
|
||||
);
|
||||
const result = JSON.parse(response.stdout);
|
||||
if (step.includes("--dry-run"))
|
||||
assert(
|
||||
result.plannedActions.every(
|
||||
(a) =>
|
||||
a.target.startsWith(profile + "/") ||
|
||||
a.target.startsWith(sharedSkills + "/"),
|
||||
),
|
||||
);
|
||||
await writeFile(
|
||||
join(lab, `${step[0]}${step.includes("--dry-run") ? "-plan" : ""}.json`),
|
||||
response.stdout,
|
||||
);
|
||||
}
|
||||
assert.equal(
|
||||
await readFile(join(profile, "settings.json.rea.backup"), "utf8"),
|
||||
original,
|
||||
);
|
||||
const configured = JSON.parse(
|
||||
await readFile(join(profile, "settings.json"), "utf8"),
|
||||
);
|
||||
assert.deepEqual(configured.model, { name: "rea-client-fixture" });
|
||||
assert.deepEqual(configured.skills, { directories: [sharedSkills] });
|
||||
assert.deepEqual(configured.mcpServers.other, {
|
||||
command: "unrelated-server",
|
||||
disabled: true,
|
||||
});
|
||||
const target = join(lab, "fixture α测试");
|
||||
await mkdir(target, { recursive: true });
|
||||
await writeFile(
|
||||
`${target}/app.js`,
|
||||
'export function result() { return { clientCompatibility: "REA_CLIENT_FIXTURE", count: 7 }; }\n',
|
||||
);
|
||||
let evidenceSeen = false;
|
||||
let skillSeen = false;
|
||||
let finalSeen = false;
|
||||
let skillRequested = false;
|
||||
let searchRequested = false;
|
||||
let artifactReadByAgent = false;
|
||||
let artifactPath;
|
||||
let selectedSkillLoaded = false;
|
||||
let artifactDigest;
|
||||
const extractEvidence = (value) => {
|
||||
if (typeof value === "string") {
|
||||
try {
|
||||
return extractEvidence(JSON.parse(value));
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
if (Array.isArray(value)) {
|
||||
for (const item of value) {
|
||||
const result = extractEvidence(item);
|
||||
if (result) return result;
|
||||
}
|
||||
} else if (value && typeof value === "object") {
|
||||
if (
|
||||
typeof value.evidence_id === "string" &&
|
||||
value.normalized_result?.graph?.nodes?.length > 0
|
||||
)
|
||||
return value;
|
||||
for (const nested of Object.values(value)) {
|
||||
const result = extractEvidence(nested);
|
||||
if (result) return result;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
};
|
||||
const fixture = await createOpenAiModelFixture({
|
||||
directory: lab,
|
||||
prefix: `${client}-${mode}`,
|
||||
onRequest: async ({ body, tools, requestIndex }) => {
|
||||
skillSeen ||= JSON.stringify(body.messages ?? []).includes(
|
||||
"reverse-engineer-anything",
|
||||
);
|
||||
selectedSkillLoaded ||= JSON.stringify(body.messages ?? []).includes(
|
||||
join(sharedSkills, "reverse-engineer-anything"),
|
||||
);
|
||||
const toolText = JSON.stringify(
|
||||
(body.messages ?? []).filter((m) => m.role === "tool"),
|
||||
);
|
||||
if (
|
||||
toolText.includes("REA_CLIENT_FIXTURE") &&
|
||||
/(?:\\?"REA_CLIENT_FIXTURE\\?"\s*:\s*true)/u.test(toolText)
|
||||
) {
|
||||
artifactReadByAgent = true;
|
||||
assert(artifactPath);
|
||||
const artifact = await readFile(artifactPath);
|
||||
artifactDigest = createHash("sha256").update(artifact).digest("hex");
|
||||
const evidence = toolContract(
|
||||
"analyze_javascript_application",
|
||||
).outputSchema.parse(JSON.parse(artifact.toString("utf8")));
|
||||
assert(extractEvidence(evidence));
|
||||
assert.equal(evidence.subject.local_path, target);
|
||||
const shape = evidence.normalized_result.graph.nodes
|
||||
.flatMap((n) => n.observations)
|
||||
.map((o) => o.properties)
|
||||
.find(
|
||||
(p) =>
|
||||
p.semantic_role === "export-return-shapes" &&
|
||||
p.exported_name === "result",
|
||||
);
|
||||
assert(shape?.return_shape_coverage?.projection_complete);
|
||||
const fields = shape.static_return_shapes.flatMap((s) => s.fields);
|
||||
assert(
|
||||
fields.some(
|
||||
(f) =>
|
||||
f.path === "/clientCompatibility" &&
|
||||
f.state === "literal" &&
|
||||
f.value === "REA_CLIENT_FIXTURE",
|
||||
),
|
||||
);
|
||||
assert(
|
||||
fields.some(
|
||||
(f) => f.path === "/count" && f.state === "literal" && f.value === 7,
|
||||
),
|
||||
);
|
||||
evidenceSeen = true;
|
||||
await writeFile(
|
||||
`${lab}/${client}-${mode}-evidence.json`,
|
||||
JSON.stringify(evidence),
|
||||
);
|
||||
}
|
||||
for (const message of body.messages ?? []) {
|
||||
if (message.role !== "tool") continue;
|
||||
const evidence = extractEvidence(message.content);
|
||||
if (evidence) {
|
||||
assert(
|
||||
JSON.stringify(evidence.normalized_result).includes(
|
||||
"REA_CLIENT_FIXTURE",
|
||||
),
|
||||
);
|
||||
evidenceSeen = true;
|
||||
await writeFile(
|
||||
`${lab}/${client}-${mode}-evidence.json`,
|
||||
JSON.stringify(evidence),
|
||||
);
|
||||
}
|
||||
}
|
||||
const selected = tools.find((t) =>
|
||||
t.name.includes("analyze_javascript_application"),
|
||||
);
|
||||
const reviewedNames = [
|
||||
...JSON.stringify(
|
||||
(body.messages ?? []).filter((m) => m.role === "tool"),
|
||||
).matchAll(
|
||||
/(?:\\?"name\\?"\s*:\s*\\?")([^"\\]*analyze_javascript_application)(?:\\?")/g,
|
||||
),
|
||||
]
|
||||
.map((m) => m[1])
|
||||
.sort((a, b) => b.length - a.length);
|
||||
const search = tools.find((t) => t.name === "tool_search");
|
||||
const invoke = tools.find((t) => t.name === "tool_call");
|
||||
const skill = tools.find((t) => t.name === "skill");
|
||||
const shell = tools.find((t) => t.name === "run_shell_command");
|
||||
const persisted = toolText.match(
|
||||
/Full output saved to: ([^\\"\n]+)(?:\\n|\n)/u,
|
||||
);
|
||||
const call = (name, args) => ({
|
||||
role: "assistant",
|
||||
content: null,
|
||||
tool_calls: [
|
||||
{
|
||||
id: `call_rea_fixture_${requestIndex}`,
|
||||
type: "function",
|
||||
function: { name, arguments: JSON.stringify(args) },
|
||||
},
|
||||
],
|
||||
});
|
||||
let message;
|
||||
if (mode === "chat" || evidenceSeen) {
|
||||
message = { role: "assistant", content: "REA_CLIENT_COMPATIBILITY_OK" };
|
||||
finalSeen = true;
|
||||
} else if (skill && !skillRequested) {
|
||||
skillRequested = true;
|
||||
message = call(skill.name, { skill: "reverse-engineer-anything" });
|
||||
} else if (persisted && shell) {
|
||||
artifactPath = persisted[1];
|
||||
assert(
|
||||
(await realpath(artifactPath)).startsWith(
|
||||
(await realpath(runtime)) + "/",
|
||||
),
|
||||
);
|
||||
const script = `const fs=require('node:fs');const e=JSON.parse(fs.readFileSync(${JSON.stringify(artifactPath)},'utf8'));console.log(JSON.stringify({REA_CLIENT_FIXTURE:JSON.stringify(e.normalized_result).includes('REA_CLIENT_FIXTURE'),evidence_id:e.evidence_id,graph_nodes:e.normalized_result.graph.nodes.length}));`;
|
||||
const quote = (value) => "'" + value.replaceAll("'", "'\\''") + "'";
|
||||
message = call(shell.name, {
|
||||
command: `node -e ${quote(script)}`,
|
||||
description:
|
||||
"Read the complete saved REA Evidence and verify the fixture facts.",
|
||||
});
|
||||
} else if (selected)
|
||||
message = call(selected.name, {
|
||||
input_path: target,
|
||||
format: "directory",
|
||||
});
|
||||
else if (reviewedNames.length && invoke)
|
||||
message = call(invoke.name, {
|
||||
name: reviewedNames[0],
|
||||
arguments: { input_path: target, format: "directory" },
|
||||
});
|
||||
else if (search && !searchRequested) {
|
||||
searchRequested = true;
|
||||
message = call(search.name, { query: "analyze_javascript_application" });
|
||||
} else
|
||||
throw new Error(
|
||||
`REA analysis tool not exposed; tools: ${tools.map((t) => t.name).join(", ")}`,
|
||||
);
|
||||
return message;
|
||||
},
|
||||
});
|
||||
const endpoint = fixture.baseUrl;
|
||||
const prompt =
|
||||
mode === "chat"
|
||||
? "Say hello briefly."
|
||||
: `Use REA to analyze the JavaScript directory ${target}. Report the result.`;
|
||||
const env = {
|
||||
...baseEnvironment,
|
||||
OPENAI_API_KEY: "local-fixture-key",
|
||||
OPENAI_BASE_URL: endpoint,
|
||||
};
|
||||
const args = [
|
||||
"--auth-type",
|
||||
"openai",
|
||||
"--openai-base-url",
|
||||
endpoint,
|
||||
"--model",
|
||||
"rea-client-fixture",
|
||||
"--approval-mode",
|
||||
mode === "chat" ? "default" : "yolo",
|
||||
"--allowed-mcp-server-names",
|
||||
"rea",
|
||||
"--output-format",
|
||||
"stream-json",
|
||||
"--max-session-turns",
|
||||
"8",
|
||||
"-p",
|
||||
prompt,
|
||||
];
|
||||
let response;
|
||||
let failure;
|
||||
try {
|
||||
response = await exec(command, args, {
|
||||
env,
|
||||
cwd: workspace,
|
||||
timeout: 120_000,
|
||||
maxBuffer: 10 * 1024 * 1024,
|
||||
});
|
||||
await writeFile(join(lab, `${mode}.stdout`), response.stdout);
|
||||
await writeFile(join(lab, `${mode}.stderr`), response.stderr);
|
||||
const initialized = response.stdout
|
||||
.split("\n")
|
||||
.filter(Boolean)
|
||||
.map((line) => JSON.parse(line))
|
||||
.find((v) => v.type === "system" && v.subtype === "init");
|
||||
assert(initialized);
|
||||
const observed = initialized.tools
|
||||
.filter((n) => n.startsWith("mcp__rea__"))
|
||||
.sort();
|
||||
assert.deepEqual(
|
||||
observed,
|
||||
TOOL_CONTRACTS.map((c) => `mcp__rea__${c.name}`).sort(),
|
||||
"Qwen must discover the complete REA catalog",
|
||||
);
|
||||
assert.equal(fixture.failure, undefined);
|
||||
assert(finalSeen);
|
||||
if (mode === "call") {
|
||||
assert(evidenceSeen);
|
||||
assert(selectedSkillLoaded);
|
||||
assert(artifactReadByAgent);
|
||||
}
|
||||
} catch (cause) {
|
||||
failure = cause;
|
||||
if (typeof cause?.stdout === "string")
|
||||
await writeFile(join(lab, `${mode}.stdout`), cause.stdout);
|
||||
if (typeof cause?.stderr === "string")
|
||||
await writeFile(join(lab, `${mode}.stderr`), cause.stderr);
|
||||
} finally {
|
||||
await fixture.close();
|
||||
const lineage = await completeVerifierRun(verifier);
|
||||
const receipt = {
|
||||
client: "Qwen Code",
|
||||
clientVersion,
|
||||
mode,
|
||||
passed: failure === undefined,
|
||||
artifacts: lab,
|
||||
runtimeRoot,
|
||||
host: {
|
||||
platform: process.platform,
|
||||
architecture: process.arch,
|
||||
node: process.versions.node,
|
||||
},
|
||||
modelFixture:
|
||||
"loopback deterministic OpenAI-compatible endpoint; no live model-provider claim",
|
||||
skillDiscovery:
|
||||
"caller-configured isolated shared directory; default OS home discovery unverified",
|
||||
catalogSize: TOOL_CONTRACTS.length,
|
||||
requests: fixture.requests,
|
||||
probes: fixture.probes,
|
||||
evidenceSeen,
|
||||
selectedSkillLoaded,
|
||||
artifactReadByAgent,
|
||||
artifactPath,
|
||||
artifactDigest,
|
||||
finalSeen,
|
||||
fixtureFailure: fixture.failure,
|
||||
verifier: lineage,
|
||||
};
|
||||
await writeFile(join(lab, "receipt.json"), JSON.stringify(receipt, null, 2));
|
||||
console.log(JSON.stringify(receipt));
|
||||
}
|
||||
if (failure !== undefined) throw failure;
|
||||
@@ -29,7 +29,7 @@ const LINK_MODES = [
|
||||
{
|
||||
mode: "dyld-info",
|
||||
flags: ["-Wl,-no_fixup_chains"],
|
||||
fixups: "LC_DYLD_INFO bind opcodes",
|
||||
fixups: "LC_DYLD_INFO rebase opcodes were not decoded",
|
||||
swift: true,
|
||||
},
|
||||
{ mode: "chained", flags: [], fixups: "chained fixups:", swift: true },
|
||||
@@ -135,7 +135,12 @@ const checkObjc = (metadata, mode) => {
|
||||
item.selector === "reaFixtureLength",
|
||||
) ||
|
||||
objcCoverage?.status !== "complete" ||
|
||||
fixupCoverage(metadata)?.reason?.startsWith(mode.fixups) !== true
|
||||
fixupCoverage(metadata)?.reason?.startsWith(mode.fixups) !== true ||
|
||||
(mode.mode === "dyld-info") !==
|
||||
(fixupCoverage(metadata)?.reason?.includes(
|
||||
"LC_DYLD_INFO bind opcodes",
|
||||
) ===
|
||||
true)
|
||||
)
|
||||
throw new Error(
|
||||
`Apple metadata fixture drifted (${mode.mode}): ${JSON.stringify(metadata)}`,
|
||||
@@ -165,7 +170,12 @@ const checkSwift = (metadata, mode) => {
|
||||
item.owner === "ReaWitnessFixture: ReaWitnessFixtureProtocol" &&
|
||||
item.implementation_address !== null,
|
||||
) ||
|
||||
fixupCoverage(metadata)?.reason?.startsWith(mode.fixups) !== true
|
||||
fixupCoverage(metadata)?.reason?.startsWith(mode.fixups) !== true ||
|
||||
(mode.mode === "dyld-info") !==
|
||||
(fixupCoverage(metadata)?.reason?.includes(
|
||||
"LC_DYLD_INFO bind opcodes",
|
||||
) ===
|
||||
true)
|
||||
)
|
||||
throw new Error(
|
||||
`Swift metadata fixture drifted (${mode.mode}): ${JSON.stringify(metadata)}`,
|
||||
|
||||
@@ -0,0 +1,104 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { createServer } from "node:http";
|
||||
import { writeFile } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
|
||||
/** Exercise the native Gemini API adapter against a local model, without cloud claims. */
|
||||
export const createGeminiModelFixture = async ({ directory, onRequest }) => {
|
||||
const requests = [];
|
||||
const probes = [];
|
||||
let incoming = 0;
|
||||
let failure;
|
||||
const server = createServer(async (request, response) => {
|
||||
try {
|
||||
const chunks = [];
|
||||
let bytes = 0;
|
||||
for await (const chunk of request) {
|
||||
bytes += chunk.length;
|
||||
assert(
|
||||
bytes <= 10 * 1024 * 1024,
|
||||
"Model fixture request exceeds 10 MiB budget",
|
||||
);
|
||||
chunks.push(chunk);
|
||||
}
|
||||
const raw = Buffer.concat(chunks).toString("utf8");
|
||||
const body = raw.length === 0 ? null : JSON.parse(raw);
|
||||
await writeFile(
|
||||
join(directory, `gemini-incoming-${++incoming}.json`),
|
||||
JSON.stringify({ method: request.method, path: request.url, body }),
|
||||
);
|
||||
const path = new URL(request.url, "http://localhost").pathname;
|
||||
if (path.endsWith(":countTokens")) {
|
||||
probes.push({ path, kind: "synthetic-token-count" });
|
||||
response.setHeader("Content-Type", "application/json");
|
||||
response.end(JSON.stringify({ totalTokens: 100 }));
|
||||
return;
|
||||
}
|
||||
if (
|
||||
!/\/models\/[^/]+:(?:streamGenerateContent|generateContent)$/u.test(
|
||||
path,
|
||||
)
|
||||
) {
|
||||
probes.push({ path, status: 404 });
|
||||
response.statusCode = 404;
|
||||
response.end(
|
||||
JSON.stringify({
|
||||
error: { message: "Unsupported fixture endpoint" },
|
||||
}),
|
||||
);
|
||||
return;
|
||||
}
|
||||
assert.equal(request.method, "POST");
|
||||
assert(body && typeof body === "object" && !Array.isArray(body));
|
||||
assert(Array.isArray(body.contents));
|
||||
const tools = (body.tools ?? []).flatMap(
|
||||
(t) => t.functionDeclarations ?? [],
|
||||
);
|
||||
requests.push({
|
||||
path,
|
||||
tools: tools.map((t) => t.name),
|
||||
contentRoles: body.contents.map((c) => c.role),
|
||||
});
|
||||
const parts = await onRequest({
|
||||
body,
|
||||
tools,
|
||||
requestIndex: requests.length,
|
||||
});
|
||||
const result = {
|
||||
candidates: [
|
||||
{ index: 0, content: { role: "model", parts }, finishReason: "STOP" },
|
||||
],
|
||||
usageMetadata: {
|
||||
promptTokenCount: 100,
|
||||
candidatesTokenCount: 20,
|
||||
totalTokenCount: 120,
|
||||
},
|
||||
modelVersion: "local-rea-fixture",
|
||||
};
|
||||
if (path.endsWith(":streamGenerateContent")) {
|
||||
response.setHeader("Content-Type", "text/event-stream");
|
||||
response.end(`data: ${JSON.stringify(result)}\n\n`);
|
||||
} else {
|
||||
response.setHeader("Content-Type", "application/json");
|
||||
response.end(JSON.stringify(result));
|
||||
}
|
||||
} catch (cause) {
|
||||
failure = String(cause);
|
||||
response.statusCode = 400;
|
||||
response.setHeader("Content-Type", "application/json");
|
||||
response.end(JSON.stringify({ error: { message: failure, code: 400 } }));
|
||||
}
|
||||
});
|
||||
await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve));
|
||||
const address = server.address();
|
||||
assert(address !== null && typeof address !== "string");
|
||||
return {
|
||||
baseUrl: `http://127.0.0.1:${address.port}`,
|
||||
requests,
|
||||
probes,
|
||||
get failure() {
|
||||
return failure;
|
||||
},
|
||||
close: () => new Promise((resolve) => server.close(resolve)),
|
||||
};
|
||||
};
|
||||
@@ -0,0 +1,171 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { createServer } from "node:http";
|
||||
import { writeFile } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
|
||||
/** A loopback model fixture exercises real clients without claiming live-provider compatibility. */
|
||||
export const createOpenAiModelFixture = async ({
|
||||
directory,
|
||||
prefix,
|
||||
onRequest,
|
||||
}) => {
|
||||
const requests = [];
|
||||
const probes = [];
|
||||
let incoming = 0;
|
||||
let failure;
|
||||
const server = createServer(async (request, response) => {
|
||||
try {
|
||||
if (request.method === "GET" && request.url?.endsWith("/models")) {
|
||||
response.setHeader("Content-Type", "application/json");
|
||||
response.end(
|
||||
JSON.stringify({
|
||||
object: "list",
|
||||
data: [
|
||||
{
|
||||
id: "rea-client-fixture",
|
||||
object: "model",
|
||||
owned_by: "local-fixture",
|
||||
},
|
||||
],
|
||||
}),
|
||||
);
|
||||
return;
|
||||
}
|
||||
const chunks = [];
|
||||
let bytes = 0;
|
||||
for await (const chunk of request) {
|
||||
bytes += chunk.length;
|
||||
assert(
|
||||
bytes <= 10 * 1024 * 1024,
|
||||
"Local model fixture exceeded its 10 MiB request resource budget",
|
||||
);
|
||||
chunks.push(chunk);
|
||||
}
|
||||
const rawBody = Buffer.concat(chunks).toString("utf8");
|
||||
const body = rawBody.length === 0 ? null : JSON.parse(rawBody);
|
||||
await writeFile(
|
||||
join(directory, `${prefix}-incoming-${++incoming}.json`),
|
||||
JSON.stringify({ method: request.method, path: request.url, body }),
|
||||
);
|
||||
if (request.url !== "/v1/chat/completions") {
|
||||
// Hermes probes custom endpoints for Ollama/llama.cpp metadata.
|
||||
// Unsupported HTTP endpoints return 404, as on an OpenAI-only server.
|
||||
probes.push({
|
||||
method: request.method,
|
||||
path: request.url,
|
||||
body,
|
||||
status: 404,
|
||||
});
|
||||
response.statusCode = 404;
|
||||
response.setHeader("Content-Type", "application/json");
|
||||
response.end(
|
||||
JSON.stringify({
|
||||
error:
|
||||
"This OpenAI fixture exposes models and chat completions only.",
|
||||
}),
|
||||
);
|
||||
return;
|
||||
}
|
||||
assert.equal(request.method, "POST");
|
||||
assert(body !== null && typeof body === "object" && !Array.isArray(body));
|
||||
assert(Array.isArray(body.messages));
|
||||
const tools = (body.tools ?? [])
|
||||
.filter((t) => t.type === "function")
|
||||
.map((t) => t.function ?? t);
|
||||
requests.push({
|
||||
path: request.url,
|
||||
tools: tools.map((t) => t.name),
|
||||
stream: body.stream,
|
||||
messageRoles: body.messages.map((m) => m.role),
|
||||
});
|
||||
await writeFile(
|
||||
join(directory, `${prefix}-request-${requests.length}.json`),
|
||||
JSON.stringify(body),
|
||||
);
|
||||
const message = await onRequest({
|
||||
body,
|
||||
tools,
|
||||
requestIndex: requests.length,
|
||||
});
|
||||
const base = {
|
||||
id: `chatcmpl-rea-${requests.length}`,
|
||||
object: "chat.completion",
|
||||
created: Math.floor(Date.now() / 1000),
|
||||
model: body.model,
|
||||
};
|
||||
const finish = message.tool_calls ? "tool_calls" : "stop";
|
||||
const usage = {
|
||||
prompt_tokens: 100,
|
||||
completion_tokens: 20,
|
||||
total_tokens: 120,
|
||||
};
|
||||
if (body.stream) {
|
||||
response.setHeader("Content-Type", "text/event-stream");
|
||||
const emit = (delta, finishReason = null) =>
|
||||
response.write(
|
||||
`data: ${JSON.stringify({ ...base, object: "chat.completion.chunk", choices: [{ index: 0, delta, finish_reason: finishReason }] })}\n\n`,
|
||||
);
|
||||
if (message.tool_calls) {
|
||||
const fragments = message.tool_calls.map((call, index) => {
|
||||
const split = Math.floor(call.function.arguments.length / 2);
|
||||
return {
|
||||
first: {
|
||||
index,
|
||||
...call,
|
||||
function: {
|
||||
name: call.function.name,
|
||||
arguments: call.function.arguments.slice(0, split),
|
||||
},
|
||||
},
|
||||
second: {
|
||||
index,
|
||||
function: { arguments: call.function.arguments.slice(split) },
|
||||
},
|
||||
};
|
||||
});
|
||||
emit({
|
||||
role: "assistant",
|
||||
content: null,
|
||||
tool_calls: fragments.map((f) => f.first),
|
||||
});
|
||||
emit({ tool_calls: fragments.map((f) => f.second) });
|
||||
} else emit(message);
|
||||
emit({}, finish);
|
||||
response.write(
|
||||
`data: ${JSON.stringify({ ...base, object: "chat.completion.chunk", choices: [], usage })}\n\n`,
|
||||
);
|
||||
response.end("data: [DONE]\n\n");
|
||||
} else {
|
||||
response.setHeader("Content-Type", "application/json");
|
||||
response.end(
|
||||
JSON.stringify({
|
||||
...base,
|
||||
choices: [{ index: 0, message, finish_reason: finish }],
|
||||
usage,
|
||||
}),
|
||||
);
|
||||
}
|
||||
} catch (cause) {
|
||||
failure = String(cause);
|
||||
response.statusCode = 400;
|
||||
response.setHeader("Content-Type", "application/json");
|
||||
response.end(
|
||||
JSON.stringify({
|
||||
error: { message: failure, type: "invalid_request_error" },
|
||||
}),
|
||||
);
|
||||
}
|
||||
});
|
||||
await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve));
|
||||
const address = server.address();
|
||||
assert(address !== null && typeof address !== "string");
|
||||
return {
|
||||
baseUrl: `http://127.0.0.1:${address.port}/v1`,
|
||||
requests,
|
||||
probes,
|
||||
get failure() {
|
||||
return failure;
|
||||
},
|
||||
close: () => new Promise((resolve) => server.close(resolve)),
|
||||
};
|
||||
};
|
||||
@@ -88,7 +88,10 @@ window.loadFile("renderer.html");require("node:fs").writeFileSync(${JSON.stringi
|
||||
ajv.compile(tool.inputSchema);
|
||||
return [
|
||||
tool.name,
|
||||
tool.outputSchema ? ajv.compile(tool.outputSchema) : undefined,
|
||||
tool.outputSchema
|
||||
? (tool.outputSchema.$id && ajv.getSchema(tool.outputSchema.$id)) ||
|
||||
ajv.compile(tool.outputSchema)
|
||||
: undefined,
|
||||
];
|
||||
}),
|
||||
);
|
||||
|
||||
@@ -1413,3 +1413,45 @@ export const clientRegistrationEntry = (
|
||||
};
|
||||
}
|
||||
};
|
||||
|
||||
/** Parse one Gemini policy at its source boundary before consolidating scopes. */
|
||||
export const parseGeminiMcpPolicy = (parsed: ClientConfigurationDocument) =>
|
||||
z
|
||||
.object({
|
||||
allowed: z.array(z.string()).optional(),
|
||||
excluded: z.array(z.string()).optional(),
|
||||
})
|
||||
.passthrough()
|
||||
.parse(parsed.document.mcp ?? {});
|
||||
|
||||
/** Gemini consolidates excludes by union and allowlists by normalized intersection. */
|
||||
export const geminiServerPolicyBlock = (
|
||||
policies: readonly ReturnType<typeof parseGeminiMcpPolicy>[],
|
||||
serverKey: string,
|
||||
): string | undefined => {
|
||||
if (policies.some(({ excluded }) => excluded?.includes(serverKey)))
|
||||
return `Gemini mcp.excluded blocks ${serverKey}. Review the selected settings policies before rerunning setup.`;
|
||||
const allowlists = policies.flatMap(({ allowed }) =>
|
||||
allowed === undefined ? [] : [allowed],
|
||||
);
|
||||
const allowed = allowlists.reduce<string[] | undefined>(
|
||||
(previous, current) => {
|
||||
if (previous === undefined) return current;
|
||||
const normalized = new Set(
|
||||
current.map((name) => name.toLowerCase().trim()),
|
||||
);
|
||||
return previous.filter((name) =>
|
||||
normalized.has(name.toLowerCase().trim()),
|
||||
);
|
||||
},
|
||||
undefined,
|
||||
);
|
||||
// Gemini's manager applies the consolidated allowlist only when nonempty.
|
||||
if (
|
||||
allowed !== undefined &&
|
||||
allowed.length > 0 &&
|
||||
!allowed.includes(serverKey)
|
||||
)
|
||||
return `Gemini mcp.allowed does not include ${serverKey}. Review the selected settings policies before rerunning setup.`;
|
||||
return undefined;
|
||||
};
|
||||
|
||||
@@ -0,0 +1,108 @@
|
||||
import { err, ok, type Result } from "../domain/result.js";
|
||||
import { readFile } from "node:fs/promises";
|
||||
import {
|
||||
effectiveClientServer,
|
||||
parseClientConfiguration,
|
||||
type ClientConfigurationDocument,
|
||||
} from "./ClientConfigurationDocument.js";
|
||||
import type { SetupClient } from "./SupportedClients.js";
|
||||
|
||||
/** A named source failure that setup can explain without losing the bad path. */
|
||||
export interface ClientConfigurationFileError {
|
||||
readonly kind: "unreadable" | "malformed";
|
||||
readonly path: string;
|
||||
readonly detail: string;
|
||||
}
|
||||
|
||||
/** Preserve named read failures and distinguish a missing source from an empty file. */
|
||||
export const readConfigurationText = async (
|
||||
path: string,
|
||||
): Promise<Result<string | undefined, ClientConfigurationFileError>> => {
|
||||
try {
|
||||
return ok(await readFile(path, "utf8"));
|
||||
} catch (cause: unknown) {
|
||||
if (cause instanceof Error && "code" in cause && cause.code === "ENOENT")
|
||||
return ok(undefined);
|
||||
return err({
|
||||
kind: "unreadable",
|
||||
path,
|
||||
detail: `Configuration ${path} could not be read: ${cause instanceof Error ? cause.message : String(cause)}`,
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
/** Read and parse one configuration at its source boundary. */
|
||||
export const readConfigurationFile = async (
|
||||
path: string,
|
||||
format: SetupClient["format"],
|
||||
): Promise<
|
||||
Result<ClientConfigurationDocument | undefined, ClientConfigurationFileError>
|
||||
> => {
|
||||
const source = await readConfigurationText(path);
|
||||
if (!source.ok) return source;
|
||||
const content = source.value;
|
||||
if (content === undefined) return ok(undefined);
|
||||
try {
|
||||
return ok(parseClientConfiguration(content, format));
|
||||
} catch (cause: unknown) {
|
||||
return err({
|
||||
kind: "malformed",
|
||||
path,
|
||||
detail: `Configuration ${path} is malformed: ${cause instanceof Error ? cause.message : String(cause)}`,
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
/** Read every applicable user file; malformed lower-priority files also stop setup. */
|
||||
export const readClientConfigurationFiles = async (
|
||||
client: SetupClient,
|
||||
): Promise<
|
||||
Result<readonly ClientConfigurationDocument[], ClientConfigurationFileError>
|
||||
> => {
|
||||
const documents: ClientConfigurationDocument[] = [];
|
||||
for (const path of client.configPaths ?? [client.configPath]) {
|
||||
const document = await readConfigurationFile(path, client.format);
|
||||
if (!document.ok) return document;
|
||||
if (document.value !== undefined) documents.push(document.value);
|
||||
}
|
||||
return ok(documents);
|
||||
};
|
||||
|
||||
/** OpenCode merges server objects in file order, replacing arrays and scalar values. */
|
||||
const mergeConfigurationValue = (previous: unknown, next: unknown): unknown => {
|
||||
if (!isObject(previous) || !isObject(next)) return next;
|
||||
return Object.fromEntries(
|
||||
[...new Set([...Object.keys(previous), ...Object.keys(next)])].map(
|
||||
(key) => [
|
||||
key,
|
||||
Object.hasOwn(next, key)
|
||||
? mergeConfigurationValue(previous[key], next[key])
|
||||
: previous[key],
|
||||
],
|
||||
),
|
||||
);
|
||||
};
|
||||
const isObject = (value: unknown): value is Record<string, unknown> =>
|
||||
typeof value === "object" && value !== null && !Array.isArray(value);
|
||||
|
||||
/** Project a client's effective user registration without modifying its source files. */
|
||||
export const effectiveClientConfiguration = (
|
||||
documents: readonly ClientConfigurationDocument[],
|
||||
): ClientConfigurationDocument | undefined => {
|
||||
const last = documents.at(-1);
|
||||
if (last === undefined) return undefined;
|
||||
if (documents.length === 1) return last;
|
||||
const servers: Record<string, unknown> = {};
|
||||
for (const document of documents) {
|
||||
const names = new Set([
|
||||
...Object.keys(document.legacyServers),
|
||||
...Object.keys(document.servers),
|
||||
]);
|
||||
for (const name of names)
|
||||
servers[name] = mergeConfigurationValue(
|
||||
servers[name],
|
||||
effectiveClientServer(document, name),
|
||||
);
|
||||
}
|
||||
return { ...last, servers, legacyServers: {} };
|
||||
};
|
||||
@@ -1,10 +1,14 @@
|
||||
import { readClientPolicyBlock } from "./GeminiClientSettings.js";
|
||||
import {
|
||||
effectiveClientConfiguration,
|
||||
readClientConfigurationFiles,
|
||||
} from "./ClientConfigurationFiles.js";
|
||||
import {
|
||||
clientServerForcedEnabled,
|
||||
clientServerListedDisabled,
|
||||
effectiveClientServer,
|
||||
parseClientConfiguration,
|
||||
} from "./ClientConfigurationDocument.js";
|
||||
import { access, readFile } from "node:fs/promises";
|
||||
import { access } from "node:fs/promises";
|
||||
import { resolve } from "node:path";
|
||||
|
||||
import { z } from "zod";
|
||||
@@ -16,6 +20,7 @@ import {
|
||||
npxRegistrationCommand,
|
||||
} from "./ClientRegistrationIdentity.js";
|
||||
import {
|
||||
clientEvidencePaths,
|
||||
manualRegistrationRemediation,
|
||||
supportedClients,
|
||||
} from "./SupportedClients.js";
|
||||
@@ -90,11 +95,7 @@ export const readClientRegistrationStatuses = async (
|
||||
});
|
||||
continue;
|
||||
}
|
||||
if (
|
||||
!(await exists(client.markerPath)) &&
|
||||
!(await exists(client.configPath))
|
||||
)
|
||||
continue;
|
||||
if (!(await clientEvidencePresent(client))) continue;
|
||||
const manualRemediation = manualRegistrationRemediation(client.name);
|
||||
if (client.format === "unsupported") {
|
||||
if (manualRemediation !== undefined)
|
||||
@@ -108,8 +109,36 @@ export const readClientRegistrationStatuses = async (
|
||||
continue;
|
||||
}
|
||||
try {
|
||||
const content = await readFile(client.configPath, "utf8");
|
||||
const parsed = parseClientConfiguration(content, client.format);
|
||||
const files = await readClientConfigurationFiles(client);
|
||||
if (!files.ok) {
|
||||
statuses.push({
|
||||
client: client.name,
|
||||
config_path: client.configPath,
|
||||
command: [],
|
||||
state: "invalid",
|
||||
remediation: files.error.detail,
|
||||
});
|
||||
continue;
|
||||
}
|
||||
const parsed = effectiveClientConfiguration(files.value);
|
||||
if (parsed === undefined) {
|
||||
statuses.push(
|
||||
unavailableStatus(client.name, client.configPath, "missing"),
|
||||
);
|
||||
continue;
|
||||
}
|
||||
const policy = await readClientPolicyBlock(client, parsed);
|
||||
if (!policy.ok) {
|
||||
statuses.push({
|
||||
client: client.name,
|
||||
config_path: client.configPath,
|
||||
command: [],
|
||||
state: "invalid",
|
||||
remediation: policy.error.detail,
|
||||
});
|
||||
continue;
|
||||
}
|
||||
const policyBlock = policy.value;
|
||||
const raw = effectiveClientServer(parsed, PRODUCT_IDENTITY.mcpServerKey);
|
||||
if (raw === undefined) {
|
||||
statuses.push(
|
||||
@@ -134,9 +163,14 @@ export const readClientRegistrationStatuses = async (
|
||||
options.platform ?? process.platform,
|
||||
clientServerForcedEnabled(parsed, PRODUCT_IDENTITY.mcpServerKey),
|
||||
) &&
|
||||
!clientServerListedDisabled(parsed, PRODUCT_IDENTITY.mcpServerKey)
|
||||
!clientServerListedDisabled(
|
||||
parsed,
|
||||
PRODUCT_IDENTITY.mcpServerKey,
|
||||
) &&
|
||||
policyBlock === undefined
|
||||
? "aligned"
|
||||
: "stale",
|
||||
policyBlock,
|
||||
),
|
||||
);
|
||||
} catch (cause: unknown) {
|
||||
@@ -163,7 +197,7 @@ const registrationAligned = (
|
||||
): boolean => {
|
||||
const command = [registration.command, ...registration.args];
|
||||
if (
|
||||
registration.disabled === true ||
|
||||
(client.format !== "grok" && registration.disabled === true) ||
|
||||
(registration.enabled === false && !forcedEnabled)
|
||||
)
|
||||
return false;
|
||||
@@ -256,10 +290,17 @@ const configuredStatus = (
|
||||
configPath: string,
|
||||
command: RegistrationCommand,
|
||||
state: "aligned" | "stale",
|
||||
policyRemediation?: string,
|
||||
): ClientRegistrationStatus =>
|
||||
state === "aligned"
|
||||
? { client, config_path: configPath, command, state, remediation: null }
|
||||
: { client, config_path: configPath, command, state, remediation };
|
||||
: {
|
||||
client,
|
||||
config_path: configPath,
|
||||
command,
|
||||
state,
|
||||
remediation: policyRemediation ?? remediation,
|
||||
};
|
||||
|
||||
const unavailableStatus = (
|
||||
client: string,
|
||||
@@ -273,6 +314,12 @@ const unavailableStatus = (
|
||||
remediation,
|
||||
});
|
||||
|
||||
const clientEvidencePresent = async (client: SetupClient): Promise<boolean> => {
|
||||
for (const path of clientEvidencePaths(client))
|
||||
if (await exists(path)) return true;
|
||||
return false;
|
||||
};
|
||||
|
||||
const exists = async (path: string | undefined): Promise<boolean> => {
|
||||
if (path === undefined) return false;
|
||||
try {
|
||||
|
||||
@@ -0,0 +1,202 @@
|
||||
import { realpath } from "node:fs/promises";
|
||||
import { dirname, resolve } from "node:path";
|
||||
import { stripComments } from "jsonc-parser";
|
||||
import { z } from "zod";
|
||||
import { err, ok, type Result } from "../domain/result.js";
|
||||
import { PRODUCT_IDENTITY } from "../identity.js";
|
||||
import { npxRegistrationCommand } from "./ClientRegistrationIdentity.js";
|
||||
import {
|
||||
clientConfigurationValuesEqual,
|
||||
clientRegistrationEntry,
|
||||
geminiServerPolicyBlock,
|
||||
parseGeminiMcpPolicy,
|
||||
type ClientConfigurationDocument,
|
||||
} from "./ClientConfigurationDocument.js";
|
||||
import {
|
||||
readConfigurationFile,
|
||||
readConfigurationText,
|
||||
type ClientConfigurationFileError,
|
||||
} from "./ClientConfigurationFiles.js";
|
||||
import type { SetupClient } from "./SupportedClients.js";
|
||||
|
||||
type SettingsSource = { path: string; document: ClientConfigurationDocument };
|
||||
const folderTrustSchema = z
|
||||
.object({
|
||||
security: z
|
||||
.object({
|
||||
folderTrust: z
|
||||
.object({ enabled: z.boolean().optional() })
|
||||
.passthrough()
|
||||
.optional(),
|
||||
})
|
||||
.passthrough()
|
||||
.optional(),
|
||||
})
|
||||
.passthrough();
|
||||
const trustRulesSchema = z.record(
|
||||
z.string(),
|
||||
z.enum(["TRUST_FOLDER", "TRUST_PARENT", "DO_NOT_TRUST"]),
|
||||
);
|
||||
const sourceFailure = (
|
||||
path: string,
|
||||
cause: unknown,
|
||||
): ClientConfigurationFileError => ({
|
||||
kind: "malformed",
|
||||
path,
|
||||
detail: `Gemini settings in ${path} are malformed: ${cause instanceof Error ? cause.message : String(cause)}`,
|
||||
});
|
||||
|
||||
// Gemini resolves symlinks when possible and compares macOS/Windows paths without case.
|
||||
const normalizedPath = async (
|
||||
path: string,
|
||||
platform: NodeJS.Platform,
|
||||
): Promise<string> => {
|
||||
const physical = await realpath(path).catch(() => path);
|
||||
const absolute = resolve(physical).replaceAll("\\", "/");
|
||||
return platform === "win32" || platform === "darwin"
|
||||
? absolute.toLowerCase()
|
||||
: absolute;
|
||||
};
|
||||
|
||||
const workspaceTrusted = async (
|
||||
settings: NonNullable<SetupClient["geminiSettings"]>,
|
||||
sources: readonly SettingsSource[],
|
||||
): Promise<Result<boolean, ClientConfigurationFileError>> => {
|
||||
let enabled = true;
|
||||
for (const { path, document } of sources) {
|
||||
const parsed = folderTrustSchema.safeParse(document.document);
|
||||
if (!parsed.success) return err(sourceFailure(path, parsed.error));
|
||||
enabled = parsed.data.security?.folderTrust?.enabled ?? enabled;
|
||||
}
|
||||
if (settings.trustOverride !== undefined) return ok(settings.trustOverride);
|
||||
if (!enabled) return ok(true);
|
||||
const text = await readConfigurationText(settings.trustedFoldersPath);
|
||||
if (!text.ok) return text;
|
||||
if (text.value === undefined) return ok(false);
|
||||
let rules: z.output<typeof trustRulesSchema>;
|
||||
try {
|
||||
// Gemini strips comments, but rejects trailing commas, BOMs and invalid trust levels.
|
||||
rules = trustRulesSchema.parse(JSON.parse(stripComments(text.value)));
|
||||
} catch (cause: unknown) {
|
||||
return err(sourceFailure(settings.trustedFoldersPath, cause));
|
||||
}
|
||||
const workspace = await normalizedPath(
|
||||
settings.workspaceDirectory,
|
||||
settings.platform,
|
||||
);
|
||||
let longest = -1;
|
||||
let trusted = false;
|
||||
for (const [path, level] of Object.entries(rules)) {
|
||||
const effective = await normalizedPath(
|
||||
level === "TRUST_PARENT" ? dirname(path) : path,
|
||||
settings.platform,
|
||||
);
|
||||
if (
|
||||
(workspace === effective ||
|
||||
workspace.startsWith(
|
||||
effective.endsWith("/") ? effective : `${effective}/`,
|
||||
)) &&
|
||||
path.length > longest
|
||||
) {
|
||||
longest = path.length;
|
||||
trusted = level !== "DO_NOT_TRUST";
|
||||
}
|
||||
}
|
||||
return ok(trusted);
|
||||
};
|
||||
|
||||
/** Check effective Gemini scopes while preserving their authority and source identity. */
|
||||
export const readClientPolicyBlock = async (
|
||||
client: SetupClient,
|
||||
userDocument?: ClientConfigurationDocument,
|
||||
desiredRegistration?: unknown,
|
||||
): Promise<Result<string | undefined, ClientConfigurationFileError>> => {
|
||||
if (client.name !== "gemini_cli" || client.geminiSettings === undefined)
|
||||
return ok(undefined);
|
||||
const settings = client.geminiSettings;
|
||||
const sources: SettingsSource[] = [];
|
||||
const higher: SettingsSource[] = [];
|
||||
for (const path of [
|
||||
settings.defaultsPath,
|
||||
client.configPath,
|
||||
settings.systemPath,
|
||||
]) {
|
||||
const parsed =
|
||||
path === client.configPath && userDocument !== undefined
|
||||
? ok(userDocument)
|
||||
: await readConfigurationFile(path, "json");
|
||||
if (!parsed.ok) return parsed;
|
||||
if (parsed.value !== undefined)
|
||||
sources.push({ path, document: parsed.value });
|
||||
}
|
||||
const workspace = await normalizedPath(
|
||||
settings.workspaceDirectory,
|
||||
settings.platform,
|
||||
);
|
||||
const home = await normalizedPath(
|
||||
dirname(dirname(client.configPath)),
|
||||
settings.platform,
|
||||
);
|
||||
if (workspace !== home) {
|
||||
const path = resolve(
|
||||
settings.workspaceDirectory,
|
||||
".gemini",
|
||||
"settings.json",
|
||||
);
|
||||
const document = await readConfigurationFile(path, "json");
|
||||
if (!document.ok) return document;
|
||||
if (document.value !== undefined) {
|
||||
const trusted = await workspaceTrusted(settings, sources);
|
||||
if (!trusted.ok) return trusted;
|
||||
if (trusted.value) {
|
||||
const source = { path, document: document.value };
|
||||
const systemIndex = sources.findIndex(
|
||||
(item) => item.path === settings.systemPath,
|
||||
);
|
||||
sources.splice(
|
||||
systemIndex < 0 ? sources.length : systemIndex,
|
||||
0,
|
||||
source,
|
||||
);
|
||||
higher.push(source);
|
||||
}
|
||||
}
|
||||
}
|
||||
const system = sources.find(({ path }) => path === settings.systemPath);
|
||||
if (system !== undefined) higher.push(system);
|
||||
const policies: ReturnType<typeof parseGeminiMcpPolicy>[] = [];
|
||||
for (const { path, document } of sources) {
|
||||
try {
|
||||
policies.push(parseGeminiMcpPolicy(document));
|
||||
} catch (cause: unknown) {
|
||||
return err(sourceFailure(path, cause));
|
||||
}
|
||||
}
|
||||
const policy = geminiServerPolicyBlock(
|
||||
policies,
|
||||
PRODUCT_IDENTITY.mcpServerKey,
|
||||
);
|
||||
if (policy !== undefined)
|
||||
return ok(
|
||||
`${policy} Settings sources: ${sources.map(({ path }) => path).join(", ")}.`,
|
||||
);
|
||||
// mcpServers uses shallow merge: the entire same-name definition is replaced.
|
||||
const override = higher.findLast(({ document }) =>
|
||||
Object.hasOwn(document.servers, PRODUCT_IDENTITY.mcpServerKey),
|
||||
);
|
||||
const desired =
|
||||
desiredRegistration ??
|
||||
userDocument?.servers[PRODUCT_IDENTITY.mcpServerKey] ??
|
||||
clientRegistrationEntry("json", npxRegistrationCommand(), {});
|
||||
if (
|
||||
override !== undefined &&
|
||||
!clientConfigurationValuesEqual(
|
||||
override.document.servers[PRODUCT_IDENTITY.mcpServerKey],
|
||||
desired,
|
||||
)
|
||||
)
|
||||
return ok(
|
||||
`Gemini mcpServers.${PRODUCT_IDENTITY.mcpServerKey} in ${override.path} overrides the user registration. Review that settings file before rerunning setup; REA does not modify system or workspace settings.`,
|
||||
);
|
||||
return ok(undefined);
|
||||
};
|
||||
@@ -0,0 +1,150 @@
|
||||
import { lstatSync, readFileSync, realpathSync, statSync } from "node:fs";
|
||||
import {
|
||||
basename,
|
||||
dirname,
|
||||
isAbsolute,
|
||||
join,
|
||||
relative,
|
||||
resolve,
|
||||
sep,
|
||||
} from "node:path";
|
||||
|
||||
/** An unresolved Hermes profile must not be configured as the default home. */
|
||||
export class HermesClientPathError extends Error {
|
||||
constructor(
|
||||
readonly path: string,
|
||||
message: string,
|
||||
) {
|
||||
super(message);
|
||||
this.name = "HermesClientPathError";
|
||||
}
|
||||
}
|
||||
|
||||
const defaultDirectory = (
|
||||
home: string,
|
||||
platform: NodeJS.Platform,
|
||||
env: Readonly<NodeJS.ProcessEnv>,
|
||||
): string => {
|
||||
const suffix = env.HERMES_DATA_DIR_SUFFIX ?? "";
|
||||
return platform === "win32"
|
||||
? join(
|
||||
env.LOCALAPPDATA?.trim() || join(home, "AppData", "Local"),
|
||||
`hermes${suffix}`,
|
||||
)
|
||||
: join(home, `.hermes${suffix}`);
|
||||
};
|
||||
const expandedDirectory = (
|
||||
home: string,
|
||||
platform: NodeJS.Platform,
|
||||
env: Readonly<NodeJS.ProcessEnv>,
|
||||
): string | undefined => {
|
||||
const override = env.HERMES_HOME?.trim();
|
||||
if (!override) return undefined;
|
||||
const variables = override.replace(
|
||||
/\$(\w+|\{[^}]*\})/gu,
|
||||
(match, variable: string) => {
|
||||
const key = variable.startsWith("{") ? variable.slice(1, -1) : variable;
|
||||
return env[key] ?? match;
|
||||
},
|
||||
);
|
||||
const expanded =
|
||||
platform === "win32"
|
||||
? variables.replace(
|
||||
/%([^%]+)%/gu,
|
||||
(match, variable: string) => env[variable] ?? match,
|
||||
)
|
||||
: variables;
|
||||
if (expanded === "~") return home;
|
||||
if (
|
||||
expanded.startsWith("~/") ||
|
||||
(platform === "win32" && expanded.startsWith("~\\"))
|
||||
)
|
||||
return join(home, expanded.slice(2));
|
||||
return resolve(expanded);
|
||||
};
|
||||
const physicalPath = (path: string): string => {
|
||||
try {
|
||||
return realpathSync(path);
|
||||
} catch {
|
||||
return resolve(path);
|
||||
}
|
||||
};
|
||||
const within = (root: string, path: string): boolean => {
|
||||
const difference = relative(physicalPath(root), physicalPath(path));
|
||||
return (
|
||||
difference === "" ||
|
||||
(!isAbsolute(difference) &&
|
||||
difference !== ".." &&
|
||||
!difference.startsWith(`..${sep}`))
|
||||
);
|
||||
};
|
||||
const PROFILE_MARKERS = [
|
||||
"config.yaml",
|
||||
".env",
|
||||
"SOUL.md",
|
||||
"profile.yaml",
|
||||
"auth.json",
|
||||
"state.db",
|
||||
];
|
||||
const RESERVED_PROFILES = new Set(["hermes", "test", "tmp", "root", "sudo"]);
|
||||
const liveProfile = (path: string): boolean => {
|
||||
try {
|
||||
if (!statSync(path).isDirectory()) return false;
|
||||
try {
|
||||
statSync(join(dirname(path), ".deleted", basename(path)));
|
||||
return false;
|
||||
} catch {
|
||||
/* No tombstone. */
|
||||
}
|
||||
return PROFILE_MARKERS.some((marker) => {
|
||||
try {
|
||||
const status = lstatSync(join(path, marker));
|
||||
return status.isFile() || status.isSymbolicLink();
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
});
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
};
|
||||
|
||||
/** Match Hermes CLI sticky-profile selection while retaining the configured path spelling. */
|
||||
export const resolveHermesClientHome = (
|
||||
home: string,
|
||||
platform: NodeJS.Platform,
|
||||
env: Readonly<NodeJS.ProcessEnv>,
|
||||
): string => {
|
||||
const native = defaultDirectory(home, platform, env);
|
||||
const root = expandedDirectory(home, platform, env) ?? native;
|
||||
// An explicit named profile bypasses the root's sticky selection in Hermes CLI.
|
||||
if (basename(dirname(root)) === "profiles") return root;
|
||||
const stickyRoot = within(native, root) ? native : root;
|
||||
let name: string;
|
||||
try {
|
||||
// Python uses utf-8-sig and ignores unreadable / invalid-UTF8 active_profile files.
|
||||
name = new TextDecoder("utf-8", { fatal: true })
|
||||
.decode(readFileSync(join(stickyRoot, "active_profile")))
|
||||
.trim();
|
||||
} catch {
|
||||
return root;
|
||||
}
|
||||
if (name === "" || name === "default") return root;
|
||||
const canonical = name.toLowerCase();
|
||||
if (canonical === "default") return root;
|
||||
if (
|
||||
!/^[a-z0-9][a-z0-9_-]{0,63}$/u.test(canonical) ||
|
||||
RESERVED_PROFILES.has(canonical)
|
||||
)
|
||||
throw new HermesClientPathError(
|
||||
join(stickyRoot, "active_profile"),
|
||||
`Invalid saved Hermes profile ${JSON.stringify(name)}. Run 'hermes profile use default' or repair active_profile.`,
|
||||
);
|
||||
const selected = join(root, "profiles", canonical);
|
||||
if (!liveProfile(selected))
|
||||
throw new HermesClientPathError(
|
||||
selected,
|
||||
`Saved Hermes profile ${JSON.stringify(name)} is missing, deleted, or has no profile identity. Run 'hermes profile use default' or restore that profile before configuring it.`,
|
||||
);
|
||||
return selected;
|
||||
};
|
||||
@@ -0,0 +1,33 @@
|
||||
import { expect, it } from "vitest";
|
||||
|
||||
import { createProgressReporter } from "./ProgressReporter.js";
|
||||
|
||||
it("limits intermediate sends to 100 ms and always delivers terminal status", async () => {
|
||||
let now = 0;
|
||||
const sent: number[] = [];
|
||||
const reporter = createProgressReporter(
|
||||
async () => {
|
||||
sent.push(now);
|
||||
},
|
||||
{ now: () => now },
|
||||
);
|
||||
for (const [completed, elapsed] of [0, 25, 50, 99, 100, 199, 200].entries()) {
|
||||
now = elapsed;
|
||||
await reporter.report({
|
||||
phase: "running",
|
||||
completed,
|
||||
total: null,
|
||||
message: "Observed capture status",
|
||||
});
|
||||
}
|
||||
expect(sent).toEqual([0, 100, 200]);
|
||||
now = 201;
|
||||
await reporter.report({
|
||||
phase: "cleanup",
|
||||
completed: 7,
|
||||
total: null,
|
||||
message: "Owned cleanup complete",
|
||||
terminal: true,
|
||||
});
|
||||
expect(sent).toEqual([0, 100, 200, 201]);
|
||||
});
|
||||
@@ -1,6 +1,10 @@
|
||||
import { createHash } from "node:crypto";
|
||||
import { posix } from "node:path";
|
||||
|
||||
import {
|
||||
logicalPathEscapesRoot,
|
||||
normalizeJoinedLogicalPath,
|
||||
} from "../domain/artifactIdentity.js";
|
||||
import {
|
||||
classifyReferenceSourcePath,
|
||||
detectReferenceSourceLanguage,
|
||||
@@ -83,10 +87,10 @@ const resolveInternalSpecifier = (
|
||||
): { to: string; resolution: "internal" | "unresolved" } => {
|
||||
if (posix.isAbsolute(specifier))
|
||||
return { to: specifier, resolution: "unresolved" };
|
||||
const normalized = posix.normalize(
|
||||
const normalized = normalizeJoinedLogicalPath(
|
||||
posix.join(posix.dirname(fromPath), specifier),
|
||||
);
|
||||
if (normalized.startsWith("../") || normalized === "..")
|
||||
if (logicalPathEscapesRoot(normalized))
|
||||
return { to: normalized, resolution: "unresolved" };
|
||||
const suffixes = ["", ".ts", ".js", ".mjs", ".cjs", ".tsx", ".jsx"];
|
||||
const candidates = suffixes.flatMap((suffix) => [
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
import { readClientPolicyBlock } from "./GeminiClientSettings.js";
|
||||
import { readClientConfigurationFiles } from "./ClientConfigurationFiles.js";
|
||||
import {
|
||||
clientRegistrationEntry,
|
||||
clientConfigurationValuesEqual,
|
||||
@@ -46,6 +48,15 @@ const configureClientDocument = async (
|
||||
command: readonly string[],
|
||||
format: NonNullable<SetupClient["format"]>,
|
||||
): Promise<ClientConfigurationResult> => {
|
||||
const files = await readClientConfigurationFiles(client);
|
||||
if (!files.ok) return { status: "failed", reason: "readback" };
|
||||
const policy = await readClientPolicyBlock(
|
||||
client,
|
||||
undefined,
|
||||
clientConfigurationDesired(client, environment, command, client.format),
|
||||
);
|
||||
if (!policy.ok || policy.value !== undefined)
|
||||
return { status: "failed", reason: "readback" };
|
||||
const transactionPath = await resolveClientConfigTransactionPath(
|
||||
client.configPath,
|
||||
);
|
||||
@@ -63,6 +74,13 @@ const configureClientDocument = async (
|
||||
original ?? (format === "toml" || format === "grok" ? "" : "{}"),
|
||||
format,
|
||||
);
|
||||
const policy = await readClientPolicyBlock(
|
||||
client,
|
||||
parsed,
|
||||
clientConfigurationDesired(client, environment, command, parsed.dialect),
|
||||
);
|
||||
if (!policy.ok || policy.value !== undefined)
|
||||
return { status: "failed", reason: "readback" };
|
||||
} catch (cause: unknown) {
|
||||
// Malformed existing configuration fails the readback gate.
|
||||
void cause;
|
||||
@@ -158,9 +176,12 @@ export const clientConfigurationAligned = async (
|
||||
): Promise<boolean> => {
|
||||
if (client.configPathError !== undefined) return false;
|
||||
try {
|
||||
const files = await readClientConfigurationFiles(client);
|
||||
if (!files.ok) return false;
|
||||
const original = await readFile(client.configPath, "utf8");
|
||||
const parsed = parseClientConfiguration(original, client.format);
|
||||
return registrationCurrent(
|
||||
const policy = await readClientPolicyBlock(
|
||||
client,
|
||||
parsed,
|
||||
clientConfigurationDesired(
|
||||
client,
|
||||
@@ -169,6 +190,19 @@ export const clientConfigurationAligned = async (
|
||||
parsed.dialect,
|
||||
),
|
||||
);
|
||||
return (
|
||||
policy.ok &&
|
||||
policy.value === undefined &&
|
||||
registrationCurrent(
|
||||
parsed,
|
||||
clientConfigurationDesired(
|
||||
client,
|
||||
providerEnvironment,
|
||||
command,
|
||||
parsed.dialect,
|
||||
),
|
||||
)
|
||||
);
|
||||
} catch (cause: unknown) {
|
||||
// Unreadable configuration is treated as not aligned so setup repairs it.
|
||||
void cause;
|
||||
@@ -185,6 +219,22 @@ export const inspectClientConfiguration = async (
|
||||
if (client.configPathError !== undefined)
|
||||
return { status: "invalid", remediation: client.configPathError };
|
||||
if (client.format === "unsupported") return { status: "already_current" };
|
||||
const files = await readClientConfigurationFiles(client);
|
||||
if (!files.ok) return { status: "invalid", remediation: files.error.detail };
|
||||
const policy = await readClientPolicyBlock(
|
||||
client,
|
||||
undefined,
|
||||
clientConfigurationDesired(
|
||||
client,
|
||||
providerEnvironment,
|
||||
command,
|
||||
client.format,
|
||||
),
|
||||
);
|
||||
if (!policy.ok)
|
||||
return { status: "invalid", remediation: policy.error.detail };
|
||||
if (policy.value !== undefined)
|
||||
return { status: "invalid", remediation: policy.value };
|
||||
const transactionPath = await resolveClientConfigTransactionPath(
|
||||
client.configPath,
|
||||
);
|
||||
@@ -207,6 +257,20 @@ export const inspectClientConfiguration = async (
|
||||
}
|
||||
try {
|
||||
const parsed = parseClientConfiguration(original, client.format);
|
||||
const policy = await readClientPolicyBlock(
|
||||
client,
|
||||
parsed,
|
||||
clientConfigurationDesired(
|
||||
client,
|
||||
providerEnvironment,
|
||||
command,
|
||||
parsed.dialect,
|
||||
),
|
||||
);
|
||||
if (!policy.ok)
|
||||
return { status: "invalid", remediation: policy.error.detail };
|
||||
if (policy.value !== undefined)
|
||||
return { status: "invalid", remediation: policy.value };
|
||||
const desired = clientConfigurationDesired(
|
||||
client,
|
||||
providerEnvironment,
|
||||
|
||||
@@ -14,7 +14,11 @@ import {
|
||||
systemLinuxHopperInstallHost,
|
||||
} from "./LinuxHopper.js";
|
||||
import { installMacHopper, systemMacHopperInstallHost } from "./MacHopper.js";
|
||||
import { supportedClients, type SetupClient } from "./SupportedClients.js";
|
||||
import {
|
||||
clientEvidencePaths,
|
||||
supportedClients,
|
||||
type SetupClient,
|
||||
} from "./SupportedClients.js";
|
||||
import {
|
||||
skillDestinations,
|
||||
canonicalSkillNeedsInstall,
|
||||
@@ -168,17 +172,19 @@ export const detectClients = async (
|
||||
const detected: SetupClient[] = [];
|
||||
for (const candidate of supportedClients(home, platform, environment)) {
|
||||
if (candidate.configPathError !== undefined) continue;
|
||||
const [hasConfig, hasMarker] = await Promise.all([
|
||||
exists(candidate.configPath),
|
||||
candidate.markerPath === undefined ? false : exists(candidate.markerPath),
|
||||
]);
|
||||
if (hasConfig || hasMarker) detected.push(candidate);
|
||||
if (await clientEvidencePresent(candidate)) detected.push(candidate);
|
||||
}
|
||||
return detected;
|
||||
};
|
||||
|
||||
const major = (version: string): number =>
|
||||
Number.parseInt(version.split(".")[0] ?? "0", 10);
|
||||
const clientEvidencePresent = async (client: SetupClient): Promise<boolean> => {
|
||||
for (const path of clientEvidencePaths(client))
|
||||
if (await exists(path)) return true;
|
||||
return false;
|
||||
};
|
||||
|
||||
const exists = async (path: string): Promise<boolean> => {
|
||||
try {
|
||||
await access(path);
|
||||
|
||||
@@ -1,12 +1,27 @@
|
||||
import { isAbsolute, join, resolve, win32 } from "node:path";
|
||||
import { dirname, isAbsolute, join, resolve, win32 } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { lstatSync } from "node:fs";
|
||||
import {
|
||||
HermesClientPathError,
|
||||
resolveHermesClientHome,
|
||||
} from "./HermesClientHome.js";
|
||||
|
||||
/** One supported client configuration location. */
|
||||
export interface SetupClient {
|
||||
readonly name: string;
|
||||
readonly displayName?: string;
|
||||
readonly configPath: string;
|
||||
/** User configuration files in increasing precedence, including the write target. */
|
||||
readonly configPaths?: readonly string[];
|
||||
/** Read-only Gemini scopes and trust inputs; never setup write targets. */
|
||||
readonly geminiSettings?: {
|
||||
readonly systemPath: string;
|
||||
readonly defaultsPath: string;
|
||||
readonly workspaceDirectory: string;
|
||||
readonly trustedFoldersPath: string;
|
||||
readonly trustOverride?: boolean;
|
||||
readonly platform: NodeJS.Platform;
|
||||
};
|
||||
/** Unresolved path evidence only; consumers must not perform I/O on it. */
|
||||
readonly configPathError?: string;
|
||||
readonly markerPath?: string;
|
||||
@@ -39,6 +54,10 @@ interface ClientDefinition {
|
||||
readonly markerPath: ClientPath;
|
||||
readonly format: NonNullable<SetupClient["format"]>;
|
||||
readonly skillPath?: ClientPath;
|
||||
readonly configPaths?: (context: ClientPathContext) => readonly string[];
|
||||
readonly geminiSettings?: (
|
||||
context: ClientPathContext,
|
||||
) => NonNullable<SetupClient["geminiSettings"]>;
|
||||
}
|
||||
|
||||
const vscodeUserDirectory = ({
|
||||
@@ -76,7 +95,7 @@ const claudeCodeMarkerDirectory = ({ home, env }: ClientPathContext): string =>
|
||||
env.CLAUDE_CONFIG_DIR ?? join(home, ".claude");
|
||||
|
||||
const codexDirectory = ({ home, env }: ClientPathContext): string =>
|
||||
env.CODEX_HOME ?? join(home, ".codex");
|
||||
env.CODEX_HOME || join(home, ".codex");
|
||||
|
||||
/** Match Qwen Code's home override, including tilde and cwd-relative paths. */
|
||||
const qwenCodeDirectory = ({ home, env }: ClientPathContext): string => {
|
||||
@@ -98,51 +117,10 @@ const qwenCodeDirectory = ({ home, env }: ClientPathContext): string => {
|
||||
};
|
||||
|
||||
const grokDirectory = ({ home, env }: ClientPathContext): string =>
|
||||
env.GROK_HOME ?? join(home, ".grok");
|
||||
env.GROK_HOME || join(home, ".grok");
|
||||
|
||||
/**
|
||||
* Hermes Agent home. `HERMES_HOME` wins; otherwise the platform default mirrors
|
||||
* Hermes's own resolver (`%LOCALAPPDATA%\\hermes` on Windows, `~/.hermes`
|
||||
* elsewhere), including the optional `HERMES_DATA_DIR_SUFFIX` that is appended
|
||||
* to the literal `hermes`/`.hermes` directory name.
|
||||
*/
|
||||
const hermesDirectory = ({
|
||||
home,
|
||||
platform,
|
||||
env,
|
||||
}: ClientPathContext): string => {
|
||||
const override = env.HERMES_HOME?.trim();
|
||||
if (override) {
|
||||
const expandedVariables = override.replace(
|
||||
/\$(\w+|\{[^}]*\})/gu,
|
||||
(match, variable: string) => {
|
||||
const key = variable.startsWith("{") ? variable.slice(1, -1) : variable;
|
||||
return env[key] ?? match;
|
||||
},
|
||||
);
|
||||
const expanded =
|
||||
platform === "win32"
|
||||
? expandedVariables.replace(
|
||||
/%([^%]+)%/gu,
|
||||
(match, variable: string) => env[variable] ?? match,
|
||||
)
|
||||
: expandedVariables;
|
||||
if (expanded === "~") return home;
|
||||
if (
|
||||
expanded.startsWith("~/") ||
|
||||
(platform === "win32" && expanded.startsWith("~\\"))
|
||||
)
|
||||
return join(home, expanded.slice(2));
|
||||
return resolve(expanded);
|
||||
}
|
||||
const suffix = env.HERMES_DATA_DIR_SUFFIX ?? "";
|
||||
return platform === "win32"
|
||||
? join(
|
||||
env.LOCALAPPDATA?.trim() || join(home, "AppData", "Local"),
|
||||
`hermes${suffix}`,
|
||||
)
|
||||
: join(home, `.hermes${suffix}`);
|
||||
};
|
||||
const hermesDirectory = ({ home, platform, env }: ClientPathContext): string =>
|
||||
resolveHermesClientHome(home, platform, env);
|
||||
|
||||
/** Grok Bot uses an absolute SAND_DATA_ROOT; anything else stays ~/.grokbot. */
|
||||
const grokBotDirectory = ({ home, env }: ClientPathContext): string => {
|
||||
@@ -188,17 +166,14 @@ const ompProfile = ({ env }: ClientPathContext): string | undefined => {
|
||||
/**
|
||||
* The OMP agent directory whose `mcp.json` holds user-scope MCP servers. OMP
|
||||
* joins PI_CONFIG_DIR to the home directory, and a named profile ignores
|
||||
* PI_CODING_AGENT_DIR. OMP resolves a relative agent directory against each
|
||||
* process's working directory, so only an absolute override names one file.
|
||||
* PI_CODING_AGENT_DIR. Relative overrides use the setup process working directory.
|
||||
*/
|
||||
const ompAgentDirectory = (context: ClientPathContext): string => {
|
||||
const root = join(context.home, context.env.PI_CONFIG_DIR || ".omp");
|
||||
const profile = ompProfile(context);
|
||||
if (profile !== undefined) return join(root, "profiles", profile, "agent");
|
||||
const override = context.env.PI_CODING_AGENT_DIR;
|
||||
return override !== undefined && override !== "" && isAbsolute(override)
|
||||
? override
|
||||
: join(root, "agent");
|
||||
return override ? resolve(override) : join(root, "agent");
|
||||
};
|
||||
|
||||
/**
|
||||
@@ -242,21 +217,66 @@ const copilotDirectory = ({ home, env }: ClientPathContext): string =>
|
||||
const commandCodeDirectory = ({ home }: ClientPathContext): string =>
|
||||
join(home, ".commandcode");
|
||||
|
||||
const opencodeDirectory = ({ home, env }: ClientPathContext): string =>
|
||||
join(env.XDG_CONFIG_HOME ?? join(home, ".config"), "opencode");
|
||||
/**
|
||||
* OpenCode appends `OPENCODE_CONFIG_DIR` only when the value is non-empty.
|
||||
* An empty variable is the same as unset and leaves the XDG directory in place.
|
||||
*/
|
||||
const openCodeOverrideDirectory = (
|
||||
env: ClientPathContext["env"],
|
||||
): string | undefined => {
|
||||
const directory = env.OPENCODE_CONFIG_DIR;
|
||||
return directory === undefined || directory === "" ? undefined : directory;
|
||||
};
|
||||
|
||||
const opencodeDirectory = (context: ClientPathContext): string =>
|
||||
openCodeOverrideDirectory(context.env) ??
|
||||
join(
|
||||
context.env.XDG_CONFIG_HOME ?? join(context.home, ".config"),
|
||||
"opencode",
|
||||
);
|
||||
|
||||
/** OpenCode loads global JSON before JSONC, then explicit file and directory overrides. */
|
||||
const openCodeConfigurationPaths = (
|
||||
context: ClientPathContext,
|
||||
): readonly string[] => {
|
||||
const directory = join(
|
||||
context.env.XDG_CONFIG_HOME ?? join(context.home, ".config"),
|
||||
"opencode",
|
||||
);
|
||||
const overrideDirectory = openCodeOverrideDirectory(context.env);
|
||||
const paths = [
|
||||
join(directory, "config.json"),
|
||||
join(directory, "opencode.json"),
|
||||
join(directory, "opencode.jsonc"),
|
||||
...(context.env.OPENCODE_CONFIG ? [context.env.OPENCODE_CONFIG] : []),
|
||||
...(overrideDirectory !== undefined
|
||||
? [
|
||||
join(overrideDirectory, "opencode.json"),
|
||||
join(overrideDirectory, "opencode.jsonc"),
|
||||
]
|
||||
: []),
|
||||
];
|
||||
const resolved = paths.map((path) => resolve(path));
|
||||
return resolved.filter((path, index) => resolved.lastIndexOf(path) === index);
|
||||
};
|
||||
|
||||
const existingOpenCodeConfigPath = (context: ClientPathContext): string => {
|
||||
if (context.env.OPENCODE_CONFIG !== undefined)
|
||||
return context.env.OPENCODE_CONFIG;
|
||||
const directory = opencodeDirectory(context);
|
||||
const candidates = [
|
||||
"opencode.json",
|
||||
"opencode.jsonc",
|
||||
".opencode.json",
|
||||
".opencode.jsonc",
|
||||
].map((filename) => join(directory, filename));
|
||||
// The selected directory is loaded after an explicit file by OpenCode V1.
|
||||
const overrideDirectory = openCodeOverrideDirectory(context.env);
|
||||
const candidates =
|
||||
overrideDirectory !== undefined
|
||||
? [
|
||||
join(overrideDirectory, "opencode.json"),
|
||||
join(overrideDirectory, "opencode.jsonc"),
|
||||
].map((path) => resolve(path))
|
||||
: context.env.OPENCODE_CONFIG
|
||||
? [resolve(context.env.OPENCODE_CONFIG)]
|
||||
: [
|
||||
join(opencodeDirectory(context), "opencode.json"),
|
||||
join(opencodeDirectory(context), "opencode.jsonc"),
|
||||
].map((path) => resolve(path));
|
||||
return (
|
||||
candidates.find((path) => {
|
||||
[...candidates].reverse().find((path) => {
|
||||
try {
|
||||
lstatSync(path);
|
||||
return true;
|
||||
@@ -268,15 +288,52 @@ const existingOpenCodeConfigPath = (context: ClientPathContext): string => {
|
||||
);
|
||||
}
|
||||
}) ??
|
||||
candidates[0] ??
|
||||
join(directory, "opencode.json")
|
||||
resolve(
|
||||
overrideDirectory !== undefined
|
||||
? join(overrideDirectory, "opencode.json")
|
||||
: context.env.OPENCODE_CONFIG ||
|
||||
join(opencodeDirectory(context), "opencode.json"),
|
||||
)
|
||||
);
|
||||
};
|
||||
|
||||
const geminiSettings = ({
|
||||
home,
|
||||
platform,
|
||||
env,
|
||||
}: ClientPathContext): NonNullable<SetupClient["geminiSettings"]> => {
|
||||
const system =
|
||||
env.GEMINI_CLI_SYSTEM_SETTINGS_PATH ||
|
||||
(platform === "darwin"
|
||||
? "/Library/Application Support/GeminiCli/settings.json"
|
||||
: platform === "win32"
|
||||
? "C:\\ProgramData\\gemini-cli\\settings.json"
|
||||
: "/etc/gemini-cli/settings.json");
|
||||
const trustOverride =
|
||||
env.GEMINI_RESTRICTED_MODE === "true" ||
|
||||
env.GEMINI_CLI_TRUST_WORKSPACE === "false"
|
||||
? false
|
||||
: env.GEMINI_CLI_TRUST_WORKSPACE === "true"
|
||||
? true
|
||||
: undefined;
|
||||
return {
|
||||
systemPath: system,
|
||||
defaultsPath:
|
||||
env.GEMINI_CLI_SYSTEM_DEFAULTS_PATH ||
|
||||
join(dirname(system), "system-defaults.json"),
|
||||
workspaceDirectory: process.cwd(),
|
||||
trustedFoldersPath:
|
||||
env.GEMINI_CLI_TRUSTED_FOLDERS_PATH ||
|
||||
join(env.GEMINI_CLI_HOME || home, ".gemini", "trustedFolders.json"),
|
||||
...(trustOverride === undefined ? {} : { trustOverride }),
|
||||
platform,
|
||||
};
|
||||
};
|
||||
|
||||
const devinDirectory = ({ home, platform, env }: ClientPathContext): string =>
|
||||
platform === "win32"
|
||||
? join(env.APPDATA ?? join(home, "AppData", "Roaming"), "devin")
|
||||
: join(home, ".config", "devin");
|
||||
: join(env.XDG_CONFIG_HOME || join(home, ".config"), "devin");
|
||||
|
||||
/** Stable product metadata used to derive setup discovery and documentation. */
|
||||
export const SUPPORTED_CLIENT_DEFINITIONS = [
|
||||
@@ -316,8 +373,13 @@ export const SUPPORTED_CLIENT_DEFINITIONS = [
|
||||
{
|
||||
name: "gemini_cli",
|
||||
displayName: "Gemini CLI",
|
||||
configPath: [".gemini", "settings.json"],
|
||||
markerPath: [".gemini"],
|
||||
geminiSettings,
|
||||
configPath: ({ home, env }: ClientPathContext) =>
|
||||
join(env.GEMINI_CLI_HOME || home, ".gemini", "settings.json"),
|
||||
markerPath: ({ home, env }: ClientPathContext) =>
|
||||
join(env.GEMINI_CLI_HOME || home, ".gemini"),
|
||||
skillPath: ({ home, env }: ClientPathContext) =>
|
||||
join(env.GEMINI_CLI_HOME || home, ".agents", "skills"),
|
||||
format: "json",
|
||||
},
|
||||
{
|
||||
@@ -339,12 +401,14 @@ export const SUPPORTED_CLIENT_DEFINITIONS = [
|
||||
name: "opencode",
|
||||
displayName: "OpenCode",
|
||||
configPath: existingOpenCodeConfigPath,
|
||||
configPaths: openCodeConfigurationPaths,
|
||||
markerPath: opencodeDirectory,
|
||||
format: "opencode",
|
||||
},
|
||||
{
|
||||
name: "antigravity",
|
||||
displayName: "Antigravity",
|
||||
skillPath: [".gemini", "config", "skills"],
|
||||
configPath: [".gemini", "config", "mcp_config.json"],
|
||||
markerPath: [".gemini", "config"],
|
||||
format: "json",
|
||||
@@ -427,6 +491,23 @@ export const SUPPORTED_CLIENT_DEFINITIONS = [
|
||||
},
|
||||
] as const satisfies readonly ClientDefinition[];
|
||||
|
||||
/**
|
||||
* Paths whose presence means doctor and setup should inspect the client.
|
||||
* Layered sources count: OpenCode still loads its XDG files when a custom
|
||||
* directory override does not exist.
|
||||
*/
|
||||
export const clientEvidencePaths = (
|
||||
client: Pick<SetupClient, "configPath" | "configPaths" | "markerPath">,
|
||||
): readonly string[] => [
|
||||
...new Set(
|
||||
[
|
||||
client.configPath,
|
||||
...(client.configPaths ?? []),
|
||||
client.markerPath,
|
||||
].filter((path): path is string => path !== undefined && path.length > 0),
|
||||
),
|
||||
];
|
||||
|
||||
const resolvePath = (path: ClientPath, context: ClientPathContext): string =>
|
||||
typeof path === "function" ? path(context) : join(context.home, ...path);
|
||||
|
||||
@@ -437,16 +518,36 @@ export const supportedClients = (
|
||||
env: ClientPathContext["env"] = process.env,
|
||||
): readonly SetupClient[] => {
|
||||
const context = { home, platform, env };
|
||||
return SUPPORTED_CLIENT_DEFINITIONS.map((definition) => {
|
||||
const definitions: readonly ClientDefinition[] = SUPPORTED_CLIENT_DEFINITIONS;
|
||||
return definitions.map((definition) => {
|
||||
try {
|
||||
const configPath = resolvePath(definition.configPath, context);
|
||||
return {
|
||||
name: definition.name,
|
||||
displayName: definition.displayName,
|
||||
configPath: resolvePath(definition.configPath, context),
|
||||
configPath,
|
||||
...(definition.geminiSettings === undefined
|
||||
? {}
|
||||
: { geminiSettings: definition.geminiSettings(context) }),
|
||||
...(definition.configPaths === undefined
|
||||
? {}
|
||||
: {
|
||||
configPaths: [
|
||||
...new Set([...definition.configPaths(context), configPath]),
|
||||
],
|
||||
}),
|
||||
markerPath: resolvePath(definition.markerPath, context),
|
||||
format: definition.format,
|
||||
};
|
||||
} catch (cause: unknown) {
|
||||
if (cause instanceof HermesClientPathError)
|
||||
return {
|
||||
name: definition.name,
|
||||
displayName: definition.displayName,
|
||||
format: definition.format,
|
||||
configPath: cause.path,
|
||||
configPathError: cause.message,
|
||||
};
|
||||
if (definition.name !== "pi") throw cause;
|
||||
return {
|
||||
name: definition.name,
|
||||
@@ -481,14 +582,21 @@ export const clientSkillDirectories = (
|
||||
const destinations = new Map<string, { client: string; directory: string }>();
|
||||
for (const client of selected) {
|
||||
const definition = definitions.find(({ name }) => name === client);
|
||||
const directory =
|
||||
definition?.skillPath === undefined
|
||||
? join(home, ".agents", "skills")
|
||||
: resolvePath(definition.skillPath, {
|
||||
home,
|
||||
platform,
|
||||
env: environment,
|
||||
});
|
||||
let directory: string;
|
||||
try {
|
||||
directory =
|
||||
definition?.skillPath === undefined
|
||||
? join(home, ".agents", "skills")
|
||||
: resolvePath(definition.skillPath, {
|
||||
home,
|
||||
platform,
|
||||
env: environment,
|
||||
});
|
||||
} catch (cause: unknown) {
|
||||
// Registration diagnostics own an unresolved profile; never fall back to another skill root.
|
||||
if (cause instanceof HermesClientPathError) continue;
|
||||
throw cause;
|
||||
}
|
||||
if (!destinations.has(directory))
|
||||
destinations.set(directory, {
|
||||
client:
|
||||
|
||||
@@ -124,11 +124,12 @@ export const runUninstall = async (
|
||||
status: "retained",
|
||||
detail: "Hopper is not owned by REA uninstall.",
|
||||
});
|
||||
const reported = collapseUninstallItems(items);
|
||||
return {
|
||||
status: items.some(({ status }) => status === "failed")
|
||||
status: reported.some(({ status }) => status === "failed")
|
||||
? "failed"
|
||||
: "complete",
|
||||
items,
|
||||
items: reported,
|
||||
};
|
||||
};
|
||||
|
||||
@@ -171,7 +172,16 @@ export const systemUninstallHost = (
|
||||
selectedHome ?? homeDirectoryFromEnvironment(environment, platform);
|
||||
return {
|
||||
clients: () =>
|
||||
Promise.resolve(supportedClients(home, platform, environment)),
|
||||
Promise.resolve(
|
||||
supportedClients(home, platform, environment).flatMap((client) =>
|
||||
client.configPaths === undefined
|
||||
? [client]
|
||||
: client.configPaths.map((configPath) => ({
|
||||
...client,
|
||||
configPath,
|
||||
})),
|
||||
),
|
||||
),
|
||||
inspectClient: async (client) => {
|
||||
const read = await readClientConfiguration(client, fileSystem);
|
||||
return read.kind === "item" && read.item.status === "failed"
|
||||
@@ -452,6 +462,68 @@ const item = (
|
||||
status: UninstallItem["status"],
|
||||
detail: string,
|
||||
): UninstallItem => ({ name, status, detail });
|
||||
|
||||
/**
|
||||
* Layered clients are removed file by file, but callers look up one status by
|
||||
* client name. A missing earlier file must not hide a later removal.
|
||||
*/
|
||||
const collapseUninstallItems = (
|
||||
items: readonly UninstallItem[],
|
||||
): UninstallItem[] => {
|
||||
const collapsed: UninstallItem[] = [];
|
||||
for (const next of items) {
|
||||
const index = collapsed.findIndex(({ name }) => name === next.name);
|
||||
const current = collapsed[index];
|
||||
if (current === undefined) {
|
||||
collapsed.push(next);
|
||||
continue;
|
||||
}
|
||||
collapsed[index] = mergeUninstallItems(current, next);
|
||||
}
|
||||
return collapsed;
|
||||
};
|
||||
|
||||
const mergeUninstallItems = (
|
||||
current: UninstallItem,
|
||||
next: UninstallItem,
|
||||
): UninstallItem => {
|
||||
const status = preferredUninstallStatus(current.status, next.status);
|
||||
const details = [current, next]
|
||||
.filter(
|
||||
(entry) =>
|
||||
entry.status === status ||
|
||||
(status === "removed" && entry.status === "retained"),
|
||||
)
|
||||
.map(({ detail }) => detail);
|
||||
return item(current.name, status, [...new Set(details)].join(" "));
|
||||
};
|
||||
|
||||
const preferredUninstallStatus = (
|
||||
current: UninstallItem["status"],
|
||||
next: UninstallItem["status"],
|
||||
): UninstallItem["status"] =>
|
||||
uninstallStatusPriority(next) > uninstallStatusPriority(current)
|
||||
? next
|
||||
: current;
|
||||
|
||||
const uninstallStatusPriority = (status: UninstallItem["status"]): number => {
|
||||
switch (status) {
|
||||
case "failed":
|
||||
return 3;
|
||||
case "removed":
|
||||
return 2;
|
||||
case "retained":
|
||||
return 1;
|
||||
case "skipped":
|
||||
return 0;
|
||||
default: {
|
||||
const exhaustive: never = status;
|
||||
throw new TypeError(
|
||||
`Unhandled uninstall item status: ${String(exhaustive)}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
};
|
||||
const registrationSchema = z
|
||||
.object({ command: z.string(), args: z.array(z.string()) })
|
||||
.passthrough();
|
||||
|
||||
@@ -42,6 +42,46 @@ describe("JavaScript analysis cancellation before publication", () => {
|
||||
expect(phases).not.toContain("validate_javascript_application_result");
|
||||
});
|
||||
|
||||
it("handles control turns and cancellation inside one large source file (#1462)", async () => {
|
||||
const root = await createTestTempDirectory("rea-js-cancel-in-file-");
|
||||
await writeFile(
|
||||
join(root, "index.js"),
|
||||
Array.from(
|
||||
{ length: 1200 },
|
||||
(_, index) =>
|
||||
`export function f${index}(){return {name:'f${index}',value:${index}};}`,
|
||||
).join("\n"),
|
||||
);
|
||||
const controller = new AbortController();
|
||||
const phases: string[] = [];
|
||||
let turns = 0;
|
||||
// Count event-loop turns once the file starts; parsing, analysis and
|
||||
// projection of this one file must leave room for them.
|
||||
const tick = (): void => {
|
||||
turns += 1;
|
||||
if (turns === 3) controller.abort();
|
||||
else if (!controller.signal.aborted) setImmediate(tick);
|
||||
};
|
||||
const result = await analyzeJavaScriptApplication(
|
||||
{ input_path: root, format: "directory" },
|
||||
{
|
||||
signal: controller.signal,
|
||||
progress: {
|
||||
async report(event) {
|
||||
phases.push(event.phase);
|
||||
if (event.phase === "parse_javascript_source") setImmediate(tick);
|
||||
},
|
||||
},
|
||||
},
|
||||
);
|
||||
if (result.ok) throw new Error("Cancelled analysis must not publish");
|
||||
expect(projectAnalysisError(result.error)).toMatchObject({
|
||||
code: "cancelled",
|
||||
});
|
||||
expect(phases).toContain("parse_javascript_source");
|
||||
expect(phases).not.toContain("build_javascript_application_graph");
|
||||
});
|
||||
|
||||
it.each([
|
||||
"parse_javascript_source",
|
||||
"build_javascript_application_graph",
|
||||
|
||||
@@ -4,7 +4,7 @@ import { setImmediate } from "node:timers/promises";
|
||||
import { resolveJavaScriptSourceMapReference } from "../../domain/javascript/javascriptSourceMapPaths.js";
|
||||
import { analyzeParsedJavaScriptStaticSource } from "../../domain/javascript/javascriptStaticAnalysis.js";
|
||||
import {
|
||||
analyzeParsedJavaScriptSemantics,
|
||||
analyzeParsedJavaScriptSemanticsSteps,
|
||||
classifyParsedJavaScriptOpenReceivers,
|
||||
} from "../../domain/javascript/javascriptSemanticAnalysis.js";
|
||||
import type { JavaScriptSemanticIr } from "../../domain/javascript/javascriptSemanticIr.js";
|
||||
@@ -33,6 +33,7 @@ import type {
|
||||
JavaScriptSourceMapOriginal,
|
||||
} from "./JavaScriptArtifactAnalysisTypes.js";
|
||||
import { analyzeJavaScriptJsonModule } from "./JavaScriptJsonModules.js";
|
||||
import { completeJavaScriptAnalysisSteps } from "./JavaScriptAnalysisControl.js";
|
||||
import { htmlArtifactReferences } from "../../domain/javascript/htmlArtifactReferences.js";
|
||||
|
||||
interface MutableArtifactAnalysis<
|
||||
@@ -56,34 +57,44 @@ interface ArtifactAnalysisContext<
|
||||
readonly projectSemantics: (
|
||||
file: JavaScriptArtifactFile,
|
||||
ir: JavaScriptSemanticIr,
|
||||
) => SemanticIr;
|
||||
) => Generator<void, SemanticIr>;
|
||||
}
|
||||
|
||||
/** Analyze every text source from the artifact without a result quota. */
|
||||
export const analyzeJavaScriptArtifactFiles = (
|
||||
fileSet: JavaScriptArtifactFileSet,
|
||||
): JavaScriptArtifactAnalysis =>
|
||||
analyzeArtifactFiles(fileSet, (_file, ir) => ir);
|
||||
analyzeArtifactFiles(fileSet, function* (_file, ir) {
|
||||
// The full IR needs no projection; one step keeps the shared protocol.
|
||||
yield;
|
||||
return ir;
|
||||
});
|
||||
|
||||
/** Project each file's full IR immediately, retaining only later module facts. */
|
||||
/**
|
||||
* Project each file's full IR immediately, retaining only later module facts.
|
||||
* Parsing, analysis and each projection pass run as separate steps, so control
|
||||
* messages and cancellation are handled within one large file as well.
|
||||
*/
|
||||
export const analyzeAndProjectJavaScriptArtifactFiles = async (
|
||||
fileSet: JavaScriptArtifactFileSet,
|
||||
projectSemantics: (
|
||||
file: JavaScriptArtifactFile,
|
||||
ir: JavaScriptSemanticIr,
|
||||
) => void,
|
||||
) => Iterator<void, void> | void,
|
||||
beforeFile?: (
|
||||
file: JavaScriptArtifactFile,
|
||||
completed: number,
|
||||
total: number,
|
||||
) => Promise<void>,
|
||||
signal?: AbortSignal,
|
||||
): Promise<JavaScriptModuleArtifactAnalysis> => {
|
||||
const state = emptyArtifactAnalysis<JavaScriptModuleSemanticIr>();
|
||||
const projectFileSemantics = (
|
||||
const projectFileSemantics = function* (
|
||||
file: JavaScriptArtifactFile,
|
||||
ir: JavaScriptSemanticIr,
|
||||
): JavaScriptModuleSemanticIr => {
|
||||
projectSemantics(file, ir);
|
||||
): Generator<void, JavaScriptModuleSemanticIr> {
|
||||
const projection = projectSemantics(file, ir);
|
||||
if (projection !== undefined) while (projection.next().done !== true) yield;
|
||||
const programScopes = ir.scopes.filter(({ kind }) => kind === "program");
|
||||
const programScopeId = programScopes[0]?.scopeId;
|
||||
const localNames = new Set(
|
||||
@@ -111,7 +122,10 @@ export const analyzeAndProjectJavaScriptArtifactFiles = async (
|
||||
// Let progress, cancellation and garbage collection run between files.
|
||||
await setImmediate();
|
||||
await beforeFile?.(file, index, fileSet.files.length);
|
||||
analyzeArtifactFile(file, context);
|
||||
await completeJavaScriptAnalysisSteps(
|
||||
analyzeArtifactFileSteps(file, context),
|
||||
signal,
|
||||
);
|
||||
}
|
||||
return finalizeArtifactAnalysis(state);
|
||||
};
|
||||
@@ -122,7 +136,10 @@ const analyzeArtifactFiles = <SemanticIr extends JavaScriptModuleSemanticIr>(
|
||||
): JavaScriptArtifactAnalysis<SemanticIr> => {
|
||||
const state = emptyArtifactAnalysis<SemanticIr>();
|
||||
const context = { state, projectSemantics };
|
||||
for (const file of fileSet.files) analyzeArtifactFile(file, context);
|
||||
for (const file of fileSet.files) {
|
||||
const steps = analyzeArtifactFileSteps(file, context);
|
||||
while (steps.next().done !== true);
|
||||
}
|
||||
return finalizeArtifactAnalysis(state);
|
||||
};
|
||||
|
||||
@@ -148,10 +165,12 @@ const finalizeArtifactAnalysis = <
|
||||
};
|
||||
};
|
||||
|
||||
const analyzeArtifactFile = <SemanticIr extends JavaScriptModuleSemanticIr>(
|
||||
function* analyzeArtifactFileSteps<
|
||||
SemanticIr extends JavaScriptModuleSemanticIr,
|
||||
>(
|
||||
file: JavaScriptArtifactFile,
|
||||
context: ArtifactAnalysisContext<SemanticIr>,
|
||||
): void => {
|
||||
): Generator<void, void> {
|
||||
const { state } = context;
|
||||
addStructuredObservations(file, state);
|
||||
if (file.kind === "html" && file.text.included)
|
||||
@@ -162,6 +181,7 @@ const analyzeArtifactFile = <SemanticIr extends JavaScriptModuleSemanticIr>(
|
||||
return;
|
||||
}
|
||||
const parsed = parseJavaScriptSource(file.text.value, file.path);
|
||||
yield;
|
||||
if (parsed === null) {
|
||||
const analysis = failedJavaScriptStaticAnalysis();
|
||||
state.files.push({ file, javascript: analysis, semantic: null });
|
||||
@@ -169,23 +189,26 @@ const analyzeArtifactFile = <SemanticIr extends JavaScriptModuleSemanticIr>(
|
||||
return;
|
||||
}
|
||||
const openReceiverFacts = classifyParsedJavaScriptOpenReceivers(parsed);
|
||||
yield;
|
||||
const analysis = analyzeParsedJavaScriptStaticSource(
|
||||
file.text.value,
|
||||
parsed,
|
||||
openReceiverFacts,
|
||||
);
|
||||
const staticFindings = findingCount(analysis);
|
||||
yield;
|
||||
const semantics =
|
||||
analysis.parse_status === "complete" || analysis.parse_status === "partial"
|
||||
? analyzeParsedJavaScriptSemantics(parsed)
|
||||
? yield* analyzeParsedJavaScriptSemanticsSteps(parsed)
|
||||
: null;
|
||||
const projected =
|
||||
semantics === null
|
||||
? null
|
||||
: yield* context.projectSemantics(file, semantics);
|
||||
state.files.push({
|
||||
file,
|
||||
javascript: analysis,
|
||||
semantic:
|
||||
semantics === null
|
||||
? null
|
||||
: { ir: context.projectSemantics(file, semantics) },
|
||||
semantic: projected === null ? null : { ir: projected },
|
||||
});
|
||||
state.visitedNodes += analysis.visited_ast_nodes;
|
||||
state.findings += staticFindings + (semantics?.moduleLinks.length ?? 0);
|
||||
@@ -194,7 +217,7 @@ const analyzeArtifactFile = <SemanticIr extends JavaScriptModuleSemanticIr>(
|
||||
0,
|
||||
);
|
||||
if (analysis.parse_status === "failed") state.parseFailures += 1;
|
||||
};
|
||||
}
|
||||
|
||||
const addSourceMap = (
|
||||
file: JavaScriptArtifactFile,
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import type { ArtifactInventorySnapshot } from "../../domain/artifactInventorySnapshot.js";
|
||||
import {
|
||||
createJavaScriptApplicationGraph,
|
||||
createImmutableJavaScriptApplicationGraphSteps,
|
||||
sealTransferredJavaScriptApplicationGraphSteps,
|
||||
type JavaScriptApplicationGraph,
|
||||
} from "../../domain/javascript/javascriptApplicationGraph.js";
|
||||
import type { JavaScriptModuleArtifactAnalysis } from "./JavaScriptArtifactAnalysisTypes.js";
|
||||
@@ -15,9 +15,9 @@ import {
|
||||
addJavaScriptHtmlRoles,
|
||||
addJavaScriptSourceMapOriginals,
|
||||
} from "./JavaScriptArtifactGraphDocuments.js";
|
||||
import { addJavaScriptStaticFindings } from "./JavaScriptArtifactGraphFindings.js";
|
||||
import { addJavaScriptStaticFindingsSteps } from "./JavaScriptArtifactGraphFindings.js";
|
||||
import {
|
||||
addJavaScriptModuleRelationships,
|
||||
addJavaScriptModuleRelationshipsSteps,
|
||||
type JavaScriptModuleRelationshipOmissions,
|
||||
addJavaScriptSourceModules,
|
||||
} from "./JavaScriptModuleRelationships.js";
|
||||
@@ -51,24 +51,42 @@ export const buildJavaScriptArtifactGraph = (
|
||||
analysis: JavaScriptModuleArtifactAnalysis,
|
||||
): JavaScriptApplicationGraph =>
|
||||
createJavaScriptApplicationGraph(
|
||||
buildJavaScriptArtifactGraphInput(snapshot, fileSet, analysis),
|
||||
completeSynchronously(
|
||||
buildJavaScriptArtifactGraphInputSteps(snapshot, fileSet, analysis),
|
||||
),
|
||||
);
|
||||
|
||||
/** Build a validated application graph with cooperative immutable ownership. */
|
||||
export const buildImmutableJavaScriptArtifactGraphSteps = (
|
||||
/**
|
||||
* Build a validated application graph with cooperative immutable ownership.
|
||||
* Constructing the graph input also runs in steps: projecting every module
|
||||
* relationship and finding at once held the event loop for seconds.
|
||||
*/
|
||||
export function* buildImmutableJavaScriptArtifactGraphSteps(
|
||||
snapshot: ArtifactInventorySnapshot,
|
||||
fileSet: JavaScriptArtifactFileSet,
|
||||
analysis: JavaScriptModuleArtifactAnalysis,
|
||||
): Generator<void, JavaScriptApplicationGraph> =>
|
||||
createImmutableJavaScriptApplicationGraphSteps(
|
||||
buildJavaScriptArtifactGraphInput(snapshot, fileSet, analysis),
|
||||
): Generator<void, JavaScriptApplicationGraph> {
|
||||
const input = yield* buildJavaScriptArtifactGraphInputSteps(
|
||||
snapshot,
|
||||
fileSet,
|
||||
analysis,
|
||||
);
|
||||
// The input was built here and is not retained, so it can be transferred.
|
||||
return yield* sealTransferredJavaScriptApplicationGraphSteps(input);
|
||||
}
|
||||
|
||||
const buildJavaScriptArtifactGraphInput = (
|
||||
const completeSynchronously = <Value>(steps: Iterator<void, Value>): Value => {
|
||||
for (;;) {
|
||||
const step = steps.next();
|
||||
if (step.done === true) return step.value;
|
||||
}
|
||||
};
|
||||
|
||||
function* buildJavaScriptArtifactGraphInputSteps(
|
||||
snapshot: ArtifactInventorySnapshot,
|
||||
fileSet: JavaScriptArtifactFileSet,
|
||||
analysis: JavaScriptModuleArtifactAnalysis,
|
||||
): unknown => {
|
||||
): Generator<void, unknown> {
|
||||
const accumulator = new JavaScriptArtifactGraphAccumulator();
|
||||
const root = createJavaScriptArtifactRootNode(accumulator, snapshot);
|
||||
const context: JavaScriptArtifactGraphContext = {
|
||||
@@ -86,15 +104,23 @@ const buildJavaScriptArtifactGraphInput = (
|
||||
containerNodes: new Map([[snapshot.manifest.root_sha256, root]]),
|
||||
};
|
||||
addJavaScriptArtifactContainers(context);
|
||||
yield;
|
||||
addJavaScriptArtifactFiles(context);
|
||||
yield;
|
||||
const packageRoots = addJavaScriptPackageNodes(context);
|
||||
yield;
|
||||
addJavaScriptSourceModules(context);
|
||||
yield;
|
||||
const bundlerLimitations = addJavaScriptBundlerNodes(context);
|
||||
const relationshipOmissions = addJavaScriptModuleRelationships(context);
|
||||
const findingLimitations = addJavaScriptStaticFindings(context);
|
||||
const relationshipOmissions =
|
||||
yield* addJavaScriptModuleRelationshipsSteps(context);
|
||||
const findingLimitations = yield* addJavaScriptStaticFindingsSteps(context);
|
||||
addElectronBoundaries(context);
|
||||
yield;
|
||||
addJavaScriptHtmlRoles(context);
|
||||
yield;
|
||||
addJavaScriptSourceMapOriginals(context);
|
||||
yield;
|
||||
const coverage = graphCoverage(context);
|
||||
return {
|
||||
schema: "JavaScriptApplicationGraph",
|
||||
@@ -111,7 +137,7 @@ const buildJavaScriptArtifactGraphInput = (
|
||||
...graphLimitations(context, coverage.status, relationshipOmissions),
|
||||
],
|
||||
};
|
||||
};
|
||||
}
|
||||
|
||||
const graphCoverage = (context: JavaScriptArtifactGraphContext) => {
|
||||
const resourceLimits = semanticResourceLimits(context);
|
||||
|
||||
@@ -44,11 +44,12 @@ interface ResolvedReference {
|
||||
}
|
||||
|
||||
/** Project imports, workers, roles, endpoints, storage, and source-map links. */
|
||||
export const addJavaScriptStaticFindings = (
|
||||
export function* addJavaScriptStaticFindingsSteps(
|
||||
context: JavaScriptArtifactGraphContext,
|
||||
): readonly string[] => {
|
||||
): Generator<void, readonly string[]> {
|
||||
const omissions = { selfReferences: 0 };
|
||||
for (const analyzed of context.analysis.files) {
|
||||
yield;
|
||||
const { file, javascript } = analyzed;
|
||||
const asset = context.assetNodes.get(file.path);
|
||||
if (javascript === null || asset === undefined) continue;
|
||||
@@ -69,7 +70,7 @@ export const addJavaScriptStaticFindings = (
|
||||
["static reference", "static references"],
|
||||
"the referencing module",
|
||||
);
|
||||
};
|
||||
}
|
||||
|
||||
const addReference = (
|
||||
context: JavaScriptArtifactGraphContext,
|
||||
|
||||
@@ -67,6 +67,36 @@ describe("artifact-local graph identity", () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe("logical path identity", () => {
|
||||
it("resolves an NFD relative import to the NFC inventory path and rejects a root escape", () => {
|
||||
const nfc = "caf\u00e9".normalize("NFC");
|
||||
const nfd = nfc.normalize("NFD");
|
||||
const files = fileMap([
|
||||
file("dir/index.js", "root"),
|
||||
file(`dir/${nfc}.js`, "root"),
|
||||
]);
|
||||
expect(
|
||||
resolve({
|
||||
declaredPath: `./${nfd}.js`,
|
||||
sourcePath: "dir/index.js",
|
||||
context: "module-specifier",
|
||||
files,
|
||||
}),
|
||||
).toMatchObject({
|
||||
resolution_status: "resolved",
|
||||
resolved_path: `dir/${nfc}.js`,
|
||||
});
|
||||
expect(
|
||||
resolve({
|
||||
declaredPath: "../escape.js",
|
||||
sourcePath: "index.js",
|
||||
context: "module-specifier",
|
||||
files,
|
||||
}),
|
||||
).toMatchObject({ resolution_status: "rejected", resolved_path: null });
|
||||
});
|
||||
});
|
||||
|
||||
describe("contextual JavaScript artifact path resolution", () => {
|
||||
it.each([
|
||||
["main.js", "app/main.js"],
|
||||
|
||||
@@ -1,6 +1,10 @@
|
||||
import { isBuiltin } from "node:module";
|
||||
import { posix } from "node:path";
|
||||
|
||||
import {
|
||||
normalizeJoinedLogicalPath,
|
||||
logicalPathEscapesRoot,
|
||||
} from "../../domain/artifactIdentity.js";
|
||||
import type { JavaScriptArtifactFile } from "../../domain/javascript/javascriptArtifactFiles.js";
|
||||
import {
|
||||
admitsCanonicalPathSyntax,
|
||||
@@ -326,12 +330,8 @@ const confineCandidate = (
|
||||
input: ResolveArtifactPathInput,
|
||||
candidate: string,
|
||||
): string | ArtifactPathResolution => {
|
||||
const normalized = posix.normalize(candidate);
|
||||
if (
|
||||
normalized === ".." ||
|
||||
normalized.startsWith("../") ||
|
||||
normalized.startsWith("/")
|
||||
)
|
||||
const normalized = normalizeJoinedLogicalPath(candidate);
|
||||
if (logicalPathEscapesRoot(normalized))
|
||||
return unresolvedOutcome(input, "rejected", [
|
||||
"The resolved candidate escapes the canonical artifact root.",
|
||||
]);
|
||||
|
||||
@@ -90,7 +90,7 @@ export const reconstructJavaScriptArtifact = async (
|
||||
const semanticProjection = createJavaScriptSemanticGraphProjection();
|
||||
const analysis = await analyzeAndProjectJavaScriptArtifactFiles(
|
||||
files,
|
||||
semanticProjection.projectFile,
|
||||
semanticProjection.projectFileSteps,
|
||||
async (file, completed, total) => {
|
||||
abortIfNeeded(signal);
|
||||
await progress?.report({
|
||||
@@ -101,6 +101,7 @@ export const reconstructJavaScriptArtifact = async (
|
||||
});
|
||||
abortIfNeeded(signal);
|
||||
},
|
||||
signal,
|
||||
);
|
||||
await checkpointJavaScriptAnalysis(signal);
|
||||
abortIfNeeded(signal);
|
||||
|
||||
@@ -136,15 +136,17 @@ export interface JavaScriptModuleRelationshipOmissions {
|
||||
type RelationshipOmissionCounter = { selfImports: number };
|
||||
|
||||
/** Compose bounded CommonJS and ESM binding relationships across artifact files. */
|
||||
export const addJavaScriptModuleRelationships = (
|
||||
export function* addJavaScriptModuleRelationshipsSteps(
|
||||
context: JavaScriptArtifactGraphContext,
|
||||
): JavaScriptModuleRelationshipOmissions => {
|
||||
): Generator<void, JavaScriptModuleRelationshipOmissions> {
|
||||
const omissions = { selfImports: 0 };
|
||||
for (const analyzed of context.analysis.files) {
|
||||
const { file, semantic } = analyzed;
|
||||
const source = context.sourceModuleNodes.get(file.path);
|
||||
if (semantic === null || source === undefined) continue;
|
||||
for (const link of semantic.ir.moduleLinks) {
|
||||
// One module can export thousands of bindings, each with return shapes.
|
||||
yield;
|
||||
const input = { context, file, semantic, source, link };
|
||||
if (isExportLink(link)) addExportRelationship(input, omissions);
|
||||
else if (link.specifier !== null)
|
||||
@@ -155,7 +157,7 @@ export const addJavaScriptModuleRelationships = (
|
||||
}
|
||||
}
|
||||
return omissions;
|
||||
};
|
||||
}
|
||||
|
||||
const addExportRelationship = (
|
||||
input: RelationshipInput,
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import {
|
||||
createJavaScriptSemanticGraph,
|
||||
createImmutableJavaScriptSemanticGraphSteps,
|
||||
sealTransferredJavaScriptSemanticGraphSteps,
|
||||
type JavaScriptSemanticGraph,
|
||||
} from "../../domain/javascript/javascriptSemanticGraph.js";
|
||||
import type {
|
||||
@@ -130,6 +130,11 @@ export interface JavaScriptSemanticGraphProjection {
|
||||
file: JavaScriptArtifactFile,
|
||||
ir: JavaScriptSemanticIr,
|
||||
) => void;
|
||||
/** Project one file in steps between its projection passes. */
|
||||
readonly projectFileSteps: (
|
||||
file: JavaScriptArtifactFile,
|
||||
ir: JavaScriptSemanticIr,
|
||||
) => Generator<void, void>;
|
||||
readonly finish: (
|
||||
rootArtifactSha256: string,
|
||||
applicationGraph: BuilderInput["applicationGraph"],
|
||||
@@ -146,10 +151,10 @@ export const createJavaScriptSemanticGraphProjection =
|
||||
const state = emptyState({ nodes: [] });
|
||||
const fingerprints: JavaScriptSemanticFingerprint[] = [];
|
||||
let truncatedFiles = 0;
|
||||
const projectSource = (
|
||||
const projectSourceSteps = function* (
|
||||
file: JavaScriptArtifactFile,
|
||||
ir: JavaScriptSemanticIr,
|
||||
): void => {
|
||||
): Generator<void, void> {
|
||||
if (state.nodes.size >= SEMANTIC_GRAPH_NODE_CEILING) {
|
||||
truncatedFiles += 1;
|
||||
return;
|
||||
@@ -161,7 +166,7 @@ export const createJavaScriptSemanticGraphProjection =
|
||||
remainingTreeBudget,
|
||||
);
|
||||
state.fileNodesDropped = false;
|
||||
fingerprints.push(...projectFile(file, ir, state));
|
||||
fingerprints.push(...(yield* projectFileSteps(file, ir, state)));
|
||||
// Nodes were dropped only if the budget actually blocked creation. A file
|
||||
// that exactly fills its share ends with a zero budget without dropping
|
||||
// anything, so the remaining budget alone cannot decide truncation.
|
||||
@@ -198,7 +203,8 @@ export const createJavaScriptSemanticGraphProjection =
|
||||
evidence_contexts: state.evidenceContexts.contexts,
|
||||
nodes: [...state.nodes.values()],
|
||||
relations,
|
||||
fingerprints,
|
||||
// The input is transferred to the factory; `fingerprints` is cleared below.
|
||||
fingerprints: [...fingerprints],
|
||||
unknowns,
|
||||
coverage: {
|
||||
status: truncatedFiles > 0 ? "partial" : "unknown",
|
||||
@@ -246,7 +252,11 @@ export const createJavaScriptSemanticGraphProjection =
|
||||
return graph;
|
||||
};
|
||||
return {
|
||||
projectFile: projectSource,
|
||||
projectFile: (file, ir) => {
|
||||
const steps = projectSourceSteps(file, ir);
|
||||
while (steps.next().done !== true);
|
||||
},
|
||||
projectFileSteps: projectSourceSteps,
|
||||
finish: (rootArtifactSha256, applicationGraph) =>
|
||||
finishWith(
|
||||
createJavaScriptSemanticGraph,
|
||||
@@ -255,18 +265,19 @@ export const createJavaScriptSemanticGraphProjection =
|
||||
),
|
||||
finishImmutableSteps: (rootArtifactSha256, applicationGraph) =>
|
||||
finishWith(
|
||||
createImmutableJavaScriptSemanticGraphSteps,
|
||||
sealTransferredJavaScriptSemanticGraphSteps,
|
||||
rootArtifactSha256,
|
||||
applicationGraph,
|
||||
),
|
||||
};
|
||||
};
|
||||
|
||||
const projectFile = (
|
||||
function* projectFileSteps(
|
||||
file: JavaScriptArtifactFile,
|
||||
ir: JavaScriptSemanticIr,
|
||||
state: BuilderState,
|
||||
): JavaScriptSemanticFingerprint[] => {
|
||||
): Generator<void, JavaScriptSemanticFingerprint[]> {
|
||||
yield;
|
||||
const moduleNode = retainNode(state, file, {
|
||||
kind: "module",
|
||||
roleKey: "module",
|
||||
@@ -276,38 +287,50 @@ const projectFile = (
|
||||
});
|
||||
if (moduleNode === null) return [];
|
||||
state.roots.add(moduleNode.node_id);
|
||||
const callableNodes = new Map(
|
||||
ir.callables.flatMap((callable) => {
|
||||
const node = retainNode(state, file, {
|
||||
kind: "function",
|
||||
roleKey: `callable:${callable.callableId}`,
|
||||
location: callable.location,
|
||||
label: callable.name,
|
||||
functionNodeId: null,
|
||||
// The label is display text; keep the exact name, which may be "".
|
||||
properties: { name: callable.name },
|
||||
});
|
||||
return node === null ? [] : [[callable.callableId, node] as const];
|
||||
}),
|
||||
);
|
||||
// Node creation keeps source order, which the per-file node budget relies
|
||||
// on; yields between nodes do not change it.
|
||||
const callableNodes = new Map<string, JavaScriptSemanticGraphNode>();
|
||||
for (const [index, callable] of ir.callables.entries()) {
|
||||
if (index % 64 === 0) yield;
|
||||
const node = retainNode(state, file, {
|
||||
kind: "function",
|
||||
roleKey: `callable:${callable.callableId}`,
|
||||
location: callable.location,
|
||||
label: callable.name,
|
||||
functionNodeId: null,
|
||||
// The label is display text; keep the exact name, which may be "".
|
||||
properties: { name: callable.name },
|
||||
});
|
||||
if (node !== null) callableNodes.set(callable.callableId, node);
|
||||
}
|
||||
const callableOwnerAt = createSemanticCallableOwnerLookup(ir, callableNodes);
|
||||
const bindingNodes = new Map(
|
||||
ir.bindings.flatMap((binding) => {
|
||||
const location = binding.definitions[0]?.location ?? null;
|
||||
const kind = binding.kind === "parameter" ? "parameter" : "binding";
|
||||
const owner = callableOwnerAt(location);
|
||||
const node = retainNode(state, file, {
|
||||
kind,
|
||||
roleKey: `binding:${binding.bindingId}`,
|
||||
location,
|
||||
label: binding.name,
|
||||
functionNodeId: owner?.node_id ?? null,
|
||||
});
|
||||
return node === null ? [] : [[binding.bindingId, node] as const];
|
||||
}),
|
||||
const bindingNodes = new Map<string, JavaScriptSemanticGraphNode>();
|
||||
for (const [index, binding] of ir.bindings.entries()) {
|
||||
if (index % 64 === 0) yield;
|
||||
const location = binding.definitions[0]?.location ?? null;
|
||||
const kind = binding.kind === "parameter" ? "parameter" : "binding";
|
||||
const owner = callableOwnerAt(location);
|
||||
const node = retainNode(state, file, {
|
||||
kind,
|
||||
roleKey: `binding:${binding.bindingId}`,
|
||||
location,
|
||||
label: binding.name,
|
||||
functionNodeId: owner?.node_id ?? null,
|
||||
});
|
||||
if (node !== null) bindingNodes.set(binding.bindingId, node);
|
||||
}
|
||||
const returnSiteNodes = yield* createReturnSiteNodes(
|
||||
file,
|
||||
ir,
|
||||
callableNodes,
|
||||
state,
|
||||
);
|
||||
const callSiteNodes = yield* createCallSiteNodes(
|
||||
file,
|
||||
ir,
|
||||
callableNodes,
|
||||
state,
|
||||
);
|
||||
const returnSiteNodes = createReturnSiteNodes(file, ir, callableNodes, state);
|
||||
const callSiteNodes = createCallSiteNodes(file, ir, callableNodes, state);
|
||||
const definitions: Omit<FileContext, "referenceNodesWithin"> = {
|
||||
file,
|
||||
ir,
|
||||
@@ -328,42 +351,38 @@ const projectFile = (
|
||||
callableOwnerAt,
|
||||
callSiteAt: createSemanticCallSiteLookup(ir, callSiteNodes),
|
||||
};
|
||||
projectDefinitionsAndReferences(definitions);
|
||||
yield* projectDefinitionsAndReferences(definitions);
|
||||
const context: FileContext = {
|
||||
...definitions,
|
||||
referenceNodesWithin: createSemanticNodeRangeLookup(
|
||||
definitions.referenceNodes,
|
||||
),
|
||||
};
|
||||
projectSemanticValues(context);
|
||||
projectSemanticObjects(context);
|
||||
projectCalls(context);
|
||||
projectSemanticPromises(context);
|
||||
projectSemanticEvents(context);
|
||||
projectSemanticTimers(context);
|
||||
projectSemanticChildProcesses(context);
|
||||
projectSemanticConfiguration(context);
|
||||
projectSemanticRequests(context);
|
||||
projectSemanticBoundaries(context);
|
||||
projectSemanticResources(context);
|
||||
projectSemanticClosureCaptures(context);
|
||||
projectSemanticFrontiers(context);
|
||||
// Each pass completes before the next starts, so yielding between passes
|
||||
// keeps projection order and results unchanged.
|
||||
for (const project of FILE_PROJECTION_PASSES) {
|
||||
yield;
|
||||
const steps = project(context);
|
||||
if (steps !== undefined) while (steps.next().done !== true) yield;
|
||||
}
|
||||
yield;
|
||||
return projectSemanticFunctionFingerprints(
|
||||
file,
|
||||
ir,
|
||||
callableNodes,
|
||||
state.evidenceContexts,
|
||||
);
|
||||
};
|
||||
}
|
||||
|
||||
const createReturnSiteNodes = (
|
||||
function* createReturnSiteNodes(
|
||||
file: JavaScriptArtifactFile,
|
||||
ir: JavaScriptSemanticIr,
|
||||
callables: ReadonlyMap<string, JavaScriptSemanticGraphNode>,
|
||||
state: BuilderState,
|
||||
): Map<string, JavaScriptSemanticGraphNode> => {
|
||||
): Generator<void, Map<string, JavaScriptSemanticGraphNode>> {
|
||||
const result = new Map<string, JavaScriptSemanticGraphNode>();
|
||||
for (const callable of ir.callables) {
|
||||
for (const [index, callable] of ir.callables.entries()) {
|
||||
if (index % 64 === 0) yield;
|
||||
const owner = callables.get(callable.callableId);
|
||||
if (owner === undefined) continue;
|
||||
for (const site of callable.returnSites) {
|
||||
@@ -379,16 +398,17 @@ const createReturnSiteNodes = (
|
||||
}
|
||||
}
|
||||
return result;
|
||||
};
|
||||
}
|
||||
|
||||
const createCallSiteNodes = (
|
||||
function* createCallSiteNodes(
|
||||
file: JavaScriptArtifactFile,
|
||||
ir: JavaScriptSemanticIr,
|
||||
callables: ReadonlyMap<string, JavaScriptSemanticGraphNode>,
|
||||
state: BuilderState,
|
||||
): Map<string, JavaScriptSemanticGraphNode> => {
|
||||
): Generator<void, Map<string, JavaScriptSemanticGraphNode>> {
|
||||
const result = new Map<string, JavaScriptSemanticGraphNode>();
|
||||
for (const call of ir.callSites) {
|
||||
for (const [index, call] of ir.callSites.entries()) {
|
||||
if (index % 64 === 0) yield;
|
||||
const owner =
|
||||
call.callerCallableId === null
|
||||
? null
|
||||
@@ -403,12 +423,13 @@ const createCallSiteNodes = (
|
||||
if (node !== null) result.set(call.callSiteId, node);
|
||||
}
|
||||
return result;
|
||||
};
|
||||
}
|
||||
|
||||
const projectDefinitionsAndReferences = (
|
||||
function* projectDefinitionsAndReferences(
|
||||
context: Omit<FileContext, "referenceNodesWithin">,
|
||||
): void => {
|
||||
for (const binding of context.ir.bindings) {
|
||||
): Generator<void, void> {
|
||||
for (const [bindingIndex, binding] of context.ir.bindings.entries()) {
|
||||
if (bindingIndex % 64 === 0) yield;
|
||||
const bindingNode = context.bindingNodes.get(binding.bindingId);
|
||||
if (bindingNode === undefined) continue;
|
||||
for (const [index, definition] of binding.definitions.entries()) {
|
||||
@@ -428,6 +449,7 @@ const projectDefinitionsAndReferences = (
|
||||
}
|
||||
}
|
||||
for (const [index, reference] of context.ir.references.entries()) {
|
||||
if (index % 64 === 0) yield;
|
||||
const expression = retainNode(context.state, context.file, {
|
||||
kind: "expression",
|
||||
roleKey: `reference:${String(index)}:${reference.role}:${reference.name}`,
|
||||
@@ -458,7 +480,7 @@ const projectDefinitionsAndReferences = (
|
||||
resolution: "resolved",
|
||||
});
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
const projectCalls = (context: FileContext): void => {
|
||||
for (const call of context.ir.callSites) {
|
||||
@@ -543,3 +565,22 @@ const createArgumentNode = (
|
||||
: `argument ${String(argument.index)}`,
|
||||
functionNodeId: callNode.function_node_id,
|
||||
});
|
||||
|
||||
/** Per-file projection passes, run in this order with a yield between each. */
|
||||
const FILE_PROJECTION_PASSES: readonly ((
|
||||
context: FileContext,
|
||||
) => Iterator<void, void> | void)[] = [
|
||||
projectSemanticValues,
|
||||
projectSemanticObjects,
|
||||
projectCalls,
|
||||
projectSemanticPromises,
|
||||
projectSemanticEvents,
|
||||
projectSemanticTimers,
|
||||
projectSemanticChildProcesses,
|
||||
projectSemanticConfiguration,
|
||||
projectSemanticRequests,
|
||||
projectSemanticBoundaries,
|
||||
projectSemanticResources,
|
||||
projectSemanticClosureCaptures,
|
||||
projectSemanticFrontiers,
|
||||
];
|
||||
|
||||
@@ -240,10 +240,11 @@ export const projectSemanticClosureCaptures = (
|
||||
};
|
||||
|
||||
/** Retain bounded unresolved dynamic-call, dynamic-property and dynamic-scope frontiers. */
|
||||
export const projectSemanticFrontiers = (
|
||||
export function* projectSemanticFrontiers(
|
||||
context: SemanticFlowProjectionContext,
|
||||
): void => {
|
||||
for (const frontier of context.ir.frontiers) {
|
||||
): Generator<void, void> {
|
||||
for (const [index, frontier] of context.ir.frontiers.entries()) {
|
||||
if (index % 64 === 0) yield;
|
||||
// Dynamic environments leave identifier reads and writes unresolved.
|
||||
const family =
|
||||
frontier.kind === "dynamic-call"
|
||||
@@ -275,4 +276,4 @@ export const projectSemanticFrontiers = (
|
||||
);
|
||||
addSemanticGraphUnknown(context.state, unknown);
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
@@ -20,13 +20,14 @@ import type { SemanticFlowProjectionContext } from "./JavaScriptSemanticGraphFlo
|
||||
import { semanticPropertySlot } from "./JavaScriptSemanticGraphValueProjection.js";
|
||||
|
||||
/** Project static object reads, writes, spreads, and destructuring. */
|
||||
export const projectSemanticObjects = (
|
||||
export function* projectSemanticObjects(
|
||||
context: SemanticFlowProjectionContext,
|
||||
): void => {
|
||||
): Generator<void, void> {
|
||||
const values = new Map(
|
||||
context.ir.bindings.map((binding) => [binding.bindingId, binding.value]),
|
||||
);
|
||||
for (const operation of context.ir.objectOperations) {
|
||||
for (const [index, operation] of context.ir.objectOperations.entries()) {
|
||||
if (index % 64 === 0) yield;
|
||||
const occurrence = addObjectOccurrence(context, operation);
|
||||
const target = objectTarget(context, values, operation);
|
||||
const ends = objectRelationEnds(context, operation, occurrence, target);
|
||||
@@ -43,7 +44,7 @@ export const projectSemanticObjects = (
|
||||
if (!resolved)
|
||||
addObjectUnknown(context, operation, target.slot ?? occurrence, target);
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
const OBJECT_RELATIONS = {
|
||||
read: "reads-property",
|
||||
|
||||
@@ -73,15 +73,27 @@ export const traceNativeUiAction = async (
|
||||
),
|
||||
);
|
||||
graph = joinInterfaceBuilderDispatch(decoded.data.graph, metadata.result);
|
||||
const archiveDecode = decoded.data.graph.coverage.find(
|
||||
(facet) => facet.facet === "archive_decode",
|
||||
);
|
||||
const documentsFound = decoded.data.documents.length > 0;
|
||||
coverage.push({
|
||||
facet: "interface_builder_archives",
|
||||
status: decoded.data.documents.length > 0 ? "complete" : "unsupported",
|
||||
status:
|
||||
archiveDecode?.status === "partial" ||
|
||||
archiveDecode?.status === "complete" ||
|
||||
archiveDecode?.status === "unsupported"
|
||||
? archiveDecode.status
|
||||
: documentsFound
|
||||
? "complete"
|
||||
: "unsupported",
|
||||
reason:
|
||||
decoded.data.documents.length > 0
|
||||
archiveDecode?.reason ??
|
||||
(documentsFound
|
||||
? null
|
||||
: "no_compiled_interface_builder_archives_found",
|
||||
examined: decoded.data.documents.length,
|
||||
omitted: 0,
|
||||
: "no_compiled_interface_builder_archives_found"),
|
||||
examined: archiveDecode?.examined ?? decoded.data.documents.length,
|
||||
omitted: archiveDecode?.omitted ?? 0,
|
||||
});
|
||||
const bounded = boundUiGraph(graph, input.max_nodes, input.max_edges);
|
||||
graph = bounded.graph;
|
||||
|
||||
@@ -15,6 +15,7 @@ import { processTraceSpecificationSchema } from "../../domain/process/processTra
|
||||
import { processScenarioSchema } from "../../domain/process/processScenario.js";
|
||||
import { compareProcessCaptures } from "../../domain/process/processComparison.js";
|
||||
import { parseProcessCapture } from "../../domain/process/processCaptureParsing.js";
|
||||
import { createProgressReporter } from "../ProgressReporter.js";
|
||||
import { captureProcessScenario } from "../../process/capture/ProcessHarness.js";
|
||||
import { PROCESS_PROVIDER } from "../../domain/process/processEvidenceProvider.js";
|
||||
import { createProcessCaptureEvidence } from "./ProcessEvidence.js";
|
||||
@@ -56,11 +57,20 @@ export const captureProcessScenarioFile = async (
|
||||
input,
|
||||
{ cause: parsed.error },
|
||||
);
|
||||
const progress = createProgressReporter(
|
||||
async (update) => {
|
||||
process.stderr.write(`${JSON.stringify({ rea_progress: update })}\n`);
|
||||
},
|
||||
{ minimumIntervalMs: 100 },
|
||||
);
|
||||
const captured = await captureProcessScenario(
|
||||
parsed.data,
|
||||
signal,
|
||||
process.platform,
|
||||
environment,
|
||||
undefined,
|
||||
undefined,
|
||||
progress,
|
||||
);
|
||||
if (!captured.ok) return cliAnalysisError(captured.error);
|
||||
return createProcessCaptureEvidence(parsed.data, captured.value);
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import { posix } from "node:path";
|
||||
|
||||
import { logicalPathNfc } from "../domain/artifactIdentity.js";
|
||||
import { compareUnicodeCodePoints } from "../domain/unicodeCodePointOrder.js";
|
||||
import { ArtifactReaderFailure } from "./ArtifactReader.js";
|
||||
|
||||
@@ -31,7 +32,7 @@ export const normalizeArtifactPath = (input: string): string => {
|
||||
"path",
|
||||
`Artifact path is absolute or unsafe: ${JSON.stringify(input)}`,
|
||||
);
|
||||
const normalized = input.normalize("NFC").replace(/\/+$/u, "");
|
||||
const normalized = logicalPathNfc(input).replace(/\/+$/u, "");
|
||||
const parts = normalized.split("/");
|
||||
if (
|
||||
normalized.length === 0 ||
|
||||
|
||||
@@ -6,7 +6,10 @@ import {
|
||||
import type { FileHandle } from "node:fs/promises";
|
||||
import type { Stats } from "node:fs";
|
||||
|
||||
import { classifyArtifactContent } from "./ArtifactGraphConstruction.js";
|
||||
import {
|
||||
classifyArtifactBytes,
|
||||
classifyArtifactContent,
|
||||
} from "./ArtifactGraphConstruction.js";
|
||||
import { ARTIFACT_CLASSIFICATION_PREFIX_BYTES } from "../ArtifactHash.js";
|
||||
import type { ArtifactOccurrence } from "../../domain/artifactGraph.js";
|
||||
import {
|
||||
@@ -17,17 +20,6 @@ import { ArtifactReaderFailure } from "../ArtifactReader.js";
|
||||
import type { HashResult } from "../ArtifactHash.js";
|
||||
import { hashStableRootArtifactHandle } from "./hashStableRootArtifact.js";
|
||||
|
||||
const classifyContainerExtension = (
|
||||
path: string,
|
||||
): ArtifactOccurrence["artifact_format"] | undefined => {
|
||||
const lower = path.toLowerCase();
|
||||
const zipPackage = zipPackageFormatForPath(lower);
|
||||
if (zipPackage !== undefined) return zipPackage;
|
||||
for (const format of ["asar", "dmg", "pkg"] as const)
|
||||
if (lower.endsWith(`.${format}`)) return format;
|
||||
return undefined;
|
||||
};
|
||||
|
||||
/** Classify and hash one file root through the same stable open descriptor. */
|
||||
export const classifyAndHashRoot = async (
|
||||
path: string,
|
||||
@@ -47,9 +39,7 @@ export const classifyAndHashRoot = async (
|
||||
"integrity",
|
||||
`Root artifact changed before inventory: ${path}`,
|
||||
);
|
||||
const format =
|
||||
classifyContainerExtension(path) ??
|
||||
(await classifyOpenedRoot(path, handle));
|
||||
const format = await classifyRootFormat(path, handle);
|
||||
const digest = await hashStableRootArtifactHandle(
|
||||
path,
|
||||
handle,
|
||||
@@ -79,14 +69,50 @@ const openRootFile = async (
|
||||
}
|
||||
};
|
||||
|
||||
const classifyOpenedRoot = async (
|
||||
const classifyRootFormat = async (
|
||||
path: string,
|
||||
handle: FileHandle,
|
||||
): Promise<ArtifactOccurrence["artifact_format"]> => {
|
||||
const magic = Buffer.alloc(ARTIFACT_CLASSIFICATION_PREFIX_BYTES);
|
||||
const observed = await handle.read(magic, 0, magic.length, 0);
|
||||
const prefix = magic.subarray(0, observed.bytesRead);
|
||||
if (hasZipSignature(prefix)) return "zip";
|
||||
return classifyArtifactContent(path, prefix, (await handle.stat()).size)
|
||||
.format;
|
||||
const lower = path.toLowerCase();
|
||||
if (hasZipSignature(prefix)) return zipPackageFormatForPath(lower) ?? "zip";
|
||||
if (lower.endsWith(".asar") && hasAsarHeader(prefix)) return "asar";
|
||||
if (
|
||||
lower.endsWith(".pkg") &&
|
||||
prefix.subarray(0, 4).toString("ascii") === "xar!"
|
||||
)
|
||||
return "pkg";
|
||||
if (lower.endsWith(".dmg") && (await hasKolyTrailer(handle))) return "dmg";
|
||||
const size = (await handle.stat()).size;
|
||||
const format = classifyArtifactContent(path, prefix, size).format;
|
||||
// Path suffixes stay role hints for nested members. A root whose container
|
||||
// magic did not match is classified from the bytes that were read.
|
||||
if (format === "asar" || format === "pkg" || format === "dmg")
|
||||
return classifyArtifactBytes(prefix, size);
|
||||
return format;
|
||||
};
|
||||
|
||||
/**
|
||||
* Electron ASAR starts with a size pickle whose payload is one uint32, then
|
||||
* a header pickle whose payload begins with a JSON object.
|
||||
*/
|
||||
const hasAsarHeader = (prefix: Buffer): boolean => {
|
||||
if (prefix.length < 16 || prefix.readUInt32LE(0) !== 4) return false;
|
||||
const headerSize = prefix.readUInt32LE(4);
|
||||
const payloadSize = prefix.readUInt32LE(8);
|
||||
if (headerSize < 16 || payloadSize !== headerSize - 4) return false;
|
||||
const stringLength = prefix.readUInt32LE(12);
|
||||
return (
|
||||
stringLength >= 2 && stringLength <= payloadSize - 4 && prefix[16] === 0x7b
|
||||
);
|
||||
};
|
||||
|
||||
const hasKolyTrailer = async (handle: FileHandle): Promise<boolean> => {
|
||||
const size = (await handle.stat()).size;
|
||||
if (size < 512) return false;
|
||||
const trailer = Buffer.alloc(4);
|
||||
const read = await handle.read(trailer, 0, trailer.length, size - 512);
|
||||
return read.bytesRead === 4 && trailer.toString("ascii") === "koly";
|
||||
};
|
||||
|
||||
@@ -5,6 +5,7 @@ import { MachOSliceArtifactReader } from "../MachOSliceArtifactReader.js";
|
||||
import { NativeDmgArtifactReader } from "../NativeDmgArtifactReader.js";
|
||||
import { ZipArtifactReader } from "../ZipArtifactReader.js";
|
||||
import type { ArtifactOccurrence } from "../../domain/artifactGraph.js";
|
||||
import { ZIP_NON_ENTRY_TAIL_LIMITATION } from "../../domain/zipPackageFormat.js";
|
||||
|
||||
export const createReader = async (
|
||||
path: string,
|
||||
@@ -39,7 +40,15 @@ export const inventoryLimitations = (
|
||||
format: ArtifactOccurrence["artifact_format"],
|
||||
reader: ArtifactReader | undefined,
|
||||
): string[] => {
|
||||
if (reader !== undefined) return [];
|
||||
if (reader !== undefined)
|
||||
return reader.format === "zip" ||
|
||||
reader.format === "ipa" ||
|
||||
reader.format === "apk" ||
|
||||
reader.format === "msix" ||
|
||||
reader.format === "appx"
|
||||
? [ZIP_NON_ENTRY_TAIL_LIMITATION]
|
||||
: [];
|
||||
// dmg and pkg reach here only after classifyRootFormat matched their bytes.
|
||||
if (format === "dmg" || format === "pkg")
|
||||
return [
|
||||
`${format.toUpperCase()} root hash is observed; child inventory requires a native macOS adapter.`,
|
||||
|
||||
@@ -56,4 +56,39 @@ describe("CDP execution-context retention", () => {
|
||||
});
|
||||
expect(electronFrames.size).toBe(0);
|
||||
});
|
||||
|
||||
it("keeps a script on the frame stored when it was parsed", () => {
|
||||
const request = inspectWebPageInputSchema.parse({
|
||||
cdp_endpoint: "http://127.0.0.1:9222",
|
||||
allowed_origins: ["https://app.example.test"],
|
||||
target_id: "page-1",
|
||||
});
|
||||
const browser = new CdpCaptureEvents(
|
||||
request,
|
||||
new Set(["https://app.example.test"]),
|
||||
);
|
||||
browser.beginAuthorizedFrame("frame-old");
|
||||
browser.beginAuthorizedFrame("frame-new");
|
||||
browser.ingest({
|
||||
method: "Runtime.executionContextCreated",
|
||||
params: { context: { id: 7, auxData: { frameId: "frame-old" } } },
|
||||
});
|
||||
browser.ingest({
|
||||
method: "Debugger.scriptParsed",
|
||||
params: {
|
||||
scriptId: "script-1",
|
||||
url: "https://app.example.test/app.js",
|
||||
executionContextId: 7,
|
||||
},
|
||||
});
|
||||
browser.ingest({
|
||||
method: "Runtime.executionContextCreated",
|
||||
params: { context: { id: 7, auxData: { frameId: "frame-new" } } },
|
||||
});
|
||||
const script = browser.scripts.get("script-1");
|
||||
if (script === undefined) throw new TypeError("Expected captured script");
|
||||
expect(
|
||||
browser.frameForScript(script, new Set(["frame-old", "frame-new"])),
|
||||
).toBe("frame-old");
|
||||
});
|
||||
});
|
||||
|
||||
@@ -198,6 +198,23 @@ describe("empty form destinations with a document base URL", () => {
|
||||
},
|
||||
);
|
||||
|
||||
it("keeps the first value when an attribute name repeats", () => {
|
||||
const value = snapshot();
|
||||
const later = `${origin}/later`;
|
||||
value.strings.push(later);
|
||||
const href = value.strings.indexOf("href");
|
||||
const guide = value.strings.indexOf("guide");
|
||||
const nodes = value.documents[0]?.nodes;
|
||||
if (nodes === undefined) throw new TypeError("Expected a DOM snapshot");
|
||||
nodes.attributes[2] = [href, guide, href, value.strings.length - 1];
|
||||
const result = capture(value);
|
||||
expect(result.nodes[2]?.attribute_names).toEqual(["href", "href"]);
|
||||
expect(result.urls.map(({ url }) => url)).toContain(
|
||||
`${origin}/screens/guide`,
|
||||
);
|
||||
expect(result.urls.map(({ url }) => url)).not.toContain(later);
|
||||
});
|
||||
|
||||
it("does not apply form semantics to an action attribute on another element", () => {
|
||||
const baseUrl = `${origin}/assets/`;
|
||||
const value = snapshot(baseUrl);
|
||||
|
||||
@@ -387,7 +387,8 @@ const domMetadata = (
|
||||
for (let index = 0; index + 1 < attributes.length; index += 2) {
|
||||
const name = indexedString(strings, attributes[index]).toLowerCase();
|
||||
const value = indexedString(strings, attributes[index + 1]);
|
||||
pairs.set(name, value);
|
||||
// HTML keeps the first declaration. A later repeat must not move dom_urls.
|
||||
if (!pairs.has(name)) pairs.set(name, value);
|
||||
}
|
||||
const urls: WebPageInspection["metadata"]["dom_urls"] = [];
|
||||
for (const attribute of domUrlAttributes) {
|
||||
|
||||
@@ -47,6 +47,7 @@ export const handleExecutionContextCreated = (
|
||||
)
|
||||
return;
|
||||
const key = String(identifier);
|
||||
// New contexts may reuse an id. Scripts already parsed keep the frame stored on them.
|
||||
state.executionContextFrames.set(key, frameId);
|
||||
};
|
||||
|
||||
@@ -80,6 +81,7 @@ export const handleScriptParsed = (
|
||||
state.completeness.exclude("scripts", exclusionReasonForUrl(rawUrl));
|
||||
return;
|
||||
}
|
||||
const executionContext = executionContextKey(params.executionContextId);
|
||||
const baseScript = {
|
||||
scriptId,
|
||||
rawUrl,
|
||||
@@ -87,7 +89,11 @@ export const handleScriptParsed = (
|
||||
origin: sanitized.origin,
|
||||
...scriptMetadataValues(params),
|
||||
language: cdpStringValue(params.scriptLanguage) ?? null,
|
||||
executionContextKey: executionContextKey(params.executionContextId),
|
||||
executionContextKey: executionContext,
|
||||
frameId:
|
||||
executionContext === null
|
||||
? null
|
||||
: (state.executionContextFrames.get(executionContext) ?? null),
|
||||
};
|
||||
const baseBytes = scriptMetadataStringBytes(baseScript);
|
||||
const sourceMapDeclared =
|
||||
@@ -131,6 +137,7 @@ const scriptMetadataStringBytes = (script: {
|
||||
readonly hash: string | null;
|
||||
readonly language: string | null;
|
||||
readonly executionContextKey: string | null;
|
||||
readonly frameId: string | null;
|
||||
}): number =>
|
||||
stringByteLength(script.scriptId) +
|
||||
stringByteLength(script.rawUrl) +
|
||||
@@ -138,7 +145,8 @@ const scriptMetadataStringBytes = (script: {
|
||||
stringByteLength(script.origin) +
|
||||
stringByteLength(script.hash) +
|
||||
stringByteLength(script.language) +
|
||||
stringByteLength(script.executionContextKey);
|
||||
stringByteLength(script.executionContextKey) +
|
||||
stringByteLength(script.frameId);
|
||||
|
||||
const stringByteLength = (value: string | null): number =>
|
||||
value === null ? 0 : Buffer.byteLength(value, "utf8");
|
||||
|
||||
@@ -12,6 +12,8 @@ export interface CapturedScript {
|
||||
readonly sourceMapUrl: string | null;
|
||||
readonly sourceMapRawUrl: string | null;
|
||||
readonly executionContextKey: string | null;
|
||||
/** Frame observed when this script was parsed. Later context reuse does not move it. */
|
||||
readonly frameId: string | null;
|
||||
}
|
||||
|
||||
export type NetworkState = WebPageInspection["network"]["requests"][number];
|
||||
|
||||
@@ -159,10 +159,8 @@ export class CdpCaptureEvents implements CdpCaptureEventsState {
|
||||
script: CapturedScript,
|
||||
allowedFrameIds: ReadonlySet<string>,
|
||||
): string | null {
|
||||
if (script.executionContextKey === null) return null;
|
||||
const frameId = this.executionContextFrames.get(script.executionContextKey);
|
||||
return frameId !== undefined && allowedFrameIds.has(frameId)
|
||||
? frameId
|
||||
return script.frameId !== null && allowedFrameIds.has(script.frameId)
|
||||
? script.frameId
|
||||
: null;
|
||||
}
|
||||
|
||||
|
||||
@@ -3,12 +3,22 @@ import type { Browser } from "playwright-core";
|
||||
|
||||
import { browserScenarioSchema } from "../domain/browserScenario.js";
|
||||
import {
|
||||
attachedScenarioLeaseKey,
|
||||
openPlaywrightScenarioBrowser,
|
||||
PlaywrightScenarioBrowserCleanupOwner,
|
||||
} from "./PlaywrightScenarioBrowser.js";
|
||||
|
||||
afterEach(() => vi.unstubAllEnvs());
|
||||
|
||||
it("locks one attached browser for spellings that discover the same endpoint", () => {
|
||||
expect(attachedScenarioLeaseKey("http://127.0.0.1:9222", "page-1")).toBe(
|
||||
attachedScenarioLeaseKey("http://127.0.0.1:9222/", "page-1"),
|
||||
);
|
||||
expect(attachedScenarioLeaseKey("http://127.0.0.1:9222", "page-1")).not.toBe(
|
||||
attachedScenarioLeaseKey("http://127.0.0.1:9222", "page-2"),
|
||||
);
|
||||
});
|
||||
|
||||
const deferred = <Value>() => {
|
||||
let resolve: (value: Value | PromiseLike<Value>) => void = () => undefined;
|
||||
const promise = new Promise<Value>((resolvePromise) => {
|
||||
|
||||
@@ -182,6 +182,13 @@ const createAttachedScenarioRelease = (
|
||||
};
|
||||
};
|
||||
|
||||
/** One browser admission. The key is the discovery URL this endpoint already parses. */
|
||||
export const attachedScenarioLeaseKey = (
|
||||
endpoint: string,
|
||||
targetId: string,
|
||||
): string =>
|
||||
JSON.stringify([new URL("/json/version", endpoint).href, targetId]);
|
||||
|
||||
const acquireAttachedScenario = (
|
||||
scenario: Extract<BrowserScenario["browser"], { readonly mode: "connect" }>,
|
||||
signal: AbortSignal | undefined,
|
||||
@@ -189,7 +196,10 @@ const acquireAttachedScenario = (
|
||||
if (signal?.aborted === true)
|
||||
return Promise.reject(new BrowserObservationError(OPERATION, "cancelled"));
|
||||
|
||||
const key = JSON.stringify([scenario.cdp_endpoint, scenario.target_id]);
|
||||
const key = attachedScenarioLeaseKey(
|
||||
scenario.cdp_endpoint,
|
||||
scenario.target_id,
|
||||
);
|
||||
const admission =
|
||||
attachedScenarioAdmissions.get(key) ??
|
||||
({
|
||||
|
||||
@@ -2,20 +2,26 @@ import { createHash } from "node:crypto";
|
||||
import { join } from "node:path";
|
||||
import { Readable } from "node:stream";
|
||||
|
||||
import { ArtifactReaderFailure } from "../../artifacts/ArtifactReader.js";
|
||||
import { DESTINATION_CASE_COLLISION_PREFIX } from "../../artifacts/ArtifactPaths.js";
|
||||
import { SafeOutputTree } from "../../artifacts/SafeOutputTree.js";
|
||||
import {
|
||||
webScriptExportManifestSchema,
|
||||
webScriptExportResultSchema,
|
||||
type ExportWebScriptsInput,
|
||||
type ExportedWebScript,
|
||||
type WebScriptExportResult,
|
||||
} from "../../domain/webScriptExport.js";
|
||||
import { planWebScriptExport } from "../../domain/webScriptExportPlan.js";
|
||||
import {
|
||||
planWebScriptExport,
|
||||
type PlannedWebScript,
|
||||
} from "../../domain/webScriptExportPlan.js";
|
||||
import { WebScriptExportError } from "../../domain/webScriptExportError.js";
|
||||
import type { SelectedScriptCapture } from "./ScriptCaptureAdapters.js";
|
||||
|
||||
const LIMITATIONS = [
|
||||
"Only script bytes retained in this capture are exported. Sources are not refetched or executed, and a response or Debugger source does not prove execution.",
|
||||
"Ordinary unambiguous URL paths preserve relative module layouts by origin. Query variants, collisions, inline sources, and unrepresentable paths are isolated; their browser module relationships remain unknown.",
|
||||
"Ordinary unambiguous URL paths preserve relative module layouts by origin, including case-distinct spellings. Query variants, exact path collisions, file/directory prefixes, inline sources, and unrepresentable paths are isolated. A case-distinct pair is isolated only when the destination filesystem cannot store both spellings; the record names that pair. Browser module relationships of isolated paths remain unknown.",
|
||||
"Local JavaScript analysis does not implement browser import maps, root-relative URL resolution, remote URL loading, or browser runtime behavior. It may strip import queries and fragments; isolated variants are not canonical targets.",
|
||||
"Source-map declarations are retained as metadata. No source maps or missing dependencies are fetched or reconstructed.",
|
||||
"Captured response bytes are browser-decoded bytes; Debugger sources are the exposed text encoded as UTF-8. Redacted bytes are exported exactly as retained, and may no longer parse as JavaScript.",
|
||||
@@ -32,7 +38,11 @@ export const publishWebScripts = async (
|
||||
const tree = await SafeOutputTree.create(input.output_directory);
|
||||
try {
|
||||
signal?.throwIfAborted();
|
||||
const records = planned.map(({ record }) => record);
|
||||
const records: ExportedWebScript[] = [];
|
||||
for (const [index, item] of planned.entries()) {
|
||||
signal?.throwIfAborted();
|
||||
records.push(await publishPlannedScript(tree, item, index, signal));
|
||||
}
|
||||
const manifest = webScriptExportManifestSchema.parse({
|
||||
capture_path: input.capture_path,
|
||||
capture_sha256: captureSha256,
|
||||
@@ -66,19 +76,6 @@ export const publishWebScripts = async (
|
||||
bytes: bytes.length,
|
||||
},
|
||||
});
|
||||
for (const { script, record } of planned) {
|
||||
signal?.throwIfAborted();
|
||||
if (
|
||||
script.content.state === "captured" &&
|
||||
record.content.state === "exported"
|
||||
)
|
||||
await tree.write(
|
||||
`files/${record.content.relative_path}`,
|
||||
Readable.from([script.content.bytes]),
|
||||
script.content.sha256,
|
||||
signal,
|
||||
);
|
||||
}
|
||||
await tree.write("manifest.json", Readable.from([bytes]), digest, signal);
|
||||
signal?.throwIfAborted();
|
||||
await tree.commit();
|
||||
@@ -98,5 +95,56 @@ export const publishWebScripts = async (
|
||||
}
|
||||
};
|
||||
|
||||
const publishPlannedScript = async (
|
||||
tree: SafeOutputTree,
|
||||
item: PlannedWebScript,
|
||||
index: number,
|
||||
signal: AbortSignal | undefined,
|
||||
): Promise<ExportedWebScript> => {
|
||||
const { script, record } = item;
|
||||
const content = script.content;
|
||||
if (content.state !== "captured" || record.content.state !== "exported")
|
||||
return record;
|
||||
const write = (relativePath: string): Promise<unknown> =>
|
||||
tree.write(
|
||||
`files/${relativePath}`,
|
||||
Readable.from([content.bytes]),
|
||||
content.sha256,
|
||||
signal,
|
||||
);
|
||||
try {
|
||||
await write(record.content.relative_path);
|
||||
return record;
|
||||
} catch (cause: unknown) {
|
||||
const collision = destinationCollisionReason(cause);
|
||||
if (collision === undefined || record.content.layout !== "url-path")
|
||||
throw cause;
|
||||
const relativePath = isolatedScriptPath(script, index);
|
||||
await write(relativePath);
|
||||
return {
|
||||
...record,
|
||||
content: {
|
||||
...record.content,
|
||||
relative_path: relativePath,
|
||||
layout: "isolated",
|
||||
layout_reason: collision,
|
||||
},
|
||||
};
|
||||
}
|
||||
};
|
||||
|
||||
const isolatedScriptPath = (
|
||||
script: PlannedWebScript["script"],
|
||||
index: number,
|
||||
): string =>
|
||||
`isolated/source-${index + 1}-${createHash("sha256").update(JSON.stringify(script.source)).digest("hex")}.js`;
|
||||
|
||||
const destinationCollisionReason = (cause: unknown): string | undefined =>
|
||||
cause instanceof ArtifactReaderFailure &&
|
||||
cause.reason === "path" &&
|
||||
cause.message.startsWith(DESTINATION_CASE_COLLISION_PREFIX)
|
||||
? cause.message
|
||||
: undefined;
|
||||
|
||||
const message = (cause: unknown): string =>
|
||||
cause instanceof Error ? cause.message : "Unknown publication failure";
|
||||
|
||||
@@ -5,11 +5,13 @@ import { ConfigurationError } from "../domain/configurationErrors.js";
|
||||
import { err, ok, type Result } from "../domain/result.js";
|
||||
import { analysisProviderSelectorSchema } from "../contracts/providerSelection.js";
|
||||
import { ghidraStartupTimeoutSchema } from "./ghidraStartupTimeout.js";
|
||||
import { mcpInputSchemaProfileSchema } from "./mcpInputSchemaProfile.js";
|
||||
import { mcpResponseBudgetSchema } from "./mcpResponseBudget.js";
|
||||
|
||||
const environmentSchema = z.object({
|
||||
REA_ANALYSIS_PROVIDER: analysisProviderSelectorSchema.default("auto"),
|
||||
REA_MCP_MAX_RESPONSE_BYTES: mcpResponseBudgetSchema.optional(),
|
||||
REA_MCP_INPUT_SCHEMA_PROFILE: mcpInputSchemaProfileSchema.optional(),
|
||||
REA_IDA_MCP_CONFIG: z
|
||||
.string()
|
||||
.min(1)
|
||||
|
||||
@@ -0,0 +1,38 @@
|
||||
import { z } from "zod";
|
||||
|
||||
import { ConfigurationError } from "../domain/configurationErrors.js";
|
||||
import { err, ok, type Result } from "../domain/result.js";
|
||||
|
||||
// Moonshot's function-calling API rejects requests while any single tool
|
||||
// parameters schema exceeds its literal serialized size cap. Community
|
||||
// measurements bound the cap between ~13.7 KB (accepted) and ~15 KB
|
||||
// (rejected); 13 KiB leaves margin under the lower bound. See
|
||||
// https://github.com/morluto/rea/issues/1484 for the measured evidence.
|
||||
export const COMPACT_INPUT_SCHEMA_BUDGET_BYTES = 13 * 1024;
|
||||
|
||||
export const mcpInputSchemaProfileSchema = z.enum(["full", "compact"]);
|
||||
|
||||
const profileConstraint =
|
||||
'REA_MCP_INPUT_SCHEMA_PROFILE must be "full" (default) or "compact". The compact profile advertises input schemas without nested annotation prose for providers that cap each tool schema\'s serialized size.';
|
||||
|
||||
export type McpInputSchemaProfile = z.infer<typeof mcpInputSchemaProfileSchema>;
|
||||
|
||||
/** Parse the advertised input schema presentation selection. */
|
||||
export const parseMcpInputSchemaProfile = (
|
||||
value: string | undefined,
|
||||
): Result<McpInputSchemaProfile | undefined, ConfigurationError> => {
|
||||
if (value === undefined) return ok(undefined);
|
||||
const parsed = mcpInputSchemaProfileSchema.safeParse(value);
|
||||
return parsed.success
|
||||
? ok(parsed.data)
|
||||
: err(
|
||||
new ConfigurationError(profileConstraint, {
|
||||
settings: [
|
||||
{
|
||||
setting: "REA_MCP_INPUT_SCHEMA_PROFILE",
|
||||
constraint: profileConstraint,
|
||||
},
|
||||
],
|
||||
}),
|
||||
);
|
||||
};
|
||||
@@ -50,6 +50,9 @@ export const parseConfig = (
|
||||
...(env.REA_MCP_MAX_RESPONSE_BYTES === undefined
|
||||
? {}
|
||||
: { mcpMaxResponseBytes: env.REA_MCP_MAX_RESPONSE_BYTES }),
|
||||
...(env.REA_MCP_INPUT_SCHEMA_PROFILE === undefined
|
||||
? {}
|
||||
: { mcpInputSchemaProfile: env.REA_MCP_INPUT_SCHEMA_PROFILE }),
|
||||
analysisProvider: env.REA_ANALYSIS_PROVIDER,
|
||||
...(env.REA_IDA_MCP_CONFIG === undefined
|
||||
? {}
|
||||
|
||||
@@ -2,8 +2,11 @@ import type { LogLevel } from "../logger.js";
|
||||
import type { ReferenceSourcePolicy } from "../domain/referenceSourcePolicy.js";
|
||||
import type { AnalysisProviderSelector } from "../contracts/providerSelection.js";
|
||||
|
||||
import type { McpInputSchemaProfile } from "./mcpInputSchemaProfile.js";
|
||||
|
||||
export interface AppConfig {
|
||||
readonly mcpMaxResponseBytes?: number;
|
||||
readonly mcpInputSchemaProfile?: McpInputSchemaProfile;
|
||||
readonly analysisProvider: AnalysisProviderSelector;
|
||||
readonly idaMcpConfigPath?: string;
|
||||
readonly ghidraInstallDir: string | undefined;
|
||||
|
||||
@@ -0,0 +1,314 @@
|
||||
import { Ajv2020 } from "ajv/dist/2020.js";
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import { compactAdvertisedInputSchema } from "./compactInputSchema.js";
|
||||
|
||||
const serializedBytes = (value: unknown): number =>
|
||||
Buffer.byteLength(JSON.stringify(value), "utf8");
|
||||
|
||||
const repeatedObservation = {
|
||||
type: "object",
|
||||
properties: {
|
||||
captured_url: { type: "string", minLength: 1 },
|
||||
captured_at: { type: "string", minLength: 1 },
|
||||
status_code: { type: "integer", minimum: 100, maximum: 599 },
|
||||
headers: {
|
||||
type: "object",
|
||||
additionalProperties: { type: "string" },
|
||||
minProperties: 1,
|
||||
},
|
||||
body_digest: { type: "string", minLength: 64, maxLength: 64 },
|
||||
},
|
||||
required: ["captured_url", "captured_at", "status_code"],
|
||||
additionalProperties: false,
|
||||
} as const;
|
||||
|
||||
const annotatedSchema = {
|
||||
type: "object",
|
||||
description: "Root group guidance stays.",
|
||||
properties: {
|
||||
query: {
|
||||
type: "string",
|
||||
description: "Field guidance leaves.",
|
||||
examples: ["seed"],
|
||||
default: "seed",
|
||||
title: "Query",
|
||||
minLength: 1,
|
||||
},
|
||||
limits: {
|
||||
type: "object",
|
||||
description: "Nested guidance leaves.",
|
||||
properties: {
|
||||
max_results: {
|
||||
type: "integer",
|
||||
description: "Leaves.",
|
||||
minimum: 1,
|
||||
maximum: 500,
|
||||
default: 50,
|
||||
},
|
||||
},
|
||||
required: ["max_results"],
|
||||
additionalProperties: false,
|
||||
},
|
||||
},
|
||||
required: ["query"],
|
||||
dependentRequired: { limits: ["query"] },
|
||||
additionalProperties: false,
|
||||
} as const;
|
||||
|
||||
const oversizedPairSchema = (): Record<string, unknown> => {
|
||||
const deepStructure = {
|
||||
type: "object",
|
||||
description: "Nested guidance that leaves.",
|
||||
properties: Object.fromEntries(
|
||||
Array.from({ length: 40 }, (_, index) => [
|
||||
`field_${index}`,
|
||||
{ type: "string", minLength: 1, description: `Field ${index}` },
|
||||
]),
|
||||
),
|
||||
required: Array.from({ length: 40 }, (_, index) => `field_${index}`),
|
||||
additionalProperties: false,
|
||||
};
|
||||
return {
|
||||
type: "object",
|
||||
description: "Provide a complete capture pair.",
|
||||
properties: {
|
||||
before: structuredClone(deepStructure),
|
||||
after: structuredClone(deepStructure),
|
||||
},
|
||||
required: ["before", "after"],
|
||||
additionalProperties: false,
|
||||
};
|
||||
};
|
||||
|
||||
describe("compact advertised input schema", () => {
|
||||
it("drops nested annotation prose while keeping validation keywords", () => {
|
||||
const { schema, presentation } = compactAdvertisedInputSchema(
|
||||
structuredClone(annotatedSchema),
|
||||
13 * 1024,
|
||||
);
|
||||
expect(presentation).toBe("inline");
|
||||
expect(schema.description).toBe("Root group guidance stays.");
|
||||
const query = (schema.properties as Record<string, unknown>)
|
||||
.query as Record<string, unknown>;
|
||||
expect(query).toEqual({ type: "string", minLength: 1 });
|
||||
const limits = (schema.properties as Record<string, unknown>)
|
||||
.limits as Record<string, unknown>;
|
||||
expect(limits.description).toBeUndefined();
|
||||
expect(limits.required).toEqual(["max_results"]);
|
||||
expect(schema.dependentRequired).toEqual({ limits: ["query"] });
|
||||
expect(schema.required).toEqual(["query"]);
|
||||
});
|
||||
|
||||
it("keeps annotation-shaped keys inside literal const and enum data", () => {
|
||||
const literalMode = {
|
||||
description: "saved data",
|
||||
default: { keep: "me" },
|
||||
};
|
||||
const { schema } = compactAdvertisedInputSchema(
|
||||
{
|
||||
type: "object",
|
||||
properties: {
|
||||
mode: { enum: [literalMode] },
|
||||
pinned: { const: { description: "literal" } },
|
||||
},
|
||||
},
|
||||
13 * 1024,
|
||||
);
|
||||
const properties = schema.properties as Record<string, unknown>;
|
||||
expect((properties.mode as Record<string, unknown>).enum).toEqual([
|
||||
literalMode,
|
||||
]);
|
||||
expect((properties.pinned as Record<string, unknown>).const).toEqual({
|
||||
description: "literal",
|
||||
});
|
||||
});
|
||||
|
||||
it("inlines local references without changing validation or literal data", () => {
|
||||
const original = {
|
||||
type: "object",
|
||||
$defs: { "observation/~": structuredClone(repeatedObservation) },
|
||||
properties: {
|
||||
before: { $ref: "#/$defs/observation~1~0" },
|
||||
after: { $ref: "#/$defs/observation~1~0" },
|
||||
literal: { const: { $ref: "saved data", $defs: { keep: true } } },
|
||||
},
|
||||
required: ["before", "after"],
|
||||
additionalProperties: false,
|
||||
};
|
||||
const { schema, presentation } = compactAdvertisedInputSchema(
|
||||
original,
|
||||
13312,
|
||||
);
|
||||
expect(presentation).toBe("inline");
|
||||
expect(schema.$defs).toBeUndefined();
|
||||
const ajv = new Ajv2020({ strict: false });
|
||||
const canonical = ajv.compile(original);
|
||||
const compact = ajv.compile(schema);
|
||||
const observation = {
|
||||
captured_url: "https://example.com",
|
||||
captured_at: "now",
|
||||
status_code: 200,
|
||||
};
|
||||
for (const input of [
|
||||
{ before: observation, after: observation },
|
||||
{ before: observation, after: { ...observation, status_code: 99 } },
|
||||
{
|
||||
before: observation,
|
||||
after: observation,
|
||||
literal: original.properties.literal.const,
|
||||
},
|
||||
])
|
||||
expect(compact(input)).toBe(canonical(input));
|
||||
});
|
||||
|
||||
it("keeps unconstrained JSON values explicit for client type inference", () => {
|
||||
const { schema } = compactAdvertisedInputSchema(
|
||||
{
|
||||
type: "object",
|
||||
properties: { normalized_result: {}, literal: { const: {} } },
|
||||
},
|
||||
13312,
|
||||
);
|
||||
const properties = schema.properties as Record<string, unknown>;
|
||||
expect(properties.normalized_result).toEqual({
|
||||
type: ["null", "boolean", "object", "array", "number", "string"],
|
||||
});
|
||||
expect(properties.literal).toEqual({ const: {} });
|
||||
const validate = new Ajv2020({ strict: false }).compile(schema);
|
||||
for (const normalized_result of [null, true, {}, [], 42, "result"])
|
||||
expect(validate({ normalized_result })).toBe(true);
|
||||
});
|
||||
|
||||
it("preserves independent validation constraints on reference siblings", () => {
|
||||
const original = {
|
||||
type: "object",
|
||||
$defs: { text: { type: "string", minLength: 3 } },
|
||||
properties: {
|
||||
query: { $ref: "#/$defs/text", minLength: 1, maxLength: 5 },
|
||||
},
|
||||
};
|
||||
const { schema } = compactAdvertisedInputSchema(original, 13312);
|
||||
const validate = new Ajv2020({ strict: false }).compile(schema);
|
||||
expect(validate({ query: "a" })).toBe(false);
|
||||
expect(validate({ query: "abc" })).toBe(true);
|
||||
expect(validate({ query: "abcdef" })).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe("compact input schema budget and reduction", () => {
|
||||
it("budgets repeated-reference expansion before materializing it", () => {
|
||||
const definitions: Record<string, unknown> = {
|
||||
leaf: structuredClone(repeatedObservation),
|
||||
};
|
||||
for (let index = 0; index < 30; index++) {
|
||||
const $ref = `#/$defs/${index === 0 ? "leaf" : `level${index - 1}`}`;
|
||||
definitions[`level${index}`] = {
|
||||
type: "object",
|
||||
properties: { left: { $ref }, right: { $ref } },
|
||||
};
|
||||
}
|
||||
const { schema, presentation } = compactAdvertisedInputSchema(
|
||||
{
|
||||
type: "object",
|
||||
$defs: definitions,
|
||||
properties: { tree: { $ref: "#/$defs/level29" } },
|
||||
},
|
||||
1024,
|
||||
);
|
||||
expect(presentation).toBe("reduced");
|
||||
expect(serializedBytes(schema)).toBeLessThanOrEqual(1024);
|
||||
expect(schema.properties).toEqual({ tree: { type: "object" } });
|
||||
});
|
||||
|
||||
it("retains referenced object, array and nullable types in reduced inputs", () => {
|
||||
const original = {
|
||||
type: "object",
|
||||
$defs: {
|
||||
pair: oversizedPairSchema(),
|
||||
observations: {
|
||||
type: "array",
|
||||
items: structuredClone(repeatedObservation),
|
||||
},
|
||||
nullable: { anyOf: [{ type: "string" }, { type: "null" }] },
|
||||
},
|
||||
properties: {
|
||||
capture: { $ref: "#/$defs/pair", description: "Complete capture." },
|
||||
observations: { $ref: "#/$defs/observations" },
|
||||
label: { $ref: "#/$defs/nullable" },
|
||||
},
|
||||
};
|
||||
const { schema, presentation } = compactAdvertisedInputSchema(
|
||||
original,
|
||||
1024,
|
||||
);
|
||||
expect(presentation).toBe("reduced");
|
||||
expect(schema.properties).toEqual({
|
||||
capture: { type: "object", description: "Complete capture." },
|
||||
observations: { type: "array" },
|
||||
label: { type: ["string", "null"] },
|
||||
});
|
||||
const validate = new Ajv2020({ strict: false }).compile(schema);
|
||||
expect(validate({ capture: {}, observations: [], label: null })).toBe(true);
|
||||
expect(validate({ capture: {}, observations: [], label: "capture" })).toBe(
|
||||
true,
|
||||
);
|
||||
expect(validate({ capture: "{}", observations: [], label: null })).toBe(
|
||||
false,
|
||||
);
|
||||
});
|
||||
|
||||
it("reduces structurally oversized schemas to the shallow property presentation", () => {
|
||||
const compacted = compactAdvertisedInputSchema(oversizedPairSchema(), 1024);
|
||||
expect(compacted.presentation).toBe("reduced");
|
||||
expect(serializedBytes(compacted.schema)).toBeLessThanOrEqual(1024);
|
||||
expect(compacted.schema.type).toBe("object");
|
||||
expect(compacted.schema.description).toContain(
|
||||
"Provide a complete capture pair.",
|
||||
);
|
||||
expect(compacted.schema.description).toContain(
|
||||
"compact input schema profile",
|
||||
);
|
||||
const properties = compacted.schema.properties as Record<string, unknown>;
|
||||
expect(properties.before).toEqual({
|
||||
type: "object",
|
||||
description: "Nested guidance that leaves.",
|
||||
});
|
||||
expect(properties.after).toEqual(properties.before);
|
||||
expect(compacted.schema.required).toEqual(["before", "after"]);
|
||||
});
|
||||
|
||||
it("produces deterministic output for equal input", () => {
|
||||
const schema = {
|
||||
type: "object",
|
||||
properties: {
|
||||
before: structuredClone(repeatedObservation),
|
||||
after: structuredClone(repeatedObservation),
|
||||
},
|
||||
};
|
||||
expect(compactAdvertisedInputSchema(schema, 13 * 1024)).toEqual(
|
||||
compactAdvertisedInputSchema(structuredClone(schema), 13 * 1024),
|
||||
);
|
||||
});
|
||||
|
||||
it("keeps inline presentations under the measured provider budget", () => {
|
||||
const { schema, presentation } = compactAdvertisedInputSchema(
|
||||
{
|
||||
type: "object",
|
||||
description: "Group guidance.",
|
||||
properties: {
|
||||
before: structuredClone(repeatedObservation),
|
||||
after: structuredClone(repeatedObservation),
|
||||
normalization: {
|
||||
type: "object",
|
||||
description: "Rules.",
|
||||
properties: { rules: { type: "array", items: { type: "string" } } },
|
||||
},
|
||||
},
|
||||
},
|
||||
13312,
|
||||
);
|
||||
expect(presentation).toBe("inline");
|
||||
expect(serializedBytes(schema)).toBeLessThanOrEqual(13312);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,311 @@
|
||||
import {
|
||||
schemaArrays,
|
||||
schemaChildren,
|
||||
schemaMaps,
|
||||
} from "./inputSchemaPresentation.js";
|
||||
|
||||
// Kimi expands schema-local references before sending tools to Moonshot.
|
||||
// Compact advertisements must fit after expansion, independently of client
|
||||
// reference handling. Canonical server-side validation remains unchanged.
|
||||
|
||||
/** Annotation keys that carry guidance but no validation semantics. */
|
||||
const annotationKeys = new Set(["description", "examples", "default", "title"]);
|
||||
|
||||
/** Keys whose values are literal data, never nested schemas to rewrite. */
|
||||
const dataKeys = new Set(["const", "enum", "examples", "default"]);
|
||||
const definitionKeys = new Set(["$defs", "definitions"]);
|
||||
// An empty schema accepts every JSON value. Kimi otherwise infers string.
|
||||
const jsonValueTypes = [
|
||||
"null",
|
||||
"boolean",
|
||||
"object",
|
||||
"array",
|
||||
"number",
|
||||
"string",
|
||||
];
|
||||
|
||||
export type AdvertisedInputSchemaPresentation = "inline" | "reduced";
|
||||
|
||||
export interface CompactedInputSchema {
|
||||
readonly schema: Record<string, unknown>;
|
||||
readonly presentation: AdvertisedInputSchemaPresentation;
|
||||
}
|
||||
|
||||
const isSchemaObject = (value: unknown): value is Record<string, unknown> =>
|
||||
typeof value === "object" && value !== null && !Array.isArray(value);
|
||||
|
||||
const serializedBytes = (value: unknown): number =>
|
||||
Buffer.byteLength(JSON.stringify(value), "utf8");
|
||||
|
||||
/** Rewrite every schema-valued position; literal data stays opaque. */
|
||||
const rebuildSchemaPositions = (
|
||||
node: unknown,
|
||||
rebuild: (child: unknown) => unknown,
|
||||
): unknown => {
|
||||
if (Array.isArray(node)) return node.map((child) => rebuild(child));
|
||||
if (!isSchemaObject(node)) return node;
|
||||
const out: Record<string, unknown> = {};
|
||||
for (const [key, value] of Object.entries(node)) {
|
||||
if (dataKeys.has(key)) {
|
||||
out[key] = value;
|
||||
continue;
|
||||
}
|
||||
if (schemaMaps.has(key) && isSchemaObject(value)) {
|
||||
out[key] = Object.fromEntries(
|
||||
Object.entries(value).map(([name, child]) => [name, rebuild(child)]),
|
||||
);
|
||||
continue;
|
||||
}
|
||||
if (schemaArrays.has(key) && Array.isArray(value)) {
|
||||
out[key] = value.map((child) => rebuild(child));
|
||||
continue;
|
||||
}
|
||||
if (schemaChildren.has(key)) {
|
||||
out[key] = Array.isArray(value)
|
||||
? value.map((child) => rebuild(child))
|
||||
: rebuild(value);
|
||||
continue;
|
||||
}
|
||||
out[key] = value;
|
||||
}
|
||||
return out;
|
||||
};
|
||||
|
||||
/**
|
||||
* Drop annotation prose from every subschema while preserving all validation
|
||||
* keywords. The root keeps its own description because the root-union
|
||||
* projection encodes accepted input groups there; the root's echoed literal
|
||||
* examples leave with the rest of the annotation prose.
|
||||
*/
|
||||
const stripAnnotations = (
|
||||
schema: Record<string, unknown>,
|
||||
): Record<string, unknown> => {
|
||||
const rebuilt = rebuildSchemaPositions(
|
||||
schema,
|
||||
stripChildAnnotations,
|
||||
) as Record<string, unknown>;
|
||||
const out: Record<string, unknown> = {};
|
||||
for (const [key, value] of Object.entries(rebuilt)) {
|
||||
if (annotationKeys.has(key) && key !== "description") continue;
|
||||
out[key] = value;
|
||||
}
|
||||
return out;
|
||||
};
|
||||
|
||||
const stripChildAnnotations = (child: unknown): unknown => {
|
||||
const rebuilt = rebuildSchemaPositions(child, stripChildAnnotations);
|
||||
if (!isSchemaObject(rebuilt)) return rebuilt;
|
||||
const kept: Record<string, unknown> = {};
|
||||
for (const [key, value] of Object.entries(rebuilt))
|
||||
if (!annotationKeys.has(key)) kept[key] = value;
|
||||
return kept;
|
||||
};
|
||||
|
||||
const resolveLocalReference = (
|
||||
root: Record<string, unknown>,
|
||||
reference: string,
|
||||
): unknown => {
|
||||
if (reference === "#") return root;
|
||||
if (!reference.startsWith("#/")) return undefined;
|
||||
let value: unknown = root;
|
||||
for (const token of reference.slice(2).split("/")) {
|
||||
const key = token.replaceAll("~1", "/").replaceAll("~0", "~");
|
||||
if (!isSchemaObject(value) || !Object.hasOwn(value, key)) return undefined;
|
||||
value = value[key];
|
||||
}
|
||||
return value;
|
||||
};
|
||||
|
||||
/** Stop serialization at the budget before allocating an expanded schema. */
|
||||
const inlineWithinBudget = (
|
||||
root: Record<string, unknown>,
|
||||
budgetBytes: number,
|
||||
): Record<string, unknown> | undefined => {
|
||||
let resolved = true;
|
||||
function* serialize(
|
||||
value: unknown,
|
||||
position: "schema" | "map" | "data",
|
||||
active: ReadonlySet<string>,
|
||||
): Generator<string> {
|
||||
if (Array.isArray(value)) {
|
||||
yield "[";
|
||||
for (const [index, child] of value.entries()) {
|
||||
if (index > 0) yield ",";
|
||||
yield* serialize(child, position, active);
|
||||
}
|
||||
yield "]";
|
||||
} else if (isSchemaObject(value)) {
|
||||
if (position === "schema" && Object.keys(value).length === 0) {
|
||||
yield JSON.stringify({ type: jsonValueTypes });
|
||||
return;
|
||||
}
|
||||
if (position === "schema" && typeof value.$ref === "string") {
|
||||
const target = resolveLocalReference(root, value.$ref);
|
||||
if (target === undefined || active.has(value.$ref)) {
|
||||
resolved = false;
|
||||
return;
|
||||
}
|
||||
const { $ref, ...siblings } = value;
|
||||
// Reference siblings are independent constraints in draft 2020-12.
|
||||
// Merging their keys into the target would overwrite validation rules.
|
||||
yield* serialize(
|
||||
Object.keys(siblings).length === 0
|
||||
? target
|
||||
: { allOf: [target, siblings] },
|
||||
"schema",
|
||||
new Set([...active, $ref]),
|
||||
);
|
||||
return;
|
||||
}
|
||||
yield "{";
|
||||
let first = true;
|
||||
for (const [key, child] of Object.entries(value)) {
|
||||
if (position === "schema" && definitionKeys.has(key)) continue;
|
||||
if (!first) yield ",";
|
||||
first = false;
|
||||
yield JSON.stringify(key) + ":";
|
||||
const childPosition =
|
||||
position === "map"
|
||||
? "schema"
|
||||
: position === "schema" && schemaMaps.has(key)
|
||||
? "map"
|
||||
: position === "schema" &&
|
||||
(schemaArrays.has(key) || schemaChildren.has(key))
|
||||
? "schema"
|
||||
: "data";
|
||||
yield* serialize(child, childPosition, active);
|
||||
}
|
||||
yield "}";
|
||||
} else {
|
||||
yield JSON.stringify(value);
|
||||
}
|
||||
}
|
||||
const chunks: string[] = [];
|
||||
let bytes = 0;
|
||||
for (const chunk of serialize(root, "schema", new Set())) {
|
||||
bytes += Buffer.byteLength(chunk, "utf8");
|
||||
if (bytes > budgetBytes || !resolved) return undefined;
|
||||
chunks.push(chunk);
|
||||
}
|
||||
return resolved
|
||||
? (JSON.parse(chunks.join("")) as Record<string, unknown>)
|
||||
: undefined;
|
||||
};
|
||||
|
||||
/** Preserve every possible type, including referenced and nullable unions. */
|
||||
const detectedTypes = (
|
||||
schema: unknown,
|
||||
root: Record<string, unknown>,
|
||||
active: ReadonlySet<string> = new Set(),
|
||||
): readonly string[] | undefined => {
|
||||
if (!isSchemaObject(schema)) return undefined;
|
||||
if (typeof schema.$ref === "string") {
|
||||
if (active.has(schema.$ref)) return undefined;
|
||||
return detectedTypes(
|
||||
resolveLocalReference(root, schema.$ref),
|
||||
root,
|
||||
new Set([...active, schema.$ref]),
|
||||
);
|
||||
}
|
||||
if (typeof schema.type === "string") return [schema.type];
|
||||
if (
|
||||
Array.isArray(schema.type) &&
|
||||
schema.type.every((type) => typeof type === "string")
|
||||
)
|
||||
return schema.type;
|
||||
const alternatives = Array.isArray(schema.anyOf)
|
||||
? schema.anyOf
|
||||
: Array.isArray(schema.oneOf)
|
||||
? schema.oneOf
|
||||
: [];
|
||||
if (alternatives.length === 0) return undefined;
|
||||
const types = alternatives.map((branch) =>
|
||||
detectedTypes(branch, root, active),
|
||||
);
|
||||
if (types.some((type) => type === undefined)) return undefined;
|
||||
return [...new Set(types.flatMap((type) => type ?? []))];
|
||||
};
|
||||
|
||||
const REDUCED_ANNOTATION =
|
||||
"Advertised in reduced form by the compact input schema profile; the server still validates complete canonical inputs.";
|
||||
|
||||
/**
|
||||
* Present only the root property shape with per-property guidance. Used for
|
||||
* schemas whose validation structure alone exceeds the compact budget; the
|
||||
* canonical schema still rejects malformed calls server-side.
|
||||
*/
|
||||
const reduceToShallowProperties = (
|
||||
schema: Record<string, unknown>,
|
||||
): Record<string, unknown> => {
|
||||
const properties = isSchemaObject(schema.properties) ? schema.properties : {};
|
||||
const description = [
|
||||
typeof schema.description === "string" ? schema.description : undefined,
|
||||
REDUCED_ANNOTATION,
|
||||
]
|
||||
.filter((text) => text !== undefined)
|
||||
.join(" ");
|
||||
const $schema =
|
||||
typeof schema.$schema === "string" ? { $schema: schema.$schema } : {};
|
||||
const shallow = Object.fromEntries(
|
||||
Object.entries(properties).map(([name, value]) => {
|
||||
const types = detectedTypes(value, schema);
|
||||
const type = types?.length === 1 ? types[0] : types;
|
||||
const description =
|
||||
isSchemaObject(value) && typeof value.description === "string"
|
||||
? { description: value.description }
|
||||
: {};
|
||||
return [
|
||||
name,
|
||||
{ ...(type === undefined ? {} : { type }), ...description },
|
||||
];
|
||||
}),
|
||||
);
|
||||
return {
|
||||
...$schema,
|
||||
type: "object",
|
||||
description,
|
||||
properties: shallow,
|
||||
...(Array.isArray(schema.required)
|
||||
? { required: [...schema.required] }
|
||||
: {}),
|
||||
...(typeof schema.minProperties === "number"
|
||||
? { minProperties: schema.minProperties }
|
||||
: {}),
|
||||
...(isSchemaObject(schema.dependentRequired)
|
||||
? { dependentRequired: { ...schema.dependentRequired } }
|
||||
: {}),
|
||||
...(schema.additionalProperties === false
|
||||
? { additionalProperties: false }
|
||||
: {}),
|
||||
};
|
||||
};
|
||||
|
||||
/**
|
||||
* Render an advertised input schema within a per-schema byte budget for
|
||||
* providers that validate the literal serialized schema size. Validation
|
||||
* keywords and the canonical contract are preserved; only
|
||||
* annotation prose and, for structurally oversized schemas, nested
|
||||
* validation detail leave the advertised form.
|
||||
*/
|
||||
export const compactAdvertisedInputSchema = (
|
||||
schema: Record<string, unknown>,
|
||||
budgetBytes: number,
|
||||
): CompactedInputSchema => {
|
||||
const inline = inlineWithinBudget(stripAnnotations(schema), budgetBytes);
|
||||
if (inline !== undefined) return { schema: inline, presentation: "inline" };
|
||||
const reduced = reduceToShallowProperties(schema);
|
||||
if (serializedBytes(reduced) <= budgetBytes)
|
||||
return { schema: reduced, presentation: "reduced" };
|
||||
// A future contract with an extreme root-property count still advertises
|
||||
// an honest, budget-fitting shape instead of an oversized one.
|
||||
return {
|
||||
schema: reduceToShallowProperties({
|
||||
type: "object",
|
||||
description:
|
||||
(typeof schema.description === "string"
|
||||
? `${schema.description} `
|
||||
: "") + REDUCED_ANNOTATION,
|
||||
}),
|
||||
presentation: "reduced",
|
||||
};
|
||||
};
|
||||
@@ -1,6 +1,6 @@
|
||||
import { isDeepStrictEqual } from "node:util";
|
||||
|
||||
const schemaMaps = new Set([
|
||||
export const schemaMaps = new Set([
|
||||
"$defs",
|
||||
"definitions",
|
||||
"properties",
|
||||
@@ -8,8 +8,8 @@ const schemaMaps = new Set([
|
||||
"dependentSchemas",
|
||||
"dependencies",
|
||||
]);
|
||||
const schemaArrays = new Set(["allOf", "anyOf", "oneOf", "prefixItems"]);
|
||||
const schemaChildren = new Set([
|
||||
export const schemaArrays = new Set(["allOf", "anyOf", "oneOf", "prefixItems"]);
|
||||
export const schemaChildren = new Set([
|
||||
"items",
|
||||
"additionalItems",
|
||||
"contains",
|
||||
|
||||
@@ -95,6 +95,46 @@ export const toolOutputSchemaWithMetadata = <Contract extends ToolContract>(
|
||||
},
|
||||
);
|
||||
|
||||
/**
|
||||
* Apply a wire-presentation transform to one schema's advertised input
|
||||
* projection. The wrapper preserves the schema's own parser and standard
|
||||
* metadata, so both Zod registrations and plain standard schemas keep their
|
||||
* canonical validation while only the advertised JSON changes.
|
||||
*/
|
||||
export const transformAdvertisedInputJsonSchema = <Schema>(
|
||||
schema: Schema,
|
||||
transform: (projected: Record<string, unknown>) => Record<string, unknown>,
|
||||
): Schema => {
|
||||
const standard = Reflect.get(schema as object, "~standard");
|
||||
if (!isSchemaRecord(standard)) return schema;
|
||||
const jsonSchema = Reflect.get(standard, "jsonSchema");
|
||||
if (!isSchemaRecord(jsonSchema)) return schema;
|
||||
const project = Reflect.get(jsonSchema, "input");
|
||||
if (typeof project !== "function") return schema;
|
||||
const byTarget = new Map<string, Record<string, unknown>>();
|
||||
const transformedInput = (options: unknown): unknown => {
|
||||
const target = Reflect.get(options as object, "target");
|
||||
if (typeof target === "string") {
|
||||
const cached = byTarget.get(target);
|
||||
if (cached !== undefined) return cached;
|
||||
}
|
||||
const projected = (project as (options: unknown) => unknown)(options);
|
||||
const transformed = isSchemaRecord(projected)
|
||||
? transform(projected)
|
||||
: projected;
|
||||
if (typeof target === "string" && isSchemaRecord(transformed))
|
||||
byTarget.set(target, transformed);
|
||||
return transformed;
|
||||
};
|
||||
return {
|
||||
...(schema as object),
|
||||
"~standard": {
|
||||
...standard,
|
||||
jsonSchema: { ...jsonSchema, input: transformedInput },
|
||||
},
|
||||
} as Schema;
|
||||
};
|
||||
|
||||
const hasRelativeSchemaReference = (
|
||||
schema: Record<string, unknown>,
|
||||
): boolean => {
|
||||
@@ -164,6 +204,9 @@ const memoizeByTarget = (
|
||||
};
|
||||
};
|
||||
|
||||
const isSchemaRecord = (value: unknown): value is Record<string, unknown> =>
|
||||
typeof value === "object" && value !== null && !Array.isArray(value);
|
||||
|
||||
const fallbackPropertyDescription = (property: string): string | undefined => {
|
||||
const explicit = Object.hasOwn(PROPERTY_DESCRIPTIONS, property)
|
||||
? PROPERTY_DESCRIPTIONS[property]
|
||||
|
||||
@@ -98,6 +98,7 @@ import type { ArtifactInventoryPartialObservation } from "./artifactPartialObser
|
||||
import type { BrowserScenarioPartialObservation } from "./browserScenarioCapture.js";
|
||||
import type { Evidence } from "./evidence.js";
|
||||
import type { AndroidPartialObservation } from "./android/androidPartialObservation.js";
|
||||
import type { FirmwarePartialObservation } from "./firmware/firmwareAnalysis.js";
|
||||
|
||||
/** Provider-neutral evidence collected before a typed analysis failure. */
|
||||
export type AnalysisPartialObservation =
|
||||
@@ -109,4 +110,5 @@ export type AnalysisPartialObservation =
|
||||
| JavaScriptRuntimeObservation
|
||||
| ArtifactInventoryPartialObservation
|
||||
| Evidence
|
||||
| AndroidPartialObservation;
|
||||
| AndroidPartialObservation
|
||||
| FirmwarePartialObservation;
|
||||
|
||||
@@ -11,6 +11,7 @@ import {
|
||||
bridgeCandidateCoverageSchema,
|
||||
projectCartesianCandidates,
|
||||
} from "../bridgeCandidateProjection.js";
|
||||
import { nativeRuntimeConvention } from "../nativeConvention.js";
|
||||
|
||||
const evidenceIdSchema = prefixedDigestSchema("ev");
|
||||
const pathSchema = z.string().min(1);
|
||||
@@ -176,7 +177,6 @@ const classify = (all: readonly Component[]) => ({
|
||||
});
|
||||
|
||||
const runtimeFamilies = (all: readonly Component[]) => {
|
||||
const paths = all.map(({ path }) => path.toLowerCase());
|
||||
const families = new Set<
|
||||
AndroidApplicationProjectionResult["runtime_families"][number]
|
||||
>();
|
||||
@@ -187,15 +187,10 @@ const runtimeFamilies = (all: readonly Component[]) => {
|
||||
if (all.some(({ format }) => format === "elf")) families.add("native");
|
||||
if (all.some(({ format }) => format === "javascript-bundle"))
|
||||
families.add("javascript");
|
||||
if (
|
||||
paths.some(
|
||||
(path) => path.includes("reactnative") || path.includes("hermes"),
|
||||
)
|
||||
)
|
||||
families.add("react-native");
|
||||
if (paths.some((path) => path.includes("libflutter.so")))
|
||||
families.add("flutter");
|
||||
if (paths.some((path) => path.includes("libunity.so"))) families.add("unity");
|
||||
for (const { path } of all) {
|
||||
const convention = nativeRuntimeConvention(path);
|
||||
if (convention !== null) families.add(convention);
|
||||
}
|
||||
return [...families].sort(compare);
|
||||
};
|
||||
|
||||
@@ -216,12 +211,12 @@ const bridgeCandidates = (
|
||||
const bridgeBasis = (
|
||||
path: string,
|
||||
): AndroidApplicationProjectionResult["bridge_candidates"][number]["basis"] => {
|
||||
const lower = path.toLowerCase();
|
||||
if (lower.includes("react") || lower.includes("hermes"))
|
||||
return "react-native-convention";
|
||||
if (lower.includes("flutter")) return "flutter-convention";
|
||||
if (lower.includes("unity")) return "unity-convention";
|
||||
if (/lib\/[^/]+\/lib[^/]+\.so$/u.test(lower)) return "jni-library-convention";
|
||||
const convention = nativeRuntimeConvention(path);
|
||||
if (convention === "react-native") return "react-native-convention";
|
||||
if (convention === "flutter") return "flutter-convention";
|
||||
if (convention === "unity") return "unity-convention";
|
||||
if (/lib\/[^/]+\/lib[^/]+\.so$/u.test(path.toLowerCase()))
|
||||
return "jni-library-convention";
|
||||
return "managed-and-native-content";
|
||||
};
|
||||
|
||||
|
||||
@@ -572,3 +572,80 @@ describe("Apple application bridge candidates", () => {
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("Apple path conventions", () => {
|
||||
it("recognizes anatomy names by English case and reports the archive spelling", () => {
|
||||
const result = project(
|
||||
inventoryEvidence("directory", "Fixture.app", [
|
||||
{ path: "contents/info.plist" },
|
||||
macho("contents/macos/Fixture"),
|
||||
{ path: "contents/_codesignature/coderesources" },
|
||||
]),
|
||||
);
|
||||
expect(result.bundles[0]).toMatchObject({
|
||||
layout: "macos-deep",
|
||||
info_plist_path: "contents/info.plist",
|
||||
executable_candidates: ["contents/macos/Fixture"],
|
||||
signing_paths: ["contents/_codesignature/coderesources"],
|
||||
});
|
||||
expect(result.components.bundle_metadata.map(({ path }) => path)).toEqual([
|
||||
"contents/info.plist",
|
||||
]);
|
||||
expect(result.components.signing.map(({ path }) => path)).toEqual([
|
||||
"contents/_codesignature/coderesources",
|
||||
]);
|
||||
});
|
||||
|
||||
it("limits runtime families and bridge bases to path segments", () => {
|
||||
const result = project(
|
||||
inventoryEvidence("ipa", "Fixture.ipa", [
|
||||
{ path: "Payload/Fixture.app/Info.plist" },
|
||||
macho("Payload/Fixture.app/Fixture"),
|
||||
macho("Payload/Fixture.app/Frameworks/libfoo.dylib"),
|
||||
macho("SwiftSupport/libswiftCore.dylib"),
|
||||
macho("Payload/Fixture.app/Frameworks/MyApp.framework/MyApp"),
|
||||
macho("Payload/Fixture.app/community/libcommunity.so"),
|
||||
{
|
||||
path: "Payload/Fixture.app/main.js",
|
||||
format: "javascript-bundle",
|
||||
},
|
||||
]),
|
||||
);
|
||||
expect(result.runtime_families).toEqual(
|
||||
expect.arrayContaining(["javascript", "native", "swift-objective-c"]),
|
||||
);
|
||||
expect(result.runtime_families).not.toContain("flutter");
|
||||
expect(result.runtime_families).not.toContain("unity");
|
||||
expect(result.runtime_families).not.toContain("react-native");
|
||||
expect(
|
||||
result.bridge_candidates.every(
|
||||
({ basis }) => basis === "javascript-and-native-content",
|
||||
),
|
||||
).toBe(true);
|
||||
expect(result.limitations).toEqual(
|
||||
expect.arrayContaining([
|
||||
"Runtime families are inferred from inventory formats and paths.",
|
||||
"A bridge basis is inferred from the native path and is repeated for every managed component.",
|
||||
]),
|
||||
);
|
||||
});
|
||||
|
||||
it("recognizes framework segments and does not treat a bare dylib as Swift", () => {
|
||||
const result = project(
|
||||
inventoryEvidence("ipa", "Fixture.ipa", [
|
||||
{ path: "Payload/Fixture.app/Info.plist" },
|
||||
macho("Payload/Fixture.app/Fixture"),
|
||||
macho("Payload/Fixture.app/Frameworks/Flutter.framework/Flutter"),
|
||||
macho(
|
||||
"Payload/Fixture.app/Frameworks/UnityFramework.framework/UnityFramework",
|
||||
),
|
||||
macho("Payload/Fixture.app/Frameworks/hermes.framework/hermes"),
|
||||
macho("Payload/Fixture.app/Frameworks/libfoo.dylib"),
|
||||
]),
|
||||
);
|
||||
expect(result.runtime_families).toEqual(
|
||||
expect.arrayContaining(["flutter", "unity", "react-native", "native"]),
|
||||
);
|
||||
expect(result.runtime_families).not.toContain("swift-objective-c");
|
||||
});
|
||||
});
|
||||
|
||||
@@ -4,6 +4,8 @@ import canonicalize from "canonicalize";
|
||||
import { z } from "zod";
|
||||
|
||||
import {
|
||||
anatomyNameIs,
|
||||
anatomyTailMatches,
|
||||
appleBundleSchema,
|
||||
appleComponentSchema,
|
||||
applicationRoots,
|
||||
@@ -14,6 +16,10 @@ import {
|
||||
platformsOf,
|
||||
type AppleInventoryEntry,
|
||||
} from "./appleBundleAnatomy.js";
|
||||
import {
|
||||
isSwiftRuntimeLibrary,
|
||||
nativeRuntimeConvention,
|
||||
} from "../nativeConvention.js";
|
||||
import { parseArtifactInventoryEvidence } from "../artifactInventoryEvidence.js";
|
||||
import { evidenceSchema } from "../evidence.js";
|
||||
import { digestSchema } from "../digests.js";
|
||||
@@ -239,19 +245,20 @@ const classifyComponents = (
|
||||
) => {
|
||||
const withinApp = (path: string): boolean =>
|
||||
roots.some((root) => isWithin(path, root));
|
||||
const anatomy = (pattern: RegExp): Component[] =>
|
||||
const matching = (predicate: (path: string) => boolean): Component[] =>
|
||||
entries.flatMap(({ path, kind, component }) =>
|
||||
kind === "file" &&
|
||||
component !== undefined &&
|
||||
!isSidecar(path) &&
|
||||
withinApp(path) &&
|
||||
pattern.test(path)
|
||||
predicate(path)
|
||||
? [component]
|
||||
: [],
|
||||
);
|
||||
const lastSegment = (path: string): string => path.split("/").at(-1) ?? "";
|
||||
return {
|
||||
bundle_metadata: all.filter(({ path }) =>
|
||||
/(?:^|\/)Info\.plist$/u.test(path),
|
||||
anatomyNameIs(lastSegment(path), "Info.plist"),
|
||||
),
|
||||
executables: all.filter(
|
||||
({ path, format }) => withinApp(path) && isMachOFormat(format),
|
||||
@@ -259,23 +266,41 @@ const classifyComponents = (
|
||||
frameworks: all.filter(({ path }) => /\.framework\//u.test(path)),
|
||||
native_libraries: all.filter(({ path }) => /\.(?:dylib|so)$/iu.test(path)),
|
||||
javascript: all.filter(({ format }) => format === "javascript-bundle"),
|
||||
signing: all.filter(({ path }) =>
|
||||
/(?:^|\/)(?:embedded\.(?:mobileprovision|provisionprofile)|_CodeSignature\/CodeResources)$/u.test(
|
||||
path,
|
||||
),
|
||||
signing: all.filter(
|
||||
({ path }) =>
|
||||
anatomyTailMatches(path, ["_CodeSignature", "CodeResources"]) ||
|
||||
anatomyNameIs(lastSegment(path), "embedded.mobileprovision") ||
|
||||
anatomyNameIs(lastSegment(path), "embedded.provisionprofile"),
|
||||
),
|
||||
privileged_helpers: anatomy(
|
||||
/(?:^|\/)Contents\/Library\/LaunchServices\/[^/]+$/u,
|
||||
privileged_helpers: matching((path) =>
|
||||
anatomyTailMatches(path, ["Contents", "Library", "LaunchServices", "*"]),
|
||||
),
|
||||
launchd_plists: anatomy(
|
||||
/(?:^|\/)Contents\/Library\/Launch(?:Agents|Daemons)\/[^/]+\.plist$/u,
|
||||
launchd_plists: matching(
|
||||
(path) =>
|
||||
path.endsWith(".plist") &&
|
||||
(anatomyTailMatches(path, [
|
||||
"Contents",
|
||||
"Library",
|
||||
"LaunchAgents",
|
||||
"*",
|
||||
]) ||
|
||||
anatomyTailMatches(path, [
|
||||
"Contents",
|
||||
"Library",
|
||||
"LaunchDaemons",
|
||||
"*",
|
||||
])),
|
||||
).map((component) => ({
|
||||
...component,
|
||||
domain: component.path.includes("/LaunchDaemons/")
|
||||
domain: component.path
|
||||
.split("/")
|
||||
.some((segment) => anatomyNameIs(segment, "LaunchDaemons"))
|
||||
? ("daemon" as const)
|
||||
: ("agent" as const),
|
||||
})),
|
||||
helpers: anatomy(/(?:^|\/)Contents\/Helpers\/[^/]+$/u),
|
||||
helpers: matching((path) =>
|
||||
anatomyTailMatches(path, ["Contents", "Helpers", "*"]),
|
||||
),
|
||||
};
|
||||
};
|
||||
|
||||
@@ -285,7 +310,6 @@ const deduplicateComponents = (values: readonly Component[]): Component[] =>
|
||||
);
|
||||
|
||||
const identifyRuntimeFamilies = (all: readonly Component[]) => {
|
||||
const paths = all.map(({ path }) => path.toLowerCase());
|
||||
const families = new Set<
|
||||
z.infer<
|
||||
typeof appleApplicationProjectionResultSchema
|
||||
@@ -293,28 +317,14 @@ const identifyRuntimeFamilies = (all: readonly Component[]) => {
|
||||
>();
|
||||
if (all.some(({ format }) => ["mach-o", "mach-o-universal"].includes(format)))
|
||||
families.add("native");
|
||||
if (paths.some((path) => /\.(?:dylib|framework\/[^/]+)$/u.test(path)))
|
||||
if (all.some(({ path }) => isSwiftRuntimeLibrary(path)))
|
||||
families.add("swift-objective-c");
|
||||
if (all.some(({ format }) => format === "javascript-bundle"))
|
||||
families.add("javascript");
|
||||
if (
|
||||
paths.some(
|
||||
(path) =>
|
||||
path.includes("reactnative") ||
|
||||
path.includes("react.framework") ||
|
||||
path.includes("hermes"),
|
||||
)
|
||||
)
|
||||
families.add("react-native");
|
||||
if (
|
||||
paths.some(
|
||||
(path) =>
|
||||
path.includes("flutter.framework") || path.includes("app.framework"),
|
||||
)
|
||||
)
|
||||
families.add("flutter");
|
||||
if (paths.some((path) => path.includes("unityframework.framework")))
|
||||
families.add("unity");
|
||||
for (const { path } of all) {
|
||||
const convention = nativeRuntimeConvention(path);
|
||||
if (convention !== null) families.add(convention);
|
||||
}
|
||||
return [...families].sort(compare);
|
||||
};
|
||||
|
||||
@@ -361,11 +371,10 @@ const bridgeBasis = (
|
||||
| "react-native-convention"
|
||||
| "flutter-convention"
|
||||
| "unity-convention" => {
|
||||
const lower = path.toLowerCase();
|
||||
if (lower.includes("react")) return "react-native-convention";
|
||||
if (lower.includes("flutter") || lower.includes("app.framework"))
|
||||
return "flutter-convention";
|
||||
if (lower.includes("unity")) return "unity-convention";
|
||||
const convention = nativeRuntimeConvention(path);
|
||||
if (convention === "react-native") return "react-native-convention";
|
||||
if (convention === "flutter") return "flutter-convention";
|
||||
if (convention === "unity") return "unity-convention";
|
||||
return "javascript-and-native-content";
|
||||
};
|
||||
|
||||
@@ -422,7 +431,9 @@ const projectionLimitations = (facts: {
|
||||
: []),
|
||||
"Bundle roles follow path conventions. Read each info_plist_path with inspect_plist for CFBundleExecutable, identifiers, and declared services.",
|
||||
"Bundle identifiers and signing claims require dedicated plist and CMS parsing; this projection reports only exact paths and hashes.",
|
||||
"Runtime families are inferred from inventory formats and paths.",
|
||||
"Bridge candidates are path-based hypotheses, not observed runtime calls.",
|
||||
"A bridge basis is inferred from the native path and is repeated for every managed component.",
|
||||
];
|
||||
|
||||
const compare = (left: string, right: string): number =>
|
||||
|
||||
@@ -67,6 +67,50 @@ export const isSidecar = (path: string): boolean => {
|
||||
export const isWithin = (path: string, root: string): boolean =>
|
||||
root === "." || path === root || path.startsWith(`${root}/`);
|
||||
|
||||
/**
|
||||
* English (en-US) case fold used only for Apple's fixed anatomy vocabulary.
|
||||
* Reported paths keep the archive spelling. Inventory identity stays case-sensitive.
|
||||
*/
|
||||
export const anatomyNameIs = (actual: string, expected: string): boolean =>
|
||||
actual.toLocaleLowerCase("en-US") === expected.toLocaleLowerCase("en-US");
|
||||
|
||||
/** Directory entry that folds to one vocabulary word. The exact spelling wins. */
|
||||
export const anatomyChild = (
|
||||
names: ReadonlySet<string> | undefined,
|
||||
expected: string,
|
||||
): string | null => {
|
||||
if (names === undefined) return null;
|
||||
let exact: string | undefined;
|
||||
let folded: string | undefined;
|
||||
for (const name of names) {
|
||||
if (name === expected) exact = name;
|
||||
else if (
|
||||
anatomyNameIs(name, expected) &&
|
||||
(folded === undefined || compare(name, folded) < 0)
|
||||
)
|
||||
folded = name;
|
||||
}
|
||||
return exact ?? folded ?? null;
|
||||
};
|
||||
|
||||
/**
|
||||
* Whether `path`'s tail matches anatomy vocabulary. `*` matches one segment
|
||||
* that is not itself a vocabulary comparison.
|
||||
*/
|
||||
export const anatomyTailMatches = (
|
||||
path: string,
|
||||
expected: readonly string[],
|
||||
): boolean => {
|
||||
const actual = path.split("/");
|
||||
if (actual.length < expected.length) return false;
|
||||
const tail = actual.slice(-expected.length);
|
||||
return expected.every((name, index) => {
|
||||
const segment = tail[index];
|
||||
if (segment === undefined || segment.length === 0) return false;
|
||||
return name === "*" || anatomyNameIs(segment, name);
|
||||
});
|
||||
};
|
||||
|
||||
/**
|
||||
* Outermost iOS (`Payload/X.app`) and macOS (`X.app/Contents/…`) applications.
|
||||
* An inventoried `.app` directory is its own root, `.`.
|
||||
@@ -82,15 +126,20 @@ export const applicationRoots = (
|
||||
const found = new Set<string>();
|
||||
for (const { path } of entries) {
|
||||
if (isSidecar(path)) continue;
|
||||
const ios = /^(Payload\/[^/]+\.app)(?:\/|$)/u.exec(path);
|
||||
if (ios?.[1] !== undefined) {
|
||||
found.add(ios[1]);
|
||||
const segments = path.split("/");
|
||||
const app = segments[1];
|
||||
if (
|
||||
anatomyNameIs(segments[0] ?? "", "Payload") &&
|
||||
app !== undefined &&
|
||||
/\.app$/iu.test(app)
|
||||
) {
|
||||
found.add(`${segments[0]}/${app}`);
|
||||
continue;
|
||||
}
|
||||
const segments = path.split("/");
|
||||
const index = segments.findIndex(
|
||||
(segment, position) =>
|
||||
/\.app$/iu.test(segment) && segments[position + 1] === "Contents",
|
||||
/\.app$/iu.test(segment) &&
|
||||
anatomyNameIs(segments[position + 1] ?? "", "Contents"),
|
||||
);
|
||||
if (index >= 0) found.add(segments.slice(0, index + 1).join("/"));
|
||||
}
|
||||
@@ -186,8 +235,8 @@ const enclosingBundle = (
|
||||
|
||||
const bundleLayout = (bundle: string, tree: Tree): Bundle["layout"] => {
|
||||
const names = tree.children.get(directoryOf(bundle));
|
||||
if (names?.has("Contents") === true) return "macos-deep";
|
||||
if (names?.has("Versions") === true) return "versioned-framework";
|
||||
if (anatomyChild(names, "Contents") !== null) return "macos-deep";
|
||||
if (anatomyChild(names, "Versions") !== null) return "versioned-framework";
|
||||
return "shallow";
|
||||
};
|
||||
|
||||
@@ -201,13 +250,18 @@ const bundleRole = (bundle: string): Bundle["role"] => {
|
||||
if (name.endsWith(".systemextension")) return "system-extension";
|
||||
if (name.endsWith(".dext")) return "driver-extension";
|
||||
if (name.endsWith(".bundle")) {
|
||||
if (container === "PlugIns") return "plug-in";
|
||||
return container === "Resources" ? "resource-bundle" : "bundle";
|
||||
if (anatomyNameIs(container ?? "", "PlugIns")) return "plug-in";
|
||||
return anatomyNameIs(container ?? "", "Resources")
|
||||
? "resource-bundle"
|
||||
: "bundle";
|
||||
}
|
||||
if (!name.endsWith(".app")) return "plug-in";
|
||||
if (container === "LoginItems" && segments.at(-3) === "Library")
|
||||
if (
|
||||
anatomyNameIs(container ?? "", "LoginItems") &&
|
||||
anatomyNameIs(segments.at(-3) ?? "", "Library")
|
||||
)
|
||||
return "login-item";
|
||||
if (container === "Helpers") return "helper-application";
|
||||
if (anatomyNameIs(container ?? "", "Helpers")) return "helper-application";
|
||||
return "nested-application";
|
||||
};
|
||||
|
||||
@@ -225,18 +279,32 @@ const contentDirectories = (
|
||||
tree: Tree,
|
||||
): string[] => {
|
||||
const directory = directoryOf(bundle);
|
||||
if (layout === "macos-deep") return [joinPath(directory, "Contents")];
|
||||
if (layout === "macos-deep") {
|
||||
const contents = namedChild(tree, directory, "Contents");
|
||||
return contents === null ? [] : [contents];
|
||||
}
|
||||
if (layout === "shallow") return [directory];
|
||||
const versions = joinPath(directory, "Versions");
|
||||
const versions = namedChild(tree, directory, "Versions");
|
||||
if (versions === null) return [];
|
||||
// Only real version directories count: Current is a symlink, and files
|
||||
// such as .DS_Store are not versions.
|
||||
return [...(tree.children.get(versions) ?? [])]
|
||||
.filter((version) => version !== "Current")
|
||||
.filter((version) => !anatomyNameIs(version, "Current"))
|
||||
.map((version) => joinPath(versions, version))
|
||||
.filter((version) => tree.directories.has(version))
|
||||
.sort(compare);
|
||||
};
|
||||
|
||||
/** Child path using the archive's spelling of one anatomy name. */
|
||||
const namedChild = (
|
||||
tree: Tree,
|
||||
directory: string,
|
||||
expected: string,
|
||||
): string | null => {
|
||||
const name = anatomyChild(tree.children.get(directory), expected);
|
||||
return name === null ? null : joinPath(directory, name);
|
||||
};
|
||||
|
||||
const infoPlistPath = (
|
||||
bundle: string,
|
||||
layout: Bundle["layout"],
|
||||
@@ -246,17 +314,15 @@ const infoPlistPath = (
|
||||
// With several real versions, Versions/Current (a symlink outside the
|
||||
// inventory) decides which plist applies, so the plist stays unknown.
|
||||
if (directories.length !== 1) return null;
|
||||
const candidates = directories
|
||||
.map((directory) =>
|
||||
joinPath(
|
||||
directory,
|
||||
layout === "versioned-framework"
|
||||
? "Resources/Info.plist"
|
||||
: "Info.plist",
|
||||
),
|
||||
)
|
||||
.filter((path) => tree.files.has(path));
|
||||
return candidates.length === 1 ? (candidates[0] ?? null) : null;
|
||||
const directory = directories[0];
|
||||
if (directory === undefined) return null;
|
||||
const parent =
|
||||
layout === "versioned-framework"
|
||||
? namedChild(tree, directory, "Resources")
|
||||
: directory;
|
||||
if (parent === null) return null;
|
||||
const info = namedChild(tree, parent, "Info.plist");
|
||||
return info !== null && tree.files.has(info) ? info : null;
|
||||
};
|
||||
|
||||
const executableCandidates = (
|
||||
@@ -265,12 +331,15 @@ const executableCandidates = (
|
||||
tree: Tree,
|
||||
): string[] =>
|
||||
contentDirectories(bundle, layout, tree)
|
||||
.flatMap((directory) =>
|
||||
directFiles(
|
||||
layout === "macos-deep" ? joinPath(directory, "MacOS") : directory,
|
||||
tree,
|
||||
),
|
||||
)
|
||||
.flatMap((directory) => {
|
||||
const executableDirectory =
|
||||
layout === "macos-deep"
|
||||
? namedChild(tree, directory, "MacOS")
|
||||
: directory;
|
||||
return executableDirectory === null
|
||||
? []
|
||||
: directFiles(executableDirectory, tree);
|
||||
})
|
||||
.filter((path) =>
|
||||
MACH_O_FORMATS.includes(tree.files.get(path)?.format ?? ""),
|
||||
);
|
||||
@@ -281,12 +350,17 @@ const signingPaths = (
|
||||
tree: Tree,
|
||||
): string[] =>
|
||||
contentDirectories(bundle, layout, tree)
|
||||
.flatMap((directory) => [
|
||||
joinPath(directory, "_CodeSignature/CodeResources"),
|
||||
joinPath(directory, "embedded.provisionprofile"),
|
||||
joinPath(directory, "embedded.mobileprovision"),
|
||||
])
|
||||
.filter((path) => tree.files.has(path))
|
||||
.flatMap((directory) => {
|
||||
const signature = namedChild(tree, directory, "_CodeSignature");
|
||||
return [
|
||||
signature === null
|
||||
? null
|
||||
: namedChild(tree, signature, "CodeResources"),
|
||||
namedChild(tree, directory, "embedded.provisionprofile"),
|
||||
namedChild(tree, directory, "embedded.mobileprovision"),
|
||||
];
|
||||
})
|
||||
.filter((path): path is string => path !== null && tree.files.has(path))
|
||||
.sort(compare);
|
||||
|
||||
/** Platforms of application roots whose layout was observed, not assumed. */
|
||||
|
||||
@@ -13,6 +13,7 @@ import {
|
||||
} from "./artifactInventoryEvidence.js";
|
||||
import { prefixedDigestSchema } from "./../domain/digests.js";
|
||||
import { canonicalJson } from "./comparisonSemantics.js";
|
||||
import { ZIP_NON_ENTRY_TAIL_LIMITATION } from "./zipPackageFormat.js";
|
||||
|
||||
const evidenceIdSchema = prefixedDigestSchema("ev");
|
||||
const comparisonStatusSchema = z.enum([
|
||||
@@ -122,6 +123,11 @@ export const compareArtifacts = (
|
||||
],
|
||||
leftCovered && rightCovered,
|
||||
);
|
||||
const rootChange = rootHashChange(left.inventory, right.inventory, changes, [
|
||||
...left.evidence.map(({ evidence_id: id }) => id),
|
||||
...right.evidence.map(({ evidence_id: id }) => id),
|
||||
]);
|
||||
if (rootChange !== null) changes.push(rootChange);
|
||||
const summary = summarize(
|
||||
left.inventory,
|
||||
right.inventory,
|
||||
@@ -431,5 +437,38 @@ const summarize = (
|
||||
const hasCoverageLimitation = (limitations: readonly string[]): boolean =>
|
||||
limitations.some(
|
||||
(limitation) =>
|
||||
limitation !== ZIP_NON_ENTRY_TAIL_LIMITATION &&
|
||||
!/integrity contradiction\(s\) were recorded/u.test(limitation),
|
||||
);
|
||||
|
||||
const rootHashChange = (
|
||||
left: InventorySet,
|
||||
right: InventorySet,
|
||||
changes: readonly ArtifactChange[],
|
||||
evidenceLinks: readonly string[],
|
||||
): ArtifactChange | null => {
|
||||
if (left.manifest.root_sha256 === right.manifest.root_sha256) return null;
|
||||
if (
|
||||
changes.some(
|
||||
(change) =>
|
||||
change.logical_path === "." && change.dimensions.includes("content"),
|
||||
)
|
||||
)
|
||||
return null;
|
||||
const leftRoot = left.occurrences.find(
|
||||
(occurrence) => occurrence.logical_path === ".",
|
||||
);
|
||||
const rightRoot = right.occurrences.find(
|
||||
(occurrence) => occurrence.logical_path === ".",
|
||||
);
|
||||
return {
|
||||
classification: "changed",
|
||||
logical_path: ".",
|
||||
dimensions: ["content"],
|
||||
left_occurrence_id: leftRoot?.occurrence_id ?? null,
|
||||
right_occurrence_id: rightRoot?.occurrence_id ?? null,
|
||||
left_artifact_id: leftRoot?.artifact_id ?? null,
|
||||
right_artifact_id: rightRoot?.artifact_id ?? null,
|
||||
evidence_links: [...evidenceLinks],
|
||||
};
|
||||
};
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
import { posix } from "node:path";
|
||||
|
||||
import { digestCanonicalValue } from "./canonicalDigest.js";
|
||||
import type {
|
||||
ArtifactEdge,
|
||||
@@ -6,6 +8,29 @@ import type {
|
||||
IntegrityContradiction,
|
||||
} from "./artifactGraph.js";
|
||||
|
||||
/** NFC spelling of a logical archive path. Inventory and path joins share it. */
|
||||
export const logicalPathNfc = (value: string): string => value.normalize("NFC");
|
||||
|
||||
/**
|
||||
* Lexical join against an inventoried logical path. The result is NFC.
|
||||
* `"."` is the archive root. A path that escapes the root, or that contains
|
||||
* NUL, is left in a form the caller can report; `logicalPathEscapesRoot`
|
||||
* is then true.
|
||||
*/
|
||||
export const normalizeJoinedLogicalPath = (value: string): string => {
|
||||
if (value.includes("\0")) return value;
|
||||
return logicalPathNfc(
|
||||
posix.normalize(value.replaceAll("\\", "/")).replace(/^\.\//u, ""),
|
||||
);
|
||||
};
|
||||
|
||||
/** Whether a joined logical path leaves the archive root. */
|
||||
export const logicalPathEscapesRoot = (path: string): boolean =>
|
||||
path.includes("\0") ||
|
||||
path === ".." ||
|
||||
path.startsWith("../") ||
|
||||
path.startsWith("/");
|
||||
|
||||
/** Content-address an artifact independently of where it was observed. */
|
||||
export const artifactIdForContent = (sha256: string): string =>
|
||||
`art_${digestCanonicalValue({ sha256 }, "Artifact")}`;
|
||||
|
||||
@@ -6,7 +6,7 @@
|
||||
* Policy divergence (deliberate, not yet unified):
|
||||
* - Local artifact paths resolve against an inventoried container root and
|
||||
* report rejected | external | not-found. Confinement is lexical
|
||||
* (posix.normalize must stay inside the root).
|
||||
* (normalizeJoinedLogicalPath must stay inside the root).
|
||||
* - Live DOM destinations resolve against the document base URL and report
|
||||
* approved | outside_policy | unsupported against allowedOrigins
|
||||
* (see CdpCaptureDocuments.domDestination).
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import { z } from "zod";
|
||||
import { isAbsolute } from "node:path";
|
||||
import { digestSchema } from "../digests.js";
|
||||
import { jsonValueSchema } from "../jsonValue.js";
|
||||
import { jsonValueSchema, type JsonValue } from "../jsonValue.js";
|
||||
import { localPathStringSchema } from "../localPath.js";
|
||||
|
||||
const bytes = z.number().int().min(0).max(Number.MAX_SAFE_INTEGER);
|
||||
@@ -95,6 +95,12 @@ const chunk = z.strictObject({
|
||||
root_file_range: range.nullable(),
|
||||
});
|
||||
|
||||
/** Completed firmware result retained when private-runtime cleanup cannot be confirmed. */
|
||||
export interface FirmwarePartialObservation {
|
||||
readonly kind: "firmware";
|
||||
readonly result: JsonValue;
|
||||
}
|
||||
|
||||
/** Portable results retain reported boundaries, unknowns and extraction lineage. */
|
||||
export const firmwareResultSchemas = {
|
||||
inspect_firmware_regions: z.strictObject({
|
||||
|
||||
@@ -11,11 +11,13 @@ import {
|
||||
createJavaScriptApplicationGraph,
|
||||
createImmutableJavaScriptApplicationGraphSteps,
|
||||
isValidatedImmutableJavaScriptApplicationGraph,
|
||||
sealTransferredJavaScriptApplicationGraphSteps,
|
||||
} from "./javascriptApplicationGraph.js";
|
||||
import {
|
||||
createJavaScriptSemanticGraph,
|
||||
createImmutableJavaScriptSemanticGraphSteps,
|
||||
isValidatedImmutableJavaScriptSemanticGraph,
|
||||
sealTransferredJavaScriptSemanticGraphSteps,
|
||||
} from "./javascriptSemanticGraph.js";
|
||||
|
||||
const complete = <Value>(steps: Iterator<void, Value>): Value => {
|
||||
@@ -265,3 +267,60 @@ describe("owned JavaScript result bindings", () => {
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe("graphs transferred to their sealing steps", () => {
|
||||
it("seal the same validated graphs as the copying factories", () => {
|
||||
const input = example();
|
||||
const { graph_id: _applicationId, ...application } = input.graph;
|
||||
const { graph_id: _semanticId, ...semantic } = input.semantic_graph;
|
||||
const expectedApplication = createJavaScriptApplicationGraph(application);
|
||||
const expectedSemantic = createJavaScriptSemanticGraph(semantic);
|
||||
const sealedApplication = complete(
|
||||
sealTransferredJavaScriptApplicationGraphSteps(
|
||||
structuredClone(application),
|
||||
),
|
||||
);
|
||||
const sealedSemantic = complete(
|
||||
sealTransferredJavaScriptSemanticGraphSteps(structuredClone(semantic)),
|
||||
);
|
||||
expect(sealedApplication).toEqual(expectedApplication);
|
||||
expect(sealedSemantic).toEqual(expectedSemantic);
|
||||
expect(
|
||||
isValidatedImmutableJavaScriptApplicationGraph(sealedApplication),
|
||||
).toBe(true);
|
||||
expect(isValidatedImmutableJavaScriptSemanticGraph(sealedSemantic)).toBe(
|
||||
true,
|
||||
);
|
||||
});
|
||||
|
||||
it("report the issues a whole-graph parse reports", () => {
|
||||
const input = example();
|
||||
const { graph_id: _applicationId, ...application } = input.graph;
|
||||
const { graph_id: _semanticId, ...semantic } = input.semantic_graph;
|
||||
const node = semantic.nodes[0];
|
||||
const applicationNode = application.nodes[0];
|
||||
if (node === undefined || applicationNode === undefined)
|
||||
throw new Error("Expected semantic and application nodes");
|
||||
const malformed = {
|
||||
...semantic,
|
||||
nodes: [{ ...node, label: 7 }, ...semantic.nodes.slice(1)],
|
||||
};
|
||||
expect(
|
||||
issuesFrom(() =>
|
||||
complete(sealTransferredJavaScriptSemanticGraphSteps(malformed)),
|
||||
),
|
||||
).toEqual(issuesFrom(() => createJavaScriptSemanticGraph(malformed)));
|
||||
const stale = {
|
||||
...application,
|
||||
nodes: [
|
||||
{ ...applicationNode, node_id: `jag_node_${"f".repeat(64)}` },
|
||||
...application.nodes.slice(1),
|
||||
],
|
||||
};
|
||||
expect(
|
||||
issuesFrom(() =>
|
||||
complete(sealTransferredJavaScriptApplicationGraphSteps(stale)),
|
||||
),
|
||||
).toEqual(issuesFrom(() => createJavaScriptApplicationGraph(stale)));
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,4 +1,9 @@
|
||||
import { z } from "zod";
|
||||
import {
|
||||
arrayContainerSchema,
|
||||
parseElementSteps,
|
||||
} from "../zodCollectionSteps.js";
|
||||
import { canonicalJsonDigestSteps } from "../canonicalJsonDigestSteps.js";
|
||||
import { canonicalJson } from "../comparisonSemantics.js";
|
||||
import { digestCanonicalValue } from "../canonicalDigest.js";
|
||||
import { uniqueSorted } from "../canonicalOrdering.js";
|
||||
@@ -93,11 +98,14 @@ type EdgeSemantic = Omit<ApplicationEdge, "edge_id">;
|
||||
const edgeId = (edge: EdgeSemantic): string =>
|
||||
`jag_edge_${digestCanonicalValue(edge, "JavaScript Application Graph")}`;
|
||||
|
||||
/** The issue sink shared by schema refinement and stepwise checking. */
|
||||
type GraphIssueContext = Pick<z.RefinementCtx, "addIssue">;
|
||||
|
||||
const sortedUniqueIssue = (
|
||||
values: readonly string[],
|
||||
path: PropertyKey[],
|
||||
label: string,
|
||||
context: z.RefinementCtx,
|
||||
context: GraphIssueContext,
|
||||
): void => {
|
||||
for (let index = 1; index < values.length; index += 1) {
|
||||
if (
|
||||
@@ -116,7 +124,7 @@ const sortedUniqueIssue = (
|
||||
const checkEvidenceNormalization = (
|
||||
evidence: ApplicationGraphEvidence,
|
||||
path: PropertyKey[],
|
||||
context: z.RefinementCtx,
|
||||
context: GraphIssueContext,
|
||||
): void => {
|
||||
sortedUniqueIssue(
|
||||
evidence.limitations,
|
||||
@@ -226,7 +234,7 @@ const observationMatchesIdentity = (
|
||||
const checkNode = (
|
||||
node: ApplicationNode,
|
||||
index: number,
|
||||
context: z.RefinementCtx,
|
||||
context: GraphIssueContext,
|
||||
): void => {
|
||||
if (node.node_id !== nodeId(node))
|
||||
context.addIssue({
|
||||
@@ -306,8 +314,16 @@ type GraphRecord = z.infer<typeof javascriptApplicationGraphRecordSchema>;
|
||||
|
||||
const checkGraphContent = (
|
||||
graph: JavaScriptApplicationGraphInput,
|
||||
context: z.RefinementCtx,
|
||||
context: GraphIssueContext,
|
||||
): void => {
|
||||
const steps = checkGraphContentSteps(graph, context);
|
||||
while (steps.next().done !== true);
|
||||
};
|
||||
|
||||
function* checkGraphContentSteps(
|
||||
graph: JavaScriptApplicationGraphInput,
|
||||
context: GraphIssueContext,
|
||||
): Generator<void, void> {
|
||||
sortedUniqueIssue(
|
||||
graph.root_node_ids,
|
||||
["root_node_ids"],
|
||||
@@ -353,9 +369,12 @@ const checkGraphContent = (
|
||||
message: "Root identifier must name a graph node",
|
||||
path: ["root_node_ids"],
|
||||
});
|
||||
for (const [index, node] of graph.nodes.entries())
|
||||
for (const [index, node] of graph.nodes.entries()) {
|
||||
if (index % 64 === 0) yield;
|
||||
checkNode(node, index, context);
|
||||
}
|
||||
for (const [index, edge] of graph.edges.entries()) {
|
||||
if (index % 64 === 0) yield;
|
||||
const { edge_id: identifier, ...semantic } = edge;
|
||||
if (identifier !== edgeId(semantic))
|
||||
context.addIssue({
|
||||
@@ -394,11 +413,11 @@ const checkGraphContent = (
|
||||
context,
|
||||
);
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
const checkGraphInvariants = (
|
||||
graph: GraphRecord,
|
||||
context: z.RefinementCtx,
|
||||
context: GraphIssueContext,
|
||||
): void => {
|
||||
checkGraphContent(graph, context);
|
||||
const { graph_id: identifier, ...semantic } = graph;
|
||||
@@ -475,22 +494,9 @@ export const createJavaScriptApplicationEdge = (
|
||||
export const createJavaScriptApplicationGraph = (
|
||||
input: unknown,
|
||||
): JavaScriptApplicationGraph => {
|
||||
const parsed = javascriptApplicationGraphInputSchema.parse(input);
|
||||
const semantic: JavaScriptApplicationGraphInput = {
|
||||
...parsed,
|
||||
root_node_ids: uniqueSorted(parsed.root_node_ids),
|
||||
nodes: [...parsed.nodes].sort((left, right) =>
|
||||
compareUnicodeCodePoints(left.node_id, right.node_id),
|
||||
),
|
||||
edges: [...parsed.edges].sort((left, right) =>
|
||||
compareUnicodeCodePoints(left.edge_id, right.edge_id),
|
||||
),
|
||||
coverage: {
|
||||
...parsed.coverage,
|
||||
limits: normalizeLimits(parsed.coverage.limits),
|
||||
},
|
||||
limitations: uniqueSorted(parsed.limitations),
|
||||
};
|
||||
const semantic = normalizeGraphInput(
|
||||
javascriptApplicationGraphInputSchema.parse(input),
|
||||
);
|
||||
const content = javascriptApplicationGraphContentSchema.parse(semantic);
|
||||
return {
|
||||
...content,
|
||||
@@ -498,6 +504,24 @@ export const createJavaScriptApplicationGraph = (
|
||||
};
|
||||
};
|
||||
|
||||
const normalizeGraphInput = (
|
||||
parsed: JavaScriptApplicationGraphInput,
|
||||
): JavaScriptApplicationGraphInput => ({
|
||||
...parsed,
|
||||
root_node_ids: uniqueSorted(parsed.root_node_ids),
|
||||
nodes: [...parsed.nodes].sort((left, right) =>
|
||||
compareUnicodeCodePoints(left.node_id, right.node_id),
|
||||
),
|
||||
edges: [...parsed.edges].sort((left, right) =>
|
||||
compareUnicodeCodePoints(left.edge_id, right.edge_id),
|
||||
),
|
||||
coverage: {
|
||||
...parsed.coverage,
|
||||
limits: normalizeLimits(parsed.coverage.limits),
|
||||
},
|
||||
limitations: uniqueSorted(parsed.limitations),
|
||||
});
|
||||
|
||||
const validatedImmutableApplicationGraphs = new WeakSet<object>();
|
||||
|
||||
/** Clone and validate synchronously, then return steps to seal the owned graph. */
|
||||
@@ -506,6 +530,65 @@ export const createImmutableJavaScriptApplicationGraphSteps = (
|
||||
): Generator<void, JavaScriptApplicationGraph> =>
|
||||
sealValidatedApplicationGraphSteps(createJavaScriptApplicationGraph(input));
|
||||
|
||||
const applicationGraphShape = javascriptApplicationGraphInputSchema.shape;
|
||||
const applicationGraphContainerSchema =
|
||||
javascriptApplicationGraphInputSchema.extend({
|
||||
nodes: arrayContainerSchema(applicationGraphShape.nodes),
|
||||
edges: arrayContainerSchema(applicationGraphShape.edges),
|
||||
});
|
||||
|
||||
/**
|
||||
* Seal a graph whose input the caller transfers: the caller neither retains nor
|
||||
* mutates any part of it afterwards. Only the graph's own fields are validated
|
||||
* synchronously; nodes and edges are validated, checked, hashed and frozen in
|
||||
* steps, because doing so at once held the event loop for large applications.
|
||||
*/
|
||||
export const sealTransferredJavaScriptApplicationGraphSteps = (
|
||||
input: unknown,
|
||||
): Generator<void, JavaScriptApplicationGraph> =>
|
||||
parseAndSealApplicationGraphSteps(
|
||||
applicationGraphContainerSchema.parse(input),
|
||||
);
|
||||
|
||||
function* parseAndSealApplicationGraphSteps(
|
||||
container: z.output<typeof applicationGraphContainerSchema>,
|
||||
): Generator<void, JavaScriptApplicationGraph> {
|
||||
const issues: z.core.$ZodIssue[] = [];
|
||||
const parsed: JavaScriptApplicationGraphInput = {
|
||||
...container,
|
||||
nodes: yield* parseElementSteps(
|
||||
applicationGraphShape.nodes.element,
|
||||
container.nodes,
|
||||
["nodes"],
|
||||
issues,
|
||||
),
|
||||
edges: yield* parseElementSteps(
|
||||
applicationGraphShape.edges.element,
|
||||
container.edges,
|
||||
["edges"],
|
||||
issues,
|
||||
),
|
||||
};
|
||||
if (issues.length > 0) throw new z.ZodError(issues);
|
||||
const content = normalizeGraphInput(parsed);
|
||||
let invalid = false;
|
||||
yield* checkGraphContentSteps(content, {
|
||||
addIssue: () => {
|
||||
invalid = true;
|
||||
},
|
||||
});
|
||||
// An invalid owned graph is a construction defect; report it with the exact
|
||||
// issues the content schema produces.
|
||||
if (invalid) {
|
||||
javascriptApplicationGraphContentSchema.parse(content);
|
||||
throw new TypeError("Application graph checks disagree with its schema");
|
||||
}
|
||||
return yield* sealValidatedApplicationGraphSteps({
|
||||
...content,
|
||||
graph_id: `jag_${yield* canonicalJsonDigestSteps(content)}`,
|
||||
});
|
||||
}
|
||||
|
||||
function* sealValidatedApplicationGraphSteps(
|
||||
graph: JavaScriptApplicationGraph,
|
||||
): Generator<void, JavaScriptApplicationGraph> {
|
||||
|
||||
@@ -2,6 +2,10 @@ import { posix, relative, win32 } from "node:path";
|
||||
|
||||
import type { ApplicationNode } from "./javascriptApplicationGraphSchemas.js";
|
||||
import type { JavaScriptApplicationGraph } from "./javascriptApplicationGraph.js";
|
||||
import {
|
||||
logicalPathEscapesRoot,
|
||||
normalizeJoinedLogicalPath,
|
||||
} from "../artifactIdentity.js";
|
||||
import { uniqueSorted } from "../canonicalOrdering.js";
|
||||
import { compareUnicodeCodePoints } from "../unicodeCodePointOrder.js";
|
||||
import type { ParsedStaticLayer } from "./javascriptRuntimeReconciliationParsing.js";
|
||||
@@ -174,7 +178,7 @@ const nodePaths = (node: ApplicationNode): string[] => {
|
||||
paths.push(observation.evidence.location.value.path);
|
||||
}
|
||||
return paths
|
||||
.map(normalizeArtifactPath)
|
||||
.map(relativeLogicalPath)
|
||||
.filter(
|
||||
(path): path is string => path !== null && path !== "artifact-root",
|
||||
);
|
||||
@@ -221,7 +225,7 @@ const pathBelowRoot = (root: string, value: string): string | null => {
|
||||
portableRemainder.startsWith("../")
|
||||
)
|
||||
return null;
|
||||
return normalizeArtifactPath(portableRemainder);
|
||||
return relativeLogicalPath(portableRemainder);
|
||||
};
|
||||
|
||||
const absoluteFilePathStyle = (path: string): "posix" | "windows" | null => {
|
||||
@@ -250,7 +254,7 @@ const pathBelowUrlPrefix = (prefix: string, value: string): string | null => {
|
||||
)
|
||||
return null;
|
||||
try {
|
||||
return normalizeArtifactPath(
|
||||
return relativeLogicalPath(
|
||||
decodeURIComponent(candidate.pathname.slice(base.pathname.length)),
|
||||
);
|
||||
} catch (cause: unknown) {
|
||||
@@ -292,17 +296,13 @@ const pathWithinPrefix = (path: string, prefix: string): boolean =>
|
||||
prefix === "" || path === prefix || path.startsWith(`${prefix}/`);
|
||||
|
||||
const safeArtifactPath = (prefix: string, suffix: string): string | null =>
|
||||
normalizeArtifactPath(prefix === "" ? suffix : posix.join(prefix, suffix));
|
||||
relativeLogicalPath(prefix === "" ? suffix : posix.join(prefix, suffix));
|
||||
|
||||
const normalizeArtifactPath = (value: string): string | null => {
|
||||
const normalized = posix
|
||||
.normalize(value.replaceAll("\\", "/"))
|
||||
.replace(/^\.\//u, "");
|
||||
const relativeLogicalPath = (value: string): string | null => {
|
||||
const normalized = normalizeJoinedLogicalPath(value);
|
||||
return normalized !== "" &&
|
||||
normalized !== "." &&
|
||||
normalized !== ".." &&
|
||||
!normalized.startsWith("/") &&
|
||||
!normalized.startsWith("../")
|
||||
!logicalPathEscapesRoot(normalized)
|
||||
? normalized
|
||||
: null;
|
||||
};
|
||||
|
||||
@@ -597,11 +597,9 @@ describe("property mutation collection on minified bundles", () => {
|
||||
{ length: 2000 },
|
||||
(_, index) => `source.p${index} = ${index};`,
|
||||
).join(" ");
|
||||
const start = performance.now();
|
||||
const value = resultValue(
|
||||
`const source = { token: "TOKEN" }; ${writes} return [source.token, source.p1999];`,
|
||||
);
|
||||
expect(performance.now() - start).toBeLessThan(2000);
|
||||
expect(value).toMatchObject({
|
||||
status: "array",
|
||||
items: [
|
||||
@@ -662,3 +660,39 @@ describe("results of methods on an escaped receiver", () => {
|
||||
});
|
||||
}, 30000);
|
||||
});
|
||||
|
||||
describe("results of methods on an escaped receiver", () => {
|
||||
it("makes every method's distinct result uncertain, not only the called one", () => {
|
||||
expect(
|
||||
resultValue(
|
||||
'const first = { t: "A" }; const second = { t: "B" }; const kept = { t: "C" }; const box = { a() { return first; }, b() { return second; }, c() { return first; } }; box.a(); return [first.t, second.t, kept.t];',
|
||||
),
|
||||
).toMatchObject({
|
||||
status: "array",
|
||||
items: [
|
||||
{ value: { status: "unknown" } },
|
||||
{ value: { status: "unknown" } },
|
||||
{ value: { status: "literal", value: "C" } },
|
||||
],
|
||||
});
|
||||
});
|
||||
|
||||
it("keeps a shared result uncertain across a wide escaped receiver (#1495)", () => {
|
||||
const count = 2000;
|
||||
const methods = Array.from(
|
||||
{ length: count },
|
||||
(_, index) => `m${index}() { return shared; },`,
|
||||
).join(" ");
|
||||
const calls = Array.from(
|
||||
{ length: count },
|
||||
(_, index) => `box.m${index}().p${index} = ${index};`,
|
||||
).join(" ");
|
||||
const value = resultValue(
|
||||
`const shared = { token: "TOKEN" }; const box = { ${methods} }; ${calls} return [shared.token];`,
|
||||
);
|
||||
expect(value).toMatchObject({
|
||||
status: "array",
|
||||
items: [{ value: { status: "unknown" } }],
|
||||
});
|
||||
}, 30000);
|
||||
});
|
||||
|
||||
@@ -38,7 +38,7 @@ import {
|
||||
collectSemanticReturns,
|
||||
resolveSemanticModuleCallables,
|
||||
} from "./javascriptSemanticReturns.js";
|
||||
import { collectJavaScriptDerivedSemantics } from "./javascriptSemanticDerivedAnalysis.js";
|
||||
import { collectJavaScriptDerivedSemanticsSteps } from "./javascriptSemanticDerivedAnalysis.js";
|
||||
import { range } from "./javascriptStaticAnalysisHelpers.js";
|
||||
import { propertyName } from "./javascriptAstValues.js";
|
||||
import { semanticCoverage } from "./javascriptSemanticCoverage.js";
|
||||
@@ -107,18 +107,40 @@ export const analyzeParsedJavaScriptReferences = (
|
||||
export const analyzeParsedJavaScriptSemantics = (
|
||||
file: ParsedJavaScriptSource,
|
||||
): JavaScriptSemanticIr => {
|
||||
const steps = analyzeParsedJavaScriptSemanticsSteps(file);
|
||||
for (;;) {
|
||||
const step = steps.next();
|
||||
if (step.done === true) return step.value;
|
||||
}
|
||||
};
|
||||
|
||||
/**
|
||||
* Recover semantics from a parsed artifact with a yield between analysis
|
||||
* phases, so a caller can serve control messages within one large file.
|
||||
* Each phase completes before the next starts; results equal the synchronous
|
||||
* form.
|
||||
*/
|
||||
export function* analyzeParsedJavaScriptSemanticsSteps(
|
||||
file: ParsedJavaScriptSource,
|
||||
): Generator<void, JavaScriptSemanticIr> {
|
||||
const state = createState(file.program);
|
||||
collectDefinitions(file.program, state);
|
||||
yield;
|
||||
collectSemanticMemberMutations(file.program, state);
|
||||
yield;
|
||||
traverseJavaScriptAst(file.program, {
|
||||
enter: (node) => collectSemanticModuleLink(node, state),
|
||||
});
|
||||
yield;
|
||||
const references = collectSemanticReferences(file.program, state);
|
||||
yield;
|
||||
const parserPartial = file.errors.length > 0;
|
||||
const bindings = immutableSemanticBindings(state);
|
||||
yield;
|
||||
const callables = collectSemanticReturns(file.program, state, parserPartial);
|
||||
yield;
|
||||
const moduleLinks = resolveSemanticModuleCallables(state, callables);
|
||||
const derived = collectJavaScriptDerivedSemantics(
|
||||
const derived = yield* collectJavaScriptDerivedSemanticsSteps(
|
||||
file.program,
|
||||
state,
|
||||
callables,
|
||||
@@ -154,7 +176,7 @@ export const analyzeParsedJavaScriptSemantics = (
|
||||
"Cross-function mutation and dynamic property resolution remain unknown.",
|
||||
],
|
||||
};
|
||||
};
|
||||
}
|
||||
|
||||
/** Receiver identity facts for the overloaded `.open` syntax only. */
|
||||
export type JavaScriptOpenReceiverFact =
|
||||
|
||||
@@ -0,0 +1,34 @@
|
||||
import { expect, it } from "vitest";
|
||||
|
||||
import {
|
||||
analyzeParsedJavaScriptSemantics,
|
||||
analyzeParsedJavaScriptSemanticsSteps,
|
||||
} from "./javascriptSemanticAnalysis.js";
|
||||
import { parseJavaScriptSource } from "./javascriptSourceParser.js";
|
||||
|
||||
const source = `
|
||||
import { readFile } from "node:fs";
|
||||
const shared = { token: "TOKEN" };
|
||||
export function load(path) {
|
||||
return new Promise((resolve) => setTimeout(() => resolve(readFile(path)), 1));
|
||||
}
|
||||
export const box = { get() { return shared; } };
|
||||
box.get().value = process.env.MODE ?? "default";
|
||||
`;
|
||||
|
||||
it("yields between semantic phases and returns the synchronous result (#1462)", () => {
|
||||
const parsed = parseJavaScriptSource(source, "main.js");
|
||||
if (parsed === null) throw new Error("Expected parsed source");
|
||||
const steps = analyzeParsedJavaScriptSemanticsSteps(parsed);
|
||||
let yields = 0;
|
||||
let step = steps.next();
|
||||
while (step.done !== true) {
|
||||
yields += 1;
|
||||
step = steps.next();
|
||||
}
|
||||
// Every top-level phase and derived collector is a separate step.
|
||||
expect(yields).toBeGreaterThanOrEqual(13);
|
||||
const reparsed = parseJavaScriptSource(source, "main.js");
|
||||
if (reparsed === null) throw new Error("Expected parsed source");
|
||||
expect(step.value).toEqual(analyzeParsedJavaScriptSemantics(reparsed));
|
||||
});
|
||||
@@ -36,43 +36,54 @@ type DerivedSemanticAnalysis = Pick<
|
||||
>;
|
||||
|
||||
/** Compose bounded semantic passes after lexical definitions are available. */
|
||||
export const collectJavaScriptDerivedSemantics = (
|
||||
/**
|
||||
* Collect derived semantics with a yield between collectors. Each collector
|
||||
* completes before the next starts, so results equal the synchronous form.
|
||||
*/
|
||||
export function* collectJavaScriptDerivedSemanticsSteps(
|
||||
program: t.Program,
|
||||
state: JavaScriptSemanticAnalysisState,
|
||||
callables: readonly JavaScriptSemanticCallable[],
|
||||
parserPartial: boolean,
|
||||
): DerivedSemanticAnalysis => {
|
||||
): Generator<void, DerivedSemanticAnalysis> {
|
||||
const calls = collectJavaScriptSemanticCalls(program, state, callables);
|
||||
yield;
|
||||
const promiseOperations = collectJavaScriptSemanticPromises(
|
||||
program,
|
||||
state,
|
||||
callables,
|
||||
);
|
||||
yield;
|
||||
const asyncEffects = collectJavaScriptSemanticAsyncEffects(
|
||||
program,
|
||||
state,
|
||||
callables,
|
||||
);
|
||||
yield;
|
||||
const childProcesses = collectJavaScriptSemanticChildProcesses(
|
||||
program,
|
||||
state,
|
||||
callables,
|
||||
);
|
||||
yield;
|
||||
const dataEffects = collectJavaScriptSemanticDataEffects(
|
||||
program,
|
||||
state,
|
||||
callables,
|
||||
);
|
||||
yield;
|
||||
const resourceOperations = collectJavaScriptSemanticResources(
|
||||
program,
|
||||
state,
|
||||
callables,
|
||||
);
|
||||
yield;
|
||||
const objectOperations = collectJavaScriptSemanticObjects(
|
||||
program,
|
||||
state,
|
||||
callables,
|
||||
);
|
||||
yield;
|
||||
const functionFingerprints = collectJavaScriptSemanticFingerprints({
|
||||
state,
|
||||
callables,
|
||||
@@ -96,4 +107,4 @@ export const collectJavaScriptDerivedSemantics = (
|
||||
objectOperations,
|
||||
functionFingerprints,
|
||||
};
|
||||
};
|
||||
}
|
||||
|
||||
@@ -6,6 +6,10 @@ import { uniqueSorted } from "../canonicalOrdering.js";
|
||||
import { compareUnicodeCodePoints } from "../unicodeCodePointOrder.js";
|
||||
import { canonicalJsonDigestSteps } from "../canonicalJsonDigestSteps.js";
|
||||
import { freezeOwnedJsonSnapshotSteps } from "../immutableJson.js";
|
||||
import {
|
||||
arrayContainerSchema,
|
||||
parseElementSteps,
|
||||
} from "../zodCollectionSteps.js";
|
||||
import type {
|
||||
ApplicationGraphEvidence,
|
||||
ApplicationGraphEvidenceContext,
|
||||
@@ -765,6 +769,15 @@ export const createJavaScriptSemanticGraph = (
|
||||
|
||||
const validatedImmutableSemanticGraphs = new WeakSet<object>();
|
||||
|
||||
const semanticGraphShape = javaScriptSemanticGraphInputSchema.shape;
|
||||
const semanticGraphContainerSchema = javaScriptSemanticGraphInputSchema.extend({
|
||||
evidence_contexts: arrayContainerSchema(semanticGraphShape.evidence_contexts),
|
||||
nodes: arrayContainerSchema(semanticGraphShape.nodes),
|
||||
relations: arrayContainerSchema(semanticGraphShape.relations),
|
||||
fingerprints: arrayContainerSchema(semanticGraphShape.fingerprints),
|
||||
unknowns: arrayContainerSchema(semanticGraphShape.unknowns),
|
||||
});
|
||||
|
||||
/** Clone input synchronously; commit, check and seal the owned graph in steps. */
|
||||
export const createImmutableJavaScriptSemanticGraphSteps = (
|
||||
input: unknown,
|
||||
@@ -773,6 +786,58 @@ export const createImmutableJavaScriptSemanticGraphSteps = (
|
||||
normalizeGraphInput(javaScriptSemanticGraphInputSchema.parse(input)),
|
||||
);
|
||||
|
||||
/**
|
||||
* Seal a graph whose input the caller transfers: the caller neither retains nor
|
||||
* mutates any part of it afterwards. Only the graph's own fields are validated
|
||||
* synchronously; collection elements are validated and cloned in steps, because
|
||||
* a whole-graph parse of a large application held the event loop for seconds.
|
||||
*/
|
||||
export const sealTransferredJavaScriptSemanticGraphSteps = (
|
||||
input: unknown,
|
||||
): Generator<void, JavaScriptSemanticGraph> =>
|
||||
parseAndSealSemanticGraphSteps(semanticGraphContainerSchema.parse(input));
|
||||
|
||||
function* parseAndSealSemanticGraphSteps(
|
||||
container: z.output<typeof semanticGraphContainerSchema>,
|
||||
): Generator<void, JavaScriptSemanticGraph> {
|
||||
const issues: z.core.$ZodIssue[] = [];
|
||||
const parsed: JavaScriptSemanticGraphInput = {
|
||||
...container,
|
||||
evidence_contexts: yield* parseElementSteps(
|
||||
semanticGraphShape.evidence_contexts.element,
|
||||
container.evidence_contexts,
|
||||
["evidence_contexts"],
|
||||
issues,
|
||||
),
|
||||
nodes: yield* parseElementSteps(
|
||||
semanticGraphShape.nodes.element,
|
||||
container.nodes,
|
||||
["nodes"],
|
||||
issues,
|
||||
),
|
||||
relations: yield* parseElementSteps(
|
||||
semanticGraphShape.relations.element,
|
||||
container.relations,
|
||||
["relations"],
|
||||
issues,
|
||||
),
|
||||
fingerprints: yield* parseElementSteps(
|
||||
semanticGraphShape.fingerprints.element,
|
||||
container.fingerprints,
|
||||
["fingerprints"],
|
||||
issues,
|
||||
),
|
||||
unknowns: yield* parseElementSteps(
|
||||
semanticGraphShape.unknowns.element,
|
||||
container.unknowns,
|
||||
["unknowns"],
|
||||
issues,
|
||||
),
|
||||
};
|
||||
if (issues.length > 0) throw new z.ZodError(issues);
|
||||
return yield* validateAndSealSemanticGraphSteps(normalizeGraphInput(parsed));
|
||||
}
|
||||
|
||||
function* validateAndSealSemanticGraphSteps(
|
||||
semantic: JavaScriptSemanticGraphInput,
|
||||
): Generator<void, JavaScriptSemanticGraph> {
|
||||
|
||||
@@ -251,19 +251,9 @@ export const collectSemanticMemberMutations = (
|
||||
originAt: current.originAt,
|
||||
});
|
||||
} else {
|
||||
if (
|
||||
t.isArrayExpression(expression) &&
|
||||
((current.path[0] === "__proto__" &&
|
||||
(effect === "escape" || current.path.length > 1)) ||
|
||||
(current.path[0] === "constructor" &&
|
||||
current.path[1] === "prototype"))
|
||||
)
|
||||
arrayIterationUnknown = true;
|
||||
for (const value of referencedValues(
|
||||
expression,
|
||||
current.path,
|
||||
effect,
|
||||
)) {
|
||||
for (const value of effect === "escape"
|
||||
? escapedReferenceLeaves(expression, current.path)
|
||||
: expandedValues(expression, current.path, effect)) {
|
||||
if (value.iterableFallbackPath !== undefined)
|
||||
deferIterable({
|
||||
node: value.node,
|
||||
@@ -290,6 +280,78 @@ export const collectSemanticMemberMutations = (
|
||||
mutation: t.Node,
|
||||
path: PropertyPath = [],
|
||||
): void => markValue(node, path, new Set(), "escape", mutation);
|
||||
function expandedValues(
|
||||
expression: t.Node,
|
||||
path: PropertyPath,
|
||||
effect: ValueEffect,
|
||||
): readonly ReferencedValue[] {
|
||||
if (
|
||||
t.isArrayExpression(expression) &&
|
||||
((path[0] === "__proto__" && (effect === "escape" || path.length > 1)) ||
|
||||
(path[0] === "constructor" && path[1] === "prototype"))
|
||||
)
|
||||
arrayIterationUnknown = true;
|
||||
return referencedValues(expression, path, effect);
|
||||
}
|
||||
// Expanding a value below its references depends only on syntax, not on the
|
||||
// escape site. Escaping an object reaches every method's results, so
|
||||
// re-expanding it at each call through that object made N calls over N
|
||||
// methods quadratic. Identifiers that resolve to the same binding and path
|
||||
// are processed identically, so one stands for all of them.
|
||||
const escapedLeaves = new WeakMap<
|
||||
t.Node,
|
||||
Map<string, readonly ReferencedValue[]>
|
||||
>();
|
||||
function escapedReferenceLeaves(
|
||||
root: t.Node,
|
||||
rootPath: PropertyPath,
|
||||
): readonly ReferencedValue[] {
|
||||
// Parser recovery can leave an argument hole, which expands to nothing.
|
||||
if (!t.isNode(root)) return [];
|
||||
const rootKey = JSON.stringify(rootPath);
|
||||
const cached = escapedLeaves.get(root)?.get(rootKey);
|
||||
if (cached !== undefined) return cached;
|
||||
const leaves: ReferencedValue[] = [];
|
||||
const leafIdentities = new Set<string>();
|
||||
const expanded = new WeakMap<t.Node, Set<string>>();
|
||||
const pending = [...expandedValues(root, rootPath, "escape")];
|
||||
for (let cursor = 0; cursor < pending.length; cursor++) {
|
||||
const value = pending[cursor];
|
||||
if (value === undefined) break;
|
||||
const node = unwrapJavaScriptExpression(value.node).node;
|
||||
if (!t.isNode(node)) continue;
|
||||
const pathKey = JSON.stringify(value.path);
|
||||
if (t.isIdentifier(node) && value.iterableFallbackPath === undefined) {
|
||||
const binding = resolveSemanticBindingState(state, node, node.name);
|
||||
const identity = JSON.stringify([
|
||||
binding === undefined ? null : binding.bindingId,
|
||||
binding === undefined ? node.name : null,
|
||||
pathKey,
|
||||
]);
|
||||
if (leafIdentities.has(identity)) continue;
|
||||
leafIdentities.add(identity);
|
||||
leaves.push(value);
|
||||
} else if (
|
||||
value.iterableFallbackPath !== undefined ||
|
||||
t.isIdentifier(node) ||
|
||||
t.isMemberExpression(node) ||
|
||||
t.isOptionalMemberExpression(node)
|
||||
)
|
||||
leaves.push(value);
|
||||
else {
|
||||
const paths = expanded.get(node) ?? new Set<string>();
|
||||
if (paths.has(pathKey)) continue;
|
||||
paths.add(pathKey);
|
||||
expanded.set(node, paths);
|
||||
for (const reference of expandedValues(node, value.path, "escape"))
|
||||
pending.push(reference);
|
||||
}
|
||||
}
|
||||
const byPath = escapedLeaves.get(root) ?? new Map();
|
||||
byPath.set(rootKey, leaves);
|
||||
escapedLeaves.set(root, byPath);
|
||||
return leaves;
|
||||
}
|
||||
const markReceiver = (callee: t.Node, mutation: t.Node): void => {
|
||||
const expression = unwrapJavaScriptExpression(callee).node;
|
||||
if (
|
||||
|
||||
@@ -0,0 +1,38 @@
|
||||
/** Path-segment conventions. These are inferences, not observations of runtime behavior. */
|
||||
|
||||
export type NativeRuntimeConvention = "react-native" | "flutter" | "unity";
|
||||
|
||||
const foldSegment = (segment: string): string =>
|
||||
segment.toLocaleLowerCase("en-US");
|
||||
|
||||
const REACT_NATIVE_SEGMENTS = new Set([
|
||||
"react.framework",
|
||||
"libhermes.so",
|
||||
"hermes.framework",
|
||||
]);
|
||||
const FLUTTER_SEGMENTS = new Set(["libflutter.so", "flutter.framework"]);
|
||||
const UNITY_SEGMENTS = new Set(["libunity.so", "unityframework.framework"]);
|
||||
|
||||
/**
|
||||
* A runtime family named by one filename or directory segment.
|
||||
* `reactnative` may occur inside a segment (`libreactnativejni.so`).
|
||||
* The other names match the whole segment.
|
||||
*/
|
||||
export const nativeRuntimeConvention = (
|
||||
path: string,
|
||||
): NativeRuntimeConvention | null => {
|
||||
for (const segment of path.split("/")) {
|
||||
const folded = foldSegment(segment);
|
||||
if (folded.includes("reactnative") || REACT_NATIVE_SEGMENTS.has(folded))
|
||||
return "react-native";
|
||||
if (FLUTTER_SEGMENTS.has(folded)) return "flutter";
|
||||
if (UNITY_SEGMENTS.has(folded)) return "unity";
|
||||
}
|
||||
return null;
|
||||
};
|
||||
|
||||
/** Swift runtime libraries are `libswift*.dylib` segments, not every dynamic library. */
|
||||
export const isSwiftRuntimeLibrary = (path: string): boolean =>
|
||||
path
|
||||
.split("/")
|
||||
.some((segment) => /^libswift.+\.dylib$/u.test(foldSegment(segment)));
|
||||
@@ -108,6 +108,9 @@ export const processComparisonContract = (
|
||||
...(scenario.normalization.ports
|
||||
? { port_normalization_version: "contextual-endpoints-v2" }
|
||||
: {}),
|
||||
...(scenario.normalization.patterns.length > 0
|
||||
? { pattern_normalization_version: "literal-replacements-v1" }
|
||||
: {}),
|
||||
rendered_line_format: "trailing-spaces-trimmed-v1",
|
||||
});
|
||||
|
||||
|
||||
@@ -46,7 +46,6 @@ describe("captured script path projection", () => {
|
||||
it.each([
|
||||
["https://a.test/main.js", "https://a.test/main.js"],
|
||||
["https://a.test/main.js", "https://a.test/main.js?v=2"],
|
||||
["https://a.test/A/main.js", "https://a.test/a/other.js"],
|
||||
["https://a.test/main.js", "https://a.test/main.js/child.js"],
|
||||
])(
|
||||
"isolates every version in a conflicting layout (%s, %s)",
|
||||
@@ -99,6 +98,25 @@ describe("captured script path projection", () => {
|
||||
},
|
||||
);
|
||||
|
||||
it("keeps case-distinct URL paths until the destination collides", () => {
|
||||
const planned = planWebScriptExport([
|
||||
source("https://a.test/A/main.js"),
|
||||
source("https://a.test/a/other.js", 2),
|
||||
]);
|
||||
expect(
|
||||
planned.every(
|
||||
({ record }) =>
|
||||
record.content.state === "exported" &&
|
||||
record.content.layout === "url-path",
|
||||
),
|
||||
).toBe(true);
|
||||
const paths = planned.map(({ record }) =>
|
||||
record.content.state === "exported" ? record.content.relative_path : "",
|
||||
);
|
||||
expect(paths[0]).toMatch(/\/A\/main\.js$/u);
|
||||
expect(paths[1]).toMatch(/\/a\/other\.js$/u);
|
||||
});
|
||||
|
||||
it("does not choose a canonical version when a competing source is unavailable", () => {
|
||||
const missing: CapturedWebScript = {
|
||||
...source("https://a.test/main.js?v=2", 2),
|
||||
|
||||
@@ -34,7 +34,7 @@ export const planWebScriptExport = (
|
||||
};
|
||||
const candidate = candidates[index];
|
||||
const reason = conflicts.has(index)
|
||||
? "Captured paths collide by content version, case, or file/directory prefix."
|
||||
? "Captured paths collide by content version or file/directory prefix."
|
||||
: (candidate?.reason ?? null);
|
||||
const path = reason === null ? candidate?.path : undefined;
|
||||
return {
|
||||
@@ -103,40 +103,27 @@ const candidatePath = (script: CapturedWebScript): CandidatePath => {
|
||||
};
|
||||
};
|
||||
|
||||
// Register every prefix so case collisions in parent directories and a file
|
||||
// conflicting with a directory isolate all affected sources, not just the last.
|
||||
// Register every exact prefix so a repeated path, or a file that is also a
|
||||
// directory prefix, isolates every affected source. Case-distinct spellings
|
||||
// stay distinct until the destination filesystem reports a collision.
|
||||
const conflictingPaths = (
|
||||
candidates: readonly CandidatePath[],
|
||||
): Set<number> => {
|
||||
const prefixes = new Map<
|
||||
string,
|
||||
{
|
||||
spellings: Set<string>;
|
||||
indices: number[];
|
||||
files: number[];
|
||||
}
|
||||
>();
|
||||
const prefixes = new Map<string, { indices: number[]; files: number[] }>();
|
||||
candidates.forEach(({ path }, index) => {
|
||||
if (path === undefined) return;
|
||||
const parts = path.split("/");
|
||||
parts.forEach((_, position) => {
|
||||
const spelling = parts.slice(0, position + 1).join("/");
|
||||
const key = spelling.toLowerCase();
|
||||
const entry = prefixes.get(key) ?? {
|
||||
spellings: new Set<string>(),
|
||||
indices: [],
|
||||
files: [],
|
||||
};
|
||||
entry.spellings.add(spelling);
|
||||
const entry = prefixes.get(spelling) ?? { indices: [], files: [] };
|
||||
entry.indices.push(index);
|
||||
if (position === parts.length - 1) entry.files.push(index);
|
||||
prefixes.set(key, entry);
|
||||
prefixes.set(spelling, entry);
|
||||
});
|
||||
});
|
||||
const conflicts = new Set<number>();
|
||||
for (const entry of prefixes.values())
|
||||
if (
|
||||
entry.spellings.size > 1 ||
|
||||
entry.files.length > 1 ||
|
||||
(entry.files.length > 0 && entry.indices.length > entry.files.length)
|
||||
)
|
||||
|
||||
@@ -28,3 +28,7 @@ export const zipPackageFormatForPath = (
|
||||
if (lower.endsWith(".appx") || lower.endsWith(".appxbundle")) return "appx";
|
||||
return undefined;
|
||||
};
|
||||
|
||||
/** Inventory cannot see bytes that are not central-directory entries. */
|
||||
export const ZIP_NON_ENTRY_TAIL_LIMITATION =
|
||||
"ZIP inventory lists central-directory entries only. Bytes after the central directory, including an APK Signing Block or a ZIP comment, are not occurrences.";
|
||||
|
||||
@@ -0,0 +1,33 @@
|
||||
import { z } from "zod";
|
||||
|
||||
/**
|
||||
* An array schema's own constraints, such as a minimum length, without its
|
||||
* element schema. Pair it with {@link parseElementSteps} to validate a large
|
||||
* collection in cooperative steps.
|
||||
*/
|
||||
export const arrayContainerSchema = (
|
||||
array: z.ZodArray<z.ZodType>,
|
||||
): z.ZodType<unknown[]> => array.clone({ ...array.def, element: z.unknown() });
|
||||
|
||||
/** Elements validated between yields, in the issues and paths a whole-array parse reports. */
|
||||
export function* parseElementSteps<Element extends z.ZodType>(
|
||||
element: Element,
|
||||
values: readonly unknown[],
|
||||
path: readonly PropertyKey[],
|
||||
issues: z.core.$ZodIssue[],
|
||||
): Generator<void, z.output<Element>[]> {
|
||||
const parsed: z.output<Element>[] = [];
|
||||
for (const [index, value] of values.entries()) {
|
||||
if (index % 64 === 0) yield;
|
||||
const result = element.safeParse(value);
|
||||
if (result.success) parsed.push(result.data);
|
||||
else
|
||||
issues.push(
|
||||
...result.error.issues.map((issue) => ({
|
||||
...issue,
|
||||
path: [...path, index, ...issue.path],
|
||||
})),
|
||||
);
|
||||
}
|
||||
return parsed;
|
||||
}
|
||||
@@ -0,0 +1,98 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
|
||||
import { projectAnalysisError } from "../domain/analysisErrorProjection.js";
|
||||
import { ProviderAdapterError } from "../domain/providerAdapterError.js";
|
||||
import { ProviderCleanupError } from "../domain/providerCleanupError.js";
|
||||
import { err, ok } from "../domain/result.js";
|
||||
import { firmwareRequestSchema } from "../domain/firmware/firmwareAnalysis.js";
|
||||
import { finishFirmwareWorkspace } from "./FirmwareProvider.js";
|
||||
|
||||
const request = firmwareRequestSchema.parse({
|
||||
operation: "extract_firmware",
|
||||
input: {
|
||||
path: "/tmp/firmware.bin",
|
||||
output_directory: "/tmp/extracted",
|
||||
},
|
||||
});
|
||||
|
||||
const execution = ok({
|
||||
result: { files: ["kept"] },
|
||||
rawResult: null,
|
||||
provider: { id: "unblob", name: "unblob", version: "1" },
|
||||
limitations: [],
|
||||
locations: [],
|
||||
subject: null,
|
||||
});
|
||||
|
||||
describe("firmware workspace cleanup", () => {
|
||||
it("returns the completed extraction beside a cleanup failure and does not delete the leaked root", async () => {
|
||||
const close = vi.fn(async () => {
|
||||
throw new Error("cleanup failed");
|
||||
});
|
||||
const finished = await finishFirmwareWorkspace(
|
||||
{ path: "/tmp/runtime", close },
|
||||
execution,
|
||||
"unblob",
|
||||
request,
|
||||
);
|
||||
expect(close).toHaveBeenCalledOnce();
|
||||
if (finished.ok || !(finished.error instanceof ProviderAdapterError))
|
||||
throw new Error("Expected cleanup failure");
|
||||
expect(finished.error.cleanupResources).toEqual([
|
||||
"/tmp/runtime",
|
||||
"/tmp/extracted",
|
||||
]);
|
||||
expect(finished.error.partialObservation).toEqual({
|
||||
kind: "firmware",
|
||||
result: { files: ["kept"] },
|
||||
});
|
||||
expect(finished.error.diagnostics).toEqual({ reason: "cleanup failed" });
|
||||
expect(projectAnalysisError(finished.error).details).toMatchObject({
|
||||
partial_observation: {
|
||||
kind: "firmware",
|
||||
result: { files: ["kept"] },
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it("does not replay a cleanup failure for a later workspace", async () => {
|
||||
const failed = await finishFirmwareWorkspace(
|
||||
{
|
||||
path: "/tmp/runtime",
|
||||
close: async () => {
|
||||
throw new Error("cleanup failed");
|
||||
},
|
||||
},
|
||||
execution,
|
||||
"unblob",
|
||||
request,
|
||||
);
|
||||
expect(failed.ok).toBe(false);
|
||||
const next = await finishFirmwareWorkspace(
|
||||
{ path: "/tmp/next", close: async () => undefined },
|
||||
execution,
|
||||
"unblob",
|
||||
request,
|
||||
);
|
||||
expect(next).toBe(execution);
|
||||
});
|
||||
|
||||
it("leaves an uncertain workspace recorded and does not remove it", async () => {
|
||||
const close = vi.fn(async () => undefined);
|
||||
const cleanup = new ProviderCleanupError(
|
||||
"unblob",
|
||||
["/tmp/runtime"],
|
||||
{ reason: "ownership uncertain" },
|
||||
{ operation: "extract_firmware" },
|
||||
);
|
||||
const outcome = err(cleanup);
|
||||
const finished = await finishFirmwareWorkspace(
|
||||
{ path: "/tmp/runtime", close },
|
||||
outcome,
|
||||
"unblob",
|
||||
request,
|
||||
);
|
||||
expect(close).not.toHaveBeenCalled();
|
||||
expect(finished).toBe(outcome);
|
||||
});
|
||||
});
|
||||
@@ -65,7 +65,6 @@ const limitations = [
|
||||
/** Queued bring-your-own firmware adapters with private snapshots and owned cleanup. */
|
||||
export class FirmwareProvider implements FirmwareAnalysisPort {
|
||||
#tail: Promise<void> = Promise.resolve();
|
||||
#cleanupFailure: AnalysisError | undefined;
|
||||
constructor(
|
||||
readonly environment: Readonly<
|
||||
Record<string, string | undefined>
|
||||
@@ -115,7 +114,6 @@ export class FirmwareProvider implements FirmwareAnalysisPort {
|
||||
: AbortSignal.any([options.signal, deadline]);
|
||||
let outcome: Outcome;
|
||||
try {
|
||||
if (this.#cleanupFailure !== undefined) return err(this.#cleanupFailure);
|
||||
if (signal.aborted) throw new AnalysisCancelledError(request.operation);
|
||||
const command = await resolveFirmwareCommand(
|
||||
this.environment,
|
||||
@@ -316,29 +314,49 @@ export class FirmwareProvider implements FirmwareAnalysisPort {
|
||||
},
|
||||
}),
|
||||
);
|
||||
if (!outcome.ok && outcome.error.cleanupIncomplete) {
|
||||
this.#cleanupFailure =
|
||||
root === undefined ||
|
||||
outcome.error.cleanupResources.includes(root.path)
|
||||
? outcome.error
|
||||
: new ProviderCleanupError(
|
||||
engineName,
|
||||
[...outcome.error.cleanupResources, root.path],
|
||||
outcome.error instanceof ProviderAdapterError
|
||||
? (outcome.error.diagnostics ?? {
|
||||
reason: outcome.error.message,
|
||||
})
|
||||
: { reason: outcome.error.message },
|
||||
{ cause: outcome.error },
|
||||
);
|
||||
}
|
||||
}
|
||||
// Never remove a workspace while process ownership remains uncertain.
|
||||
if (this.#cleanupFailure !== undefined) return err(this.#cleanupFailure);
|
||||
try {
|
||||
await root?.close();
|
||||
} catch (cause: unknown) {
|
||||
this.#cleanupFailure = new ProviderCleanupError(
|
||||
// A leaked root stays recorded on this result and is not deleted. The next
|
||||
// operation may create a new root; this failure is not replayed for it.
|
||||
return finishFirmwareWorkspace(root, outcome, engineName, request);
|
||||
}
|
||||
}
|
||||
|
||||
/** Keep a completed extraction beside cleanup failure, or skip removal when ownership is uncertain. */
|
||||
export const finishFirmwareWorkspace = async (
|
||||
root: { readonly path: string; close(): Promise<void> } | undefined,
|
||||
outcome: Outcome,
|
||||
engineName: string,
|
||||
request: FirmwareRequest,
|
||||
): Promise<Outcome> => {
|
||||
if (!outcome.ok && outcome.error.cleanupIncomplete) {
|
||||
if (
|
||||
root === undefined ||
|
||||
outcome.error.cleanupResources.includes(root.path)
|
||||
)
|
||||
return outcome;
|
||||
return err(
|
||||
new ProviderCleanupError(
|
||||
engineName,
|
||||
[...outcome.error.cleanupResources, root.path],
|
||||
outcome.error instanceof ProviderAdapterError
|
||||
? (outcome.error.diagnostics ?? { reason: outcome.error.message })
|
||||
: { reason: outcome.error.message },
|
||||
{
|
||||
cause: outcome.error,
|
||||
operation: request.operation,
|
||||
...(outcome.error.partialObservation === undefined
|
||||
? {}
|
||||
: { partialObservation: outcome.error.partialObservation }),
|
||||
},
|
||||
),
|
||||
);
|
||||
}
|
||||
try {
|
||||
await root?.close();
|
||||
} catch (cause: unknown) {
|
||||
return err(
|
||||
new ProviderCleanupError(
|
||||
engineName,
|
||||
[
|
||||
root?.path ?? "unknown",
|
||||
@@ -346,13 +364,23 @@ export class FirmwareProvider implements FirmwareAnalysisPort {
|
||||
? [request.input.output_directory]
|
||||
: []),
|
||||
],
|
||||
{ reason: cause instanceof Error ? cause.message : String(cause) },
|
||||
{
|
||||
reason: cause instanceof Error ? cause.message : String(cause),
|
||||
previous_result: outcome.ok ? outcome.value.result : null,
|
||||
cause,
|
||||
operation: request.operation,
|
||||
...(outcome.ok
|
||||
? {
|
||||
partialObservation: {
|
||||
kind: "firmware",
|
||||
result: outcome.value.result,
|
||||
},
|
||||
}
|
||||
: outcome.error.partialObservation === undefined
|
||||
? {}
|
||||
: { partialObservation: outcome.error.partialObservation }),
|
||||
},
|
||||
);
|
||||
return err(this.#cleanupFailure);
|
||||
}
|
||||
return outcome;
|
||||
),
|
||||
);
|
||||
}
|
||||
}
|
||||
return outcome;
|
||||
};
|
||||
|
||||
@@ -4,11 +4,19 @@ import type { DecodeIssue } from "./AppleDispatchDecodeFacts.js";
|
||||
import type { FacetDecodeFacts } from "./AppleDispatchDecodeFacts.js";
|
||||
|
||||
const describeFixups = (fixups: PointerFixups): string => {
|
||||
if (fixups.kind === "chained")
|
||||
return `chained fixups: ${fixups.formats.join(", ") || "no fixup segments"}`;
|
||||
return fixups.kind === "dyld-info"
|
||||
? "LC_DYLD_INFO bind opcodes"
|
||||
: "no fixup load commands";
|
||||
const unread =
|
||||
fixups.unreadRebaseBytes > 0
|
||||
? `LC_DYLD_INFO rebase opcodes were not decoded (${fixups.unreadRebaseBytes} bytes)`
|
||||
: undefined;
|
||||
const kind =
|
||||
fixups.kind === "chained"
|
||||
? `chained fixups: ${fixups.formats.join(", ") || "no fixup segments"}`
|
||||
: fixups.kind === "dyld-info"
|
||||
? "LC_DYLD_INFO bind opcodes"
|
||||
: unread === undefined
|
||||
? "no fixup load commands"
|
||||
: undefined;
|
||||
return [unread, kind].filter((line) => line !== undefined).join("; ");
|
||||
};
|
||||
|
||||
/** Append the per-facet coverage of one Apple dispatch metadata decode. */
|
||||
@@ -83,18 +91,26 @@ export const pushDispatchCoverage = (input: {
|
||||
({ decode }) => decode.status === "decoded",
|
||||
).length,
|
||||
});
|
||||
const fixupReason = [
|
||||
describeFixups(fixups),
|
||||
...fixups.issues.map(
|
||||
({ code, location, message }) =>
|
||||
`${code}${location === undefined ? "" : ` at ${location}`}: ${message}`,
|
||||
),
|
||||
]
|
||||
.filter((line) => line.length > 0)
|
||||
.join("; ");
|
||||
result.coverage.push({
|
||||
facet: "pointer_fixups",
|
||||
status: fixups.issues.length > 0 ? "partial" : "complete",
|
||||
reason: [
|
||||
describeFixups(fixups),
|
||||
...fixups.issues.map(
|
||||
({ code, location, message }) =>
|
||||
`${code}${location === undefined ? "" : ` at ${location}`}: ${message}`,
|
||||
),
|
||||
].join("; "),
|
||||
examined: 0,
|
||||
decoded: 0,
|
||||
status:
|
||||
fixups.issues.length > 0 || fixups.unreadRebaseBytes > 0
|
||||
? "partial"
|
||||
: fixups.examined === 0
|
||||
? "unsupported"
|
||||
: "complete",
|
||||
reason: fixupReason.length > 0 ? fixupReason : null,
|
||||
examined: fixups.examined,
|
||||
decoded: fixups.issues.length > 0 ? 0 : fixups.examined,
|
||||
});
|
||||
result.coverage.push({
|
||||
facet: "swift_generic_resilient_witnesses_overrides_async_coroutines",
|
||||
|
||||
@@ -354,7 +354,7 @@ describe("byte-swapped universal Apple dispatch metadata", () => {
|
||||
const bytes = wrapped(false);
|
||||
bytes.writeUInt32BE(0xfeedface, 256);
|
||||
expect(() => decodeAppleDispatchMetadata(bytes, 100, provenance)).toThrow(
|
||||
"Only little-endian 64-bit Mach-O metadata is supported",
|
||||
"Unsupported 32-bit Mach-O metadata for requested CPU type arm64 (FAT header byte order: little-endian)",
|
||||
);
|
||||
});
|
||||
|
||||
|
||||
@@ -149,13 +149,37 @@ describe("pointer fixup tables", () => {
|
||||
data.writeUInt16LE(99, 46);
|
||||
const fixups = parsePointerFixups(
|
||||
data,
|
||||
{ chained: { offset: 0, size: data.length }, binds: [], dylibs: [] },
|
||||
{
|
||||
chained: { offset: 0, size: data.length },
|
||||
binds: [],
|
||||
rebases: [],
|
||||
dylibs: [],
|
||||
},
|
||||
SEGMENTS,
|
||||
BASE,
|
||||
);
|
||||
expect(fixups.failures).toContain("Unsupported chained pointer format 99");
|
||||
});
|
||||
|
||||
it("keeps unread LC_DYLD_INFO rebase bytes when chained fixups are present", () => {
|
||||
const data = chainedData(1);
|
||||
const fixups = parsePointerFixups(
|
||||
data,
|
||||
{
|
||||
chained: { offset: 0, size: data.length },
|
||||
binds: [],
|
||||
rebases: [{ offset: 8, size: 12 }],
|
||||
dylibs: [],
|
||||
},
|
||||
SEGMENTS,
|
||||
BASE,
|
||||
);
|
||||
expect(fixups).toMatchObject({
|
||||
kind: "chained",
|
||||
unreadRebaseBytes: 12,
|
||||
});
|
||||
});
|
||||
|
||||
it("does not decode unreferenced imports while constructing fixup facts", () => {
|
||||
const data = chainedData(1);
|
||||
// The second import is outside the table bytes; only the first is needed
|
||||
@@ -164,7 +188,12 @@ describe("pointer fixup tables", () => {
|
||||
expect(() =>
|
||||
parsePointerFixups(
|
||||
data,
|
||||
{ chained: { offset: 0, size: data.length }, binds: [], dylibs: [] },
|
||||
{
|
||||
chained: { offset: 0, size: data.length },
|
||||
binds: [],
|
||||
rebases: [],
|
||||
dylibs: [],
|
||||
},
|
||||
SEGMENTS,
|
||||
BASE,
|
||||
),
|
||||
@@ -179,6 +208,7 @@ describe("pointer fixup tables", () => {
|
||||
{
|
||||
chained: { offset: 0, size: data.length },
|
||||
binds: [],
|
||||
rebases: [],
|
||||
dylibs: ["/usr/lib/libobjc.A.dylib", "/usr/lib/libSystem.B.dylib"],
|
||||
},
|
||||
SEGMENTS,
|
||||
@@ -241,6 +271,7 @@ describe("pointer fixup tables", () => {
|
||||
{ offset: 0, size: opcodes.length, stream: "bind" },
|
||||
{ offset: opcodes.length, size: lazy.length, stream: "lazy" },
|
||||
],
|
||||
rebases: [],
|
||||
dylibs: ["/usr/lib/libobjc.A.dylib", "/usr/lib/libSystem.B.dylib"],
|
||||
},
|
||||
SEGMENTS,
|
||||
|
||||
@@ -6,6 +6,11 @@ import type { DecodeIssue } from "./AppleDispatchDecodeFacts.js";
|
||||
export interface FixupCommands {
|
||||
/** Absolute file range of `LC_DYLD_CHAINED_FIXUPS` data. */
|
||||
readonly chained: { readonly offset: number; readonly size: number } | null;
|
||||
/** Absolute file ranges of `LC_DYLD_INFO(_ONLY)` rebase streams. */
|
||||
readonly rebases: readonly {
|
||||
readonly offset: number;
|
||||
readonly size: number;
|
||||
}[];
|
||||
/** Absolute file ranges of `LC_DYLD_INFO(_ONLY)` bind streams. */
|
||||
readonly binds: readonly {
|
||||
readonly offset: number;
|
||||
@@ -35,6 +40,10 @@ export interface PointerFixups {
|
||||
readonly formats: readonly string[];
|
||||
readonly failures: readonly string[];
|
||||
readonly issues: readonly DecodeIssue[];
|
||||
/** Fixup segments or bind streams this reader walked. */
|
||||
readonly examined: number;
|
||||
/** Rebase opcode bytes present in the load command and not decoded. */
|
||||
readonly unreadRebaseBytes: number;
|
||||
/** Decode the pointer stored at `address` whose raw 64-bit value is `raw`. */
|
||||
decode(address: bigint, raw: bigint): DecodedPointer;
|
||||
}
|
||||
@@ -250,6 +259,8 @@ const chainedFixups = (
|
||||
formats,
|
||||
failures: issues.map(({ message }) => message),
|
||||
issues,
|
||||
examined: Math.max(ranges.length, 1),
|
||||
unreadRebaseBytes: 0,
|
||||
decode(address, raw) {
|
||||
const segment = ranges.find(
|
||||
({ from, to }) => address >= from && address < to,
|
||||
@@ -420,13 +431,24 @@ export const parsePointerFixups = (
|
||||
baseAddress: bigint,
|
||||
): PointerFixups => {
|
||||
const image = { bytes, segments, dylibs: commands.dylibs, baseAddress };
|
||||
if (commands.chained !== null) return chainedFixups(image, commands.chained);
|
||||
const unreadRebaseBytes = commands.rebases.reduce(
|
||||
(total, rebase) => total + rebase.size,
|
||||
0,
|
||||
);
|
||||
if (commands.chained !== null) {
|
||||
const chained = chainedFixups(image, commands.chained);
|
||||
return unreadRebaseBytes === 0
|
||||
? chained
|
||||
: { ...chained, unreadRebaseBytes };
|
||||
}
|
||||
if (commands.binds.length === 0)
|
||||
return {
|
||||
kind: "none",
|
||||
formats: [],
|
||||
failures: [],
|
||||
issues: [],
|
||||
examined: 0,
|
||||
unreadRebaseBytes,
|
||||
decode: (_address, raw) => ({ kind: "rebase", target: raw }),
|
||||
};
|
||||
const binds = new Map<bigint, Import>();
|
||||
@@ -446,6 +468,8 @@ export const parsePointerFixups = (
|
||||
issues: failures.map((message) =>
|
||||
issue("pointer_fixups", "bind_stream_decode_failed", message),
|
||||
),
|
||||
examined: commands.binds.length,
|
||||
unreadRebaseBytes,
|
||||
decode(address, raw) {
|
||||
const bound = binds.get(address);
|
||||
return bound === undefined
|
||||
|
||||
@@ -16,6 +16,27 @@ describe("native Mach-O slice selection", () => {
|
||||
expect(parseMachoLayout(Buffer.from(fat), "arm64").segments).toEqual([]);
|
||||
});
|
||||
|
||||
it("selects only the admitted CPU type and rejects a 32-bit slice", () => {
|
||||
const sixtyFour = fatImage([
|
||||
{ cpu: CPU.x86_64, bytes: machoImage({ cpu: CPU.x86_64 }) },
|
||||
]);
|
||||
expect(() => parseMachoLayout(Buffer.from(sixtyFour), "x86")).toThrow(
|
||||
/Requested FAT architecture is absent/u,
|
||||
);
|
||||
expect(() => parseMachoLayout(Buffer.from(sixtyFour), "arm")).toThrow(
|
||||
/Requested FAT architecture is absent/u,
|
||||
);
|
||||
const thirtyTwo = fatImage([
|
||||
{
|
||||
cpu: CPU.i386,
|
||||
bytes: machoImage({ cpu: CPU.i386, wide: false }),
|
||||
},
|
||||
]);
|
||||
expect(() => parseMachoLayout(Buffer.from(thirtyTwo), "x86")).toThrow(
|
||||
/Unsupported 32-bit Mach-O metadata for requested CPU type x86/u,
|
||||
);
|
||||
});
|
||||
|
||||
it("rejects FAT subtype disagreement and unaligned slice offsets", () => {
|
||||
const original = fatImage([{ cpu: CPU.arm64, bytes: machoImage({}) }]);
|
||||
const mismatched = original.slice();
|
||||
|
||||
@@ -90,7 +90,8 @@ export const selectMachoSlice = (
|
||||
const headerEnd = 8 + count * stride;
|
||||
if (count > 128 || headerEnd > bytes.length)
|
||||
throw malformed("Malformed FAT architecture table");
|
||||
const cpu = architecture === "arm64" ? 0x0100000c : 0x01000007;
|
||||
const cpu = MACHO_CPU_TYPE[architecture];
|
||||
if (cpu === undefined) throw invalid("Requested FAT architecture is absent");
|
||||
const declarations = readFatSliceDeclarations(bytes.subarray(8, headerEnd), {
|
||||
count,
|
||||
wide: fat64,
|
||||
@@ -115,9 +116,26 @@ export const selectMachoSlice = (
|
||||
}
|
||||
if (selected === undefined)
|
||||
throw invalid("Requested FAT architecture is absent");
|
||||
if (selected.slice + 4 > bytes.length)
|
||||
throw malformed("Truncated Mach-O header");
|
||||
const sliceMagic = bytes.readUInt32LE(selected.slice);
|
||||
if (sliceMagic === MH_MAGIC || sliceMagic === MH_CIGAM)
|
||||
throw invalid(
|
||||
`Unsupported 32-bit Mach-O metadata for requested CPU type ${architecture}`,
|
||||
);
|
||||
return selected;
|
||||
};
|
||||
|
||||
/** CPU types admitted by `machArchitecture` in the domain binary target parser. */
|
||||
const MACHO_CPU_TYPE: Readonly<Record<string, number>> = {
|
||||
x86: 7,
|
||||
x86_64: 0x01000007,
|
||||
arm: 12,
|
||||
arm64: 0x0100000c,
|
||||
};
|
||||
const MH_MAGIC = 0xfeedface;
|
||||
const MH_CIGAM = 0xcefaedfe;
|
||||
|
||||
/** Parse segments/sections for a little-endian 64-bit Mach-O architecture slice. */
|
||||
export const parseMachoLayout = (
|
||||
bytes: Buffer,
|
||||
@@ -151,6 +169,7 @@ export const parseMachoLayout = (
|
||||
sections: [],
|
||||
dylibs: [],
|
||||
binds: [],
|
||||
rebases: [],
|
||||
chained: null,
|
||||
baseAddress: undefined,
|
||||
};
|
||||
@@ -168,7 +187,7 @@ export const parseMachoLayout = (
|
||||
else collectFixupCommand(bytes, command, state);
|
||||
cursor += command.size;
|
||||
}
|
||||
const { segments, sections, chained, binds, dylibs } = state;
|
||||
const { segments, sections, chained, binds, rebases, dylibs } = state;
|
||||
const offset = (address: bigint, size = 1): number => {
|
||||
const matches = segments.filter(
|
||||
(segment) =>
|
||||
@@ -187,7 +206,7 @@ export const parseMachoLayout = (
|
||||
segments,
|
||||
sections,
|
||||
offset,
|
||||
fixups: { chained, binds, dylibs },
|
||||
fixups: { chained, binds, rebases, dylibs },
|
||||
baseAddress: state.baseAddress ?? 0n,
|
||||
};
|
||||
};
|
||||
@@ -200,6 +219,7 @@ interface LayoutState {
|
||||
readonly sections: Section[];
|
||||
readonly dylibs: string[];
|
||||
readonly binds: FixupCommands["binds"][number][];
|
||||
readonly rebases: FixupCommands["rebases"][number][];
|
||||
chained: FixupCommands["chained"];
|
||||
baseAddress: bigint | undefined;
|
||||
}
|
||||
@@ -273,6 +293,8 @@ const collectFixupCommand = (
|
||||
state.chained = range(cursor + 8);
|
||||
} else if (kind === LC_DYLD_INFO || kind === LC_DYLD_INFO_ONLY) {
|
||||
if (size < 48) throw new RangeError("Truncated dyld-info command");
|
||||
const rebase = range(cursor + 8);
|
||||
if (rebase.size > 0) state.rebases.push(rebase);
|
||||
for (const [at, stream] of [
|
||||
[cursor + 16, "bind"],
|
||||
[cursor + 24, "weak"],
|
||||
|
||||
@@ -66,6 +66,8 @@ describe("Swift vtable coverage facts: unresolved Swift dispatch slots", () => {
|
||||
failures: [],
|
||||
issues: [],
|
||||
decode: (_address, raw) => ({ kind: "rebase", target: raw }),
|
||||
examined: 0,
|
||||
unreadRebaseBytes: 0,
|
||||
},
|
||||
swift: {
|
||||
facet: "swift_conformances_static_witness_slots",
|
||||
@@ -178,6 +180,8 @@ describe("Swift vtable coverage facts: unsupported pointer fixup coverage", () =
|
||||
kind: "unsupported",
|
||||
reason: "Unsupported chained pointer format 99",
|
||||
}),
|
||||
examined: 0,
|
||||
unreadRebaseBytes: 0,
|
||||
},
|
||||
swift: {
|
||||
facet: "swift_conformances_static_witness_slots",
|
||||
|
||||
@@ -0,0 +1,73 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import { AnalysisCancelledError } from "../domain/analysisErrorCore.js";
|
||||
import { err, ok } from "../domain/result.js";
|
||||
import type { NativeCommandCapture } from "./CommandRunner.js";
|
||||
import { inspectNativeMacho } from "./NativeMachoInspection.js";
|
||||
|
||||
const capture = (tool: string, stdout: string): NativeCommandCapture => ({
|
||||
tool,
|
||||
executable: tool,
|
||||
executableSha256: "a".repeat(64),
|
||||
toolVersion: null,
|
||||
versionReason: null,
|
||||
arguments: [],
|
||||
stdout,
|
||||
stderr: "",
|
||||
stdoutBytes: stdout.length,
|
||||
stderrBytes: 0,
|
||||
exitCode: 0,
|
||||
signal: null,
|
||||
});
|
||||
|
||||
const inspect = async (lipo: string, architecture: "x86" | "arm64") => {
|
||||
const commands: {
|
||||
readonly tool: string;
|
||||
readonly args: readonly string[];
|
||||
}[] = [];
|
||||
const result = await inspectNativeMacho({
|
||||
target: {
|
||||
kind: "executable",
|
||||
format: "mach-o",
|
||||
path: "/fixture/app",
|
||||
sha256: "b".repeat(64),
|
||||
architecture,
|
||||
availableArchitectures: ["x86", "arm64"],
|
||||
},
|
||||
run: (tool, args) => {
|
||||
commands.push({ tool, args });
|
||||
if (tool === "lipo") return Promise.resolve(ok(capture(tool, lipo)));
|
||||
if (tool === "file") return Promise.resolve(ok(capture(tool, "Mach-O")));
|
||||
return Promise.resolve(err(new AnalysisCancelledError("inspect_native")));
|
||||
},
|
||||
invocation: () => {
|
||||
throw new Error("invocation is not reached");
|
||||
},
|
||||
});
|
||||
return { result, commands };
|
||||
};
|
||||
|
||||
describe("Mach-O tool architecture selection", () => {
|
||||
it("rejects a CPU lipo did not list instead of reading another slice", async () => {
|
||||
const { result, commands } = await inspect("architecture arm64\n", "x86");
|
||||
expect(result.ok).toBe(false);
|
||||
if (result.ok) return;
|
||||
expect(result.error).toMatchObject({
|
||||
_tag: "AnalysisOutputError",
|
||||
reason: "lipo did not list i386; refusing to read another slice",
|
||||
});
|
||||
expect(commands.map(({ tool }) => tool)).toEqual(["file", "lipo"]);
|
||||
});
|
||||
|
||||
it("passes the arm64e slice lipo listed for an arm64 target", async () => {
|
||||
const { commands } = await inspect("architecture arm64e\n", "arm64");
|
||||
const otool = commands.find(({ tool }) => tool === "otool");
|
||||
expect(otool?.args).toEqual([
|
||||
"-h",
|
||||
"-l",
|
||||
"-arch",
|
||||
"arm64e",
|
||||
"/fixture/app",
|
||||
]);
|
||||
});
|
||||
});
|
||||
@@ -1,6 +1,7 @@
|
||||
import type { BinaryTarget } from "../domain/binaryTargetTypes.js";
|
||||
import type { EvidenceLocation } from "../domain/evidence.js";
|
||||
import type { AnalysisError } from "../domain/analysisErrorBase.js";
|
||||
import { AnalysisOutputError } from "../domain/analysisErrorCore.js";
|
||||
import {
|
||||
inspectMachoSchema,
|
||||
type NativeCommandInvocation,
|
||||
@@ -65,10 +66,19 @@ export const inspectNativeMacho = async (
|
||||
}
|
||||
const lipo = captures.find(({ tool }) => tool === "lipo");
|
||||
if (lipo === undefined) throw new TypeError("Missing lipo capture");
|
||||
const selectedArchitecture = selectArchitecture(
|
||||
context.target,
|
||||
parseLipoArchitectures(lipo.stdout).map(({ name }) => name),
|
||||
const availableArchitectures = parseLipoArchitectures(lipo.stdout).map(
|
||||
({ name }) => name,
|
||||
);
|
||||
const selection = selectArchitecture(context.target, availableArchitectures);
|
||||
if (selection.status === "absent")
|
||||
return err(
|
||||
new AnalysisOutputError(
|
||||
"inspect_macho",
|
||||
`lipo did not list ${selection.architecture}; refusing to read another slice`,
|
||||
),
|
||||
);
|
||||
const selectedArchitecture =
|
||||
selection.status === "selected" ? selection.architecture : null;
|
||||
for (const [tool, prefix] of REQUIRED_SLICE_COMMANDS) {
|
||||
const captured = await context.run(
|
||||
tool,
|
||||
@@ -339,24 +349,32 @@ const selectedArchitectureArguments = (
|
||||
: ["-arch", toolArchitecture];
|
||||
};
|
||||
|
||||
type ArchitectureSelection =
|
||||
| { readonly status: "unfiltered" }
|
||||
| { readonly status: "selected"; readonly architecture: string }
|
||||
| { readonly status: "absent"; readonly architecture: string };
|
||||
|
||||
const selectArchitecture = (
|
||||
target: BinaryTarget,
|
||||
available: readonly string[],
|
||||
): string | null => {
|
||||
): ArchitectureSelection => {
|
||||
if (target.kind !== "executable" || target.availableArchitectures.length < 2)
|
||||
return null;
|
||||
return { status: "unfiltered" };
|
||||
const normalized =
|
||||
target.architecture === "x86" ? "i386" : target.architecture;
|
||||
if (available.includes(normalized)) return normalized;
|
||||
if (available.includes(normalized))
|
||||
return { status: "selected", architecture: normalized };
|
||||
// BinaryTarget intentionally normalizes ARM64 CPU subtypes. Recover the
|
||||
// concrete arm64e slice name from lipo before invoking slice-aware tools.
|
||||
if (normalized === "arm64") {
|
||||
const arm64e = available.find((architecture) => architecture === "arm64e");
|
||||
if (arm64e !== undefined) return arm64e;
|
||||
if (arm64e !== undefined)
|
||||
return { status: "selected", architecture: arm64e };
|
||||
const variant = available.find((architecture) =>
|
||||
/^arm64e\.[A-Za-z0-9_]+$/u.test(architecture),
|
||||
);
|
||||
if (variant !== undefined) return variant;
|
||||
if (variant !== undefined)
|
||||
return { status: "selected", architecture: variant };
|
||||
}
|
||||
return normalized;
|
||||
return { status: "absent", architecture: normalized };
|
||||
};
|
||||
|
||||
@@ -75,6 +75,7 @@ interface TerminalExitOptions {
|
||||
readonly interactions: InteractionEvent[];
|
||||
readonly dispatchedEventIndexes: ReadonlySet<number>;
|
||||
readonly recordEvent: RecordProcessCaptureEvent;
|
||||
readonly onLiveProgress?: () => void;
|
||||
}
|
||||
|
||||
interface CaptureResultOptions {
|
||||
@@ -601,8 +602,10 @@ export const observeSettlement = async (
|
||||
recordEvent: RecordProcessCaptureEvent = () => undefined,
|
||||
platform: NodeJS.Platform = process.platform,
|
||||
signal?: AbortSignal,
|
||||
onLiveProgress?: () => void,
|
||||
): Promise<ObservedProcessSettlement> => {
|
||||
assertNotCancelled(signal);
|
||||
onLiveProgress?.();
|
||||
if (platform === "win32") {
|
||||
recordEvent("lifecycle", 1);
|
||||
return { state: "unverifiable", elapsed_ms: 0 };
|
||||
@@ -611,6 +614,7 @@ export const observeSettlement = async (
|
||||
let consecutiveEmpty = 0;
|
||||
let deadlineReached = false;
|
||||
while (!deadlineReached) {
|
||||
onLiveProgress?.();
|
||||
assertNotCancelled(signal);
|
||||
const observations = await Promise.all(
|
||||
[...new Set(processGroupIds)].map((processGroupId) =>
|
||||
@@ -651,6 +655,7 @@ export const awaitTerminalExit = async ({
|
||||
interactions,
|
||||
dispatchedEventIndexes,
|
||||
recordEvent,
|
||||
onLiveProgress,
|
||||
}: TerminalExitOptions): Promise<{
|
||||
exitCode: number;
|
||||
signal?: number;
|
||||
@@ -661,6 +666,7 @@ export const awaitTerminalExit = async ({
|
||||
// Keep the initiating lifecycle reason so comparisons distinguish a target
|
||||
// exit from harness-owned timeout, idle-timeout, and cancellation cleanup.
|
||||
let reason: "exited" | "timeout" | "idle_timeout" | "cancelled" = "exited";
|
||||
onLiveProgress?.();
|
||||
terminal.onExit((exit) => {
|
||||
recordEvent("lifecycle", 0);
|
||||
for (const [eventIndex, event] of scenario.events.entries()) {
|
||||
@@ -688,6 +694,7 @@ export const awaitTerminalExit = async ({
|
||||
resolveExit({ ...exit, reason });
|
||||
});
|
||||
const timeout = scheduleProcessInterval(() => {
|
||||
onLiveProgress?.();
|
||||
if (signal?.aborted === true) {
|
||||
reason = "cancelled";
|
||||
terminal.kill("SIGKILL");
|
||||
|
||||
@@ -0,0 +1,113 @@
|
||||
import { expect, it } from "vitest";
|
||||
|
||||
import {
|
||||
createProcessCaptureProgressTracker,
|
||||
type ProcessCaptureProgress,
|
||||
} from "./ProcessCaptureProgress.js";
|
||||
|
||||
const blockedProgress = () => {
|
||||
let release: () => void = () => undefined;
|
||||
const blocked = new Promise<void>((resolve) => {
|
||||
release = resolve;
|
||||
});
|
||||
const delivered: Parameters<ProcessCaptureProgress["report"]>[0][] = [];
|
||||
const progress: ProcessCaptureProgress = {
|
||||
async report(update) {
|
||||
delivered.push(update);
|
||||
if (delivered.length === 1) await blocked;
|
||||
},
|
||||
};
|
||||
return { progress, delivered, release: () => release() };
|
||||
};
|
||||
|
||||
it("coalesces intermediate observations while the progress receiver is blocked", async () => {
|
||||
const receiver = blockedProgress();
|
||||
const tracker = createProcessCaptureProgressTracker(receiver.progress);
|
||||
void tracker.report({
|
||||
phase: "running",
|
||||
counts: { frames: 1, samples: 0, interactions: 0 },
|
||||
});
|
||||
await Promise.resolve();
|
||||
let latest: Promise<void> | undefined;
|
||||
for (let frames = 2; frames <= 1_000; frames += 1) {
|
||||
latest = tracker.report({
|
||||
phase: "running",
|
||||
counts: { frames, samples: 0, interactions: 0 },
|
||||
});
|
||||
}
|
||||
expect(receiver.delivered).toHaveLength(1);
|
||||
receiver.release();
|
||||
await latest;
|
||||
expect(receiver.delivered.map((update) => update.completed)).toEqual([
|
||||
1, 1_000,
|
||||
]);
|
||||
expect(receiver.delivered[1]?.message).toContain("frames=1000");
|
||||
});
|
||||
|
||||
it("discards stale live observations before delivering final cleanup", async () => {
|
||||
const receiver = blockedProgress();
|
||||
const tracker = createProcessCaptureProgressTracker(receiver.progress);
|
||||
void tracker.report({
|
||||
phase: "running",
|
||||
counts: { frames: 1, samples: 0, interactions: 0 },
|
||||
});
|
||||
await Promise.resolve();
|
||||
void tracker.report({
|
||||
phase: "running",
|
||||
counts: { frames: 2, samples: 0, interactions: 0 },
|
||||
});
|
||||
void tracker.report({
|
||||
phase: "settling",
|
||||
counts: { frames: 2, samples: 1, interactions: 0 },
|
||||
});
|
||||
tracker.closeLive();
|
||||
await tracker.report({
|
||||
phase: "running",
|
||||
counts: { frames: 10, samples: 10, interactions: 10 },
|
||||
});
|
||||
const terminal = tracker.report({
|
||||
phase: "cleanup",
|
||||
counts: { frames: 2, samples: 2, interactions: 0 },
|
||||
disposition: "exited",
|
||||
terminal: true,
|
||||
});
|
||||
receiver.release();
|
||||
await terminal;
|
||||
expect(receiver.delivered.map((update) => update.phase)).toEqual([
|
||||
"running",
|
||||
"cleanup",
|
||||
]);
|
||||
expect(receiver.delivered.at(-1)).toMatchObject({
|
||||
completed: 4,
|
||||
total: null,
|
||||
terminal: true,
|
||||
});
|
||||
});
|
||||
|
||||
it("still delivers terminal status after an observer rejects an update", async () => {
|
||||
const delivered: Parameters<ProcessCaptureProgress["report"]>[0][] = [];
|
||||
const tracker = createProcessCaptureProgressTracker({
|
||||
async report(update) {
|
||||
delivered.push(update);
|
||||
if (delivered.length === 1) throw new Error("Observer disconnected");
|
||||
},
|
||||
});
|
||||
await tracker.report({
|
||||
phase: "prepare",
|
||||
counts: { frames: 0, samples: 0, interactions: 0 },
|
||||
});
|
||||
tracker.closeLive();
|
||||
await expect(
|
||||
tracker.report({
|
||||
phase: "cleanup",
|
||||
counts: { frames: 1, samples: 1, interactions: 0 },
|
||||
disposition: "failed",
|
||||
terminal: true,
|
||||
}),
|
||||
).resolves.toBeUndefined();
|
||||
expect(delivered.at(-1)).toMatchObject({
|
||||
phase: "cleanup",
|
||||
completed: 2,
|
||||
terminal: true,
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,121 @@
|
||||
import type { ProcessCaptureCleanupReport } from "../../domain/process/processCapture.js";
|
||||
|
||||
/**
|
||||
* Progress port accepted by process capture.
|
||||
* Structurally matches the shared application `ProgressReporter`.
|
||||
*/
|
||||
export interface ProcessCaptureProgress {
|
||||
report(update: {
|
||||
readonly phase: string;
|
||||
readonly completed: number;
|
||||
readonly total: null;
|
||||
readonly message: string;
|
||||
readonly terminal?: boolean;
|
||||
}): Promise<void>;
|
||||
}
|
||||
|
||||
/** Observed collection counts. These are not a percentage of an unknown total. */
|
||||
export interface ProcessCaptureProgressCounts {
|
||||
readonly frames: number;
|
||||
readonly samples: number;
|
||||
readonly interactions: number;
|
||||
}
|
||||
|
||||
export type ProcessCaptureProgressPhase =
|
||||
| "prepare"
|
||||
| "running"
|
||||
| "settling"
|
||||
| "cleanup";
|
||||
|
||||
export type ProcessCaptureProgressDisposition =
|
||||
| "exited"
|
||||
| "timeout"
|
||||
| "idle_timeout"
|
||||
| "cancelled"
|
||||
| "failed";
|
||||
|
||||
export interface ProcessCaptureProgressReport {
|
||||
readonly phase: ProcessCaptureProgressPhase;
|
||||
readonly counts: ProcessCaptureProgressCounts;
|
||||
readonly terminal?: boolean;
|
||||
readonly disposition?: ProcessCaptureProgressDisposition;
|
||||
readonly cleanup?: ProcessCaptureCleanupReport;
|
||||
}
|
||||
|
||||
const formatProgressMessage = (
|
||||
elapsedMs: number,
|
||||
counts: ProcessCaptureProgressCounts,
|
||||
disposition: ProcessCaptureProgressDisposition | undefined,
|
||||
cleanup: ProcessCaptureCleanupReport | undefined,
|
||||
): string => {
|
||||
const status = `elapsed_ms=${String(elapsedMs)} frames=${String(counts.frames)} samples=${String(counts.samples)} interactions=${String(counts.interactions)}`;
|
||||
const dispositionText =
|
||||
disposition === undefined ? "" : ` disposition=${disposition}`;
|
||||
const cleanupText =
|
||||
cleanup === undefined
|
||||
? ""
|
||||
: ` owned_process_group=${cleanup.owned_process_group.state} terminal_renderer=${cleanup.terminal_renderer.state} temporary_root=${cleanup.temporary_root.state}`;
|
||||
return `${status}${dispositionText}${cleanupText}`;
|
||||
};
|
||||
|
||||
/** Serialize observed capture status through one rate-limited reporter. */
|
||||
export const createProcessCaptureProgressTracker = (
|
||||
progress: ProcessCaptureProgress | undefined,
|
||||
now: () => number = Date.now,
|
||||
) => {
|
||||
const started = now();
|
||||
let completed = 0;
|
||||
let live = true;
|
||||
let next: Parameters<ProcessCaptureProgress["report"]>[0] | undefined;
|
||||
let pending: Promise<void> | undefined;
|
||||
const flush = async (): Promise<void> => {
|
||||
try {
|
||||
while (next !== undefined) {
|
||||
const update = next;
|
||||
next = undefined;
|
||||
try {
|
||||
await progress?.report(update);
|
||||
} catch {
|
||||
// Progress delivery is observational; capture and cleanup still finish.
|
||||
continue;
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
pending = undefined;
|
||||
}
|
||||
};
|
||||
return {
|
||||
closeLive(): void {
|
||||
live = false;
|
||||
if (next?.terminal !== true) next = undefined;
|
||||
},
|
||||
report(update: ProcessCaptureProgressReport): Promise<void> {
|
||||
if (progress === undefined) return Promise.resolve();
|
||||
if (update.terminal !== true && !live) return Promise.resolve();
|
||||
const units =
|
||||
update.counts.frames +
|
||||
update.counts.samples +
|
||||
update.counts.interactions;
|
||||
if (units > completed) completed = units;
|
||||
// A slow observer needs the latest observation, not a queue of old states.
|
||||
next = {
|
||||
phase: update.phase,
|
||||
completed,
|
||||
total: null as null,
|
||||
message: formatProgressMessage(
|
||||
Math.max(0, now() - started),
|
||||
update.counts,
|
||||
update.disposition,
|
||||
update.cleanup,
|
||||
),
|
||||
...(update.terminal === true ? { terminal: true as const } : {}),
|
||||
};
|
||||
pending ??= Promise.resolve().then(flush);
|
||||
return pending;
|
||||
},
|
||||
};
|
||||
};
|
||||
|
||||
export type ProcessCaptureProgressTracker = ReturnType<
|
||||
typeof createProcessCaptureProgressTracker
|
||||
>;
|
||||
@@ -55,6 +55,13 @@ import {
|
||||
import { makeProcessCaptureEnvironment } from "./ProcessCaptureEnvironment.js";
|
||||
import { classifyFilesystemEffects } from "./ProcessFilesystemEffects.js";
|
||||
import { processCaptureOwnershipUnavailableReason } from "./ProcessCaptureCapability.js";
|
||||
import {
|
||||
createProcessCaptureProgressTracker,
|
||||
type ProcessCaptureProgress,
|
||||
type ProcessCaptureProgressCounts,
|
||||
type ProcessCaptureProgressDisposition,
|
||||
type ProcessCaptureProgressTracker,
|
||||
} from "./ProcessCaptureProgress.js";
|
||||
import type { ProcessOwnershipBaseline } from "../ProcessOwnership.js";
|
||||
|
||||
interface StartedCaptureRuntime {
|
||||
@@ -262,7 +269,10 @@ const finishProcessRun = async (options: {
|
||||
readonly executionFailure: unknown;
|
||||
readonly observations?: ProcessCaptureObservationBuffer;
|
||||
readonly rootPid?: number;
|
||||
readonly progress: ProcessCaptureProgressTracker;
|
||||
readonly progressCounts: () => ProcessCaptureProgressCounts;
|
||||
}): Promise<ProcessCapture> => {
|
||||
options.progress.closeLive();
|
||||
await options.stopSampler();
|
||||
const sampledProcessGroupIds =
|
||||
options.runtime === undefined
|
||||
@@ -281,6 +291,20 @@ const finishProcessRun = async (options: {
|
||||
temporaryRoot: options.temporaryRoot,
|
||||
...(options.cleanupHost === undefined ? {} : { host: options.cleanupHost }),
|
||||
});
|
||||
const observedExit =
|
||||
options.observations?.exit.state === "available"
|
||||
? options.observations.exit.value.reason
|
||||
: options.capture?.exit.reason;
|
||||
await options.progress.report({
|
||||
phase: "cleanup",
|
||||
counts: options.progressCounts(),
|
||||
terminal: true,
|
||||
disposition: captureProgressDisposition(
|
||||
observedExit,
|
||||
options.executionFailure,
|
||||
),
|
||||
cleanup,
|
||||
});
|
||||
const cleanupFailure = cleanupReportFailure(cleanup);
|
||||
let { capture, executionFailure } = options;
|
||||
let verifiedCapture: ProcessCapture | undefined;
|
||||
@@ -336,6 +360,7 @@ const completeCapture = async (options: {
|
||||
readonly eventJournal: readonly ProcessCaptureEventJournalEntry[];
|
||||
readonly observationBuffer: ProcessCaptureObservationBuffer;
|
||||
readonly recordEvent: RecordProcessCaptureEvent;
|
||||
readonly onLiveProgress?: () => void;
|
||||
}): Promise<PendingProcessCapture> => {
|
||||
const unavailableReason = (stage: string, cause: unknown): string =>
|
||||
`${stage} failed: ${cause instanceof Error ? cause.message : "unknown failure"}`;
|
||||
@@ -351,6 +376,7 @@ const completeCapture = async (options: {
|
||||
options.recordEvent,
|
||||
options.hostPlatform,
|
||||
options.signal,
|
||||
options.onLiveProgress,
|
||||
);
|
||||
} catch (cause: unknown) {
|
||||
options.observationBuffer.settlement = {
|
||||
@@ -460,6 +486,25 @@ const completeCapture = async (options: {
|
||||
});
|
||||
};
|
||||
|
||||
const captureProgressDisposition = (
|
||||
exitReason: string | undefined,
|
||||
executionFailure: unknown,
|
||||
): ProcessCaptureProgressDisposition => {
|
||||
if (
|
||||
exitReason === "exited" ||
|
||||
exitReason === "timeout" ||
|
||||
exitReason === "idle_timeout" ||
|
||||
exitReason === "cancelled"
|
||||
)
|
||||
return exitReason;
|
||||
if (
|
||||
executionFailure instanceof ProcessCaptureError &&
|
||||
executionFailure.reason === "cancelled"
|
||||
)
|
||||
return "cancelled";
|
||||
return "failed";
|
||||
};
|
||||
|
||||
/** Execute one caller-selected scenario and return bounded observations. */
|
||||
const runProcessScenario = async (
|
||||
scenario: ProcessScenario,
|
||||
@@ -468,12 +513,44 @@ const runProcessScenario = async (
|
||||
hostPlatform: NodeJS.Platform = process.platform,
|
||||
captureSnapshot: typeof snapshotRoots = snapshotRoots,
|
||||
cleanupHost?: ProcessCaptureCleanupHost,
|
||||
progress?: ProcessCaptureProgress,
|
||||
): Promise<ProcessCapture> => {
|
||||
const { temporaryRoot, runId, ownershipBaseline, before } =
|
||||
await prepareProcessCapture(scenario, signal, captureSnapshot);
|
||||
const progressTracker = createProcessCaptureProgressTracker(progress);
|
||||
const frames: TerminalFrame[] = [];
|
||||
const samples: ProcessSample[] = [];
|
||||
const interactions: InteractionEvent[] = [];
|
||||
const progressCounts = (): ProcessCaptureProgressCounts => ({
|
||||
frames: frames.length,
|
||||
samples: samples.length,
|
||||
interactions: interactions.length,
|
||||
});
|
||||
const reportRunning = () => {
|
||||
void progressTracker.report({ phase: "running", counts: progressCounts() });
|
||||
};
|
||||
const reportSettling = () => {
|
||||
void progressTracker.report({
|
||||
phase: "settling",
|
||||
counts: progressCounts(),
|
||||
});
|
||||
};
|
||||
let prepared: Awaited<ReturnType<typeof prepareProcessCapture>>;
|
||||
try {
|
||||
await progressTracker.report({
|
||||
phase: "prepare",
|
||||
counts: progressCounts(),
|
||||
});
|
||||
prepared = await prepareProcessCapture(scenario, signal, captureSnapshot);
|
||||
} catch (cause: unknown) {
|
||||
progressTracker.closeLive();
|
||||
await progressTracker.report({
|
||||
phase: "cleanup",
|
||||
counts: progressCounts(),
|
||||
terminal: true,
|
||||
disposition: signal?.aborted === true ? "cancelled" : "failed",
|
||||
});
|
||||
throw cause;
|
||||
}
|
||||
const { temporaryRoot, runId, ownershipBaseline, before } = prepared;
|
||||
const journal = createProcessCaptureJournal();
|
||||
const { entries: eventJournal, recordEvent } = journal;
|
||||
recordEvent("filesystem_checkpoints", 0);
|
||||
@@ -523,6 +600,7 @@ const runProcessScenario = async (
|
||||
interactions,
|
||||
dispatchedEventIndexes,
|
||||
recordEvent,
|
||||
...(progress === undefined ? {} : { onLiveProgress: reportRunning }),
|
||||
});
|
||||
observations.exit = {
|
||||
state: "available",
|
||||
@@ -549,6 +627,7 @@ const runProcessScenario = async (
|
||||
recordEvent,
|
||||
captureSnapshot,
|
||||
...(signal === undefined ? {} : { signal }),
|
||||
...(progress === undefined ? {} : { onLiveProgress: reportSettling }),
|
||||
});
|
||||
} catch (cause: unknown) {
|
||||
executionFailure = normalizeCaptureFailure(cause, signal);
|
||||
@@ -590,6 +669,8 @@ const runProcessScenario = async (
|
||||
executionFailure,
|
||||
observations,
|
||||
...(actualRootPid === undefined ? {} : { rootPid: actualRootPid }),
|
||||
progress: progressTracker,
|
||||
progressCounts,
|
||||
});
|
||||
};
|
||||
|
||||
@@ -599,8 +680,9 @@ export const captureProcessScenario = async (
|
||||
signal?: AbortSignal,
|
||||
platform: NodeJS.Platform = process.platform,
|
||||
environment: Readonly<Record<string, string | undefined>> = process.env,
|
||||
captureSnapshot: typeof snapshotRoots = snapshotRoots,
|
||||
captureSnapshot?: typeof snapshotRoots,
|
||||
cleanupHost?: ProcessCaptureCleanupHost,
|
||||
progress?: ProcessCaptureProgress,
|
||||
): Promise<Result<ProcessCapture, ProcessCaptureError | AnalysisError>> => {
|
||||
const ownershipReason = processCaptureOwnershipUnavailableReason(platform);
|
||||
if (ownershipReason !== undefined)
|
||||
@@ -624,8 +706,9 @@ export const captureProcessScenario = async (
|
||||
signal,
|
||||
environment,
|
||||
platform,
|
||||
captureSnapshot,
|
||||
captureSnapshot ?? snapshotRoots,
|
||||
cleanupHost,
|
||||
progress,
|
||||
),
|
||||
);
|
||||
} catch (cause: unknown) {
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user