chore(review): integrate main for returned-target index verification

This commit is contained in:
N0zoM1z0
2026-10-10 19:51:10 +08:00
133 changed files with 9459 additions and 1062 deletions
@@ -31,7 +31,7 @@ When tools are absent or registration is stale:
`npx -y rea-agents@latest doctor --client codex --json`. Substitute the current
supported client: `claude_code`, `claude_desktop`, `codex`, `cursor`,
`gemini_cli`, `windsurf`, `devin`, `opencode`, `antigravity`, `copilot_cli`,
`commandcode`, `vscode`, `grok_build`, `omp`, or `grok_bot`. `grok_bot` has no local
`commandcode`, `qwen_code`, `vscode`, `grok_build`, `omp`, `pi`, `hermes`, or `grok_bot`. `grok_bot` has no local
file registration: doctor reports the Grok Bot chat step, and setup does not
write `mcp.json`. If the client is unknown, use `doctor --json` and
inspect its registration results before choosing a setup scope.
+73 -7
View File
@@ -175,9 +175,26 @@ Setup and doctor follow Qwen Code's tilde and working-directory-relative home
overrides. Qwen Code also discovers personal skills in `~/.agents/skills`,
where setup installs the bundled workflow.
Gemini CLI uses `$GEMINI_CLI_HOME/.gemini/settings.json` when that home override
is nonempty, including its shared personal skills at
`$GEMINI_CLI_HOME/.agents/skills`. Setup and doctor check MCP allow/exclude
policies from system defaults, user settings, trusted workspace settings, and
system overrides. Workspace trust follows `GEMINI_CLI_TRUST_WORKSPACE`,
`GEMINI_RESTRICTED_MODE`, the effective `security.folderTrust.enabled` setting,
and the selected `trustedFolders.json` (including
`GEMINI_CLI_TRUSTED_FOLDERS_PATH`). Settings in an untrusted workspace do not
participate. A workspace or system `mcpServers.rea` replaces the entire user
entry; setup stops if the winning definition conflicts with its planned entry.
Review those files manually: setup never edits administrator/workspace settings
or grants workspace trust. Trust supplied only by a running IDE is outside this
standalone CLI check.
Codex treats an unset or empty `CODEX_HOME` as `~/.codex`. Devin honors
`XDG_CONFIG_HOME` for its `devin/mcp_config.json` on non-Windows systems.
For OMP, setup writes a `type: "stdio"` entry to the user-level
`~/.omp/agent/mcp.json`. It follows `PI_CONFIG_DIR`, an absolute
`PI_CODING_AGENT_DIR`, and the profile selected by `OMP_PROFILE` or
`~/.omp/agent/mcp.json`. It follows `PI_CONFIG_DIR` and
`PI_CODING_AGENT_DIR` (relative to the setup working directory), and the profile selected by `OMP_PROFILE` or
`PI_PROFILE` (`~/.omp/profiles/<name>/agent/mcp.json`). Setup also removes
`rea` from that file's `disabledServers` list, which would otherwise hide the
registration. Doctor treats an `enabled: false` entry as active when
@@ -191,17 +208,65 @@ Windows, `~/.hermes` elsewhere — including any `HERMES_DATA_DIR_SUFFIX`. Herme
connects an entry unless `enabled` is false, so setup states `enabled: true`.
Hermes personal skills install under the active Hermes home's `skills` directory.
Setup, doctor and uninstall follow the root's `active_profile` selection, as
plain Hermes CLI commands do. An explicit `HERMES_HOME` pointing to
`profiles/<name>` selects that named profile directly. An invalid, missing or
deleted sticky profile is reported instead of writing into the default home;
repair the profile or run `hermes profile use default` before retrying.
REA follows `HERMES_HOME` whitespace trimming, environment-variable expansion,
and home expansion before resolving that profile. Setup previews the resolved
configuration and skill destinations; doctor and uninstall use those same paths.
### DeepSeek Harness
DeepSeek Harness uses a Cordis YAML patch sequence rather than a `mcpServers`
object. REA's `setup --client` does not edit this format. For Harness
`0.2.0-rc.2`, append the following entry to `$DSH_HOME/cordis.patch.yml`
(`~/.dsh/cordis.patch.yml` when `DSH_HOME` is unset). Preserve existing patches
and replace the executable paths with absolute paths from your installation:
```yaml
- insert:
- id: rea-mcp
name: "@deepseek-ai/dsh-mcp-client"
config:
serverName: rea
transport: stdio
command: /absolute/path/to/node
args: ["/absolute/path/to/rea/scripts/rea.mjs", mcp]
cwd: !!js process.cwd()
```
For a global npm installation, the launcher is under
`$(npm root -g)/rea-agents/scripts/rea.mjs`. Check the Node executable with
`command -v node`. Register `serverName: rea` only once in each active profile,
then restart Harness. It exposes tools as `mcp__rea__<tool_name>`.
Run `rea setup --skill --dry-run`, review the destination, then
`rea setup --skill --yes` to install the workflow in `~/.agents/skills`.
Harness discovers that directory, or `$DSH_AGENTS_HOME/skills` when configured;
its native `skill` tool can load `reverse-engineer-anything`.
REA's optional [native client verification](testing.md#end-to-end-integration-and-golden-evidence)
checks headless chat, skill loading, all tool input schemas and actual analysis
using the installed Harness and a local model fixture. This establishes the
Harness integration, with live DeepSeek API compatibility remaining unverified.
See the [Harness MCP guide](https://github.com/deepseek-ai/deepseek-harness/blob/639ed015397290b3745d163aafe02ffee4aa3f84/packages/mcp/mcp-client/README.md)
for its native configuration contract.
### Other client settings
For OpenCode, setup writes the V1 `mcp.rea` entry, which OpenCode V1 and V2
both load. If the configuration already uses OpenCode V2's native
`mcp.servers` table, setup registers REA there instead and replaces any earlier
`mcp.rea` entry from REA.
`mcp.rea` entry from REA. Setup honors `OPENCODE_CONFIG_DIR` and
`OPENCODE_CONFIG`, preferring the later-loaded JSONC file when both JSON and
JSONC exist. The ordinary XDG global configuration still participates when a
custom directory is selected. Doctor checks the merged user registration; uninstall removes
REA-owned entries from every applicable user file so an older entry cannot
become active again. Malformed applicable files block setup before changes.
Grok Build loads `[mcp_servers.rea]` from `$GROK_HOME/config.toml`, or from
`~/.grok/config.toml` when `GROK_HOME` is unset. Setup edits that server
`~/.grok/config.toml` when `GROK_HOME` is unset or empty. Setup edits that server
table, `[mcp_servers.rea.env]`, and a root `disabled_mcp_servers` entry that
names `rea`. It sets `startup_timeout_sec = 30` and leaves every other name
in that list. The shared skill installed under `~/.agents/skills` is already
@@ -241,9 +306,10 @@ save verified paths for an existing Ghidra installation.
The bundled skill is installed where each selected client discovers personal
skills: Claude Code uses `~/.claude/skills` (or
`$CLAUDE_CONFIG_DIR/skills` when configured), while other supported clients,
including Qwen Code, use the shared `~/.agents/skills` directory. A mixed selection
plans the relevant paths. Selecting the skill without a client uses the shared
`$CLAUDE_CONFIG_DIR/skills` when configured). Gemini CLI uses `.agents/skills`
under its selected home; Antigravity 2.0 uses `~/.gemini/config/skills`.
Other integrations, including Qwen Code, use `~/.agents/skills`.
A mixed selection plans each distinct destination. Selecting the skill without a client uses the shared
directory. Setup leaves existing skill copies in other locations untouched.
`doctor --skill --json` verifies the selected copies against the bundled
+49
View File
@@ -36,6 +36,49 @@ after SDK conversion and check advertised validation and actual calls.
Individual model APIs can impose additional nesting limits; complete producer
captures can exceed ten structural levels.
### Compact input schema profile
Some function-calling providers reject a request while any single tool's
`parameters` schema exceeds a literal serialized size cap; Moonshot clients
such as kimi-cli fail every request while the default catalog is attached
([#1484](https://github.com/morluto/rea/issues/1484)). Setting the server's
`REA_MCP_INPUT_SCHEMA_PROFILE` environment variable to `compact` renders every
advertised input schema within a 13,312-byte budget, measured to sit below
Moonshot's observed cap (community measurements: a 13.7 KB schema is accepted
while a ~15 KB one is rejected). The setting is read at startup; restart the
server to apply changes, as with `REA_MCP_MAX_RESPONSE_BYTES`. The default
value `full` keeps the complete advertisement unchanged.
The compact profile renders the same tool set, output schemas, annotations,
and canonical server-side validation. It changes only the advertised input
JSON Schema:
- Annotation prose leaves the advertised form: nested property descriptions,
literal examples, defaults, and titles are dropped. The root keeps its
description because it states the accepted input groups.
- Schema-local references are inlined within the budget. Kimi Code expands
references before sending provider requests, so sharing definitions cannot
establish the provider-facing size. Expansion stops at the budget before
allocating an oversized presentation. Unconstrained JSON fields advertise
all JSON value types explicitly so Kimi cannot infer string for Evidence
objects or arrays.
- The few schemas whose validation structure alone exceeds the budget
(`compare_web_captures`, `build_reconstruction_obligation_ledger`,
`evaluate_reconstruction_coverage`, `project_managed_application_graph`,
`compare_application_versions`, `capture_browser_scenario`) are advertised
in a reduced form: property names, reference-resolved types (including union
alternatives), first-level guidance, and the root's required-field constraints,
with nested validation detail elided. The
canonical schema still rejects malformed calls server-side, so callers
following the reduced advertisement receive the canonical typed error
instead of silent acceptance.
Boundary tests pin every advertised compact schema to the budget, keep them
valid JSON Schema that still accepts every canonical example, and hold the
set of six reduced presentations as contracts evolve. Client verification
also checks provider-facing sizes and canonical examples after conversion by
the actual Kimi Code CLI; local captures do not establish Moonshot acceptance.
Self-contained output schemas advertise a content-bound `$id`, including their
declared dialect. SDK validators can reuse compiled schemas across complete
catalog refreshes and equivalent tool outputs; changing the schema changes its
@@ -101,6 +144,12 @@ if retention is unavailable or fails, the diagnostic reports that reason.
REA accepts ordinary `tools/call` progress tokens. Updates are monotonic,
rate-bounded to at most one intermediate update per 100 ms, and always allow a
terminal update. Unknown totals are omitted; REA does not fabricate percentages.
`capture_process_scenario` uses that boundary only when the client supplies a
progress token. Notifications set `progress` to the count of collected terminal
frames, process samples, and interaction events, omit `total`, and put the
lifecycle phase, elapsed time, and final cleanup status in `message`. A client
that does not send a progress token receives no progress notifications. The
tool result is unchanged either way.
Provider calls receive the request cancellation signal. Artifact traversal,
hashing, version comparisons, Hopper requests, and process capture
check the same signal. Cancellation is distinct from timeout. A cleanup failure
+36
View File
@@ -51,6 +51,34 @@ format is TOON. The file contains the complete capture Evidence record.
Choose an output file distinct from the scenario input: shell redirection opens
and truncates the output before REA reads the scenario.
While the capture runs, REA writes live status to stderr as one JSON object per
line. Stdout stays the final Evidence document:
```json
{
"rea_progress": {
"phase": "running",
"completed": 3,
"total": null,
"message": "elapsed_ms=1200 frames=1 samples=2 interactions=0",
"sequence": 4
}
}
```
`phase` is the observed lifecycle stage: `prepare`, `running`, `settling`, or
`cleanup`. `completed` counts collected terminal frames, process samples, and
interaction events. `total` stays null; REA does not invent a unit total or a
percentage. `message` reports elapsed time and those counts. The final line uses
`phase` `cleanup`, sets `terminal` to true, and adds `disposition` (`exited`,
`timeout`, `idle_timeout`, `cancelled`, or `failed`) plus the owned-process,
renderer, and temporary-root cleanup states. Updates use the shared progress
boundary and are limited to one intermediate line per 100 ms; the terminal line
is always emitted. If a receiver is slow, intermediate observations are merged
into the latest status, and stale live observations are discarded before final
cleanup status. Progress never copies child output. Child output remains in the
capture Evidence on stdout.
For example:
```json
@@ -119,6 +147,14 @@ selected normalization. Original admitted PTY chunks are available through
comparison contract, so a capture whose lines were padded to full width does not
compare as the same contract.
Custom `normalization.patterns` replace literal text with the caller's literal
replacement, including dollar sequences such as `$&` and `$$`. An empty pattern
also matches the first and last string positions. Captures with custom patterns
commit `pattern_normalization_version: "literal-replacements-v1"`. Legacy captures
remain readable. Cross-version comparison is rejected; capture both sides with
the same replacement semantics. Captures without custom patterns keep their
existing comparison contract.
Port normalization recognizes explicit `port`, `tcp_port`, `udp_port`, and
`listen` fields, URL authorities, IP endpoints, and `localhost` endpoints, with
ports from 0 through 65535. A port may end a sentence, as in `port: 8080.`;
+45
View File
@@ -88,6 +88,51 @@ entrypoint and include its verifier/fixture files in their cache inputs.
## End-to-end, integration and golden evidence
The optional `verify:qwen-client`, `verify:pi-client`, and `verify:hermes-client`
lanes require an installed native client. Select its executable with
`REA_VERIFY_QWEN_COMMAND`, `REA_VERIFY_PI_COMMAND`, or `REA_VERIFY_HERMES_COMMAND`.
They configure disposable profiles through the public REA CLI, preserve caller
settings and backups, and exercise real stdio MCP calls and skill loading. The
`call` mode checks complete catalog discovery and actual JavaScript Evidence;
pass `-- chat` to check plain chat with REA enabled. Qwen also reads the full
result that its client offloads, and Pi exercises default codemode execution.
Use `REA_VERIFY_RUNTIME_ROOT` to select a production-only installed REA package.
Receipts retain client versions, result digests, host coverage and owned-process
cleanup. These POSIX lanes use a deterministic loopback model, so they do not
prove live model-provider or native Windows compatibility. Qwen and Pi use
caller-configured skill directories to isolate the fixture account from the OS
home; this does not prove their default home-directory discovery.
Set `REA_VERIFY_HERMES_STICKY_PROFILE=1` to exercise native Hermes selection of
a named sticky profile with `HERMES_HOME` still pointing to its root.
`verify:gemini-client` requires an installed Gemini CLI (verified with
`@google/[email protected]`); select it with `REA_VERIFY_GEMINI_COMMAND`.
The optional POSIX lane uses native `GEMINI_CLI_HOME` discovery in an isolated
Git project, checks setup plans, backups and idempotence, activates the installed
personal skill, validates all forwarded input JSON Schemas against their declared
dialect, forwards the complete REA catalog and checks full JavaScript
Evidence for a Unicode path. Use `-- chat` for ordinary chat with REA enabled,
or `REA_VERIFY_RUNTIME_ROOT` for a production-only installed REA package.
The local fixture exercises the real Gemini API adapter, including its
`untrusted_context` tool-result envelope. Token counts are synthetic; live
Google API and native Windows compatibility remain unverified. The lane disables
the client's memory-based relaunch to preserve the caller's Node heap budget.
`verify:deepseek-client` requires an installed DeepSeek Harness (`dsh`;
verified with `@deepseek-ai/[email protected]`) and Git. Set
`REA_VERIFY_DEEPSEEK_COMMAND` to select its executable. This optional POSIX lane
installs the skill through `rea setup --skill`, writes the native Cordis MCP
patch into an isolated `DSH_HOME`, and uses `DSH_AGENTS_HOME` for native personal
skill discovery. A separate Git root prevents inherited project skills from
masking a missing installation. The default `call` mode checks skill loading,
complete catalog discovery, the actual forwarded input schemas and regexes,
and full JavaScript Evidence for a Unicode path. Use `-- chat` for ordinary
chat with REA enabled, or `REA_VERIFY_RUNTIME_ROOT` for an installed package.
The fixture uses Harness's custom OpenAI adapter and the one-million-token
context capacity declared by its default DeepSeek Flash model; it does not
establish live DeepSeek API or native Windows coverage. Receipts include
request artifacts, result digests and owned-process lineage.
Full E2E tests invoke the production command dispatcher and real providers,
without fake launchers, runners or responses. `verify:keyed-archive` writes an
actual Foundation binary archive, runs the CLI and a separate stdio MCP
+5
View File
@@ -179,6 +179,11 @@
"verify:readiness": "npm run build:cached && npm run verify:fixtures && node scripts/verify-reconstruction-readiness.mjs",
"verify:package": "npm run build:cached && node scripts/verify-package.mjs",
"verify:package:windows": "npm run build:cached && node scripts/verify-windows-package.mjs",
"verify:qwen-client": "npm run build:cached && node scripts/verify-qwen-client.mjs",
"verify:gemini-client": "npm run build:cached && node scripts/verify-gemini-client.mjs",
"verify:pi-client": "npm run build:cached && node scripts/verify-native-agent-client.mjs pi",
"verify:hermes-client": "npm run build:cached && node scripts/verify-native-agent-client.mjs hermes",
"verify:deepseek-client": "npm run build:cached && node scripts/verify-deepseek-client.mjs",
"probe:windows-capabilities": "npm run build:cached && node scripts/windows-capabilities.mjs",
"verify:published": "node scripts/verify-published-package.mjs",
"config:print": "node scripts/print-mcp-config.mjs",
+5 -1
View File
@@ -40,7 +40,11 @@ export async function verifyGhidraBoundaries(
ajv.validateSchema(tool[key]),
`${tool.name} ${key}: ${JSON.stringify(ajv.errors)}`,
);
validators.set(tool.name, ajv.compile(tool.outputSchema));
validators.set(
tool.name,
(tool.outputSchema?.$id && ajv.getSchema(tool.outputSchema.$id)) ||
ajv.compile(tool.outputSchema),
);
}
let successfulCalls = 0;
let rejectedCalls = 0;
+5 -1
View File
@@ -42,7 +42,11 @@ export async function verifyHopperBoundaryContracts(
ajv.validateSchema(tool.outputSchema),
`${tool.name} output schema`,
);
validators.set(tool.name, ajv.compile(tool.outputSchema));
validators.set(
tool.name,
(tool.outputSchema?.$id && ajv.getSchema(tool.outputSchema.$id)) ||
ajv.compile(tool.outputSchema),
);
}
let successfulCalls = 0;
let rejectedCalls = 0;
+346
View File
@@ -0,0 +1,346 @@
import assert from "node:assert/strict";
import { createHash } from "node:crypto";
import { mkdir, mkdtemp, readFile, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import { dirname, join } from "node:path";
import { fileURLToPath } from "node:url";
import { Ajv2020 } from "ajv/dist/2020.js";
import {
TOOL_CONTRACTS,
toolContract,
} from "../dist/contracts/toolContracts.js";
import { exec } from "./lib/verify-package-core.mjs";
import { createVerifierRun, completeVerifierRun } from "./lib/verifier-run.mjs";
import { createOpenAiModelFixture } from "./verify/clients/model-fixture.mjs";
const mode = process.argv[2] ?? "call";
assert(["chat", "call"].includes(mode), "Select chat or call.");
assert(process.platform !== "win32", "Native Windows coverage is unverified.");
const repo = fileURLToPath(new URL("..", import.meta.url));
const runtimeRoot = process.env.REA_VERIFY_RUNTIME_ROOT ?? repo;
const command = process.env.REA_VERIFY_DEEPSEEK_COMMAND ?? "dsh";
const lab = await mkdtemp(join(tmpdir(), "rea-deepseek-client-"));
const account = join(lab, "account");
const profile = join(lab, "selected profile");
const workspace = join(lab, "workspace");
const target = join(workspace, "fixture α测试");
const skillPath = join(
account,
".agents",
"skills",
"reverse-engineer-anything",
"SKILL.md",
);
const cli = join(runtimeRoot, "scripts", "rea.mjs");
const verifier = createVerifierRun();
await Promise.all([
mkdir(account),
mkdir(profile),
mkdir(target, { recursive: true }),
]);
const environment = {
PATH: process.env.PATH ?? "",
USERPROFILE: account,
DSH_HOME: profile,
DSH_AGENTS_HOME: join(account, ".agents"),
OPENAI_API_KEY: "local-fixture-key",
REA_PROCESS_RUN_ID: verifier.run_id,
};
// A temporary directory under the checkout must not inherit its project skills or AGENTS.md.
await exec(
"git",
["-c", "init.defaultBranch=main", "init", "--quiet", workspace],
{
env: environment,
timeout: 30_000,
},
);
await writeFile(
join(target, "app.js"),
'export function result() { return { clientCompatibility: "REA_CLIENT_FIXTURE", count: 7 }; }\n',
);
const clientVersion = (
await exec(command, ["--version"], {
env: environment,
cwd: workspace,
timeout: 30_000,
})
).stdout.trim();
for (const args of [
["setup", "--skill", "--dry-run"],
["setup", "--skill", "--yes"],
]) {
const result = await exec(process.execPath, [cli, ...args, "--json"], {
env: environment,
cwd: workspace,
timeout: 60_000,
});
const parsed = JSON.parse(result.stdout);
assert.equal(parsed.status, args.includes("--dry-run") ? "planned" : "ready");
if (args.includes("--dry-run")) {
assert.equal(parsed.plannedActions.length, 1);
assert.equal(parsed.plannedActions[0].kind, "install_skill");
assert.equal(parsed.plannedActions[0].target, dirname(skillPath));
}
await writeFile(
join(lab, args.includes("--dry-run") ? "setup-plan.json" : "setup.json"),
result.stdout,
);
}
assert(
(await readFile(skillPath, "utf8")).includes(
"name: reverse-engineer-anything",
),
);
let catalog;
let patternCount = 0;
let skillRequested = false;
let skillLoaded = false;
let evidenceSeen = false;
let finalSeen = false;
let artifactDigest;
let mainRequests = 0;
const ajv = new Ajv2020({ strict: false, validateFormats: false });
const patterns = new Set();
const validatePatterns = (value) => {
if (!value || typeof value !== "object") return;
for (const [key, child] of Object.entries(value)) {
if (key === "pattern" && typeof child === "string") {
// Check the representation the native client actually forwards to its model adapter.
for (const flags of ["", "u", "v"]) new RegExp(child, flags);
patterns.add(child);
patternCount++;
} else validatePatterns(child);
}
};
const extractEvidence = (value) => {
if (typeof value === "string") {
try {
return extractEvidence(JSON.parse(value));
} catch {
return undefined;
}
}
if (value && typeof value === "object") {
if (
typeof value.evidence_id === "string" &&
value.normalized_result?.graph?.nodes
)
return value;
for (const child of Object.values(value)) {
const found = extractEvidence(child);
if (found) return found;
}
}
};
const fixture = await createOpenAiModelFixture({
directory: lab,
prefix: `deepseek-${mode}`,
onRequest: async ({ body, tools, requestIndex }) => {
// Harness generates a session title in a separate, tool-free model request.
if (tools.length === 0)
return { role: "assistant", content: "REA validation" };
assert(
++mainRequests <= 4,
"Harness did not complete the bounded fixture workflow.",
);
const rea = tools.filter((tool) => tool.name.startsWith("mcp__rea__"));
const observedCatalog = rea
.map((tool) => tool.name.slice("mcp__rea__".length))
.sort();
assert.deepEqual(
observedCatalog,
TOOL_CONTRACTS.map((tool) => tool.name).sort(),
);
if (catalog === undefined) {
catalog = observedCatalog;
for (const tool of rea) {
assert(
ajv.validateSchema(tool.parameters),
`${tool.name}: ${ajv.errorsText(ajv.errors)}`,
);
validatePatterns(tool.parameters);
}
assert(
patternCount > 0,
"The advertised regex constraints must reach the model.",
);
}
const toolMessages = (body.messages ?? []).filter(
(message) => message.role === "tool",
);
const serialized = JSON.stringify(toolMessages);
skillLoaded ||=
serialized.includes(
'<skill_content name=\\"reverse-engineer-anything\\">',
) && serialized.includes(dirname(skillPath));
for (const message of toolMessages) {
const found = extractEvidence(message.content);
if (found === undefined) continue;
const evidence = toolContract(
"analyze_javascript_application",
).outputSchema.parse(found);
assert.equal(evidence.subject.local_path, target);
const shape = evidence.normalized_result.graph.nodes
.flatMap((node) => node.observations)
.map((observation) => observation.properties)
.find(
(properties) =>
properties.semantic_role === "export-return-shapes" &&
properties.exported_name === "result",
);
assert(shape?.return_shape_coverage?.projection_complete);
const fields = shape.static_return_shapes.flatMap(
(entry) => entry.fields,
);
assert(
fields.some(
(field) =>
field.path === "/clientCompatibility" &&
field.state === "literal" &&
field.value === "REA_CLIENT_FIXTURE",
),
);
assert(
fields.some(
(field) =>
field.path === "/count" &&
field.state === "literal" &&
field.value === 7,
),
);
const text = JSON.stringify(evidence);
await writeFile(join(lab, "evidence.json"), text);
artifactDigest = createHash("sha256").update(text).digest("hex");
evidenceSeen = true;
}
if (mode === "chat" || evidenceSeen) {
finalSeen = true;
return { role: "assistant", content: "REA_CLIENT_COMPATIBILITY_OK" };
}
const call = (name, args) => ({
role: "assistant",
content: null,
tool_calls: [
{
id: `call_rea_${requestIndex}`,
type: "function",
function: { name, arguments: JSON.stringify(args) },
},
],
});
if (!skillRequested) {
assert(tools.some((tool) => tool.name === "skill"));
skillRequested = true;
return call("skill", { name: "reverse-engineer-anything" });
}
assert(
skillLoaded,
"Harness must load the installed skill before analysis.",
);
return call("mcp__rea__analyze_javascript_application", {
input_path: target,
format: "directory",
});
},
});
let failure;
try {
// Use the client's native Cordis patch DSL; generic `rea setup --client` does not edit it.
await writeFile(
join(profile, "cordis.patch.yml"),
`
- id: llm-pi-ai
config:
providers:
rea-fixture:
apiKeyEnv: OPENAI_API_KEY
api: openai-completions
baseURL: ${fixture.baseUrl}
defaultMaxTokens: 4096
retryPolicy:
mode: normal
maxRetries: 0
models:
- id: rea-client-fixture
contextWindow: 1000000
- id: agent-default-model
config:
provider: rea-fixture
model: rea-client-fixture
- insert:
- id: rea-mcp
name: '@deepseek-ai/dsh-mcp-client'
config:
serverName: rea
transport: stdio
command: ${JSON.stringify(process.execPath)}
args: ${JSON.stringify([cli, "mcp"])}
cwd: !!js process.cwd()
env:
REA_PROCESS_RUN_ID: ${JSON.stringify(verifier.run_id)}
`,
);
const execution = exec(
command,
[
"--profile",
"headless",
"--json",
mode === "chat" ? "Say hello briefly." : `Use REA to analyze ${target}.`,
],
{
env: environment,
cwd: workspace,
timeout: 120_000,
maxBuffer: 10 * 1024 * 1024,
},
);
execution.child.stdin.end();
const result = await execution;
await writeFile(join(lab, "client.stdout"), result.stdout);
await writeFile(join(lab, "client.stderr"), result.stderr);
assert.equal(fixture.failure, undefined);
assert(finalSeen && result.stdout.includes("REA_CLIENT_COMPATIBILITY_OK"));
if (mode === "call") assert(evidenceSeen && skillLoaded);
} catch (cause) {
failure = cause;
if (typeof cause?.stdout === "string")
await writeFile(join(lab, "client.stdout"), cause.stdout);
if (typeof cause?.stderr === "string")
await writeFile(join(lab, "client.stderr"), cause.stderr);
} finally {
await fixture.close();
const receipt = {
client: "deepseek-harness",
clientVersion,
mode,
passed: failure === undefined,
artifacts: lab,
runtimeRoot,
host: {
platform: process.platform,
architecture: process.arch,
node: process.versions.node,
},
modelFixture:
"loopback custom OpenAI-compatible adapter; no live DeepSeek provider claim",
modelContextWindow: 1000000,
skillDiscovery:
"native DSH_AGENTS_HOME personal directory; isolated project root",
catalogSize: catalog?.length,
patternCount,
uniquePatterns: patterns.size,
skillLoaded,
evidenceSeen,
artifactDigest,
finalSeen,
requests: fixture.requests,
fixtureFailure: fixture.failure,
verifier: await completeVerifierRun(verifier),
};
await writeFile(join(lab, "receipt.json"), JSON.stringify(receipt, null, 2));
console.log(JSON.stringify(receipt));
}
if (failure !== undefined) throw failure;
+378
View File
@@ -0,0 +1,378 @@
import assert from "node:assert/strict";
import { createHash } from "node:crypto";
import { mkdir, mkdtemp, readFile, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { fileURLToPath } from "node:url";
import Ajv2020 from "ajv/dist/2020.js";
import {
TOOL_CONTRACTS,
toolContract,
} from "../dist/contracts/toolContracts.js";
import { exec } from "./lib/verify-package-core.mjs";
import { createVerifierRun, completeVerifierRun } from "./lib/verifier-run.mjs";
import { createGeminiModelFixture } from "./verify/clients/gemini-model-fixture.mjs";
assert(
process.platform !== "win32",
"Native Gemini verifier requires POSIX; Windows is unverified",
);
const repo = fileURLToPath(new URL("..", import.meta.url));
const runtimeRoot = process.env.REA_VERIFY_RUNTIME_ROOT ?? repo;
const command = process.env.REA_VERIFY_GEMINI_COMMAND ?? "gemini";
const mode = process.argv[2] ?? "call";
assert(
["chat", "call"].includes(mode),
"Usage: verify-gemini-client.mjs [chat|call]",
);
const verifier = createVerifierRun();
const lab = await mkdtemp(join(tmpdir(), "rea-gemini-client-"));
const account = join(lab, "account");
const profile = join(lab, "selected profile");
const workspace = join(lab, "workspace");
const configPath = join(profile, ".gemini", "settings.json");
const skillPath = join(
profile,
".agents",
"skills",
"reverse-engineer-anything",
"SKILL.md",
);
await Promise.all([
mkdir(account),
mkdir(workspace),
mkdir(join(profile, ".gemini"), { recursive: true }),
]);
await exec("git", ["init", "--quiet", workspace], { timeout: 30_000 });
await writeFile(join(lab, "system-settings.json"), "{}\n");
await writeFile(join(lab, "system-defaults.json"), "{}\n");
const environment = {
PATH: process.env.PATH ?? "",
USERPROFILE: account,
GEMINI_CLI_HOME: profile,
GEMINI_CLI_SYSTEM_SETTINGS_PATH: join(lab, "system-settings.json"),
GEMINI_CLI_SYSTEM_DEFAULTS_PATH: join(lab, "system-defaults.json"),
// Preserve the caller's bounded Node heap instead of the launcher's host-memory auto-sizing.
GEMINI_CLI_NO_RELAUNCH: "1",
GEMINI_CLI_TRUST_WORKSPACE: "true",
REA_PROCESS_RUN_ID: verifier.run_id,
};
let clientVersion;
try {
clientVersion = (
await exec(command, ["--version"], { env: environment, timeout: 30_000 })
).stdout.trim();
} catch (cause) {
throw new Error(
`Gemini preflight failed for ${command}; install Gemini CLI or set REA_VERIFY_GEMINI_COMMAND. ${String(cause)}`,
);
}
const original = JSON.stringify(
{
security: {
auth: { selectedType: "gemini-api-key" },
folderTrust: { enabled: false },
},
telemetry: { enabled: false },
advanced: { autoConfigureMemory: false },
model: { name: "gemini-2.5-flash", maxSessionTurns: 8 },
mcpServers: { other: { command: "unrelated-server" } },
},
null,
2,
);
await writeFile(configPath, original);
for (const args of [
["setup", "--client", "gemini_cli", "--dry-run"],
["setup", "--client", "gemini_cli", "--yes"],
["doctor", "--client", "gemini_cli", "--skill"],
]) {
const result = await exec(
process.execPath,
[join(runtimeRoot, "scripts/rea.mjs"), ...args, "--json"],
{ env: environment, cwd: workspace, timeout: 60_000 },
);
const parsed = JSON.parse(result.stdout);
if (args[0] === "doctor") assert.equal(parsed.healthy, true);
else {
assert.equal(
parsed.status,
args.includes("--dry-run") ? "planned" : "ready",
);
if (args.includes("--dry-run")) {
assert(parsed.plannedActions.some((a) => a.target === configPath));
assert(
parsed.plannedActions.some((a) =>
a.target.startsWith(join(profile, ".agents", "skills") + "/"),
),
);
assert(
parsed.plannedActions.every((a) => a.target.startsWith(profile + "/")),
);
}
}
await writeFile(
join(lab, `${args[0]}${args.includes("--dry-run") ? "-plan" : ""}.json`),
result.stdout,
);
}
assert.equal(await readFile(`${configPath}.rea.backup`, "utf8"), original);
const configured = await readFile(configPath, "utf8");
assert.deepEqual(JSON.parse(configured).mcpServers.other, {
command: "unrelated-server",
});
const repeated = await exec(
process.execPath,
[
join(runtimeRoot, "scripts/rea.mjs"),
"setup",
"--client",
"gemini_cli",
"--yes",
"--json",
],
{ env: environment, cwd: workspace, timeout: 60_000 },
);
assert.deepEqual(JSON.parse(repeated.stdout).appliedActions, []);
assert.equal(await readFile(configPath, "utf8"), configured);
assert.equal(await readFile(`${configPath}.rea.backup`, "utf8"), original);
assert(
(await readFile(skillPath, "utf8")).includes(
"name: reverse-engineer-anything",
),
);
const target = join(lab, "fixture α测试");
await mkdir(target);
await writeFile(
join(target, "app.js"),
'export function result() { return { clientCompatibility: "REA_CLIENT_FIXTURE", count: 7 }; }\n',
);
let evidenceSeen = false;
let skillLoaded = false;
let skillRequested = false;
let finalSeen = false;
let catalogSize = 0;
let schemasChecked = 0;
let artifactDigest;
const schemaValidator = new Ajv2020({ strict: false, validateFormats: false });
const extractEvidence = (value) => {
if (typeof value === "string") {
if (
value.startsWith("<untrusted_context>\n") &&
value.endsWith("\n</untrusted_context>")
) {
return extractEvidence(
value.slice(
"<untrusted_context>\n".length,
-"\n</untrusted_context>".length,
),
);
}
try {
return extractEvidence(JSON.parse(value));
} catch {
return undefined;
}
}
if (value && typeof value === "object") {
if (
typeof value.evidence_id === "string" &&
value.normalized_result?.graph?.nodes?.length > 0
)
return value;
for (const nested of Object.values(value)) {
const evidence = extractEvidence(nested);
if (evidence) return evidence;
}
}
return undefined;
};
const fixture = await createGeminiModelFixture({
directory: lab,
onRequest: async ({ body, tools }) => {
const analysis = tools.find((t) =>
t.name.endsWith("analyze_javascript_application"),
);
assert(analysis, "Gemini must expose the REA analysis tool");
const prefix = analysis.name.slice(
0,
-"analyze_javascript_application".length,
);
const expected = TOOL_CONTRACTS.map((t) => prefix + t.name).sort();
assert.deepEqual(
tools
.filter((t) => expected.includes(t.name))
.map((t) => t.name)
.sort(),
expected,
"Gemini must forward the complete REA tool catalog",
);
catalogSize = expected.length;
if (schemasChecked === 0) {
for (const tool of tools.filter((t) => expected.includes(t.name))) {
assert(
tool.parametersJsonSchema,
`${tool.name}: missing advertised JSON Schema`,
);
assert(
schemaValidator.validateSchema(tool.parametersJsonSchema),
`${tool.name}: invalid advertised JSON Schema: ${JSON.stringify(schemaValidator.errors)}`,
);
schemasChecked++;
}
}
const responses = body.contents
.flatMap((c) => c.parts ?? [])
.filter((p) => p.functionResponse)
.map((p) => p.functionResponse);
skillLoaded ||= responses.some(
(r) =>
r.name === "activate_skill" &&
r.response?.output?.includes(
`<activated_skill name="reverse-engineer-anything">`,
) &&
r.response.output.includes(
join(profile, ".agents", "skills", "reverse-engineer-anything"),
),
);
const observed = extractEvidence(responses);
if (observed) {
const evidence = toolContract(
"analyze_javascript_application",
).outputSchema.parse(observed);
assert.equal(evidence.subject.local_path, target);
const shape = evidence.normalized_result.graph.nodes
.flatMap((n) => n.observations)
.map((o) => o.properties)
.find(
(p) =>
p.semantic_role === "export-return-shapes" &&
p.exported_name === "result",
);
assert(shape?.return_shape_coverage?.projection_complete);
const fields = shape.static_return_shapes.flatMap((s) => s.fields);
assert(
fields.some(
(f) =>
f.path === "/clientCompatibility" &&
f.state === "literal" &&
f.value === "REA_CLIENT_FIXTURE",
),
);
assert(
fields.some(
(f) => f.path === "/count" && f.state === "literal" && f.value === 7,
),
);
const bytes = JSON.stringify(evidence);
artifactDigest = createHash("sha256").update(bytes).digest("hex");
await writeFile(join(lab, "gemini-evidence.json"), bytes);
evidenceSeen = true;
}
if (mode === "chat" || evidenceSeen) {
finalSeen = true;
return [{ text: "REA_CLIENT_COMPATIBILITY_OK" }];
}
if (!skillRequested) {
const skill = tools.find((t) => t.name === "activate_skill");
assert(skill, "Gemini must expose its native skill activation tool");
skillRequested = true;
return [
{
functionCall: {
name: skill.name,
args: { name: "reverse-engineer-anything" },
},
},
];
}
assert(skillLoaded, "Gemini must load the installed personal REA skill");
return [
{
functionCall: {
name: analysis.name,
args: { input_path: target, format: "directory" },
},
},
];
},
});
let failure;
try {
const response = await exec(
command,
[
"--model",
"gemini-2.5-flash",
"--approval-mode",
mode === "chat" ? "default" : "yolo",
"--allowed-mcp-server-names",
"rea",
"--output-format",
"stream-json",
"-p",
mode === "chat"
? "Say hello briefly."
: `Use REA to analyze the JavaScript directory ${target}. Report the result.`,
],
{
env: {
...environment,
GEMINI_API_KEY: "local-fixture-key",
GOOGLE_GEMINI_BASE_URL: fixture.baseUrl,
},
cwd: workspace,
timeout: 120_000,
maxBuffer: 10 * 1024 * 1024,
},
);
await writeFile(join(lab, `${mode}.stdout`), response.stdout);
await writeFile(join(lab, `${mode}.stderr`), response.stderr);
const events = response.stdout
.split("\n")
.filter(Boolean)
.map((line) => JSON.parse(line));
assert.equal(events.findLast((e) => e.type === "result")?.status, "success");
assert.equal(fixture.failure, undefined);
assert(finalSeen && response.stdout.includes("REA_CLIENT_COMPATIBILITY_OK"));
if (mode === "call") assert(evidenceSeen && skillLoaded);
} catch (cause) {
failure = cause;
if (typeof cause?.stdout === "string")
await writeFile(join(lab, `${mode}.stdout`), cause.stdout);
if (typeof cause?.stderr === "string")
await writeFile(join(lab, `${mode}.stderr`), cause.stderr);
} finally {
await fixture.close();
const lineage = await completeVerifierRun(verifier);
const receipt = {
client: "Gemini CLI",
clientVersion,
mode,
passed: failure === undefined,
artifacts: lab,
runtimeRoot,
host: {
platform: process.platform,
architecture: process.arch,
node: process.versions.node,
},
modelFixture:
"loopback native Gemini API adapter; token counts synthetic; no live Google API claim",
skillDiscovery:
"native GEMINI_CLI_HOME personal shared directory; isolated Git project",
catalogSize,
schemasChecked,
evidenceSeen,
skillLoaded,
artifactDigest,
finalSeen,
requests: fixture.requests,
probes: fixture.probes,
fixtureFailure: fixture.failure,
verifier: lineage,
};
await writeFile(join(lab, "receipt.json"), JSON.stringify(receipt, null, 2));
console.log(JSON.stringify(receipt));
}
if (failure !== undefined) throw failure;
+467
View File
@@ -0,0 +1,467 @@
import assert from "node:assert/strict";
import { createHash } from "node:crypto";
import { mkdir, mkdtemp, readFile, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { fileURLToPath } from "node:url";
import {
TOOL_CONTRACTS,
toolContract,
} from "../dist/contracts/toolContracts.js";
import { exec } from "./lib/verify-package-core.mjs";
import { createVerifierRun, completeVerifierRun } from "./lib/verifier-run.mjs";
import { createOpenAiModelFixture } from "./verify/clients/model-fixture.mjs";
const client = process.argv[2];
const mode = process.argv[3] ?? "call";
assert(["pi", "hermes"].includes(client), "Select pi or hermes.");
assert(["chat", "call"].includes(mode), "Select chat or call.");
assert(
process.platform !== "win32",
"This real-client lane requires POSIX; native Windows is unverified.",
);
const repo = fileURLToPath(new URL("..", import.meta.url));
const runtimeRoot = process.env.REA_VERIFY_RUNTIME_ROOT ?? repo;
const command =
process.env[`REA_VERIFY_${client.toUpperCase()}_COMMAND`] ?? client;
const lab = await mkdtemp(join(tmpdir(), `rea-${client}-client-`));
const account = join(lab, "account");
const profile = join(lab, "selected profile");
const stickyHermes =
client === "hermes" && process.env.REA_VERIFY_HERMES_STICKY_PROFILE === "1";
const selectedProfile = stickyHermes
? join(profile, "profiles", "coder")
: profile;
const workspace = join(lab, "workspace");
const target = join(workspace, "fixture α测试");
const artifactPath = join(workspace, "evidence.json");
const skillPath = join(
client === "pi" ? join(account, ".agents") : selectedProfile,
"skills",
"reverse-engineer-anything",
"SKILL.md",
);
const verifier = createVerifierRun();
await Promise.all([
mkdir(account),
mkdir(profile, { recursive: true }),
...(stickyHermes ? [mkdir(selectedProfile, { recursive: true })] : []),
mkdir(target, { recursive: true }),
]);
await writeFile(
join(target, "app.js"),
'export function result() { return { clientCompatibility: "REA_CLIENT_FIXTURE", count: 7 }; }\n',
);
const environment = {
PATH: process.env.PATH ?? "",
USERPROFILE: account,
[client === "pi" ? "PI_CODING_AGENT_DIR" : "HERMES_HOME"]: profile,
PI_OFFLINE: "1",
REA_PROCESS_RUN_ID: verifier.run_id,
};
const configPath = join(
selectedProfile,
client === "pi" ? "mcp.json" : "config.yaml",
);
if (stickyHermes) await writeFile(join(profile, "active_profile"), "coder\n");
const original =
client === "pi"
? JSON.stringify({
mcpServers: { other: { command: "unrelated-server", enabled: false } },
})
: "# Keep caller settings.\nmcp_servers:\n other:\n command: unrelated-server\n enabled: false\n";
await writeFile(configPath, original);
if (client === "pi") {
// The CLI fixture account and native os.homedir() differ; use Pi's caller-owned skill setting.
await writeFile(
join(profile, "settings.json"),
JSON.stringify({ skills: [join(account, ".agents", "skills")] }),
);
}
const clientVersion = (
await exec(command, ["--version"], { env: environment, timeout: 30_000 })
).stdout.trim();
for (const args of [
["setup", "--client", client, "--dry-run"],
["setup", "--client", client, "--yes"],
["doctor", "--client", client, "--skill"],
]) {
const result = await exec(
process.execPath,
[join(runtimeRoot, "scripts/rea.mjs"), ...args, "--json"],
{ env: environment, cwd: workspace, timeout: 60_000 },
);
const parsed = JSON.parse(result.stdout);
if (args[0] === "doctor") assert.equal(parsed.healthy, true);
else
assert.equal(
parsed.status,
args.includes("--dry-run") ? "planned" : "ready",
);
await writeFile(
join(lab, `${args[0]}${args.includes("--dry-run") ? "-plan" : ""}.json`),
result.stdout,
);
}
assert.equal(await readFile(`${configPath}.rea.backup`, "utf8"), original);
const installedConfiguration = await readFile(configPath, "utf8");
assert(installedConfiguration.includes("unrelated-server"));
if (client === "hermes")
assert(installedConfiguration.includes("# Keep caller settings."));
const repeated = await exec(
process.execPath,
[
join(runtimeRoot, "scripts/rea.mjs"),
"setup",
"--client",
client,
"--yes",
"--json",
],
{ env: environment, cwd: workspace, timeout: 60_000 },
);
assert.deepEqual(JSON.parse(repeated.stdout).appliedActions, []);
assert.equal(await readFile(configPath, "utf8"), installedConfiguration);
assert.equal(await readFile(`${configPath}.rea.backup`, "utf8"), original);
assert(
(await readFile(skillPath, "utf8")).includes(
"name: reverse-engineer-anything",
),
);
let evidenceSeen = false;
let skillLoaded = false;
let skillRequested = false;
let callRequested = false;
let searchRequested = false;
let describeRequested = false;
let finalSeen = false;
let catalog;
let artifactDigest;
const extractEvidence = (value) => {
if (typeof value === "string") {
if (
value.startsWith("<untrusted_tool_result ") &&
value.endsWith("</untrusted_tool_result>")
) {
// Hermes wraps external MCP content with its own instruction banner.
// Decode only that observed envelope, retaining the original request artifact.
const start = value.indexOf("\n\n");
if (start >= 0)
return extractEvidence(
value
.slice(start + 2, value.lastIndexOf("</untrusted_tool_result>"))
.trim(),
);
}
try {
return extractEvidence(JSON.parse(value));
} catch {
return undefined;
}
}
if (value && typeof value === "object") {
if (
typeof value.evidence_id === "string" &&
value.normalized_result?.graph?.nodes
)
return value;
for (const child of Object.values(value)) {
const found = extractEvidence(child);
if (found) return found;
}
}
};
const validateEvidence = async (value) => {
const evidence = toolContract(
"analyze_javascript_application",
).outputSchema.parse(value);
assert.equal(evidence.subject.local_path, target);
const shape = evidence.normalized_result.graph.nodes
.flatMap((n) => n.observations)
.map((o) => o.properties)
.find(
(p) =>
p.semantic_role === "export-return-shapes" &&
p.exported_name === "result",
);
assert(shape?.return_shape_coverage?.projection_complete);
const fields = shape.static_return_shapes.flatMap((s) => s.fields);
assert(
fields.some(
(f) =>
f.path === "/clientCompatibility" &&
f.state === "literal" &&
f.value === "REA_CLIENT_FIXTURE",
),
);
assert(
fields.some(
(f) => f.path === "/count" && f.state === "literal" && f.value === 7,
),
);
const serialized = JSON.stringify(evidence);
await writeFile(join(lab, "evidence.json"), serialized);
artifactDigest = createHash("sha256").update(serialized).digest("hex");
evidenceSeen = true;
};
const fixture = await createOpenAiModelFixture({
directory: lab,
prefix: `${client}-${mode}`,
onRequest: async ({ body, tools, requestIndex }) => {
assert(
requestIndex <= 6,
"Client did not complete the bounded fixture workflow.",
);
const messages = body.messages ?? [];
const toolMessages = messages.filter((m) => m.role === "tool");
const serializedTools = JSON.stringify(toolMessages);
const call = (name, args) => ({
role: "assistant",
content: null,
tool_calls: [
{
id: `call_rea_${requestIndex}`,
type: "function",
function: { name, arguments: JSON.stringify(args) },
},
],
});
if (
client === "pi" &&
callRequested &&
serializedTools.includes("REA_CLIENT_FIXTURE")
) {
const fact = toolMessages
.flatMap((m) => {
const text =
typeof m.content === "string"
? m.content
: JSON.stringify(m.content);
const marker = text.indexOf('{"REA_CLIENT_FIXTURE":true');
if (marker < 0) return [];
const end = text.indexOf("}\n", marker);
try {
return [
JSON.parse(text.slice(marker, end < 0 ? undefined : end + 1)),
];
} catch {
return [];
}
})
.find((v) => Array.isArray(v.catalog));
assert(
fact,
"Pi codemode must return its actual catalog and skill-read receipt.",
);
catalog = fact.catalog;
skillLoaded = fact.skill_loaded === true;
await validateEvidence(JSON.parse(await readFile(artifactPath, "utf8")));
} else if (client === "hermes") {
const search = tools.find((t) => t.name === "tool_search");
const exposed = tools
.filter((t) => t.name.startsWith("mcp__rea__"))
.map((t) => t.name.slice(10));
const deferred = [
...(search?.description ?? "").matchAll(/^- mcp__rea__(\w+):/gmu),
].map((m) => m[1]);
const native = [...new Set([...exposed, ...deferred])];
// Hermes advertises MCP prompt utilities beside server tools.
if (native.length)
catalog = native.filter(
(name) => !["get_prompt", "list_prompts"].includes(name),
);
skillLoaded ||=
serializedTools.includes(skillPath) &&
serializedTools.includes("reverse-engineer-anything");
for (const message of toolMessages) {
const evidence = extractEvidence(message.content);
if (evidence) await validateEvidence(evidence);
}
}
if (mode === "chat" || evidenceSeen) {
finalSeen = true;
return { role: "assistant", content: "REA_CLIENT_COMPATIBILITY_OK" };
}
if (client === "pi") {
const tool = tools.find((t) => t.name === "codemode");
assert(
tool,
`Native Pi codemode unavailable: ${tools.map((t) => t.name).join(", ")}`,
);
assert(
JSON.stringify(messages).includes(skillPath),
"Pi must advertise the installed REA skill.",
);
callRequested = true;
return call(tool.name, {
code: `await describeNamespace("rea"); const skill = await tools.read({path:${JSON.stringify(skillPath)}}); const result = await tools.mcp__rea__analyze_javascript_application({input_path:${JSON.stringify(target)},format:"directory"}); if(result.isError) throw new Error(JSON.stringify(result)); const evidence = result.structuredContent ?? JSON.parse(result.content.find(c=>c.type==="text").text); await tools.write({path:${JSON.stringify(artifactPath)},content:JSON.stringify(evidence)}); text({REA_CLIENT_FIXTURE:true,skill_loaded:typeof skill==="string"&&skill.includes("name: reverse-engineer-anything"),catalog:ALL_TOOLS.filter(t=>t.name.startsWith("mcp__rea__")).map(t=>t.name.slice(10))});`,
});
}
if (!skillRequested) {
const tool = tools.find((t) => t.name === "skill_view");
assert(tool, "Hermes skill_view must be available.");
skillRequested = true;
return call(tool.name, { name: "reverse-engineer-anything" });
}
if (!searchRequested) {
const tool = tools.find((t) => t.name === "tool_search");
assert(tool, "Hermes native deferred tool_search must be available.");
searchRequested = true;
return call(tool.name, { queries: ["analyze_javascript_application"] });
}
assert(
serializedTools.includes("mcp__rea__analyze_javascript_application"),
"Hermes must discover the analysis tool.",
);
if (!describeRequested) {
const tool = tools.find((t) => t.name === "tool_describe");
assert(tool, "Hermes must load the actual parameter schema.");
describeRequested = true;
return call(tool.name, {
names: ["mcp__rea__analyze_javascript_application"],
});
}
const tool = tools.find((t) => t.name === "tool_call");
assert(tool, "Hermes native deferred tool_call must be available.");
callRequested = true;
return call(tool.name, {
calls: [
{
name: "mcp__rea__analyze_javascript_application",
arguments: { input_path: target, format: "directory" },
},
],
});
},
});
if (client === "pi") {
await writeFile(
join(profile, "models.json"),
JSON.stringify({
providers: {
"rea-fixture": {
baseUrl: fixture.baseUrl,
api: "openai-completions",
apiKey: "local-fixture-key",
models: [
{
id: "rea-client-fixture",
reasoning: false,
contextWindow: 200000,
maxTokens: 4096,
},
],
},
},
}),
);
} else {
const configured = await readFile(configPath, "utf8");
await writeFile(
configPath,
configured +
`\nmodel:\n default: rea-client-fixture\n provider: custom\n base_url: ${fixture.baseUrl}\nagent:\n max_turns: 6\n`,
);
}
const args =
client === "pi"
? [
"--offline",
"--provider",
"rea-fixture",
"--model",
"rea-client-fixture",
"--no-session",
"--no-context-files",
"--mode",
"json",
"-p",
mode === "chat"
? "Say hello briefly."
: `Use REA to analyze ${target}.`,
]
: [
"--oneshot",
mode === "chat"
? "Say hello briefly."
: `Use REA to analyze ${target}.`,
"--provider",
"custom",
"--model",
"rea-client-fixture",
"--reasoning",
"none",
];
let failure;
try {
const execution = exec(command, args, {
env: {
...environment,
OPENAI_API_KEY: "local-fixture-key",
OPENAI_BASE_URL: fixture.baseUrl,
},
cwd: workspace,
timeout: 120_000,
maxBuffer: 10 * 1024 * 1024,
});
// Pi consumes piped input before its first prompt; an unused open pipe is not EOF.
execution.child.stdin.end();
const result = await execution;
await writeFile(join(lab, "client.stdout"), result.stdout);
await writeFile(join(lab, "client.stderr"), result.stderr);
assert.equal(fixture.failure, undefined);
assert(finalSeen);
assert(result.stdout.includes("REA_CLIENT_COMPATIBILITY_OK"));
if (mode === "call") {
assert(evidenceSeen);
assert(
skillLoaded,
"Native client must actually read the installed skill.",
);
assert.deepEqual(
catalog?.sort(),
TOOL_CONTRACTS.map((t) => t.name).sort(),
"The complete REA catalog must survive native client discovery.",
);
}
} catch (cause) {
failure = cause;
if (typeof cause?.stdout === "string")
await writeFile(join(lab, "client.stdout"), cause.stdout);
if (typeof cause?.stderr === "string")
await writeFile(join(lab, "client.stderr"), cause.stderr);
} finally {
await fixture.close();
const receipt = {
client,
clientVersion,
mode,
passed: failure === undefined,
artifacts: lab,
runtimeRoot,
host: {
platform: process.platform,
architecture: process.arch,
node: process.versions.node,
},
modelFixture:
"loopback deterministic OpenAI-compatible endpoint; no live provider claim",
skillDiscovery:
client === "pi"
? "caller-configured isolated shared directory; default OS home discovery unverified"
: "native HERMES_HOME personal directory",
stickyHermes,
catalogSize: catalog?.length,
evidenceSeen,
skillLoaded,
artifactDigest,
finalSeen,
requests: fixture.requests,
probes: fixture.probes,
fixtureFailure: fixture.failure,
verifier: await completeVerifierRun(verifier),
};
await writeFile(join(lab, "receipt.json"), JSON.stringify(receipt, null, 2));
console.log(JSON.stringify(receipt));
}
if (failure !== undefined) throw failure;
+4
View File
@@ -87,6 +87,10 @@ export async function verifyPackageEnvironment({
PI_PROFILE: undefined,
QWEN_HOME: undefined,
OPENCODE_CONFIG: undefined,
OPENCODE_CONFIG_DIR: undefined,
GEMINI_CLI_HOME: undefined,
GEMINI_CLI_SYSTEM_SETTINGS_PATH: undefined,
GEMINI_CLI_SYSTEM_DEFAULTS_PATH: undefined,
XDG_CONFIG_HOME: join(home, ".config"),
PATH: `${fakeBin}:${process.env.PATH ?? ""}`,
REA_ANALYSIS_PROVIDER: "auto",
+403
View File
@@ -0,0 +1,403 @@
import assert from "node:assert/strict";
import { createOpenAiModelFixture } from "./verify/clients/model-fixture.mjs";
import {
mkdir,
writeFile,
readFile,
mkdtemp,
realpath,
} from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { fileURLToPath } from "node:url";
import { createHash } from "node:crypto";
import { exec } from "./lib/verify-package-core.mjs";
import { createVerifierRun, completeVerifierRun } from "./lib/verifier-run.mjs";
import {
TOOL_CONTRACTS,
toolContract,
} from "../dist/contracts/toolContracts.js";
assert(
process.platform !== "win32",
"The Qwen real-client verifier currently requires a POSIX host; native Windows is unverified.",
);
const repo = fileURLToPath(new URL("..", import.meta.url));
const runtimeRoot = process.env.REA_VERIFY_RUNTIME_ROOT ?? repo;
const lab = await mkdtemp(join(tmpdir(), "rea-qwen-client-"));
const client = "qwen";
const mode = process.argv[2] ?? "call";
assert(
["chat", "call"].includes(mode),
"Usage: node scripts/verify-qwen-client.mjs [chat|call]",
);
const command = process.env.REA_VERIFY_QWEN_COMMAND ?? "qwen";
const verifier = createVerifierRun();
const profile = join(lab, "selected profile");
const account = join(lab, "account");
const runtime = join(profile, "runtime");
const workspace = join(lab, "workspace");
const sharedSkills = join(account, ".agents", "skills");
await Promise.all([
mkdir(profile, { recursive: true }),
mkdir(account),
mkdir(workspace),
]);
const baseEnvironment = {
PATH: process.env.PATH ?? "",
USERPROFILE: account,
QWEN_HOME: profile,
QWEN_RUNTIME_DIR: runtime,
REA_PROCESS_RUN_ID: verifier.run_id,
};
let clientVersion;
try {
clientVersion = (
await exec(command, ["--version"], {
env: baseEnvironment,
timeout: 30_000,
})
).stdout.trim();
} catch (cause) {
throw new Error(
`Qwen client preflight failed for ${command}; install Qwen Code or set REA_VERIFY_QWEN_COMMAND. ${String(cause)}`,
);
}
const original = JSON.stringify(
{
security: {
auth: { selectedType: "openai" },
folderTrust: { enabled: false },
},
telemetry: { enabled: false },
model: { name: "rea-client-fixture" },
// Qwen uses os.homedir(), while REA's isolated CLI account uses USERPROFILE.
// This caller-owned setting lets the real client load the REA-installed bundle.
skills: { directories: [sharedSkills] },
mcpServers: { other: { command: "unrelated-server", disabled: true } },
},
null,
2,
);
await writeFile(join(profile, "settings.json"), original);
for (const step of [
["setup", "--client", "qwen_code", "--dry-run"],
["setup", "--client", "qwen_code", "--yes"],
["doctor", "--client", "qwen_code", "--skill"],
]) {
const response = await exec(
process.execPath,
[join(runtimeRoot, "scripts/rea.mjs"), ...step, "--json"],
{ env: baseEnvironment, cwd: repo, timeout: 60_000 },
);
const result = JSON.parse(response.stdout);
if (step.includes("--dry-run"))
assert(
result.plannedActions.every(
(a) =>
a.target.startsWith(profile + "/") ||
a.target.startsWith(sharedSkills + "/"),
),
);
await writeFile(
join(lab, `${step[0]}${step.includes("--dry-run") ? "-plan" : ""}.json`),
response.stdout,
);
}
assert.equal(
await readFile(join(profile, "settings.json.rea.backup"), "utf8"),
original,
);
const configured = JSON.parse(
await readFile(join(profile, "settings.json"), "utf8"),
);
assert.deepEqual(configured.model, { name: "rea-client-fixture" });
assert.deepEqual(configured.skills, { directories: [sharedSkills] });
assert.deepEqual(configured.mcpServers.other, {
command: "unrelated-server",
disabled: true,
});
const target = join(lab, "fixture α测试");
await mkdir(target, { recursive: true });
await writeFile(
`${target}/app.js`,
'export function result() { return { clientCompatibility: "REA_CLIENT_FIXTURE", count: 7 }; }\n',
);
let evidenceSeen = false;
let skillSeen = false;
let finalSeen = false;
let skillRequested = false;
let searchRequested = false;
let artifactReadByAgent = false;
let artifactPath;
let selectedSkillLoaded = false;
let artifactDigest;
const extractEvidence = (value) => {
if (typeof value === "string") {
try {
return extractEvidence(JSON.parse(value));
} catch {
return null;
}
}
if (Array.isArray(value)) {
for (const item of value) {
const result = extractEvidence(item);
if (result) return result;
}
} else if (value && typeof value === "object") {
if (
typeof value.evidence_id === "string" &&
value.normalized_result?.graph?.nodes?.length > 0
)
return value;
for (const nested of Object.values(value)) {
const result = extractEvidence(nested);
if (result) return result;
}
}
return null;
};
const fixture = await createOpenAiModelFixture({
directory: lab,
prefix: `${client}-${mode}`,
onRequest: async ({ body, tools, requestIndex }) => {
skillSeen ||= JSON.stringify(body.messages ?? []).includes(
"reverse-engineer-anything",
);
selectedSkillLoaded ||= JSON.stringify(body.messages ?? []).includes(
join(sharedSkills, "reverse-engineer-anything"),
);
const toolText = JSON.stringify(
(body.messages ?? []).filter((m) => m.role === "tool"),
);
if (
toolText.includes("REA_CLIENT_FIXTURE") &&
/(?:\\?"REA_CLIENT_FIXTURE\\?"\s*:\s*true)/u.test(toolText)
) {
artifactReadByAgent = true;
assert(artifactPath);
const artifact = await readFile(artifactPath);
artifactDigest = createHash("sha256").update(artifact).digest("hex");
const evidence = toolContract(
"analyze_javascript_application",
).outputSchema.parse(JSON.parse(artifact.toString("utf8")));
assert(extractEvidence(evidence));
assert.equal(evidence.subject.local_path, target);
const shape = evidence.normalized_result.graph.nodes
.flatMap((n) => n.observations)
.map((o) => o.properties)
.find(
(p) =>
p.semantic_role === "export-return-shapes" &&
p.exported_name === "result",
);
assert(shape?.return_shape_coverage?.projection_complete);
const fields = shape.static_return_shapes.flatMap((s) => s.fields);
assert(
fields.some(
(f) =>
f.path === "/clientCompatibility" &&
f.state === "literal" &&
f.value === "REA_CLIENT_FIXTURE",
),
);
assert(
fields.some(
(f) => f.path === "/count" && f.state === "literal" && f.value === 7,
),
);
evidenceSeen = true;
await writeFile(
`${lab}/${client}-${mode}-evidence.json`,
JSON.stringify(evidence),
);
}
for (const message of body.messages ?? []) {
if (message.role !== "tool") continue;
const evidence = extractEvidence(message.content);
if (evidence) {
assert(
JSON.stringify(evidence.normalized_result).includes(
"REA_CLIENT_FIXTURE",
),
);
evidenceSeen = true;
await writeFile(
`${lab}/${client}-${mode}-evidence.json`,
JSON.stringify(evidence),
);
}
}
const selected = tools.find((t) =>
t.name.includes("analyze_javascript_application"),
);
const reviewedNames = [
...JSON.stringify(
(body.messages ?? []).filter((m) => m.role === "tool"),
).matchAll(
/(?:\\?"name\\?"\s*:\s*\\?")([^"\\]*analyze_javascript_application)(?:\\?")/g,
),
]
.map((m) => m[1])
.sort((a, b) => b.length - a.length);
const search = tools.find((t) => t.name === "tool_search");
const invoke = tools.find((t) => t.name === "tool_call");
const skill = tools.find((t) => t.name === "skill");
const shell = tools.find((t) => t.name === "run_shell_command");
const persisted = toolText.match(
/Full output saved to: ([^\\"\n]+)(?:\\n|\n)/u,
);
const call = (name, args) => ({
role: "assistant",
content: null,
tool_calls: [
{
id: `call_rea_fixture_${requestIndex}`,
type: "function",
function: { name, arguments: JSON.stringify(args) },
},
],
});
let message;
if (mode === "chat" || evidenceSeen) {
message = { role: "assistant", content: "REA_CLIENT_COMPATIBILITY_OK" };
finalSeen = true;
} else if (skill && !skillRequested) {
skillRequested = true;
message = call(skill.name, { skill: "reverse-engineer-anything" });
} else if (persisted && shell) {
artifactPath = persisted[1];
assert(
(await realpath(artifactPath)).startsWith(
(await realpath(runtime)) + "/",
),
);
const script = `const fs=require('node:fs');const e=JSON.parse(fs.readFileSync(${JSON.stringify(artifactPath)},'utf8'));console.log(JSON.stringify({REA_CLIENT_FIXTURE:JSON.stringify(e.normalized_result).includes('REA_CLIENT_FIXTURE'),evidence_id:e.evidence_id,graph_nodes:e.normalized_result.graph.nodes.length}));`;
const quote = (value) => "'" + value.replaceAll("'", "'\\''") + "'";
message = call(shell.name, {
command: `node -e ${quote(script)}`,
description:
"Read the complete saved REA Evidence and verify the fixture facts.",
});
} else if (selected)
message = call(selected.name, {
input_path: target,
format: "directory",
});
else if (reviewedNames.length && invoke)
message = call(invoke.name, {
name: reviewedNames[0],
arguments: { input_path: target, format: "directory" },
});
else if (search && !searchRequested) {
searchRequested = true;
message = call(search.name, { query: "analyze_javascript_application" });
} else
throw new Error(
`REA analysis tool not exposed; tools: ${tools.map((t) => t.name).join(", ")}`,
);
return message;
},
});
const endpoint = fixture.baseUrl;
const prompt =
mode === "chat"
? "Say hello briefly."
: `Use REA to analyze the JavaScript directory ${target}. Report the result.`;
const env = {
...baseEnvironment,
OPENAI_API_KEY: "local-fixture-key",
OPENAI_BASE_URL: endpoint,
};
const args = [
"--auth-type",
"openai",
"--openai-base-url",
endpoint,
"--model",
"rea-client-fixture",
"--approval-mode",
mode === "chat" ? "default" : "yolo",
"--allowed-mcp-server-names",
"rea",
"--output-format",
"stream-json",
"--max-session-turns",
"8",
"-p",
prompt,
];
let response;
let failure;
try {
response = await exec(command, args, {
env,
cwd: workspace,
timeout: 120_000,
maxBuffer: 10 * 1024 * 1024,
});
await writeFile(join(lab, `${mode}.stdout`), response.stdout);
await writeFile(join(lab, `${mode}.stderr`), response.stderr);
const initialized = response.stdout
.split("\n")
.filter(Boolean)
.map((line) => JSON.parse(line))
.find((v) => v.type === "system" && v.subtype === "init");
assert(initialized);
const observed = initialized.tools
.filter((n) => n.startsWith("mcp__rea__"))
.sort();
assert.deepEqual(
observed,
TOOL_CONTRACTS.map((c) => `mcp__rea__${c.name}`).sort(),
"Qwen must discover the complete REA catalog",
);
assert.equal(fixture.failure, undefined);
assert(finalSeen);
if (mode === "call") {
assert(evidenceSeen);
assert(selectedSkillLoaded);
assert(artifactReadByAgent);
}
} catch (cause) {
failure = cause;
if (typeof cause?.stdout === "string")
await writeFile(join(lab, `${mode}.stdout`), cause.stdout);
if (typeof cause?.stderr === "string")
await writeFile(join(lab, `${mode}.stderr`), cause.stderr);
} finally {
await fixture.close();
const lineage = await completeVerifierRun(verifier);
const receipt = {
client: "Qwen Code",
clientVersion,
mode,
passed: failure === undefined,
artifacts: lab,
runtimeRoot,
host: {
platform: process.platform,
architecture: process.arch,
node: process.versions.node,
},
modelFixture:
"loopback deterministic OpenAI-compatible endpoint; no live model-provider claim",
skillDiscovery:
"caller-configured isolated shared directory; default OS home discovery unverified",
catalogSize: TOOL_CONTRACTS.length,
requests: fixture.requests,
probes: fixture.probes,
evidenceSeen,
selectedSkillLoaded,
artifactReadByAgent,
artifactPath,
artifactDigest,
finalSeen,
fixtureFailure: fixture.failure,
verifier: lineage,
};
await writeFile(join(lab, "receipt.json"), JSON.stringify(receipt, null, 2));
console.log(JSON.stringify(receipt));
}
if (failure !== undefined) throw failure;
+13 -3
View File
@@ -29,7 +29,7 @@ const LINK_MODES = [
{
mode: "dyld-info",
flags: ["-Wl,-no_fixup_chains"],
fixups: "LC_DYLD_INFO bind opcodes",
fixups: "LC_DYLD_INFO rebase opcodes were not decoded",
swift: true,
},
{ mode: "chained", flags: [], fixups: "chained fixups:", swift: true },
@@ -135,7 +135,12 @@ const checkObjc = (metadata, mode) => {
item.selector === "reaFixtureLength",
) ||
objcCoverage?.status !== "complete" ||
fixupCoverage(metadata)?.reason?.startsWith(mode.fixups) !== true
fixupCoverage(metadata)?.reason?.startsWith(mode.fixups) !== true ||
(mode.mode === "dyld-info") !==
(fixupCoverage(metadata)?.reason?.includes(
"LC_DYLD_INFO bind opcodes",
) ===
true)
)
throw new Error(
`Apple metadata fixture drifted (${mode.mode}): ${JSON.stringify(metadata)}`,
@@ -165,7 +170,12 @@ const checkSwift = (metadata, mode) => {
item.owner === "ReaWitnessFixture: ReaWitnessFixtureProtocol" &&
item.implementation_address !== null,
) ||
fixupCoverage(metadata)?.reason?.startsWith(mode.fixups) !== true
fixupCoverage(metadata)?.reason?.startsWith(mode.fixups) !== true ||
(mode.mode === "dyld-info") !==
(fixupCoverage(metadata)?.reason?.includes(
"LC_DYLD_INFO bind opcodes",
) ===
true)
)
throw new Error(
`Swift metadata fixture drifted (${mode.mode}): ${JSON.stringify(metadata)}`,
@@ -0,0 +1,104 @@
import assert from "node:assert/strict";
import { createServer } from "node:http";
import { writeFile } from "node:fs/promises";
import { join } from "node:path";
/** Exercise the native Gemini API adapter against a local model, without cloud claims. */
export const createGeminiModelFixture = async ({ directory, onRequest }) => {
const requests = [];
const probes = [];
let incoming = 0;
let failure;
const server = createServer(async (request, response) => {
try {
const chunks = [];
let bytes = 0;
for await (const chunk of request) {
bytes += chunk.length;
assert(
bytes <= 10 * 1024 * 1024,
"Model fixture request exceeds 10 MiB budget",
);
chunks.push(chunk);
}
const raw = Buffer.concat(chunks).toString("utf8");
const body = raw.length === 0 ? null : JSON.parse(raw);
await writeFile(
join(directory, `gemini-incoming-${++incoming}.json`),
JSON.stringify({ method: request.method, path: request.url, body }),
);
const path = new URL(request.url, "http://localhost").pathname;
if (path.endsWith(":countTokens")) {
probes.push({ path, kind: "synthetic-token-count" });
response.setHeader("Content-Type", "application/json");
response.end(JSON.stringify({ totalTokens: 100 }));
return;
}
if (
!/\/models\/[^/]+:(?:streamGenerateContent|generateContent)$/u.test(
path,
)
) {
probes.push({ path, status: 404 });
response.statusCode = 404;
response.end(
JSON.stringify({
error: { message: "Unsupported fixture endpoint" },
}),
);
return;
}
assert.equal(request.method, "POST");
assert(body && typeof body === "object" && !Array.isArray(body));
assert(Array.isArray(body.contents));
const tools = (body.tools ?? []).flatMap(
(t) => t.functionDeclarations ?? [],
);
requests.push({
path,
tools: tools.map((t) => t.name),
contentRoles: body.contents.map((c) => c.role),
});
const parts = await onRequest({
body,
tools,
requestIndex: requests.length,
});
const result = {
candidates: [
{ index: 0, content: { role: "model", parts }, finishReason: "STOP" },
],
usageMetadata: {
promptTokenCount: 100,
candidatesTokenCount: 20,
totalTokenCount: 120,
},
modelVersion: "local-rea-fixture",
};
if (path.endsWith(":streamGenerateContent")) {
response.setHeader("Content-Type", "text/event-stream");
response.end(`data: ${JSON.stringify(result)}\n\n`);
} else {
response.setHeader("Content-Type", "application/json");
response.end(JSON.stringify(result));
}
} catch (cause) {
failure = String(cause);
response.statusCode = 400;
response.setHeader("Content-Type", "application/json");
response.end(JSON.stringify({ error: { message: failure, code: 400 } }));
}
});
await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve));
const address = server.address();
assert(address !== null && typeof address !== "string");
return {
baseUrl: `http://127.0.0.1:${address.port}`,
requests,
probes,
get failure() {
return failure;
},
close: () => new Promise((resolve) => server.close(resolve)),
};
};
+171
View File
@@ -0,0 +1,171 @@
import assert from "node:assert/strict";
import { createServer } from "node:http";
import { writeFile } from "node:fs/promises";
import { join } from "node:path";
/** A loopback model fixture exercises real clients without claiming live-provider compatibility. */
export const createOpenAiModelFixture = async ({
directory,
prefix,
onRequest,
}) => {
const requests = [];
const probes = [];
let incoming = 0;
let failure;
const server = createServer(async (request, response) => {
try {
if (request.method === "GET" && request.url?.endsWith("/models")) {
response.setHeader("Content-Type", "application/json");
response.end(
JSON.stringify({
object: "list",
data: [
{
id: "rea-client-fixture",
object: "model",
owned_by: "local-fixture",
},
],
}),
);
return;
}
const chunks = [];
let bytes = 0;
for await (const chunk of request) {
bytes += chunk.length;
assert(
bytes <= 10 * 1024 * 1024,
"Local model fixture exceeded its 10 MiB request resource budget",
);
chunks.push(chunk);
}
const rawBody = Buffer.concat(chunks).toString("utf8");
const body = rawBody.length === 0 ? null : JSON.parse(rawBody);
await writeFile(
join(directory, `${prefix}-incoming-${++incoming}.json`),
JSON.stringify({ method: request.method, path: request.url, body }),
);
if (request.url !== "/v1/chat/completions") {
// Hermes probes custom endpoints for Ollama/llama.cpp metadata.
// Unsupported HTTP endpoints return 404, as on an OpenAI-only server.
probes.push({
method: request.method,
path: request.url,
body,
status: 404,
});
response.statusCode = 404;
response.setHeader("Content-Type", "application/json");
response.end(
JSON.stringify({
error:
"This OpenAI fixture exposes models and chat completions only.",
}),
);
return;
}
assert.equal(request.method, "POST");
assert(body !== null && typeof body === "object" && !Array.isArray(body));
assert(Array.isArray(body.messages));
const tools = (body.tools ?? [])
.filter((t) => t.type === "function")
.map((t) => t.function ?? t);
requests.push({
path: request.url,
tools: tools.map((t) => t.name),
stream: body.stream,
messageRoles: body.messages.map((m) => m.role),
});
await writeFile(
join(directory, `${prefix}-request-${requests.length}.json`),
JSON.stringify(body),
);
const message = await onRequest({
body,
tools,
requestIndex: requests.length,
});
const base = {
id: `chatcmpl-rea-${requests.length}`,
object: "chat.completion",
created: Math.floor(Date.now() / 1000),
model: body.model,
};
const finish = message.tool_calls ? "tool_calls" : "stop";
const usage = {
prompt_tokens: 100,
completion_tokens: 20,
total_tokens: 120,
};
if (body.stream) {
response.setHeader("Content-Type", "text/event-stream");
const emit = (delta, finishReason = null) =>
response.write(
`data: ${JSON.stringify({ ...base, object: "chat.completion.chunk", choices: [{ index: 0, delta, finish_reason: finishReason }] })}\n\n`,
);
if (message.tool_calls) {
const fragments = message.tool_calls.map((call, index) => {
const split = Math.floor(call.function.arguments.length / 2);
return {
first: {
index,
...call,
function: {
name: call.function.name,
arguments: call.function.arguments.slice(0, split),
},
},
second: {
index,
function: { arguments: call.function.arguments.slice(split) },
},
};
});
emit({
role: "assistant",
content: null,
tool_calls: fragments.map((f) => f.first),
});
emit({ tool_calls: fragments.map((f) => f.second) });
} else emit(message);
emit({}, finish);
response.write(
`data: ${JSON.stringify({ ...base, object: "chat.completion.chunk", choices: [], usage })}\n\n`,
);
response.end("data: [DONE]\n\n");
} else {
response.setHeader("Content-Type", "application/json");
response.end(
JSON.stringify({
...base,
choices: [{ index: 0, message, finish_reason: finish }],
usage,
}),
);
}
} catch (cause) {
failure = String(cause);
response.statusCode = 400;
response.setHeader("Content-Type", "application/json");
response.end(
JSON.stringify({
error: { message: failure, type: "invalid_request_error" },
}),
);
}
});
await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve));
const address = server.address();
assert(address !== null && typeof address !== "string");
return {
baseUrl: `http://127.0.0.1:${address.port}/v1`,
requests,
probes,
get failure() {
return failure;
},
close: () => new Promise((resolve) => server.close(resolve)),
};
};
+4 -1
View File
@@ -88,7 +88,10 @@ window.loadFile("renderer.html");require("node:fs").writeFileSync(${JSON.stringi
ajv.compile(tool.inputSchema);
return [
tool.name,
tool.outputSchema ? ajv.compile(tool.outputSchema) : undefined,
tool.outputSchema
? (tool.outputSchema.$id && ajv.getSchema(tool.outputSchema.$id)) ||
ajv.compile(tool.outputSchema)
: undefined,
];
}),
);
@@ -1413,3 +1413,45 @@ export const clientRegistrationEntry = (
};
}
};
/** Parse one Gemini policy at its source boundary before consolidating scopes. */
export const parseGeminiMcpPolicy = (parsed: ClientConfigurationDocument) =>
z
.object({
allowed: z.array(z.string()).optional(),
excluded: z.array(z.string()).optional(),
})
.passthrough()
.parse(parsed.document.mcp ?? {});
/** Gemini consolidates excludes by union and allowlists by normalized intersection. */
export const geminiServerPolicyBlock = (
policies: readonly ReturnType<typeof parseGeminiMcpPolicy>[],
serverKey: string,
): string | undefined => {
if (policies.some(({ excluded }) => excluded?.includes(serverKey)))
return `Gemini mcp.excluded blocks ${serverKey}. Review the selected settings policies before rerunning setup.`;
const allowlists = policies.flatMap(({ allowed }) =>
allowed === undefined ? [] : [allowed],
);
const allowed = allowlists.reduce<string[] | undefined>(
(previous, current) => {
if (previous === undefined) return current;
const normalized = new Set(
current.map((name) => name.toLowerCase().trim()),
);
return previous.filter((name) =>
normalized.has(name.toLowerCase().trim()),
);
},
undefined,
);
// Gemini's manager applies the consolidated allowlist only when nonempty.
if (
allowed !== undefined &&
allowed.length > 0 &&
!allowed.includes(serverKey)
)
return `Gemini mcp.allowed does not include ${serverKey}. Review the selected settings policies before rerunning setup.`;
return undefined;
};
+108
View File
@@ -0,0 +1,108 @@
import { err, ok, type Result } from "../domain/result.js";
import { readFile } from "node:fs/promises";
import {
effectiveClientServer,
parseClientConfiguration,
type ClientConfigurationDocument,
} from "./ClientConfigurationDocument.js";
import type { SetupClient } from "./SupportedClients.js";
/** A named source failure that setup can explain without losing the bad path. */
export interface ClientConfigurationFileError {
readonly kind: "unreadable" | "malformed";
readonly path: string;
readonly detail: string;
}
/** Preserve named read failures and distinguish a missing source from an empty file. */
export const readConfigurationText = async (
path: string,
): Promise<Result<string | undefined, ClientConfigurationFileError>> => {
try {
return ok(await readFile(path, "utf8"));
} catch (cause: unknown) {
if (cause instanceof Error && "code" in cause && cause.code === "ENOENT")
return ok(undefined);
return err({
kind: "unreadable",
path,
detail: `Configuration ${path} could not be read: ${cause instanceof Error ? cause.message : String(cause)}`,
});
}
};
/** Read and parse one configuration at its source boundary. */
export const readConfigurationFile = async (
path: string,
format: SetupClient["format"],
): Promise<
Result<ClientConfigurationDocument | undefined, ClientConfigurationFileError>
> => {
const source = await readConfigurationText(path);
if (!source.ok) return source;
const content = source.value;
if (content === undefined) return ok(undefined);
try {
return ok(parseClientConfiguration(content, format));
} catch (cause: unknown) {
return err({
kind: "malformed",
path,
detail: `Configuration ${path} is malformed: ${cause instanceof Error ? cause.message : String(cause)}`,
});
}
};
/** Read every applicable user file; malformed lower-priority files also stop setup. */
export const readClientConfigurationFiles = async (
client: SetupClient,
): Promise<
Result<readonly ClientConfigurationDocument[], ClientConfigurationFileError>
> => {
const documents: ClientConfigurationDocument[] = [];
for (const path of client.configPaths ?? [client.configPath]) {
const document = await readConfigurationFile(path, client.format);
if (!document.ok) return document;
if (document.value !== undefined) documents.push(document.value);
}
return ok(documents);
};
/** OpenCode merges server objects in file order, replacing arrays and scalar values. */
const mergeConfigurationValue = (previous: unknown, next: unknown): unknown => {
if (!isObject(previous) || !isObject(next)) return next;
return Object.fromEntries(
[...new Set([...Object.keys(previous), ...Object.keys(next)])].map(
(key) => [
key,
Object.hasOwn(next, key)
? mergeConfigurationValue(previous[key], next[key])
: previous[key],
],
),
);
};
const isObject = (value: unknown): value is Record<string, unknown> =>
typeof value === "object" && value !== null && !Array.isArray(value);
/** Project a client's effective user registration without modifying its source files. */
export const effectiveClientConfiguration = (
documents: readonly ClientConfigurationDocument[],
): ClientConfigurationDocument | undefined => {
const last = documents.at(-1);
if (last === undefined) return undefined;
if (documents.length === 1) return last;
const servers: Record<string, unknown> = {};
for (const document of documents) {
const names = new Set([
...Object.keys(document.legacyServers),
...Object.keys(document.servers),
]);
for (const name of names)
servers[name] = mergeConfigurationValue(
servers[name],
effectiveClientServer(document, name),
);
}
return { ...last, servers, legacyServers: {} };
};
+59 -12
View File
@@ -1,10 +1,14 @@
import { readClientPolicyBlock } from "./GeminiClientSettings.js";
import {
effectiveClientConfiguration,
readClientConfigurationFiles,
} from "./ClientConfigurationFiles.js";
import {
clientServerForcedEnabled,
clientServerListedDisabled,
effectiveClientServer,
parseClientConfiguration,
} from "./ClientConfigurationDocument.js";
import { access, readFile } from "node:fs/promises";
import { access } from "node:fs/promises";
import { resolve } from "node:path";
import { z } from "zod";
@@ -16,6 +20,7 @@ import {
npxRegistrationCommand,
} from "./ClientRegistrationIdentity.js";
import {
clientEvidencePaths,
manualRegistrationRemediation,
supportedClients,
} from "./SupportedClients.js";
@@ -90,11 +95,7 @@ export const readClientRegistrationStatuses = async (
});
continue;
}
if (
!(await exists(client.markerPath)) &&
!(await exists(client.configPath))
)
continue;
if (!(await clientEvidencePresent(client))) continue;
const manualRemediation = manualRegistrationRemediation(client.name);
if (client.format === "unsupported") {
if (manualRemediation !== undefined)
@@ -108,8 +109,36 @@ export const readClientRegistrationStatuses = async (
continue;
}
try {
const content = await readFile(client.configPath, "utf8");
const parsed = parseClientConfiguration(content, client.format);
const files = await readClientConfigurationFiles(client);
if (!files.ok) {
statuses.push({
client: client.name,
config_path: client.configPath,
command: [],
state: "invalid",
remediation: files.error.detail,
});
continue;
}
const parsed = effectiveClientConfiguration(files.value);
if (parsed === undefined) {
statuses.push(
unavailableStatus(client.name, client.configPath, "missing"),
);
continue;
}
const policy = await readClientPolicyBlock(client, parsed);
if (!policy.ok) {
statuses.push({
client: client.name,
config_path: client.configPath,
command: [],
state: "invalid",
remediation: policy.error.detail,
});
continue;
}
const policyBlock = policy.value;
const raw = effectiveClientServer(parsed, PRODUCT_IDENTITY.mcpServerKey);
if (raw === undefined) {
statuses.push(
@@ -134,9 +163,14 @@ export const readClientRegistrationStatuses = async (
options.platform ?? process.platform,
clientServerForcedEnabled(parsed, PRODUCT_IDENTITY.mcpServerKey),
) &&
!clientServerListedDisabled(parsed, PRODUCT_IDENTITY.mcpServerKey)
!clientServerListedDisabled(
parsed,
PRODUCT_IDENTITY.mcpServerKey,
) &&
policyBlock === undefined
? "aligned"
: "stale",
policyBlock,
),
);
} catch (cause: unknown) {
@@ -163,7 +197,7 @@ const registrationAligned = (
): boolean => {
const command = [registration.command, ...registration.args];
if (
registration.disabled === true ||
(client.format !== "grok" && registration.disabled === true) ||
(registration.enabled === false && !forcedEnabled)
)
return false;
@@ -256,10 +290,17 @@ const configuredStatus = (
configPath: string,
command: RegistrationCommand,
state: "aligned" | "stale",
policyRemediation?: string,
): ClientRegistrationStatus =>
state === "aligned"
? { client, config_path: configPath, command, state, remediation: null }
: { client, config_path: configPath, command, state, remediation };
: {
client,
config_path: configPath,
command,
state,
remediation: policyRemediation ?? remediation,
};
const unavailableStatus = (
client: string,
@@ -273,6 +314,12 @@ const unavailableStatus = (
remediation,
});
const clientEvidencePresent = async (client: SetupClient): Promise<boolean> => {
for (const path of clientEvidencePaths(client))
if (await exists(path)) return true;
return false;
};
const exists = async (path: string | undefined): Promise<boolean> => {
if (path === undefined) return false;
try {
+202
View File
@@ -0,0 +1,202 @@
import { realpath } from "node:fs/promises";
import { dirname, resolve } from "node:path";
import { stripComments } from "jsonc-parser";
import { z } from "zod";
import { err, ok, type Result } from "../domain/result.js";
import { PRODUCT_IDENTITY } from "../identity.js";
import { npxRegistrationCommand } from "./ClientRegistrationIdentity.js";
import {
clientConfigurationValuesEqual,
clientRegistrationEntry,
geminiServerPolicyBlock,
parseGeminiMcpPolicy,
type ClientConfigurationDocument,
} from "./ClientConfigurationDocument.js";
import {
readConfigurationFile,
readConfigurationText,
type ClientConfigurationFileError,
} from "./ClientConfigurationFiles.js";
import type { SetupClient } from "./SupportedClients.js";
type SettingsSource = { path: string; document: ClientConfigurationDocument };
const folderTrustSchema = z
.object({
security: z
.object({
folderTrust: z
.object({ enabled: z.boolean().optional() })
.passthrough()
.optional(),
})
.passthrough()
.optional(),
})
.passthrough();
const trustRulesSchema = z.record(
z.string(),
z.enum(["TRUST_FOLDER", "TRUST_PARENT", "DO_NOT_TRUST"]),
);
const sourceFailure = (
path: string,
cause: unknown,
): ClientConfigurationFileError => ({
kind: "malformed",
path,
detail: `Gemini settings in ${path} are malformed: ${cause instanceof Error ? cause.message : String(cause)}`,
});
// Gemini resolves symlinks when possible and compares macOS/Windows paths without case.
const normalizedPath = async (
path: string,
platform: NodeJS.Platform,
): Promise<string> => {
const physical = await realpath(path).catch(() => path);
const absolute = resolve(physical).replaceAll("\\", "/");
return platform === "win32" || platform === "darwin"
? absolute.toLowerCase()
: absolute;
};
const workspaceTrusted = async (
settings: NonNullable<SetupClient["geminiSettings"]>,
sources: readonly SettingsSource[],
): Promise<Result<boolean, ClientConfigurationFileError>> => {
let enabled = true;
for (const { path, document } of sources) {
const parsed = folderTrustSchema.safeParse(document.document);
if (!parsed.success) return err(sourceFailure(path, parsed.error));
enabled = parsed.data.security?.folderTrust?.enabled ?? enabled;
}
if (settings.trustOverride !== undefined) return ok(settings.trustOverride);
if (!enabled) return ok(true);
const text = await readConfigurationText(settings.trustedFoldersPath);
if (!text.ok) return text;
if (text.value === undefined) return ok(false);
let rules: z.output<typeof trustRulesSchema>;
try {
// Gemini strips comments, but rejects trailing commas, BOMs and invalid trust levels.
rules = trustRulesSchema.parse(JSON.parse(stripComments(text.value)));
} catch (cause: unknown) {
return err(sourceFailure(settings.trustedFoldersPath, cause));
}
const workspace = await normalizedPath(
settings.workspaceDirectory,
settings.platform,
);
let longest = -1;
let trusted = false;
for (const [path, level] of Object.entries(rules)) {
const effective = await normalizedPath(
level === "TRUST_PARENT" ? dirname(path) : path,
settings.platform,
);
if (
(workspace === effective ||
workspace.startsWith(
effective.endsWith("/") ? effective : `${effective}/`,
)) &&
path.length > longest
) {
longest = path.length;
trusted = level !== "DO_NOT_TRUST";
}
}
return ok(trusted);
};
/** Check effective Gemini scopes while preserving their authority and source identity. */
export const readClientPolicyBlock = async (
client: SetupClient,
userDocument?: ClientConfigurationDocument,
desiredRegistration?: unknown,
): Promise<Result<string | undefined, ClientConfigurationFileError>> => {
if (client.name !== "gemini_cli" || client.geminiSettings === undefined)
return ok(undefined);
const settings = client.geminiSettings;
const sources: SettingsSource[] = [];
const higher: SettingsSource[] = [];
for (const path of [
settings.defaultsPath,
client.configPath,
settings.systemPath,
]) {
const parsed =
path === client.configPath && userDocument !== undefined
? ok(userDocument)
: await readConfigurationFile(path, "json");
if (!parsed.ok) return parsed;
if (parsed.value !== undefined)
sources.push({ path, document: parsed.value });
}
const workspace = await normalizedPath(
settings.workspaceDirectory,
settings.platform,
);
const home = await normalizedPath(
dirname(dirname(client.configPath)),
settings.platform,
);
if (workspace !== home) {
const path = resolve(
settings.workspaceDirectory,
".gemini",
"settings.json",
);
const document = await readConfigurationFile(path, "json");
if (!document.ok) return document;
if (document.value !== undefined) {
const trusted = await workspaceTrusted(settings, sources);
if (!trusted.ok) return trusted;
if (trusted.value) {
const source = { path, document: document.value };
const systemIndex = sources.findIndex(
(item) => item.path === settings.systemPath,
);
sources.splice(
systemIndex < 0 ? sources.length : systemIndex,
0,
source,
);
higher.push(source);
}
}
}
const system = sources.find(({ path }) => path === settings.systemPath);
if (system !== undefined) higher.push(system);
const policies: ReturnType<typeof parseGeminiMcpPolicy>[] = [];
for (const { path, document } of sources) {
try {
policies.push(parseGeminiMcpPolicy(document));
} catch (cause: unknown) {
return err(sourceFailure(path, cause));
}
}
const policy = geminiServerPolicyBlock(
policies,
PRODUCT_IDENTITY.mcpServerKey,
);
if (policy !== undefined)
return ok(
`${policy} Settings sources: ${sources.map(({ path }) => path).join(", ")}.`,
);
// mcpServers uses shallow merge: the entire same-name definition is replaced.
const override = higher.findLast(({ document }) =>
Object.hasOwn(document.servers, PRODUCT_IDENTITY.mcpServerKey),
);
const desired =
desiredRegistration ??
userDocument?.servers[PRODUCT_IDENTITY.mcpServerKey] ??
clientRegistrationEntry("json", npxRegistrationCommand(), {});
if (
override !== undefined &&
!clientConfigurationValuesEqual(
override.document.servers[PRODUCT_IDENTITY.mcpServerKey],
desired,
)
)
return ok(
`Gemini mcpServers.${PRODUCT_IDENTITY.mcpServerKey} in ${override.path} overrides the user registration. Review that settings file before rerunning setup; REA does not modify system or workspace settings.`,
);
return ok(undefined);
};
+150
View File
@@ -0,0 +1,150 @@
import { lstatSync, readFileSync, realpathSync, statSync } from "node:fs";
import {
basename,
dirname,
isAbsolute,
join,
relative,
resolve,
sep,
} from "node:path";
/** An unresolved Hermes profile must not be configured as the default home. */
export class HermesClientPathError extends Error {
constructor(
readonly path: string,
message: string,
) {
super(message);
this.name = "HermesClientPathError";
}
}
const defaultDirectory = (
home: string,
platform: NodeJS.Platform,
env: Readonly<NodeJS.ProcessEnv>,
): string => {
const suffix = env.HERMES_DATA_DIR_SUFFIX ?? "";
return platform === "win32"
? join(
env.LOCALAPPDATA?.trim() || join(home, "AppData", "Local"),
`hermes${suffix}`,
)
: join(home, `.hermes${suffix}`);
};
const expandedDirectory = (
home: string,
platform: NodeJS.Platform,
env: Readonly<NodeJS.ProcessEnv>,
): string | undefined => {
const override = env.HERMES_HOME?.trim();
if (!override) return undefined;
const variables = override.replace(
/\$(\w+|\{[^}]*\})/gu,
(match, variable: string) => {
const key = variable.startsWith("{") ? variable.slice(1, -1) : variable;
return env[key] ?? match;
},
);
const expanded =
platform === "win32"
? variables.replace(
/%([^%]+)%/gu,
(match, variable: string) => env[variable] ?? match,
)
: variables;
if (expanded === "~") return home;
if (
expanded.startsWith("~/") ||
(platform === "win32" && expanded.startsWith("~\\"))
)
return join(home, expanded.slice(2));
return resolve(expanded);
};
const physicalPath = (path: string): string => {
try {
return realpathSync(path);
} catch {
return resolve(path);
}
};
const within = (root: string, path: string): boolean => {
const difference = relative(physicalPath(root), physicalPath(path));
return (
difference === "" ||
(!isAbsolute(difference) &&
difference !== ".." &&
!difference.startsWith(`..${sep}`))
);
};
const PROFILE_MARKERS = [
"config.yaml",
".env",
"SOUL.md",
"profile.yaml",
"auth.json",
"state.db",
];
const RESERVED_PROFILES = new Set(["hermes", "test", "tmp", "root", "sudo"]);
const liveProfile = (path: string): boolean => {
try {
if (!statSync(path).isDirectory()) return false;
try {
statSync(join(dirname(path), ".deleted", basename(path)));
return false;
} catch {
/* No tombstone. */
}
return PROFILE_MARKERS.some((marker) => {
try {
const status = lstatSync(join(path, marker));
return status.isFile() || status.isSymbolicLink();
} catch {
return false;
}
});
} catch {
return false;
}
};
/** Match Hermes CLI sticky-profile selection while retaining the configured path spelling. */
export const resolveHermesClientHome = (
home: string,
platform: NodeJS.Platform,
env: Readonly<NodeJS.ProcessEnv>,
): string => {
const native = defaultDirectory(home, platform, env);
const root = expandedDirectory(home, platform, env) ?? native;
// An explicit named profile bypasses the root's sticky selection in Hermes CLI.
if (basename(dirname(root)) === "profiles") return root;
const stickyRoot = within(native, root) ? native : root;
let name: string;
try {
// Python uses utf-8-sig and ignores unreadable / invalid-UTF8 active_profile files.
name = new TextDecoder("utf-8", { fatal: true })
.decode(readFileSync(join(stickyRoot, "active_profile")))
.trim();
} catch {
return root;
}
if (name === "" || name === "default") return root;
const canonical = name.toLowerCase();
if (canonical === "default") return root;
if (
!/^[a-z0-9][a-z0-9_-]{0,63}$/u.test(canonical) ||
RESERVED_PROFILES.has(canonical)
)
throw new HermesClientPathError(
join(stickyRoot, "active_profile"),
`Invalid saved Hermes profile ${JSON.stringify(name)}. Run 'hermes profile use default' or repair active_profile.`,
);
const selected = join(root, "profiles", canonical);
if (!liveProfile(selected))
throw new HermesClientPathError(
selected,
`Saved Hermes profile ${JSON.stringify(name)} is missing, deleted, or has no profile identity. Run 'hermes profile use default' or restore that profile before configuring it.`,
);
return selected;
};
+33
View File
@@ -0,0 +1,33 @@
import { expect, it } from "vitest";
import { createProgressReporter } from "./ProgressReporter.js";
it("limits intermediate sends to 100 ms and always delivers terminal status", async () => {
let now = 0;
const sent: number[] = [];
const reporter = createProgressReporter(
async () => {
sent.push(now);
},
{ now: () => now },
);
for (const [completed, elapsed] of [0, 25, 50, 99, 100, 199, 200].entries()) {
now = elapsed;
await reporter.report({
phase: "running",
completed,
total: null,
message: "Observed capture status",
});
}
expect(sent).toEqual([0, 100, 200]);
now = 201;
await reporter.report({
phase: "cleanup",
completed: 7,
total: null,
message: "Owned cleanup complete",
terminal: true,
});
expect(sent).toEqual([0, 100, 200, 201]);
});
@@ -1,6 +1,10 @@
import { createHash } from "node:crypto";
import { posix } from "node:path";
import {
logicalPathEscapesRoot,
normalizeJoinedLogicalPath,
} from "../domain/artifactIdentity.js";
import {
classifyReferenceSourcePath,
detectReferenceSourceLanguage,
@@ -83,10 +87,10 @@ const resolveInternalSpecifier = (
): { to: string; resolution: "internal" | "unresolved" } => {
if (posix.isAbsolute(specifier))
return { to: specifier, resolution: "unresolved" };
const normalized = posix.normalize(
const normalized = normalizeJoinedLogicalPath(
posix.join(posix.dirname(fromPath), specifier),
);
if (normalized.startsWith("../") || normalized === "..")
if (logicalPathEscapesRoot(normalized))
return { to: normalized, resolution: "unresolved" };
const suffixes = ["", ".ts", ".js", ".mjs", ".cjs", ".tsx", ".jsx"];
const candidates = suffixes.flatMap((suffix) => [
+65 -1
View File
@@ -1,3 +1,5 @@
import { readClientPolicyBlock } from "./GeminiClientSettings.js";
import { readClientConfigurationFiles } from "./ClientConfigurationFiles.js";
import {
clientRegistrationEntry,
clientConfigurationValuesEqual,
@@ -46,6 +48,15 @@ const configureClientDocument = async (
command: readonly string[],
format: NonNullable<SetupClient["format"]>,
): Promise<ClientConfigurationResult> => {
const files = await readClientConfigurationFiles(client);
if (!files.ok) return { status: "failed", reason: "readback" };
const policy = await readClientPolicyBlock(
client,
undefined,
clientConfigurationDesired(client, environment, command, client.format),
);
if (!policy.ok || policy.value !== undefined)
return { status: "failed", reason: "readback" };
const transactionPath = await resolveClientConfigTransactionPath(
client.configPath,
);
@@ -63,6 +74,13 @@ const configureClientDocument = async (
original ?? (format === "toml" || format === "grok" ? "" : "{}"),
format,
);
const policy = await readClientPolicyBlock(
client,
parsed,
clientConfigurationDesired(client, environment, command, parsed.dialect),
);
if (!policy.ok || policy.value !== undefined)
return { status: "failed", reason: "readback" };
} catch (cause: unknown) {
// Malformed existing configuration fails the readback gate.
void cause;
@@ -158,9 +176,12 @@ export const clientConfigurationAligned = async (
): Promise<boolean> => {
if (client.configPathError !== undefined) return false;
try {
const files = await readClientConfigurationFiles(client);
if (!files.ok) return false;
const original = await readFile(client.configPath, "utf8");
const parsed = parseClientConfiguration(original, client.format);
return registrationCurrent(
const policy = await readClientPolicyBlock(
client,
parsed,
clientConfigurationDesired(
client,
@@ -169,6 +190,19 @@ export const clientConfigurationAligned = async (
parsed.dialect,
),
);
return (
policy.ok &&
policy.value === undefined &&
registrationCurrent(
parsed,
clientConfigurationDesired(
client,
providerEnvironment,
command,
parsed.dialect,
),
)
);
} catch (cause: unknown) {
// Unreadable configuration is treated as not aligned so setup repairs it.
void cause;
@@ -185,6 +219,22 @@ export const inspectClientConfiguration = async (
if (client.configPathError !== undefined)
return { status: "invalid", remediation: client.configPathError };
if (client.format === "unsupported") return { status: "already_current" };
const files = await readClientConfigurationFiles(client);
if (!files.ok) return { status: "invalid", remediation: files.error.detail };
const policy = await readClientPolicyBlock(
client,
undefined,
clientConfigurationDesired(
client,
providerEnvironment,
command,
client.format,
),
);
if (!policy.ok)
return { status: "invalid", remediation: policy.error.detail };
if (policy.value !== undefined)
return { status: "invalid", remediation: policy.value };
const transactionPath = await resolveClientConfigTransactionPath(
client.configPath,
);
@@ -207,6 +257,20 @@ export const inspectClientConfiguration = async (
}
try {
const parsed = parseClientConfiguration(original, client.format);
const policy = await readClientPolicyBlock(
client,
parsed,
clientConfigurationDesired(
client,
providerEnvironment,
command,
parsed.dialect,
),
);
if (!policy.ok)
return { status: "invalid", remediation: policy.error.detail };
if (policy.value !== undefined)
return { status: "invalid", remediation: policy.value };
const desired = clientConfigurationDesired(
client,
providerEnvironment,
+12 -6
View File
@@ -14,7 +14,11 @@ import {
systemLinuxHopperInstallHost,
} from "./LinuxHopper.js";
import { installMacHopper, systemMacHopperInstallHost } from "./MacHopper.js";
import { supportedClients, type SetupClient } from "./SupportedClients.js";
import {
clientEvidencePaths,
supportedClients,
type SetupClient,
} from "./SupportedClients.js";
import {
skillDestinations,
canonicalSkillNeedsInstall,
@@ -168,17 +172,19 @@ export const detectClients = async (
const detected: SetupClient[] = [];
for (const candidate of supportedClients(home, platform, environment)) {
if (candidate.configPathError !== undefined) continue;
const [hasConfig, hasMarker] = await Promise.all([
exists(candidate.configPath),
candidate.markerPath === undefined ? false : exists(candidate.markerPath),
]);
if (hasConfig || hasMarker) detected.push(candidate);
if (await clientEvidencePresent(candidate)) detected.push(candidate);
}
return detected;
};
const major = (version: string): number =>
Number.parseInt(version.split(".")[0] ?? "0", 10);
const clientEvidencePresent = async (client: SetupClient): Promise<boolean> => {
for (const path of clientEvidencePaths(client))
if (await exists(path)) return true;
return false;
};
const exists = async (path: string): Promise<boolean> => {
try {
await access(path);
+186 -78
View File
@@ -1,12 +1,27 @@
import { isAbsolute, join, resolve, win32 } from "node:path";
import { dirname, isAbsolute, join, resolve, win32 } from "node:path";
import { fileURLToPath } from "node:url";
import { lstatSync } from "node:fs";
import {
HermesClientPathError,
resolveHermesClientHome,
} from "./HermesClientHome.js";
/** One supported client configuration location. */
export interface SetupClient {
readonly name: string;
readonly displayName?: string;
readonly configPath: string;
/** User configuration files in increasing precedence, including the write target. */
readonly configPaths?: readonly string[];
/** Read-only Gemini scopes and trust inputs; never setup write targets. */
readonly geminiSettings?: {
readonly systemPath: string;
readonly defaultsPath: string;
readonly workspaceDirectory: string;
readonly trustedFoldersPath: string;
readonly trustOverride?: boolean;
readonly platform: NodeJS.Platform;
};
/** Unresolved path evidence only; consumers must not perform I/O on it. */
readonly configPathError?: string;
readonly markerPath?: string;
@@ -39,6 +54,10 @@ interface ClientDefinition {
readonly markerPath: ClientPath;
readonly format: NonNullable<SetupClient["format"]>;
readonly skillPath?: ClientPath;
readonly configPaths?: (context: ClientPathContext) => readonly string[];
readonly geminiSettings?: (
context: ClientPathContext,
) => NonNullable<SetupClient["geminiSettings"]>;
}
const vscodeUserDirectory = ({
@@ -76,7 +95,7 @@ const claudeCodeMarkerDirectory = ({ home, env }: ClientPathContext): string =>
env.CLAUDE_CONFIG_DIR ?? join(home, ".claude");
const codexDirectory = ({ home, env }: ClientPathContext): string =>
env.CODEX_HOME ?? join(home, ".codex");
env.CODEX_HOME || join(home, ".codex");
/** Match Qwen Code's home override, including tilde and cwd-relative paths. */
const qwenCodeDirectory = ({ home, env }: ClientPathContext): string => {
@@ -98,51 +117,10 @@ const qwenCodeDirectory = ({ home, env }: ClientPathContext): string => {
};
const grokDirectory = ({ home, env }: ClientPathContext): string =>
env.GROK_HOME ?? join(home, ".grok");
env.GROK_HOME || join(home, ".grok");
/**
* Hermes Agent home. `HERMES_HOME` wins; otherwise the platform default mirrors
* Hermes's own resolver (`%LOCALAPPDATA%\\hermes` on Windows, `~/.hermes`
* elsewhere), including the optional `HERMES_DATA_DIR_SUFFIX` that is appended
* to the literal `hermes`/`.hermes` directory name.
*/
const hermesDirectory = ({
home,
platform,
env,
}: ClientPathContext): string => {
const override = env.HERMES_HOME?.trim();
if (override) {
const expandedVariables = override.replace(
/\$(\w+|\{[^}]*\})/gu,
(match, variable: string) => {
const key = variable.startsWith("{") ? variable.slice(1, -1) : variable;
return env[key] ?? match;
},
);
const expanded =
platform === "win32"
? expandedVariables.replace(
/%([^%]+)%/gu,
(match, variable: string) => env[variable] ?? match,
)
: expandedVariables;
if (expanded === "~") return home;
if (
expanded.startsWith("~/") ||
(platform === "win32" && expanded.startsWith("~\\"))
)
return join(home, expanded.slice(2));
return resolve(expanded);
}
const suffix = env.HERMES_DATA_DIR_SUFFIX ?? "";
return platform === "win32"
? join(
env.LOCALAPPDATA?.trim() || join(home, "AppData", "Local"),
`hermes${suffix}`,
)
: join(home, `.hermes${suffix}`);
};
const hermesDirectory = ({ home, platform, env }: ClientPathContext): string =>
resolveHermesClientHome(home, platform, env);
/** Grok Bot uses an absolute SAND_DATA_ROOT; anything else stays ~/.grokbot. */
const grokBotDirectory = ({ home, env }: ClientPathContext): string => {
@@ -188,17 +166,14 @@ const ompProfile = ({ env }: ClientPathContext): string | undefined => {
/**
* The OMP agent directory whose `mcp.json` holds user-scope MCP servers. OMP
* joins PI_CONFIG_DIR to the home directory, and a named profile ignores
* PI_CODING_AGENT_DIR. OMP resolves a relative agent directory against each
* process's working directory, so only an absolute override names one file.
* PI_CODING_AGENT_DIR. Relative overrides use the setup process working directory.
*/
const ompAgentDirectory = (context: ClientPathContext): string => {
const root = join(context.home, context.env.PI_CONFIG_DIR || ".omp");
const profile = ompProfile(context);
if (profile !== undefined) return join(root, "profiles", profile, "agent");
const override = context.env.PI_CODING_AGENT_DIR;
return override !== undefined && override !== "" && isAbsolute(override)
? override
: join(root, "agent");
return override ? resolve(override) : join(root, "agent");
};
/**
@@ -242,21 +217,66 @@ const copilotDirectory = ({ home, env }: ClientPathContext): string =>
const commandCodeDirectory = ({ home }: ClientPathContext): string =>
join(home, ".commandcode");
const opencodeDirectory = ({ home, env }: ClientPathContext): string =>
join(env.XDG_CONFIG_HOME ?? join(home, ".config"), "opencode");
/**
* OpenCode appends `OPENCODE_CONFIG_DIR` only when the value is non-empty.
* An empty variable is the same as unset and leaves the XDG directory in place.
*/
const openCodeOverrideDirectory = (
env: ClientPathContext["env"],
): string | undefined => {
const directory = env.OPENCODE_CONFIG_DIR;
return directory === undefined || directory === "" ? undefined : directory;
};
const opencodeDirectory = (context: ClientPathContext): string =>
openCodeOverrideDirectory(context.env) ??
join(
context.env.XDG_CONFIG_HOME ?? join(context.home, ".config"),
"opencode",
);
/** OpenCode loads global JSON before JSONC, then explicit file and directory overrides. */
const openCodeConfigurationPaths = (
context: ClientPathContext,
): readonly string[] => {
const directory = join(
context.env.XDG_CONFIG_HOME ?? join(context.home, ".config"),
"opencode",
);
const overrideDirectory = openCodeOverrideDirectory(context.env);
const paths = [
join(directory, "config.json"),
join(directory, "opencode.json"),
join(directory, "opencode.jsonc"),
...(context.env.OPENCODE_CONFIG ? [context.env.OPENCODE_CONFIG] : []),
...(overrideDirectory !== undefined
? [
join(overrideDirectory, "opencode.json"),
join(overrideDirectory, "opencode.jsonc"),
]
: []),
];
const resolved = paths.map((path) => resolve(path));
return resolved.filter((path, index) => resolved.lastIndexOf(path) === index);
};
const existingOpenCodeConfigPath = (context: ClientPathContext): string => {
if (context.env.OPENCODE_CONFIG !== undefined)
return context.env.OPENCODE_CONFIG;
const directory = opencodeDirectory(context);
const candidates = [
"opencode.json",
"opencode.jsonc",
".opencode.json",
".opencode.jsonc",
].map((filename) => join(directory, filename));
// The selected directory is loaded after an explicit file by OpenCode V1.
const overrideDirectory = openCodeOverrideDirectory(context.env);
const candidates =
overrideDirectory !== undefined
? [
join(overrideDirectory, "opencode.json"),
join(overrideDirectory, "opencode.jsonc"),
].map((path) => resolve(path))
: context.env.OPENCODE_CONFIG
? [resolve(context.env.OPENCODE_CONFIG)]
: [
join(opencodeDirectory(context), "opencode.json"),
join(opencodeDirectory(context), "opencode.jsonc"),
].map((path) => resolve(path));
return (
candidates.find((path) => {
[...candidates].reverse().find((path) => {
try {
lstatSync(path);
return true;
@@ -268,15 +288,52 @@ const existingOpenCodeConfigPath = (context: ClientPathContext): string => {
);
}
}) ??
candidates[0] ??
join(directory, "opencode.json")
resolve(
overrideDirectory !== undefined
? join(overrideDirectory, "opencode.json")
: context.env.OPENCODE_CONFIG ||
join(opencodeDirectory(context), "opencode.json"),
)
);
};
const geminiSettings = ({
home,
platform,
env,
}: ClientPathContext): NonNullable<SetupClient["geminiSettings"]> => {
const system =
env.GEMINI_CLI_SYSTEM_SETTINGS_PATH ||
(platform === "darwin"
? "/Library/Application Support/GeminiCli/settings.json"
: platform === "win32"
? "C:\\ProgramData\\gemini-cli\\settings.json"
: "/etc/gemini-cli/settings.json");
const trustOverride =
env.GEMINI_RESTRICTED_MODE === "true" ||
env.GEMINI_CLI_TRUST_WORKSPACE === "false"
? false
: env.GEMINI_CLI_TRUST_WORKSPACE === "true"
? true
: undefined;
return {
systemPath: system,
defaultsPath:
env.GEMINI_CLI_SYSTEM_DEFAULTS_PATH ||
join(dirname(system), "system-defaults.json"),
workspaceDirectory: process.cwd(),
trustedFoldersPath:
env.GEMINI_CLI_TRUSTED_FOLDERS_PATH ||
join(env.GEMINI_CLI_HOME || home, ".gemini", "trustedFolders.json"),
...(trustOverride === undefined ? {} : { trustOverride }),
platform,
};
};
const devinDirectory = ({ home, platform, env }: ClientPathContext): string =>
platform === "win32"
? join(env.APPDATA ?? join(home, "AppData", "Roaming"), "devin")
: join(home, ".config", "devin");
: join(env.XDG_CONFIG_HOME || join(home, ".config"), "devin");
/** Stable product metadata used to derive setup discovery and documentation. */
export const SUPPORTED_CLIENT_DEFINITIONS = [
@@ -316,8 +373,13 @@ export const SUPPORTED_CLIENT_DEFINITIONS = [
{
name: "gemini_cli",
displayName: "Gemini CLI",
configPath: [".gemini", "settings.json"],
markerPath: [".gemini"],
geminiSettings,
configPath: ({ home, env }: ClientPathContext) =>
join(env.GEMINI_CLI_HOME || home, ".gemini", "settings.json"),
markerPath: ({ home, env }: ClientPathContext) =>
join(env.GEMINI_CLI_HOME || home, ".gemini"),
skillPath: ({ home, env }: ClientPathContext) =>
join(env.GEMINI_CLI_HOME || home, ".agents", "skills"),
format: "json",
},
{
@@ -339,12 +401,14 @@ export const SUPPORTED_CLIENT_DEFINITIONS = [
name: "opencode",
displayName: "OpenCode",
configPath: existingOpenCodeConfigPath,
configPaths: openCodeConfigurationPaths,
markerPath: opencodeDirectory,
format: "opencode",
},
{
name: "antigravity",
displayName: "Antigravity",
skillPath: [".gemini", "config", "skills"],
configPath: [".gemini", "config", "mcp_config.json"],
markerPath: [".gemini", "config"],
format: "json",
@@ -427,6 +491,23 @@ export const SUPPORTED_CLIENT_DEFINITIONS = [
},
] as const satisfies readonly ClientDefinition[];
/**
* Paths whose presence means doctor and setup should inspect the client.
* Layered sources count: OpenCode still loads its XDG files when a custom
* directory override does not exist.
*/
export const clientEvidencePaths = (
client: Pick<SetupClient, "configPath" | "configPaths" | "markerPath">,
): readonly string[] => [
...new Set(
[
client.configPath,
...(client.configPaths ?? []),
client.markerPath,
].filter((path): path is string => path !== undefined && path.length > 0),
),
];
const resolvePath = (path: ClientPath, context: ClientPathContext): string =>
typeof path === "function" ? path(context) : join(context.home, ...path);
@@ -437,16 +518,36 @@ export const supportedClients = (
env: ClientPathContext["env"] = process.env,
): readonly SetupClient[] => {
const context = { home, platform, env };
return SUPPORTED_CLIENT_DEFINITIONS.map((definition) => {
const definitions: readonly ClientDefinition[] = SUPPORTED_CLIENT_DEFINITIONS;
return definitions.map((definition) => {
try {
const configPath = resolvePath(definition.configPath, context);
return {
name: definition.name,
displayName: definition.displayName,
configPath: resolvePath(definition.configPath, context),
configPath,
...(definition.geminiSettings === undefined
? {}
: { geminiSettings: definition.geminiSettings(context) }),
...(definition.configPaths === undefined
? {}
: {
configPaths: [
...new Set([...definition.configPaths(context), configPath]),
],
}),
markerPath: resolvePath(definition.markerPath, context),
format: definition.format,
};
} catch (cause: unknown) {
if (cause instanceof HermesClientPathError)
return {
name: definition.name,
displayName: definition.displayName,
format: definition.format,
configPath: cause.path,
configPathError: cause.message,
};
if (definition.name !== "pi") throw cause;
return {
name: definition.name,
@@ -481,14 +582,21 @@ export const clientSkillDirectories = (
const destinations = new Map<string, { client: string; directory: string }>();
for (const client of selected) {
const definition = definitions.find(({ name }) => name === client);
const directory =
definition?.skillPath === undefined
? join(home, ".agents", "skills")
: resolvePath(definition.skillPath, {
home,
platform,
env: environment,
});
let directory: string;
try {
directory =
definition?.skillPath === undefined
? join(home, ".agents", "skills")
: resolvePath(definition.skillPath, {
home,
platform,
env: environment,
});
} catch (cause: unknown) {
// Registration diagnostics own an unresolved profile; never fall back to another skill root.
if (cause instanceof HermesClientPathError) continue;
throw cause;
}
if (!destinations.has(directory))
destinations.set(directory, {
client:
+75 -3
View File
@@ -124,11 +124,12 @@ export const runUninstall = async (
status: "retained",
detail: "Hopper is not owned by REA uninstall.",
});
const reported = collapseUninstallItems(items);
return {
status: items.some(({ status }) => status === "failed")
status: reported.some(({ status }) => status === "failed")
? "failed"
: "complete",
items,
items: reported,
};
};
@@ -171,7 +172,16 @@ export const systemUninstallHost = (
selectedHome ?? homeDirectoryFromEnvironment(environment, platform);
return {
clients: () =>
Promise.resolve(supportedClients(home, platform, environment)),
Promise.resolve(
supportedClients(home, platform, environment).flatMap((client) =>
client.configPaths === undefined
? [client]
: client.configPaths.map((configPath) => ({
...client,
configPath,
})),
),
),
inspectClient: async (client) => {
const read = await readClientConfiguration(client, fileSystem);
return read.kind === "item" && read.item.status === "failed"
@@ -452,6 +462,68 @@ const item = (
status: UninstallItem["status"],
detail: string,
): UninstallItem => ({ name, status, detail });
/**
* Layered clients are removed file by file, but callers look up one status by
* client name. A missing earlier file must not hide a later removal.
*/
const collapseUninstallItems = (
items: readonly UninstallItem[],
): UninstallItem[] => {
const collapsed: UninstallItem[] = [];
for (const next of items) {
const index = collapsed.findIndex(({ name }) => name === next.name);
const current = collapsed[index];
if (current === undefined) {
collapsed.push(next);
continue;
}
collapsed[index] = mergeUninstallItems(current, next);
}
return collapsed;
};
const mergeUninstallItems = (
current: UninstallItem,
next: UninstallItem,
): UninstallItem => {
const status = preferredUninstallStatus(current.status, next.status);
const details = [current, next]
.filter(
(entry) =>
entry.status === status ||
(status === "removed" && entry.status === "retained"),
)
.map(({ detail }) => detail);
return item(current.name, status, [...new Set(details)].join(" "));
};
const preferredUninstallStatus = (
current: UninstallItem["status"],
next: UninstallItem["status"],
): UninstallItem["status"] =>
uninstallStatusPriority(next) > uninstallStatusPriority(current)
? next
: current;
const uninstallStatusPriority = (status: UninstallItem["status"]): number => {
switch (status) {
case "failed":
return 3;
case "removed":
return 2;
case "retained":
return 1;
case "skipped":
return 0;
default: {
const exhaustive: never = status;
throw new TypeError(
`Unhandled uninstall item status: ${String(exhaustive)}`,
);
}
}
};
const registrationSchema = z
.object({ command: z.string(), args: z.array(z.string()) })
.passthrough();
@@ -42,6 +42,46 @@ describe("JavaScript analysis cancellation before publication", () => {
expect(phases).not.toContain("validate_javascript_application_result");
});
it("handles control turns and cancellation inside one large source file (#1462)", async () => {
const root = await createTestTempDirectory("rea-js-cancel-in-file-");
await writeFile(
join(root, "index.js"),
Array.from(
{ length: 1200 },
(_, index) =>
`export function f${index}(){return {name:'f${index}',value:${index}};}`,
).join("\n"),
);
const controller = new AbortController();
const phases: string[] = [];
let turns = 0;
// Count event-loop turns once the file starts; parsing, analysis and
// projection of this one file must leave room for them.
const tick = (): void => {
turns += 1;
if (turns === 3) controller.abort();
else if (!controller.signal.aborted) setImmediate(tick);
};
const result = await analyzeJavaScriptApplication(
{ input_path: root, format: "directory" },
{
signal: controller.signal,
progress: {
async report(event) {
phases.push(event.phase);
if (event.phase === "parse_javascript_source") setImmediate(tick);
},
},
},
);
if (result.ok) throw new Error("Cancelled analysis must not publish");
expect(projectAnalysisError(result.error)).toMatchObject({
code: "cancelled",
});
expect(phases).toContain("parse_javascript_source");
expect(phases).not.toContain("build_javascript_application_graph");
});
it.each([
"parse_javascript_source",
"build_javascript_application_graph",
@@ -4,7 +4,7 @@ import { setImmediate } from "node:timers/promises";
import { resolveJavaScriptSourceMapReference } from "../../domain/javascript/javascriptSourceMapPaths.js";
import { analyzeParsedJavaScriptStaticSource } from "../../domain/javascript/javascriptStaticAnalysis.js";
import {
analyzeParsedJavaScriptSemantics,
analyzeParsedJavaScriptSemanticsSteps,
classifyParsedJavaScriptOpenReceivers,
} from "../../domain/javascript/javascriptSemanticAnalysis.js";
import type { JavaScriptSemanticIr } from "../../domain/javascript/javascriptSemanticIr.js";
@@ -33,6 +33,7 @@ import type {
JavaScriptSourceMapOriginal,
} from "./JavaScriptArtifactAnalysisTypes.js";
import { analyzeJavaScriptJsonModule } from "./JavaScriptJsonModules.js";
import { completeJavaScriptAnalysisSteps } from "./JavaScriptAnalysisControl.js";
import { htmlArtifactReferences } from "../../domain/javascript/htmlArtifactReferences.js";
interface MutableArtifactAnalysis<
@@ -56,34 +57,44 @@ interface ArtifactAnalysisContext<
readonly projectSemantics: (
file: JavaScriptArtifactFile,
ir: JavaScriptSemanticIr,
) => SemanticIr;
) => Generator<void, SemanticIr>;
}
/** Analyze every text source from the artifact without a result quota. */
export const analyzeJavaScriptArtifactFiles = (
fileSet: JavaScriptArtifactFileSet,
): JavaScriptArtifactAnalysis =>
analyzeArtifactFiles(fileSet, (_file, ir) => ir);
analyzeArtifactFiles(fileSet, function* (_file, ir) {
// The full IR needs no projection; one step keeps the shared protocol.
yield;
return ir;
});
/** Project each file's full IR immediately, retaining only later module facts. */
/**
* Project each file's full IR immediately, retaining only later module facts.
* Parsing, analysis and each projection pass run as separate steps, so control
* messages and cancellation are handled within one large file as well.
*/
export const analyzeAndProjectJavaScriptArtifactFiles = async (
fileSet: JavaScriptArtifactFileSet,
projectSemantics: (
file: JavaScriptArtifactFile,
ir: JavaScriptSemanticIr,
) => void,
) => Iterator<void, void> | void,
beforeFile?: (
file: JavaScriptArtifactFile,
completed: number,
total: number,
) => Promise<void>,
signal?: AbortSignal,
): Promise<JavaScriptModuleArtifactAnalysis> => {
const state = emptyArtifactAnalysis<JavaScriptModuleSemanticIr>();
const projectFileSemantics = (
const projectFileSemantics = function* (
file: JavaScriptArtifactFile,
ir: JavaScriptSemanticIr,
): JavaScriptModuleSemanticIr => {
projectSemantics(file, ir);
): Generator<void, JavaScriptModuleSemanticIr> {
const projection = projectSemantics(file, ir);
if (projection !== undefined) while (projection.next().done !== true) yield;
const programScopes = ir.scopes.filter(({ kind }) => kind === "program");
const programScopeId = programScopes[0]?.scopeId;
const localNames = new Set(
@@ -111,7 +122,10 @@ export const analyzeAndProjectJavaScriptArtifactFiles = async (
// Let progress, cancellation and garbage collection run between files.
await setImmediate();
await beforeFile?.(file, index, fileSet.files.length);
analyzeArtifactFile(file, context);
await completeJavaScriptAnalysisSteps(
analyzeArtifactFileSteps(file, context),
signal,
);
}
return finalizeArtifactAnalysis(state);
};
@@ -122,7 +136,10 @@ const analyzeArtifactFiles = <SemanticIr extends JavaScriptModuleSemanticIr>(
): JavaScriptArtifactAnalysis<SemanticIr> => {
const state = emptyArtifactAnalysis<SemanticIr>();
const context = { state, projectSemantics };
for (const file of fileSet.files) analyzeArtifactFile(file, context);
for (const file of fileSet.files) {
const steps = analyzeArtifactFileSteps(file, context);
while (steps.next().done !== true);
}
return finalizeArtifactAnalysis(state);
};
@@ -148,10 +165,12 @@ const finalizeArtifactAnalysis = <
};
};
const analyzeArtifactFile = <SemanticIr extends JavaScriptModuleSemanticIr>(
function* analyzeArtifactFileSteps<
SemanticIr extends JavaScriptModuleSemanticIr,
>(
file: JavaScriptArtifactFile,
context: ArtifactAnalysisContext<SemanticIr>,
): void => {
): Generator<void, void> {
const { state } = context;
addStructuredObservations(file, state);
if (file.kind === "html" && file.text.included)
@@ -162,6 +181,7 @@ const analyzeArtifactFile = <SemanticIr extends JavaScriptModuleSemanticIr>(
return;
}
const parsed = parseJavaScriptSource(file.text.value, file.path);
yield;
if (parsed === null) {
const analysis = failedJavaScriptStaticAnalysis();
state.files.push({ file, javascript: analysis, semantic: null });
@@ -169,23 +189,26 @@ const analyzeArtifactFile = <SemanticIr extends JavaScriptModuleSemanticIr>(
return;
}
const openReceiverFacts = classifyParsedJavaScriptOpenReceivers(parsed);
yield;
const analysis = analyzeParsedJavaScriptStaticSource(
file.text.value,
parsed,
openReceiverFacts,
);
const staticFindings = findingCount(analysis);
yield;
const semantics =
analysis.parse_status === "complete" || analysis.parse_status === "partial"
? analyzeParsedJavaScriptSemantics(parsed)
? yield* analyzeParsedJavaScriptSemanticsSteps(parsed)
: null;
const projected =
semantics === null
? null
: yield* context.projectSemantics(file, semantics);
state.files.push({
file,
javascript: analysis,
semantic:
semantics === null
? null
: { ir: context.projectSemantics(file, semantics) },
semantic: projected === null ? null : { ir: projected },
});
state.visitedNodes += analysis.visited_ast_nodes;
state.findings += staticFindings + (semantics?.moduleLinks.length ?? 0);
@@ -194,7 +217,7 @@ const analyzeArtifactFile = <SemanticIr extends JavaScriptModuleSemanticIr>(
0,
);
if (analysis.parse_status === "failed") state.parseFailures += 1;
};
}
const addSourceMap = (
file: JavaScriptArtifactFile,
@@ -1,7 +1,7 @@
import type { ArtifactInventorySnapshot } from "../../domain/artifactInventorySnapshot.js";
import {
createJavaScriptApplicationGraph,
createImmutableJavaScriptApplicationGraphSteps,
sealTransferredJavaScriptApplicationGraphSteps,
type JavaScriptApplicationGraph,
} from "../../domain/javascript/javascriptApplicationGraph.js";
import type { JavaScriptModuleArtifactAnalysis } from "./JavaScriptArtifactAnalysisTypes.js";
@@ -15,9 +15,9 @@ import {
addJavaScriptHtmlRoles,
addJavaScriptSourceMapOriginals,
} from "./JavaScriptArtifactGraphDocuments.js";
import { addJavaScriptStaticFindings } from "./JavaScriptArtifactGraphFindings.js";
import { addJavaScriptStaticFindingsSteps } from "./JavaScriptArtifactGraphFindings.js";
import {
addJavaScriptModuleRelationships,
addJavaScriptModuleRelationshipsSteps,
type JavaScriptModuleRelationshipOmissions,
addJavaScriptSourceModules,
} from "./JavaScriptModuleRelationships.js";
@@ -51,24 +51,42 @@ export const buildJavaScriptArtifactGraph = (
analysis: JavaScriptModuleArtifactAnalysis,
): JavaScriptApplicationGraph =>
createJavaScriptApplicationGraph(
buildJavaScriptArtifactGraphInput(snapshot, fileSet, analysis),
completeSynchronously(
buildJavaScriptArtifactGraphInputSteps(snapshot, fileSet, analysis),
),
);
/** Build a validated application graph with cooperative immutable ownership. */
export const buildImmutableJavaScriptArtifactGraphSteps = (
/**
* Build a validated application graph with cooperative immutable ownership.
* Constructing the graph input also runs in steps: projecting every module
* relationship and finding at once held the event loop for seconds.
*/
export function* buildImmutableJavaScriptArtifactGraphSteps(
snapshot: ArtifactInventorySnapshot,
fileSet: JavaScriptArtifactFileSet,
analysis: JavaScriptModuleArtifactAnalysis,
): Generator<void, JavaScriptApplicationGraph> =>
createImmutableJavaScriptApplicationGraphSteps(
buildJavaScriptArtifactGraphInput(snapshot, fileSet, analysis),
): Generator<void, JavaScriptApplicationGraph> {
const input = yield* buildJavaScriptArtifactGraphInputSteps(
snapshot,
fileSet,
analysis,
);
// The input was built here and is not retained, so it can be transferred.
return yield* sealTransferredJavaScriptApplicationGraphSteps(input);
}
const buildJavaScriptArtifactGraphInput = (
const completeSynchronously = <Value>(steps: Iterator<void, Value>): Value => {
for (;;) {
const step = steps.next();
if (step.done === true) return step.value;
}
};
function* buildJavaScriptArtifactGraphInputSteps(
snapshot: ArtifactInventorySnapshot,
fileSet: JavaScriptArtifactFileSet,
analysis: JavaScriptModuleArtifactAnalysis,
): unknown => {
): Generator<void, unknown> {
const accumulator = new JavaScriptArtifactGraphAccumulator();
const root = createJavaScriptArtifactRootNode(accumulator, snapshot);
const context: JavaScriptArtifactGraphContext = {
@@ -86,15 +104,23 @@ const buildJavaScriptArtifactGraphInput = (
containerNodes: new Map([[snapshot.manifest.root_sha256, root]]),
};
addJavaScriptArtifactContainers(context);
yield;
addJavaScriptArtifactFiles(context);
yield;
const packageRoots = addJavaScriptPackageNodes(context);
yield;
addJavaScriptSourceModules(context);
yield;
const bundlerLimitations = addJavaScriptBundlerNodes(context);
const relationshipOmissions = addJavaScriptModuleRelationships(context);
const findingLimitations = addJavaScriptStaticFindings(context);
const relationshipOmissions =
yield* addJavaScriptModuleRelationshipsSteps(context);
const findingLimitations = yield* addJavaScriptStaticFindingsSteps(context);
addElectronBoundaries(context);
yield;
addJavaScriptHtmlRoles(context);
yield;
addJavaScriptSourceMapOriginals(context);
yield;
const coverage = graphCoverage(context);
return {
schema: "JavaScriptApplicationGraph",
@@ -111,7 +137,7 @@ const buildJavaScriptArtifactGraphInput = (
...graphLimitations(context, coverage.status, relationshipOmissions),
],
};
};
}
const graphCoverage = (context: JavaScriptArtifactGraphContext) => {
const resourceLimits = semanticResourceLimits(context);
@@ -44,11 +44,12 @@ interface ResolvedReference {
}
/** Project imports, workers, roles, endpoints, storage, and source-map links. */
export const addJavaScriptStaticFindings = (
export function* addJavaScriptStaticFindingsSteps(
context: JavaScriptArtifactGraphContext,
): readonly string[] => {
): Generator<void, readonly string[]> {
const omissions = { selfReferences: 0 };
for (const analyzed of context.analysis.files) {
yield;
const { file, javascript } = analyzed;
const asset = context.assetNodes.get(file.path);
if (javascript === null || asset === undefined) continue;
@@ -69,7 +70,7 @@ export const addJavaScriptStaticFindings = (
["static reference", "static references"],
"the referencing module",
);
};
}
const addReference = (
context: JavaScriptArtifactGraphContext,
@@ -67,6 +67,36 @@ describe("artifact-local graph identity", () => {
});
});
describe("logical path identity", () => {
it("resolves an NFD relative import to the NFC inventory path and rejects a root escape", () => {
const nfc = "caf\u00e9".normalize("NFC");
const nfd = nfc.normalize("NFD");
const files = fileMap([
file("dir/index.js", "root"),
file(`dir/${nfc}.js`, "root"),
]);
expect(
resolve({
declaredPath: `./${nfd}.js`,
sourcePath: "dir/index.js",
context: "module-specifier",
files,
}),
).toMatchObject({
resolution_status: "resolved",
resolved_path: `dir/${nfc}.js`,
});
expect(
resolve({
declaredPath: "../escape.js",
sourcePath: "index.js",
context: "module-specifier",
files,
}),
).toMatchObject({ resolution_status: "rejected", resolved_path: null });
});
});
describe("contextual JavaScript artifact path resolution", () => {
it.each([
["main.js", "app/main.js"],
@@ -1,6 +1,10 @@
import { isBuiltin } from "node:module";
import { posix } from "node:path";
import {
normalizeJoinedLogicalPath,
logicalPathEscapesRoot,
} from "../../domain/artifactIdentity.js";
import type { JavaScriptArtifactFile } from "../../domain/javascript/javascriptArtifactFiles.js";
import {
admitsCanonicalPathSyntax,
@@ -326,12 +330,8 @@ const confineCandidate = (
input: ResolveArtifactPathInput,
candidate: string,
): string | ArtifactPathResolution => {
const normalized = posix.normalize(candidate);
if (
normalized === ".." ||
normalized.startsWith("../") ||
normalized.startsWith("/")
)
const normalized = normalizeJoinedLogicalPath(candidate);
if (logicalPathEscapesRoot(normalized))
return unresolvedOutcome(input, "rejected", [
"The resolved candidate escapes the canonical artifact root.",
]);
@@ -90,7 +90,7 @@ export const reconstructJavaScriptArtifact = async (
const semanticProjection = createJavaScriptSemanticGraphProjection();
const analysis = await analyzeAndProjectJavaScriptArtifactFiles(
files,
semanticProjection.projectFile,
semanticProjection.projectFileSteps,
async (file, completed, total) => {
abortIfNeeded(signal);
await progress?.report({
@@ -101,6 +101,7 @@ export const reconstructJavaScriptArtifact = async (
});
abortIfNeeded(signal);
},
signal,
);
await checkpointJavaScriptAnalysis(signal);
abortIfNeeded(signal);
@@ -136,15 +136,17 @@ export interface JavaScriptModuleRelationshipOmissions {
type RelationshipOmissionCounter = { selfImports: number };
/** Compose bounded CommonJS and ESM binding relationships across artifact files. */
export const addJavaScriptModuleRelationships = (
export function* addJavaScriptModuleRelationshipsSteps(
context: JavaScriptArtifactGraphContext,
): JavaScriptModuleRelationshipOmissions => {
): Generator<void, JavaScriptModuleRelationshipOmissions> {
const omissions = { selfImports: 0 };
for (const analyzed of context.analysis.files) {
const { file, semantic } = analyzed;
const source = context.sourceModuleNodes.get(file.path);
if (semantic === null || source === undefined) continue;
for (const link of semantic.ir.moduleLinks) {
// One module can export thousands of bindings, each with return shapes.
yield;
const input = { context, file, semantic, source, link };
if (isExportLink(link)) addExportRelationship(input, omissions);
else if (link.specifier !== null)
@@ -155,7 +157,7 @@ export const addJavaScriptModuleRelationships = (
}
}
return omissions;
};
}
const addExportRelationship = (
input: RelationshipInput,
@@ -1,6 +1,6 @@
import {
createJavaScriptSemanticGraph,
createImmutableJavaScriptSemanticGraphSteps,
sealTransferredJavaScriptSemanticGraphSteps,
type JavaScriptSemanticGraph,
} from "../../domain/javascript/javascriptSemanticGraph.js";
import type {
@@ -130,6 +130,11 @@ export interface JavaScriptSemanticGraphProjection {
file: JavaScriptArtifactFile,
ir: JavaScriptSemanticIr,
) => void;
/** Project one file in steps between its projection passes. */
readonly projectFileSteps: (
file: JavaScriptArtifactFile,
ir: JavaScriptSemanticIr,
) => Generator<void, void>;
readonly finish: (
rootArtifactSha256: string,
applicationGraph: BuilderInput["applicationGraph"],
@@ -146,10 +151,10 @@ export const createJavaScriptSemanticGraphProjection =
const state = emptyState({ nodes: [] });
const fingerprints: JavaScriptSemanticFingerprint[] = [];
let truncatedFiles = 0;
const projectSource = (
const projectSourceSteps = function* (
file: JavaScriptArtifactFile,
ir: JavaScriptSemanticIr,
): void => {
): Generator<void, void> {
if (state.nodes.size >= SEMANTIC_GRAPH_NODE_CEILING) {
truncatedFiles += 1;
return;
@@ -161,7 +166,7 @@ export const createJavaScriptSemanticGraphProjection =
remainingTreeBudget,
);
state.fileNodesDropped = false;
fingerprints.push(...projectFile(file, ir, state));
fingerprints.push(...(yield* projectFileSteps(file, ir, state)));
// Nodes were dropped only if the budget actually blocked creation. A file
// that exactly fills its share ends with a zero budget without dropping
// anything, so the remaining budget alone cannot decide truncation.
@@ -198,7 +203,8 @@ export const createJavaScriptSemanticGraphProjection =
evidence_contexts: state.evidenceContexts.contexts,
nodes: [...state.nodes.values()],
relations,
fingerprints,
// The input is transferred to the factory; `fingerprints` is cleared below.
fingerprints: [...fingerprints],
unknowns,
coverage: {
status: truncatedFiles > 0 ? "partial" : "unknown",
@@ -246,7 +252,11 @@ export const createJavaScriptSemanticGraphProjection =
return graph;
};
return {
projectFile: projectSource,
projectFile: (file, ir) => {
const steps = projectSourceSteps(file, ir);
while (steps.next().done !== true);
},
projectFileSteps: projectSourceSteps,
finish: (rootArtifactSha256, applicationGraph) =>
finishWith(
createJavaScriptSemanticGraph,
@@ -255,18 +265,19 @@ export const createJavaScriptSemanticGraphProjection =
),
finishImmutableSteps: (rootArtifactSha256, applicationGraph) =>
finishWith(
createImmutableJavaScriptSemanticGraphSteps,
sealTransferredJavaScriptSemanticGraphSteps,
rootArtifactSha256,
applicationGraph,
),
};
};
const projectFile = (
function* projectFileSteps(
file: JavaScriptArtifactFile,
ir: JavaScriptSemanticIr,
state: BuilderState,
): JavaScriptSemanticFingerprint[] => {
): Generator<void, JavaScriptSemanticFingerprint[]> {
yield;
const moduleNode = retainNode(state, file, {
kind: "module",
roleKey: "module",
@@ -276,38 +287,50 @@ const projectFile = (
});
if (moduleNode === null) return [];
state.roots.add(moduleNode.node_id);
const callableNodes = new Map(
ir.callables.flatMap((callable) => {
const node = retainNode(state, file, {
kind: "function",
roleKey: `callable:${callable.callableId}`,
location: callable.location,
label: callable.name,
functionNodeId: null,
// The label is display text; keep the exact name, which may be "".
properties: { name: callable.name },
});
return node === null ? [] : [[callable.callableId, node] as const];
}),
);
// Node creation keeps source order, which the per-file node budget relies
// on; yields between nodes do not change it.
const callableNodes = new Map<string, JavaScriptSemanticGraphNode>();
for (const [index, callable] of ir.callables.entries()) {
if (index % 64 === 0) yield;
const node = retainNode(state, file, {
kind: "function",
roleKey: `callable:${callable.callableId}`,
location: callable.location,
label: callable.name,
functionNodeId: null,
// The label is display text; keep the exact name, which may be "".
properties: { name: callable.name },
});
if (node !== null) callableNodes.set(callable.callableId, node);
}
const callableOwnerAt = createSemanticCallableOwnerLookup(ir, callableNodes);
const bindingNodes = new Map(
ir.bindings.flatMap((binding) => {
const location = binding.definitions[0]?.location ?? null;
const kind = binding.kind === "parameter" ? "parameter" : "binding";
const owner = callableOwnerAt(location);
const node = retainNode(state, file, {
kind,
roleKey: `binding:${binding.bindingId}`,
location,
label: binding.name,
functionNodeId: owner?.node_id ?? null,
});
return node === null ? [] : [[binding.bindingId, node] as const];
}),
const bindingNodes = new Map<string, JavaScriptSemanticGraphNode>();
for (const [index, binding] of ir.bindings.entries()) {
if (index % 64 === 0) yield;
const location = binding.definitions[0]?.location ?? null;
const kind = binding.kind === "parameter" ? "parameter" : "binding";
const owner = callableOwnerAt(location);
const node = retainNode(state, file, {
kind,
roleKey: `binding:${binding.bindingId}`,
location,
label: binding.name,
functionNodeId: owner?.node_id ?? null,
});
if (node !== null) bindingNodes.set(binding.bindingId, node);
}
const returnSiteNodes = yield* createReturnSiteNodes(
file,
ir,
callableNodes,
state,
);
const callSiteNodes = yield* createCallSiteNodes(
file,
ir,
callableNodes,
state,
);
const returnSiteNodes = createReturnSiteNodes(file, ir, callableNodes, state);
const callSiteNodes = createCallSiteNodes(file, ir, callableNodes, state);
const definitions: Omit<FileContext, "referenceNodesWithin"> = {
file,
ir,
@@ -328,42 +351,38 @@ const projectFile = (
callableOwnerAt,
callSiteAt: createSemanticCallSiteLookup(ir, callSiteNodes),
};
projectDefinitionsAndReferences(definitions);
yield* projectDefinitionsAndReferences(definitions);
const context: FileContext = {
...definitions,
referenceNodesWithin: createSemanticNodeRangeLookup(
definitions.referenceNodes,
),
};
projectSemanticValues(context);
projectSemanticObjects(context);
projectCalls(context);
projectSemanticPromises(context);
projectSemanticEvents(context);
projectSemanticTimers(context);
projectSemanticChildProcesses(context);
projectSemanticConfiguration(context);
projectSemanticRequests(context);
projectSemanticBoundaries(context);
projectSemanticResources(context);
projectSemanticClosureCaptures(context);
projectSemanticFrontiers(context);
// Each pass completes before the next starts, so yielding between passes
// keeps projection order and results unchanged.
for (const project of FILE_PROJECTION_PASSES) {
yield;
const steps = project(context);
if (steps !== undefined) while (steps.next().done !== true) yield;
}
yield;
return projectSemanticFunctionFingerprints(
file,
ir,
callableNodes,
state.evidenceContexts,
);
};
}
const createReturnSiteNodes = (
function* createReturnSiteNodes(
file: JavaScriptArtifactFile,
ir: JavaScriptSemanticIr,
callables: ReadonlyMap<string, JavaScriptSemanticGraphNode>,
state: BuilderState,
): Map<string, JavaScriptSemanticGraphNode> => {
): Generator<void, Map<string, JavaScriptSemanticGraphNode>> {
const result = new Map<string, JavaScriptSemanticGraphNode>();
for (const callable of ir.callables) {
for (const [index, callable] of ir.callables.entries()) {
if (index % 64 === 0) yield;
const owner = callables.get(callable.callableId);
if (owner === undefined) continue;
for (const site of callable.returnSites) {
@@ -379,16 +398,17 @@ const createReturnSiteNodes = (
}
}
return result;
};
}
const createCallSiteNodes = (
function* createCallSiteNodes(
file: JavaScriptArtifactFile,
ir: JavaScriptSemanticIr,
callables: ReadonlyMap<string, JavaScriptSemanticGraphNode>,
state: BuilderState,
): Map<string, JavaScriptSemanticGraphNode> => {
): Generator<void, Map<string, JavaScriptSemanticGraphNode>> {
const result = new Map<string, JavaScriptSemanticGraphNode>();
for (const call of ir.callSites) {
for (const [index, call] of ir.callSites.entries()) {
if (index % 64 === 0) yield;
const owner =
call.callerCallableId === null
? null
@@ -403,12 +423,13 @@ const createCallSiteNodes = (
if (node !== null) result.set(call.callSiteId, node);
}
return result;
};
}
const projectDefinitionsAndReferences = (
function* projectDefinitionsAndReferences(
context: Omit<FileContext, "referenceNodesWithin">,
): void => {
for (const binding of context.ir.bindings) {
): Generator<void, void> {
for (const [bindingIndex, binding] of context.ir.bindings.entries()) {
if (bindingIndex % 64 === 0) yield;
const bindingNode = context.bindingNodes.get(binding.bindingId);
if (bindingNode === undefined) continue;
for (const [index, definition] of binding.definitions.entries()) {
@@ -428,6 +449,7 @@ const projectDefinitionsAndReferences = (
}
}
for (const [index, reference] of context.ir.references.entries()) {
if (index % 64 === 0) yield;
const expression = retainNode(context.state, context.file, {
kind: "expression",
roleKey: `reference:${String(index)}:${reference.role}:${reference.name}`,
@@ -458,7 +480,7 @@ const projectDefinitionsAndReferences = (
resolution: "resolved",
});
}
};
}
const projectCalls = (context: FileContext): void => {
for (const call of context.ir.callSites) {
@@ -543,3 +565,22 @@ const createArgumentNode = (
: `argument ${String(argument.index)}`,
functionNodeId: callNode.function_node_id,
});
/** Per-file projection passes, run in this order with a yield between each. */
const FILE_PROJECTION_PASSES: readonly ((
context: FileContext,
) => Iterator<void, void> | void)[] = [
projectSemanticValues,
projectSemanticObjects,
projectCalls,
projectSemanticPromises,
projectSemanticEvents,
projectSemanticTimers,
projectSemanticChildProcesses,
projectSemanticConfiguration,
projectSemanticRequests,
projectSemanticBoundaries,
projectSemanticResources,
projectSemanticClosureCaptures,
projectSemanticFrontiers,
];
@@ -240,10 +240,11 @@ export const projectSemanticClosureCaptures = (
};
/** Retain bounded unresolved dynamic-call, dynamic-property and dynamic-scope frontiers. */
export const projectSemanticFrontiers = (
export function* projectSemanticFrontiers(
context: SemanticFlowProjectionContext,
): void => {
for (const frontier of context.ir.frontiers) {
): Generator<void, void> {
for (const [index, frontier] of context.ir.frontiers.entries()) {
if (index % 64 === 0) yield;
// Dynamic environments leave identifier reads and writes unresolved.
const family =
frontier.kind === "dynamic-call"
@@ -275,4 +276,4 @@ export const projectSemanticFrontiers = (
);
addSemanticGraphUnknown(context.state, unknown);
}
};
}
@@ -20,13 +20,14 @@ import type { SemanticFlowProjectionContext } from "./JavaScriptSemanticGraphFlo
import { semanticPropertySlot } from "./JavaScriptSemanticGraphValueProjection.js";
/** Project static object reads, writes, spreads, and destructuring. */
export const projectSemanticObjects = (
export function* projectSemanticObjects(
context: SemanticFlowProjectionContext,
): void => {
): Generator<void, void> {
const values = new Map(
context.ir.bindings.map((binding) => [binding.bindingId, binding.value]),
);
for (const operation of context.ir.objectOperations) {
for (const [index, operation] of context.ir.objectOperations.entries()) {
if (index % 64 === 0) yield;
const occurrence = addObjectOccurrence(context, operation);
const target = objectTarget(context, values, operation);
const ends = objectRelationEnds(context, operation, occurrence, target);
@@ -43,7 +44,7 @@ export const projectSemanticObjects = (
if (!resolved)
addObjectUnknown(context, operation, target.slot ?? occurrence, target);
}
};
}
const OBJECT_RELATIONS = {
read: "reads-property",
+17 -5
View File
@@ -73,15 +73,27 @@ export const traceNativeUiAction = async (
),
);
graph = joinInterfaceBuilderDispatch(decoded.data.graph, metadata.result);
const archiveDecode = decoded.data.graph.coverage.find(
(facet) => facet.facet === "archive_decode",
);
const documentsFound = decoded.data.documents.length > 0;
coverage.push({
facet: "interface_builder_archives",
status: decoded.data.documents.length > 0 ? "complete" : "unsupported",
status:
archiveDecode?.status === "partial" ||
archiveDecode?.status === "complete" ||
archiveDecode?.status === "unsupported"
? archiveDecode.status
: documentsFound
? "complete"
: "unsupported",
reason:
decoded.data.documents.length > 0
archiveDecode?.reason ??
(documentsFound
? null
: "no_compiled_interface_builder_archives_found",
examined: decoded.data.documents.length,
omitted: 0,
: "no_compiled_interface_builder_archives_found"),
examined: archiveDecode?.examined ?? decoded.data.documents.length,
omitted: archiveDecode?.omitted ?? 0,
});
const bounded = boundUiGraph(graph, input.max_nodes, input.max_edges);
graph = bounded.graph;
+10
View File
@@ -15,6 +15,7 @@ import { processTraceSpecificationSchema } from "../../domain/process/processTra
import { processScenarioSchema } from "../../domain/process/processScenario.js";
import { compareProcessCaptures } from "../../domain/process/processComparison.js";
import { parseProcessCapture } from "../../domain/process/processCaptureParsing.js";
import { createProgressReporter } from "../ProgressReporter.js";
import { captureProcessScenario } from "../../process/capture/ProcessHarness.js";
import { PROCESS_PROVIDER } from "../../domain/process/processEvidenceProvider.js";
import { createProcessCaptureEvidence } from "./ProcessEvidence.js";
@@ -56,11 +57,20 @@ export const captureProcessScenarioFile = async (
input,
{ cause: parsed.error },
);
const progress = createProgressReporter(
async (update) => {
process.stderr.write(`${JSON.stringify({ rea_progress: update })}\n`);
},
{ minimumIntervalMs: 100 },
);
const captured = await captureProcessScenario(
parsed.data,
signal,
process.platform,
environment,
undefined,
undefined,
progress,
);
if (!captured.ok) return cliAnalysisError(captured.error);
return createProcessCaptureEvidence(parsed.data, captured.value);
+2 -1
View File
@@ -1,5 +1,6 @@
import { posix } from "node:path";
import { logicalPathNfc } from "../domain/artifactIdentity.js";
import { compareUnicodeCodePoints } from "../domain/unicodeCodePointOrder.js";
import { ArtifactReaderFailure } from "./ArtifactReader.js";
@@ -31,7 +32,7 @@ export const normalizeArtifactPath = (input: string): string => {
"path",
`Artifact path is absolute or unsafe: ${JSON.stringify(input)}`,
);
const normalized = input.normalize("NFC").replace(/\/+$/u, "");
const normalized = logicalPathNfc(input).replace(/\/+$/u, "");
const parts = normalized.split("/");
if (
normalized.length === 0 ||
+45 -19
View File
@@ -6,7 +6,10 @@ import {
import type { FileHandle } from "node:fs/promises";
import type { Stats } from "node:fs";
import { classifyArtifactContent } from "./ArtifactGraphConstruction.js";
import {
classifyArtifactBytes,
classifyArtifactContent,
} from "./ArtifactGraphConstruction.js";
import { ARTIFACT_CLASSIFICATION_PREFIX_BYTES } from "../ArtifactHash.js";
import type { ArtifactOccurrence } from "../../domain/artifactGraph.js";
import {
@@ -17,17 +20,6 @@ import { ArtifactReaderFailure } from "../ArtifactReader.js";
import type { HashResult } from "../ArtifactHash.js";
import { hashStableRootArtifactHandle } from "./hashStableRootArtifact.js";
const classifyContainerExtension = (
path: string,
): ArtifactOccurrence["artifact_format"] | undefined => {
const lower = path.toLowerCase();
const zipPackage = zipPackageFormatForPath(lower);
if (zipPackage !== undefined) return zipPackage;
for (const format of ["asar", "dmg", "pkg"] as const)
if (lower.endsWith(`.${format}`)) return format;
return undefined;
};
/** Classify and hash one file root through the same stable open descriptor. */
export const classifyAndHashRoot = async (
path: string,
@@ -47,9 +39,7 @@ export const classifyAndHashRoot = async (
"integrity",
`Root artifact changed before inventory: ${path}`,
);
const format =
classifyContainerExtension(path) ??
(await classifyOpenedRoot(path, handle));
const format = await classifyRootFormat(path, handle);
const digest = await hashStableRootArtifactHandle(
path,
handle,
@@ -79,14 +69,50 @@ const openRootFile = async (
}
};
const classifyOpenedRoot = async (
const classifyRootFormat = async (
path: string,
handle: FileHandle,
): Promise<ArtifactOccurrence["artifact_format"]> => {
const magic = Buffer.alloc(ARTIFACT_CLASSIFICATION_PREFIX_BYTES);
const observed = await handle.read(magic, 0, magic.length, 0);
const prefix = magic.subarray(0, observed.bytesRead);
if (hasZipSignature(prefix)) return "zip";
return classifyArtifactContent(path, prefix, (await handle.stat()).size)
.format;
const lower = path.toLowerCase();
if (hasZipSignature(prefix)) return zipPackageFormatForPath(lower) ?? "zip";
if (lower.endsWith(".asar") && hasAsarHeader(prefix)) return "asar";
if (
lower.endsWith(".pkg") &&
prefix.subarray(0, 4).toString("ascii") === "xar!"
)
return "pkg";
if (lower.endsWith(".dmg") && (await hasKolyTrailer(handle))) return "dmg";
const size = (await handle.stat()).size;
const format = classifyArtifactContent(path, prefix, size).format;
// Path suffixes stay role hints for nested members. A root whose container
// magic did not match is classified from the bytes that were read.
if (format === "asar" || format === "pkg" || format === "dmg")
return classifyArtifactBytes(prefix, size);
return format;
};
/**
* Electron ASAR starts with a size pickle whose payload is one uint32, then
* a header pickle whose payload begins with a JSON object.
*/
const hasAsarHeader = (prefix: Buffer): boolean => {
if (prefix.length < 16 || prefix.readUInt32LE(0) !== 4) return false;
const headerSize = prefix.readUInt32LE(4);
const payloadSize = prefix.readUInt32LE(8);
if (headerSize < 16 || payloadSize !== headerSize - 4) return false;
const stringLength = prefix.readUInt32LE(12);
return (
stringLength >= 2 && stringLength <= payloadSize - 4 && prefix[16] === 0x7b
);
};
const hasKolyTrailer = async (handle: FileHandle): Promise<boolean> => {
const size = (await handle.stat()).size;
if (size < 512) return false;
const trailer = Buffer.alloc(4);
const read = await handle.read(trailer, 0, trailer.length, size - 512);
return read.bytesRead === 4 && trailer.toString("ascii") === "koly";
};
+10 -1
View File
@@ -5,6 +5,7 @@ import { MachOSliceArtifactReader } from "../MachOSliceArtifactReader.js";
import { NativeDmgArtifactReader } from "../NativeDmgArtifactReader.js";
import { ZipArtifactReader } from "../ZipArtifactReader.js";
import type { ArtifactOccurrence } from "../../domain/artifactGraph.js";
import { ZIP_NON_ENTRY_TAIL_LIMITATION } from "../../domain/zipPackageFormat.js";
export const createReader = async (
path: string,
@@ -39,7 +40,15 @@ export const inventoryLimitations = (
format: ArtifactOccurrence["artifact_format"],
reader: ArtifactReader | undefined,
): string[] => {
if (reader !== undefined) return [];
if (reader !== undefined)
return reader.format === "zip" ||
reader.format === "ipa" ||
reader.format === "apk" ||
reader.format === "msix" ||
reader.format === "appx"
? [ZIP_NON_ENTRY_TAIL_LIMITATION]
: [];
// dmg and pkg reach here only after classifyRootFormat matched their bytes.
if (format === "dmg" || format === "pkg")
return [
`${format.toUpperCase()} root hash is observed; child inventory requires a native macOS adapter.`,
@@ -56,4 +56,39 @@ describe("CDP execution-context retention", () => {
});
expect(electronFrames.size).toBe(0);
});
it("keeps a script on the frame stored when it was parsed", () => {
const request = inspectWebPageInputSchema.parse({
cdp_endpoint: "http://127.0.0.1:9222",
allowed_origins: ["https://app.example.test"],
target_id: "page-1",
});
const browser = new CdpCaptureEvents(
request,
new Set(["https://app.example.test"]),
);
browser.beginAuthorizedFrame("frame-old");
browser.beginAuthorizedFrame("frame-new");
browser.ingest({
method: "Runtime.executionContextCreated",
params: { context: { id: 7, auxData: { frameId: "frame-old" } } },
});
browser.ingest({
method: "Debugger.scriptParsed",
params: {
scriptId: "script-1",
url: "https://app.example.test/app.js",
executionContextId: 7,
},
});
browser.ingest({
method: "Runtime.executionContextCreated",
params: { context: { id: 7, auxData: { frameId: "frame-new" } } },
});
const script = browser.scripts.get("script-1");
if (script === undefined) throw new TypeError("Expected captured script");
expect(
browser.frameForScript(script, new Set(["frame-old", "frame-new"])),
).toBe("frame-old");
});
});
@@ -198,6 +198,23 @@ describe("empty form destinations with a document base URL", () => {
},
);
it("keeps the first value when an attribute name repeats", () => {
const value = snapshot();
const later = `${origin}/later`;
value.strings.push(later);
const href = value.strings.indexOf("href");
const guide = value.strings.indexOf("guide");
const nodes = value.documents[0]?.nodes;
if (nodes === undefined) throw new TypeError("Expected a DOM snapshot");
nodes.attributes[2] = [href, guide, href, value.strings.length - 1];
const result = capture(value);
expect(result.nodes[2]?.attribute_names).toEqual(["href", "href"]);
expect(result.urls.map(({ url }) => url)).toContain(
`${origin}/screens/guide`,
);
expect(result.urls.map(({ url }) => url)).not.toContain(later);
});
it("does not apply form semantics to an action attribute on another element", () => {
const baseUrl = `${origin}/assets/`;
const value = snapshot(baseUrl);
+2 -1
View File
@@ -387,7 +387,8 @@ const domMetadata = (
for (let index = 0; index + 1 < attributes.length; index += 2) {
const name = indexedString(strings, attributes[index]).toLowerCase();
const value = indexedString(strings, attributes[index + 1]);
pairs.set(name, value);
// HTML keeps the first declaration. A later repeat must not move dom_urls.
if (!pairs.has(name)) pairs.set(name, value);
}
const urls: WebPageInspection["metadata"]["dom_urls"] = [];
for (const attribute of domUrlAttributes) {
+10 -2
View File
@@ -47,6 +47,7 @@ export const handleExecutionContextCreated = (
)
return;
const key = String(identifier);
// New contexts may reuse an id. Scripts already parsed keep the frame stored on them.
state.executionContextFrames.set(key, frameId);
};
@@ -80,6 +81,7 @@ export const handleScriptParsed = (
state.completeness.exclude("scripts", exclusionReasonForUrl(rawUrl));
return;
}
const executionContext = executionContextKey(params.executionContextId);
const baseScript = {
scriptId,
rawUrl,
@@ -87,7 +89,11 @@ export const handleScriptParsed = (
origin: sanitized.origin,
...scriptMetadataValues(params),
language: cdpStringValue(params.scriptLanguage) ?? null,
executionContextKey: executionContextKey(params.executionContextId),
executionContextKey: executionContext,
frameId:
executionContext === null
? null
: (state.executionContextFrames.get(executionContext) ?? null),
};
const baseBytes = scriptMetadataStringBytes(baseScript);
const sourceMapDeclared =
@@ -131,6 +137,7 @@ const scriptMetadataStringBytes = (script: {
readonly hash: string | null;
readonly language: string | null;
readonly executionContextKey: string | null;
readonly frameId: string | null;
}): number =>
stringByteLength(script.scriptId) +
stringByteLength(script.rawUrl) +
@@ -138,7 +145,8 @@ const scriptMetadataStringBytes = (script: {
stringByteLength(script.origin) +
stringByteLength(script.hash) +
stringByteLength(script.language) +
stringByteLength(script.executionContextKey);
stringByteLength(script.executionContextKey) +
stringByteLength(script.frameId);
const stringByteLength = (value: string | null): number =>
value === null ? 0 : Buffer.byteLength(value, "utf8");
+2
View File
@@ -12,6 +12,8 @@ export interface CapturedScript {
readonly sourceMapUrl: string | null;
readonly sourceMapRawUrl: string | null;
readonly executionContextKey: string | null;
/** Frame observed when this script was parsed. Later context reuse does not move it. */
readonly frameId: string | null;
}
export type NetworkState = WebPageInspection["network"]["requests"][number];
+2 -4
View File
@@ -159,10 +159,8 @@ export class CdpCaptureEvents implements CdpCaptureEventsState {
script: CapturedScript,
allowedFrameIds: ReadonlySet<string>,
): string | null {
if (script.executionContextKey === null) return null;
const frameId = this.executionContextFrames.get(script.executionContextKey);
return frameId !== undefined && allowedFrameIds.has(frameId)
? frameId
return script.frameId !== null && allowedFrameIds.has(script.frameId)
? script.frameId
: null;
}
@@ -3,12 +3,22 @@ import type { Browser } from "playwright-core";
import { browserScenarioSchema } from "../domain/browserScenario.js";
import {
attachedScenarioLeaseKey,
openPlaywrightScenarioBrowser,
PlaywrightScenarioBrowserCleanupOwner,
} from "./PlaywrightScenarioBrowser.js";
afterEach(() => vi.unstubAllEnvs());
it("locks one attached browser for spellings that discover the same endpoint", () => {
expect(attachedScenarioLeaseKey("http://127.0.0.1:9222", "page-1")).toBe(
attachedScenarioLeaseKey("http://127.0.0.1:9222/", "page-1"),
);
expect(attachedScenarioLeaseKey("http://127.0.0.1:9222", "page-1")).not.toBe(
attachedScenarioLeaseKey("http://127.0.0.1:9222", "page-2"),
);
});
const deferred = <Value>() => {
let resolve: (value: Value | PromiseLike<Value>) => void = () => undefined;
const promise = new Promise<Value>((resolvePromise) => {
+11 -1
View File
@@ -182,6 +182,13 @@ const createAttachedScenarioRelease = (
};
};
/** One browser admission. The key is the discovery URL this endpoint already parses. */
export const attachedScenarioLeaseKey = (
endpoint: string,
targetId: string,
): string =>
JSON.stringify([new URL("/json/version", endpoint).href, targetId]);
const acquireAttachedScenario = (
scenario: Extract<BrowserScenario["browser"], { readonly mode: "connect" }>,
signal: AbortSignal | undefined,
@@ -189,7 +196,10 @@ const acquireAttachedScenario = (
if (signal?.aborted === true)
return Promise.reject(new BrowserObservationError(OPERATION, "cancelled"));
const key = JSON.stringify([scenario.cdp_endpoint, scenario.target_id]);
const key = attachedScenarioLeaseKey(
scenario.cdp_endpoint,
scenario.target_id,
);
const admission =
attachedScenarioAdmissions.get(key) ??
({
+64 -16
View File
@@ -2,20 +2,26 @@ import { createHash } from "node:crypto";
import { join } from "node:path";
import { Readable } from "node:stream";
import { ArtifactReaderFailure } from "../../artifacts/ArtifactReader.js";
import { DESTINATION_CASE_COLLISION_PREFIX } from "../../artifacts/ArtifactPaths.js";
import { SafeOutputTree } from "../../artifacts/SafeOutputTree.js";
import {
webScriptExportManifestSchema,
webScriptExportResultSchema,
type ExportWebScriptsInput,
type ExportedWebScript,
type WebScriptExportResult,
} from "../../domain/webScriptExport.js";
import { planWebScriptExport } from "../../domain/webScriptExportPlan.js";
import {
planWebScriptExport,
type PlannedWebScript,
} from "../../domain/webScriptExportPlan.js";
import { WebScriptExportError } from "../../domain/webScriptExportError.js";
import type { SelectedScriptCapture } from "./ScriptCaptureAdapters.js";
const LIMITATIONS = [
"Only script bytes retained in this capture are exported. Sources are not refetched or executed, and a response or Debugger source does not prove execution.",
"Ordinary unambiguous URL paths preserve relative module layouts by origin. Query variants, collisions, inline sources, and unrepresentable paths are isolated; their browser module relationships remain unknown.",
"Ordinary unambiguous URL paths preserve relative module layouts by origin, including case-distinct spellings. Query variants, exact path collisions, file/directory prefixes, inline sources, and unrepresentable paths are isolated. A case-distinct pair is isolated only when the destination filesystem cannot store both spellings; the record names that pair. Browser module relationships of isolated paths remain unknown.",
"Local JavaScript analysis does not implement browser import maps, root-relative URL resolution, remote URL loading, or browser runtime behavior. It may strip import queries and fragments; isolated variants are not canonical targets.",
"Source-map declarations are retained as metadata. No source maps or missing dependencies are fetched or reconstructed.",
"Captured response bytes are browser-decoded bytes; Debugger sources are the exposed text encoded as UTF-8. Redacted bytes are exported exactly as retained, and may no longer parse as JavaScript.",
@@ -32,7 +38,11 @@ export const publishWebScripts = async (
const tree = await SafeOutputTree.create(input.output_directory);
try {
signal?.throwIfAborted();
const records = planned.map(({ record }) => record);
const records: ExportedWebScript[] = [];
for (const [index, item] of planned.entries()) {
signal?.throwIfAborted();
records.push(await publishPlannedScript(tree, item, index, signal));
}
const manifest = webScriptExportManifestSchema.parse({
capture_path: input.capture_path,
capture_sha256: captureSha256,
@@ -66,19 +76,6 @@ export const publishWebScripts = async (
bytes: bytes.length,
},
});
for (const { script, record } of planned) {
signal?.throwIfAborted();
if (
script.content.state === "captured" &&
record.content.state === "exported"
)
await tree.write(
`files/${record.content.relative_path}`,
Readable.from([script.content.bytes]),
script.content.sha256,
signal,
);
}
await tree.write("manifest.json", Readable.from([bytes]), digest, signal);
signal?.throwIfAborted();
await tree.commit();
@@ -98,5 +95,56 @@ export const publishWebScripts = async (
}
};
const publishPlannedScript = async (
tree: SafeOutputTree,
item: PlannedWebScript,
index: number,
signal: AbortSignal | undefined,
): Promise<ExportedWebScript> => {
const { script, record } = item;
const content = script.content;
if (content.state !== "captured" || record.content.state !== "exported")
return record;
const write = (relativePath: string): Promise<unknown> =>
tree.write(
`files/${relativePath}`,
Readable.from([content.bytes]),
content.sha256,
signal,
);
try {
await write(record.content.relative_path);
return record;
} catch (cause: unknown) {
const collision = destinationCollisionReason(cause);
if (collision === undefined || record.content.layout !== "url-path")
throw cause;
const relativePath = isolatedScriptPath(script, index);
await write(relativePath);
return {
...record,
content: {
...record.content,
relative_path: relativePath,
layout: "isolated",
layout_reason: collision,
},
};
}
};
const isolatedScriptPath = (
script: PlannedWebScript["script"],
index: number,
): string =>
`isolated/source-${index + 1}-${createHash("sha256").update(JSON.stringify(script.source)).digest("hex")}.js`;
const destinationCollisionReason = (cause: unknown): string | undefined =>
cause instanceof ArtifactReaderFailure &&
cause.reason === "path" &&
cause.message.startsWith(DESTINATION_CASE_COLLISION_PREFIX)
? cause.message
: undefined;
const message = (cause: unknown): string =>
cause instanceof Error ? cause.message : "Unknown publication failure";
+2
View File
@@ -5,11 +5,13 @@ import { ConfigurationError } from "../domain/configurationErrors.js";
import { err, ok, type Result } from "../domain/result.js";
import { analysisProviderSelectorSchema } from "../contracts/providerSelection.js";
import { ghidraStartupTimeoutSchema } from "./ghidraStartupTimeout.js";
import { mcpInputSchemaProfileSchema } from "./mcpInputSchemaProfile.js";
import { mcpResponseBudgetSchema } from "./mcpResponseBudget.js";
const environmentSchema = z.object({
REA_ANALYSIS_PROVIDER: analysisProviderSelectorSchema.default("auto"),
REA_MCP_MAX_RESPONSE_BYTES: mcpResponseBudgetSchema.optional(),
REA_MCP_INPUT_SCHEMA_PROFILE: mcpInputSchemaProfileSchema.optional(),
REA_IDA_MCP_CONFIG: z
.string()
.min(1)
+38
View File
@@ -0,0 +1,38 @@
import { z } from "zod";
import { ConfigurationError } from "../domain/configurationErrors.js";
import { err, ok, type Result } from "../domain/result.js";
// Moonshot's function-calling API rejects requests while any single tool
// parameters schema exceeds its literal serialized size cap. Community
// measurements bound the cap between ~13.7 KB (accepted) and ~15 KB
// (rejected); 13 KiB leaves margin under the lower bound. See
// https://github.com/morluto/rea/issues/1484 for the measured evidence.
export const COMPACT_INPUT_SCHEMA_BUDGET_BYTES = 13 * 1024;
export const mcpInputSchemaProfileSchema = z.enum(["full", "compact"]);
const profileConstraint =
'REA_MCP_INPUT_SCHEMA_PROFILE must be "full" (default) or "compact". The compact profile advertises input schemas without nested annotation prose for providers that cap each tool schema\'s serialized size.';
export type McpInputSchemaProfile = z.infer<typeof mcpInputSchemaProfileSchema>;
/** Parse the advertised input schema presentation selection. */
export const parseMcpInputSchemaProfile = (
value: string | undefined,
): Result<McpInputSchemaProfile | undefined, ConfigurationError> => {
if (value === undefined) return ok(undefined);
const parsed = mcpInputSchemaProfileSchema.safeParse(value);
return parsed.success
? ok(parsed.data)
: err(
new ConfigurationError(profileConstraint, {
settings: [
{
setting: "REA_MCP_INPUT_SCHEMA_PROFILE",
constraint: profileConstraint,
},
],
}),
);
};
+3
View File
@@ -50,6 +50,9 @@ export const parseConfig = (
...(env.REA_MCP_MAX_RESPONSE_BYTES === undefined
? {}
: { mcpMaxResponseBytes: env.REA_MCP_MAX_RESPONSE_BYTES }),
...(env.REA_MCP_INPUT_SCHEMA_PROFILE === undefined
? {}
: { mcpInputSchemaProfile: env.REA_MCP_INPUT_SCHEMA_PROFILE }),
analysisProvider: env.REA_ANALYSIS_PROVIDER,
...(env.REA_IDA_MCP_CONFIG === undefined
? {}
+3
View File
@@ -2,8 +2,11 @@ import type { LogLevel } from "../logger.js";
import type { ReferenceSourcePolicy } from "../domain/referenceSourcePolicy.js";
import type { AnalysisProviderSelector } from "../contracts/providerSelection.js";
import type { McpInputSchemaProfile } from "./mcpInputSchemaProfile.js";
export interface AppConfig {
readonly mcpMaxResponseBytes?: number;
readonly mcpInputSchemaProfile?: McpInputSchemaProfile;
readonly analysisProvider: AnalysisProviderSelector;
readonly idaMcpConfigPath?: string;
readonly ghidraInstallDir: string | undefined;
+314
View File
@@ -0,0 +1,314 @@
import { Ajv2020 } from "ajv/dist/2020.js";
import { describe, expect, it } from "vitest";
import { compactAdvertisedInputSchema } from "./compactInputSchema.js";
const serializedBytes = (value: unknown): number =>
Buffer.byteLength(JSON.stringify(value), "utf8");
const repeatedObservation = {
type: "object",
properties: {
captured_url: { type: "string", minLength: 1 },
captured_at: { type: "string", minLength: 1 },
status_code: { type: "integer", minimum: 100, maximum: 599 },
headers: {
type: "object",
additionalProperties: { type: "string" },
minProperties: 1,
},
body_digest: { type: "string", minLength: 64, maxLength: 64 },
},
required: ["captured_url", "captured_at", "status_code"],
additionalProperties: false,
} as const;
const annotatedSchema = {
type: "object",
description: "Root group guidance stays.",
properties: {
query: {
type: "string",
description: "Field guidance leaves.",
examples: ["seed"],
default: "seed",
title: "Query",
minLength: 1,
},
limits: {
type: "object",
description: "Nested guidance leaves.",
properties: {
max_results: {
type: "integer",
description: "Leaves.",
minimum: 1,
maximum: 500,
default: 50,
},
},
required: ["max_results"],
additionalProperties: false,
},
},
required: ["query"],
dependentRequired: { limits: ["query"] },
additionalProperties: false,
} as const;
const oversizedPairSchema = (): Record<string, unknown> => {
const deepStructure = {
type: "object",
description: "Nested guidance that leaves.",
properties: Object.fromEntries(
Array.from({ length: 40 }, (_, index) => [
`field_${index}`,
{ type: "string", minLength: 1, description: `Field ${index}` },
]),
),
required: Array.from({ length: 40 }, (_, index) => `field_${index}`),
additionalProperties: false,
};
return {
type: "object",
description: "Provide a complete capture pair.",
properties: {
before: structuredClone(deepStructure),
after: structuredClone(deepStructure),
},
required: ["before", "after"],
additionalProperties: false,
};
};
describe("compact advertised input schema", () => {
it("drops nested annotation prose while keeping validation keywords", () => {
const { schema, presentation } = compactAdvertisedInputSchema(
structuredClone(annotatedSchema),
13 * 1024,
);
expect(presentation).toBe("inline");
expect(schema.description).toBe("Root group guidance stays.");
const query = (schema.properties as Record<string, unknown>)
.query as Record<string, unknown>;
expect(query).toEqual({ type: "string", minLength: 1 });
const limits = (schema.properties as Record<string, unknown>)
.limits as Record<string, unknown>;
expect(limits.description).toBeUndefined();
expect(limits.required).toEqual(["max_results"]);
expect(schema.dependentRequired).toEqual({ limits: ["query"] });
expect(schema.required).toEqual(["query"]);
});
it("keeps annotation-shaped keys inside literal const and enum data", () => {
const literalMode = {
description: "saved data",
default: { keep: "me" },
};
const { schema } = compactAdvertisedInputSchema(
{
type: "object",
properties: {
mode: { enum: [literalMode] },
pinned: { const: { description: "literal" } },
},
},
13 * 1024,
);
const properties = schema.properties as Record<string, unknown>;
expect((properties.mode as Record<string, unknown>).enum).toEqual([
literalMode,
]);
expect((properties.pinned as Record<string, unknown>).const).toEqual({
description: "literal",
});
});
it("inlines local references without changing validation or literal data", () => {
const original = {
type: "object",
$defs: { "observation/~": structuredClone(repeatedObservation) },
properties: {
before: { $ref: "#/$defs/observation~1~0" },
after: { $ref: "#/$defs/observation~1~0" },
literal: { const: { $ref: "saved data", $defs: { keep: true } } },
},
required: ["before", "after"],
additionalProperties: false,
};
const { schema, presentation } = compactAdvertisedInputSchema(
original,
13312,
);
expect(presentation).toBe("inline");
expect(schema.$defs).toBeUndefined();
const ajv = new Ajv2020({ strict: false });
const canonical = ajv.compile(original);
const compact = ajv.compile(schema);
const observation = {
captured_url: "https://example.com",
captured_at: "now",
status_code: 200,
};
for (const input of [
{ before: observation, after: observation },
{ before: observation, after: { ...observation, status_code: 99 } },
{
before: observation,
after: observation,
literal: original.properties.literal.const,
},
])
expect(compact(input)).toBe(canonical(input));
});
it("keeps unconstrained JSON values explicit for client type inference", () => {
const { schema } = compactAdvertisedInputSchema(
{
type: "object",
properties: { normalized_result: {}, literal: { const: {} } },
},
13312,
);
const properties = schema.properties as Record<string, unknown>;
expect(properties.normalized_result).toEqual({
type: ["null", "boolean", "object", "array", "number", "string"],
});
expect(properties.literal).toEqual({ const: {} });
const validate = new Ajv2020({ strict: false }).compile(schema);
for (const normalized_result of [null, true, {}, [], 42, "result"])
expect(validate({ normalized_result })).toBe(true);
});
it("preserves independent validation constraints on reference siblings", () => {
const original = {
type: "object",
$defs: { text: { type: "string", minLength: 3 } },
properties: {
query: { $ref: "#/$defs/text", minLength: 1, maxLength: 5 },
},
};
const { schema } = compactAdvertisedInputSchema(original, 13312);
const validate = new Ajv2020({ strict: false }).compile(schema);
expect(validate({ query: "a" })).toBe(false);
expect(validate({ query: "abc" })).toBe(true);
expect(validate({ query: "abcdef" })).toBe(false);
});
});
describe("compact input schema budget and reduction", () => {
it("budgets repeated-reference expansion before materializing it", () => {
const definitions: Record<string, unknown> = {
leaf: structuredClone(repeatedObservation),
};
for (let index = 0; index < 30; index++) {
const $ref = `#/$defs/${index === 0 ? "leaf" : `level${index - 1}`}`;
definitions[`level${index}`] = {
type: "object",
properties: { left: { $ref }, right: { $ref } },
};
}
const { schema, presentation } = compactAdvertisedInputSchema(
{
type: "object",
$defs: definitions,
properties: { tree: { $ref: "#/$defs/level29" } },
},
1024,
);
expect(presentation).toBe("reduced");
expect(serializedBytes(schema)).toBeLessThanOrEqual(1024);
expect(schema.properties).toEqual({ tree: { type: "object" } });
});
it("retains referenced object, array and nullable types in reduced inputs", () => {
const original = {
type: "object",
$defs: {
pair: oversizedPairSchema(),
observations: {
type: "array",
items: structuredClone(repeatedObservation),
},
nullable: { anyOf: [{ type: "string" }, { type: "null" }] },
},
properties: {
capture: { $ref: "#/$defs/pair", description: "Complete capture." },
observations: { $ref: "#/$defs/observations" },
label: { $ref: "#/$defs/nullable" },
},
};
const { schema, presentation } = compactAdvertisedInputSchema(
original,
1024,
);
expect(presentation).toBe("reduced");
expect(schema.properties).toEqual({
capture: { type: "object", description: "Complete capture." },
observations: { type: "array" },
label: { type: ["string", "null"] },
});
const validate = new Ajv2020({ strict: false }).compile(schema);
expect(validate({ capture: {}, observations: [], label: null })).toBe(true);
expect(validate({ capture: {}, observations: [], label: "capture" })).toBe(
true,
);
expect(validate({ capture: "{}", observations: [], label: null })).toBe(
false,
);
});
it("reduces structurally oversized schemas to the shallow property presentation", () => {
const compacted = compactAdvertisedInputSchema(oversizedPairSchema(), 1024);
expect(compacted.presentation).toBe("reduced");
expect(serializedBytes(compacted.schema)).toBeLessThanOrEqual(1024);
expect(compacted.schema.type).toBe("object");
expect(compacted.schema.description).toContain(
"Provide a complete capture pair.",
);
expect(compacted.schema.description).toContain(
"compact input schema profile",
);
const properties = compacted.schema.properties as Record<string, unknown>;
expect(properties.before).toEqual({
type: "object",
description: "Nested guidance that leaves.",
});
expect(properties.after).toEqual(properties.before);
expect(compacted.schema.required).toEqual(["before", "after"]);
});
it("produces deterministic output for equal input", () => {
const schema = {
type: "object",
properties: {
before: structuredClone(repeatedObservation),
after: structuredClone(repeatedObservation),
},
};
expect(compactAdvertisedInputSchema(schema, 13 * 1024)).toEqual(
compactAdvertisedInputSchema(structuredClone(schema), 13 * 1024),
);
});
it("keeps inline presentations under the measured provider budget", () => {
const { schema, presentation } = compactAdvertisedInputSchema(
{
type: "object",
description: "Group guidance.",
properties: {
before: structuredClone(repeatedObservation),
after: structuredClone(repeatedObservation),
normalization: {
type: "object",
description: "Rules.",
properties: { rules: { type: "array", items: { type: "string" } } },
},
},
},
13312,
);
expect(presentation).toBe("inline");
expect(serializedBytes(schema)).toBeLessThanOrEqual(13312);
});
});
+311
View File
@@ -0,0 +1,311 @@
import {
schemaArrays,
schemaChildren,
schemaMaps,
} from "./inputSchemaPresentation.js";
// Kimi expands schema-local references before sending tools to Moonshot.
// Compact advertisements must fit after expansion, independently of client
// reference handling. Canonical server-side validation remains unchanged.
/** Annotation keys that carry guidance but no validation semantics. */
const annotationKeys = new Set(["description", "examples", "default", "title"]);
/** Keys whose values are literal data, never nested schemas to rewrite. */
const dataKeys = new Set(["const", "enum", "examples", "default"]);
const definitionKeys = new Set(["$defs", "definitions"]);
// An empty schema accepts every JSON value. Kimi otherwise infers string.
const jsonValueTypes = [
"null",
"boolean",
"object",
"array",
"number",
"string",
];
export type AdvertisedInputSchemaPresentation = "inline" | "reduced";
export interface CompactedInputSchema {
readonly schema: Record<string, unknown>;
readonly presentation: AdvertisedInputSchemaPresentation;
}
const isSchemaObject = (value: unknown): value is Record<string, unknown> =>
typeof value === "object" && value !== null && !Array.isArray(value);
const serializedBytes = (value: unknown): number =>
Buffer.byteLength(JSON.stringify(value), "utf8");
/** Rewrite every schema-valued position; literal data stays opaque. */
const rebuildSchemaPositions = (
node: unknown,
rebuild: (child: unknown) => unknown,
): unknown => {
if (Array.isArray(node)) return node.map((child) => rebuild(child));
if (!isSchemaObject(node)) return node;
const out: Record<string, unknown> = {};
for (const [key, value] of Object.entries(node)) {
if (dataKeys.has(key)) {
out[key] = value;
continue;
}
if (schemaMaps.has(key) && isSchemaObject(value)) {
out[key] = Object.fromEntries(
Object.entries(value).map(([name, child]) => [name, rebuild(child)]),
);
continue;
}
if (schemaArrays.has(key) && Array.isArray(value)) {
out[key] = value.map((child) => rebuild(child));
continue;
}
if (schemaChildren.has(key)) {
out[key] = Array.isArray(value)
? value.map((child) => rebuild(child))
: rebuild(value);
continue;
}
out[key] = value;
}
return out;
};
/**
* Drop annotation prose from every subschema while preserving all validation
* keywords. The root keeps its own description because the root-union
* projection encodes accepted input groups there; the root's echoed literal
* examples leave with the rest of the annotation prose.
*/
const stripAnnotations = (
schema: Record<string, unknown>,
): Record<string, unknown> => {
const rebuilt = rebuildSchemaPositions(
schema,
stripChildAnnotations,
) as Record<string, unknown>;
const out: Record<string, unknown> = {};
for (const [key, value] of Object.entries(rebuilt)) {
if (annotationKeys.has(key) && key !== "description") continue;
out[key] = value;
}
return out;
};
const stripChildAnnotations = (child: unknown): unknown => {
const rebuilt = rebuildSchemaPositions(child, stripChildAnnotations);
if (!isSchemaObject(rebuilt)) return rebuilt;
const kept: Record<string, unknown> = {};
for (const [key, value] of Object.entries(rebuilt))
if (!annotationKeys.has(key)) kept[key] = value;
return kept;
};
const resolveLocalReference = (
root: Record<string, unknown>,
reference: string,
): unknown => {
if (reference === "#") return root;
if (!reference.startsWith("#/")) return undefined;
let value: unknown = root;
for (const token of reference.slice(2).split("/")) {
const key = token.replaceAll("~1", "/").replaceAll("~0", "~");
if (!isSchemaObject(value) || !Object.hasOwn(value, key)) return undefined;
value = value[key];
}
return value;
};
/** Stop serialization at the budget before allocating an expanded schema. */
const inlineWithinBudget = (
root: Record<string, unknown>,
budgetBytes: number,
): Record<string, unknown> | undefined => {
let resolved = true;
function* serialize(
value: unknown,
position: "schema" | "map" | "data",
active: ReadonlySet<string>,
): Generator<string> {
if (Array.isArray(value)) {
yield "[";
for (const [index, child] of value.entries()) {
if (index > 0) yield ",";
yield* serialize(child, position, active);
}
yield "]";
} else if (isSchemaObject(value)) {
if (position === "schema" && Object.keys(value).length === 0) {
yield JSON.stringify({ type: jsonValueTypes });
return;
}
if (position === "schema" && typeof value.$ref === "string") {
const target = resolveLocalReference(root, value.$ref);
if (target === undefined || active.has(value.$ref)) {
resolved = false;
return;
}
const { $ref, ...siblings } = value;
// Reference siblings are independent constraints in draft 2020-12.
// Merging their keys into the target would overwrite validation rules.
yield* serialize(
Object.keys(siblings).length === 0
? target
: { allOf: [target, siblings] },
"schema",
new Set([...active, $ref]),
);
return;
}
yield "{";
let first = true;
for (const [key, child] of Object.entries(value)) {
if (position === "schema" && definitionKeys.has(key)) continue;
if (!first) yield ",";
first = false;
yield JSON.stringify(key) + ":";
const childPosition =
position === "map"
? "schema"
: position === "schema" && schemaMaps.has(key)
? "map"
: position === "schema" &&
(schemaArrays.has(key) || schemaChildren.has(key))
? "schema"
: "data";
yield* serialize(child, childPosition, active);
}
yield "}";
} else {
yield JSON.stringify(value);
}
}
const chunks: string[] = [];
let bytes = 0;
for (const chunk of serialize(root, "schema", new Set())) {
bytes += Buffer.byteLength(chunk, "utf8");
if (bytes > budgetBytes || !resolved) return undefined;
chunks.push(chunk);
}
return resolved
? (JSON.parse(chunks.join("")) as Record<string, unknown>)
: undefined;
};
/** Preserve every possible type, including referenced and nullable unions. */
const detectedTypes = (
schema: unknown,
root: Record<string, unknown>,
active: ReadonlySet<string> = new Set(),
): readonly string[] | undefined => {
if (!isSchemaObject(schema)) return undefined;
if (typeof schema.$ref === "string") {
if (active.has(schema.$ref)) return undefined;
return detectedTypes(
resolveLocalReference(root, schema.$ref),
root,
new Set([...active, schema.$ref]),
);
}
if (typeof schema.type === "string") return [schema.type];
if (
Array.isArray(schema.type) &&
schema.type.every((type) => typeof type === "string")
)
return schema.type;
const alternatives = Array.isArray(schema.anyOf)
? schema.anyOf
: Array.isArray(schema.oneOf)
? schema.oneOf
: [];
if (alternatives.length === 0) return undefined;
const types = alternatives.map((branch) =>
detectedTypes(branch, root, active),
);
if (types.some((type) => type === undefined)) return undefined;
return [...new Set(types.flatMap((type) => type ?? []))];
};
const REDUCED_ANNOTATION =
"Advertised in reduced form by the compact input schema profile; the server still validates complete canonical inputs.";
/**
* Present only the root property shape with per-property guidance. Used for
* schemas whose validation structure alone exceeds the compact budget; the
* canonical schema still rejects malformed calls server-side.
*/
const reduceToShallowProperties = (
schema: Record<string, unknown>,
): Record<string, unknown> => {
const properties = isSchemaObject(schema.properties) ? schema.properties : {};
const description = [
typeof schema.description === "string" ? schema.description : undefined,
REDUCED_ANNOTATION,
]
.filter((text) => text !== undefined)
.join(" ");
const $schema =
typeof schema.$schema === "string" ? { $schema: schema.$schema } : {};
const shallow = Object.fromEntries(
Object.entries(properties).map(([name, value]) => {
const types = detectedTypes(value, schema);
const type = types?.length === 1 ? types[0] : types;
const description =
isSchemaObject(value) && typeof value.description === "string"
? { description: value.description }
: {};
return [
name,
{ ...(type === undefined ? {} : { type }), ...description },
];
}),
);
return {
...$schema,
type: "object",
description,
properties: shallow,
...(Array.isArray(schema.required)
? { required: [...schema.required] }
: {}),
...(typeof schema.minProperties === "number"
? { minProperties: schema.minProperties }
: {}),
...(isSchemaObject(schema.dependentRequired)
? { dependentRequired: { ...schema.dependentRequired } }
: {}),
...(schema.additionalProperties === false
? { additionalProperties: false }
: {}),
};
};
/**
* Render an advertised input schema within a per-schema byte budget for
* providers that validate the literal serialized schema size. Validation
* keywords and the canonical contract are preserved; only
* annotation prose and, for structurally oversized schemas, nested
* validation detail leave the advertised form.
*/
export const compactAdvertisedInputSchema = (
schema: Record<string, unknown>,
budgetBytes: number,
): CompactedInputSchema => {
const inline = inlineWithinBudget(stripAnnotations(schema), budgetBytes);
if (inline !== undefined) return { schema: inline, presentation: "inline" };
const reduced = reduceToShallowProperties(schema);
if (serializedBytes(reduced) <= budgetBytes)
return { schema: reduced, presentation: "reduced" };
// A future contract with an extreme root-property count still advertises
// an honest, budget-fitting shape instead of an oversized one.
return {
schema: reduceToShallowProperties({
type: "object",
description:
(typeof schema.description === "string"
? `${schema.description} `
: "") + REDUCED_ANNOTATION,
}),
presentation: "reduced",
};
};
+3 -3
View File
@@ -1,6 +1,6 @@
import { isDeepStrictEqual } from "node:util";
const schemaMaps = new Set([
export const schemaMaps = new Set([
"$defs",
"definitions",
"properties",
@@ -8,8 +8,8 @@ const schemaMaps = new Set([
"dependentSchemas",
"dependencies",
]);
const schemaArrays = new Set(["allOf", "anyOf", "oneOf", "prefixItems"]);
const schemaChildren = new Set([
export const schemaArrays = new Set(["allOf", "anyOf", "oneOf", "prefixItems"]);
export const schemaChildren = new Set([
"items",
"additionalItems",
"contains",
+43
View File
@@ -95,6 +95,46 @@ export const toolOutputSchemaWithMetadata = <Contract extends ToolContract>(
},
);
/**
* Apply a wire-presentation transform to one schema's advertised input
* projection. The wrapper preserves the schema's own parser and standard
* metadata, so both Zod registrations and plain standard schemas keep their
* canonical validation while only the advertised JSON changes.
*/
export const transformAdvertisedInputJsonSchema = <Schema>(
schema: Schema,
transform: (projected: Record<string, unknown>) => Record<string, unknown>,
): Schema => {
const standard = Reflect.get(schema as object, "~standard");
if (!isSchemaRecord(standard)) return schema;
const jsonSchema = Reflect.get(standard, "jsonSchema");
if (!isSchemaRecord(jsonSchema)) return schema;
const project = Reflect.get(jsonSchema, "input");
if (typeof project !== "function") return schema;
const byTarget = new Map<string, Record<string, unknown>>();
const transformedInput = (options: unknown): unknown => {
const target = Reflect.get(options as object, "target");
if (typeof target === "string") {
const cached = byTarget.get(target);
if (cached !== undefined) return cached;
}
const projected = (project as (options: unknown) => unknown)(options);
const transformed = isSchemaRecord(projected)
? transform(projected)
: projected;
if (typeof target === "string" && isSchemaRecord(transformed))
byTarget.set(target, transformed);
return transformed;
};
return {
...(schema as object),
"~standard": {
...standard,
jsonSchema: { ...jsonSchema, input: transformedInput },
},
} as Schema;
};
const hasRelativeSchemaReference = (
schema: Record<string, unknown>,
): boolean => {
@@ -164,6 +204,9 @@ const memoizeByTarget = (
};
};
const isSchemaRecord = (value: unknown): value is Record<string, unknown> =>
typeof value === "object" && value !== null && !Array.isArray(value);
const fallbackPropertyDescription = (property: string): string | undefined => {
const explicit = Object.hasOwn(PROPERTY_DESCRIPTIONS, property)
? PROPERTY_DESCRIPTIONS[property]
+3 -1
View File
@@ -98,6 +98,7 @@ import type { ArtifactInventoryPartialObservation } from "./artifactPartialObser
import type { BrowserScenarioPartialObservation } from "./browserScenarioCapture.js";
import type { Evidence } from "./evidence.js";
import type { AndroidPartialObservation } from "./android/androidPartialObservation.js";
import type { FirmwarePartialObservation } from "./firmware/firmwareAnalysis.js";
/** Provider-neutral evidence collected before a typed analysis failure. */
export type AnalysisPartialObservation =
@@ -109,4 +110,5 @@ export type AnalysisPartialObservation =
| JavaScriptRuntimeObservation
| ArtifactInventoryPartialObservation
| Evidence
| AndroidPartialObservation;
| AndroidPartialObservation
| FirmwarePartialObservation;
+11 -16
View File
@@ -11,6 +11,7 @@ import {
bridgeCandidateCoverageSchema,
projectCartesianCandidates,
} from "../bridgeCandidateProjection.js";
import { nativeRuntimeConvention } from "../nativeConvention.js";
const evidenceIdSchema = prefixedDigestSchema("ev");
const pathSchema = z.string().min(1);
@@ -176,7 +177,6 @@ const classify = (all: readonly Component[]) => ({
});
const runtimeFamilies = (all: readonly Component[]) => {
const paths = all.map(({ path }) => path.toLowerCase());
const families = new Set<
AndroidApplicationProjectionResult["runtime_families"][number]
>();
@@ -187,15 +187,10 @@ const runtimeFamilies = (all: readonly Component[]) => {
if (all.some(({ format }) => format === "elf")) families.add("native");
if (all.some(({ format }) => format === "javascript-bundle"))
families.add("javascript");
if (
paths.some(
(path) => path.includes("reactnative") || path.includes("hermes"),
)
)
families.add("react-native");
if (paths.some((path) => path.includes("libflutter.so")))
families.add("flutter");
if (paths.some((path) => path.includes("libunity.so"))) families.add("unity");
for (const { path } of all) {
const convention = nativeRuntimeConvention(path);
if (convention !== null) families.add(convention);
}
return [...families].sort(compare);
};
@@ -216,12 +211,12 @@ const bridgeCandidates = (
const bridgeBasis = (
path: string,
): AndroidApplicationProjectionResult["bridge_candidates"][number]["basis"] => {
const lower = path.toLowerCase();
if (lower.includes("react") || lower.includes("hermes"))
return "react-native-convention";
if (lower.includes("flutter")) return "flutter-convention";
if (lower.includes("unity")) return "unity-convention";
if (/lib\/[^/]+\/lib[^/]+\.so$/u.test(lower)) return "jni-library-convention";
const convention = nativeRuntimeConvention(path);
if (convention === "react-native") return "react-native-convention";
if (convention === "flutter") return "flutter-convention";
if (convention === "unity") return "unity-convention";
if (/lib\/[^/]+\/lib[^/]+\.so$/u.test(path.toLowerCase()))
return "jni-library-convention";
return "managed-and-native-content";
};
+77
View File
@@ -572,3 +572,80 @@ describe("Apple application bridge candidates", () => {
});
});
});
describe("Apple path conventions", () => {
it("recognizes anatomy names by English case and reports the archive spelling", () => {
const result = project(
inventoryEvidence("directory", "Fixture.app", [
{ path: "contents/info.plist" },
macho("contents/macos/Fixture"),
{ path: "contents/_codesignature/coderesources" },
]),
);
expect(result.bundles[0]).toMatchObject({
layout: "macos-deep",
info_plist_path: "contents/info.plist",
executable_candidates: ["contents/macos/Fixture"],
signing_paths: ["contents/_codesignature/coderesources"],
});
expect(result.components.bundle_metadata.map(({ path }) => path)).toEqual([
"contents/info.plist",
]);
expect(result.components.signing.map(({ path }) => path)).toEqual([
"contents/_codesignature/coderesources",
]);
});
it("limits runtime families and bridge bases to path segments", () => {
const result = project(
inventoryEvidence("ipa", "Fixture.ipa", [
{ path: "Payload/Fixture.app/Info.plist" },
macho("Payload/Fixture.app/Fixture"),
macho("Payload/Fixture.app/Frameworks/libfoo.dylib"),
macho("SwiftSupport/libswiftCore.dylib"),
macho("Payload/Fixture.app/Frameworks/MyApp.framework/MyApp"),
macho("Payload/Fixture.app/community/libcommunity.so"),
{
path: "Payload/Fixture.app/main.js",
format: "javascript-bundle",
},
]),
);
expect(result.runtime_families).toEqual(
expect.arrayContaining(["javascript", "native", "swift-objective-c"]),
);
expect(result.runtime_families).not.toContain("flutter");
expect(result.runtime_families).not.toContain("unity");
expect(result.runtime_families).not.toContain("react-native");
expect(
result.bridge_candidates.every(
({ basis }) => basis === "javascript-and-native-content",
),
).toBe(true);
expect(result.limitations).toEqual(
expect.arrayContaining([
"Runtime families are inferred from inventory formats and paths.",
"A bridge basis is inferred from the native path and is repeated for every managed component.",
]),
);
});
it("recognizes framework segments and does not treat a bare dylib as Swift", () => {
const result = project(
inventoryEvidence("ipa", "Fixture.ipa", [
{ path: "Payload/Fixture.app/Info.plist" },
macho("Payload/Fixture.app/Fixture"),
macho("Payload/Fixture.app/Frameworks/Flutter.framework/Flutter"),
macho(
"Payload/Fixture.app/Frameworks/UnityFramework.framework/UnityFramework",
),
macho("Payload/Fixture.app/Frameworks/hermes.framework/hermes"),
macho("Payload/Fixture.app/Frameworks/libfoo.dylib"),
]),
);
expect(result.runtime_families).toEqual(
expect.arrayContaining(["flutter", "unity", "react-native", "native"]),
);
expect(result.runtime_families).not.toContain("swift-objective-c");
});
});
+49 -38
View File
@@ -4,6 +4,8 @@ import canonicalize from "canonicalize";
import { z } from "zod";
import {
anatomyNameIs,
anatomyTailMatches,
appleBundleSchema,
appleComponentSchema,
applicationRoots,
@@ -14,6 +16,10 @@ import {
platformsOf,
type AppleInventoryEntry,
} from "./appleBundleAnatomy.js";
import {
isSwiftRuntimeLibrary,
nativeRuntimeConvention,
} from "../nativeConvention.js";
import { parseArtifactInventoryEvidence } from "../artifactInventoryEvidence.js";
import { evidenceSchema } from "../evidence.js";
import { digestSchema } from "../digests.js";
@@ -239,19 +245,20 @@ const classifyComponents = (
) => {
const withinApp = (path: string): boolean =>
roots.some((root) => isWithin(path, root));
const anatomy = (pattern: RegExp): Component[] =>
const matching = (predicate: (path: string) => boolean): Component[] =>
entries.flatMap(({ path, kind, component }) =>
kind === "file" &&
component !== undefined &&
!isSidecar(path) &&
withinApp(path) &&
pattern.test(path)
predicate(path)
? [component]
: [],
);
const lastSegment = (path: string): string => path.split("/").at(-1) ?? "";
return {
bundle_metadata: all.filter(({ path }) =>
/(?:^|\/)Info\.plist$/u.test(path),
anatomyNameIs(lastSegment(path), "Info.plist"),
),
executables: all.filter(
({ path, format }) => withinApp(path) && isMachOFormat(format),
@@ -259,23 +266,41 @@ const classifyComponents = (
frameworks: all.filter(({ path }) => /\.framework\//u.test(path)),
native_libraries: all.filter(({ path }) => /\.(?:dylib|so)$/iu.test(path)),
javascript: all.filter(({ format }) => format === "javascript-bundle"),
signing: all.filter(({ path }) =>
/(?:^|\/)(?:embedded\.(?:mobileprovision|provisionprofile)|_CodeSignature\/CodeResources)$/u.test(
path,
),
signing: all.filter(
({ path }) =>
anatomyTailMatches(path, ["_CodeSignature", "CodeResources"]) ||
anatomyNameIs(lastSegment(path), "embedded.mobileprovision") ||
anatomyNameIs(lastSegment(path), "embedded.provisionprofile"),
),
privileged_helpers: anatomy(
/(?:^|\/)Contents\/Library\/LaunchServices\/[^/]+$/u,
privileged_helpers: matching((path) =>
anatomyTailMatches(path, ["Contents", "Library", "LaunchServices", "*"]),
),
launchd_plists: anatomy(
/(?:^|\/)Contents\/Library\/Launch(?:Agents|Daemons)\/[^/]+\.plist$/u,
launchd_plists: matching(
(path) =>
path.endsWith(".plist") &&
(anatomyTailMatches(path, [
"Contents",
"Library",
"LaunchAgents",
"*",
]) ||
anatomyTailMatches(path, [
"Contents",
"Library",
"LaunchDaemons",
"*",
])),
).map((component) => ({
...component,
domain: component.path.includes("/LaunchDaemons/")
domain: component.path
.split("/")
.some((segment) => anatomyNameIs(segment, "LaunchDaemons"))
? ("daemon" as const)
: ("agent" as const),
})),
helpers: anatomy(/(?:^|\/)Contents\/Helpers\/[^/]+$/u),
helpers: matching((path) =>
anatomyTailMatches(path, ["Contents", "Helpers", "*"]),
),
};
};
@@ -285,7 +310,6 @@ const deduplicateComponents = (values: readonly Component[]): Component[] =>
);
const identifyRuntimeFamilies = (all: readonly Component[]) => {
const paths = all.map(({ path }) => path.toLowerCase());
const families = new Set<
z.infer<
typeof appleApplicationProjectionResultSchema
@@ -293,28 +317,14 @@ const identifyRuntimeFamilies = (all: readonly Component[]) => {
>();
if (all.some(({ format }) => ["mach-o", "mach-o-universal"].includes(format)))
families.add("native");
if (paths.some((path) => /\.(?:dylib|framework\/[^/]+)$/u.test(path)))
if (all.some(({ path }) => isSwiftRuntimeLibrary(path)))
families.add("swift-objective-c");
if (all.some(({ format }) => format === "javascript-bundle"))
families.add("javascript");
if (
paths.some(
(path) =>
path.includes("reactnative") ||
path.includes("react.framework") ||
path.includes("hermes"),
)
)
families.add("react-native");
if (
paths.some(
(path) =>
path.includes("flutter.framework") || path.includes("app.framework"),
)
)
families.add("flutter");
if (paths.some((path) => path.includes("unityframework.framework")))
families.add("unity");
for (const { path } of all) {
const convention = nativeRuntimeConvention(path);
if (convention !== null) families.add(convention);
}
return [...families].sort(compare);
};
@@ -361,11 +371,10 @@ const bridgeBasis = (
| "react-native-convention"
| "flutter-convention"
| "unity-convention" => {
const lower = path.toLowerCase();
if (lower.includes("react")) return "react-native-convention";
if (lower.includes("flutter") || lower.includes("app.framework"))
return "flutter-convention";
if (lower.includes("unity")) return "unity-convention";
const convention = nativeRuntimeConvention(path);
if (convention === "react-native") return "react-native-convention";
if (convention === "flutter") return "flutter-convention";
if (convention === "unity") return "unity-convention";
return "javascript-and-native-content";
};
@@ -422,7 +431,9 @@ const projectionLimitations = (facts: {
: []),
"Bundle roles follow path conventions. Read each info_plist_path with inspect_plist for CFBundleExecutable, identifiers, and declared services.",
"Bundle identifiers and signing claims require dedicated plist and CMS parsing; this projection reports only exact paths and hashes.",
"Runtime families are inferred from inventory formats and paths.",
"Bridge candidates are path-based hypotheses, not observed runtime calls.",
"A bridge basis is inferred from the native path and is repeated for every managed component.",
];
const compare = (left: string, right: string): number =>
+111 -37
View File
@@ -67,6 +67,50 @@ export const isSidecar = (path: string): boolean => {
export const isWithin = (path: string, root: string): boolean =>
root === "." || path === root || path.startsWith(`${root}/`);
/**
* English (en-US) case fold used only for Apple's fixed anatomy vocabulary.
* Reported paths keep the archive spelling. Inventory identity stays case-sensitive.
*/
export const anatomyNameIs = (actual: string, expected: string): boolean =>
actual.toLocaleLowerCase("en-US") === expected.toLocaleLowerCase("en-US");
/** Directory entry that folds to one vocabulary word. The exact spelling wins. */
export const anatomyChild = (
names: ReadonlySet<string> | undefined,
expected: string,
): string | null => {
if (names === undefined) return null;
let exact: string | undefined;
let folded: string | undefined;
for (const name of names) {
if (name === expected) exact = name;
else if (
anatomyNameIs(name, expected) &&
(folded === undefined || compare(name, folded) < 0)
)
folded = name;
}
return exact ?? folded ?? null;
};
/**
* Whether `path`'s tail matches anatomy vocabulary. `*` matches one segment
* that is not itself a vocabulary comparison.
*/
export const anatomyTailMatches = (
path: string,
expected: readonly string[],
): boolean => {
const actual = path.split("/");
if (actual.length < expected.length) return false;
const tail = actual.slice(-expected.length);
return expected.every((name, index) => {
const segment = tail[index];
if (segment === undefined || segment.length === 0) return false;
return name === "*" || anatomyNameIs(segment, name);
});
};
/**
* Outermost iOS (`Payload/X.app`) and macOS (`X.app/Contents/…`) applications.
* An inventoried `.app` directory is its own root, `.`.
@@ -82,15 +126,20 @@ export const applicationRoots = (
const found = new Set<string>();
for (const { path } of entries) {
if (isSidecar(path)) continue;
const ios = /^(Payload\/[^/]+\.app)(?:\/|$)/u.exec(path);
if (ios?.[1] !== undefined) {
found.add(ios[1]);
const segments = path.split("/");
const app = segments[1];
if (
anatomyNameIs(segments[0] ?? "", "Payload") &&
app !== undefined &&
/\.app$/iu.test(app)
) {
found.add(`${segments[0]}/${app}`);
continue;
}
const segments = path.split("/");
const index = segments.findIndex(
(segment, position) =>
/\.app$/iu.test(segment) && segments[position + 1] === "Contents",
/\.app$/iu.test(segment) &&
anatomyNameIs(segments[position + 1] ?? "", "Contents"),
);
if (index >= 0) found.add(segments.slice(0, index + 1).join("/"));
}
@@ -186,8 +235,8 @@ const enclosingBundle = (
const bundleLayout = (bundle: string, tree: Tree): Bundle["layout"] => {
const names = tree.children.get(directoryOf(bundle));
if (names?.has("Contents") === true) return "macos-deep";
if (names?.has("Versions") === true) return "versioned-framework";
if (anatomyChild(names, "Contents") !== null) return "macos-deep";
if (anatomyChild(names, "Versions") !== null) return "versioned-framework";
return "shallow";
};
@@ -201,13 +250,18 @@ const bundleRole = (bundle: string): Bundle["role"] => {
if (name.endsWith(".systemextension")) return "system-extension";
if (name.endsWith(".dext")) return "driver-extension";
if (name.endsWith(".bundle")) {
if (container === "PlugIns") return "plug-in";
return container === "Resources" ? "resource-bundle" : "bundle";
if (anatomyNameIs(container ?? "", "PlugIns")) return "plug-in";
return anatomyNameIs(container ?? "", "Resources")
? "resource-bundle"
: "bundle";
}
if (!name.endsWith(".app")) return "plug-in";
if (container === "LoginItems" && segments.at(-3) === "Library")
if (
anatomyNameIs(container ?? "", "LoginItems") &&
anatomyNameIs(segments.at(-3) ?? "", "Library")
)
return "login-item";
if (container === "Helpers") return "helper-application";
if (anatomyNameIs(container ?? "", "Helpers")) return "helper-application";
return "nested-application";
};
@@ -225,18 +279,32 @@ const contentDirectories = (
tree: Tree,
): string[] => {
const directory = directoryOf(bundle);
if (layout === "macos-deep") return [joinPath(directory, "Contents")];
if (layout === "macos-deep") {
const contents = namedChild(tree, directory, "Contents");
return contents === null ? [] : [contents];
}
if (layout === "shallow") return [directory];
const versions = joinPath(directory, "Versions");
const versions = namedChild(tree, directory, "Versions");
if (versions === null) return [];
// Only real version directories count: Current is a symlink, and files
// such as .DS_Store are not versions.
return [...(tree.children.get(versions) ?? [])]
.filter((version) => version !== "Current")
.filter((version) => !anatomyNameIs(version, "Current"))
.map((version) => joinPath(versions, version))
.filter((version) => tree.directories.has(version))
.sort(compare);
};
/** Child path using the archive's spelling of one anatomy name. */
const namedChild = (
tree: Tree,
directory: string,
expected: string,
): string | null => {
const name = anatomyChild(tree.children.get(directory), expected);
return name === null ? null : joinPath(directory, name);
};
const infoPlistPath = (
bundle: string,
layout: Bundle["layout"],
@@ -246,17 +314,15 @@ const infoPlistPath = (
// With several real versions, Versions/Current (a symlink outside the
// inventory) decides which plist applies, so the plist stays unknown.
if (directories.length !== 1) return null;
const candidates = directories
.map((directory) =>
joinPath(
directory,
layout === "versioned-framework"
? "Resources/Info.plist"
: "Info.plist",
),
)
.filter((path) => tree.files.has(path));
return candidates.length === 1 ? (candidates[0] ?? null) : null;
const directory = directories[0];
if (directory === undefined) return null;
const parent =
layout === "versioned-framework"
? namedChild(tree, directory, "Resources")
: directory;
if (parent === null) return null;
const info = namedChild(tree, parent, "Info.plist");
return info !== null && tree.files.has(info) ? info : null;
};
const executableCandidates = (
@@ -265,12 +331,15 @@ const executableCandidates = (
tree: Tree,
): string[] =>
contentDirectories(bundle, layout, tree)
.flatMap((directory) =>
directFiles(
layout === "macos-deep" ? joinPath(directory, "MacOS") : directory,
tree,
),
)
.flatMap((directory) => {
const executableDirectory =
layout === "macos-deep"
? namedChild(tree, directory, "MacOS")
: directory;
return executableDirectory === null
? []
: directFiles(executableDirectory, tree);
})
.filter((path) =>
MACH_O_FORMATS.includes(tree.files.get(path)?.format ?? ""),
);
@@ -281,12 +350,17 @@ const signingPaths = (
tree: Tree,
): string[] =>
contentDirectories(bundle, layout, tree)
.flatMap((directory) => [
joinPath(directory, "_CodeSignature/CodeResources"),
joinPath(directory, "embedded.provisionprofile"),
joinPath(directory, "embedded.mobileprovision"),
])
.filter((path) => tree.files.has(path))
.flatMap((directory) => {
const signature = namedChild(tree, directory, "_CodeSignature");
return [
signature === null
? null
: namedChild(tree, signature, "CodeResources"),
namedChild(tree, directory, "embedded.provisionprofile"),
namedChild(tree, directory, "embedded.mobileprovision"),
];
})
.filter((path): path is string => path !== null && tree.files.has(path))
.sort(compare);
/** Platforms of application roots whose layout was observed, not assumed. */
+39
View File
@@ -13,6 +13,7 @@ import {
} from "./artifactInventoryEvidence.js";
import { prefixedDigestSchema } from "./../domain/digests.js";
import { canonicalJson } from "./comparisonSemantics.js";
import { ZIP_NON_ENTRY_TAIL_LIMITATION } from "./zipPackageFormat.js";
const evidenceIdSchema = prefixedDigestSchema("ev");
const comparisonStatusSchema = z.enum([
@@ -122,6 +123,11 @@ export const compareArtifacts = (
],
leftCovered && rightCovered,
);
const rootChange = rootHashChange(left.inventory, right.inventory, changes, [
...left.evidence.map(({ evidence_id: id }) => id),
...right.evidence.map(({ evidence_id: id }) => id),
]);
if (rootChange !== null) changes.push(rootChange);
const summary = summarize(
left.inventory,
right.inventory,
@@ -431,5 +437,38 @@ const summarize = (
const hasCoverageLimitation = (limitations: readonly string[]): boolean =>
limitations.some(
(limitation) =>
limitation !== ZIP_NON_ENTRY_TAIL_LIMITATION &&
!/integrity contradiction\(s\) were recorded/u.test(limitation),
);
const rootHashChange = (
left: InventorySet,
right: InventorySet,
changes: readonly ArtifactChange[],
evidenceLinks: readonly string[],
): ArtifactChange | null => {
if (left.manifest.root_sha256 === right.manifest.root_sha256) return null;
if (
changes.some(
(change) =>
change.logical_path === "." && change.dimensions.includes("content"),
)
)
return null;
const leftRoot = left.occurrences.find(
(occurrence) => occurrence.logical_path === ".",
);
const rightRoot = right.occurrences.find(
(occurrence) => occurrence.logical_path === ".",
);
return {
classification: "changed",
logical_path: ".",
dimensions: ["content"],
left_occurrence_id: leftRoot?.occurrence_id ?? null,
right_occurrence_id: rightRoot?.occurrence_id ?? null,
left_artifact_id: leftRoot?.artifact_id ?? null,
right_artifact_id: rightRoot?.artifact_id ?? null,
evidence_links: [...evidenceLinks],
};
};
+25
View File
@@ -1,3 +1,5 @@
import { posix } from "node:path";
import { digestCanonicalValue } from "./canonicalDigest.js";
import type {
ArtifactEdge,
@@ -6,6 +8,29 @@ import type {
IntegrityContradiction,
} from "./artifactGraph.js";
/** NFC spelling of a logical archive path. Inventory and path joins share it. */
export const logicalPathNfc = (value: string): string => value.normalize("NFC");
/**
* Lexical join against an inventoried logical path. The result is NFC.
* `"."` is the archive root. A path that escapes the root, or that contains
* NUL, is left in a form the caller can report; `logicalPathEscapesRoot`
* is then true.
*/
export const normalizeJoinedLogicalPath = (value: string): string => {
if (value.includes("\0")) return value;
return logicalPathNfc(
posix.normalize(value.replaceAll("\\", "/")).replace(/^\.\//u, ""),
);
};
/** Whether a joined logical path leaves the archive root. */
export const logicalPathEscapesRoot = (path: string): boolean =>
path.includes("\0") ||
path === ".." ||
path.startsWith("../") ||
path.startsWith("/");
/** Content-address an artifact independently of where it was observed. */
export const artifactIdForContent = (sha256: string): string =>
`art_${digestCanonicalValue({ sha256 }, "Artifact")}`;
+1 -1
View File
@@ -6,7 +6,7 @@
* Policy divergence (deliberate, not yet unified):
* - Local artifact paths resolve against an inventoried container root and
* report rejected | external | not-found. Confinement is lexical
* (posix.normalize must stay inside the root).
* (normalizeJoinedLogicalPath must stay inside the root).
* - Live DOM destinations resolve against the document base URL and report
* approved | outside_policy | unsupported against allowedOrigins
* (see CdpCaptureDocuments.domDestination).
+7 -1
View File
@@ -1,7 +1,7 @@
import { z } from "zod";
import { isAbsolute } from "node:path";
import { digestSchema } from "../digests.js";
import { jsonValueSchema } from "../jsonValue.js";
import { jsonValueSchema, type JsonValue } from "../jsonValue.js";
import { localPathStringSchema } from "../localPath.js";
const bytes = z.number().int().min(0).max(Number.MAX_SAFE_INTEGER);
@@ -95,6 +95,12 @@ const chunk = z.strictObject({
root_file_range: range.nullable(),
});
/** Completed firmware result retained when private-runtime cleanup cannot be confirmed. */
export interface FirmwarePartialObservation {
readonly kind: "firmware";
readonly result: JsonValue;
}
/** Portable results retain reported boundaries, unknowns and extraction lineage. */
export const firmwareResultSchemas = {
inspect_firmware_regions: z.strictObject({
@@ -11,11 +11,13 @@ import {
createJavaScriptApplicationGraph,
createImmutableJavaScriptApplicationGraphSteps,
isValidatedImmutableJavaScriptApplicationGraph,
sealTransferredJavaScriptApplicationGraphSteps,
} from "./javascriptApplicationGraph.js";
import {
createJavaScriptSemanticGraph,
createImmutableJavaScriptSemanticGraphSteps,
isValidatedImmutableJavaScriptSemanticGraph,
sealTransferredJavaScriptSemanticGraphSteps,
} from "./javascriptSemanticGraph.js";
const complete = <Value>(steps: Iterator<void, Value>): Value => {
@@ -265,3 +267,60 @@ describe("owned JavaScript result bindings", () => {
);
});
});
describe("graphs transferred to their sealing steps", () => {
it("seal the same validated graphs as the copying factories", () => {
const input = example();
const { graph_id: _applicationId, ...application } = input.graph;
const { graph_id: _semanticId, ...semantic } = input.semantic_graph;
const expectedApplication = createJavaScriptApplicationGraph(application);
const expectedSemantic = createJavaScriptSemanticGraph(semantic);
const sealedApplication = complete(
sealTransferredJavaScriptApplicationGraphSteps(
structuredClone(application),
),
);
const sealedSemantic = complete(
sealTransferredJavaScriptSemanticGraphSteps(structuredClone(semantic)),
);
expect(sealedApplication).toEqual(expectedApplication);
expect(sealedSemantic).toEqual(expectedSemantic);
expect(
isValidatedImmutableJavaScriptApplicationGraph(sealedApplication),
).toBe(true);
expect(isValidatedImmutableJavaScriptSemanticGraph(sealedSemantic)).toBe(
true,
);
});
it("report the issues a whole-graph parse reports", () => {
const input = example();
const { graph_id: _applicationId, ...application } = input.graph;
const { graph_id: _semanticId, ...semantic } = input.semantic_graph;
const node = semantic.nodes[0];
const applicationNode = application.nodes[0];
if (node === undefined || applicationNode === undefined)
throw new Error("Expected semantic and application nodes");
const malformed = {
...semantic,
nodes: [{ ...node, label: 7 }, ...semantic.nodes.slice(1)],
};
expect(
issuesFrom(() =>
complete(sealTransferredJavaScriptSemanticGraphSteps(malformed)),
),
).toEqual(issuesFrom(() => createJavaScriptSemanticGraph(malformed)));
const stale = {
...application,
nodes: [
{ ...applicationNode, node_id: `jag_node_${"f".repeat(64)}` },
...application.nodes.slice(1),
],
};
expect(
issuesFrom(() =>
complete(sealTransferredJavaScriptApplicationGraphSteps(stale)),
),
).toEqual(issuesFrom(() => createJavaScriptApplicationGraph(stale)));
});
});
@@ -1,4 +1,9 @@
import { z } from "zod";
import {
arrayContainerSchema,
parseElementSteps,
} from "../zodCollectionSteps.js";
import { canonicalJsonDigestSteps } from "../canonicalJsonDigestSteps.js";
import { canonicalJson } from "../comparisonSemantics.js";
import { digestCanonicalValue } from "../canonicalDigest.js";
import { uniqueSorted } from "../canonicalOrdering.js";
@@ -93,11 +98,14 @@ type EdgeSemantic = Omit<ApplicationEdge, "edge_id">;
const edgeId = (edge: EdgeSemantic): string =>
`jag_edge_${digestCanonicalValue(edge, "JavaScript Application Graph")}`;
/** The issue sink shared by schema refinement and stepwise checking. */
type GraphIssueContext = Pick<z.RefinementCtx, "addIssue">;
const sortedUniqueIssue = (
values: readonly string[],
path: PropertyKey[],
label: string,
context: z.RefinementCtx,
context: GraphIssueContext,
): void => {
for (let index = 1; index < values.length; index += 1) {
if (
@@ -116,7 +124,7 @@ const sortedUniqueIssue = (
const checkEvidenceNormalization = (
evidence: ApplicationGraphEvidence,
path: PropertyKey[],
context: z.RefinementCtx,
context: GraphIssueContext,
): void => {
sortedUniqueIssue(
evidence.limitations,
@@ -226,7 +234,7 @@ const observationMatchesIdentity = (
const checkNode = (
node: ApplicationNode,
index: number,
context: z.RefinementCtx,
context: GraphIssueContext,
): void => {
if (node.node_id !== nodeId(node))
context.addIssue({
@@ -306,8 +314,16 @@ type GraphRecord = z.infer<typeof javascriptApplicationGraphRecordSchema>;
const checkGraphContent = (
graph: JavaScriptApplicationGraphInput,
context: z.RefinementCtx,
context: GraphIssueContext,
): void => {
const steps = checkGraphContentSteps(graph, context);
while (steps.next().done !== true);
};
function* checkGraphContentSteps(
graph: JavaScriptApplicationGraphInput,
context: GraphIssueContext,
): Generator<void, void> {
sortedUniqueIssue(
graph.root_node_ids,
["root_node_ids"],
@@ -353,9 +369,12 @@ const checkGraphContent = (
message: "Root identifier must name a graph node",
path: ["root_node_ids"],
});
for (const [index, node] of graph.nodes.entries())
for (const [index, node] of graph.nodes.entries()) {
if (index % 64 === 0) yield;
checkNode(node, index, context);
}
for (const [index, edge] of graph.edges.entries()) {
if (index % 64 === 0) yield;
const { edge_id: identifier, ...semantic } = edge;
if (identifier !== edgeId(semantic))
context.addIssue({
@@ -394,11 +413,11 @@ const checkGraphContent = (
context,
);
}
};
}
const checkGraphInvariants = (
graph: GraphRecord,
context: z.RefinementCtx,
context: GraphIssueContext,
): void => {
checkGraphContent(graph, context);
const { graph_id: identifier, ...semantic } = graph;
@@ -475,22 +494,9 @@ export const createJavaScriptApplicationEdge = (
export const createJavaScriptApplicationGraph = (
input: unknown,
): JavaScriptApplicationGraph => {
const parsed = javascriptApplicationGraphInputSchema.parse(input);
const semantic: JavaScriptApplicationGraphInput = {
...parsed,
root_node_ids: uniqueSorted(parsed.root_node_ids),
nodes: [...parsed.nodes].sort((left, right) =>
compareUnicodeCodePoints(left.node_id, right.node_id),
),
edges: [...parsed.edges].sort((left, right) =>
compareUnicodeCodePoints(left.edge_id, right.edge_id),
),
coverage: {
...parsed.coverage,
limits: normalizeLimits(parsed.coverage.limits),
},
limitations: uniqueSorted(parsed.limitations),
};
const semantic = normalizeGraphInput(
javascriptApplicationGraphInputSchema.parse(input),
);
const content = javascriptApplicationGraphContentSchema.parse(semantic);
return {
...content,
@@ -498,6 +504,24 @@ export const createJavaScriptApplicationGraph = (
};
};
const normalizeGraphInput = (
parsed: JavaScriptApplicationGraphInput,
): JavaScriptApplicationGraphInput => ({
...parsed,
root_node_ids: uniqueSorted(parsed.root_node_ids),
nodes: [...parsed.nodes].sort((left, right) =>
compareUnicodeCodePoints(left.node_id, right.node_id),
),
edges: [...parsed.edges].sort((left, right) =>
compareUnicodeCodePoints(left.edge_id, right.edge_id),
),
coverage: {
...parsed.coverage,
limits: normalizeLimits(parsed.coverage.limits),
},
limitations: uniqueSorted(parsed.limitations),
});
const validatedImmutableApplicationGraphs = new WeakSet<object>();
/** Clone and validate synchronously, then return steps to seal the owned graph. */
@@ -506,6 +530,65 @@ export const createImmutableJavaScriptApplicationGraphSteps = (
): Generator<void, JavaScriptApplicationGraph> =>
sealValidatedApplicationGraphSteps(createJavaScriptApplicationGraph(input));
const applicationGraphShape = javascriptApplicationGraphInputSchema.shape;
const applicationGraphContainerSchema =
javascriptApplicationGraphInputSchema.extend({
nodes: arrayContainerSchema(applicationGraphShape.nodes),
edges: arrayContainerSchema(applicationGraphShape.edges),
});
/**
* Seal a graph whose input the caller transfers: the caller neither retains nor
* mutates any part of it afterwards. Only the graph's own fields are validated
* synchronously; nodes and edges are validated, checked, hashed and frozen in
* steps, because doing so at once held the event loop for large applications.
*/
export const sealTransferredJavaScriptApplicationGraphSteps = (
input: unknown,
): Generator<void, JavaScriptApplicationGraph> =>
parseAndSealApplicationGraphSteps(
applicationGraphContainerSchema.parse(input),
);
function* parseAndSealApplicationGraphSteps(
container: z.output<typeof applicationGraphContainerSchema>,
): Generator<void, JavaScriptApplicationGraph> {
const issues: z.core.$ZodIssue[] = [];
const parsed: JavaScriptApplicationGraphInput = {
...container,
nodes: yield* parseElementSteps(
applicationGraphShape.nodes.element,
container.nodes,
["nodes"],
issues,
),
edges: yield* parseElementSteps(
applicationGraphShape.edges.element,
container.edges,
["edges"],
issues,
),
};
if (issues.length > 0) throw new z.ZodError(issues);
const content = normalizeGraphInput(parsed);
let invalid = false;
yield* checkGraphContentSteps(content, {
addIssue: () => {
invalid = true;
},
});
// An invalid owned graph is a construction defect; report it with the exact
// issues the content schema produces.
if (invalid) {
javascriptApplicationGraphContentSchema.parse(content);
throw new TypeError("Application graph checks disagree with its schema");
}
return yield* sealValidatedApplicationGraphSteps({
...content,
graph_id: `jag_${yield* canonicalJsonDigestSteps(content)}`,
});
}
function* sealValidatedApplicationGraphSteps(
graph: JavaScriptApplicationGraph,
): Generator<void, JavaScriptApplicationGraph> {
@@ -2,6 +2,10 @@ import { posix, relative, win32 } from "node:path";
import type { ApplicationNode } from "./javascriptApplicationGraphSchemas.js";
import type { JavaScriptApplicationGraph } from "./javascriptApplicationGraph.js";
import {
logicalPathEscapesRoot,
normalizeJoinedLogicalPath,
} from "../artifactIdentity.js";
import { uniqueSorted } from "../canonicalOrdering.js";
import { compareUnicodeCodePoints } from "../unicodeCodePointOrder.js";
import type { ParsedStaticLayer } from "./javascriptRuntimeReconciliationParsing.js";
@@ -174,7 +178,7 @@ const nodePaths = (node: ApplicationNode): string[] => {
paths.push(observation.evidence.location.value.path);
}
return paths
.map(normalizeArtifactPath)
.map(relativeLogicalPath)
.filter(
(path): path is string => path !== null && path !== "artifact-root",
);
@@ -221,7 +225,7 @@ const pathBelowRoot = (root: string, value: string): string | null => {
portableRemainder.startsWith("../")
)
return null;
return normalizeArtifactPath(portableRemainder);
return relativeLogicalPath(portableRemainder);
};
const absoluteFilePathStyle = (path: string): "posix" | "windows" | null => {
@@ -250,7 +254,7 @@ const pathBelowUrlPrefix = (prefix: string, value: string): string | null => {
)
return null;
try {
return normalizeArtifactPath(
return relativeLogicalPath(
decodeURIComponent(candidate.pathname.slice(base.pathname.length)),
);
} catch (cause: unknown) {
@@ -292,17 +296,13 @@ const pathWithinPrefix = (path: string, prefix: string): boolean =>
prefix === "" || path === prefix || path.startsWith(`${prefix}/`);
const safeArtifactPath = (prefix: string, suffix: string): string | null =>
normalizeArtifactPath(prefix === "" ? suffix : posix.join(prefix, suffix));
relativeLogicalPath(prefix === "" ? suffix : posix.join(prefix, suffix));
const normalizeArtifactPath = (value: string): string | null => {
const normalized = posix
.normalize(value.replaceAll("\\", "/"))
.replace(/^\.\//u, "");
const relativeLogicalPath = (value: string): string | null => {
const normalized = normalizeJoinedLogicalPath(value);
return normalized !== "" &&
normalized !== "." &&
normalized !== ".." &&
!normalized.startsWith("/") &&
!normalized.startsWith("../")
!logicalPathEscapesRoot(normalized)
? normalized
: null;
};
@@ -597,11 +597,9 @@ describe("property mutation collection on minified bundles", () => {
{ length: 2000 },
(_, index) => `source.p${index} = ${index};`,
).join(" ");
const start = performance.now();
const value = resultValue(
`const source = { token: "TOKEN" }; ${writes} return [source.token, source.p1999];`,
);
expect(performance.now() - start).toBeLessThan(2000);
expect(value).toMatchObject({
status: "array",
items: [
@@ -662,3 +660,39 @@ describe("results of methods on an escaped receiver", () => {
});
}, 30000);
});
describe("results of methods on an escaped receiver", () => {
it("makes every method's distinct result uncertain, not only the called one", () => {
expect(
resultValue(
'const first = { t: "A" }; const second = { t: "B" }; const kept = { t: "C" }; const box = { a() { return first; }, b() { return second; }, c() { return first; } }; box.a(); return [first.t, second.t, kept.t];',
),
).toMatchObject({
status: "array",
items: [
{ value: { status: "unknown" } },
{ value: { status: "unknown" } },
{ value: { status: "literal", value: "C" } },
],
});
});
it("keeps a shared result uncertain across a wide escaped receiver (#1495)", () => {
const count = 2000;
const methods = Array.from(
{ length: count },
(_, index) => `m${index}() { return shared; },`,
).join(" ");
const calls = Array.from(
{ length: count },
(_, index) => `box.m${index}().p${index} = ${index};`,
).join(" ");
const value = resultValue(
`const shared = { token: "TOKEN" }; const box = { ${methods} }; ${calls} return [shared.token];`,
);
expect(value).toMatchObject({
status: "array",
items: [{ value: { status: "unknown" } }],
});
}, 30000);
});
@@ -38,7 +38,7 @@ import {
collectSemanticReturns,
resolveSemanticModuleCallables,
} from "./javascriptSemanticReturns.js";
import { collectJavaScriptDerivedSemantics } from "./javascriptSemanticDerivedAnalysis.js";
import { collectJavaScriptDerivedSemanticsSteps } from "./javascriptSemanticDerivedAnalysis.js";
import { range } from "./javascriptStaticAnalysisHelpers.js";
import { propertyName } from "./javascriptAstValues.js";
import { semanticCoverage } from "./javascriptSemanticCoverage.js";
@@ -107,18 +107,40 @@ export const analyzeParsedJavaScriptReferences = (
export const analyzeParsedJavaScriptSemantics = (
file: ParsedJavaScriptSource,
): JavaScriptSemanticIr => {
const steps = analyzeParsedJavaScriptSemanticsSteps(file);
for (;;) {
const step = steps.next();
if (step.done === true) return step.value;
}
};
/**
* Recover semantics from a parsed artifact with a yield between analysis
* phases, so a caller can serve control messages within one large file.
* Each phase completes before the next starts; results equal the synchronous
* form.
*/
export function* analyzeParsedJavaScriptSemanticsSteps(
file: ParsedJavaScriptSource,
): Generator<void, JavaScriptSemanticIr> {
const state = createState(file.program);
collectDefinitions(file.program, state);
yield;
collectSemanticMemberMutations(file.program, state);
yield;
traverseJavaScriptAst(file.program, {
enter: (node) => collectSemanticModuleLink(node, state),
});
yield;
const references = collectSemanticReferences(file.program, state);
yield;
const parserPartial = file.errors.length > 0;
const bindings = immutableSemanticBindings(state);
yield;
const callables = collectSemanticReturns(file.program, state, parserPartial);
yield;
const moduleLinks = resolveSemanticModuleCallables(state, callables);
const derived = collectJavaScriptDerivedSemantics(
const derived = yield* collectJavaScriptDerivedSemanticsSteps(
file.program,
state,
callables,
@@ -154,7 +176,7 @@ export const analyzeParsedJavaScriptSemantics = (
"Cross-function mutation and dynamic property resolution remain unknown.",
],
};
};
}
/** Receiver identity facts for the overloaded `.open` syntax only. */
export type JavaScriptOpenReceiverFact =
@@ -0,0 +1,34 @@
import { expect, it } from "vitest";
import {
analyzeParsedJavaScriptSemantics,
analyzeParsedJavaScriptSemanticsSteps,
} from "./javascriptSemanticAnalysis.js";
import { parseJavaScriptSource } from "./javascriptSourceParser.js";
const source = `
import { readFile } from "node:fs";
const shared = { token: "TOKEN" };
export function load(path) {
return new Promise((resolve) => setTimeout(() => resolve(readFile(path)), 1));
}
export const box = { get() { return shared; } };
box.get().value = process.env.MODE ?? "default";
`;
it("yields between semantic phases and returns the synchronous result (#1462)", () => {
const parsed = parseJavaScriptSource(source, "main.js");
if (parsed === null) throw new Error("Expected parsed source");
const steps = analyzeParsedJavaScriptSemanticsSteps(parsed);
let yields = 0;
let step = steps.next();
while (step.done !== true) {
yields += 1;
step = steps.next();
}
// Every top-level phase and derived collector is a separate step.
expect(yields).toBeGreaterThanOrEqual(13);
const reparsed = parseJavaScriptSource(source, "main.js");
if (reparsed === null) throw new Error("Expected parsed source");
expect(step.value).toEqual(analyzeParsedJavaScriptSemantics(reparsed));
});
@@ -36,43 +36,54 @@ type DerivedSemanticAnalysis = Pick<
>;
/** Compose bounded semantic passes after lexical definitions are available. */
export const collectJavaScriptDerivedSemantics = (
/**
* Collect derived semantics with a yield between collectors. Each collector
* completes before the next starts, so results equal the synchronous form.
*/
export function* collectJavaScriptDerivedSemanticsSteps(
program: t.Program,
state: JavaScriptSemanticAnalysisState,
callables: readonly JavaScriptSemanticCallable[],
parserPartial: boolean,
): DerivedSemanticAnalysis => {
): Generator<void, DerivedSemanticAnalysis> {
const calls = collectJavaScriptSemanticCalls(program, state, callables);
yield;
const promiseOperations = collectJavaScriptSemanticPromises(
program,
state,
callables,
);
yield;
const asyncEffects = collectJavaScriptSemanticAsyncEffects(
program,
state,
callables,
);
yield;
const childProcesses = collectJavaScriptSemanticChildProcesses(
program,
state,
callables,
);
yield;
const dataEffects = collectJavaScriptSemanticDataEffects(
program,
state,
callables,
);
yield;
const resourceOperations = collectJavaScriptSemanticResources(
program,
state,
callables,
);
yield;
const objectOperations = collectJavaScriptSemanticObjects(
program,
state,
callables,
);
yield;
const functionFingerprints = collectJavaScriptSemanticFingerprints({
state,
callables,
@@ -96,4 +107,4 @@ export const collectJavaScriptDerivedSemantics = (
objectOperations,
functionFingerprints,
};
};
}
@@ -6,6 +6,10 @@ import { uniqueSorted } from "../canonicalOrdering.js";
import { compareUnicodeCodePoints } from "../unicodeCodePointOrder.js";
import { canonicalJsonDigestSteps } from "../canonicalJsonDigestSteps.js";
import { freezeOwnedJsonSnapshotSteps } from "../immutableJson.js";
import {
arrayContainerSchema,
parseElementSteps,
} from "../zodCollectionSteps.js";
import type {
ApplicationGraphEvidence,
ApplicationGraphEvidenceContext,
@@ -765,6 +769,15 @@ export const createJavaScriptSemanticGraph = (
const validatedImmutableSemanticGraphs = new WeakSet<object>();
const semanticGraphShape = javaScriptSemanticGraphInputSchema.shape;
const semanticGraphContainerSchema = javaScriptSemanticGraphInputSchema.extend({
evidence_contexts: arrayContainerSchema(semanticGraphShape.evidence_contexts),
nodes: arrayContainerSchema(semanticGraphShape.nodes),
relations: arrayContainerSchema(semanticGraphShape.relations),
fingerprints: arrayContainerSchema(semanticGraphShape.fingerprints),
unknowns: arrayContainerSchema(semanticGraphShape.unknowns),
});
/** Clone input synchronously; commit, check and seal the owned graph in steps. */
export const createImmutableJavaScriptSemanticGraphSteps = (
input: unknown,
@@ -773,6 +786,58 @@ export const createImmutableJavaScriptSemanticGraphSteps = (
normalizeGraphInput(javaScriptSemanticGraphInputSchema.parse(input)),
);
/**
* Seal a graph whose input the caller transfers: the caller neither retains nor
* mutates any part of it afterwards. Only the graph's own fields are validated
* synchronously; collection elements are validated and cloned in steps, because
* a whole-graph parse of a large application held the event loop for seconds.
*/
export const sealTransferredJavaScriptSemanticGraphSteps = (
input: unknown,
): Generator<void, JavaScriptSemanticGraph> =>
parseAndSealSemanticGraphSteps(semanticGraphContainerSchema.parse(input));
function* parseAndSealSemanticGraphSteps(
container: z.output<typeof semanticGraphContainerSchema>,
): Generator<void, JavaScriptSemanticGraph> {
const issues: z.core.$ZodIssue[] = [];
const parsed: JavaScriptSemanticGraphInput = {
...container,
evidence_contexts: yield* parseElementSteps(
semanticGraphShape.evidence_contexts.element,
container.evidence_contexts,
["evidence_contexts"],
issues,
),
nodes: yield* parseElementSteps(
semanticGraphShape.nodes.element,
container.nodes,
["nodes"],
issues,
),
relations: yield* parseElementSteps(
semanticGraphShape.relations.element,
container.relations,
["relations"],
issues,
),
fingerprints: yield* parseElementSteps(
semanticGraphShape.fingerprints.element,
container.fingerprints,
["fingerprints"],
issues,
),
unknowns: yield* parseElementSteps(
semanticGraphShape.unknowns.element,
container.unknowns,
["unknowns"],
issues,
),
};
if (issues.length > 0) throw new z.ZodError(issues);
return yield* validateAndSealSemanticGraphSteps(normalizeGraphInput(parsed));
}
function* validateAndSealSemanticGraphSteps(
semantic: JavaScriptSemanticGraphInput,
): Generator<void, JavaScriptSemanticGraph> {
@@ -251,19 +251,9 @@ export const collectSemanticMemberMutations = (
originAt: current.originAt,
});
} else {
if (
t.isArrayExpression(expression) &&
((current.path[0] === "__proto__" &&
(effect === "escape" || current.path.length > 1)) ||
(current.path[0] === "constructor" &&
current.path[1] === "prototype"))
)
arrayIterationUnknown = true;
for (const value of referencedValues(
expression,
current.path,
effect,
)) {
for (const value of effect === "escape"
? escapedReferenceLeaves(expression, current.path)
: expandedValues(expression, current.path, effect)) {
if (value.iterableFallbackPath !== undefined)
deferIterable({
node: value.node,
@@ -290,6 +280,78 @@ export const collectSemanticMemberMutations = (
mutation: t.Node,
path: PropertyPath = [],
): void => markValue(node, path, new Set(), "escape", mutation);
function expandedValues(
expression: t.Node,
path: PropertyPath,
effect: ValueEffect,
): readonly ReferencedValue[] {
if (
t.isArrayExpression(expression) &&
((path[0] === "__proto__" && (effect === "escape" || path.length > 1)) ||
(path[0] === "constructor" && path[1] === "prototype"))
)
arrayIterationUnknown = true;
return referencedValues(expression, path, effect);
}
// Expanding a value below its references depends only on syntax, not on the
// escape site. Escaping an object reaches every method's results, so
// re-expanding it at each call through that object made N calls over N
// methods quadratic. Identifiers that resolve to the same binding and path
// are processed identically, so one stands for all of them.
const escapedLeaves = new WeakMap<
t.Node,
Map<string, readonly ReferencedValue[]>
>();
function escapedReferenceLeaves(
root: t.Node,
rootPath: PropertyPath,
): readonly ReferencedValue[] {
// Parser recovery can leave an argument hole, which expands to nothing.
if (!t.isNode(root)) return [];
const rootKey = JSON.stringify(rootPath);
const cached = escapedLeaves.get(root)?.get(rootKey);
if (cached !== undefined) return cached;
const leaves: ReferencedValue[] = [];
const leafIdentities = new Set<string>();
const expanded = new WeakMap<t.Node, Set<string>>();
const pending = [...expandedValues(root, rootPath, "escape")];
for (let cursor = 0; cursor < pending.length; cursor++) {
const value = pending[cursor];
if (value === undefined) break;
const node = unwrapJavaScriptExpression(value.node).node;
if (!t.isNode(node)) continue;
const pathKey = JSON.stringify(value.path);
if (t.isIdentifier(node) && value.iterableFallbackPath === undefined) {
const binding = resolveSemanticBindingState(state, node, node.name);
const identity = JSON.stringify([
binding === undefined ? null : binding.bindingId,
binding === undefined ? node.name : null,
pathKey,
]);
if (leafIdentities.has(identity)) continue;
leafIdentities.add(identity);
leaves.push(value);
} else if (
value.iterableFallbackPath !== undefined ||
t.isIdentifier(node) ||
t.isMemberExpression(node) ||
t.isOptionalMemberExpression(node)
)
leaves.push(value);
else {
const paths = expanded.get(node) ?? new Set<string>();
if (paths.has(pathKey)) continue;
paths.add(pathKey);
expanded.set(node, paths);
for (const reference of expandedValues(node, value.path, "escape"))
pending.push(reference);
}
}
const byPath = escapedLeaves.get(root) ?? new Map();
byPath.set(rootKey, leaves);
escapedLeaves.set(root, byPath);
return leaves;
}
const markReceiver = (callee: t.Node, mutation: t.Node): void => {
const expression = unwrapJavaScriptExpression(callee).node;
if (
+38
View File
@@ -0,0 +1,38 @@
/** Path-segment conventions. These are inferences, not observations of runtime behavior. */
export type NativeRuntimeConvention = "react-native" | "flutter" | "unity";
const foldSegment = (segment: string): string =>
segment.toLocaleLowerCase("en-US");
const REACT_NATIVE_SEGMENTS = new Set([
"react.framework",
"libhermes.so",
"hermes.framework",
]);
const FLUTTER_SEGMENTS = new Set(["libflutter.so", "flutter.framework"]);
const UNITY_SEGMENTS = new Set(["libunity.so", "unityframework.framework"]);
/**
* A runtime family named by one filename or directory segment.
* `reactnative` may occur inside a segment (`libreactnativejni.so`).
* The other names match the whole segment.
*/
export const nativeRuntimeConvention = (
path: string,
): NativeRuntimeConvention | null => {
for (const segment of path.split("/")) {
const folded = foldSegment(segment);
if (folded.includes("reactnative") || REACT_NATIVE_SEGMENTS.has(folded))
return "react-native";
if (FLUTTER_SEGMENTS.has(folded)) return "flutter";
if (UNITY_SEGMENTS.has(folded)) return "unity";
}
return null;
};
/** Swift runtime libraries are `libswift*.dylib` segments, not every dynamic library. */
export const isSwiftRuntimeLibrary = (path: string): boolean =>
path
.split("/")
.some((segment) => /^libswift.+\.dylib$/u.test(foldSegment(segment)));
+3
View File
@@ -108,6 +108,9 @@ export const processComparisonContract = (
...(scenario.normalization.ports
? { port_normalization_version: "contextual-endpoints-v2" }
: {}),
...(scenario.normalization.patterns.length > 0
? { pattern_normalization_version: "literal-replacements-v1" }
: {}),
rendered_line_format: "trailing-spaces-trimmed-v1",
});
+19 -1
View File
@@ -46,7 +46,6 @@ describe("captured script path projection", () => {
it.each([
["https://a.test/main.js", "https://a.test/main.js"],
["https://a.test/main.js", "https://a.test/main.js?v=2"],
["https://a.test/A/main.js", "https://a.test/a/other.js"],
["https://a.test/main.js", "https://a.test/main.js/child.js"],
])(
"isolates every version in a conflicting layout (%s, %s)",
@@ -99,6 +98,25 @@ describe("captured script path projection", () => {
},
);
it("keeps case-distinct URL paths until the destination collides", () => {
const planned = planWebScriptExport([
source("https://a.test/A/main.js"),
source("https://a.test/a/other.js", 2),
]);
expect(
planned.every(
({ record }) =>
record.content.state === "exported" &&
record.content.layout === "url-path",
),
).toBe(true);
const paths = planned.map(({ record }) =>
record.content.state === "exported" ? record.content.relative_path : "",
);
expect(paths[0]).toMatch(/\/A\/main\.js$/u);
expect(paths[1]).toMatch(/\/a\/other\.js$/u);
});
it("does not choose a canonical version when a competing source is unavailable", () => {
const missing: CapturedWebScript = {
...source("https://a.test/main.js?v=2", 2),
+7 -20
View File
@@ -34,7 +34,7 @@ export const planWebScriptExport = (
};
const candidate = candidates[index];
const reason = conflicts.has(index)
? "Captured paths collide by content version, case, or file/directory prefix."
? "Captured paths collide by content version or file/directory prefix."
: (candidate?.reason ?? null);
const path = reason === null ? candidate?.path : undefined;
return {
@@ -103,40 +103,27 @@ const candidatePath = (script: CapturedWebScript): CandidatePath => {
};
};
// Register every prefix so case collisions in parent directories and a file
// conflicting with a directory isolate all affected sources, not just the last.
// Register every exact prefix so a repeated path, or a file that is also a
// directory prefix, isolates every affected source. Case-distinct spellings
// stay distinct until the destination filesystem reports a collision.
const conflictingPaths = (
candidates: readonly CandidatePath[],
): Set<number> => {
const prefixes = new Map<
string,
{
spellings: Set<string>;
indices: number[];
files: number[];
}
>();
const prefixes = new Map<string, { indices: number[]; files: number[] }>();
candidates.forEach(({ path }, index) => {
if (path === undefined) return;
const parts = path.split("/");
parts.forEach((_, position) => {
const spelling = parts.slice(0, position + 1).join("/");
const key = spelling.toLowerCase();
const entry = prefixes.get(key) ?? {
spellings: new Set<string>(),
indices: [],
files: [],
};
entry.spellings.add(spelling);
const entry = prefixes.get(spelling) ?? { indices: [], files: [] };
entry.indices.push(index);
if (position === parts.length - 1) entry.files.push(index);
prefixes.set(key, entry);
prefixes.set(spelling, entry);
});
});
const conflicts = new Set<number>();
for (const entry of prefixes.values())
if (
entry.spellings.size > 1 ||
entry.files.length > 1 ||
(entry.files.length > 0 && entry.indices.length > entry.files.length)
)
+4
View File
@@ -28,3 +28,7 @@ export const zipPackageFormatForPath = (
if (lower.endsWith(".appx") || lower.endsWith(".appxbundle")) return "appx";
return undefined;
};
/** Inventory cannot see bytes that are not central-directory entries. */
export const ZIP_NON_ENTRY_TAIL_LIMITATION =
"ZIP inventory lists central-directory entries only. Bytes after the central directory, including an APK Signing Block or a ZIP comment, are not occurrences.";
+33
View File
@@ -0,0 +1,33 @@
import { z } from "zod";
/**
* An array schema's own constraints, such as a minimum length, without its
* element schema. Pair it with {@link parseElementSteps} to validate a large
* collection in cooperative steps.
*/
export const arrayContainerSchema = (
array: z.ZodArray<z.ZodType>,
): z.ZodType<unknown[]> => array.clone({ ...array.def, element: z.unknown() });
/** Elements validated between yields, in the issues and paths a whole-array parse reports. */
export function* parseElementSteps<Element extends z.ZodType>(
element: Element,
values: readonly unknown[],
path: readonly PropertyKey[],
issues: z.core.$ZodIssue[],
): Generator<void, z.output<Element>[]> {
const parsed: z.output<Element>[] = [];
for (const [index, value] of values.entries()) {
if (index % 64 === 0) yield;
const result = element.safeParse(value);
if (result.success) parsed.push(result.data);
else
issues.push(
...result.error.issues.map((issue) => ({
...issue,
path: [...path, index, ...issue.path],
})),
);
}
return parsed;
}
+98
View File
@@ -0,0 +1,98 @@
import { describe, expect, it, vi } from "vitest";
import { projectAnalysisError } from "../domain/analysisErrorProjection.js";
import { ProviderAdapterError } from "../domain/providerAdapterError.js";
import { ProviderCleanupError } from "../domain/providerCleanupError.js";
import { err, ok } from "../domain/result.js";
import { firmwareRequestSchema } from "../domain/firmware/firmwareAnalysis.js";
import { finishFirmwareWorkspace } from "./FirmwareProvider.js";
const request = firmwareRequestSchema.parse({
operation: "extract_firmware",
input: {
path: "/tmp/firmware.bin",
output_directory: "/tmp/extracted",
},
});
const execution = ok({
result: { files: ["kept"] },
rawResult: null,
provider: { id: "unblob", name: "unblob", version: "1" },
limitations: [],
locations: [],
subject: null,
});
describe("firmware workspace cleanup", () => {
it("returns the completed extraction beside a cleanup failure and does not delete the leaked root", async () => {
const close = vi.fn(async () => {
throw new Error("cleanup failed");
});
const finished = await finishFirmwareWorkspace(
{ path: "/tmp/runtime", close },
execution,
"unblob",
request,
);
expect(close).toHaveBeenCalledOnce();
if (finished.ok || !(finished.error instanceof ProviderAdapterError))
throw new Error("Expected cleanup failure");
expect(finished.error.cleanupResources).toEqual([
"/tmp/runtime",
"/tmp/extracted",
]);
expect(finished.error.partialObservation).toEqual({
kind: "firmware",
result: { files: ["kept"] },
});
expect(finished.error.diagnostics).toEqual({ reason: "cleanup failed" });
expect(projectAnalysisError(finished.error).details).toMatchObject({
partial_observation: {
kind: "firmware",
result: { files: ["kept"] },
},
});
});
it("does not replay a cleanup failure for a later workspace", async () => {
const failed = await finishFirmwareWorkspace(
{
path: "/tmp/runtime",
close: async () => {
throw new Error("cleanup failed");
},
},
execution,
"unblob",
request,
);
expect(failed.ok).toBe(false);
const next = await finishFirmwareWorkspace(
{ path: "/tmp/next", close: async () => undefined },
execution,
"unblob",
request,
);
expect(next).toBe(execution);
});
it("leaves an uncertain workspace recorded and does not remove it", async () => {
const close = vi.fn(async () => undefined);
const cleanup = new ProviderCleanupError(
"unblob",
["/tmp/runtime"],
{ reason: "ownership uncertain" },
{ operation: "extract_firmware" },
);
const outcome = err(cleanup);
const finished = await finishFirmwareWorkspace(
{ path: "/tmp/runtime", close },
outcome,
"unblob",
request,
);
expect(close).not.toHaveBeenCalled();
expect(finished).toBe(outcome);
});
});
+58 -30
View File
@@ -65,7 +65,6 @@ const limitations = [
/** Queued bring-your-own firmware adapters with private snapshots and owned cleanup. */
export class FirmwareProvider implements FirmwareAnalysisPort {
#tail: Promise<void> = Promise.resolve();
#cleanupFailure: AnalysisError | undefined;
constructor(
readonly environment: Readonly<
Record<string, string | undefined>
@@ -115,7 +114,6 @@ export class FirmwareProvider implements FirmwareAnalysisPort {
: AbortSignal.any([options.signal, deadline]);
let outcome: Outcome;
try {
if (this.#cleanupFailure !== undefined) return err(this.#cleanupFailure);
if (signal.aborted) throw new AnalysisCancelledError(request.operation);
const command = await resolveFirmwareCommand(
this.environment,
@@ -316,29 +314,49 @@ export class FirmwareProvider implements FirmwareAnalysisPort {
},
}),
);
if (!outcome.ok && outcome.error.cleanupIncomplete) {
this.#cleanupFailure =
root === undefined ||
outcome.error.cleanupResources.includes(root.path)
? outcome.error
: new ProviderCleanupError(
engineName,
[...outcome.error.cleanupResources, root.path],
outcome.error instanceof ProviderAdapterError
? (outcome.error.diagnostics ?? {
reason: outcome.error.message,
})
: { reason: outcome.error.message },
{ cause: outcome.error },
);
}
}
// Never remove a workspace while process ownership remains uncertain.
if (this.#cleanupFailure !== undefined) return err(this.#cleanupFailure);
try {
await root?.close();
} catch (cause: unknown) {
this.#cleanupFailure = new ProviderCleanupError(
// A leaked root stays recorded on this result and is not deleted. The next
// operation may create a new root; this failure is not replayed for it.
return finishFirmwareWorkspace(root, outcome, engineName, request);
}
}
/** Keep a completed extraction beside cleanup failure, or skip removal when ownership is uncertain. */
export const finishFirmwareWorkspace = async (
root: { readonly path: string; close(): Promise<void> } | undefined,
outcome: Outcome,
engineName: string,
request: FirmwareRequest,
): Promise<Outcome> => {
if (!outcome.ok && outcome.error.cleanupIncomplete) {
if (
root === undefined ||
outcome.error.cleanupResources.includes(root.path)
)
return outcome;
return err(
new ProviderCleanupError(
engineName,
[...outcome.error.cleanupResources, root.path],
outcome.error instanceof ProviderAdapterError
? (outcome.error.diagnostics ?? { reason: outcome.error.message })
: { reason: outcome.error.message },
{
cause: outcome.error,
operation: request.operation,
...(outcome.error.partialObservation === undefined
? {}
: { partialObservation: outcome.error.partialObservation }),
},
),
);
}
try {
await root?.close();
} catch (cause: unknown) {
return err(
new ProviderCleanupError(
engineName,
[
root?.path ?? "unknown",
@@ -346,13 +364,23 @@ export class FirmwareProvider implements FirmwareAnalysisPort {
? [request.input.output_directory]
: []),
],
{ reason: cause instanceof Error ? cause.message : String(cause) },
{
reason: cause instanceof Error ? cause.message : String(cause),
previous_result: outcome.ok ? outcome.value.result : null,
cause,
operation: request.operation,
...(outcome.ok
? {
partialObservation: {
kind: "firmware",
result: outcome.value.result,
},
}
: outcome.error.partialObservation === undefined
? {}
: { partialObservation: outcome.error.partialObservation }),
},
);
return err(this.#cleanupFailure);
}
return outcome;
),
);
}
}
return outcome;
};
+31 -15
View File
@@ -4,11 +4,19 @@ import type { DecodeIssue } from "./AppleDispatchDecodeFacts.js";
import type { FacetDecodeFacts } from "./AppleDispatchDecodeFacts.js";
const describeFixups = (fixups: PointerFixups): string => {
if (fixups.kind === "chained")
return `chained fixups: ${fixups.formats.join(", ") || "no fixup segments"}`;
return fixups.kind === "dyld-info"
? "LC_DYLD_INFO bind opcodes"
: "no fixup load commands";
const unread =
fixups.unreadRebaseBytes > 0
? `LC_DYLD_INFO rebase opcodes were not decoded (${fixups.unreadRebaseBytes} bytes)`
: undefined;
const kind =
fixups.kind === "chained"
? `chained fixups: ${fixups.formats.join(", ") || "no fixup segments"}`
: fixups.kind === "dyld-info"
? "LC_DYLD_INFO bind opcodes"
: unread === undefined
? "no fixup load commands"
: undefined;
return [unread, kind].filter((line) => line !== undefined).join("; ");
};
/** Append the per-facet coverage of one Apple dispatch metadata decode. */
@@ -83,18 +91,26 @@ export const pushDispatchCoverage = (input: {
({ decode }) => decode.status === "decoded",
).length,
});
const fixupReason = [
describeFixups(fixups),
...fixups.issues.map(
({ code, location, message }) =>
`${code}${location === undefined ? "" : ` at ${location}`}: ${message}`,
),
]
.filter((line) => line.length > 0)
.join("; ");
result.coverage.push({
facet: "pointer_fixups",
status: fixups.issues.length > 0 ? "partial" : "complete",
reason: [
describeFixups(fixups),
...fixups.issues.map(
({ code, location, message }) =>
`${code}${location === undefined ? "" : ` at ${location}`}: ${message}`,
),
].join("; "),
examined: 0,
decoded: 0,
status:
fixups.issues.length > 0 || fixups.unreadRebaseBytes > 0
? "partial"
: fixups.examined === 0
? "unsupported"
: "complete",
reason: fixupReason.length > 0 ? fixupReason : null,
examined: fixups.examined,
decoded: fixups.issues.length > 0 ? 0 : fixups.examined,
});
result.coverage.push({
facet: "swift_generic_resilient_witnesses_overrides_async_coroutines",
+1 -1
View File
@@ -354,7 +354,7 @@ describe("byte-swapped universal Apple dispatch metadata", () => {
const bytes = wrapped(false);
bytes.writeUInt32BE(0xfeedface, 256);
expect(() => decodeAppleDispatchMetadata(bytes, 100, provenance)).toThrow(
"Only little-endian 64-bit Mach-O metadata is supported",
"Unsupported 32-bit Mach-O metadata for requested CPU type arm64 (FAT header byte order: little-endian)",
);
});
+33 -2
View File
@@ -149,13 +149,37 @@ describe("pointer fixup tables", () => {
data.writeUInt16LE(99, 46);
const fixups = parsePointerFixups(
data,
{ chained: { offset: 0, size: data.length }, binds: [], dylibs: [] },
{
chained: { offset: 0, size: data.length },
binds: [],
rebases: [],
dylibs: [],
},
SEGMENTS,
BASE,
);
expect(fixups.failures).toContain("Unsupported chained pointer format 99");
});
it("keeps unread LC_DYLD_INFO rebase bytes when chained fixups are present", () => {
const data = chainedData(1);
const fixups = parsePointerFixups(
data,
{
chained: { offset: 0, size: data.length },
binds: [],
rebases: [{ offset: 8, size: 12 }],
dylibs: [],
},
SEGMENTS,
BASE,
);
expect(fixups).toMatchObject({
kind: "chained",
unreadRebaseBytes: 12,
});
});
it("does not decode unreferenced imports while constructing fixup facts", () => {
const data = chainedData(1);
// The second import is outside the table bytes; only the first is needed
@@ -164,7 +188,12 @@ describe("pointer fixup tables", () => {
expect(() =>
parsePointerFixups(
data,
{ chained: { offset: 0, size: data.length }, binds: [], dylibs: [] },
{
chained: { offset: 0, size: data.length },
binds: [],
rebases: [],
dylibs: [],
},
SEGMENTS,
BASE,
),
@@ -179,6 +208,7 @@ describe("pointer fixup tables", () => {
{
chained: { offset: 0, size: data.length },
binds: [],
rebases: [],
dylibs: ["/usr/lib/libobjc.A.dylib", "/usr/lib/libSystem.B.dylib"],
},
SEGMENTS,
@@ -241,6 +271,7 @@ describe("pointer fixup tables", () => {
{ offset: 0, size: opcodes.length, stream: "bind" },
{ offset: opcodes.length, size: lazy.length, stream: "lazy" },
],
rebases: [],
dylibs: ["/usr/lib/libobjc.A.dylib", "/usr/lib/libSystem.B.dylib"],
},
SEGMENTS,
+25 -1
View File
@@ -6,6 +6,11 @@ import type { DecodeIssue } from "./AppleDispatchDecodeFacts.js";
export interface FixupCommands {
/** Absolute file range of `LC_DYLD_CHAINED_FIXUPS` data. */
readonly chained: { readonly offset: number; readonly size: number } | null;
/** Absolute file ranges of `LC_DYLD_INFO(_ONLY)` rebase streams. */
readonly rebases: readonly {
readonly offset: number;
readonly size: number;
}[];
/** Absolute file ranges of `LC_DYLD_INFO(_ONLY)` bind streams. */
readonly binds: readonly {
readonly offset: number;
@@ -35,6 +40,10 @@ export interface PointerFixups {
readonly formats: readonly string[];
readonly failures: readonly string[];
readonly issues: readonly DecodeIssue[];
/** Fixup segments or bind streams this reader walked. */
readonly examined: number;
/** Rebase opcode bytes present in the load command and not decoded. */
readonly unreadRebaseBytes: number;
/** Decode the pointer stored at `address` whose raw 64-bit value is `raw`. */
decode(address: bigint, raw: bigint): DecodedPointer;
}
@@ -250,6 +259,8 @@ const chainedFixups = (
formats,
failures: issues.map(({ message }) => message),
issues,
examined: Math.max(ranges.length, 1),
unreadRebaseBytes: 0,
decode(address, raw) {
const segment = ranges.find(
({ from, to }) => address >= from && address < to,
@@ -420,13 +431,24 @@ export const parsePointerFixups = (
baseAddress: bigint,
): PointerFixups => {
const image = { bytes, segments, dylibs: commands.dylibs, baseAddress };
if (commands.chained !== null) return chainedFixups(image, commands.chained);
const unreadRebaseBytes = commands.rebases.reduce(
(total, rebase) => total + rebase.size,
0,
);
if (commands.chained !== null) {
const chained = chainedFixups(image, commands.chained);
return unreadRebaseBytes === 0
? chained
: { ...chained, unreadRebaseBytes };
}
if (commands.binds.length === 0)
return {
kind: "none",
formats: [],
failures: [],
issues: [],
examined: 0,
unreadRebaseBytes,
decode: (_address, raw) => ({ kind: "rebase", target: raw }),
};
const binds = new Map<bigint, Import>();
@@ -446,6 +468,8 @@ export const parsePointerFixups = (
issues: failures.map((message) =>
issue("pointer_fixups", "bind_stream_decode_failed", message),
),
examined: commands.binds.length,
unreadRebaseBytes,
decode(address, raw) {
const bound = binds.get(address);
return bound === undefined
+21
View File
@@ -16,6 +16,27 @@ describe("native Mach-O slice selection", () => {
expect(parseMachoLayout(Buffer.from(fat), "arm64").segments).toEqual([]);
});
it("selects only the admitted CPU type and rejects a 32-bit slice", () => {
const sixtyFour = fatImage([
{ cpu: CPU.x86_64, bytes: machoImage({ cpu: CPU.x86_64 }) },
]);
expect(() => parseMachoLayout(Buffer.from(sixtyFour), "x86")).toThrow(
/Requested FAT architecture is absent/u,
);
expect(() => parseMachoLayout(Buffer.from(sixtyFour), "arm")).toThrow(
/Requested FAT architecture is absent/u,
);
const thirtyTwo = fatImage([
{
cpu: CPU.i386,
bytes: machoImage({ cpu: CPU.i386, wide: false }),
},
]);
expect(() => parseMachoLayout(Buffer.from(thirtyTwo), "x86")).toThrow(
/Unsupported 32-bit Mach-O metadata for requested CPU type x86/u,
);
});
it("rejects FAT subtype disagreement and unaligned slice offsets", () => {
const original = fatImage([{ cpu: CPU.arm64, bytes: machoImage({}) }]);
const mismatched = original.slice();
+25 -3
View File
@@ -90,7 +90,8 @@ export const selectMachoSlice = (
const headerEnd = 8 + count * stride;
if (count > 128 || headerEnd > bytes.length)
throw malformed("Malformed FAT architecture table");
const cpu = architecture === "arm64" ? 0x0100000c : 0x01000007;
const cpu = MACHO_CPU_TYPE[architecture];
if (cpu === undefined) throw invalid("Requested FAT architecture is absent");
const declarations = readFatSliceDeclarations(bytes.subarray(8, headerEnd), {
count,
wide: fat64,
@@ -115,9 +116,26 @@ export const selectMachoSlice = (
}
if (selected === undefined)
throw invalid("Requested FAT architecture is absent");
if (selected.slice + 4 > bytes.length)
throw malformed("Truncated Mach-O header");
const sliceMagic = bytes.readUInt32LE(selected.slice);
if (sliceMagic === MH_MAGIC || sliceMagic === MH_CIGAM)
throw invalid(
`Unsupported 32-bit Mach-O metadata for requested CPU type ${architecture}`,
);
return selected;
};
/** CPU types admitted by `machArchitecture` in the domain binary target parser. */
const MACHO_CPU_TYPE: Readonly<Record<string, number>> = {
x86: 7,
x86_64: 0x01000007,
arm: 12,
arm64: 0x0100000c,
};
const MH_MAGIC = 0xfeedface;
const MH_CIGAM = 0xcefaedfe;
/** Parse segments/sections for a little-endian 64-bit Mach-O architecture slice. */
export const parseMachoLayout = (
bytes: Buffer,
@@ -151,6 +169,7 @@ export const parseMachoLayout = (
sections: [],
dylibs: [],
binds: [],
rebases: [],
chained: null,
baseAddress: undefined,
};
@@ -168,7 +187,7 @@ export const parseMachoLayout = (
else collectFixupCommand(bytes, command, state);
cursor += command.size;
}
const { segments, sections, chained, binds, dylibs } = state;
const { segments, sections, chained, binds, rebases, dylibs } = state;
const offset = (address: bigint, size = 1): number => {
const matches = segments.filter(
(segment) =>
@@ -187,7 +206,7 @@ export const parseMachoLayout = (
segments,
sections,
offset,
fixups: { chained, binds, dylibs },
fixups: { chained, binds, rebases, dylibs },
baseAddress: state.baseAddress ?? 0n,
};
};
@@ -200,6 +219,7 @@ interface LayoutState {
readonly sections: Section[];
readonly dylibs: string[];
readonly binds: FixupCommands["binds"][number][];
readonly rebases: FixupCommands["rebases"][number][];
chained: FixupCommands["chained"];
baseAddress: bigint | undefined;
}
@@ -273,6 +293,8 @@ const collectFixupCommand = (
state.chained = range(cursor + 8);
} else if (kind === LC_DYLD_INFO || kind === LC_DYLD_INFO_ONLY) {
if (size < 48) throw new RangeError("Truncated dyld-info command");
const rebase = range(cursor + 8);
if (rebase.size > 0) state.rebases.push(rebase);
for (const [at, stream] of [
[cursor + 16, "bind"],
[cursor + 24, "weak"],
+4
View File
@@ -66,6 +66,8 @@ describe("Swift vtable coverage facts: unresolved Swift dispatch slots", () => {
failures: [],
issues: [],
decode: (_address, raw) => ({ kind: "rebase", target: raw }),
examined: 0,
unreadRebaseBytes: 0,
},
swift: {
facet: "swift_conformances_static_witness_slots",
@@ -178,6 +180,8 @@ describe("Swift vtable coverage facts: unsupported pointer fixup coverage", () =
kind: "unsupported",
reason: "Unsupported chained pointer format 99",
}),
examined: 0,
unreadRebaseBytes: 0,
},
swift: {
facet: "swift_conformances_static_witness_slots",
+73
View File
@@ -0,0 +1,73 @@
import { describe, expect, it } from "vitest";
import { AnalysisCancelledError } from "../domain/analysisErrorCore.js";
import { err, ok } from "../domain/result.js";
import type { NativeCommandCapture } from "./CommandRunner.js";
import { inspectNativeMacho } from "./NativeMachoInspection.js";
const capture = (tool: string, stdout: string): NativeCommandCapture => ({
tool,
executable: tool,
executableSha256: "a".repeat(64),
toolVersion: null,
versionReason: null,
arguments: [],
stdout,
stderr: "",
stdoutBytes: stdout.length,
stderrBytes: 0,
exitCode: 0,
signal: null,
});
const inspect = async (lipo: string, architecture: "x86" | "arm64") => {
const commands: {
readonly tool: string;
readonly args: readonly string[];
}[] = [];
const result = await inspectNativeMacho({
target: {
kind: "executable",
format: "mach-o",
path: "/fixture/app",
sha256: "b".repeat(64),
architecture,
availableArchitectures: ["x86", "arm64"],
},
run: (tool, args) => {
commands.push({ tool, args });
if (tool === "lipo") return Promise.resolve(ok(capture(tool, lipo)));
if (tool === "file") return Promise.resolve(ok(capture(tool, "Mach-O")));
return Promise.resolve(err(new AnalysisCancelledError("inspect_native")));
},
invocation: () => {
throw new Error("invocation is not reached");
},
});
return { result, commands };
};
describe("Mach-O tool architecture selection", () => {
it("rejects a CPU lipo did not list instead of reading another slice", async () => {
const { result, commands } = await inspect("architecture arm64\n", "x86");
expect(result.ok).toBe(false);
if (result.ok) return;
expect(result.error).toMatchObject({
_tag: "AnalysisOutputError",
reason: "lipo did not list i386; refusing to read another slice",
});
expect(commands.map(({ tool }) => tool)).toEqual(["file", "lipo"]);
});
it("passes the arm64e slice lipo listed for an arm64 target", async () => {
const { commands } = await inspect("architecture arm64e\n", "arm64");
const otool = commands.find(({ tool }) => tool === "otool");
expect(otool?.args).toEqual([
"-h",
"-l",
"-arch",
"arm64e",
"/fixture/app",
]);
});
});
+27 -9
View File
@@ -1,6 +1,7 @@
import type { BinaryTarget } from "../domain/binaryTargetTypes.js";
import type { EvidenceLocation } from "../domain/evidence.js";
import type { AnalysisError } from "../domain/analysisErrorBase.js";
import { AnalysisOutputError } from "../domain/analysisErrorCore.js";
import {
inspectMachoSchema,
type NativeCommandInvocation,
@@ -65,10 +66,19 @@ export const inspectNativeMacho = async (
}
const lipo = captures.find(({ tool }) => tool === "lipo");
if (lipo === undefined) throw new TypeError("Missing lipo capture");
const selectedArchitecture = selectArchitecture(
context.target,
parseLipoArchitectures(lipo.stdout).map(({ name }) => name),
const availableArchitectures = parseLipoArchitectures(lipo.stdout).map(
({ name }) => name,
);
const selection = selectArchitecture(context.target, availableArchitectures);
if (selection.status === "absent")
return err(
new AnalysisOutputError(
"inspect_macho",
`lipo did not list ${selection.architecture}; refusing to read another slice`,
),
);
const selectedArchitecture =
selection.status === "selected" ? selection.architecture : null;
for (const [tool, prefix] of REQUIRED_SLICE_COMMANDS) {
const captured = await context.run(
tool,
@@ -339,24 +349,32 @@ const selectedArchitectureArguments = (
: ["-arch", toolArchitecture];
};
type ArchitectureSelection =
| { readonly status: "unfiltered" }
| { readonly status: "selected"; readonly architecture: string }
| { readonly status: "absent"; readonly architecture: string };
const selectArchitecture = (
target: BinaryTarget,
available: readonly string[],
): string | null => {
): ArchitectureSelection => {
if (target.kind !== "executable" || target.availableArchitectures.length < 2)
return null;
return { status: "unfiltered" };
const normalized =
target.architecture === "x86" ? "i386" : target.architecture;
if (available.includes(normalized)) return normalized;
if (available.includes(normalized))
return { status: "selected", architecture: normalized };
// BinaryTarget intentionally normalizes ARM64 CPU subtypes. Recover the
// concrete arm64e slice name from lipo before invoking slice-aware tools.
if (normalized === "arm64") {
const arm64e = available.find((architecture) => architecture === "arm64e");
if (arm64e !== undefined) return arm64e;
if (arm64e !== undefined)
return { status: "selected", architecture: arm64e };
const variant = available.find((architecture) =>
/^arm64e\.[A-Za-z0-9_]+$/u.test(architecture),
);
if (variant !== undefined) return variant;
if (variant !== undefined)
return { status: "selected", architecture: variant };
}
return normalized;
return { status: "absent", architecture: normalized };
};
@@ -75,6 +75,7 @@ interface TerminalExitOptions {
readonly interactions: InteractionEvent[];
readonly dispatchedEventIndexes: ReadonlySet<number>;
readonly recordEvent: RecordProcessCaptureEvent;
readonly onLiveProgress?: () => void;
}
interface CaptureResultOptions {
@@ -601,8 +602,10 @@ export const observeSettlement = async (
recordEvent: RecordProcessCaptureEvent = () => undefined,
platform: NodeJS.Platform = process.platform,
signal?: AbortSignal,
onLiveProgress?: () => void,
): Promise<ObservedProcessSettlement> => {
assertNotCancelled(signal);
onLiveProgress?.();
if (platform === "win32") {
recordEvent("lifecycle", 1);
return { state: "unverifiable", elapsed_ms: 0 };
@@ -611,6 +614,7 @@ export const observeSettlement = async (
let consecutiveEmpty = 0;
let deadlineReached = false;
while (!deadlineReached) {
onLiveProgress?.();
assertNotCancelled(signal);
const observations = await Promise.all(
[...new Set(processGroupIds)].map((processGroupId) =>
@@ -651,6 +655,7 @@ export const awaitTerminalExit = async ({
interactions,
dispatchedEventIndexes,
recordEvent,
onLiveProgress,
}: TerminalExitOptions): Promise<{
exitCode: number;
signal?: number;
@@ -661,6 +666,7 @@ export const awaitTerminalExit = async ({
// Keep the initiating lifecycle reason so comparisons distinguish a target
// exit from harness-owned timeout, idle-timeout, and cancellation cleanup.
let reason: "exited" | "timeout" | "idle_timeout" | "cancelled" = "exited";
onLiveProgress?.();
terminal.onExit((exit) => {
recordEvent("lifecycle", 0);
for (const [eventIndex, event] of scenario.events.entries()) {
@@ -688,6 +694,7 @@ export const awaitTerminalExit = async ({
resolveExit({ ...exit, reason });
});
const timeout = scheduleProcessInterval(() => {
onLiveProgress?.();
if (signal?.aborted === true) {
reason = "cancelled";
terminal.kill("SIGKILL");
@@ -0,0 +1,113 @@
import { expect, it } from "vitest";
import {
createProcessCaptureProgressTracker,
type ProcessCaptureProgress,
} from "./ProcessCaptureProgress.js";
const blockedProgress = () => {
let release: () => void = () => undefined;
const blocked = new Promise<void>((resolve) => {
release = resolve;
});
const delivered: Parameters<ProcessCaptureProgress["report"]>[0][] = [];
const progress: ProcessCaptureProgress = {
async report(update) {
delivered.push(update);
if (delivered.length === 1) await blocked;
},
};
return { progress, delivered, release: () => release() };
};
it("coalesces intermediate observations while the progress receiver is blocked", async () => {
const receiver = blockedProgress();
const tracker = createProcessCaptureProgressTracker(receiver.progress);
void tracker.report({
phase: "running",
counts: { frames: 1, samples: 0, interactions: 0 },
});
await Promise.resolve();
let latest: Promise<void> | undefined;
for (let frames = 2; frames <= 1_000; frames += 1) {
latest = tracker.report({
phase: "running",
counts: { frames, samples: 0, interactions: 0 },
});
}
expect(receiver.delivered).toHaveLength(1);
receiver.release();
await latest;
expect(receiver.delivered.map((update) => update.completed)).toEqual([
1, 1_000,
]);
expect(receiver.delivered[1]?.message).toContain("frames=1000");
});
it("discards stale live observations before delivering final cleanup", async () => {
const receiver = blockedProgress();
const tracker = createProcessCaptureProgressTracker(receiver.progress);
void tracker.report({
phase: "running",
counts: { frames: 1, samples: 0, interactions: 0 },
});
await Promise.resolve();
void tracker.report({
phase: "running",
counts: { frames: 2, samples: 0, interactions: 0 },
});
void tracker.report({
phase: "settling",
counts: { frames: 2, samples: 1, interactions: 0 },
});
tracker.closeLive();
await tracker.report({
phase: "running",
counts: { frames: 10, samples: 10, interactions: 10 },
});
const terminal = tracker.report({
phase: "cleanup",
counts: { frames: 2, samples: 2, interactions: 0 },
disposition: "exited",
terminal: true,
});
receiver.release();
await terminal;
expect(receiver.delivered.map((update) => update.phase)).toEqual([
"running",
"cleanup",
]);
expect(receiver.delivered.at(-1)).toMatchObject({
completed: 4,
total: null,
terminal: true,
});
});
it("still delivers terminal status after an observer rejects an update", async () => {
const delivered: Parameters<ProcessCaptureProgress["report"]>[0][] = [];
const tracker = createProcessCaptureProgressTracker({
async report(update) {
delivered.push(update);
if (delivered.length === 1) throw new Error("Observer disconnected");
},
});
await tracker.report({
phase: "prepare",
counts: { frames: 0, samples: 0, interactions: 0 },
});
tracker.closeLive();
await expect(
tracker.report({
phase: "cleanup",
counts: { frames: 1, samples: 1, interactions: 0 },
disposition: "failed",
terminal: true,
}),
).resolves.toBeUndefined();
expect(delivered.at(-1)).toMatchObject({
phase: "cleanup",
completed: 2,
terminal: true,
});
});
@@ -0,0 +1,121 @@
import type { ProcessCaptureCleanupReport } from "../../domain/process/processCapture.js";
/**
* Progress port accepted by process capture.
* Structurally matches the shared application `ProgressReporter`.
*/
export interface ProcessCaptureProgress {
report(update: {
readonly phase: string;
readonly completed: number;
readonly total: null;
readonly message: string;
readonly terminal?: boolean;
}): Promise<void>;
}
/** Observed collection counts. These are not a percentage of an unknown total. */
export interface ProcessCaptureProgressCounts {
readonly frames: number;
readonly samples: number;
readonly interactions: number;
}
export type ProcessCaptureProgressPhase =
| "prepare"
| "running"
| "settling"
| "cleanup";
export type ProcessCaptureProgressDisposition =
| "exited"
| "timeout"
| "idle_timeout"
| "cancelled"
| "failed";
export interface ProcessCaptureProgressReport {
readonly phase: ProcessCaptureProgressPhase;
readonly counts: ProcessCaptureProgressCounts;
readonly terminal?: boolean;
readonly disposition?: ProcessCaptureProgressDisposition;
readonly cleanup?: ProcessCaptureCleanupReport;
}
const formatProgressMessage = (
elapsedMs: number,
counts: ProcessCaptureProgressCounts,
disposition: ProcessCaptureProgressDisposition | undefined,
cleanup: ProcessCaptureCleanupReport | undefined,
): string => {
const status = `elapsed_ms=${String(elapsedMs)} frames=${String(counts.frames)} samples=${String(counts.samples)} interactions=${String(counts.interactions)}`;
const dispositionText =
disposition === undefined ? "" : ` disposition=${disposition}`;
const cleanupText =
cleanup === undefined
? ""
: ` owned_process_group=${cleanup.owned_process_group.state} terminal_renderer=${cleanup.terminal_renderer.state} temporary_root=${cleanup.temporary_root.state}`;
return `${status}${dispositionText}${cleanupText}`;
};
/** Serialize observed capture status through one rate-limited reporter. */
export const createProcessCaptureProgressTracker = (
progress: ProcessCaptureProgress | undefined,
now: () => number = Date.now,
) => {
const started = now();
let completed = 0;
let live = true;
let next: Parameters<ProcessCaptureProgress["report"]>[0] | undefined;
let pending: Promise<void> | undefined;
const flush = async (): Promise<void> => {
try {
while (next !== undefined) {
const update = next;
next = undefined;
try {
await progress?.report(update);
} catch {
// Progress delivery is observational; capture and cleanup still finish.
continue;
}
}
} finally {
pending = undefined;
}
};
return {
closeLive(): void {
live = false;
if (next?.terminal !== true) next = undefined;
},
report(update: ProcessCaptureProgressReport): Promise<void> {
if (progress === undefined) return Promise.resolve();
if (update.terminal !== true && !live) return Promise.resolve();
const units =
update.counts.frames +
update.counts.samples +
update.counts.interactions;
if (units > completed) completed = units;
// A slow observer needs the latest observation, not a queue of old states.
next = {
phase: update.phase,
completed,
total: null as null,
message: formatProgressMessage(
Math.max(0, now() - started),
update.counts,
update.disposition,
update.cleanup,
),
...(update.terminal === true ? { terminal: true as const } : {}),
};
pending ??= Promise.resolve().then(flush);
return pending;
},
};
};
export type ProcessCaptureProgressTracker = ReturnType<
typeof createProcessCaptureProgressTracker
>;
+87 -4
View File
@@ -55,6 +55,13 @@ import {
import { makeProcessCaptureEnvironment } from "./ProcessCaptureEnvironment.js";
import { classifyFilesystemEffects } from "./ProcessFilesystemEffects.js";
import { processCaptureOwnershipUnavailableReason } from "./ProcessCaptureCapability.js";
import {
createProcessCaptureProgressTracker,
type ProcessCaptureProgress,
type ProcessCaptureProgressCounts,
type ProcessCaptureProgressDisposition,
type ProcessCaptureProgressTracker,
} from "./ProcessCaptureProgress.js";
import type { ProcessOwnershipBaseline } from "../ProcessOwnership.js";
interface StartedCaptureRuntime {
@@ -262,7 +269,10 @@ const finishProcessRun = async (options: {
readonly executionFailure: unknown;
readonly observations?: ProcessCaptureObservationBuffer;
readonly rootPid?: number;
readonly progress: ProcessCaptureProgressTracker;
readonly progressCounts: () => ProcessCaptureProgressCounts;
}): Promise<ProcessCapture> => {
options.progress.closeLive();
await options.stopSampler();
const sampledProcessGroupIds =
options.runtime === undefined
@@ -281,6 +291,20 @@ const finishProcessRun = async (options: {
temporaryRoot: options.temporaryRoot,
...(options.cleanupHost === undefined ? {} : { host: options.cleanupHost }),
});
const observedExit =
options.observations?.exit.state === "available"
? options.observations.exit.value.reason
: options.capture?.exit.reason;
await options.progress.report({
phase: "cleanup",
counts: options.progressCounts(),
terminal: true,
disposition: captureProgressDisposition(
observedExit,
options.executionFailure,
),
cleanup,
});
const cleanupFailure = cleanupReportFailure(cleanup);
let { capture, executionFailure } = options;
let verifiedCapture: ProcessCapture | undefined;
@@ -336,6 +360,7 @@ const completeCapture = async (options: {
readonly eventJournal: readonly ProcessCaptureEventJournalEntry[];
readonly observationBuffer: ProcessCaptureObservationBuffer;
readonly recordEvent: RecordProcessCaptureEvent;
readonly onLiveProgress?: () => void;
}): Promise<PendingProcessCapture> => {
const unavailableReason = (stage: string, cause: unknown): string =>
`${stage} failed: ${cause instanceof Error ? cause.message : "unknown failure"}`;
@@ -351,6 +376,7 @@ const completeCapture = async (options: {
options.recordEvent,
options.hostPlatform,
options.signal,
options.onLiveProgress,
);
} catch (cause: unknown) {
options.observationBuffer.settlement = {
@@ -460,6 +486,25 @@ const completeCapture = async (options: {
});
};
const captureProgressDisposition = (
exitReason: string | undefined,
executionFailure: unknown,
): ProcessCaptureProgressDisposition => {
if (
exitReason === "exited" ||
exitReason === "timeout" ||
exitReason === "idle_timeout" ||
exitReason === "cancelled"
)
return exitReason;
if (
executionFailure instanceof ProcessCaptureError &&
executionFailure.reason === "cancelled"
)
return "cancelled";
return "failed";
};
/** Execute one caller-selected scenario and return bounded observations. */
const runProcessScenario = async (
scenario: ProcessScenario,
@@ -468,12 +513,44 @@ const runProcessScenario = async (
hostPlatform: NodeJS.Platform = process.platform,
captureSnapshot: typeof snapshotRoots = snapshotRoots,
cleanupHost?: ProcessCaptureCleanupHost,
progress?: ProcessCaptureProgress,
): Promise<ProcessCapture> => {
const { temporaryRoot, runId, ownershipBaseline, before } =
await prepareProcessCapture(scenario, signal, captureSnapshot);
const progressTracker = createProcessCaptureProgressTracker(progress);
const frames: TerminalFrame[] = [];
const samples: ProcessSample[] = [];
const interactions: InteractionEvent[] = [];
const progressCounts = (): ProcessCaptureProgressCounts => ({
frames: frames.length,
samples: samples.length,
interactions: interactions.length,
});
const reportRunning = () => {
void progressTracker.report({ phase: "running", counts: progressCounts() });
};
const reportSettling = () => {
void progressTracker.report({
phase: "settling",
counts: progressCounts(),
});
};
let prepared: Awaited<ReturnType<typeof prepareProcessCapture>>;
try {
await progressTracker.report({
phase: "prepare",
counts: progressCounts(),
});
prepared = await prepareProcessCapture(scenario, signal, captureSnapshot);
} catch (cause: unknown) {
progressTracker.closeLive();
await progressTracker.report({
phase: "cleanup",
counts: progressCounts(),
terminal: true,
disposition: signal?.aborted === true ? "cancelled" : "failed",
});
throw cause;
}
const { temporaryRoot, runId, ownershipBaseline, before } = prepared;
const journal = createProcessCaptureJournal();
const { entries: eventJournal, recordEvent } = journal;
recordEvent("filesystem_checkpoints", 0);
@@ -523,6 +600,7 @@ const runProcessScenario = async (
interactions,
dispatchedEventIndexes,
recordEvent,
...(progress === undefined ? {} : { onLiveProgress: reportRunning }),
});
observations.exit = {
state: "available",
@@ -549,6 +627,7 @@ const runProcessScenario = async (
recordEvent,
captureSnapshot,
...(signal === undefined ? {} : { signal }),
...(progress === undefined ? {} : { onLiveProgress: reportSettling }),
});
} catch (cause: unknown) {
executionFailure = normalizeCaptureFailure(cause, signal);
@@ -590,6 +669,8 @@ const runProcessScenario = async (
executionFailure,
observations,
...(actualRootPid === undefined ? {} : { rootPid: actualRootPid }),
progress: progressTracker,
progressCounts,
});
};
@@ -599,8 +680,9 @@ export const captureProcessScenario = async (
signal?: AbortSignal,
platform: NodeJS.Platform = process.platform,
environment: Readonly<Record<string, string | undefined>> = process.env,
captureSnapshot: typeof snapshotRoots = snapshotRoots,
captureSnapshot?: typeof snapshotRoots,
cleanupHost?: ProcessCaptureCleanupHost,
progress?: ProcessCaptureProgress,
): Promise<Result<ProcessCapture, ProcessCaptureError | AnalysisError>> => {
const ownershipReason = processCaptureOwnershipUnavailableReason(platform);
if (ownershipReason !== undefined)
@@ -624,8 +706,9 @@ export const captureProcessScenario = async (
signal,
environment,
platform,
captureSnapshot,
captureSnapshot ?? snapshotRoots,
cleanupHost,
progress,
),
);
} catch (cause: unknown) {

Some files were not shown because too many files have changed in this diff Show More