From 4999e4be2c1669d04417fcdea582052048adbb2d Mon Sep 17 00:00:00 2001 From: Ysopwnable <161784452+N0zoM1z0@users.noreply.github.com> Date: Sun, 11 Oct 2026 07:22:50 +0800 Subject: [PATCH] fix(javascript): isolate source analysis and retain bounded partial evidence (#1651) * refactor(javascript): validate transferable file projections * test(javascript): consume decoded module and static facts * fix(javascript): isolate source analysis and retain bounded partial evidence * test(javascript): verify cleanup retention and repeated source paths --- .../references/javascript-applications.md | 16 + docs/javascript-artifact-reconstruction.md | 46 +- knip.json | 1 + package.json | 3 + .../JavaScriptApplicationService.ts | 24 +- .../javascript/JavaScriptArtifactAnalysis.ts | 206 ++++- .../JavaScriptArtifactAnalysisTypes.ts | 24 +- .../JavaScriptArtifactGraphAccumulator.ts | 39 +- .../JavaScriptArtifactGraphBuilder.ts | 55 +- .../JavaScriptArtifactGraphContext.ts | 8 +- .../JavaScriptArtifactGraphStructure.ts | 81 +- .../JavaScriptArtifactReconstruction.ts | 229 ++++-- .../JavaScriptArtifactReconstructionInput.ts | 2 + .../JavaScriptSemanticGraphBuilder.ts | 160 +++- src/artifacts/ArtifactResourceScope.test.ts | 46 ++ src/artifacts/ArtifactResourceScope.ts | 9 + .../javascript/JavaScriptSourceAnalysis.ts | 12 + src/cli/electronCommands.ts | 6 + src/cliObservationOptions.ts | 4 + .../javascriptAnalysisWorkerProcess.ts | 82 ++ src/domain/analysisErrorProjection.ts | 5 +- .../javascriptAnalysisResourceControls.ts | 56 ++ .../javascriptApplicationAnalysis.ts | 2 + .../javascript/javascriptApplicationGraph.ts | 10 +- ...avascriptApplicationProjectionResources.ts | 42 + .../javascript/javascriptModuleSemanticIr.ts | 36 + .../javascriptSemanticFileProjection.ts | 18 + .../javascript/javascriptSemanticGraph.ts | 28 + .../javascript/javascriptSourceAnalysis.ts | 40 + src/domain/jsonValue.ts | 2 + .../analysis/JavaScriptAnalysisTransfer.ts | 414 ++++++++++ .../analysis/JavaScriptAnalysisWorker.ts | 726 ++++++++++++++++++ .../JavaScriptAnalysisWorkerProtocol.ts | 73 ++ .../JavaScriptAnalysisWorkerSchemas.ts | 582 ++++++++++++++ .../JavaScriptAnalysisWorkerServer.ts | 185 +++++ src/server/registerElectronTools.ts | 17 +- src/server/toolResult.ts | 33 +- .../javascriptSourceAnalysisWorker.test.ts | 93 +++ .../javascriptAnalysisCancellation.test.ts | 41 +- .../mcp/javascriptAnalysisResources.test.ts | 331 ++++++++ .../javascriptArtifactReconstruction.test.ts | 137 +++- 41 files changed, 3748 insertions(+), 176 deletions(-) create mode 100644 src/artifacts/javascript/JavaScriptSourceAnalysis.ts create mode 100644 src/composition/javascriptAnalysisWorkerProcess.ts create mode 100644 src/domain/javascript/javascriptAnalysisResourceControls.ts create mode 100644 src/domain/javascript/javascriptApplicationProjectionResources.ts create mode 100644 src/domain/javascript/javascriptModuleSemanticIr.ts create mode 100644 src/domain/javascript/javascriptSemanticFileProjection.ts create mode 100644 src/domain/javascript/javascriptSourceAnalysis.ts create mode 100644 src/javascript/analysis/JavaScriptAnalysisTransfer.ts create mode 100644 src/javascript/analysis/JavaScriptAnalysisWorker.ts create mode 100644 src/javascript/analysis/JavaScriptAnalysisWorkerProtocol.ts create mode 100644 src/javascript/analysis/JavaScriptAnalysisWorkerSchemas.ts create mode 100644 src/javascript/analysis/JavaScriptAnalysisWorkerServer.ts create mode 100644 tests/boundary/javascript/javascriptSourceAnalysisWorker.test.ts create mode 100644 tests/boundary/mcp/javascriptAnalysisResources.test.ts diff --git a/.agents/skills/reverse-engineer-anything/references/javascript-applications.md b/.agents/skills/reverse-engineer-anything/references/javascript-applications.md index 01b49a888..4d4a1fe97 100644 --- a/.agents/skills/reverse-engineer-anything/references/javascript-applications.md +++ b/.agents/skills/reverse-engineer-anything/references/javascript-applications.md @@ -9,6 +9,22 @@ that static analysis. Read findings, coverage, limitations and unknowns, then follow a specific unanswered question. Older servers without `detail` return the complete application Evidence inline when it fits. +When the connected schema advertises `max_heap_mb` and `analysis_timeout_ms`, +these select the isolated source analyzer's old-space heap and per-source +deadline. Defaults are 1024 MiB and 300000 ms. They do not change the MCP +process heap. Resource-limited analyses preserve completed static facts and +report partial semantic coverage with specific unknown scopes; this does not +mean the source is malformed. Inspect those limits before retrying a source. +When expanded relationships exceed the parent's available memory, inspect the +file's modules item view for retained `static_analysis` and +`semantic_module_analysis`. Their facts remain available even when graph links +are unknown. Cumulative result exhaustion can stop analysis before every file +is examined; the inventory and remaining unknown scopes distinguish that case. +Timeout or cancellation after source analysis begins retains partial Evidence +in the same session. A returned `error.details.partial_observation` reference +can be supplied directly to `inspect_analysis_view`. If the SDK cancels locally, retrieve its recorded ID from +the Evidence bundle before inspecting the partial analysis. + ## Read a module without repeating analysis Use the actual returned parent ID, on the same connection, as `source.evidence_id`. diff --git a/docs/javascript-artifact-reconstruction.md b/docs/javascript-artifact-reconstruction.md index 4b988a848..bd3b3a8f5 100644 --- a/docs/javascript-artifact-reconstruction.md +++ b/docs/javascript-artifact-reconstruction.md @@ -89,9 +89,9 @@ queries use per-file range indexes, while call sites use exact-range lookups. Existing node budgets and coverage reporting remain in effect. Progress identifies the source file being parsed and projected. Cancellation is -checked between files, with event-loop yields that allow notifications and -completed file-local allocations to be released. A single file's synchronous -parser and semantic extraction still need to finish before that boundary. +checked between files and throughout result transfer. The owned analyzer can +be stopped while its synchronous parser or semantic extractor is active; +the CLI/MCP process remains available for control requests. Graph and Evidence identifiers hash canonical JSON incrementally, without assembling a single string for the whole graph. The canonical bytes and existing @@ -113,6 +113,46 @@ smaller view, for example `--format json --filter-output evidence_id,normalized_result.statistics`. Streaming output does not bound the memory needed to construct the analysis graph itself. +JavaScript source parsing, static analysis and semantic projection run in one +owned analysis process, reused sequentially across files. Target code is never +executed. The analyzer defaults to 1024 MiB of V8 old-space and a 300000 ms +deadline per source, including startup and fact transfer. Select +`max_heap_mb` and `analysis_timeout_ms` through MCP, or `--max-heap-mb` and +`--analysis-timeout-ms` on the dedicated CLI command. These choices control the +analyzer; the caller's `NODE_OPTIONS` still controls the CLI/MCP process heap. +V8 old-space limits do not impose a process RSS limit. + +A source that exhausts the analyzer heap does not terminate the CLI/MCP +process. Completed static facts are retained, other files can still be analyzed, +and unexamined semantic relationships remain explicit unknowns. The parent +also checks transfer expansion and accumulated results against its remaining +heap before accepting semantic facts. A resource-limited result has partial +coverage, the observed limits and an `analyze-javascript-source` unknown scope +with the specific failure, including captured diagnostics when available. +Resource failures do not increment syntax parse failures. There is no fixed +source-length cutoff. + +Static observations also need memory when expanded into nodes, relationships +and Evidence. If that expansion does not fit, the file node retains +`static_analysis` and any admitted `semantic_module_analysis` observations, +including their exact values and source locations. Use the public modules item +view to read them. The graph marks those unexpanded relationships as unknown. +If cumulative result capacity is exhausted, analysis stops; inventoried but +unexamined files remain present with explicit unknown scopes. + +A timeout stops the remaining analysis and returns typed failure details with +completed partial Evidence. MCP failure details return +`partial_observation: { kind: "retained-evidence", evidence_id: "..." }` +after recording succeeds, so `inspect_analysis_view` can open that analysis +directly without serializing the entire partial graph into the error response. +CLI failure output retains the complete partial Evidence. +Cancellation after source analysis begins likewise +retains completed facts; an SDK client may reject its cancelled request before +receiving that reply. On the same MCP connection, inspect the Evidence bundle +and use the retained analysis ID with `inspect_analysis_view`. Owned analyzer +cleanup is verified before its private files are released. Cleanup uncertainty +is reported separately and retains ownership for retry. + CLI workflows parse larger JSON input files incrementally from one verified regular-file handle, without constructing a document-sized string. Smaller files retain native JSON parsing. Both paths require strict JSON and valid diff --git a/knip.json b/knip.json index 15d9c51c8..f830c12f7 100644 --- a/knip.json +++ b/knip.json @@ -3,6 +3,7 @@ "entry": [ "src/main.ts", "src/cli.ts", + "src/composition/javascriptAnalysisWorkerProcess.ts", "src/contracts/errorSchemas.ts", "src/evaluation/AgentEvaluationFixtures.ts", "src/application/RealHopperAssertions.ts", diff --git a/package.json b/package.json index 13900238f..fbd1e0825 100644 --- a/package.json +++ b/package.json @@ -30,6 +30,9 @@ "access": "public" }, "type": "module", + "imports": { + "#javascript-analysis-worker": "./dist/composition/javascriptAnalysisWorkerProcess.js" + }, "bin": { "rea": "scripts/rea.mjs", "rea-agents": "scripts/rea.mjs" diff --git a/src/application/javascript/JavaScriptApplicationService.ts b/src/application/javascript/JavaScriptApplicationService.ts index 917c29dff..04a2125c0 100644 --- a/src/application/javascript/JavaScriptApplicationService.ts +++ b/src/application/javascript/JavaScriptApplicationService.ts @@ -6,14 +6,9 @@ import { analyzeJavaScriptApplicationInputSchema, parseOwnedJavaScriptApplicationAnalysisSteps, } from "../../domain/javascript/javascriptApplicationAnalysis.js"; -import { - AnalysisAccessDeniedError, - AnalysisInputError, - AnalysisOutputError, - AnalysisUnsupportedTargetError, -} from "../../domain/analysisErrorCore.js"; +import { AnalysisOutputError } from "../../domain/analysisErrorCore.js"; import { ArtifactOperationError } from "../../domain/artifactOperationError.js"; -import { type AnalysisError } from "../../domain/analysisErrorBase.js"; +import { AnalysisError } from "../../domain/analysisErrorBase.js"; import type { Evidence } from "../../domain/evidence.js"; import { analysisInputErrorFromIssues } from "../../domain/inputIssueProjection.js"; import { err, ok, type Result } from "../../domain/result.js"; @@ -64,6 +59,12 @@ export const analyzeJavaScriptApplicationValidated = async ( input_path: input.input_path, format: input.format, integrity_policy: input.integrity_policy, + ...(input.max_heap_mb === undefined + ? {} + : { max_heap_mb: input.max_heap_mb }), + ...(input.analysis_timeout_ms === undefined + ? {} + : { analysis_timeout_ms: input.analysis_timeout_ms }), }, scope, options.signal, @@ -110,13 +111,8 @@ export const analyzeJavaScriptApplicationValidated = async ( assertJavaScriptAnalysisActive(options.signal); return ok(evidence); } catch (cause: unknown) { - // Input selection failures are already typed by their cause. - if ( - cause instanceof AnalysisInputError || - cause instanceof AnalysisUnsupportedTargetError || - cause instanceof AnalysisAccessDeniedError - ) - return err(cause); + // Preserve execution constraints and their completed partial Evidence. + if (cause instanceof AnalysisError) return err(cause); if (cause instanceof ArtifactReaderFailure) return err(artifactFailureToOperationError(cause)); if (cause instanceof z.ZodError) diff --git a/src/application/javascript/JavaScriptArtifactAnalysis.ts b/src/application/javascript/JavaScriptArtifactAnalysis.ts index ec2c13932..eec68f71f 100644 --- a/src/application/javascript/JavaScriptArtifactAnalysis.ts +++ b/src/application/javascript/JavaScriptArtifactAnalysis.ts @@ -8,6 +8,7 @@ import { classifyParsedJavaScriptOpenReceiversSteps, } from "../../domain/javascript/javascriptSemanticAnalysis.js"; import type { JavaScriptSemanticIr } from "../../domain/javascript/javascriptSemanticIr.js"; +import { projectJavaScriptModuleSemantics } from "../../domain/javascript/javascriptModuleSemanticIr.js"; import { parseJavaScriptSource } from "../../domain/javascript/javascriptSourceParser.js"; import { hasValidSourceMapContents } from "../../domain/sourceMapContents.js"; import { flattenSourceMapLeaves } from "../../domain/sourceMapEnvelope.js"; @@ -35,6 +36,15 @@ import type { import { analyzeJavaScriptJsonModule } from "./JavaScriptJsonModules.js"; import { completeJavaScriptAnalysisSteps } from "./JavaScriptAnalysisControl.js"; import { htmlArtifactReferences } from "../../domain/javascript/htmlArtifactReferences.js"; +import type { JavaScriptSourceAnalysisPort } from "../../domain/javascript/javascriptSourceAnalysis.js"; +import type { JavaScriptSemanticGraphProjection } from "./JavaScriptSemanticGraphBuilder.js"; +import { AnalysisError } from "../../domain/analysisErrorBase.js"; +import { projectAnalysisError } from "../../domain/analysisErrorProjection.js"; +import { + AnalysisCancelledError, + AnalysisResourceConstraintError, + AnalysisTimeoutError, +} from "../../domain/analysisErrorCore.js"; interface MutableArtifactAnalysis< SemanticIr extends JavaScriptModuleSemanticIr, @@ -95,27 +105,7 @@ export const analyzeAndProjectJavaScriptArtifactFiles = async ( ): Generator { const projection = projectSemantics(file, ir); if (projection !== undefined) while (projection.next().done !== true) yield; - const programScopes = ir.scopes.filter(({ kind }) => kind === "program"); - const programScopeId = programScopes[0]?.scopeId; - const localNames = new Set( - ir.moduleLinks.map(({ localName }) => localName), - ); - const callableIds = new Set( - ir.moduleLinks.map(({ callableId }) => callableId), - ); - return { - scopes: programScopes, - bindings: ir.bindings.filter( - ({ name, scopeId }) => - scopeId === programScopeId && localNames.has(name), - ), - callables: ir.callables.filter(({ callableId }) => - callableIds.has(callableId), - ), - moduleLinks: ir.moduleLinks, - coverage: ir.coverage, - limitations: ir.limitations, - }; + return projectJavaScriptModuleSemantics(ir); }; const context = { state, projectSemantics: projectFileSemantics }; for (const [index, file] of fileSet.files.entries()) { @@ -130,6 +120,174 @@ export const analyzeAndProjectJavaScriptArtifactFiles = async ( return finalizeArtifactAnalysis(state); }; +/** Analyze sources behind a hard heap boundary, retaining collected facts on interruption. */ +export const analyzeAndAdoptJavaScriptArtifactFiles = async ( + fileSet: JavaScriptArtifactFileSet, + worker: Pick, + projection: JavaScriptSemanticGraphProjection, + beforeFile: ( + file: JavaScriptArtifactFile, + completed: number, + total: number, + ) => Promise, + signal?: AbortSignal, +): Promise<{ + readonly analysis: JavaScriptModuleArtifactAnalysis; + readonly interruption: AnalysisError | null; +}> => { + const state = emptyArtifactAnalysis(); + let interruption: AnalysisError | null = null; + for (const [index, file] of fileSet.files.entries()) { + try { + await setImmediate(); + await beforeFile(file, index, fileSet.files.length); + if (signal?.aborted === true) + throw new AnalysisCancelledError("analyze_javascript_application"); + if (file.kind !== "javascript" || !file.text.included) { + const steps = analyzeArtifactFileSteps(file, { + state, + projectSemantics: function* (_file, ir) { + yield; + return projectJavaScriptModuleSemantics(ir); + }, + }); + await completeJavaScriptAnalysisSteps(steps, signal); + continue; + } + const result = await worker.analyze( + file, + projection.remainingFileNodeBudget(), + signal, + ); + const facts = result.ok ? result.value : result.error; + const applicationProjectionFailure = facts.applicationProjectionFailure; + if (facts.projection !== null) + projection.adoptFileProjection(facts.projection); + const error = result.ok ? null : result.error.error; + const limits = + error instanceof AnalysisResourceConstraintError + ? error.reportedLimits + : error instanceof AnalysisTimeoutError + ? { worker_timeout_ms: error.timeoutMs } + : null; + const failure = + error === null + ? undefined + : { + reason: `${file.path}: ${error.message}`, + error: projectAnalysisError(error), + limits, + }; + state.files.push({ + file, + javascript: facts.javascript, + semantic: facts.module === null ? null : { ir: facts.module }, + ...(failure === undefined ? {} : { analysis_failure: failure }), + ...(applicationProjectionFailure === undefined + ? {} + : { + application_projection_failure: { + reason: applicationProjectionFailure.message, + error: projectAnalysisError(applicationProjectionFailure), + limits: applicationProjectionFailure.reportedLimits, + }, + }), + }); + if (failure !== undefined && facts.projection === null) + projection.recordUnavailableFile( + file, + failure.reason, + failure.limits, + error instanceof AnalysisResourceConstraintError || + error instanceof AnalysisTimeoutError + ? "resource-limit" + : "incomplete-module", + ); + if (facts.javascript !== null) { + state.visitedNodes += facts.javascript.visited_ast_nodes; + state.findings += + findingCount(facts.javascript) + + (facts.module?.moduleLinks.length ?? 0); + state.modules += facts.javascript.bundler_registrations.reduce( + (count, item) => count + item.modules.length, + 0, + ); + if (facts.javascript.parse_status === "failed") + state.parseFailures += 1; + } + if ( + error !== null && + (!(error instanceof AnalysisResourceConstraintError) || + error.reportedLimits?.parent_result_budget_exhausted === true) + ) { + interruption = error; + break; + } + } catch (cause: unknown) { + if (signal?.aborted !== true && !(cause instanceof AnalysisError)) + throw cause; + interruption = + signal?.aborted === true + ? new AnalysisCancelledError("analyze_javascript_application") + : cause instanceof AnalysisError + ? cause + : null; + if (interruption !== null) { + state.files.push({ + file, + javascript: null, + semantic: null, + analysis_failure: { + reason: `${file.path}: ${interruption.message}`, + error: projectAnalysisError(interruption), + limits: null, + }, + }); + projection.recordUnavailableFile( + file, + interruption.message, + null, + "incomplete-module", + ); + } + break; + } + } + if (interruption !== null) { + const analyzedPaths = new Set(state.files.map(({ file }) => file.path)); + const failure = { + reason: `Source analysis was interrupted: ${interruption.message}`, + error: projectAnalysisError(interruption), + limits: + interruption instanceof AnalysisResourceConstraintError + ? interruption.reportedLimits + : interruption instanceof AnalysisTimeoutError + ? { worker_timeout_ms: interruption.timeoutMs } + : null, + }; + for (const file of fileSet.files) { + if (analyzedPaths.has(file.path)) continue; + state.files.push({ + file, + javascript: null, + semantic: null, + analysis_failure: failure, + }); + if (file.kind === "javascript") + projection.recordUnavailableFile( + file, + failure.reason, + failure.limits, + interruption instanceof AnalysisResourceConstraintError || + interruption instanceof AnalysisTimeoutError + ? "resource-limit" + : "incomplete-module", + ); + } + } + return { analysis: finalizeArtifactAnalysis(state), interruption }; +}; + const analyzeArtifactFiles = ( fileSet: JavaScriptArtifactFileSet, projectSemantics: ArtifactAnalysisContext["projectSemantics"], @@ -161,6 +319,12 @@ const finalizeArtifactAnalysis = < limitations: [ "JavaScript and HTML were parsed as inert text; bundle bootstrap code was never executed.", "Static paths and relationships may remain unresolved when expressions are dynamic or obfuscated.", + ...state.files.flatMap( + ({ analysis_failure, application_projection_failure }) => + [analysis_failure, application_projection_failure].flatMap( + (failure) => (failure === undefined ? [] : [failure.reason]), + ), + ), ], }; }; diff --git a/src/application/javascript/JavaScriptArtifactAnalysisTypes.ts b/src/application/javascript/JavaScriptArtifactAnalysisTypes.ts index 261ac88ec..630a330c5 100644 --- a/src/application/javascript/JavaScriptArtifactAnalysisTypes.ts +++ b/src/application/javascript/JavaScriptArtifactAnalysisTypes.ts @@ -4,7 +4,9 @@ import type { JavaScriptStaticAnalysis, } from "../../domain/javascript/javascriptStaticAnalysisTypes.js"; import type { JavaScriptSemanticIr } from "../../domain/javascript/javascriptSemanticIr.js"; +import type { JavaScriptModuleSemanticIr } from "../../domain/javascript/javascriptModuleSemanticIr.js"; import type { JavaScriptArtifactFile } from "../../domain/javascript/javascriptArtifactFiles.js"; +import type { JsonValue } from "../../domain/jsonValue.js"; interface JavaScriptStructuredObservation { readonly path: string; @@ -89,15 +91,7 @@ export type JavaScriptJsonModuleObservation = JavaScriptStructuredObservation & ); /** Module facts needed after file-local semantic projection has finished. */ -export type JavaScriptModuleSemanticIr = Pick< - JavaScriptSemanticIr, - | "scopes" - | "bindings" - | "callables" - | "moduleLinks" - | "coverage" - | "limitations" ->; +export type { JavaScriptModuleSemanticIr } from "../../domain/javascript/javascriptModuleSemanticIr.js"; /** One relevant file plus optional AST-only JavaScript facts. */ export interface AnalyzedJavaScriptArtifactFile< @@ -108,6 +102,18 @@ export interface AnalyzedJavaScriptArtifactFile< readonly semantic: { readonly ir: SemanticIr; } | null; + /** Operational incompleteness, distinct from a malformed source file. */ + readonly analysis_failure?: { + readonly reason: string; + readonly error: JsonValue; + readonly limits: Readonly> | null; + }; + /** Facts retained inline when their expanded graph exceeds available memory. */ + readonly application_projection_failure?: { + readonly reason: string; + readonly error: JsonValue; + readonly limits: Readonly> | null; + }; } /** Complete static-analysis projection before graph construction. */ diff --git a/src/application/javascript/JavaScriptArtifactGraphAccumulator.ts b/src/application/javascript/JavaScriptArtifactGraphAccumulator.ts index 5891e00de..c492f05c6 100644 --- a/src/application/javascript/JavaScriptArtifactGraphAccumulator.ts +++ b/src/application/javascript/JavaScriptArtifactGraphAccumulator.ts @@ -12,7 +12,7 @@ export class JavaScriptArtifactGraphAccumulator { readonly #nodes = new Map(); readonly #edges = new Map(); - /** Create or merge one canonical node. */ + /** Create or merge one canonical node, retaining a stable live construction reference. */ addNode(input: Input): ApplicationNode { const created = createJavaScriptApplicationNode(displayableNode(input)); const existing = this.#nodes.get(created.node_id); @@ -23,19 +23,23 @@ export class JavaScriptArtifactGraphAccumulator { const bySemanticId = new Map( [...existing.observations, ...created.observations].map((observation) => [ observation.observation_id, - observationInput(observation), + observation, ]), ); - const observations = [...bySemanticId.entries()] - .sort(([left], [right]) => (left < right ? -1 : left > right ? 1 : 0)) - .map(([, observation]) => observation); - const merged = createJavaScriptApplicationNode({ - kind: existing.kind, - identity: existing.identity, - observations, - }); - this.#nodes.set(merged.node_id, merged); - return merged; + const observations = [...bySemanticId.values()].sort((left, right) => + left.observation_id < right.observation_id + ? -1 + : left.observation_id > right.observation_id + ? 1 + : 0, + ); + // Both factories already validated and identified these observations. + // Replacing the node retained every previous observation array through + // path/chunk/IPC indexes when multiple paths shared an entity identity. + existing.observations.length = 0; + for (const observation of observations) + existing.observations.push(observation); + return existing; } /** Create or replace one canonical edge by semantic identifier. */ @@ -56,17 +60,6 @@ export class JavaScriptArtifactGraphAccumulator { } } -const observationInput = ( - observation: ApplicationNode["observations"][number], -) => { - const { - observation_id: _observationId, - identifier_strategy: _strategy, - ...input - } = observation; - return input; -}; - /** Node fields whose display form is normalized before validation. */ interface DisplayableNodeInput { readonly observations: readonly { readonly label: string | null }[]; diff --git a/src/application/javascript/JavaScriptArtifactGraphBuilder.ts b/src/application/javascript/JavaScriptArtifactGraphBuilder.ts index 023802a79..b10b92deb 100644 --- a/src/application/javascript/JavaScriptArtifactGraphBuilder.ts +++ b/src/application/javascript/JavaScriptArtifactGraphBuilder.ts @@ -107,19 +107,33 @@ function* buildJavaScriptArtifactGraphInputSteps( yield; addJavaScriptArtifactFiles(context); yield; - const packageRoots = addJavaScriptPackageNodes(context); + // Raw observations remain on the file nodes. Only admitted expansions feed + // the relationship builders, including their Electron and bundler callers. + const projectionContext = { + ...context, + analysis: { + ...analysis, + files: analysis.files.map((file) => + file.application_projection_failure === undefined + ? file + : { ...file, javascript: null, semantic: null }, + ), + }, + }; + const packageRoots = addJavaScriptPackageNodes(projectionContext); yield; - addJavaScriptSourceModules(context); + addJavaScriptSourceModules(projectionContext); yield; - const bundlerLimitations = addJavaScriptBundlerNodes(context); + const bundlerLimitations = addJavaScriptBundlerNodes(projectionContext); const relationshipOmissions = - yield* addJavaScriptModuleRelationshipsSteps(context); - const findingLimitations = yield* addJavaScriptStaticFindingsSteps(context); - addElectronBoundaries(context); + yield* addJavaScriptModuleRelationshipsSteps(projectionContext); + const findingLimitations = + yield* addJavaScriptStaticFindingsSteps(projectionContext); + addElectronBoundaries(projectionContext); yield; - addJavaScriptHtmlRoles(context); + addJavaScriptHtmlRoles(projectionContext); yield; - addJavaScriptSourceMapOriginals(context); + addJavaScriptSourceMapOriginals(projectionContext); yield; const coverage = graphCoverage(context); return { @@ -153,14 +167,34 @@ const graphCoverage = (context: JavaScriptArtifactGraphContext) => { javascript !== null && javascript.parse_status === "partial", ); const unknownGap = + context.analysis.files.some( + ({ analysis_failure, application_projection_failure }) => + analysis_failure !== undefined || + application_projection_failure !== undefined, + ) || context.analysis.parse_failures > 0 || context.fileSet.invalid_utf8_files > 0 || sourceMapPolicyGap || malformedStructuredData || partialJavaScript; - if (resourceLimits.length > 0) + const executionLimits = context.analysis.files.flatMap( + ({ analysis_failure, application_projection_failure }) => + [analysis_failure, application_projection_failure].flatMap((failure) => + failure === undefined + ? [] + : javaScriptAnalysisCoverageLimits(failure.limits), + ), + ); + if (resourceLimits.length > 0 || executionLimits.length > 0) return partialApplicationCoverage( - semanticResourceLimitCoverage(resourceLimits), + [ + ...new Map( + [ + ...semanticResourceLimitCoverage(resourceLimits), + ...executionLimits, + ].map((limit) => [JSON.stringify(limit), limit]), + ).values(), + ], null, ); if (unknownGap) return partialApplicationCoverage([], null); @@ -260,3 +294,4 @@ const graphLimitations = ( ]), ]; }; +import { javaScriptAnalysisCoverageLimits } from "../../domain/javascript/javascriptAnalysisResourceControls.js"; diff --git a/src/application/javascript/JavaScriptArtifactGraphContext.ts b/src/application/javascript/JavaScriptArtifactGraphContext.ts index 58b72bce7..8b0fed5f1 100644 --- a/src/application/javascript/JavaScriptArtifactGraphContext.ts +++ b/src/application/javascript/JavaScriptArtifactGraphContext.ts @@ -1,4 +1,5 @@ import type { ArtifactInventorySnapshot } from "../../domain/artifactInventorySnapshot.js"; +import type { JsonValue } from "../../domain/jsonValue.js"; import { compositeKey } from "../../domain/unicodeCodePointOrder.js"; import { completeApplicationCoverage, @@ -99,6 +100,8 @@ export interface UnavailableParseInput { readonly asset: ApplicationNode; readonly operation: string; readonly limitation: string; + readonly failure?: JsonValue; + readonly limits?: JavaScriptArtifactGraphCoverage["limits"]; } /** Add a direct artifact-containment edge. */ @@ -182,12 +185,15 @@ export const addUnavailableStaticParseScope = ( path: input.file.path, operation: input.operation, limitation: input.limitation, + ...(input.failure === undefined + ? {} + : { analysis_failure: input.failure }), }, evidence: unavailableAstEvidence({ sha256: input.file.sha256, path: input.file.path, operation: input.operation, - coverage: partialApplicationCoverage([], null), + coverage: partialApplicationCoverage(input.limits ?? [], null), limitation: input.limitation, }), }, diff --git a/src/application/javascript/JavaScriptArtifactGraphStructure.ts b/src/application/javascript/JavaScriptArtifactGraphStructure.ts index ce171beb9..112a5f8b6 100644 --- a/src/application/javascript/JavaScriptArtifactGraphStructure.ts +++ b/src/application/javascript/JavaScriptArtifactGraphStructure.ts @@ -2,6 +2,9 @@ import type { ArtifactInventorySnapshot } from "../../domain/artifactInventorySn import type { ApplicationNode } from "../../domain/javascript/javascriptApplicationGraphSchemas.js"; import { completeApplicationCoverage } from "../../domain/javascript/javascriptApplicationEvidenceSchemas.js"; import type { JavaScriptModuleArtifactAnalysis } from "./JavaScriptArtifactAnalysisTypes.js"; +import { jsonValueSchema } from "../../domain/jsonValue.js"; +import { freezeJsonSnapshot } from "../../domain/immutableJson.js"; +import type { JavaScriptSourceRange } from "../../domain/javascript/javascriptStaticAnalysisTypes.js"; import type { JavaScriptArtifactContainer, JavaScriptArtifactFile, @@ -12,12 +15,14 @@ import { addUnavailableStaticParseScope, artifactFileNodeKind, artifactLocalIdentity, + javascriptAnalysisCoverage, createElectronRoleNode, linkElectronRoleToAsset, type JavaScriptArtifactGraphContext, } from "./JavaScriptArtifactGraphContext.js"; import { artifactObservationEvidence, + astObservationEvidence, staticInferenceEvidence, } from "./JavaScriptArtifactGraphEvidence.js"; import { resolveArtifactPathByContext } from "./JavaScriptArtifactPathResolution.js"; @@ -127,6 +132,40 @@ export const addJavaScriptArtifactFiles = ( analyzed.javascript, jsonValue, ); + if ( + analyzed.application_projection_failure !== undefined && + analyzed.javascript !== null + ) + context.accumulator.addNode({ + kind: target.kind, + identity: target.identity, + observations: [ + { + label: file.path, + properties: { + static_analysis: freezeJsonSnapshot( + jsonValueSchema.parse(analyzed.javascript), + ), + semantic_module_analysis: + analyzed.semantic === null + ? null + : freezeJsonSnapshot( + jsonValueSchema.parse(analyzed.semantic.ir), + ), + application_projection_failure: + analyzed.application_projection_failure.error, + }, + evidence: astObservationEvidence({ + sha256: file.sha256, + path: file.path, + operation: "retain-unexpanded-static-findings", + range: sourceFileRange(file), + coverage: javascriptAnalysisCoverage(analyzed.javascript), + limitations: [analyzed.application_projection_failure.reason], + }), + }, + ], + }); context.fileNodes.set(file.path, target); if (file.kind === "javascript") context.assetNodes.set(file.path, target); const entry = createAsarEntry(context, file); @@ -159,7 +198,18 @@ export const addJavaScriptArtifactFiles = ( }), }); } - if ( + if (analyzed.analysis_failure !== undefined) + addUnavailableStaticParseScope(context, { + file, + asset: target, + operation: "analyze-javascript-source", + limitation: analyzed.analysis_failure.reason, + failure: analyzed.analysis_failure.error, + limits: javaScriptAnalysisCoverageLimits( + analyzed.analysis_failure.limits, + ), + }); + else if ( file.kind === "javascript" && (analyzed.javascript === null || analyzed.javascript.parse_status === "failed") @@ -172,6 +222,17 @@ export const addJavaScriptArtifactFiles = ( ? "JavaScript syntax could not be parsed." : `JavaScript text was unavailable: ${file.text.reason}.`, }); + if (analyzed.application_projection_failure !== undefined) + addUnavailableStaticParseScope(context, { + file, + asset: target, + operation: "project-javascript-application", + limitation: analyzed.application_projection_failure.reason, + failure: analyzed.application_projection_failure.error, + limits: javaScriptAnalysisCoverageLimits( + analyzed.application_projection_failure.limits, + ), + }); const packageValue = packagesByPath.get(file.path); if (packageValue !== undefined && packageValue.status !== "included") addUnavailableStaticParseScope(context, { @@ -198,6 +259,23 @@ export const addJavaScriptArtifactFiles = ( } }; +const sourceFileRange = ( + file: JavaScriptArtifactFile, +): JavaScriptSourceRange => { + if (!file.text.included) + throw new TypeError("Retained static facts require their source text"); + let line = 1; + let lastLineStart = 0; + for (const match of file.text.value.matchAll(/\r\n|[\n\r\u2028\u2029]/gu)) { + line += 1; + lastLineStart = match.index + match[0].length; + } + return { + start: { line: 1, column: 0 }, + end: { line, column: file.text.value.length - lastLineStart }, + }; +}; + /** Project package metadata and its declared Electron roles. */ export const addJavaScriptPackageNodes = ( context: JavaScriptArtifactGraphContext, @@ -407,3 +485,4 @@ const containerProperties = (container: JavaScriptArtifactContainer) => ({ bytes: container.bytes, inventory_artifact_id: container.inventory_artifact_id, }); +import { javaScriptAnalysisCoverageLimits } from "../../domain/javascript/javascriptAnalysisResourceControls.js"; diff --git a/src/application/javascript/JavaScriptArtifactReconstruction.ts b/src/application/javascript/JavaScriptArtifactReconstruction.ts index 00253da3f..cd5b6c501 100644 --- a/src/application/javascript/JavaScriptArtifactReconstruction.ts +++ b/src/application/javascript/JavaScriptArtifactReconstruction.ts @@ -11,9 +11,15 @@ import { createJavaScriptArtifactReader as createReader } from "../../artifacts/ import type { JavaScriptApplicationGraph } from "../../domain/javascript/javascriptApplicationGraph.js"; import type { JavaScriptSemanticGraph } from "../../domain/javascript/javascriptSemanticGraph.js"; import type { ElectronBoundarySummary } from "../../domain/javascript/javascriptApplicationAnalysis.js"; +import { parseOwnedJavaScriptApplicationAnalysisSteps } from "../../domain/javascript/javascriptApplicationAnalysis.js"; +import { AnalysisError } from "../../domain/analysisErrorBase.js"; import type { ArtifactInventorySnapshot } from "../../domain/artifactInventorySnapshot.js"; import type { ArtifactInventoryPartialObservation } from "../../domain/artifactPartialObservation.js"; -import { analyzeAndProjectJavaScriptArtifactFiles } from "./JavaScriptArtifactAnalysis.js"; +import type { ArtifactResourceScope } from "../../artifacts/ArtifactResourceScope.js"; +import { analyzeAndAdoptJavaScriptArtifactFiles } from "./JavaScriptArtifactAnalysis.js"; +import { createJavaScriptSourceAnalysis } from "../../artifacts/javascript/JavaScriptSourceAnalysis.js"; +import { JAVASCRIPT_ANALYSIS_RESOURCE_DEFAULTS } from "../../domain/javascript/javascriptAnalysisResourceControls.js"; +import { createOwnedJavaScriptApplicationEvidenceCooperatively } from "./JavaScriptApplicationEvidence.js"; import { readJavaScriptArtifactFiles } from "../../artifacts/javascript/JavaScriptArtifactFiles.js"; import { buildImmutableJavaScriptArtifactGraphSteps } from "./JavaScriptArtifactGraphBuilder.js"; import { @@ -21,7 +27,6 @@ import { type JavaScriptArtifactReconstructionInput, } from "./JavaScriptArtifactReconstructionInput.js"; import { scanCanonicalArtifactInventoryInScope } from "../../artifacts/inventory/scanCanonical.js"; -import type { ArtifactResourceScope } from "../../artifacts/ArtifactResourceScope.js"; import { summarizeElectronBoundaries } from "./ElectronBoundaryAnalysis.js"; import { createJavaScriptSemanticGraphProjection } from "./JavaScriptSemanticGraphBuilder.js"; import type { ProgressReporter } from "../ProgressReporter.js"; @@ -63,12 +68,19 @@ export const reconstructJavaScriptArtifact = async ( progress?: ProgressReporter, ): Promise => scope.run(async () => { + let effectiveSignal = signal; + let effectiveProgress = progress; const reportPhase = async ( phase: string, message: string, ): Promise => { - await progress?.report({ phase, completed: 0, total: 1, message }); - await checkpointJavaScriptAnalysis(signal); + await effectiveProgress?.report({ + phase, + completed: 0, + total: 1, + message, + }); + await checkpointJavaScriptAnalysis(effectiveSignal); }; const input = javascriptArtifactReconstructionInputSchema.parse(rawInput); abortIfNeeded(signal); @@ -85,7 +97,13 @@ export const reconstructJavaScriptArtifact = async ( `Artifact inventory classified ${path} as ${snapshot.manifest.root_format}, not ${format}`, ); const reader = createReader(path, format); - let outcome: + const readerOwner = { + kind: "reader" as const, + reader, + resource: `JavaScript artifact reader for ${path}`, + }; + let readerOpen = true; + let reconstructionOutcome: | { readonly kind: "completed"; readonly result: JavaScriptArtifactReconstructionResult; @@ -102,28 +120,75 @@ export const reconstructJavaScriptArtifact = async ( scope, signal, ); + // Release acquisition before analysis consumes these owned file facts. + const earlyCleanup = await scope.release(readerOwner); + readerOpen = false; + if (earlyCleanup.kind === "failed") + throw ArtifactReaderFailure.withCleanup( + earlyCleanup.cause, + ArtifactReaderFailure.cleanupObservation( + earlyCleanup.cause, + readerOwner.resource, + ), + { kind: "artifact-inventory", inventory: snapshot }, + ); await reportPhase( "parse_javascript_sources", `Parsing and projecting ${String(files.files.length)} application source files`, ); const semanticProjection = createJavaScriptSemanticGraphProjection(); - const analysis = await analyzeAndProjectJavaScriptArtifactFiles( - files, - semanticProjection.projectFileSteps, - async (file, completed, total) => { - abortIfNeeded(signal); - await progress?.report({ - phase: "parse_javascript_source", - completed: 0, - total: 1, - message: `Parsing and projecting ${file.path} (${String(completed + 1)}/${String(total)})`, - }); - abortIfNeeded(signal); + const worker = createJavaScriptSourceAnalysis( + { + heapMb: + input.max_heap_mb ?? JAVASCRIPT_ANALYSIS_RESOURCE_DEFAULTS.heapMb, + timeoutMs: + input.analysis_timeout_ms ?? + JAVASCRIPT_ANALYSIS_RESOURCE_DEFAULTS.timeoutMs, }, - signal, + files.files.filter( + (file) => file.kind === "javascript" && file.text.included, + ).length, ); - await checkpointJavaScriptAnalysis(signal); - abortIfNeeded(signal); + let outcome; + let cleanupFailure: AnalysisError | undefined; + try { + outcome = await analyzeAndAdoptJavaScriptArtifactFiles( + files, + worker, + semanticProjection, + async (file, completed, total) => { + abortIfNeeded(signal); + await progress?.report({ + phase: "parse_javascript_source", + completed: 0, + total: 1, + message: `Parsing and projecting ${file.path} (${String(completed + 1)}/${String(total)})`, + }); + abortIfNeeded(signal); + }, + signal, + ); + } finally { + try { + const cleanup = await scope.release({ + kind: "javascript-source-analysis", + analysis: worker, + resource: `JavaScript source analyzer for ${path}`, + }); + if (cleanup.kind === "failed") throw cleanup.cause; + } catch (cause: unknown) { + if (!(cause instanceof AnalysisError)) throw cause; + cleanupFailure = cause; + } + } + const analysis = outcome.analysis; + const interruption = cleanupFailure ?? outcome.interruption; + if (interruption !== null && interruption !== undefined) { + effectiveSignal = undefined; + effectiveProgress = undefined; + } + await checkpointJavaScriptAnalysis(effectiveSignal); + abortIfNeeded(effectiveSignal); await reportPhase( "build_javascript_application_graph", "Constructing application and Electron boundary relationships", @@ -139,7 +204,7 @@ export const reconstructJavaScriptArtifact = async ( ); const graph = await completeJavaScriptAnalysisSteps( applicationGraphSteps, - signal, + effectiveSignal, ); await reportPhase( "build_javascript_semantic_graph", @@ -155,63 +220,83 @@ export const reconstructJavaScriptArtifact = async ( ); const semanticGraph = await completeJavaScriptAnalysisSteps( semanticGraphSteps, - signal, + effectiveSignal, ); - await checkpointJavaScriptAnalysis(signal); - outcome = { - kind: "completed", - result: { - input_path: path, - format, - root_artifact_sha256: snapshot.manifest.root_sha256, - inventory_manifest_id: snapshot.manifest.manifest_id, - inventory_graph_sha256: snapshot.manifest.graph_sha256, - integrity_contradictions: snapshot.integrity_contradictions, - graph, - semantic_graph: semanticGraph, - electron_summary: summarizeElectronBoundaries(analysis), - statistics: { - relevant_files: files.files.length, - nested_asar_containers: files.containers.length, - text_bytes_read: files.text_bytes_read, - invalid_utf8_files: files.invalid_utf8_files, - parsed_javascript_files: analysis.files.filter( - ({ javascript }) => javascript !== null, - ).length, - visited_ast_nodes: analysis.visited_ast_nodes, - findings: analysis.findings, - modules: analysis.modules, - parse_failures: analysis.parse_failures, - }, - limitations: analysis.limitations, + await checkpointJavaScriptAnalysis(effectiveSignal); + const completed = { + input_path: path, + format, + root_artifact_sha256: snapshot.manifest.root_sha256, + inventory_manifest_id: snapshot.manifest.manifest_id, + inventory_graph_sha256: snapshot.manifest.graph_sha256, + integrity_contradictions: snapshot.integrity_contradictions, + graph, + semantic_graph: semanticGraph, + electron_summary: summarizeElectronBoundaries(analysis), + statistics: { + relevant_files: files.files.length, + nested_asar_containers: files.containers.length, + text_bytes_read: files.text_bytes_read, + invalid_utf8_files: files.invalid_utf8_files, + parsed_javascript_files: analysis.files.filter( + ({ javascript }) => javascript !== null, + ).length, + visited_ast_nodes: analysis.visited_ast_nodes, + findings: analysis.findings, + modules: analysis.modules, + parse_failures: analysis.parse_failures, }, + limitations: analysis.limitations, }; + if (interruption !== null && interruption !== undefined) { + const { electron_summary: summary, ...application } = completed; + const partial = await completeJavaScriptAnalysisSteps( + parseOwnedJavaScriptApplicationAnalysisSteps({ + ...application, + summary, + limitations: completed.graph.limitations, + }), + ); + const { max_heap_mb, analysis_timeout_ms, ...coreInput } = input; + const parameters = { + ...coreInput, + ...(max_heap_mb === undefined ? {} : { max_heap_mb }), + ...(analysis_timeout_ms === undefined ? {} : { analysis_timeout_ms }), + }; + const evidence = + await createOwnedJavaScriptApplicationEvidenceCooperatively( + parameters, + partial, + ); + throw interruption.retainPartialObservation(evidence); + } + reconstructionOutcome = { kind: "completed", result: completed }; } catch (cause: unknown) { - outcome = { kind: "failed", cause }; + reconstructionOutcome = { kind: "failed", cause }; } - const owner = { - kind: "reader" as const, - reader, - resource: `JavaScript artifact reader for ${path}`, - }; - const cleanupAttempt = await scope.release(owner); - if (cleanupAttempt.kind === "failed") { - const cleanup = ArtifactReaderFailure.cleanupObservation( - cleanupAttempt.cause, - owner.resource, - ); - const partialObservation: ArtifactInventoryPartialObservation = { - kind: "artifact-inventory", - inventory: snapshot, - }; - throw ArtifactReaderFailure.withCleanup( - outcome.kind === "failed" ? outcome.cause : cleanupAttempt.cause, - cleanup, - partialObservation, - ); + if (readerOpen) { + const cleanupAttempt = await scope.release(readerOwner); + if (cleanupAttempt.kind === "failed") { + const cleanup = ArtifactReaderFailure.cleanupObservation( + cleanupAttempt.cause, + readerOwner.resource, + ); + const partialObservation: ArtifactInventoryPartialObservation = { + kind: "artifact-inventory", + inventory: snapshot, + }; + throw ArtifactReaderFailure.withCleanup( + reconstructionOutcome.kind === "failed" + ? reconstructionOutcome.cause + : cleanupAttempt.cause, + cleanup, + partialObservation, + ); + } } - if (outcome.kind === "failed") throw outcome.cause; - return outcome.result; + if (reconstructionOutcome.kind === "failed") + throw reconstructionOutcome.cause; + return reconstructionOutcome.result; }); const OPERATION = "analyze_javascript_application"; diff --git a/src/application/javascript/JavaScriptArtifactReconstructionInput.ts b/src/application/javascript/JavaScriptArtifactReconstructionInput.ts index 9c72fd36f..f2babb47a 100644 --- a/src/application/javascript/JavaScriptArtifactReconstructionInput.ts +++ b/src/application/javascript/JavaScriptArtifactReconstructionInput.ts @@ -1,11 +1,13 @@ import { z } from "zod"; import { artifactIntegrityPolicySchema } from "../../domain/artifactIntegrityPolicy.js"; +import { javaScriptAnalysisResourceControlsSchema } from "../../domain/javascript/javascriptAnalysisResourceControls.js"; /** Local ASAR/directory reconstruction request. */ export const javascriptArtifactReconstructionInputSchema = z.strictObject({ input_path: z.string().min(1), format: z.enum(["auto", "asar", "directory"]).default("auto"), integrity_policy: artifactIntegrityPolicySchema, + ...javaScriptAnalysisResourceControlsSchema.shape, }); /** Parsed local reconstruction request. */ diff --git a/src/application/javascript/JavaScriptSemanticGraphBuilder.ts b/src/application/javascript/JavaScriptSemanticGraphBuilder.ts index 3ed0cafda..374b0cb3b 100644 --- a/src/application/javascript/JavaScriptSemanticGraphBuilder.ts +++ b/src/application/javascript/JavaScriptSemanticGraphBuilder.ts @@ -1,5 +1,6 @@ import { createJavaScriptSemanticGraph, + createJavaScriptSemanticGraphUnknown, sealTransferredJavaScriptSemanticGraphSteps, type JavaScriptSemanticGraph, } from "../../domain/javascript/javascriptSemanticGraph.js"; @@ -10,6 +11,7 @@ import type { import { JAVASCRIPT_SEMANTIC_RELATION_FAMILIES, JAVASCRIPT_SEMANTIC_RELATION_FAMILY, + JAVASCRIPT_SEMANTIC_RELATIONS, } from "../../domain/javascript/javascriptSemanticGraphSchemas.js"; import type { JavaScriptSemanticCallArgument, @@ -33,6 +35,13 @@ import { projectSemanticResources } from "./JavaScriptSemanticGraphResourceProje import { projectSemanticObjects } from "./JavaScriptSemanticGraphObjectProjection.js"; import { projectSemanticValues } from "./JavaScriptSemanticGraphValueProjection.js"; import { inferredSemanticEvidenceAt } from "./JavaScriptSemanticGraphEvidence.js"; +import { observedSemanticEvidence } from "./JavaScriptSemanticGraphEvidence.js"; +import { + partialApplicationCoverage, + type ApplicationCoverage, +} from "../../domain/javascript/javascriptApplicationEvidenceSchemas.js"; +import type { JsonValue } from "../../domain/jsonValue.js"; +import { javaScriptAnalysisCoverageLimits } from "../../domain/javascript/javascriptAnalysisResourceControls.js"; import { projectSemanticFunctionFingerprints } from "./JavaScriptSemanticGraphFingerprintProjection.js"; import { projectSemanticClosureCaptures, @@ -55,6 +64,7 @@ import { bindSemanticGraphApplicationNodes, type SemanticGraphProjectionState as BuilderState, } from "./JavaScriptSemanticGraphConstruction.js"; +import type { JavaScriptSemanticFileProjection } from "../../domain/javascript/javascriptSemanticFileProjection.js"; interface BuilderInput { readonly rootArtifactSha256: string; @@ -126,6 +136,19 @@ export const buildJavaScriptSemanticGraph = ({ /** File-local semantic projection that does not retain consumed source IR. */ export interface JavaScriptSemanticGraphProjection { + /** Node capacity offered to the next independently analyzed source file. */ + readonly remainingFileNodeBudget: () => number; + /** Adopt decoded file-local facts; final validation and proof remain in this realm. */ + readonly adoptFileProjection: ( + projection: JavaScriptSemanticFileProjection, + ) => void; + /** Preserve source-bound unknowns for an execution or result-retention failure. */ + readonly recordUnavailableFile: ( + file: JavaScriptArtifactFile, + reason: string, + limits: Readonly> | null, + unknownReason: "resource-limit" | "incomplete-module", + ) => void; readonly projectFile: ( file: JavaScriptArtifactFile, ir: JavaScriptSemanticIr, @@ -151,6 +174,12 @@ export const createJavaScriptSemanticGraphProjection = const state = emptyState(); const fingerprints: JavaScriptSemanticFingerprint[] = []; let truncatedFiles = 0; + let unavailableFiles = 0; + let resourceTruncatedFiles = 0; + const executionLimits = new Map< + string, + ApplicationCoverage["limits"][number] + >(); const projectSourceSteps = function* ( file: JavaScriptArtifactFile, ir: JavaScriptSemanticIr, @@ -207,12 +236,15 @@ export const createJavaScriptSemanticGraphProjection = fingerprints: [...fingerprints], unknowns, coverage: { - status: truncatedFiles > 0 ? "partial" : "unknown", - truncated: truncatedFiles > 0, - omitted_nodes: truncatedFiles > 0 ? null : 0, - omitted_relations: truncatedFiles > 0 ? null : 0, - limits: - truncatedFiles > 0 + status: + truncatedFiles > 0 || unavailableFiles > 0 ? "partial" : "unknown", + truncated: truncatedFiles > 0 || resourceTruncatedFiles > 0, + omitted_nodes: truncatedFiles > 0 || unavailableFiles > 0 ? null : 0, + omitted_relations: + truncatedFiles > 0 || unavailableFiles > 0 ? null : 0, + limits: [ + ...executionLimits.values(), + ...(truncatedFiles > 0 ? [ { name: "semantic_graph_node_ceiling", @@ -220,12 +252,14 @@ export const createJavaScriptSemanticGraphProjection = unit: "items" as const, }, ] - : [], + : []), + ], families: JAVASCRIPT_SEMANTIC_RELATION_FAMILIES.map((family) => ({ family, status: semanticFamilyStatus(family), retained_relations: retainedByFamily.get(family) ?? 0, - omitted_relations: truncatedFiles > 0 ? null : 0, + omitted_relations: + truncatedFiles > 0 || unavailableFiles > 0 ? null : 0, unknown_ids: unknownIdsByFamily.get(family) ?? [], })), }, @@ -252,6 +286,87 @@ export const createJavaScriptSemanticGraphProjection = return graph; }; return { + remainingFileNodeBudget: () => + Math.min( + SEMANTIC_GRAPH_FILE_NODE_CEILING, + Math.max(0, SEMANTIC_GRAPH_NODE_CEILING - state.nodes.size), + ), + adoptFileProjection: (projection) => { + const newNodes = projection.nodes.filter( + ({ node_id }) => !state.nodes.has(node_id), + ); + if ( + newNodes.length > + Math.min( + SEMANTIC_GRAPH_FILE_NODE_CEILING, + SEMANTIC_GRAPH_NODE_CEILING - state.nodes.size, + ) + ) + throw new RangeError( + "Transferred semantic nodes exceed the offered tree capacity", + ); + for (const context of projection.evidenceContexts) + state.evidenceContexts.adopt(context); + for (const node of newNodes) state.nodes.set(node.node_id, node); + for (const relation of projection.relations) + state.relations.set(relation.relation_id, relation); + for (const unknown of projection.unknowns) + state.unknowns.set(unknown.unknown_id, unknown); + for (const root of projection.roots) state.roots.add(root); + fingerprints.push(...projection.fingerprints); + if (projection.truncated) truncatedFiles += 1; + }, + recordUnavailableFile: (file, reason, limits, unknownReason) => { + const retainedLimits = javaScriptAnalysisCoverageLimits(limits); + for (const limit of retainedLimits) { + executionLimits.set( + JSON.stringify([limit.name, limit.value, limit.unit]), + limit, + ); + } + state.fileNodeBudget = Math.min( + SEMANTIC_GRAPH_FILE_NODE_CEILING, + Math.max(0, SEMANTIC_GRAPH_NODE_CEILING - state.nodes.size), + ); + const module = retainNode(state, file, { + kind: "module", + roleKey: "module", + location: null, + label: file.path, + functionNodeId: null, + }); + state.fileNodeBudget = null; + if (module !== null) state.roots.add(module.node_id); + const evidence = { + ...observedSemanticEvidence(file, null), + authority: "unknown" as const, + state: "unknown" as const, + confidence: "unknown" as const, + coverage: partialApplicationCoverage(retainedLimits, null), + limitations: [reason], + }; + for (const family of JAVASCRIPT_SEMANTIC_RELATION_FAMILIES) { + const unknown = createJavaScriptSemanticGraphUnknown( + { + node_id: module?.node_id ?? null, + family, + relation_kinds: JAVASCRIPT_SEMANTIC_RELATIONS.filter( + (relation) => + JAVASCRIPT_SEMANTIC_RELATION_FAMILY[relation] === family, + ), + reason: unknownReason, + detail: reason, + candidate_node_ids: [], + evidence, + }, + state.evidenceContexts, + ); + state.unknowns.set(unknown.unknown_id, unknown); + } + unavailableFiles += 1; + if (unknownReason === "resource-limit" && retainedLimits.length > 0) + resourceTruncatedFiles += 1; + }, projectFile: (file, ir) => { const steps = projectSourceSteps(file, ir); while (steps.next().done !== true); @@ -272,6 +387,35 @@ export const createJavaScriptSemanticGraphProjection = }; }; +/** Project one source with the parent's remaining node capacity, retaining no full IR. */ +export function* projectJavaScriptSemanticFileSteps( + file: JavaScriptArtifactFile, + ir: JavaScriptSemanticIr, + nodeBudget: number, +): Generator { + if ( + !Number.isSafeInteger(nodeBudget) || + nodeBudget < 0 || + nodeBudget > SEMANTIC_GRAPH_FILE_NODE_CEILING + ) + throw new RangeError( + "File semantic projection requires a valid offered node capacity", + ); + const state = emptyState(); + state.fileNodeBudget = nodeBudget; + state.fileNodesDropped = false; + const fingerprints = yield* projectFileSteps(file, ir, state); + return { + roots: [...state.roots], + evidenceContexts: state.evidenceContexts.contexts, + nodes: [...state.nodes.values()], + relations: [...state.relations.values()], + unknowns: [...state.unknowns.values()], + fingerprints, + truncated: Boolean(state.fileNodesDropped), + }; +} + function* projectFileSteps( file: JavaScriptArtifactFile, ir: JavaScriptSemanticIr, diff --git a/src/artifacts/ArtifactResourceScope.test.ts b/src/artifacts/ArtifactResourceScope.test.ts index 9bf8fe87b..395f113b3 100644 --- a/src/artifacts/ArtifactResourceScope.test.ts +++ b/src/artifacts/ArtifactResourceScope.test.ts @@ -1,5 +1,7 @@ import { expect, it } from "vitest"; +import { err } from "../domain/result.js"; +import { AnalysisOutputError } from "../domain/analysisErrorCore.js"; import type { ArtifactReader } from "./ArtifactReader.js"; import { ArtifactResourceScope } from "./ArtifactResourceScope.js"; @@ -66,3 +68,47 @@ it("retains the same failed reader owner and retries before admitting work", asy await scope.close(); expect(closeAttempts).toBe(4); }); + +it("retains a failed source analyzer until shutdown can verify its release", async () => { + const scope = new ArtifactResourceScope(); + let releaseAllowed = false; + const failure = new AnalysisOutputError( + "analyze_javascript_application", + "Owned worker is still running", + ); + const owner = { + kind: "javascript-source-analysis" as const, + resource: "fixture source analyzer", + analysis: { + async analyze() { + return err({ + error: failure, + javascript: null, + module: null, + projection: null, + }); + }, + async close() { + if (!releaseAllowed) throw failure; + }, + }, + }; + expect(await scope.release(owner)).toEqual({ + kind: "failed", + cause: failure, + }); + let admitted = false; + await expect( + scope.run(async () => { + admitted = true; + }), + ).rejects.toMatchObject({ + cleanup: { resources: ["fixture source analyzer"] }, + }); + expect(admitted).toBe(false); + await expect(scope.close()).rejects.toMatchObject({ + cleanup: { resources: ["fixture source analyzer"] }, + }); + releaseAllowed = true; + await scope.close(); +}); diff --git a/src/artifacts/ArtifactResourceScope.ts b/src/artifacts/ArtifactResourceScope.ts index 4aa7b9e37..1f06ed254 100644 --- a/src/artifacts/ArtifactResourceScope.ts +++ b/src/artifacts/ArtifactResourceScope.ts @@ -1,3 +1,4 @@ +import type { JavaScriptSourceAnalysisPort } from "../domain/javascript/javascriptSourceAnalysis.js"; import type { OwnedFileHandle } from "../filesystem/OwnedFileHandle.js"; import type { OwnedDirectoryHandle } from "../filesystem/OwnedDirectoryHandle.js"; @@ -7,6 +8,11 @@ import type { SafeOutputTree } from "./SafeOutputTree.js"; /** One artifact resource and its format-specific cleanup owner. */ export type ArtifactResourceOwner = + | { + readonly kind: "javascript-source-analysis"; + readonly resource: string; + readonly analysis: JavaScriptSourceAnalysisPort; + } | { readonly kind: "directory-handle"; readonly resource: string; @@ -134,6 +140,9 @@ export class ArtifactResourceScope { const cleanup = async (owner: ArtifactResourceOwner): Promise => { switch (owner.kind) { + case "javascript-source-analysis": + await owner.analysis.close(); + return; case "reader": await owner.reader.close(); return; diff --git a/src/artifacts/javascript/JavaScriptSourceAnalysis.ts b/src/artifacts/javascript/JavaScriptSourceAnalysis.ts new file mode 100644 index 000000000..ac37fc75f --- /dev/null +++ b/src/artifacts/javascript/JavaScriptSourceAnalysis.ts @@ -0,0 +1,12 @@ +import type { + JavaScriptSourceAnalysisLimits, + JavaScriptSourceAnalysisPort, +} from "../../domain/javascript/javascriptSourceAnalysis.js"; +import { JavaScriptAnalysisWorker } from "../../javascript/analysis/JavaScriptAnalysisWorker.js"; + +/** Construct the inert artifact analyzer; resources are acquired on the first source. */ +export const createJavaScriptSourceAnalysis = ( + limits: JavaScriptSourceAnalysisLimits, + sourceCount: number, +): JavaScriptSourceAnalysisPort => + new JavaScriptAnalysisWorker(limits, undefined, sourceCount); diff --git a/src/cli/electronCommands.ts b/src/cli/electronCommands.ts index 93f99163a..e570cbfc5 100644 --- a/src/cli/electronCommands.ts +++ b/src/cli/electronCommands.ts @@ -198,6 +198,12 @@ const registerJavaScriptApplicationCommand = ( input_path: args.path, format: options.artifactFormat, integrity_policy: options.integrityPolicy, + ...(options.maxHeapMb === undefined + ? {} + : { max_heap_mb: options.maxHeapMb }), + ...(options.analysisTimeoutMs === undefined + ? {} + : { analysis_timeout_ms: options.analysisTimeoutMs }), }, signal, ), diff --git a/src/cliObservationOptions.ts b/src/cliObservationOptions.ts index cdb1b1f0e..4733668a2 100644 --- a/src/cliObservationOptions.ts +++ b/src/cliObservationOptions.ts @@ -1,4 +1,5 @@ import { z } from "incur"; +import { javaScriptAnalysisResourceControlsSchema } from "./domain/javascript/javascriptAnalysisResourceControls.js"; /** Accept an agent-selected browser observation duration without an artificial ceiling. */ export const observationDuration = (fallback: number, minimum = 0) => @@ -59,6 +60,9 @@ export const electronPageInspectionOptions = z.object({ }); export const javascriptApplicationOptions = z.object({ + maxHeapMb: javaScriptAnalysisResourceControlsSchema.shape.max_heap_mb, + analysisTimeoutMs: + javaScriptAnalysisResourceControlsSchema.shape.analysis_timeout_ms, artifactFormat: z .enum(["auto", "asar", "directory"]) .default("auto") diff --git a/src/composition/javascriptAnalysisWorkerProcess.ts b/src/composition/javascriptAnalysisWorkerProcess.ts new file mode 100644 index 000000000..3b94f4677 --- /dev/null +++ b/src/composition/javascriptAnalysisWorkerProcess.ts @@ -0,0 +1,82 @@ +import { parseJavaScriptSource } from "../domain/javascript/javascriptSourceParser.js"; +import { analyzeParsedJavaScriptStaticSourceSteps } from "../domain/javascript/javascriptStaticAnalysis.js"; +import { + analyzeParsedJavaScriptSemanticsSteps, + classifyParsedJavaScriptOpenReceiversSteps, +} from "../domain/javascript/javascriptSemanticAnalysis.js"; +import { failedJavaScriptStaticAnalysis } from "../domain/javascript/javascriptStaticAnalysisHelpers.js"; +import { projectJavaScriptModuleSemantics } from "../domain/javascript/javascriptModuleSemanticIr.js"; +import { measureJavaScriptModuleTransfer } from "../javascript/analysis/JavaScriptAnalysisTransfer.js"; +import { AnalysisResourceConstraintError } from "../domain/analysisErrorCore.js"; +import { projectJavaScriptSemanticFileSteps } from "../application/javascript/JavaScriptSemanticGraphBuilder.js"; +import { completeJavaScriptAnalysisSteps } from "../application/javascript/JavaScriptAnalysisControl.js"; +import { + serveJavaScriptAnalysisWorker, + type JavaScriptSourceWorkerAnalyzer, +} from "../javascript/analysis/JavaScriptAnalysisWorkerServer.js"; + +const analyze: JavaScriptSourceWorkerAnalyzer = async ( + file, + nodeBudget, + callbacks, +) => { + if (!file.text.included) + throw new TypeError("JavaScript worker requires decoded source text"); + await callbacks.phase("parse"); + let parsed = parseJavaScriptSource(file.text.value, file.path); + if (parsed === null) { + await callbacks.checkpoint(failedJavaScriptStaticAnalysis()); + return null; + } + await callbacks.phase("static"); + const receivers = await completeJavaScriptAnalysisSteps( + classifyParsedJavaScriptOpenReceiversSteps(parsed), + ); + const facts = await completeJavaScriptAnalysisSteps( + analyzeParsedJavaScriptStaticSourceSteps( + file.text.value, + parsed, + receivers, + ), + ); + const resultBudgetBytes = await callbacks.checkpoint(facts); + if (facts.parse_status === "failed") return null; + if (resultBudgetBytes === 0) + throw new AnalysisResourceConstraintError( + "analyze_javascript_application", + "memory", + `The parent reserved its remaining result capacity for static observations; semantic analysis of ${file.path} was not started`, + { semantic_result_budget_bytes: 0 }, + ); + await callbacks.phase("semantic"); + const ir = await completeJavaScriptAnalysisSteps( + analyzeParsedJavaScriptSemanticsSteps(parsed), + ); + parsed = null; + const module = projectJavaScriptModuleSemantics(ir); + const measured = measureJavaScriptModuleTransfer(module); + if (measured.retentionBytes > resultBudgetBytes) + throw new AnalysisResourceConstraintError( + "analyze_javascript_application", + "memory", + `Completed module facts for ${file.path} exceed the parent's offered result budget; static facts were retained`, + { + semantic_result_budget_bytes: resultBudgetBytes, + module_transfer_bytes: measured.bytes, + }, + ); + await callbacks.phase("projection"); + const projection = await completeJavaScriptAnalysisSteps( + projectJavaScriptSemanticFileSteps(file, ir, nodeBudget), + ); + return { module, projection }; +}; + +try { + await serveJavaScriptAnalysisWorker(analyze); +} catch (cause: unknown) { + process.stderr.write( + `${cause instanceof Error ? cause.message : String(cause)}\n`, + ); + process.exitCode = 1; +} diff --git a/src/domain/analysisErrorProjection.ts b/src/domain/analysisErrorProjection.ts index 4eaac5176..be7b431b6 100644 --- a/src/domain/analysisErrorProjection.ts +++ b/src/domain/analysisErrorProjection.ts @@ -1,4 +1,5 @@ import { jsonValueSchema, type JsonValue } from "./jsonValue.js"; +import { isImmutableJsonSnapshot } from "./immutableJson.js"; import { analysisErrorCategory, @@ -71,7 +72,9 @@ export const projectAnalysisError = ( ...(error.partialObservation === undefined ? {} : { - partial_observation: jsonValueSchema.parse(error.partialObservation), + partial_observation: isImmutableJsonSnapshot(error.partialObservation) + ? error.partialObservation + : jsonValueSchema.parse(error.partialObservation), }), }; return { diff --git a/src/domain/javascript/javascriptAnalysisResourceControls.ts b/src/domain/javascript/javascriptAnalysisResourceControls.ts new file mode 100644 index 000000000..56886457b --- /dev/null +++ b/src/domain/javascript/javascriptAnalysisResourceControls.ts @@ -0,0 +1,56 @@ +import { z } from "zod"; +import type { JsonValue } from "../jsonValue.js"; +import type { ApplicationCoverage } from "./javascriptApplicationEvidenceSchemas.js"; + +/** Shared operational defaults for the isolated source-analysis boundary. */ +export const JAVASCRIPT_ANALYSIS_RESOURCE_DEFAULTS = { + heapMb: 1024, + timeoutMs: 300_000, +} as const; + +/** Project observed execution limits without inventing unmeasured values. */ +export const javaScriptAnalysisCoverageLimits = ( + limits: Readonly> | null, +): ApplicationCoverage["limits"] => { + const result: ApplicationCoverage["limits"] = []; + for (const key of [ + "worker_heap_limit_bytes", + "worker_timeout_ms", + "parent_result_budget_bytes", + "semantic_result_budget_bytes", + "transfer_record_bytes", + "transfer_file_bytes", + ]) { + const value = limits?.[key]; + if (typeof value !== "number" || !Number.isSafeInteger(value) || value < 0) + continue; + result.push({ + name: `javascript_analysis_${key}`, + value, + unit: key.endsWith("_ms") ? "milliseconds" : "bytes", + }); + } + return result; +}; + +/** Optional operational choices; omission keeps them out of Evidence parameters. */ +export const javaScriptAnalysisResourceControlsSchema = z.strictObject({ + max_heap_mb: z + .number() + .int() + .min(128) + .max(16_384) + .optional() + .describe( + "Maximum V8 old-space heap in MiB for the isolated JavaScript source analyzer; defaults to 1024. This does not change the CLI/MCP process heap.", + ), + analysis_timeout_ms: z + .number() + .int() + .min(1) + .max(2_147_483_647) + .optional() + .describe( + "Maximum startup, analysis and transfer time per JavaScript source, in milliseconds; defaults to 300000. On timeout, return collected partial Evidence and stop the analysis.", + ), +}); diff --git a/src/domain/javascript/javascriptApplicationAnalysis.ts b/src/domain/javascript/javascriptApplicationAnalysis.ts index 6320ab20e..c085e857e 100644 --- a/src/domain/javascript/javascriptApplicationAnalysis.ts +++ b/src/domain/javascript/javascriptApplicationAnalysis.ts @@ -15,6 +15,7 @@ import { prefixedDigestSchema } from "../digests.js"; import { integrityContradictionSchema } from "../artifactGraph.js"; import { artifactIntegrityPolicySchema } from "../artifactIntegrityPolicy.js"; import { localPathStringSchema } from "../localPath.js"; +import { javaScriptAnalysisResourceControlsSchema } from "./javascriptAnalysisResourceControls.js"; const countSchema = z.number().int().min(0); @@ -23,6 +24,7 @@ export const analyzeJavaScriptApplicationInputSchema = z.strictObject({ input_path: localPathStringSchema, format: z.enum(["auto", "asar", "directory"]).default("auto"), integrity_policy: artifactIntegrityPolicySchema, + ...javaScriptAnalysisResourceControlsSchema.shape, }); /** Compact counts for the high-level Electron architecture/security surface. */ diff --git a/src/domain/javascript/javascriptApplicationGraph.ts b/src/domain/javascript/javascriptApplicationGraph.ts index ea84585f1..f4b53503b 100644 --- a/src/domain/javascript/javascriptApplicationGraph.ts +++ b/src/domain/javascript/javascriptApplicationGraph.ts @@ -8,7 +8,10 @@ import { canonicalJson } from "../comparisonSemantics.js"; import { digestCanonicalValue } from "../canonicalDigest.js"; import { uniqueSorted } from "../canonicalOrdering.js"; import { compareUnicodeCodePoints } from "../unicodeCodePointOrder.js"; -import { freezeOwnedJsonSnapshotSteps } from "../immutableJson.js"; +import { + freezeJsonSnapshot, + freezeOwnedJsonSnapshotSteps, +} from "../immutableJson.js"; import type { ApplicationGraphEvidence, @@ -449,6 +452,10 @@ export const createJavaScriptApplicationNode = ( input: unknown, ): ApplicationNode => { const parsed = applicationNodeInputSchema.parse(input); + // These properties are newly parsed and exclusively owned. Authenticate + // them before the observation/node schemas and final graph validate again. + for (const observation of parsed.observations) + freezeJsonSnapshot(observation.properties); const identity = normalizeIdentity(parsed.identity); const identifier = nodeId({ kind: parsed.kind, identity }); const observations = parsed.observations @@ -479,6 +486,7 @@ export const createJavaScriptApplicationEdge = ( input: unknown, ): ApplicationEdge => { const parsed = applicationEdgeInputSchema.parse(input); + freezeJsonSnapshot(parsed.properties); const semantic = { ...parsed, evidence: normalizeEvidence(parsed.evidence), diff --git a/src/domain/javascript/javascriptApplicationProjectionResources.ts b/src/domain/javascript/javascriptApplicationProjectionResources.ts new file mode 100644 index 000000000..0e5ba7570 --- /dev/null +++ b/src/domain/javascript/javascriptApplicationProjectionResources.ts @@ -0,0 +1,42 @@ +import type { JavaScriptStaticAnalysis } from "./javascriptStaticAnalysisTypes.js"; + +/** Forecast static-fact expansion into observations, nodes and relationships. */ +export const estimateJavaScriptStaticApplicationProjection = ( + facts: JavaScriptStaticAnalysis, + checkpointBytes: number, + sourcePath: string, +): number => { + // A finding can produce a node, a separate observation and several edges. + // Allow for identifiers, locations, authority, coverage and schema copies, + // independently of the producer's compact checkpoint representation. + const findings = + facts.references.length + + facts.endpoints.length + + facts.storage.length + + facts.role_paths.length + + facts.source_map_urls.length + + facts.electron.browser_windows.length + + facts.electron.context_bridge_apis.length + + facts.electron.ipc.length + + facts.electron.sender_validations.length + + facts.electron.utility_processes.length + + facts.electron.native_addon_bindings.length; + let records = findings * 8; + let repeatedBytes = 0; + for (const registration of facts.bundler_registrations) { + records += + 3 + + registration.modules.length * 3 + + registration.entry_module_keys.length * 3 + + registration.async_chunk_keys.length * 3; + // Every module observation and containment edge repeats its chunk keys. + repeatedBytes += + Buffer.byteLength(JSON.stringify(registration.chunk_keys)) * + registration.modules.length * + 2; + } + for (const binding of facts.electron.native_addon_bindings) + records += binding.members.length * 3; + const metadataBytes = 2048 + Buffer.byteLength(sourcePath) * 4; + return (checkpointBytes * 4 + repeatedBytes + records * metadataBytes) * 8; +}; diff --git a/src/domain/javascript/javascriptModuleSemanticIr.ts b/src/domain/javascript/javascriptModuleSemanticIr.ts new file mode 100644 index 000000000..e2fc94e1f --- /dev/null +++ b/src/domain/javascript/javascriptModuleSemanticIr.ts @@ -0,0 +1,36 @@ +import type { JavaScriptSemanticIr } from "./javascriptSemanticIr.js"; + +/** Module facts needed after file-local semantic projection has finished. */ +export type JavaScriptModuleSemanticIr = Pick< + JavaScriptSemanticIr, + | "scopes" + | "bindings" + | "callables" + | "moduleLinks" + | "coverage" + | "limitations" +>; + +/** Retain facts consumed by source-module and export-return composition. */ +export const projectJavaScriptModuleSemantics = ( + ir: JavaScriptSemanticIr, +): JavaScriptModuleSemanticIr => { + const scopes = ir.scopes.filter(({ kind }) => kind === "program"); + const programScopeId = scopes[0]?.scopeId; + const localNames = new Set(ir.moduleLinks.map(({ localName }) => localName)); + const callableIds = new Set( + ir.moduleLinks.map(({ callableId }) => callableId), + ); + return { + scopes, + bindings: ir.bindings.filter( + ({ name, scopeId }) => scopeId === programScopeId && localNames.has(name), + ), + callables: ir.callables.filter(({ callableId }) => + callableIds.has(callableId), + ), + moduleLinks: ir.moduleLinks, + coverage: ir.coverage, + limitations: ir.limitations, + }; +}; diff --git a/src/domain/javascript/javascriptSemanticFileProjection.ts b/src/domain/javascript/javascriptSemanticFileProjection.ts new file mode 100644 index 000000000..8a7a4bb5e --- /dev/null +++ b/src/domain/javascript/javascriptSemanticFileProjection.ts @@ -0,0 +1,18 @@ +import type { + JavaScriptSemanticEvidenceContext, + JavaScriptSemanticFingerprint, + JavaScriptSemanticGraphNode, + JavaScriptSemanticGraphRelation, + JavaScriptSemanticGraphUnknown, +} from "./javascriptSemanticGraphSchemas.js"; + +/** File-local facts transported before binding them to the containing application graph. */ +export interface JavaScriptSemanticFileProjection { + readonly roots: readonly string[]; + readonly evidenceContexts: readonly JavaScriptSemanticEvidenceContext[]; + readonly nodes: readonly JavaScriptSemanticGraphNode[]; + readonly relations: readonly JavaScriptSemanticGraphRelation[]; + readonly unknowns: readonly JavaScriptSemanticGraphUnknown[]; + readonly fingerprints: readonly JavaScriptSemanticFingerprint[]; + readonly truncated: boolean; +} diff --git a/src/domain/javascript/javascriptSemanticGraph.ts b/src/domain/javascript/javascriptSemanticGraph.ts index ce7eb6ae8..cdaa8c28b 100644 --- a/src/domain/javascript/javascriptSemanticGraph.ts +++ b/src/domain/javascript/javascriptSemanticGraph.ts @@ -19,6 +19,7 @@ import { JAVASCRIPT_SEMANTIC_RELATION_FAMILIES, JAVASCRIPT_SEMANTIC_RELATION_FAMILY, javaScriptSemanticFingerprintInputSchema, + javaScriptSemanticEvidenceContextSchema, javaScriptSemanticGraphInputSchema, javaScriptSemanticGraphRecordSchema, javaScriptSemanticNodeInputSchema, @@ -109,6 +110,33 @@ export class JavaScriptSemanticEvidenceContextRegistry { this.#contexts.clear(); this.#idsByCanonicalContext.clear(); } + + /** Validate and intern provenance decoded in this realm from an analysis worker. */ + adopt(input: unknown): void { + const context = javaScriptSemanticEvidenceContextSchema.parse(input); + const { context_id: identifier, ...value } = context; + if (identifier !== evidenceContextId(value)) + throw new TypeError( + "Transferred semantic evidence context identifier is stale", + ); + const canonical = canonicalJson( + value, + "JavaScript semantic evidence context", + ); + const existing = this.#contexts.get(identifier); + if (existing !== undefined) { + const { context_id: _existingIdentifier, ...existingValue } = existing; + if ( + canonical !== + canonicalJson(existingValue, "JavaScript semantic evidence context") + ) + throw new TypeError( + "Transferred semantic evidence context digest collision", + ); + } + this.#contexts.set(identifier, context); + this.#idsByCanonicalContext.set(canonical, identifier); + } } /** Derive a semantic node ID from its kind and exact artifact identity. */ diff --git a/src/domain/javascript/javascriptSourceAnalysis.ts b/src/domain/javascript/javascriptSourceAnalysis.ts new file mode 100644 index 000000000..36bc3c632 --- /dev/null +++ b/src/domain/javascript/javascriptSourceAnalysis.ts @@ -0,0 +1,40 @@ +import type { AnalysisError } from "../analysisErrorBase.js"; +import type { AnalysisResourceConstraintError } from "../analysisErrorCore.js"; +import type { Result } from "../result.js"; +import type { JavaScriptArtifactFile } from "./javascriptArtifactFiles.js"; +import type { JavaScriptStaticAnalysis } from "./javascriptStaticAnalysisTypes.js"; +import type { JavaScriptModuleSemanticIr } from "./javascriptModuleSemanticIr.js"; +import type { JavaScriptSemanticFileProjection } from "./javascriptSemanticFileProjection.js"; + +/** Portable facts collected from one inert source. */ +export interface JavaScriptAnalyzedSource { + readonly javascript: JavaScriptStaticAnalysis; + readonly module: JavaScriptModuleSemanticIr | null; + readonly projection: JavaScriptSemanticFileProjection | null; + readonly applicationProjectionFailure?: AnalysisResourceConstraintError; +} + +/** Execution failure alongside facts already collected by the consumer. */ +export interface JavaScriptSourceAnalysisFailure { + readonly error: AnalysisError; + readonly javascript: JavaScriptStaticAnalysis | null; + readonly module: JavaScriptModuleSemanticIr | null; + readonly projection: JavaScriptSemanticFileProjection | null; + readonly applicationProjectionFailure?: AnalysisResourceConstraintError; +} + +/** Operational choices for source analysis, independent of transport framing. */ +export interface JavaScriptSourceAnalysisLimits { + readonly heapMb: number; + readonly timeoutMs: number; +} + +/** Sequential source-analysis authority with independently verified cleanup. */ +export interface JavaScriptSourceAnalysisPort { + analyze( + file: JavaScriptArtifactFile, + nodeBudget: number, + signal?: AbortSignal, + ): Promise>; + close(): Promise; +} diff --git a/src/domain/jsonValue.ts b/src/domain/jsonValue.ts index a6111a6c5..adaf2017f 100644 --- a/src/domain/jsonValue.ts +++ b/src/domain/jsonValue.ts @@ -78,6 +78,8 @@ export function* jsonValueValidationSteps( } const preserveJsonValue = (value: JsonValue): JsonValue => { + const immutableCandidate: unknown = value; + if (isImmutableJsonSnapshot(immutableCandidate)) return value; if (Array.isArray(value)) return value.map(preserveJsonValue); if (value !== null && typeof value === "object") return Object.fromEntries( diff --git a/src/javascript/analysis/JavaScriptAnalysisTransfer.ts b/src/javascript/analysis/JavaScriptAnalysisTransfer.ts new file mode 100644 index 000000000..75a3f1692 --- /dev/null +++ b/src/javascript/analysis/JavaScriptAnalysisTransfer.ts @@ -0,0 +1,414 @@ +import { createHash } from "node:crypto"; +import { createReadStream } from "node:fs"; +import { open, stat } from "node:fs/promises"; +import { z } from "zod"; +import { AnalysisResourceConstraintError } from "../../domain/analysisErrorCore.js"; + +import type { JavaScriptArtifactFile } from "../../domain/javascript/javascriptArtifactFiles.js"; +import type { JavaScriptModuleSemanticIr } from "../../domain/javascript/javascriptModuleSemanticIr.js"; +import type { JavaScriptSemanticFileProjection } from "../../domain/javascript/javascriptSemanticFileProjection.js"; +import { digestSchema } from "../../domain/digests.js"; +import { + javaScriptWorkerGraphRecordSchema, + javaScriptWorkerModuleHeaderSchema, + javaScriptWorkerModuleRecordSchema, + javaScriptWorkerProjectionHeaderSchema, +} from "./JavaScriptAnalysisWorkerSchemas.js"; + +// Budgets bound transfer representation, independently of source length. +/** Record bound whose transient decoding expansion is reserved by the consumer. */ +export const JAVASCRIPT_ANALYSIS_TRANSFER_RECORD_BYTES = 8 * 1024 * 1024; +const FRAME_BYTES = JAVASCRIPT_ANALYSIS_TRANSFER_RECORD_BYTES; +const TRANSFER_BYTES = 512 * 1024 * 1024; +const BATCH_BYTES = 1024 * 1024; +const recordSchema = z.discriminatedUnion("kind", [ + z.strictObject({ + kind: z.literal("projection-header"), + value: javaScriptWorkerProjectionHeaderSchema, + }), + z.strictObject({ + kind: z.literal("module-header"), + value: javaScriptWorkerModuleHeaderSchema, + }), + ...javaScriptWorkerGraphRecordSchema.options, + ...javaScriptWorkerModuleRecordSchema.options, +]); + +/** Complete-file coordinates authenticated before adopting any worker facts. */ +export const javaScriptAnalysisTransferDescriptorSchema = z.strictObject({ + bytes: z.number().int().nonnegative().max(TRANSFER_BYTES), + sha256: digestSchema, + records: z.number().int().nonnegative(), + module_bytes: z.number().int().nonnegative(), + module_records: z.number().int().nonnegative(), + graph_bytes: z.number().int().nonnegative(), + graph_records: z.number().int().nonnegative(), +}); +type Descriptor = z.output; + +/** Conservative retained-object and sealing expansion for validated transfer rows. */ +export const javaScriptTransferRetentionBytes = ( + bytes: number, + records: number, +): number => bytes * 8 + records * 64; + +const encodeTransferRecord = (input: unknown, previousBytes: number) => { + const record = recordSchema.parse(input); + const line = `${JSON.stringify(record)}\n`; + const size = Buffer.byteLength(line); + if (size > FRAME_BYTES || previousBytes + size > TRANSFER_BYTES) + throw new AnalysisResourceConstraintError( + "analyze_javascript_application", + "transport", + size > FRAME_BYTES + ? "A complete JavaScript fact exceeds the transfer record byte budget" + : "Complete JavaScript file facts exceed the transfer byte budget", + { + transfer_record_bytes: FRAME_BYTES, + transfer_file_bytes: TRANSFER_BYTES, + record_bytes: size, + transfer_bytes_at_least: previousBytes + size, + }, + ); + return { line, size, kind: record.kind }; +}; + +function* moduleTransferRecords( + module: JavaScriptModuleSemanticIr, +): Generator { + yield { + kind: "module-header", + value: { coverage: module.coverage, limitations: module.limitations }, + }; + for (const value of module.scopes) yield { kind: "scope", value }; + for (const value of module.bindings) yield { kind: "binding", value }; + for (const value of module.callables) yield { kind: "callable", value }; + for (const value of module.moduleLinks) yield { kind: "module-link", value }; +} + +/** Measure actual validated module rows before allocating an unadmittable graph. */ +export const measureJavaScriptModuleTransfer = ( + module: JavaScriptModuleSemanticIr, +) => { + let bytes = 0; + let records = 0; + for (const record of moduleTransferRecords(module)) { + bytes += encodeTransferRecord(record, bytes).size; + records += 1; + } + return { + bytes, + records, + retentionBytes: javaScriptTransferRetentionBytes(bytes, records), + }; +}; + +/** Encode validated facts in bounded records, without serializing a whole graph or IR. */ +export const writeJavaScriptAnalysisTransfer = async ( + path: string, + module: JavaScriptModuleSemanticIr, + projection: JavaScriptSemanticFileProjection, +): Promise => { + const output = await open(path, "wx", 0o600); + const digest = createHash("sha256"); + let bytes = 0; + let records = 0; + let moduleBytes = 0; + let moduleRecords = 0; + let graphBytes = 0; + let graphRecords = 0; + let batch: string[] = []; + let batchBytes = 0; + const flush = async (): Promise => { + if (batch.length === 0) return; + const buffer = Buffer.from(batch.join("")); + // FileHandle.write can be short; writeFile completes the buffer. + await output.writeFile(buffer); + digest.update(buffer); + batch = []; + batchBytes = 0; + }; + const append = async (input: unknown): Promise => { + const { line, size, kind } = encodeTransferRecord(input, bytes); + bytes += size; + records += 1; + if ( + ["module-header", "scope", "binding", "callable", "module-link"].includes( + kind, + ) + ) { + moduleBytes += size; + moduleRecords += 1; + } else { + graphBytes += size; + graphRecords += 1; + } + batch.push(line); + batchBytes += size; + if (batchBytes >= BATCH_BYTES) await flush(); + }; + try { + await append({ + kind: "projection-header", + value: { roots: projection.roots, truncated: projection.truncated }, + }); + for (const record of moduleTransferRecords(module)) await append(record); + for (const value of projection.evidenceContexts) + await append({ kind: "evidence-context", value }); + for (const value of projection.nodes) await append({ kind: "node", value }); + for (const value of projection.relations) + await append({ kind: "relation", value }); + for (const value of projection.unknowns) + await append({ kind: "unknown", value }); + for (const value of projection.fingerprints) + await append({ kind: "fingerprint", value }); + await flush(); + return { + bytes, + sha256: digest.digest("hex"), + records, + module_bytes: moduleBytes, + module_records: moduleRecords, + graph_bytes: graphBytes, + graph_records: graphRecords, + }; + } finally { + await output.close(); + } +}; + +/** Decode producer records into realm-local, source-bound facts; sealing follows in the consumer. */ +export function readJavaScriptAnalysisTransfer( + path: string, + descriptor: Descriptor, + source: Pick, + signal?: AbortSignal, +): Promise<{ + readonly module: JavaScriptModuleSemanticIr; + readonly projection: JavaScriptSemanticFileProjection; +}>; +/** Validate the transfer while retaining only module facts when graph admission fails. */ +export function readJavaScriptAnalysisTransfer( + path: string, + descriptor: Descriptor, + source: Pick, + signal: AbortSignal | undefined, + retainGraph: boolean, +): Promise<{ + readonly module: JavaScriptModuleSemanticIr; + readonly projection: JavaScriptSemanticFileProjection | null; +}>; +export async function readJavaScriptAnalysisTransfer( + path: string, + descriptor: Descriptor, + source: Pick, + signal?: AbortSignal, + retainGraph = true, +): Promise<{ + readonly module: JavaScriptModuleSemanticIr; + readonly projection: JavaScriptSemanticFileProjection | null; +}> { + const expected = javaScriptAnalysisTransferDescriptorSchema.parse(descriptor); + if ((await stat(path)).size !== expected.bytes) + throw new TypeError( + "JavaScript analysis transfer byte count does not match", + ); + const scopes: JavaScriptModuleSemanticIr["scopes"][number][] = []; + const bindings: JavaScriptModuleSemanticIr["bindings"][number][] = []; + const callables: JavaScriptModuleSemanticIr["callables"][number][] = []; + const moduleLinks: JavaScriptModuleSemanticIr["moduleLinks"][number][] = []; + const evidenceContexts: JavaScriptSemanticFileProjection["evidenceContexts"][number][] = + []; + const nodes: JavaScriptSemanticFileProjection["nodes"][number][] = []; + const relations: JavaScriptSemanticFileProjection["relations"][number][] = []; + const unknowns: JavaScriptSemanticFileProjection["unknowns"][number][] = []; + const fingerprints: JavaScriptSemanticFileProjection["fingerprints"][number][] = + []; + let projectionHeader: + | z.output + | undefined; + let moduleHeader: + | z.output + | undefined; + const identities = new Map>(); + const retainIdentity = (kind: string, identifier: string): void => { + let seen = identities.get(kind); + if (seen === undefined) { + seen = new Set(); + identities.set(kind, seen); + } + if (seen.has(identifier)) + throw new TypeError( + `Duplicate JavaScript transfer ${kind} identity: ${identifier}`, + ); + seen.add(identifier); + }; + let records = 0; + let moduleBytes = 0; + let moduleRecords = 0; + let graphBytes = 0; + let graphRecords = 0; + const digest = createHash("sha256"); + const stream = createReadStream(path, { + highWaterMark: 64 * 1024, + ...(signal === undefined ? {} : { signal }), + }); + let bytes = 0; + let pending = ""; + stream.setEncoding("utf8"); + try { + for await (const chunk of stream) { + signal?.throwIfAborted(); + if (typeof chunk !== "string") + throw new TypeError("JavaScript transfer stream is not UTF-8 text"); + bytes += Buffer.byteLength(chunk); + if (bytes > expected.bytes) + throw new RangeError("JavaScript transfer grew while being read"); + digest.update(chunk); + pending += chunk; + let newline: number; + while ((newline = pending.indexOf("\n")) >= 0) { + const line = pending.slice(0, newline); + pending = pending.slice(newline + 1); + if (Buffer.byteLength(line) + 1 > FRAME_BYTES) + throw new RangeError( + "JavaScript transfer record exceeds its byte budget", + ); + const input: unknown = JSON.parse(line); + const record = recordSchema.parse(input); + if ( + [ + "module-header", + "scope", + "binding", + "callable", + "module-link", + ].includes(record.kind) + ) { + moduleBytes += Buffer.byteLength(line) + 1; + moduleRecords += 1; + } else { + graphBytes += Buffer.byteLength(line) + 1; + graphRecords += 1; + } + if (record.kind === "projection-header") { + if (records !== 0 || projectionHeader !== undefined) + throw new TypeError( + "Duplicate or late JavaScript projection header", + ); + projectionHeader = record.value; + } else if (record.kind === "module-header") { + if (records !== 1 || moduleHeader !== undefined) + throw new TypeError("Duplicate or late JavaScript module header"); + moduleHeader = record.value; + } else { + if (projectionHeader === undefined || moduleHeader === undefined) + throw new TypeError( + "JavaScript transfer facts precede their headers", + ); + if ( + !retainGraph && + [ + "evidence-context", + "node", + "relation", + "unknown", + "fingerprint", + ].includes(record.kind) + ) { + if ( + record.kind === "node" && + (record.value.identity.artifact_sha256 !== source.sha256 || + record.value.identity.module_path !== source.path) + ) + throw new TypeError( + "JavaScript transfer node belongs to a different artifact source", + ); + records += 1; + continue; + } + switch (record.kind) { + case "scope": + retainIdentity(record.kind, record.value.scopeId); + scopes.push(record.value); + break; + case "binding": + retainIdentity(record.kind, record.value.bindingId); + bindings.push(record.value); + break; + case "callable": + retainIdentity(record.kind, record.value.callableId); + callables.push(record.value); + break; + case "module-link": + moduleLinks.push(record.value); + break; + case "evidence-context": + retainIdentity(record.kind, record.value.context_id); + evidenceContexts.push(record.value); + break; + case "node": { + retainIdentity(record.kind, record.value.node_id); + if ( + record.value.identity.artifact_sha256 !== source.sha256 || + record.value.identity.module_path !== source.path + ) + throw new TypeError( + "JavaScript transfer node belongs to a different artifact source", + ); + nodes.push(record.value); + break; + } + case "relation": + retainIdentity(record.kind, record.value.relation_id); + relations.push(record.value); + break; + case "unknown": + retainIdentity(record.kind, record.value.unknown_id); + unknowns.push(record.value); + break; + case "fingerprint": + retainIdentity(record.kind, record.value.fingerprint_id); + fingerprints.push(record.value); + break; + } + } + records += 1; + } + if (Buffer.byteLength(pending) + 1 > FRAME_BYTES) + throw new RangeError( + "Unterminated JavaScript transfer record exceeds its byte budget", + ); + } + } finally { + stream.destroy(); + } + if ( + pending !== "" || + records !== expected.records || + bytes !== expected.bytes || + moduleBytes !== expected.module_bytes || + moduleRecords !== expected.module_records || + graphBytes !== expected.graph_bytes || + graphRecords !== expected.graph_records || + digest.digest("hex") !== expected.sha256 + ) + throw new TypeError( + "JavaScript analysis transfer is incomplete or its digest does not match", + ); + if (projectionHeader === undefined || moduleHeader === undefined) + throw new TypeError("JavaScript analysis transfer is missing its headers"); + return { + module: { ...moduleHeader, scopes, bindings, callables, moduleLinks }, + projection: retainGraph + ? { + ...projectionHeader, + evidenceContexts, + nodes, + relations, + unknowns, + fingerprints, + } + : null, + }; +} diff --git a/src/javascript/analysis/JavaScriptAnalysisWorker.ts b/src/javascript/analysis/JavaScriptAnalysisWorker.ts new file mode 100644 index 000000000..f8c544c99 --- /dev/null +++ b/src/javascript/analysis/JavaScriptAnalysisWorker.ts @@ -0,0 +1,726 @@ +import { createHash, randomUUID } from "node:crypto"; +import { readFile, rm, stat, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { StringDecoder } from "node:string_decoder"; +import { fileURLToPath } from "node:url"; +import { getHeapStatistics } from "node:v8"; +import { z } from "zod"; +import type { JavaScriptArtifactFile } from "../../domain/javascript/javascriptArtifactFiles.js"; +import type { JavaScriptStaticAnalysis } from "../../domain/javascript/javascriptStaticAnalysisTypes.js"; +import { estimateJavaScriptStaticApplicationProjection } from "../../domain/javascript/javascriptApplicationProjectionResources.js"; +import type { JavaScriptModuleSemanticIr } from "../../domain/javascript/javascriptModuleSemanticIr.js"; +import type { JavaScriptSemanticFileProjection } from "../../domain/javascript/javascriptSemanticFileProjection.js"; +import type { + JavaScriptAnalyzedSource, + JavaScriptSourceAnalysisFailure, + JavaScriptSourceAnalysisLimits, +} from "../../domain/javascript/javascriptSourceAnalysis.js"; +import { AnalysisError } from "../../domain/analysisErrorBase.js"; +import { + AnalysisCancelledError, + AnalysisTimeoutError, + AnalysisOutputError, + AnalysisResourceConstraintError, + AnalysisCapabilityUnavailableError, +} from "../../domain/analysisErrorCore.js"; +import { ProviderAdapterError } from "../../domain/providerAdapterError.js"; +import { ProviderCleanupError } from "../../domain/providerCleanupError.js"; +import { ok, err, type Result } from "../../domain/result.js"; +import { + PrivateRuntimeRoot, + PrivateRuntimeRootUnavailableError, +} from "../../process/PrivateRuntimeRoot.js"; +import { + ProviderProcessSupervisor, + spawnOwnedProviderProcess, +} from "../../process/ProviderProcess.js"; +import { cleanupOwnedProcessGroup } from "../../process/ProcessOwnership.js"; +import { prepareProcessOwnershipInspection } from "../../process/ProcessOwnershipObservation.js"; +import { + ProviderStartupDeadline, + waitForAbortableDelay, +} from "../../process/ProviderDeadline.js"; +import { + readJavaScriptAnalysisTransfer, + JAVASCRIPT_ANALYSIS_TRANSFER_RECORD_BYTES, + javaScriptTransferRetentionBytes, +} from "./JavaScriptAnalysisTransfer.js"; +import { javaScriptWorkerStaticAnalysisSchema } from "./JavaScriptAnalysisWorkerSchemas.js"; +import { + JAVASCRIPT_ANALYSIS_WORKER_DEFAULTS as DEFAULTS, + javaScriptWorkerMessageSchema, + javaScriptWorkerRequestSchema, + javaScriptWorkerAdmissionSchema, + type JavaScriptWorkerMessage, +} from "./JavaScriptAnalysisWorkerProtocol.js"; + +const OPERATION = "analyze_javascript_application"; +const PROVIDER = "javascript-static-analysis"; +const MIB = 1024 * 1024; +const pendingCleanup = new Set(); + +/** One sequential owned worker, retained between source files and closed after analysis. */ +export class JavaScriptAnalysisWorker { + readonly #runId = `rea-javascript-analysis-${randomUUID()}`; + #root: PrivateRuntimeRoot | undefined; + #supervisor: ProviderProcessSupervisor | undefined; + #ready = false; + #heapLimitBytes = 0; + #frames: JavaScriptWorkerMessage[] = []; + #pending = ""; + #decoder = new StringDecoder("utf8"); + #protocolFailure: string | undefined; + #phase = "startup"; + #lastFailure: AnalysisError | undefined; + #estimatedRetainedBytes = 0; + #applicationProjectionFailure: AnalysisResourceConstraintError | undefined; + #parentCapacityExhausted = false; + #retainedStaticBytes = 0; + #sourcesRemaining: number | undefined; + readonly #resultBudgetBytes: number; + readonly #onStdout = (chunk: Buffer | string): void => { + try { + this.#pending += + typeof chunk === "string" ? chunk : this.#decoder.write(chunk); + let newline: number; + while ((newline = this.#pending.indexOf("\n")) >= 0) { + const line = this.#pending.slice(0, newline); + this.#pending = this.#pending.slice(newline + 1); + if ( + Buffer.byteLength(line) > DEFAULTS.protocolBytes || + this.#frames.length >= 16 + ) + throw new RangeError( + "JavaScript worker exceeded its control-frame budget", + ); + const raw: unknown = JSON.parse(line); + this.#frames.push(javaScriptWorkerMessageSchema.parse(raw)); + } + if (Buffer.byteLength(this.#pending) > DEFAULTS.protocolBytes) + throw new RangeError( + "JavaScript worker control frame is unterminated or oversized", + ); + } catch (cause: unknown) { + this.#protocolFailure = message(cause); + } + }; + + constructor( + readonly limits: JavaScriptSourceAnalysisLimits = DEFAULTS, + readonly launcher: typeof spawnOwnedProviderProcess = spawnOwnedProviderProcess, + sourceCount?: number, + ) { + if ( + !Number.isSafeInteger(limits.heapMb) || + limits.heapMb < 128 || + limits.heapMb > 16_384 || + !Number.isSafeInteger(limits.timeoutMs) || + limits.timeoutMs <= 0 || + limits.timeoutMs > 2_147_483_647 + ) + throw new RangeError( + "JavaScript worker requires valid heap and execution limits", + ); + if ( + sourceCount !== undefined && + (!Number.isSafeInteger(sourceCount) || sourceCount < 0) + ) + throw new RangeError( + "JavaScript worker source count must be a nonnegative safe integer", + ); + this.#sourcesRemaining = sourceCount; + // Reserve host-side control, schema/Evidence work and a maximum record's + // transient copies. Admission also accounts for retained earlier files. + this.#resultBudgetBytes = Math.max( + 0, + Math.floor( + (getHeapStatistics().heap_size_limit - + process.memoryUsage().heapUsed - + 24 * JAVASCRIPT_ANALYSIS_TRANSFER_RECORD_BYTES) / + 2, + ), + ); + } + + /** Analyze inert source; hard worker failure does not terminate this process. */ + async analyze( + file: JavaScriptArtifactFile, + nodeBudget: number, + signal?: AbortSignal, + ): Promise< + Result + > { + let javascript: JavaScriptStaticAnalysis | null = null; + let module: JavaScriptModuleSemanticIr | null = null; + let projection: JavaScriptSemanticFileProjection | null = null; + const id = randomUUID(); + this.#applicationProjectionFailure = undefined; + this.#parentCapacityExhausted = false; + if (this.#sourcesRemaining !== undefined) + this.#sourcesRemaining = Math.max(0, this.#sourcesRemaining - 1); + let deadline: ProviderStartupDeadline | undefined; + let result: Result< + JavaScriptAnalyzedSource, + JavaScriptSourceAnalysisFailure + >; + try { + if (signal?.aborted === true) throw new AnalysisCancelledError(OPERATION); + if (!file.text.included || file.kind !== "javascript") + throw new TypeError( + "JavaScript analysis worker requires included JavaScript text", + ); + // Host ownership preparation is outside the parser execution deadline. + await prepareProcessOwnershipInspection(signal); + for (const owner of pendingCleanup) await owner.close(); + deadline = new ProviderStartupDeadline(this.limits.timeoutMs, signal); + await this.#start(deadline); + const root = this.#root; + const supervisor = this.#supervisor; + if (root === undefined || supervisor === undefined) + throw new TypeError("JavaScript worker resources were not acquired"); + if (this.#frames.length !== 0 || this.#pending !== "") + throw new AnalysisOutputError( + OPERATION, + "JavaScript worker has unsolicited or incomplete control frames", + ); + supervisor.resetOutput(); + this.#phase = "source-transfer"; + await writeFile(join(root.path, `${id}.source`), file.text.value, { + flag: "wx", + mode: 0o600, + signal: deadline.signal, + }); + const { text: _text, ...metadata } = file; + let offeredResultBudget = 0; + const request = javaScriptWorkerRequestSchema.parse({ + id, + node_budget: nodeBudget, + text_sha256: createHash("sha256").update(file.text.value).digest("hex"), + file: metadata, + }); + const frame = `${JSON.stringify(request)}\n`; + if (Buffer.byteLength(frame) > DEFAULTS.protocolBytes) + throw new AnalysisResourceConstraintError( + OPERATION, + "transport", + `Source identity for ${file.path} exceeds the worker request byte budget`, + { worker_request_bytes: DEFAULTS.protocolBytes }, + ); + const stdin = supervisor.launch.process.stdin; + if (stdin === undefined || stdin === null) + throw new TypeError("JavaScript worker has no writable request stream"); + await new Promise((resolve, reject) => + stdin.write(frame, (error) => + error === null || error === undefined ? resolve() : reject(error), + ), + ); + for (;;) { + const response = await this.#next(deadline, file.path); + if (response.kind === "ready" || response.id !== id) + throw new AnalysisOutputError( + OPERATION, + "JavaScript worker response names a different source job", + ); + if (response.kind === "phase") { + this.#phase = response.phase; + continue; + } + if (response.kind === "static") { + if (javascript !== null) + throw new AnalysisOutputError( + OPERATION, + "JavaScript worker emitted a duplicate static checkpoint", + ); + javascript = await this.#readCheckpoint(id, file.path, response); + if (this.#parentCapacityExhausted) + throw new AnalysisResourceConstraintError( + OPERATION, + "memory", + `The parent's result capacity was exhausted after retaining static facts for ${file.path}; remaining sources were not analyzed`, + { + ...this.#limits(), + parent_result_budget_bytes: this.#resultBudgetBytes, + parent_result_budget_exhausted: true, + }, + ); + offeredResultBudget = + this.#applicationProjectionFailure !== undefined && + this.#sourcesRemaining !== undefined && + this.#sourcesRemaining > 0 + ? 0 + : this.#remainingResultBudget(); + const admission = javaScriptWorkerAdmissionSchema.parse({ + id, + semantic_result_budget_bytes: offeredResultBudget, + }); + await new Promise((resolve, reject) => + stdin.write(`${JSON.stringify(admission)}\n`, (error) => + error === null || error === undefined ? resolve() : reject(error), + ), + ); + continue; + } + if (response.kind === "failure") { + if (response.reason === "resource") { + result = err({ + javascript, + module, + projection, + error: new AnalysisResourceConstraintError( + OPERATION, + response.resource ?? "memory", + `${file.path}: ${response.message}`, + { ...this.#limits(), ...response.limits }, + ), + }); + break; + } + throw new AnalysisOutputError( + OPERATION, + `${file.path} during ${this.#phase}: ${response.message}`, + ); + } + if (javascript === null) + throw new AnalysisOutputError( + OPERATION, + "JavaScript worker completed before a static checkpoint", + ); + if (response.descriptor === null) { + result = ok({ javascript, module: null, projection: null }); + break; + } + const descriptor = response.descriptor; + // JSON bytes alone omit object/array overhead and the later graph/Evidence + // sealing/serialization copies. Reserve a conservative expansion budget. + const moduleEstimate = javaScriptTransferRetentionBytes( + descriptor.module_bytes, + descriptor.module_records, + ); + const graphEstimate = javaScriptTransferRetentionBytes( + descriptor.graph_bytes, + descriptor.graph_records, + ); + const remaining = Math.min( + offeredResultBudget, + this.#remainingResultBudget(), + ); + if (moduleEstimate > remaining) { + result = err({ + javascript, + module: null, + projection: null, + error: new AnalysisResourceConstraintError( + OPERATION, + "memory", + `Completed module facts for ${file.path} exceed the parent's remaining result budget; static facts were retained`, + { + ...this.#limits(), + parent_result_budget_bytes: this.#resultBudgetBytes, + module_transfer_bytes: descriptor.module_bytes, + }, + ), + }); + break; + } + const retainGraph = moduleEstimate + graphEstimate <= remaining; + this.#phase = "result-admission"; + const decoded = await readJavaScriptAnalysisTransfer( + join(root.path, `${id}.records`), + descriptor, + file, + deadline.signal, + retainGraph, + ); + module = decoded.module; + projection = decoded.projection; + this.#estimatedRetainedBytes += + moduleEstimate + (retainGraph ? graphEstimate : 0); + result = retainGraph + ? ok({ javascript, module, projection: decoded.projection }) + : err({ + javascript, + module, + projection, + error: new AnalysisResourceConstraintError( + OPERATION, + "memory", + `Semantic graph for ${file.path} exceeds the parent's remaining result budget; static and module facts were retained`, + { + ...this.#limits(), + parent_result_budget_bytes: this.#resultBudgetBytes, + graph_transfer_bytes: descriptor.graph_bytes, + graph_records: descriptor.graph_records, + }, + ), + }); + break; + } + } catch (cause: unknown) { + const error = this.#failure(cause, file.path, deadline); + this.#lastFailure = error; + // Complete checkpoints remain readable after a child crash or cancellation. + const cleanup = await this.#stop(error); + if ( + cleanup === undefined && + javascript === null && + this.#root !== undefined + ) { + try { + javascript = await this.#readCheckpoint(id, file.path); + } catch (checkpointCause: unknown) { + void checkpointCause; + } + } + result = err({ error: cleanup ?? error, javascript, module, projection }); + } finally { + deadline?.dispose(); + } + if (result.ok) this.#lastFailure = undefined; + else if (!(result.error.error instanceof ProviderCleanupError)) + this.#lastFailure = result.error.error; + // Child stop must be verified before temporary files are released. On + // cleanup uncertainty, close retains this owner for a later retry. + if (this.#root !== undefined && !pendingCleanup.has(this)) { + try { + for (const suffix of ["source", "static.json", "records"]) + await rm(join(this.#root.path, `${id}.${suffix}`), { force: true }); + } catch (cause: unknown) { + pendingCleanup.add(this); + return err({ + javascript, + module, + projection, + error: this.#cleanupError(message(cause)), + }); + } + } + if (this.#applicationProjectionFailure !== undefined) + return result.ok + ? ok({ + ...result.value, + applicationProjectionFailure: this.#applicationProjectionFailure, + }) + : err({ + ...result.error, + applicationProjectionFailure: this.#applicationProjectionFailure, + }); + return result; + } + + /** Verify process cleanup, then release the private runtime; failed close is retryable. */ + async close(): Promise { + const failure = await this.#stop(); + if (failure !== undefined) throw failure; + if (this.#root !== undefined) { + try { + await this.#root.close(); + this.#root = undefined; + } catch (cause: unknown) { + pendingCleanup.add(this); + throw this.#cleanupError(message(cause)); + } + } + pendingCleanup.delete(this); + } + + async #start(deadline: ProviderStartupDeadline): Promise { + if (this.#supervisor !== undefined && this.#ready) return; + this.#phase = "startup"; + this.#heapLimitBytes = 0; + this.#root ??= await PrivateRuntimeRoot.create({ + prefix: "rea-javascript-analysis-", + }); + const environment: NodeJS.ProcessEnv = { + ...process.env, + UV_THREADPOOL_SIZE: "1", + }; + delete environment.NODE_OPTIONS; + const spawned = await this.launcher({ + command: process.execPath, + arguments: [ + `--max-old-space-size=${String(this.limits.heapMb)}`, + "--max-semi-space-size=8", + "--v8-pool-size=1", + "--expose-gc", + "--stack_size=2048", + fileURLToPath(import.meta.resolve("#javascript-analysis-worker")), + ], + runId: `${this.#runId}-${randomUUID()}`, + cwd: this.#root.path, + hostEnvironment: environment, + stdin: "pipe", + signal: deadline.signal, + }); + this.#supervisor = new ProviderProcessSupervisor( + { + ...spawned, + ownsProcessLifetime: true, + cleanup: + spawned.cleanup ?? + (() => cleanupOwnedProcessGroup(spawned.ownership)), + }, + { captureStdout: false, maxDiagnosticBytes: 256 * 1024 }, + ); + spawned.process.stdout?.on("data", this.#onStdout); + const ready = await this.#next(deadline, "worker startup"); + if (ready.kind !== "ready") + throw new AnalysisOutputError( + OPERATION, + "JavaScript worker emitted facts before its startup handshake", + ); + this.#heapLimitBytes = ready.heap_limit_bytes; + this.#ready = true; + } + + async #next( + deadline: ProviderStartupDeadline, + path: string, + ): Promise { + for (;;) { + if (deadline.interruption === "cancelled") + throw new AnalysisCancelledError(OPERATION); + if (deadline.interruption === "timeout") + throw new AnalysisTimeoutError(OPERATION, this.limits.timeoutMs); + if (this.#protocolFailure !== undefined) + throw new AnalysisOutputError(OPERATION, this.#protocolFailure); + const frame = this.#frames.shift(); + if (frame !== undefined) return frame; + const snapshot = this.#supervisor?.snapshot(); + if ( + snapshot !== undefined && + (snapshot.exitCode !== undefined || snapshot.signal !== undefined) + ) { + // Exit is not complete output: let any final control frame drain first. + if (!(await this.#supervisor?.waitForOutputClose(10))) continue; + if (this.#frames.length !== 0) continue; + const drained = this.#supervisor?.snapshot() ?? snapshot; + throw /heap out of memory|heap limit|allocation failed/iu.test( + drained.stderr.text, + ) + ? new AnalysisResourceConstraintError( + OPERATION, + "memory", + `Owned JavaScript worker exhausted its heap while analyzing ${path} during ${this.#phase}`, + this.#limits(), + { capturedOutput: this.#captured() }, + ) + : new ProviderAdapterError(PROVIDER, OPERATION, { + capturedOutput: this.#captured(), + diagnostics: { + source_path: path, + phase: this.#phase, + exit_code: drained.exitCode ?? null, + signal: drained.signal ?? null, + }, + }); + } + await waitForAbortableDelay(10, deadline.signal); + } + } + + async #readCheckpoint( + id: string, + sourcePath: string, + expected?: { readonly bytes: number; readonly sha256: string }, + ): Promise { + if (this.#root === undefined) + throw new TypeError("JavaScript checkpoint has no owned runtime"); + const path = join(this.#root.path, `${id}.static.json`); + const bytes = (await stat(path)).size; + if ( + bytes > DEFAULTS.checkpointBytes || + (expected !== undefined && expected.bytes !== bytes) + ) + throw new AnalysisOutputError( + OPERATION, + "JavaScript static checkpoint byte count exceeds or differs from its declared budget", + ); + if ( + bytes * 16 + 4 * MIB > + getHeapStatistics().heap_size_limit - + process.memoryUsage().heapUsed - + 64 * MIB + ) + throw new AnalysisResourceConstraintError( + OPERATION, + "memory", + "The parent's remaining heap cannot safely decode the completed JavaScript static checkpoint", + { ...this.#limits(), checkpoint_bytes: bytes }, + ); + const text = await readFile(path, "utf8"); + if ( + expected !== undefined && + createHash("sha256").update(text).digest("hex") !== expected.sha256 + ) + throw new AnalysisOutputError( + OPERATION, + "JavaScript static checkpoint digest does not match", + ); + const value: unknown = JSON.parse(text); + const facts = javaScriptWorkerStaticAnalysisSchema.parse(value); + const remaining = this.#remainingTotalResultBudget(); + const retainedBytes = bytes * 6; + this.#parentCapacityExhausted = retainedBytes > remaining; + this.#estimatedRetainedBytes += retainedBytes; + this.#retainedStaticBytes += retainedBytes; + const expandedBytes = estimateJavaScriptStaticApplicationProjection( + facts, + bytes, + sourcePath, + ); + if (expandedBytes <= this.#remainingResultBudget()) + this.#estimatedRetainedBytes += expandedBytes; + else + this.#applicationProjectionFailure = new AnalysisResourceConstraintError( + OPERATION, + "memory", + `Expanded application relationships for ${sourcePath} exceed the parent's result budget; complete static facts were retained on the file node`, + { + parent_result_budget_bytes: this.#resultBudgetBytes, + application_projection_required_bytes: expandedBytes, + }, + ); + return facts; + } + + #limits() { + return { + worker_old_space_mb: this.limits.heapMb, + ...(this.#heapLimitBytes === 0 + ? {} + : { worker_heap_limit_bytes: this.#heapLimitBytes }), + worker_timeout_ms: this.limits.timeoutMs, + }; + } + #remainingResultBudget(): number { + // Future sources need their compact observations before current modules + // consume capacity with expanded relationships. A final or sole source + // can use the unused reserve as well. + const staticReserve = + this.#sourcesRemaining !== undefined && this.#sourcesRemaining > 0 + ? Math.max( + 0, + Math.floor((this.#resultBudgetBytes * 4) / 5) - + this.#retainedStaticBytes, + ) + : 0; + return Math.max(0, this.#remainingTotalResultBudget() - staticReserve); + } + #remainingTotalResultBudget(): number { + return Math.max( + 0, + Math.min( + this.#resultBudgetBytes - this.#estimatedRetainedBytes, + getHeapStatistics().heap_size_limit - + process.memoryUsage().heapUsed - + 24 * JAVASCRIPT_ANALYSIS_TRANSFER_RECORD_BYTES, + ), + ); + } + #captured() { + const snapshot = this.#supervisor?.snapshot(); + return { + stdout: "", + stderr: snapshot?.stderr.text ?? "", + truncated: snapshot?.diagnosticTruncated ?? false, + stderr_bytes: snapshot?.stderr.observedBytes ?? 0, + exit_code: snapshot?.exitCode ?? null, + signal: snapshot?.signal ?? null, + }; + } + #failure( + cause: unknown, + path: string, + deadline: ProviderStartupDeadline | undefined, + ): AnalysisError { + if (deadline?.interruption === "cancelled") + return new AnalysisCancelledError(OPERATION, { + capturedOutput: this.#captured(), + }); + if (deadline?.interruption === "timeout") + return new AnalysisTimeoutError(OPERATION, this.limits.timeoutMs, { + capturedOutput: this.#captured(), + }); + if (cause instanceof AnalysisError) return cause; + if ( + cause instanceof z.ZodError || + cause instanceof TypeError || + cause instanceof SyntaxError + ) + return new AnalysisOutputError( + OPERATION, + `${path} during ${this.#phase}: ${message(cause)}`, + { cause, capturedOutput: this.#captured() }, + ); + return cause instanceof PrivateRuntimeRootUnavailableError + ? new AnalysisCapabilityUnavailableError( + PROVIDER, + OPERATION, + cause.message, + { cause }, + ) + : new ProviderAdapterError(PROVIDER, OPERATION, { + cause, + capturedOutput: this.#captured(), + diagnostics: { + source_path: path, + phase: this.#phase, + message: message(cause), + }, + }); + } + #cleanupError( + reason: string, + original: AnalysisError | undefined = this.#lastFailure, + ): ProviderCleanupError { + return new ProviderCleanupError( + PROVIDER, + [this.#runId, ...(this.#root === undefined ? [] : [this.#root.path])], + { + reason, + worker_pid: this.#supervisor?.launch.process.pid ?? null, + ...(original === undefined + ? {} + : { + execution_failure: { + tag: original._tag, + message: original.message, + captured_output: { + stdout: original.capturedOutput?.stdout ?? "", + stderr: original.capturedOutput?.stderr ?? "", + truncated: original.capturedOutput?.truncated ?? false, + }, + }, + }), + }, + { + operation: OPERATION, + ...(original === undefined ? {} : { cause: original }), + }, + ); + } + async #stop( + original: AnalysisError | undefined = this.#lastFailure, + ): Promise { + if (this.#supervisor !== undefined) { + const supervisor = this.#supervisor; + const stopped = await supervisor.stop(); + if (stopped.status === "incomplete") { + pendingCleanup.add(this); + return this.#cleanupError(stopped.reason, original); + } + supervisor.launch.process.stdout?.off("data", this.#onStdout); + supervisor.dispose(); + this.#supervisor = undefined; + } + this.#ready = false; + this.#frames = []; + this.#pending = ""; + this.#decoder = new StringDecoder("utf8"); + this.#protocolFailure = undefined; + return undefined; + } +} + +const message = (cause: unknown): string => + cause instanceof Error ? cause.message : String(cause); diff --git a/src/javascript/analysis/JavaScriptAnalysisWorkerProtocol.ts b/src/javascript/analysis/JavaScriptAnalysisWorkerProtocol.ts new file mode 100644 index 000000000..cf8e905bd --- /dev/null +++ b/src/javascript/analysis/JavaScriptAnalysisWorkerProtocol.ts @@ -0,0 +1,73 @@ +import { z } from "zod"; +import { jsonValueSchema } from "../../domain/jsonValue.js"; +import { digestSchema } from "../../domain/digests.js"; +import { JAVASCRIPT_ANALYSIS_RESOURCE_DEFAULTS } from "../../domain/javascript/javascriptAnalysisResourceControls.js"; +import { javaScriptAnalysisTransferDescriptorSchema } from "./JavaScriptAnalysisTransfer.js"; + +/** Independent worker execution budget; source length is not an admission rule. */ +export const JAVASCRIPT_ANALYSIS_WORKER_DEFAULTS = { + ...JAVASCRIPT_ANALYSIS_RESOURCE_DEFAULTS, + checkpointBytes: 8 * 1024 * 1024, + protocolBytes: 64 * 1024, +} as const; + +/** Caller-selected artifact identity, separate from the decoded source-text digest. */ +export const javaScriptWorkerRequestSchema = z.strictObject({ + id: z.uuid(), + node_budget: z.number().int().min(0).max(20_000), + text_sha256: digestSchema, + file: z.strictObject({ + path: z.string(), + sha256: digestSchema, + container_sha256: digestSchema, + bytes: z.number().int().nonnegative(), + inventory_artifact_id: z.string(), + kind: z.literal("javascript"), + unpacked: z.boolean(), + }), +}); + +/** Parent admission after validated static observations have been retained. */ +export const javaScriptWorkerAdmissionSchema = z.strictObject({ + id: z.uuid(), + semantic_result_budget_bytes: z.number().int().nonnegative(), +}); + +const identity = { id: z.uuid() }; + +/** Control frames never contain AST, full IR, graph rows or source text. */ +export const javaScriptWorkerMessageSchema = z.discriminatedUnion("kind", [ + z.strictObject({ + kind: z.literal("ready"), + heap_limit_bytes: z.number().int().positive(), + }), + z.strictObject({ + kind: z.literal("phase"), + ...identity, + phase: z.enum(["parse", "static", "semantic", "projection", "transfer"]), + }), + z.strictObject({ + kind: z.literal("static"), + ...identity, + bytes: z.number().int().nonnegative(), + sha256: digestSchema, + }), + z.strictObject({ + kind: z.literal("complete"), + ...identity, + descriptor: javaScriptAnalysisTransferDescriptorSchema.nullable(), + }), + z.strictObject({ + kind: z.literal("failure"), + ...identity, + reason: z.enum(["resource", "output", "io"]), + resource: z.enum(["memory", "cpu", "file-size", "transport"]).nullable(), + limits: z.record(z.string(), jsonValueSchema).nullable(), + message: z.string(), + }), +]); + +/** One decoded control frame from the independently owned worker. */ +export type JavaScriptWorkerMessage = z.output< + typeof javaScriptWorkerMessageSchema +>; diff --git a/src/javascript/analysis/JavaScriptAnalysisWorkerSchemas.ts b/src/javascript/analysis/JavaScriptAnalysisWorkerSchemas.ts new file mode 100644 index 000000000..92bfaec35 --- /dev/null +++ b/src/javascript/analysis/JavaScriptAnalysisWorkerSchemas.ts @@ -0,0 +1,582 @@ +import { z } from "zod"; +import { digestSchema } from "../../domain/digests.js"; +import type { JavaScriptStaticAnalysis } from "../../domain/javascript/javascriptStaticAnalysisTypes.js"; +import type { + JavaScriptSemanticProperty, + JavaScriptSemanticValue, +} from "../../domain/javascript/javascriptSemanticValueTypes.js"; +import { + javaScriptSemanticNodeSchema, + javaScriptSemanticRelationSchema, + javaScriptSemanticUnknownSchema, + javaScriptSemanticFingerprintSchema, + javaScriptSemanticEvidenceContextSchema, +} from "../../domain/javascript/javascriptSemanticGraphSchemas.js"; + +const range = + javaScriptSemanticNodeSchema.shape.identity.shape.source_range.unwrap(); +const count = z.number().int().nonnegative(); +const strings = z.array(z.string()); +const pathContext = z.enum(["module-specifier", "filesystem-expression"]); +const findingContext = { + module_key: z.string().nullable(), + module_runtime: z.string().optional(), + location: range, +}; +const withRuntime = ( + value: Value, +) => { + const { module_runtime, ...rest } = value; + return { + ...rest, + ...(module_runtime === undefined ? {} : { module_runtime }), + }; +}; +const staticValue = z.discriminatedUnion("status", [ + z.strictObject({ + status: z.literal("literal"), + value: z.union([z.string(), z.number(), z.boolean(), z.null()]), + expression: z.null(), + }), + z.strictObject({ + status: z.literal("dynamic"), + value: z.null(), + expression: z.string(), + }), +]); +const nullHandler = { handler_kind: z.null(), handler_location: z.null() }; +const missingHandler = { + handler_kind: z.literal("missing"), + handler_location: z.null(), +}; +const locatedHandler = { + handler_kind: z.enum([ + "inline-function", + "identifier", + "member-expression", + "dynamic-expression", + ]), + handler_location: range, +}; +const ipcChannel = (shape: Shape) => + z.union([ + z.strictObject({ + ...findingContext, + ...shape, + channel: z.string(), + channel_expression: z.null(), + }), + z.strictObject({ + ...findingContext, + ...shape, + channel: z.null(), + channel_expression: z.string(), + }), + ]); +const ipc = z + .union([ + ipcChannel({ + side: z.literal("renderer"), + operation: z.enum(["send", "send-sync", "post-message", "send-to-host"]), + mode: z.literal("send"), + ...nullHandler, + }), + ipcChannel({ + side: z.literal("renderer"), + operation: z.literal("invoke"), + mode: z.literal("invoke"), + ...nullHandler, + }), + ipcChannel({ + side: z.enum(["renderer", "main"]), + operation: z.enum(["on", "once"]), + mode: z.literal("listen"), + ...missingHandler, + }), + ipcChannel({ + side: z.enum(["renderer", "main"]), + operation: z.enum(["on", "once"]), + mode: z.literal("listen"), + ...locatedHandler, + }), + ipcChannel({ + side: z.literal("main"), + operation: z.enum(["handle", "handle-once"]), + mode: z.literal("handle"), + ...missingHandler, + }), + ipcChannel({ + side: z.literal("main"), + operation: z.enum(["handle", "handle-once"]), + mode: z.literal("handle"), + ...locatedHandler, + }), + ]) + .transform((value) => { + const { module_runtime, ...rest } = value; + return { + ...rest, + ...(module_runtime === undefined ? {} : { module_runtime }), + }; + }); +const windowShape = { + ...findingContext, + options_status: z.enum(["object-literal", "dynamic", "missing"]), + web_preferences_status: z.enum(["object-literal", "dynamic", "missing"]), + web_preferences: z.array( + z.strictObject({ name: z.string(), value: staticValue }), + ), +}; +const bridgeShape = { + ...findingContext, + world: z.enum(["main", "isolated"]), + world_id: staticValue.nullable(), + api_status: z.enum(["object-literal", "dynamic", "missing"]), + members: strings, + unknown_members: count, +}; +const utilityShape = { ...findingContext, service_name: z.string().nullable() }; +const electron = z.strictObject({ + browser_windows: z.array( + z.union([ + z + .strictObject({ + ...windowShape, + preload_path: z.string(), + preload_resolution_context: pathContext, + }) + .transform(withRuntime), + z + .strictObject({ + ...windowShape, + preload_path: z.null(), + preload_resolution_context: z.null(), + }) + .transform(withRuntime), + ]), + ), + context_bridge_apis: z.array( + z.union([ + z + .strictObject({ + ...bridgeShape, + api_key: z.string(), + api_key_expression: z.null(), + }) + .transform(withRuntime), + z + .strictObject({ + ...bridgeShape, + api_key: z.null(), + api_key_expression: z.string(), + }) + .transform(withRuntime), + z + .strictObject({ + ...bridgeShape, + api_key: z.null(), + api_key_expression: z.null(), + }) + .transform(withRuntime), + ]), + ), + ipc: z.array(ipc), + sender_validations: z.array( + z.strictObject({ + ...findingContext, + subject: z.enum([ + "sender-url", + "sender-origin", + "sender-frame", + "sender-id", + "frame-id", + "process-id", + ]), + mechanism: z.string(), + expected: staticValue, + enforcement: z.literal("unknown"), + }), + ), + utility_processes: z.array( + z.union([ + z + .strictObject({ + ...utilityShape, + module_path: z.string(), + module_resolution_context: pathContext, + module_expression: z.null(), + }) + .transform(withRuntime), + z + .strictObject({ + ...utilityShape, + module_path: z.null(), + module_resolution_context: z.null(), + module_expression: z.string(), + }) + .transform(withRuntime), + ]), + ), + native_addon_bindings: z.array( + z.strictObject({ + ...findingContext, + specifier: z.string(), + binding_kind: z.enum(["import", "require", "re-export"]), + module_kind: z.enum(["import", "require"]), + members: strings, + namespace_access: z.boolean().optional(), + dynamic_member_access: z.boolean().optional(), + }), + ), +}); +const referenceShape = { + ...findingContext, + kind: z.enum([ + "static-import", + "dynamic-import", + "require", + "worker", + "service-worker", + ]), +}; + +/** Exact producer facts recoverable before a later semantic worker failure. */ +export const javaScriptWorkerStaticAnalysisSchema = z + .strictObject({ + parse_status: z.enum(["complete", "partial", "failed"]), + parse_error_count: count, + visited_ast_nodes: count, + references: z.array( + z.union([ + z + .strictObject({ + ...referenceShape, + specifier: z.string(), + expression: z.null(), + }) + .transform(withRuntime), + z + .strictObject({ + ...referenceShape, + specifier: z.null(), + expression: z.string(), + }) + .transform(withRuntime), + ]), + ), + endpoints: z.array( + z.strictObject({ + ...findingContext, + kind: z.enum(["route", "network"]), + value: z.string(), + mechanism: z.string(), + }), + ), + storage: z.array( + z.strictObject({ + ...findingContext, + kind: z.enum([ + "local-storage", + "session-storage", + "indexed-db", + "cache-storage", + "sqlite", + ]), + name: z.string().nullable(), + mechanism: z.string(), + }), + ), + bundler_registrations: z.array( + z.strictObject({ + bundler: z.enum(["webpack", "rspack", "esbuild"]), + runtime: z.string(), + chunk_keys: strings, + unknown_chunk_keys: count, + runtime_require_name: z.string().nullable(), + runtime_module_cache_status: z.enum(["observed", "not-observed"]), + entry_module_keys: strings, + unknown_entry_module_keys: count, + async_chunk_keys: strings, + unknown_async_chunk_keys: count, + modules: z.array( + z.strictObject({ + module_key: z.string(), + factory_require_name: z.string().nullable(), + source_sha256: digestSchema, + exports: strings, + location: range, + structural_fingerprint_sha256: digestSchema, + structural_fingerprint_algorithm: z.literal("babel-ast-v1"), + }), + ), + location: range, + }), + ), + role_paths: z.array( + z.strictObject({ + ...findingContext, + role: z.enum(["preload", "renderer"]), + path: z.string(), + resolution_context: pathContext, + mechanism: z.string(), + }), + ), + source_map_urls: z.array( + z.strictObject({ declared_url: z.string(), location: range }), + ), + vendors: strings, + electron, + limitations: strings, + }) + .transform((analysis) => ({ + ...analysis, + endpoints: analysis.endpoints.map(withRuntime), + storage: analysis.storage.map(withRuntime), + role_paths: analysis.role_paths.map(withRuntime), + electron: { + ...analysis.electron, + sender_validations: analysis.electron.sender_validations.map(withRuntime), + native_addon_bindings: analysis.electron.native_addon_bindings.map( + (value) => { + const { namespace_access, dynamic_member_access, ...rest } = + withRuntime(value); + return { + ...rest, + ...(namespace_access === undefined ? {} : { namespace_access }), + ...(dynamic_member_access === undefined + ? {} + : { dynamic_member_access }), + }; + }, + ), + }, + })) satisfies z.ZodType; + +const resourceLimit = z.enum([ + "primitive-candidates", + "primitive-bytes", + "expression-depth", +]); +const primitive = z.union([z.string(), z.number(), z.boolean(), z.null()]); +const property: z.ZodType = z.lazy(() => + z.strictObject({ + name: z.string(), + value: semanticValue, + presence: z.enum(["present", "absent", "unknown-coverage"]), + }), +); +const semanticValue: z.ZodType = z.lazy(() => + z + .discriminatedUnion("status", [ + z.strictObject({ status: z.literal("literal"), value: primitive }), + z.strictObject({ + status: z.literal("union"), + values: z.array(primitive), + }), + z.discriminatedUnion("unknownProperties", [ + z.strictObject({ + status: z.literal("object"), + properties: z.array(property), + unknownProperties: z.literal(false), + omittedProperties: z.literal(0), + }), + z.strictObject({ + status: z.literal("object"), + properties: z.array(property), + unknownProperties: z.literal(true), + omittedProperties: count.nullable(), + }), + ]), + z.discriminatedUnion("unknownItems", [ + z.strictObject({ + status: z.literal("array"), + items: z.array(property), + unknownItems: z.literal(false), + omittedItems: z.literal(0), + }), + z.strictObject({ + status: z.literal("array"), + items: z.array(property), + unknownItems: z.literal(true), + omittedItems: count.nullable(), + }), + ]), + z.strictObject({ + status: z.enum(["unknown", "ambiguous", "cycle"]), + reason: z.string(), + resourceLimit: resourceLimit.optional(), + }), + ]) + .transform((value) => { + if ( + value.status === "literal" || + value.status === "union" || + value.status === "object" || + value.status === "array" + ) + return value; + const { resourceLimit: limit, ...rest } = value; + return { + ...rest, + ...(limit === undefined ? {} : { resourceLimit: limit }), + }; + }), +); +const origin = z.strictObject({ specifier: z.string(), importedPath: strings }); +const provenance = z.discriminatedUnion("status", [ + z.strictObject({ + status: z.literal("module"), + origins: z.tuple([origin]), + reason: z.null(), + }), + z.strictObject({ + status: z.literal("local"), + origins: z.tuple([]), + reason: z.null(), + }), + z.strictObject({ + status: z.literal("ambiguous"), + origins: z.array(origin), + reason: z.string(), + }), + z.strictObject({ + status: z.enum(["unknown", "cycle"]), + origins: z.tuple([]), + reason: z.string(), + }), +]); +const coverage = z + .discriminatedUnion("status", [ + z.strictObject({ + status: z.literal("complete"), + omittedCount: z.literal(0), + resourceLimits: z.array(resourceLimit).optional(), + }), + z.strictObject({ + status: z.literal("partial"), + omittedCount: z.union([z.literal(0), z.null()]), + resourceLimits: z.array(resourceLimit).optional(), + }), + z.strictObject({ + status: z.literal("failed"), + omittedCount: z.null(), + resourceLimits: z.array(resourceLimit).optional(), + }), + ]) + .transform((value) => { + const { resourceLimits, ...rest } = value; + return { + ...rest, + ...(resourceLimits === undefined ? {} : { resourceLimits }), + }; + }); +const moduleLink = z.strictObject({ + kind: z.enum(["import", "require", "export", "re-export", "commonjs-export"]), + specifier: z.string().nullable(), + importedName: z.string().nullable(), + localName: z.string().nullable(), + exportedName: z.string().nullable(), + callableId: z.string().nullable(), + location: range, +}); +const definitionKind = z.enum([ + "import", + "variable", + "parameter", + "function", + "class", + "catch", + "assignment", +]); +const scope = z.strictObject({ + scopeId: z.string(), + parentScopeId: z.string().nullable(), + kind: z.enum([ + "program", + "function", + "block", + "static-block", + "class", + "catch", + ]), + location: range, + bindingsComplete: z.boolean(), + bindingIds: strings, +}); +const binding = z.strictObject({ + bindingId: z.string(), + scopeId: z.string(), + name: z.string(), + kind: definitionKind, + mutable: z.boolean(), + definitions: z.array( + z.strictObject({ kind: definitionKind, location: range }), + ), + value: semanticValue, + provenance, +}); +const callable = z.strictObject({ + callableId: z.string(), + kind: z.enum(["function", "class", "method"]), + name: z.string().nullable(), + containerScopeId: z.string(), + bodyScopeId: z.string().nullable(), + location: range, + returnSites: z.array( + z.strictObject({ + returnSiteId: z.string(), + location: range, + identityReferenceLocation: range.nullable(), + value: semanticValue, + }), + ), + returnCoverage: z.strictObject({ + retainedCount: count, + status: z.enum(["complete", "partial"]), + omittedCount: z.union([z.literal(0), z.null()]), + }), +}); + +/** Validate each transferred graph fact independently of its emitting realm. */ +export const javaScriptWorkerGraphRecordSchema = z.discriminatedUnion("kind", [ + z.strictObject({ + kind: z.literal("evidence-context"), + value: javaScriptSemanticEvidenceContextSchema, + }), + z.strictObject({ + kind: z.literal("node"), + value: javaScriptSemanticNodeSchema, + }), + z.strictObject({ + kind: z.literal("relation"), + value: javaScriptSemanticRelationSchema, + }), + z.strictObject({ + kind: z.literal("unknown"), + value: javaScriptSemanticUnknownSchema, + }), + z.strictObject({ + kind: z.literal("fingerprint"), + value: javaScriptSemanticFingerprintSchema, + }), +]); + +/** Portable per-file header, before any graph binding or immutable proof. */ +export const javaScriptWorkerProjectionHeaderSchema = z.strictObject({ + roots: z.array(javaScriptSemanticNodeSchema.shape.node_id), + truncated: z.boolean(), +}); + +/** Module facts are transferred individually to avoid a whole-IR JSON string. */ +export const javaScriptWorkerModuleRecordSchema = z.discriminatedUnion("kind", [ + z.strictObject({ kind: z.literal("scope"), value: scope }), + z.strictObject({ kind: z.literal("binding"), value: binding }), + z.strictObject({ kind: z.literal("callable"), value: callable }), + z.strictObject({ kind: z.literal("module-link"), value: moduleLink }), +]); + +/** Coverage and limitations are independent of module collection sizes. */ +export const javaScriptWorkerModuleHeaderSchema = z.strictObject({ + coverage, + limitations: strings, +}); diff --git a/src/javascript/analysis/JavaScriptAnalysisWorkerServer.ts b/src/javascript/analysis/JavaScriptAnalysisWorkerServer.ts new file mode 100644 index 000000000..65f69829b --- /dev/null +++ b/src/javascript/analysis/JavaScriptAnalysisWorkerServer.ts @@ -0,0 +1,185 @@ +import { createHash } from "node:crypto"; +import { readFile, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { createInterface } from "node:readline"; +import { getHeapStatistics } from "node:v8"; +import { AnalysisResourceConstraintError } from "../../domain/analysisErrorCore.js"; +import type { JavaScriptArtifactFile } from "../../domain/javascript/javascriptArtifactFiles.js"; +import type { JavaScriptStaticAnalysis } from "../../domain/javascript/javascriptStaticAnalysisTypes.js"; +import type { JavaScriptModuleSemanticIr } from "../../domain/javascript/javascriptModuleSemanticIr.js"; +import type { JavaScriptSemanticFileProjection } from "../../domain/javascript/javascriptSemanticFileProjection.js"; +import { writeJavaScriptAnalysisTransfer } from "./JavaScriptAnalysisTransfer.js"; +import { javaScriptWorkerStaticAnalysisSchema } from "./JavaScriptAnalysisWorkerSchemas.js"; +import { + JAVASCRIPT_ANALYSIS_WORKER_DEFAULTS as DEFAULTS, + javaScriptWorkerRequestSchema, + javaScriptWorkerAdmissionSchema, + javaScriptWorkerMessageSchema, + type JavaScriptWorkerMessage, +} from "./JavaScriptAnalysisWorkerProtocol.js"; + +/** Trusted composition callback; this provider owns framing and temporary I/O. */ +export interface JavaScriptSourceWorkerAnalyzer { + ( + file: JavaScriptArtifactFile, + nodeBudget: number, + callbacks: { + readonly phase: ( + phase: "parse" | "static" | "semantic" | "projection", + ) => Promise; + readonly checkpoint: (facts: JavaScriptStaticAnalysis) => Promise; + }, + ): Promise<{ + readonly module: JavaScriptModuleSemanticIr; + readonly projection: JavaScriptSemanticFileProjection; + } | null>; +} + +const send = async (input: JavaScriptWorkerMessage): Promise => { + const value = javaScriptWorkerMessageSchema.parse(input); + const frame = `${JSON.stringify(value)}\n`; + if (Buffer.byteLength(frame) > DEFAULTS.protocolBytes) + throw new RangeError( + "JavaScript worker control frame exceeds its byte budget", + ); + await new Promise((resolve, reject) => { + process.stdout.write(frame, (error) => + error === null || error === undefined ? resolve() : reject(error), + ); + }); +}; + +/** Serve sequential inert-source requests from a parent-owned private runtime root. */ +export const serveJavaScriptAnalysisWorker = async ( + analyze: JavaScriptSourceWorkerAnalyzer, +): Promise => { + const root = process.cwd(); + const input = createInterface({ input: process.stdin, crlfDelay: Infinity }); + const requests = input[Symbol.asyncIterator](); + await send({ + kind: "ready", + heap_limit_bytes: getHeapStatistics().heap_size_limit, + }); + try { + for (;;) { + const next = await requests.next(); + if (next.done) break; + const line = next.value; + if (Buffer.byteLength(line) > DEFAULTS.protocolBytes) + throw new RangeError( + "JavaScript worker request exceeds its byte budget", + ); + const raw: unknown = JSON.parse(line); + const request = javaScriptWorkerRequestSchema.parse(raw); + // Function scope releases source AST/IR/results before accepting another job. + await analyzeJob(root, request, analyze, requests); + if ( + process.memoryUsage().heapUsed > + getHeapStatistics().heap_size_limit / 2 + ) + global.gc?.(); + } + } finally { + input.close(); + } +}; + +const analyzeJob = async ( + root: string, + request: ReturnType, + analyze: JavaScriptSourceWorkerAnalyzer, + requests: AsyncIterator, +): Promise => { + try { + const source = await readFile(join(root, `${request.id}.source`), "utf8"); + if ( + createHash("sha256").update(source).digest("hex") !== request.text_sha256 + ) + throw new TypeError( + "JavaScript worker source text digest does not match its request", + ); + const result = await analyze( + { + ...request.file, + text: { included: true, value: source }, + }, + request.node_budget, + { + phase: async (phase) => send({ kind: "phase", id: request.id, phase }), + checkpoint: async (facts) => { + const value = javaScriptWorkerStaticAnalysisSchema.parse(facts); + const text = JSON.stringify(value); + const bytes = Buffer.byteLength(text); + if (bytes > DEFAULTS.checkpointBytes) + throw new AnalysisResourceConstraintError( + "analyze_javascript_application", + "transport", + "JavaScript static checkpoint exceeds its complete-fact byte budget", + { + checkpoint_bytes: bytes, + checkpoint_limit_bytes: DEFAULTS.checkpointBytes, + }, + ); + await writeFile(join(root, `${request.id}.static.json`), text, { + flag: "wx", + mode: 0o600, + }); + await send({ + kind: "static", + id: request.id, + bytes, + sha256: createHash("sha256").update(text).digest("hex"), + }); + const next = await requests.next(); + if ( + next.done || + Buffer.byteLength(next.value) > DEFAULTS.protocolBytes + ) + throw new TypeError( + "JavaScript worker admission frame is missing or oversized", + ); + const raw: unknown = JSON.parse(next.value); + const admission = javaScriptWorkerAdmissionSchema.parse(raw); + if (admission.id !== request.id) + throw new TypeError( + "JavaScript worker admission names a different source job", + ); + return admission.semantic_result_budget_bytes; + }, + }, + ); + await send({ kind: "phase", id: request.id, phase: "transfer" }); + const descriptor = + result === null + ? null + : await writeJavaScriptAnalysisTransfer( + join(root, `${request.id}.records`), + result.module, + result.projection, + ); + await send({ kind: "complete", id: request.id, descriptor }); + } catch (cause: unknown) { + await send({ + kind: "failure", + id: request.id, + reason: + cause instanceof AnalysisResourceConstraintError || + cause instanceof RangeError + ? "resource" + : cause instanceof TypeError + ? "output" + : "io", + resource: + cause instanceof AnalysisResourceConstraintError + ? cause.resource + : cause instanceof RangeError + ? "memory" + : null, + limits: + cause instanceof AnalysisResourceConstraintError + ? cause.reportedLimits + : null, + message: cause instanceof Error ? cause.message : String(cause), + }); + } +}; diff --git a/src/server/registerElectronTools.ts b/src/server/registerElectronTools.ts index a0478f310..fbb19f37c 100644 --- a/src/server/registerElectronTools.ts +++ b/src/server/registerElectronTools.ts @@ -184,8 +184,23 @@ const runElectronTool = async ( progress: mcpProgressReporter(context), }), ); - if (!result.ok) + if (!result.ok) { + const partial = result.error.partialObservation; + if ( + partial !== undefined && + "evidence_id" in partial && + typeof partial.evidence_id === "string" + ) { + const recorded = options.recordEvidence?.(partial); + if (recorded !== undefined) + return options.delivery.toRecordedPartialErrorToolResult( + result.error, + partial.evidence_id, + recorded, + ); + } return options.delivery.toCallToolResult(result, contract); + } const recorded = options.recordEvidence?.(result.value); return options.delivery.toEvidenceToolResult( result.value, diff --git a/src/server/toolResult.ts b/src/server/toolResult.ts index 1f24f0a0e..6daddc695 100644 --- a/src/server/toolResult.ts +++ b/src/server/toolResult.ts @@ -3,7 +3,10 @@ import { STDIO_DEFAULT_MAX_BUFFER_SIZE } from "@modelcontextprotocol/server"; import type { ToolContract } from "../contracts/toolContractTypes.js"; import type { Evidence } from "../domain/evidence.js"; -import { projectAnalysisError } from "../domain/analysisErrorProjection.js"; +import { + projectAnalysisError, + type AnalysisErrorProjection, +} from "../domain/analysisErrorProjection.js"; import { type AnalysisError } from "../domain/analysisErrorBase.js"; import type { JsonValue } from "../domain/jsonValue.js"; import type { Result } from "../domain/result.js"; @@ -64,7 +67,33 @@ export class ToolResultDelivery { /** Project an error without allocating oversized repeated MCP text. */ toErrorToolResult(error: AnalysisError): CallToolResult { - const structuredContent = { error: projectAnalysisError(error) }; + return this.projectedErrorResult(projectAnalysisError(error)); + } + + /** Deliver the original failure with the producer-acknowledged partial Evidence reference. */ + toRecordedPartialErrorToolResult( + error: AnalysisError, + evidenceId: string, + recorded: Result, + ): CallToolResult { + if (!recorded.ok) return this.toErrorToolResult(error); + const projected = projectAnalysisError(error); + return this.projectedErrorResult({ + ...projected, + details: { + ...projected.details, + partial_observation: { + kind: "retained-evidence", + evidence_id: evidenceId, + }, + }, + }); + } + + private projectedErrorResult( + projected: AnalysisErrorProjection, + ): CallToolResult { + const structuredContent = { error: projected }; // Errors reach the wire as text only; the structured copy is a private // carrier the transport removes after its own budget check. const encoded = encodeToolResult( diff --git a/tests/boundary/javascript/javascriptSourceAnalysisWorker.test.ts b/tests/boundary/javascript/javascriptSourceAnalysisWorker.test.ts new file mode 100644 index 000000000..46987c877 --- /dev/null +++ b/tests/boundary/javascript/javascriptSourceAnalysisWorker.test.ts @@ -0,0 +1,93 @@ +import { createHash } from "node:crypto"; +import { expect, it } from "vitest"; +import { JavaScriptAnalysisWorker } from "../../../src/javascript/analysis/JavaScriptAnalysisWorker.js"; +import { spawnOwnedProviderProcess } from "../../../src/process/ProviderProcess.js"; +import type { JavaScriptArtifactFile } from "../../../src/domain/javascript/javascriptArtifactFiles.js"; + +const source = (path: string, value: string): JavaScriptArtifactFile => ({ + path, + sha256: createHash("sha256").update(value).digest("hex"), + container_sha256: "1".repeat(64), + bytes: Buffer.byteLength(value), + inventory_artifact_id: "fixture-source", + kind: "javascript", + unpacked: false, + text: { included: true, value }, +}); + +it("reuses one real owned analyzer while retaining distinct source identities and exported returns", async () => { + const launches: number[] = []; + const worker = new JavaScriptAnalysisWorker( + { heapMb: 128, timeoutMs: 30_000 }, + async (request) => { + const spawned = await spawnOwnedProviderProcess(request); + const pid = spawned.process.pid; + if (pid === undefined) + throw new Error("Owned analyzer has no observed PID"); + launches.push(pid); + return spawned; + }, + ); + try { + const first = source( + "first.mjs", + "export function create() { return { observed: 7 }; }", + ); + const second = source( + "second.mjs", + "export function next() { return { observed: 8 }; }", + ); + for (const [index, file] of [first, second].entries()) { + const result = await worker.analyze(file, 100); + if (!result.ok) throw result.error.error; + expect(result.value.javascript.parse_status).toBe("complete"); + expect( + result.value.module?.callables[0]?.returnSites[0]?.value, + ).toMatchObject({ + status: "object", + properties: [ + { name: "observed", value: { status: "literal", value: index + 7 } }, + ], + }); + expect( + result.value.projection?.nodes.every( + ({ identity }) => identity.artifact_sha256 === file.sha256, + ), + ).toBe(true); + expect( + result.value.projection?.nodes.some( + ({ identity }) => identity.module_path === file.path, + ), + ).toBe(true); + } + expect(launches).toHaveLength(1); + } finally { + await worker.close(); + } +}); + +it("reports an actual startup deadline and closes its owned resources before subsequent analysis", async () => { + const file = source("app.mjs", "export const observed = 7;"); + const expired = new JavaScriptAnalysisWorker({ heapMb: 128, timeoutMs: 1 }); + try { + const result = await expired.analyze(file, 100); + if (result.ok) + throw new Error( + "A one-millisecond fresh process deadline unexpectedly completed", + ); + expect(result.error.error).toMatchObject({ + _tag: "AnalysisTimeoutError", + timeoutMs: 1, + }); + } finally { + await expired.close(); + } + const next = new JavaScriptAnalysisWorker({ heapMb: 128, timeoutMs: 30_000 }); + try { + const result = await next.analyze(file, 100); + if (!result.ok) throw result.error.error; + expect(result.value.javascript.parse_status).toBe("complete"); + } finally { + await next.close(); + } +}); diff --git a/tests/boundary/mcp/javascriptAnalysisCancellation.test.ts b/tests/boundary/mcp/javascriptAnalysisCancellation.test.ts index 3325b1706..eb3763a81 100644 --- a/tests/boundary/mcp/javascriptAnalysisCancellation.test.ts +++ b/tests/boundary/mcp/javascriptAnalysisCancellation.test.ts @@ -12,9 +12,10 @@ import { } from "../../fixtures/binarySession.js"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; -it("stops a cancelled SDK request on the server, retains prior Evidence, and accepts subsequent work", async () => { +it("retains completed source facts and prior Evidence after SDK cancellation, then accepts subsequent work", async () => { const root = await createTestTempDirectory("rea-mcp-js-cancellation-"); await writeFile(join(root, "main.js"), "export const observed = 1;\n"); + await writeFile(join(root, "second.js"), "export const pending = 2;\n"); const session = createTestBinarySession(() => { throw new Error( "Static JavaScript analysis must not start a binary provider", @@ -74,7 +75,12 @@ it("stops a cancelled SDK request on the server, retains prior Evidence, and acc }, { signal: controller.signal, - onprogress() { + onprogress(update) { + if ( + !update.message?.startsWith("parse_javascript_source:") || + !update.message.includes("second.js") + ) + return; requested = true; controller.abort(new Error("SDK cancellation verification")); }, @@ -89,15 +95,38 @@ it("stops a cancelled SDK request on the server, retains prior Evidence, and acc name: "get_evidence_bundle", arguments: {}, }); + const records = z + .object({ + result: z.object({ + records: z.array(z.object({ evidence_id: z.string() })), + }), + }) + .parse(bundle.structuredContent).result.records; + expect(records).toContainEqual({ evidence_id: prior.evidence_id }); + const partial = records.find( + ({ evidence_id }) => evidence_id !== prior.evidence_id, + ); + if (partial === undefined) + throw new Error("Cancelled analysis discarded its completed source facts"); expect( z .object({ - result: z.object({ - records: z.array(z.object({ evidence_id: z.string() })), + statistics: z.object({ + parsed_javascript_files: z.number(), + parse_failures: z.number(), }), }) - .parse(bundle.structuredContent).result.records, - ).toEqual([{ evidence_id: prior.evidence_id }]); + .parse(session.evidenceById(partial.evidence_id)?.normalized_result) + .statistics, + ).toEqual({ parsed_javascript_files: 1, parse_failures: 0 }); + const view = await client.callTool({ + name: "inspect_analysis_view", + arguments: { + source: { kind: "retained-evidence", evidence_id: partial.evidence_id }, + view: { kind: "summary" }, + }, + }); + expect(view.isError, JSON.stringify(view)).not.toBe(true); const next = await client.callTool({ name: "analyze_javascript_application", arguments: { diff --git a/tests/boundary/mcp/javascriptAnalysisResources.test.ts b/tests/boundary/mcp/javascriptAnalysisResources.test.ts new file mode 100644 index 000000000..a2d172183 --- /dev/null +++ b/tests/boundary/mcp/javascriptAnalysisResources.test.ts @@ -0,0 +1,331 @@ +import { Client } from "@modelcontextprotocol/client"; +import { StdioClientTransport } from "@modelcontextprotocol/client/stdio"; +import { createHash } from "node:crypto"; +import { readFile, writeFile } from "node:fs/promises"; +import { join, resolve } from "node:path"; +import { expect, it } from "vitest"; +import { z } from "zod"; + +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; + +it("retains every static finding through public module views when application expansion exceeds memory, then accepts more work", async () => { + const root = await createTestTempDirectory("rea-js-resource-boundary-"); + const nextRoot = await createTestTempDirectory("rea-js-resource-recovery-"); + await writeFile(join(nextRoot, "main.js"), "export const recovered = 9;\n"); + const source = Array.from( + { length: 4000 }, + (_, index) => + `import { observed as binding${String(index)} } from "./target.js";`, + ).join("\n"); + const path = join(root, "main.js"); + await writeFile(path, source); + await writeFile(join(root, "target.js"), "export const observed = 7;\n"); + const digest = createHash("sha256").update(source).digest("hex"); + const { client, transport, diagnostics } = createResourceClient(); + try { + await client.connect(transport); + const response = await client.callTool( + { + name: "analyze_javascript_application", + arguments: { + input_path: root, + format: "directory", + max_heap_mb: 128, + detail: "summary", + }, + }, + { timeout: 60000 }, + ); + expect(response.isError).not.toBe(true); + const summary = z + .object({ + normalized_result: z.object({ + parent_evidence_id: z.string(), + summary: z.object({ + coverage: z.object({ + application: z.object({ status: z.string() }), + }), + }), + }), + }) + .parse(response.structuredContent).normalized_result; + expect(summary.summary.coverage.application.status).toBe("partial"); + const retained = { + kind: "retained-evidence", + evidence_id: summary.parent_evidence_id, + }; + const pageResponse = await client.callTool({ + name: "inspect_analysis_view", + arguments: { + source: retained, + view: { kind: "page", collection: "modules", offset: 0, limit: 100 }, + }, + }); + expect(pageResponse.isError).not.toBe(true); + const page = z + .object({ + normalized_result: z.object({ + items: z.array( + z.object({ + node_id: z.string(), + kind: z.string(), + path: z.string().nullable(), + }), + ), + }), + }) + .parse(pageResponse.structuredContent).normalized_result; + const module = page.items.find( + (item) => item.kind === "javascript-asset" && item.path === "main.js", + ); + if (module === undefined) + throw new Error("The analyzed file disappeared from public module views"); + const itemResponse = await client.callTool({ + name: "inspect_analysis_view", + arguments: { + source: retained, + view: { + kind: "item", + collection: "modules", + selector: { node_id: module.node_id }, + }, + }, + }); + expect(itemResponse.isError).not.toBe(true); + const item = z + .object({ + normalized_result: z.object({ + item: z.object({ + identity: z.object({ sha256: z.string() }), + observations: z.array( + z.object({ + properties: z.object({ + static_analysis: z + .object({ + parse_status: z.string(), + references: z.array( + z.object({ + specifier: z.string(), + location: z.object({ + start: z.object({ line: z.number() }), + }), + }), + ), + }) + .optional(), + }), + }), + ), + }), + }), + }) + .parse(itemResponse.structuredContent).normalized_result.item; + expect(item.identity.sha256).toBe(digest); + const facts = item.observations.find( + (observation) => observation.properties.static_analysis !== undefined, + )?.properties.static_analysis; + if (facts === undefined) + throw new Error("Completed static observations were discarded"); + expect(facts.parse_status).toBe("complete"); + expect( + facts.references.map((reference) => [ + reference.specifier, + reference.location.start.line, + ]), + ).toEqual( + Array.from({ length: 4000 }, (_, index) => ["./target.js", index + 1]), + ); + await client.ping(); + const next = await client.callTool({ + name: "analyze_javascript_application", + arguments: { + input_path: nextRoot, + format: "directory", + max_heap_mb: 128, + detail: "summary", + }, + }); + expect(next.isError).not.toBe(true); + await client.ping(); + expect( + createHash("sha256") + .update(await readFile(path)) + .digest("hex"), + ).toBe(digest); + expect(diagnostics()).not.toMatch(/FATAL ERROR:.*heap/iu); + } finally { + await client.close(); + await transport.close(); + } +}, 90000); + +it("returns a directly usable partial analysis reference for an oversized timeout observation", async () => { + const root = await createTestTempDirectory("rea-js-large-timeout-"); + for (let offset = 0; offset < 1500; offset += 50) + await Promise.all( + Array.from({ length: 50 }, (_, index) => + writeFile( + join(root, `file-${String(offset + index)}.js`), + "export const observed = 7;\n", + ), + ), + ); + const nextRoot = await createTestTempDirectory("rea-js-timeout-recovery-"); + await writeFile(join(nextRoot, "main.js"), "export const recovered = 9;\n"); + const { client, transport, diagnostics } = createResourceClient(); + try { + await client.connect(transport); + const response = await client.callTool( + { + name: "analyze_javascript_application", + arguments: { + input_path: root, + format: "directory", + max_heap_mb: 128, + analysis_timeout_ms: 1, + }, + }, + { timeout: 60000 }, + ); + expect(response.isError).toBe(true); + const text = response.content.find((item) => item.type === "text"); + if (text?.type !== "text") + throw new Error("Missing actionable timeout response"); + const projected = z + .object({ + error: z.object({ + code: z.string(), + details: z.object({ + partial_observation: z.object({ + kind: z.literal("retained-evidence"), + evidence_id: z.string(), + }), + }), + }), + }) + .parse(JSON.parse(text.text)); + expect(projected.error.code).toBe("provider_timeout"); + const summaryResponse = await client.callTool({ + name: "inspect_analysis_view", + arguments: { + source: projected.error.details.partial_observation, + view: { kind: "summary" }, + }, + }); + expect(summaryResponse.isError).not.toBe(true); + const summary = z + .object({ + normalized_result: z.object({ + summary: z.object({ + statistics: z.object({ + relevant_files: z.number(), + parsed_javascript_files: z.number(), + parse_failures: z.number(), + }), + coverage: z.object({ semantic_unknowns: z.number() }), + }), + }), + }) + .parse(summaryResponse.structuredContent).normalized_result.summary; + expect(summary.statistics).toEqual({ + relevant_files: 1500, + parsed_javascript_files: 0, + parse_failures: 0, + }); + expect(summary.coverage.semantic_unknowns).toBeGreaterThan(1500); + await verifyRepeatedSourcePaths( + client, + projected.error.details.partial_observation, + ); + await client.ping(); + const next = await client.callTool({ + name: "analyze_javascript_application", + arguments: { + input_path: nextRoot, + format: "directory", + max_heap_mb: 128, + detail: "summary", + }, + }); + expect(next.isError).not.toBe(true); + await client.ping(); + expect(diagnostics()).not.toMatch(/FATAL ERROR:.*heap/iu); + } finally { + await client.close(); + await transport.close(); + } +}, 90000); + +const createResourceClient = () => { + const client = new Client({ name: "javascript-resources", version: "1" }); + const transport = new StdioClientTransport({ + command: process.execPath, + args: [resolve("scripts/rea.mjs"), "mcp"], + cwd: process.cwd(), + env: { + PATH: process.env.PATH ?? "", + NODE_OPTIONS: "--max-old-space-size=512 --max-semi-space-size=8", + }, + stderr: "pipe", + }); + let diagnostics = ""; + transport.stderr?.on("data", (chunk: Buffer) => { + if (diagnostics.length < 262144) diagnostics += chunk.toString(); + }); + return { client, transport, diagnostics: () => diagnostics }; +}; + +const verifyRepeatedSourcePaths = async ( + client: Client, + source: { readonly kind: "retained-evidence"; readonly evidence_id: string }, +): Promise => { + const page = await client.callTool({ + name: "inspect_analysis_view", + arguments: { + source, + view: { kind: "page", collection: "modules", offset: 0, limit: 4 }, + }, + }); + expect(page.isError).not.toBe(true); + const modules = z + .object({ + normalized_result: z.object({ + items: z.array(z.object({ kind: z.string(), node_id: z.string() })), + }), + }) + .parse(page.structuredContent); + const asset = modules.normalized_result.items.find( + ({ kind }) => kind === "javascript-asset", + ); + if (asset === undefined) + throw new Error( + "Identical source files disappeared from the retained graph", + ); + const item = await client.callTool({ + name: "inspect_analysis_view", + arguments: { + source, + view: { + kind: "item", + collection: "modules", + selector: { node_id: asset.node_id }, + }, + }, + }); + expect(item.isError).not.toBe(true); + const observations = z + .object({ + normalized_result: z.object({ + item: z.object({ + observations: z.array(z.object({ label: z.string() })), + }), + }), + }) + .parse(item.structuredContent).normalized_result.item.observations; + expect(observations.map(({ label }) => label).sort()).toEqual( + Array.from( + { length: 1500 }, + (_, index) => `file-${String(index)}.js`, + ).sort(), + ); +}; diff --git a/tests/composition/javascript-applications/javascriptArtifactReconstruction.test.ts b/tests/composition/javascript-applications/javascriptArtifactReconstruction.test.ts index bb41d5c0b..65dd20a35 100644 --- a/tests/composition/javascript-applications/javascriptArtifactReconstruction.test.ts +++ b/tests/composition/javascript-applications/javascriptArtifactReconstruction.test.ts @@ -3,7 +3,7 @@ import { primitiveByteExpansionSource, } from "../../fixtures/javascriptPrimitiveExpansion.js"; import { createHash } from "node:crypto"; -import { mkdir, rm, writeFile } from "node:fs/promises"; +import { mkdir, readFile, rm, writeFile } from "node:fs/promises"; import { join } from "node:path"; import { createPackageWithOptions } from "@electron/asar"; @@ -23,7 +23,20 @@ import { buildJavaScriptArtifactGraph } from "../../../src/application/javascrip import { buildJavaScriptSemanticGraph, createJavaScriptSemanticGraphProjection, + projectJavaScriptSemanticFileSteps, } from "../../../src/application/javascript/JavaScriptSemanticGraphBuilder.js"; +import { + projectJavaScriptModuleSemantics, + type JavaScriptModuleSemanticIr, +} from "../../../src/domain/javascript/javascriptModuleSemanticIr.js"; +import type { JavaScriptStaticAnalysis } from "../../../src/domain/javascript/javascriptStaticAnalysisTypes.js"; +import type { JavaScriptArtifactFile } from "../../../src/domain/javascript/javascriptArtifactFiles.js"; +import { completeJavaScriptAnalysisSteps } from "../../../src/application/javascript/JavaScriptAnalysisControl.js"; +import { + writeJavaScriptAnalysisTransfer, + readJavaScriptAnalysisTransfer, +} from "../../../src/javascript/analysis/JavaScriptAnalysisTransfer.js"; +import { javaScriptWorkerStaticAnalysisSchema } from "../../../src/javascript/analysis/JavaScriptAnalysisWorkerSchemas.js"; import { createJavaScriptArtifactReader } from "../../../src/artifacts/javascript/JavaScriptArtifactReader.js"; import { scanCanonicalArtifactInventory, @@ -31,7 +44,10 @@ import { } from "../../fixtures/artifactInventory.js"; import { parseJavaScriptApplicationGraph } from "../../../src/domain/javascript/javascriptApplicationGraph.js"; import { javascriptApplicationAnalysisResultSchema } from "../../../src/domain/javascript/javascriptApplicationAnalysis.js"; -import { createJavaScriptSemanticGraph } from "../../../src/domain/javascript/javascriptSemanticGraph.js"; +import { + createJavaScriptSemanticGraph, + isValidatedImmutableJavaScriptSemanticGraph, +} from "../../../src/domain/javascript/javascriptSemanticGraph.js"; import { parseJavaScriptSemanticGraph } from "../../../src/domain/javascript/javascriptSemanticGraphSerialization.js"; import { writeJavaScriptArtifactFixture } from "../../fixtures/javascriptArtifactApplication.js"; @@ -48,7 +64,10 @@ it("reports file progress and honors cancellation before the next source", async } }, }), - ).rejects.toMatchObject({ reason: "cancelled" }); + ).rejects.toMatchObject({ + _tag: "AnalysisCancelledError", + partialObservation: { operation: "analyze_javascript_application" }, + }); expect(progress).toHaveLength(2); expect(progress[0]).not.toBe(progress[1]); }); @@ -61,6 +80,7 @@ it("preserves graph commitments and export shapes when consuming file-local IR", import { readFile } from "node:fs"; export { readFile as read }; export function create(value) { return { value, nested: { enabled: true } }; } + export function configured(condition) { return { values: [1, null, undefined], choice: condition ? 1 : 2, ...external }; } function unexported() { return "private"; } `, ); @@ -96,6 +116,7 @@ it("preserves graph commitments and export shapes when consuming file-local IR", ); expect(exported?.semantic?.ir.callables.map(({ name }) => name)).toEqual([ "create", + "configured", ]); expect( full.files @@ -103,11 +124,121 @@ it("preserves graph commitments and export shapes when consuming file-local IR", ?.semantic?.ir.callables.map(({ name }) => name), ).toContain("unexported"); expect(parseJavaScriptSemanticGraph(semantic)).toEqual(semantic); + const transferred = createJavaScriptSemanticGraphProjection(); + const wireRoot = await createTestTempDirectory("rea-javascript-transfer-"); + const decodedModules = new Map(); + const decodedStatic = new Map(); + let rejectedInvalidTransfer = false; + for (const [index, analyzed] of full.files.entries()) { + if (analyzed.javascript !== null) { + const raw: unknown = JSON.parse(JSON.stringify(analyzed.javascript)); + const decodedFacts = javaScriptWorkerStaticAnalysisSchema.parse(raw); + expect(decodedFacts).toEqual(analyzed.javascript); + decodedStatic.set(analyzed.file.path, decodedFacts); + } + if (analyzed.semantic === null) continue; + const fragment = await completeJavaScriptAnalysisSteps( + projectJavaScriptSemanticFileSteps( + analyzed.file, + analyzed.semantic.ir, + transferred.remainingFileNodeBudget(), + ), + ); + const module = projectJavaScriptModuleSemantics(analyzed.semantic.ir); + const wirePath = join(wireRoot, `${String(index)}.jsonl`); + const descriptor = await writeJavaScriptAnalysisTransfer( + wirePath, + module, + fragment, + ); + const decoded = await readJavaScriptAnalysisTransfer( + wirePath, + descriptor, + analyzed.file, + ); + expect(decoded.module).toEqual(module); + decodedModules.set(analyzed.file.path, decoded.module); + transferred.adoptFileProjection(decoded.projection); + if (!rejectedInvalidTransfer) { + await rejectInvalidTransfer( + wirePath, + descriptor, + analyzed.file, + wireRoot, + ); + rejectedInvalidTransfer = true; + } + } + const decodedGraph = buildJavaScriptArtifactGraph(snapshot, files, { + ...compact, + files: compact.files.map((analyzed) => { + const module = decodedModules.get(analyzed.file.path); + return { + ...analyzed, + javascript: + decodedStatic.get(analyzed.file.path) ?? analyzed.javascript, + semantic: module === undefined ? analyzed.semantic : { ir: module }, + }; + }), + }); + expect(decodedGraph).toEqual(graph); + const resealed = await completeJavaScriptAnalysisSteps( + transferred.finishImmutableSteps( + snapshot.manifest.root_sha256, + decodedGraph, + ), + ); + expect(resealed).toEqual(semantic); + expect(isValidatedImmutableJavaScriptSemanticGraph(resealed)).toBe(true); } finally { await reader.close(); } }); +const rejectInvalidTransfer = async ( + wirePath: string, + descriptor: Awaited>, + file: JavaScriptArtifactFile, + wireRoot: string, +): Promise => { + await expect( + readJavaScriptAnalysisTransfer(wirePath, descriptor, { + ...file, + sha256: "0".repeat(64), + }), + ).rejects.toThrow("different artifact source"); + await expect( + readJavaScriptAnalysisTransfer( + wirePath, + { ...descriptor, sha256: "0".repeat(64) }, + file, + ), + ).rejects.toThrow("digest does not match"); + const wire = await readFile(wirePath, "utf8"); + const duplicate = wire + .split("\n") + .find((line) => line.startsWith('{"kind":"node"')); + if (duplicate === undefined) + throw new Error("Fixture emitted no semantic node record"); + const invalid = `${wire}${duplicate}\n`; + const invalidPath = join(wireRoot, "duplicate.jsonl"); + await writeFile(invalidPath, invalid); + await expect( + readJavaScriptAnalysisTransfer( + invalidPath, + { + ...descriptor, + bytes: Buffer.byteLength(invalid), + records: descriptor.records + 1, + graph_bytes: descriptor.graph_bytes + Buffer.byteLength(duplicate) + 1, + graph_records: descriptor.graph_records + 1, + sha256: createHash("sha256").update(invalid).digest("hex"), + }, + file, + ), + ).rejects.toThrow("Duplicate JavaScript transfer node identity"); +}; + it("reports semantic value resource limits in application graph coverage", async () => { const root = await createTestTempDirectory("rea-javascript-semantic-limit-"); await writeFile(join(root, "app.js"), primitiveCandidateExpansionSource());