mirror of
https://github.com/morluto/rea.git
synced 2026-10-12 02:19:24 +00:00
fix(browser): bound script metadata and preserve omissions
This commit is contained in:
@@ -138,7 +138,7 @@ REA retains local observation data and removes only URL userinfo credentials, ex
|
||||
- Console observations with a stack source retain call type, argument types, timestamp, and source location. When console text capture is selected, already-delivered primitive values are retained verbatim; objects, getters, and remote properties are never expanded.
|
||||
- WebSocket observations retain direction, opcode, and payload byte length. When shape capture is selected, text frames are classified as text or complete value-free JSON shape; binary bytes, hashes, prefixes, and raw frames are never retained.
|
||||
- Storage observations always redact values. Key names, IndexedDB names, and cache names require `include_storage_keys`; stable content fingerprints require the additional `include_storage_fingerprints` selection. Cookie fingerprints cover cookies applicable to the current main-frame URL, including its path, rather than every cookie path on that origin. Other storage remains scoped to the origin. Cache bodies above 64 KiB and partial IndexedDB remote objects make fingerprint coverage incomplete, so identical observations remain `unknown`.
|
||||
- Script metadata is included only when CDP supplies a URL on an allowed origin. Stable keys exclude transient CDP script IDs, and exact transient raw URLs are used only during script/resource reconciliation. URL-less scripts are excluded because their origin cannot be established. When CDP supplies an execution-context association, the accepted script retains its authorized frame ID for later attribution. Source content is omitted unless explicitly requested and is returned inline as a self-verifying artifact with its SHA-256 digest.
|
||||
- Script metadata is included only when CDP supplies a URL on an allowed origin. Stable keys exclude transient CDP script IDs, and exact transient raw URLs are used only during script/resource reconciliation. URL-less scripts are excluded because their origin cannot be established. When CDP supplies an execution-context association, the accepted script retains its authorized frame ID for later attribution. Script metadata admission bounds retained identity and source-map strings to 8 MiB and 50,000 records. When a source-map URL cannot fit, the script identity remains and source-map coverage reports `resource_budget_exhausted`; rejected script observations are counted without retaining their IDs. Source content is omitted unless explicitly requested and is returned inline as a self-verifying artifact with its SHA-256 digest.
|
||||
|
||||
Frames, resources, scripts, events, and workers outside the selected origin scope are excluded. An explicit `allowed_origins` list can include additional origins; excluded target details are counted without being exposed.
|
||||
Retained workers include validated opener-target and parent-frame IDs when CDP
|
||||
@@ -148,20 +148,21 @@ prove static module ownership.
|
||||
## Completeness and limits
|
||||
|
||||
The default inspection window is 500 ms; callers choose its duration. The
|
||||
capture returns all in-scope frames, DOM and accessibility nodes, scripts,
|
||||
capture returns in-scope frames, DOM and accessibility nodes, admitted scripts,
|
||||
resources, workers, storage names, and observed network, console, and WebSocket
|
||||
events. Full selected text, script sources, and JSON shapes are returned inline.
|
||||
The result distinguishes
|
||||
`complete_within_window`, `policy_filtered`, `attach_limited`, and `truncated`
|
||||
coverage. It reports excluded items and any values omitted or truncated by
|
||||
validation; it does not drop records to meet an internal collection quota.
|
||||
validation and resource bounds. Omitted script metadata makes its affected
|
||||
section explicitly truncated.
|
||||
Disallowed-origin entries are filtered before retention.
|
||||
|
||||
CDP discovery, WebSocket connection, and commands continue until the peer responds or the caller cancels the operation; REA has no internal byte, message-count, or pending-command quota for these exchanges. Without caller cancellation, a stalled peer can leave an operation waiting. On cancellation, REA closes only its own WebSocket, waiting up to one second before terminating it. Malformed unsolicited events and malformed replies to pending commands poison the connection so later commands fail closed. When a correlated reply reports a session ID, it must match the command's selected session. An optional method is reported unavailable only when the browser rejects it as unknown; other command rejections retain their reported failure.
|
||||
|
||||
Network and console coverage starts only after REA attaches and enables the relevant CDP domains. `prior_activity_available` is always `false`; absence from these arrays is not evidence that an event never occurred. If CDP reuses a request ID without supplying redirect metadata, REA retains the predecessor observation and marks network coverage incomplete.
|
||||
|
||||
Source maps are not fetched by inspection. Bundle analysis fetches them only when `fetch_source_maps` is selected, with exact-origin redirect checks and no credentials, cookies, or referrer. Each fetch operation has a 30-second deadline and a 64 MiB aggregate streamed-response budget across redirects. A deadline or byte-limit failure remains explicitly unavailable and makes bundle completeness partial. Every requested map, mapping, original source, AST node, and finding from parsed content is returned inline. Original source artifacts with `.ts`, `.tsx`, `.mts`, or `.cts` suffixes carry `text/typescript`; other source names retain `text/javascript`.
|
||||
Source maps are not fetched by inspection. Bundle analysis fetches them only when `fetch_source_maps` is selected, with exact-origin redirect checks and no credentials, cookies, or referrer. Each fetch operation has a 30-second deadline and a 64 MiB aggregate streamed-response budget across redirects. A deadline or byte-limit failure remains explicitly unavailable and makes bundle completeness partial. Source-map decoding also bounds raw resolved identities to 64 MiB and aggregate inline evidence to 32 MiB. Failed maps preserve their failure context when it fits; if another context cannot fit, collection stops with the full requested count, the returned processed count, and an explicit unknown-results limitation. Retained mappings and original sources are returned inline. See [source-location limits](web-source-location.md) for the structural decoder bounds. Original source artifacts with `.ts`, `.tsx`, `.mts`, or `.cts` suffixes carry `text/typescript`; other source names retain `text/javascript`.
|
||||
|
||||
## Non-goals and threat model
|
||||
|
||||
|
||||
@@ -246,6 +246,8 @@ Do not call handcrafted utility output or synthetic binary builders real-data
|
||||
goldens. Keep unsupported binary layouts and malformed boundaries as targeted
|
||||
regressions until a real fixture establishes equivalent coverage.
|
||||
|
||||
`verify:browser` exercises source-map failure isolation and expanded-output limits through the compiled CLI and stdio MCP with real Chrome. It also submits five 2 MiB source-map annotations, checks retained script identities and explicit map omissions, and closes its owned fixture target. These public workflows replace redundant metadata-admission assertions; replacement and document-reset boundaries remain focused producer regressions.
|
||||
|
||||
`verify:browser` also captures a source-owned noise canvas as a real PNG above
|
||||
8 MiB through the CLI and stdio MCP, with complete byte/digest parity and real PNG
|
||||
decoding. Its SDK client explicitly permits the larger inline JSON response;
|
||||
|
||||
@@ -0,0 +1,209 @@
|
||||
import { createServer } from "node:http";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { Client } from "@modelcontextprotocol/client";
|
||||
import { StdioClientTransport } from "@modelcontextprotocol/client/stdio";
|
||||
import { artifactCliEvidence, artifactMcpResult } from "./artifact-e2e.mjs";
|
||||
|
||||
const ANNOTATION_BYTES = 2 * 1024 * 1024;
|
||||
const SCRIPT_COUNT = 5;
|
||||
|
||||
/** Verify real Chrome scriptParsed metadata is bounded before public capture retention. */
|
||||
export async function verifyBrowserCaptureMetadataBudget(endpoint) {
|
||||
const marker = "rea-capture-metadata-budget";
|
||||
let page = "";
|
||||
let pageReady = false;
|
||||
let fixtureOrigin = "";
|
||||
const server = createServer(
|
||||
{ maxHeaderSize: 3 * 1024 * 1024 },
|
||||
(request, response) => {
|
||||
if (request.url?.startsWith("/maps/") === true) {
|
||||
response.writeHead(200, {
|
||||
"content-type": "application/source-map+json",
|
||||
});
|
||||
response.end(
|
||||
JSON.stringify({
|
||||
version: 3,
|
||||
names: [],
|
||||
sources: [`${fixtureOrigin}/source.ts`],
|
||||
sourcesContent: ["export const retainedSource = true;"],
|
||||
mappings: "AAAA",
|
||||
}),
|
||||
);
|
||||
return;
|
||||
}
|
||||
if (request.url === "/capture-metadata-ready") {
|
||||
pageReady = true;
|
||||
response.writeHead(204).end();
|
||||
return;
|
||||
}
|
||||
if (request.url !== "/capture-metadata-budget") {
|
||||
response.writeHead(404).end();
|
||||
return;
|
||||
}
|
||||
response.writeHead(200, { "content-type": "text/html; charset=utf-8" });
|
||||
response.end(page);
|
||||
},
|
||||
);
|
||||
await new Promise((resolve, reject) => {
|
||||
server.once("error", reject);
|
||||
server.listen(0, "127.0.0.1", resolve);
|
||||
});
|
||||
let targetId;
|
||||
try {
|
||||
const address = server.address();
|
||||
if (address === null || typeof address === "string")
|
||||
throw new Error("Browser metadata fixture did not bind a TCP port");
|
||||
fixtureOrigin = `http://127.0.0.1:${String(address.port)}`;
|
||||
const annotations = Array.from(
|
||||
{ length: SCRIPT_COUNT },
|
||||
(_, index) =>
|
||||
`<script>globalThis["${marker}${index}"]=true;\n//# sourceMappingURL=${fixtureOrigin}/maps/${String(index)}-${"x".repeat(ANNOTATION_BYTES)}.map\n</script>`,
|
||||
).join("");
|
||||
page = `<!doctype html><meta charset="utf-8">${annotations}<script>fetch("/capture-metadata-ready")</script>`;
|
||||
const fixtureUrl = `${fixtureOrigin}/capture-metadata-budget`;
|
||||
const created = await fetch(
|
||||
`${endpoint}/json/new?${encodeURIComponent(fixtureUrl)}`,
|
||||
{ method: "PUT" },
|
||||
);
|
||||
if (!created.ok)
|
||||
throw new Error(`Chrome target creation failed: HTTP ${created.status}`);
|
||||
const target = await created.json();
|
||||
if (typeof target.id !== "string" || target.id.length === 0)
|
||||
throw new Error("Chrome target creation returned no target ID");
|
||||
targetId = target.id;
|
||||
|
||||
const deadline = Date.now() + 15_000;
|
||||
let served = false;
|
||||
while (!served && Date.now() < deadline) {
|
||||
await new Promise((resolve) => setTimeout(resolve, 20));
|
||||
served = pageReady;
|
||||
}
|
||||
if (!served) throw new Error("Chrome did not request the metadata fixture");
|
||||
|
||||
const evidence = await artifactCliEvidence("inspect-web-page", endpoint, [
|
||||
targetId,
|
||||
"--allowed-origins",
|
||||
fixtureOrigin,
|
||||
"--observation-ms",
|
||||
"0",
|
||||
]);
|
||||
const cliFacts = assertMetadataBudget(evidence.normalized_result);
|
||||
const bundle = await artifactCliEvidence("analyze-web-bundle", endpoint, [
|
||||
targetId,
|
||||
"--allowed-origins",
|
||||
fixtureOrigin,
|
||||
"--observation-ms",
|
||||
"0",
|
||||
"--fetch-source-maps",
|
||||
]);
|
||||
assertBundleMetadataOmissions(
|
||||
bundle.normalized_result,
|
||||
cliFacts.omittedMaps,
|
||||
);
|
||||
const transport = new StdioClientTransport({
|
||||
command: process.execPath,
|
||||
args: [fileURLToPath(new URL("../rea.mjs", import.meta.url)), "mcp"],
|
||||
env: { PATH: process.env.PATH ?? "", REA_LOG_LEVEL: "silent" },
|
||||
stderr: "pipe",
|
||||
});
|
||||
const client = new Client({ name: "browser-metadata-e2e", version: "1" });
|
||||
try {
|
||||
await client.connect(transport);
|
||||
assertMetadataBudget(
|
||||
await artifactMcpResult(client, "inspect_web_page", {
|
||||
cdp_endpoint: endpoint,
|
||||
allowed_origins: [fixtureOrigin],
|
||||
target_id: targetId,
|
||||
observation_ms: 0,
|
||||
}),
|
||||
);
|
||||
assertBundleMetadataOmissions(
|
||||
await artifactMcpResult(client, "analyze_web_bundle", {
|
||||
cdp_endpoint: endpoint,
|
||||
allowed_origins: [fixtureOrigin],
|
||||
target_id: targetId,
|
||||
observation_ms: 0,
|
||||
fetch_source_maps: true,
|
||||
}),
|
||||
cliFacts.omittedMaps,
|
||||
);
|
||||
} finally {
|
||||
try {
|
||||
await client.close();
|
||||
} finally {
|
||||
await transport.close();
|
||||
}
|
||||
}
|
||||
return { ...cliFacts, public_cli: true, public_stdio_mcp: true };
|
||||
} finally {
|
||||
try {
|
||||
if (targetId !== undefined) {
|
||||
const closed = await fetch(
|
||||
`${endpoint}/json/close/${encodeURIComponent(targetId)}`,
|
||||
);
|
||||
if (!closed.ok)
|
||||
throw new Error(
|
||||
`Chrome fixture target cleanup failed: HTTP ${closed.status}`,
|
||||
);
|
||||
}
|
||||
} finally {
|
||||
server.closeAllConnections();
|
||||
await new Promise((resolve, reject) =>
|
||||
server.close((error) =>
|
||||
error === undefined ? resolve() : reject(error),
|
||||
),
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function assertMetadataBudget(result) {
|
||||
const scripts = result.scripts.items.filter(
|
||||
(script) =>
|
||||
script.url.includes("capture-metadata-budget") &&
|
||||
script.length >= ANNOTATION_BYTES,
|
||||
);
|
||||
const retainedMaps = scripts.filter(
|
||||
(script) => script.source_map_url !== null,
|
||||
).length;
|
||||
const omittedMaps = SCRIPT_COUNT - retainedMaps;
|
||||
const exclusion = result.completeness.excluded.find(
|
||||
(item) =>
|
||||
item.section === "source_maps" &&
|
||||
item.reason === "resource_budget_exhausted",
|
||||
);
|
||||
if (
|
||||
scripts.length !== SCRIPT_COUNT ||
|
||||
retainedMaps <= 0 ||
|
||||
retainedMaps >= SCRIPT_COUNT ||
|
||||
exclusion?.count !== omittedMaps ||
|
||||
!result.completeness.truncated_sections.includes("source_maps")
|
||||
)
|
||||
throw new Error(
|
||||
`Real Chrome source-map metadata admission was not reported accurately: scripts=${String(scripts.length)}, retained_maps=${String(retainedMaps)}, excluded=${String(exclusion?.count)}`,
|
||||
);
|
||||
return {
|
||||
scripts: scripts.length,
|
||||
retainedMaps,
|
||||
omittedMaps,
|
||||
declaredBytes: ANNOTATION_BYTES * SCRIPT_COUNT,
|
||||
};
|
||||
}
|
||||
|
||||
function assertBundleMetadataOmissions(result, omittedMaps) {
|
||||
const unknown = result.unknowns.find(
|
||||
({ dimension, reason }) =>
|
||||
dimension === "source_maps" &&
|
||||
reason.includes(String(omittedMaps)) &&
|
||||
reason.includes("source-map declarations"),
|
||||
);
|
||||
if (
|
||||
result.observations.source_maps.status !== "included" ||
|
||||
result.observations.source_maps.items.length === 0 ||
|
||||
result.completeness.status !== "partial" ||
|
||||
unknown === undefined
|
||||
)
|
||||
throw new Error(
|
||||
"Bundle analysis lost capture omissions despite valid retained maps",
|
||||
);
|
||||
}
|
||||
@@ -41,6 +41,7 @@ import { verifyBrowserNetworkEvidence } from "./lib/browser-network-e2e.mjs";
|
||||
import { verifyBrowserScriptExport } from "./lib/browser-script-export-e2e.mjs";
|
||||
import { verifyBrowserModules } from "./lib/browser-module-e2e.mjs";
|
||||
import { verifyBrowserDomDestinations } from "./lib/browser-dom-destinations-e2e.mjs";
|
||||
import { verifyBrowserCaptureMetadataBudget } from "./lib/browser-capture-metadata-budget-e2e.mjs";
|
||||
import { artifactCliEvidence, artifactMcpResult } from "./lib/artifact-e2e.mjs";
|
||||
|
||||
const REAL_BROWSER_STARTUP_TIMEOUT_MS = 60_000;
|
||||
@@ -332,6 +333,7 @@ try {
|
||||
target_id: target,
|
||||
});
|
||||
const moduleTrace = await verifyBrowserModules(executable);
|
||||
const captureMetadata = await verifyBrowserCaptureMetadataBudget(endpoint);
|
||||
report = {
|
||||
browser: observed.value.browser.product,
|
||||
endpoint,
|
||||
@@ -353,6 +355,7 @@ try {
|
||||
networkEvidence,
|
||||
scriptExport,
|
||||
moduleTrace,
|
||||
captureMetadata,
|
||||
domDestinations,
|
||||
browserScenarioCli: true,
|
||||
browserScenarioAttachCleanup: "disconnected-external",
|
||||
|
||||
@@ -65,6 +65,7 @@ export class CdpCaptureCompleteness {
|
||||
reason === "invalid_protocol_value"
|
||||
)
|
||||
this.#unavailable.add(section);
|
||||
if (reason === "resource_budget_exhausted") this.#truncated.add(section);
|
||||
const key = `${section}\0${reason}`;
|
||||
const existing = this.#excluded.get(key);
|
||||
const combined =
|
||||
|
||||
@@ -26,7 +26,11 @@ import {
|
||||
isMainFrameNavigation,
|
||||
} from "./CdpCaptureEventHelpers.js";
|
||||
import type { CdpCaptureEventsState } from "./CdpCaptureEventState.js";
|
||||
import type { CapturedScript, NetworkState } from "./CdpCaptureEventTypes.js";
|
||||
import {
|
||||
CDP_CAPTURE_SCRIPT_METADATA_LIMITS,
|
||||
type CapturedScript,
|
||||
type NetworkState,
|
||||
} from "./CdpCaptureEventTypes.js";
|
||||
|
||||
export const handleExecutionContextCreated = (
|
||||
state: CdpCaptureEventsState,
|
||||
@@ -64,12 +68,18 @@ export const handleScriptParsed = (
|
||||
state.completeness.exclude("scripts", "invalid_protocol_value");
|
||||
return;
|
||||
}
|
||||
const replacedBytes = state.scriptMetadataBytesById.get(scriptId) ?? 0;
|
||||
state.scriptMetadataBudgetOmissionsById.delete(scriptId);
|
||||
if (replacedBytes > 0) {
|
||||
state.scripts.delete(scriptId);
|
||||
state.scriptMetadataBytesById.delete(scriptId);
|
||||
state.retainedScriptMetadataBytes -= replacedBytes;
|
||||
}
|
||||
if (sanitized === undefined) {
|
||||
state.completeness.exclude("scripts", exclusionReasonForUrl(rawUrl));
|
||||
return;
|
||||
}
|
||||
const sourceMap = sourceMapForScript(params.sourceMapURL, rawUrl, state);
|
||||
const script: CapturedScript = {
|
||||
const baseScript = {
|
||||
scriptId,
|
||||
rawUrl,
|
||||
url: sanitized.url,
|
||||
@@ -84,21 +94,79 @@ export const handleScriptParsed = (
|
||||
),
|
||||
isModule: params.isModule === true,
|
||||
language: cdpStringValue(params.scriptLanguage) ?? null,
|
||||
sourceMapUrl: sourceMap.sanitized,
|
||||
sourceMapRawUrl: sourceMap.raw,
|
||||
executionContextKey: executionContextKey(params.executionContextId),
|
||||
};
|
||||
const baseBytes = scriptMetadataStringBytes(baseScript);
|
||||
const sourceMapDeclared =
|
||||
typeof params.sourceMapURL === "string" && params.sourceMapURL.length > 0;
|
||||
if (
|
||||
state.scripts.size >= CDP_CAPTURE_SCRIPT_METADATA_LIMITS.scripts ||
|
||||
baseBytes >
|
||||
CDP_CAPTURE_SCRIPT_METADATA_LIMITS.retainedBytes -
|
||||
state.retainedScriptMetadataBytes
|
||||
) {
|
||||
state.rejectedScriptMetadataBudgetCount += 1;
|
||||
if (sourceMapDeclared) state.rejectedSourceMapMetadataBudgetCount += 1;
|
||||
return;
|
||||
}
|
||||
const sourceMap = sourceMapForScript(
|
||||
params.sourceMapURL,
|
||||
rawUrl,
|
||||
state,
|
||||
CDP_CAPTURE_SCRIPT_METADATA_LIMITS.retainedBytes -
|
||||
state.retainedScriptMetadataBytes -
|
||||
baseBytes,
|
||||
);
|
||||
const { budgetExhausted, ...sourceMapFields } = sourceMap;
|
||||
const script: CapturedScript = { ...baseScript, ...sourceMapFields };
|
||||
if (budgetExhausted)
|
||||
state.scriptMetadataBudgetOmissionsById.set(scriptId, true);
|
||||
const retainedBytes =
|
||||
baseBytes +
|
||||
stringByteLength(sourceMap.sourceMapUrl) +
|
||||
stringByteLength(sourceMap.sourceMapRawUrl);
|
||||
state.retainedScriptMetadataBytes += retainedBytes;
|
||||
state.scriptMetadataBytesById.set(scriptId, retainedBytes);
|
||||
state.scripts.set(scriptId, script);
|
||||
};
|
||||
|
||||
const scriptMetadataStringBytes = (script: {
|
||||
readonly scriptId: string;
|
||||
readonly rawUrl: string;
|
||||
readonly url: string;
|
||||
readonly origin: string | null;
|
||||
readonly hash: string;
|
||||
readonly language: string | null;
|
||||
readonly executionContextKey: string | null;
|
||||
}): number =>
|
||||
stringByteLength(script.scriptId) +
|
||||
stringByteLength(script.rawUrl) +
|
||||
stringByteLength(script.url) +
|
||||
stringByteLength(script.origin) +
|
||||
stringByteLength(script.hash) +
|
||||
stringByteLength(script.language) +
|
||||
stringByteLength(script.executionContextKey);
|
||||
|
||||
const stringByteLength = (value: string | null): number =>
|
||||
value === null ? 0 : Buffer.byteLength(value, "utf8");
|
||||
|
||||
const sourceMapForScript = (
|
||||
value: unknown,
|
||||
scriptUrl: string,
|
||||
state: CdpCaptureEventsState,
|
||||
): { readonly sanitized: string | null; readonly raw: string | null } => {
|
||||
remainingBytes: number,
|
||||
): {
|
||||
readonly sourceMapUrl: string | null;
|
||||
readonly sourceMapRawUrl: string | null;
|
||||
readonly budgetExhausted: boolean;
|
||||
} => {
|
||||
const declaredUrl = cdpStringValue(value);
|
||||
if (declaredUrl === undefined || declaredUrl === "")
|
||||
return { sanitized: null, raw: null };
|
||||
return {
|
||||
sourceMapUrl: null,
|
||||
sourceMapRawUrl: null,
|
||||
budgetExhausted: false,
|
||||
};
|
||||
let rawUrl: string;
|
||||
try {
|
||||
rawUrl = new URL(declaredUrl, scriptUrl).href;
|
||||
@@ -106,12 +174,36 @@ const sourceMapForScript = (
|
||||
// Unparseable URLs are recorded as unsupported, not as failures.
|
||||
void cause;
|
||||
state.completeness.exclude("source_maps", "unsupported_url");
|
||||
return { sanitized: null, raw: null };
|
||||
return {
|
||||
sourceMapUrl: null,
|
||||
sourceMapRawUrl: null,
|
||||
budgetExhausted: false,
|
||||
};
|
||||
}
|
||||
const sanitized = allowedSanitizedUrl(rawUrl, state.allowedOrigins);
|
||||
if (sanitized !== undefined) return { sanitized: sanitized.url, raw: rawUrl };
|
||||
if (sanitized !== undefined) {
|
||||
if (
|
||||
stringByteLength(rawUrl) + stringByteLength(sanitized.url) >
|
||||
remainingBytes
|
||||
) {
|
||||
return {
|
||||
sourceMapUrl: null,
|
||||
sourceMapRawUrl: null,
|
||||
budgetExhausted: true,
|
||||
};
|
||||
}
|
||||
return {
|
||||
sourceMapUrl: sanitized.url,
|
||||
sourceMapRawUrl: rawUrl,
|
||||
budgetExhausted: false,
|
||||
};
|
||||
}
|
||||
state.completeness.exclude("source_maps", exclusionReasonForUrl(rawUrl));
|
||||
return { sanitized: null, raw: null };
|
||||
return {
|
||||
sourceMapUrl: null,
|
||||
sourceMapRawUrl: null,
|
||||
budgetExhausted: false,
|
||||
};
|
||||
};
|
||||
|
||||
export const handleRequestWillBeSent = (
|
||||
|
||||
@@ -9,6 +9,11 @@ export interface CdpCaptureEventsState {
|
||||
readonly input: InspectWebPageInput;
|
||||
readonly allowedOrigins: ReadonlySet<string>;
|
||||
readonly scripts: Map<string, CapturedScript>;
|
||||
readonly scriptMetadataBudgetOmissionsById: Map<string, true>;
|
||||
rejectedScriptMetadataBudgetCount: number;
|
||||
rejectedSourceMapMetadataBudgetCount: number;
|
||||
readonly scriptMetadataBytesById: Map<string, number>;
|
||||
retainedScriptMetadataBytes: number;
|
||||
readonly executionContextFrames: Map<string, string>;
|
||||
readonly network: Map<string, NetworkState>;
|
||||
readonly networkRequestTimestamps: Map<string, number>;
|
||||
|
||||
@@ -15,3 +15,9 @@ export interface CapturedScript {
|
||||
}
|
||||
|
||||
export type NetworkState = WebPageInspection["network"]["requests"][number];
|
||||
|
||||
/** Bounds retained CDP script identities independently from source text capture. */
|
||||
export const CDP_CAPTURE_SCRIPT_METADATA_LIMITS = {
|
||||
retainedBytes: 8 * 1024 * 1024,
|
||||
scripts: 50_000,
|
||||
} as const;
|
||||
|
||||
@@ -15,7 +15,13 @@ export { type CapturedScript } from "./CdpCaptureEventTypes.js";
|
||||
|
||||
/** Event accumulator that validates and normalizes payloads at ingestion. */
|
||||
export class CdpCaptureEvents implements CdpCaptureEventsState {
|
||||
#scriptMetadataBudgetExclusionsRecorded = false;
|
||||
readonly scripts = new Map<string, CapturedScript>();
|
||||
readonly scriptMetadataBudgetOmissionsById = new Map<string, true>();
|
||||
rejectedScriptMetadataBudgetCount = 0;
|
||||
rejectedSourceMapMetadataBudgetCount = 0;
|
||||
readonly scriptMetadataBytesById = new Map<string, number>();
|
||||
retainedScriptMetadataBytes = 0;
|
||||
readonly executionContextFrames = new Map<string, string>();
|
||||
readonly network = new Map<string, NetworkState>();
|
||||
readonly networkRequestTimestamps = new Map<string, number>();
|
||||
@@ -51,6 +57,12 @@ export class CdpCaptureEvents implements CdpCaptureEventsState {
|
||||
this.navigationDuringCapture = false;
|
||||
this.committedDocument = undefined;
|
||||
this.scripts.clear();
|
||||
this.scriptMetadataBudgetOmissionsById.clear();
|
||||
this.rejectedScriptMetadataBudgetCount = 0;
|
||||
this.rejectedSourceMapMetadataBudgetCount = 0;
|
||||
this.scriptMetadataBytesById.clear();
|
||||
this.retainedScriptMetadataBytes = 0;
|
||||
this.#scriptMetadataBudgetExclusionsRecorded = false;
|
||||
this.executionContextFrames.clear();
|
||||
this.network.clear();
|
||||
this.networkRequestTimestamps.clear();
|
||||
@@ -81,6 +93,28 @@ export class CdpCaptureEvents implements CdpCaptureEventsState {
|
||||
this.committedDocument = undefined;
|
||||
}
|
||||
|
||||
/** Record final script metadata omissions after all scriptParsed replacements. */
|
||||
recordScriptMetadataBudgetExclusions(): void {
|
||||
if (this.#scriptMetadataBudgetExclusionsRecorded) return;
|
||||
this.#scriptMetadataBudgetExclusionsRecorded = true;
|
||||
const omittedScripts = this.rejectedScriptMetadataBudgetCount;
|
||||
const omittedSourceMaps =
|
||||
this.rejectedSourceMapMetadataBudgetCount +
|
||||
this.scriptMetadataBudgetOmissionsById.size;
|
||||
if (omittedScripts > 0)
|
||||
this.completeness.exclude(
|
||||
"scripts",
|
||||
"resource_budget_exhausted",
|
||||
omittedScripts,
|
||||
);
|
||||
if (omittedSourceMaps > 0)
|
||||
this.completeness.exclude(
|
||||
"source_maps",
|
||||
"resource_budget_exhausted",
|
||||
omittedSourceMaps,
|
||||
);
|
||||
}
|
||||
|
||||
ingest(event: CdpEvent): void {
|
||||
const params = recordValue(event.params);
|
||||
if (params === undefined) return;
|
||||
|
||||
@@ -162,6 +162,7 @@ const captureAuthorizedPage = async (
|
||||
if (state.events.navigationDuringCapture || completedUrl !== attachedUrl)
|
||||
throw new BrowserObservationError("inspect_web_page", "target_changed");
|
||||
await report(context.progress, 3, "Normalizing browser evidence");
|
||||
state.events.recordScriptMetadataBudgetExclusions();
|
||||
return {
|
||||
inspection: normalizedInspection(state, {
|
||||
attachedUrl,
|
||||
|
||||
@@ -36,6 +36,7 @@ const browserExclusionReasonSchema = z.enum([
|
||||
"out_of_target_scope",
|
||||
"provider_unavailable",
|
||||
"invalid_protocol_value",
|
||||
"resource_budget_exhausted",
|
||||
]);
|
||||
export type BrowserExclusionReason = z.infer<
|
||||
typeof browserExclusionReasonSchema
|
||||
|
||||
@@ -17,12 +17,19 @@ export const buildWebBundleAnalysis = (
|
||||
const unavailable = inspection.scripts.items
|
||||
.filter((script) => !script.source.included)
|
||||
.map(({ script_key }) => script_key);
|
||||
const omittedScripts = captureBudgetOmission(inspection, "scripts");
|
||||
const omittedSourceMaps =
|
||||
sourceMaps.status === "not_requested"
|
||||
? undefined
|
||||
: captureBudgetOmission(inspection, "source_maps");
|
||||
const sourceMapIncomplete =
|
||||
sourceMaps.status === "partial" || sourceMaps.status === "unavailable";
|
||||
const partial =
|
||||
accumulator.parseFailures > 0 ||
|
||||
unavailable.length > 0 ||
|
||||
sourceMapIncomplete;
|
||||
sourceMapIncomplete ||
|
||||
omittedScripts !== undefined ||
|
||||
omittedSourceMaps !== undefined;
|
||||
return webBundleAnalysisSchema.parse({
|
||||
capture: buildCaptureObservation(inspection, sourceScripts),
|
||||
observations: {
|
||||
@@ -41,6 +48,8 @@ export const buildWebBundleAnalysis = (
|
||||
unavailable,
|
||||
parseFailures: accumulator.parseFailures,
|
||||
sourceMaps,
|
||||
omittedScripts,
|
||||
omittedSourceMaps,
|
||||
}),
|
||||
completeness: buildCompleteness({
|
||||
partial,
|
||||
@@ -83,9 +92,20 @@ interface UnknownsInput {
|
||||
readonly unavailable: string[];
|
||||
readonly parseFailures: number;
|
||||
readonly sourceMaps: WebBundleAnalysis["observations"]["source_maps"];
|
||||
readonly omittedScripts: number | null | undefined;
|
||||
readonly omittedSourceMaps: number | null | undefined;
|
||||
}
|
||||
|
||||
const buildUnknowns = (input: UnknownsInput): WebBundleAnalysis["unknowns"] => [
|
||||
...(input.omittedScripts === undefined
|
||||
? []
|
||||
: [
|
||||
{
|
||||
dimension: "script_inventory",
|
||||
reason: captureBudgetReason("script records", input.omittedScripts),
|
||||
affected_script_keys: [],
|
||||
},
|
||||
]),
|
||||
...(input.unavailable.length === 0
|
||||
? []
|
||||
: [
|
||||
@@ -106,6 +126,18 @@ const buildUnknowns = (input: UnknownsInput): WebBundleAnalysis["unknowns"] => [
|
||||
),
|
||||
},
|
||||
]),
|
||||
...(input.omittedSourceMaps === undefined
|
||||
? []
|
||||
: [
|
||||
{
|
||||
dimension: "source_maps" as const,
|
||||
reason: captureBudgetReason(
|
||||
"source-map declarations",
|
||||
input.omittedSourceMaps,
|
||||
),
|
||||
affected_script_keys: [],
|
||||
},
|
||||
]),
|
||||
...(input.sourceMaps.status === "not_requested" ||
|
||||
input.sourceMaps.status === "included"
|
||||
? []
|
||||
@@ -123,6 +155,25 @@ const buildUnknowns = (input: UnknownsInput): WebBundleAnalysis["unknowns"] => [
|
||||
]),
|
||||
];
|
||||
|
||||
const captureBudgetOmission = (
|
||||
inspection: WebPageInspection,
|
||||
section: "scripts" | "source_maps",
|
||||
): number | null | undefined => {
|
||||
const exclusions = inspection.completeness.excluded.filter(
|
||||
(exclusion) =>
|
||||
exclusion.section === section &&
|
||||
exclusion.reason === "resource_budget_exhausted",
|
||||
);
|
||||
if (exclusions.length === 0) return undefined;
|
||||
if (exclusions.some(({ count }) => count === null)) return null;
|
||||
return exclusions.reduce((sum, { count }) => sum + (count ?? 0), 0);
|
||||
};
|
||||
|
||||
const captureBudgetReason = (label: string, count: number | null): string =>
|
||||
count === null
|
||||
? `Capture omitted an unknown number of ${label} because its resource budget was exhausted.`
|
||||
: `Capture omitted ${String(count)} ${label} because its resource budget was exhausted.`;
|
||||
|
||||
interface CompletenessInput {
|
||||
readonly partial: boolean;
|
||||
readonly parsedScripts: number;
|
||||
|
||||
@@ -0,0 +1,120 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import { CdpCaptureEvents } from "../../../src/browser/CdpCaptureEvents.js";
|
||||
import { inspectWebPageInputSchema } from "../../../src/domain/browserObservation.js";
|
||||
import { CDP_CAPTURE_SCRIPT_METADATA_LIMITS } from "../../../src/browser/CdpCaptureEventTypes.js";
|
||||
|
||||
describe("CDP script metadata admission", () => {
|
||||
it("releases a replaced script ID's prior bytes and resets admission on a new document", () => {
|
||||
const events = captureEvents();
|
||||
const origin = "https://app.example";
|
||||
events.beginAuthorizedFrame("frame-1");
|
||||
const emit = (scriptId: string, bytes: number): void => {
|
||||
events.ingest({
|
||||
method: "Debugger.scriptParsed",
|
||||
params: {
|
||||
scriptId,
|
||||
url: `${origin}/app.js`,
|
||||
sourceMapURL: `/maps/${"x".repeat(bytes)}.map`,
|
||||
},
|
||||
});
|
||||
};
|
||||
|
||||
emit("same-id", 3 * 1024 * 1024);
|
||||
emit("same-id", 2 * 1024 * 1024);
|
||||
emit("next-id", 1024 * 1024);
|
||||
|
||||
expect(events.scripts.get("same-id")?.sourceMapUrl).toContain(
|
||||
"x".repeat(2 * 1024 * 1024),
|
||||
);
|
||||
expect(events.scripts.get("next-id")?.sourceMapUrl).toContain(
|
||||
"x".repeat(1024 * 1024),
|
||||
);
|
||||
expect(events.retainedScriptMetadataBytes).toBeLessThanOrEqual(
|
||||
CDP_CAPTURE_SCRIPT_METADATA_LIMITS.retainedBytes,
|
||||
);
|
||||
const normalizedAway = `./`.repeat(1_200_000) + "map.js.map";
|
||||
events.ingest({
|
||||
method: "Debugger.scriptParsed",
|
||||
params: {
|
||||
scriptId: "normalized-url",
|
||||
url: `${origin}/app.js`,
|
||||
sourceMapURL: normalizedAway,
|
||||
},
|
||||
});
|
||||
expect(events.scripts.get("normalized-url")?.sourceMapUrl).toBe(
|
||||
`${origin}/map.js.map`,
|
||||
);
|
||||
|
||||
events.beginAuthorizedFrame("frame-2");
|
||||
emit("new-document", 5 * 1024 * 1024);
|
||||
emit("new-document", 1024 * 1024);
|
||||
events.recordScriptMetadataBudgetExclusions();
|
||||
expect(
|
||||
events.completeness
|
||||
.snapshot()
|
||||
.excluded.find(
|
||||
({ section, reason }) =>
|
||||
section === "source_maps" && reason === "resource_budget_exhausted",
|
||||
),
|
||||
).toBeUndefined();
|
||||
|
||||
events.beginAuthorizedFrame("frame-3");
|
||||
emit("new-document", 3 * 1024 * 1024);
|
||||
expect(events.scripts.get("new-document")?.sourceMapUrl).toContain(
|
||||
"x".repeat(3 * 1024 * 1024),
|
||||
);
|
||||
});
|
||||
|
||||
it("does not retain IDs for scripts rejected by byte or record limits", () => {
|
||||
const events = captureEvents();
|
||||
const origin = "https://app.example";
|
||||
events.beginAuthorizedFrame("frame-1");
|
||||
const emit = (scriptId: string): void => {
|
||||
events.ingest({
|
||||
method: "Debugger.scriptParsed",
|
||||
params: { scriptId, url: `${origin}/app.js` },
|
||||
});
|
||||
};
|
||||
|
||||
emit("x".repeat(CDP_CAPTURE_SCRIPT_METADATA_LIMITS.retainedBytes + 1));
|
||||
for (
|
||||
let index = 0;
|
||||
index < CDP_CAPTURE_SCRIPT_METADATA_LIMITS.scripts;
|
||||
index += 1
|
||||
)
|
||||
emit(`script-${String(index)}`);
|
||||
for (let index = 0; index < 10; index += 1)
|
||||
emit(`rejected-${String(index)}`);
|
||||
|
||||
expect(events.scripts.size).toBe(
|
||||
CDP_CAPTURE_SCRIPT_METADATA_LIMITS.scripts,
|
||||
);
|
||||
expect(events.scriptMetadataBudgetOmissionsById.size).toBe(0);
|
||||
expect(events.rejectedScriptMetadataBudgetCount).toBe(11);
|
||||
events.recordScriptMetadataBudgetExclusions();
|
||||
expect(
|
||||
events.completeness
|
||||
.snapshot()
|
||||
.excluded.find(
|
||||
({ section, reason }) =>
|
||||
section === "scripts" && reason === "resource_budget_exhausted",
|
||||
),
|
||||
).toEqual({
|
||||
section: "scripts",
|
||||
reason: "resource_budget_exhausted",
|
||||
count: 11,
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
const captureEvents = (): CdpCaptureEvents =>
|
||||
new CdpCaptureEvents(
|
||||
inspectWebPageInputSchema.parse({
|
||||
cdp_endpoint: "http://127.0.0.1:9222",
|
||||
allowed_origins: ["https://app.example"],
|
||||
target_id: "page",
|
||||
observation_ms: 0,
|
||||
}),
|
||||
new Set(["https://app.example"]),
|
||||
);
|
||||
Reference in New Issue
Block a user