diff --git a/.github/workflows/real-web-network-captures.yml b/.github/workflows/real-web-network-captures.yml new file mode 100644 index 000000000..91681043a --- /dev/null +++ b/.github/workflows/real-web-network-captures.yml @@ -0,0 +1,75 @@ +name: Real historical web network captures + +on: + workflow_dispatch: + pull_request: + paths: + - ".github/workflows/real-web-network-captures.yml" + - "bridge/mitmproxy/**" + - "src/browser/history/**" + - "src/domain/webNetworkCapture*.ts" + - "src/domain/analysisError*.ts" + - "src/contracts/errorSchemas.ts" + - "src/domain/explicitSensitive*.ts" + - "src/application/WebNetworkCapture*" + - "src/composition/webNetworkCaptures.ts" + - "src/cli/webNetworkCaptureCommands.ts" + - "src/server/registerWebNetworkCaptureTool.ts" + - "src/server/createServer.ts" + - "src/server/toolRegistrationOptions.ts" + - "src/cli.ts" + - "src/cliCommandNames.ts" + - "src/contracts/toolContracts.ts" + - "src/contracts/toolEffects.ts" + - "src/process/OwnedCommand.ts" + - "src/process/ProviderProcess.ts" + - "src/process/ProviderDeadline.ts" + - "src/process/ProcessOwnership*.ts" + - "src/process/PrivateRuntimeRoot.ts" + - "src/process/capture/ProcessCaptureLifecycle.ts" + - "src/artifacts/readStableArtifact*.ts" + - "src/artifacts/ArtifactReader.ts" + - "src/contracts/webNetworkCaptureToolContracts.ts" + - "scripts/verify-web-network-captures.mjs" + - "scripts/fixtures/generate-mitmproxy-capture.py" + - "package.json" + - "package-lock.json" + +permissions: + contents: read + +concurrency: + group: real-web-network-captures-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +env: + HUSKY: "0" + NODE_OPTIONS: --max-old-space-size=768 + +jobs: + real-web-network-captures: + runs-on: ubuntu-24.04 + timeout-minutes: 15 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version-file: .nvmrc + cache: npm + - name: Use pinned package manager + run: npm install --global npm@11.16.0 + - run: npm ci + - name: Acquire fixed upstream native profile in runner temporary storage + shell: bash + run: | + capture_tool_root="${RUNNER_TEMP}/rea-mitmproxy-12.2.3" + mkdir -p "${capture_tool_root}" + curl --fail --location --retry 3 --max-time 180 \ + https://downloads.mitmproxy.org/12.2.3/mitmproxy-12.2.3-linux-x86_64.tar.gz \ + --output "${capture_tool_root}/mitmproxy.tar.gz" + echo '2e95286b618fa6fd33e5e62a78c2e5112571d85f42ec2bac29b97ee242bdb5c5 mitmproxy.tar.gz' \ + | (cd "${capture_tool_root}" && sha256sum --check --strict) + tar -xzf "${capture_tool_root}/mitmproxy.tar.gz" -C "${capture_tool_root}" + echo "REA_MITMDUMP_COMMAND=${capture_tool_root}/mitmdump" >> "${GITHUB_ENV}" + - name: Verify upstream producer fixtures through public CLI and MCP + run: npm run verify:web:network-captures diff --git a/README.md b/README.md index b130e6367..e02ce7e6c 100644 --- a/README.md +++ b/README.md @@ -410,7 +410,7 @@ See [native investigation](docs/native-investigation.md) for keyed archives, ins | Managed PE/CLI | 7 | .NET identity, metadata, CIL instructions, native dependencies, reconstruction imports, and build comparisons | | Firmware | 2 | Linux firmware region inspection and explicit extraction | | Android APK | 5 | package and manifest declarations, class search, member inventories, method decompilation, and incoming static references | -| Browser observation | 11 | page structure, network metadata, scripts, source maps, WebMCP discovery, screenshots, and capture comparisons | +| Browser observation | 12 | page structure, network metadata, scripts, source maps, WebMCP discovery, screenshots, and capture comparisons | | Electron analysis | 5 | renderer observation, static app mapping, and static/runtime reconciliation | | JavaScript runtime | 2 | Node/Electron Inspector target discovery, script locations, and execution-context events | | Application workflows | 13 | captured website script export; Android/Apple inventory projections; cross-layer feature traces, build comparisons, historical source mapping, static return-shape comparison, and reconstruction checks | @@ -473,6 +473,14 @@ rea inspect-web-page http://127.0.0.1:9222 TARGET_ID --json The eight passive browser tools work through both CLI and MCP. They inspect the selected page without navigating, clicking, or evaluating its JavaScript. Credentials, cookies, authorization headers, and raw payload values are not retained. A request selects whether to include script sources, accessibility text, screenshots, or console and payload summaries. REA cannot observe activity that happened before it attached. See [browser observation](docs/browser-observation.md) for browser startup, capture options, and limits. +### Historical web network captures + +For retained HAR or native mitmproxy evidence, use `inspect_web_network_capture` +or `inspect-web-network-capture`. It runs offline, preserves producer identities +and exposed bytes, and does not invent live attribution. See +[historical web network captures](docs/web-network-captures.md) for upstream +profiles, caller-selected records and credential exclusions. + ### Controlled browser scenarios `capture_browser_scenario` runs a caller-declared sequence of browser actions through diff --git a/README_ar.md b/README_ar.md index 9cb606d39..b96ea451a 100644 --- a/README_ar.md +++ b/README_ar.md @@ -222,7 +222,7 @@ rea setup | PE/CLI المُدار | 7 | هوية تجميعات .NET وبياناتها الوصفية وتعليمات CIL والاعتماديات الأصلية واستيراد إعادة البناء ومقارنة الإصدارات | | البرامج الثابتة | 2 | فحص مناطق البرامج الثابتة على Linux واستخراجها الصريح | | Android APK | 5 | بيانات الحزمة وmanifest والبحث عن الأصناف وقوائم الأعضاء وفك ترجمة الأساليب والمراجع الثابتة | -| مراقبة المتصفح | 11 | بنية الصفحات وبيانات الشبكة والبرامج النصية وخرائط المصدر واكتشاف WebMCP والصور ومقارنة الالتقاط | +| مراقبة المتصفح | 12 | بنية الصفحات وبيانات الشبكة والبرامج النصية وخرائط المصدر واكتشاف WebMCP والصور ومقارنة الالتقاط | | تحليل Electron | 5 | مراقبة العارضين وخريطة التطبيق الثابتة والمواءمة بين نتائج الثبات والتشغيل | | وقت تشغيل JavaScript | 2 | اكتشاف أهداف Node/Electron Inspector ومواضع البرامج وأحداث سياقات التنفيذ | | سير عمل التطبيقات | 13 | تصدير النصوص البرمجية الملتقطة، تتبع الميزات عبر الطبقات ومقارنة الإصدارات ومواءمة المصدر التاريخي ومقارنة أشكال القيم المعادة والتحقق من إعادة الإنشاء | diff --git a/README_ja.md b/README_ja.md index 5e0efc78d..0a54ad635 100644 --- a/README_ja.md +++ b/README_ja.md @@ -245,7 +245,7 @@ TypeScript と SQLite を使って私のプロジェクト向けに実装して | マネージド PE/CLI | 7 | .NET の識別情報、メタデータ、CIL 命令、ネイティブ依存関係、再構築結果のインポート、ビルド比較 | | ファームウェア | 2 | Linux ファームウェア領域の検査と明示的な展開 | | Android APK | 5 | パッケージと manifest の宣言、クラス検索、メンバー一覧、メソッドの逆コンパイル、静的な被参照 | -| ブラウザー観察 | 11 | ページ構成、ネットワークメタデータ、スクリプト、ソースマップ、WebMCP の検出、スクリーンショット、キャプチャ比較 | +| ブラウザー観察 | 12 | ページ構成、ネットワークメタデータ、スクリプト、ソースマップ、WebMCP の検出、スクリーンショット、キャプチャ比較 | | Electron 解析 | 5 | レンダラーの観察、静的なアプリのマッピング、静的結果と実行時結果の照合 | | JavaScript 実行時 | 2 | Node/Electron Inspector の対象検出、スクリプトの位置、実行コンテキストのイベント | | アプリワークフロー | 13 | 取得した Web スクリプトのエクスポート、Android/Apple の目録の投影、層をまたぐ機能トレース、ビルド比較、過去のソースとの対応付け、静的な戻り値形状の比較、再構築の検証 | diff --git a/README_ko.md b/README_ko.md index 56de8102a..7e1cee874 100644 --- a/README_ko.md +++ b/README_ko.md @@ -212,7 +212,7 @@ REA는 1–5단계의 바이너리 분석을 처리합니다. 6단계는 에이 | 관리 PE/CLI | 7 | .NET 식별, 메타데이터, CIL 명령어, 네이티브 종속성, 재구성 가져오기, 빌드 비교 | | 펌웨어 | 2 | Linux 펌웨어 영역 검사 및 명시적 추출 | | Android APK | 5 | 패키지와 manifest 선언, 클래스 검색, 멤버 목록, 메서드 디컴파일, 정적 참조 | -| 브라우저 관찰 | 11 | 페이지 구조, 네트워크 메타데이터, 스크립트, 소스 맵, WebMCP 탐색, 스크린샷, 캡처 비교 | +| 브라우저 관찰 | 12 | 페이지 구조, 네트워크 메타데이터, 스크립트, 소스 맵, WebMCP 탐색, 스크린샷, 캡처 비교 | | Electron 분석 | 5 | 렌더러 관찰, 정적 앱 매핑, 정적·런타임 결과 연결 | | JavaScript 런타임 | 2 | Node/Electron Inspector 대상 탐색, 스크립트 위치, 실행 컨텍스트 이벤트 | | 앱 워크플로 | 13 | 캡처한 웹 스크립트 내보내기, 계층 간 기능 추적, 빌드 비교, 히스토리 소스 매핑, 정적 반환 구조 비교, 재구현 검증 | diff --git a/README_zh.md b/README_zh.md index 02540890a..d2eec3158 100644 --- a/README_zh.md +++ b/README_zh.md @@ -211,7 +211,7 @@ REA 负责第 1–5 步中的二进制分析。第 6 步由智能体使用其常 | 托管 PE/CLI | 7 | .NET 程序集身份、元数据、CIL 指令、原生依赖声明、重建导入与构建比较 | | 固件 | 2 | Linux 固件区域检查与显式提取 | | Android APK | 5 | 包与 manifest 声明、类搜索、成员清单、方法反编译与静态引用 | -| 浏览器观察 | 11 | 页面结构、网络元数据、脚本、来源映射、WebMCP 发现、截图与捕获比较 | +| 浏览器观察 | 12 | 页面结构、网络元数据、脚本、来源映射、WebMCP 发现、截图与捕获比较 | | Electron 分析 | 5 | 渲染进程观察、静态应用映射、静态/运行时结果关联 | | JavaScript 运行时 | 2 | Node/Electron Inspector 目标发现、脚本位置与执行上下文事件 | | 应用工作流 | 13 | 捕获的网站脚本导出、跨层功能追踪、构建比较、历史源码映射、静态返回结构比较与重建验证 | diff --git a/bridge/mitmproxy/capture.py b/bridge/mitmproxy/capture.py new file mode 100644 index 000000000..cc24331fe --- /dev/null +++ b/bridge/mitmproxy/capture.py @@ -0,0 +1,210 @@ +"""REA offline adapter using mitmproxy's unchanged native tnetstring decoder. + +No FlowReader migrations, HAR conversion, proxy listener, or target requests. +The containing mitmdump process and its configuration directory are REA-owned. +""" + +import base64 +import hashlib +import json +import math +import re +import resource +from pathlib import Path + +from mitmproxy import ctx, version +from mitmproxy.io import tnetstring + +PROFILE = "12.2.3" +OUTPUT_BYTES = 96 * 1024 * 1024 +CREDENTIALS = {b"authorization", b"proxy-authorization", b"cookie", b"set-cookie"} + + +class CaptureFailure(Exception): + def __init__(self, reason, message, pointer=""): + self.reason, self.pointer = reason, pointer + super().__init__(message) + + +def pointer(parent, key): + return parent + "/" + str(key).replace("~", "~0").replace("/", "~1") + + +class BoundedReader: + """Refuse malicious declared lengths before the upstream parser allocates.""" + def __init__(self, handle, size): + self.handle, self.size = handle, size + + def read(self, count): + if count < 0 or count > self.size - self.handle.tell(): + raise CaptureFailure("format", "Native record declares bytes beyond the retained capture.") + return self.handle.read(count) + + +def project_record(state, secrets): + binaries, numbers, redactions = [], [], [] + byte_secrets = [] + for secret in secrets: + try: + byte_secrets.append(secret.encode("utf-8", errors="strict")) + except UnicodeEncodeError: + # Non-scalar JSON strings have no UTF-8 byte representation. + # Keep them in secrets for the ordinary string projection. + pass + + def redact(text, path, original): + if any(secret in text or secret in original for secret in secrets): + redactions.append({"pointer": path, "reason": "explicit-sensitive-value"}) + return None + return text + + def transport_value(value, path, transport_url): + if not (transport_url or re.fullmatch(r"/(?:backup/)*request/(?:path|authority)", path)): + return value, False + binary = isinstance(value, bytes) + pattern = rb"^([\x00-\x20]*(?:[a-z][a-z0-9+.-]*:)?//)[^/?#]*@" if binary else r"^([\x00-\x20]*(?:[a-z][a-z0-9+.-]*:)?//)[^/?#]*@" + replacement = rb"\1" if binary else r"\1" + separator = b"@" if binary else "@" + safe_url = re.sub(pattern, replacement, value, flags=re.I) + if path.endswith("/authority") and separator in safe_url: + safe_url = safe_url.rsplit(separator, 1)[1] + changed = safe_url != value + if changed: + redactions.append({"pointer": path, "reason": "transport-credential"}) + return safe_url, changed + + def validate_omitted(value, path, depth): + before = len(binaries), len(numbers), len(redactions) + visit(value, path, depth) + del binaries[before[0]:] + del numbers[before[1]:] + del redactions[before[2]:] + + def visit(value, path, depth=0, credential=False, transport_url=False): + if depth > 64: + raise CaptureFailure("input-limit", "Native capture exceeds the 64-level evidence nesting budget.", path) + if credential: + validate_omitted(value, path, depth) + redactions.append({"pointer": path, "reason": "transport-credential"}) + if isinstance(value, bytes): + binaries.append({"pointer": path, "representation": "producer-bytes", "state": "redacted", "content_base64": None, "bytes": None, "sha256": None}) + return None + if isinstance(value, bytes): + safe_bytes, credential_url = transport_value(value, path, transport_url) + content_base64 = base64.b64encode(value).decode("ascii") + digest = hashlib.sha256(value).hexdigest() + hidden = any(secret in value or secret in safe_bytes for secret in byte_secrets) or any(secret in content_base64 or secret in digest for secret in secrets) + safe_text = None + try: + safe_text = safe_bytes.decode("utf-8", errors="strict") + except UnicodeDecodeError: + pass + if hidden: + redactions.append({"pointer": path, "reason": "explicit-sensitive-value"}) + exclude = hidden or credential_url + binaries.append({"pointer": path, "representation": "producer-bytes", "state": "redacted" if exclude else "retained", "content_base64": None if exclude else content_base64, "bytes": None if exclude else len(value), "sha256": None if exclude else digest}) + if hidden: + return None + return safe_text + if isinstance(value, str): + safe_text, _ = transport_value(value, path, transport_url) + return redact(safe_text, path, value) + if isinstance(value, bool) or value is None: + return value + if isinstance(value, (int, float)): + literal = repr(value) + if any(secret in literal for secret in secrets): + redactions.append({"pointer": path, "reason": "explicit-sensitive-value"}) + return None + numbers.append({"pointer": path, "producer_type": "integer" if isinstance(value, int) else "float", "literal": literal}) + return value if (abs(value) <= 9007199254740991 if isinstance(value, int) else math.isfinite(value)) else None + if isinstance(value, (list, tuple)): + fields = re.fullmatch(r"/(?:backup/)*(?:request|response)/(?:headers|trailers)", path) + if fields: + result = [] + for index, field in enumerate(value): + if not isinstance(field, (list, tuple)) or len(field) != 2 or not isinstance(field[0], bytes): + raise CaptureFailure("format", "Native header/trailer is not an ordered byte pair.", pointer(path, index)) + result.append([visit(field[0], pointer(pointer(path, index), 0), depth + 2), visit(field[1], pointer(pointer(path, index), 1), depth + 2, field[0].lower() in CREDENTIALS, field[0].lower() in {b"location", b"referer", b"origin"})]) + return result + return [visit(item, pointer(path, index), depth + 1) for index, item in enumerate(value)] + if isinstance(value, dict): + result = {} + for key, item in value.items(): + if not isinstance(key, str): + raise CaptureFailure("format", "Native record has a non-string dictionary key.", path) + if key == "__proto__": + raise CaptureFailure("unsupported", "Native object contains a __proto__ member that the current JSON schema boundary cannot preserve.", path) + if any(secret in key for secret in secrets): + # Validate the omitted subtree too, then discard all of its + # coordinates rather than inventing a replacement identity. + validate_omitted(item, pointer(path, key), depth + 1) + redactions.append({"pointer": path, "reason": "explicit-sensitive-value", "scope": "property-name"}) + continue + result[key] = visit(item, pointer(path, key), depth + 1) + return result + raise CaptureFailure("decoder", "Native parser returned an unexpected value type: " + type(value).__name__ + ".", path) + + return {"reported": visit(state, ""), "binary_fields": binaries, "numeric_literals": numbers, "redactions": redactions} + + +def decode(request): + if version.VERSION != PROFILE: + raise CaptureFailure("unsupported", "Expected mitmproxy " + PROFILE + "; observed " + version.VERSION + ".") + records = [] + capture = Path(request["snapshot_path"]) + size = capture.stat().st_size + with capture.open("rb") as handle: + reader = BoundedReader(handle, size) + while handle.tell() < size: + start = handle.tell() + try: + state = tnetstring.load(reader) + except CaptureFailure: + raise + except RecursionError: + raise CaptureFailure("input-limit", "Native parser exceeded its recursion budget at byte offset " + str(start) + ".") from None + except (ValueError, IndexError, UnicodeError, EOFError): + raise CaptureFailure("format", "Malformed native record at byte offset " + str(start) + ".") from None + if not isinstance(state, dict): + raise CaptureFailure("format", "Native flow record must be a dictionary at byte offset " + str(start) + ".") + records.append({"ordinal": len(records), "location": {"kind": "byte-range", "offset": start, "bytes": handle.tell() - start}, **project_record(state, request["sensitive_values"]), "limitations": ["Fields are original native states without FlowReader migration. Binary fields contain exact producer bytes; reported UTF-8 strings for those fields are derived display views. Unknown flow/version extensions are retained without interpretation."]}) + return {"decoder": {"id": "mitmproxy-native-tnetstring", "name": "REA offline native mitmproxy adapter", "version": PROFILE}, "container": {"reported": None, "numeric_literals": [], "redactions": [], "records_pointer": None}, "total_records": len(records), "records": records} + + +def failure_reply(error): + """Keep producer constraints, resource exhaustion and adapter defects distinct.""" + if isinstance(error, CaptureFailure): + return {"ok": False, "reason": error.reason, "message": str(error), "pointer": error.pointer} + if isinstance(error, MemoryError): + return {"ok": False, "reason": "resource-limit", "message": "Native decoder exhausted its 768 MiB address-space budget.", "pointer": ""} + return {"ok": False, "reason": "decoder", "message": "Offline native decoder failed unexpectedly: " + type(error).__name__ + ".", "pointer": ""} + + +class OfflineCapture: + def load(self, loader): + loader.add_option("rea_request_path", str, "", "REA private offline capture request") + + def running(self): + resource.setrlimit(resource.RLIMIT_AS, (768 * 1024 * 1024, 768 * 1024 * 1024)) + resource.setrlimit(resource.RLIMIT_CPU, (30, 30)) + resource.setrlimit(resource.RLIMIT_FSIZE, (OUTPUT_BYTES, OUTPUT_BYTES)) + resource.setrlimit(resource.RLIMIT_CORE, (0, 0)) + request = json.loads(Path(ctx.options.rea_request_path).read_text(encoding="utf-8")) + try: + response = {"ok": True, "value": decode(request)} + except Exception as error: + response = failure_reply(error) + try: + try: + output = json.dumps(response, ensure_ascii=True, allow_nan=False, separators=(",", ":")).encode("ascii") + except Exception as error: + output = json.dumps(failure_reply(error), separators=(",", ":")).encode("ascii") + if len(output) > OUTPUT_BYTES: + output = b'{"ok":false,"reason":"limit","message":"Native reply exceeds the complete-evidence output budget.","pointer":""}' + Path(request["reply_path"]).write_bytes(output) + finally: + ctx.master.shutdown() + + +addons = [OfflineCapture()] diff --git a/docs/error-contract.schema.json b/docs/error-contract.schema.json index 5f276b551..c1286e5d0 100644 --- a/docs/error-contract.schema.json +++ b/docs/error-contract.schema.json @@ -35,6 +35,74 @@ "required": ["code", "category", "message", "retryable", "remediation"], "additionalProperties": false }, + { + "type": "object", + "properties": { + "code": { "type": "string", "const": "access_denied" }, + "category": { + "type": "string", + "enum": [ + "invalid_input", + "unsupported_provider", + "integrity_mismatch", + "truncated", + "cancelled", + "timeout", + "unavailable", + "execution_failure" + ] + }, + "message": { "type": "string", "minLength": 1 }, + "retryable": { "type": "boolean" }, + "remediation": { + "type": "object", + "properties": { "action": { "type": "string", "minLength": 1 } }, + "required": ["action"], + "additionalProperties": false + }, + "details": { + "type": "object", + "propertyNames": { "type": "string" }, + "additionalProperties": { "$ref": "#/$defs/__schema0" } + } + }, + "required": ["code", "category", "message", "retryable", "remediation"], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "code": { "type": "string", "const": "artifact_changed" }, + "category": { + "type": "string", + "enum": [ + "invalid_input", + "unsupported_provider", + "integrity_mismatch", + "truncated", + "cancelled", + "timeout", + "unavailable", + "execution_failure" + ] + }, + "message": { "type": "string", "minLength": 1 }, + "retryable": { "type": "boolean" }, + "remediation": { + "type": "object", + "properties": { "action": { "type": "string", "minLength": 1 } }, + "required": ["action"], + "additionalProperties": false + }, + "details": { + "type": "object", + "propertyNames": { "type": "string" }, + "additionalProperties": { "$ref": "#/$defs/__schema0" } + } + }, + "required": ["code", "category", "message", "retryable", "remediation"], + "additionalProperties": false + }, { "type": "object", "properties": { diff --git a/docs/product-catalog.json b/docs/product-catalog.json index 138ec5bf1..4da83b0a4 100644 --- a/docs/product-catalog.json +++ b/docs/product-catalog.json @@ -7,10 +7,10 @@ "client": "2.3.1", "core": "2.3.1" }, - "skill_version": "27" + "skill_version": "28" }, "tools": { - "total": 133, + "total": 134, "families": [ { "id": "direct", @@ -142,7 +142,7 @@ { "id": "browser", "surface": "browser-provider", - "count": 11, + "count": 12, "tools": [ "analyze_web_bundle", "capture_browser_scenario", @@ -151,6 +151,7 @@ "compare_web_screenshots", "discover_webmcp_tools", "inspect_web_event_listeners", + "inspect_web_network_capture", "inspect_web_page", "list_browser_targets", "observe_web_execution", @@ -268,6 +269,12 @@ "xrefs" ] }, + { + "id": "har-schema/lossless-json", + "name": "REA HAR capture adapter", + "version": "har-schema@2.0.0;jsonc-parser@3.3.1;lossless-json@4.3.1", + "capabilities": ["inspect_web_network_capture"] + }, { "id": "hopper", "name": "Hopper Disassembler", @@ -343,6 +350,12 @@ "trace_android_references" ] }, + { + "id": "mitmproxy-native-tnetstring", + "name": "REA offline native mitmproxy adapter", + "version": "12.2.3", + "capabilities": ["inspect_web_network_capture"] + }, { "id": "native-macos", "name": "macOS native inspection utilities", @@ -570,7 +583,7 @@ } ], "cli": { - "primary_count": 88, + "primary_count": 89, "commands": [ "address-to-file-offset", "analyze", @@ -627,6 +640,7 @@ "inspect-plist", "inspect-signature", "inspect-web-event-listeners", + "inspect-web-network-capture", "inspect-web-page", "instructions", "list-architectures", @@ -670,15 +684,15 @@ }, "runtime_catalog": { "counts": { - "cli_commands": 88, - "mcp_tools": 133, + "cli_commands": 89, + "mcp_tools": 134, "mcp_prompts": 6 }, "digests": { - "tools_sha256": "a493c05cd1618b5626b7f5ef3dfa214886917ac1b04716054ab9de100bddd468", + "tools_sha256": "d6f2493e9ad693a762f1bd3daf9b6736571f2a190b2a41986e045bf6ff48af89", "prompts_sha256": "ef2de8814390b340623c32001610aa2709d90bed51537b6e562ab3abf144ab4d", - "combined_sha256": "8c32372537b3d12e1f0befc51b8b9a5d083d37041eac3ceb1ba9d0a3f1853b2c", - "providers_sha256": "aea15f517c822d80918ccaba57a4fa06557f484cf27b8acbaed4e1a660fac896" + "combined_sha256": "e41a84b460b65f3a8d0d33498d958d249a1de81a9660c18580a0e3ab5d6941e1", + "providers_sha256": "e601cdceb5981ee04d6f431492f961f8867b1faa4f08ca0ecd29be2a239bc467" } } } diff --git a/docs/verification/managed-conformance-manifest.json b/docs/verification/managed-conformance-manifest.json index 494ad1ae3..78be54868 100644 --- a/docs/verification/managed-conformance-manifest.json +++ b/docs/verification/managed-conformance-manifest.json @@ -1,5 +1,5 @@ { - "manifest_id": "ecm_c139eb1f5cdc61ec60c8a73c9c0e50c7d9c78937a5677bf4bd392bb6269cf23e", + "manifest_id": "ecm_0615df0eb6f041eb2d145edbca9c4068a7dc9cbe53801dd237c7dd9394a2fa40", "verifier": { "id": "rea-managed-conformance", "version": "1" @@ -13,7 +13,7 @@ "skill_digests": [ { "skill_id": "reverse-engineer-anything", - "sha256": "4ee0e4f65a6d7643536da971f3da5532bd0b5de65dbc4ee01623259609c3196c" + "sha256": "49845719d9ea2d229fb954ba92203921e843908b2f116c94ecb94636812edc42" } ], "claims": [ diff --git a/docs/web-network-captures.md b/docs/web-network-captures.md new file mode 100644 index 000000000..5216c2031 --- /dev/null +++ b/docs/web-network-captures.md @@ -0,0 +1,147 @@ +# Historical web network captures + +Use `inspect_web_network_capture` / `inspect-web-network-capture` to inspect +retained producer evidence. This is an offline operation: recorded URLs are +never fetched, and no browser or proxy listener is started. + +```sh +rea inspect-web-network-capture /absolute/session.har har --json +rea inspect-web-network-capture /absolute/session.har har --record 2 --record 0 --json +``` + +```json +{ + "name": "inspect_web_network_capture", + "arguments": { + "capture_path": "/absolute/session.har", + "format": "har", + "record_ordinals": [2, 0] + } +} +``` + +Omit ordinals to return every record. Explicit selection preserves caller order +and original zero-based producer ordinals; duplicate or out-of-range selections +are input errors. Every record is validated before selection, so selecting one +entry does not hide a malformed entry elsewhere. + +## Formats and upstream profiles + +HAR uses unchanged `har-schema@2.0.0` draft-06 schemas, `ajv@8.20.0`, +`ajv-formats@3.0.1`, `jsonc-parser@3.3.1` JSON visitor, and +`lossless-json@4.3.1` numeric representations, pinned in the npm lockfile. +Only the HAR 1.2 profile is supported. Unknown extensions remain reported +evidence. Duplicate object keys, including equal values and escaped spellings, +are rejected before materialization. A `__proto__` member returns an explicit +unsupported-boundary error in both formats: the current shared JSON schema +cannot preserve this member. Original capture files remain unchanged. +The observed mitmproxy 12.2.3 `SaveHar` profile can emit `postData.text: null` +for a missing request body. REA preserves that reported null, omitting the +optional field only from its validation copy for this exact producer profile; +it does not invent an empty string or body bytes. + +Native mitmproxy decoding requires a caller-supplied **mitmdump 12.2.3 on +Linux**. Set its absolute executable path: + +```sh +REA_MITMDUMP_COMMAND=/absolute/mitmdump rea inspect-web-network-capture /absolute/session.mitm mitmproxy --json +``` + +REA does not install mitmproxy. Its owned process uses a private configuration +directory, `--no-server`, and the unchanged upstream `mitmproxy.io.tnetstring` +decoder. It does not run `FlowReader` migrations or convert native states to +HAR. Raw IDs, state versions, flow types, backup states and unknown extensions +are preserved without synthesizing connections, timestamps or content. The +tested upstream source is tag `v12.2.3`, commit +`6c09d56e4c29a92f5ad01b03199977584b8ea14f`; other native tool versions return +an explicit unsupported-profile diagnostic. Non-Linux native decoding is not +supported by this adapter. + +## Reading the evidence + +- `artifact` identifies the caller-selected file and stable snapshot digest. +- `records` preserve producer order/ordinals and original JSON pointers or byte + ranges. No live browser transaction or scenario IDs are invented. +- `reported` retains the producer's fields. HAR container metadata is returned + separately, with `log.entries` replaced by `null` and `records_pointer` + identifying the record collection. This is an inspection representation, + not a reconstructed HAR file. +- `numeric_literals` preserve HAR number lexemes and native integer/float + representations, including unsafe integers and non-finite native floats. + Values that cannot be safely represented as JSON numbers are `null` in the + reported view. Unsafe standard HAR numeric fields fail schema validation; + unsafe extension numbers remain available through their sidecars. +- `binary_fields` retain exposed bytes, base64 and SHA-256 independently of + declared sizes. Valid HAR `content.encoding: "base64"` is decoded exactly; + producer-decoded Unicode text, missing text and unknown encodings do not + establish original body bytes. Post-data parameters do not reconstruct a + multipart body. Native byte fields also have an exact UTF-8 display view + when valid; binary sidecars establish their actual representation. Native + missing content and empty content remain distinct. +- `redactions` identify known transport authentication exclusions and explicit + sensitive-value exclusions. Redacted byte fields retain neither original + bytes nor their digest. Unknown extension names and token-looking query + parameters do not establish sensitivity. + +Mark sensitive literal string/UTF-8 byte values with MCP `sensitive_values`, or +repeat CLI `--sensitive-value`. These declarations are never persisted in +Evidence parameters; the count is retained. Explicit text/UTF-8 byte matches +exclude the entire field with a reported `null` rather than substitute a text +marker or rewrite payload bytes. This prevents replacement markers from +reintroducing declared literals. The file digest identifies the original artifact +unless it matches an explicit declaration; in that case `artifact.sha256` and +Evidence `subject` are null with an explanation. A sensitive property name excludes that +entire property and its sidecars; a `scope: "property-name"` redaction points +to its actual parent. No substitute property name or child pointer is invented. +Omitted subtrees are still validated, including canonical HAR base64. Explicitly sensitive artifact +paths are empty in Evidence and omitted from locations, while the original +SHA-256 and size remain available unless the digest itself is explicitly excluded. Failure messages, diagnostics and cleanup +resources follow the same explicit declarations without changing error types; +sensitive issue pointers identify a real ancestor instead of a fabricated +coordinate. Header/cookie authentication values and known +transport URL userinfo are excluded structurally, including native backups. + +Generated free-text limitations follow the same literal declarations, including +global, per-record and Evidence explanations added during projection. Contract +field names, typed discriminators and exact decoder identities keep their +canonical meanings; declarations do not rename formats or providers. + +Historical evidence does not prove runtime attribution, execution or deployment +authenticity. Mitmproxy messages are upstream-assembled messages, not recovered +wire fragments. Native dictionary duplicate keys have the unchanged upstream +decoder's last-value semantics; this adapter does not claim to recover discarded +duplicates or the original numeric spelling of parsed native floats. + +## Resource and lifecycle boundaries + +Complete evidence has a 32 MiB input, 96 MiB decoder reply and 64-level nesting +budget. Exceeding a budget returns no partial success. +Oversized selected captures return an input error identifying `capture_path` +and the byte budget; owned reply/output failures remain output or provider errors. +HAR runs in an owned +192 MiB Node old-generation heap with one V8 worker. Native decoding applies +Linux limits of 768 MiB address space, 30 seconds CPU and 96 MiB file output; +both commands have a 30-second wall deadline and independently supervised +process cleanup. Owned decoder diagnostics retain at most 1 MiB across stdout +and stderr; the collector drops overflow as it arrives, reports the limit +failure and verifies cleanup. Heap limits alone do not establish aggregate RSS limits. + +Stable regular-file reads reject symlinks, replacement and concurrent changes. +Private snapshots, declarations and tool configuration are removed before the +result returns. Cleanup uncertainty retains the selected capture, resource and +original failure information. The original capture is never changed. + +## Excluded coordinates and read failures + +Explicitly marked values also exclude matching source coordinates and their sidecars. A hidden record coordinate is reported as `location.kind: "unknown"`; REA does not invent a replacement pointer. Original record ordinals and artifact digests remain available. + +Host filesystem read denials return `code: "access_denied"`, `category: "unavailable"`, the actual `EACCES`/`EPERM` code and read-access guidance. Missing files remain input failures. Cleanup failures retain the prior structured error, including its input constraints or provider diagnostics, with explicitly marked text excluded. + +An observed replacement or in-place change during stable artifact acquisition +returns `code: "artifact_changed"`, `category: "integrity_mismatch"` and +`retryable: true`, retaining the selected path and reader's reason. Wait until +the capture file is stable before retrying. No unstable snapshot is decoded. +Command cleanup failures retain the original status and both diagnostic streams; +the decoder's ownership preparation receives the same cancellation signal. + +MCP advertises the same strict named input contract used by the application. Historical inspection delegates argument validation to that shared application boundary so accepted explicit-sensitive declarations also cover invalid argument names and other correction details. The SDK receives raw arguments through its Standard Schema interface; malformed inputs still fail before decoder or filesystem effects and return the normal structured REA error. Protocol-envelope validation remains owned by MCP. diff --git a/package-lock.json b/package-lock.json index e4d1c0a19..e47340616 100644 --- a/package-lock.json +++ b/package-lock.json @@ -20,13 +20,17 @@ "@xterm/addon-serialize": "0.14.0", "@xterm/headless": "6.0.0", "@zip.js/zip.js": "2.15.0", + "ajv": "8.20.0", + "ajv-formats": "3.0.1", "canonicalize": "3.0.0", "cross-spawn": "7.0.6", "diff": "9.0.0", + "har-schema": "2.0.0", "ignore": "7.0.6", "incur": "0.6.1", "isomorphic-git": "1.38.7", "jsonc-parser": "3.3.1", + "lossless-json": "4.3.1", "pino": "10.4.0", "playwright-core": "1.58.2", "plist": "5.0.0", @@ -48,7 +52,6 @@ "@types/write-file-atomic": "4.0.3", "@types/ws": "8.18.1", "@vitest/coverage-v8": "5.0.3", - "ajv": "8.20.0", "husky": "9.1.7", "jscpd": "5.0.12", "knip": "6.39.0", @@ -3971,7 +3974,6 @@ "version": "8.20.0", "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.20.0.tgz", "integrity": "sha512-Thbli+OlOj+iMPYFBVBfJ3OmCAnaSyNn4M1vz9T6Gka5Jt9ba/HIR56joy65tY6kx/FCF5VXNB819Y7/GUrBGA==", - "dev": true, "license": "MIT", "dependencies": { "fast-deep-equal": "^3.1.3", @@ -3984,6 +3986,23 @@ "url": "https://github.com/sponsors/epoberezkin" } }, + "node_modules/ajv-formats": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/ajv-formats/-/ajv-formats-3.0.1.tgz", + "integrity": "sha512-8iUql50EUR+uUcdRQ3HDqa6EVyo3docL8g5WJ3FNcWmu62IbkGUue/pEyLBW8VGKKucTPgqeks4fIU1DA4yowQ==", + "license": "MIT", + "dependencies": { + "ajv": "^8.0.0" + }, + "peerDependencies": { + "ajv": "^8.0.0" + }, + "peerDependenciesMeta": { + "ajv": { + "optional": true + } + } + }, "node_modules/algoliasearch": { "version": "5.59.0", "resolved": "https://registry.npmjs.org/algoliasearch/-/algoliasearch-5.59.0.tgz", @@ -4660,7 +4679,6 @@ "version": "3.1.3", "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==", - "dev": true, "license": "MIT" }, "node_modules/fast-string-truncated-width": { @@ -4682,7 +4700,6 @@ "version": "3.1.8", "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.8.tgz", "integrity": "sha512-GZMtZUTNRpOVIECoXwLNZS5xUGE+mVNbTB8h/7Rwh2TFWcBQiPzTgyZi05BF9UMZKkLJv8XBRJTlU7zg8+ZfMg==", - "dev": true, "funding": [ { "type": "github", @@ -4890,6 +4907,15 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/har-schema": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/har-schema/-/har-schema-2.0.0.tgz", + "integrity": "sha512-Oqluz6zhGX8cyRaTQlFMPw80bSJVG2x/cFb8ZPhUILGgHka9SsokCCOQgpveePerqidZOrT14ipqfJb7ILcW5Q==", + "license": "ISC", + "engines": { + "node": ">=4" + } + }, "node_modules/has-property-descriptors": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/has-property-descriptors/-/has-property-descriptors-1.0.2.tgz", @@ -5315,7 +5341,6 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==", - "dev": true, "license": "MIT" }, "node_modules/jsonc-parser": { @@ -5743,6 +5768,12 @@ "url": "https://github.com/chalk/slice-ansi?sponsor=1" } }, + "node_modules/lossless-json": { + "version": "4.3.1", + "resolved": "https://registry.npmjs.org/lossless-json/-/lossless-json-4.3.1.tgz", + "integrity": "sha512-SqD/Bg3ZfltBJ2Z14hJ/BihnvtV553WO4g9/ePtlp4lrnl9jF3AdIJt53A/Wkg/0Li+LMfxaBqgx1MiFZdQlpQ==", + "license": "MIT" + }, "node_modules/lru-cache": { "version": "11.5.2", "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-11.5.2.tgz", @@ -6609,7 +6640,6 @@ "version": "2.0.2", "resolved": "https://registry.npmjs.org/require-from-string/-/require-from-string-2.0.2.tgz", "integrity": "sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==", - "dev": true, "license": "MIT", "engines": { "node": ">=0.10.0" diff --git a/package.json b/package.json index 52109d452..8643ff966 100644 --- a/package.json +++ b/package.json @@ -50,6 +50,7 @@ "bridge/ghidra/ReaGhidraExtensions.java", "bridge/ghidra/ReaGhidraPrepareCom.java", "bridge/android/ReaJadxBridge.java", + "bridge/mitmproxy/capture.py", "scripts/rea.mjs", "scripts/electron-active-hook.cjs", "scripts/electron-active-hook-boundaries.cjs", @@ -191,7 +192,8 @@ "verify:javascript:recovery": "node scripts/verify/javascript/recovery.mjs", "verify:module-boundaries": "node scripts/verify-module-boundaries.mjs", "verify:browser:source-maps": "npm run build:cached && node scripts/verify-browser-source-maps.mjs", - "verify:browser:runtime": "npm run build:cached && node scripts/verify-browser-runtime.mjs" + "verify:browser:runtime": "npm run build:cached && node scripts/verify-browser-runtime.mjs", + "verify:web:network-captures": "npm run build:cached && node scripts/verify-web-network-captures.mjs" }, "lint-staged": { "*.{cjs,cts,js,jsx,mjs,mts,ts,tsx}": [ @@ -212,13 +214,17 @@ "@xterm/addon-serialize": "0.14.0", "@xterm/headless": "6.0.0", "@zip.js/zip.js": "2.15.0", + "ajv": "8.20.0", + "ajv-formats": "3.0.1", "canonicalize": "3.0.0", "cross-spawn": "7.0.6", "diff": "9.0.0", + "har-schema": "2.0.0", "ignore": "7.0.6", "incur": "0.6.1", "isomorphic-git": "1.38.7", "jsonc-parser": "3.3.1", + "lossless-json": "4.3.1", "pino": "10.4.0", "playwright-core": "1.58.2", "plist": "5.0.0", @@ -241,7 +247,6 @@ "@types/write-file-atomic": "4.0.3", "@types/ws": "8.18.1", "@vitest/coverage-v8": "5.0.3", - "ajv": "8.20.0", "husky": "9.1.7", "jscpd": "5.0.12", "knip": "6.39.0", diff --git a/scripts/fixtures/generate-mitmproxy-capture.py b/scripts/fixtures/generate-mitmproxy-capture.py new file mode 100644 index 000000000..4be3dd4a7 --- /dev/null +++ b/scripts/fixtures/generate-mitmproxy-capture.py @@ -0,0 +1,151 @@ +import importlib.util +import json +from pathlib import Path +from mitmproxy import connection, ctx, http +from mitmproxy.io import FlowWriter +from mitmproxy.io import tnetstring +from mitmproxy.addons.savehar import SaveHar +from mitmproxy.websocket import WebSocketData, WebSocketMessage + +class Generator: + def load(self, loader): + loader.add_option("rea_fixture_root", str, "", "Owned synthetic fixture root") + loader.add_option("rea_adapter_path", str, "", "REA native adapter under verification") + + def running(self): + root = Path(ctx.options.rea_fixture_root) + spec = importlib.util.spec_from_file_location("rea_capture_under_test", ctx.options.rea_adapter_path) + adapter = importlib.util.module_from_spec(spec) + spec.loader.exec_module(adapter) + classifications = [ + (adapter.CaptureFailure("format", "Malformed retained record.", "/record"), "format"), + (adapter.CaptureFailure("input-limit", "Nesting budget.", "/extension"), "input-limit"), + (MemoryError("payload must not enter diagnostics"), "resource-limit"), + (RuntimeError("payload must not enter diagnostics"), "decoder"), + (OSError("payload must not enter diagnostics"), "decoder"), + ] + for error, expected in classifications: + reply = adapter.failure_reply(error) + assert reply["reason"] == expected, reply + assert "payload must not enter diagnostics" not in json.dumps(reply) + for state, expected in [({"metadata": {1: b"bad-key"}}, "format"), ({"metadata": object()}, "decoder")]: + try: + adapter.project_record(state, []) + except adapter.CaptureFailure as error: + assert error.reason == expected, (error.reason, expected) + assert error.pointer == "/metadata", error.pointer + else: + raise AssertionError("Invalid native representation was accepted") + non_utf8 = adapter.project_record({"request": {"headers": [(b"Referer", b"https://byte-user:byte-secret@example.test/path\xff")] }}, []) + assert non_utf8["reported"]["request"]["headers"][0][1] is None, non_utf8 + assert non_utf8["binary_fields"][-1]["state"] == "redacted", non_utf8 + for value in [" \tHTTPS://ows-user:ows-secret@example.test/path\t ", b" \tHTTPS://ows-user:ows-secret@example.test/path\t "]: + projected = adapter.project_record({"request": {"headers": [(b"Referer", value)]}}, []) + assert projected["reported"]["request"]["headers"][0][1] == " \tHTTPS://example.test/path\t ", projected + assert "ows-secret" not in json.dumps(projected), projected + if isinstance(value, bytes): + assert projected["binary_fields"][-1]["state"] == "redacted", projected + (root / "native-error-classifications.json").write_text(json.dumps({"passed": len(classifications) + 2})) + first = http.HTTPFlow(connection.Client(peername=("127.0.0.1", 1), sockname=("127.0.0.1", 2)), connection.Server(address=("example.test", 80))) + first.id = "original-producer-id" + first.request = http.Request.make("POST", "http://example.test/a?token=ordinary", b"\x00\xffbody", [(b"Authorization", b"Bearer native-transport-secret"), (b"X-Duplicate", b"one"), (b"X-Duplicate", b"two")]) + first.request.path = "http://user:password@example.test/a?token=ordinary#fragment" + first.response = http.Response.make(200, b"\x00\xfeanswer", [(b"Set-Cookie", b"session=cookie-secret"), (b"Location", b"https://redirect-user:redirect-password@example.test/b?token=ordinary")]) + first.metadata = {"big_integer": 9007199254740993, "nonfinite": float("inf"), "isLosslessNumber": True, "value": "ordinary-extension", "response": {"headers": [["Authorization", "ordinary-extension-credential-name"]]}} + first.websocket = WebSocketData(messages=[WebSocketMessage(2, True, b"\x00\xfdmessage", 123.5)]) + first.backup() + first.comment = "caller-selected-fixture" + second = first.copy() + second.id = first.id + second.request.raw_content = None + second.response.raw_content = b"" + second.websocket = None + with (root / "flows.mitm").open("wb") as handle: + writer = FlowWriter(handle) + writer.add(first) + writer.add(second) + har_first, har_second = first.copy(), second.copy() + har_first.request.path = "/a?token=ordinary#fragment" + har_second.request.path = "/a?token=ordinary#fragment" + har_first.response.raw_content = b"\x00\xff\x01\xfe" + markers = {"sensitive": "REDACTED", "bracket": "literal[", "overlap": "secret", "ordinary": "unmarked"} + har = SaveHar().make_har([har_first, har_second]) + har["log"]["entries"][0]["_markers"] = markers + har["log"]["entries"][0]["_private_properties"] = {"private-property/~": {"number": 123, "text": "private-property"}, "kept": 7} + (root / "producer.har").write_text(json.dumps(har)) + har_text = json.dumps(har) + for name, members in [ + ("equal-number", '"_duplicate": 1, "_duplicate": 1'), + ("equal-null", '"_duplicate": null, "_duplicate": null'), + ("equal-object", '"_duplicate": {"value": 1}, "_duplicate": {"value": 1}'), + ("escaped-key", '"_duplicate": 1, "\\u005fduplicate": 1'), + ]: + duplicate = har_text.replace('"cache": {}', '"cache": {}, ' + members, 1) + assert duplicate != har_text + (root / ("duplicate-" + name + ".har")).write_text(duplicate) + duplicate = har_text.replace('"version": "1.2"', '"version": "1.2", "version": "1.2"', 1) + assert duplicate != har_text + (root / "duplicate-core.har").write_text(duplicate) + prototype = json.loads(har_text) + prototype["_extension"] = {"__proto__": {"preserved": 7}, "constructor": "ordinary"} + (root / "prototype.har").write_text(json.dumps(prototype)) + prototype_native = first.get_state() + prototype_native["metadata"]["__proto__"] = {"preserved": 7} + with (root / "prototype.mitm").open("wb") as handle: + tnetstring.dump(prototype_native, handle) + invalid_har = json.loads(json.dumps(har)) + invalid_har["log"]["entries"][0]["response"]["content"].update({"encoding": "base64", "text": "AR=="}) + (root / "invalid-private-parent.har").write_text(json.dumps(invalid_har)) + strings = first.get_state() + for state in [strings, strings["backup"]]: + state["request"]["path"] = "https://native-user:string-password@example.test/string-path" + state["request"]["authority"] = "authority-user:authority-password@example.test" + state["request"]["headers"] = [(b"Referer", "https://referer-user:referer-password@example.test/from"), (b"Origin", "//origin-user:origin-password@example.test")] + state["response"]["headers"] = [(b"Location", "https://location-user:location-password@example.test/to")] + state["metadata"]["_markers"] = dict(markers) + state["metadata"]["_private_properties"] = {"private-property/~": {"number": 123, "bytes": b"private-property"}, "kept": 7} + with (root / "string-urls.mitm").open("wb") as handle: + tnetstring.dump(strings, handle) + strings["metadata"]["_private_properties"]["private-property/~"] = {1: b"unsupported-key"} + with (root / "invalid-private-key.mitm").open("wb") as handle: + tnetstring.dump(strings, handle) + extension = "leaf" + for _ in range(66): + extension = {"child": extension} + har["_extension"] = extension + (root / "deep.har").write_text(json.dumps(har)) + deep = first.get_state() + deep["_extension"] = extension + with (root / "deep.mitm").open("wb") as handle: + tnetstring.dump(deep, handle) + for name, value in [("invalid-credential-key", {1: b"bad-key"}), ("deep-credential", extension)]: + malformed = first.get_state() + malformed["request"]["headers"] = [(b"Authorization", value)] + with (root / (name + ".mitm")).open("wb") as handle: + tnetstring.dump(malformed, handle) + for prefix, name in [(" \t", "ows"), ("\r\n \v\f\x00\x1f", "controls")]: + url = prefix + "HTTPS://ows-user:ows-secret@example.test/path?token=ordinary#fragment\t " + relative = prefix + "//ows-user:ows-secret@example.test/path?token=ordinary#fragment\t " + whitespace_har = json.loads((root / "producer.har").read_text()) + entry = whitespace_har["log"]["entries"][0] + for side, header in [("response", "Location"), ("request", "Referer"), ("request", "Origin")]: + entry[side]["headers"] = [{"name": header, "value": url}] + entry[side]["headers"] + (root / (name + ".har")).write_text(json.dumps(whitespace_har)) + whitespace_native = first.get_state() + for state in [whitespace_native, whitespace_native["backup"]]: + state["request"]["headers"] = [(b"Referer", url.encode("utf-8")), (b"Origin", relative)] + state["response"]["headers"] = [(b"Location", url)] + with (root / (name + ".mitm")).open("wb") as handle: + tnetstring.dump(whitespace_native, handle) + non_utf8_native = first.get_state() + for state in [non_utf8_native, non_utf8_native["backup"]]: + state["request"]["path"] = b"https://byte-user:byte-secret@example.test/path\xff" + state["request"]["authority"] = b"byte-user:byte-secret@example.test\xff" + state["request"]["headers"] = [(b"Referer", b" https://byte-user:byte-secret@example.test/from\xff"), (b"Origin", b"//byte-user:byte-secret@example.test\xff")] + state["response"]["headers"] = [(b"Location", b"https://byte-user:byte-secret@example.test/to\xff")] + with (root / "non-utf8-urls.mitm").open("wb") as handle: + tnetstring.dump(non_utf8_native, handle) + (root / "oracle.json").write_text(json.dumps({"records": 2, "request_base64": "AP9ib2R5", "response_base64": "AP5hbnN3ZXI=", "websocket_base64": "AP1tZXNzYWdl", "id": first.id})) + ctx.master.shutdown() + +addons = [Generator()] diff --git a/scripts/lib/catalog-builders.mjs b/scripts/lib/catalog-builders.mjs index 1e5ddfab6..3664e28aa 100644 --- a/scripts/lib/catalog-builders.mjs +++ b/scripts/lib/catalog-builders.mjs @@ -48,6 +48,8 @@ export const toolFamilyCatalog = (sources) => { ...sources.browserContracts.BROWSER_TOOL_CONTRACTS, ...sources.webRuntimeContracts.WEB_RUNTIME_TOOL_CONTRACTS, ...sources.browserScenarioContracts.BROWSER_SCENARIO_TOOL_CONTRACTS, + ...sources.webNetworkCaptureContracts + .WEB_NETWORK_CAPTURE_TOOL_CONTRACTS, ], }, { @@ -170,6 +172,17 @@ export const providerCatalog = (sources) => { identity: sources.androidProvider.JADX_PROVIDER_IDENTITY, contracts: sources.androidContracts.ANDROID_TOOL_CONTRACTS, }, + { + identity: sources.webNetworkCaptureProvider.HAR_CAPTURE_PROVIDER_IDENTITY, + contracts: + sources.webNetworkCaptureContracts.WEB_NETWORK_CAPTURE_TOOL_CONTRACTS, + }, + { + identity: + sources.webNetworkCaptureProvider.MITMPROXY_CAPTURE_PROVIDER_IDENTITY, + contracts: + sources.webNetworkCaptureContracts.WEB_NETWORK_CAPTURE_TOOL_CONTRACTS, + }, { identity: sources.browserProvider.CDP_BROWSER_PROVIDER_IDENTITY, contracts: [ diff --git a/scripts/lib/catalog-core.mjs b/scripts/lib/catalog-core.mjs index 8b0649e4b..c2b2018f8 100644 --- a/scripts/lib/catalog-core.mjs +++ b/scripts/lib/catalog-core.mjs @@ -51,6 +51,9 @@ const SOURCE_PATHS = { "dist/contracts/managed/managedWorkflowToolContracts.js", browserContracts: "dist/contracts/browserToolContracts.js", webRuntimeContracts: "dist/contracts/webRuntimeToolContracts.js", + webNetworkCaptureContracts: + "dist/contracts/webNetworkCaptureToolContracts.js", + webNetworkCaptureProvider: "dist/browser/history/CaptureRelease.js", browserScenarioContracts: "dist/contracts/browserScenarioToolContracts.js", electronContracts: "dist/contracts/javascript/electronToolContracts.js", javascriptRuntimeObservationContracts: diff --git a/scripts/verify-web-network-captures.mjs b/scripts/verify-web-network-captures.mjs new file mode 100644 index 000000000..d5d1e2609 --- /dev/null +++ b/scripts/verify-web-network-captures.mjs @@ -0,0 +1,994 @@ +#!/usr/bin/env node +import assert from "node:assert/strict"; +import { execFile } from "node:child_process"; +import { createHash, randomUUID } from "node:crypto"; +import { + access, + copyFile, + chmod, + readFile, + truncate, + writeFile, +} from "node:fs/promises"; +import { dirname, isAbsolute, join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import { promisify } from "node:util"; +import { Client } from "@modelcontextprotocol/client"; +import { StdioClientTransport } from "@modelcontextprotocol/client/stdio"; +import { PrivateRuntimeRoot } from "../dist/process/PrivateRuntimeRoot.js"; +import { runOwnedCommand } from "../dist/process/OwnedCommand.js"; +import { WEB_NETWORK_CAPTURE_LIMITS } from "../dist/domain/webNetworkCapture.js"; +import { mcpTextValue } from "./lib/mcp-verifier-results.mjs"; +import { createVerifierRun, completeVerifierRun } from "./lib/verifier-run.mjs"; + +const executable = process.env.REA_MITMDUMP_COMMAND; +if ( + process.platform !== "linux" || + executable === undefined || + !isAbsolute(executable) +) + throw new Error( + "verify:web:network-captures requires Linux and absolute REA_MITMDUMP_COMMAND for caller-supplied mitmdump 12.2.3", + ); +await access(executable); +const entrypoint = + process.argv[2] ?? fileURLToPath(new URL("./rea.mjs", import.meta.url)); +const run = createVerifierRun(); +const runtime = await PrivateRuntimeRoot.create({ + prefix: "rea-history-verifier-", +}); +const environment = Object.fromEntries( + Object.entries(process.env).filter(([, value]) => typeof value === "string"), +); +const client = new Client({ + name: "historical-web-capture-verifier", + version: "1", +}); +const transport = new StdioClientTransport({ + command: process.execPath, + args: [entrypoint, "mcp"], + env: environment, + stderr: "pipe", +}); +const failures = []; +let cases = 0; +try { + await runOwnedCommand( + { + command: executable, + arguments: [ + "--no-server", + "-q", + "--set", + `confdir=${join(runtime.path, "config")}`, + "--set", + `rea_fixture_root=${runtime.path}`, + "--set", + `rea_adapter_path=${resolve(dirname(entrypoint), "../bridge/mitmproxy/capture.py")}`, + "-s", + fileURLToPath( + new URL("./fixtures/generate-mitmproxy-capture.py", import.meta.url), + ), + ], + cwd: runtime.path, + runId: `rea-history-fixture-${randomUUID()}`, + hostEnvironment: environment, + }, + { timeoutMs: 30_000, diagnosticBytes: 1024 * 1024 }, + ); + assert.deepEqual( + JSON.parse( + await readFile( + join(runtime.path, "native-error-classifications.json"), + "utf8", + ), + ), + { passed: 7 }, + ); + await client.connect(transport); + for (const format of ["har", "mitmproxy"]) { + const path = join( + runtime.path, + format === "har" ? "producer.har" : "flows.mitm", + ); + const raw = await readFile(path); + const input = { capture_path: path, format, record_ordinals: [1, 0] }; + for (const mode of ["cli", "mcp"]) { + const value = await inspect(mode, input); + assert.equal(value.total_records, 2); + assert.deepEqual( + value.records.map((record) => record.ordinal), + [1, 0], + ); + assert.equal( + value.artifact.sha256, + createHash("sha256").update(raw).digest("hex"), + ); + assert.equal(value.runtime_attribution, "unknown"); + const first = value.records[1]; + if (format === "mitmproxy") { + assert.equal(first.reported.id, "original-producer-id"); + assert.equal(value.records[0].reported.id, first.reported.id); + assert.equal(first.location.offset, 0); + assert.equal( + value.records[0].location.offset + value.records[0].location.bytes, + raw.length, + ); + assert.equal( + first.reported.request.path, + "http://example.test/a?token=ordinary#fragment", + ); + assertBinary( + first, + "/request/content", + Buffer.from([0, 255, ...Buffer.from("body")]), + ); + assertBinary( + first, + "/response/content", + Buffer.from([0, 254, ...Buffer.from("answer")]), + ); + assertBinary( + first, + "/websocket/messages/0/2", + Buffer.from([0, 253, ...Buffer.from("message")]), + ); + assert.equal(value.records[0].reported.request.content, null); + assertBinary(value.records[0], "/response/content", Buffer.alloc(0)); + assert.ok( + first.numeric_literals.some( + (number) => + number.pointer === "/metadata/big_integer" && + number.literal === "9007199254740993", + ), + ); + assert.ok( + first.numeric_literals.some( + (number) => + number.pointer === "/metadata/nonfinite" && + number.literal === "inf", + ), + ); + assert.equal( + first.reported.metadata.response.headers[0][1], + "ordinary-extension-credential-name", + ); + } else { + assert.equal(value.container.reported.log.creator.name, "mitmproxy"); + assert.equal(value.container.reported.log.creator.version, "12.2.3"); + assert.equal(value.records[0].reported.request.postData.text, null); + assertBinary( + first, + "/response/content/text", + Buffer.from([0, 255, 1, 254]), + ); + } + const text = JSON.stringify(value); + for (const secret of [ + "native-transport-secret", + "cookie-secret", + "redirect-password", + "user:password", + ]) + assert.ok( + !text.includes(secret), + `Credential leaked through ${mode}/${format}`, + ); + cases++; + } + } + for (const mode of ["cli", "mcp"]) { + const value = await inspect(mode, { + capture_path: join(runtime.path, "string-urls.mitm"), + format: "mitmproxy", + }); + const first = value.records[0]; + for (const state of [first.reported, first.reported.backup]) { + assert.equal(state.request.path, "https://example.test/string-path"); + assert.equal(state.request.authority, "example.test"); + assert.deepEqual( + state.request.headers.map((field) => field[1]), + ["https://example.test/from", "//example.test"], + ); + assert.equal(state.response.headers[0][1], "https://example.test/to"); + } + assert.ok(!JSON.stringify(value).includes("password")); + assert.ok( + first.redactions.some( + (redaction) => + redaction.pointer === "/backup/request/authority" && + redaction.reason === "transport-credential", + ), + ); + cases++; + } + for (const sensitive_values of [ + ["REDACTED", "["], + ["secret", "REDACTED"], + ]) { + for (const format of ["har", "mitmproxy"]) { + for (const mode of ["cli", "mcp"]) { + const value = await inspect(mode, { + capture_path: join( + runtime.path, + format === "har" ? "producer.har" : "string-urls.mitm", + ), + format, + record_ordinals: [0], + sensitive_values, + }); + const first = value.records[0]; + const markers = + format === "har" + ? first.reported._markers + : first.reported.metadata._markers; + assert.equal(markers.sensitive, null); + assert.equal(markers.ordinary, "unmarked"); + assert.equal( + sensitive_values.includes("[") ? markers.bracket : markers.overlap, + null, + ); + if (format === "mitmproxy") + assert.equal(first.reported.backup.metadata._markers.sensitive, null); + cases++; + } + } + } + for (const name of [ + "equal-number", + "equal-null", + "equal-object", + "escaped-key", + "core", + ]) { + const path = join(runtime.path, `duplicate-${name}.har`); + const original = await readFile(path); + for (const mode of ["cli", "mcp"]) { + await inspect( + mode, + { capture_path: path, format: "har" }, + "invalid_input", + ); + assert.deepEqual(await readFile(path), original); + cases++; + } + } + for (const format of ["har", "mitmproxy"]) { + const path = join( + runtime.path, + format === "har" ? "prototype.har" : "prototype.mitm", + ); + const original = await readFile(path); + for (const mode of ["cli", "mcp"]) { + const error = await inspect( + mode, + { capture_path: path, format }, + "unsupported_provider", + ); + assert.ok( + JSON.stringify(error).includes("JSON schema boundary cannot preserve"), + ); + assert.deepEqual(await readFile(path), original); + cases++; + } + } + for (const format of ["har", "mitmproxy"]) { + const path = join(runtime.path, `malformed-${format}`); + await writeFile(path, format === "har" ? "{invalid-json" : "999999999999:"); + for (const mode of ["cli", "mcp"]) { + await inspect(mode, { capture_path: path, format }, "invalid_input"); + cases++; + } + } + for (const format of ["har", "mitmproxy"]) { + for (const mode of ["cli", "mcp"]) { + const value = await inspect(mode, { + capture_path: join( + runtime.path, + format === "har" ? "producer.har" : "string-urls.mitm", + ), + format, + sensitive_values: ["private-property"], + }); + const record = value.records[0]; + const properties = + format === "har" + ? record.reported._private_properties + : record.reported.metadata._private_properties; + assert.deepEqual(properties, { kept: 7 }); + assert.ok(!JSON.stringify(value).includes("private-property")); + assert.ok( + record.redactions.some( + (item) => + item.scope === "property-name" && + item.pointer === + (format === "har" + ? "/_private_properties" + : "/metadata/_private_properties"), + ), + ); + if (format === "mitmproxy") + assert.deepEqual(record.reported.backup.metadata._private_properties, { + kept: 7, + }); + cases++; + } + } + for (const [fixture, prefix] of [ + ["ows", " \t"], + ["controls", "\r\n \v\f\x00\x1f"], + ]) { + for (const format of ["har", "mitmproxy"]) { + for (const mode of ["cli", "mcp"]) { + const value = await inspect(mode, { + capture_path: join( + runtime.path, + `${fixture}.${format === "har" ? "har" : "mitm"}`, + ), + format, + }); + assert.ok(!JSON.stringify(value).includes("ows-secret")); + const record = value.records[0]; + const states = + format === "har" + ? [record.reported] + : [record.reported, record.reported.backup]; + for (const state of states) { + if (format === "har") { + for (const side of ["request", "response"]) + assert.ok( + state[side].headers.some( + (header) => + header.value === + `${prefix}HTTPS://example.test/path?token=ordinary#fragment\t `, + ), + ); + } else { + assert.deepEqual(state.request.headers, [ + [ + "Referer", + `${prefix}HTTPS://example.test/path?token=ordinary#fragment\t `, + ], + [ + "Origin", + `${prefix}//example.test/path?token=ordinary#fragment\t `, + ], + ]); + assert.deepEqual(state.response.headers, [ + [ + "Location", + `${prefix}HTTPS://example.test/path?token=ordinary#fragment\t `, + ], + ]); + } + } + if (format === "mitmproxy") { + for (const pointer of [ + "/request/headers/0/1", + "/backup/request/headers/0/1", + ]) + assert.deepEqual( + record.binary_fields.find((field) => field.pointer === pointer), + { + pointer, + representation: "producer-bytes", + state: "redacted", + content_base64: null, + bytes: null, + sha256: null, + }, + ); + } + cases++; + } + } + } + for (const mode of ["cli", "mcp"]) { + const value = await inspect(mode, { + capture_path: join(runtime.path, "non-utf8-urls.mitm"), + format: "mitmproxy", + }); + const record = value.records[0]; + assert.ok(!JSON.stringify(value).includes("byte-secret")); + for (const prefix of ["", "/backup"]) { + for (const suffix of [ + "/request/path", + "/request/authority", + "/request/headers/0/1", + "/request/headers/1/1", + "/response/headers/0/1", + ]) { + const pointer = prefix + suffix; + assert.deepEqual( + record.binary_fields.find((field) => field.pointer === pointer), + { + pointer, + representation: "producer-bytes", + state: "redacted", + content_base64: null, + bytes: null, + sha256: null, + }, + ); + } + } + cases++; + } + const nativeCoordinateInput = { + capture_path: join(runtime.path, "flows.mitm"), + format: "mitmproxy", + }; + const nativeCoordinates = await inspect("mcp", nativeCoordinateInput); + const originalLocation = nativeCoordinates.records[1].location; + assert.equal(originalLocation.kind, "byte-range"); + for (const literal of [ + String(originalLocation.offset), + String(originalLocation.bytes), + ]) { + for (const mode of ["cli", "mcp"]) { + const excluded = await inspect(mode, { + ...nativeCoordinateInput, + sensitive_values: [literal], + }); + assert.deepEqual(excluded.records[1].location, { + kind: "unknown", + reason: "explicit-sensitive-value", + }); + cases++; + } + } + cases++; + const deniedPath = join(runtime.path, "unreadable.har"); + await copyFile(join(runtime.path, "producer.har"), deniedPath); + await chmod(deniedPath, 0o000); + for (const mode of ["cli", "mcp"]) { + for (const format of ["har", "mitmproxy"]) { + const failure = await inspect( + mode, + { capture_path: deniedPath, format }, + "unavailable", + ); + assert.equal(failure.code, "access_denied"); + assert.equal(failure.details.system_code, "EACCES"); + assert.equal(failure.details.path, deniedPath); + cases++; + } + for (const literal of ["log", "entries"]) { + const value = await inspect(mode, { + capture_path: join(runtime.path, "producer.har"), + format: "har", + sensitive_values: [literal], + }); + assert.equal(value.container.records_pointer, null); + assert.ok( + value.records.every( + (record) => + record.location.kind === "unknown" && + record.location.reason === "explicit-sensitive-value", + ), + ); + assert.ok(!JSON.stringify(value).includes(literal)); + cases++; + } + for (const format of ["har", "mitmproxy"]) { + const value = await inspect(mode, { + capture_path: join( + runtime.path, + format === "har" ? "producer.har" : "string-urls.mitm", + ), + format, + sensitive_values: ["~1"], + }); + assert.ok(!JSON.stringify(value).includes("~1")); + cases++; + } + } + for (const mode of ["cli", "mcp"]) { + for (const format of ["har", "mitmproxy"]) { + for (const sensitive_values of [ + ["producer"], + ["producer", "REDACTED", "…"], + ]) { + await inspect(mode, { + capture_path: join( + runtime.path, + format === "har" ? "producer.har" : "flows.mitm", + ), + format, + sensitive_values, + }); + cases++; + } + } + } + for (const mode of ["cli", "mcp"]) { + for (const selection of [ + { record_ordinals: [12345], sensitive_values: ["12345"] }, + { record_ordinals: [0, 0], sensitive_values: ["Each"] }, + ]) { + await inspect( + mode, + { + capture_path: join(runtime.path, "producer.har"), + format: "har", + ...selection, + }, + "invalid_input", + ); + cases++; + } + } + await inspect( + "mcp", + { + capture_path: "relative.har", + format: "har", + sensitive_values: ["absolute"], + }, + "invalid_input", + ); + cases++; + const privatePath = join(runtime.path, "REDACTED.har"); + for (const name of ["mysecret", "/ordinary/secret"]) { + const unknownArgument = await inspect( + "mcp", + { + capture_path: join(runtime.path, "producer.har"), + format: "har", + sensitive_values: ["secret"], + [name]: true, + }, + "invalid_input", + ); + assert.deepEqual(unknownArgument.details.issues[0].path, []); + cases++; + } + await inspect("mcp", { + capture_path: join(runtime.path, "flows.mitm"), + format: "mitmproxy", + sensitive_values: ["\ud800", "ordinary-scalar"], + }); + cases++; + const replacementHar = JSON.parse( + await readFile(join(runtime.path, "producer.har"), "utf8"), + ); + replacementHar.log.entries[0].response.content = { + size: 3, + mimeType: "application/octet-stream", + encoding: "base64", + text: "77+9", + }; + const replacementHarPath = join(runtime.path, "unicode-scalar.har"); + await writeFile(replacementHarPath, JSON.stringify(replacementHar)); + for (const literal of ["\ud800", "\udfff", "\ufffd"]) { + const unicode = await inspect("mcp", { + capture_path: replacementHarPath, + format: "har", + sensitive_values: [literal], + }); + const body = unicode.records[0].binary_fields.find( + (field) => field.pointer === "/response/content/text", + ); + assert.equal(body?.state, literal === "\ufffd" ? "redacted" : "retained"); + assert.equal(body.content_base64, literal === "\ufffd" ? null : "77+9"); + cases++; + } + const numericHar = ( + await readFile(join(runtime.path, "producer.har"), "utf8") + ).replace('"cache": {}', '"cache": {}, "_big": 9007199254740993'); + const numericPath = join(runtime.path, "numeric.har"); + await writeFile(numericPath, numericHar); + assert.ok(numericHar.includes('"_big": 9007199254740993')); + for (const mode of ["cli", "mcp"]) { + for (const format of ["har", "mitmproxy"]) { + for (const literal of ["9007199254740993", "4740993"]) { + const value = await inspect(mode, { + capture_path: + format === "har" ? numericPath : join(runtime.path, "flows.mitm"), + format, + sensitive_values: [literal], + }); + assert.ok(!JSON.stringify(value).includes(literal)); + const first = value.records[0]; + const pointer = format === "har" ? "/_big" : "/metadata/big_integer"; + assert.ok( + first.redactions.some( + (redaction) => + redaction.pointer === pointer && + redaction.reason === "explicit-sensitive-value", + ), + ); + assert.ok( + first.numeric_literals.every( + (number) => !number.literal.includes(literal), + ), + ); + cases++; + } + } + } + for (const mode of ["cli", "mcp"]) { + for (const format of ["har", "mitmproxy"]) { + await inspect(mode, { + capture_path: join( + runtime.path, + format === "har" ? "producer.har" : "flows.mitm", + ), + format, + sensitive_values: [format], + }); + cases++; + } + } + const encodedHar = JSON.parse( + await readFile(join(runtime.path, "producer.har"), "utf8"), + ); + encodedHar.log.entries[0].response.content = { + size: 6, + mimeType: "application/octet-stream", + encoding: "base64", + text: "c2 VjcmV0", + }; + const encodedHarPath = join(runtime.path, "encoded-sensitive.har"); + await writeFile(encodedHarPath, JSON.stringify(encodedHar)); + for (const mode of ["cli", "mcp"]) { + for (const format of ["har", "mitmproxy"]) { + const bytes = + format === "har" + ? Buffer.from("secret") + : Buffer.from([0, 255, ...Buffer.from("body")]); + for (const literal of [ + bytes.toString("base64"), + bytes.toString("base64").slice(0, 4), + createHash("sha256").update(bytes).digest("hex"), + ]) { + const value = await inspect(mode, { + capture_path: + format === "har" + ? encodedHarPath + : join(runtime.path, "flows.mitm"), + format, + sensitive_values: [literal], + }); + const pointer = + format === "har" ? "/response/content/text" : "/request/content"; + const field = value.records[0].binary_fields.find( + (field) => field.pointer === pointer, + ); + assert.equal(field?.state, "redacted"); + assert.equal(field.content_base64, null); + assert.equal(field.sha256, null); + assert.ok(!JSON.stringify(value).includes(literal)); + cases++; + } + } + } + for (const mode of ["cli", "mcp"]) { + for (const format of ["har", "mitmproxy"]) { + const capturePath = join( + runtime.path, + format === "har" ? "producer.har" : "flows.mitm", + ); + const digest = createHash("sha256") + .update(await readFile(capturePath)) + .digest("hex"); + for (const literal of [digest, digest.slice(0, 12)]) { + const value = await inspect(mode, { + capture_path: capturePath, + format, + sensitive_values: [literal], + }); + assert.equal(value.artifact.sha256, null); + assert.equal(value.artifact.path, capturePath); + assert.ok(!JSON.stringify(value).includes(literal)); + cases++; + } + } + } + const oversizedPath = join(runtime.path, "oversized-capture"); + for (const mode of ["cli", "mcp"]) { + for (const literal of ["log", "entries"]) { + const hidden = await inspect(mode, { + capture_path: join(runtime.path, "producer.har"), + format: "har", + sensitive_values: [literal], + }); + assert.equal(hidden.total_records, 2); + assert.equal(hidden.container.records_pointer, null); + for (const [ordinal, record] of hidden.records.entries()) { + assert.equal(record.ordinal, ordinal); + assert.deepEqual(record.location, { + kind: "unknown", + reason: "explicit-sensitive-value", + }); + assert.equal(record.reported, null); + assert.deepEqual(record.binary_fields, []); + assert.deepEqual(record.numeric_literals, []); + } + cases++; + } + } + await writeFile(oversizedPath, ""); + await truncate(oversizedPath, WEB_NETWORK_CAPTURE_LIMITS.inputBytes + 1); + await copyFile(join(runtime.path, "producer.har"), privatePath); + for (const mode of ["cli", "mcp"]) { + for (const format of ["har", "mitmproxy"]) { + const failure = await inspect( + mode, + { capture_path: oversizedPath, format }, + "invalid_input", + ); + assert.equal(failure.details.issues[0].reason, "out_of_range"); + assert.deepEqual(failure.details.issues[0].path, ["capture_path"]); + assert.equal( + failure.details.issues[0].expected.maximum_capture_bytes, + WEB_NETWORK_CAPTURE_LIMITS.inputBytes, + ); + cases++; + } + for (const format of ["har", "mitmproxy"]) { + const failure = await inspect( + mode, + { + capture_path: join( + runtime.path, + format === "har" ? "deep.har" : "deep.mitm", + ), + format, + }, + "invalid_input", + ); + const issue = failure.details.issues[0]; + assert.equal(issue.reason, "out_of_range"); + assert.equal(issue.path[0], "capture_path"); + assert.match(issue.path[1], /^\/_extension\/child/); + assert.equal( + issue.expected.maximum_capture_nesting, + WEB_NETWORK_CAPTURE_LIMITS.depth, + ); + cases++; + } + await inspect( + mode, + { + capture_path: join(runtime.path, "invalid-credential-key.mitm"), + format: "mitmproxy", + }, + "invalid_input", + ); + cases++; + const credentialDepth = await inspect( + mode, + { + capture_path: join(runtime.path, "deep-credential.mitm"), + format: "mitmproxy", + }, + "invalid_input", + ); + assert.equal(credentialDepth.details.issues[0].reason, "out_of_range"); + assert.match( + credentialDepth.details.issues[0].path[1], + /^\/request\/headers\/0\/1\/child/, + ); + cases++; + const value = await inspect(mode, { + capture_path: privatePath, + format: "har", + sensitive_values: ["REDACTED", "["], + }); + assert.equal(value.artifact.path, ""); + assert.ok(!JSON.stringify(value).includes("REDACTED")); + cases++; + await inspect( + mode, + { + capture_path: join(runtime.path, "invalid-private-parent.har"), + format: "har", + sensitive_values: ["response"], + }, + "invalid_input", + ); + cases++; + await inspect( + mode, + { + capture_path: join(runtime.path, "private-property-absent.har"), + format: "har", + sensitive_values: ["private-property", "REDACTED"], + }, + "invalid_input", + ); + cases++; + await inspect( + mode, + { + capture_path: join(runtime.path, "invalid-private-key.mitm"), + format: "mitmproxy", + sensitive_values: ["private-property"], + }, + "invalid_input", + ); + cases++; + } + for (const mode of ["cli", "mcp"]) { + const value = await inspect(mode, { + capture_path: join(runtime.path, "flows.mitm"), + format: "mitmproxy", + record_ordinals: [0], + sensitive_values: ["body"], + }); + assert.equal( + value.records[0].binary_fields.find( + (field) => field.pointer === "/request/content", + ).state, + "redacted", + ); + cases++; + } + for (const format of ["har", "mitmproxy"]) { + const path = join(runtime.path, `empty-${format}`); + await writeFile( + path, + format === "har" + ? JSON.stringify({ + log: { + version: "1.2", + creator: { name: "source-owned-empty-fixture", version: "1" }, + entries: [], + }, + }) + : "", + ); + for (const mode of ["cli", "mcp"]) { + const value = await inspect(mode, { capture_path: path, format }); + assert.equal(value.total_records, 0); + cases++; + } + } +} catch (cause) { + failures.push(cause); +} finally { + for (const cleanup of [ + () => client.close(), + () => transport.close(), + () => runtime.close(), + ]) { + try { + await cleanup(); + } catch (cause) { + failures.push(cause); + } + } +} +const verifier = await completeVerifierRun(run); +try { + assert.equal(verifier.process_lineage.status, "verified"); + assert.deepEqual(verifier.process_lineage.descendants, []); +} catch (cause) { + failures.push(cause); +} +if (failures.length > 0) { + console.error( + JSON.stringify( + { status: "failed", public_cases: cases, verifier }, + null, + 2, + ), + ); + throw new AggregateError( + failures, + "Historical capture verification failed; analysis and cleanup failures are retained.", + ); +} +console.log( + JSON.stringify( + { + status: "passed", + public_cases: cases, + upstream: "mitmproxy 12.2.3 FlowWriter + SaveHar", + native_failure_classifications: 7, + offline: true, + verifier, + }, + null, + 2, + ), +); + +async function inspect(mode, input, errorCategory) { + if (mode === "mcp") { + const result = await client.callTool({ + name: "inspect_web_network_capture", + arguments: input, + }); + const value = JSON.parse(mcpTextValue(result)); + if (errorCategory !== undefined) { + assert.equal(result.isError, true); + assert.equal(value.error.category, errorCategory); + for (const literal of input.sensitive_values ?? []) + assert.ok(!JSON.stringify(value.error).includes(literal)); + return value.error; + } + assert.notEqual(result.isError, true, mcpTextValue(result)); + assertSensitiveLimitations(value.evidence, input.sensitive_values ?? []); + return value.result; + } + const args = [ + entrypoint, + "inspect-web-network-capture", + input.capture_path, + input.format, + "--json", + ...(input.record_ordinals ?? []).flatMap((ordinal) => [ + "--record", + String(ordinal), + ]), + ...(input.sensitive_values ?? []).flatMap((value) => [ + "--sensitive-value", + value, + ]), + ]; + let result; + try { + result = await promisify(execFile)(process.execPath, args, { + env: environment, + timeout: 40_000, + maxBuffer: 96 * 1024 * 1024, + }); + } catch (cause) { + if (errorCategory === undefined) throw cause; + assert.equal(typeof cause.code, "number"); + const value = JSON.parse(cause.stdout); + assert.equal(value.category, errorCategory); + for (const literal of input.sensitive_values ?? []) + assert.ok(!JSON.stringify(value).includes(literal)); + return value; + } + assert.equal(errorCategory, undefined, "Expected malformed capture to fail"); + const evidence = JSON.parse(result.stdout); + assertSensitiveLimitations(evidence, input.sensitive_values ?? []); + return evidence.normalized_result; +} + +function assertSensitiveLimitations(evidence, literals) { + for (const value of Object.values(evidence.parameters)) + if (typeof value === "string") + for (const literal of literals) + assert.ok( + !value.includes(literal), + "Declared literal retained in Evidence parameter", + ); + const result = evidence.normalized_result; + if (result.artifact.sha256 === null) assert.equal(evidence.subject, null); + else assert.equal(evidence.subject.digest.sha256, result.artifact.sha256); + assert.deepEqual( + evidence.provider, + result.decoder, + "Evidence provider tuple changed", + ); + assert.equal( + result.decoder.name, + result.format === "har" + ? "REA HAR capture adapter" + : "REA offline native mitmproxy adapter", + ); + const texts = [ + evidence.limitations, + result.limitations, + ...result.records.map((record) => record.limitations), + ].flat(); + for (const literal of literals) + assert.ok( + texts.every((text) => !text.includes(literal)), + "Declared literal retained in authored limitations", + ); +} + +function assertBinary(record, pointer, bytes) { + const field = record.binary_fields.find((item) => item.pointer === pointer); + assert.ok(field, `Missing native byte field ${pointer}`); + assert.equal(field.state, "retained"); + assert.equal(field.content_base64, bytes.toString("base64")); + assert.equal(field.bytes, bytes.length); + assert.equal(field.sha256, createHash("sha256").update(bytes).digest("hex")); +} diff --git a/skills/reverse-engineer-anything/SKILL.md b/skills/reverse-engineer-anything/SKILL.md index 3004ed5b2..54ed16b34 100644 --- a/skills/reverse-engineer-anything/SKILL.md +++ b/skills/reverse-engineer-anything/SKILL.md @@ -2,9 +2,9 @@ name: reverse-engineer-anything description: Reverse engineer native, managed, Electron/JavaScript, packaged, firmware, and browser targets with REA. Use shipped-artifact or requested runtime evidence to explain features, compare versions, decompile code, or guide a reconstruction. Skip REA for ordinary source-repository architecture analysis. metadata: - version: "27" - tool_count: 133 - catalog_digest: "8c32372537b3d12e1f0befc51b8b9a5d083d37041eac3ceb1ba9d0a3f1853b2c" + version: "28" + tool_count: 134 + catalog_digest: "e41a84b460b65f3a8d0d33498d958d249a1de81a9660c18580a0e3ab5d6941e1" --- # REA @@ -93,6 +93,11 @@ explicit path or endpoint and do not need it. consult the NativeAOT workflow in `references/native-and-artifacts.md` for optional metadata recovery and its supported host/layout boundary. - User-owned browser page already open: `list_browser_targets`. +- Retained HAR/native mitmproxy capture: `inspect_web_network_capture`. Select + original record ordinals when useful; inspect producer fields and byte/number + sidecars without fetching recorded URLs or claiming live attribution. See + [historical captures](https://github.com/morluto/rea/blob/main/docs/web-network-captures.md) for the exact upstream + profile and credential exclusions. - User-owned Electron runtime already open: `list_electron_targets`. - Native executable, library, or analysis database: `open_binary`, then use focused analysis tools directly; call `binary_overview` when metadata or diff --git a/src/application/WebNetworkCapturePort.ts b/src/application/WebNetworkCapturePort.ts new file mode 100644 index 000000000..5f9cd5545 --- /dev/null +++ b/src/application/WebNetworkCapturePort.ts @@ -0,0 +1,15 @@ +import type { ExecutionOptions } from "./AnalysisProvider.js"; +import type { AnalysisError } from "../domain/analysisErrorBase.js"; +import type { Result } from "../domain/result.js"; +import type { + InspectWebNetworkCaptureInput, + WebNetworkCapture, +} from "../domain/webNetworkCapture.js"; + +/** Replaceable offline capture port; producer protocols remain in format adapters. */ +export interface WebNetworkCapturePort { + inspect( + input: InspectWebNetworkCaptureInput, + options?: ExecutionOptions, + ): Promise>; +} diff --git a/src/application/WebNetworkCaptureService.test.ts b/src/application/WebNetworkCaptureService.test.ts new file mode 100644 index 000000000..0105e2d3f --- /dev/null +++ b/src/application/WebNetworkCaptureService.test.ts @@ -0,0 +1,340 @@ +import { expect, it } from "vitest"; +import { WebNetworkCaptureService } from "./WebNetworkCaptureService.js"; +import { historicalHar } from "../../tests/fixtures/historicalHar.js"; +import { ok } from "../domain/result.js"; +import { projectAnalysisError } from "../domain/analysisErrorProjection.js"; +import { parseEvidence } from "../domain/evidence.js"; +import { + webNetworkCaptureSchema, + type WebNetworkCapture, +} from "../domain/webNetworkCapture.js"; + +it.each(["REDACTED", "[", "]", "/capture.har"])( + "excludes sensitive path text even when a replacement marker would collide: %s", + async (literal) => { + const report = fixture(); + report.artifact.path = `/capture.har/${literal}`; + const result = await new WebNetworkCaptureService({ + inspect: () => Promise.resolve(ok(report)), + }).inspect({ + capture_path: report.artifact.path, + format: "har", + sensitive_values: [literal], + }); + if (!result.ok) throw result.error; + expect(result.value.subject?.local_path).not.toContain(literal); + expect(result.value.parameters.capture_path).not.toContain(literal); + expect( + result.value.locations.every( + (location) => + location.kind !== "artifact-path" || !location.path.includes(literal), + ), + ).toBe(true); + expect(result.value.subject?.digest.sha256).toBe(report.artifact.sha256); + }, +); + +it("keeps observed artifact identity when the entire sensitive path is excluded", async () => { + const result = await new WebNetworkCaptureService({ + inspect: () => Promise.resolve(ok(fixture())), + }).inspect({ + capture_path: "/capture.har", + format: "har", + sensitive_values: ["/", "[", "REDACTED"], + }); + if (!result.ok) throw result.error; + expect(result.value.subject).toMatchObject({ + local_path: "", + digest: { sha256: "a".repeat(64) }, + }); + expect(result.value.locations).toEqual([]); +}); + +const fixture = (): WebNetworkCapture => ({ + decoder: { id: "test-historical-port", name: "test port", version: "1" }, + container: { + reported: null, + numeric_literals: [], + redactions: [], + records_pointer: "/log/entries", + }, + total_records: 1, + records: [ + { + ordinal: 0, + location: { kind: "json-pointer", pointer: "/log/entries/0" }, + reported: historicalHar().log.entries[0] ?? null, + numeric_literals: [], + binary_fields: [], + redactions: [], + limitations: [], + }, + ], + artifact: { path: "/capture.har", sha256: "a".repeat(64), bytes: 10 }, + format: "har", + runtime_attribution: "unknown", + limitations: ["historical fixture"], +}); + +it.each(["a".repeat(64), "aaaa"])( + "marks an explicitly sensitive artifact digest unknown without inventing another identity: %s", + async (literal) => { + const result = await new WebNetworkCaptureService({ + inspect: () => Promise.resolve(ok(fixture())), + }).inspect({ + capture_path: "/capture.har", + format: "har", + sensitive_values: [literal], + }); + if (!result.ok) throw result.error; + expect(result.value.subject).toBeNull(); + expect(result.value.normalized_result).toMatchObject({ + artifact: { path: "/capture.har", sha256: null, bytes: 10 }, + }); + expect(result.value.parameters.capture_path).toBe("/capture.har"); + expect(result.value.locations).toEqual([ + { kind: "artifact-path", path: "/capture.har" }, + ]); + expect(parseEvidence(result.value)).toEqual(result.value); + }, +); + +it("projects the unknown-subject explanation before generating its Evidence ID", async () => { + const result = await new WebNetworkCaptureService({ + inspect: () => Promise.resolve(ok(fixture())), + }).inspect({ + capture_path: "/capture.har", + format: "har", + sensitive_values: ["a"], + }); + if (!result.ok) throw result.error; + expect(result.value.subject).toBeNull(); + expect(result.value.limitations.every((text) => !text.includes("a"))).toBe( + true, + ); + expect(parseEvidence(result.value)).toEqual(result.value); +}); + +it("rejects an adapter omitting its original observed digest before caller projection", async () => { + const report = fixture(); + report.artifact.sha256 = null; + const result = await new WebNetworkCaptureService({ + inspect: () => Promise.resolve(ok(report)), + }).inspect({ capture_path: report.artifact.path, format: "har" }); + expect(result.ok).toBe(false); + if (!result.ok) expect(result.error._tag).toBe("AnalysisOutputError"); +}); + +it.each([ + ["producer"], + ["artifact"], + ["coordinates"], + ["producer", "REDACTED", "…"], + ["producer", "[", "]"], +])( + "excludes declared literals from all authored limitations: %j", + async (...literals) => { + const report = fixture(); + report.artifact.path = "/artifact/capture.har"; + report.limitations = [ + "Historical producer artifact coordinates are retained.", + ]; + const first = report.records[0]; + if (first === undefined) throw new Error("fixture missing record"); + first.limitations = ["This is producer-decoded artifact evidence."]; + const result = await new WebNetworkCaptureService({ + inspect: () => Promise.resolve(ok(report)), + }).inspect({ + capture_path: report.artifact.path, + format: "har", + sensitive_values: literals, + }); + if (!result.ok) throw result.error; + const projected = webNetworkCaptureSchema.parse( + result.value.normalized_result, + ); + const texts = [ + result.value.limitations, + projected.limitations, + ...projected.records.map((record) => record.limitations), + ].flat(); + for (const literal of literals) + expect(texts.every((text) => !text.includes(literal))).toBe(true); + expect(result.value.subject?.digest.sha256).toBe(report.artifact.sha256); + expect(projected.decoder).toEqual(report.decoder); + expect(projected.records[0]?.reported).toEqual(first.reported); + }, +); + +it("keeps explicitly sensitive values out of Evidence parameters and preserves historical authority", async () => { + const service = new WebNetworkCaptureService({ + inspect: () => Promise.resolve(ok(fixture())), + }); + const result = await service.inspect({ + capture_path: "/capture.har", + format: "har", + sensitive_values: ["explicit-private-value"], + }); + if (!result.ok) throw result.error; + expect(result.value.parameters).toMatchObject({ + sensitive_value_count: 1, + record_ordinals: [0], + }); + expect(JSON.stringify(result.value)).not.toContain("explicit-private-value"); + expect(result.value.authority).toBe("historical-reference"); + expect(result.value.provider).toEqual(fixture().decoder); +}); + +it.each(["har", "ha"])( + "omits a format parameter containing a declared sensitive literal: %s", + async (literal) => { + const result = await new WebNetworkCaptureService({ + inspect: () => Promise.resolve(ok(fixture())), + }).inspect({ + capture_path: "/capture.har", + format: "har", + sensitive_values: [literal], + }); + if (!result.ok) throw result.error; + expect(result.value.parameters).not.toHaveProperty("format"); + expect(JSON.stringify(result.value.parameters)).not.toContain(literal); + expect(result.value.normalized_result).toMatchObject({ format: "har" }); + }, +); + +it.each([ + { record_ordinals: [0, 0] }, + { record_ordinals: [1] }, + { capture_path: "relative.har" }, +])("rejects invalid caller selection %j", async (extra) => { + const result = await new WebNetworkCaptureService({ + inspect: () => Promise.resolve(ok(fixture())), + }).inspect({ capture_path: "/capture.har", format: "har", ...extra }); + expect(result.ok).toBe(false); + if (!result.ok) expect(result.error._tag).toBe("AnalysisInputError"); +}); + +it("rejects a port changing the artifact identity or producer ordinal sequence", async () => { + const report = fixture(); + report.artifact.path = "/different.har"; + const result = await new WebNetworkCaptureService({ + inspect: () => Promise.resolve(ok(report)), + }).inspect({ capture_path: "/capture.har", format: "har" }); + expect(result.ok).toBe(false); + if (!result.ok) expect(result.error._tag).toBe("AnalysisOutputError"); +}); + +it("applies an explicit sensitive path literal after verifying the selected artifact", async () => { + const result = await new WebNetworkCaptureService({ + inspect: () => Promise.resolve(ok(fixture())), + }).inspect({ + capture_path: "/capture.har", + format: "har", + sensitive_values: ["capture.har"], + }); + if (!result.ok) throw result.error; + expect(JSON.stringify(result.value)).not.toContain("capture.har"); + expect(result.value.parameters.capture_path).toBe(""); +}); + +it.each(["log", "entries"])( + "excludes a sensitive producer coordinate without inventing a replacement: %s", + async (literal) => { + const result = await new WebNetworkCaptureService({ + inspect: () => Promise.resolve(ok(fixture())), + }).inspect({ + capture_path: "/capture.har", + format: "har", + sensitive_values: [literal], + }); + if (!result.ok) throw result.error; + expect(result.value.normalized_result).toMatchObject({ + container: { records_pointer: null }, + records: [ + { + ordinal: 0, + location: { kind: "unknown", reason: "explicit-sensitive-value" }, + }, + ], + }); + expect(JSON.stringify(result.value)).not.toContain(literal); + }, +); + +it("excludes an escaped sensitive sidecar coordinate while preserving its reported field", async () => { + const report = fixture(); + report.container.reported = { "field/name": 42 }; + report.container.numeric_literals = [ + { pointer: "/field~1name", producer_type: "json-number", literal: "42" }, + ]; + report.container.redactions = [ + { pointer: "/field~1name", reason: "explicit-sensitive-value" }, + ]; + const result = await new WebNetworkCaptureService({ + inspect: () => Promise.resolve(ok(report)), + }).inspect({ + capture_path: "/capture.har", + format: "har", + sensitive_values: ["~1"], + }); + if (!result.ok) throw result.error; + expect(result.value.normalized_result).toMatchObject({ + container: { + reported: { "field/name": 42 }, + numeric_literals: [], + redactions: [], + }, + }); + expect(JSON.stringify(result.value)).not.toContain("~1"); +}); + +it.each([ + { literal: "12345", input: { record_ordinals: [12345] } }, + { literal: "Each", input: { record_ordinals: [0, 0] } }, + { literal: "absolute", input: { capture_path: "relative.har" } }, + { literal: "secret", input: { mysecret: true } }, +])( + "excludes declared text from application-generated input errors: $literal", + async ({ literal, input }) => { + const result = await new WebNetworkCaptureService({ + inspect: () => Promise.resolve(ok(fixture())), + }).inspect({ + capture_path: "/capture.har", + format: "har", + sensitive_values: [literal], + ...input, + }); + if (result.ok) throw new Error("Expected invalid selection"); + expect(result.error._tag).toBe("AnalysisInputError"); + expect(JSON.stringify(projectAnalysisError(result.error))).not.toContain( + literal, + ); + }, +); + +it.each(["12345", "67890", "234", "789"])( + "excludes sensitive native byte-range coordinates: %s", + async (literal) => { + const report = fixture(); + report.format = "mitmproxy"; + report.artifact.path = "/capture.mitm"; + const record = report.records[0]; + if (record === undefined) throw new Error("record missing"); + record.location = { kind: "byte-range", offset: 12345, bytes: 67890 }; + const result = await new WebNetworkCaptureService({ + inspect: () => Promise.resolve(ok(report)), + }).inspect({ + capture_path: report.artifact.path, + format: "mitmproxy", + sensitive_values: [literal], + }); + if (!result.ok) throw result.error; + expect(result.value.normalized_result).toMatchObject({ + records: [ + { location: { kind: "unknown", reason: "explicit-sensitive-value" } }, + ], + }); + expect(result.value.raw_result).toBeNull(); + expect(parseEvidence(result.value)).toEqual(result.value); + }, +); diff --git a/src/application/WebNetworkCaptureService.ts b/src/application/WebNetworkCaptureService.ts new file mode 100644 index 000000000..ca86b7288 --- /dev/null +++ b/src/application/WebNetworkCaptureService.ts @@ -0,0 +1,202 @@ +import { redactExplicitFailure } from "../domain/explicitSensitiveFailure.js"; +import { excludeCaptureCoordinates } from "../domain/webNetworkCaptureCoordinates.js"; +import { redactExplicitText } from "../domain/explicitSensitiveValues.js"; +import { isAbsolute } from "node:path"; +import type { ExecutionOptions } from "./AnalysisProvider.js"; +import type { WebNetworkCapturePort } from "./WebNetworkCapturePort.js"; +import type { AnalysisError } from "../domain/analysisErrorBase.js"; +import { + AnalysisCancelledError, + AnalysisInputError, + AnalysisOutputError, +} from "../domain/analysisErrorCore.js"; +import { createEvidence, type Evidence } from "../domain/evidence.js"; +import { projectInputIssues } from "../domain/inputIssueProjection.js"; +import { jsonObjectSchema } from "../domain/jsonValue.js"; +import { err, ok, type Result } from "../domain/result.js"; +import { + inspectWebNetworkCaptureInputSchema, + webNetworkCaptureSchema, +} from "../domain/webNetworkCapture.js"; + +const OPERATION = "inspect_web_network_capture"; + +/** Shared CLI/MCP historical inspection preserves caller selection and producer record identity. */ +export class WebNetworkCaptureService { + constructor(readonly port: WebNetworkCapturePort) {} + + /** Return selected complete records inline, without inventing live browser attribution. */ + async inspect( + rawInput: unknown, + options?: ExecutionOptions, + ): Promise> { + if (options?.signal?.aborted) + return err(new AnalysisCancelledError(OPERATION)); + const declarations = + inspectWebNetworkCaptureInputSchema.shape.sensitive_values.safeParse( + typeof rawInput === "object" && + rawInput !== null && + "sensitive_values" in rawInput + ? rawInput.sensitive_values + : undefined, + ); + const failure = (error: AnalysisError): Result => + err( + redactExplicitFailure( + error, + declarations.success ? declarations.data : [], + ), + ); + const parsed = inspectWebNetworkCaptureInputSchema.safeParse(rawInput); + if (!parsed.success) + return failure( + new AnalysisInputError( + OPERATION, + { cause: parsed.error }, + projectInputIssues(parsed.error.issues, rawInput), + ), + ); + const input = parsed.data; + if (!isAbsolute(input.capture_path)) + return failure( + new AnalysisInputError(OPERATION, undefined, [ + { + path: ["capture_path"], + reason: "invalid_format", + message: "Expected an absolute filesystem path on this host.", + }, + ]), + ); + if ( + input.record_ordinals !== undefined && + new Set(input.record_ordinals).size !== input.record_ordinals.length + ) + return failure( + new AnalysisInputError(OPERATION, undefined, [ + { + path: ["record_ordinals"], + reason: "invalid_format", + message: "Each requested producer ordinal must occur once.", + }, + ]), + ); + const loaded = await this.port.inspect(input, options); + if (!loaded.ok) return failure(loaded.error); + if (options?.signal?.aborted) + return failure(new AnalysisCancelledError(OPERATION)); + const report = webNetworkCaptureSchema.safeParse(loaded.value); + if (!report.success) + return failure( + new AnalysisOutputError( + OPERATION, + "Capture adapter returned malformed historical evidence.", + ), + ); + const value = report.data; + const artifactSha256 = value.artifact.sha256; + if (artifactSha256 === null) + return failure( + new AnalysisOutputError( + OPERATION, + "Capture adapter omitted its observed artifact digest before explicit projection.", + ), + ); + if ( + value.artifact.path !== input.capture_path || + value.format !== input.format || + value.total_records !== value.records.length || + value.records.some((record, index) => record.ordinal !== index) + ) + return failure( + new AnalysisOutputError( + OPERATION, + "Capture adapter changed the selected artifact, format or original record sequence.", + ), + ); + const safeValue = excludeCaptureCoordinates(value, input.sensitive_values); + const redactText = (text: string): string => + redactExplicitText(text, input.sensitive_values); + const selected = + input.record_ordinals ?? value.records.map(({ ordinal }) => ordinal); + const records = []; + for (const ordinal of selected) { + const record = safeValue.records[ordinal]; + if (record === undefined) + return failure( + new AnalysisInputError(OPERATION, undefined, [ + { + path: ["record_ordinals"], + reason: "invalid_format", + message: `Producer ordinal ${ordinal} is outside the retained capture (${value.total_records} records).`, + }, + ]), + ); + records.push({ + ...record, + limitations: record.limitations.map(redactText), + }); + } + const safePath = input.sensitive_values.some((literal) => + input.capture_path.includes(literal), + ) + ? "" + : input.capture_path; + const safeSha256 = input.sensitive_values.some((literal) => + artifactSha256.includes(literal), + ) + ? null + : artifactSha256; + const subjectUnavailableReason = redactText( + "The explicitly sensitive artifact digest is excluded; its cryptographic identity is unknown in this Evidence.", + ); + const limitations = [ + ...safeValue.limitations, + ...(safePath === "" + ? [ + "The explicitly sensitive artifact path is excluded; its size remains available.", + ] + : []), + ].map(redactText); + const parameters = jsonObjectSchema.parse({ + capture_path: safePath, + ...(input.sensitive_values.some((literal) => + input.format.includes(literal), + ) + ? {} + : { format: input.format }), + record_ordinals: selected, + sensitive_value_count: input.sensitive_values.length, + }); + return ok( + createEvidence( + safeSha256 === null + ? undefined + : { + path: safePath, + sha256: safeSha256, + format: "file", + }, + value.decoder, + { + operation: OPERATION, + parameters, + result: { + ...safeValue, + artifact: { ...value.artifact, path: safePath, sha256: safeSha256 }, + records, + limitations: + safeSha256 === null + ? [...limitations, subjectUnavailableReason] + : limitations, + }, + confidence: "observed", + authority: "historical-reference", + limitations, + ...(safeSha256 === null ? { subjectUnavailableReason } : {}), + locations: + safePath === "" ? [] : [{ kind: "artifact-path", path: safePath }], + }, + ), + ); + } +} diff --git a/src/artifacts/readStableArtifact.test.ts b/src/artifacts/readStableArtifact.test.ts new file mode 100644 index 000000000..6c02e1b98 --- /dev/null +++ b/src/artifacts/readStableArtifact.test.ts @@ -0,0 +1,62 @@ +import { + lstat, + mkdtemp, + open, + rm, + symlink, + unlink, + writeFile, +} from "node:fs/promises"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; +import { expect, it } from "vitest"; +import { readStableArtifact } from "./readStableArtifact.js"; + +const checkChange = async ( + stage: "before-open" | "after-read" | "symlink-replacement", +): Promise => { + const root = await mkdtemp(join(tmpdir(), "rea-stable-read-")); + const path = join(root, "selected"); + await writeFile(path, "original evidence"); + let inspected = 0; + try { + const reading = readStableArtifact(path, 1024, undefined, { + async lstat(selected) { + inspected++; + if (stage === "after-read" && inspected === 2) await unlink(selected); + const stat = await lstat(selected, { bigint: true }); + if (inspected === 1 && stage !== "after-read") { + await unlink(selected); + if (stage === "symlink-replacement") + await symlink("missing", selected); + } + return stat; + }, + open, + }); + await expect(reading).rejects.toMatchObject({ reason: "integrity" }); + } finally { + await rm(root, { recursive: true, force: true }); + } +}; + +it.each(["before-open", "after-read"] as const)( + "retains an actual post-validation %s change as integrity failure", + checkChange, +); + +it.skipIf(process.platform === "win32")( + "retains a post-validation symlink replacement as an integrity failure", + () => checkChange("symlink-replacement"), +); + +it("keeps initially missing inputs distinguishable from later changes", async () => { + const root = await mkdtemp(join(tmpdir(), "rea-stable-missing-")); + try { + await expect( + readStableArtifact(join(root, "missing"), 1024), + ).rejects.toMatchObject({ code: "ENOENT" }); + } finally { + await rm(root, { recursive: true, force: true }); + } +}); diff --git a/src/artifacts/readStableArtifact.ts b/src/artifacts/readStableArtifact.ts index 83280550e..560d00c95 100644 --- a/src/artifacts/readStableArtifact.ts +++ b/src/artifacts/readStableArtifact.ts @@ -1,16 +1,50 @@ import { createHash } from "node:crypto"; -import { constants } from "node:fs"; -import { lstat, open } from "node:fs/promises"; +import { constants, type BigIntStats } from "node:fs"; +import { lstat, open, type FileHandle } from "node:fs/promises"; import { ArtifactReaderFailure } from "./ArtifactReader.js"; +/** Filesystem boundary for stable reads, including deterministic replacement probes. */ +export interface StableArtifactFileSystem { + lstat(path: string): Promise; + open(path: string, flags: number): Promise; +} + +const FILE_SYSTEM: StableArtifactFileSystem = { + lstat: (path) => lstat(path, { bigint: true }), + open, +}; + +const afterValidation = async ( + path: string, + stage: string, + read: () => Promise, +): Promise => { + try { + return await read(); + } catch (cause) { + if ( + cause instanceof Error && + "code" in cause && + ["ENOENT", "ENOTDIR", "ELOOP"].includes(String(cause.code)) + ) + throw new ArtifactReaderFailure( + "integrity", + `Artifact changed ${stage}: ${path}`, + { cause }, + ); + throw cause; + } +}; + /** Read one bounded regular file and reject replacement or in-place changes. */ export const readStableArtifact = async ( path: string, maximumBytes: number, signal?: AbortSignal, + fileSystem: StableArtifactFileSystem = FILE_SYSTEM, ): Promise<{ readonly bytes: Buffer; readonly sha256: string }> => { signal?.throwIfAborted(); - const before = await lstat(path, { bigint: true }); + const before = await fileSystem.lstat(path); if (!before.isFile() || before.isSymbolicLink()) throw new ArtifactReaderFailure( "path", @@ -21,9 +55,11 @@ export const readStableArtifact = async ( "limit", `Selected artifact exceeds ${String(maximumBytes)} bytes: ${path}`, ); - const file = await open( - path, - constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK, + const file = await afterValidation(path, "before open", () => + fileSystem.open( + path, + constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK, + ), ); try { const initial = await file.stat({ bigint: true }); @@ -59,7 +95,9 @@ export const readStableArtifact = async ( chunks.push(bytes); } const after = await file.stat({ bigint: true }); - const current = await lstat(path, { bigint: true }); + const current = await afterValidation(path, "during read", () => + fileSystem.lstat(path), + ); if ( [after, current].some( (stat) => diff --git a/src/browser/history/CaptureFailures.test.ts b/src/browser/history/CaptureFailures.test.ts new file mode 100644 index 000000000..71026ed6c --- /dev/null +++ b/src/browser/history/CaptureFailures.test.ts @@ -0,0 +1,101 @@ +import { expect, it } from "vitest"; +import { ArtifactReaderFailure } from "../../artifacts/ArtifactReader.js"; +import { projectAnalysisError } from "../../domain/analysisErrorProjection.js"; +import { analysisErrorProjectionSchema } from "../../contracts/errorSchemas.js"; +import { inspectWebNetworkCaptureInputSchema } from "../../domain/webNetworkCapture.js"; +import { historicalCaptureFailure } from "./CaptureFailures.js"; +import { redactExplicitFailure } from "../../domain/explicitSensitiveFailure.js"; +import { OwnedCommandFailure } from "../../process/OwnedCommand.js"; + +it.each(["before open", "during read"])( + "preserves observed capture changes %s as retryable acquisition integrity failures", + (stage) => { + const path = "/selected/private-marker.har"; + const input = inspectWebNetworkCaptureInputSchema.parse({ + capture_path: path, + format: "har", + }); + const cause = new ArtifactReaderFailure( + "integrity", + `Artifact changed ${stage}: ${path}`, + ); + const error = historicalCaptureFailure(input, cause, "capture-read"); + const projected = projectAnalysisError(error); + expect(analysisErrorProjectionSchema.parse(projected)).toMatchObject({ + code: "artifact_changed", + category: "integrity_mismatch", + retryable: true, + details: { + operation: "inspect_web_network_capture", + path, + boundary: "stable-artifact-read", + reason: cause.message, + }, + }); + expect(projected.remediation.action).toContain("stable"); + const redacted = projectAnalysisError( + redactExplicitFailure(error, ["private-marker", "REDACTED"]), + ); + expect(redacted.code).toBe("artifact_changed"); + expect(JSON.stringify(redacted)).not.toContain("private-marker"); + expect(JSON.stringify(redacted)).not.toContain("REDACTED"); + }, +); + +it("retains actual command status and both diagnostic streams when cleanup also fails", () => { + const input = inspectWebNetworkCaptureInputSchema.parse({ + capture_path: "/selected/input.har", + format: "har", + }); + const cause = new OwnedCommandFailure( + "process", + "Observed command failure", + { + stdout: { text: "original provider output", bytes: 24 }, + stderr: { text: "private-marker provider reason", bytes: 30 }, + exitCode: 2, + signal: null, + }, + "Observed cleanup failure", + undefined, + ["pid:42"], + ); + const error = historicalCaptureFailure(input, cause, "decoder"); + expect(projectAnalysisError(error)).toMatchObject({ + code: "cleanup_incomplete", + details: { + diagnostics: { + previous_error: { + failure_kind: "process", + exit_code: 2, + stdout: "original provider output", + stderr: "private-marker provider reason", + }, + }, + }, + }); + const redacted = projectAnalysisError( + redactExplicitFailure(error, ["private-marker"]), + ); + expect(JSON.stringify(redacted)).not.toContain("private-marker"); + expect(redacted.code).toBe("cleanup_incomplete"); +}); +it.each(["io", "unavailable"] as const)( + "keeps %s read failures separate from malformed capture input", + (reason) => { + const input = inspectWebNetworkCaptureInputSchema.parse({ + capture_path: "/selected/input.har", + format: "har", + }); + const error = historicalCaptureFailure( + input, + new ArtifactReaderFailure(reason, "Observed acquisition failure"), + "capture-read", + ); + expect(projectAnalysisError(error)).toMatchObject({ + code: "execution_failure", + category: "execution_failure", + details: { diagnostics: { phase: "capture-read", failure_kind: reason } }, + }); + }, +); diff --git a/src/browser/history/CaptureFailures.ts b/src/browser/history/CaptureFailures.ts new file mode 100644 index 000000000..fec40d549 --- /dev/null +++ b/src/browser/history/CaptureFailures.ts @@ -0,0 +1,146 @@ +import type { ExecutionOptions } from "../../application/AnalysisProvider.js"; +import { ArtifactReaderFailure } from "../../artifacts/ArtifactReader.js"; +import { AnalysisError } from "../../domain/analysisErrorBase.js"; +import { + AnalysisAccessDeniedError, + AnalysisArtifactChangedError, + AnalysisCancelledError, + AnalysisCapabilityUnavailableError, + AnalysisInputError, + AnalysisOutputError, + AnalysisTimeoutError, +} from "../../domain/analysisErrorCore.js"; +import { ProviderAdapterError } from "../../domain/providerAdapterError.js"; +import { ProviderCleanupError } from "../../domain/providerCleanupError.js"; +import { + WEB_NETWORK_CAPTURE_LIMITS, + type InspectWebNetworkCaptureInput, +} from "../../domain/webNetworkCapture.js"; +import { OwnedCommandFailure } from "../../process/OwnedCommand.js"; +const OPERATION = "inspect_web_network_capture"; + +/** Preserve the origin and typed cause of a historical capture failure. */ +export const historicalCaptureFailure = ( + input: InspectWebNetworkCaptureInput, + cause: unknown, + phase: "capture-read" | "decoder", + options?: ExecutionOptions, +): AnalysisError => { + if (cause instanceof AnalysisError) return cause; + if (cause instanceof OwnedCommandFailure) { + if (cause.cleanupFailure !== null) + return new ProviderCleanupError( + input.format, + cause.resources, + { + reason: cause.cleanupFailure, + previous_error: { + failure_kind: cause.reason, + message: cause.message, + exit_code: cause.snapshot?.exitCode ?? null, + signal: cause.snapshot?.signal ?? null, + stdout: cause.snapshot?.stdout.text ?? null, + stderr: cause.snapshot?.stderr.text ?? null, + diagnostic_truncated: cause.snapshot?.diagnosticTruncated ?? null, + }, + }, + { operation: OPERATION, cause }, + ); + if (cause.reason === "cancelled") + return new AnalysisCancelledError(OPERATION); + if (cause.reason === "timeout") + return new AnalysisTimeoutError( + OPERATION, + WEB_NETWORK_CAPTURE_LIMITS.timeoutMs, + ); + if (cause.reason === "output-limit") + return new AnalysisOutputError(OPERATION, cause.message); + } + if (options?.signal?.aborted) return new AnalysisCancelledError(OPERATION); + if ( + phase === "capture-read" && + cause instanceof ArtifactReaderFailure && + cause.reason === "integrity" + ) + return new AnalysisArtifactChangedError( + OPERATION, + input.capture_path, + cause.message, + { cause }, + ); + if ( + phase === "capture-read" && + cause instanceof ArtifactReaderFailure && + cause.reason === "cancelled" + ) + return new AnalysisCancelledError(OPERATION); + if ( + phase === "capture-read" && + cause instanceof ArtifactReaderFailure && + (cause.reason === "limit" || + cause.reason === "format" || + cause.reason === "path") + ) + return new AnalysisInputError(OPERATION, { cause }, [ + { + path: ["capture_path"], + reason: cause.reason === "limit" ? "out_of_range" : "invalid_format", + message: + cause.reason === "limit" + ? `${cause.message}. Select a capture within the ${WEB_NETWORK_CAPTURE_LIMITS.inputBytes}-byte input budget.` + : cause.message, + ...(cause.reason === "limit" + ? { + expected: { + maximum_capture_bytes: WEB_NETWORK_CAPTURE_LIMITS.inputBytes, + }, + } + : {}), + }, + ]); + if ( + phase === "capture-read" && + cause instanceof Error && + "code" in cause && + (cause.code === "EACCES" || cause.code === "EPERM") + ) + return new AnalysisAccessDeniedError( + OPERATION, + input.capture_path, + cause.code, + { cause }, + ); + if ( + phase === "capture-read" && + cause instanceof Error && + "code" in cause && + ["ENOENT", "ENOTDIR"].includes(String(cause.code)) + ) + return new AnalysisInputError(OPERATION, { cause }, [ + { + path: ["capture_path"], + reason: "invalid_value", + message: `Selected capture could not be read (${String(cause.code)}): ${input.capture_path}.`, + }, + ]); + return new ProviderAdapterError(input.format, OPERATION, { + cause, + diagnostics: { + phase, + capture_path: input.capture_path, + reason: cause instanceof Error ? cause.message : String(cause), + ...(cause instanceof ArtifactReaderFailure + ? { failure_kind: cause.reason } + : {}), + ...(cause instanceof OwnedCommandFailure && cause.snapshot !== null + ? { + exit_code: cause.snapshot.exitCode ?? null, + signal: cause.snapshot.signal ?? null, + stdout: cause.snapshot.stdout.text, + stderr: cause.snapshot.stderr.text, + diagnostic_truncated: cause.snapshot.diagnosticTruncated ?? false, + } + : {}), + }, + }); +}; diff --git a/src/browser/history/CaptureFormatError.ts b/src/browser/history/CaptureFormatError.ts new file mode 100644 index 000000000..fd7a9272d --- /dev/null +++ b/src/browser/history/CaptureFormatError.ts @@ -0,0 +1,10 @@ +/** Expected malformed, unsupported or oversized producer data at a historical decoder boundary. */ +export class CaptureFormatError extends Error { + constructor( + readonly reason: "format" | "unsupported" | "input-limit", + message: string, + readonly pointer = "", + ) { + super(message); + } +} diff --git a/src/browser/history/CaptureRedaction.test.ts b/src/browser/history/CaptureRedaction.test.ts new file mode 100644 index 000000000..d6ce3bb1c --- /dev/null +++ b/src/browser/history/CaptureRedaction.test.ts @@ -0,0 +1,61 @@ +import { expect, it } from "vitest"; +import { CaptureRedaction } from "./CaptureRedaction.js"; + +it.each([ + { + value: " https://user:password@example.test/path ", + expected: " https://example.test/path ", + }, + { + value: "\tHTTPS://user:password@example.test/path\t", + expected: "\tHTTPS://example.test/path\t", + }, + { + value: " \t//user:password@example.test/path?token=ordinary#fragment\t ", + expected: " \t//example.test/path?token=ordinary#fragment\t ", + }, + { + value: "\r\n \v\fhttps://user:password@example.test/path", + expected: "\r\n \v\fhttps://example.test/path", + }, + { + value: "\u0000\u001f//user:password@example.test/path", + expected: "\u0000\u001f//example.test/path", + }, +])( + "excludes URL userinfo after optional whitespace while preserving the original header text: $value", + ({ value, expected }) => { + const redactor = new CaptureRedaction([]); + expect(redactor.text(value, "/response/headers/0/value", true)).toBe( + expected, + ); + expect(redactor.redactions).toEqual([ + { pointer: "/response/headers/0/value", reason: "transport-credential" }, + ]); + }, +); + +it("preserves the same URL-looking text in an ordinary extension", () => { + const redactor = new CaptureRedaction([]); + const value = " \thttps://user:password@example.test/path\t "; + expect(redactor.text(value, "/_extension")).toBe(value); + expect(redactor.redactions).toEqual([]); +}); + +it.each(["\ud800", "\udfff"])( + "does not invent replacement UTF-8 bytes for a non-scalar declaration", + (literal) => { + const redactor = new CaptureRedaction([literal]); + expect(redactor.sensitiveBytes(Buffer.from("\ufffd"))).toBe(false); + expect(redactor.text(literal, "/reported")).toBe(null); + }, +); + +it("matches scalar UTF-8 declarations and complete surrogate pairs exactly", () => { + expect( + new CaptureRedaction(["\ufffd"]).sensitiveBytes(Buffer.from("\ufffd")), + ).toBe(true); + expect( + new CaptureRedaction(["\ud83d\ude00"]).sensitiveBytes(Buffer.from("😀")), + ).toBe(true); +}); diff --git a/src/browser/history/CaptureRedaction.ts b/src/browser/history/CaptureRedaction.ts new file mode 100644 index 000000000..86945aed0 --- /dev/null +++ b/src/browser/history/CaptureRedaction.ts @@ -0,0 +1,93 @@ +import type { WebNetworkCaptureRecord } from "../../domain/webNetworkCapture.js"; + +type Redaction = WebNetworkCaptureRecord["redactions"][number]; +const CREDENTIAL_HEADERS = new Set([ + "authorization", + "proxy-authorization", + "cookie", + "set-cookie", +]); + +/** Literal redaction and structural authentication exclusions preserve unrelated local evidence. */ +export class CaptureRedaction { + readonly redactions: Redaction[] = []; + readonly #sensitiveByteValues: readonly Buffer[]; + constructor(readonly sensitiveValues: readonly string[]) { + // An unpaired surrogate has no UTF-8 representation. Buffer.from would + // invent replacement-character bytes and incorrectly discard real evidence. + this.#sensitiveByteValues = sensitiveValues.flatMap((value) => { + const encoded = Buffer.from(value, "utf8"); + return encoded.toString("utf8") === value ? [encoded] : []; + }); + } + + /** Preserve exact text unless explicitly sensitive or known transport URL userinfo. */ + text(value: string, pointer: string, transportUrl = false): string | null { + let result = value; + if (transportUrl) { + const match = + /^([\x00-\x20]*(?:[a-z][a-z0-9+.-]*:)?\/\/)([^/?#]*@)/i.exec(result); + if (match !== null) { + result = `${match[1]}${result.slice(match[0].length)}`; + this.redactions.push({ pointer, reason: "transport-credential" }); + } + } + if ( + this.sensitiveValues.some( + (literal) => value.includes(literal) || result.includes(literal), + ) + ) { + this.redactions.push({ pointer, reason: "explicit-sensitive-value" }); + return null; + } + return result; + } + + /** Remove a credential field's value without retaining its digest or encoded form. */ + credential(pointer: string): null { + this.redactions.push({ pointer, reason: "transport-credential" }); + return null; + } + + /** Known HAR credential surfaces; unknown extensions are ordinary caller-selected evidence. */ + harCredential( + pointer: string, + parent: Readonly>, + ): boolean { + return ( + (/^\/(request|response)\/headers\/\d+\/value$/.test(pointer) && + typeof parent.name === "string" && + CREDENTIAL_HEADERS.has(parent.name.toLowerCase())) || + /^\/(request|response)\/cookies\/\d+\/value$/.test(pointer) + ); + } + + /** Only actual request/redirect fields and known transport header URLs declare URL semantics. */ + harTransportUrl( + pointer: string, + parent?: Readonly>, + ): boolean { + return ( + pointer === "/request/url" || + pointer === "/response/redirectURL" || + (/^\/(request|response)\/headers\/\d+\/value$/.test(pointer) && + typeof parent?.name === "string" && + ["location", "referer", "origin"].includes(parent.name.toLowerCase())) + ); + } + + /** Preserve bytes unless an explicitly declared UTF-8 literal occurs in them. */ + sensitiveBytes(bytes: Buffer): boolean { + return this.#sensitiveByteValues.some((value) => bytes.includes(value)); + } +} + +/** Escape a producer property for an RFC 6901 location. */ +export const capturePointer = (parent: string, key: string): string => + `${parent}/${key.replaceAll("~", "~0").replaceAll("/", "~1")}`; + +/** Structural record guard does not duck-type number/byte sidecar markers. */ +export const captureObject = ( + value: unknown, +): value is Record => + typeof value === "object" && value !== null && !Array.isArray(value); diff --git a/src/browser/history/CaptureRelease.ts b/src/browser/history/CaptureRelease.ts new file mode 100644 index 000000000..4463befe9 --- /dev/null +++ b/src/browser/history/CaptureRelease.ts @@ -0,0 +1,13 @@ +/** Exact upstream package profile for unchanged offline HAR validation and numeric parsing. */ +export const HAR_CAPTURE_PROVIDER_IDENTITY = { + id: "har-schema/lossless-json", + name: "REA HAR capture adapter", + version: "har-schema@2.0.0;jsonc-parser@3.3.1;lossless-json@4.3.1", +} as const; + +/** Native executable profile verified with unchanged mitmproxy raw-state decoding. */ +export const MITMPROXY_CAPTURE_PROVIDER_IDENTITY = { + id: "mitmproxy-native-tnetstring", + name: "REA offline native mitmproxy adapter", + version: "12.2.3", +} as const; diff --git a/src/browser/history/HarCapture.test.ts b/src/browser/history/HarCapture.test.ts new file mode 100644 index 000000000..7e525ad5a --- /dev/null +++ b/src/browser/history/HarCapture.test.ts @@ -0,0 +1,546 @@ +import { createHash } from "node:crypto"; +import { expect, it } from "vitest"; +import { historicalHar } from "../../../tests/fixtures/historicalHar.js"; +import { decodeHarCapture } from "./HarCapture.js"; + +it.each([ + { values: [] }, + { values: ["response"] }, + { values: ["content"] }, + { values: ["log"] }, + { values: ["entries"] }, +])( + "validates canonical base64 even when its parent property is sensitive: %j", + ({ values }) => { + const fixture = historicalHar(); + const first = fixture.log.entries[0]; + if (first === undefined) throw new Error("fixture missing"); + const input = { + log: { + ...fixture.log, + entries: [ + { + ...first, + response: { + ...first.response, + content: { + size: 1, + mimeType: "application/octet-stream", + encoding: "base64", + text: "AR==", + }, + }, + }, + ], + }, + }; + expect(() => decodeHarCapture(JSON.stringify(input), values)).toThrow( + "not valid canonical base64", + ); + }, +); + +it.each(["log", "entries", "ntries"])( + "excludes extracted record payloads beneath the marked structural property %s", + (literal) => { + const decoded = decodeHarCapture(JSON.stringify(historicalHar()), [ + literal, + ]); + expect(decoded.total_records).toBeGreaterThan(0); + expect(decoded.container.records_pointer).toBe(null); + for (const [ordinal, record] of decoded.records.entries()) { + expect(record).toMatchObject({ + ordinal, + location: { kind: "unknown", reason: "explicit-sensitive-value" }, + reported: null, + binary_fields: [], + numeric_literals: [], + redactions: [{ pointer: "", reason: "explicit-sensitive-value" }], + }); + } + }, +); + +it("excludes sensitive property identities before emitting keys and sidecar coordinates", () => { + const fixture = historicalHar(); + const first = fixture.log.entries[0]; + if (first === undefined) throw new Error("fixture missing"); + const result = decodeHarCapture( + JSON.stringify({ + ...fixture, + _extension: { + "api-secret/~": { numeric: 123, text: "api-secret" }, + kept: 7, + }, + log: { ...fixture.log, entries: [{ ...first, "api-secret": 42 }] }, + }), + ["api-secret"], + ); + expect(JSON.stringify(result)).not.toContain("api-secret"); + expect(result.container.reported).toMatchObject({ _extension: { kept: 7 } }); + expect(result.container.redactions).toContainEqual({ + pointer: "/_extension", + reason: "explicit-sensitive-value", + scope: "property-name", + }); + expect(result.records[0]?.redactions).toContainEqual({ + pointer: "", + reason: "explicit-sensitive-value", + scope: "property-name", + }); +}); + +it.each([ + { value: "REDACTED", sensitive: ["REDACTED"] }, + { value: "literal[", sensitive: ["["] }, + { value: "before secret after", sensitive: ["secret", "REDACTED"] }, + { value: "before secret after", sensitive: ["secret", "[RED"] }, +])( + "excludes sensitive text without introducing replacement literals: $value", + ({ value, sensitive }) => { + const fixture = historicalHar(); + const result = decodeHarCapture( + JSON.stringify({ ...fixture, _sensitive: value, _ordinary: "unmarked" }), + sensitive, + ); + expect(result.container.reported).toMatchObject({ + _sensitive: null, + _ordinary: "unmarked", + }); + expect(result.container.redactions).toContainEqual({ + pointer: "/_sensitive", + reason: "explicit-sensitive-value", + }); + }, +); + +it("excludes a declared literal introduced by stripping transport URL userinfo", () => { + const fixture = historicalHar(); + const entry = fixture.log.entries[0]; + if (entry === undefined) throw new Error("fixture missing"); + const url = "https://example.test/sensitive"; + const input = { + log: { + ...fixture.log, + entries: [ + { + ...entry, + request: { + ...entry.request, + url: "https://user:password@example.test/sensitive", + }, + }, + ], + }, + }; + const result = decodeHarCapture(JSON.stringify(input), [url]); + expect(result.records[0]?.reported).toMatchObject({ request: { url: null } }); +}); + +it.each([ + '"_duplicate":1,"_duplicate":1', + '"_duplicate":null,"_duplicate":null', + '"_duplicate":{"value":1},"_duplicate":{"value":1}', + '"_duplicate":1,"\\u005fduplicate":1', +])( + "rejects equal-valued duplicate JSON members before projection: %s", + (members) => { + const text = JSON.stringify(historicalHar()).replace( + '"cache":{}', + `"cache":{},${members}`, + ); + expect(() => decodeHarCapture(text, [])).toThrow("duplicate"); + }, +); + +it("rejects identical duplicate core fields before schema validation", () => { + const text = JSON.stringify(historicalHar()).replace( + '"version":"1.2"', + '"version":"1.2","version":"1.2"', + ); + expect(() => decodeHarCapture(text, [])).toThrow("duplicate"); +}); + +it("reports an unsupported JSON schema boundary instead of dropping a producer property", () => { + const text = JSON.stringify(historicalHar()).replace( + '"cache":{}', + '"cache":{},"_extension":{"__proto__":{"preserved":7},"constructor":"ordinary"}', + ); + expect(() => decodeHarCapture(text, [])).toThrow( + "JSON schema boundary cannot preserve", + ); +}); + +it("preserves Unicode, offsets, sizes, duplicate URLs and opaque extensions without inventing bytes", () => { + const fixture = historicalHar(); + const first = fixture.log.entries[0]; + if (first === undefined) throw new Error("fixture missing"); + fixture.log.entries.push(structuredClone(first)); + const result = decodeHarCapture( + JSON.stringify({ + ...fixture, + _extension: { isLosslessNumber: true, value: "ordinary" }, + }), + [], + ); + expect(result.total_records).toBe(2); + expect(result.records[0]?.reported).toEqual(fixture.log.entries[0]); + expect(result.records[0]?.binary_fields).toEqual([]); + expect(result.records[0]?.numeric_literals).toContainEqual({ + pointer: "/response/bodySize", + producer_type: "json-number", + literal: "99", + }); + expect(result.records[1]?.location).toEqual({ + kind: "json-pointer", + pointer: "/log/entries/1", + }); + expect(result.container.reported).toMatchObject({ + _extension: { isLosslessNumber: true, value: "ordinary" }, + log: { entries: null }, + }); +}); + +it.each(["9007199254740993", "4740993", "99"])( + "excludes explicitly sensitive numeric lexemes and their reported values: %s", + (literal) => { + const text = JSON.stringify(historicalHar()).replace( + '"cache":{}', + '"cache":{},"_big":9007199254740993', + ); + const result = decodeHarCapture(text, [literal]); + for (const record of result.records) { + expect( + record.numeric_literals.every( + (number) => !number.literal.includes(literal), + ), + ).toBe(true); + } + const pointer = literal === "99" ? "/response/bodySize" : "/_big"; + expect(result.records[0]?.redactions).toContainEqual({ + pointer, + reason: "explicit-sensitive-value", + }); + if (literal === "99") + expect(result.records[0]?.reported).toMatchObject({ + response: { bodySize: null }, + }); + else expect(result.records[0]?.reported).toMatchObject({ _big: null }); + }, +); + +it("keeps unsafe extension numeric lexemes and rejects unsafe mandatory schema numbers", () => { + const text = JSON.stringify(historicalHar()); + const result = decodeHarCapture( + text.replace('"cache":{}', '"cache":{},"_big":9007199254740993'), + [], + ); + expect(result.records[0]?.numeric_literals).toContainEqual({ + pointer: "/_big", + producer_type: "json-number", + literal: "9007199254740993", + }); + expect(result.records[0]?.reported).toMatchObject({ _big: null }); + expect(() => + decodeHarCapture( + text.replace('"bodySize":0', '"bodySize":9007199254740993'), + [], + ), + ).toThrow("HAR schema validation failed"); +}); + +it.each(["container", "record"] as const)( + "retains %s extension content without interpreting it as an HTTP body", + (location) => { + const fixture = historicalHar(); + const extension = { + response: { content: { encoding: "base64", text: "opaque extension" } }, + }; + const input = + location === "container" + ? { ...fixture, _extension: extension } + : { + log: { + ...fixture.log, + entries: fixture.log.entries.map((entry) => ({ + ...entry, + _extension: extension, + })), + }, + }; + const result = decodeHarCapture(JSON.stringify(input), []); + expect( + location === "container" + ? result.container.reported + : result.records[0]?.reported, + ).toMatchObject({ _extension: extension }); + expect(result.records[0]?.binary_fields).toEqual([]); + }, +); + +it("retains valid base64 bytes independently of declared sizes", () => { + const fixture = historicalHar(); + const entry = fixture.log.entries[0]; + if (entry === undefined) throw new Error("fixture missing"); + const text = JSON.stringify({ + log: { + ...fixture.log, + entries: [ + { + ...entry, + response: { + ...entry.response, + content: { + size: 999, + mimeType: "application/octet-stream", + encoding: "base64", + text: "AP9B\n", + }, + }, + }, + ], + }, + }); + const result = decodeHarCapture(text, []); + expect(result.records[0]?.binary_fields[0]).toMatchObject({ + pointer: "/response/content/text", + state: "retained", + content_base64: "AP9B", + bytes: 3, + }); + expect(result.records[0]?.reported).toMatchObject({ + response: { content: { size: 999, text: "AP9B\n" } }, + }); + expect(() => decodeHarCapture(text.replace("AP9B", "AR=="), [])).toThrow( + "not valid canonical base64", + ); +}); + +it("redacts actual authentication fields and explicit binary values while preserving similarly named extensions", () => { + const fixture = historicalHar(); + const entry = fixture.log.entries[0]; + if (entry === undefined) throw new Error("fixture missing"); + const raw = { + log: { + ...fixture.log, + entries: [ + { + ...entry, + request: { + ...entry.request, + url: "https://user:password@example.test/a?token=ordinary#fragment", + headers: [ + { name: "Authorization", value: "Bearer transport-secret" }, + { name: "X-Test", value: "a" }, + { name: "X-Test", value: "b" }, + ], + cookies: [{ name: "session", value: "cookie-secret" }], + }, + response: { + ...entry.response, + content: { + size: 5, + mimeType: "application/octet-stream", + encoding: "base64", + text: Buffer.from("chosen-private").toString("base64"), + }, + }, + _extension: { + response: { + headers: [{ name: "Authorization", value: "ordinary-extension" }], + }, + }, + }, + ], + }, + }; + const result = decodeHarCapture(JSON.stringify(raw), ["chosen-private"]); + const output = JSON.stringify(result); + for (const secret of [ + "password", + "transport-secret", + "cookie-secret", + Buffer.from("chosen-private").toString("base64"), + ]) + expect(output).not.toContain(secret); + expect(output).toContain("token=ordinary#fragment"); + expect(output).toContain("ordinary-extension"); + expect(result.records[0]?.reported).toMatchObject({ + request: { + headers: [ + { name: "Authorization", value: null }, + { name: "X-Test", value: "a" }, + { name: "X-Test", value: "b" }, + ], + }, + }); + expect(result.records[0]?.binary_fields[0]).toMatchObject({ + state: "redacted", + sha256: null, + content_base64: null, + }); +}); + +it.each([ + "{", + '{"log":{},"log":{}}', + JSON.stringify({ log: { version: "1.2", entries: [] } }), +])("rejects malformed producer data %s", (text) => { + expect(() => decodeHarCapture(text, [])).toThrow(); +}); + +it("preserves observed SaveHar null post-data without broadening arbitrary HAR producers", () => { + const fixture = historicalHar(); + const first = fixture.log.entries[0]; + if (first === undefined) throw new Error("fixture missing"); + const log = { + ...fixture.log, + creator: { name: "mitmproxy", version: "12.2.3" }, + entries: [ + { + ...first, + request: { + ...first.request, + postData: { mimeType: "text/plain", text: null }, + }, + }, + ], + }; + expect( + decodeHarCapture(JSON.stringify({ log }), []).records[0]?.reported, + ).toMatchObject({ request: { postData: { text: null } } }); + expect(() => + decodeHarCapture( + JSON.stringify({ + log: { ...log, creator: { name: "other", version: "1" } }, + }), + [], + ), + ).toThrow("HAR schema validation failed"); +}); + +it.each([ + "c2VjcmV0", + "c2Vj", + createHash("sha256").update("secret").digest("hex"), +])( + "excludes sensitive derived byte representations from spaced HAR text: %s", + (literal) => { + const fixture = historicalHar(); + const first = fixture.log.entries[0]; + if (first === undefined) throw new Error("fixture missing"); + const result = decodeHarCapture( + JSON.stringify({ + log: { + ...fixture.log, + entries: [ + { + ...first, + response: { + ...first.response, + content: { + size: 6, + mimeType: "application/octet-stream", + encoding: "base64", + text: "c2 VjcmV0", + }, + }, + }, + ], + }, + }), + [literal], + ); + expect(result.records[0]?.reported).toMatchObject({ + response: { content: { text: null } }, + }); + expect(result.records[0]?.binary_fields).toContainEqual({ + pointer: "/response/content/text", + representation: "har-base64-content", + state: "redacted", + content_base64: null, + bytes: null, + sha256: null, + }); + }, +); + +it("excludes a declared literal in encoded HAR text from its binary sidecar too", () => { + const fixture = historicalHar(); + const first = fixture.log.entries[0]; + if (first === undefined) throw new Error("fixture missing"); + const log = { + ...fixture.log, + entries: [ + { + ...first, + response: { + ...first.response, + content: { + size: 3, + mimeType: "application/octet-stream", + encoding: "base64", + text: "AP9B", + }, + }, + }, + ], + }; + const result = decodeHarCapture(JSON.stringify({ log }), ["AP9B"]); + expect(JSON.stringify(result)).not.toContain("AP9B"); + expect(result.records[0]?.binary_fields[0]?.state).toBe("redacted"); +}); + +it("rejects excessive extension nesting instead of returning a partial capture", () => { + let extension: unknown = null; + for (let index = 0; index < 66; index++) extension = { child: extension }; + expect(() => + decodeHarCapture( + JSON.stringify({ ...historicalHar(), _extension: extension }), + [], + ), + ).toThrow("nesting budget"); +}); + +it.each([ + { name: "Location", side: "response" as const }, + { name: "Referer", side: "request" as const }, + { name: "Origin", side: "request" as const }, +])( + "excludes leading-OWS userinfo from a producer $name header", + ({ name, side }) => { + const fixture = historicalHar(); + const first = fixture.log.entries[0]; + if (first === undefined) throw new Error("fixture missing"); + const value = + " \tHTTPS://user:password@example.test/path?token=ordinary#fragment\t "; + const result = decodeHarCapture( + JSON.stringify({ + log: { + ...fixture.log, + entries: [ + { + ...first, + [side]: { ...first[side], headers: [{ name, value }] }, + }, + ], + }, + }), + [], + ); + expect(result.records[0]?.reported).toMatchObject({ + [side]: { + headers: [ + { + name, + value: " \tHTTPS://example.test/path?token=ordinary#fragment\t ", + }, + ], + }, + }); + expect(result.records[0]?.redactions).toContainEqual({ + pointer: `/${side}/headers/0/value`, + reason: "transport-credential", + }); + }, +); diff --git a/src/browser/history/HarCapture.ts b/src/browser/history/HarCapture.ts new file mode 100644 index 000000000..ddc31567c --- /dev/null +++ b/src/browser/history/HarCapture.ts @@ -0,0 +1,244 @@ +import { createHash } from "node:crypto"; +import { isSafeNumber, LosslessNumber } from "lossless-json"; +import { parseHarJson } from "./HarJson.js"; +import type { JsonValue } from "../../domain/jsonValue.js"; +import { + WEB_NETWORK_CAPTURE_LIMITS, + type WebNetworkCapture, + type WebNetworkCaptureRecord, +} from "../../domain/webNetworkCapture.js"; +import { CaptureFormatError } from "./CaptureFormatError.js"; +import { + CaptureRedaction, + captureObject, + capturePointer, +} from "./CaptureRedaction.js"; +import { createHarValidator } from "./HarSchema.js"; +import { HAR_CAPTURE_PROVIDER_IDENTITY } from "./CaptureRelease.js"; + +type DecodedCapture = Omit< + WebNetworkCapture, + "artifact" | "format" | "runtime_attribution" | "limitations" +>; +type Projection = Pick< + WebNetworkCaptureRecord, + "reported" | "numeric_literals" | "binary_fields" | "redactions" +>; + +/** Decode a retained HAR with exact numeric lexemes and unchanged upstream schema validation. */ +export const decodeHarCapture = ( + text: string, + sensitiveValues: readonly string[], +): DecodedCapture => { + const raw = parseHarJson(text); + const validation = projectHar(raw, [], false); + createHarValidator()(validation.reported); + if ( + !captureObject(raw) || + !captureObject(raw.log) || + !Array.isArray(raw.log.entries) + ) + throw new CaptureFormatError("format", "HAR must contain log.entries."); + if (raw.log.version !== "1.2") + throw new CaptureFormatError( + "unsupported", + "Only the pinned HAR 1.2 schema profile is supported.", + "/log/version", + ); + const entries = raw.log.entries; + const recordsExcluded = ["log", "entries"].some((key) => + sensitiveValues.some((literal) => key.includes(literal)), + ); + const container = projectHar( + { ...raw, log: { ...raw.log, entries: null } }, + sensitiveValues, + true, + ); + const records = entries.map((entry, ordinal): WebNetworkCaptureRecord => { + // Validate each original record even when a structural ancestor excludes it. + const projected = projectHar(entry, sensitiveValues, true); + const projection: Projection = recordsExcluded + ? { + reported: null, + numeric_literals: [], + binary_fields: [], + redactions: [{ pointer: "", reason: "explicit-sensitive-value" }], + } + : projected; + return { + ordinal, + location: recordsExcluded + ? { kind: "unknown", reason: "explicit-sensitive-value" } + : { kind: "json-pointer", pointer: `/log/entries/${ordinal}` }, + ...projection, + limitations: [ + "HAR text is producer-decoded Unicode. Original body bytes remain unknown unless content explicitly uses valid base64; declared sizes are preserved independently of retained bytes. mitmproxy 12.2.3 SaveHar may report null optional postData.text; that null is retained and omitted only from the upstream validation copy.", + ...(recordsExcluded + ? [ + "This record's payload and sidecars are excluded because its structural ancestor property was explicitly marked sensitive.", + ] + : []), + ], + }; + }); + return { + decoder: { ...HAR_CAPTURE_PROVIDER_IDENTITY }, + container: { + reported: container.reported, + numeric_literals: container.numeric_literals, + redactions: container.redactions, + records_pointer: recordsExcluded ? null : "/log/entries", + }, + total_records: records.length, + records, + }; +}; + +const projectHar = ( + value: unknown, + sensitiveValues: readonly string[], + redact: boolean, +): Projection => { + const redactor = new CaptureRedaction(sensitiveValues); + const numeric_literals: Projection["numeric_literals"] = []; + const binary_fields: Projection["binary_fields"] = []; + const visit = ( + item: unknown, + pointer: string, + depth: number, + parent?: Readonly>, + ): JsonValue => { + if (depth > WEB_NETWORK_CAPTURE_LIMITS.depth) + throw new CaptureFormatError( + "input-limit", + "HAR exceeds the 64-level complete-evidence nesting budget.", + pointer, + ); + if ( + redact && + parent !== undefined && + redactor.harCredential(pointer, parent) + ) + return redactor.credential(pointer); + if (item instanceof LosslessNumber) { + if (redact && redactor.text(item.value, pointer) === null) return null; + numeric_literals.push({ + pointer, + producer_type: "json-number", + literal: item.value, + }); + return isSafeNumber(item.value) ? Number(item.value) : null; + } + if (typeof item === "string") { + if ( + redact && + pointer === "/response/content/text" && + parent?.encoding === "base64" + ) { + const bytes = decodeHarBase64(item, pointer); + const contentBase64 = bytes.toString("base64"); + const sha256 = createHash("sha256").update(bytes).digest("hex"); + const hidden = + redactor.sensitiveBytes(bytes) || + sensitiveValues.some( + (literal) => + item.includes(literal) || + contentBase64.includes(literal) || + sha256.includes(literal), + ); + binary_fields.push( + hidden + ? { + pointer, + representation: "har-base64-content", + state: "redacted", + content_base64: null, + bytes: null, + sha256: null, + } + : { + pointer, + representation: "har-base64-content", + state: "retained", + content_base64: contentBase64, + bytes: bytes.length, + sha256, + }, + ); + if (hidden) { + redactor.redactions.push({ + pointer, + reason: "explicit-sensitive-value", + }); + return null; + } + } + return redact + ? redactor.text( + item, + pointer, + redactor.harTransportUrl(pointer, parent), + ) + : item; + } + if (item === null || typeof item === "boolean") return item; + if (Array.isArray(item)) + return item.map((child, index) => + visit(child, capturePointer(pointer, String(index)), depth + 1), + ); + if (captureObject(item)) + return Object.fromEntries( + Object.entries(item).flatMap(([key, child]) => { + if ( + redact && + sensitiveValues.some((literal) => key.includes(literal)) + ) { + const beforeNumbers = numeric_literals.length; + const beforeBinaries = binary_fields.length; + const beforeRedactions = redactor.redactions.length; + visit(child, capturePointer(pointer, key), depth + 1, item); + numeric_literals.splice(beforeNumbers); + binary_fields.splice(beforeBinaries); + redactor.redactions.splice(beforeRedactions); + redactor.redactions.push({ + pointer, + reason: "explicit-sensitive-value", + scope: "property-name", + }); + return []; + } + return [ + [key, visit(child, capturePointer(pointer, key), depth + 1, item)], + ]; + }), + ); + throw new CaptureFormatError( + "format", + "HAR parser returned a non-JSON value.", + pointer, + ); + }; + return { + reported: visit(value, "", 0), + numeric_literals, + binary_fields, + redactions: redactor.redactions, + }; +}; + +const decodeHarBase64 = (value: string, pointer: string): Buffer => { + const compact = value.replace(/[\t\n\r ]/g, ""); + const bytes = Buffer.from(compact, "base64"); + if ( + !/^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/.test( + compact, + ) || + bytes.toString("base64") !== compact + ) + throw new CaptureFormatError( + "format", + "HAR content declares base64 but its text is not valid canonical base64 (ASCII whitespace is allowed).", + pointer, + ); + return bytes; +}; diff --git a/src/browser/history/HarCaptureAdapter.ts b/src/browser/history/HarCaptureAdapter.ts new file mode 100644 index 000000000..b66167a5f --- /dev/null +++ b/src/browser/history/HarCaptureAdapter.ts @@ -0,0 +1,24 @@ +import { fileURLToPath } from "node:url"; +import type { HistoricalCaptureFormatAdapter } from "./HistoricalCaptureFormatAdapter.js"; +import { HAR_CAPTURE_PROVIDER_IDENTITY } from "./CaptureRelease.js"; + +/** HAR's parser/schema packages run in one bounded Node heap, without network or target execution. */ +export class HarCaptureAdapter implements HistoricalCaptureFormatAdapter { + readonly format = "har"; + readonly identity = HAR_CAPTURE_PROVIDER_IDENTITY; + async command( + requestPath: string, + ): Promise<{ command: string; arguments: readonly string[] }> { + return { + command: process.execPath, + arguments: [ + "--max-old-space-size=192", + "--max-semi-space-size=8", + "--v8-pool-size=1", + "--single-threaded", + fileURLToPath(new URL("./HarCaptureProcess.js", import.meta.url)), + requestPath, + ], + }; + } +} diff --git a/src/browser/history/HarCaptureProcess.ts b/src/browser/history/HarCaptureProcess.ts new file mode 100644 index 000000000..0751d0b80 --- /dev/null +++ b/src/browser/history/HarCaptureProcess.ts @@ -0,0 +1,64 @@ +import { readFile, writeFile } from "node:fs/promises"; +import { TextDecoder } from "node:util"; +import { z } from "zod"; +import { readStableArtifact } from "../../artifacts/readStableArtifact.js"; +import { WEB_NETWORK_CAPTURE_LIMITS } from "../../domain/webNetworkCapture.js"; +import { CaptureFormatError } from "./CaptureFormatError.js"; +import { decodeHarCapture } from "./HarCapture.js"; + +const requestSchema = z.strictObject({ + snapshot_path: z.string(), + reply_path: z.string(), + sensitive_values: z.array(z.string().min(1)), +}); + +/** Decode untrusted HAR in an owned, bounded heap rather than the long-lived MCP process. */ +const main = async (): Promise => { + const request = requestSchema.parse( + JSON.parse(await readFile(process.argv[2] ?? "", "utf8")), + ); + let reply: unknown; + try { + const snapshot = await readStableArtifact( + request.snapshot_path, + WEB_NETWORK_CAPTURE_LIMITS.inputBytes, + ); + let text: string; + try { + text = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode( + snapshot.bytes, + ); + } catch { + throw new CaptureFormatError("format", "HAR is not valid UTF-8 JSON."); + } + reply = { + ok: true, + value: decodeHarCapture(text, request.sensitive_values), + }; + } catch (cause: unknown) { + reply = + cause instanceof CaptureFormatError + ? { + ok: false, + reason: cause.reason, + message: cause.message, + pointer: cause.pointer, + } + : { + ok: false, + reason: "decoder", + message: `HAR decoder failed unexpectedly (${cause instanceof Error ? cause.name : "unknown exception"}).`, + pointer: "", + }; + } + let text = JSON.stringify(reply); + if (Buffer.byteLength(text) > WEB_NETWORK_CAPTURE_LIMITS.outputBytes) + text = JSON.stringify({ + ok: false, + reason: "limit", + message: "HAR reply exceeds the complete-evidence output budget.", + pointer: "", + }); + await writeFile(request.reply_path, text, { flag: "wx", mode: 0o600 }); +}; +await main(); diff --git a/src/browser/history/HarJson.ts b/src/browser/history/HarJson.ts new file mode 100644 index 000000000..93f25e7cd --- /dev/null +++ b/src/browser/history/HarJson.ts @@ -0,0 +1,118 @@ +import { visit } from "jsonc-parser"; +import { LosslessNumber } from "lossless-json"; +import { WEB_NETWORK_CAPTURE_LIMITS } from "../../domain/webNetworkCapture.js"; +import { CaptureFormatError } from "./CaptureFormatError.js"; + +type Frame = + | { + kind: "object"; + pointer: string; + members: Map; + pendingKey?: string; + } + | { kind: "array"; pointer: string; items: unknown[] }; + +/** Materialize upstream JSON events without duplicate folding, prototype setters or numeric rounding. */ +export const parseHarJson = (text: string): unknown => { + const frames: Frame[] = []; + let result: unknown; + const valuePointer = (): string => { + const parent = frames.at(-1); + if (parent === undefined) return ""; + if (parent.kind === "array") + return `${parent.pointer}/${parent.items.length}`; + return parent.pendingKey === undefined + ? parent.pointer + : `${parent.pointer}/${parent.pendingKey.replaceAll("~", "~0").replaceAll("/", "~1")}`; + }; + const checkDepth = (): void => { + if (frames.length > WEB_NETWORK_CAPTURE_LIMITS.depth) + throw new CaptureFormatError( + "input-limit", + "HAR exceeds the 64-level complete-evidence nesting budget.", + valuePointer(), + ); + }; + const append = (value: unknown): void => { + const parent = frames.at(-1); + if (parent === undefined) result = value; + else if (parent.kind === "array") parent.items.push(value); + else { + if (parent.pendingKey === undefined) + throw new TypeError( + "JSON visitor omitted an object property identity.", + ); + parent.members.set(parent.pendingKey, value); + delete parent.pendingKey; + } + }; + visit( + text, + { + onObjectBegin: () => { + checkDepth(); + frames.push({ + kind: "object", + pointer: valuePointer(), + members: new Map(), + }); + }, + onObjectProperty: (key) => { + if (key === "__proto__") + throw new CaptureFormatError( + "unsupported", + "HAR object contains a __proto__ member that the current JSON schema boundary cannot preserve.", + ); + const frame = frames.at(-1); + if (frame?.kind !== "object") + throw new TypeError( + "JSON visitor emitted a property outside an object.", + ); + if (frame.members.has(key)) + throw new CaptureFormatError( + "format", + "HAR JSON contains duplicate object members.", + ); + frame.pendingKey = key; + }, + onObjectEnd: () => { + const frame = frames.pop(); + if (frame?.kind !== "object") + throw new TypeError("JSON visitor ended an unexpected object."); + append(Object.fromEntries(frame.members)); + }, + onArrayBegin: () => { + checkDepth(); + frames.push({ kind: "array", pointer: valuePointer(), items: [] }); + }, + onArrayEnd: () => { + const frame = frames.pop(); + if (frame?.kind !== "array") + throw new TypeError("JSON visitor ended an unexpected array."); + append(frame.items); + }, + onLiteralValue: (value: unknown, offset, length) => { + checkDepth(); + if (typeof value === "number") + append(new LosslessNumber(text.slice(offset, offset + length))); + else if ( + value === null || + typeof value === "boolean" || + typeof value === "string" + ) + append(value); + else + throw new TypeError("JSON visitor emitted an unsupported literal."); + }, + onError: () => { + throw new CaptureFormatError("format", "HAR is malformed JSON."); + }, + }, + { + disallowComments: true, + allowTrailingComma: false, + allowEmptyContent: false, + }, + ); + return result; +}; diff --git a/src/browser/history/HarSchema.ts b/src/browser/history/HarSchema.ts new file mode 100644 index 000000000..069377f8a --- /dev/null +++ b/src/browser/history/HarSchema.ts @@ -0,0 +1,87 @@ +import { createRequire } from "node:module"; +import { Ajv } from "ajv"; +import { z } from "zod"; +import { CaptureFormatError } from "./CaptureFormatError.js"; + +/** Validate using the unchanged, pinned upstream HAR 1.2 draft-06 schemas. */ +export const createHarValidator = (): ((value: unknown) => void) => { + const require = createRequire(import.meta.url); + const schemas = z + .record(z.string(), z.record(z.string(), z.unknown())) + .parse(require("har-schema")); + const ajv = new Ajv({ strict: false, allErrors: false, ownProperties: true }); + ajv.addMetaSchema(require("ajv/dist/refs/json-schema-draft-06.json")); + const plugin: unknown = require("ajv-formats"); + if (typeof plugin !== "function") + throw new Error("Pinned AJV formats plugin is unavailable"); + plugin(ajv); + for (const schema of Object.values(schemas)) ajv.addSchema(schema); + const validate = ajv.getSchema("har.json#"); + if (validate === undefined) + throw new Error("Pinned HAR root schema is unavailable"); + return (value) => { + allowMitmproxyAbsentPostData(value); + if (!validate(value)) { + const error = validate.errors?.[0]; + throw new CaptureFormatError( + "format", + `HAR schema validation failed: ${error?.keyword ?? "unknown constraint"}.`, + error?.instancePath ?? "", + ); + } + }; +}; + +/** Observed SaveHar 12.2.3 compatibility: validate omitted optional text while retaining reported null. */ +const allowMitmproxyAbsentPostData = (value: unknown): void => { + const parsed = z + .object({ + log: z.object({ + creator: z.object({ + name: z.literal("mitmproxy"), + version: z.literal("12.2.3"), + }), + entries: z.array(z.unknown()), + }), + }) + .safeParse(value); + if ( + !parsed.success || + typeof value !== "object" || + value === null || + !("log" in value) + ) + return; + const log = value.log; + if ( + typeof log !== "object" || + log === null || + !("entries" in log) || + !Array.isArray(log.entries) + ) + return; + for (const entry of log.entries) { + const candidate: unknown = entry; + if ( + typeof candidate !== "object" || + candidate === null || + !("request" in candidate) + ) + continue; + const request = candidate.request; + if ( + typeof request !== "object" || + request === null || + !("postData" in request) + ) + continue; + const body = request.postData; + if ( + typeof body === "object" && + body !== null && + "text" in body && + body.text === null + ) + delete body.text; + } +}; diff --git a/src/browser/history/HistoricalCaptureDecoder.ts b/src/browser/history/HistoricalCaptureDecoder.ts new file mode 100644 index 000000000..e180ad6ad --- /dev/null +++ b/src/browser/history/HistoricalCaptureDecoder.ts @@ -0,0 +1,250 @@ +import { historicalCaptureFailure } from "./CaptureFailures.js"; +import { projectAnalysisError } from "../../domain/analysisErrorProjection.js"; +import type { HistoricalCaptureFormatAdapter } from "./HistoricalCaptureFormatAdapter.js"; +import { randomUUID } from "node:crypto"; +import { writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { z } from "zod"; +import type { ExecutionOptions } from "../../application/AnalysisProvider.js"; +import { readStableArtifact } from "../../artifacts/readStableArtifact.js"; +import { ArtifactReaderFailure } from "../../artifacts/ArtifactReader.js"; +import type { AnalysisError } from "../../domain/analysisErrorBase.js"; +import { + AnalysisCapabilityUnavailableError, + AnalysisCancelledError, + AnalysisInputError, + AnalysisOutputError, +} from "../../domain/analysisErrorCore.js"; +import { ProviderAdapterError } from "../../domain/providerAdapterError.js"; +import { ProviderCleanupError } from "../../domain/providerCleanupError.js"; +import { err, ok, type Result } from "../../domain/result.js"; +import { + WEB_NETWORK_CAPTURE_LIMITS, + webNetworkCaptureSchema, + type InspectWebNetworkCaptureInput, + type WebNetworkCapture, +} from "../../domain/webNetworkCapture.js"; +import { runOwnedCommand } from "../../process/OwnedCommand.js"; +import { PrivateRuntimeRoot } from "../../process/PrivateRuntimeRoot.js"; +import { redactExplicitFailure } from "../../domain/explicitSensitiveFailure.js"; + +const OPERATION = "inspect_web_network_capture"; +const decodedSchema = webNetworkCaptureSchema.omit({ + artifact: true, + format: true, + runtime_attribution: true, + limitations: true, +}); +const replySchema = z.discriminatedUnion("ok", [ + z.strictObject({ ok: z.literal(true), value: decodedSchema }), + z.strictObject({ + ok: z.literal(false), + reason: z.enum([ + "format", + "unsupported", + "input-limit", + "resource-limit", + "decoder", + "limit", + ]), + message: z.string(), + pointer: z.string(), + }), +]); + +/** One owned snapshot/process/root lifecycle for replaceable historical format decoders. */ +export class HistoricalCaptureDecoder { + constructor( + readonly adapters: readonly HistoricalCaptureFormatAdapter[], + readonly environment: Readonly, + readonly createRuntime: () => Promise< + Pick + > = () => PrivateRuntimeRoot.create({ prefix: "rea-web-capture-" }), + ) {} + + /** Snapshot the selected artifact, decode it offline, and release private credentials and processes. */ + async inspect( + input: InspectWebNetworkCaptureInput, + options?: ExecutionOptions, + ): Promise> { + let runtime: Pick | undefined; + let result: Result; + let phase: "capture-read" | "decoder" = "capture-read"; + try { + const adapter = this.adapters.find( + (candidate) => candidate.format === input.format, + ); + if (adapter === undefined) + throw new AnalysisCapabilityUnavailableError( + input.format, + OPERATION, + "No adapter is configured for the selected capture format.", + ); + const snapshot = await readStableArtifact( + input.capture_path, + WEB_NETWORK_CAPTURE_LIMITS.inputBytes, + options?.signal, + ); + phase = "decoder"; + runtime = await this.createRuntime(); + const snapshotPath = join(runtime.path, "capture.snapshot"); + const requestPath = join(runtime.path, "request.json"); + const replyPath = join(runtime.path, "reply.json"); + await writeFile(snapshotPath, snapshot.bytes, { + mode: 0o600, + flag: "wx", + ...(options?.signal === undefined ? {} : { signal: options.signal }), + }); + await writeFile( + requestPath, + JSON.stringify({ + snapshot_path: snapshotPath, + reply_path: replyPath, + sensitive_values: input.sensitive_values, + }), + { mode: 0o600, flag: "wx" }, + ); + const command = await adapter.command(requestPath, runtime.path); + const environment: NodeJS.ProcessEnv = { + ...this.environment, + UV_THREADPOOL_SIZE: "1", + }; + delete environment.NODE_OPTIONS; + await runOwnedCommand( + { + ...command, + runId: `rea-web-capture-${randomUUID()}`, + cwd: runtime.path, + hostEnvironment: environment, + }, + { + timeoutMs: WEB_NETWORK_CAPTURE_LIMITS.timeoutMs, + diagnosticBytes: 1024 * 1024, + }, + options?.signal === undefined ? {} : { signal: options.signal }, + ); + let reply: z.output; + try { + const replyFile = await readStableArtifact( + replyPath, + WEB_NETWORK_CAPTURE_LIMITS.outputBytes, + options?.signal, + ); + reply = replySchema.parse(JSON.parse(replyFile.bytes.toString("utf8"))); + } catch (cause: unknown) { + if (options?.signal?.aborted) + throw new AnalysisCancelledError(OPERATION); + throw new AnalysisOutputError( + OPERATION, + `Owned capture decoder reply failed for ${input.capture_path}: ${replyFailureReason(cause)}`, + { cause }, + ); + } + if (!reply.ok) { + if (reply.reason === "format") + throw new AnalysisInputError(OPERATION, undefined, [ + { + path: ["capture_path", reply.pointer], + reason: "invalid_format", + message: reply.message, + }, + ]); + if (reply.reason === "input-limit") + throw new AnalysisInputError(OPERATION, undefined, [ + { + path: ["capture_path", reply.pointer], + reason: "out_of_range", + message: reply.message, + expected: { + maximum_capture_nesting: WEB_NETWORK_CAPTURE_LIMITS.depth, + }, + }, + ]); + if (reply.reason === "decoder" || reply.reason === "resource-limit") + throw new ProviderAdapterError(input.format, OPERATION, { + diagnostics: { + phase: "decoder", + failure_kind: reply.reason, + reason: reply.message, + pointer: reply.pointer, + }, + }); + if (reply.reason === "unsupported") + throw new AnalysisCapabilityUnavailableError( + input.format, + OPERATION, + reply.message, + ); + throw new AnalysisOutputError(OPERATION, reply.message); + } + if ( + reply.value.decoder.id !== adapter.identity.id || + reply.value.decoder.name !== adapter.identity.name || + reply.value.decoder.version !== adapter.identity.version + ) + throw new AnalysisOutputError( + OPERATION, + "Capture decoder changed the selected upstream profile identity.", + ); + result = ok({ + ...reply.value, + artifact: { + path: input.capture_path, + sha256: snapshot.sha256, + bytes: snapshot.bytes.length, + }, + format: input.format, + runtime_attribution: "unknown", + limitations: [ + "This is retained producer evidence, without live browser transaction IDs, scenario provenance, execution attribution or deployment authenticity. Recorded URLs are never fetched.", + "Every record is decoded before selection; no partial success is returned on malformed input or a resource limit.", + "Known authentication headers/cookies and explicitly marked literal values are excluded. Unknown extension names do not establish sensitivity.", + "A 32 MiB input, 96 MiB reply and 64-level nesting budget bound complete evidence. HAR uses a 192 MiB Node old-generation heap; native decoding uses a 768 MiB Linux address-space limit. Each owned command has a 30-second deadline with independent cleanup.", + ], + }); + } catch (cause: unknown) { + result = err( + redactExplicitFailure( + historicalCaptureFailure(input, cause, phase, options), + input.sensitive_values, + ), + ); + } + if (runtime !== undefined) { + try { + await runtime.close(); + } catch (cause: unknown) { + return err( + redactExplicitFailure( + new ProviderCleanupError( + input.format, + [runtime.path], + { + capture_path: input.capture_path, + previous_error: result.ok + ? null + : projectAnalysisError(result.error), + reason: cause instanceof Error ? cause.message : String(cause), + }, + { operation: OPERATION }, + ), + input.sensitive_values, + ), + ); + } + } + return result.ok && options?.signal?.aborted + ? err(new AnalysisCancelledError(OPERATION)) + : result; + } +} + +const replyFailureReason = (cause: unknown): string => { + if (cause instanceof ArtifactReaderFailure) return cause.message; + if (cause instanceof z.ZodError) + return `Reply violates its schema at ${cause.issues[0]?.path.map(String).join(".") ?? "root"}.`; + if (cause instanceof SyntaxError) return "Reply is malformed JSON."; + if (cause instanceof Error && "code" in cause) + return `Reply file could not be read (${String(cause.code)}).`; + return "Reply could not be validated."; +}; diff --git a/src/browser/history/HistoricalCaptureFormatAdapter.ts b/src/browser/history/HistoricalCaptureFormatAdapter.ts new file mode 100644 index 000000000..55281b5ec --- /dev/null +++ b/src/browser/history/HistoricalCaptureFormatAdapter.ts @@ -0,0 +1,15 @@ +import type { InspectWebNetworkCaptureInput } from "../../domain/webNetworkCapture.js"; + +/** Format-specific process profile keeps mature upstream code behind a replaceable adapter seam. */ +export interface HistoricalCaptureFormatAdapter { + readonly format: InspectWebNetworkCaptureInput["format"]; + readonly identity: { + readonly id: string; + readonly name: string; + readonly version: string; + }; + command( + requestPath: string, + runtimePath: string, + ): Promise<{ command: string; arguments: readonly string[] }>; +} diff --git a/src/browser/history/MitmproxyCaptureAdapter.ts b/src/browser/history/MitmproxyCaptureAdapter.ts new file mode 100644 index 000000000..a3ebd66f7 --- /dev/null +++ b/src/browser/history/MitmproxyCaptureAdapter.ts @@ -0,0 +1,56 @@ +import { access } from "node:fs/promises"; +import { constants } from "node:fs"; +import { isAbsolute } from "node:path"; +import { fileURLToPath } from "node:url"; +import { AnalysisCapabilityUnavailableError } from "../../domain/analysisErrorCore.js"; +import type { HistoricalCaptureFormatAdapter } from "./HistoricalCaptureFormatAdapter.js"; +import { MITMPROXY_CAPTURE_PROVIDER_IDENTITY } from "./CaptureRelease.js"; +const OPERATION = "inspect_web_network_capture"; + +/** Native mitmproxy decoding is offline and uses only an explicitly configured Linux tool. */ +export class MitmproxyCaptureAdapter implements HistoricalCaptureFormatAdapter { + readonly format = "mitmproxy"; + readonly identity = MITMPROXY_CAPTURE_PROVIDER_IDENTITY; + constructor(readonly environment: Readonly) {} + async command( + requestPath: string, + runtimePath: string, + ): Promise<{ command: string; arguments: readonly string[] }> { + const command = this.environment.REA_MITMDUMP_COMMAND; + if ( + process.platform !== "linux" || + command === undefined || + !isAbsolute(command) + ) + throw new AnalysisCapabilityUnavailableError( + "mitmproxy", + OPERATION, + "Native capture decoding requires Linux and an absolute REA_MITMDUMP_COMMAND path to caller-supplied mitmdump 12.2.3; HAR needs no external engine.", + ); + try { + await access(command, constants.R_OK | constants.X_OK); + } catch (cause: unknown) { + throw new AnalysisCapabilityUnavailableError( + "mitmproxy", + OPERATION, + `Configured mitmdump executable is unavailable: ${command}.`, + { cause }, + ); + } + return { + command, + arguments: [ + "--no-server", + "-q", + "--set", + `confdir=${runtimePath}`, + "--set", + `rea_request_path=${requestPath}`, + "-s", + fileURLToPath( + new URL("../../../bridge/mitmproxy/capture.py", import.meta.url), + ), + ], + }; + } +} diff --git a/src/cli.ts b/src/cli.ts index cd9b683e5..10f1a0949 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -16,6 +16,7 @@ import { registerWebScriptCommands } from "./cli/webScriptCommands.js"; import { registerWebModuleCommands } from "./cli/webModuleCommands.js"; import { registerWebSourceLocationCommands } from "./cli/webSourceLocationCommands.js"; import { registerWebRuntimeCommands } from "./cli/webRuntimeCommands.js"; +import { registerWebNetworkCaptureCommands } from "./cli/webNetworkCaptureCommands.js"; import { registerJavaScriptRecoveryCommands } from "./cli/javascriptRecoveryCommands.js"; import { registerAdvancedBrowserCommands } from "./cli/browserAdvancedCommands.js"; import { registerBrowserScenarioCommands } from "./cliBrowserScenarioCommands.js"; @@ -65,6 +66,7 @@ export const createCli = ( registerWebModuleCommands(cli, logger, environment); registerWebSourceLocationCommands(cli, logger, environment); registerWebRuntimeCommands(cli, logger); + registerWebNetworkCaptureCommands(cli, logger, environment); registerJavaScriptRecoveryCommands(cli, logger, environment); registerAdvancedBrowserCommands(cli, logger); registerBrowserScenarioCommands(cli, logger); diff --git a/src/cli/webNetworkCaptureCommands.ts b/src/cli/webNetworkCaptureCommands.ts new file mode 100644 index 000000000..a5601b57b --- /dev/null +++ b/src/cli/webNetworkCaptureCommands.ts @@ -0,0 +1,60 @@ +import { z } from "incur"; +import { createWebNetworkCaptureService } from "../composition/webNetworkCaptures.js"; +import { CLI_COMMANDS } from "../cliCommandNames.js"; +import { logCliCommand } from "../cliLogging.js"; +import { projectAnalysisError } from "../domain/analysisErrorProjection.js"; +import type { Logger } from "../logger.js"; +import type { CliInstance } from "./types.js"; +import { withCommandCancellation } from "./commandCancellation.js"; + +/** Expose offline historical capture inspection with the same selection contract as MCP. */ +export const registerWebNetworkCaptureCommands = ( + cli: CliInstance, + logger: Logger, + environment: Readonly, +): void => { + const service = createWebNetworkCaptureService(environment); + cli.command(CLI_COMMANDS.inspectWebNetworkCapture, { + description: + "Inspect retained HAR or native mitmproxy records without fetching recorded URLs", + args: z.object({ + capturePath: z.string().describe("Absolute local capture path"), + format: z.enum(["har", "mitmproxy"]).describe("Producer capture format"), + }), + options: z.object({ + record: z + .array(z.coerce.number().int().nonnegative()) + .optional() + .describe("Zero-based producer ordinals; repeat to select several"), + sensitiveValue: z + .array(z.string().min(1)) + .default([]) + .describe( + "Literal string values explicitly marked sensitive; repeat as needed", + ), + }), + run: ({ args, options }) => + withCommandCancellation((signal) => + logCliCommand( + logger, + CLI_COMMANDS.inspectWebNetworkCapture, + async () => { + const result = await service.inspect( + { + capture_path: args.capturePath, + format: args.format, + sensitive_values: options.sensitiveValue, + ...(options.record === undefined + ? {} + : { record_ordinals: options.record }), + }, + { signal }, + ); + return result.ok + ? result.value + : projectAnalysisError(result.error); + }, + ), + ), + }); +}; diff --git a/src/cliCommandNames.ts b/src/cliCommandNames.ts index 9cc5cac8f..76ba42882 100644 --- a/src/cliCommandNames.ts +++ b/src/cliCommandNames.ts @@ -1,5 +1,6 @@ /** Canonical names used by every one-shot CLI command registration. */ export const CLI_COMMANDS = Object.freeze({ + inspectWebNetworkCapture: "inspect-web-network-capture", observeWebExecution: "observe-web-execution", inspectWebEventListeners: "inspect-web-event-listeners", traceWebSourceLocation: "trace-web-source-location", diff --git a/src/composition/webNetworkCaptures.ts b/src/composition/webNetworkCaptures.ts new file mode 100644 index 000000000..6dc7668d4 --- /dev/null +++ b/src/composition/webNetworkCaptures.ts @@ -0,0 +1,15 @@ +import { WebNetworkCaptureService } from "../application/WebNetworkCaptureService.js"; +import { HistoricalCaptureDecoder } from "../browser/history/HistoricalCaptureDecoder.js"; +import { HarCaptureAdapter } from "../browser/history/HarCaptureAdapter.js"; +import { MitmproxyCaptureAdapter } from "../browser/history/MitmproxyCaptureAdapter.js"; + +/** Compose offline format adapters without import-time reads, installation or process startup. */ +export const createWebNetworkCaptureService = ( + environment: Readonly = process.env, +): WebNetworkCaptureService => + new WebNetworkCaptureService( + new HistoricalCaptureDecoder( + [new HarCaptureAdapter(), new MitmproxyCaptureAdapter(environment)], + environment, + ), + ); diff --git a/src/contracts/contractSnapshot.test.ts b/src/contracts/contractSnapshot.test.ts index 1de10c515..f923cc3e9 100644 --- a/src/contracts/contractSnapshot.test.ts +++ b/src/contracts/contractSnapshot.test.ts @@ -16,6 +16,7 @@ import { ELECTRON_TOOL_CONTRACTS } from "./javascript/electronToolContracts.js"; import { JAVASCRIPT_RUNTIME_OBSERVATION_TOOL_CONTRACTS } from "./javascript/javascriptRuntimeObservationToolContracts.js"; import { APPLICATION_TOOL_CONTRACTS } from "./applicationToolContracts.js"; import { WEB_RUNTIME_TOOL_CONTRACTS } from "./webRuntimeToolContracts.js"; +import { WEB_NETWORK_CAPTURE_TOOL_CONTRACTS } from "./webNetworkCaptureToolContracts.js"; import { WEB_SCRIPT_TOOL_CONTRACTS } from "./webScriptToolContracts.js"; import { JAVASCRIPT_RECOVERY_TOOL_CONTRACTS } from "./javascript/javascriptRecoveryToolContracts.js"; import { TOOL_EFFECTS } from "./toolEffects.js"; @@ -80,6 +81,7 @@ describe("tool contract surface", () => { ...APPLICATION_TOOL_CONTRACTS, ...WEB_SCRIPT_TOOL_CONTRACTS, ...WEB_RUNTIME_TOOL_CONTRACTS, + ...WEB_NETWORK_CAPTURE_TOOL_CONTRACTS, ...JAVASCRIPT_RECOVERY_TOOL_CONTRACTS, ...SESSION_TOOL_CONTRACTS, ].map(({ name }) => name); diff --git a/src/contracts/errorSchemas.ts b/src/contracts/errorSchemas.ts index 7da9c1916..d5c0e78ac 100644 --- a/src/contracts/errorSchemas.ts +++ b/src/contracts/errorSchemas.ts @@ -30,6 +30,8 @@ const generic = (code: Code) => /** Stable discriminated schema shared by every CLI and MCP error surface. */ export const analysisErrorProjectionSchema = z.discriminatedUnion("code", [ generic("invalid_request"), + generic("access_denied"), + generic("artifact_changed"), generic("unreadable_output"), generic("capability_unavailable"), generic("provider_unavailable"), diff --git a/src/contracts/toolContracts.test.ts b/src/contracts/toolContracts.test.ts index 6fec3a226..449e5f785 100644 --- a/src/contracts/toolContracts.test.ts +++ b/src/contracts/toolContracts.test.ts @@ -17,6 +17,7 @@ import { APPLICATION_TOOL_CONTRACTS } from "./applicationToolContracts.js"; import { JAVASCRIPT_RECOVERY_TOOL_CONTRACTS } from "./javascript/javascriptRecoveryToolContracts.js"; import { WEB_SCRIPT_TOOL_CONTRACTS } from "./webScriptToolContracts.js"; import { WEB_RUNTIME_TOOL_CONTRACTS } from "./webRuntimeToolContracts.js"; +import { WEB_NETWORK_CAPTURE_TOOL_CONTRACTS } from "./webNetworkCaptureToolContracts.js"; const GROUPS = { official: OFFICIAL_TOOL_CONTRACTS, @@ -34,6 +35,7 @@ const GROUPS = { javascript_recovery: JAVASCRIPT_RECOVERY_TOOL_CONTRACTS, web_script_export: WEB_SCRIPT_TOOL_CONTRACTS, web_runtime: WEB_RUNTIME_TOOL_CONTRACTS, + web_network_capture: WEB_NETWORK_CAPTURE_TOOL_CONTRACTS, session: SESSION_TOOL_CONTRACTS, } as const; diff --git a/src/contracts/toolContracts.ts b/src/contracts/toolContracts.ts index dd3896c43..40ed7a6db 100644 --- a/src/contracts/toolContracts.ts +++ b/src/contracts/toolContracts.ts @@ -10,6 +10,7 @@ import { JAVASCRIPT_RUNTIME_OBSERVATION_TOOL_CONTRACTS } from "./javascript/java import { APPLICATION_TOOL_CONTRACTS } from "./applicationToolContracts.js"; import { WEB_SCRIPT_TOOL_CONTRACTS } from "./webScriptToolContracts.js"; import { WEB_RUNTIME_TOOL_CONTRACTS } from "./webRuntimeToolContracts.js"; +import { WEB_NETWORK_CAPTURE_TOOL_CONTRACTS } from "./webNetworkCaptureToolContracts.js"; import { JAVASCRIPT_RECOVERY_TOOL_CONTRACTS } from "./javascript/javascriptRecoveryToolContracts.js"; import { OFFICIAL_TOOL_CONTRACTS } from "./officialToolContracts.js"; import { ENHANCED_TOOL_CONTRACTS } from "./enhancedToolContracts.js"; @@ -33,6 +34,7 @@ export const TOOL_CONTRACTS = [ ...APPLICATION_TOOL_CONTRACTS, ...WEB_SCRIPT_TOOL_CONTRACTS, ...WEB_RUNTIME_TOOL_CONTRACTS, + ...WEB_NETWORK_CAPTURE_TOOL_CONTRACTS, ...JAVASCRIPT_RECOVERY_TOOL_CONTRACTS, ...SESSION_TOOL_CONTRACTS, ] as const; diff --git a/src/contracts/toolEffects.ts b/src/contracts/toolEffects.ts index f1547b979..2468c0d8b 100644 --- a/src/contracts/toolEffects.ts +++ b/src/contracts/toolEffects.ts @@ -41,6 +41,11 @@ const sessionEvidence = effects({ mutatesSession: true }); /** Explicit effect audit for every public tool. */ export const TOOL_EFFECTS: Readonly> = { + inspect_web_network_capture: effects({ + mutatesSession: true, + writesFilesystem: true, + launchesProcess: true, + }), observe_web_execution: effects({ mutatesTarget: true, mutatesSession: true, diff --git a/src/contracts/webNetworkCaptureToolContracts.ts b/src/contracts/webNetworkCaptureToolContracts.ts new file mode 100644 index 000000000..a9e45d606 --- /dev/null +++ b/src/contracts/webNetworkCaptureToolContracts.ts @@ -0,0 +1,30 @@ +import { + inspectWebNetworkCaptureInputSchema, + webNetworkCaptureSchema, +} from "../domain/webNetworkCapture.js"; +import type { ToolContract } from "./toolContracts.js"; +import { toolContractMetadata } from "./toolEffects.js"; +import { evidenceResultOf } from "./toolOutputSchemas.js"; + +/** Historical inspection is distinct from live browser capture and runtime authority. */ +export const WEB_NETWORK_CAPTURE_TOOL_CONTRACTS = [ + { + name: "inspect_web_network_capture", + ...toolContractMetadata("inspect_web_network_capture"), + description: + "Inspect retained local HAR 1.2 or native mitmproxy capture records without contacting recorded URLs. Returns original ordinals, pointers/byte ranges, preserved producer fields, precise numeric representations and exact exposed binary bytes inline. Excludes known transport credentials and explicitly marked values. Decodes every record before optional ordinal selection; malformed or oversized captures return no partial success. Duplicate HAR members are invalid; prototype-named __proto__ members are unsupported by the current shared JSON boundary. Uses pinned upstream parsers, owned temporary files and bounded offline processes. Native mitmproxy requires caller-supplied mitmdump 12.2.3 on Linux. Browser transaction IDs, runtime attribution and original bytes of HAR Unicode text remain unknown.", + kind: "browser-provider", + inputSchema: inspectWebNetworkCaptureInputSchema, + outputSchema: evidenceResultOf(webNetworkCaptureSchema), + examples: [ + { + title: "Inspect historical HAR entries", + input: { + capture_path: "/captures/session.har", + format: "har", + record_ordinals: [0, 2], + }, + }, + ], + }, +] as const satisfies readonly ToolContract[]; diff --git a/src/domain/analysisAccessDenied.test.ts b/src/domain/analysisAccessDenied.test.ts new file mode 100644 index 000000000..2f5ce5c43 --- /dev/null +++ b/src/domain/analysisAccessDenied.test.ts @@ -0,0 +1,29 @@ +import { expect, it } from "vitest"; +import { AnalysisAccessDeniedError } from "./analysisErrorCore.js"; +import { projectAnalysisError } from "./analysisErrorProjection.js"; +import { analysisErrorProjectionSchema } from "../contracts/errorSchemas.js"; + +it.each(["EACCES", "EPERM"] as const)( + "projects %s as host access denial with correction guidance and a valid public schema", + (systemCode) => { + const error = new AnalysisAccessDeniedError( + "inspect_web_network_capture", + "/selected/capture.har", + systemCode, + { cause: new Error("internal detail") }, + ); + const projected = projectAnalysisError(error); + expect(analysisErrorProjectionSchema.parse(projected)).toMatchObject({ + code: "access_denied", + category: "unavailable", + retryable: false, + details: { + path: "/selected/capture.har", + system_code: systemCode, + boundary: "filesystem-read", + }, + }); + expect(projected.remediation.action).toContain("read access"); + expect(JSON.stringify(projected)).not.toContain("internal detail"); + }, +); diff --git a/src/domain/analysisErrorBase.ts b/src/domain/analysisErrorBase.ts index 8c5c40cd6..33db95a58 100644 --- a/src/domain/analysisErrorBase.ts +++ b/src/domain/analysisErrorBase.ts @@ -2,6 +2,8 @@ const ANALYSIS_ERROR_TAGS = [ "AnalysisProtocolError", "AnalysisInputError", + "AnalysisAccessDeniedError", + "AnalysisArtifactChangedError", "AnalysisOutputError", "AnalysisCapabilityUnavailableError", "AnalysisCancelledError", diff --git a/src/domain/analysisErrorCore.ts b/src/domain/analysisErrorCore.ts index e709dabc2..dd89321f0 100644 --- a/src/domain/analysisErrorCore.ts +++ b/src/domain/analysisErrorCore.ts @@ -90,3 +90,32 @@ export class AnalysisTimeoutError extends AnalysisError { ); } } + +/** Host filesystem permissions denied the selected read; this is not malformed caller input. */ +export class AnalysisAccessDeniedError extends AnalysisError { + readonly _tag = "AnalysisAccessDeniedError"; + constructor( + readonly operation: string, + readonly path: string, + readonly systemCode: "EACCES" | "EPERM", + options?: ErrorOptions, + ) { + super( + `Host filesystem read access denied (${systemCode}) for ${path} during ${operation}`, + options, + ); + } +} + +/** Selected artifact acquisition observed a change, so no stable identity can be reported. */ +export class AnalysisArtifactChangedError extends AnalysisError { + readonly _tag = "AnalysisArtifactChangedError"; + constructor( + readonly operation: string, + readonly path: string, + readonly reason: string, + options?: ErrorOptions, + ) { + super(`Selected artifact changed during acquisition: ${reason}`, options); + } +} diff --git a/src/domain/analysisErrorPresentation.ts b/src/domain/analysisErrorPresentation.ts index faa7b3e8b..f4baf9f18 100644 --- a/src/domain/analysisErrorPresentation.ts +++ b/src/domain/analysisErrorPresentation.ts @@ -1,4 +1,8 @@ -import { AnalysisInputError } from "./analysisErrorCore.js"; +import { + AnalysisAccessDeniedError, + AnalysisArtifactChangedError, + AnalysisInputError, +} from "./analysisErrorCore.js"; import { ArtifactOperationError } from "./artifactOperationError.js"; import { BinaryTargetError, @@ -47,6 +51,10 @@ export const analysisErrorRemediationAction = ( error.constraint === "directory_requires_file" ) return "For a JavaScript/Electron application directory, call analyze_javascript_application with input_path or run `rea analyze `. For binary analysis, select its executable file."; + if (error instanceof AnalysisAccessDeniedError) + return "Check the current process's read access to the selected path. Retry with a readable local file."; + if (error instanceof AnalysisArtifactChangedError) + return "Wait until the selected file is stable, then retry this operation."; if (error instanceof AnalysisInputError) return "Correct the listed arguments and retry."; if (error instanceof UnknownRegistryError && error.reason === "not-found") @@ -107,6 +115,8 @@ const STATIC_ERROR_CATEGORIES: Readonly< Partial> > = { AnalysisInputError: "invalid_input", + AnalysisAccessDeniedError: "unavailable", + AnalysisArtifactChangedError: "integrity_mismatch", AnalysisCapabilityUnavailableError: "unsupported_provider", ProviderSelectionError: "unsupported_provider", EvidenceIntegrityError: "integrity_mismatch", @@ -119,6 +129,10 @@ const STATIC_ERROR_CATEGORIES: Readonly< }; export const analysisErrorUserMessage = (error: AnalysisError): string => { + if (error instanceof AnalysisAccessDeniedError) + return "Host filesystem permissions denied read access to the selected path."; + if (error instanceof AnalysisArtifactChangedError) + return "The selected artifact changed during acquisition; no stable snapshot was decoded."; if (error instanceof AnalysisInputError) return "Analysis input is invalid. Check the arguments and try again."; const hopperMessage = hopperErrorUserMessage(error); @@ -243,6 +257,8 @@ const evidenceFileMessage = (reason: EvidenceFileError["reason"]): string => { const KNOWN_ERROR_TAGS = { AnalysisProtocolError: true, AnalysisInputError: true, + AnalysisAccessDeniedError: true, + AnalysisArtifactChangedError: true, AnalysisOutputError: true, AnalysisCapabilityUnavailableError: true, AnalysisCancelledError: true, diff --git a/src/domain/analysisErrorProjection.ts b/src/domain/analysisErrorProjection.ts index 85ff1901f..2f61a932f 100644 --- a/src/domain/analysisErrorProjection.ts +++ b/src/domain/analysisErrorProjection.ts @@ -7,6 +7,8 @@ import { assertKnownAnalysisErrorTag, } from "./analysisErrorPresentation.js"; import { + AnalysisAccessDeniedError, + AnalysisArtifactChangedError, AnalysisCancelledError, AnalysisCapabilityUnavailableError, AnalysisInputError, @@ -130,6 +132,8 @@ const STATIC_ERROR_CODES = { AnalysisOutputError: "unreadable_output", HopperProtocolError: "unreadable_output", AnalysisInputError: "invalid_request", + AnalysisAccessDeniedError: "access_denied", + AnalysisArtifactChangedError: "artifact_changed", AnalysisCapabilityUnavailableError: "capability_unavailable", AnalysisCancelledError: "cancelled", HopperCancelledError: "cancelled", @@ -177,6 +181,20 @@ const errorDetails = ( const requestErrorDetails = ( error: AnalysisError, ): Readonly> | undefined => { + if (error instanceof AnalysisArtifactChangedError) + return { + operation: error.operation, + path: error.path, + reason: error.reason, + boundary: "stable-artifact-read", + }; + if (error instanceof AnalysisAccessDeniedError) + return { + operation: error.operation, + path: error.path, + system_code: error.systemCode, + boundary: "filesystem-read", + }; if (error instanceof AnalysisOutputError) return { operation: error.operation, reason: error.reason }; if (error instanceof AnalysisInputError && error.issues.length > 0) @@ -372,6 +390,7 @@ const lifecycleErrorDetails = ( }; const RETRYABLE_CODES: ReadonlySet = new Set([ + "artifact_changed", "invalid_request", "provider_timeout", "cancelled", @@ -384,6 +403,8 @@ export interface AnalysisErrorProjection extends Readonly< > { readonly code: | "invalid_request" + | "access_denied" + | "artifact_changed" | "unreadable_output" | "capability_unavailable" | "provider_unavailable" diff --git a/src/domain/evidence.test.ts b/src/domain/evidence.test.ts index c13663709..29a629d3d 100644 --- a/src/domain/evidence.test.ts +++ b/src/domain/evidence.test.ts @@ -156,6 +156,23 @@ describe("analysis evidence identity", () => { }); }); +it("binds an explicit unknown-subject reason into the semantic Evidence identity", () => { + const reason = "Caller excluded the observed digest."; + const observation = { operation: "health", parameters: {}, result: true }; + const explicit = createEvidence(undefined, PROVIDER, { + ...observation, + subjectUnavailableReason: reason, + }); + const defaulted = createEvidence(undefined, PROVIDER, observation); + expect(explicit.subject).toBeNull(); + expect(explicit.limitations).toEqual([reason]); + expect(defaulted.limitations).toEqual([ + "Artifact identity is unavailable for this observation.", + ]); + expect(explicit.evidence_id).not.toBe(defaulted.evidence_id); + expect(parseEvidence(explicit)).toEqual(explicit); +}); + it("derives byte-stable bundle manifests independent of record order", () => { const artifactEvidence = createEvidence(TARGET, PROVIDER, { operation: "health", diff --git a/src/domain/evidence.ts b/src/domain/evidence.ts index 014acb7ed..6348fc38a 100644 --- a/src/domain/evidence.ts +++ b/src/domain/evidence.ts @@ -169,6 +169,8 @@ export interface EvidenceObservation { readonly authority?: EvidenceAuthority; readonly environment?: ExecutionEnvironment | null; readonly limitations?: readonly string[]; + /** Reason the subject identity is unavailable, when no target identity is supplied. */ + readonly subjectUnavailableReason?: string; readonly locations?: readonly EvidenceLocation[]; readonly evidenceLinks?: readonly string[]; } @@ -229,7 +231,7 @@ export const createEvidence = ( target === undefined ? null : { - name: target.path.split("/").at(-1) ?? target.path, + name: target.path.split("/").at(-1) || "artifact", digest: { sha256: target.sha256 }, format: target.format, architecture: target.architecture ?? null, @@ -262,7 +264,10 @@ export const createEvidence = ( environment: observation.environment ?? null, limitations: [ ...(target === undefined - ? ["Artifact identity is unavailable for this observation."] + ? [ + observation.subjectUnavailableReason ?? + "Artifact identity is unavailable for this observation.", + ] : []), ...(observation.limitations ?? []), ], diff --git a/src/domain/explicitSensitiveFailure.test.ts b/src/domain/explicitSensitiveFailure.test.ts new file mode 100644 index 000000000..28d935b82 --- /dev/null +++ b/src/domain/explicitSensitiveFailure.test.ts @@ -0,0 +1,126 @@ +import { AnalysisAccessDeniedError } from "./analysisErrorCore.js"; +import { expect, it } from "vitest"; +import { + AnalysisInputError, + AnalysisCapabilityUnavailableError, +} from "./analysisErrorCore.js"; +import { projectAnalysisError } from "./analysisErrorProjection.js"; +import { ProviderCleanupError } from "./providerCleanupError.js"; +import { + redactExplicitFailure, + redactExplicitPointer, +} from "./explicitSensitiveFailure.js"; + +it.each([ + ["/metadata/private~1~0/numeric", ["private/~"], "/metadata"], + ["/metadata/private/child", ["private"], "/metadata"], + ["/ordinary/path", [], "/ordinary/path"], +])( + "preserves a real parent for excluded pointer identities: %s", + (pointer, values, expected) => { + expect(redactExplicitPointer(pointer, values)).toBe(expected); + }, +); + +it("preserves cleanup failure authority and resources while excluding diagnostics", () => { + const error = new ProviderCleanupError( + "har", + ["/root/private/capture", "pid:123"], + { + capture_path: "/root/private/capture", + nested: { "private-key": 5, ordinary: "private ordinary" }, + exit_code: 2, + }, + { operation: "inspect_web_network_capture" }, + ); + const projected = projectAnalysisError( + redactExplicitFailure(error, ["private", "REDACTED"]), + ); + expect(projected.code).toBe("cleanup_incomplete"); + expect(JSON.stringify(projected)).not.toContain("private"); + expect(JSON.stringify(projected)).not.toContain("REDACTED"); + expect(JSON.stringify(projected)).toContain("pid:123"); + expect(JSON.stringify(projected)).toContain('"exit_code":2'); +}); + +it("keeps unsupported profile reasons distinct from malformed input", () => { + const error = new AnalysisCapabilityUnavailableError( + "mitmproxy", + "inspect_web_network_capture", + "Unsupported private profile", + { userMessage: "Select private-compatible profile" }, + ); + const result = redactExplicitFailure(error, ["private"]); + expect(result).toBeInstanceOf(AnalysisCapabilityUnavailableError); + expect(projectAnalysisError(result).code).toBe("capability_unavailable"); + expect(JSON.stringify(projectAnalysisError(result))).not.toContain("private"); +}); + +it("coarsens format issue pointers without creating replacement coordinates", () => { + const error = new AnalysisInputError( + "inspect_web_network_capture", + undefined, + [ + { + path: ["capture_path", "/metadata/private~1~0/nested"], + reason: "invalid_format", + message: "Malformed private field", + }, + ], + ); + const result = redactExplicitFailure(error, ["private/~", "private"]); + expect(result).toBeInstanceOf(AnalysisInputError); + if (!(result instanceof AnalysisInputError)) + throw new Error("Input error required"); + expect(result.issues[0]?.path).toEqual(["capture_path", "/metadata"]); +}); + +it("excludes a marked permission-denied path without losing its host error code", () => { + const error = redactExplicitFailure( + new AnalysisAccessDeniedError( + "inspect_web_network_capture", + "/declared-path-value/capture.har", + "EACCES", + ), + ["declared-path-value"], + ); + expect(error._tag).toBe("AnalysisAccessDeniedError"); + const projected = projectAnalysisError(error); + expect(projected).toMatchObject({ + code: "access_denied", + category: "unavailable", + details: { system_code: "EACCES" }, + }); + expect(JSON.stringify(projected)).not.toContain("declared-path-value"); +}); + +it("coarsens ordinary sensitive issue-path segments to their real parent", () => { + const input = new AnalysisInputError( + "inspect_web_network_capture", + undefined, + [ + { + path: ["input", "mysecret", "child"], + reason: "unknown_argument", + message: "Unrecognized mysecret", + }, + ], + ); + const result = redactExplicitFailure(input, ["secret"]); + if (!(result instanceof AnalysisInputError)) + throw new Error("Input error required"); + expect(result.issues[0]?.path).toEqual(["input"]); + expect(JSON.stringify(projectAnalysisError(result))).not.toContain("secret"); +}); + +it("does not reinterpret an unknown argument name as a JSON pointer", () => { + const input = new AnalysisInputError( + "inspect_web_network_capture", + undefined, + [{ path: ["/ordinary/secret"], reason: "unknown_argument" }], + ); + const result = redactExplicitFailure(input, ["secret"]); + if (!(result instanceof AnalysisInputError)) + throw new Error("Input error required"); + expect(result.issues[0]?.path).toEqual([]); +}); diff --git a/src/domain/explicitSensitiveFailure.ts b/src/domain/explicitSensitiveFailure.ts new file mode 100644 index 000000000..f26324f9b --- /dev/null +++ b/src/domain/explicitSensitiveFailure.ts @@ -0,0 +1,136 @@ +import type { AnalysisError } from "./analysisErrorBase.js"; +import { + AnalysisAccessDeniedError, + AnalysisArtifactChangedError, + AnalysisInputError, + AnalysisOutputError, + AnalysisCapabilityUnavailableError, +} from "./analysisErrorCore.js"; +import { redactExplicitText } from "./explicitSensitiveValues.js"; +import type { JsonValue } from "./jsonValue.js"; +import { ProviderAdapterError } from "./providerAdapterError.js"; +import { ProviderCleanupError } from "./providerCleanupError.js"; + +/** Keep a real JSON-pointer ancestor when a property identity is explicitly excluded. */ +export const redactExplicitPointer = ( + pointer: string, + values: readonly string[], +): string => { + let parent = ""; + for (const segment of pointer.split("/").slice(1)) { + const decoded = segment.replaceAll("~1", "/").replaceAll("~0", "~"); + const child = `${parent}/${segment}`; + if ( + values.some( + (literal) => decoded.includes(literal) || child.includes(literal), + ) + ) + return parent; + parent = child; + } + return parent; +}; + +/** Exclude explicit literals from common acquisition/decoder failures, preserving typed diagnostics. */ +export const redactExplicitFailure = ( + error: AnalysisError, + values: readonly string[], +): AnalysisError => { + if (values.length === 0) return error; + const text = (value: string): string => redactExplicitText(value, values); + const path = ( + parts: readonly (string | number)[], + ordinaryNames: boolean, + ): (string | number)[] => { + const retained: (string | number)[] = []; + for (const part of parts) { + if (typeof part !== "string") { + retained.push(part); + } else if (!ordinaryNames && part.startsWith("/")) { + const pointer = redactExplicitPointer(part, values); + retained.push(pointer); + if (pointer !== part) break; + } else { + if (values.some((literal) => part.includes(literal))) break; + retained.push(part); + } + } + return retained; + }; + const diagnostics = ( + value: Readonly>, + ): Record => + Object.fromEntries( + Object.entries(value) + .filter(([key]) => !values.some((literal) => key.includes(literal))) + .map(([key, item]) => [key, visit(item)]), + ); + const visit = (value: JsonValue): JsonValue => { + if (typeof value === "string") return text(value); + if (Array.isArray(value)) return value.map(visit); + if (typeof value === "object" && value !== null) return diagnostics(value); + return value; + }; + if (error instanceof AnalysisAccessDeniedError) + return new AnalysisAccessDeniedError( + error.operation, + text(error.path), + error.systemCode, + { cause: error }, + ); + if (error instanceof AnalysisArtifactChangedError) + return new AnalysisArtifactChangedError( + error.operation, + text(error.path), + text(error.reason), + { cause: error }, + ); + if (error instanceof AnalysisInputError) + return new AnalysisInputError( + error.operation, + { cause: error }, + error.issues.map((issue) => ({ + ...issue, + // Unknown-argument paths contain literal field names, including names + // beginning with '/'; they are not producer JSON-pointer coordinates. + path: path(issue.path, issue.reason === "unknown_argument"), + ...(issue.message === undefined + ? {} + : { message: text(issue.message) }), + ...(issue.expected === undefined + ? {} + : { expected: visit(issue.expected) }), + })), + ); + if (error instanceof AnalysisOutputError) + return new AnalysisOutputError(error.operation, text(error.reason), { + cause: error, + }); + if (error instanceof AnalysisCapabilityUnavailableError) + return new AnalysisCapabilityUnavailableError( + error.providerId, + error.operation, + text(error.reason), + { + cause: error, + ...(error.userMessage === undefined + ? {} + : { userMessage: text(error.userMessage) }), + }, + ); + if (error instanceof ProviderCleanupError) + return new ProviderCleanupError( + error.providerId, + error.cleanupResources.map(text), + diagnostics(error.diagnostics ?? {}), + { operation: error.operation, cause: error }, + ); + if (error instanceof ProviderAdapterError) + return new ProviderAdapterError(error.providerId, error.operation, { + cause: error, + ...(error.diagnostics === undefined + ? {} + : { diagnostics: diagnostics(error.diagnostics) }), + }); + return error; +}; diff --git a/src/domain/explicitSensitiveValues.test.ts b/src/domain/explicitSensitiveValues.test.ts new file mode 100644 index 000000000..64c03a9d7 --- /dev/null +++ b/src/domain/explicitSensitiveValues.test.ts @@ -0,0 +1,22 @@ +import { expect, it } from "vitest"; +import { redactExplicitText } from "./explicitSensitiveValues.js"; + +it.each([ + ["secret context", ["secret"]], + ["secret context", ["secret", "REDACTED"]], + ["secret context", ["secret", "[", "…"]], + ["secretable context", ["secret", "able", "table"]], + ["secretsecret context", ["secret", "…", "REDACTED"]], +] as const)( + "keeps declared literals out of replacement text: %s", + (input, values) => { + const result = redactExplicitText(input, values); + for (const value of values) expect(result).not.toContain(value); + }, +); + +it("does not guess sensitivity or treat empty declarations as matches", () => { + expect(redactExplicitText("API_SECRET ordinary evidence", [""])).toBe( + "API_SECRET ordinary evidence", + ); +}); diff --git a/src/domain/explicitSensitiveValues.ts b/src/domain/explicitSensitiveValues.ts new file mode 100644 index 000000000..52c115c81 --- /dev/null +++ b/src/domain/explicitSensitiveValues.ts @@ -0,0 +1,20 @@ +/** Replace caller-declared literal text values without guessing sensitivity from names. */ +export const redactExplicitText = ( + value: string, + sensitiveValues: readonly string[], +): string => { + const literals = [...new Set(sensitiveValues)].filter( + (literal) => literal !== "", + ); + const replacement = + ["[REDACTED]", "…", ""].find((candidate) => + literals.every((literal) => !candidate.includes(literal)), + ) ?? ""; + let result = value; + for (const literal of literals.sort( + (left, right) => right.length - left.length, + )) + result = result.replaceAll(literal, replacement); + // Replacement boundaries can form another declaration; exclude that text. + return literals.some((literal) => result.includes(literal)) ? "" : result; +}; diff --git a/src/domain/webNetworkCapture.ts b/src/domain/webNetworkCapture.ts new file mode 100644 index 000000000..bda9334bb --- /dev/null +++ b/src/domain/webNetworkCapture.ts @@ -0,0 +1,141 @@ +import { z } from "zod"; +import { jsonValueSchema } from "./jsonValue.js"; + +/** Complete historical evidence budgets; exceeding a budget returns no partial capture. */ +export const WEB_NETWORK_CAPTURE_LIMITS = { + inputBytes: 32 * 1024 * 1024, + outputBytes: 96 * 1024 * 1024, + timeoutMs: 30_000, + depth: 64, +} as const; + +/** Inspect an explicit retained capture without contacting its recorded destinations. */ +export const inspectWebNetworkCaptureInputSchema = z.strictObject({ + capture_path: z + .string() + .min(1) + .describe("Absolute local HAR or mitmproxy capture path."), + format: z.enum(["har", "mitmproxy"]), + record_ordinals: z + .array(z.number().int().nonnegative()) + .min(1) + .optional() + .describe( + "Zero-based producer record ordinals, in caller-selected order. Omit to inspect every record.", + ), + sensitive_values: z + .array(z.string().min(1)) + .default([]) + .describe( + "Literal text excluded from capture payloads, byte/numeric projections, producer coordinates, artifact paths/digests and diagnostic text. Declaration strings are never persisted in Evidence parameters. Canonical contract keys, enums and provider identities remain intact.", + ), +}); + +const pointerSchema = z + .string() + .describe("RFC 6901 pointer in the producer's reported object."); +const redactionSchema = z.union([ + z.strictObject({ + pointer: pointerSchema, + reason: z.enum(["transport-credential", "explicit-sensitive-value"]), + }), + z.strictObject({ + pointer: pointerSchema, + reason: z.literal("explicit-sensitive-value"), + scope: z + .literal("property-name") + .describe( + "An omitted property name; pointer identifies its actual parent, not an invented child coordinate.", + ), + }), +]); +const binaryBase = { + pointer: pointerSchema, + representation: z.enum(["producer-bytes", "har-base64-content"]), +}; +const binarySchema = z.discriminatedUnion("state", [ + z.strictObject({ + ...binaryBase, + state: z.literal("retained"), + content_base64: z.string(), + bytes: z.number().int().nonnegative(), + sha256: z.string().regex(/^[0-9a-f]{64}$/), + }), + z.strictObject({ + ...binaryBase, + state: z.literal("redacted"), + content_base64: z.null(), + bytes: z.null(), + sha256: z.null(), + }), +]); +const numberSchema = z.strictObject({ + pointer: pointerSchema, + producer_type: z.enum(["json-number", "integer", "float"]), + literal: z.string(), +}); + +/** Preserve producer representations with independent sidecars rather than invented browser IDs. */ +export const webNetworkCaptureRecordSchema = z.strictObject({ + ordinal: z.number().int().nonnegative(), + location: z.discriminatedUnion("kind", [ + z.strictObject({ + kind: z.literal("unknown"), + reason: z.literal("explicit-sensitive-value"), + }), + z.strictObject({ kind: z.literal("json-pointer"), pointer: pointerSchema }), + z.strictObject({ + kind: z.literal("byte-range"), + offset: z.number().int().nonnegative(), + bytes: z.number().int().positive(), + }), + ]), + reported: jsonValueSchema, + binary_fields: z.array(binarySchema), + numeric_literals: z.array(numberSchema), + redactions: z.array(redactionSchema), + limitations: z.array(z.string()), +}); + +/** Historical capture inspection has file identity, producer provenance and explicit unknowns. */ +export const webNetworkCaptureSchema = z.strictObject({ + artifact: z.strictObject({ + path: z + .string() + .describe( + "Selected artifact path; empty when explicitly excluded from Evidence.", + ), + sha256: z + .string() + .regex(/^[0-9a-f]{64}$/) + .nullable() + .describe( + "Observed artifact SHA-256; null only when explicitly excluded by a sensitive declaration.", + ), + bytes: z.number().int().nonnegative(), + }), + format: z.enum(["har", "mitmproxy"]), + decoder: z.strictObject({ + id: z.string().min(1), + name: z.string().min(1), + version: z.string().min(1), + }), + container: z.strictObject({ + reported: jsonValueSchema, + numeric_literals: z.array(numberSchema), + redactions: z.array(redactionSchema), + records_pointer: pointerSchema.nullable(), + }), + total_records: z.number().int().nonnegative(), + records: z.array(webNetworkCaptureRecordSchema), + runtime_attribution: z.literal("unknown"), + limitations: z.array(z.string()), +}); + +export type InspectWebNetworkCaptureInput = z.output< + typeof inspectWebNetworkCaptureInputSchema +>; +export type WebNetworkCapture = z.output; +export type WebNetworkCaptureRecord = z.output< + typeof webNetworkCaptureRecordSchema +>; diff --git a/src/domain/webNetworkCaptureCoordinates.ts b/src/domain/webNetworkCaptureCoordinates.ts new file mode 100644 index 000000000..6cba46cd9 --- /dev/null +++ b/src/domain/webNetworkCaptureCoordinates.ts @@ -0,0 +1,68 @@ +import { redactExplicitText } from "./explicitSensitiveValues.js"; +import type { WebNetworkCapture } from "./webNetworkCapture.js"; + +/** Exclude marked source coordinates without inventing a substitute record identity. */ +export const excludeCaptureCoordinates = ( + value: WebNetworkCapture, + sensitiveValues: readonly string[], +): WebNetworkCapture => { + if (sensitiveValues.length === 0) return value; + let excluded = false; + const sensitive = (pointer: string): boolean => + sensitiveValues.some((literal) => pointer.includes(literal)); + const sidecars = ( + items: readonly T[], + ): T[] => + items.filter((item) => { + if (!sensitive(item.pointer)) return true; + excluded = true; + return false; + }); + const records = value.records.map((record) => { + const location = + (record.location.kind === "json-pointer" && + sensitive(record.location.pointer)) || + (record.location.kind === "byte-range" && + (sensitive(String(record.location.offset)) || + sensitive(String(record.location.bytes)))) + ? { + kind: "unknown" as const, + reason: "explicit-sensitive-value" as const, + } + : record.location; + if (location !== record.location) excluded = true; + return { + ...record, + location, + binary_fields: sidecars(record.binary_fields), + numeric_literals: sidecars(record.numeric_literals), + redactions: sidecars(record.redactions), + }; + }); + const recordsPointer = value.container.records_pointer; + const records_pointer = + recordsPointer !== null && sensitive(recordsPointer) + ? null + : recordsPointer; + if (records_pointer !== recordsPointer) excluded = true; + const container = { + ...value.container, + records_pointer, + numeric_literals: sidecars(value.container.numeric_literals), + redactions: sidecars(value.container.redactions), + }; + return { + ...value, + container, + records, + limitations: excluded + ? [ + ...value.limitations, + redactExplicitText( + "Marked source coordinates and their sidecars are excluded; affected record locations remain unknown.", + sensitiveValues, + ), + ] + : value.limitations, + }; +}; diff --git a/src/generatedPackageMetadata.ts b/src/generatedPackageMetadata.ts index c049e92da..1f3712348 100644 --- a/src/generatedPackageMetadata.ts +++ b/src/generatedPackageMetadata.ts @@ -5,5 +5,5 @@ export const PACKAGE_METADATA = { serverSdkVersion: "2.3.1", clientSdkVersion: "2.3.1", coreSdkVersion: "2.3.1", - skillVersion: "27", + skillVersion: "28", } as const; diff --git a/src/process/OwnedCommand.ts b/src/process/OwnedCommand.ts new file mode 100644 index 000000000..1fa054c70 --- /dev/null +++ b/src/process/OwnedCommand.ts @@ -0,0 +1,198 @@ +import type { + OwnedProviderProcessSpawnOptions, + ProviderProcessSnapshot, + SpawnedOwnedProviderProcess, +} from "./ProviderProcess.js"; +import { + ProviderProcessSupervisor, + spawnOwnedProviderProcess, +} from "./ProviderProcess.js"; +import { cleanupOwnedProcessGroup } from "./ProcessOwnership.js"; +import { + ProviderStartupDeadline, + waitForAbortableDelay, +} from "./ProviderDeadline.js"; + +/** A short-lived command failed its lifecycle or complete-output contract. */ +export class OwnedCommandFailure extends Error { + constructor( + readonly reason: "cancelled" | "timeout" | "output-limit" | "process", + message: string, + readonly snapshot: ProviderProcessSnapshot | null = null, + readonly cleanupFailure: string | null = null, + options?: ErrorOptions, + readonly resources: readonly string[] = [], + ) { + super(message, options); + } +} + +/** Execute a single owned process with independent cleanup and bounded diagnostic retention. */ +export const runOwnedCommand = async ( + spawn: OwnedProviderProcessSpawnOptions, + limits: { readonly timeoutMs: number; readonly diagnosticBytes: number }, + options: { + readonly signal?: AbortSignal; + readonly launcher?: ( + input: OwnedProviderProcessSpawnOptions, + ) => Promise; + } = {}, +): Promise => { + const callerSignal = options.signal ?? spawn.signal; + if (callerSignal?.aborted) + throw new OwnedCommandFailure( + "cancelled", + "Command cancelled before launch.", + ); + const deadline = new ProviderStartupDeadline(limits.timeoutMs, callerSignal); + try { + return await collectOwnedCommand( + spawn, + limits, + options.launcher ?? spawnOwnedProviderProcess, + deadline, + callerSignal, + ); + } finally { + deadline.dispose(); + } +}; + +const collectOwnedCommand = async ( + spawn: OwnedProviderProcessSpawnOptions, + limits: { readonly diagnosticBytes: number }, + launcher: ( + input: OwnedProviderProcessSpawnOptions, + ) => Promise, + deadline: ProviderStartupDeadline, + callerSignal?: AbortSignal, +): Promise => { + const signal = deadline.signal; + let launched: SpawnedOwnedProviderProcess; + try { + launched = await launcher({ + ...spawn, + signal, + }); + } catch (cause: unknown) { + const interruption = deadline.interruption; + if (interruption !== undefined) + throw new OwnedCommandFailure( + interruption, + interruption === "cancelled" + ? "Command cancelled during ownership preparation." + : "Command deadline elapsed during ownership preparation.", + null, + null, + { cause }, + ); + throw cause; + } + let exceeded = false; + let processFailure: string | undefined; + const supervisor = new ProviderProcessSupervisor( + { + ...launched, + ownsProcessLifetime: true, + cleanup: + launched.cleanup ?? + (() => cleanupOwnedProcessGroup(launched.ownership)), + }, + { + maxDiagnosticBytes: limits.diagnosticBytes, + onDiagnostic: (event) => { + if (event.type === "output" && event.truncated === true) + exceeded = true; + if (event.type === "error") processFailure = event.message; + }, + }, + ); + let failure: OwnedCommandFailure | undefined; + try { + while (!(await supervisor.waitForOutputClose(10))) { + const interruption = deadline.interruption; + if (interruption !== undefined) + throw new OwnedCommandFailure( + interruption, + interruption === "cancelled" + ? "Command cancelled." + : "Command deadline elapsed.", + ); + if (exceeded) + throw new OwnedCommandFailure( + "output-limit", + "Command diagnostic output exceeded its complete-output budget.", + ); + if (processFailure !== undefined) + throw new OwnedCommandFailure("process", processFailure); + await waitForAbortableDelay(25, signal); + } + const snapshot = supervisor.snapshot(); + const interruption = deadline.interruption; + if (interruption !== undefined) + throw new OwnedCommandFailure( + interruption, + interruption === "cancelled" + ? "Command cancelled." + : "Command deadline elapsed.", + snapshot, + ); + if (exceeded) + throw new OwnedCommandFailure( + "output-limit", + "Command diagnostic output exceeded its complete-output budget.", + snapshot, + ); + if ( + processFailure !== undefined || + snapshot.exitCode !== 0 || + snapshot.signal !== null + ) + throw new OwnedCommandFailure( + "process", + processFailure ?? + "Command failed; inspect its exit status and diagnostics.", + snapshot, + ); + } catch (cause: unknown) { + failure = + cause instanceof OwnedCommandFailure + ? cause + : new OwnedCommandFailure( + "process", + "Command collection failed.", + null, + null, + { cause }, + ); + } + const stopped = await supervisor.stop(); + const snapshot = supervisor.snapshot(); + supervisor.dispose(); + if (stopped.status === "incomplete") + throw new OwnedCommandFailure( + failure?.reason ?? "process", + failure?.message ?? "Command cleanup failed.", + snapshot, + stopped.reason, + failure === undefined ? undefined : { cause: failure }, + [ + spawn.runId, + ...(spawn.cwd === undefined ? [] : [spawn.cwd]), + `pid:${launched.ownership.leaderPid}`, + `process-group:${launched.ownership.processGroupId}`, + ], + ); + if (failure !== undefined) + throw new OwnedCommandFailure( + failure.reason, + failure.message, + snapshot, + failure.cleanupFailure, + { cause: failure }, + failure.resources, + ); + if (callerSignal?.aborted) + throw new OwnedCommandFailure("cancelled", "Command cancelled.", snapshot); + return snapshot; +}; diff --git a/src/process/ProviderProcess.ts b/src/process/ProviderProcess.ts index 07b0ba5be..ef5ff8262 100644 --- a/src/process/ProviderProcess.ts +++ b/src/process/ProviderProcess.ts @@ -72,6 +72,8 @@ export type ProviderProcessLaunch = /** Detached diagnostics captured for one supervised provider process. */ export interface ProviderProcessSnapshot { + /** True only when a configured diagnostic budget excluded output bytes. */ + readonly diagnosticTruncated?: boolean; readonly stdout: { readonly text: string; readonly bytes: number; @@ -91,6 +93,7 @@ export type ProviderProcessDiagnostic = readonly stream: "stdout" | "stderr"; readonly bytes: number; readonly totalBytes: number; + readonly truncated?: boolean; } | { readonly type: "exit"; @@ -104,6 +107,8 @@ export type ProviderProcessDiagnostic = export interface ProviderProcessSupervisorOptions { /** Retain stdout for diagnostics; disable when it carries a parsed protocol. Defaults to true. */ readonly captureStdout?: boolean; + /** Aggregate retained stdout/stderr byte budget; overflow is drained and reported, not retained. */ + readonly maxDiagnosticBytes?: number; readonly onDiagnostic?: (event: ProviderProcessDiagnostic) => void; } @@ -209,6 +214,9 @@ export class ProviderProcessSupervisor { }>(); readonly #outputClose = deferred(); #outputClosed = false; + #stdoutObservedBytes = 0; + #stderrObservedBytes = 0; + #diagnosticTruncated = false; readonly #onStdout = (chunk: Buffer | string): void => { this.#capture("stdout", this.#stdout, chunk); }; @@ -253,9 +261,10 @@ export class ProviderProcessSupervisor { this.#recordExit(launch.process.exitCode, launch.process.signalCode); } - /** Latest complete output and exit observation. */ + /** Latest retained output and exit observation, with explicit configured truncation. */ snapshot(): ProviderProcessSnapshot { return { + ...(this.#diagnosticTruncated ? { diagnosticTruncated: true } : {}), stdout: this.#stdout.snapshot(), stderr: this.#stderr.snapshot(), exitCode: this.#exitObservation?.code, @@ -267,6 +276,9 @@ export class ProviderProcessSupervisor { resetOutput(): void { this.#stdout.reset(); this.#stderr.reset(); + this.#stdoutObservedBytes = 0; + this.#stderrObservedBytes = 0; + this.#diagnosticTruncated = false; } /** Wait for process exit up to a caller-owned bounded interval. */ @@ -388,12 +400,29 @@ export class ProviderProcessSupervisor { chunk: Buffer | string, ): void { const bytes = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk); - capture.append(bytes); + if (stream === "stdout") this.#stdoutObservedBytes += bytes.byteLength; + else this.#stderrObservedBytes += bytes.byteLength; + const remaining = + this.#options.maxDiagnosticBytes === undefined + ? bytes.byteLength + : Math.max( + 0, + this.#options.maxDiagnosticBytes - + this.#stdout.bytes - + this.#stderr.bytes, + ); + const retained = Math.min(bytes.byteLength, remaining); + if (retained < bytes.byteLength) this.#diagnosticTruncated = true; + if (retained > 0) capture.append(bytes.subarray(0, retained)); this.#options.onDiagnostic?.({ type: "output", stream, bytes: bytes.byteLength, - totalBytes: capture.bytes, + totalBytes: + stream === "stdout" + ? this.#stdoutObservedBytes + : this.#stderrObservedBytes, + ...(retained < bytes.byteLength ? { truncated: true } : {}), }); } diff --git a/src/process/capture/ProcessCaptureLifecycle.ts b/src/process/capture/ProcessCaptureLifecycle.ts index 1dcec19d8..5cd75bf1a 100644 --- a/src/process/capture/ProcessCaptureLifecycle.ts +++ b/src/process/capture/ProcessCaptureLifecycle.ts @@ -364,15 +364,30 @@ const filesystemFailure = (cause: unknown): string | undefined => { return `Executable file observation failed: ${cause.message}`; }; -/** Sample the caller-selected executable contents and stable file identity before launch. */ +/** Metadata IO seam for deterministic prelaunch cancellation and identity checks. */ +export interface SelectedExecutableFileSystem { + open(path: string, flags: number): ReturnType; + stat(path: string): Promise; +} + +const selectedExecutableFileSystem: SelectedExecutableFileSystem = { + open, + stat: (path) => stat(path, { bigint: true }), +}; + +/** Sample the selected file without launching it; retain cancellation and file identity. */ export const observeSelectedExecutable = async ( path: string, signal?: AbortSignal, + fileSystem: SelectedExecutableFileSystem = selectedExecutableFileSystem, ): Promise => { assertNotCancelled(signal); let handle: Awaited> | undefined; try { - handle = await open(path, constants.O_RDONLY | constants.O_NONBLOCK); + handle = await fileSystem.open( + path, + constants.O_RDONLY | constants.O_NONBLOCK, + ); const openedStats = await handle.stat({ bigint: true }); if (!openedStats.isFile()) return { @@ -381,9 +396,7 @@ export const observeSelectedExecutable = async ( reason: "Selected executable is not a regular file.", }; const openedIdentity = identityFromStats(openedStats); - const selectedPathIdentity = identityFromStats( - await stat(path, { bigint: true }), - ); + const selectedPathIdentity = identityFromStats(await fileSystem.stat(path)); if (!sameExecutableIdentity(openedIdentity, selectedPathIdentity)) return { sha256: null, @@ -391,6 +404,9 @@ export const observeSelectedExecutable = async ( reason: "Selected executable path changed while it was being opened.", }; + // IO above can settle after cancellation. Do not construct a stream with an + // already-aborted signal: Node can emit a second, unhandled AbortError. + assertNotCancelled(signal); const hash = createHash("sha256"); for await (const chunk of handle.createReadStream({ autoClose: false, diff --git a/src/server/createServer.ts b/src/server/createServer.ts index 68b1aeb08..acc5dca05 100644 --- a/src/server/createServer.ts +++ b/src/server/createServer.ts @@ -25,6 +25,9 @@ import type { WebSourceLocationService } from "../application/WebSourceLocationS import type { WebRuntimeService } from "../application/WebRuntimeService.js"; import { createWebRuntimeService } from "../composition/webRuntime.js"; import { registerWebRuntimeTools } from "./registerWebRuntimeTools.js"; +import type { WebNetworkCaptureService } from "../application/WebNetworkCaptureService.js"; +import { createWebNetworkCaptureService } from "../composition/webNetworkCaptures.js"; +import { registerWebNetworkCaptureTool } from "./registerWebNetworkCaptureTool.js"; import { registerJavaScriptRecoveryTool } from "./registerJavaScriptRecoveryTool.js"; import { JavaScriptRecoveryService } from "../application/javascript/JavaScriptRecoveryService.js"; import type { JavaScriptRecoveryPort } from "../application/javascript/JavaScriptRecoveryPort.js"; @@ -62,6 +65,7 @@ export interface CreateServerOptions { readonly webModuleTrace?: WebModuleTraceService; readonly webSourceLocation?: WebSourceLocationService; readonly webRuntime?: WebRuntimeService; + readonly webNetworkCapture?: WebNetworkCaptureService; readonly androidAnalysis?: AndroidAnalysisPort; readonly browserObservation?: BrowserObservationPort; readonly browserScenarioCapture?: BrowserScenarioCapturePort; @@ -213,6 +217,12 @@ export const createServer = ( toolLogger, recordEvidence, ); + registerWebNetworkCaptureTool( + server, + options.webNetworkCapture ?? createWebNetworkCaptureService(), + toolLogger, + recordEvidence, + ); registerJavaScriptRecoveryTool( server, new JavaScriptRecoveryService( diff --git a/src/server/registerWebNetworkCaptureTool.ts b/src/server/registerWebNetworkCaptureTool.ts new file mode 100644 index 000000000..2c1a9e5c5 --- /dev/null +++ b/src/server/registerWebNetworkCaptureTool.ts @@ -0,0 +1,46 @@ +import type { + McpServer, + StandardSchemaWithJSON, +} from "@modelcontextprotocol/server"; +import type { WebNetworkCaptureService } from "../application/WebNetworkCaptureService.js"; +import type { EvidenceWriter } from "../application/investigation/InvestigationRecordPort.js"; +import { toolContract } from "../contracts/toolContracts.js"; +import type { Logger } from "../logger.js"; +import { logToolExecution } from "./toolLogging.js"; +import { toolRegistrationOptions } from "./toolRegistrationOptions.js"; +import { toCallToolResult } from "./toolResult.js"; + +/** Bind historical inspection to its named contract and caller-owned Evidence writer. */ +export const registerWebNetworkCaptureTool = ( + server: McpServer, + service: WebNetworkCaptureService, + logger: Logger, + recordEvidence?: EvidenceWriter["recordEvidence"], +): void => { + const contract = toolContract("inspect_web_network_capture"); + const registration = toolRegistrationOptions(contract); + // Keep the exact advertised contract; the shared service validates raw input + // before effects so caller-declared sensitivity also covers invalid arguments. + // SDK pre-validation otherwise emits unprojected issue paths as plain text. + const inputSchema: StandardSchemaWithJSON = { + "~standard": { + ...registration.inputSchema["~standard"], + vendor: "rea-application", + validate: (value: unknown) => ({ value }), + }, + }; + server.registerTool( + contract.name, + { ...registration, inputSchema }, + async (input, context) => { + const result = await logToolExecution(logger, contract.name, () => + service.inspect(input, { signal: context.mcpReq.signal }), + ); + if (!result.ok) return toCallToolResult(result, contract); + const recorded = recordEvidence?.(result.value); + return recorded !== undefined && !recorded.ok + ? toCallToolResult(recorded, contract) + : toCallToolResult(result, contract); + }, + ); +}; diff --git a/tests/boundary/browser/historicalCaptureDecoder.test.ts b/tests/boundary/browser/historicalCaptureDecoder.test.ts new file mode 100644 index 000000000..5187d1f6a --- /dev/null +++ b/tests/boundary/browser/historicalCaptureDecoder.test.ts @@ -0,0 +1,481 @@ +import { PrivateRuntimeRoot } from "../../../src/process/PrivateRuntimeRoot.js"; +import { + access, + chmod, + mkdtemp, + rm, + truncate, + writeFile, +} from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { expect, it, onTestFinished } from "vitest"; +import { HistoricalCaptureDecoder } from "../../../src/browser/history/HistoricalCaptureDecoder.js"; +import { HAR_CAPTURE_PROVIDER_IDENTITY } from "../../../src/browser/history/CaptureRelease.js"; +import { + inspectWebNetworkCaptureInputSchema, + WEB_NETWORK_CAPTURE_LIMITS, +} from "../../../src/domain/webNetworkCapture.js"; +import { historicalHar } from "../../fixtures/historicalHar.js"; +import { decodeHarCapture } from "../../../src/browser/history/HarCapture.js"; +import { projectAnalysisError } from "../../../src/domain/analysisErrorProjection.js"; + +it.each([true, false])( + "binds decoder replies to the complete selected provider identity (matching name: %s)", + async (matching) => { + const root = await mkdtemp(join(tmpdir(), "rea-capture-identity-")); + onTestFinished(() => rm(root, { recursive: true, force: true })); + const path = join(root, "capture.har"); + const text = JSON.stringify(historicalHar()); + await writeFile(path, text); + const decoder = new HistoricalCaptureDecoder( + [ + { + format: "har", + identity: HAR_CAPTURE_PROVIDER_IDENTITY, + command: async (_, runtimePath) => { + const value = decodeHarCapture(text, []); + if (!matching) value.decoder.name = "different adapter"; + await writeFile( + join(runtimePath, "reply.json"), + JSON.stringify({ ok: true, value }), + ); + return { command: process.execPath, arguments: ["-e", ""] }; + }, + }, + ], + process.env, + ); + const result = await decoder.inspect( + inspectWebNetworkCaptureInputSchema.parse({ + capture_path: path, + format: "har", + }), + ); + expect(result.ok).toBe(matching); + if (!result.ok) expect(result.error._tag).toBe("AnalysisOutputError"); + else expect(result.value.decoder).toEqual(HAR_CAPTURE_PROVIDER_IDENTITY); + }, +); + +it("classifies an oversized selected capture before creating an owned decoder", async () => { + const root = await mkdtemp(join(tmpdir(), "rea-historical-limit-")); + onTestFinished(() => rm(root, { recursive: true, force: true })); + const path = join(root, "input.har"); + await writeFile(path, ""); + await truncate(path, WEB_NETWORK_CAPTURE_LIMITS.inputBytes + 1); + const decoder = new HistoricalCaptureDecoder( + [ + { + format: "har", + identity: HAR_CAPTURE_PROVIDER_IDENTITY, + command: async () => { + throw new Error("Oversized input must not launch decoding"); + }, + }, + ], + process.env, + ); + const result = await decoder.inspect( + inspectWebNetworkCaptureInputSchema.parse({ + capture_path: path, + format: "har", + }), + ); + if (result.ok) throw new Error("Oversized input must fail"); + expect(result.error._tag).toBe("AnalysisInputError"); + expect(projectAnalysisError(result.error)).toMatchObject({ + category: "invalid_input", + details: { + issues: [ + { + path: ["capture_path"], + reason: "out_of_range", + expected: { + maximum_capture_bytes: WEB_NETWORK_CAPTURE_LIMITS.inputBytes, + }, + }, + ], + }, + }); +}); + +it("does not misclassify decoder filesystem failures as missing selected input", async () => { + const root = await mkdtemp(join(tmpdir(), "rea-historical-command-")); + onTestFinished(() => rm(root, { recursive: true, force: true })); + const path = join(root, "input.har"); + await writeFile(path, JSON.stringify(historicalHar())); + const decoder = new HistoricalCaptureDecoder( + [ + { + format: "har", + identity: HAR_CAPTURE_PROVIDER_IDENTITY, + command: async () => { + throw Object.assign( + new Error("Configured decoder command file missing"), + { code: "ENOENT" }, + ); + }, + }, + ], + process.env, + ); + const result = await decoder.inspect( + inspectWebNetworkCaptureInputSchema.parse({ + capture_path: path, + format: "har", + }), + ); + if (result.ok) throw new Error("Failed decoder command must fail"); + expect(result.error._tag).toBe("ProviderAdapterError"); + expect(projectAnalysisError(result.error)).toMatchObject({ + details: { + diagnostics: { + phase: "decoder", + reason: "Configured decoder command file missing", + }, + }, + }); +}); + +it.each(["missing-capture", "missing-reply", "failed-command"])( + "redacts declared paths and diagnostics while preserving the %s failure", + async (mode) => { + const root = await mkdtemp(join(tmpdir(), "rea-historical-sensitive-")); + onTestFinished(() => rm(root, { recursive: true, force: true })); + const path = join(root, "explicit-private-value.har"); + if (mode !== "missing-capture") + await writeFile(path, JSON.stringify(historicalHar())); + const decoder = new HistoricalCaptureDecoder( + [ + { + format: "har", + identity: HAR_CAPTURE_PROVIDER_IDENTITY, + command: async () => ({ + command: process.execPath, + arguments: [ + "-e", + mode === "failed-command" + ? 'process.stderr.write("explicit-private-value ordinary diagnostic"); process.exit(2)' + : "process.exit(0)", + ], + }), + }, + ], + process.env, + ); + const result = await decoder.inspect( + inspectWebNetworkCaptureInputSchema.parse({ + capture_path: path, + format: "har", + sensitive_values: ["explicit-private-value", "REDACTED"], + }), + ); + if (result.ok) throw new Error("Failure required"); + const projected = projectAnalysisError(result.error); + const serialized = JSON.stringify(projected); + expect(serialized).not.toContain("explicit-private-value"); + expect(serialized).not.toContain("REDACTED"); + expect(result.error._tag).toBe( + mode === "missing-capture" + ? "AnalysisInputError" + : mode === "missing-reply" + ? "AnalysisOutputError" + : "ProviderAdapterError", + ); + if (mode === "missing-capture") expect(serialized).toContain("ENOENT"); + if (mode === "failed-command") + expect(serialized).toContain('"exit_code":2'); + }, +); + +it("reports an absent owned reply as output failure and removes its private snapshot root", async () => { + const root = await mkdtemp(join(tmpdir(), "rea-historical-decoder-")); + onTestFinished(() => rm(root, { recursive: true, force: true })); + const path = join(root, "input.har"); + await writeFile(path, JSON.stringify(historicalHar())); + let privatePath: string | undefined; + const decoder = new HistoricalCaptureDecoder( + [ + { + format: "har", + identity: HAR_CAPTURE_PROVIDER_IDENTITY, + command: async (_, runtimePath) => { + privatePath = runtimePath; + return { + command: process.execPath, + arguments: ["-e", "process.exit(0)"], + }; + }, + }, + ], + process.env, + ); + const result = await decoder.inspect( + inspectWebNetworkCaptureInputSchema.parse({ + capture_path: path, + format: "har", + }), + ); + if (result.ok) throw new Error("Owned reply is required"); + expect(result.error._tag).toBe("AnalysisOutputError"); + expect(result.error.message).toContain("ENOENT"); + if (privatePath === undefined) + throw new Error("Private root was not acquired"); + await expect(access(privatePath)).rejects.toMatchObject({ code: "ENOENT" }); +}); + +it("cancels after snapshot acquisition before launch and removes private input declarations", async () => { + const root = await mkdtemp(join(tmpdir(), "rea-historical-cancel-")); + onTestFinished(() => rm(root, { recursive: true, force: true })); + const path = join(root, "input.har"); + await writeFile(path, JSON.stringify(historicalHar())); + let privatePath: string | undefined; + const controller = new AbortController(); + const decoder = new HistoricalCaptureDecoder( + [ + { + format: "har", + identity: HAR_CAPTURE_PROVIDER_IDENTITY, + command: async (_, runtimePath) => { + privatePath = runtimePath; + controller.abort(); + return { + command: process.execPath, + arguments: ["-e", "process.exit(3)"], + }; + }, + }, + ], + process.env, + ); + const result = await decoder.inspect( + inspectWebNetworkCaptureInputSchema.parse({ + capture_path: path, + format: "har", + sensitive_values: ["explicit-private-declaration"], + }), + { signal: controller.signal }, + ); + if (result.ok) throw new Error("Cancelled acquisition must fail"); + expect(result.error._tag).toBe("AnalysisCancelledError"); + if (privatePath === undefined) + throw new Error("Private root was not acquired"); + await expect(access(privatePath)).rejects.toMatchObject({ code: "ENOENT" }); +}); + +it.each([ + { reason: "input-limit", tag: "AnalysisInputError" }, + { reason: "resource-limit", tag: "ProviderAdapterError" }, + { reason: "decoder", tag: "ProviderAdapterError" }, + { reason: "limit", tag: "AnalysisOutputError" }, +])( + "preserves $reason decoder failures through the public error boundary", + async ({ reason, tag }) => { + const root = await mkdtemp(join(tmpdir(), "rea-historical-reply-")); + onTestFinished(() => rm(root, { recursive: true, force: true })); + const path = join(root, "input.har"); + await writeFile(path, JSON.stringify(historicalHar())); + let privatePath: string | undefined; + const decoder = new HistoricalCaptureDecoder( + [ + { + format: "har", + identity: HAR_CAPTURE_PROVIDER_IDENTITY, + command: async (_, runtimePath) => { + privatePath = runtimePath; + await writeFile( + join(runtimePath, "reply.json"), + JSON.stringify({ + ok: false, + reason, + message: "Specific decoder constraint", + pointer: "/extension/child", + }), + ); + return { + command: process.execPath, + arguments: ["-e", "process.exit(0)"], + }; + }, + }, + ], + process.env, + ); + const result = await decoder.inspect( + inspectWebNetworkCaptureInputSchema.parse({ + capture_path: path, + format: "har", + }), + ); + if (result.ok) throw new Error("Failure required"); + expect(result.error._tag).toBe(tag); + const projected = projectAnalysisError(result.error); + if (reason === "input-limit") { + expect(projected).toMatchObject({ + category: "invalid_input", + details: { + issues: [ + { + path: ["capture_path", "/extension/child"], + reason: "out_of_range", + expected: { + maximum_capture_nesting: WEB_NETWORK_CAPTURE_LIMITS.depth, + }, + }, + ], + }, + }); + } else if (tag === "ProviderAdapterError") { + expect(projected).toMatchObject({ + details: { + diagnostics: { + phase: "decoder", + reason: "Specific decoder constraint", + failure_kind: reason, + pointer: "/extension/child", + }, + }, + }); + } + if (privatePath === undefined) + throw new Error("Private root was not acquired"); + await expect(access(privatePath)).rejects.toMatchObject({ code: "ENOENT" }); + }, +); + +it.skipIf(process.platform === "win32" || process.getuid?.() === 0)( + "reports a real filesystem permission denial separately from missing capture input", + async () => { + const root = await mkdtemp(join(tmpdir(), "rea-historical-denied-")); + onTestFinished(() => rm(root, { recursive: true, force: true })); + const path = join(root, "unreadable.har"); + await writeFile(path, JSON.stringify(historicalHar()), { mode: 0o000 }); + const decoder = new HistoricalCaptureDecoder( + [ + { + format: "har", + identity: HAR_CAPTURE_PROVIDER_IDENTITY, + command: async () => { + throw new Error("Permission-denied input must not launch decoding"); + }, + }, + ], + process.env, + ); + const result = await decoder.inspect( + inspectWebNetworkCaptureInputSchema.parse({ + capture_path: path, + format: "har", + }), + ); + await chmod(path, 0o600); + if (result.ok) throw new Error("Permission denial required"); + expect(projectAnalysisError(result.error)).toMatchObject({ + code: "access_denied", + category: "unavailable", + retryable: false, + details: { + operation: "inspect_web_network_capture", + path, + system_code: "EACCES", + }, + }); + expect(projectAnalysisError(result.error).remediation.action).toContain( + "read access", + ); + }, +); + +it.each(["format", "process"])( + "preserves the structured %s failure when runtime cleanup fails", + async (kind) => { + const root = await mkdtemp(join(tmpdir(), "rea-historical-cleanup-")); + onTestFinished(() => rm(root, { recursive: true, force: true })); + const path = join(root, "capture-private.har"); + await writeFile(path, JSON.stringify(historicalHar())); + const decoder = new HistoricalCaptureDecoder( + [ + { + format: "har", + identity: HAR_CAPTURE_PROVIDER_IDENTITY, + command: async (_, runtimePath) => { + if (kind === "format") + await writeFile( + join(runtimePath, "reply.json"), + JSON.stringify({ + ok: false, + reason: "format", + message: "Malformed retained record", + pointer: "/metadata/capture-private", + }), + ); + return { + command: process.execPath, + arguments: [ + "-e", + kind === "format" + ? "process.exit(0)" + : 'process.stderr.write("Specific provider fault"); process.exit(3)', + ], + }; + }, + }, + ], + process.env, + async () => { + const runtime = await PrivateRuntimeRoot.create({ + prefix: "rea-cleanup-proof-", + }); + onTestFinished(() => runtime.close()); + return { + path: runtime.path, + close: async () => { + throw new Error("Deliberate cleanup denial"); + }, + }; + }, + ); + const result = await decoder.inspect( + inspectWebNetworkCaptureInputSchema.parse({ + capture_path: path, + format: "har", + sensitive_values: ["capture-private"], + }), + ); + if (result.ok) throw new Error("Cleanup failure required"); + const projected = projectAnalysisError(result.error); + expect(projected).toMatchObject({ + code: "cleanup_incomplete", + details: { + diagnostics: { + previous_error: + kind === "format" + ? { + category: "invalid_input", + details: { + issues: [ + { + path: ["capture_path", "/metadata"], + reason: "invalid_format", + message: "Malformed retained record", + }, + ], + }, + } + : { + code: "execution_failure", + details: { + diagnostics: { + phase: "decoder", + exit_code: 3, + stderr: "Specific provider fault", + }, + }, + }, + }, + }, + }); + expect(JSON.stringify(projected)).not.toContain("capture-private"); + }, +); diff --git a/tests/boundary/browser/historicalWebCapture.test.ts b/tests/boundary/browser/historicalWebCapture.test.ts new file mode 100644 index 000000000..a6905bdb3 --- /dev/null +++ b/tests/boundary/browser/historicalWebCapture.test.ts @@ -0,0 +1,109 @@ +import { createHash } from "node:crypto"; +import { mkdtemp, writeFile, rm } from "node:fs/promises"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; +import { expect, it, onTestFinished } from "vitest"; +import { execFile } from "node:child_process"; +import { fileURLToPath } from "node:url"; +import { promisify } from "node:util"; +import { parseEvidence } from "../../../src/domain/evidence.js"; +const entrypoint = fileURLToPath( + new URL("../../../scripts/rea.mjs", import.meta.url), +); +const cli = async ( + path: string, + format: string, + extra: readonly string[] = [], +) => { + try { + const result = await promisify(execFile)( + process.execPath, + [ + entrypoint, + "inspect-web-network-capture", + path, + format, + "--json", + ...extra, + ], + { env: { ...process.env, REA_MITMDUMP_COMMAND: "" }, timeout: 40_000 }, + ); + return { ok: true, stdout: result.stdout }; + } catch (cause: unknown) { + if ( + cause instanceof Error && + "stdout" in cause && + typeof cause.stdout === "string" + ) + return { ok: false, stdout: cause.stdout }; + throw cause; + } +}; +import { historicalHar } from "../../fixtures/historicalHar.js"; + +it("reads actual upstream HAR packages in an owned process and retains selected producer order", async () => { + const root = await mkdtemp(join(tmpdir(), "rea-history-har-")); + onTestFinished(() => rm(root, { recursive: true, force: true })); + const fixture = historicalHar(); + const first = fixture.log.entries[0]; + if (first === undefined) throw new Error("fixture missing"); + fixture.log.entries.push(structuredClone(first)); + const text = JSON.stringify(fixture); + const path = join(root, "capture.har"); + await writeFile(path, text); + const called = await cli(path, "har", ["--record", "1", "--record", "0"]); + expect(called.ok, called.stdout).toBe(true); + const result = { value: parseEvidence(JSON.parse(called.stdout)) }; + expect(result.value.subject?.digest).toMatchObject({ + sha256: createHash("sha256").update(text).digest("hex"), + }); + expect(result.value.normalized_result).toMatchObject({ + total_records: 2, + records: [{ ordinal: 1 }, { ordinal: 0 }], + runtime_attribution: "unknown", + }); +}); + +it("keeps malformed JSON separate from absent external mitmproxy capability", async () => { + const root = await mkdtemp(join(tmpdir(), "rea-history-errors-")); + onTestFinished(() => rm(root, { recursive: true, force: true })); + const path = join(root, "capture.har"); + await writeFile(path, "{malformed-private-json"); + const malformed = await cli(path, "har"); + expect(malformed.ok).toBe(false); + expect(JSON.parse(malformed.stdout)).toMatchObject({ + category: "invalid_input", + }); + expect(malformed.stdout).not.toContain("malformed-private-json"); + const missing = await cli(path, "mitmproxy"); + expect(missing.ok).toBe(false); + expect(JSON.parse(missing.stdout)).toMatchObject({ + category: "unsupported_provider", + }); +}); + +it("reports the actual HAR decoder nesting constraint as a typed selected-input issue", async () => { + const root = await mkdtemp(join(tmpdir(), "rea-history-depth-")); + onTestFinished(() => rm(root, { recursive: true, force: true })); + let extension: unknown = "leaf"; + for (let depth = 0; depth < 66; depth++) extension = { child: extension }; + const path = join(root, "deep.har"); + await writeFile( + path, + JSON.stringify({ ...historicalHar(), _extension: extension }), + ); + const result = await cli(path, "har"); + expect(result.ok).toBe(false); + expect(JSON.parse(result.stdout)).toMatchObject({ + category: "invalid_input", + details: { + issues: [ + { + path: ["capture_path", expect.stringMatching(/^\/_extension\/child/)], + reason: "out_of_range", + expected: { maximum_capture_nesting: 64 }, + }, + ], + }, + }); +}); diff --git a/tests/boundary/cli/productCatalog.test.ts b/tests/boundary/cli/productCatalog.test.ts index f3a483ecf..d3bbc10de 100644 --- a/tests/boundary/cli/productCatalog.test.ts +++ b/tests/boundary/cli/productCatalog.test.ts @@ -19,6 +19,10 @@ import { MANAGED_WORKFLOW_PROVIDER, } from "../../../src/application/InvestigationProviders.js"; import { CDP_BROWSER_PROVIDER_IDENTITY } from "../../../src/browser/CdpBrowserProvider.js"; +import { + HAR_CAPTURE_PROVIDER_IDENTITY, + MITMPROXY_CAPTURE_PROVIDER_IDENTITY, +} from "../../../src/browser/history/CaptureRelease.js"; import { CDP_ELECTRON_PROVIDER_IDENTITY } from "../../../src/browser/CdpElectronProvider.js"; import { PLAYWRIGHT_ELECTRON_ACTIVE_PROVIDER_IDENTITY } from "../../../src/browser/PlaywrightElectronActiveProvider.js"; import { PLAYWRIGHT_BROWSER_SCENARIO_PROVIDER_IDENTITY } from "../../../src/browser/PlaywrightBrowserScenarioProvider.js"; @@ -106,6 +110,8 @@ describe("canonical product catalog", () => { BINWALK_PROVIDER_IDENTITY, UNBLOB_PROVIDER_IDENTITY, WAKARU_PROVIDER_IDENTITY, + HAR_CAPTURE_PROVIDER_IDENTITY, + MITMPROXY_CAPTURE_PROVIDER_IDENTITY, GHIDRA_PROVIDER_IDENTITY, IDA_PROVIDER_IDENTITY, NATIVE_MACOS_PROVIDER_IDENTITY, diff --git a/tests/boundary/mcp/webNetworkCaptureMcp.test.ts b/tests/boundary/mcp/webNetworkCaptureMcp.test.ts new file mode 100644 index 000000000..602479ea7 --- /dev/null +++ b/tests/boundary/mcp/webNetworkCaptureMcp.test.ts @@ -0,0 +1,91 @@ +import { Client, InMemoryTransport } from "@modelcontextprotocol/client"; +import { Ajv2020 } from "ajv/dist/2020.js"; +import { expect, it, onTestFinished } from "vitest"; +import { WebNetworkCaptureService } from "../../../src/application/WebNetworkCaptureService.js"; +import { toolContract } from "../../../src/contracts/toolContracts.js"; +import { parseEvidence } from "../../../src/domain/evidence.js"; +import { ok } from "../../../src/domain/result.js"; +import { createServer } from "../../../src/server/createServer.js"; +import { createTestBinarySession } from "../../fixtures/binarySession.js"; + +it("publishes valid SDK schemas and retains historical inline Evidence with the supplied session", async () => { + const service = new WebNetworkCaptureService({ + inspect: () => + Promise.resolve( + ok({ + artifact: { path: "/capture.har", sha256: "a".repeat(64), bytes: 10 }, + format: "har", + decoder: { id: "test-capture-port", name: "test port", version: "1" }, + container: { + reported: { creator: "fixture" }, + numeric_literals: [], + redactions: [], + records_pointer: "/log/entries", + }, + total_records: 0, + records: [], + runtime_attribution: "unknown", + limitations: ["synthetic application port, no real decoder claim"], + }), + ), + }); + const session = createTestBinarySession(() => { + throw new Error("Binary provider must not start"); + }); + const server = createServer(session, session, { webNetworkCapture: service }); + const client = new Client({ + name: "historical-capture-contract", + version: "1", + }); + onTestFinished(async () => { + await client.close(); + await server.close(); + await session.close(); + }); + const [clientTransport, serverTransport] = + InMemoryTransport.createLinkedPair(); + await server.connect(serverTransport); + await client.connect(clientTransport); + const advertised = (await client.listTools()).tools.find( + (tool) => tool.name === "inspect_web_network_capture", + ); + if (advertised?.outputSchema === undefined) + throw new Error("Capture inspection must advertise both schemas"); + const ajv = new Ajv2020({ strict: false, validateFormats: false }); + const inputSchema: Record = advertised.inputSchema; + const outputSchema: Record = advertised.outputSchema; + expect(ajv.validateSchema(inputSchema)).toBe(true); + expect(ajv.validateSchema(outputSchema)).toBe(true); + const args = { capture_path: "/capture.har", format: "har" }; + expect(ajv.validate(inputSchema, args)).toBe(true); + const response = await client.callTool({ + name: "inspect_web_network_capture", + arguments: args, + }); + expect(response.isError).not.toBe(true); + expect( + ajv.validate(outputSchema, response.structuredContent), + JSON.stringify(ajv.errors), + ).toBe(true); + const parsed = toolContract("inspect_web_network_capture").outputSchema.parse( + response.structuredContent, + ); + const evidence = parseEvidence(parsed.evidence); + expect(parsed.result).toEqual(evidence.normalized_result); + expect(session.evidenceById(evidence.evidence_id)).toEqual(evidence); + const invalid = await client.callTool({ + name: "inspect_web_network_capture", + arguments: { ...args, capture_path: "relative.har" }, + }); + expect(invalid.isError).toBe(true); + const invalidSensitive = await client.callTool({ + name: "inspect_web_network_capture", + arguments: { ...args, sensitive_values: ["secret"], mysecret: true }, + }); + expect(invalidSensitive.isError).toBe(true); + expect(invalidSensitive.structuredContent).toMatchObject({ + error: { code: "invalid_request", details: { issues: [{ path: [] }] } }, + }); + expect(JSON.stringify(invalidSensitive)).not.toContain("secret"); + expect(ajv.validate(inputSchema, { ...args, mysecret: true })).toBe(false); +}); diff --git a/tests/boundary/process/ownedCommand.test.ts b/tests/boundary/process/ownedCommand.test.ts new file mode 100644 index 000000000..c40e5fd57 --- /dev/null +++ b/tests/boundary/process/ownedCommand.test.ts @@ -0,0 +1,186 @@ +import { randomUUID } from "node:crypto"; +import { setTimeout as delay } from "node:timers/promises"; +import { expect, it } from "vitest"; +import { + OwnedCommandFailure, + runOwnedCommand, +} from "../../../src/process/OwnedCommand.js"; +import { spawnOwnedProviderProcess } from "../../../src/process/ProviderProcess.js"; +import { cleanupOwnedProcessGroup } from "../../../src/process/ProcessOwnership.js"; +import { waitForProviderProcessReady } from "../../fixtures/providerProcess.js"; + +const command = (script: string) => ({ + command: process.execPath, + arguments: ["-e", script], + runId: `rea-owned-command-test-${randomUUID()}`, +}); + +it("includes ownership preparation in the command deadline before provider creation", async () => { + let prepared = false; + await expect( + runOwnedCommand( + command("process.exit(0)"), + { + timeoutMs: 20, + diagnosticBytes: 1024, + }, + { + launcher: async (input) => { + await delay(80, undefined, { signal: input.signal }); + prepared = true; + throw new Error( + "Slow preparation must be aborted before provider creation", + ); + }, + }, + ), + ).rejects.toMatchObject({ reason: "timeout", snapshot: null }); + expect(prepared).toBe(false); +}); + +it("passes cancellation into native ownership preparation before provider creation", async () => { + const controller = new AbortController(); + await expect( + runOwnedCommand( + command("process.exit(0)"), + { timeoutMs: 2000, diagnosticBytes: 1024 }, + { + signal: controller.signal, + launcher: async (input) => { + expect(input.signal?.aborted).toBe(false); + controller.abort(); + input.signal?.throwIfAborted(); + throw new Error("Cancelled preparation must not launch a provider"); + }, + }, + ), + ).rejects.toMatchObject({ reason: "cancelled", snapshot: null }); +}); + +it("bounds retained diagnostics across both streams during a real output burst", async () => { + try { + await runOwnedCommand( + command( + 'process.stdout.write("a".repeat(256 * 1024)); process.stderr.write("b".repeat(256 * 1024))', + ), + { timeoutMs: 2_000, diagnosticBytes: 1024 }, + ); + throw new Error("An oversized producer must fail"); + } catch (cause: unknown) { + if (!(cause instanceof OwnedCommandFailure)) throw cause; + expect(cause.reason).toBe("output-limit"); + expect(cause.snapshot).not.toBeNull(); + expect( + (cause.snapshot?.stdout.bytes ?? 0) + (cause.snapshot?.stderr.bytes ?? 0), + ).toBeLessThanOrEqual(1024); + } +}); + +it("collects a real owned command and verifies its exit before returning diagnostics", async () => { + const result = await runOwnedCommand( + command('process.stdout.write("observed-output")'), + { timeoutMs: 2_000, diagnosticBytes: 1024 }, + ); + expect(result).toMatchObject({ + exitCode: 0, + signal: null, + stdout: { text: "observed-output", bytes: 15 }, + }); +}); + +it("cancels a real acquired process and independently releases it", async () => { + const controller = new AbortController(); + let pid: number | undefined; + const result = runOwnedCommand( + command( + 'process.stdout.write("ready\\n"); setInterval(() => undefined, 1000)', + ), + { timeoutMs: 5_000, diagnosticBytes: 1024 }, + { + signal: controller.signal, + launcher: async (input) => { + const process = await spawnOwnedProviderProcess(input); + pid = process.process.pid; + await waitForProviderProcessReady(process.process); + setImmediate(() => controller.abort()); + return process; + }, + }, + ); + await expect(result).rejects.toMatchObject({ + reason: "cancelled", + cleanupFailure: null, + }); + if (pid === undefined) throw new Error("Process was not acquired"); + const acquiredPid = pid; + expect(() => process.kill(acquiredPid, 0)).toThrow(); +}); + +it("returns a timeout for a running process after independent cleanup", async () => { + await expect( + runOwnedCommand(command("setInterval(() => undefined, 1000)"), { + timeoutMs: 50, + diagnosticBytes: 1024, + }), + ).rejects.toMatchObject({ reason: "timeout", cleanupFailure: null }); +}); + +it("honors cancellation during cleanup after a successful command", async () => { + const controller = new AbortController(); + await expect( + runOwnedCommand( + command("process.exit(0)"), + { timeoutMs: 2_000, diagnosticBytes: 1024 }, + { + signal: controller.signal, + launcher: async (input) => { + const launched = await spawnOwnedProviderProcess(input); + return { + ...launched, + cleanup: async () => { + const result = await cleanupOwnedProcessGroup(launched.ownership); + controller.abort(); + return result; + }, + }; + }, + }, + ), + ).rejects.toMatchObject({ reason: "cancelled", cleanupFailure: null }); +}); + +it("rejects oversized diagnostics even when the child exits immediately", async () => { + await expect( + runOwnedCommand(command('process.stdout.write("x".repeat(2048))'), { + timeoutMs: 2_000, + diagnosticBytes: 1024, + }), + ).rejects.toMatchObject({ reason: "output-limit", cleanupFailure: null }); +}); + +it("preserves original command failure together with failed cleanup resource identities", async () => { + const request = command("process.exit(2)"); + await expect( + runOwnedCommand( + request, + { timeoutMs: 2_000, diagnosticBytes: 1024 }, + { + launcher: async (input) => { + const process = await spawnOwnedProviderProcess(input); + return { + ...process, + cleanup: async () => ({ + cleaned: false as const, + reason: "synthetic cleanup could not be confirmed", + }), + }; + }, + }, + ), + ).rejects.toMatchObject({ + reason: "process", + cleanupFailure: "synthetic cleanup could not be confirmed", + resources: expect.arrayContaining([request.runId]), + snapshot: { exitCode: 2 }, + }); +}); diff --git a/tests/boundary/process/processCaptureNormalization.test.ts b/tests/boundary/process/processCaptureNormalization.test.ts index 23730e6dc..dbaef7c28 100644 --- a/tests/boundary/process/processCaptureNormalization.test.ts +++ b/tests/boundary/process/processCaptureNormalization.test.ts @@ -41,7 +41,6 @@ itWithLinuxCaptureCapability.each([false, true])( idle_timeout_ms: 10_000, }); const result = await captureProcessScenario(scenario); - expect(result.ok).toBe(true); if (!result.ok) throw result.error; const capture = result.value; const identity = identitySchema.parse( diff --git a/tests/boundary/process/providerProcessOutput.test.ts b/tests/boundary/process/providerProcessOutput.test.ts new file mode 100644 index 000000000..888394c99 --- /dev/null +++ b/tests/boundary/process/providerProcessOutput.test.ts @@ -0,0 +1,42 @@ +import { expect, it } from "vitest"; +import { + ProviderProcessSupervisor, + type ProviderProcessDiagnostic, +} from "../../../src/process/ProviderProcess.js"; +import { + spawnProviderProcessFixture, + stopProviderProcessFixture, +} from "../../fixtures/providerProcess.js"; + +it("bounds capture before diagnostic callbacks while draining both output streams", async () => { + const child = spawnProviderProcessFixture("burst", 70_000); + const diagnostics: ProviderProcessDiagnostic[] = []; + const supervisor = new ProviderProcessSupervisor( + { process: child, ownsProcessLifetime: true }, + { + maxDiagnosticBytes: 1024, + onDiagnostic: (event) => { + const snapshot = supervisor.snapshot(); + expect( + snapshot.stdout.bytes + snapshot.stderr.bytes, + ).toBeLessThanOrEqual(1024); + diagnostics.push(event); + }, + }, + ); + try { + expect(await supervisor.waitForOutputClose(2_000)).toBe(true); + expect( + diagnostics.some((event) => event.type === "output" && event.truncated), + ).toBe(true); + for (const stream of ["stdout", "stderr"]) + expect( + diagnostics + .filter((event) => event.type === "output" && event.stream === stream) + .at(-1), + ).toMatchObject({ totalBytes: 70_000 }); + } finally { + await supervisor.stop(); + await stopProviderProcessFixture(child); + } +}); diff --git a/tests/boundary/process/selectedExecutableCancellation.test.ts b/tests/boundary/process/selectedExecutableCancellation.test.ts new file mode 100644 index 000000000..238737a58 --- /dev/null +++ b/tests/boundary/process/selectedExecutableCancellation.test.ts @@ -0,0 +1,22 @@ +import { open, stat, type FileHandle } from "node:fs/promises"; +import { expect, it } from "vitest"; +import { observeSelectedExecutable } from "../../../src/process/capture/ProcessCaptureLifecycle.js"; + +it("cancels after real metadata IO without creating an already-aborted stream or leaking the file handle", async () => { + const controller = new AbortController(); + let opened: FileHandle | undefined; + await expect( + observeSelectedExecutable(process.execPath, controller.signal, { + async open(path, flags) { + opened = await open(path, flags); + return opened; + }, + async stat(path) { + const metadata = await stat(path, { bigint: true }); + controller.abort(); + return metadata; + }, + }), + ).rejects.toMatchObject({ reason: "cancelled" }); + expect(opened?.fd).toBe(-1); +}); diff --git a/tests/fixtures/historicalHar.ts b/tests/fixtures/historicalHar.ts new file mode 100644 index 000000000..1c2542105 --- /dev/null +++ b/tests/fixtures/historicalHar.ts @@ -0,0 +1,36 @@ +/** Synthetic HAR 1.2 fixture with producer-declared sizes independent of Unicode content. */ +export const historicalHar = () => ({ + log: { + version: "1.2", + creator: { name: "rea-source-owned-fixture", version: "1" }, + entries: [ + { + startedDateTime: "2026-10-07T13:00:00.123+08:00", + time: 1.25, + request: { + method: "GET", + url: "https://example.test/a?token=ordinary#fragment", + httpVersion: "HTTP/2", + cookies: [], + headers: [], + queryString: [], + headersSize: -1, + bodySize: 0, + }, + response: { + status: 200, + statusText: "OK", + httpVersion: "HTTP/2", + cookies: [], + headers: [], + content: { size: 99, mimeType: "text/plain", text: "雪😀\r\n" }, + redirectURL: "", + headersSize: -1, + bodySize: 99, + }, + cache: {}, + timings: { send: 0, wait: 1.25, receive: 0 }, + }, + ], + }, +}); diff --git a/third_party/web-network-captures/README.md b/third_party/web-network-captures/README.md new file mode 100644 index 000000000..49d0bc110 --- /dev/null +++ b/third_party/web-network-captures/README.md @@ -0,0 +1,36 @@ +# Historical network upstream profiles + +REA uses unmodified upstream packages or a caller-supplied tool. The adapter +code lives under `src/browser/history/` and `bridge/mitmproxy/`; this directory +records provenance rather than vendoring a second parser implementation. + +| Upstream | Fixed profile | Use | +| --- | --- | --- | +| [har-schema](https://github.com/ahmadnassri/har-schema) | npm 2.0.0, ISC | HAR 1.2 draft-06 schemas | +| [jsonc-parser](https://github.com/microsoft/node-jsonc-parser) | npm 3.3.1, MIT | Strict JSON visitor; member sequence, duplicate detection and prototype-safe materialization | +| [lossless-json](https://github.com/josdejong/lossless-json) | npm 4.3.1 | Original JSON numeric lexemes and safe-number classification | +| [AJV](https://github.com/ajv-validator/ajv) | npm 8.20.0 | Upstream schema validation | +| [ajv-formats](https://github.com/ajv-validator/ajv-formats) | npm 3.0.1 | Upstream URI, timestamp and address formats | +| [mitmproxy](https://github.com/mitmproxy/mitmproxy) | v12.2.3, commit `6c09d56e4c29a92f5ad01b03199977584b8ea14f`, MIT | Unchanged native `mitmproxy.io.tnetstring.load`; upstream `FlowWriter` / `SaveHar` for ephemeral verification fixtures | + +The npm lockfile pins exact package versions and distribution integrity hashes; +their original code and license files remain in the installed packages. Native +mitmproxy is bring-your-own. REA never installs or upgrades it and rejects a +different runtime version. The Linux verification lane downloads the official +standalone release into runner temporary storage with fixed archive SHA-256 +`2e95286b618fa6fd33e5e62a78c2e5112571d85f42ec2bac29b97ee242bdb5c5`. +This is a pinned archive digest, not a publisher signature claim. + +No native captures or game/application binaries are checked in. Verification +uses source-owned Python fixtures and upstream writers. The original file +digest/record representation is preserved independently of the decoded view; +upstream conversion defaults are not imported into native states. + +The observed `SaveHar` 12.2.3 optional `postData.text: null` quirk is admitted +only in a validation copy for that declared producer profile, with the original +null retained. The upstream schemas themselves remain unchanged. + +The strict JSON visitor supplies member events before object materialization; +REA rejects all duplicate names and retains original numeric token spelling. +The shared JSON schema currently drops `__proto__` members, so both adapters +report that representation as unsupported instead of returning incomplete data.