From 2e2f981ccfe0e675f561949eb7eb9aca0346f0f6 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Sat, 7 Nov 2026 22:46:17 +0800 Subject: [PATCH] test: assert absolute-path facts without frozen sentences (#9231) * test: assert absolute-path facts without frozen sentences Sibling path tests now require the field path, invalid_format, and regex expectation, and they record that each negative case ran. NUL rejection follows the argument that contains the byte instead of searching for the word NUL. * test: retain actionable NUL path diagnostics --------- Co-authored-by: Cursor Agent --- src/cliJsonPaths.test.ts | 2 +- src/domain/localPath.test.ts | 21 ++++- .../applications/electronInvalidPaths.test.ts | 6 +- .../tools/localToolContracts.test.ts | 78 +++++++++++++++++-- .../mcp/absoluteLocalPathSchema.test.ts | 66 ++++++++++++---- tests/boundary/mcp/sessionMcp.test.ts | 30 ++++++- tests/boundary/mcp/wasmArtifactMcp.test.ts | 36 +++++---- 7 files changed, 194 insertions(+), 45 deletions(-) diff --git a/src/cliJsonPaths.test.ts b/src/cliJsonPaths.test.ts index 56d881cbf9..12e781b252 100644 --- a/src/cliJsonPaths.test.ts +++ b/src/cliJsonPaths.test.ts @@ -123,7 +123,7 @@ describe("CLI JSON path normalization", () => { expect.objectContaining({ path: ["executable_path"], code: "invalid_format", - message: expect.stringContaining("absolute local filesystem path"), + format: "regex", }), ]); }); diff --git a/src/domain/localPath.test.ts b/src/domain/localPath.test.ts index 0cc8721870..073f6cc74f 100644 --- a/src/domain/localPath.test.ts +++ b/src/domain/localPath.test.ts @@ -21,11 +21,20 @@ const PATHS = [ ]; it("matches the host path module for absolute local paths", () => { + const checked: string[] = []; for (const path of PATHS) { + checked.push(path); expect(absoluteLocalPathStringSchema.safeParse(path).success).toBe( isAbsolute(path), ); } + expect(checked).toEqual( + expect.arrayContaining([ + "/absolute/path", + "relative/path", + "file:///absolute/path", + ]), + ); expect(absoluteLocalPathStringSchema.safeParse("").success).toBe(false); expect( absoluteLocalPathStringSchema.safeParse("/absolute/with\0nul").success, @@ -40,8 +49,18 @@ it("keeps the Windows absolute-path pattern valid in every client mode", () => { () => new RegExp(windowsAbsolutePathPattern.source, flags), flags, ).not.toThrow(); - for (const path of PATHS) + const checked: string[] = []; + for (const path of PATHS) { + checked.push(path); expect(windowsAbsolutePathPattern.test(path), path).toBe( win32.isAbsolute(path), ); + } + expect(checked).toEqual( + expect.arrayContaining([ + "C:\\absolute\\path", + "C:relative/path", + "/absolute/path", + ]), + ); }); diff --git a/tests/acceptance/applications/electronInvalidPaths.test.ts b/tests/acceptance/applications/electronInvalidPaths.test.ts index 41031746c3..31724ce058 100644 --- a/tests/acceptance/applications/electronInvalidPaths.test.ts +++ b/tests/acceptance/applications/electronInvalidPaths.test.ts @@ -255,11 +255,11 @@ cliTest( code: "invalid_request", details: { issues: [ - { + expect.objectContaining({ path: ["executable_path"], reason: "invalid_format", - message: expect.stringContaining("absolute local filesystem path"), - }, + expected: "regex", + }), ], }, }); diff --git a/tests/acceptance/tools/localToolContracts.test.ts b/tests/acceptance/tools/localToolContracts.test.ts index 5389ab81ea..61276ac348 100644 --- a/tests/acceptance/tools/localToolContracts.test.ts +++ b/tests/acceptance/tools/localToolContracts.test.ts @@ -4,6 +4,7 @@ import { join } from "node:path"; import { fileURLToPath } from "node:url"; import { promisify } from "node:util"; import { expect, it } from "vitest"; +import { z } from "zod"; import { toolContract } from "../../../src/contracts/toolContracts.js"; import { createEvidence, parseEvidence } from "../../../src/domain/evidence.js"; @@ -329,21 +330,84 @@ const pathRequests: readonly [string, Record][] = [ ["start_gdb_session", { inferior_tty: nul }], ]; +const nulIssuePaths = ( + value: unknown, + path: readonly (string | number)[] = [], +): (string | number)[][] => { + if (typeof value === "string") return value.includes("\0") ? [[...path]] : []; + if (Array.isArray(value)) + return value.flatMap((item, index) => + nulIssuePaths(item, [...path, index]), + ); + if (typeof value === "object" && value !== null) + return Object.entries(value).flatMap(([key, item]) => + nulIssuePaths(item, [...path, key]), + ); + return []; +}; + it("rejects OS-invalid local inputs through the real advertised MCP contracts", async () => { const { call } = await connectLocalToolsMcp(); + const checked: string[] = []; for (const [name, arguments_] of pathRequests) { - const response = await call(name, arguments_); - expect(response.isError, name).toBe(true); - expect(JSON.stringify(response.content), name).toContain("NUL"); + const paths = nulIssuePaths(arguments_); + expect(paths.length, name).toBeGreaterThan(0); + const error = parseMcpToolError(await call(name, arguments_)).error; + expect(error, name).toMatchObject({ + code: "invalid_request", + details: { operation: name }, + }); + const issues = z + .object({ + issues: z.array( + z.object({ + path: z.array(z.union([z.string(), z.number()])), + reason: z.string(), + message: z.string(), + expected: z.unknown().optional(), + }), + ), + }) + .parse(error.details).issues; + expect(issues, name).toEqual( + expect.arrayContaining( + paths.map((path) => + expect.objectContaining({ + path, + reason: "invalid_format", + expected: "regex", + message: expect.stringMatching(/\bNUL\b|null byte|zero byte/i), + }), + ), + ), + ); + checked.push(name); } + expect(checked).toEqual( + expect.arrayContaining([ + "extract_firmware", + "inspect_wasm_artifact", + "demangle_swift", + "capture_electron_scenario", + ]), + ); const relative = await call("extract_firmware", { path: "/tmp/firmware.bin", output_directory: "relative-output", }); - expect(relative.isError).toBe(true); - expect(JSON.stringify(relative.content)).toContain( - "absolute local filesystem path", - ); + expect(parseMcpToolError(relative).error).toMatchObject({ + code: "invalid_request", + details: { + operation: "extract_firmware", + issues: [ + expect.objectContaining({ + path: ["output_directory"], + reason: "invalid_format", + expected: "regex", + }), + ], + }, + }); }, 60_000); const ignoredOptions = [ diff --git a/tests/boundary/mcp/absoluteLocalPathSchema.test.ts b/tests/boundary/mcp/absoluteLocalPathSchema.test.ts index c935805e7a..329daf21c8 100644 --- a/tests/boundary/mcp/absoluteLocalPathSchema.test.ts +++ b/tests/boundary/mcp/absoluteLocalPathSchema.test.ts @@ -13,7 +13,7 @@ interface PathInputCase { readonly field: string; readonly valid: Readonly>; readonly invalid: Readonly>; - readonly issuePath?: readonly string[]; + readonly issuePath: readonly string[]; } const createMcp = async ( @@ -104,7 +104,13 @@ const inputCases = ( field: string, valid: Readonly>, invalid: Readonly>, - ): PathInputCase => ({ name, field, valid, invalid }); + ): PathInputCase => ({ + name, + field, + valid, + invalid, + issuePath: field.split("."), + }); return [ pathCase("open_binary", "path", { path: absolute }, { path: relative }), @@ -246,8 +252,7 @@ it("advertises full host-absolute path constraints after SDK schema conversion", const ajv = new Ajv2020({ strict: false, validateFormats: false }); const absolute = resolve("rea-absolute-input"); const relative = "relative-input"; - - for (const { name, field, valid, invalid } of [ + const schemaCases = [ ...inputCases(absolute, relative), { name: "inspect_go_binary", @@ -276,7 +281,9 @@ it("advertises full host-absolute path constraints after SDK schema conversion", right_bundle_path: absolute, }, }, - ]) { + ]; + const checked: string[] = []; + for (const { name, field, valid, invalid } of schemaCases) { const schema = listed.tools.find( (tool) => tool.name === name, )?.inputSchema; @@ -288,7 +295,16 @@ it("advertises full host-absolute path constraints after SDK schema conversion", validate(valid), `${name}.${field}: ${JSON.stringify(validate.errors)}`, ).toBe(true); + checked.push(`${name}.${field}`); } + expect(checked).toEqual( + expect.arrayContaining([ + "open_binary.path", + "extract_firmware.output_directory", + "inspect_go_binary.path NUL", + "compare_bundles.left_bundle_path empty", + ]), + ); expect(firmwareCalls).toEqual([]); } finally { await close(); @@ -299,19 +315,39 @@ it("rejects relative paths through in-memory MCP handlers", async () => { const firmwareCalls: string[] = []; const { client, close } = await createMcp(firmwareCalls); try { - for (const { name, field, invalid, issuePath } of inputCases( - resolve("rea-absolute-input"), - "relative-input", - )) { + const cases = inputCases(resolve("rea-absolute-input"), "relative-input"); + const checked: string[] = []; + for (const { name, field, invalid, issuePath } of cases) { const result = await client.callTool({ name, arguments: invalid }); expect(result.isError, `${name}.${field}`).toBe(true); - expect(JSON.stringify(result), `${name}.${field}`).toContain("absolute"); - if (issuePath !== undefined) - expect(parseMcpToolError(result).error.details).toMatchObject({ - operation: name, - issues: [{ path: issuePath, reason: "invalid_format" }], - }); + expect(parseMcpToolError(result).error, `${name}.${field}`).toMatchObject( + { + code: "invalid_request", + details: { + operation: name, + issues: [ + expect.objectContaining({ + path: issuePath, + reason: "invalid_format", + expected: "regex", + }), + ], + }, + }, + ); + checked.push(`${name}.${field}`); } + expect(checked).toEqual( + expect.arrayContaining([ + "open_binary.path", + "open_binary.snapshot_path", + "extract_firmware.path", + "extract_firmware.output_directory", + "inspect_analysis_view.source.path", + "capture_browser_scenario.browser.executable_path", + "capture_electron_scenario.application_root", + ]), + ); expect(firmwareCalls).toEqual([]); } finally { await close(); diff --git a/tests/boundary/mcp/sessionMcp.test.ts b/tests/boundary/mcp/sessionMcp.test.ts index 5001a9ae23..5fa38952e7 100644 --- a/tests/boundary/mcp/sessionMcp.test.ts +++ b/tests/boundary/mcp/sessionMcp.test.ts @@ -709,8 +709,19 @@ describe("session filesystem path boundaries over MCP", () => { name: "export_evidence_bundle", arguments: { path: "relative-bundle.json" }, }); - expect(exported.isError, JSON.stringify(exported.content)).toBe(true); - expect(JSON.stringify(exported.content)).toContain("absolute"); + expect(parseMcpToolError(exported).error).toMatchObject({ + code: "invalid_request", + details: { + operation: "export_evidence_bundle", + issues: [ + expect.objectContaining({ + path: ["path"], + reason: "invalid_format", + expected: "regex", + }), + ], + }, + }); const opened = await mcp.callTool({ name: "open_binary", @@ -721,8 +732,19 @@ describe("session filesystem path boundaries over MCP", () => { name: "close_binary", arguments: { snapshot_path: "relative-analysis.json" }, }); - expect(closed.isError, JSON.stringify(closed.content)).toBe(true); - expect(JSON.stringify(closed.content)).toContain("absolute"); + expect(parseMcpToolError(closed).error).toMatchObject({ + code: "invalid_request", + details: { + operation: "close_binary", + issues: [ + expect.objectContaining({ + path: ["snapshot_path"], + reason: "invalid_format", + expected: "regex", + }), + ], + }, + }); await mcp.callTool({ name: "close_binary", arguments: {} }); }, 10_000); }); diff --git a/tests/boundary/mcp/wasmArtifactMcp.test.ts b/tests/boundary/mcp/wasmArtifactMcp.test.ts index 0af626a51a..dede3d394f 100644 --- a/tests/boundary/mcp/wasmArtifactMcp.test.ts +++ b/tests/boundary/mcp/wasmArtifactMcp.test.ts @@ -383,7 +383,7 @@ it("attributes rejected WASM paths to their selected fields before provider work }, }); onTestFinished(harness.close); - for (const [input, paths] of [ + const relativeCases = [ [{ path: "relative.wasm" }, [["path"]]], [ { path: value.artifact.path, glue_paths: ["relative.js"] }, @@ -412,7 +412,9 @@ it("attributes rejected WASM paths to their selected fields before provider work }, [["path"], ["glue_paths", 0], ["candidate_paths", 0]], ], - ] as const) { + ] as const; + const checked: string[] = []; + for (const [input, paths] of relativeCases) { const response = await harness.client.callTool({ name: "inspect_wasm_artifact", arguments: input, @@ -423,14 +425,19 @@ it("attributes rejected WASM paths to their selected fields before provider work .object({ issues: z.array(z.unknown()) }) .parse(failure.details); expect(issues).toEqual( - paths.map((path) => ({ - path, - reason: "invalid_format", - expected: "regex", - message: "must be an absolute local filesystem path", - })), + paths.map((path) => + expect.objectContaining({ + path, + reason: "invalid_format", + expected: "regex", + }), + ), ); + checked.push(paths.map((path) => path.join(".")).join("+")); } + expect(checked).toEqual( + expect.arrayContaining(["path", "glue_paths.0", "candidate_paths.0"]), + ); expect(inspect).not.toHaveBeenCalled(); const valid = await harness.client.callTool({ name: "inspect_wasm_artifact", @@ -760,12 +767,13 @@ it("rejects malformed paths, refuses WASM summary delivery before provider work expect( z.object({ issues: z.array(z.unknown()) }).parse(rejected.details).issues, ).toEqual( - [["path"], ["glue_paths", 0]].map((path) => ({ - path, - reason: "invalid_format", - expected: "regex", - message: "must be an absolute local filesystem path", - })), + [["path"], ["glue_paths", 0]].map((path) => + expect.objectContaining({ + path, + reason: "invalid_format", + expected: "regex", + }), + ), ); expect(inspect).not.toHaveBeenCalled(); for (const input of [