rea/tests/boundary/mcp/processComparisonDiagnosticsMcp.test.ts

import { parseMcpToolError } from "../../fixtures/mcpToolError.js";
import { writeFile } from "node:fs/promises";
import { join } from "node:path";

import { Client, InMemoryTransport } from "@modelcontextprotocol/client";
import { expect, it, onTestFinished } from "vitest";

import { compareProcessEvidenceFiles } from "../../../src/application/process/ProcessCli.js";
import { PROCESS_PROVIDER } from "../../../src/domain/process/processEvidenceProvider.js";
import { EMPTY_PROCESS_CAPTURE_EXAMPLE } from "../../../src/domain/process/processCaptureExample.js";
import {
  createEvidence,
  parseEvidence,
  type EvidenceProvider,
} from "../../../src/domain/evidence.js";
import { jsonValueSchema } from "../../../src/domain/jsonValue.js";
import { digestProcessCommitment } from "../../../src/domain/process/processScenario.js";
import { createServer } from "../../../src/server/createServer.js";
import { createTestBinarySession } from "../../fixtures/binarySession.js";
import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js";

const captureEvidence = (
  side: "left" | "right",
  comparisonContract: Readonly<Record<string, string>> = {},
  provider: EvidenceProvider = PROCESS_PROVIDER,
) =>
  createEvidence(undefined, provider, {
    predicateType: "rea.process-capture",
    operation: "capture_process_scenario",
    parameters: { side },
    result: jsonValueSchema.parse({
      ...EMPTY_PROCESS_CAPTURE_EXAMPLE,
      manifest: {
        ...EMPTY_PROCESS_CAPTURE_EXAMPLE.manifest,
        comparison_contract: comparisonContract,
        comparison_contract_sha256: digestProcessCommitment(comparisonContract),
      },
    }),
    confidence: "observed",
    authority: "controlled-replay",
  });

const left = captureEvidence("left", { working_directory: "/work/reference" });
const right = captureEvidence("right", {
  working_directory: "/work/reconstruction",
});

const connect = async () => {
  const session = createTestBinarySession(() => {
    throw new Error("Process comparison must not launch a provider");
  });
  const server = createServer({ kind: "session", session });
  const client = new Client({ name: "process-comparison-test", version: "1" });
  onTestFinished(async () => {
    await Promise.allSettled([client.close(), server.close(), session.close()]);
  });
  const [clientTransport, serverTransport] =
    InMemoryTransport.createLinkedPair();
  await server.connect(serverTransport);
  await client.connect(clientTransport);
  return client;
};

const compareViaCli = async (
  left: ReturnType<typeof captureEvidence>,
  right: ReturnType<typeof captureEvidence>,
) => {
  const root = await createTestTempDirectory("rea-process-provider-");
  const leftPath = join(root, "left.json");
  const rightPath = join(root, "right.json");
  await Promise.all([
    writeFile(leftPath, JSON.stringify(left)),
    writeFile(rightPath, JSON.stringify(right)),
  ]);
  return compareProcessEvidenceFiles(leftPath, rightPath);
};

it.each(
  (["left", "right"] as const).flatMap((side) =>
    [
      { id: "unrelated-provider" },
      { version: "1" },
      { name: "unrelated provider" },
    ].map((change) => ({ side, change })),
  ),
)(
  "rejects an authenticated $side capture with provider $change",
  async ({ side, change }) => {
    const wrongProvider = captureEvidence(
      side,
      {},
      { ...PROCESS_PROVIDER, ...change },
    );
    expect(parseEvidence(wrongProvider)).toEqual(wrongProvider);
    const left = side === "left" ? wrongProvider : captureEvidence("left");
    const right = side === "right" ? wrongProvider : captureEvidence("right");
    const issue = {
      path: [side, "provider"],
      reason: "invalid_value",
      message: expect.stringContaining("process capture provider"),
    };
    expect(await compareViaCli(left, right)).toMatchObject({
      category: "invalid_input",
      details: { operation: "compare_process_captures", issues: [issue] },
    });
    const client = await connect();
    const response = await client.callTool({
      name: "compare_process_captures",
      arguments: { left, right },
    });
    expect(response.isError).toBe(true);
    expect(parseMcpToolError(response)).toMatchObject({
      error: { code: "invalid_request", details: { issues: [issue] } },
    });
    await client.ping();
  },
);

it.each([PROCESS_PROVIDER.name, "REA deterministic process harness"])(
  "preserves authenticated CLI/MCP comparison Evidence for provider name %s",
  async (name) => {
    const provider = { ...PROCESS_PROVIDER, name };
    const left = captureEvidence("left", {}, provider);
    const right = captureEvidence("right", {}, provider);
    const cli = parseEvidence(await compareViaCli(left, right));
    const client = await connect();
    const response = await client.callTool({
      name: "compare_process_captures",
      arguments: { left, right },
    });
    expect(response.isError).not.toBe(true);
    expect(parseEvidence(response.structuredContent)).toEqual(cli);
    const text = response.content.find((block) => block.type === "text");
    expect(text?.type).toBe("text");
    if (text?.type !== "text") throw new Error("Missing comparison text");
    expect(JSON.parse(text.text)).toEqual(cli);
    expect(cli.evidence_links).toEqual([left.evidence_id, right.evidence_id]);
  },
);

it.each(["left", "right"] as const)(
  "authenticates the %s Evidence before provider admission",
  async (side) => {
    const mismatched = captureEvidence(
      side,
      {},
      { ...PROCESS_PROVIDER, id: "unrelated-provider" },
    );
    const tampered = { ...mismatched, parameters: { side: "tampered" } };
    const left = side === "left" ? tampered : captureEvidence("left");
    const right = side === "right" ? tampered : captureEvidence("right");
    expect(await compareViaCli(left, right)).toMatchObject({
      category: "invalid_input",
      details: {
        issues: [
          {
            path: ["evidence_id"],
            message: expect.stringContaining(
              "Evidence semantic identifier does not match its record",
            ),
          },
        ],
      },
    });
    const client = await connect();
    const response = await client.callTool({
      name: "compare_process_captures",
      arguments: { left, right },
    });
    expect(parseMcpToolError(response)).toMatchObject({
      error: { code: "evidence_integrity_mismatch" },
    });
  },
);

it("names differing comparison contract fields through MCP and the CLI", async () => {
  const client = await connect();
  const response = await client.callTool({
    name: "compare_process_captures",
    arguments: { left, right },
  });
  expect(response.isError).toBe(true);
  const issue = {
    path: ["right"],
    reason: "invalid_value",
    message: expect.stringContaining(
      "these scenario fields differ: working_directory",
    ),
    expected: ["working_directory"],
  };
  expect(parseMcpToolError(response)).toMatchObject({
    error: { code: "invalid_request", details: { issues: [issue] } },
  });

  const root = await createTestTempDirectory("rea-process-contract-");
  const leftPath = join(root, "left.json");
  const rightPath = join(root, "right.json");
  await writeFile(leftPath, JSON.stringify(left));
  await writeFile(rightPath, JSON.stringify(right));
  expect(await compareProcessEvidenceFiles(leftPath, rightPath)).toMatchObject({
    error: "Process command failed",
    code: "invalid_request",
    details: { operation: "compare_process_captures", issues: [issue] },
  });
});

it("names the stale capture that exceeds max_capture_age_ms", async () => {
  const client = await connect();
  const response = await client.callTool({
    name: "compare_process_captures",
    arguments: {
      left: captureEvidence("left"),
      right: captureEvidence("right"),
      max_capture_age_ms: 1000,
    },
  });
  expect(response.isError).toBe(true);
  expect(parseMcpToolError(response)).toMatchObject({
    error: {
      code: "invalid_request",
      details: {
        issues: [
          {
            path: ["max_capture_age_ms"],
            reason: "out_of_range",
            message: expect.stringContaining(
              `left completed at ${EMPTY_PROCESS_CAPTURE_EXAMPLE.manifest.completed_at}`,
            ),
          },
        ],
      },
    },
  });
});

it("reports CLI trace specification issues like the MCP trace_spec input", async () => {
  const root = await createTestTempDirectory("rea-process-trace-spec-");
  const capturePath = join(root, "capture.json");
  const specPath = join(root, "spec.json");
  await writeFile(capturePath, JSON.stringify(captureEvidence("left")));
  await writeFile(specPath, JSON.stringify({ language: {} }));
  expect(
    await compareProcessEvidenceFiles(capturePath, capturePath, specPath),
  ).toMatchObject({
    code: "invalid_request",
    details: {
      operation: "compare_process_captures",
      issues: expect.arrayContaining([
        expect.objectContaining({ path: ["trace_spec", "events"] }),
      ]),
    },
  });
});

it("names the side and constraint when Evidence is not a usable capture", async () => {
  const client = await connect();
  const capture = captureEvidence("right");
  const unrelated = createEvidence(undefined, PROCESS_PROVIDER, {
    operation: "other",
    parameters: {},
    result: {},
  });
  const wrongKind = await client.callTool({
    name: "compare_process_captures",
    arguments: { left: unrelated, right: capture },
  });
  expect(parseMcpToolError(wrongKind)).toMatchObject({
    error: {
      code: "invalid_request",
      details: {
        issues: [
          {
            path: ["left"],
            reason: "invalid_value",
            message: expect.stringContaining(
              "the left Evidence has operation other",
            ),
          },
        ],
      },
    },
  });

  const tampered = await client.callTool({
    name: "compare_process_captures",
    arguments: {
      left: capture,
      right: { ...capture, parameters: { side: "changed" } },
    },
  });
  expect(parseMcpToolError(tampered)).toMatchObject({
    error: {
      code: "evidence_integrity_mismatch",
      message: expect.stringContaining(
        "The right Evidence failed validation (at evidence_id: Evidence semantic identifier does not match its record)",
      ),
    },
  });

  const invalidResult = createEvidence(undefined, PROCESS_PROVIDER, {
    predicateType: "rea.process-capture",
    operation: "capture_process_scenario",
    parameters: {},
    result: { manifest: {} },
  });
  const malformed = await client.callTool({
    name: "compare_process_captures",
    arguments: { left: capture, right: invalidResult },
  });
  expect(parseMcpToolError(malformed)).toMatchObject({
    error: {
      code: "evidence_integrity_mismatch",
      message: expect.stringContaining(
        "The right Evidence has an invalid process capture result (at ",
      ),
    },
  });

  const root = await createTestTempDirectory("rea-process-malformed-");
  const leftPath = join(root, "left.json");
  const rightPath = join(root, "right.json");
  await writeFile(leftPath, JSON.stringify(capture));
  await writeFile(rightPath, JSON.stringify(invalidResult));
  expect(await compareProcessEvidenceFiles(leftPath, rightPath)).toMatchObject({
    category: "invalid_input",
    message: expect.stringContaining("Capture evidence is malformed (at "),
  });
});