Compare commits

...
Author SHA1 Message Date
morluto 5c42969cb8 Merge commit 'ac30fbd' into HEAD 2026-11-07 23:16:26 +08:00
morluto 092f09c29b fix(compare): validate same-Evidence contradictions 2026-11-07 22:41:41 +08:00
Cursor Agent 0e9f2f002f test(compare): admit the partial same-evidence fixture explicitly 2026-11-07 12:03:20 +00:00
Cursor Agent 8d8f50cd08 fix(compare): reject impossible same-evidence changes
A comparison of one Evidence record cannot report a changed dimension,
unequal subject hashes, or unequal digests on an unchanged dimension.
2026-11-07 12:02:56 +00:00
Cursor Agent a139b063bf merge(main): refresh identical function evidence onto current main 2026-11-07 11:39:29 +00:00
morluto 8041857acc test(native): verify same-evidence dimension citations 2026-11-01 13:30:52 +08:00
morluto 12d5954653 fix(native)!: recognize identical function evidence 2026-11-01 13:23:35 +08:00
9 changed files with 378 additions and 5 deletions

No files matched your search

+5
View File
@@ -1245,6 +1245,11 @@ and slice commitments establish the same canonical entry. The result reports
remain separate dimensions. Equal numeric addresses without applicable coordinate
commitments do not establish same-entry correspondence. Inspect the retained
source Evidence and coverage when correspondence remains ambiguous.
When both inputs authenticate to the same Evidence ID and an applicable entry
comparison is unavailable, the match reports `method: "same-evidence"`.
This identifies one selected observation, not a correspondence between distinct
records or proof of complete analysis. Unavailable facets and unestablished
call coverage remain unknown.
To aggregate already-retained comparison results, pass their exact IDs to
`find_changed_behavior`, for example:
+1 -1
View File
@@ -123,7 +123,7 @@ export const SESSION_TOOL_CONTRACTS = [
),
session(
"compare_functions",
"Compare two explicit analyze_function Evidence records or exact retained session references across identity, exact provider text, calls, references, strings, and address-normalized CFG topology. Authenticated same-artifact entry coordinates establish correspondence independently of renamed labels; metadata and text changes remain separate. Missing or provider-incompatible facets remain truncated or unknown; every conclusion cites both Evidence sets.",
"Compare two explicit analyze_function Evidence records or exact retained session references across identity, exact provider text, calls, references, strings, and address-normalized CFG topology. The same authenticated Evidence record or compatible same-artifact entry coordinates establish correspondence independently of symbol origin; metadata and text changes remain separate. Missing or provider-incompatible facets remain truncated or unknown; every conclusion cites both Evidence sets.",
functionComparisonInputSchema,
),
session(
+108
View File
@@ -0,0 +1,108 @@
import { describe, expect, it } from "vitest";
import { readComparisonResult } from "./comparisonEvidenceResult.js";
import type { Evidence } from "./evidence.js";
import {
functionComparisonDimensionNameSchema,
functionComparisonResultSchema,
} from "./functionComparisonSchemas.js";
const source = `ev_${"a".repeat(64)}`;
const hash = "b".repeat(64);
const other = "c".repeat(64);
const evidence = (result: unknown): Evidence =>
({
evidence_id: `ev_${"d".repeat(64)}`,
evidence_links: [source],
parameters: { left_evidence_id: source, right_evidence_id: source },
normalized_result: result,
}) as unknown as Evidence;
const baseline = () => ({
status: "unchanged" as const,
function_match: {
status: "matched" as const,
method: "same-evidence" as const,
left_name: "main",
right_name: "main",
},
left_subject_sha256: hash,
right_subject_sha256: hash,
summary: { unchanged: 8, changed: 0, unknown: 0 },
dimensions: functionComparisonDimensionNameSchema.options.map(
(dimension) => ({
dimension,
left_count: 1,
right_count: 1,
evidence_links: [source, source],
limitations: [],
status: "unchanged" as const,
left_digest: hash,
right_digest: hash,
text_delta: null,
conclusion_kind: "derived_relationship" as const,
}),
),
changes: [],
limitations: [],
});
describe("same-evidence comparison admission", () => {
it("accepts a consistent complete comparison", () => {
const result = baseline();
expect(functionComparisonResultSchema.safeParse(result).success).toBe(true);
expect(readComparisonResult(evidence(result), "compare_functions").ok).toBe(
true,
);
});
it("rejects a changed dimension for one source", () => {
const result = baseline();
const changed = {
...result.dimensions[0]!,
status: "changed" as const,
right_digest: other,
};
const candidate = {
...result,
status: "changed" as const,
summary: { unchanged: 7, changed: 1, unknown: 0 },
dimensions: [changed, ...result.dimensions.slice(1)],
changes: [changed],
};
expect(functionComparisonResultSchema.safeParse(candidate).success).toBe(
true,
);
expect(
readComparisonResult(evidence(candidate), "compare_functions").ok,
).toBe(false);
});
it("rejects unequal subject hashes for one source", () => {
const candidate = { ...baseline(), right_subject_sha256: other };
expect(functionComparisonResultSchema.safeParse(candidate).success).toBe(
true,
);
expect(
readComparisonResult(evidence(candidate), "compare_functions").ok,
).toBe(false);
});
it("rejects unequal unchanged digests for one source", () => {
const result = baseline();
const candidate = {
...result,
dimensions: [
{ ...result.dimensions[0]!, right_digest: other },
...result.dimensions.slice(1),
],
};
expect(functionComparisonResultSchema.safeParse(candidate).success).toBe(
true,
);
expect(
readComparisonResult(evidence(candidate), "compare_functions").ok,
).toBe(false);
});
});
+34
View File
@@ -15,6 +15,7 @@ import {
type ProcessCaptureComparison,
} from "./process/processComparison.js";
import { err, ok, type Result } from "./result.js";
import { comparisonSourceEvidenceSides } from "./comparisonSourceEvidence.js";
type ComparisonResults = {
readonly compare_artifacts: ArtifactComparisonResult;
@@ -58,6 +59,39 @@ export function readComparisonResult(
evidence.normalized_result,
);
if (!parsed.ok) return parsed;
if (parsed.value.function_match.method === "same-evidence") {
try {
const { left, right } = comparisonSourceEvidenceSides(evidence);
const sourceId = left[0];
const sameSource =
left.length === 1 &&
right.length === 1 &&
sourceId === right[0] &&
sourceId !== undefined &&
evidence.evidence_links.includes(sourceId) &&
parsed.value.dimensions.every(({ evidence_links }) =>
evidence_links.every((id) => id === sourceId),
);
const impossibleChange =
parsed.value.left_subject_sha256 !==
parsed.value.right_subject_sha256 ||
parsed.value.dimensions.some(
(dimension) =>
dimension.status === "changed" ||
(dimension.status === "unchanged" &&
dimension.left_digest !== dimension.right_digest),
);
if (!sameSource || impossibleChange)
return err(
new EvidenceIntegrityError(
"Same-Evidence function comparison must cite the same source on both sides",
),
);
} catch (cause) {
if (cause instanceof EvidenceIntegrityError) return err(cause);
throw cause;
}
}
const links = validateFindingLinks(
evidence,
parsed.value.dimensions.flatMap(({ evidence_links }) => evidence_links),
+2 -1
View File
@@ -26,7 +26,8 @@ export const compareFunctions = (
const dimensions = compareDimensions(left, right, {
links,
providersDiffer,
sameEntry: match.method === "same-entry",
sameIdentity:
match.method === "same-entry" || match.method === "same-evidence",
coordinateContext,
});
const changes = dimensions.filter(({ status }) => status !== "unchanged");
+2 -2
View File
@@ -34,7 +34,7 @@ type ComparisonContext = Pick<
DimensionResultInput,
"links" | "providersDiffer"
> & {
readonly sameEntry: boolean;
readonly sameIdentity: boolean;
readonly coordinateContext: FunctionSnapshotCoordinateContext;
};
@@ -554,7 +554,7 @@ const compareIdentity = (
context: ComparisonContext,
): FunctionDimension => {
if (
!context.sameEntry &&
!context.sameIdentity &&
(!hasStableNativeSymbol(
left.dossier.procedure.name,
left.dossier.procedure.symbol,
@@ -49,13 +49,21 @@ export const functionMatch = (
left_name: leftName,
right_name: rightName,
};
}
if (left.evidence.evidence_id === right.evidence.evidence_id)
return {
status: "matched",
method: "same-evidence",
left_name: leftName,
right_name: rightName,
};
if (coordinateContext === "common")
return {
status: "ambiguous",
method: "explicit",
left_name: leftName,
right_name: rightName,
};
}
const stableNames =
hasStableNativeSymbol(leftName, left.dossier.procedure.symbol) &&
hasStableNativeSymbol(rightName, right.dossier.procedure.symbol);
+6
View File
@@ -79,6 +79,12 @@ const functionMatchSchema = z.union([
left_name: z.string(),
right_name: z.string(),
}),
z.object({
status: z.literal("matched"),
method: z.literal("same-evidence"),
left_name: z.string(),
right_name: z.string(),
}),
z.object({
status: z.enum(["mismatched", "ambiguous"]),
method: z.literal("explicit"),
@@ -0,0 +1,211 @@
import { afterAll, expect, it } from "vitest";
import { Ajv2020 } from "ajv/dist/2020.js";
import type { AnySchema } from "ajv";
import type { Client } from "@modelcontextprotocol/client";
import { createApplicationMcpHarness } from "../../fixtures/applicationMcpHarness.js";
import { createTestBinarySession } from "../../fixtures/binarySession.js";
import { createMcpSchemaValidation } from "../../fixtures/mcpSchemaValidation.js";
import { parseMcpToolError } from "../../fixtures/mcpToolError.js";
import { FUNCTION_COMPARISON_EXAMPLE } from "../../../src/contracts/functionComparisonExample.js";
import {
createEvidence,
parseEvidence,
type Evidence,
} from "../../../src/domain/evidence.js";
import { functionDossierSchema } from "../../../src/domain/native/functionDossier.js";
import { functionComparisonResultSchema } from "../../../src/domain/functionComparisonSchemas.js";
const example = FUNCTION_COMPARISON_EXAMPLE.left;
const dossier = functionDossierSchema.parse(example.normalized_result);
const { symbol: _symbol, ...procedure } = dossier.procedure;
const observation = (request: string) =>
createEvidence(
{ path: "/tmp/fixture", sha256: "0".repeat(64), format: "mach-o" },
example.provider,
{
operation: "analyze_function",
parameters: { procedure: request },
result: {
...dossier,
procedure,
comments: [],
facet_availability: {
...dossier.facet_availability,
comments: { available: false, reason: "Fixture has no comments." },
},
},
confidence: "derived",
authority: "shipped-artifact",
relationshipCoverage: { status: "complete" },
},
);
const main = observation("main");
const another = observation("another-selector");
const schemaValidation = createMcpSchemaValidation();
afterAll(() => schemaValidation.clear());
const ref = (evidence_id: string) => ({
kind: "retained-evidence",
evidence_id,
});
const expectRejectedDimensionCitation = async (
client: Client,
comparison: Evidence,
) => {
const original = functionComparisonResultSchema.parse(
comparison.normalized_result,
);
const altered = (dimension: (typeof original.dimensions)[number]) =>
dimension.dimension === "comments"
? {
...dimension,
evidence_links: [main.evidence_id, another.evidence_id],
}
: dimension;
const result = functionComparisonResultSchema.parse({
...original,
dimensions: original.dimensions.map(altered),
changes: original.changes.map(altered),
});
const invalid = createEvidence(undefined, comparison.provider, {
predicateType: comparison.predicate_type,
operation: comparison.operation,
parameters: comparison.parameters,
result,
confidence: comparison.confidence,
authority: comparison.authority,
limitations: comparison.limitations,
evidenceLinks: [main.evidence_id, another.evidence_id],
});
const response = await client.callTool({
name: "find_changed_behavior",
arguments: { comparisons: [invalid] },
});
expect(response.isError).toBe(true);
expect(parseMcpToolError(response).error.code).toBe(
"evidence_integrity_mismatch",
);
};
it.each(["full", "compact"] as const)(
"compares the same authenticated observation while preserving incomplete facets (%s)",
async (profile) => {
const session = createTestBinarySession(() => {
throw new Error("offline comparison must not launch a provider");
});
expect(session.recordEvidence(main).ok).toBe(true);
expect(session.recordEvidence(another).ok).toBe(true);
const { client } = await createApplicationMcpHarness({
session,
schemaValidator: schemaValidation,
serverOptions: {
environment: { REA_MCP_OUTPUT_SCHEMA_PROFILE: profile },
},
});
const schema = (await client.listTools()).tools.find(
({ name }) => name === "compare_functions",
)?.outputSchema;
if (schema === undefined) throw new Error("Missing output schema");
const reply = await client.callTool({
name: "compare_functions",
arguments: { left: ref(main.evidence_id), right: ref(main.evidence_id) },
});
expect(reply.isError, JSON.stringify(reply.content)).not.toBe(true);
expect(
new Ajv2020({ strict: false }).validate(
schema as AnySchema,
reply.structuredContent,
),
).toBe(true);
const result = parseEvidence(reply.structuredContent).normalized_result;
expect(result).toMatchObject({
status: "unknown",
function_match: { status: "matched", method: "same-evidence" },
dimensions: expect.arrayContaining([
expect.objectContaining({ dimension: "identity", status: "unchanged" }),
expect.objectContaining({ dimension: "comments", status: "unknown" }),
]),
});
const comparison = parseEvidence(reply.structuredContent);
const behavior = await client.callTool({
name: "find_changed_behavior",
arguments: { comparisons: [ref(comparison.evidence_id)] },
});
expect(behavior.isError, JSON.stringify(behavior.content)).not.toBe(true);
const behaviorResult = parseEvidence(
behavior.structuredContent,
).normalized_result;
expect(behaviorResult).toMatchObject({
behavior_status: "unknown",
findings: {
items: expect.arrayContaining([
expect.objectContaining({ dimension: "comments", status: "unknown" }),
]),
},
});
expect(behaviorResult).toMatchObject({
findings: {
items: expect.not.arrayContaining([
expect.objectContaining({ dimension: "identity" }),
]),
},
});
const contradictory = createEvidence(undefined, comparison.provider, {
predicateType: comparison.predicate_type,
operation: comparison.operation,
parameters: {
left_evidence_id: main.evidence_id,
right_evidence_id: another.evidence_id,
},
result: comparison.normalized_result,
confidence: comparison.confidence,
authority: comparison.authority,
limitations: comparison.limitations,
evidenceLinks: [main.evidence_id, another.evidence_id],
});
const contradictoryBehavior = await client.callTool({
name: "find_changed_behavior",
arguments: { comparisons: [contradictory] },
});
expect(contradictoryBehavior.isError).toBe(true);
expect(parseMcpToolError(contradictoryBehavior).error.code).toBe(
"evidence_integrity_mismatch",
);
await expectRejectedDimensionCitation(client, comparison);
const distinct = await client.callTool({
name: "compare_functions",
arguments: {
left: ref(main.evidence_id),
right: ref(another.evidence_id),
},
});
expect(
parseEvidence(distinct.structuredContent).normalized_result,
).toMatchObject({
function_match: { status: "ambiguous" },
});
const rebound = await client.callTool({
name: "compare_functions",
arguments: {
left: main,
right: { ...main, caller_input_path: "/tmp/alternate-spelling" },
},
});
expect(
parseEvidence(rebound.structuredContent).normalized_result,
).toMatchObject({
function_match: { status: "matched", method: "same-evidence" },
});
const tampered = await client.callTool({
name: "compare_functions",
arguments: {
left: ref(main.evidence_id),
right: { ...main, normalized_result: { ...dossier, procedure } },
},
});
expect(tampered.isError).toBe(true);
expect(parseMcpToolError(tampered).error.code).toBe(
"evidence_integrity_mismatch",
);
},
);