fix(evidence): order canonical projections independently of locale

This commit is contained in:
morluto
2026-10-10 05:13:53 +08:00
parent 0f10499245
commit 614fb7681e
22 changed files with 123 additions and 81 deletions
@@ -2,6 +2,9 @@ import { createHash } from "node:crypto";
import canonicalize from "canonicalize";
import { canonicalJson } from "../domain/comparisonSemantics.js";
import { compareUnicodeCodePoints } from "../domain/unicodeCodePointOrder.js";
import type {
ReconstructionObligation,
ReconstructionObligationLedger,
@@ -118,7 +121,8 @@ export const obligationLedgerStatus = (
export const obligationEdgeOrder = (
left: { readonly obligation_id: string },
right: { readonly obligation_id: string },
): number => JSON.stringify(left).localeCompare(JSON.stringify(right));
): number =>
compareUnicodeCodePoints(canonicalJson(left), canonicalJson(right));
export const digestObligationLedgerValue = (value: unknown): string => {
const encoded = canonicalize(value);
@@ -30,6 +30,7 @@ import type {
CapabilityDescriptor,
} from "../AnalysisProvider.js";
import { OFFICIAL_TOOL_CONTRACTS } from "../../contracts/officialToolContracts.js";
import { compareUnicodeCodePoints } from "../../domain/unicodeCodePointOrder.js";
const STATEFUL_OPERATIONS: ReadonlySet<AnalysisOperation> = new Set([
"health",
@@ -144,10 +145,11 @@ export class AnalysisSnapshotCache {
const snapshotTargetIdentity = snapshotTarget(target);
const binding = snapshotBinding(profile);
const entries = [...this.#entries.values()].sort((left, right) =>
left.query_id.localeCompare(right.query_id),
compareUnicodeCodePoints(left.query_id, right.query_id),
);
const workflows = [...this.#workflowEntries.values()].sort(
(left, right) => left.query_id.localeCompare(right.query_id),
(left, right) =>
compareUnicodeCodePoints(left.query_id, right.query_id),
);
// Parsing owns the returned JSON and metadata; pre-cloning the same
// payloads here only adds another full materialization.
@@ -209,14 +211,18 @@ export class AnalysisSnapshotCache {
/** Return canonical entries for persistence. */
entries(): AnalysisSnapshotEntry[] {
return [...this.#entries.values()]
.sort((left, right) => left.query_id.localeCompare(right.query_id))
.sort((left, right) =>
compareUnicodeCodePoints(left.query_id, right.query_id),
)
.map((entry) => structuredClone(entry));
}
/** Return canonical composed-workflow entries for persistence. */
workflowEntries(): AnalysisSnapshotWorkflowEntry[] {
return [...this.#workflowEntries.values()]
.sort((left, right) => left.query_id.localeCompare(right.query_id))
.sort((left, right) =>
compareUnicodeCodePoints(left.query_id, right.query_id),
)
.map((entry) => structuredClone(entry));
}
+6 -1
View File
@@ -3,6 +3,8 @@ import { createHash } from "node:crypto";
import canonicalize from "canonicalize";
import { z } from "zod";
import { compareUnicodeCodePoints } from "./unicodeCodePointOrder.js";
import {
analysisProfileSchema,
analysisProfilesEqual,
@@ -463,7 +465,10 @@ const hasCanonicalQueryOrder = (
): boolean => {
let previous: string | undefined;
for (const { query_id: queryId } of entries) {
if (previous !== undefined && previous.localeCompare(queryId) > 0)
if (
previous !== undefined &&
compareUnicodeCodePoints(previous, queryId) > 0
)
return false;
previous = queryId;
}
+4 -2
View File
@@ -1,5 +1,7 @@
import { z } from "zod";
import { compareUnicodeCodePoints } from "./unicodeCodePointOrder.js";
/** Stable browser sections used by coverage and omission accounting. */
const browserSectionSchema = z.enum([
"frames",
@@ -138,6 +140,6 @@ const canonicalExclusions = (
): BrowserExclusion[] =>
[...exclusions].sort(
(left, right) =>
left.section.localeCompare(right.section) ||
left.reason.localeCompare(right.reason),
compareUnicodeCodePoints(left.section, right.section) ||
compareUnicodeCodePoints(left.reason, right.reason),
);
+5 -8
View File
@@ -10,6 +10,7 @@ import {
parseFunctionEvidence,
type FunctionSnapshot,
} from "./functionDossierEvidence.js";
import { compareUnicodeCodePoints } from "./unicodeCodePointOrder.js";
/** Minimal directed caller-to-callee adjacency for call-path search. */
class CallGraph {
@@ -104,9 +105,7 @@ export const buildCallPath = (input: CallPathInput): CallPathResult => {
goal: parsed.goal.address,
explored: summarizeSearch(graph, search.reached),
evidence_links: uniqueEvidence(snapshots.values()),
limitations: [...new Set(limitations)].sort((left, right) =>
left.localeCompare(right),
),
limitations: [...new Set(limitations)].sort(compareUnicodeCodePoints),
};
const searchScope = { exhaustive };
if (found)
@@ -242,9 +241,7 @@ const inspectSearch = ({
);
continue;
}
const neighbors = graph
.outNeighbors(node)
.sort((left, right) => left.localeCompare(right));
const neighbors = graph.outNeighbors(node).sort(compareUnicodeCodePoints);
for (const neighbor of neighbors)
if (!reached.has(neighbor)) {
reached.set(neighbor, depth + 1);
@@ -289,7 +286,7 @@ const enumeratePaths = ({
distance !== undefined &&
goalDistances.get(neighbor) === distance - 1,
)
.sort((left, right) => left.localeCompare(right));
.sort(compareUnicodeCodePoints);
neighbors.set(node, candidates);
}
return { neighbors: candidates, nextIndex: 0 };
@@ -370,4 +367,4 @@ const snapshotLinks = (snapshot: FunctionSnapshot | undefined): string[] => {
const uniqueEvidence = (snapshots: Iterable<FunctionSnapshot>): string[] =>
[
...new Set([...snapshots].flatMap((snapshot) => snapshotLinks(snapshot))),
].sort((left, right) => left.localeCompare(right));
].sort(compareUnicodeCodePoints);
+8 -7
View File
@@ -6,7 +6,7 @@ import { describe, expect } from "vitest";
import { digestCanonicalValue } from "./canonicalDigest.js";
const legacyDigest = (value: unknown): string => {
const canonicalizeDigest = (value: unknown): string => {
const encoded = canonicalize(value);
if (encoded === undefined) throw new TypeError("Unserializable fixture");
return createHash("sha256").update(encoded).digest("hex");
@@ -16,7 +16,7 @@ describe("incremental canonical digest", () => {
it.prop([fc.jsonValue()])(
"preserves canonicalize digests for JSON values",
(value) => {
expect(digestCanonicalValue(value)).toBe(legacyDigest(value));
expect(digestCanonicalValue(value)).toBe(canonicalizeDigest(value));
},
);
@@ -25,6 +25,7 @@ describe("incremental canonical digest", () => {
const sparse: unknown[] = [undefined, Symbol("omit"), () => undefined];
sparse.length = 6;
const fixtures: unknown[] = [
{ "\ue000": "bmp", 𐀀: "supplementary" },
{
"€": "\ud800",
"😀": "\udc00",
@@ -40,7 +41,7 @@ describe("incremental canonical digest", () => {
{ nested: { toJSON: () => undefined } },
];
for (const value of fixtures)
expect(digestCanonicalValue(value)).toBe(legacyDigest(value));
expect(digestCanonicalValue(value)).toBe(canonicalizeDigest(value));
});
it("rejects unsupported roots, non-finite numbers, BigInt and ancestor cycles", () => {
@@ -59,14 +60,14 @@ describe("incremental canonical digest", () => {
const value = {
payload: Array.from({ length: 64 }, () => "x".repeat(16_384)),
};
expect(digestCanonicalValue(value)).toBe(legacyDigest(value));
expect(digestCanonicalValue(value)).toBe(canonicalizeDigest(value));
});
it("keeps digests identical when buffered parts cross flush boundaries", () => {
const manyKeys: Record<string, unknown> = {};
for (let i = 0; i < 2000; i += 1) manyKeys[`k${i}`] = i;
const value = { keys: manyKeys, tail: ["€", "😀", "x".repeat(9000)] };
expect(digestCanonicalValue(value)).toBe(legacyDigest(value));
expect(digestCanonicalValue(value)).toBe(canonicalizeDigest(value));
});
it("preserves canonicalize property access and toJSON cycle behavior", () => {
@@ -80,11 +81,11 @@ describe("incremental canonical digest", () => {
};
};
expect(digestCanonicalValue(createAccessor())).toBe(
legacyDigest(createAccessor()),
canonicalizeDigest(createAccessor()),
);
const shared = { toJSON: () => ({ answer: 42 }) };
expect(digestCanonicalValue([shared, shared])).toBe(
legacyDigest([shared, shared]),
canonicalizeDigest([shared, shared]),
);
});
});
+7 -6
View File
@@ -2,6 +2,7 @@ import { z } from "zod";
import { artifactComparisonResultSchema } from "./artifactComparison.js";
import { uniqueSorted } from "./canonicalOrdering.js";
import { compareUnicodeCodePoints } from "./unicodeCodePointOrder.js";
import { comparisonSourceEvidenceIds } from "./comparisonSourceEvidence.js";
import { evidenceSchema, parseEvidence, type Evidence } from "./evidence.js";
import { functionComparisonResultSchema } from "./functionComparisonSchemas.js";
@@ -388,12 +389,12 @@ const count = (
.length;
const compareFindings = (left: Finding, right: Finding): number =>
[left.scope, left.dimension, left.source_comparison_id]
.join("\0")
.localeCompare(
[right.scope, right.dimension, right.source_comparison_id].join("\0"),
"en",
);
compareUnicodeCodePoints(left.scope, right.scope) ||
compareUnicodeCodePoints(left.dimension, right.dimension) ||
compareUnicodeCodePoints(
left.source_comparison_id,
right.source_comparison_id,
);
const assertUnique = (values: readonly string[], label: string): void => {
if (new Set(values).size !== values.length)
+5 -3
View File
@@ -1,6 +1,8 @@
import { z } from "zod";
import canonicalize from "canonicalize";
import { compareUnicodeCodePoints } from "./unicodeCodePointOrder.js";
import {
evidenceSchema,
immutableEvidence,
@@ -72,7 +74,7 @@ export const createEvidenceBundle = (
unknowns: readonly ResidualUnknown[] = [],
): EvidenceBundle => {
const sortedRecords = [...records].sort((left, right) =>
left.evidence_id.localeCompare(right.evidence_id),
compareUnicodeCodePoints(left.evidence_id, right.evidence_id),
);
return {
artifacts: uniqueSorted(
@@ -117,7 +119,7 @@ export const createEvidenceBundle = (
),
unknowns: [...unknowns].sort(
(left, right) =>
left.unknown_id.localeCompare(right.unknown_id) ||
compareUnicodeCodePoints(left.unknown_id, right.unknown_id) ||
left.revision - right.revision,
),
records: sortedRecords,
@@ -517,6 +519,6 @@ const uniqueSorted = <Value>(values: readonly Value[]): Value[] => {
const unique = new Map<string, Value>();
for (const value of values) unique.set(JSON.stringify(value), value);
return [...unique.entries()]
.sort(([left], [right]) => left.localeCompare(right))
.sort(([left], [right]) => compareUnicodeCodePoints(left, right))
.map(([, value]) => value);
};
+2 -1
View File
@@ -1,4 +1,5 @@
import { canonicalJson } from "./comparisonSemantics.js";
import { compareUnicodeCodePoints } from "./unicodeCodePointOrder.js";
import { parseFunctionEvidence } from "./functionDossierEvidence.js";
import { functionMatch } from "./functionComparisonNormalization.js";
import {
@@ -40,6 +41,6 @@ export const compareFunctions = (
]
: []),
]),
].sort((a, b) => a.localeCompare(b)),
].sort((a, b) => compareUnicodeCodePoints(a, b)),
});
};
+2 -1
View File
@@ -1,4 +1,5 @@
import { parseEvidence, type Evidence } from "./evidence.js";
import { compareUnicodeCodePoints } from "./unicodeCodePointOrder.js";
import { functionDossierSchema, type FunctionDossier } from "./hopperValues.js";
/** Complete function observation parsed from one analyze_function Evidence record. */
@@ -25,6 +26,6 @@ export const parseFunctionEvidence = (input: unknown): FunctionSnapshot => {
dossier,
limitations: [
...new Set([...evidence.limitations, ...dossier.limitations]),
].sort((left, right) => left.localeCompare(right)),
].sort((left, right) => compareUnicodeCodePoints(left, right)),
};
};
@@ -326,11 +326,11 @@ const uniquePaths = (paths: readonly CurrentPath[]): CurrentPath[] =>
path,
]),
).values(),
].sort((left, right) =>
compareUnicodeCodePoints(
`${left.kind}\0${left.value}`,
`${right.kind}\0${right.value}`,
),
].sort(
(left, right) =>
compareUnicodeCodePoints(left.kind, right.kind) ||
compareUnicodeCodePoints(left.value, right.value) ||
Number(left.allowSuffix) - Number(right.allowSuffix),
);
const pathSuffixes = (path: string): string[] => {
+4 -8
View File
@@ -102,15 +102,11 @@ describe("inferJsonShape", () => {
},
);
it("preserves the existing locale order for paths that do not tie", () => {
it("orders paths deterministically by Unicode code point", () => {
const names = ["z", "é", "A", "_", "a"];
const expected = names
.map((name) => `/${name}`)
.sort((left, right) => left.localeCompare(right));
expect(new Set(expected).size).toBe(names.length);
for (const [index, left] of expected.entries())
for (const right of expected.slice(index + 1))
expect(left.localeCompare(right)).not.toBe(0);
// Code-point order (locale-independent): A < _ < a < z < é.
// `localeCompare` order varies by host ICU and must not leak into evidence.
const expected = ["/A", "/_", "/a", "/z", "/é"];
const shape = inferJsonShape(
JSON.stringify(Object.fromEntries(names.map((name) => [name, true]))),
);
+1 -5
View File
@@ -90,11 +90,7 @@ export const inferJsonShape = (text: string): JsonShape | null => {
types: [...value.types].sort(),
observations: value.observations,
}))
.sort(
(left, right) =>
left.path.localeCompare(right.path) ||
compareUnicodeCodePoints(left.path, right.path),
),
.sort((left, right) => compareUnicodeCodePoints(left.path, right.path)),
});
};
+4 -3
View File
@@ -1,5 +1,7 @@
import { z } from "zod";
import { compareUnicodeCodePoints } from "../unicodeCodePointOrder.js";
/** Objective-C method type. */
export const objcMethodTypeSchema = z.enum([
"instance",
@@ -395,9 +397,8 @@ export const inspectNativeDispatchMetadata = (
}));
const decodedImplementations = [...implementations.values()].sort(
(left, right) =>
`${left.class_name}:${left.selector}`.localeCompare(
`${right.class_name}:${right.selector}`,
),
compareUnicodeCodePoints(left.class_name, right.class_name) ||
compareUnicodeCodePoints(left.selector, right.selector),
);
const swift = swiftSymbols.length > 0;
+4 -3
View File
@@ -3,6 +3,8 @@ import { createHash } from "node:crypto";
import canonicalize from "canonicalize";
import { z } from "zod";
import { compareUnicodeCodePoints } from "./unicodeCodePointOrder.js";
import { evidenceBundleSchema, parseEvidenceBundle } from "./evidenceBundle.js";
import {
evaluateReconstructionClaims,
@@ -403,9 +405,8 @@ const closureStatus = (
};
const reasonOrder = (left: ClosureReason, right: ClosureReason): number =>
`${left.code}:${left.subject_id}`.localeCompare(
`${right.code}:${right.subject_id}`,
);
compareUnicodeCodePoints(left.code, right.code) ||
compareUnicodeCodePoints(left.subject_id, right.subject_id);
const digest = (value: unknown): string => {
const encoded = canonicalize(value);
+6 -6
View File
@@ -1,4 +1,5 @@
import { uniqueSorted } from "./canonicalOrdering.js";
import { compareUnicodeCodePoints } from "./unicodeCodePointOrder.js";
import type {
ReconstructionClaim,
ReconstructionClaimResult,
@@ -63,7 +64,7 @@ export const reconstructionClaimUnknowns = (
return unknown === undefined ? [] : [unknown];
})
.sort((left, right) =>
left.unknown_id.localeCompare(right.unknown_id, "en"),
compareUnicodeCodePoints(left.unknown_id, right.unknown_id),
);
};
@@ -103,11 +104,10 @@ export const reconstructionProbes = (
output.set(key, current);
}
}
const sorted = [...output.values()].sort((left, right) =>
`${left.operation}\0${left.rationale}`.localeCompare(
`${right.operation}\0${right.rationale}`,
"en",
),
const sorted = [...output.values()].sort(
(left, right) =>
compareUnicodeCodePoints(left.operation, right.operation) ||
compareUnicodeCodePoints(left.rationale, right.rationale),
);
return sorted;
};
+5 -2
View File
@@ -6,6 +6,7 @@ import { z } from "zod";
import { artifactComparisonResultSchema } from "./artifactComparison.js";
import { artifactInventoryResultSchema } from "./artifactGraph.js";
import { uniqueSorted } from "./canonicalOrdering.js";
import { compareUnicodeCodePoints } from "./unicodeCodePointOrder.js";
import { comparisonSourceEvidenceSides } from "./comparisonSourceEvidence.js";
import { parseEvidence, type Evidence } from "./evidence.js";
import { parseEvidenceBundle } from "./evidenceBundle.js";
@@ -76,7 +77,9 @@ export const verifyReconstruction = (
const heads = reconstructionUnknownHeads(bundle.unknowns);
const results = specification.claims
.map((claim) => evaluateClaim(claim, records, heads))
.sort((left, right) => left.claim_id.localeCompare(right.claim_id, "en"));
.sort((left, right) =>
compareUnicodeCodePoints(left.claim_id, right.claim_id),
);
const failed = count(results, "fail");
const unresolved = count(results, "unknown");
const probes = reconstructionProbes(results, heads);
@@ -88,7 +91,7 @@ export const verifyReconstruction = (
specification_sha256: digest({
...specification,
claims: [...specification.claims].sort((left, right) =>
left.claim_id.localeCompare(right.claim_id, "en"),
compareUnicodeCodePoints(left.claim_id, right.claim_id),
),
}),
summary: {
+6 -5
View File
@@ -2,6 +2,7 @@ import { createHash } from "node:crypto";
import canonicalize from "canonicalize";
import { z } from "zod";
import { compareUnicodeCodePoints } from "./unicodeCodePointOrder.js";
import { prefixedDigestSchema } from "./../domain/digests.js";
const evidenceIdSchema = prefixedDigestSchema("ev");
@@ -287,13 +288,13 @@ const canonicalJson = (value: unknown): string => {
};
const sortedUnique = (values: readonly string[]): string[] =>
[...new Set(values)].sort((left, right) => left.localeCompare(right));
[...new Set(values)].sort(compareUnicodeCodePoints);
const sortedRelationships = (
values: readonly z.infer<typeof relationshipSchema>[],
): z.infer<typeof relationshipSchema>[] =>
[...values].sort((left, right) =>
`${left.type}:${left.unknown_id}`.localeCompare(
`${right.type}:${right.unknown_id}`,
),
[...values].sort(
(left, right) =>
compareUnicodeCodePoints(left.type, right.type) ||
compareUnicodeCodePoints(left.unknown_id, right.unknown_id),
);
+2 -1
View File
@@ -5,6 +5,7 @@ import { z } from "zod";
import { artifactComparisonResultSchema } from "./artifactComparison.js";
import { uniqueSorted } from "./canonicalOrdering.js";
import { compareUnicodeCodePoints } from "./unicodeCodePointOrder.js";
import { comparisonSourceEvidenceIds } from "./comparisonSourceEvidence.js";
import { evidenceSchema, parseEvidence, type Evidence } from "./evidence.js";
import { functionComparisonResultSchema } from "./functionComparisonSchemas.js";
@@ -175,7 +176,7 @@ export const correlateStaticAndRuntime = (
const correlations = parsed.mappings
.map((mapping) => correlate(mapping, staticById, runtimeById))
.sort((left, right) =>
left.correlation_id.localeCompare(right.correlation_id, "en"),
compareUnicodeCodePoints(left.correlation_id, right.correlation_id),
);
if (
new Set(correlations.map(({ correlation_id: id }) => id)).size !==
+9 -3
View File
@@ -1,4 +1,5 @@
import { digestCanonicalValue } from "./canonicalDigest.js";
import { compareUnicodeCodePoints } from "./unicodeCodePointOrder.js";
import type { WebPageInspection } from "./browserObservationSchemas.js";
import {
webCaptureDiffSchema,
@@ -59,8 +60,8 @@ const compareIdentities = (
if (!before.has(identity)) changes.push({ identity, change: "added" });
return changes.sort(
(left, right) =>
left.identity.localeCompare(right.identity) ||
left.change.localeCompare(right.change),
compareUnicodeCodePoints(left.identity, right.identity) ||
compareUnicodeCodePoints(left.change, right.change),
);
};
@@ -301,7 +302,12 @@ const incompleteSections = (completeness: {
const domProjection = (inspection: WebPageInspection) => ({
frames: inspection.frames
.map(({ url, origin }) => ({ url, origin }))
.sort((left, right) => left.url.localeCompare(right.url)),
.sort(
(left, right) =>
compareUnicodeCodePoints(left.url, right.url) ||
compareUnicodeCodePoints(left.origin ?? "", right.origin ?? "") ||
Number(left.origin !== null) - Number(right.origin !== null),
),
nodes: inspection.dom.nodes.map(({ index: _index, ...node }) => node),
});
+4 -2
View File
@@ -2,6 +2,8 @@ import { createHash } from "node:crypto";
import canonicalize from "canonicalize";
import { compareUnicodeCodePoints } from "./unicodeCodePointOrder.js";
export interface WebInventoryResourceInput {
readonly url: string;
readonly origin: string | null;
@@ -42,8 +44,8 @@ export const stableWebResources = (
).values(),
].sort(
(left, right) =>
left.resource_key.localeCompare(right.resource_key) ||
left.url.localeCompare(right.url),
compareUnicodeCodePoints(left.resource_key, right.resource_key) ||
compareUnicodeCodePoints(left.url, right.url),
);
};
@@ -59,15 +59,31 @@ describe("browser comparison input boundary", () => {
inspectWebPageInputSchema.parse({
cdp_endpoint: browser.endpoint,
target_id: "allowed-page",
allowed_origins: [
browser.allowedOrigin,
"https://private.example.test",
],
observation_ms: 0,
include_json_body_shapes: true,
include_websocket_shapes: true,
}),
);
if (!produced.ok) throw produced.error;
// The same frame URL can occur under different origins. Capture order
// must not turn identical frame identities into a DOM change.
produced.value.frames = [null, browser.allowedOrigin, ""].map(
(origin, index) => ({
frame_id: String(index),
parent_frame_id: null,
url: "about:blank",
origin,
}),
);
const reordered = structuredClone(produced.value);
reordered.frames.reverse();
const passive = {
before: { inspection: produced.value },
after: { inspection: produced.value },
after: { inspection: reordered },
};
const scenario = contract.examples[0]?.input;
if (scenario === undefined) throw new Error("Missing scenario example");
@@ -131,9 +147,8 @@ describe("browser comparison input boundary", () => {
});
}
const parsedPassive = browserCaptureComparisonInputSchema.parse(passive);
expect(parsedPassive).toEqual({
before: { inspection: produced.value, webmcp: null },
after: { inspection: produced.value, webmcp: null },
expect(compareBrowserCaptures(parsedPassive)).toMatchObject({
dimensions: { dom_structure: { status: "unchanged" } },
});
expect(compareBrowserCaptures(parsedPassive).overall_status).toBe(
"unknown",