morluto/rea
1
0
Fork 0
mirror of https://github.com/morluto/rea.git synced 2026-10-12 02:19:24 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(evaluation): authenticate delivered Evidence citations

Browse Source 
Authenticate completed successful REA Evidence records for citation accounting and shared fixture grading. Preserve genuine structured and complete text-only delivery, including authenticated inventory substeps.

Closes #1728.
This commit is contained in:
Ysopwnable
2026-10-11 14:48:46 +08:00
committed by GitHub
parent e36c82becd
commit 844fa52772
4 changed files with 269 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/evaluation/CodexAgentEval.ts
+15 -12
View File
@@ -1,4 +1,5 @@
import {
deliveredCodexEvidence,
evaluateKnownAnswers,
type FixtureClaimExpectation,
type KnownAnswerAssessment,
@@ -67,12 +68,6 @@ const canonicalValue = (value: unknown): unknown => {
);
};
const evidenceIdsFrom = (value: unknown): readonly string[] => {
const encoded = JSON.stringify(value);
if (encoded === undefined) return [];
return [...new Set(encoded.match(/ev_[a-f0-9]{64}/gu) ?? [])];
};
const toolResultError = (
item: Record<string, unknown>,
): Record<string, unknown> | undefined => {
@@ -118,8 +113,16 @@ const toolResultFailed = (
);
};
const callFromItem = (value: unknown): CodexMcpCall | undefined => {
const item = record(value);
const callFromEvent = (
event: Record<string, unknown>,
): CodexMcpCall | undefined => {
const item = record(
event.type === "item.completed"
? event.item
: event.type === "mcp_tool_call"
? event
: undefined,
);
if (item?.type !== "mcp_tool_call") return undefined;
const tool = textValue(item.tool) ?? textValue(item.name);
if (tool === undefined) return undefined;
@@ -129,7 +132,9 @@ const callFromItem = (value: unknown): CodexMcpCall | undefined => {
server: textValue(item.server) ?? textValue(item.server_name) ?? null,
tool,
arguments: item.arguments ?? item.input ?? {},
evidenceIds: evidenceIdsFrom(item.result ?? item.output),
evidenceIds: deliveredCodexEvidence(event).map(
({ evidence_id }) => evidence_id,
),
error: toolResultFailed(item, error),
errorCode: textValue(error?.code) ?? null,
};
@@ -163,9 +168,7 @@ const consumeEvent = (state: EvaluationState, value: unknown): void => {
if (event === undefined) return;
const completedItem =
event.type === "item.completed" ? event.item : undefined;
const call =
callFromItem(completedItem) ??
(event.type === "mcp_tool_call" ? callFromItem(event) : undefined);
const call = callFromEvent(event);
if (call !== undefined) {
const identity =
call.id ??
src/evaluation/KnownAnswerEvaluation.ts
+61 -19
View File
@@ -124,7 +124,7 @@ const sameJson = (left: unknown, right: unknown): boolean => {
);
};
const parseClosedAnswer = (text: string): unknown => {
const parseClosedJson = (text: string): unknown => {
const objects: Set<string>[] = [];
let invalid = false;
try {
@@ -156,7 +156,7 @@ const parseClosedAnswer = (text: string): unknown => {
// The JSON visitor checked syntax and duplicate members before JSON.parse can fold them.
return JSON.parse(text);
} catch (error) {
// Syntax and nesting failures are malformed final-answer input, not an evaluation crash.
// Syntax and nesting failures are malformed transcript input, not an evaluation crash.
if (error instanceof SyntaxError || error instanceof RangeError)
return undefined;
throw error;
@@ -224,20 +224,29 @@ const evidenceFromCall = (
(result.error !== undefined && result.error !== null)
)
return undefined;
const structured = record(
result.structuredContent ?? result.structured_content,
);
if (
structured === undefined ||
(structured.error !== undefined && structured.error !== null)
)
return undefined;
if (
result.structuredContent !== undefined &&
result.structured_content !== undefined &&
!sameJson(result.structuredContent, result.structured_content)
)
return undefined;
let value = result.structuredContent ?? result.structured_content;
if (value === undefined || value === null) {
// Some Codex transcripts retain only the complete MCP JSON text block.
// Prose, multiple blocks, and truncated previews cannot establish delivery.
if (!Array.isArray(result.content) || result.content.length !== 1)
return undefined;
const block = record(result.content[0]);
if (block?.type !== "text" || typeof block.text !== "string")
return undefined;
value = parseClosedJson(block.text);
}
const structured = record(value);
if (
structured === undefined ||
(structured.error !== undefined && structured.error !== null)
)
return undefined;
let evidence: Evidence;
try {
evidence = parseEvidence(structured);
@@ -249,20 +258,53 @@ const evidenceFromCall = (
return evidence;
};
/** Authenticate completed REA deliveries for both citations and fixture grading. */
export const deliveredCodexEvidence = (event: unknown): readonly Evidence[] => {
const item = completedCall(event);
if (item === undefined) return [];
const evidence = evidenceFromCall(item);
if (evidence === undefined) return [];
const records = [evidence];
// Artifact inspection delivers its inventory Evidence inline. Bare links and
// arbitrary IDs in normalized facts are references, not delivered records.
if (evidence.operation !== "inspect_artifact") return records;
const substeps = record(evidence.normalized_result)?.substeps;
if (!Array.isArray(substeps)) return records;
for (const value of substeps) {
const substep = record(value);
if (
substep?.status !== "completed" ||
substep.operation !== "inventory_artifact" ||
typeof substep.evidence_id !== "string" ||
!evidence.evidence_links.includes(substep.evidence_id)
)
continue;
try {
const nested = parseEvidence(substep.evidence);
if (
nested.evidence_id === substep.evidence_id &&
nested.operation === substep.operation
)
records.push(nested);
} catch {
// An authenticated parent does not authenticate an embedded record's ID.
}
}
return records;
};
const collectDeliveredEvidence = (
events: readonly unknown[],
): DeliveredEvidence => {
const records = new Map<string, Evidence>();
const conflicts = new Set<string>();
for (const event of events) {
const item = completedCall(event);
if (item === undefined) continue;
const evidence = evidenceFromCall(item);
if (evidence === undefined) continue;
const previous = records.get(evidence.evidence_id);
if (previous !== undefined && !sameJson(previous, evidence))
conflicts.add(evidence.evidence_id);
else records.set(evidence.evidence_id, evidence);
for (const evidence of deliveredCodexEvidence(event)) {
const previous = records.get(evidence.evidence_id);
if (previous !== undefined && !sameJson(previous, evidence))
conflicts.add(evidence.evidence_id);
else records.set(evidence.evidence_id, evidence);
}
}
return { records, conflicts };
};
@@ -382,7 +424,7 @@ export const evaluateKnownAnswers = (
manifestFailures: ["duplicate_fixture_claim_ids"],
claims: [],
};
const manifest = finalAnswerSchema.safeParse(parseClosedAnswer(finalMessage));
const manifest = finalAnswerSchema.safeParse(parseClosedJson(finalMessage));
if (!manifest.success)
return {
scope,
tests/evaluation/model-evals/codexAgentEval.test.ts
+180 -6
View File
@@ -1,7 +1,56 @@
import { describe, expect, it } from "vitest";
import { createEvidence } from "../../../src/domain/evidence.js";
import type { JsonValue } from "../../../src/domain/jsonValue.js";
import { evaluateCodexEvents } from "../../../src/evaluation/CodexAgentEval.js";
const observation = (
operation = "inspect_managed_artifact",
result: JsonValue = { observed: true },
evidenceLinks: readonly string[] = [],
) =>
createEvidence(
{ path: "/tmp/app.dll", sha256: "1".repeat(64), format: "pe" },
{ id: "managed", name: "Managed static analysis", version: null },
{
operation,
parameters: {},
result,
evidenceLinks,
authority: "shipped-artifact",
},
);
const citationMetrics = (
result: unknown,
evidenceId: string,
tool = "inspect_managed_artifact",
) =>
evaluateCodexEvents(
[
{
type: "item.completed",
item: {
type: "mcp_tool_call",
status: "completed",
server: "rea",
tool,
arguments: {},
result,
},
},
{
type: "item.completed",
item: {
type: "agent_message",
text: `Observed artifact evidence ${evidenceId} is cited here; runtime behavior remains unknown because no runtime authority was supplied.`,
},
},
],
tool,
{ requireEvidence: true },
);
describe("Codex agent release evaluation", () => {
it("measures natural routing, repeated calls, tokens, and answer heuristics", () => {
const call = {
@@ -50,7 +99,8 @@ describe("Codex agent release evaluation", () => {
});
it("requires the final answer to cite produced Evidence when requested", () => {
const evidenceId = `ev_${"a".repeat(64)}`;
const evidence = observation();
const evidenceId = evidence.evidence_id;
const events = [
{
type: "item.completed",
@@ -60,7 +110,7 @@ describe("Codex agent release evaluation", () => {
server: "rea",
tool: "inspect_managed_artifact",
arguments: { path: "/tmp/app.dll" },
result: { evidence_id: evidenceId },
result: { structuredContent: evidence },
},
},
{
@@ -82,7 +132,126 @@ describe("Codex agent release evaluation", () => {
answerHeuristicsMet: true,
});
});
});
describe("Codex Evidence delivery", () => {
it("rejects diagnostic IDs and failed results as delivered Evidence", () => {
const evidenceId = `ev_${"a".repeat(64)}`;
for (const result of [
{ content: [{ type: "text", text: `diagnostic: ${evidenceId}` }] },
{ evidence_id: evidenceId },
{ structuredContent: { metadata: { evidence_id: evidenceId } } },
{
isError: true,
content: [
{
type: "text",
text: JSON.stringify({ error: { message: evidenceId } }),
},
],
},
]) {
expect(citationMetrics(result, evidenceId)).toMatchObject({
evidenceIds: [],
finalCitesEvidence: false,
answerHeuristicsMet: false,
});
}
});
it("accepts complete text-only Evidence while rejecting truncated and tampered copies", () => {
const evidence = observation();
const text = JSON.stringify(evidence);
const textResult = (value: string) => ({
content: [{ type: "text", text: value }],
structured_content: null,
});
expect(
citationMetrics(textResult(text), evidence.evidence_id),
).toMatchObject({
evidenceIds: [evidence.evidence_id],
finalCitesEvidence: true,
answerHeuristicsMet: true,
});
for (const result of [
textResult(text.slice(0, -1)),
textResult(
JSON.stringify({
...evidence,
normalized_result: { fabricated: true },
}),
),
textResult(
text.replace(
`"evidence_id":`,
`"evidence_id":"${evidence.evidence_id}","evidence_id":`,
),
),
]) {
expect(citationMetrics(result, evidence.evidence_id).evidenceIds).toEqual(
[],
);
}
});
it("retains authenticated nested inspection Evidence without accepting bare or tampered links", () => {
const nested = observation("inventory_artifact");
const substep = {
status: "completed",
operation: nested.operation,
evidence_id: nested.evidence_id,
evidence: nested,
};
const inspection = observation(
"inspect_artifact",
{ substeps: [substep] },
[nested.evidence_id],
);
expect(
citationMetrics(
{ structuredContent: inspection },
nested.evidence_id,
"inspect_artifact",
),
).toMatchObject({
evidenceIds: [inspection.evidence_id, nested.evidence_id],
finalCitesEvidence: true,
});
for (const [step, links] of [
[substep, []],
[
{
...substep,
evidence: { ...nested, normalized_result: { fabricated: true } },
},
[nested.evidence_id],
],
[{ ...substep, operation: "other_tool" }, [nested.evidence_id]],
[
{ ...substep, evidence_id: `ev_${"b".repeat(64)}` },
[nested.evidence_id],
],
] as const) {
const parent = observation(
"inspect_artifact",
{ substeps: [step] },
links,
);
expect(
citationMetrics(
{ structuredContent: parent },
nested.evidence_id,
"inspect_artifact",
),
).toMatchObject({
evidenceIds: [parent.evidence_id],
finalCitesEvidence: false,
});
}
});
});
describe("Codex request accounting", () => {
it("flags identical calls with distinct item identities", () => {
const event = (id: string) => ({
type: "item.completed",
@@ -269,7 +438,8 @@ describe("Codex agent routing and error evaluation", () => {
describe("Codex agent completion evaluation", () => {
it("does not present matching answer terms as factual correctness or honesty", () => {
const evidenceId = `ev_${"a".repeat(64)}`;
const evidence = observation("analyze_javascript_application");
const evidenceId = evidence.evidence_id;
const metrics = evaluateCodexEvents(
[
{
@@ -280,7 +450,7 @@ describe("Codex agent completion evaluation", () => {
server: "rea",
tool: "analyze_javascript_application",
arguments: { input_path: "/tmp/app.asar" },
result: { evidence_id: evidenceId },
result: { structuredContent: evidence },
},
},
{
@@ -380,11 +550,15 @@ describe("Codex agent completion evaluation", () => {
describe("Codex agent workflow evaluation", () => {
it("requires an ordered tool subsequence and rejects input validation failures", () => {
const evidenceId = `ev_${"b".repeat(64)}`;
const comparison = observation("compare_javascript_export_shapes");
const evidenceId = comparison.evidence_id;
const call = (
id: string,
tool: string,
result: unknown = { evidence_id: evidenceId },
result: unknown = {
structuredContent:
tool === comparison.operation ? comparison : observation(tool),
},
) => ({
type: "item.completed",
item: {
tests/evaluation/model-evals/knownAnswerEvaluation.test.ts
+13
View File
@@ -351,6 +351,19 @@ describe("successful Evidence grounding", () => {
).toBe("passed");
});
it("grades a complete text-only production Evidence response", () => {
const evidence = createObservation();
const event = delivered(evidence, {
result: {
content: resultFor(evidence).content,
structured_content: null,
},
});
expect(
assess([event], answer([claimFor(evidence)])).factualCorrectness,
).toBe("passed");
});
it("rejects authentic unrelated Evidence containing the same value elsewhere", () => {
const evidence = createObservation({ unrelated: expected().expectedValue });
expect(