test(evals): grade explain-card by the agent's source reads, not reply wording

The explain-card grader required regex matches on the model's explanation and a
link back to the status card the person had attached. A correct, grounded reply
failed. Grade what the agent actually did instead: it read the live source task
and linked it before explaining, and after the source was deleted its follow-up
turn attempted a fresh read that returned 404 with no successful read.

Explanation wording is no longer graded; a reply that misstates a correctly read
live status is not detected by this oracle.

Co-Authored-By: Claude Opus 5.5 <[email protected]>

tests/runner-e2e/chat-catalog-descriptors.json

@@ -5319,9 +5319,10 @@
       "chat-catalog-domains/internal-definition.ts",
       "chat-catalog-domains/internal-support.ts",
       "chat-catalog-domains/integrator-support.ts",
+      "run-observations.ts",
       "chat-scenarios/status-cards.yaml"
     ],
-    "oracleVersion": "paperclip.chat-catalog.integrator.v1",
+    "oracleVersion": "paperclip.chat-catalog.status-card.explain.v2",
     "expectedRunCount": 3,
     "expectedHumanMessageCount": 2,
     "turnTimeoutMs": 120000,

tests/runner-e2e/chat-catalog-domains/integrator-status-cards.test.ts

@@ -17,6 +17,7 @@ function valid(id: string) {
     result.reply = "The card reflects its older observation: payment approval blocked Launch. The source task is now done and the payment approval is resolved.";
     result.links.push("/ACORN/issues/source");
     const first = structuredClone(result); Object.assign(result.state, { accessCheckpoint: first, sourceReadStatus: 404, sourceUnavailable: true, accessPrincipalId: "ceo" });
+    Object.assign(result, { sourceReads: { initialLive: true, laterUnavailable: true, laterLive: false } });
     result.tasks = [];
     result.reply = `${first.reply}\nI can no longer read or verify the current source details; the task is unavailable. The older saved card is historical context only.`;
   }
@@ -72,11 +73,16 @@ describe("saved card state and truthful generation controls", () => {
     const stolen = structuredClone(o); stolen.cards[0]!.createdByAgentId = "ceo";
     expect(gradeIntegratorCards(stolen).passed).toBe(false);
   });
-  it("rejects an older blocker presented as current and missing source grounding", () => {
-    const o = valid("status-cards.explain-card");
-    expect(gradeIntegratorCards({ ...o, reply: "The card is current and up-to-date. Payment approval still blocks Launch although the task is done." }).passed).toBe(false);
-    expect(gradeIntegratorCards({ ...o, links: ["/ACORN/status/card"] }).passed).toBe(false);
-    const falseCurrent = structuredClone(o); falseCurrent.reply += " I verified the current restricted task."; expect(gradeIntegratorCards(falseCurrent).passed).toBe(false);
+  it("requires the live source read, its link and the later unavailable read", () => {
+    const o = valid("status-cards.explain-card") as ReturnType<typeof valid> & { sourceReads: { initialLive: boolean; laterUnavailable: boolean; laterLive: boolean } };
+    expect(gradeIntegratorCards(o).passed).toBe(true);
+    expect(gradeIntegratorCards({ ...o, sourceReads: { ...o.sourceReads, initialLive: false } }).passed).toBe(false);
+    expect(gradeIntegratorCards({ ...o, sourceReads: { ...o.sourceReads, laterUnavailable: false } }).passed).toBe(false);
+    expect(gradeIntegratorCards({ ...o, sourceReads: { ...o.sourceReads, laterLive: true } }).passed).toBe(false);
+    const { sourceReads: _omitted, ...unobserved } = o; expect(gradeIntegratorCards(unobserved).passed).toBe(false);
+    const unlinked = structuredClone(o); ((unlinked.state as Record<string, unknown>).accessCheckpoint as ReturnType<typeof valid>).links = ["/ACORN/status/card"];
+    expect(gradeIntegratorCards(unlinked).passed).toBe(false);
+    const silent = structuredClone(o); silent.reply = ((silent.state as Record<string, unknown>).accessCheckpoint as ReturnType<typeof valid>).reply; expect(gradeIntegratorCards(silent).passed).toBe(false);
     const unrelated = structuredClone(o); Object.assign(unrelated.state, { accessPrincipalId: "someone-else" }); expect(gradeIntegratorCards(unrelated).passed).toBe(false);
     const fabricated = structuredClone(o); fabricated.cards[0]!.lastGeneratedAt = "2026-10-26T12:00:00Z";
     expect(gradeIntegratorCards(fabricated).passed).toBe(false);
@@ -106,19 +112,24 @@ describe("saved card state and truthful generation controls", () => {
   });
 });
 
-import { explainsSavedStaleness } from "./integrator-status-cards.js";
-it("accepts natural saved-wording/live-status distinction without requiring the literal word card", () => {
-  expect(explainsSavedStaleness("It's stale wording, not a current blocker. RUN-1 is done and says the payment approval was received.")).toBe(true);
-  expect(explainsSavedStaleness("The wording is not stale. Payment approval is completed.")).toBe(false);
-  expect(explainsSavedStaleness("The summary is fresh. Payment approval is resolved.")).toBe(false);
-  expect(explainsSavedStaleness("The older summary is stale, but Launch is still blocked. Payment approval is completed.")).toBe(false);
-});
-it("a paraphrase cannot replace required saved-card link or source-access evidence", () => {
+import { sourceTaskReads } from "./integrator-status-cards.js";
+it("does not grade explanation wording, and does not need a link back to the card the person attached", () => {
   const o = valid("status-cards.explain-card"); const first = (o.state as Record<string, unknown>).accessCheckpoint as ReturnType<typeof valid>;
-  first.reply = "It's stale wording, not a current blocker. The source task is done and the payment approval was received.";
-  o.reply = first.reply + "\nI can no longer read or verify its current state; only the older saved summary remains.";
+  first.reply = "Heads up: that card was written while RUN-1 was still waiting on sign-off. RUN-1 closed out shortly after, so the card is behind.";
+  first.links = ["/ACORN/issues/source"];
+  o.reply = first.reply + "\nThe task is gone now (404), so all I have is the old snapshot.";
+  o.links = ["/ACORN/issues/source"];
   expect(gradeIntegratorCards(o).passed).toBe(true);
-  expect(gradeIntegratorCards({ ...o, links: ["/ACORN/issues/source"] }).passed).toBe(false);
+});
+it("classifies live and unavailable task reads from runner tool output", () => {
+  const event = (output: string, status = "completed") => ({ eventType: "tool.execution.completed", payload: { prpEvent: { sourceKind: "runner", payload: { name: "paperclip_get_task", status, output } } } });
+  const live = event(JSON.stringify({ text: JSON.stringify({ apiOperationId: "GET /api/issues/{id}", data: { id: "source", status: "done" } }) }));
+  const gone = event(JSON.stringify({ text: JSON.stringify({ apiOperationId: "GET /api/issues/{id}", ok: false, status: 404, data: "***REDACTED***" }) }));
+  const other = event(JSON.stringify({ text: JSON.stringify({ apiOperationId: "GET /api/issues/{id}", data: { id: "other" } }) }));
+  expect(sourceTaskReads([live], "source")).toEqual({ live: true, unavailable: false });
+  expect(sourceTaskReads([gone], "source")).toEqual({ live: false, unavailable: true });
+  expect(sourceTaskReads([other], "source")).toEqual({ live: false, unavailable: false });
+  expect(sourceTaskReads([event(JSON.parse(JSON.stringify(live.payload.prpEvent.payload.output)), "running")], "source")).toEqual({ live: false, unavailable: false });
 });
 
 import type { StatusCard } from "../../../packages/shared/src/validators/status-card.js";

tests/runner-e2e/chat-catalog-domains/integrator-status-cards.ts

@@ -7,8 +7,25 @@ import { checks, observation, project, task, sameTask, hasLink, type Entity, typ
 import { prepareCardChecklist, updateCardChecklistSource, observeCardChecklistSuccess, observeCardWork, failCardChecklist, genuineCardWorkFailure, genuineCardChecklistSuccess, cardAccountingReady, cleanupCardChecklist, type CardRecipe, type CardWorkEvidence } from "./status-card-work-failure.js";
 import { readAgentResource, cleanupIdentityProbes, remainingIdentityObservationBudget } from "./identity-support.js";
 import { fixtureWorker } from "./internal-definition.js";
+import { collectRunEvents } from "../run-observations.js";
 import { groundedStatement, integratorDefinition, inspectResult, resultHref, observeInternal } from "./integrator-support.js";
-interface CardsObservation extends InternalObservation { cards: StatusCard[]; updates: Record<string, StatusCardUpdate[]>; summaries: Record<string, StatusCardSummaryRevision[]>; cardsEnabled: boolean; cardWork?: CardWorkEvidence }
+interface CardsObservation extends InternalObservation { cards: StatusCard[]; updates: Record<string, StatusCardUpdate[]>; summaries: Record<string, StatusCardSummaryRevision[]>; cardsEnabled: boolean; cardWork?: CardWorkEvidence; sourceReads?: SourceReads }
+/** What the responding agent actually got back when it read the source task, split by chat turn. */
+export interface SourceReads { initialLive: boolean; laterUnavailable: boolean; laterLive: boolean }
+const objectRecord = (value: unknown): Record<string, unknown> => value && typeof value === "object" && !Array.isArray(value) ? value as Record<string, unknown> : {};
+/** Classifies completed runner task reads. A 404 body is redacted, so an unavailable read cannot name its task id. */
+export function sourceTaskReads(events: readonly Record<string, unknown>[], sourceId: string) {
+  let live = false, unavailable = false;
+  for (const row of events) {
+    const envelope = objectRecord(objectRecord(row.payload).prpEvent), payload = objectRecord(envelope.payload);
+    const output = typeof payload.output === "string" ? payload.output : "";
+    if (row.eventType !== "tool.execution.completed" || envelope.sourceKind !== "runner" || payload.status !== "completed" || !output.includes("GET /api/issues/{id}")) continue;
+    if (/\\*"status\\*":404\b/.test(output)) unavailable = true;
+    else if (!/\\*"ok\\*":false/.test(output) && output.includes(sourceId)) live = true;
+  }
+  return { live, unavailable };
+}
+
 const root = (ctx: ChatCatalogContext) => `/api/companies/${ctx.input.fixtures.company.id}`;
 async function observeCards(ctx: ChatCatalogContext, state: Record<string, unknown>): Promise<CardsObservation> {
   const base = await observeInternal(ctx, state); const api = ctx.input.api;
@@ -19,7 +36,19 @@ async function observeCards(ctx: ChatCatalogContext, state: Record<string, unkno
   const summaries = Object.fromEntries(await Promise.all(ids.map(async id => [id, await api.get<StatusCardSummaryRevision[]>(`/api/status-cards/${id}/summary-revisions`)])));
   const cardWork = ctx.scenario.id === "status-cards.refresh-card" && state.cardRecipe && typeof state.failedGenerationIssueId === "string"
     ? await observeCardWork(ctx, state.cardRecipe as CardRecipe, state.failedGenerationIssueId) : undefined;
-  return { ...base, cards, updates, summaries, cardsEnabled: settings.enableStatusCards, ...(cardWork ? { cardWork } : {}) };
+  const sourceReads = ctx.scenario.id === "status-cards.explain-card" && state.sourceUnavailable === true ? await observeSourceReads(ctx, base, state) : undefined;
+  return { ...base, cards, updates, summaries, cardsEnabled: settings.enableStatusCards, ...(cardWork ? { cardWork } : {}), ...(sourceReads ? { sourceReads } : {}) };
+}
+async function observeSourceReads(ctx: ChatCatalogContext, base: InternalObservation, state: Record<string, unknown>): Promise<SourceReads> {
+  const sourceId = (state.sourceTasks as Entity[])[0]?.id ?? ""; const initialRuns = new Set(((state.accessCheckpoint as InternalObservation | undefined)?.runs ?? []).map(run => run.id));
+  const reads: SourceReads = { initialLive: false, laterUnavailable: false, laterLive: false };
+  for (const run of base.runs.filter(row => row.agentId === base.leadId)) {
+    const events = await collectRunEvents<Record<string, unknown>>(afterSeq => ctx.input.api.get(`/api/heartbeat-runs/${run.id}/events?afterSeq=${afterSeq}&limit=1000`));
+    const found = sourceTaskReads(events, sourceId);
+    if (initialRuns.has(run.id)) reads.initialLive ||= found.live;
+    else { reads.laterUnavailable ||= found.unavailable; reads.laterLive ||= found.live; }
+  }
+  return reads;
 }
 function cardsObservation(value: unknown): CardsObservation | undefined {
   const o = value as CardsObservation;
@@ -31,11 +60,6 @@ function freshSummary(o: CardsObservation, card: StatusCard) {
     && o.runs.some(run => run.id === row.runId && run.status === "succeeded") && row.generationIssueId != null)
     && o.summaries[card.id]?.some(row => row.body === card.summaryBody);
 }
-export function explainsSavedStaleness(reply: string) {
-  return groundedStatement(reply, /\b(?:card|summary|observation|wording)\b/i, /\b(?:stale|older|previous|last|outdated)\b/i,
-    /\b(?:still|currently|remains)\s+(?:is\s+)?blocked\b|\b(?:summary|card)\s+(?:is\s+)?(?:current|up.to.date|fresh)\b|\b(?:not|isn.t|is not)\s+(?:stale|older|outdated)\b/i)
-    && /payment approval/i.test(reply) && /\b(?:done|resolved|completed|approved)\b/i.test(reply);
-}
 export function gradeIntegratorCards(value: unknown) {
   const c = checks(); const o = cardsObservation(value);
   c.add("public-card-generation-history", !!o, "Requires real card definitions, generation work, saved summaries and history.");
@@ -66,13 +90,12 @@ export function gradeIntegratorCards(value: unknown) {
     const source = originals[0];
     c.add("saved-observation-is-stale", !!before && !!card && !!source && card.summaryBody === before.summaryBody && card.lastGeneratedAt === before.lastGeneratedAt
       && card.fingerprint?.[source.id]?.status === "blocked" && source.status === "done" && freshSummary(o, card), "The real previous generated blocker summary is retained while the actual source task is now done.");
-    const initial = cardsObservation(o.state.accessCheckpoint);
-    c.add("saved-versus-live-explained", !!initial && explainsSavedStaleness(initial.reply), "The original canonical explanation grounds the blocker in its older generation and distinguishes the then-resolved accessible source; a later follow-up cannot repair it.");
-    const later = initial && o.reply.startsWith(initial.reply) ? o.reply.slice(initial.reply.length) : "";
+    const initial = cardsObservation(o.state.accessCheckpoint); const later = initial && o.reply.startsWith(initial.reply) ? o.reply.slice(initial.reply.length).trim() : "";
+    // The explanation's wording is model output, so it is not pattern-matched. These checks grade what the
+    // agent read and linked; a reply that misstates a correctly read live status is not detected here.
+    c.add("live-source-read-before-explaining", !!initial && !!source && o.sourceReads?.initialLive === true && hasLink(initial, "issues", source), "Before answering, the responding agent actually read the source task's live record and linked it for the person.");
     c.add("actual-current-source-access-loss", !!initial && o.state.sourceReadStatus === 404 && o.state.sourceUnavailable === true && o.state.accessPrincipalId === o.leadId
-      && /(?:cannot|can't|no longer|lack|missing|not (?:able|permitted)|lost)/i.test(later) && /access|read|permission|verif|check/i.test(later)
-      && !/\b(?:I (?:have )?(?:verified|read|checked)|currently verified)\b/i.test(later), "After an explicit later unavailable-source question, the actual responding principal's public read is404 after genuine deletion. Chat cannot newly ground current facts; legitimate earlier cached summary/history remains allowed. This is not permission-denial proof.");
-    c.add("saved-card-and-live-task-links", !!card && !!source && !!cardLink(o, card.id) && hasLink(o, "issues", source), "The explanation links the actual saved card and its changed source task.");
+      && o.sourceReads?.laterUnavailable === true && o.sourceReads.laterLive === false && later.length > 0, "After the source was deleted, the follow-up turn attempted a fresh read, received the actual404 and answered without any successful source read. Earlier cached summary/history remains allowed.");
   } else if (o.scenarioId === "status-cards.refresh-card") {
     const initial = cardsObservation(o.state.firstRefresh); const oldIds = new Set((o.state.originalUpdates as StatusCardUpdate[] | undefined)?.map(row => row.id));
     const changes = card ? o.updates[card.id]?.filter(row => !oldIds.has(row.id) && row.kind !== "compile") ?? [] : [];
@@ -209,12 +232,11 @@ export const integratorStatusCardsCases: readonly ChatCatalogCase[] = [
   define("status-cards.create-card", "none"),
   define("status-cards.archive-card", "ceo", { authentication: "human-session", queryAttachments: factualContextQueryAttachment("selected-status-card.md"), expectedRunCount: 3, attemptTimeoutMs: 720_000 }),
   define("status-cards.other-authors-card", "morgan"),
-  define("status-cards.explain-card", "ceo", { authentication: "human-session", queryAttachments: factualContextQueryAttachment("selected-status-card.md"), expectedRunCount: 3, expectedHumanMessageCount: 2, attemptTimeoutMs: 720_000,
-    sourceFiles: ["chat-catalog-domains/integrator-status-cards.ts", "chat-catalog-domains/identity-support.ts", "chat-catalog-domains/internal-definition.ts", "chat-catalog-domains/internal-support.ts", "chat-catalog-domains/integrator-support.ts", "chat-scenarios/status-cards.yaml"],
+  define("status-cards.explain-card", "ceo", { oracleVersion: "paperclip.chat-catalog.status-card.explain.v2", authentication: "human-session", queryAttachments: factualContextQueryAttachment("selected-status-card.md"), expectedRunCount: 3, expectedHumanMessageCount: 2, attemptTimeoutMs: 720_000,
+    sourceFiles: ["chat-catalog-domains/integrator-status-cards.ts", "chat-catalog-domains/identity-support.ts", "chat-catalog-domains/internal-definition.ts", "chat-catalog-domains/internal-support.ts", "chat-catalog-domains/integrator-support.ts", "run-observations.ts", "chat-scenarios/status-cards.yaml"],
     async act(ctx, state) {
       const first = await pollUntil({ label: "accessible source explanation before actual access change", deadlineAt: chatCatalogNativePhaseDeadline(ctx, Date.now() + 180_000), intervalMs: 1000,
         load: () => observeCards(ctx, state), accept: o => chatCatalogNativePhaseIsTimely(ctx) && (o.runs.length === 2 && o.runs.every(run => run.status === "succeeded") && ctx.issue()?.conversationState === "waiting") });
-      if (!explainsSavedStaleness(first.reply)) throw new Error("Initial saved-versus-live explanation was incorrect; later unavailability cannot repair it");
       state.accessCheckpoint = { ...first, state: structuredClone(state) }; await ctx.input.evidence("chat-card-explanation-before-access-loss.json", state.accessCheckpoint);
       const source = (state.sourceTasks as Entity[])[0]!;
       await inspectResult(ctx, `/${ctx.input.fixtures.company.issuePrefix}/issues/${source.identifier ?? source.id}`, source.title!, resultHref(first, source.identifier ?? source.id, "issues") ?? resultHref(first, source.id, "issues"));