fix(sentry): restore the report window on master and cover native resumes

Reapply the task-interruption report window and run_error_code tag on
top of master's native same-run resume, queued Interrupt hand-off,
admission-wait and root-fallback fingerprint changes.

A native same-run resume that lapses or is exhausted on an old run is a
new failed state, so its due time and the coordinator's terminal
finalization (native_run_finalizations.updatedAt for phase
terminal_failure) now count as recent changes. The new test fails
without this and passes with it.

Co-Authored-By: Claude Opus 5.5 <[email protected]>

server/src/__tests__/task-interruption-query.test.ts

@@ -1,8 +1,10 @@
 import { randomUUID } from "node:crypto";
-import { eq } from "drizzle-orm";
+import { and, eq } from "drizzle-orm";
 import { createRunSecretRedactionRegistry } from "../services/run-secret-redaction.js";
 import { afterAll, beforeAll, describe, expect, it, vi } from "vitest";
-import { agents, agentWakeupRequests, companies, createDb, applyPendingMigrations, heartbeatRuns, issues, nativeRunFinalizations, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS } from "@paperclipai/db";
+import { agents, agentWakeupRequests, companies, createDb, applyPendingMigrations, heartbeatRuns, issueRecoveryActions, issues,
+  nativeRunFinalizations, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS } from "@paperclipai/db";
+import { createSystemRecovery } from "../services/system-recovery-state.js";
 import { taskInterruptionPresentation, taskInterruptionPresentations } from "../services/task-interruption-presentation.js";
 import { readNativeSessionResumptionAgent } from "../services/native-runtime/native-session-resumption-admission.js";
 import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
@@ -24,6 +26,9 @@ describe.skipIf(!support.supported)("task interruption latest run query", () =>
     db = createDb(database.connectionString);
   }, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS);
   afterAll(async () => { await db?.$client.end(); await database?.cleanup(); });
+  // The shared fixture's failures are historical; reporting tests need a current failure.
+  const markFailuresFresh = (companyId: string) => db.update(heartbeatRuns).set({ finishedAt: new Date() })
+    .where(and(eq(heartbeatRuns.companyId, companyId), eq(heartbeatRuns.status, "failed")));
 
   it("shows a pending native same-run resume instead of a competing Retry", async () => {
     const companyId = randomUUID(), agentId = randomUUID(), runId = randomUUID();
@@ -290,6 +295,7 @@ describe.skipIf(!support.supported)("task interruption latest run query", () =>
     visit(explain["QUERY PLAN"][0].Plan);
     expect(inspected.length).toBeGreaterThan(0);
     expect(Math.max(...inspected)).toBeLessThan(30);
+    await markFailuresFresh(f.companyId);
     const presentations = await taskInterruptionPresentations(db, f.companyId, f.input);
     for (const issue of f.input) expect(presentations.get(issue.id)).toMatchObject({ state: "failed", canRetry: true });
     for (const issue of f.input) expect(captureTaskInterruptionFailure).toHaveBeenCalledWith(expect.objectContaining({
@@ -362,7 +368,7 @@ describe.skipIf(!support.supported)("task interruption latest run query", () =>
       expect(result.get(input[n]!.id)).toMatchObject({ responsibility: "system", state: "failed" });
       expect(captureTaskInterruptionFailure).toHaveBeenCalledWith({
         companyId, agentId, taskId: input[n]!.id, runId: runs[n]!.id,
-        runStatus: runs[n]!.status, agentAdapter: "process", intentId: n === 1 ? wait!.id : null,
+        runStatus: runs[n]!.status, agentAdapter: "process", intentId: n === 1 ? wait!.id : null, runErrorCode: null,
       });
     }
     expect(result.get(input[2]!.id)?.state).toBe("working");
@@ -371,6 +377,89 @@ describe.skipIf(!support.supported)("task interruption latest run query", () =>
     for (const n of [6, 8]) expect(result.get(input[n]!.id)?.responsibility).toBe("user");
   });
 
+  it("reports a retained failure once at its state change, not on every later process's read", async () => {
+    const companyId = randomUUID(), agentId = randomUUID();
+    await db.insert(companies).values({ id: companyId, name: "Retained failures", issuePrefix: `R${companyId.slice(0, 7)}` });
+    await db.insert(agents).values({ id: agentId, companyId, name: "Worker", adapterType: "process" });
+    const input = await db.insert(issues).values(Array.from({ length: 3 }, (_, n) => ({
+      companyId, title: `Retained ${n}`, status: "in_progress", assigneeAgentId: agentId,
+    }))).returning();
+    const old = new Date(Date.now() - 22 * 60 * 60_000);
+    const runs = await db.insert(heartbeatRuns).values(input.map((issue, n) => ({
+      companyId, agentId, issueId: issue.id, status: "failed", createdAt: old,
+      finishedAt: n === 2 ? new Date(Date.now() - 60_000) : old,
+      errorCode: n === 0 ? "native_workspace_sync_out_retry_exhausted" : null,
+    }))).returning();
+    // A new saved message is a new state change for an old failed run.
+    const [wait] = await db.insert(agentWakeupRequests).values({
+      companyId, agentId, source: "on_demand", status: "deferred_issue_execution",
+      requestedByActorType: "user", payload: { issueId: input[1]!.id }, requestedAt: new Date(Date.now() - 20_000),
+    }).returning();
+    captureTaskInterruptionFailure.mockClear();
+    const result = await taskInterruptionPresentations(db, companyId, input);
+    for (const issue of input) expect(result.get(issue.id)).toMatchObject({ responsibility: "system", state: "failed" });
+    expect(captureTaskInterruptionFailure).toHaveBeenCalledTimes(2);
+    expect(captureTaskInterruptionFailure).toHaveBeenCalledWith(expect.objectContaining({
+      taskId: input[1]!.id, runId: runs[1]!.id, intentId: wait!.id }));
+    expect(captureTaskInterruptionFailure).toHaveBeenCalledWith(expect.objectContaining({
+      taskId: input[2]!.id, runId: runs[2]!.id, intentId: null }));
+    await db.update(heartbeatRuns).set({ finishedAt: new Date() }).where(eq(heartbeatRuns.id, runs[0]!.id));
+    captureTaskInterruptionFailure.mockClear();
+    await taskInterruptionPresentations(db, companyId, [input[0]!]);
+    expect(captureTaskInterruptionFailure).toHaveBeenCalledExactlyOnceWith(expect.objectContaining({
+      taskId: input[0]!.id, runErrorCode: "native_workspace_sync_out_retry_exhausted" }));
+  });
+
+  it("reports recovery that newly fails on an old run once, then treats it as retained", async () => {
+    const companyId = randomUUID(), agentId = randomUUID();
+    await db.insert(companies).values({ id: companyId, name: "Old run recovery", issuePrefix: `O${companyId.slice(0, 7)}` });
+    await db.insert(agents).values({ id: agentId, companyId, name: "Worker", adapterType: "process" });
+    const [issue] = await db.insert(issues).values({ companyId, title: "Recovery", status: "in_progress",
+      assigneeAgentId: agentId }).returning();
+    const old = new Date(Date.now() - 22 * 60 * 60_000);
+    const [run] = await db.insert(heartbeatRuns).values({ companyId, agentId, issueId: issue!.id, status: "failed",
+      createdAt: old, finishedAt: old }).returning();
+    // Exhausted recovery clears nextAttemptAt; only the saved transition time is recent.
+    const exhausted = { ...createSystemRecovery(run!.id), state: "exhausted" as const, attempts: 6, retriesUsed: 5,
+      firstFailureAt: old.getTime(), deadlineAt: old.getTime() + 30 * 60_000, nextAttemptAt: null };
+    const [action] = await db.insert(issueRecoveryActions).values({ companyId, sourceIssueId: issue!.id,
+      kind: "workspace_recovery", status: "active", ownerType: "system", cause: "native_workspace_sync_out_failed",
+      fingerprint: `recovery:${run!.id}`, nextAction: "Recover saved files.", createdAt: old, updatedAt: new Date(),
+      evidence: { runId: run!.id, systemRecovery: exhausted } }).returning();
+    captureTaskInterruptionFailure.mockClear();
+    expect((await taskInterruptionPresentations(db, companyId, [issue!])).get(issue!.id))
+      .toMatchObject({ responsibility: "system", state: "failed" });
+    expect(captureTaskInterruptionFailure).toHaveBeenCalledExactlyOnceWith(expect.objectContaining({
+      taskId: issue!.id, runId: run!.id }));
+    await db.update(issueRecoveryActions).set({ updatedAt: old }).where(eq(issueRecoveryActions.id, action!.id));
+    captureTaskInterruptionFailure.mockClear();
+    await taskInterruptionPresentations(db, companyId, [issue!]);
+    expect(captureTaskInterruptionFailure).not.toHaveBeenCalled();
+  });
+
+  it("reports a native resume that is newly exhausted on an old run once, then treats it as retained", async () => {
+    const companyId = randomUUID(), agentId = randomUUID(), runId = randomUUID();
+    await db.insert(companies).values({ id: companyId, name: "Old native run", issuePrefix: `N${companyId.slice(0, 7)}` });
+    await db.insert(agents).values({ id: agentId, companyId, name: "Native worker", adapterType: "paperclip_runner" });
+    const [issue] = await db.insert(issues).values({ companyId, title: "Native resume", status: "in_progress",
+      assigneeAgentId: agentId }).returning();
+    const old = new Date(Date.now() - 22 * 60 * 60_000);
+    await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, nativeIssueId: issue!.id, scopeKind: "issue",
+      status: "failed", runtimeMode: "native", errorCode: "native_session_interrupted", createdAt: old, finishedAt: old });
+    // Only the coordinator's saved terminal transition is recent.
+    await db.insert(nativeRunFinalizations).values({ companyId, issueId: issue!.id, runId, phase: "terminal_failure",
+      attempt: 3, nextAttemptAt: null, createdAt: old, updatedAt: new Date() });
+    captureTaskInterruptionFailure.mockClear();
+    expect((await taskInterruptionPresentations(db, companyId, [issue!])).get(issue!.id))
+      .toMatchObject({ responsibility: "system", state: "failed" });
+    expect(captureTaskInterruptionFailure).toHaveBeenCalledExactlyOnceWith(expect.objectContaining({
+      taskId: issue!.id, runId, runErrorCode: "native_session_interrupted" }));
+    await db.update(nativeRunFinalizations).set({ updatedAt: old }).where(eq(nativeRunFinalizations.runId, runId));
+    captureTaskInterruptionFailure.mockClear();
+    await taskInterruptionPresentations(db, companyId, [issue!]);
+    expect(captureTaskInterruptionFailure).not.toHaveBeenCalled();
+  });
+
   it("keeps the failed run's adapter after the task is reassigned", async () => {
     const f = await seedInterruptionFixture(db, { selected: 1, history: 1, unrelated: 0, payloadBytes: 0 });
     const [nextAgent] = await db.insert(agents).values({
@@ -378,6 +467,7 @@ describe.skipIf(!support.supported)("task interruption latest run query", () =>
     }).returning();
     const [reassigned] = await db.update(issues).set({ assigneeAgentId: nextAgent!.id })
       .where(eq(issues.id, f.input[0]!.id)).returning();
+    await markFailuresFresh(f.companyId);
     captureTaskInterruptionFailure.mockClear();
     const result = await taskInterruptionPresentations(db, f.companyId, [reassigned!]);
     expect(result.get(reassigned!.id)?.state).toBe("failed");

server/src/sentry.ts

@@ -220,6 +220,8 @@ export interface RunFailureEvent {
   /** The run status that triggered this report. */
   runStatus: HeartbeatRunStatus;
   failureSource?: "task_interruption";
+  /** A task-state event's observed run error code. It is a tag, not a fingerprint part. */
+  runErrorCode?: string | null;
   /** Bounded, redacted diagnostics selected by the run failure reporter. */
   diagnostics?: RunFailureDiagnostics;
   /** Recorded process exit evidence, when available. Validated before capture. */
@@ -269,6 +271,8 @@ function captureRunFailureEvent(event: RunFailureEvent): string | null {
       diagnostics?.exceptions.at(-1)?.code;
     const rootErrorCode = typeof rootCodeCandidate === "string" && /^[a-zA-Z][a-zA-Z0-9_]{1,99}$/.test(rootCodeCandidate)
       ? rootCodeCandidate : undefined;
+    const runErrorCode = typeof event.runErrorCode === "string" && /^[a-zA-Z][a-zA-Z0-9_]{1,99}$/.test(event.runErrorCode)
+      ? event.runErrorCode : undefined;
     // Without a usable root code, unrelated failures that share the error code,
     // adapter and phase would collapse into one group. Fall back to the root
     // exception's class name plus its HTTP status or numeric code. These are
@@ -337,6 +341,7 @@ function captureRunFailureEvent(event: RunFailureEvent): string | null {
         event_kind: "run_failure",
         ...(event.failureSource ? { failure_source: event.failureSource } : {}),
         ...(rootErrorCode ? { root_error_code: rootErrorCode } : {}),
+        ...(runErrorCode ? { run_error_code: runErrorCode } : {}),
         ...(failurePhase ? { failure_phase: failurePhase } : {}),
         run_id: event.runId,
         ...(event.taskId ? { task_id: event.taskId } : {}),
@@ -393,7 +398,7 @@ function confirmTaskInterruptionDelivery(eventId: string | undefined, statusCode
 }
 
 export function captureTaskInterruptionFailure(event: Pick<RunFailureEvent,
-  "companyId" | "agentId" | "taskId" | "runId" | "runStatus" | "agentAdapter"> & { intentId: string | null }): void {
+  "companyId" | "agentId" | "taskId" | "runId" | "runStatus" | "agentAdapter" | "runErrorCode"> & { intentId: string | null }): void {
   const key = JSON.stringify([event.companyId, event.taskId, event.runId, event.intentId]);
   if (reportedTaskInterruptions.has(key)) return;
   const now = Date.now();
@@ -411,6 +416,7 @@ export function captureTaskInterruptionFailure(event: Pick<RunFailureEvent,
     errorMessage: "Couldn’t finish this attempt. Your message is saved.",
     errorCode: "task_attempt_failed",
     failureSource: "task_interruption",
+    runErrorCode: event.runErrorCode,
   });
   if (!eventId) return;
   pendingTaskInterruptions.set(key, { eventId, retryAt: now + TASK_INTERRUPTION_DELIVERY_RETRY_MS });

server/src/services/task-interruption-presentation.ts

@@ -36,9 +36,11 @@ type ProjectionIssue = Pick<typeof issues.$inferSelect, "id" | "companyId" | "st
 type ProjectionRun = Pick<typeof heartbeatRuns.$inferSelect,
   "id" | "status" | "scheduledRetryAt" | "errorCode" | "livenessState" | "resultJson"> &
   Partial<Pick<typeof heartbeatRuns.$inferSelect, "agentId" | "error" | "companyId" | "nativeIssueId" | "runtimeMode" | "startedAt"
-    | "scheduledRetryReason">> & {
+    | "scheduledRetryReason" | "createdAt" | "finishedAt">> & {
   /** Due time of the native coordinator's own same-run resume, if one is pending. */
   nativeResumeAt?: Date | null;
+  /** When the native coordinator saved this run's terminal failure, if it did. */
+  nativeFailedAt?: Date | null;
 };
 type ProjectionWait = { id: string; requestedAt: Date; sourceRunId?: string | null; executionWaitReason?: string | null;
   preparationStatus?: string | null; preparationRetryAt?: number | string | null;
@@ -65,6 +67,27 @@ type ProjectionState = {
   admissionWait?: { id: string; status: string; reason: string | null; agentId: string };
 };
 
+// A task read reports the change that made a task fail, not every later read of
+// retained history. The delivery cache is process-local, so a restart or another
+// replica would otherwise resend each old failure whenever a list shows it.
+const TASK_INTERRUPTION_REPORT_WINDOW_MS = 60 * 60_000;
+
+/** Latest saved fact that can move a task into the failed state, if known. The
+ * recovery row's update time covers recovery that newly fails on an old run:
+ * each saved recovery transition stamps it, while activity heartbeats do not.
+ * The native resume due time and terminal finalization cover a same-run resume
+ * that lapses or is exhausted on an old run. */
+export function taskInterruptionChangedAt(input: Pick<ProjectionState, "run" | "action" | "wait" | "retryWait">): number | null {
+  const { run, wait, action } = input;
+  const retryWait = input.retryWait ?? wait;
+  const times: unknown[] = [run?.finishedAt ?? run?.createdAt, run?.scheduledRetryAt, run?.nativeResumeAt, run?.nativeFailedAt,
+    wait?.requestedAt, retryWait?.requestedAt, action?.updatedAt].map(value => value?.getTime());
+  times.push(readSystemRecovery(action?.evidence)?.nextAttemptAt,
+    retryWait?.preparationRetryAt == null ? null : Number(retryWait.preparationRetryAt));
+  const known = times.filter((time): time is number => typeof time === "number" && Number.isFinite(time));
+  return known.length ? Math.max(...known) : null;
+}
+
 function boundedJsonString(value: SQL, limit: number) {
   return sql<string | null>`case when jsonb_typeof(${value}) = 'string' then left(${value} #>> '{}', ${limit}) else null end`;
 }
@@ -256,7 +279,7 @@ export async function taskInterruptionPresentations(db: Db, companyId: string, i
   const startingIssueIds = selected.filter(issue => issue.status === "todo" && issue.assigneeAgentId).map(issue => issue.id);
   const [runs, waits, agentRows, projectRows, pauseRows, approvalRows, companyRows, admissionWaits] = await Promise.all([
     db.select({ issueId: requestedIssueId, id: heartbeatRuns.id, status: heartbeatRuns.status,
-      agentId: heartbeatRuns.agentId, companyId: heartbeatRuns.companyId,
+      createdAt: heartbeatRuns.createdAt, finishedAt: heartbeatRuns.finishedAt, agentId: heartbeatRuns.agentId, companyId: heartbeatRuns.companyId,
       nativeIssueId: heartbeatRuns.nativeIssueId, runtimeMode: heartbeatRuns.runtimeMode, startedAt: heartbeatRuns.startedAt,
       agentAdapter: agents.adapterType,
       error: sql<string | null>`case when ${heartbeatRuns.errorCode} = 'setup_failed'
@@ -273,6 +296,9 @@ export async function taskInterruptionPresentations(db: Db, companyId: string, i
           and ${nativeRunFinalizations.resultId} is null and ${nativeRunFinalizations.phase} = 'retryable_failure'
           and ${heartbeatRuns.contextSnapshot}->'nativeQuestionCancellation' is null
           and ${heartbeatRuns.runnerProfileJson}->'legacyWorkspaceRecoveryMigration' is null)`.mapWith(value => value == null ? null : new Date(value)),
+      nativeFailedAt: sql<Date | null>`(select ${nativeRunFinalizations.updatedAt} from ${nativeRunFinalizations}
+        where ${nativeRunFinalizations.companyId} = ${heartbeatRuns.companyId} and ${nativeRunFinalizations.runId} = ${heartbeatRuns.id}
+          and ${nativeRunFinalizations.phase} = 'terminal_failure')`.mapWith(value => value == null ? null : new Date(value)),
       // The run whose resultJson supplied providerFailure. A scheduled retry
       // shows its predecessor's reason, so redaction must use that run's registry.
       providerFailureRunId: sql<string>`case when ${heartbeatRuns.resultJson}->'providerFailure' is null
@@ -440,26 +466,30 @@ export async function taskInterruptionPresentations(db: Db, companyId: string, i
   const projectMap = new Map(projectRows.map(project => [project.id, project]));
   const pauses = new Set(pauseRows.map(row => row.issueId)), approvalsSet = new Set(approvalRows.map(row => row.issueId));
   const admissionWaitMap = byIssue(admissionWaits);
+  const now = Date.now();
   for (const issue of selected) {
     const required = requirements.get(issue.id);
     const denied = required?.projectIds.filter(projectId => currentAccess.get(`${issue.assigneeAgentId}:${projectId}`) !== true);
-    const presentation = projectTaskInterruption(issue, {
-      run: runMap.get(issue.id), action: actionMap.get(issue.id), wait: waitMap.get(issue.id), retryWait: retryWaitMap.get(issue.id),
+    const facts = { run: runMap.get(issue.id), action: actionMap.get(issue.id), wait: waitMap.get(issue.id),
+      retryWait: retryWaitMap.get(issue.id) };
+    const presentation = projectTaskInterruption(issue, { ...facts,
       interruptWait: interruptWaitFor(issue.id),
       projectAccessAllowed: denied?.length === 0,
       projectAccessProjectId: denied?.includes(issue.projectId ?? "") ? issue.projectId : null,
       workspaceSelectionAvailable: required?.available ?? false,
       agent: agentMap.get(issue.assigneeAgentId ?? ""), project: projectMap.get(issue.projectId ?? ""),
       company: companyRows[0], paused: pauses.has(issue.id), approval: approvalsSet.has(issue.id),
-      ownerStopRequired: ownerStopRequired.has(`${issue.id}:${runMap.get(issue.id)?.id}`),
+      ownerStopRequired: ownerStopRequired.has(`${issue.id}:${facts.run?.id}`),
       admissionWait: admissionWaitMap.get(issue.id),
-    });
+    }, now);
     result.set(issue.id, presentation);
-    const run = runMap.get(issue.id);
+    const run = facts.run;
     const runStatus = HEARTBEAT_RUN_STATUSES.find(status => status === run?.status);
-    if (presentation?.responsibility === "system" && presentation.state === "failed" && run && runStatus) {
+    const changedAt = taskInterruptionChangedAt(facts);
+    if (presentation?.responsibility === "system" && presentation.state === "failed" && run && runStatus
+        && (changedAt === null || changedAt > now - TASK_INTERRUPTION_REPORT_WINDOW_MS)) {
       captureTaskInterruptionFailure({ companyId, taskId: issue.id, agentId: run.agentId,
-        runId: run.id, runStatus, intentId: presentation.intentId,
+        runId: run.id, runStatus, intentId: presentation.intentId, runErrorCode: run.errorCode ?? null,
         agentAdapter: run.agentAdapter ?? "unknown" });
     }
   }