fix(evals): correct stale chat-catalog setup and UI expectations

Repairs setup/observation defects found in TestOps launch 1474 evidence,
on top of candidate b59e295c7f:

- instance-settings / chat-disabled: match the shipped Agent Chat
  disabled copy ("disabled for this instance … enable it in
  Experimental settings") instead of text the UI never renders.
- interrupt-with-direction, claude-account, cloud-choice: replace the
  dead [data-interaction-id] selector (no component renders it) with the
  real hooks: inline #interaction-<id> or the docked composer takeover.
  The old fallback clicked the timeline chip, not Approve.
- registered artifacts: identify a deliverable by its run-attributed
  attachment filename; the work-product title is a display label.
  Run, status, duplicate, filename and byte checks are unchanged.
- multiple-handoffs: deliver the prerequisite logo as logo.html with an
  inline <svg>; image/svg+xml is deliberately not an allowed upload type.

Co-Authored-By: Claude Opus 5.5 <[email protected]>

tests/runner-e2e/chat-catalog-domains/privileged-configuration.ts

@@ -50,7 +50,7 @@ const settings = privilegedDefinition("instance-operations.instance-settings", "
     await expect.poll(async () => (await ctx.input.api.get<{ enableAgentChat: boolean }>(endpoint, { timeout: remaining() })).enableAgentChat, { timeout: remaining() }).toBe(false);
     state.savedDisabled = true;
     await page.goto(source, { waitUntil: "domcontentloaded", timeout: remaining() });
-    await expect(page.getByText(/Agent Chat is disabled(?:\.| in Experimental settings)/).first()).toBeVisible({ timeout: remaining() });
+    await expect(page.getByText(/Agent Chat is disabled(?:\.| in Experimental settings| for this instance)/).first()).toBeVisible({ timeout: remaining() });
     state.disabledRouteClear = true;
     await page.goto(settingsUrl, { waitUntil: "domcontentloaded", timeout: remaining() });
     await expect(toggle).toHaveAttribute("aria-checked", "false", { timeout: remaining() }); await toggle.click({ timeout: remaining() });

tests/runner-e2e/chat-catalog-domains/runtime-chat-disabled.ts

@@ -62,10 +62,10 @@ export const runtimeDisabledChatCases: readonly IdentityBoundaryCase[] = [{
     person.bindApiSession(ctx.api); // Capture verification uses the same person through private Node fetch.
     const page = ctx.page;
     await page.goto(`/${fixtures.company.issuePrefix}/chats/${fixtures.agent.id}`, { waitUntil: "domcontentloaded" });
-    await expect(page.getByText(/Agent Chat is disabled\. Enable it in Experimental settings/)).toBeVisible();
+    await expect(page.getByText(/Agent Chat is disabled for this instance\. An administrator can enable it in Experimental settings/)).toBeVisible();
     await expect(page.getByTestId("task-chat-composer-input")).toHaveCount(0);
     const unavailableVisible = await page.getByText(/Agent Chat is disabled/).isVisible();
-    const supportedOperatorStepVisible = await page.getByText(/Enable it in Experimental settings/).isVisible();
+    const supportedOperatorStepVisible = await page.getByText(/can enable it in Experimental settings/).isVisible();
     ctx.observeResources({ companyId: fixtures.company.id, agentId: fixtures.agent.id, runIds: [] });
     await ctx.capture("disabled-agent-chat", "Morgan Chat is genuinely unavailable; canonical intent remains unsent", "runtime-disabled-agent-chat.png");
     const key = await api.postSensitive<{ id: string; token: string }>(`/api/agents/${fixtures.agent.id}/keys`, { name: "disabled-chat-permission-probe" });

tests/runner-e2e/chat-catalog-domains/runtime-claude-account.ts

@@ -12,7 +12,7 @@ import { checks, type Entity } from "./internal-support.js";
 import { waitForReply } from "./identity-support.js";
 import { matchesClaudeFixtureTarget, privateClaudeOperation, type ClaudePrivateReceipt } from "./runtime-claude-private.js";
 import { selectedRuntimeInputs, runtimePrivateSpec } from "./runtime-private.js";
-import { openRuntimeAgentProfile, observeRuntime, runtimeDefinition, runtimeObservation, type RuntimeAgent } from "./runtime-support.js";
+import { clickInteractionAction, openRuntimeAgentProfile, observeRuntime, runtimeDefinition, runtimeObservation, type RuntimeAgent } from "./runtime-support.js";
 const continueHire = "The Anthropic account is connected now. Continue the same Morgan frontend engineer hire using Claude Code.";
 export function supportedClaudeSetupLink(href: string, origin: string, prefix: string) {
   try { const url = new URL(href, origin); return url.origin === origin && url.pathname.startsWith(`/${prefix}/`) && (url.pathname.endsWith("/company/settings/connections") || url.pathname.endsWith("/apps/connect") && ["anthropic", "claude"].includes(url.searchParams.get("source") ?? url.searchParams.get("provider") ?? "")); } catch { return false; }
@@ -69,7 +69,7 @@ export const runtimeClaudeAccountCases = [runtimeDefinition("agents.hire-needs-a
     if (target) await ctx.input.page.locator(`[id^="comment-"] a[href=${JSON.stringify(target)}]`).first().click();
     else {
       if (!request) throw new Error("The canonical waiting response exposed neither a supported account setup link nor an actual addressed Anthropic setup request");
-      await ctx.input.page.locator(`[data-interaction-id=${JSON.stringify(request.id)}]`).getByRole("button", { name: /Connect|Set up|Continue/i }).first().click();
+      await clickInteractionAction(ctx.input.page, request, /Connect|Set up|Continue/i);
     }
     await expect(ctx.input.page.getByRole("heading", { name: /connections|Connect.*Anthropic|Connect a model provider/i }).first()).toBeVisible(); state.supportedSetupVisible = true;
     const selected = selectedRuntimeInputs(ctx, "agents.hire-needs-account");
@@ -92,7 +92,7 @@ export const runtimeClaudeAccountCases = [runtimeDefinition("agents.hire-needs-a
       state.setupRequestReconciled = completedClaudeSetupMatches({ request: resolved, sourceId: ctx.issue()!.id, sourceRunId: firstRun.id, companyId: before.companyId, humanId: session.user.id, connectionId: String(state.connectionId) });
       if (!state.setupRequestReconciled) throw new Error("The actual addressed Anthropic setup request did not reconcile against established human account access");
       await ctx.input.page.reload({ waitUntil: "domcontentloaded" });
-      await expect(ctx.input.page.locator(`[data-interaction-id=${JSON.stringify(String(state.setupRequestId))}]`)).toContainText(/connected|ready|complete/i);
+      await expect(ctx.input.page.locator(`[id=${JSON.stringify(`interaction-${String(state.setupRequestId)}`)}]`)).toContainText(/connected|ready|complete/i);
       await ctx.refreshIssue();
       state.setupAutoWakeExpected = ["in_progress", "in_review"].includes(ctx.issue()!.status) && ctx.issue()!.assigneeAgentId === resolved!.payload.requestingAgentId;
       if (state.setupAutoWakeExpected) {

tests/runner-e2e/chat-catalog-domains/runtime-cloud-choice.ts

@@ -5,7 +5,7 @@ import type { IssueThreadInteraction } from "../../../packages/shared/src/types/
 import { expect } from "@playwright/test";
 import { checks } from "./internal-support.js";
 import { waitForReply } from "./identity-support.js";
-import { runtimeDefinition, runtimeObservation, observeRuntime } from "./runtime-support.js";
+import { interactionSurface, runtimeDefinition, runtimeObservation, observeRuntime } from "./runtime-support.js";
 import { runtimePrivateSpec, selectedRuntimeInputs } from "./runtime-private.js";
 import { createOwnedDaytonaTarget, deleteOwnedDaytonaTarget, type OwnedDaytonaTarget } from "./runtime-daytona-target.js";
 import { privateClaudeOperation } from "./runtime-claude-private.js";
@@ -48,7 +48,7 @@ export const runtimeCloudChoiceCases = [runtimeDefinition("agents.ambiguous-clou
       const runs = await ctx.allRuns();
       if (questions.some(row => row.sourceRunId !== runs[0]?.id || row.issueId !== source.id)) throw new Error("Clarification card is not attributed to the actual canonical source run");
       await expect(ctx.input.page.getByTestId("task-chat-unanswered-question").first()).toBeVisible();
-      state.visibleChoiceText = await ctx.input.page.locator(`[data-interaction-id=${JSON.stringify(questions[0]!.id)}]`).innerText();
+      state.visibleChoiceText = await interactionSurface(ctx.input.page, questions[0]!).first().innerText();
     }
     else await expect(ctx.input.page.getByTestId("task-chat-composer-input")).toBeVisible();
     state.questionVisible = questions.length > 0 || /\?|which|do you mean/i.test((await observeRuntime(ctx, state)).reply); state.sourceWaiting = source.conversationState === "waiting";

tests/runner-e2e/chat-catalog-domains/runtime-interruption.ts

@@ -10,6 +10,7 @@ import { readRegisteredArtifacts } from "../registered-artifact.js";
 import { createIssueThreadInteractionSchema } from "../../../packages/shared/src/validators/issue.js";
 import { checks, type Entity } from "./internal-support.js";
 import { busyRuntimeWorker, releaseRuntimeWorker } from "./runtime-busy-worker.js";
+import { clickInteractionAction } from "./runtime-support.js";
 import { runtimeQueueCases } from "./runtime-queue.js";
 import { runtimeDefinition, runtimeObservation, observeRuntime } from "./runtime-support.js";
 export function gradeInterruption(value: unknown) {
@@ -44,11 +45,8 @@ export const runtimeInterruptionCases = [runtimeDefinition("steering.interrupt-w
     payload: { version: 1, prompt: "Use this saved simpler layout on the next eligible attempt?", supersedeOnUserComment: false,
       target: { type: "issue_document", issueId: task.id, documentId: plan.id, key: "plan", revisionId: plan.latestRevisionId, revisionNumber: plan.latestRevisionNumber } } }));
   await ctx.input.page.goto(`/${ctx.input.fixtures.company.issuePrefix}/issues/${task.identifier ?? task.id}`, { waitUntil: "domcontentloaded" });
-  const card = ctx.input.page.locator(`[data-interaction-id=${JSON.stringify(confirmation.id)}]`);
   const accept = ctx.input.page.waitForResponse(r => r.request().method() === "POST" && new URL(r.url()).pathname === `/api/issues/${task.id}/interactions/${confirmation.id}/accept`);
-  const acceptButton = card.getByRole("button", { name: /Approve|Accept/i });
-  if (await acceptButton.count()) await acceptButton.first().click();
-  else await ctx.input.page.getByRole("button", { name: /Approve|Accept/i }).first().click();
+  await clickInteractionAction(ctx.input.page, confirmation, "Approve");
   if ((await accept).status() !== 200) throw new Error("Actual human simpler-plan approval did not succeed");
   const session = await ctx.input.api.get<{ user: { id: string } }>("/api/auth/get-session");
   const approved = (await ctx.input.api.get<IssueThreadInteraction[]>(`/api/issues/${task.id}/interactions`)).find(row => row.id === confirmation.id)!;

tests/runner-e2e/chat-catalog-domains/runtime-multiple-handoffs.ts

@@ -23,7 +23,7 @@ export function gradeMultipleHandoffs(value: unknown) {
   c.add("logo-task-and-delivered-logo-unchanged", !!logoBefore && !!logo && sameTask(logoBefore, logo) && logo.status === "done" && logo.assigneeAgentId === (s.logoWorker as Entity).id
     && before?.contentVerified === true && after?.contentVerified === true && before.attachmentId === after.attachmentId && before.sha256 === after.sha256
     && before.content === after.content && after.content.includes(String(s.logoMarker)) && /<svg\b/i.test(after.content) && s.logoRevisionsUnchanged === true,
-  "The separate real Logo task, SVG bytes, artifact identity, saved revisions and original worker attribution remain unchanged.");
+  "The separate real Logo task, logo bytes, artifact identity, saved revisions and original worker attribution remain unchanged.");
   c.add("canonical-reply-honestly-identifies-affected-pending-work", /website/i.test(String(s.queryReply)) && /queue|pending|wait|next|forward|saved/i.test(String(s.queryReply))
     && !/already (?:implemented|consumed)|headline (?:is|has been) (?:already )?(?:changed|updated)|I(?:'ve| have)? implemented/i.test(String(s.queryReply)),
   "The canonical reply identifies Website and accurately describes feedback saved behind active work rather than claiming it already executed.");
@@ -39,16 +39,16 @@ export const runtimeMultipleHandoffCases = [runtimeDefinition("agent-chat.multip
   for (const scope of [{ scopeType: "company", scopeId: ctx.input.fixtures.company.id }, { scopeType: "agent", scopeId: logoWorker.id }])
     await ctx.input.api.post(`${root}/budgets/policies`, { ...scope, amount: 500, hardStopEnabled: true, isActive: true, unpricedUsagePolicy: "allow" });
   const logo = await ctx.input.api.post<Entity>(`${root}/issues`, { title: "Logo", status: "todo", assigneeAgentId: logoWorker.id,
-    description: `Create an actual small SVG logo file logo.svg with a simple acorn/circle mark, visible text Acorn and the exact reference ${logoMarker} in its SVG metadata or text. Save a short non-plan task delivery document describing it and register the real SVG file as a downloadable artifact work product titled logo.svg. Then complete this same task. No other task, publication or website change is authorized.` });
+    description: `Create an actual small logo file logo.html containing an inline SVG with a simple acorn/circle mark, visible text Acorn and the exact reference ${logoMarker} in its SVG metadata or text. Save a short non-plan task delivery document describing it and register the real logo.html file as a downloadable artifact work product titled logo.html. Then complete this same task. No other task, publication or website change is authorized.` });
   const complete = await pollUntil({ label: "Actual separate Logo work completed before the Website conversation", deadlineAt: Date.now() + 240_000,
     load: async () => ({ task: await ctx.input.api.get<Entity>(`/api/issues/${logo.id}`), runs: await ctx.allRuns() }),
     accept: v => v.task.status === "done" && v.runs.length === 1 && v.runs[0]?.agentId === logoWorker.id && v.runs[0].status === "succeeded" });
-  const [artifact] = await readRegisteredArtifacts(ctx.input.api, logo.id, complete.runs[0]!.id, ["logo.svg"]);
-  if (!artifact?.contentVerified || !artifact.content.includes(logoMarker) || !/<svg\b/i.test(artifact.content)) throw new Error("Completed Logo prerequisite lacks its actual native-created SVG");
+  const [artifact] = await readRegisteredArtifacts(ctx.input.api, logo.id, complete.runs[0]!.id, ["logo.html"]);
+  if (!artifact?.contentVerified || !artifact.content.includes(logoMarker) || !/<svg\b/i.test(artifact.content)) throw new Error("Completed Logo prerequisite lacks its actual native-created logo");
   const documents = await ctx.input.api.get<Entity[]>(`/api/issues/${logo.id}/documents`), revisions: Entity[] = [];
   for (const doc of documents) revisions.push(...await ctx.input.api.get<Entity[]>(`/api/issues/${logo.id}/documents/${encodeURIComponent(doc.key!)}/revisions`));
   const busy = await busyRuntimeWorker(ctx); await openStandingChat(ctx);
-  const message = `Keep these two existing work references in this conversation: Website ${busy.task.identifier ?? busy.task.id} is assigned to Morgan and actively waiting for the brief; Logo ${logo.identifier ?? logo.id} was completed by Riley and its actual delivered SVG is ${new URL(`/api/attachments/${artifact.attachmentId}/content?download=1`, ctx.input.api.baseURL).href}. Acknowledge these separate references, without creating, reassigning, editing or starting any task. Keep the Logo deliverable unchanged.`;
+  const message = `Keep these two existing work references in this conversation: Website ${busy.task.identifier ?? busy.task.id} is assigned to Morgan and actively waiting for the brief; Logo ${logo.identifier ?? logo.id} was completed by Riley and its actual delivered logo is ${new URL(`/api/attachments/${artifact.attachmentId}/content?download=1`, ctx.input.api.baseURL).href}. Acknowledge these separate references, without creating, reassigning, editing or starting any task. Keep the Logo deliverable unchanged.`;
   await sendChatMessage(ctx.input.page, message);
   const saved = await pollUntil({ label: "Real source history retains separate active Website and completed Logo", deadlineAt: Date.now() + 150_000,
     load: async () => { await ctx.refreshIssue(); return { source: ctx.issue(), runs: await ctx.allRuns() }; },
@@ -85,7 +85,7 @@ export const runtimeMultipleHandoffCases = [runtimeDefinition("agent-chat.multip
   async observe(ctx, state) {
     const o = await observeRuntime(ctx, state), logo = state.logoBefore as Entity;
     state.logoAfter = await ctx.input.api.get<Entity>(`/api/issues/${logo.id}`);
-    const [artifact] = await readRegisteredArtifacts(ctx.input.api, logo.id, String(state.logoRunId), ["logo.svg"]); state.logoArtifactAfter = artifact;
+    const [artifact] = await readRegisteredArtifacts(ctx.input.api, logo.id, String(state.logoRunId), ["logo.html"]); state.logoArtifactAfter = artifact;
     const documents = await ctx.input.api.get<Entity[]>(`/api/issues/${logo.id}/documents`), revisions: Entity[] = [];
     for (const doc of documents) revisions.push(...await ctx.input.api.get<Entity[]>(`/api/issues/${logo.id}/documents/${encodeURIComponent(doc.key!)}/revisions`));
     state.logoRevisionsUnchanged = JSON.stringify(documents.map(doc => [doc.id, doc.latestRevisionId]).sort()) === JSON.stringify((state.logoDocumentsBefore as Entity[]).map(doc => [doc.id, doc.latestRevisionId]).sort())

tests/runner-e2e/chat-catalog-domains/runtime-support.ts

@@ -1,4 +1,4 @@
-import { expect } from "@playwright/test";
+import { expect, type Page } from "@playwright/test";
 import { pollUntil } from "../api.js";
 import type { ChatCatalogCase, ChatCatalogContext } from "../chat-catalog-contract.js";
 import { codexSubscriptionAgentConfig } from "../codex-subscription-fixture.js";
@@ -72,6 +72,24 @@ export async function verifyRuntimeResult(ctx: ChatCatalogContext, state: Record
   await ctx.input.evidence("chat-catalog-runtime-profile-opened.json", { agentId: target.id, companyId: target.companyId, url: ctx.input.page.url() });
   await ctx.input.page.goto(source, { waitUntil: "domcontentloaded" });
 }
+/**
+ * A pending interaction renders inline in the thread as `#interaction-<id>`,
+ * or docked in the composer takeover, which carries its title but no id.
+ */
+export function interactionSurface(page: Page, interaction: { id: string; title?: string | null }) {
+  const inline = page.locator(`[id=${JSON.stringify(`interaction-${interaction.id}`)}]`);
+  return interaction.title
+    ? inline.or(page.getByTestId("task-chat-composer-takeover").filter({ hasText: interaction.title }))
+    : inline;
+}
+/** Open a pending request from its timeline chip when it is not already docked, then use its own action. */
+export async function clickInteractionAction(page: Page, interaction: { id: string; title?: string | null }, name: string | RegExp) {
+  const docked = page.getByTestId("task-chat-composer-takeover").filter({ hasText: interaction.title ?? "" });
+  const action = (interaction.title ? docked : interactionSurface(page, interaction)).getByRole("button", typeof name === "string" ? { name, exact: true } : { name });
+  await expect(interactionSurface(page, interaction).first()).toBeVisible();
+  if (!(await action.first().isVisible())) await page.locator(`[id=${JSON.stringify(`interaction-${interaction.id}`)}]`).click();
+  await action.first().click();
+}
 export function runtimeDefinition(scenarioId: string, setup: ChatCatalogCase["setup"], grade: ChatCatalogCase["grade"], extra: Partial<ChatCatalogCase> = {}): ChatCatalogCase {
   return { scenarioId, setup, grade, observe: observeRuntime, verifyResult: verifyRuntimeResult,
     oracleVersion: "paperclip.chat-catalog.runtime.v2", coverageGaps: [], expectedRunCount: 1,

tests/runner-e2e/registered-artifact.test.ts

@@ -28,7 +28,7 @@ it.each(["draft", "changes_requested", "merged", "closed", "failed", "archived",
   expect(download).not.toHaveBeenCalled();
 });
 
-it.each(["unregistered", "duplicate", "old-run", "unattributed", "wrong-issue", "wrong-attachment-issue", "wrong-type", "wrong-title", "wrong-filename", "wrong-attachment-id", "negative-size", "excessive-size", "size-mismatch", "hash-mismatch", "download-failure"])("refuses %s output evidence even when ready for review", async reason => {
+it.each(["unregistered", "duplicate", "old-run", "unattributed", "wrong-issue", "wrong-attachment-issue", "wrong-type", "wrong-filename", "wrong-attachment-id", "negative-size", "excessive-size", "size-mismatch", "hash-mismatch", "download-failure"])("refuses %s output evidence even when ready for review", async reason => {
   const { api, product, attachment, get, download } = fixture();
   product.status = "ready_for_review";
   if (reason === "unregistered") get.mockImplementation(async (path: string) => path.endsWith("work-products") ? [] : [attachment]);
@@ -38,7 +38,6 @@ it.each(["unregistered", "duplicate", "old-run", "unattributed", "wrong-issue",
   if (reason === "wrong-issue") product.issueId = "other-issue";
   if (reason === "wrong-attachment-issue") attachment.issueId = "other-issue";
   if (reason === "wrong-type") product.type = "document";
-  if (reason === "wrong-title") product.title = "other.txt";
   if (reason === "wrong-filename") attachment.originalFilename = "other.txt";
   if (reason === "wrong-attachment-id") product.metadata.attachmentId = "other-attachment";
   if (reason === "negative-size") attachment.byteSize = -1;
@@ -49,6 +48,16 @@ it.each(["unregistered", "duplicate", "old-run", "unattributed", "wrong-issue",
   await expect(readRegisteredArtifacts(api, "issue", "run", ["proof.txt"])).rejects.toThrow();
 });
 
+it("identifies the deliverable by its run-attributed file, not its display title", async () => {
+  const { api, product, attachment, download } = fixture();
+  product.title = "Acorn Notes proof (proof.txt)";
+  expect((await readRegisteredArtifacts(api, "issue", "run", ["proof.txt"]))[0]?.contentVerified).toBe(true);
+  attachment.originalFilename = "other.txt";
+  download.mockClear();
+  await expect(readRegisteredArtifacts(api, "issue", "run", ["proof.txt"])).rejects.toBeInstanceOf(RegisteredArtifactNotFoundError);
+  expect(download).not.toHaveBeenCalled();
+});
+
 it("does not accept a filename or unchecked content as downloaded evidence", async () => {
   const matcher = { kind: "artifact_exact" as const, name: "proof.txt", expected: "verified\n" };
   expect((await evaluateMatcher(matcher, { artifacts: [{ name: "proof.txt", content: "verified\n" }] })).passed).toBe(false);

tests/runner-e2e/registered-artifact.ts

@@ -19,10 +19,14 @@ export async function readRegisteredArtifacts(api: RunnerApi, issueId: string, r
     api.get<IssueWorkProduct[]>(`/api/issues/${encodeURIComponent(issueId)}/work-products`),
     api.get<IssueAttachment[]>(`/api/issues/${encodeURIComponent(issueId)}/attachments`),
   ]);
+  // The file name identifies a deliverable; its work-product title is a
+  // human-facing label the agent may choose, so it is not part of identity.
+  const filenameOf = (product: IssueWorkProduct) =>
+    attachments.find(attachment => attachment.id === product.metadata?.attachmentId)?.originalFilename;
   return Promise.all(names.map(async name => {
     const registered = products.filter(product => product.issueId === issueId &&
       product.createdByRunId === runId && product.type === "artifact" &&
-      REGISTERED_ARTIFACT_STATUSES.has(product.status) && product.title === name);
+      REGISTERED_ARTIFACT_STATUSES.has(product.status) && (product.title === name || filenameOf(product) === name));
     if (registered.length === 0) throw new RegisteredArtifactNotFoundError(name);
     if (registered.length !== 1) throw new Error(`Expected one registered artifact ${name} from the tested run; observed ${registered.length}`);
     const product = registered[0]!;