refactor(core): narrow AI SDK media fallback

packages/core/src/aisdk.ts

@@ -435,7 +435,22 @@ function prompt(request: LLMRequest): LanguageModelV3Prompt {
     .map((part) => part.text)
     .filter(Boolean)
     .join("\n\n")
-  const messages = request.messages.flatMap(message)
+  const pending: UserContent = []
+  const messages = request.messages.flatMap((input, index) => {
+    if (input.role !== "tool") return message(input)
+    const lowered = toolMessage(input)
+    pending.push(...lowered.media)
+    if (request.messages[index + 1]?.role === "tool" || pending.length === 0) return lowered.messages
+    const media = [...pending]
+    pending.length = 0
+    return [
+      ...lowered.messages,
+      {
+        role: "user" as const,
+        content: [{ type: "text" as const, text: "Attached media from tool result:" }, ...media],
+      },
+    ]
+  })
   if (!system.length) return messages
   return [{ role: "system", content: system }, ...messages]
 }
@@ -448,10 +463,33 @@ function message(input: LLMRequest["messages"][number]): LanguageModelV3Message[
       return [{ role: "user", content: input.content.flatMap(userPart) }]
     case "assistant":
       return [{ role: "assistant", content: input.content.flatMap(assistantPart) }]
-    case "tool": {
-      const content = input.content.flatMap(toolResultPart)
-      return content.length ? [{ role: "tool", content }] : []
-    }
+    case "tool":
+      return toolMessage(input).messages
+  }
+}
+
+function toolMessage(input: LLMRequest["messages"][number]) {
+  const media: UserContent = []
+  const content = input.content.flatMap((part) => {
+    if (part.type !== "tool-result" || part.result.type !== "content") return toolResultPart(part)
+    const value = part.result.value.filter((item) => {
+      if (item.type !== "file") return true
+      if (!item.mime.startsWith("image/") && item.mime !== "application/pdf") return true
+      const data = /^data:[^;,]+(?:;[^,]*)*;base64,(.*)$/s.exec(item.uri)?.[1] ?? item.uri
+      media.push({ type: "file", mediaType: item.mime, data, filename: item.name })
+      return false
+    })
+    return toolResultPart({
+      ...part,
+      result:
+        value.length === 0
+          ? { type: "text", value: "Media attached in the following user message." }
+          : { ...part.result, value },
+    })
+  })
+  return {
+    messages: content.length ? ([{ role: "tool", content }] satisfies LanguageModelV3Message[]) : [],
+    media,
   }
 }
 

packages/core/src/session/model-request.ts

@@ -1,6 +1,6 @@
 export * as SessionModelRequest from "./model-request.js"
 
-import { LLM, Message, SystemPart, type ContentPart, type LLMRequest } from "@opencode-ai/ai"
+import { LLM, Message, SystemPart, type LLMRequest } from "@opencode-ai/ai"
 import type { StreamOptions } from "@opencode-ai/ai/route"
 import type { Content } from "@opencode-ai/schema/tool"
 import { Cause, Config, Context, Effect, Layer, Result } from "effect"
@@ -26,8 +26,6 @@ const IMAGE_BYTES_TRIGGER = 25 * 1024 * 1024 // 25 MiB
 const IMAGE_BYTES_TARGET = 15 * 1024 * 1024 // 15 MiB
 const IMAGE_REMOVED =
   "[This image was removed to reduce the request size and is no longer visible. Do not make claims about its contents from memory. If needed, retrieve it again with an available tool or ask the user to attach it again.]"
-const TOOL_RESULT_MEDIA_PROMPT = "Attached media from tool result:"
-const TOOL_RESULT_MEDIA_MOVED = "Media attached in the following user message."
 
 /** Failures a prepared execution can surface: infrastructure errors plus user declines resurfaced from the defect tunnel. */
 export type ExecuteError = Tool.Error | Permission.DeclinedError | QuestionTool.CancelledError
@@ -104,31 +102,6 @@ export const unsupportedParts = (messages: LLMRequest["messages"], capabilities:
     }),
   )
 
-export const toolResultMediaFallback = (messages: LLMRequest["messages"], protocol: string) => {
-  if (protocol !== "ai-sdk") return messages
-  return messages.flatMap((message) => {
-    const media: ContentPart[] = []
-    const content = message.content.map((part) => {
-      if (part.type !== "tool-result" || part.result.type !== "content") return part
-      const value = part.result.value.filter((item) => {
-        if (item.type !== "file") return true
-        if (!item.mime.startsWith("image/") && item.mime !== "application/pdf") return true
-        const data = /^data:[^;,]+(?:;[^,]*)*;base64,(.*)$/s.exec(item.uri)?.[1] ?? item.uri
-        media.push({ type: "media", mediaType: item.mime, data, filename: item.name })
-        return false
-      })
-      return {
-        ...part,
-        result:
-          value.length === 0 ? { type: "text" as const, value: TOOL_RESULT_MEDIA_MOVED } : { ...part.result, value },
-      }
-    })
-    const transformed = Message.make({ ...message, content })
-    if (media.length === 0) return [transformed]
-    return [transformed, Message.user([Message.text(TOOL_RESULT_MEDIA_PROMPT), ...media])]
-  })
-}
-
 export const boundImages = (messages: LLMRequest["messages"]) => {
   const isImage = (mime: string) => mime.toLowerCase().startsWith("image/")
   const size = (data: string | Uint8Array) =>
@@ -253,8 +226,6 @@ export const layer = Layer.effect(
           return [[name, { ...tool, description: definition.description, inputSchema: definition.input }] as const]
         }),
       )
-      const supported = unsupportedParts(context.messages, resolved.capabilities)
-      const preparedMessages = toolResultMediaFallback(supported, model.route.protocol)
       const request = LLM.request({
         model,
         http: {
@@ -262,7 +233,7 @@ export const layer = Layer.effect(
         },
         promptCacheKey: SessionPromptCacheKey.make(session.id),
         system: context.system,
-        messages: boundImages(preparedMessages),
+        messages: boundImages(unsupportedParts(context.messages, resolved.capabilities)),
         tools: Array.from(hooked, ([name, tool]) => ({ ...tool, name })),
         toolChoice: stepLimitReached ? "none" : undefined,
       })

packages/core/test/aisdk.test.ts

@@ -2,7 +2,6 @@ import { APICallError } from "@ai-sdk/provider"
 import type { LanguageModelV3, LanguageModelV3StreamPart } from "@ai-sdk/provider"
 import { createMistral } from "@ai-sdk/mistral"
 import { AISDK } from "@opencode-ai/core/aisdk"
-import { toolResultMediaFallback } from "@opencode-ai/core/session/model-request"
 import { SessionRunnerRetry } from "@opencode-ai/core/session/runner/retry"
 import { toSessionError } from "@opencode-ai/core/session/to-session-error"
 import { Model } from "@opencode-ai/core/model"
@@ -279,71 +278,7 @@ it.effect("projects replay metadata onto AI SDK prompt parts", () =>
   }),
 )
 
-it.effect("preserves multimodal messages in AI SDK prompts", () =>
-  Effect.gen(function* () {
-    const aisdk = yield* AISDK.Service
-    yield* aisdk.hook.sdk((event) => {
-      event.sdk = { languageModel: () => ({ provider: event.model.providerID }) }
-    })
-
-    const resolved = yield* aisdk.model(model("test-ai-sdk"))
-    const prepared = yield* compileRequest(
-      LLM.request({
-        model: resolved,
-        system: "Inspect every attachment.",
-        messages: [
-          Message.user([
-            { type: "text", text: "What is in these files?" },
-            { type: "media", mediaType: "image/png", data: "AAAA", filename: "pixel.png" },
-            {
-              type: "media",
-              mediaType: "application/pdf",
-              data: "https://example.com/document.pdf",
-              filename: "document.pdf",
-            },
-            { type: "media", mediaType: "audio/mpeg", data: new Uint8Array([73, 68, 51]), filename: "clip.mp3" },
-          ]),
-          Message.assistant([
-            { type: "text", text: "I found this reference image." },
-            { type: "media", mediaType: "image/webp", data: "BBBB", filename: "reference.webp" },
-          ]),
-        ],
-      }),
-    )
-
-    expect(prepared.body.prompt).toEqual([
-      { role: "system", content: "Inspect every attachment." },
-      {
-        role: "user",
-        content: [
-          { type: "text", text: "What is in these files?" },
-          { type: "file", mediaType: "image/png", data: "AAAA", filename: "pixel.png" },
-          {
-            type: "file",
-            mediaType: "application/pdf",
-            data: "https://example.com/document.pdf",
-            filename: "document.pdf",
-          },
-          {
-            type: "file",
-            mediaType: "audio/mpeg",
-            data: new Uint8Array([73, 68, 51]),
-            filename: "clip.mp3",
-          },
-        ],
-      },
-      {
-        role: "assistant",
-        content: [
-          { type: "text", text: "I found this reference image." },
-          { type: "file", mediaType: "image/webp", data: "BBBB", filename: "reference.webp" },
-        ],
-      },
-    ])
-  }),
-)
-
-it.effect("sends images through the real Mistral AI SDK provider", () =>
+it.effect("moves a tool image through the real Mistral provider as a user message", () =>
   Effect.gen(function* () {
     const aisdk = yield* AISDK.Service
     let body: { messages?: unknown[] } | undefined
@@ -383,154 +318,48 @@ it.effect("sends images through the real Mistral AI SDK provider", () =>
       LLM.request({
         model: resolved,
         messages: [
-          Message.user([
-            { type: "text", text: "What is in this image?" },
-            { type: "media", mediaType: "image/png", data: "AAAA", filename: "pixel.png" },
-          ]),
-        ],
-      }),
-    ).pipe(Effect.provide(client))
-
-    expect(body?.messages).toEqual([
-      {
-        role: "user",
-        content: [
-          { type: "text", text: "What is in this image?" },
-          { type: "image_url", image_url: "data:image/png;base64,AAAA" },
-        ],
-      },
-    ])
-  }),
-)
-
-it.effect("sends tool result images to Mistral as a following user message", () =>
-  Effect.gen(function* () {
-    const aisdk = yield* AISDK.Service
-    let body: { messages?: unknown[] } | undefined
-    const mockFetch = Object.assign(
-      async (_input: Parameters<typeof fetch>[0], init?: RequestInit) => {
-        body = JSON.parse(String(init?.body))
-        const chunks = [
-          {
-            id: "response-1",
-            created: 0,
-            model: "pixtral-large-latest",
-            choices: [{ index: 0, delta: { content: [{ type: "text", text: "I see it." }] } }],
-          },
-          {
-            id: "response-1",
-            created: 0,
-            model: "pixtral-large-latest",
-            choices: [{ index: 0, delta: {}, finish_reason: "stop" }],
-            usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 },
-          },
-        ]
-        return new Response(chunks.map((chunk) => `data: ${JSON.stringify(chunk)}\n\n`).join(""), {
-          headers: { "Content-Type": "text/event-stream" },
-        })
-      },
-      { preconnect: fetch.preconnect },
-    )
-    yield* aisdk.hook.sdk((event) => {
-      event.sdk = createMistral({ apiKey: "test", fetch: mockFetch })
-    })
-
-    const resolved = yield* aisdk.model({
-      ...model("@ai-sdk/mistral"),
-      modelID: Model.ID.make("pixtral-large-latest"),
-    })
-    const messages = toolResultMediaFallback(
-      [
-        Message.user("Capture the screen."),
-        Message.assistant({ type: "tool-call", id: "call_1", name: "screenshot", input: {} }),
-        Message.tool({
-          type: "tool-result",
-          id: "call_1",
-          name: "screenshot",
-          result: {
-            type: "content",
-            value: [
-              { type: "text", text: "Image captured" },
-              { type: "file", uri: "data:image/png;base64,AAAA", mime: "image/png", name: "screen.png" },
-            ],
-          },
-        }),
-      ],
-      "ai-sdk",
-    )
-    yield* LLMClient.generate(LLM.request({ model: resolved, messages })).pipe(Effect.provide(client))
-
-    expect(body?.messages?.at(-1)).toEqual({
-      role: "user",
-      content: [
-        { type: "text", text: "Attached media from tool result:" },
-        { type: "image_url", image_url: "data:image/png;base64,AAAA" },
-      ],
-    })
-  }),
-)
-
-it.effect("preserves tool result content in AI SDK prompts", () =>
-  Effect.gen(function* () {
-    const aisdk = yield* AISDK.Service
-    yield* aisdk.hook.sdk((event) => {
-      event.sdk = { languageModel: () => ({ provider: event.model.providerID }) }
-    })
-
-    const resolved = yield* aisdk.model(model("test-ai-sdk"))
-    const prepared = yield* compileRequest(
-      LLM.request({
-        model: resolved,
-        messages: [
+          Message.user("Inspect the screenshot."),
+          Message.assistant({ type: "tool-call", id: "call_1", name: "screenshot", input: {} }),
           Message.tool({
+            type: "tool-result",
             id: "call_1",
-            name: "read",
+            name: "screenshot",
             result: {
               type: "content",
               value: [
-                { type: "text", text: "attachments" },
-                { type: "file", uri: "data:image/png;base64,AAAA", mime: "image/png", name: "pixel.png" },
-                {
-                  type: "file",
-                  uri: "data:application/pdf;charset=utf-8;base64,JVBERg==",
-                  mime: "application/pdf",
-                  name: "document.pdf",
-                },
-                { type: "file", uri: "data:audio/mpeg;base64,SUQz", mime: "audio/mpeg", name: "clip.mp3" },
-                { type: "file", uri: "https://example.com/pixel.png", mime: "image/png" },
-                { type: "file", uri: "https://example.com/document.pdf", mime: "application/pdf" },
+                { type: "text", text: "Screenshot captured" },
+                { type: "file", uri: "data:image/png;base64,AAAA", mime: "image/png", name: "screen.png" },
               ],
             },
           }),
         ],
       }),
-    )
+    ).pipe(Effect.provide(client))
 
-    expect(prepared.body.prompt).toEqual([
+    expect(body?.messages).toEqual([
+      { role: "user", content: [{ type: "text", text: "Inspect the screenshot." }] },
+      {
+        role: "assistant",
+        content: "",
+        tool_calls: [
+          {
+            id: "call_1",
+            type: "function",
+            function: { name: "screenshot", arguments: "{}" },
+          },
+        ],
+      },
       {
         role: "tool",
+        name: "screenshot",
+        tool_call_id: "call_1",
+        content: '[{"type":"text","text":"Screenshot captured"}]',
+      },
+      {
+        role: "user",
         content: [
-          {
-            type: "tool-result",
-            toolCallId: "call_1",
-            toolName: "read",
-            output: {
-              type: "content",
-              value: [
-                { type: "text", text: "attachments" },
-                { type: "image-data", data: "AAAA", mediaType: "image/png" },
-                {
-                  type: "file-data",
-                  data: "JVBERg==",
-                  mediaType: "application/pdf",
-                  filename: "document.pdf",
-                },
-                { type: "file-data", data: "SUQz", mediaType: "audio/mpeg", filename: "clip.mp3" },
-                { type: "image-url", url: "https://example.com/pixel.png" },
-                { type: "file-url", url: "https://example.com/document.pdf" },
-              ],
-            },
-          },
+          { type: "text", text: "Attached media from tool result:" },
+          { type: "image_url", image_url: "data:image/png;base64,AAAA" },
         ],
       },
     ])

packages/core/test/session-model-request.test.ts

@@ -1,6 +1,6 @@
 import { describe, expect, test } from "bun:test"
 import { Message, ToolResultPart } from "@opencode-ai/ai"
-import { boundImages, toolResultMediaFallback, unsupportedParts } from "@opencode-ai/core/session/model-request"
+import { boundImages, unsupportedParts } from "@opencode-ai/core/session/model-request"
 
 const capabilities = (input: string[]) => ({ tools: true, input, output: ["text"] })
 
@@ -110,95 +110,3 @@ describe("SessionModelRequest.boundImages", () => {
     })
   })
 })
-
-describe("SessionModelRequest.toolResultMediaFallback", () => {
-  test("moves image and PDF tool results into a following user message", () => {
-    const messages = toolResultMediaFallback(
-      [
-        Message.tool(
-          ToolResultPart.make({
-            id: "call_1",
-            name: "read",
-            result: {
-              type: "content",
-              value: [
-                { type: "text", text: "Attachments read successfully" },
-                { type: "file", uri: "data:image/png;base64,AAAA", mime: "image/png", name: "pixel.png" },
-                {
-                  type: "file",
-                  uri: "data:application/pdf;base64,JVBERg==",
-                  mime: "application/pdf",
-                  name: "document.pdf",
-                },
-                { type: "file", uri: "data:audio/mpeg;base64,SUQz", mime: "audio/mpeg", name: "clip.mp3" },
-              ],
-            },
-          }),
-        ),
-      ],
-      "ai-sdk",
-    )
-
-    expect(messages).toEqual([
-      Message.tool(
-        ToolResultPart.make({
-          id: "call_1",
-          name: "read",
-          result: {
-            type: "content",
-            value: [
-              { type: "text", text: "Attachments read successfully" },
-              { type: "file", uri: "data:audio/mpeg;base64,SUQz", mime: "audio/mpeg", name: "clip.mp3" },
-            ],
-          },
-        }),
-      ),
-      Message.user([
-        Message.text("Attached media from tool result:"),
-        { type: "media", mediaType: "image/png", data: "AAAA", filename: "pixel.png" },
-        { type: "media", mediaType: "application/pdf", data: "JVBERg==", filename: "document.pdf" },
-      ]),
-    ])
-  })
-
-  test("leaves a valid tool result when all content is moved", () => {
-    const messages = toolResultMediaFallback(
-      [
-        Message.tool(
-          ToolResultPart.make({
-            id: "call_1",
-            name: "screenshot",
-            result: {
-              type: "content",
-              value: [{ type: "file", uri: "data:image/png;base64,AAAA", mime: "image/png" }],
-            },
-          }),
-        ),
-      ],
-      "ai-sdk",
-    )
-
-    expect(messages[0]?.content[0]).toMatchObject({
-      type: "tool-result",
-      result: { type: "text", value: "Media attached in the following user message." },
-    })
-    expect(messages[1]?.role).toBe("user")
-  })
-
-  test("leaves native protocol tool results untouched", () => {
-    const messages = [
-      Message.tool(
-        ToolResultPart.make({
-          id: "call_1",
-          name: "screenshot",
-          result: {
-            type: "content",
-            value: [{ type: "file", uri: "data:image/png;base64,AAAA", mime: "image/png" }],
-          },
-        }),
-      ),
-    ]
-
-    expect(toolResultMediaFallback(messages, "openai-chat")).toBe(messages)
-  })
-})