Compare commits

...

5 Commits

Author SHA1 Message Date
Aiden Cline 34585dce41 Merge branch 'v2' into empty-variants 2026-08-16 16:03:04 -05:00
Aiden Cline ec10d71f22 fix(core): keep plan mode reminders in sync with session agent (#42652) 2026-08-16 16:01:02 -05:00
opencode-agent[bot] ca589273c7 fix(core): preserve tool-result images for AI SDK providers (#42849)
Co-authored-by: Aiden Cline <rekram1-node@users.noreply.github.com>
Co-authored-by: Aiden Cline <63023139+rekram1-node@users.noreply.github.com>
2026-08-16 15:20:37 -05:00
opencode-agent[bot] 75a979ec5c fix(ai): pass media through protocol lowering (#42921)
Co-authored-by: Aiden Cline <rekram1-node@users.noreply.github.com>
2026-08-16 15:18:43 -05:00
Aiden Cline 654f4ede55 fix(tui): drop unavailable saved variants 2026-08-16 04:14:49 +00:00
17 changed files with 458 additions and 177 deletions
@@ -30,7 +30,6 @@ import { ToolSchemaProjection } from "./utils/tool-schema.js"
import { ToolStream } from "./utils/tool-stream.js"
const ADAPTER = "anthropic-messages"
const MEDIA_MIMES = new Set<string>([...ProviderShared.IMAGE_MIMES, ...ProviderShared.PDF_MIMES])
export const DEFAULT_BASE_URL = "https://api.anthropic.com/v1"
export const PATH = "/messages"
@@ -400,7 +399,7 @@ const lowerServerToolResult = Effect.fn("AnthropicMessages.lowerServerToolResult
})
const lowerMedia = Effect.fn("AnthropicMessages.lowerMedia")(function* (part: MediaPart) {
const media = yield* ProviderShared.validateMedia("Anthropic Messages", part, MEDIA_MIMES)
const media = ProviderShared.normalizeMedia(part)
if (media.mime === "application/pdf")
return {
type: "document" as const,
@@ -410,6 +409,8 @@ const lowerMedia = Effect.fn("AnthropicMessages.lowerMedia")(function* (part: Me
data: media.base64,
},
} satisfies AnthropicDocumentBlock
if (!media.mime.startsWith("image/"))
return yield* invalid(`Anthropic Messages does not support media type ${part.mediaType}`)
return {
type: "image" as const,
source: {
+2 -3
View File
@@ -24,7 +24,6 @@ import { Lifecycle } from "./utils/lifecycle.js"
import { ToolSchemaProjection } from "./utils/tool-schema.js"
const ADAPTER = "gemini"
const MEDIA_MIMES = new Set<string>(ProviderShared.MEDIA_MIMES)
// Google documents this sentinel for replaying Gemini 3 function calls after their original signature was lost.
const SKIP_THOUGHT_SIGNATURE_VALIDATOR = "skip_thought_signature_validator"
export const DEFAULT_BASE_URL = "https://generativelanguage.googleapis.com/v1beta"
@@ -248,7 +247,7 @@ const lowerToolConfig = (toolChoice: NonNullable<LLMRequest["toolChoice"]>) =>
const lowerUserPart = Effect.fn("Gemini.lowerUserPart")(function* (part: TextPart | MediaPart) {
if (part.type === "text") return { text: part.text }
const media = yield* ProviderShared.validateMedia("Gemini", part, MEDIA_MIMES)
const media = ProviderShared.normalizeMedia(part)
return { inlineData: { mimeType: media.mime, data: media.base64 } }
})
@@ -353,7 +352,7 @@ const lowerMessages = Effect.fn("Gemini.lowerMessages")(function* (request: LLMR
const media: GeminiInlineDataPart[] = []
for (const item of content) {
if (item.type === "text") continue
const value = yield* ProviderShared.validateToolFile("Gemini", item, MEDIA_MIMES)
const value = ProviderShared.normalizeToolFile(item)
media.push({ inlineData: { mimeType: value.mime, data: value.base64 } })
}
parts.push({
+4 -5
View File
@@ -26,7 +26,6 @@ import { ToolStream } from "./utils/tool-stream.js"
const ADAPTER = "open-responses"
const NAME = "Open Responses"
const MEDIA_MIMES = new Set<string>([...ProviderShared.IMAGE_MIMES, ...ProviderShared.PDF_MIMES])
export const PATH = "/responses"
// =============================================================================
@@ -285,7 +284,7 @@ export interface Extension {
readonly name: string
readonly lowerMedia?: (input: {
readonly part: MediaPart
readonly media: ProviderShared.ValidatedMedia
readonly media: ProviderShared.NormalizedMedia
readonly request: LLMRequest
}) => MediaInput | undefined
readonly messagePhase?: (value: unknown) => MessagePhase | null | undefined
@@ -380,13 +379,13 @@ const lowerMedia = Effect.fn("OpenResponses.lowerMedia")(function* (
request: LLMRequest,
extension: Extension,
) {
const media = yield* ProviderShared.validateMedia(extension.name, part, MEDIA_MIMES)
const media = ProviderShared.normalizeMedia(part)
const extended = extension.lowerMedia?.({ part, media, request })
if (extended) return extended
if (media.mime === "application/pdf") {
if (!media.mime.startsWith("image/")) {
return {
type: "input_file" as const,
filename: part.filename ?? "document.pdf",
filename: part.filename ?? (media.mime === "application/pdf" ? "document.pdf" : "file"),
file_data: media.dataUrl,
}
}
+3 -2
View File
@@ -28,7 +28,6 @@ import { ToolSchemaProjection } from "./utils/tool-schema.js"
import { ToolStream } from "./utils/tool-stream.js"
const ADAPTER = "openai-chat"
const IMAGE_MIMES = new Set<string>(ProviderShared.IMAGE_MIMES)
const RESERVED_REASONING_FIELDS = new Set(["role", "content", "tool_calls"])
export const DEFAULT_BASE_URL = "https://api.openai.com/v1"
export const PATH = "/chat/completions"
@@ -284,7 +283,9 @@ const lowerToolCall = (part: ToolCallPart): OpenAIChatAssistantToolCall => ({
})
const lowerMedia = Effect.fn("OpenAIChat.lowerMedia")(function* (part: MediaPart) {
const media = yield* ProviderShared.validateMedia("OpenAI Chat", part, IMAGE_MIMES)
const media = ProviderShared.normalizeMedia(part)
if (!media.mime.startsWith("image/"))
return yield* ProviderShared.invalidRequest(`OpenAI Chat does not support media type ${part.mediaType}`)
return { type: "image_url" as const, image_url: { url: media.dataUrl } }
})
+9 -44
View File
@@ -155,59 +155,24 @@ export const wrappedSystemUpdate = Effect.fn("ProviderShared.wrappedSystemUpdate
export const parseToolInput = (route: string, name: string, raw: string) =>
parseJson(route, raw || "{}", `Invalid JSON input for ${route} tool call ${name}`)
export const IMAGE_MIMES = ["image/png", "image/jpeg", "image/gif", "image/webp"] as const
export const VIDEO_MIMES = ["video/mp4", "video/webm", "video/quicktime"] as const
export const AUDIO_MIMES = ["audio/wav", "audio/mp3", "audio/aiff", "audio/aac", "audio/ogg", "audio/flac"] as const
export const PDF_MIMES = ["application/pdf"] as const
export const MEDIA_MIMES = [...IMAGE_MIMES, ...VIDEO_MIMES, ...AUDIO_MIMES, ...PDF_MIMES] as const
export const MAX_MEDIA_ENCODED_BYTES = 28 * 1024 * 1024
export const MAX_MEDIA_DECODED_BYTES = 20 * 1024 * 1024
const base64Pattern = /^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/
export interface ValidatedMedia {
export interface NormalizedMedia {
readonly mime: string
readonly base64: string
readonly dataUrl: string
readonly bytes: Uint8Array
}
export const validateMedia = Effect.fn("ProviderShared.validateMedia")(function* (
route: string,
part: MediaPart,
supportedMimes: ReadonlySet<string>,
) {
export const normalizeMedia = (part: MediaPart): NormalizedMedia => {
const mime = part.mediaType.toLowerCase()
if (!supportedMimes.has(mime)) return yield* invalidRequest(`${route} does not support media type ${part.mediaType}`)
let base64: string
if (typeof part.data !== "string") {
if (part.data.byteLength > MAX_MEDIA_DECODED_BYTES)
return yield* invalidRequest(`${route} media exceeds the ${MAX_MEDIA_DECODED_BYTES} byte decoded limit`)
base64 = Buffer.from(part.data).toString("base64")
} else if (part.data.startsWith("data:")) {
const match = /^data:([^;,]+);base64,([A-Za-z0-9+/]*={0,2})$/s.exec(part.data)
if (!match) return yield* invalidRequest(`${route} media data URL must contain valid base64`)
if (match[1]!.toLowerCase() !== mime)
return yield* invalidRequest(`${route} media type ${part.mediaType} does not match data URL type ${match[1]}`)
base64 = match[2]!
} else {
base64 = part.data
const base64 = Buffer.from(part.data).toString("base64")
return { mime, base64, dataUrl: `data:${mime};base64,${base64}` }
}
if (!part.data.startsWith("data:")) return { mime, base64: part.data, dataUrl: `data:${mime};base64,${part.data}` }
return { mime, base64: part.data.slice(part.data.indexOf(",") + 1), dataUrl: part.data }
}
if (Buffer.byteLength(base64, "utf8") > MAX_MEDIA_ENCODED_BYTES)
return yield* invalidRequest(`${route} media exceeds the ${MAX_MEDIA_ENCODED_BYTES} byte encoded limit`)
if (!base64 || base64.length % 4 !== 0 || !base64Pattern.test(base64))
return yield* invalidRequest(`${route} media must contain valid base64`)
const bytes = Buffer.from(base64, "base64")
if (bytes.byteLength > MAX_MEDIA_DECODED_BYTES)
return yield* invalidRequest(`${route} media exceeds the ${MAX_MEDIA_DECODED_BYTES} byte decoded limit`)
if (bytes.toString("base64") !== base64) return yield* invalidRequest(`${route} media must contain canonical base64`)
return { mime, base64, dataUrl: `data:${mime};base64,${base64}`, bytes } satisfies ValidatedMedia
})
export const validateToolFile = (route: string, part: Tool.FileContent, supportedMimes: ReadonlySet<string>) =>
validateMedia(route, { type: "media", mediaType: part.mime, data: part.uri, filename: part.name }, supportedMimes)
export const normalizeToolFile = (part: Tool.FileContent) =>
normalizeMedia({ type: "media", mediaType: part.mime, data: part.uri, filename: part.name })
export const trimBaseUrl = (value: string) => value.replace(/\/+$/, "")
@@ -66,11 +66,7 @@ export const lower = Effect.fn("BedrockMedia.lower")(function* (part: MediaPart)
const mime = part.mediaType.toLowerCase()
const imageFormat = IMAGE_FORMATS[mime as keyof typeof IMAGE_FORMATS]
if (imageFormat) {
const media = yield* ProviderShared.validateMedia(
"Bedrock Converse",
part,
new Set<string>(Object.keys(IMAGE_FORMATS)),
)
const media = ProviderShared.normalizeMedia(part)
return { image: { format: imageFormat, source: { bytes: media.base64 } } } satisfies ImageBlock
}
if (mime.startsWith("image/"))
@@ -79,11 +75,7 @@ export const lower = Effect.fn("BedrockMedia.lower")(function* (part: MediaPart)
if (documentFormat) {
if (!part.filename)
return yield* ProviderShared.invalidRequest("Bedrock Converse document media requires a filename")
const media = yield* ProviderShared.validateMedia(
"Bedrock Converse",
part,
new Set<string>(Object.keys(DOCUMENT_FORMATS)),
)
const media = ProviderShared.normalizeMedia(part)
return documentBlock(part.filename, documentFormat, media.base64)
}
return yield* ProviderShared.invalidRequest(`Bedrock Converse does not support media type ${part.mediaType}`)
@@ -399,7 +399,7 @@ describe("Anthropic Messages route", () => {
}),
)
it.effect("rejects unsupported media in tool-result content with a clear error", () =>
it.effect("rejects tool-result media that cannot be lowered", () =>
Effect.gen(function* () {
const error = yield* compileRequest(
LLM.request({
@@ -418,8 +418,7 @@ describe("Anthropic Messages route", () => {
}),
).pipe(Effect.flip)
expect(error.message).toContain("Anthropic Messages")
expect(error.message).toContain("audio/mpeg")
expect(error.message).toContain("Anthropic Messages does not support media type audio/mpeg")
}),
)
+18 -24
View File
@@ -4,7 +4,6 @@ import { LLM, AIError, LLMRequest, Message, ToolCallPart, ToolDefinition, Usage
import { Auth, LLMClient } from "../../src/route.js"
import { compileRequest } from "../../src/route/client.js"
import * as Gemini from "../../src/protocols/gemini.js"
import { ProviderShared } from "../../src/protocols/shared.js"
import { it } from "../lib/effect.js"
import { fixedResponse } from "../lib/http.js"
import { sseEvents, sseRaw } from "../lib/sse.js"
@@ -291,35 +290,30 @@ describe("Gemini route", () => {
}),
)
for (const [name, media] of [
["mismatched data URL MIME", { mediaType: "image/png", data: "data:image/jpeg;base64,/9j/" }],
["malformed base64", { mediaType: "image/png", data: "%%%=" }],
["unsupported SVG", { mediaType: "image/svg+xml", data: "PHN2Zz4=" }],
] as const)
it.effect(`rejects ${name}`, () =>
Effect.gen(function* () {
const error = yield* compileRequest(
LLM.request({ model, messages: [Message.user({ type: "media", ...media })] }),
).pipe(Effect.flip)
expect(error.message).toMatch(/does not support|does not match|valid base64/)
}),
)
it.effect("rejects oversized image input", () =>
it.effect("passes encoded media through without local validation", () =>
Effect.gen(function* () {
const error = yield* compileRequest(
const prepared = yield* compileRequest(
LLM.request({
model,
messages: [
Message.user({
type: "media",
mediaType: "image/png",
data: "A".repeat(ProviderShared.MAX_MEDIA_ENCODED_BYTES + 4),
}),
Message.user([
{ type: "media", mediaType: "image/png", data: "%%%=" },
{ type: "media", mediaType: "image/png", data: "data:image/jpeg;base64,/9j/" },
{ type: "media", mediaType: "image/svg+xml", data: "PHN2Zz4=" },
]),
],
}),
).pipe(Effect.flip)
expect(error.message).toContain("encoded limit")
)
expect(prepared.body.contents).toEqual([
{
role: "user",
parts: [
{ inlineData: { mimeType: "image/png", data: "%%%=" } },
{ inlineData: { mimeType: "image/png", data: "/9j/" } },
{ inlineData: { mimeType: "image/svg+xml", data: "PHN2Zz4=" } },
],
},
])
}),
)
+29 -22
View File
@@ -527,35 +527,42 @@ describe("OpenAI Chat route", () => {
}),
)
for (const [name, media] of [
["mismatched data URL MIME", { mediaType: "image/png", data: "data:image/jpeg;base64,/9j/" }],
["malformed base64", { mediaType: "image/png", data: "not-base64" }],
["unsupported SVG", { mediaType: "image/svg+xml", data: "PHN2Zz4=" }],
] as const)
it.effect(`rejects ${name}`, () =>
Effect.gen(function* () {
const error = yield* compileRequest(
LLM.request({ model, messages: [Message.user({ type: "media", ...media })] }),
).pipe(Effect.flip)
expect(error.message).toMatch(/does not support|does not match|valid base64/)
}),
)
it.effect("passes encoded image media through without local validation", () =>
Effect.gen(function* () {
const prepared = yield* compileRequest(
LLM.request({
model,
messages: [
Message.user([
{ type: "media", mediaType: "image/png", data: "not-base64" },
{ type: "media", mediaType: "image/png", data: "data:image/jpeg;base64,/9j/" },
{ type: "media", mediaType: "image/svg+xml", data: "PHN2Zz4=" },
]),
],
}),
)
expect(prepared.body.messages).toEqual([
{
role: "user",
content: [
{ type: "image_url", image_url: { url: "data:image/png;base64,not-base64" } },
{ type: "image_url", image_url: { url: "data:image/jpeg;base64,/9j/" } },
{ type: "image_url", image_url: { url: "data:image/svg+xml;base64,PHN2Zz4=" } },
],
},
])
}),
)
it.effect("rejects oversized image input", () =>
it.effect("rejects non-image media that cannot be lowered", () =>
Effect.gen(function* () {
const error = yield* compileRequest(
LLM.request({
model,
messages: [
Message.user({
type: "media",
mediaType: "image/png",
data: "A".repeat(ProviderShared.MAX_MEDIA_ENCODED_BYTES + 4),
}),
],
messages: [Message.user({ type: "media", mediaType: "audio/mpeg", data: "AAECAw==" })],
}),
).pipe(Effect.flip)
expect(error.message).toContain("encoded limit")
expect(error.message).toContain("OpenAI Chat does not support media type audio/mpeg")
}),
)
@@ -1149,6 +1149,32 @@ describe("OpenAI Responses route", () => {
}),
)
it.effect("passes large PDF tool-result content through", () =>
Effect.gen(function* () {
const base64 = "A".repeat(8_125_844)
const dataUrl = `data:application/pdf;base64,${base64}`
const prepared = yield* compileRequest(
LLM.request({
id: "req_tool_result_large_pdf",
model,
messages: [
Message.assistant([ToolCallPart.make({ id: "call_1", name: "read", input: {} })]),
Message.tool({
id: "call_1",
name: "read",
resultType: "content",
result: [{ type: "file", uri: dataUrl, mime: "application/pdf", name: "report.pdf" }],
}),
],
}),
)
expect(expectToolOutput(prepared.body).output).toEqual([
{ type: "input_file", filename: "report.pdf", file_data: dataUrl },
])
}),
)
it.effect("uses xAI inline file encoding for PDF tool results", () =>
Effect.gen(function* () {
const prepared = yield* compileRequest(
@@ -1184,9 +1210,9 @@ describe("OpenAI Responses route", () => {
}),
)
it.effect("rejects unsupported media in tool-result content with a clear error", () =>
it.effect("passes non-image tool-result content through as an input file", () =>
Effect.gen(function* () {
const error = yield* compileRequest(
const prepared = yield* compileRequest(
LLM.request({
id: "req_tool_result_unsupported_media",
model,
@@ -1200,10 +1226,11 @@ describe("OpenAI Responses route", () => {
}),
],
}),
).pipe(Effect.flip)
)
expect(error.message).toContain("OpenAI Responses")
expect(error.message).toContain("audio/mpeg")
expect(expectToolOutput(prepared.body).output).toEqual([
{ type: "input_file", filename: "file", file_data: "data:audio/mpeg;base64,AAECAw==" },
])
}),
)
@@ -2394,17 +2421,28 @@ describe("OpenAI Responses route", () => {
}),
)
it.effect("rejects unsupported user media content", () =>
it.effect("passes non-image user media through as an input file", () =>
Effect.gen(function* () {
const error = yield* compileRequest(
const prepared = yield* compileRequest(
LLM.request({
id: "req_media",
model,
messages: [Message.user({ type: "media", mediaType: "application/x-tar", data: "AAECAw==" })],
}),
).pipe(Effect.flip)
)
expect(error.message).toContain("OpenAI Responses does not support media type application/x-tar")
expect(prepared.body.input).toEqual([
{
role: "user",
content: [
{
type: "input_file",
filename: "file",
file_data: "data:application/x-tar;base64,AAECAw==",
},
],
},
])
}),
)
+43 -5
View File
@@ -435,7 +435,22 @@ function prompt(request: LLMRequest): LanguageModelV3Prompt {
.map((part) => part.text)
.filter(Boolean)
.join("\n\n")
const messages = request.messages.flatMap(message)
const pending: UserContent = []
const messages = request.messages.flatMap((input, index) => {
if (input.role !== "tool") return message(input)
const lowered = toolMessage(input)
pending.push(...lowered.media)
if (request.messages[index + 1]?.role === "tool" || pending.length === 0) return lowered.messages
const media = [...pending]
pending.length = 0
return [
...lowered.messages,
{
role: "user" as const,
content: [{ type: "text" as const, text: "Attached media from tool result:" }, ...media],
},
]
})
if (!system.length) return messages
return [{ role: "system", content: system }, ...messages]
}
@@ -448,10 +463,33 @@ function message(input: LLMRequest["messages"][number]): LanguageModelV3Message[
return [{ role: "user", content: input.content.flatMap(userPart) }]
case "assistant":
return [{ role: "assistant", content: input.content.flatMap(assistantPart) }]
case "tool": {
const content = input.content.flatMap(toolResultPart)
return content.length ? [{ role: "tool", content }] : []
}
case "tool":
return toolMessage(input).messages
}
}
function toolMessage(input: LLMRequest["messages"][number]) {
const media: UserContent = []
const content = input.content.flatMap((part) => {
if (part.type !== "tool-result" || part.result.type !== "content") return toolResultPart(part)
const value = part.result.value.filter((item) => {
if (item.type !== "file") return true
if (!item.mime.startsWith("image/") && item.mime !== "application/pdf") return true
const data = /^data:[^;,]+(?:;[^,]*)*;base64,(.*)$/s.exec(item.uri)?.[1] ?? item.uri
media.push({ type: "file", mediaType: item.mime, data, filename: item.name })
return false
})
return toolResultPart({
...part,
result:
value.length === 0
? { type: "text", value: "Media attached in the following user message." }
: { ...part.result, value },
})
})
return {
messages: content.length ? ([{ role: "tool", content }] satisfies LanguageModelV3Message[]) : [],
media,
}
}
+31 -3
View File
@@ -1,6 +1,6 @@
export * as PlanPlugin from "./plan.js"
import { ToolFailure } from "@opencode-ai/ai"
import { Message, ToolFailure } from "@opencode-ai/ai"
import { define } from "@opencode-ai/plugin/effect/plugin"
import { Effect, Stream } from "effect"
import { Agent } from "../agent.js"
@@ -38,13 +38,33 @@ export const Plugin = define({
})
})
// Compaction and committed reverts can strip reminders while the session's agent stays
// put. Reconcile per request, appending near the tail so the cached prefix stays warm.
yield* ctx.session.hook("context", (event) => {
const reminder = lastReminder(event.messages)
const missing = event.agent === plan && reminder !== enter
const stale = event.agent !== plan && reminder === enter
const text = missing ? enter : stale ? leave : undefined
if (!text) return Effect.void
// Before the user's prompt, matching where agent-switch reminders land.
const at = event.messages.at(-1)?.role === "user" ? event.messages.length - 1 : event.messages.length
event.messages.splice(at, 0, Message.user(text))
return ctx.session
.synthetic({ sessionID: event.sessionID, text, resume: false })
.pipe(
Effect.catchCause((cause) =>
Effect.logWarning("failed to persist Plan mode reminder", { sessionID: event.sessionID, cause }),
),
)
})
yield* ctx.event.subscribe().pipe(
Stream.filter(
(event): event is SessionEvent.Created | SessionEvent.AgentSelected =>
event.type === "session.created" || event.type === "session.agent.selected",
),
Stream.runForEach((event) => {
const text = reminder(event)
const text = switchReminder(event)
if (!text) return Effect.void
return ctx.session
.synthetic({
@@ -63,7 +83,7 @@ export const Plugin = define({
}),
})
function reminder(event: SessionEvent.Created | SessionEvent.AgentSelected) {
function switchReminder(event: SessionEvent.Created | SessionEvent.AgentSelected) {
if (event.type === "session.created") {
if (event.data.agent !== plan) return
return enter
@@ -72,3 +92,11 @@ function reminder(event: SessionEvent.Created | SessionEvent.AgentSelected) {
if (event.data.agent === plan) return enter
if (event.data.previous === plan) return leave
}
function lastReminder(messages: ReadonlyArray<Message>) {
return messages.reduce<string | undefined>((found, message) => {
const part = message.role === "user" && message.content.length === 1 ? message.content[0] : undefined
if (part?.type !== "text") return found
return part.text === enter || part.text === leave ? part.text : found
}, undefined)
}
+61 -39
View File
@@ -1,5 +1,6 @@
import { APICallError } from "@ai-sdk/provider"
import type { LanguageModelV3, LanguageModelV3StreamPart } from "@ai-sdk/provider"
import { createMistral } from "@ai-sdk/mistral"
import { AISDK } from "@opencode-ai/core/aisdk"
import { SessionRunnerRetry } from "@opencode-ai/core/session/runner/retry"
import { toSessionError } from "@opencode-ai/core/session/to-session-error"
@@ -277,67 +278,88 @@ it.effect("projects replay metadata onto AI SDK prompt parts", () =>
}),
)
it.effect("preserves tool result content in AI SDK prompts", () =>
it.effect("moves a tool image through the real Mistral provider as a user message", () =>
Effect.gen(function* () {
const aisdk = yield* AISDK.Service
let body: { messages?: unknown[] } | undefined
const mockFetch = Object.assign(
async (_input: Parameters<typeof fetch>[0], init?: RequestInit) => {
body = JSON.parse(String(init?.body))
const chunks = [
{
id: "response-1",
created: 0,
model: "pixtral-large-latest",
choices: [{ index: 0, delta: { content: [{ type: "text", text: "I see it." }] } }],
},
{
id: "response-1",
created: 0,
model: "pixtral-large-latest",
choices: [{ index: 0, delta: {}, finish_reason: "stop" }],
usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 },
},
]
return new Response(chunks.map((chunk) => `data: ${JSON.stringify(chunk)}\n\n`).join(""), {
headers: { "Content-Type": "text/event-stream" },
})
},
{ preconnect: fetch.preconnect },
)
yield* aisdk.hook.sdk((event) => {
event.sdk = { languageModel: () => ({ provider: event.model.providerID }) }
event.sdk = createMistral({ apiKey: "test", fetch: mockFetch })
})
const resolved = yield* aisdk.model(model("test-ai-sdk"))
const prepared = yield* compileRequest(
const resolved = yield* aisdk.model({
...model("@ai-sdk/mistral"),
modelID: Model.ID.make("pixtral-large-latest"),
})
yield* LLMClient.generate(
LLM.request({
model: resolved,
messages: [
Message.user("Inspect the screenshot."),
Message.assistant({ type: "tool-call", id: "call_1", name: "screenshot", input: {} }),
Message.tool({
type: "tool-result",
id: "call_1",
name: "read",
name: "screenshot",
result: {
type: "content",
value: [
{ type: "text", text: "attachments" },
{ type: "file", uri: "data:image/png;base64,AAAA", mime: "image/png", name: "pixel.png" },
{
type: "file",
uri: "data:application/pdf;charset=utf-8;base64,JVBERg==",
mime: "application/pdf",
name: "document.pdf",
},
{ type: "file", uri: "data:audio/mpeg;base64,SUQz", mime: "audio/mpeg", name: "clip.mp3" },
{ type: "file", uri: "https://example.com/pixel.png", mime: "image/png" },
{ type: "file", uri: "https://example.com/document.pdf", mime: "application/pdf" },
{ type: "text", text: "Screenshot captured" },
{ type: "file", uri: "data:image/png;base64,AAAA", mime: "image/png", name: "screen.png" },
],
},
}),
],
}),
)
).pipe(Effect.provide(client))
expect(prepared.body.prompt).toEqual([
expect(body?.messages).toEqual([
{ role: "user", content: [{ type: "text", text: "Inspect the screenshot." }] },
{
role: "assistant",
content: "",
tool_calls: [
{
id: "call_1",
type: "function",
function: { name: "screenshot", arguments: "{}" },
},
],
},
{
role: "tool",
name: "screenshot",
tool_call_id: "call_1",
content: '[{"type":"text","text":"Screenshot captured"}]',
},
{
role: "user",
content: [
{
type: "tool-result",
toolCallId: "call_1",
toolName: "read",
output: {
type: "content",
value: [
{ type: "text", text: "attachments" },
{ type: "image-data", data: "AAAA", mediaType: "image/png" },
{
type: "file-data",
data: "JVBERg==",
mediaType: "application/pdf",
filename: "document.pdf",
},
{ type: "file-data", data: "SUQz", mediaType: "audio/mpeg", filename: "clip.mp3" },
{ type: "image-url", url: "https://example.com/pixel.png" },
{ type: "file-url", url: "https://example.com/document.pdf" },
],
},
},
{ type: "text", text: "Attached media from tool result:" },
{ type: "image_url", image_url: "data:image/png;base64,AAAA" },
],
},
])
+180
View File
@@ -0,0 +1,180 @@
import { describe, expect } from "bun:test"
import { Message } from "@opencode-ai/ai"
import { DateTime, Effect, Stream } from "effect"
import type { SessionContext } from "@opencode-ai/plugin/effect/session"
import { Agent } from "@opencode-ai/core/agent"
import { Event } from "@opencode-ai/schema/event"
import { Model } from "@opencode-ai/core/model"
import { PlanPlugin } from "@opencode-ai/core/plugin/plan"
import { Provider } from "@opencode-ai/core/provider"
import { Session } from "@opencode-ai/core/session"
import { SessionEvent } from "@opencode-ai/core/session/event"
import { SessionInbox } from "@opencode-ai/core/session/inbox"
import { SessionMessage } from "@opencode-ai/core/session/message"
import { it } from "../lib/effect"
import { host } from "./host"
const sessionID = Session.ID.make("ses_plan_test")
const plan = Agent.ID.make("plan")
const build = Agent.ID.make("build")
const agentSelected = (agent: Agent.ID, previous: Agent.ID): SessionEvent.AgentSelected => ({
id: Event.ID.create(),
created: 0,
durable: { aggregateID: sessionID, seq: Event.Seq.make(0), version: Event.Version.make(1) },
type: "session.agent.selected",
data: { sessionID, agent, previous },
})
/** Runs the plan plugin against stubbed domains, capturing persisted reminders and the context hook. */
const run = Effect.fnUntraced(function* (events: ReadonlyArray<SessionEvent.AgentSelected> = []) {
const persisted = new Array<string>()
let contextHook: ((input: SessionContext) => Effect.Effect<void>) | undefined
yield* PlanPlugin.Plugin.effect(
host({
agent: {
get: () => Effect.die("unused agent.get"),
list: () => Effect.die("unused agent.list"),
reload: () => Effect.die("unused agent.reload"),
transform: () => Effect.succeed({ dispose: Effect.void }),
},
tool: {
transform: () => Effect.die("unused tool.transform"),
hook: () => Effect.succeed({ dispose: Effect.void }),
},
event: {
subscribe: () => Stream.fromIterable(events),
},
session: {
hook: (name, callback) => {
if (name === "context") contextHook = callback as (input: SessionContext) => Effect.Effect<void>
return Effect.succeed({ dispose: Effect.void })
},
synthetic: (input) => {
persisted.push(input.text)
return Effect.succeed(
SessionInbox.Synthetic.make({
id: SessionMessage.ID.make("msg_plan_test"),
sessionID,
timeCreated: DateTime.makeUnsafe(0),
type: "synthetic",
payload: { text: input.text },
delivery: "steer",
}),
)
},
},
}),
)
if (!contextHook) return yield* Effect.die("plan plugin did not register a context hook")
return { persisted, contextHook }
})
const request = (agent: Agent.ID, messages: Array<Message>): SessionContext => ({
sessionID,
agent,
model: { id: Model.ID.make("test-model"), providerID: Provider.ID.make("test") },
system: [],
messages,
tools: {},
})
const settle = (persisted: ReadonlyArray<string>, expected: number, remaining = 1000): Effect.Effect<void, Error> =>
Effect.gen(function* () {
if (persisted.length >= expected) return
if (remaining === 0) {
return yield* Effect.fail(new Error(`Timed out waiting for ${expected} reminders, saw ${persisted.length}`))
}
yield* Effect.promise(() => Bun.sleep(1))
yield* settle(persisted, expected, remaining - 1)
})
/** The exact reminder texts, derived from plugin behavior rather than duplicated here. */
const reminders = Effect.gen(function* () {
const planRun = yield* run()
yield* planRun.contextHook(request(plan, []))
const buildRun = yield* run()
yield* buildRun.contextHook(request(build, [Message.user(planRun.persisted[0]!)]))
return { enter: planRun.persisted[0]!, leave: buildRun.persisted[0]! }
})
describe("plan plugin reminders", () => {
it.effect("injects enter and leave reminders on agent switches", () =>
Effect.gen(function* () {
const { persisted } = yield* run([agentSelected(plan, build), agentSelected(build, plan)])
yield* settle(persisted, 2)
expect(persisted[0]).toContain("You are in Plan mode")
expect(persisted[1]).toContain("NO LONGER in Plan mode")
}),
)
it.effect("reconciles a missing enter reminder into the request and persists it", () =>
Effect.gen(function* () {
const { persisted, contextHook } = yield* run()
const messages = [Message.user("what agent are you?")]
yield* contextHook(request(plan, messages))
expect(messages).toHaveLength(2)
// Inserted before the user's prompt, matching where agent-switch reminders land.
const first = messages[0]?.content[0]
expect(first?.type === "text" && first.text).toContain("You are in Plan mode")
expect(persisted).toHaveLength(1)
}),
)
it.effect("does nothing when the transcript already has a live enter reminder", () =>
Effect.gen(function* () {
const { enter } = yield* reminders
const { persisted, contextHook } = yield* run()
const messages = [Message.user(enter), Message.user("hello")]
yield* contextHook(request(plan, messages))
expect(messages).toHaveLength(2)
expect(persisted).toHaveLength(0)
}),
)
it.effect("reconciles a stale enter reminder with a leave reminder", () =>
Effect.gen(function* () {
const { enter } = yield* reminders
const { persisted, contextHook } = yield* run()
const messages = [Message.user(enter), Message.user("ok implement it")]
yield* contextHook(request(build, messages))
expect(messages).toHaveLength(3)
const middle = messages[1]?.content[0]
expect(middle?.type === "text" && middle.text).toContain("NO LONGER in Plan mode")
expect(persisted).toHaveLength(1)
}),
)
it.effect("does nothing for non-plan sessions without plan history", () =>
Effect.gen(function* () {
const { persisted, contextHook } = yield* run()
const messages = [Message.user("hello")]
yield* contextHook(request(build, messages))
expect(messages).toHaveLength(1)
expect(persisted).toHaveLength(0)
}),
)
it.effect("does nothing when a leave reminder already follows the enter reminder", () =>
Effect.gen(function* () {
const { enter, leave } = yield* reminders
const { persisted, contextHook } = yield* run()
const messages = [Message.user(enter), Message.user(leave), Message.user("continue")]
yield* contextHook(request(build, messages))
expect(messages).toHaveLength(3)
expect(persisted).toHaveLength(0)
}),
)
it.effect("treats reminder text quoted inside a larger message as not live", () =>
Effect.gen(function* () {
const { enter } = yield* reminders
const { persisted, contextHook } = yield* run()
// Mirrors a compaction checkpoint quoting the reminder inside <recent-context>.
const messages = [Message.user(`<conversation-checkpoint>\n${enter}\n</conversation-checkpoint>`)]
yield* contextHook(request(plan, messages))
expect(messages).toHaveLength(2)
expect(persisted).toHaveLength(1)
}),
)
})
+12 -5
View File
@@ -10,6 +10,7 @@ import { useClient } from "./client"
import { RGBA } from "@opentui/core"
import { readJson, writeJsonAtomic } from "../util/persistence"
import {
availableModelVariant,
createModelPreferenceRepository,
cycleModelVariant,
modelPreferenceKey,
@@ -221,7 +222,14 @@ export const { use: useLocal, provider: LocalProvider } = createSimpleContext({
if (route.data.type === "session") return sessionSelection(route.data.sessionID)
const model = newSessionModel()
if (!model) return
return { ...model, variant: normalizeModelVariant(preferences.variant[modelPreferenceKey(model)]) }
const info = models()?.find((item) => item.providerID === model.providerID && item.id === model.modelID)
return {
...model,
variant: availableModelVariant(
preferences.variant[modelPreferenceKey(model)],
info?.variants?.map((item) => item.id) ?? [],
),
}
})
const currentModel = createMemo(() => {
@@ -262,13 +270,12 @@ export const { use: useLocal, provider: LocalProvider } = createSimpleContext({
if (route.data.type === "session") {
const sessionID = route.data.sessionID
const current = sessionSelection(sessionID)
const preferred = normalizeModelVariant(
const preferred =
current?.providerID === model.providerID && current.modelID === model.modelID
? current.variant
: preferences.variant[modelPreferenceKey(model)],
)
: preferences.variant[modelPreferenceKey(model)]
const info = models()?.find((item) => item.providerID === model.providerID && item.id === model.modelID)
const variant = preferred && info?.variants?.some((item) => item.id === preferred) ? preferred : undefined
const variant = availableModelVariant(preferred, info?.variants?.map((item) => item.id) ?? [])
setSessionDraft(sessionID, { ...model, variant })
return true
}
+5
View File
@@ -39,6 +39,11 @@ export function normalizeModelVariant(value: string | undefined) {
return value === "default" ? undefined : value
}
export function availableModelVariant(value: string | undefined, variants: string[]) {
const variant = normalizeModelVariant(value)
return variant && variants.includes(variant) ? variant : undefined
}
export function modelPreferenceKey(model: ModelPreferenceModel) {
return `${model.providerID}/${model.modelID}`
}
+7 -1
View File
@@ -1,6 +1,6 @@
import { expect, test } from "bun:test"
import path from "node:path"
import { createModelPreferenceRepository, decodeModelPreference } from "../src/model-preference"
import { availableModelVariant, createModelPreferenceRepository, decodeModelPreference } from "../src/model-preference"
import { tmpdir } from "./fixture/fixture"
test("repairs known model preferences and preserves unrelated fields", () => {
@@ -19,6 +19,12 @@ test("repairs known model preferences and preserves unrelated fields", () => {
})
})
test("drops a saved variant that is no longer available", () => {
expect(availableModelVariant("medium", [])).toBeUndefined()
expect(availableModelVariant("medium", ["low", "high"])).toBeUndefined()
expect(availableModelVariant("medium", ["low", "medium", "high"])).toBe("medium")
})
test("atomically serializes patches and variant updates", async () => {
await using tmp = await tmpdir()
const file = path.join(tmp.path, "model.json")