From f7593836cd0cb837da2e5bf0f86ee9eb79c52f06 Mon Sep 17 00:00:00 2001 From: Aiden Cline Date: Wed, 22 Jul 2026 00:12:00 -0500 Subject: [PATCH] feat(ai): support PDF inputs --- .../ai/src/protocols/anthropic-messages.ts | 52 +++++++++++++--- packages/ai/src/protocols/openai-responses.ts | 37 ++++++++++-- packages/ai/src/protocols/shared.ts | 3 +- .../test/provider/anthropic-messages.test.ts | 12 ++-- packages/ai/test/provider/gemini.test.ts | 11 +++- .../ai/test/provider/openai-responses.test.ts | 59 +++++++++++++++++-- 6 files changed, 146 insertions(+), 28 deletions(-) diff --git a/packages/ai/src/protocols/anthropic-messages.ts b/packages/ai/src/protocols/anthropic-messages.ts index 26d127eb0e..181da7b1d2 100644 --- a/packages/ai/src/protocols/anthropic-messages.ts +++ b/packages/ai/src/protocols/anthropic-messages.ts @@ -56,6 +56,17 @@ const AnthropicImageBlock = Schema.Struct({ }) type AnthropicImageBlock = Schema.Schema.Type +const AnthropicDocumentBlock = Schema.Struct({ + type: Schema.tag("document"), + source: Schema.Struct({ + type: Schema.tag("base64"), + media_type: Schema.Literal("application/pdf"), + data: Schema.String, + }), + cache_control: Schema.optional(AnthropicCacheControl), +}) +type AnthropicDocumentBlock = Schema.Schema.Type + const AnthropicThinkingBlock = Schema.Struct({ type: Schema.tag("thinking"), thinking: Schema.String, @@ -101,13 +112,13 @@ const AnthropicServerToolResultBlock = Schema.Struct({ }) type AnthropicServerToolResultBlock = Schema.Schema.Type -// Anthropic accepts either a plain string or an ordered array of text/image -// blocks inside `tool_result.content`. The array form is required when a tool +// Anthropic accepts either a plain string or an ordered array of text, image, and +// document blocks inside `tool_result.content`. The array form is required when a tool // returns image bytes (screenshot, image search, etc.) so they can be passed // to the model as proper image inputs instead of being JSON-stringified into // the prompt — which silently inflates context by megabytes and can push the // conversation over the model's token limit. -const AnthropicToolResultContent = Schema.Union([AnthropicTextBlock, AnthropicImageBlock]) +const AnthropicToolResultContent = Schema.Union([AnthropicTextBlock, AnthropicImageBlock, AnthropicDocumentBlock]) const AnthropicToolResultBlock = Schema.Struct({ type: Schema.tag("tool_result"), @@ -117,7 +128,12 @@ const AnthropicToolResultBlock = Schema.Struct({ cache_control: Schema.optional(AnthropicCacheControl), }) -const AnthropicUserBlock = Schema.Union([AnthropicTextBlock, AnthropicImageBlock, AnthropicToolResultBlock]) +const AnthropicUserBlock = Schema.Union([ + AnthropicTextBlock, + AnthropicImageBlock, + AnthropicDocumentBlock, + AnthropicToolResultBlock, +]) type AnthropicUserBlock = Schema.Schema.Type const AnthropicAssistantBlock = Schema.Union([ AnthropicTextBlock, @@ -319,12 +335,21 @@ const lowerServerToolResult = Effect.fn("AnthropicMessages.lowerServerToolResult return { type: wireType, tool_use_id: part.id, content: part.result.value } satisfies AnthropicServerToolResultBlock }) -const lowerImage = Effect.fn("AnthropicMessages.lowerImage")(function* (part: MediaPart) { +const lowerMedia = Effect.fn("AnthropicMessages.lowerMedia")(function* (part: MediaPart) { const media = yield* ProviderShared.validateMedia( "Anthropic Messages", part, - new Set(ProviderShared.IMAGE_MIMES), + new Set([...ProviderShared.IMAGE_MIMES, ...ProviderShared.DOCUMENT_MIMES]), ) + if (media.mime === "application/pdf") + return { + type: "document" as const, + source: { + type: "base64" as const, + media_type: "application/pdf" as const, + data: media.base64, + }, + } satisfies AnthropicDocumentBlock return { type: "image" as const, source: { @@ -335,7 +360,7 @@ const lowerImage = Effect.fn("AnthropicMessages.lowerImage")(function* (part: Me } satisfies AnthropicImageBlock }) -// Tool results may carry structured text/images. Keep media as provider-native +// Tool results may carry structured text, images, and documents. Keep media as provider-native // content instead of JSON-stringifying base64 into a prompt string. const lowerToolResultContentItem = Effect.fn("AnthropicMessages.lowerToolResultContentItem")(function* ( item: ToolContent, @@ -344,8 +369,17 @@ const lowerToolResultContentItem = Effect.fn("AnthropicMessages.lowerToolResultC const media = yield* ProviderShared.validateToolFile( "Anthropic Messages", item, - new Set(ProviderShared.IMAGE_MIMES), + new Set([...ProviderShared.IMAGE_MIMES, ...ProviderShared.DOCUMENT_MIMES]), ) + if (media.mime === "application/pdf") + return { + type: "document" as const, + source: { + type: "base64" as const, + media_type: "application/pdf" as const, + data: media.base64, + }, + } satisfies AnthropicDocumentBlock return { type: "image" as const, source: { @@ -445,7 +479,7 @@ const lowerMessages = Effect.fn("AnthropicMessages.lowerMessages")(function* ( continue } if (part.type === "media") { - content.push(yield* lowerImage(part)) + content.push(yield* lowerMedia(part)) continue } return yield* ProviderShared.unsupportedContent("Anthropic Messages", "user", ["text", "media"]) diff --git a/packages/ai/src/protocols/openai-responses.ts b/packages/ai/src/protocols/openai-responses.ts index f8e11b3cc2..9c730359ed 100644 --- a/packages/ai/src/protocols/openai-responses.ts +++ b/packages/ai/src/protocols/openai-responses.ts @@ -42,7 +42,16 @@ const OpenAIResponsesInputImage = Schema.Struct({ type: Schema.tag("input_image"), image_url: Schema.String, }) -const OpenAIResponsesInputContent = Schema.Union([OpenAIResponsesInputText, OpenAIResponsesInputImage]) +const OpenAIResponsesInputFile = Schema.Struct({ + type: Schema.tag("input_file"), + filename: Schema.String, + file_data: Schema.String, +}) +const OpenAIResponsesInputContent = Schema.Union([ + OpenAIResponsesInputText, + OpenAIResponsesInputImage, + OpenAIResponsesInputFile, +]) type OpenAIResponsesInputContent = Schema.Schema.Type const OpenAIResponsesOutputText = Schema.Struct({ @@ -68,9 +77,13 @@ const OpenAIResponsesItemReference = Schema.Struct({ }) // `function_call_output.output` accepts either a plain string or an ordered -// array of content items so tools can return images in addition to text. +// array of content items so tools can return images and files in addition to text. // https://platform.openai.com/docs/api-reference/responses/object -const OpenAIResponsesFunctionCallOutputContent = Schema.Union([OpenAIResponsesInputText, OpenAIResponsesInputImage]) +const OpenAIResponsesFunctionCallOutputContent = Schema.Union([ + OpenAIResponsesInputText, + OpenAIResponsesInputImage, + OpenAIResponsesInputFile, +]) const OpenAIResponsesFunctionCallOutput = Schema.Union([ Schema.String, @@ -351,14 +364,20 @@ const lowerUserContent = Effect.fn("OpenAIResponses.lowerUserContent")(function* const media = yield* ProviderShared.validateMedia( "OpenAI Responses", part, - new Set(ProviderShared.IMAGE_MIMES), + new Set([...ProviderShared.IMAGE_MIMES, ...ProviderShared.DOCUMENT_MIMES]), ) + if (media.mime === "application/pdf") + return { + type: "input_file" as const, + filename: part.filename ?? "document.pdf", + file_data: media.dataUrl, + } return { type: "input_image" as const, image_url: media.dataUrl } } return yield* ProviderShared.unsupportedContent("OpenAI Responses", "user", ["text", "media"]) }) -// Tool results may carry structured text/images. Keep media as provider-native +// Tool results may carry structured text, images, and files. Keep media as provider-native // content instead of JSON-stringifying base64 into a prompt string. const lowerToolResultContentItem = Effect.fn("OpenAIResponses.lowerToolResultContentItem")(function* ( item: ToolContent, @@ -367,8 +386,14 @@ const lowerToolResultContentItem = Effect.fn("OpenAIResponses.lowerToolResultCon const media = yield* ProviderShared.validateToolFile( "OpenAI Responses", item, - new Set(ProviderShared.IMAGE_MIMES), + new Set([...ProviderShared.IMAGE_MIMES, ...ProviderShared.DOCUMENT_MIMES]), ) + if (media.mime === "application/pdf") + return { + type: "input_file" as const, + filename: item.name ?? "document.pdf", + file_data: media.dataUrl, + } return { type: "input_image" as const, image_url: media.dataUrl } }) diff --git a/packages/ai/src/protocols/shared.ts b/packages/ai/src/protocols/shared.ts index 173dc511bb..1b60aa666e 100644 --- a/packages/ai/src/protocols/shared.ts +++ b/packages/ai/src/protocols/shared.ts @@ -158,7 +158,8 @@ export const parseToolInput = (route: string, name: string, raw: string) => export const IMAGE_MIMES = ["image/png", "image/jpeg", "image/gif", "image/webp"] as const export const VIDEO_MIMES = ["video/mp4", "video/webm", "video/quicktime"] as const export const AUDIO_MIMES = ["audio/wav", "audio/mp3", "audio/aiff", "audio/aac", "audio/ogg", "audio/flac"] as const -export const MEDIA_MIMES = [...IMAGE_MIMES, ...VIDEO_MIMES, ...AUDIO_MIMES] as const +export const DOCUMENT_MIMES = ["application/pdf"] as const +export const MEDIA_MIMES = [...IMAGE_MIMES, ...VIDEO_MIMES, ...AUDIO_MIMES, ...DOCUMENT_MIMES] as const export const MAX_MEDIA_ENCODED_BYTES = 28 * 1024 * 1024 export const MAX_MEDIA_DECODED_BYTES = 20 * 1024 * 1024 diff --git a/packages/ai/test/provider/anthropic-messages.test.ts b/packages/ai/test/provider/anthropic-messages.test.ts index 69a27e6228..9aead6445c 100644 --- a/packages/ai/test/provider/anthropic-messages.test.ts +++ b/packages/ai/test/provider/anthropic-messages.test.ts @@ -235,9 +235,9 @@ describe("Anthropic Messages route", () => { }), ) - // Regression: screenshot/read tool results must stay structured so base64 - // image data is not JSON-stringified into `tool_result.content`. - it.effect("lowers image tool-result content as structured image blocks", () => + // Regression: read tool results must stay structured so base64 media data is + // not JSON-stringified into `tool_result.content`. + it.effect("lowers media tool-result content as structured blocks", () => Effect.gen(function* () { const prepared = yield* LLMClient.prepare( LLM.request({ @@ -253,6 +253,7 @@ describe("Anthropic Messages route", () => { result: [ { type: "text", text: "Image read successfully" }, { type: "file", uri: "data:image/png;base64,AAECAw==", mime: "image/png" }, + { type: "file", uri: "data:application/pdf;base64,JVBERi0xLjQ=", mime: "application/pdf" }, ], }), ], @@ -263,6 +264,7 @@ describe("Anthropic Messages route", () => { expect(expectToolResult(prepared.body).content).toEqual([ { type: "text", text: "Image read successfully" }, { type: "image", source: { type: "base64", media_type: "image/png", data: "AAECAw==" } }, + { type: "document", source: { type: "base64", media_type: "application/pdf", data: "JVBERi0xLjQ=" } }, ]) }), ) @@ -756,7 +758,7 @@ describe("Anthropic Messages route", () => { }), ) - it.effect("continues a conversation with user image content", () => + it.effect("continues a conversation with user media content", () => Effect.gen(function* () { const response = yield* LLMClient.generate( LLM.request({ @@ -766,6 +768,7 @@ describe("Anthropic Messages route", () => { Message.user([ { type: "text", text: "What is in this image?" }, { type: "media", mediaType: "image/png", data: "AAECAw==" }, + { type: "media", mediaType: "application/pdf", data: "JVBERi0xLjQ=", filename: "report.pdf" }, ]), ], }), @@ -781,6 +784,7 @@ describe("Anthropic Messages route", () => { content: [ { type: "text", text: "What is in this image?" }, { type: "image", source: { type: "base64", media_type: "image/png", data: "AAECAw==" } }, + { type: "document", source: { type: "base64", media_type: "application/pdf", data: "JVBERi0xLjQ=" } }, ], }, ], diff --git a/packages/ai/test/provider/gemini.test.ts b/packages/ai/test/provider/gemini.test.ts index 1dc253c0ea..087ca234ce 100644 --- a/packages/ai/test/provider/gemini.test.ts +++ b/packages/ai/test/provider/gemini.test.ts @@ -70,6 +70,7 @@ describe("Gemini route", () => { Message.user([ { type: "text", text: "What is in this image?" }, { type: "media", mediaType: "image/png", data: "AAECAw==" }, + { type: "media", mediaType: "application/pdf", data: "JVBERi0xLjQ=" }, ]), Message.assistant([ToolCallPart.make({ id: "call_1", name: "lookup", input: { query: "weather" } })]), Message.tool({ id: "call_1", name: "lookup", result: { forecast: "sunny" } }), @@ -81,7 +82,11 @@ describe("Gemini route", () => { contents: [ { role: "user", - parts: [{ text: "What is in this image?" }, { inlineData: { mimeType: "image/png", data: "AAECAw==" } }], + parts: [ + { text: "What is in this image?" }, + { inlineData: { mimeType: "image/png", data: "AAECAw==" } }, + { inlineData: { mimeType: "application/pdf", data: "JVBERi0xLjQ=" } }, + ], }, { role: "model", @@ -110,7 +115,7 @@ describe("Gemini route", () => { }), ) - it.effect("continues image tool results as inline vision input without base64 text", () => + it.effect("continues media tool results as inline model input without base64 text", () => Effect.gen(function* () { const prepared = yield* LLMClient.prepare( LLM.request({ @@ -125,6 +130,7 @@ describe("Gemini route", () => { value: [ { type: "text", text: "Image read successfully" }, { type: "file", uri: "data:image/png;base64,AAECAw==", mime: "image/png", name: "pixel.png" }, + { type: "file", uri: "data:application/pdf;base64,JVBERi0xLjQ=", mime: "application/pdf" }, ], }, }), @@ -144,6 +150,7 @@ describe("Gemini route", () => { }, }, { inlineData: { mimeType: "image/png", data: "AAECAw==" } }, + { inlineData: { mimeType: "application/pdf", data: "JVBERi0xLjQ=" } }, ], }, ]) diff --git a/packages/ai/test/provider/openai-responses.test.ts b/packages/ai/test/provider/openai-responses.test.ts index e345cd59b1..c6bf70b9ca 100644 --- a/packages/ai/test/provider/openai-responses.test.ts +++ b/packages/ai/test/provider/openai-responses.test.ts @@ -524,7 +524,42 @@ describe("OpenAI Responses route", () => { }), ) - it.effect("rejects non-image media in tool-result content with a clear error", () => + it.effect("lowers PDF tool-result content as structured input_file array", () => + Effect.gen(function* () { + const prepared = yield* LLMClient.prepare( + LLM.request({ + id: "req_tool_result_pdf", + model, + messages: [ + Message.assistant([ToolCallPart.make({ id: "call_1", name: "read", input: {} })]), + Message.tool({ + id: "call_1", + name: "read", + resultType: "content", + result: [ + { + type: "file", + uri: "data:application/pdf;base64,JVBERi0xLjQ=", + mime: "application/pdf", + name: "report.pdf", + }, + ], + }), + ], + }), + ) + + expect(expectToolOutput(prepared.body).output).toEqual([ + { + type: "input_file", + filename: "report.pdf", + file_data: "data:application/pdf;base64,JVBERi0xLjQ=", + }, + ]) + }), + ) + + it.effect("rejects unsupported media in tool-result content with a clear error", () => Effect.gen(function* () { const error = yield* LLMClient.prepare( LLM.request({ @@ -1526,20 +1561,32 @@ describe("OpenAI Responses route", () => { }), ) - it.effect("lowers user image content", () => + it.effect("lowers user image and PDF content", () => Effect.gen(function* () { const prepared = yield* LLMClient.prepare( LLM.request({ id: "req_media", model, - messages: [Message.user({ type: "media", mediaType: "image/png", data: "AAECAw==" })], + messages: [ + Message.user([ + { type: "media", mediaType: "image/png", data: "AAECAw==" }, + { type: "media", mediaType: "application/pdf", data: "JVBERi0xLjQ=", filename: "report.pdf" }, + ]), + ], }), ) expect(prepared.body.input).toEqual([ { role: "user", - content: [{ type: "input_image", image_url: "data:image/png;base64,AAECAw==" }], + content: [ + { type: "input_image", image_url: "data:image/png;base64,AAECAw==" }, + { + type: "input_file", + filename: "report.pdf", + file_data: "data:application/pdf;base64,JVBERi0xLjQ=", + }, + ], }, ]) }), @@ -1551,11 +1598,11 @@ describe("OpenAI Responses route", () => { LLM.request({ id: "req_media", model, - messages: [Message.user({ type: "media", mediaType: "application/pdf", data: "AAECAw==" })], + messages: [Message.user({ type: "media", mediaType: "application/x-tar", data: "AAECAw==" })], }), ).pipe(Effect.flip) - expect(error.message).toContain("OpenAI Responses does not support media type application/pdf") + expect(error.message).toContain("OpenAI Responses does not support media type application/x-tar") }), )