From 2271f9b222434972700559cd903bc79c4e08d83f Mon Sep 17 00:00:00 2001 From: Aiden Cline <63023139+rekram1-node@users.noreply.github.com> Date: Wed, 22 Jul 2026 13:42:29 -0500 Subject: [PATCH] feat(ai): support PDF inputs (#38253) --- .../ai/src/protocols/anthropic-messages.ts | 65 +++--- packages/ai/src/protocols/bedrock-converse.ts | 3 +- packages/ai/src/protocols/gemini.ts | 33 ++- packages/ai/src/protocols/openai-responses.ts | 103 ++++++--- packages/ai/src/protocols/shared.ts | 3 +- .../ai/src/protocols/utils/bedrock-media.ts | 8 +- packages/ai/src/tool-runtime.ts | 25 ++- .../recordings/pdf/anthropic-tool-result.json | 35 +++ .../recordings/pdf/anthropic-user-input.json | 34 +++ .../recordings/pdf/bedrock-tool-result.json | 36 +++ .../recordings/pdf/bedrock-user-input.json | 35 +++ .../recordings/pdf/gemini-tool-result.json | 53 +++++ .../recordings/pdf/gemini-user-input.json | 34 +++ .../recordings/pdf/openai-tool-result.json | 35 +++ .../recordings/pdf/openai-user-input.json | 34 +++ .../recordings/pdf/xai-tool-result.json | 35 +++ .../recordings/pdf/xai-user-input.json | 34 +++ packages/ai/test/lib/tool-runtime.ts | 7 +- .../test/provider/anthropic-messages.test.ts | 14 +- .../ai/test/provider/bedrock-converse.test.ts | 99 ++++++++- packages/ai/test/provider/gemini.test.ts | 75 ++++++- .../ai/test/provider/openai-responses.test.ts | 129 ++++++++++- .../ai/test/provider/pdf.recorded.test.ts | 207 ++++++++++++++++++ packages/ai/test/tool-runtime.test.ts | 45 ++++ 24 files changed, 1074 insertions(+), 107 deletions(-) create mode 100644 packages/ai/test/fixtures/recordings/pdf/anthropic-tool-result.json create mode 100644 packages/ai/test/fixtures/recordings/pdf/anthropic-user-input.json create mode 100644 packages/ai/test/fixtures/recordings/pdf/bedrock-tool-result.json create mode 100644 packages/ai/test/fixtures/recordings/pdf/bedrock-user-input.json create mode 100644 packages/ai/test/fixtures/recordings/pdf/gemini-tool-result.json create mode 100644 packages/ai/test/fixtures/recordings/pdf/gemini-user-input.json create mode 100644 packages/ai/test/fixtures/recordings/pdf/openai-tool-result.json create mode 100644 packages/ai/test/fixtures/recordings/pdf/openai-user-input.json create mode 100644 packages/ai/test/fixtures/recordings/pdf/xai-tool-result.json create mode 100644 packages/ai/test/fixtures/recordings/pdf/xai-user-input.json create mode 100644 packages/ai/test/provider/pdf.recorded.test.ts diff --git a/packages/ai/src/protocols/anthropic-messages.ts b/packages/ai/src/protocols/anthropic-messages.ts index 26d127eb0e..f5b9ea67d9 100644 --- a/packages/ai/src/protocols/anthropic-messages.ts +++ b/packages/ai/src/protocols/anthropic-messages.ts @@ -27,6 +27,7 @@ import { ToolSchemaProjection } from "./utils/tool-schema" import { ToolStream } from "./utils/tool-stream" const ADAPTER = "anthropic-messages" +const MEDIA_MIMES = new Set([...ProviderShared.IMAGE_MIMES, ...ProviderShared.PDF_MIMES]) export const DEFAULT_BASE_URL = "https://api.anthropic.com/v1" export const PATH = "/messages" @@ -56,6 +57,17 @@ const AnthropicImageBlock = Schema.Struct({ }) type AnthropicImageBlock = Schema.Schema.Type +const AnthropicDocumentBlock = Schema.Struct({ + type: Schema.tag("document"), + source: Schema.Struct({ + type: Schema.tag("base64"), + media_type: Schema.Literal("application/pdf"), + data: Schema.String, + }), + cache_control: Schema.optional(AnthropicCacheControl), +}) +type AnthropicDocumentBlock = Schema.Schema.Type + const AnthropicThinkingBlock = Schema.Struct({ type: Schema.tag("thinking"), thinking: Schema.String, @@ -101,13 +113,10 @@ const AnthropicServerToolResultBlock = Schema.Struct({ }) type AnthropicServerToolResultBlock = Schema.Schema.Type -// Anthropic accepts either a plain string or an ordered array of text/image -// blocks inside `tool_result.content`. The array form is required when a tool -// returns image bytes (screenshot, image search, etc.) so they can be passed -// to the model as proper image inputs instead of being JSON-stringified into -// the prompt — which silently inflates context by megabytes and can push the -// conversation over the model's token limit. -const AnthropicToolResultContent = Schema.Union([AnthropicTextBlock, AnthropicImageBlock]) +// Anthropic accepts either a plain string or an ordered array of text, image, and +// document blocks inside `tool_result.content`. The array form keeps media as native +// model input instead of JSON-stringifying base64 into prompt text. +const AnthropicToolResultContent = Schema.Union([AnthropicTextBlock, AnthropicImageBlock, AnthropicDocumentBlock]) const AnthropicToolResultBlock = Schema.Struct({ type: Schema.tag("tool_result"), @@ -117,7 +126,12 @@ const AnthropicToolResultBlock = Schema.Struct({ cache_control: Schema.optional(AnthropicCacheControl), }) -const AnthropicUserBlock = Schema.Union([AnthropicTextBlock, AnthropicImageBlock, AnthropicToolResultBlock]) +const AnthropicUserBlock = Schema.Union([ + AnthropicTextBlock, + AnthropicImageBlock, + AnthropicDocumentBlock, + AnthropicToolResultBlock, +]) type AnthropicUserBlock = Schema.Schema.Type const AnthropicAssistantBlock = Schema.Union([ AnthropicTextBlock, @@ -319,12 +333,17 @@ const lowerServerToolResult = Effect.fn("AnthropicMessages.lowerServerToolResult return { type: wireType, tool_use_id: part.id, content: part.result.value } satisfies AnthropicServerToolResultBlock }) -const lowerImage = Effect.fn("AnthropicMessages.lowerImage")(function* (part: MediaPart) { - const media = yield* ProviderShared.validateMedia( - "Anthropic Messages", - part, - new Set(ProviderShared.IMAGE_MIMES), - ) +const lowerMedia = Effect.fn("AnthropicMessages.lowerMedia")(function* (part: MediaPart) { + const media = yield* ProviderShared.validateMedia("Anthropic Messages", part, MEDIA_MIMES) + if (media.mime === "application/pdf") + return { + type: "document" as const, + source: { + type: "base64" as const, + media_type: "application/pdf" as const, + data: media.base64, + }, + } satisfies AnthropicDocumentBlock return { type: "image" as const, source: { @@ -335,25 +354,13 @@ const lowerImage = Effect.fn("AnthropicMessages.lowerImage")(function* (part: Me } satisfies AnthropicImageBlock }) -// Tool results may carry structured text/images. Keep media as provider-native +// Tool results may carry structured text, images, and documents. Keep media as provider-native // content instead of JSON-stringifying base64 into a prompt string. const lowerToolResultContentItem = Effect.fn("AnthropicMessages.lowerToolResultContentItem")(function* ( item: ToolContent, ) { if (item.type === "text") return { type: "text" as const, text: item.text } satisfies AnthropicTextBlock - const media = yield* ProviderShared.validateToolFile( - "Anthropic Messages", - item, - new Set(ProviderShared.IMAGE_MIMES), - ) - return { - type: "image" as const, - source: { - type: "base64" as const, - media_type: media.mime, - data: media.base64, - }, - } satisfies AnthropicImageBlock + return yield* lowerMedia({ type: "media", mediaType: item.mime, data: item.uri, filename: item.name }) }) const lowerToolResultContent = Effect.fn("AnthropicMessages.lowerToolResultContent")(function* (part: ToolResultPart) { @@ -445,7 +452,7 @@ const lowerMessages = Effect.fn("AnthropicMessages.lowerMessages")(function* ( continue } if (part.type === "media") { - content.push(yield* lowerImage(part)) + content.push(yield* lowerMedia(part)) continue } return yield* ProviderShared.unsupportedContent("Anthropic Messages", "user", ["text", "media"]) diff --git a/packages/ai/src/protocols/bedrock-converse.ts b/packages/ai/src/protocols/bedrock-converse.ts index b94f23d62f..801fb8a98a 100644 --- a/packages/ai/src/protocols/bedrock-converse.ts +++ b/packages/ai/src/protocols/bedrock-converse.ts @@ -52,6 +52,7 @@ const BedrockToolResultContentItem = Schema.Union([ Schema.Struct({ text: Schema.String }), Schema.Struct({ json: Schema.Unknown }), BedrockMedia.ImageBlock, + BedrockMedia.DocumentBlock, ]) const BedrockToolResultBlock = Schema.Struct({ @@ -283,8 +284,6 @@ const lowerToolResultContent = Effect.fn("BedrockConverse.lowerToolResultContent data: item.uri, filename: item.name, }) - if (!("image" in media)) - return yield* ProviderShared.invalidRequest("Bedrock Converse only supports image media in tool results") content.push(media) } return content diff --git a/packages/ai/src/protocols/gemini.ts b/packages/ai/src/protocols/gemini.ts index 82a69b059e..6cba2bc7f9 100644 --- a/packages/ai/src/protocols/gemini.ts +++ b/packages/ai/src/protocols/gemini.ts @@ -41,9 +41,11 @@ const GeminiInlineDataPart = Schema.Struct({ data: Schema.String, }), }) +type GeminiInlineDataPart = Schema.Schema.Type const GeminiFunctionCallPart = Schema.Struct({ functionCall: Schema.Struct({ + id: Schema.optional(Schema.String), name: Schema.String, args: Schema.Unknown, }), @@ -52,8 +54,10 @@ const GeminiFunctionCallPart = Schema.Struct({ const GeminiFunctionResponsePart = Schema.Struct({ functionResponse: Schema.Struct({ + id: Schema.optional(Schema.String), name: Schema.String, response: Schema.Unknown, + parts: Schema.optional(Schema.Array(GeminiInlineDataPart)), }), }) @@ -197,8 +201,13 @@ const thoughtSignature = (providerMetadata: ProviderMetadata | undefined) => { : undefined } +const functionCallId = (providerMetadata: ProviderMetadata | undefined) => { + const google = providerMetadata?.google + return ProviderShared.isRecord(google) && typeof google.functionCallId === "string" ? google.functionCallId : undefined +} + const lowerToolCall = (part: ToolCallPart) => ({ - functionCall: { name: part.name, args: part.input }, + functionCall: { id: functionCallId(part.providerMetadata), name: part.name, args: part.input }, thoughtSignature: thoughtSignature(part.providerMetadata), }) @@ -255,6 +264,7 @@ const lowerMessages = Effect.fn("Gemini.lowerMessages")(function* (request: LLMR if (part.result.type !== "content") { parts.push({ functionResponse: { + id: functionCallId(part.providerMetadata), name: part.name, response: { name: part.name, @@ -266,20 +276,23 @@ const lowerMessages = Effect.fn("Gemini.lowerMessages")(function* (request: LLMR } const content: ReadonlyArray = part.result.value const text = content.filter((item) => item.type === "text").map((item) => item.text) + const media: GeminiInlineDataPart[] = [] + for (const item of content) { + if (item.type === "text") continue + const value = yield* ProviderShared.validateToolFile("Gemini", item, MEDIA_MIMES) + media.push({ inlineData: { mimeType: value.mime, data: value.base64 } }) + } parts.push({ functionResponse: { + id: functionCallId(part.providerMetadata), name: part.name, response: { name: part.name, content: text.join("\n"), }, + parts: media.length > 0 ? media : undefined, }, }) - for (const item of content) { - if (item.type === "text") continue - const media = yield* ProviderShared.validateToolFile("Gemini", item, MEDIA_MIMES) - parts.push({ inlineData: { mimeType: media.mime, data: media.base64 } }) - } } contents.push({ role: "user", parts }) } @@ -441,6 +454,10 @@ const step = (state: ParserState, event: GeminiEvent) => { if ("functionCall" in part) { const input = part.functionCall.args const id = `tool_${nextToolCallId++}` + const metadata = { + ...(part.functionCall.id === undefined ? {} : { functionCallId: part.functionCall.id }), + ...(part.thoughtSignature === undefined ? {} : { thoughtSignature: part.thoughtSignature }), + } lifecycle = Lifecycle.reasoningEnd( lifecycle, events, @@ -453,9 +470,7 @@ const step = (state: ParserState, event: GeminiEvent) => { id, name: part.functionCall.name, input, - providerMetadata: part.thoughtSignature - ? googleMetadata({ thoughtSignature: part.thoughtSignature }) - : undefined, + providerMetadata: Object.keys(metadata).length > 0 ? googleMetadata(metadata) : undefined, }), ) hasToolCalls = true diff --git a/packages/ai/src/protocols/openai-responses.ts b/packages/ai/src/protocols/openai-responses.ts index f8e11b3cc2..87ca8c75e1 100644 --- a/packages/ai/src/protocols/openai-responses.ts +++ b/packages/ai/src/protocols/openai-responses.ts @@ -11,6 +11,7 @@ import { type FinishReason, type JsonSchema, type LLMRequest, + type MediaPart, type ProviderMetadata, type ReasoningPart, type TextPart, @@ -28,6 +29,7 @@ import { ToolStream } from "./utils/tool-stream" import { OpenAIImage } from "./utils/openai-image" const ADAPTER = "openai-responses" +const MEDIA_MIMES = new Set([...ProviderShared.IMAGE_MIMES, ...ProviderShared.PDF_MIMES]) export const DEFAULT_BASE_URL = "https://api.openai.com/v1" export const PATH = "/responses" @@ -42,7 +44,17 @@ const OpenAIResponsesInputImage = Schema.Struct({ type: Schema.tag("input_image"), image_url: Schema.String, }) -const OpenAIResponsesInputContent = Schema.Union([OpenAIResponsesInputText, OpenAIResponsesInputImage]) +const OpenAIResponsesInputFile = Schema.Struct({ + type: Schema.tag("input_file"), + filename: Schema.String, + file_data: Schema.String, + mime_type: Schema.optional(Schema.String), +}) +const OpenAIResponsesInputContent = Schema.Union([ + OpenAIResponsesInputText, + OpenAIResponsesInputImage, + OpenAIResponsesInputFile, +]) type OpenAIResponsesInputContent = Schema.Schema.Type const OpenAIResponsesOutputText = Schema.Struct({ @@ -68,9 +80,13 @@ const OpenAIResponsesItemReference = Schema.Struct({ }) // `function_call_output.output` accepts either a plain string or an ordered -// array of content items so tools can return images in addition to text. +// array of content items so tools can return images and files in addition to text. // https://platform.openai.com/docs/api-reference/responses/object -const OpenAIResponsesFunctionCallOutputContent = Schema.Union([OpenAIResponsesInputText, OpenAIResponsesInputImage]) +const OpenAIResponsesFunctionCallOutputContent = Schema.Union([ + OpenAIResponsesInputText, + OpenAIResponsesInputImage, + OpenAIResponsesInputFile, +]) const OpenAIResponsesFunctionCallOutput = Schema.Union([ Schema.String, @@ -343,42 +359,58 @@ const hostedToolItemID = (part: ToolResultPart) => { : undefined } -const lowerUserContent = Effect.fn("OpenAIResponses.lowerUserContent")(function* ( - part: LLMRequest["messages"][number]["content"][number], -) { - if (part.type === "text") return { type: "input_text" as const, text: part.text } - if (part.type === "media") { - const media = yield* ProviderShared.validateMedia( - "OpenAI Responses", - part, - new Set(ProviderShared.IMAGE_MIMES), - ) - return { type: "input_image" as const, image_url: media.dataUrl } +const lowerMedia = Effect.fn("OpenAIResponses.lowerMedia")(function* (part: MediaPart, provider: string) { + const media = yield* ProviderShared.validateMedia("OpenAI Responses", part, MEDIA_MIMES) + if (media.mime === "application/pdf") { + // xAI models inline bytes and MIME separately; OpenAI uses a data URL in file_data. + if (provider === "xai") + return { + type: "input_file" as const, + filename: part.filename ?? "document.pdf", + file_data: media.base64, + mime_type: media.mime, + } + return { + type: "input_file" as const, + filename: part.filename ?? "document.pdf", + file_data: media.dataUrl, + } } - return yield* ProviderShared.unsupportedContent("OpenAI Responses", "user", ["text", "media"]) -}) - -// Tool results may carry structured text/images. Keep media as provider-native -// content instead of JSON-stringifying base64 into a prompt string. -const lowerToolResultContentItem = Effect.fn("OpenAIResponses.lowerToolResultContentItem")(function* ( - item: ToolContent, -) { - if (item.type === "text") return { type: "input_text" as const, text: item.text } - const media = yield* ProviderShared.validateToolFile( - "OpenAI Responses", - item, - new Set(ProviderShared.IMAGE_MIMES), - ) return { type: "input_image" as const, image_url: media.dataUrl } }) -const lowerToolResultOutput = Effect.fn("OpenAIResponses.lowerToolResultOutput")(function* (part: ToolResultPart) { +const lowerUserContent = Effect.fn("OpenAIResponses.lowerUserContent")(function* ( + part: LLMRequest["messages"][number]["content"][number], + provider: string, +) { + if (part.type === "text") return { type: "input_text" as const, text: part.text } + if (part.type === "media") return yield* lowerMedia(part, provider) + return yield* ProviderShared.unsupportedContent("OpenAI Responses", "user", ["text", "media"]) +}) + +// Tool results may carry structured text, images, and files. Keep media as provider-native +// content instead of JSON-stringifying base64 into a prompt string. +const lowerToolResultContentItem = Effect.fn("OpenAIResponses.lowerToolResultContentItem")(function* ( + item: ToolContent, + provider: string, +) { + if (item.type === "text") return { type: "input_text" as const, text: item.text } + return yield* lowerMedia( + { type: "media", mediaType: item.mime, data: item.uri, filename: item.name }, + provider, + ) +}) + +const lowerToolResultOutput = Effect.fn("OpenAIResponses.lowerToolResultOutput")(function* ( + part: ToolResultPart, + provider: string, +) { // Text/json/error results are encoded as a plain string for backward // compatibility with existing cassettes and provider expectations. if (part.result.type !== "content") return ProviderShared.toolResultText(part) // Preserve the narrowed array element type when compiled through a consumer package. const content: ReadonlyArray = part.result.value - return yield* Effect.forEach(content, lowerToolResultContentItem) + return yield* Effect.forEach(content, (item) => lowerToolResultContentItem(item, provider)) }) const lowerMessages = Effect.fn("OpenAIResponses.lowerMessages")(function* (request: LLMRequest) { @@ -401,7 +433,10 @@ const lowerMessages = Effect.fn("OpenAIResponses.lowerMessages")(function* (requ } if (message.role === "user") { - input.push({ role: "user", content: yield* Effect.forEach(message.content, lowerUserContent) }) + input.push({ + role: "user", + content: yield* Effect.forEach(message.content, (part) => lowerUserContent(part, request.model.provider)), + }) continue } @@ -460,7 +495,9 @@ const lowerMessages = Effect.fn("OpenAIResponses.lowerMessages")(function* (requ const content: ReadonlyArray = part.result.value input.push({ role: "user", - content: yield* Effect.forEach(content, lowerToolResultContentItem), + content: yield* Effect.forEach(content, (item) => + lowerToolResultContentItem(item, request.model.provider), + ), }) } if (itemID) hostedToolReferences.add(itemID) @@ -483,7 +520,7 @@ const lowerMessages = Effect.fn("OpenAIResponses.lowerMessages")(function* (requ input.push({ type: "function_call_output", call_id: part.id, - output: yield* lowerToolResultOutput(part), + output: yield* lowerToolResultOutput(part, request.model.provider), }) } } diff --git a/packages/ai/src/protocols/shared.ts b/packages/ai/src/protocols/shared.ts index 173dc511bb..478088b081 100644 --- a/packages/ai/src/protocols/shared.ts +++ b/packages/ai/src/protocols/shared.ts @@ -158,7 +158,8 @@ export const parseToolInput = (route: string, name: string, raw: string) => export const IMAGE_MIMES = ["image/png", "image/jpeg", "image/gif", "image/webp"] as const export const VIDEO_MIMES = ["video/mp4", "video/webm", "video/quicktime"] as const export const AUDIO_MIMES = ["audio/wav", "audio/mp3", "audio/aiff", "audio/aac", "audio/ogg", "audio/flac"] as const -export const MEDIA_MIMES = [...IMAGE_MIMES, ...VIDEO_MIMES, ...AUDIO_MIMES] as const +export const PDF_MIMES = ["application/pdf"] as const +export const MEDIA_MIMES = [...IMAGE_MIMES, ...VIDEO_MIMES, ...AUDIO_MIMES, ...PDF_MIMES] as const export const MAX_MEDIA_ENCODED_BYTES = 28 * 1024 * 1024 export const MAX_MEDIA_DECODED_BYTES = 20 * 1024 * 1024 diff --git a/packages/ai/src/protocols/utils/bedrock-media.ts b/packages/ai/src/protocols/utils/bedrock-media.ts index 6fda6c4fbb..6a12966b7b 100644 --- a/packages/ai/src/protocols/utils/bedrock-media.ts +++ b/packages/ai/src/protocols/utils/bedrock-media.ts @@ -49,10 +49,10 @@ const DOCUMENT_FORMATS = { "text/markdown": "md", } as const satisfies Record -const documentBlock = (part: MediaPart, format: DocumentFormat, bytes: string): DocumentBlock => ({ +const documentBlock = (name: string, format: DocumentFormat, bytes: string): DocumentBlock => ({ document: { format, - name: part.filename ?? `document.${format}`, + name, source: { bytes }, }, }) @@ -77,12 +77,14 @@ export const lower = Effect.fn("BedrockMedia.lower")(function* (part: MediaPart) return yield* ProviderShared.invalidRequest(`Bedrock Converse does not support image media type ${part.mediaType}`) const documentFormat = DOCUMENT_FORMATS[mime as keyof typeof DOCUMENT_FORMATS] if (documentFormat) { + if (!part.filename) + return yield* ProviderShared.invalidRequest("Bedrock Converse document media requires a filename") const media = yield* ProviderShared.validateMedia( "Bedrock Converse", part, new Set(Object.keys(DOCUMENT_FORMATS)), ) - return documentBlock(part, documentFormat, media.base64) + return documentBlock(part.filename, documentFormat, media.base64) } return yield* ProviderShared.invalidRequest(`Bedrock Converse does not support media type ${part.mediaType}`) }) diff --git a/packages/ai/src/tool-runtime.ts b/packages/ai/src/tool-runtime.ts index d69bbb9d47..c483950c12 100644 --- a/packages/ai/src/tool-runtime.ts +++ b/packages/ai/src/tool-runtime.ts @@ -68,10 +68,29 @@ const result = (call: ToolCallPart, value: ToolResultValueType | ToolSettlement, events: settlement.result.type === "error" ? [ - LLMEvent.toolError({ id: call.id, name: call.name, message: String(settlement.result.value), error }), - LLMEvent.toolResult({ id: call.id, name: call.name, result: settlement.result }), + LLMEvent.toolError({ + id: call.id, + name: call.name, + message: String(settlement.result.value), + error, + providerMetadata: call.providerMetadata, + }), + LLMEvent.toolResult({ + id: call.id, + name: call.name, + result: settlement.result, + providerMetadata: call.providerMetadata, + }), ] - : [LLMEvent.toolResult({ id: call.id, name: call.name, result: settlement.result, output: settlement.output })], + : [ + LLMEvent.toolResult({ + id: call.id, + name: call.name, + result: settlement.result, + output: settlement.output, + providerMetadata: call.providerMetadata, + }), + ], } } diff --git a/packages/ai/test/fixtures/recordings/pdf/anthropic-tool-result.json b/packages/ai/test/fixtures/recordings/pdf/anthropic-tool-result.json new file mode 100644 index 0000000000..69281239e5 --- /dev/null +++ b/packages/ai/test/fixtures/recordings/pdf/anthropic-tool-result.json @@ -0,0 +1,35 @@ +{ + "version": 1, + "metadata": { + "tags": [ + "prefix:pdf", + "pdf", + "provider:anthropic", + "protocol:anthropic-messages", + "tool", + "tool-result" + ], + "name": "pdf/anthropic-tool-result", + "recordedAt": "2026-07-22T18:15:39.002Z" + }, + "interactions": [ + { + "transport": "http", + "request": { + "method": "POST", + "url": "https://api.anthropic.com/v1/messages", + "headers": { + "content-type": "application/json" + }, + "body": "{\"model\":\"claude-haiku-4-5-20251001\",\"system\":[{\"type\":\"text\",\"text\":\"Read the PDF returned by the tool and follow the user's response format exactly.\"}],\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"text\",\"text\":\"Return only the verification code from the PDF.\"}]},{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"call_pdf_1\",\"name\":\"read_pdf\",\"input\":{}}]},{\"role\":\"user\",\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"call_pdf_1\",\"content\":[{\"type\":\"text\",\"text\":\"PDF read successfully\"},{\"type\":\"document\",\"source\":{\"type\":\"base64\",\"media_type\":\"application/pdf\",\"data\":\"JVBERi0xLjQKMSAwIG9iago8PCAvVHlwZSAvQ2F0YWxvZyAvUGFnZXMgMiAwIFIgPj4KZW5kb2JqCjIgMCBvYmoKPDwgL1R5cGUgL1BhZ2VzIC9LaWRzIFszIDAgUl0gL0NvdW50IDEgPj4KZW5kb2JqCjMgMCBvYmoKPDwgL1R5cGUgL1BhZ2UgL1BhcmVudCAyIDAgUiAvTWVkaWFCb3ggWzAgMCA2MTIgNzkyXSAvUmVzb3VyY2VzIDw8IC9Gb250IDw8IC9GMSA1IDAgUiA+PiA+PiAvQ29udGVudHMgNCAwIFIgPj4KZW5kb2JqCjQgMCBvYmoKPDwgL0xlbmd0aCA3NSA+PgpzdHJlYW0KQlQKL0YxIDE4IFRmCjcyIDcyMCBUZAooUERGIGNhc3NldHRlIHZlcmlmaWNhdGlvbiBjb2RlOiBPUkNISUQtNzM5MSkgVGoKRVQKZW5kc3RyZWFtCmVuZG9iago1IDAgb2JqCjw8IC9UeXBlIC9Gb250IC9TdWJ0eXBlIC9UeXBlMSAvQmFzZUZvbnQgL0hlbHZldGljYSA+PgplbmRvYmoKeHJlZgowIDYKMDAwMDAwMDAwMCA2NTUzNSBmIAowMDAwMDAwMDA5IDAwMDAwIG4gCjAwMDAwMDAwNTggMDAwMDAgbiAKMDAwMDAwMDExNSAwMDAwMCBuIAowMDAwMDAwMjQxIDAwMDAwIG4gCjAwMDAwMDAzNjUgMDAwMDAgbiAKdHJhaWxlcgo8PCAvU2l6ZSA2IC9Sb290IDEgMCBSID4+CnN0YXJ0eHJlZgo0MzUKJSVFT0YK\"}}]}]}],\"tools\":[{\"name\":\"read_pdf\",\"description\":\"Read the attached PDF.\",\"input_schema\":{\"type\":\"object\",\"properties\":{},\"additionalProperties\":false}}],\"stream\":true,\"max_tokens\":40,\"temperature\":0}" + }, + "response": { + "status": 200, + "headers": { + "content-type": "text/event-stream; charset=utf-8" + }, + "body": "event: message_start\ndata: {\"type\":\"message_start\",\"message\":{\"model\":\"claude-haiku-4-5-20251001\",\"id\":\"msg_011CdHYzxyRpSVFgwTm6ccUr\",\"type\":\"message\",\"role\":\"assistant\",\"content\":[],\"stop_reason\":null,\"stop_sequence\":null,\"stop_details\":null,\"usage\":{\"input_tokens\":2229,\"cache_creation_input_tokens\":0,\"cache_read_input_tokens\":0,\"cache_creation\":{\"ephemeral_5m_input_tokens\":0,\"ephemeral_1h_input_tokens\":0},\"output_tokens\":2,\"service_tier\":\"standard\",\"inference_geo\":\"not_available\"}} }\n\nevent: content_block_start\ndata: {\"type\":\"content_block_start\",\"index\":0,\"content_block\":{\"type\":\"text\",\"text\":\"\"} }\n\nevent: ping\ndata: {\"type\": \"ping\"}\n\nevent: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"text_delta\",\"text\":\"ORCH\"} }\n\nevent: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"text_delta\",\"text\":\"ID-7391\"} }\n\nevent: content_block_stop\ndata: {\"type\":\"content_block_stop\",\"index\":0 }\n\nevent: message_delta\ndata: {\"type\":\"message_delta\",\"delta\":{\"stop_reason\":\"end_turn\",\"stop_sequence\":null,\"stop_details\":null},\"usage\":{\"input_tokens\":2229,\"cache_creation_input_tokens\":0,\"cache_read_input_tokens\":0,\"output_tokens\":9} }\n\nevent: message_stop\ndata: {\"type\":\"message_stop\" }\n\n" + } + } + ] +} diff --git a/packages/ai/test/fixtures/recordings/pdf/anthropic-user-input.json b/packages/ai/test/fixtures/recordings/pdf/anthropic-user-input.json new file mode 100644 index 0000000000..372474e11f --- /dev/null +++ b/packages/ai/test/fixtures/recordings/pdf/anthropic-user-input.json @@ -0,0 +1,34 @@ +{ + "version": 1, + "metadata": { + "tags": [ + "prefix:pdf", + "pdf", + "provider:anthropic", + "protocol:anthropic-messages", + "user-input" + ], + "name": "pdf/anthropic-user-input", + "recordedAt": "2026-07-22T18:15:37.979Z" + }, + "interactions": [ + { + "transport": "http", + "request": { + "method": "POST", + "url": "https://api.anthropic.com/v1/messages", + "headers": { + "content-type": "application/json" + }, + "body": "{\"model\":\"claude-haiku-4-5-20251001\",\"messages\":[{\"role\":\"user\",\"content\":[{\"type\":\"document\",\"source\":{\"type\":\"base64\",\"media_type\":\"application/pdf\",\"data\":\"JVBERi0xLjQKMSAwIG9iago8PCAvVHlwZSAvQ2F0YWxvZyAvUGFnZXMgMiAwIFIgPj4KZW5kb2JqCjIgMCBvYmoKPDwgL1R5cGUgL1BhZ2VzIC9LaWRzIFszIDAgUl0gL0NvdW50IDEgPj4KZW5kb2JqCjMgMCBvYmoKPDwgL1R5cGUgL1BhZ2UgL1BhcmVudCAyIDAgUiAvTWVkaWFCb3ggWzAgMCA2MTIgNzkyXSAvUmVzb3VyY2VzIDw8IC9Gb250IDw8IC9GMSA1IDAgUiA+PiA+PiAvQ29udGVudHMgNCAwIFIgPj4KZW5kb2JqCjQgMCBvYmoKPDwgL0xlbmd0aCA3NSA+PgpzdHJlYW0KQlQKL0YxIDE4IFRmCjcyIDcyMCBUZAooUERGIGNhc3NldHRlIHZlcmlmaWNhdGlvbiBjb2RlOiBPUkNISUQtNzM5MSkgVGoKRVQKZW5kc3RyZWFtCmVuZG9iago1IDAgb2JqCjw8IC9UeXBlIC9Gb250IC9TdWJ0eXBlIC9UeXBlMSAvQmFzZUZvbnQgL0hlbHZldGljYSA+PgplbmRvYmoKeHJlZgowIDYKMDAwMDAwMDAwMCA2NTUzNSBmIAowMDAwMDAwMDA5IDAwMDAwIG4gCjAwMDAwMDAwNTggMDAwMDAgbiAKMDAwMDAwMDExNSAwMDAwMCBuIAowMDAwMDAwMjQxIDAwMDAwIG4gCjAwMDAwMDAzNjUgMDAwMDAgbiAKdHJhaWxlcgo8PCAvU2l6ZSA2IC9Sb290IDEgMCBSID4+CnN0YXJ0eHJlZgo0MzUKJSVFT0YK\"}},{\"type\":\"text\",\"text\":\"Return only the verification code from the PDF.\"}]}],\"stream\":true,\"max_tokens\":40,\"temperature\":0}" + }, + "response": { + "status": 200, + "headers": { + "content-type": "text/event-stream; charset=utf-8" + }, + "body": "event: message_start\ndata: {\"type\":\"message_start\",\"message\":{\"model\":\"claude-haiku-4-5-20251001\",\"id\":\"msg_011CdHYzsayb45rgfamcjFt3\",\"type\":\"message\",\"role\":\"assistant\",\"content\":[],\"stop_reason\":null,\"stop_sequence\":null,\"stop_details\":null,\"usage\":{\"input_tokens\":1602,\"cache_creation_input_tokens\":0,\"cache_read_input_tokens\":0,\"cache_creation\":{\"ephemeral_5m_input_tokens\":0,\"ephemeral_1h_input_tokens\":0},\"output_tokens\":2,\"service_tier\":\"standard\",\"inference_geo\":\"not_available\"}} }\n\nevent: content_block_start\ndata: {\"type\":\"content_block_start\",\"index\":0,\"content_block\":{\"type\":\"text\",\"text\":\"\"} }\n\nevent: ping\ndata: {\"type\": \"ping\"}\n\nevent: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"text_delta\",\"text\":\"ORCH\"} }\n\nevent: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"text_delta\",\"text\":\"ID-7391\"}}\n\nevent: content_block_stop\ndata: {\"type\":\"content_block_stop\",\"index\":0 }\n\nevent: message_delta\ndata: {\"type\":\"message_delta\",\"delta\":{\"stop_reason\":\"end_turn\",\"stop_sequence\":null,\"stop_details\":null},\"usage\":{\"input_tokens\":1602,\"cache_creation_input_tokens\":0,\"cache_read_input_tokens\":0,\"output_tokens\":9} }\n\nevent: message_stop\ndata: {\"type\":\"message_stop\" }\n\n" + } + } + ] +} diff --git a/packages/ai/test/fixtures/recordings/pdf/bedrock-tool-result.json b/packages/ai/test/fixtures/recordings/pdf/bedrock-tool-result.json new file mode 100644 index 0000000000..3225e019d6 --- /dev/null +++ b/packages/ai/test/fixtures/recordings/pdf/bedrock-tool-result.json @@ -0,0 +1,36 @@ +{ + "version": 1, + "metadata": { + "tags": [ + "prefix:pdf", + "pdf", + "provider:amazon-bedrock", + "protocol:bedrock-converse", + "tool", + "tool-result" + ], + "name": "pdf/bedrock-tool-result", + "recordedAt": "2026-07-22T18:15:52.400Z" + }, + "interactions": [ + { + "transport": "http", + "request": { + "method": "POST", + "url": "https://bedrock-runtime.us-east-1.amazonaws.com/model/us.anthropic.claude-haiku-4-5-20251001-v1%3A0/converse-stream", + "headers": { + "content-type": "application/json" + }, + "body": "{\"modelId\":\"us.anthropic.claude-haiku-4-5-20251001-v1:0\",\"messages\":[{\"role\":\"user\",\"content\":[{\"text\":\"Return only the verification code from the PDF.\"}]},{\"role\":\"assistant\",\"content\":[{\"toolUse\":{\"toolUseId\":\"call_pdf_1\",\"name\":\"read_pdf\",\"input\":{}}}]},{\"role\":\"user\",\"content\":[{\"toolResult\":{\"toolUseId\":\"call_pdf_1\",\"content\":[{\"text\":\"PDF read successfully\"},{\"document\":{\"format\":\"pdf\",\"name\":\"verification\",\"source\":{\"bytes\":\"JVBERi0xLjQKMSAwIG9iago8PCAvVHlwZSAvQ2F0YWxvZyAvUGFnZXMgMiAwIFIgPj4KZW5kb2JqCjIgMCBvYmoKPDwgL1R5cGUgL1BhZ2VzIC9LaWRzIFszIDAgUl0gL0NvdW50IDEgPj4KZW5kb2JqCjMgMCBvYmoKPDwgL1R5cGUgL1BhZ2UgL1BhcmVudCAyIDAgUiAvTWVkaWFCb3ggWzAgMCA2MTIgNzkyXSAvUmVzb3VyY2VzIDw8IC9Gb250IDw8IC9GMSA1IDAgUiA+PiA+PiAvQ29udGVudHMgNCAwIFIgPj4KZW5kb2JqCjQgMCBvYmoKPDwgL0xlbmd0aCA3NSA+PgpzdHJlYW0KQlQKL0YxIDE4IFRmCjcyIDcyMCBUZAooUERGIGNhc3NldHRlIHZlcmlmaWNhdGlvbiBjb2RlOiBPUkNISUQtNzM5MSkgVGoKRVQKZW5kc3RyZWFtCmVuZG9iago1IDAgb2JqCjw8IC9UeXBlIC9Gb250IC9TdWJ0eXBlIC9UeXBlMSAvQmFzZUZvbnQgL0hlbHZldGljYSA+PgplbmRvYmoKeHJlZgowIDYKMDAwMDAwMDAwMCA2NTUzNSBmIAowMDAwMDAwMDA5IDAwMDAwIG4gCjAwMDAwMDAwNTggMDAwMDAgbiAKMDAwMDAwMDExNSAwMDAwMCBuIAowMDAwMDAwMjQxIDAwMDAwIG4gCjAwMDAwMDAzNjUgMDAwMDAgbiAKdHJhaWxlcgo8PCAvU2l6ZSA2IC9Sb290IDEgMCBSID4+CnN0YXJ0eHJlZgo0MzUKJSVFT0YK\"}}}],\"status\":\"success\"}}]}],\"system\":[{\"text\":\"Read the PDF returned by the tool and follow the user's response format exactly.\"}],\"inferenceConfig\":{\"maxTokens\":40,\"temperature\":0},\"toolConfig\":{\"tools\":[{\"toolSpec\":{\"name\":\"read_pdf\",\"description\":\"Read the attached PDF.\",\"inputSchema\":{\"json\":{\"type\":\"object\",\"properties\":{},\"additionalProperties\":false}}}}]}}" + }, + "response": { + "status": 200, + "headers": { + "content-type": "application/vnd.amazon.eventstream" + }, + "body": "AAAAqgAAAFLa0GiGCzpldmVudC10eXBlBwAMbWVzc2FnZVN0YXJ0DTpjb250ZW50LXR5cGUHABBhcHBsaWNhdGlvbi9qc29uDTptZXNzYWdlLXR5cGUHAAVldmVudHsicCI6ImFiY2RlZmdoaWprbG1ub3BxcnN0dXZ3eHl6QUJDREVGR0hJSktMTU5PUFFSUyIsInJvbGUiOiJhc3Npc3RhbnQifXIDPnsAAADIAAAAV0lIuCQLOmV2ZW50LXR5cGUHABFjb250ZW50QmxvY2tEZWx0YQ06Y29udGVudC10eXBlBwAQYXBwbGljYXRpb24vanNvbg06bWVzc2FnZS10eXBlBwAFZXZlbnR7ImNvbnRlbnRCbG9ja0luZGV4IjowLCJkZWx0YSI6eyJ0ZXh0IjoiT1JDSCJ9LCJwIjoiYWJjZGVmZ2hpamtsbW5vcHFyc3R1dnd4eXpBQkNERUZHSElKS0xNTk9QUSJ9z2HHHgAAAMUAAABXsdh8lQs6ZXZlbnQtdHlwZQcAEWNvbnRlbnRCbG9ja0RlbHRhDTpjb250ZW50LXR5cGUHABBhcHBsaWNhdGlvbi9qc29uDTptZXNzYWdlLXR5cGUHAAVldmVudHsiY29udGVudEJsb2NrSW5kZXgiOjAsImRlbHRhIjp7InRleHQiOiJJRC0ifSwicCI6ImFiY2RlZmdoaWprbG1ub3BxcnN0dXZ3eHl6QUJDREVGR0hJSktMTU5PIn2+8d8RAAAAywAAAFcO6ML0CzpldmVudC10eXBlBwARY29udGVudEJsb2NrRGVsdGENOmNvbnRlbnQtdHlwZQcAEGFwcGxpY2F0aW9uL2pzb24NOm1lc3NhZ2UtdHlwZQcABWV2ZW50eyJjb250ZW50QmxvY2tJbmRleCI6MCwiZGVsdGEiOnsidGV4dCI6IjcifSwicCI6ImFiY2RlZmdoaWprbG1ub3BxcnN0dXZ3eHl6QUJDREVGR0hJSktMTU5PUFFSU1RVVlcifSIeZ+kAAACzAAAAV8dKafoLOmV2ZW50LXR5cGUHABFjb250ZW50QmxvY2tEZWx0YQ06Y29udGVudC10eXBlBwAQYXBwbGljYXRpb24vanNvbg06bWVzc2FnZS10eXBlBwAFZXZlbnR7ImNvbnRlbnRCbG9ja0luZGV4IjowLCJkZWx0YSI6eyJ0ZXh0IjoiMzkxIn0sInAiOiJhYmNkZWZnaGlqa2xtbm9wcXJzdHV2dyJ9HAStJQAAAMAAAABWDj/Dcws6ZXZlbnQtdHlwZQcAEGNvbnRlbnRCbG9ja1N0b3ANOmNvbnRlbnQtdHlwZQcAEGFwcGxpY2F0aW9uL2pzb24NOm1lc3NhZ2UtdHlwZQcABWV2ZW50eyJjb250ZW50QmxvY2tJbmRleCI6MCwicCI6ImFiY2RlZmdoaWprbG1ub3BxcnN0dXZ3eHl6QUJDREVGR0hJSktMTU5PUFFSU1RVVldYWVowMTIzNDU2NyJ9EPTSwQAAALAAAABRaYm2Hws6ZXZlbnQtdHlwZQcAC21lc3NhZ2VTdG9wDTpjb250ZW50LXR5cGUHABBhcHBsaWNhdGlvbi9qc29uDTptZXNzYWdlLXR5cGUHAAVldmVudHsicCI6ImFiY2RlZmdoaWprbG1ub3BxcnN0dXZ3eHl6QUJDREVGR0hJSktMTU5PUFFSU1RVIiwic3RvcFJlYXNvbiI6ImVuZF90dXJuIn0SuCAcAAAA+AAAAE6MAqhiCzpldmVudC10eXBlBwAIbWV0YWRhdGENOmNvbnRlbnQtdHlwZQcAEGFwcGxpY2F0aW9uL2pzb24NOm1lc3NhZ2UtdHlwZQcABWV2ZW50eyJtZXRyaWNzIjp7ImxhdGVuY3lNcyI6MzkyMn0sInAiOiJhYmNkZWZnaGlqa2xtbm9wcXJzdHV2d3h5ekFCQ0RFIiwidXNhZ2UiOnsiaW5wdXRUb2tlbnMiOjIyNDEsIm91dHB1dFRva2VucyI6Niwic2VydmVyVG9vbFVzYWdlIjp7fSwidG90YWxUb2tlbnMiOjIyNDd9fcd35Hw=", + "bodyEncoding": "base64" + } + } + ] +} diff --git a/packages/ai/test/fixtures/recordings/pdf/bedrock-user-input.json b/packages/ai/test/fixtures/recordings/pdf/bedrock-user-input.json new file mode 100644 index 0000000000..53f04e5777 --- /dev/null +++ b/packages/ai/test/fixtures/recordings/pdf/bedrock-user-input.json @@ -0,0 +1,35 @@ +{ + "version": 1, + "metadata": { + "tags": [ + "prefix:pdf", + "pdf", + "provider:amazon-bedrock", + "protocol:bedrock-converse", + "user-input" + ], + "name": "pdf/bedrock-user-input", + "recordedAt": "2026-07-22T18:15:48.408Z" + }, + "interactions": [ + { + "transport": "http", + "request": { + "method": "POST", + "url": "https://bedrock-runtime.us-east-1.amazonaws.com/model/us.anthropic.claude-haiku-4-5-20251001-v1%3A0/converse-stream", + "headers": { + "content-type": "application/json" + }, + "body": "{\"modelId\":\"us.anthropic.claude-haiku-4-5-20251001-v1:0\",\"messages\":[{\"role\":\"user\",\"content\":[{\"document\":{\"format\":\"pdf\",\"name\":\"verification\",\"source\":{\"bytes\":\"JVBERi0xLjQKMSAwIG9iago8PCAvVHlwZSAvQ2F0YWxvZyAvUGFnZXMgMiAwIFIgPj4KZW5kb2JqCjIgMCBvYmoKPDwgL1R5cGUgL1BhZ2VzIC9LaWRzIFszIDAgUl0gL0NvdW50IDEgPj4KZW5kb2JqCjMgMCBvYmoKPDwgL1R5cGUgL1BhZ2UgL1BhcmVudCAyIDAgUiAvTWVkaWFCb3ggWzAgMCA2MTIgNzkyXSAvUmVzb3VyY2VzIDw8IC9Gb250IDw8IC9GMSA1IDAgUiA+PiA+PiAvQ29udGVudHMgNCAwIFIgPj4KZW5kb2JqCjQgMCBvYmoKPDwgL0xlbmd0aCA3NSA+PgpzdHJlYW0KQlQKL0YxIDE4IFRmCjcyIDcyMCBUZAooUERGIGNhc3NldHRlIHZlcmlmaWNhdGlvbiBjb2RlOiBPUkNISUQtNzM5MSkgVGoKRVQKZW5kc3RyZWFtCmVuZG9iago1IDAgb2JqCjw8IC9UeXBlIC9Gb250IC9TdWJ0eXBlIC9UeXBlMSAvQmFzZUZvbnQgL0hlbHZldGljYSA+PgplbmRvYmoKeHJlZgowIDYKMDAwMDAwMDAwMCA2NTUzNSBmIAowMDAwMDAwMDA5IDAwMDAwIG4gCjAwMDAwMDAwNTggMDAwMDAgbiAKMDAwMDAwMDExNSAwMDAwMCBuIAowMDAwMDAwMjQxIDAwMDAwIG4gCjAwMDAwMDAzNjUgMDAwMDAgbiAKdHJhaWxlcgo8PCAvU2l6ZSA2IC9Sb290IDEgMCBSID4+CnN0YXJ0eHJlZgo0MzUKJSVFT0YK\"}}},{\"text\":\"Return only the verification code from the PDF.\"}]}],\"inferenceConfig\":{\"maxTokens\":40,\"temperature\":0}}" + }, + "response": { + "status": 200, + "headers": { + "content-type": "application/vnd.amazon.eventstream" + }, + "body": "AAAAtgAAAFJ/wBIFCzpldmVudC10eXBlBwAMbWVzc2FnZVN0YXJ0DTpjb250ZW50LXR5cGUHABBhcHBsaWNhdGlvbi9qc29uDTptZXNzYWdlLXR5cGUHAAVldmVudHsicCI6ImFiY2RlZmdoaWprbG1ub3BxcnN0dXZ3eHl6QUJDREVGR0hJSktMTU5PUFFSU1RVVldYWVowMTIzNCIsInJvbGUiOiJhc3Npc3RhbnQifURlAvAAAADGAAAAV/Z4BkULOmV2ZW50LXR5cGUHABFjb250ZW50QmxvY2tEZWx0YQ06Y29udGVudC10eXBlBwAQYXBwbGljYXRpb24vanNvbg06bWVzc2FnZS10eXBlBwAFZXZlbnR7ImNvbnRlbnRCbG9ja0luZGV4IjowLCJkZWx0YSI6eyJ0ZXh0IjoiT1JDSCJ9LCJwIjoiYWJjZGVmZ2hpamtsbW5vcHFyc3R1dnd4eXpBQkNERUZHSElKS0xNTk8ifU1V/fQAAADWAAAAV5aYkccLOmV2ZW50LXR5cGUHABFjb250ZW50QmxvY2tEZWx0YQ06Y29udGVudC10eXBlBwAQYXBwbGljYXRpb24vanNvbg06bWVzc2FnZS10eXBlBwAFZXZlbnR7ImNvbnRlbnRCbG9ja0luZGV4IjowLCJkZWx0YSI6eyJ0ZXh0IjoiSUQtIn0sInAiOiJhYmNkZWZnaGlqa2xtbm9wcXJzdHV2d3h5ekFCQ0RFRkdISUpLTE1OT1BRUlNUVVZXWFlaMDEyMzQ1In1Rr1g8AAAAoAAAAFfgCoSoCzpldmVudC10eXBlBwARY29udGVudEJsb2NrRGVsdGENOmNvbnRlbnQtdHlwZQcAEGFwcGxpY2F0aW9uL2pzb24NOm1lc3NhZ2UtdHlwZQcABWV2ZW50eyJjb250ZW50QmxvY2tJbmRleCI6MCwiZGVsdGEiOnsidGV4dCI6IjcifSwicCI6ImFiY2RlZiJ9UwQMPQAAAM8AAABX+2hkNAs6ZXZlbnQtdHlwZQcAEWNvbnRlbnRCbG9ja0RlbHRhDTpjb250ZW50LXR5cGUHABBhcHBsaWNhdGlvbi9qc29uDTptZXNzYWdlLXR5cGUHAAVldmVudHsiY29udGVudEJsb2NrSW5kZXgiOjAsImRlbHRhIjp7InRleHQiOiIzOTEifSwicCI6ImFiY2RlZmdoaWprbG1ub3BxcnN0dXZ3eHl6QUJDREVGR0hJSktMTU5PUFFSU1RVVldYWSJ9ZmoyCwAAAJAAAABWNiwMuAs6ZXZlbnQtdHlwZQcAEGNvbnRlbnRCbG9ja1N0b3ANOmNvbnRlbnQtdHlwZQcAEGFwcGxpY2F0aW9uL2pzb24NOm1lc3NhZ2UtdHlwZQcABWV2ZW50eyJjb250ZW50QmxvY2tJbmRleCI6MCwicCI6ImFiY2RlZmdoaWprbCJ9wtmmXgAAAIgAAABR+NhFWAs6ZXZlbnQtdHlwZQcAC21lc3NhZ2VTdG9wDTpjb250ZW50LXR5cGUHABBhcHBsaWNhdGlvbi9qc29uDTptZXNzYWdlLXR5cGUHAAVldmVudHsicCI6ImFiY2RlZmciLCJzdG9wUmVhc29uIjoiZW5kX3R1cm4ifa8D/doAAADvAAAATl7C4/ALOmV2ZW50LXR5cGUHAAhtZXRhZGF0YQ06Y29udGVudC10eXBlBwAQYXBwbGljYXRpb24vanNvbg06bWVzc2FnZS10eXBlBwAFZXZlbnR7Im1ldHJpY3MiOnsibGF0ZW5jeU1zIjo0NTQ1fSwicCI6ImFiY2RlZmdoaWprbG1ub3BxcnN0dXYiLCJ1c2FnZSI6eyJpbnB1dFRva2VucyI6MTYxNCwib3V0cHV0VG9rZW5zIjo2LCJzZXJ2ZXJUb29sVXNhZ2UiOnt9LCJ0b3RhbFRva2VucyI6MTYyMH19db4j2Q==", + "bodyEncoding": "base64" + } + } + ] +} diff --git a/packages/ai/test/fixtures/recordings/pdf/gemini-tool-result.json b/packages/ai/test/fixtures/recordings/pdf/gemini-tool-result.json new file mode 100644 index 0000000000..1ac7bc5f6e --- /dev/null +++ b/packages/ai/test/fixtures/recordings/pdf/gemini-tool-result.json @@ -0,0 +1,53 @@ +{ + "version": 1, + "metadata": { + "tags": [ + "prefix:pdf", + "pdf", + "provider:google", + "protocol:gemini", + "tool", + "tool-result" + ], + "name": "pdf/gemini-tool-result", + "recordedAt": "2026-07-22T18:21:59.606Z" + }, + "interactions": [ + { + "transport": "http", + "request": { + "method": "POST", + "url": "https://generativelanguage.googleapis.com/v1beta/models/gemini-3.5-flash:streamGenerateContent?alt=sse", + "headers": { + "content-type": "application/json" + }, + "body": "{\"contents\":[{\"role\":\"user\",\"parts\":[{\"text\":\"Use read_pdf with path verification.pdf and return the verification code.\"}]}],\"systemInstruction\":{\"parts\":[{\"text\":\"Call read_pdf exactly once with path verification.pdf, then reply only with the verification code from its PDF.\"}]},\"tools\":[{\"functionDeclarations\":[{\"name\":\"read_pdf\",\"description\":\"Read the attached PDF.\",\"parameters\":{\"required\":[\"path\"],\"type\":\"object\",\"properties\":{\"path\":{\"type\":\"string\"}}}}]}],\"generationConfig\":{\"maxOutputTokens\":256,\"temperature\":0}}" + }, + "response": { + "status": 200, + "headers": { + "content-type": "text/event-stream" + }, + "body": "data: {\"candidates\": [{\"content\": {\"parts\": [{\"functionCall\": {\"name\": \"read_pdf\",\"args\": {\"path\": \"verification.pdf\"},\"id\": \"58shgmez\"},\"thoughtSignature\": \"EqkCCqYCARFNMg/JrCTv5i3zYENFBVpZNFL3pbzJmi5Eu387ncF703xFMB4pwyaP7a1gi49EqBhCI2hWOpesU5nZQOLAhGgExKGa2GM+HzpEB5g62r0NFblm/BGkVZaImTuHR7bytfRC5jHQlHKo4OS27OLUVjvkMkBIYsvjhDErY7niERbXJVpyxTVqUf1GgZMSu8kC9/5WDlMs9xVKNT/6KMW4PhhSR9nXg4KZUa+bC03/ydhsWWgBa5aLCgvTq7WPj217xIsmUkSiRedIffPsUSNjYdMHUvWi8bOlvM1veEEP6GIfv5h9gXXzjnHbEHfQxV8PZuBAyY7iM6nqyfkJNdkZ1HdB7DXMBsMsRN6SgrIrFoXX2WaGrkoEI5tdZx1t/gdwF1jEVT6k\"}],\"role\": \"model\"},\"index\": 0}],\"usageMetadata\": {\"promptTokenCount\": 81,\"candidatesTokenCount\": 18,\"totalTokenCount\": 151,\"promptTokensDetails\": [{\"modality\": \"TEXT\",\"tokenCount\": 81}],\"thoughtsTokenCount\": 52,\"serviceTier\": \"standard\"},\"modelVersion\": \"gemini-3.5-flash\",\"responseId\": \"RAphaui3OaSHz7IPy8Kb4Ak\"}\r\n\r\ndata: {\"candidates\": [{\"content\": {\"parts\": [{\"text\": \"\"}],\"role\": \"model\"},\"finishReason\": \"STOP\",\"index\": 0}],\"usageMetadata\": {\"promptTokenCount\": 81,\"candidatesTokenCount\": 18,\"totalTokenCount\": 151,\"promptTokensDetails\": [{\"modality\": \"TEXT\",\"tokenCount\": 81}],\"thoughtsTokenCount\": 52,\"serviceTier\": \"standard\"},\"modelVersion\": \"gemini-3.5-flash\",\"responseId\": \"RAphaui3OaSHz7IPy8Kb4Ak\"}\r\n\r\n" + } + }, + { + "transport": "http", + "request": { + "method": "POST", + "url": "https://generativelanguage.googleapis.com/v1beta/models/gemini-3.5-flash:streamGenerateContent?alt=sse", + "headers": { + "content-type": "application/json" + }, + "body": "{\"contents\":[{\"role\":\"user\",\"parts\":[{\"text\":\"Use read_pdf with path verification.pdf and return the verification code.\"}]},{\"role\":\"model\",\"parts\":[{\"functionCall\":{\"id\":\"58shgmez\",\"name\":\"read_pdf\",\"args\":{\"path\":\"verification.pdf\"}},\"thoughtSignature\":\"EqkCCqYCARFNMg/JrCTv5i3zYENFBVpZNFL3pbzJmi5Eu387ncF703xFMB4pwyaP7a1gi49EqBhCI2hWOpesU5nZQOLAhGgExKGa2GM+HzpEB5g62r0NFblm/BGkVZaImTuHR7bytfRC5jHQlHKo4OS27OLUVjvkMkBIYsvjhDErY7niERbXJVpyxTVqUf1GgZMSu8kC9/5WDlMs9xVKNT/6KMW4PhhSR9nXg4KZUa+bC03/ydhsWWgBa5aLCgvTq7WPj217xIsmUkSiRedIffPsUSNjYdMHUvWi8bOlvM1veEEP6GIfv5h9gXXzjnHbEHfQxV8PZuBAyY7iM6nqyfkJNdkZ1HdB7DXMBsMsRN6SgrIrFoXX2WaGrkoEI5tdZx1t/gdwF1jEVT6k\"}]},{\"role\":\"user\",\"parts\":[{\"functionResponse\":{\"id\":\"58shgmez\",\"name\":\"read_pdf\",\"response\":{\"name\":\"read_pdf\",\"content\":\"PDF read successfully\"},\"parts\":[{\"inlineData\":{\"mimeType\":\"application/pdf\",\"data\":\"JVBERi0xLjQKMSAwIG9iago8PCAvVHlwZSAvQ2F0YWxvZyAvUGFnZXMgMiAwIFIgPj4KZW5kb2JqCjIgMCBvYmoKPDwgL1R5cGUgL1BhZ2VzIC9LaWRzIFszIDAgUl0gL0NvdW50IDEgPj4KZW5kb2JqCjMgMCBvYmoKPDwgL1R5cGUgL1BhZ2UgL1BhcmVudCAyIDAgUiAvTWVkaWFCb3ggWzAgMCA2MTIgNzkyXSAvUmVzb3VyY2VzIDw8IC9Gb250IDw8IC9GMSA1IDAgUiA+PiA+PiAvQ29udGVudHMgNCAwIFIgPj4KZW5kb2JqCjQgMCBvYmoKPDwgL0xlbmd0aCA3NSA+PgpzdHJlYW0KQlQKL0YxIDE4IFRmCjcyIDcyMCBUZAooUERGIGNhc3NldHRlIHZlcmlmaWNhdGlvbiBjb2RlOiBPUkNISUQtNzM5MSkgVGoKRVQKZW5kc3RyZWFtCmVuZG9iago1IDAgb2JqCjw8IC9UeXBlIC9Gb250IC9TdWJ0eXBlIC9UeXBlMSAvQmFzZUZvbnQgL0hlbHZldGljYSA+PgplbmRvYmoKeHJlZgowIDYKMDAwMDAwMDAwMCA2NTUzNSBmIAowMDAwMDAwMDA5IDAwMDAwIG4gCjAwMDAwMDAwNTggMDAwMDAgbiAKMDAwMDAwMDExNSAwMDAwMCBuIAowMDAwMDAwMjQxIDAwMDAwIG4gCjAwMDAwMDAzNjUgMDAwMDAgbiAKdHJhaWxlcgo8PCAvU2l6ZSA2IC9Sb290IDEgMCBSID4+CnN0YXJ0eHJlZgo0MzUKJSVFT0YK\"}}]}}]}],\"systemInstruction\":{\"parts\":[{\"text\":\"Call read_pdf exactly once with path verification.pdf, then reply only with the verification code from its PDF.\"}]},\"tools\":[{\"functionDeclarations\":[{\"name\":\"read_pdf\",\"description\":\"Read the attached PDF.\",\"parameters\":{\"required\":[\"path\"],\"type\":\"object\",\"properties\":{\"path\":{\"type\":\"string\"}}}}]}],\"generationConfig\":{\"maxOutputTokens\":256,\"temperature\":0}}" + }, + "response": { + "status": 200, + "headers": { + "content-type": "text/event-stream" + }, + "body": "data: {\"candidates\": [{\"content\": {\"parts\": [{\"text\": \"ORCHID-7391\"}],\"role\": \"model\"},\"index\": 0}],\"usageMetadata\": {\"promptTokenCount\": 123,\"candidatesTokenCount\": 8,\"totalTokenCount\": 184,\"promptTokensDetails\": [{\"modality\": \"TEXT\",\"tokenCount\": 123}],\"thoughtsTokenCount\": 53,\"serviceTier\": \"standard\"},\"modelVersion\": \"gemini-3.5-flash\",\"responseId\": \"RgphaoL6CMjQz7IPjOnEmQI\"}\r\n\r\ndata: {\"candidates\": [{\"content\": {\"parts\": [{\"text\": \"\",\"thoughtSignature\": \"EqECCp4CARFNMg9obBl8O6iU9lawUIWiE+1vztZm9NtaT9FuyJz343hd9ruz+xPco4Q1DY1GF81ZiSI2ElBkt8Wfwsqtix9LNGSMvbZhhk/ZnB54t05M/Dft1kujcMvEdZUWUI/jWaJ349tO1bKVH9MacG5+gl0n4y8DwyQZSV3xIcet547drSkcA/TM03RB+yj1/dcLHsvUjmv9EnO897vZgO2Dk4tbZ2NyCtOeQ3JKVhUTLg2pjkGk+POCNiOdESWiUzxdQKw9LiV6nnzi071tXNiMeVimq6d7xAzRVNapI2uXynvn9Uk3eyn85purOFa8cKriK9oD6vcyGMqgd9+gu2m3to0IHqd7o+2YSr1m5qV1xT1R2/WRQEtb1b1AuOAU6w==\"}],\"role\": \"model\"},\"finishReason\": \"STOP\",\"index\": 0}],\"usageMetadata\": {\"promptTokenCount\": 1277,\"candidatesTokenCount\": 8,\"totalTokenCount\": 1338,\"promptTokensDetails\": [{\"modality\": \"IMAGE\",\"tokenCount\": 1102},{\"modality\": \"TEXT\",\"tokenCount\": 175}],\"thoughtsTokenCount\": 53,\"serviceTier\": \"standard\"},\"modelVersion\": \"gemini-3.5-flash\",\"responseId\": \"RgphaoL6CMjQz7IPjOnEmQI\"}\r\n\r\n" + } + } + ] +} diff --git a/packages/ai/test/fixtures/recordings/pdf/gemini-user-input.json b/packages/ai/test/fixtures/recordings/pdf/gemini-user-input.json new file mode 100644 index 0000000000..dbe73618a0 --- /dev/null +++ b/packages/ai/test/fixtures/recordings/pdf/gemini-user-input.json @@ -0,0 +1,34 @@ +{ + "version": 1, + "metadata": { + "tags": [ + "prefix:pdf", + "pdf", + "provider:google", + "protocol:gemini", + "user-input" + ], + "name": "pdf/gemini-user-input", + "recordedAt": "2026-07-22T18:20:55.140Z" + }, + "interactions": [ + { + "transport": "http", + "request": { + "method": "POST", + "url": "https://generativelanguage.googleapis.com/v1beta/models/gemini-3.5-flash:streamGenerateContent?alt=sse", + "headers": { + "content-type": "application/json" + }, + "body": "{\"contents\":[{\"role\":\"user\",\"parts\":[{\"inlineData\":{\"mimeType\":\"application/pdf\",\"data\":\"JVBERi0xLjQKMSAwIG9iago8PCAvVHlwZSAvQ2F0YWxvZyAvUGFnZXMgMiAwIFIgPj4KZW5kb2JqCjIgMCBvYmoKPDwgL1R5cGUgL1BhZ2VzIC9LaWRzIFszIDAgUl0gL0NvdW50IDEgPj4KZW5kb2JqCjMgMCBvYmoKPDwgL1R5cGUgL1BhZ2UgL1BhcmVudCAyIDAgUiAvTWVkaWFCb3ggWzAgMCA2MTIgNzkyXSAvUmVzb3VyY2VzIDw8IC9Gb250IDw8IC9GMSA1IDAgUiA+PiA+PiAvQ29udGVudHMgNCAwIFIgPj4KZW5kb2JqCjQgMCBvYmoKPDwgL0xlbmd0aCA3NSA+PgpzdHJlYW0KQlQKL0YxIDE4IFRmCjcyIDcyMCBUZAooUERGIGNhc3NldHRlIHZlcmlmaWNhdGlvbiBjb2RlOiBPUkNISUQtNzM5MSkgVGoKRVQKZW5kc3RyZWFtCmVuZG9iago1IDAgb2JqCjw8IC9UeXBlIC9Gb250IC9TdWJ0eXBlIC9UeXBlMSAvQmFzZUZvbnQgL0hlbHZldGljYSA+PgplbmRvYmoKeHJlZgowIDYKMDAwMDAwMDAwMCA2NTUzNSBmIAowMDAwMDAwMDA5IDAwMDAwIG4gCjAwMDAwMDAwNTggMDAwMDAgbiAKMDAwMDAwMDExNSAwMDAwMCBuIAowMDAwMDAwMjQxIDAwMDAwIG4gCjAwMDAwMDAzNjUgMDAwMDAgbiAKdHJhaWxlcgo8PCAvU2l6ZSA2IC9Sb290IDEgMCBSID4+CnN0YXJ0eHJlZgo0MzUKJSVFT0YK\"}},{\"text\":\"Return only the verification code from the PDF.\"}]}],\"generationConfig\":{\"maxOutputTokens\":256,\"temperature\":0}}" + }, + "response": { + "status": 200, + "headers": { + "content-type": "text/event-stream" + }, + "body": "data: {\"candidates\": [{\"content\": {\"parts\": [{\"text\": \"ORCH\"}],\"role\": \"model\"},\"index\": 0}],\"usageMetadata\": {\"promptTokenCount\": 10,\"candidatesTokenCount\": 2,\"totalTokenCount\": 127,\"promptTokensDetails\": [{\"modality\": \"TEXT\",\"tokenCount\": 10}],\"thoughtsTokenCount\": 115,\"serviceTier\": \"standard\"},\"modelVersion\": \"gemini-3.5-flash\",\"responseId\": \"BQpharW2KaPgz7IP6uSLiAw\"}\r\n\r\ndata: {\"candidates\": [{\"content\": {\"parts\": [{\"text\": \"ID-7391\"}],\"role\": \"model\"},\"index\": 0}],\"usageMetadata\": {\"promptTokenCount\": 10,\"candidatesTokenCount\": 8,\"totalTokenCount\": 133,\"promptTokensDetails\": [{\"modality\": \"TEXT\",\"tokenCount\": 10}],\"thoughtsTokenCount\": 115,\"serviceTier\": \"standard\"},\"modelVersion\": \"gemini-3.5-flash\",\"responseId\": \"BQpharW2KaPgz7IP6uSLiAw\"}\r\n\r\ndata: {\"candidates\": [{\"content\": {\"parts\": [{\"text\": \"\",\"thoughtSignature\": \"EokECoYEARFNMg8L4fpLqaX8tIQZcvw2vLt3WsFjGqpuJGgna0/AGczwuzndRcf3LGIEaliCf4ijVOb1AG4/VPBh1kMzfjeAyHhvWIe4yQVoBwI7BjpFyLie+SnGTXQXKKy5ygRqRLFsV6DcAixNXXBHJw2x/2Nhtriryqs4fhWrL/P7ppHC10sMnTwN6Mw5x20NKwgT+rrw6lvYmQe9rdQsBJ6Zmp0GpPlwZZiAgzvwPfVoNwHSGb54xe/T9wjISjwWNgpedhbsIBDRZFDwruS4x57KBKeMPO69GLfeMP8PJ7rpR0HgT7nRbrl/OdykG/jqSMTvoRSqxawsD+Yr/DukgGatyfB5Ic+X4RhD07URpkGTAu/cakBtzhSmM/hpzKU9m/cId1UCjopLTtonUqSAKkroPdp8kIYw0MI2OZCNVwbDrdClUPmjRKfcTkcC2jNj1rS+WDFbm+mo+SP3rDSvvCdyJuiXHGKiM2EhbYnu42aHVC6w7eAe4Gv3Fq/0faW47r0ihbiAohFB9XUA+fD07g83EjIuc9Q6BRVTTcBfoRkrR/yFZKt3qwPq02W6rPD13/1wAnMtabNcxePMMGk7Dlxwng9yPS0NEge2KD+miOj9SC4aTvOTq2451tfK1x3UZqqb205zGOPbjizhH/CA/PGkG84hdkAG4mrUK0rEHqeWwRXDsxpyfto=\"}],\"role\": \"model\"},\"finishReason\": \"STOP\",\"index\": 0}],\"usageMetadata\": {\"promptTokenCount\": 530,\"candidatesTokenCount\": 8,\"totalTokenCount\": 653,\"promptTokensDetails\": [{\"modality\": \"IMAGE\",\"tokenCount\": 520},{\"modality\": \"TEXT\",\"tokenCount\": 10}],\"thoughtsTokenCount\": 115,\"serviceTier\": \"standard\"},\"modelVersion\": \"gemini-3.5-flash\",\"responseId\": \"BQpharW2KaPgz7IP6uSLiAw\"}\r\n\r\n" + } + } + ] +} diff --git a/packages/ai/test/fixtures/recordings/pdf/openai-tool-result.json b/packages/ai/test/fixtures/recordings/pdf/openai-tool-result.json new file mode 100644 index 0000000000..42363b5ec8 --- /dev/null +++ b/packages/ai/test/fixtures/recordings/pdf/openai-tool-result.json @@ -0,0 +1,35 @@ +{ + "version": 1, + "metadata": { + "tags": [ + "prefix:pdf", + "pdf", + "provider:openai", + "protocol:openai-responses", + "tool", + "tool-result" + ], + "name": "pdf/openai-tool-result", + "recordedAt": "2026-07-22T18:15:36.438Z" + }, + "interactions": [ + { + "transport": "http", + "request": { + "method": "POST", + "url": "https://api.openai.com/v1/responses", + "headers": { + "content-type": "application/json" + }, + "body": "{\"model\":\"gpt-4o-mini\",\"input\":[{\"role\":\"system\",\"content\":\"Read the PDF returned by the tool and follow the user's response format exactly.\"},{\"role\":\"user\",\"content\":[{\"type\":\"input_text\",\"text\":\"Return only the verification code from the PDF.\"}]},{\"type\":\"function_call\",\"call_id\":\"call_pdf_1\",\"name\":\"read_pdf\",\"arguments\":\"{}\"},{\"type\":\"function_call_output\",\"call_id\":\"call_pdf_1\",\"output\":[{\"type\":\"input_text\",\"text\":\"PDF read successfully\"},{\"type\":\"input_file\",\"filename\":\"verification.pdf\",\"file_data\":\"data:application/pdf;base64,JVBERi0xLjQKMSAwIG9iago8PCAvVHlwZSAvQ2F0YWxvZyAvUGFnZXMgMiAwIFIgPj4KZW5kb2JqCjIgMCBvYmoKPDwgL1R5cGUgL1BhZ2VzIC9LaWRzIFszIDAgUl0gL0NvdW50IDEgPj4KZW5kb2JqCjMgMCBvYmoKPDwgL1R5cGUgL1BhZ2UgL1BhcmVudCAyIDAgUiAvTWVkaWFCb3ggWzAgMCA2MTIgNzkyXSAvUmVzb3VyY2VzIDw8IC9Gb250IDw8IC9GMSA1IDAgUiA+PiA+PiAvQ29udGVudHMgNCAwIFIgPj4KZW5kb2JqCjQgMCBvYmoKPDwgL0xlbmd0aCA3NSA+PgpzdHJlYW0KQlQKL0YxIDE4IFRmCjcyIDcyMCBUZAooUERGIGNhc3NldHRlIHZlcmlmaWNhdGlvbiBjb2RlOiBPUkNISUQtNzM5MSkgVGoKRVQKZW5kc3RyZWFtCmVuZG9iago1IDAgb2JqCjw8IC9UeXBlIC9Gb250IC9TdWJ0eXBlIC9UeXBlMSAvQmFzZUZvbnQgL0hlbHZldGljYSA+PgplbmRvYmoKeHJlZgowIDYKMDAwMDAwMDAwMCA2NTUzNSBmIAowMDAwMDAwMDA5IDAwMDAwIG4gCjAwMDAwMDAwNTggMDAwMDAgbiAKMDAwMDAwMDExNSAwMDAwMCBuIAowMDAwMDAwMjQxIDAwMDAwIG4gCjAwMDAwMDAzNjUgMDAwMDAgbiAKdHJhaWxlcgo8PCAvU2l6ZSA2IC9Sb290IDEgMCBSID4+CnN0YXJ0eHJlZgo0MzUKJSVFT0YK\"}]}],\"tools\":[{\"type\":\"function\",\"name\":\"read_pdf\",\"description\":\"Read the attached PDF.\",\"parameters\":{\"type\":\"object\",\"properties\":{},\"additionalProperties\":false},\"strict\":false}],\"store\":false,\"max_output_tokens\":40,\"temperature\":0,\"stream\":true}" + }, + "response": { + "status": 200, + "headers": { + "content-type": "text/event-stream; charset=utf-8" + }, + "body": "event: response.created\ndata: {\"type\":\"response.created\",\"response\":{\"id\":\"resp_0cd823aeeb11ef0e016a6108c703b88193872d736c48d60e2c\",\"object\":\"response\",\"created_at\":1784744135,\"status\":\"in_progress\",\"background\":false,\"completed_at\":null,\"error\":null,\"frequency_penalty\":0.0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":40,\"max_tool_calls\":null,\"model\":\"gpt-4o-mini-2024-07-18\",\"moderation\":null,\"output\":[],\"parallel_tool_calls\":true,\"presence_penalty\":0.0,\"previous_response_id\":null,\"prompt_cache_key\":null,\"prompt_cache_retention\":\"in_memory\",\"reasoning\":{\"context\":null,\"effort\":null,\"summary\":null},\"safety_identifier\":null,\"service_tier\":\"auto\",\"store\":false,\"temperature\":0.0,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tool_usage\":{\"image_gen\":{\"input_tokens\":0,\"input_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"output_tokens\":0,\"output_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"total_tokens\":0},\"web_search\":{\"num_requests\":0}},\"tools\":[{\"type\":\"function\",\"description\":\"Read the attached PDF.\",\"name\":\"read_pdf\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"properties\":{},\"additionalProperties\":false},\"strict\":false}],\"top_logprobs\":0,\"top_p\":1.0,\"truncation\":\"disabled\",\"usage\":null,\"user\":null,\"metadata\":{}},\"sequence_number\":0}\n\nevent: response.in_progress\ndata: {\"type\":\"response.in_progress\",\"response\":{\"id\":\"resp_0cd823aeeb11ef0e016a6108c703b88193872d736c48d60e2c\",\"object\":\"response\",\"created_at\":1784744135,\"status\":\"in_progress\",\"background\":false,\"completed_at\":null,\"error\":null,\"frequency_penalty\":0.0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":40,\"max_tool_calls\":null,\"model\":\"gpt-4o-mini-2024-07-18\",\"moderation\":null,\"output\":[],\"parallel_tool_calls\":true,\"presence_penalty\":0.0,\"previous_response_id\":null,\"prompt_cache_key\":null,\"prompt_cache_retention\":\"in_memory\",\"reasoning\":{\"context\":null,\"effort\":null,\"summary\":null},\"safety_identifier\":null,\"service_tier\":\"auto\",\"store\":false,\"temperature\":0.0,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tool_usage\":{\"image_gen\":{\"input_tokens\":0,\"input_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"output_tokens\":0,\"output_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"total_tokens\":0},\"web_search\":{\"num_requests\":0}},\"tools\":[{\"type\":\"function\",\"description\":\"Read the attached PDF.\",\"name\":\"read_pdf\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"properties\":{},\"additionalProperties\":false},\"strict\":false}],\"top_logprobs\":0,\"top_p\":1.0,\"truncation\":\"disabled\",\"usage\":null,\"user\":null,\"metadata\":{}},\"sequence_number\":1}\n\nevent: response.output_item.added\ndata: {\"type\":\"response.output_item.added\",\"item\":{\"id\":\"msg_0cd823aeeb11ef0e016a6108c83f8081938114e89ac5d99c7e\",\"type\":\"message\",\"status\":\"in_progress\",\"content\":[],\"role\":\"assistant\"},\"output_index\":0,\"sequence_number\":2}\n\nevent: response.content_part.added\ndata: {\"type\":\"response.content_part.added\",\"content_index\":0,\"item_id\":\"msg_0cd823aeeb11ef0e016a6108c83f8081938114e89ac5d99c7e\",\"output_index\":0,\"part\":{\"type\":\"output_text\",\"annotations\":[],\"logprobs\":[],\"text\":\"\"},\"sequence_number\":3}\n\nevent: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"The\",\"item_id\":\"msg_0cd823aeeb11ef0e016a6108c83f8081938114e89ac5d99c7e\",\"logprobs\":[],\"obfuscation\":\"ztkMbqje3EBwS\",\"output_index\":0,\"sequence_number\":4}\n\nevent: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" verification\",\"item_id\":\"msg_0cd823aeeb11ef0e016a6108c83f8081938114e89ac5d99c7e\",\"logprobs\":[],\"obfuscation\":\"mEL\",\"output_index\":0,\"sequence_number\":5}\n\nevent: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" code\",\"item_id\":\"msg_0cd823aeeb11ef0e016a6108c83f8081938114e89ac5d99c7e\",\"logprobs\":[],\"obfuscation\":\"l7H1kZM2CjW\",\"output_index\":0,\"sequence_number\":6}\n\nevent: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" from\",\"item_id\":\"msg_0cd823aeeb11ef0e016a6108c83f8081938114e89ac5d99c7e\",\"logprobs\":[],\"obfuscation\":\"zoQfdh40bG4\",\"output_index\":0,\"sequence_number\":7}\n\nevent: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" the\",\"item_id\":\"msg_0cd823aeeb11ef0e016a6108c83f8081938114e89ac5d99c7e\",\"logprobs\":[],\"obfuscation\":\"5fXB3VBJUbAr\",\"output_index\":0,\"sequence_number\":8}\n\nevent: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" PDF\",\"item_id\":\"msg_0cd823aeeb11ef0e016a6108c83f8081938114e89ac5d99c7e\",\"logprobs\":[],\"obfuscation\":\"BpJQrXS6gY3c\",\"output_index\":0,\"sequence_number\":9}\n\nevent: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" is\",\"item_id\":\"msg_0cd823aeeb11ef0e016a6108c83f8081938114e89ac5d99c7e\",\"logprobs\":[],\"obfuscation\":\"xmGyHaXtMBy6i\",\"output_index\":0,\"sequence_number\":10}\n\nevent: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\":\",\"item_id\":\"msg_0cd823aeeb11ef0e016a6108c83f8081938114e89ac5d99c7e\",\"logprobs\":[],\"obfuscation\":\"oJeWezxDkzq3MRx\",\"output_index\":0,\"sequence_number\":11}\n\nevent: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" OR\",\"item_id\":\"msg_0cd823aeeb11ef0e016a6108c83f8081938114e89ac5d99c7e\",\"logprobs\":[],\"obfuscation\":\"LpS12bymKrO3H\",\"output_index\":0,\"sequence_number\":12}\n\nevent: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"CH\",\"item_id\":\"msg_0cd823aeeb11ef0e016a6108c83f8081938114e89ac5d99c7e\",\"logprobs\":[],\"obfuscation\":\"EvCpePUNmEXVil\",\"output_index\":0,\"sequence_number\":13}\n\nevent: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"ID\",\"item_id\":\"msg_0cd823aeeb11ef0e016a6108c83f8081938114e89ac5d99c7e\",\"logprobs\":[],\"obfuscation\":\"IS5u0zHGO1f3DZ\",\"output_index\":0,\"sequence_number\":14}\n\nevent: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"-\",\"item_id\":\"msg_0cd823aeeb11ef0e016a6108c83f8081938114e89ac5d99c7e\",\"logprobs\":[],\"obfuscation\":\"sQxTFLJOfQQxxil\",\"output_index\":0,\"sequence_number\":15}\n\nevent: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"739\",\"item_id\":\"msg_0cd823aeeb11ef0e016a6108c83f8081938114e89ac5d99c7e\",\"logprobs\":[],\"obfuscation\":\"iIJRJFEo0kyEU\",\"output_index\":0,\"sequence_number\":16}\n\nevent: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"1\",\"item_id\":\"msg_0cd823aeeb11ef0e016a6108c83f8081938114e89ac5d99c7e\",\"logprobs\":[],\"obfuscation\":\"lwMVw41zdSRsVxm\",\"output_index\":0,\"sequence_number\":17}\n\nevent: response.output_text.done\ndata: {\"type\":\"response.output_text.done\",\"content_index\":0,\"item_id\":\"msg_0cd823aeeb11ef0e016a6108c83f8081938114e89ac5d99c7e\",\"logprobs\":[],\"output_index\":0,\"sequence_number\":18,\"text\":\"The verification code from the PDF is: ORCHID-7391\"}\n\nevent: response.content_part.done\ndata: {\"type\":\"response.content_part.done\",\"content_index\":0,\"item_id\":\"msg_0cd823aeeb11ef0e016a6108c83f8081938114e89ac5d99c7e\",\"output_index\":0,\"part\":{\"type\":\"output_text\",\"annotations\":[],\"logprobs\":[],\"text\":\"The verification code from the PDF is: ORCHID-7391\"},\"sequence_number\":19}\n\nevent: response.output_item.done\ndata: {\"type\":\"response.output_item.done\",\"item\":{\"id\":\"msg_0cd823aeeb11ef0e016a6108c83f8081938114e89ac5d99c7e\",\"type\":\"message\",\"status\":\"completed\",\"content\":[{\"type\":\"output_text\",\"annotations\":[],\"logprobs\":[],\"text\":\"The verification code from the PDF is: ORCHID-7391\"}],\"role\":\"assistant\"},\"output_index\":0,\"sequence_number\":20}\n\nevent: response.completed\ndata: {\"type\":\"response.completed\",\"response\":{\"id\":\"resp_0cd823aeeb11ef0e016a6108c703b88193872d736c48d60e2c\",\"object\":\"response\",\"created_at\":1784744135,\"status\":\"completed\",\"background\":false,\"completed_at\":1784744136,\"error\":null,\"frequency_penalty\":0.0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":40,\"max_tool_calls\":null,\"model\":\"gpt-4o-mini-2024-07-18\",\"moderation\":null,\"output\":[{\"id\":\"msg_0cd823aeeb11ef0e016a6108c83f8081938114e89ac5d99c7e\",\"type\":\"message\",\"status\":\"completed\",\"content\":[{\"type\":\"output_text\",\"annotations\":[],\"logprobs\":[],\"text\":\"The verification code from the PDF is: ORCHID-7391\"}],\"role\":\"assistant\"}],\"parallel_tool_calls\":true,\"presence_penalty\":0.0,\"previous_response_id\":null,\"prompt_cache_key\":null,\"prompt_cache_retention\":\"in_memory\",\"reasoning\":{\"context\":null,\"effort\":null,\"summary\":null},\"safety_identifier\":null,\"service_tier\":\"default\",\"store\":false,\"temperature\":0.0,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tool_usage\":{\"image_gen\":{\"input_tokens\":0,\"input_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"output_tokens\":0,\"output_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"total_tokens\":0},\"web_search\":{\"num_requests\":0}},\"tools\":[{\"type\":\"function\",\"description\":\"Read the attached PDF.\",\"name\":\"read_pdf\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"properties\":{},\"additionalProperties\":false},\"strict\":false}],\"top_logprobs\":0,\"top_p\":1.0,\"truncation\":\"disabled\",\"usage\":{\"input_tokens\":107,\"input_tokens_details\":{\"cache_write_tokens\":0,\"cached_tokens\":0},\"output_tokens\":16,\"output_tokens_details\":{\"reasoning_tokens\":0},\"total_tokens\":123},\"user\":null,\"metadata\":{}},\"sequence_number\":21}\n\n" + } + } + ] +} diff --git a/packages/ai/test/fixtures/recordings/pdf/openai-user-input.json b/packages/ai/test/fixtures/recordings/pdf/openai-user-input.json new file mode 100644 index 0000000000..ab2b4a9dd7 --- /dev/null +++ b/packages/ai/test/fixtures/recordings/pdf/openai-user-input.json @@ -0,0 +1,34 @@ +{ + "version": 1, + "metadata": { + "tags": [ + "prefix:pdf", + "pdf", + "provider:openai", + "protocol:openai-responses", + "user-input" + ], + "name": "pdf/openai-user-input", + "recordedAt": "2026-07-22T18:15:34.867Z" + }, + "interactions": [ + { + "transport": "http", + "request": { + "method": "POST", + "url": "https://api.openai.com/v1/responses", + "headers": { + "content-type": "application/json" + }, + "body": "{\"model\":\"gpt-4o-mini\",\"input\":[{\"role\":\"user\",\"content\":[{\"type\":\"input_file\",\"filename\":\"verification.pdf\",\"file_data\":\"data:application/pdf;base64,JVBERi0xLjQKMSAwIG9iago8PCAvVHlwZSAvQ2F0YWxvZyAvUGFnZXMgMiAwIFIgPj4KZW5kb2JqCjIgMCBvYmoKPDwgL1R5cGUgL1BhZ2VzIC9LaWRzIFszIDAgUl0gL0NvdW50IDEgPj4KZW5kb2JqCjMgMCBvYmoKPDwgL1R5cGUgL1BhZ2UgL1BhcmVudCAyIDAgUiAvTWVkaWFCb3ggWzAgMCA2MTIgNzkyXSAvUmVzb3VyY2VzIDw8IC9Gb250IDw8IC9GMSA1IDAgUiA+PiA+PiAvQ29udGVudHMgNCAwIFIgPj4KZW5kb2JqCjQgMCBvYmoKPDwgL0xlbmd0aCA3NSA+PgpzdHJlYW0KQlQKL0YxIDE4IFRmCjcyIDcyMCBUZAooUERGIGNhc3NldHRlIHZlcmlmaWNhdGlvbiBjb2RlOiBPUkNISUQtNzM5MSkgVGoKRVQKZW5kc3RyZWFtCmVuZG9iago1IDAgb2JqCjw8IC9UeXBlIC9Gb250IC9TdWJ0eXBlIC9UeXBlMSAvQmFzZUZvbnQgL0hlbHZldGljYSA+PgplbmRvYmoKeHJlZgowIDYKMDAwMDAwMDAwMCA2NTUzNSBmIAowMDAwMDAwMDA5IDAwMDAwIG4gCjAwMDAwMDAwNTggMDAwMDAgbiAKMDAwMDAwMDExNSAwMDAwMCBuIAowMDAwMDAwMjQxIDAwMDAwIG4gCjAwMDAwMDAzNjUgMDAwMDAgbiAKdHJhaWxlcgo8PCAvU2l6ZSA2IC9Sb290IDEgMCBSID4+CnN0YXJ0eHJlZgo0MzUKJSVFT0YK\"},{\"type\":\"input_text\",\"text\":\"Return only the verification code from the PDF.\"}]}],\"store\":false,\"max_output_tokens\":40,\"temperature\":0,\"stream\":true}" + }, + "response": { + "status": 200, + "headers": { + "content-type": "text/event-stream; charset=utf-8" + }, + "body": "event: response.created\ndata: {\"type\":\"response.created\",\"response\":{\"id\":\"resp_098ccdfefefe7eee016a6108c55ad48194a250d21618f41130\",\"object\":\"response\",\"created_at\":1784744134,\"status\":\"in_progress\",\"background\":false,\"completed_at\":null,\"error\":null,\"frequency_penalty\":0.0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":40,\"max_tool_calls\":null,\"model\":\"gpt-4o-mini-2024-07-18\",\"moderation\":null,\"output\":[],\"parallel_tool_calls\":true,\"presence_penalty\":0.0,\"previous_response_id\":null,\"prompt_cache_key\":null,\"prompt_cache_retention\":\"in_memory\",\"reasoning\":{\"context\":null,\"effort\":null,\"summary\":null},\"safety_identifier\":null,\"service_tier\":\"auto\",\"store\":false,\"temperature\":0.0,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tool_usage\":{\"image_gen\":{\"input_tokens\":0,\"input_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"output_tokens\":0,\"output_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"total_tokens\":0},\"web_search\":{\"num_requests\":0}},\"tools\":[],\"top_logprobs\":0,\"top_p\":1.0,\"truncation\":\"disabled\",\"usage\":null,\"user\":null,\"metadata\":{}},\"sequence_number\":0}\n\nevent: response.in_progress\ndata: {\"type\":\"response.in_progress\",\"response\":{\"id\":\"resp_098ccdfefefe7eee016a6108c55ad48194a250d21618f41130\",\"object\":\"response\",\"created_at\":1784744134,\"status\":\"in_progress\",\"background\":false,\"completed_at\":null,\"error\":null,\"frequency_penalty\":0.0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":40,\"max_tool_calls\":null,\"model\":\"gpt-4o-mini-2024-07-18\",\"moderation\":null,\"output\":[],\"parallel_tool_calls\":true,\"presence_penalty\":0.0,\"previous_response_id\":null,\"prompt_cache_key\":null,\"prompt_cache_retention\":\"in_memory\",\"reasoning\":{\"context\":null,\"effort\":null,\"summary\":null},\"safety_identifier\":null,\"service_tier\":\"auto\",\"store\":false,\"temperature\":0.0,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tool_usage\":{\"image_gen\":{\"input_tokens\":0,\"input_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"output_tokens\":0,\"output_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"total_tokens\":0},\"web_search\":{\"num_requests\":0}},\"tools\":[],\"top_logprobs\":0,\"top_p\":1.0,\"truncation\":\"disabled\",\"usage\":null,\"user\":null,\"metadata\":{}},\"sequence_number\":1}\n\nevent: response.output_item.added\ndata: {\"type\":\"response.output_item.added\",\"item\":{\"id\":\"msg_098ccdfefefe7eee016a6108c6b6e48194bdd91a4de2cb7bbe\",\"type\":\"message\",\"status\":\"in_progress\",\"content\":[],\"role\":\"assistant\"},\"output_index\":0,\"sequence_number\":2}\n\nevent: response.content_part.added\ndata: {\"type\":\"response.content_part.added\",\"content_index\":0,\"item_id\":\"msg_098ccdfefefe7eee016a6108c6b6e48194bdd91a4de2cb7bbe\",\"output_index\":0,\"part\":{\"type\":\"output_text\",\"annotations\":[],\"logprobs\":[],\"text\":\"\"},\"sequence_number\":3}\n\nevent: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"OR\",\"item_id\":\"msg_098ccdfefefe7eee016a6108c6b6e48194bdd91a4de2cb7bbe\",\"logprobs\":[],\"obfuscation\":\"OT7X1BjsfOw0Y7\",\"output_index\":0,\"sequence_number\":4}\n\nevent: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"CH\",\"item_id\":\"msg_098ccdfefefe7eee016a6108c6b6e48194bdd91a4de2cb7bbe\",\"logprobs\":[],\"obfuscation\":\"XpcQS0WDba72lQ\",\"output_index\":0,\"sequence_number\":5}\n\nevent: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"ID\",\"item_id\":\"msg_098ccdfefefe7eee016a6108c6b6e48194bdd91a4de2cb7bbe\",\"logprobs\":[],\"obfuscation\":\"IfrmlWWbGSpVPM\",\"output_index\":0,\"sequence_number\":6}\n\nevent: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"-\",\"item_id\":\"msg_098ccdfefefe7eee016a6108c6b6e48194bdd91a4de2cb7bbe\",\"logprobs\":[],\"obfuscation\":\"Jpf9w1TEeBMJR6X\",\"output_index\":0,\"sequence_number\":7}\n\nevent: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"739\",\"item_id\":\"msg_098ccdfefefe7eee016a6108c6b6e48194bdd91a4de2cb7bbe\",\"logprobs\":[],\"obfuscation\":\"J1beY5zFoug8n\",\"output_index\":0,\"sequence_number\":8}\n\nevent: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"1\",\"item_id\":\"msg_098ccdfefefe7eee016a6108c6b6e48194bdd91a4de2cb7bbe\",\"logprobs\":[],\"obfuscation\":\"kG1Isnrwy5QR4QB\",\"output_index\":0,\"sequence_number\":9}\n\nevent: response.output_text.done\ndata: {\"type\":\"response.output_text.done\",\"content_index\":0,\"item_id\":\"msg_098ccdfefefe7eee016a6108c6b6e48194bdd91a4de2cb7bbe\",\"logprobs\":[],\"output_index\":0,\"sequence_number\":10,\"text\":\"ORCHID-7391\"}\n\nevent: response.content_part.done\ndata: {\"type\":\"response.content_part.done\",\"content_index\":0,\"item_id\":\"msg_098ccdfefefe7eee016a6108c6b6e48194bdd91a4de2cb7bbe\",\"output_index\":0,\"part\":{\"type\":\"output_text\",\"annotations\":[],\"logprobs\":[],\"text\":\"ORCHID-7391\"},\"sequence_number\":11}\n\nevent: response.output_item.done\ndata: {\"type\":\"response.output_item.done\",\"item\":{\"id\":\"msg_098ccdfefefe7eee016a6108c6b6e48194bdd91a4de2cb7bbe\",\"type\":\"message\",\"status\":\"completed\",\"content\":[{\"type\":\"output_text\",\"annotations\":[],\"logprobs\":[],\"text\":\"ORCHID-7391\"}],\"role\":\"assistant\"},\"output_index\":0,\"sequence_number\":12}\n\nevent: response.completed\ndata: {\"type\":\"response.completed\",\"response\":{\"id\":\"resp_098ccdfefefe7eee016a6108c55ad48194a250d21618f41130\",\"object\":\"response\",\"created_at\":1784744134,\"status\":\"completed\",\"background\":false,\"completed_at\":1784744134,\"error\":null,\"frequency_penalty\":0.0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":40,\"max_tool_calls\":null,\"model\":\"gpt-4o-mini-2024-07-18\",\"moderation\":null,\"output\":[{\"id\":\"msg_098ccdfefefe7eee016a6108c6b6e48194bdd91a4de2cb7bbe\",\"type\":\"message\",\"status\":\"completed\",\"content\":[{\"type\":\"output_text\",\"annotations\":[],\"logprobs\":[],\"text\":\"ORCHID-7391\"}],\"role\":\"assistant\"}],\"parallel_tool_calls\":true,\"presence_penalty\":0.0,\"previous_response_id\":null,\"prompt_cache_key\":null,\"prompt_cache_retention\":\"in_memory\",\"reasoning\":{\"context\":null,\"effort\":null,\"summary\":null},\"safety_identifier\":null,\"service_tier\":\"default\",\"store\":false,\"temperature\":0.0,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tool_usage\":{\"image_gen\":{\"input_tokens\":0,\"input_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"output_tokens\":0,\"output_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"total_tokens\":0},\"web_search\":{\"num_requests\":0}},\"tools\":[],\"top_logprobs\":0,\"top_p\":1.0,\"truncation\":\"disabled\",\"usage\":{\"input_tokens\":44,\"input_tokens_details\":{\"cache_write_tokens\":0,\"cached_tokens\":0},\"output_tokens\":7,\"output_tokens_details\":{\"reasoning_tokens\":0},\"total_tokens\":51},\"user\":null,\"metadata\":{}},\"sequence_number\":13}\n\n" + } + } + ] +} diff --git a/packages/ai/test/fixtures/recordings/pdf/xai-tool-result.json b/packages/ai/test/fixtures/recordings/pdf/xai-tool-result.json new file mode 100644 index 0000000000..71833b93a5 --- /dev/null +++ b/packages/ai/test/fixtures/recordings/pdf/xai-tool-result.json @@ -0,0 +1,35 @@ +{ + "version": 1, + "metadata": { + "tags": [ + "prefix:pdf", + "pdf", + "provider:xai", + "protocol:openai-responses", + "tool", + "tool-result" + ], + "name": "pdf/xai-tool-result", + "recordedAt": "2026-07-22T18:15:43.608Z" + }, + "interactions": [ + { + "transport": "http", + "request": { + "method": "POST", + "url": "https://api.x.ai/v1/responses", + "headers": { + "content-type": "application/json" + }, + "body": "{\"model\":\"grok-4.5\",\"input\":[{\"role\":\"system\",\"content\":\"Read the PDF returned by the tool and follow the user's response format exactly.\"},{\"role\":\"user\",\"content\":[{\"type\":\"input_text\",\"text\":\"Return only the verification code from the PDF.\"}]},{\"type\":\"function_call\",\"call_id\":\"call_pdf_1\",\"name\":\"read_pdf\",\"arguments\":\"{}\"},{\"type\":\"function_call_output\",\"call_id\":\"call_pdf_1\",\"output\":[{\"type\":\"input_text\",\"text\":\"PDF read successfully\"},{\"type\":\"input_file\",\"filename\":\"verification.pdf\",\"file_data\":\"JVBERi0xLjQKMSAwIG9iago8PCAvVHlwZSAvQ2F0YWxvZyAvUGFnZXMgMiAwIFIgPj4KZW5kb2JqCjIgMCBvYmoKPDwgL1R5cGUgL1BhZ2VzIC9LaWRzIFszIDAgUl0gL0NvdW50IDEgPj4KZW5kb2JqCjMgMCBvYmoKPDwgL1R5cGUgL1BhZ2UgL1BhcmVudCAyIDAgUiAvTWVkaWFCb3ggWzAgMCA2MTIgNzkyXSAvUmVzb3VyY2VzIDw8IC9Gb250IDw8IC9GMSA1IDAgUiA+PiA+PiAvQ29udGVudHMgNCAwIFIgPj4KZW5kb2JqCjQgMCBvYmoKPDwgL0xlbmd0aCA3NSA+PgpzdHJlYW0KQlQKL0YxIDE4IFRmCjcyIDcyMCBUZAooUERGIGNhc3NldHRlIHZlcmlmaWNhdGlvbiBjb2RlOiBPUkNISUQtNzM5MSkgVGoKRVQKZW5kc3RyZWFtCmVuZG9iago1IDAgb2JqCjw8IC9UeXBlIC9Gb250IC9TdWJ0eXBlIC9UeXBlMSAvQmFzZUZvbnQgL0hlbHZldGljYSA+PgplbmRvYmoKeHJlZgowIDYKMDAwMDAwMDAwMCA2NTUzNSBmIAowMDAwMDAwMDA5IDAwMDAwIG4gCjAwMDAwMDAwNTggMDAwMDAgbiAKMDAwMDAwMDExNSAwMDAwMCBuIAowMDAwMDAwMjQxIDAwMDAwIG4gCjAwMDAwMDAzNjUgMDAwMDAgbiAKdHJhaWxlcgo8PCAvU2l6ZSA2IC9Sb290IDEgMCBSID4+CnN0YXJ0eHJlZgo0MzUKJSVFT0YK\",\"mime_type\":\"application/pdf\"}]}],\"tools\":[{\"type\":\"function\",\"name\":\"read_pdf\",\"description\":\"Read the attached PDF.\",\"parameters\":{\"type\":\"object\",\"properties\":{},\"additionalProperties\":false},\"strict\":false}],\"store\":false,\"max_output_tokens\":40,\"temperature\":0,\"stream\":true}" + }, + "response": { + "status": 200, + "headers": { + "content-type": "text/event-stream" + }, + "body": "event: response.created\ndata: {\"sequence_number\":0,\"type\":\"response.created\",\"response\":{\"created_at\":0,\"completed_at\":null,\"id\":\"dab13d84-6060-9eb7-b898-cbe7a3475ad7\",\"max_output_tokens\":40,\"model\":\"grok-4.5\",\"object\":\"response\",\"output\":[],\"parallel_tool_calls\":true,\"previous_response_id\":null,\"reasoning\":{\"effort\":null,\"summary\":null},\"temperature\":0.0,\"text\":{\"format\":{\"type\":\"text\"}},\"tool_choice\":\"auto\",\"tools\":[{\"type\":\"function\",\"description\":\"Read the attached PDF.\",\"name\":\"read_pdf\",\"parameters\":{\"type\":\"object\",\"properties\":{},\"additionalProperties\":false},\"strict\":false}],\"top_p\":0.949999988079071,\"usage\":null,\"user\":null,\"incomplete_details\":null,\"status\":\"in_progress\",\"store\":false,\"metadata\":{\"system_fingerprint\":\"fp_a39489019fa99b6e\"},\"background\":false,\"service_tier\":\"default\",\"truncation\":\"disabled\",\"top_logprobs\":0,\"presence_penalty\":0.0,\"frequency_penalty\":0.0,\"prompt_cache_key\":null,\"max_tool_calls\":null,\"safety_identifier\":null,\"error\":null,\"instructions\":null}}\n\nevent: response.in_progress\ndata: {\"sequence_number\":1,\"type\":\"response.in_progress\",\"response\":{\"created_at\":0,\"completed_at\":null,\"id\":\"dab13d84-6060-9eb7-b898-cbe7a3475ad7\",\"max_output_tokens\":40,\"model\":\"grok-4.5\",\"object\":\"response\",\"output\":[],\"parallel_tool_calls\":true,\"previous_response_id\":null,\"reasoning\":{\"effort\":null,\"summary\":null},\"temperature\":0.0,\"text\":{\"format\":{\"type\":\"text\"}},\"tool_choice\":\"auto\",\"tools\":[{\"type\":\"function\",\"description\":\"Read the attached PDF.\",\"name\":\"read_pdf\",\"parameters\":{\"type\":\"object\",\"properties\":{},\"additionalProperties\":false},\"strict\":false}],\"top_p\":0.949999988079071,\"usage\":null,\"user\":null,\"incomplete_details\":null,\"status\":\"in_progress\",\"store\":false,\"metadata\":{\"system_fingerprint\":\"fp_a39489019fa99b6e\"},\"background\":false,\"service_tier\":\"default\",\"truncation\":\"disabled\",\"top_logprobs\":0,\"presence_penalty\":0.0,\"frequency_penalty\":0.0,\"prompt_cache_key\":null,\"max_tool_calls\":null,\"safety_identifier\":null,\"error\":null,\"instructions\":null}}\n\nevent: response.output_item.added\ndata: {\"sequence_number\":2,\"type\":\"response.output_item.added\",\"item\":{\"content\":[],\"id\":\"msg_dab13d84-6060-9eb7-b898-cbe7a3475ad7\",\"role\":\"assistant\",\"type\":\"message\",\"status\":\"in_progress\"},\"output_index\":0}\n\nevent: response.content_part.added\ndata: {\"sequence_number\":3,\"type\":\"response.content_part.added\",\"content_index\":0,\"item_id\":\"msg_dab13d84-6060-9eb7-b898-cbe7a3475ad7\",\"output_index\":0,\"part\":{\"type\":\"output_text\",\"text\":\"\",\"logprobs\":[],\"annotations\":[]}}\n\nevent: response.output_text.delta\ndata: {\"sequence_number\":4,\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"OR\",\"item_id\":\"msg_dab13d84-6060-9eb7-b898-cbe7a3475ad7\",\"output_index\":0,\"logprobs\":[]}\n\nevent: response.output_text.delta\ndata: {\"sequence_number\":5,\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"CH\",\"item_id\":\"msg_dab13d84-6060-9eb7-b898-cbe7a3475ad7\",\"output_index\":0,\"logprobs\":[]}\n\nevent: response.output_text.delta\ndata: {\"sequence_number\":6,\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"ID\",\"item_id\":\"msg_dab13d84-6060-9eb7-b898-cbe7a3475ad7\",\"output_index\":0,\"logprobs\":[]}\n\nevent: response.output_text.delta\ndata: {\"sequence_number\":7,\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"-\",\"item_id\":\"msg_dab13d84-6060-9eb7-b898-cbe7a3475ad7\",\"output_index\":0,\"logprobs\":[]}\n\nevent: response.output_text.delta\ndata: {\"sequence_number\":8,\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"739\",\"item_id\":\"msg_dab13d84-6060-9eb7-b898-cbe7a3475ad7\",\"output_index\":0,\"logprobs\":[]}\n\nevent: response.output_text.delta\ndata: {\"sequence_number\":9,\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"1\",\"item_id\":\"msg_dab13d84-6060-9eb7-b898-cbe7a3475ad7\",\"output_index\":0,\"logprobs\":[]}\n\nevent: response.output_text.done\ndata: {\"sequence_number\":10,\"type\":\"response.output_text.done\",\"content_index\":0,\"item_id\":\"msg_dab13d84-6060-9eb7-b898-cbe7a3475ad7\",\"output_index\":0,\"text\":\"ORCHID-7391\",\"logprobs\":[]}\n\nevent: response.content_part.done\ndata: {\"sequence_number\":11,\"type\":\"response.content_part.done\",\"content_index\":0,\"item_id\":\"msg_dab13d84-6060-9eb7-b898-cbe7a3475ad7\",\"output_index\":0,\"part\":{\"type\":\"output_text\",\"text\":\"ORCHID-7391\",\"logprobs\":[],\"annotations\":[]}}\n\nevent: response.output_item.done\ndata: {\"sequence_number\":12,\"type\":\"response.output_item.done\",\"item\":{\"content\":[{\"type\":\"output_text\",\"text\":\"ORCHID-7391\",\"logprobs\":[],\"annotations\":[]}],\"id\":\"msg_dab13d84-6060-9eb7-b898-cbe7a3475ad7\",\"role\":\"assistant\",\"type\":\"message\",\"status\":\"completed\"},\"output_index\":0}\n\nevent: response.completed\ndata: {\"sequence_number\":13,\"type\":\"response.completed\",\"response\":{\"created_at\":0,\"completed_at\":1784744143,\"id\":\"dab13d84-6060-9eb7-b898-cbe7a3475ad7\",\"max_output_tokens\":40,\"model\":\"grok-4.5\",\"object\":\"response\",\"output\":[{\"content\":[{\"type\":\"output_text\",\"text\":\"ORCHID-7391\",\"logprobs\":[],\"annotations\":[]}],\"id\":\"msg_dab13d84-6060-9eb7-b898-cbe7a3475ad7\",\"role\":\"assistant\",\"type\":\"message\",\"status\":\"completed\"}],\"parallel_tool_calls\":true,\"previous_response_id\":null,\"reasoning\":{\"effort\":null,\"summary\":null},\"temperature\":0.0,\"text\":{\"format\":{\"type\":\"text\"}},\"tool_choice\":\"auto\",\"tools\":[{\"type\":\"function\",\"description\":\"Read the attached PDF.\",\"name\":\"read_pdf\",\"parameters\":{\"type\":\"object\",\"properties\":{},\"additionalProperties\":false},\"strict\":false}],\"top_p\":0.949999988079071,\"usage\":{\"input_tokens\":1592,\"input_tokens_details\":{\"cached_tokens\":128},\"output_tokens\":10,\"output_tokens_details\":{\"reasoning_tokens\":0},\"total_tokens\":1602,\"num_sources_used\":0,\"num_server_side_tools_used\":0,\"cost_in_usd_ticks\":30264000,\"context_details\":{\"input_tokens\":1592,\"output_tokens\":11}},\"user\":null,\"incomplete_details\":null,\"status\":\"completed\",\"store\":false,\"metadata\":{\"system_fingerprint\":\"fp_a39489019fa99b6e\"},\"background\":false,\"service_tier\":\"default\",\"truncation\":\"disabled\",\"top_logprobs\":0,\"presence_penalty\":0.0,\"frequency_penalty\":0.0,\"prompt_cache_key\":null,\"max_tool_calls\":null,\"safety_identifier\":null,\"error\":null,\"instructions\":null}}\n\n" + } + } + ] +} diff --git a/packages/ai/test/fixtures/recordings/pdf/xai-user-input.json b/packages/ai/test/fixtures/recordings/pdf/xai-user-input.json new file mode 100644 index 0000000000..c3831f6e06 --- /dev/null +++ b/packages/ai/test/fixtures/recordings/pdf/xai-user-input.json @@ -0,0 +1,34 @@ +{ + "version": 1, + "metadata": { + "tags": [ + "prefix:pdf", + "pdf", + "provider:xai", + "protocol:openai-responses", + "user-input" + ], + "name": "pdf/xai-user-input", + "recordedAt": "2026-07-22T18:15:42.429Z" + }, + "interactions": [ + { + "transport": "http", + "request": { + "method": "POST", + "url": "https://api.x.ai/v1/responses", + "headers": { + "content-type": "application/json" + }, + "body": "{\"model\":\"grok-4.5\",\"input\":[{\"role\":\"user\",\"content\":[{\"type\":\"input_file\",\"filename\":\"verification.pdf\",\"file_data\":\"JVBERi0xLjQKMSAwIG9iago8PCAvVHlwZSAvQ2F0YWxvZyAvUGFnZXMgMiAwIFIgPj4KZW5kb2JqCjIgMCBvYmoKPDwgL1R5cGUgL1BhZ2VzIC9LaWRzIFszIDAgUl0gL0NvdW50IDEgPj4KZW5kb2JqCjMgMCBvYmoKPDwgL1R5cGUgL1BhZ2UgL1BhcmVudCAyIDAgUiAvTWVkaWFCb3ggWzAgMCA2MTIgNzkyXSAvUmVzb3VyY2VzIDw8IC9Gb250IDw8IC9GMSA1IDAgUiA+PiA+PiAvQ29udGVudHMgNCAwIFIgPj4KZW5kb2JqCjQgMCBvYmoKPDwgL0xlbmd0aCA3NSA+PgpzdHJlYW0KQlQKL0YxIDE4IFRmCjcyIDcyMCBUZAooUERGIGNhc3NldHRlIHZlcmlmaWNhdGlvbiBjb2RlOiBPUkNISUQtNzM5MSkgVGoKRVQKZW5kc3RyZWFtCmVuZG9iago1IDAgb2JqCjw8IC9UeXBlIC9Gb250IC9TdWJ0eXBlIC9UeXBlMSAvQmFzZUZvbnQgL0hlbHZldGljYSA+PgplbmRvYmoKeHJlZgowIDYKMDAwMDAwMDAwMCA2NTUzNSBmIAowMDAwMDAwMDA5IDAwMDAwIG4gCjAwMDAwMDAwNTggMDAwMDAgbiAKMDAwMDAwMDExNSAwMDAwMCBuIAowMDAwMDAwMjQxIDAwMDAwIG4gCjAwMDAwMDAzNjUgMDAwMDAgbiAKdHJhaWxlcgo8PCAvU2l6ZSA2IC9Sb290IDEgMCBSID4+CnN0YXJ0eHJlZgo0MzUKJSVFT0YK\",\"mime_type\":\"application/pdf\"},{\"type\":\"input_text\",\"text\":\"Return only the verification code from the PDF.\"}]}],\"store\":false,\"max_output_tokens\":40,\"temperature\":0,\"stream\":true}" + }, + "response": { + "status": 200, + "headers": { + "content-type": "text/event-stream" + }, + "body": "event: response.created\ndata: {\"sequence_number\":0,\"type\":\"response.created\",\"response\":{\"created_at\":0,\"completed_at\":null,\"id\":\"c69d4e2b-243a-909a-84b6-ee953321ef6c\",\"max_output_tokens\":40,\"model\":\"grok-4.5\",\"object\":\"response\",\"output\":[],\"parallel_tool_calls\":true,\"previous_response_id\":null,\"reasoning\":{\"effort\":null,\"summary\":null},\"temperature\":0.0,\"text\":{\"format\":{\"type\":\"text\"}},\"tool_choice\":\"auto\",\"tools\":[],\"top_p\":0.949999988079071,\"usage\":null,\"user\":null,\"incomplete_details\":null,\"status\":\"in_progress\",\"store\":false,\"metadata\":{\"system_fingerprint\":\"fp_a39489019fa99b6e\"},\"background\":false,\"service_tier\":\"default\",\"truncation\":\"disabled\",\"top_logprobs\":0,\"presence_penalty\":0.0,\"frequency_penalty\":0.0,\"prompt_cache_key\":null,\"max_tool_calls\":null,\"safety_identifier\":null,\"error\":null,\"instructions\":null}}\n\nevent: response.in_progress\ndata: {\"sequence_number\":1,\"type\":\"response.in_progress\",\"response\":{\"created_at\":0,\"completed_at\":null,\"id\":\"c69d4e2b-243a-909a-84b6-ee953321ef6c\",\"max_output_tokens\":40,\"model\":\"grok-4.5\",\"object\":\"response\",\"output\":[],\"parallel_tool_calls\":true,\"previous_response_id\":null,\"reasoning\":{\"effort\":null,\"summary\":null},\"temperature\":0.0,\"text\":{\"format\":{\"type\":\"text\"}},\"tool_choice\":\"auto\",\"tools\":[],\"top_p\":0.949999988079071,\"usage\":null,\"user\":null,\"incomplete_details\":null,\"status\":\"in_progress\",\"store\":false,\"metadata\":{\"system_fingerprint\":\"fp_a39489019fa99b6e\"},\"background\":false,\"service_tier\":\"default\",\"truncation\":\"disabled\",\"top_logprobs\":0,\"presence_penalty\":0.0,\"frequency_penalty\":0.0,\"prompt_cache_key\":null,\"max_tool_calls\":null,\"safety_identifier\":null,\"error\":null,\"instructions\":null}}\n\nevent: response.output_item.added\ndata: {\"sequence_number\":2,\"type\":\"response.output_item.added\",\"item\":{\"id\":\"rs_c69d4e2b-243a-909a-84b6-ee953321ef6c\",\"summary\":[],\"type\":\"reasoning\",\"status\":\"in_progress\"},\"output_index\":0}\n\nevent: response.reasoning_summary_part.added\ndata: {\"sequence_number\":3,\"type\":\"response.reasoning_summary_part.added\",\"item_id\":\"rs_c69d4e2b-243a-909a-84b6-ee953321ef6c\",\"output_index\":0,\"part\":{\"text\":\"\",\"type\":\"summary_text\"},\"summary_index\":0}\n\nevent: response.reasoning_summary_text.delta\ndata: {\"sequence_number\":4,\"type\":\"response.reasoning_summary_text.delta\",\"delta\":\"The\",\"item_id\":\"rs_c69d4e2b-243a-909a-84b6-ee953321ef6c\",\"output_index\":0,\"summary_index\":0}\n\nevent: response.reasoning_summary_text.delta\ndata: {\"sequence_number\":5,\"type\":\"response.reasoning_summary_text.delta\",\"delta\":\" user\",\"item_id\":\"rs_c69d4e2b-243a-909a-84b6-ee953321ef6c\",\"output_index\":0,\"summary_index\":0}\n\nevent: response.reasoning_summary_text.delta\ndata: {\"sequence_number\":6,\"type\":\"response.reasoning_summary_text.delta\",\"delta\":\" wants\",\"item_id\":\"rs_c69d4e2b-243a-909a-84b6-ee953321ef6c\",\"output_index\":0,\"summary_index\":0}\n\nevent: response.reasoning_summary_text.delta\ndata: {\"sequence_number\":7,\"type\":\"response.reasoning_summary_text.delta\",\"delta\":\" me\",\"item_id\":\"rs_c69d4e2b-243a-909a-84b6-ee953321ef6c\",\"output_index\":0,\"summary_index\":0}\n\nevent: response.reasoning_summary_text.delta\ndata: {\"sequence_number\":8,\"type\":\"response.reasoning_summary_text.delta\",\"delta\":\" to\",\"item_id\":\"rs_c69d4e2b-243a-909a-84b6-ee953321ef6c\",\"output_index\":0,\"summary_index\":0}\n\nevent: response.reasoning_summary_text.delta\ndata: {\"sequence_number\":9,\"type\":\"response.reasoning_summary_text.delta\",\"delta\":\" return\",\"item_id\":\"rs_c69d4e2b-243a-909a-84b6-ee953321ef6c\",\"output_index\":0,\"summary_index\":0}\n\nevent: response.reasoning_summary_text.delta\ndata: {\"sequence_number\":10,\"type\":\"response.reasoning_summary_text.delta\",\"delta\":\" only\",\"item_id\":\"rs_c69d4e2b-243a-909a-84b6-ee953321ef6c\",\"output_index\":0,\"summary_index\":0}\n\nevent: response.reasoning_summary_text.delta\ndata: {\"sequence_number\":11,\"type\":\"response.reasoning_summary_text.delta\",\"delta\":\" the\",\"item_id\":\"rs_c69d4e2b-243a-909a-84b6-ee953321ef6c\",\"output_index\":0,\"summary_index\":0}\n\nevent: response.reasoning_summary_text.delta\ndata: {\"sequence_number\":12,\"type\":\"response.reasoning_summary_text.delta\",\"delta\":\" verification\",\"item_id\":\"rs_c69d4e2b-243a-909a-84b6-ee953321ef6c\",\"output_index\":0,\"summary_index\":0}\n\nevent: response.reasoning_summary_text.delta\ndata: {\"sequence_number\":13,\"type\":\"response.reasoning_summary_text.delta\",\"delta\":\" code\",\"item_id\":\"rs_c69d4e2b-243a-909a-84b6-ee953321ef6c\",\"output_index\":0,\"summary_index\":0}\n\nevent: response.reasoning_summary_text.delta\ndata: {\"sequence_number\":14,\"type\":\"response.reasoning_summary_text.delta\",\"delta\":\" from\",\"item_id\":\"rs_c69d4e2b-243a-909a-84b6-ee953321ef6c\",\"output_index\":0,\"summary_index\":0}\n\nevent: response.reasoning_summary_text.delta\ndata: {\"sequence_number\":15,\"type\":\"response.reasoning_summary_text.delta\",\"delta\":\" the\",\"item_id\":\"rs_c69d4e2b-243a-909a-84b6-ee953321ef6c\",\"output_index\":0,\"summary_index\":0}\n\nevent: response.reasoning_summary_text.delta\ndata: {\"sequence_number\":16,\"type\":\"response.reasoning_summary_text.delta\",\"delta\":\" PDF\",\"item_id\":\"rs_c69d4e2b-243a-909a-84b6-ee953321ef6c\",\"output_index\":0,\"summary_index\":0}\n\nevent: response.reasoning_summary_text.delta\ndata: {\"sequence_number\":17,\"type\":\"response.reasoning_summary_text.delta\",\"delta\":\".\\n\",\"item_id\":\"rs_c69d4e2b-243a-909a-84b6-ee953321ef6c\",\"output_index\":0,\"summary_index\":0}\n\nevent: response.reasoning_summary_text.done\ndata: {\"sequence_number\":18,\"type\":\"response.reasoning_summary_text.done\",\"item_id\":\"rs_c69d4e2b-243a-909a-84b6-ee953321ef6c\",\"output_index\":0,\"summary_index\":0,\"text\":\"The user wants me to return only the verification code from the PDF.\\n\"}\n\nevent: response.reasoning_summary_part.done\ndata: {\"sequence_number\":19,\"type\":\"response.reasoning_summary_part.done\",\"item_id\":\"rs_c69d4e2b-243a-909a-84b6-ee953321ef6c\",\"output_index\":0,\"part\":{\"text\":\"The user wants me to return only the verification code from the PDF.\\n\",\"type\":\"summary_text\"},\"summary_index\":0}\n\nevent: response.output_item.done\ndata: {\"sequence_number\":20,\"type\":\"response.output_item.done\",\"item\":{\"id\":\"rs_c69d4e2b-243a-909a-84b6-ee953321ef6c\",\"summary\":[{\"text\":\"The user wants me to return only the verification code from the PDF.\\n\",\"type\":\"summary_text\"}],\"type\":\"reasoning\",\"status\":\"completed\"},\"output_index\":0}\n\nevent: response.output_item.added\ndata: {\"sequence_number\":21,\"type\":\"response.output_item.added\",\"item\":{\"content\":[],\"id\":\"msg_c69d4e2b-243a-909a-84b6-ee953321ef6c\",\"role\":\"assistant\",\"type\":\"message\",\"status\":\"in_progress\"},\"output_index\":1}\n\nevent: response.content_part.added\ndata: {\"sequence_number\":22,\"type\":\"response.content_part.added\",\"content_index\":0,\"item_id\":\"msg_c69d4e2b-243a-909a-84b6-ee953321ef6c\",\"output_index\":1,\"part\":{\"type\":\"output_text\",\"text\":\"\",\"logprobs\":[],\"annotations\":[]}}\n\nevent: response.output_text.delta\ndata: {\"sequence_number\":23,\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"OR\",\"item_id\":\"msg_c69d4e2b-243a-909a-84b6-ee953321ef6c\",\"output_index\":1,\"logprobs\":[]}\n\nevent: response.output_text.delta\ndata: {\"sequence_number\":24,\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"CH\",\"item_id\":\"msg_c69d4e2b-243a-909a-84b6-ee953321ef6c\",\"output_index\":1,\"logprobs\":[]}\n\nevent: response.output_text.delta\ndata: {\"sequence_number\":25,\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"ID\",\"item_id\":\"msg_c69d4e2b-243a-909a-84b6-ee953321ef6c\",\"output_index\":1,\"logprobs\":[]}\n\nevent: response.output_text.delta\ndata: {\"sequence_number\":26,\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"-\",\"item_id\":\"msg_c69d4e2b-243a-909a-84b6-ee953321ef6c\",\"output_index\":1,\"logprobs\":[]}\n\nevent: response.output_text.delta\ndata: {\"sequence_number\":27,\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"739\",\"item_id\":\"msg_c69d4e2b-243a-909a-84b6-ee953321ef6c\",\"output_index\":1,\"logprobs\":[]}\n\nevent: response.output_text.delta\ndata: {\"sequence_number\":28,\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"1\",\"item_id\":\"msg_c69d4e2b-243a-909a-84b6-ee953321ef6c\",\"output_index\":1,\"logprobs\":[]}\n\nevent: response.output_text.done\ndata: {\"sequence_number\":29,\"type\":\"response.output_text.done\",\"content_index\":0,\"item_id\":\"msg_c69d4e2b-243a-909a-84b6-ee953321ef6c\",\"output_index\":1,\"text\":\"ORCHID-7391\",\"logprobs\":[]}\n\nevent: response.content_part.done\ndata: {\"sequence_number\":30,\"type\":\"response.content_part.done\",\"content_index\":0,\"item_id\":\"msg_c69d4e2b-243a-909a-84b6-ee953321ef6c\",\"output_index\":1,\"part\":{\"type\":\"output_text\",\"text\":\"ORCHID-7391\",\"logprobs\":[],\"annotations\":[]}}\n\nevent: response.output_item.done\ndata: {\"sequence_number\":31,\"type\":\"response.output_item.done\",\"item\":{\"content\":[{\"type\":\"output_text\",\"text\":\"ORCHID-7391\",\"logprobs\":[],\"annotations\":[]}],\"id\":\"msg_c69d4e2b-243a-909a-84b6-ee953321ef6c\",\"role\":\"assistant\",\"type\":\"message\",\"status\":\"completed\"},\"output_index\":1}\n\nevent: response.completed\ndata: {\"sequence_number\":32,\"type\":\"response.completed\",\"response\":{\"created_at\":0,\"completed_at\":1784744142,\"id\":\"c69d4e2b-243a-909a-84b6-ee953321ef6c\",\"max_output_tokens\":40,\"model\":\"grok-4.5\",\"object\":\"response\",\"output\":[{\"id\":\"rs_c69d4e2b-243a-909a-84b6-ee953321ef6c\",\"summary\":[{\"text\":\"The user wants me to return only the verification code from the PDF.\\n\",\"type\":\"summary_text\"}],\"type\":\"reasoning\",\"status\":\"completed\"},{\"content\":[{\"type\":\"output_text\",\"text\":\"ORCHID-7391\",\"logprobs\":[],\"annotations\":[]}],\"id\":\"msg_c69d4e2b-243a-909a-84b6-ee953321ef6c\",\"role\":\"assistant\",\"type\":\"message\",\"status\":\"completed\"}],\"parallel_tool_calls\":true,\"previous_response_id\":null,\"reasoning\":{\"effort\":null,\"summary\":null},\"temperature\":0.0,\"text\":{\"format\":{\"type\":\"text\"}},\"tool_choice\":\"auto\",\"tools\":[],\"top_p\":0.949999988079071,\"usage\":{\"input_tokens\":1522,\"input_tokens_details\":{\"cached_tokens\":128},\"output_tokens\":119,\"output_tokens_details\":{\"reasoning_tokens\":109},\"total_tokens\":1641,\"num_sources_used\":0,\"num_server_side_tools_used\":0,\"cost_in_usd_ticks\":35404000,\"context_details\":{\"input_tokens\":1522,\"output_tokens\":119}},\"user\":null,\"incomplete_details\":null,\"status\":\"completed\",\"store\":false,\"metadata\":{\"system_fingerprint\":\"fp_a39489019fa99b6e\"},\"background\":false,\"service_tier\":\"default\",\"truncation\":\"disabled\",\"top_logprobs\":0,\"presence_penalty\":0.0,\"frequency_penalty\":0.0,\"prompt_cache_key\":null,\"max_tool_calls\":null,\"safety_identifier\":null,\"error\":null,\"instructions\":null}}\n\n" + } + } + ] +} diff --git a/packages/ai/test/lib/tool-runtime.ts b/packages/ai/test/lib/tool-runtime.ts index 28ebc47c71..55fce8e001 100644 --- a/packages/ai/test/lib/tool-runtime.ts +++ b/packages/ai/test/lib/tool-runtime.ts @@ -59,7 +59,12 @@ export const runTools = (options: RunOptions) => ...request.messages, Message.assistant(state.assistantContent), ...dispatched.map(([call, dispatched]) => - Message.tool({ id: call.id, name: call.name, result: dispatched.result }), + Message.tool({ + id: call.id, + name: call.name, + result: dispatched.result, + providerMetadata: call.providerMetadata, + }), ), ], }) diff --git a/packages/ai/test/provider/anthropic-messages.test.ts b/packages/ai/test/provider/anthropic-messages.test.ts index 69a27e6228..30b3c1b6b8 100644 --- a/packages/ai/test/provider/anthropic-messages.test.ts +++ b/packages/ai/test/provider/anthropic-messages.test.ts @@ -235,9 +235,9 @@ describe("Anthropic Messages route", () => { }), ) - // Regression: screenshot/read tool results must stay structured so base64 - // image data is not JSON-stringified into `tool_result.content`. - it.effect("lowers image tool-result content as structured image blocks", () => + // Regression: read tool results must stay structured so base64 media data is + // not JSON-stringified into `tool_result.content`. + it.effect("lowers media tool-result content as structured blocks", () => Effect.gen(function* () { const prepared = yield* LLMClient.prepare( LLM.request({ @@ -253,6 +253,7 @@ describe("Anthropic Messages route", () => { result: [ { type: "text", text: "Image read successfully" }, { type: "file", uri: "data:image/png;base64,AAECAw==", mime: "image/png" }, + { type: "file", uri: "data:application/pdf;base64,JVBERi0xLjQ=", mime: "application/pdf" }, ], }), ], @@ -263,6 +264,7 @@ describe("Anthropic Messages route", () => { expect(expectToolResult(prepared.body).content).toEqual([ { type: "text", text: "Image read successfully" }, { type: "image", source: { type: "base64", media_type: "image/png", data: "AAECAw==" } }, + { type: "document", source: { type: "base64", media_type: "application/pdf", data: "JVBERi0xLjQ=" } }, ]) }), ) @@ -292,7 +294,7 @@ describe("Anthropic Messages route", () => { }), ) - it.effect("rejects non-image media in tool-result content with a clear error", () => + it.effect("rejects unsupported media in tool-result content with a clear error", () => Effect.gen(function* () { const error = yield* LLMClient.prepare( LLM.request({ @@ -756,7 +758,7 @@ describe("Anthropic Messages route", () => { }), ) - it.effect("continues a conversation with user image content", () => + it.effect("continues a conversation with user media content", () => Effect.gen(function* () { const response = yield* LLMClient.generate( LLM.request({ @@ -766,6 +768,7 @@ describe("Anthropic Messages route", () => { Message.user([ { type: "text", text: "What is in this image?" }, { type: "media", mediaType: "image/png", data: "AAECAw==" }, + { type: "media", mediaType: "application/pdf", data: "JVBERi0xLjQ=", filename: "report.pdf" }, ]), ], }), @@ -781,6 +784,7 @@ describe("Anthropic Messages route", () => { content: [ { type: "text", text: "What is in this image?" }, { type: "image", source: { type: "base64", media_type: "image/png", data: "AAECAw==" } }, + { type: "document", source: { type: "base64", media_type: "application/pdf", data: "JVBERi0xLjQ=" } }, ], }, ], diff --git a/packages/ai/test/provider/bedrock-converse.test.ts b/packages/ai/test/provider/bedrock-converse.test.ts index a550d1eb90..59ca67333b 100644 --- a/packages/ai/test/provider/bedrock-converse.test.ts +++ b/packages/ai/test/provider/bedrock-converse.test.ts @@ -549,10 +549,12 @@ describe("Bedrock Converse route", () => { LLM.request({ id: "req_doc", model, + cache: "none", messages: [ Message.user([ + { type: "text", text: "Summarize these documents." }, { type: "media", mediaType: "application/pdf", data: "UERGREFUQQ==", filename: "report.pdf" }, - { type: "media", mediaType: "text/csv", data: "Q1NWREFUQQ==" }, + { type: "media", mediaType: "text/csv", data: "Q1NWREFUQQ==", filename: "data.csv" }, ]), ], }), @@ -563,10 +565,9 @@ describe("Bedrock Converse route", () => { { role: "user", content: [ - // Filename round-trips when supplied. + { text: "Summarize these documents." }, { document: { format: "pdf", name: "report.pdf", source: { bytes: "UERGREFUQQ==" } } }, - // Falls back to a stable placeholder when filename is missing. - { document: { format: "csv", name: "document.csv", source: { bytes: "Q1NWREFUQQ==" } } }, + { document: { format: "csv", name: "data.csv", source: { bytes: "Q1NWREFUQQ==" } } }, ], }, ], @@ -574,6 +575,96 @@ describe("Bedrock Converse route", () => { }), ) + it.effect("requires names for document media", () => + Effect.gen(function* () { + const error = yield* LLMClient.prepare( + LLM.request({ + model, + messages: [Message.user({ type: "media", mediaType: "application/pdf", data: "UERGREFUQQ==" })], + }), + ).pipe(Effect.flip) + + expect(error.message).toContain("document media requires a filename") + }), + ) + + it.effect("passes named document-only messages through for provider validation", () => + Effect.gen(function* () { + const prepared = yield* LLMClient.prepare( + LLM.request({ + model, + cache: "none", + messages: [ + Message.user({ + type: "media", + mediaType: "application/pdf", + data: "UERGREFUQQ==", + filename: "report.pdf", + }), + ], + }), + ) + + expect(prepared.body.messages).toEqual([ + { + role: "user", + content: [{ document: { format: "pdf", name: "report.pdf", source: { bytes: "UERGREFUQQ==" } } }], + }, + ]) + }), + ) + + it.effect("lowers document media in tool results", () => + Effect.gen(function* () { + const prepared = yield* LLMClient.prepare( + LLM.request({ + model, + messages: [ + Message.assistant([ToolCallPart.make({ id: "call_1", name: "read", input: { path: "report.pdf" } })]), + Message.tool({ + id: "call_1", + name: "read", + result: { + type: "content", + value: [ + { type: "text", text: "Read successfully" }, + { + type: "file", + uri: "data:application/pdf;base64,UERGREFUQQ==", + mime: "application/pdf", + name: "report", + }, + ], + }, + }), + ], + }), + ) + + expect(prepared.body.messages).toEqual([ + { + role: "assistant", + content: [{ toolUse: { toolUseId: "call_1", name: "read", input: { path: "report.pdf" } } }], + }, + { + role: "user", + content: [ + { + toolResult: { + toolUseId: "call_1", + status: "success", + content: [ + { text: "Read successfully" }, + { document: { format: "pdf", name: "report", source: { bytes: "UERGREFUQQ==" } } }, + ], + }, + }, + ], + }, + ]) + }), + ) + it.effect("rejects unsupported image media types", () => Effect.gen(function* () { const error = yield* LLMClient.prepare( diff --git a/packages/ai/test/provider/gemini.test.ts b/packages/ai/test/provider/gemini.test.ts index 1dc253c0ea..5195d372c9 100644 --- a/packages/ai/test/provider/gemini.test.ts +++ b/packages/ai/test/provider/gemini.test.ts @@ -70,6 +70,7 @@ describe("Gemini route", () => { Message.user([ { type: "text", text: "What is in this image?" }, { type: "media", mediaType: "image/png", data: "AAECAw==" }, + { type: "media", mediaType: "application/pdf", data: "JVBERi0xLjQ=" }, ]), Message.assistant([ToolCallPart.make({ id: "call_1", name: "lookup", input: { query: "weather" } })]), Message.tool({ id: "call_1", name: "lookup", result: { forecast: "sunny" } }), @@ -81,7 +82,11 @@ describe("Gemini route", () => { contents: [ { role: "user", - parts: [{ text: "What is in this image?" }, { inlineData: { mimeType: "image/png", data: "AAECAw==" } }], + parts: [ + { text: "What is in this image?" }, + { inlineData: { mimeType: "image/png", data: "AAECAw==" } }, + { inlineData: { mimeType: "application/pdf", data: "JVBERi0xLjQ=" } }, + ], }, { role: "model", @@ -90,7 +95,12 @@ describe("Gemini route", () => { { role: "user", parts: [ - { functionResponse: { name: "lookup", response: { name: "lookup", content: '{"forecast":"sunny"}' } } }, + { + functionResponse: { + name: "lookup", + response: { name: "lookup", content: '{"forecast":"sunny"}' }, + }, + }, ], }, ], @@ -110,7 +120,7 @@ describe("Gemini route", () => { }), ) - it.effect("continues image tool results as inline vision input without base64 text", () => + it.effect("continues media tool results as inline model input without base64 text", () => Effect.gen(function* () { const prepared = yield* LLMClient.prepare( LLM.request({ @@ -125,6 +135,7 @@ describe("Gemini route", () => { value: [ { type: "text", text: "Image read successfully" }, { type: "file", uri: "data:image/png;base64,AAECAw==", mime: "image/png", name: "pixel.png" }, + { type: "file", uri: "data:application/pdf;base64,JVBERi0xLjQ=", mime: "application/pdf" }, ], }, }), @@ -141,9 +152,12 @@ describe("Gemini route", () => { functionResponse: { name: "read", response: { name: "read", content: "Image read successfully" }, + parts: [ + { inlineData: { mimeType: "image/png", data: "AAECAw==" } }, + { inlineData: { mimeType: "application/pdf", data: "JVBERi0xLjQ=" } }, + ], }, }, - { inlineData: { mimeType: "image/png", data: "AAECAw==" } }, ], }, ]) @@ -174,8 +188,13 @@ describe("Gemini route", () => { { role: "user", parts: [ - { functionResponse: { name: "read", response: { name: "read", content: "" } } }, - { inlineData: { mimeType: "image/jpeg", data: "/9j/" } }, + { + functionResponse: { + name: "read", + response: { name: "read", content: "" }, + parts: [{ inlineData: { mimeType: "image/jpeg", data: "/9j/" } }], + }, + }, ], }, ]) @@ -372,7 +391,10 @@ describe("Gemini route", () => { parts: [ { text: "thinking", thought: true }, { text: "", thought: true, thoughtSignature: "thought_sig" }, - { functionCall: { name: "lookup", args: { query: "weather" } }, thoughtSignature: "tool_sig" }, + { + functionCall: { id: "provider_call", name: "lookup", args: { query: "weather" } }, + thoughtSignature: "tool_sig", + }, ], }, finishReason: "STOP", @@ -398,7 +420,10 @@ describe("Gemini route", () => { id: "reasoning-0", providerMetadata: { google: { thoughtSignature: "thought_sig" } }, }) - expect(toolCall).toMatchObject({ providerMetadata: { google: { thoughtSignature: "tool_sig" } } }) + expect(toolCall).toMatchObject({ + id: "tool_0", + providerMetadata: { google: { functionCallId: "provider_call", thoughtSignature: "tool_sig" } }, + }) expect(response.events.findIndex((event) => event.type === "reasoning-end")).toBeLessThan( response.events.findIndex((event) => event.type === "tool-call"), ) @@ -416,6 +441,13 @@ describe("Gemini route", () => { providerMetadata: toolCall?.providerMetadata, }), ]), + Message.tool({ + id: "tool_0", + name: "lookup", + result: "done", + resultType: "text", + providerMetadata: toolCall?.providerMetadata, + }), ], }), ) @@ -424,7 +456,22 @@ describe("Gemini route", () => { role: "model", parts: [ { text: "thinking", thought: true, thoughtSignature: "thought_sig" }, - { functionCall: { name: "lookup", args: { query: "weather" } }, thoughtSignature: "tool_sig" }, + { + functionCall: { id: "provider_call", name: "lookup", args: { query: "weather" } }, + thoughtSignature: "tool_sig", + }, + ], + }, + { + role: "user", + parts: [ + { + functionResponse: { + id: "provider_call", + name: "lookup", + response: { name: "lookup", content: "done" }, + }, + }, ], }, ]) @@ -498,7 +545,7 @@ describe("Gemini route", () => { content: { role: "model", parts: [ - { functionCall: { name: "lookup", args: { query: "weather" } } }, + { functionCall: { id: "tool_0", name: "lookup", args: { query: "weather" } } }, { functionCall: { name: "lookup", args: { query: "news" } } }, ], }, @@ -513,7 +560,13 @@ describe("Gemini route", () => { ).pipe(Effect.provide(fixedResponse(body))) expect(response.toolCalls).toEqual([ - { type: "tool-call", id: "tool_0", name: "lookup", input: { query: "weather" } }, + { + type: "tool-call", + id: "tool_0", + name: "lookup", + input: { query: "weather" }, + providerMetadata: { google: { functionCallId: "tool_0" } }, + }, { type: "tool-call", id: "tool_1", name: "lookup", input: { query: "news" } }, ]) expect(response.events.at(-1)).toMatchObject({ type: "finish", reason: "tool-calls" }) diff --git a/packages/ai/test/provider/openai-responses.test.ts b/packages/ai/test/provider/openai-responses.test.ts index e345cd59b1..11466a6896 100644 --- a/packages/ai/test/provider/openai-responses.test.ts +++ b/packages/ai/test/provider/openai-responses.test.ts @@ -5,6 +5,7 @@ import { LLM, LLMError, LLMEvent, Message, Model, ToolCallPart, ToolResultPart, import { Auth, LLMClient, RequestExecutor, WebSocketExecutor } from "../../src/route" import * as Azure from "../../src/providers/azure" import * as OpenAI from "../../src/providers/openai" +import * as XAI from "../../src/providers/xai" import * as OpenAIResponses from "../../src/protocols/openai-responses" import * as ProviderShared from "../../src/protocols/shared" import { continuationRequest, nativeOpenAIResponsesContinuation } from "../continuation-scenarios" @@ -16,6 +17,8 @@ const model = OpenAIResponses.route .with({ endpoint: { baseURL: "https://api.openai.test/v1/" }, auth: Auth.bearer("test") }) .model({ id: "gpt-4.1-mini" }) +const xaiModel = XAI.configure({ apiKey: "test", baseURL: "https://api.x.ai/v1" }).responses("grok-4.5") + const request = LLM.request({ id: "req_1", model, @@ -524,7 +527,77 @@ describe("OpenAI Responses route", () => { }), ) - it.effect("rejects non-image media in tool-result content with a clear error", () => + it.effect("lowers PDF tool-result content as structured input_file array", () => + Effect.gen(function* () { + const prepared = yield* LLMClient.prepare( + LLM.request({ + id: "req_tool_result_pdf", + model, + messages: [ + Message.assistant([ToolCallPart.make({ id: "call_1", name: "read", input: {} })]), + Message.tool({ + id: "call_1", + name: "read", + resultType: "content", + result: [ + { + type: "file", + uri: "data:application/pdf;base64,JVBERi0xLjQ=", + mime: "application/pdf", + name: "report.pdf", + }, + ], + }), + ], + }), + ) + + expect(expectToolOutput(prepared.body).output).toEqual([ + { + type: "input_file", + filename: "report.pdf", + file_data: "data:application/pdf;base64,JVBERi0xLjQ=", + }, + ]) + }), + ) + + it.effect("uses xAI inline file encoding for PDF tool results", () => + Effect.gen(function* () { + const prepared = yield* LLMClient.prepare( + LLM.request({ + model: xaiModel, + messages: [ + Message.assistant([ToolCallPart.make({ id: "call_1", name: "read", input: {} })]), + Message.tool({ + id: "call_1", + name: "read", + resultType: "content", + result: [ + { + type: "file", + uri: "data:application/pdf;base64,JVBERi0xLjQ=", + mime: "application/pdf", + name: "report.pdf", + }, + ], + }), + ], + }), + ) + + expect(expectToolOutput(prepared.body).output).toEqual([ + { + type: "input_file", + filename: "report.pdf", + file_data: "JVBERi0xLjQ=", + mime_type: "application/pdf", + }, + ]) + }), + ) + + it.effect("rejects unsupported media in tool-result content with a clear error", () => Effect.gen(function* () { const error = yield* LLMClient.prepare( LLM.request({ @@ -1526,20 +1599,64 @@ describe("OpenAI Responses route", () => { }), ) - it.effect("lowers user image content", () => + it.effect("lowers user image and PDF content", () => Effect.gen(function* () { const prepared = yield* LLMClient.prepare( LLM.request({ id: "req_media", model, - messages: [Message.user({ type: "media", mediaType: "image/png", data: "AAECAw==" })], + messages: [ + Message.user([ + { type: "media", mediaType: "image/png", data: "AAECAw==" }, + { type: "media", mediaType: "application/pdf", data: "JVBERi0xLjQ=", filename: "report.pdf" }, + ]), + ], }), ) expect(prepared.body.input).toEqual([ { role: "user", - content: [{ type: "input_image", image_url: "data:image/png;base64,AAECAw==" }], + content: [ + { type: "input_image", image_url: "data:image/png;base64,AAECAw==" }, + { + type: "input_file", + filename: "report.pdf", + file_data: "data:application/pdf;base64,JVBERi0xLjQ=", + }, + ], + }, + ]) + }), + ) + + it.effect("uses xAI inline file encoding for user PDFs", () => + Effect.gen(function* () { + const prepared = yield* LLMClient.prepare( + LLM.request({ + model: xaiModel, + messages: [ + Message.user({ + type: "media", + mediaType: "application/pdf", + data: "data:application/pdf;base64,JVBERi0xLjQ=", + filename: "report.pdf", + }), + ], + }), + ) + + expect(prepared.body.input).toEqual([ + { + role: "user", + content: [ + { + type: "input_file", + filename: "report.pdf", + file_data: "JVBERi0xLjQ=", + mime_type: "application/pdf", + }, + ], }, ]) }), @@ -1551,11 +1668,11 @@ describe("OpenAI Responses route", () => { LLM.request({ id: "req_media", model, - messages: [Message.user({ type: "media", mediaType: "application/pdf", data: "AAECAw==" })], + messages: [Message.user({ type: "media", mediaType: "application/x-tar", data: "AAECAw==" })], }), ).pipe(Effect.flip) - expect(error.message).toContain("OpenAI Responses does not support media type application/pdf") + expect(error.message).toContain("OpenAI Responses does not support media type application/x-tar") }), ) diff --git a/packages/ai/test/provider/pdf.recorded.test.ts b/packages/ai/test/provider/pdf.recorded.test.ts new file mode 100644 index 0000000000..c3827a651c --- /dev/null +++ b/packages/ai/test/provider/pdf.recorded.test.ts @@ -0,0 +1,207 @@ +import { describe, expect } from "bun:test" +import { Effect, Schema, Stream } from "effect" +import { LLM, LLMResponse, Message, ToolDefinition, type Model } from "../../src" +import { AmazonBedrock, Anthropic, Google, OpenAI, XAI } from "../../src/providers" +import { LLMClient } from "../../src/route" +import { Tool } from "../../src/tool" +import { runTools } from "../lib/tool-runtime" +import { recordedTests } from "../recorded-test" + +const CODE = "ORCHID-7391" +const PDF = + "JVBERi0xLjQKMSAwIG9iago8PCAvVHlwZSAvQ2F0YWxvZyAvUGFnZXMgMiAwIFIgPj4KZW5kb2JqCjIgMCBvYmoKPDwgL1R5cGUgL1BhZ2VzIC9LaWRzIFszIDAgUl0gL0NvdW50IDEgPj4KZW5kb2JqCjMgMCBvYmoKPDwgL1R5cGUgL1BhZ2UgL1BhcmVudCAyIDAgUiAvTWVkaWFCb3ggWzAgMCA2MTIgNzkyXSAvUmVzb3VyY2VzIDw8IC9Gb250IDw8IC9GMSA1IDAgUiA+PiA+PiAvQ29udGVudHMgNCAwIFIgPj4KZW5kb2JqCjQgMCBvYmoKPDwgL0xlbmd0aCA3NSA+PgpzdHJlYW0KQlQKL0YxIDE4IFRmCjcyIDcyMCBUZAooUERGIGNhc3NldHRlIHZlcmlmaWNhdGlvbiBjb2RlOiBPUkNISUQtNzM5MSkgVGoKRVQKZW5kc3RyZWFtCmVuZG9iago1IDAgb2JqCjw8IC9UeXBlIC9Gb250IC9TdWJ0eXBlIC9UeXBlMSAvQmFzZUZvbnQgL0hlbHZldGljYSA+PgplbmRvYmoKeHJlZgowIDYKMDAwMDAwMDAwMCA2NTUzNSBmIAowMDAwMDAwMDA5IDAwMDAwIG4gCjAwMDAwMDAwNTggMDAwMDAgbiAKMDAwMDAwMDExNSAwMDAwMCBuIAowMDAwMDAwMjQxIDAwMDAwIG4gCjAwMDAwMDAzNjUgMDAwMDAgbiAKdHJhaWxlcgo8PCAvU2l6ZSA2IC9Sb290IDEgMCBSID4+CnN0YXJ0eHJlZgo0MzUKJSVFT0YK" + +const openai = OpenAI.configure({ apiKey: process.env.OPENAI_API_KEY ?? "fixture" }) +const anthropic = Anthropic.configure({ apiKey: process.env.ANTHROPIC_API_KEY ?? "fixture" }) +const google = Google.configure({ apiKey: process.env.GOOGLE_API_KEY ?? "fixture" }) +const xai = XAI.configure({ apiKey: process.env.XAI_API_KEY ?? "fixture" }) +const bedrock = AmazonBedrock.configure({ + apiKey: process.env.AWS_BEDROCK_API_KEY ?? "fixture", + region: process.env.AWS_REGION ?? "us-east-1", +}) + +const targets: ReadonlyArray<{ + readonly id: string + readonly name: string + readonly provider: string + readonly protocol: string + readonly requires: string + readonly filename: string + readonly maxTokens: number + readonly model: Model +}> = [ + { + id: "openai", + name: "OpenAI Responses gpt-4o-mini", + provider: "openai", + protocol: "openai-responses", + requires: "OPENAI_API_KEY", + filename: "verification.pdf", + maxTokens: 40, + model: openai.responses("gpt-4o-mini"), + }, + { + id: "anthropic", + name: "Anthropic Haiku 4.5", + provider: "anthropic", + protocol: "anthropic-messages", + requires: "ANTHROPIC_API_KEY", + filename: "verification.pdf", + maxTokens: 40, + model: anthropic.model("claude-haiku-4-5-20251001"), + }, + { + id: "gemini", + name: "Gemini 3.5 Flash", + provider: "google", + protocol: "gemini", + requires: "GOOGLE_API_KEY", + filename: "verification.pdf", + maxTokens: 256, + model: google.model("gemini-3.5-flash"), + }, + { + id: "xai", + name: "xAI Grok 4.5", + provider: "xai", + protocol: "openai-responses", + requires: "XAI_API_KEY", + filename: "verification.pdf", + maxTokens: 40, + model: xai.responses("grok-4.5"), + }, + { + id: "bedrock", + name: "Bedrock Claude Haiku 4.5", + provider: "amazon-bedrock", + protocol: "bedrock-converse", + requires: "AWS_BEDROCK_API_KEY", + filename: "verification", + maxTokens: 40, + model: bedrock.model("us.anthropic.claude-haiku-4-5-20251001-v1:0"), + }, +] + +const recorded = recordedTests({ prefix: "pdf", tags: ["pdf"] }) +const prompt = "Return only the verification code from the PDF." +const readPdf = ToolDefinition.make({ + name: "read_pdf", + description: "Read the attached PDF.", + inputSchema: { type: "object", properties: {}, additionalProperties: false }, +}) +const readPdfRuntime = Tool.make({ + description: readPdf.description, + parameters: Schema.Struct({ path: Schema.String }), + success: Schema.String, + execute: () => Effect.succeed("PDF read successfully"), + toModelOutput: () => [ + { type: "text", text: "PDF read successfully" }, + { + type: "file", + uri: `data:application/pdf;base64,${PDF}`, + mime: "application/pdf", + name: "verification.pdf", + }, + ], +}) + +const expectCode = (response: LLMResponse) => { + expect(response.finishReason).toBe("stop") + expect(response.text.toUpperCase()).toContain(CODE) +} + +describe("PDF recorded", () => { + for (const target of targets) { + recorded.effect.with( + `reads a user PDF with ${target.name}`, + { + id: `${target.id}-user-input`, + provider: target.provider, + protocol: target.protocol, + requires: [target.requires], + tags: ["user-input"], + }, + Effect.gen(function* () { + expectCode( + yield* LLMClient.generate( + LLM.request({ + id: `recorded_pdf_${target.id}_user_input`, + model: target.model, + cache: "none", + generation: { maxTokens: target.maxTokens, temperature: 0 }, + messages: [ + Message.user([ + { type: "media", mediaType: "application/pdf", data: PDF, filename: target.filename }, + { type: "text", text: prompt }, + ]), + ], + }), + ), + ) + }), + ) + + recorded.effect.with( + `reads a PDF tool result with ${target.name}`, + { + id: `${target.id}-tool-result`, + provider: target.provider, + protocol: target.protocol, + requires: [target.requires], + tags: ["tool", "tool-result"], + }, + Effect.gen(function* () { + if (target.id === "gemini") { + const events = Array.from( + yield* runTools({ + request: LLM.request({ + id: "recorded_pdf_gemini_tool_result", + model: target.model, + system: + "Call read_pdf exactly once with path verification.pdf, then reply only with the verification code from its PDF.", + prompt: "Use read_pdf with path verification.pdf and return the verification code.", + cache: "none", + generation: { maxTokens: target.maxTokens, temperature: 0 }, + }), + tools: { read_pdf: readPdfRuntime }, + }).pipe(Stream.runCollect), + ) + expect(events.at(-1)).toMatchObject({ type: "finish", reason: "stop" }) + expect(LLMResponse.text({ events }).toUpperCase()).toContain(CODE) + return + } + + expectCode( + yield* LLMClient.generate( + LLM.request({ + id: `recorded_pdf_${target.id}_tool_result`, + model: target.model, + system: "Read the PDF returned by the tool and follow the user's response format exactly.", + cache: "none", + generation: { maxTokens: target.maxTokens, temperature: 0 }, + messages: [ + Message.user(prompt), + Message.assistant([{ type: "tool-call", id: "call_pdf_1", name: readPdf.name, input: {} }]), + Message.tool({ + id: "call_pdf_1", + name: readPdf.name, + resultType: "content", + result: [ + { type: "text", text: "PDF read successfully" }, + { + type: "file", + uri: `data:application/pdf;base64,${PDF}`, + mime: "application/pdf", + name: target.filename, + }, + ], + }), + ], + tools: [readPdf], + }), + ), + ) + }), + ) + } +}) diff --git a/packages/ai/test/tool-runtime.test.ts b/packages/ai/test/tool-runtime.test.ts index c03a18bd8a..e6e9788719 100644 --- a/packages/ai/test/tool-runtime.test.ts +++ b/packages/ai/test/tool-runtime.test.ts @@ -183,6 +183,51 @@ describe("LLMClient tools", () => { }), ) + it.effect("preserves provider metadata on dispatched tool results", () => + Effect.gen(function* () { + const tool = Tool.make({ + description: "Return text.", + parameters: Schema.Struct({}), + success: Schema.String, + execute: () => Effect.succeed("hello"), + }) + const providerMetadata = { google: { functionCallId: "provider_call" } } + const dispatched = yield* ToolRuntime.dispatch( + { tool }, + LLMEvent.toolCall({ id: "call_1", name: "tool", input: {}, providerMetadata }), + ) + + expect(dispatched.events).toEqual([ + LLMEvent.toolResult({ + id: "call_1", + name: "tool", + result: { type: "text", value: "hello" }, + output: { structured: "hello", content: [{ type: "text", text: "hello" }] }, + providerMetadata, + }), + ]) + + const failed = yield* ToolRuntime.dispatch( + {}, + LLMEvent.toolCall({ id: "call_2", name: "missing", input: {}, providerMetadata }), + ) + expect(failed.events).toEqual([ + LLMEvent.toolError({ + id: "call_2", + name: "missing", + message: "Unknown tool: missing", + providerMetadata, + }), + LLMEvent.toolResult({ + id: "call_2", + name: "missing", + result: { type: "error", value: "Unknown tool: missing" }, + providerMetadata, + }), + ]) + }), + ) + it.effect("uses the narrow default projection for encoded typed success", () => Effect.gen(function* () { const text = Tool.make({