feat: support file content type in message content (#1894)

.changeset/clever-shrimps-invite.md

@@ -0,0 +1,10 @@
+---
+"@llamaindex/core": patch
+"llamaindex": patch
+"@llamaindex/anthropic": patch
+"@llamaindex/google": patch
+"@llamaindex/openai": patch
+"@llamaindex/vercel": patch
+---
+
+feat: support file content type in message content

examples/anthropic/chat-with-file.ts

@@ -0,0 +1,39 @@
+import { Anthropic } from "@llamaindex/anthropic";
+import fs from "fs";
+
+// Note that: Anthropic only supports PDF files for now with limited models
+// See: https://docs.anthropic.com/en/docs/build-with-claude/pdf-support?q=pdf#supported-platforms-and-models
+
+async function main() {
+  if (!process.env.ANTHROPIC_API_KEY) {
+    throw new Error("Please set the ANTHROPIC_API_KEY environment variable.");
+  }
+
+  const llm = new Anthropic({
+    apiKey: process.env.ANTHROPIC_API_KEY,
+    model: "claude-3-7-sonnet",
+  });
+
+  const result = await llm.chat({
+    messages: [
+      {
+        role: "user",
+        content: [
+          {
+            type: "text",
+            text: "What's in this document? Describe it in detail.",
+          },
+          {
+            type: "file",
+            data: fs.readFileSync("./data/manga.pdf"),
+            mimeType: "application/pdf",
+          },
+        ],
+      },
+    ],
+  });
+
+  console.log(result.message);
+}
+
+void main().catch(console.error);

examples/gemini/chat.ts

@@ -1,11 +1,12 @@
 import { Gemini, GEMINI_MODEL } from "@llamaindex/google";
+import fs from "fs";
 
 (async () => {
   if (!process.env.GOOGLE_API_KEY) {
     throw new Error("Please set the GOOGLE_API_KEY environment variable.");
   }
   const gemini = new Gemini({
-    model: GEMINI_MODEL.GEMINI_PRO,
+    model: GEMINI_MODEL.GEMINI_PRO_1_5,
   });
   const result = await gemini.chat({
     messages: [
@@ -18,4 +19,26 @@ import { Gemini, GEMINI_MODEL } from "@llamaindex/google";
     ],
   });
   console.log(result);
+
+  // chat with file
+  const resultWithFile = await gemini.chat({
+    messages: [
+      {
+        role: "user",
+        content: [
+          {
+            type: "text",
+            text: "What's in this document? Describe it in detail.",
+          },
+          {
+            type: "file",
+            data: fs.readFileSync("./data/manga.pdf"),
+            mimeType: "application/pdf",
+          },
+        ],
+      },
+    ],
+  });
+
+  console.log(resultWithFile);
 })();

examples/openai/openai.ts

@@ -1,7 +1,8 @@
 import { OpenAI, OpenAIEmbedding } from "@llamaindex/openai";
+import fs from "fs";
 
 (async () => {
-  const llm = new OpenAI({ model: "gpt-4.5-preview", temperature: 0.1 });
+  const llm = new OpenAI({ model: "gpt-4o" });
 
   // complete api
   const response1 = await llm.complete({ prompt: "How are you?" });
@@ -13,7 +14,51 @@ import { OpenAI, OpenAIEmbedding } from "@llamaindex/openai";
   });
   console.log(response2.message.content);
 
-  // embeddings
+  // chat with file
+  const response3 = await llm.chat({
+    messages: [
+      {
+        role: "user",
+        content: [
+          {
+            type: "text",
+            text: "What's in this document? Describe it in detail.",
+          },
+          {
+            type: "file",
+            data: fs.readFileSync("./data/manga.pdf"),
+            mimeType: "application/pdf",
+          },
+        ],
+      },
+    ],
+  });
+  console.log(response3.message.content);
+
+  // chat with image
+  const response4 = await llm.chat({
+    messages: [
+      {
+        role: "user",
+        content: [
+          {
+            type: "text",
+            text: "What's in this image? Describe it in detail.",
+          },
+          {
+            type: "image_url",
+            image_url: {
+              url: "https://storage.googleapis.com/cloud-samples-data/vision/face/faces.jpeg",
+            },
+          },
+        ],
+      },
+    ],
+  });
+
+  console.log("Single Image Analysis:", response4.message.content);
+
+  // // embeddings
   const embedModel = new OpenAIEmbedding();
   const texts = ["hello", "world"];
   const embeddings = await embedModel.getTextEmbeddingsBatch(texts);

examples/openai/responses/chat-with-file.ts

@@ -0,0 +1,35 @@
+import { openaiResponses } from "@llamaindex/openai";
+import fs from "fs";
+
+async function main() {
+  if (!process.env.OPENAI_API_KEY) {
+    throw new Error("Please set the OPENAI_API_KEY environment variable.");
+  }
+
+  const llm = openaiResponses({
+    apiKey: process.env.OPENAI_API_KEY,
+  });
+
+  const result = await llm.chat({
+    messages: [
+      {
+        role: "user",
+        content: [
+          {
+            type: "text",
+            text: "What's in this document? Describe it in detail.",
+          },
+          {
+            type: "file",
+            data: fs.readFileSync("./data/manga.pdf"),
+            mimeType: "application/pdf",
+          },
+        ],
+      },
+    ],
+  });
+
+  console.log(result);
+}
+
+void main().catch(console.error);

packages/core/src/llms/index.ts

@@ -17,6 +17,7 @@ export type {
   LLMMetadata,
   MessageContent,
   MessageContentDetail,
+  MessageContentFileDetail,
   MessageContentImageDetail,
   MessageContentTextDetail,
   MessageType,

packages/core/src/llms/type.ts

@@ -163,9 +163,16 @@ export type MessageContentImageDetail = {
   detail?: "high" | "low" | "auto";
 };
 
+export type MessageContentFileDetail = {
+  type: "file";
+  data: Buffer;
+  mimeType: string;
+};
+
 export type MessageContentDetail =
   | MessageContentTextDetail
-  | MessageContentImageDetail;
+  | MessageContentImageDetail
+  | MessageContentFileDetail;
 
 /**
  * Extended type for the content of a message that allows for multi-modal messages.

packages/llamaindex/src/tools/QueryEngineTool.ts

@@ -51,7 +51,7 @@ export class QueryEngineTool implements BaseTool<QueryEngineParam> {
     const response = await this.queryEngine.query({ query });
 
     if (!this.includeSourceNodes) {
-      return { content: response.message.content };
+      return { content: response.message.content } as unknown as JSONValue;
     }
 
     return {

packages/providers/anthropic/src/llm.ts

@@ -319,6 +319,24 @@ export class Anthropic extends ToolCallLLM<
               text: content.text,
             };
           }
+
+          if (content.type === "file") {
+            if (content.mimeType !== "application/pdf") {
+              throw new Error(
+                "Only supports mimeType `application/pdf` for file content.",
+              );
+            }
+
+            return {
+              type: "document" as const,
+              source: {
+                type: "base64" as const,
+                media_type: content.mimeType,
+                data: content.data.toString("base64"),
+              },
+            };
+          }
+
           return {
             type: "image" as const,
             source: {

packages/providers/anthropic/tests/index.test.ts

@@ -164,6 +164,67 @@ describe("Message Formatting", () => {
 
       expect(anthropic.formatMessages(inputMessages)).toEqual(expectedOutput);
     });
+
+    test("Anthropic handles PDF file content", () => {
+      const anthropic = new Anthropic();
+      const pdfBuffer = Buffer.from("test PDF content");
+      const inputMessages: ChatMessage[] = [
+        {
+          content: [
+            {
+              type: "text",
+              text: "Here's a PDF document:",
+            },
+            {
+              type: "file",
+              mimeType: "application/pdf",
+              data: pdfBuffer,
+            },
+          ],
+          role: "user",
+        },
+      ];
+      const expectedOutput: MessageParam[] = [
+        {
+          role: "user",
+          content: [
+            {
+              type: "text",
+              text: "Here's a PDF document:",
+            },
+            {
+              type: "document",
+              source: {
+                type: "base64",
+                media_type: "application/pdf",
+                data: pdfBuffer.toString("base64"),
+              },
+            },
+          ],
+        },
+      ];
+
+      expect(anthropic.formatMessages(inputMessages)).toEqual(expectedOutput);
+    });
+
+    test("Anthropic throws error for non-PDF files", () => {
+      const anthropic = new Anthropic();
+      const docxBuffer = Buffer.from("fake docx content");
+      const inputMessages: ChatMessage[] = [
+        {
+          content: [
+            {
+              type: "file",
+              mimeType: "application/docx",
+              data: docxBuffer,
+            },
+          ],
+          role: "user",
+        },
+      ];
+
+      expect(() => anthropic.formatMessages(inputMessages)).toThrowError();
+    });
   });
 
   describe("Tool Message Formatting", () => {

packages/providers/google/package.json

@@ -32,7 +32,7 @@
   "dependencies": {
     "@google-cloud/vertexai": "1.9.0",
     "@google/genai": "^0.4.0",
-    "@google/generative-ai": "0.21.0",
+    "@google/generative-ai": "0.24.0",
     "@llamaindex/core": "workspace:*",
     "@llamaindex/env": "workspace:*"
   }

packages/providers/google/src/base.ts

@@ -5,10 +5,12 @@ import {
   type FunctionCall,
   type ModelParams as GoogleModelParams,
   type RequestOptions as GoogleRequestOptions,
+  type StartChatParams as GoogleStartChatParams,
   type GenerateContentStreamResult as GoogleStreamGenerateContentResult,
   type SafetySetting,
 } from "@google/generative-ai";
 
+import type { StartChatParams as VertexStartChatParams } from "@google-cloud/vertexai";
 import { wrapLLMEvent } from "@llamaindex/core/decorator";
 import type {
   CompletionResponse,
@@ -96,6 +98,8 @@ export type GeminiConfig = Partial<typeof DEFAULT_GEMINI_PARAMS> & {
   safetySettings?: SafetySetting[];
 };
 
+type StartChatParams = GoogleStartChatParams & VertexStartChatParams;
+
 /**
  * Gemini Session to manage the connection to the Gemini API
  */
@@ -254,13 +258,13 @@ export class Gemini extends ToolCallLLM<GeminiAdditionalChatOptions> {
     };
   }
 
-  private createStartChatParams(
+  private async createStartChatParams(
     params: GeminiChatParamsNonStreaming | GeminiChatParamsStreaming,
   ) {
-    const context = getChatContext(params);
+    const context = await getChatContext(params);
     const common = {
       history: context.history,
-      safetySettings: this.safetySettings,
+      safetySettings: this.safetySettings as SafetySetting[],
     };
 
     return params.tools?.length
@@ -282,12 +286,14 @@ export class Gemini extends ToolCallLLM<GeminiAdditionalChatOptions> {
   protected async nonStreamChat(
     params: GeminiChatParamsNonStreaming,
   ): Promise<GeminiChatNonStreamResponse> {
-    const context = getChatContext(params);
+    const context = await getChatContext(params);
     const client = this.session.getGenerativeModel(
       this.metadata,
       this.#requestOptions,
     );
-    const chat = client.startChat(this.createStartChatParams(params));
+    const chat = client.startChat(
+      (await this.createStartChatParams(params)) as StartChatParams,
+    );
     const { response } = await chat.sendMessage(context.message);
     const topCandidate = response.candidates![0]!;
 
@@ -311,12 +317,14 @@ export class Gemini extends ToolCallLLM<GeminiAdditionalChatOptions> {
   protected async *streamChat(
     params: GeminiChatParamsStreaming,
   ): GeminiChatStreamResponse {
-    const context = getChatContext(params);
+    const context = await getChatContext(params);
     const client = this.session.getGenerativeModel(
       this.metadata,
       this.#requestOptions,
     );
-    const chat = client.startChat(this.createStartChatParams(params));
+    const chat = client.startChat(
+      (await this.createStartChatParams(params)) as StartChatParams,
+    );
     const result = await chat.sendMessageStream(context.message);
     yield* this.session.getChatStream(result);
   }
@@ -350,7 +358,7 @@ export class Gemini extends ToolCallLLM<GeminiAdditionalChatOptions> {
     if (stream) {
       const result = await client.generateContentStream(
         getPartsText(
-          GeminiHelper.messageContentToGeminiParts({ content: prompt }),
+          await GeminiHelper.messageContentToGeminiParts({ content: prompt }),
         ),
       );
       return this.session.getCompletionStream(result);
@@ -358,7 +366,7 @@ export class Gemini extends ToolCallLLM<GeminiAdditionalChatOptions> {
 
     const result = await client.generateContent(
       getPartsText(
-        GeminiHelper.messageContentToGeminiParts({ content: prompt }),
+        await GeminiHelper.messageContentToGeminiParts({ content: prompt }),
       ),
     );
     return {

packages/providers/google/src/utils.ts

@@ -8,15 +8,18 @@ import {
 } from "@google/generative-ai";
 
 import { type GenerateContentResponse } from "@google-cloud/vertexai";
+import { FileState, GoogleAIFileManager } from "@google/generative-ai/server";
 import type {
   BaseTool,
   ChatMessage,
+  MessageContentFileDetail,
   MessageContentImageDetail,
   MessageContentTextDetail,
   MessageType,
   ToolCallLLMMessageOptions,
 } from "@llamaindex/core/llms";
 import { extractDataUrlComponents } from "@llamaindex/core/utils";
+import { getEnv } from "@llamaindex/env";
 import type {
   ChatContext,
   FileDataPart,
@@ -126,9 +129,9 @@ export const cleanParts = (
   };
 };
 
-export const getChatContext = (
+export const getChatContext = async (
   params: GeminiChatParamsStreaming | GeminiChatParamsNonStreaming,
-): ChatContext => {
+): Promise<ChatContext> => {
   // Gemini doesn't allow:
   // 1. Consecutive messages from the same role
   // 2. Parts that have empty text
@@ -145,8 +148,10 @@ export const getChatContext = (
     {} as Record<string, string>,
   );
   const messages = GeminiHelper.mergeNeighboringSameRoleMessages(
-    params.messages.map((message) =>
-      GeminiHelper.chatMessageToGemini(message, fnMap),
+    await Promise.all(
+      params.messages.map((message) =>
+        GeminiHelper.chatMessageToGemini(message, fnMap),
+      ),
     ),
   ).map(cleanParts);
 
@@ -226,13 +231,13 @@ export class GeminiHelper {
       );
   }
 
-  public static messageContentToGeminiParts({
+  public static async messageContentToGeminiParts({
     content,
     options = undefined,
     fnMap = undefined,
   }: Pick<ChatMessage<ToolCallLLMMessageOptions>, "content" | "options"> & {
     fnMap?: Record<string, string>;
-  }): Part[] {
+  }): Promise<Part[]> {
     if (options && "toolResult" in options) {
       if (!fnMap) throw Error("fnMap must be set");
       const name = fnMap[options.toolResult.id];
@@ -276,9 +281,53 @@ export class GeminiHelper {
       (i) => i.type === "text",
     ) as MessageContentTextDetail[];
     parts.push(...textContents.map((t) => ({ text: t.text })));
+
+    const fileContents = content.filter(
+      (i) => i.type === "file",
+    ) as MessageContentFileDetail[];
+
+    if (fileContents.length > 0) {
+      for (const file of fileContents) {
+        const uploadResponse = await GeminiHelper.uploadFile(
+          file.data,
+          file.mimeType,
+        );
+        parts.push({
+          fileData: {
+            mimeType: uploadResponse.file.mimeType,
+            fileUri: uploadResponse.file.uri,
+          },
+        });
+      }
+    }
+
     return parts;
   }
 
+  // Upload a file for AI processing
+  public static async uploadFile(
+    data: string | Buffer, // file name or buffer
+    mimeType: string, // eg. application/pdf
+    interval = 5_000, // time to refetch upload status
+  ) {
+    const fileManager = new GoogleAIFileManager(getEnv("GOOGLE_API_KEY")!);
+
+    const uploadResponse = await fileManager.uploadFile(data, { mimeType });
+
+    let file = await fileManager.getFile(uploadResponse.file.name);
+
+    while (file.state === FileState.PROCESSING) {
+      await new Promise((resolve) => setTimeout(resolve, interval));
+      file = await fileManager.getFile(uploadResponse.file.name);
+    }
+
+    if (file.state === FileState.FAILED) {
+      throw new Error("Failed to upload file");
+    }
+
+    return uploadResponse;
+  }
+
   public static getGeminiMessageRole(
     message: ChatMessage<ToolCallLLMMessageOptions>,
   ): GeminiMessageRole {
@@ -290,13 +339,16 @@ export class GeminiHelper {
     ];
   }
 
-  public static chatMessageToGemini(
+  public static async chatMessageToGemini(
     message: ChatMessage<ToolCallLLMMessageOptions>,
     fnMap: Record<string, string>, // mapping of fn call id to fn call name
-  ): GeminiMessageContent {
+  ): Promise<GeminiMessageContent> {
     return {
       role: GeminiHelper.getGeminiMessageRole(message),
-      parts: GeminiHelper.messageContentToGeminiParts({ ...message, fnMap }),
+      parts: await GeminiHelper.messageContentToGeminiParts({
+        ...message,
+        fnMap,
+      }),
     };
   }
 }

packages/providers/google/src/vertex.ts

@@ -1,5 +1,6 @@
 import {
   type GenerateContentResponse,
+  type SafetySetting,
   VertexAI,
   GenerativeModel as VertexGenerativeModel,
   GenerativeModelPreview as VertexGenerativeModelPreview,
@@ -62,12 +63,12 @@ export class GeminiVertexSession implements IGeminiSession {
     const safetySettings = metadata.safetySettings ?? DEFAULT_SAFETY_SETTINGS;
     if (this.preview) {
       return this.vertex.preview.getGenerativeModel({
-        safetySettings,
+        safetySettings: safetySettings as SafetySetting[],
         ...metadata,
       });
     }
     return this.vertex.getGenerativeModel({
-      safetySettings,
+      safetySettings: safetySettings as SafetySetting[],
       ...metadata,
     });
   }

packages/providers/openai/src/llm.ts

@@ -1,5 +1,6 @@
 import { wrapEventCaller, wrapLLMEvent } from "@llamaindex/core/decorator";
 import {
+  ToolCallLLM,
   type BaseTool,
   type ChatMessage,
   type ChatResponse,
@@ -9,7 +10,6 @@ import {
   type LLMMetadata,
   type MessageType,
   type PartialToolCall,
-  ToolCallLLM,
   type ToolCallLLMMessageOptions,
 } from "@llamaindex/core/llms";
 import { extractText } from "@llamaindex/core/utils";
@@ -24,6 +24,7 @@ import { zodResponseFormat } from "openai/helpers/zod";
 import type { ChatModel } from "openai/resources/chat/chat";
 import type {
   ChatCompletionAssistantMessageParam,
+  ChatCompletionContentPart,
   ChatCompletionMessageToolCall,
   ChatCompletionRole,
   ChatCompletionSystemMessageParam,
@@ -205,9 +206,29 @@ export class OpenAI extends ToolCallLLM<OpenAIAdditionalChatOptions> {
           }),
         } satisfies ChatCompletionAssistantMessageParam;
       } else if (message.role === "user") {
+        if (typeof message.content === "string") {
+          return { role: "user", content: message.content };
+        }
+
         return {
           role: "user",
-          content: message.content,
+          content: message.content.map((item, index) => {
+            if (item.type === "file") {
+              if (item.mimeType !== "application/pdf") {
+                throw new Error("Only PDF files are supported");
+              }
+              return {
+                type: "file",
+                file: {
+                  file_data: `data:${item.mimeType};base64,${item.data.toString("base64")}`,
+                  filename: `part-${index}.pdf`,
+                },
+              } satisfies ChatCompletionContentPart.File;
+            }
+
+            // keep it as is for other types
+            return item;
+          }),
         } satisfies ChatCompletionUserMessageParam;
       }
 

packages/providers/openai/src/responses.ts

@@ -682,7 +682,7 @@ export class OpenAIResponses extends ToolCallLLM<OpenAIResponsesChatOptions> {
       return content;
     }
 
-    return content.map((item) => {
+    return content.map((item, index) => {
       if (item.type === "text") {
         return {
           type: "input_text",
@@ -696,6 +696,19 @@ export class OpenAIResponses extends ToolCallLLM<OpenAIResponsesChatOptions> {
           detail: item.detail || "auto",
         };
       }
+      if (item.type === "file") {
+        if (item.mimeType !== "application/pdf") {
+          throw new Error(
+            "Only supports mimeType `application/pdf` for file content.",
+          );
+        }
+
+        return {
+          type: "input_file",
+          filename: `part-${index}.pdf`,
+          file_data: `data:${item.mimeType};base64,${item.data.toString("base64")}`,
+        };
+      }
       throw new Error("Unsupported content type");
     });
   }

packages/providers/openai/src/utils.ts

@@ -227,9 +227,17 @@ export type ResponsesMessageContentImageDetail = {
   image_url: string;
   detail: "high" | "low" | "auto";
 };
+
+export type ResponsesMessageContentFileDetail = {
+  type: "input_file";
+  filename: string;
+  file_data: string;
+};
+
 export type ResponsesMessageContentDetail =
   | ResponsesMessageContentTextDetail
-  | ResponsesMessageContentImageDetail;
+  | ResponsesMessageContentImageDetail
+  | ResponsesMessageContentFileDetail;
 
 export type ResponseMessageContent = string | ResponsesMessageContentDetail[];
 

packages/providers/openai/tests/chat.test.ts

@@ -1,3 +1,4 @@
+import { ChatMessage, ToolCallLLMMessageOptions } from "@llamaindex/core/llms";
 import { describe, expect, it } from "vitest";
 import { z } from "zod";
 import { OpenAI } from "../src/llm";
@@ -44,3 +45,189 @@ describe("OpenAI Chat Tests", () => {
     });
   });
 });
+
+describe("OpenAI Static Methods", () => {
+  describe("toOpenAIMessage", () => {
+    it("should convert simple text messages", () => {
+      const messages: ChatMessage<ToolCallLLMMessageOptions>[] = [
+        {
+          role: "user",
+          content: "Hello world",
+        },
+        {
+          role: "assistant",
+          content: "Hi there",
+        },
+        {
+          role: "system",
+          content: "You are a helpful assistant",
+        },
+      ];
+
+      const result = OpenAI.toOpenAIMessage(messages);
+      expect(result).toEqual([
+        {
+          role: "user",
+          content: "Hello world",
+        },
+        {
+          role: "assistant",
+          content: "Hi there",
+        },
+        {
+          role: "system",
+          content: "You are a helpful assistant",
+        },
+      ]);
+    });
+
+    it("should convert tool result messages", () => {
+      const messages: ChatMessage<ToolCallLLMMessageOptions>[] = [
+        {
+          role: "assistant",
+          content: "Weather result",
+          options: {
+            toolResult: {
+              id: "weather-123",
+            },
+          },
+        },
+      ];
+
+      const result = OpenAI.toOpenAIMessage(messages);
+      expect(result).toEqual([
+        {
+          role: "tool",
+          content: "Weather result",
+          tool_call_id: "weather-123",
+        },
+      ]);
+    });
+
+    it("should convert tool call messages", () => {
+      const messages: ChatMessage<ToolCallLLMMessageOptions>[] = [
+        {
+          role: "assistant",
+          content: "Let me check the weather",
+          options: {
+            toolCall: [
+              {
+                id: "weather-123",
+                name: "get_weather",
+                input: { location: "London" },
+              },
+            ],
+          },
+        },
+      ];
+
+      const result = OpenAI.toOpenAIMessage(messages);
+      expect(result).toEqual([
+        {
+          role: "assistant",
+          content: "Let me check the weather",
+          tool_calls: [
+            {
+              id: "weather-123",
+              type: "function",
+              function: {
+                name: "get_weather",
+                arguments: JSON.stringify({ location: "London" }),
+              },
+            },
+          ],
+        },
+      ]);
+    });
+
+    it("should convert user messages with file content", () => {
+      const pdfBuffer = Buffer.from("test PDF content");
+      const messages: ChatMessage<ToolCallLLMMessageOptions>[] = [
+        {
+          role: "user",
+          content: [
+            {
+              type: "file",
+              mimeType: "application/pdf",
+              data: pdfBuffer,
+            },
+          ],
+        },
+      ];
+
+      const result = OpenAI.toOpenAIMessage(messages);
+      expect(result).toEqual([
+        {
+          role: "user",
+          content: [
+            {
+              type: "file",
+              file: {
+                file_data: `data:application/pdf;base64,${pdfBuffer.toString("base64")}`,
+                filename: "part-0.pdf",
+              },
+            },
+          ],
+        },
+      ]);
+    });
+
+    it("should convert user messages with mixed content", () => {
+      const pdfBuffer = Buffer.from("test PDF content");
+      const messages: ChatMessage<ToolCallLLMMessageOptions>[] = [
+        {
+          role: "user",
+          content: [
+            {
+              type: "text",
+              text: "Here's a PDF file:",
+            },
+            {
+              type: "file",
+              mimeType: "application/pdf",
+              data: pdfBuffer,
+            },
+          ],
+        },
+      ];
+
+      const result = OpenAI.toOpenAIMessage(messages);
+      expect(result).toEqual([
+        {
+          role: "user",
+          content: [
+            {
+              type: "text",
+              text: "Here's a PDF file:",
+            },
+            {
+              type: "file",
+              file: {
+                file_data: `data:application/pdf;base64,${pdfBuffer.toString("base64")}`,
+                filename: "part-1.pdf",
+              },
+            },
+          ],
+        },
+      ]);
+    });
+
+    it("should throw error for non-PDF files", () => {
+      const fileBuffer = Buffer.from("fake file content");
+      const messages: ChatMessage<ToolCallLLMMessageOptions>[] = [
+        {
+          role: "user",
+          content: [
+            {
+              type: "file",
+              mimeType: "text/csv",
+              data: fileBuffer,
+            },
+          ],
+        },
+      ];
+
+      expect(() => OpenAI.toOpenAIMessage(messages)).toThrowError();
+    });
+  });
+});

packages/providers/openai/tests/responses.test.ts

@@ -181,6 +181,36 @@ describe("OpenAIResponses Unit Tests", () => {
         },
       ]);
     });
+
+    it("should process file content with PDF type", () => {
+      const pdfBuffer = Buffer.from("test PDF content");
+      const content = [
+        {
+          type: "file",
+          mimeType: "application/pdf",
+          data: pdfBuffer,
+        },
+      ];
+      // @ts-expect-error accessing private method
+      const result = llm.processMessageContent(content);
+      expect(result[0]).toEqual({
+        type: "input_file",
+        filename: "part-0.pdf",
+        file_data: `data:application/pdf;base64,${pdfBuffer.toString("base64")}`,
+      });
+    });
+
+    it("should throw error for non-PDF file types", () => {
+      const content = [
+        {
+          type: "file",
+          mimeType: "image/jpeg",
+          data: Buffer.from("test image content"),
+        },
+      ];
+      // @ts-expect-error accessing private method
+      expect(() => llm.processMessageContent(content)).toThrowError();
+    });
   });
 
   describe("isResponseCreatedEvent", () => {

packages/providers/vercel/src/llm.ts

@@ -96,6 +96,9 @@ export class VercelLLM extends ToolCallLLM<VercelAdditionalChatOptions> {
                       image: new URL(contentDetail.image_url.url),
                     } satisfies ImagePart;
                   }
+                  if (contentDetail.type === "file") {
+                    throw new Error("File content not supported yet");
+                  }
                   return {
                     type: "text",
                     text: contentDetail.text,

pnpm-lock.yaml

@@ -1207,8 +1207,8 @@ importers:
         specifier: ^0.4.0
         version: 0.4.0(bufferutil@4.0.9)
       '@google/generative-ai':
-        specifier: 0.21.0
-        version: 0.21.0
+        specifier: 0.24.0
+        version: 0.24.0
       '@llamaindex/core':
         specifier: workspace:*
         version: link:../../core
@@ -3194,8 +3194,8 @@ packages:
     resolution: {integrity: sha512-Cm4uJX1sKarpm1mje/MiOIinM7zdUUrQp/5/qGPAgznbdd/B9zup5ehT6c1qGqycFcSopTA1J1HpqHS5kJR8hQ==}
     engines: {node: '>=18.0.0'}
 
-  '@google/generative-ai@0.21.0':
-    resolution: {integrity: sha512-7XhUbtnlkSEZK15kN3t+tzIMxsbKm/dSkKBFalj+20NvPKe1kBY7mR2P7vuijEn+f06z5+A8bVGKO0v39cr6Wg==}
+  '@google/generative-ai@0.24.0':
+    resolution: {integrity: sha512-fnEITCGEB7NdX0BhoYZ/cq/7WPZ1QS5IzJJfC3Tg/OwkvBetMiVJciyaan297OvE4B9Jg1xvo0zIazX/9sGu1Q==}
     engines: {node: '>=18.0.0'}
 
   '@graphql-typed-document-node/core@3.2.0':
@@ -15337,7 +15337,7 @@ snapshots:
   '@google/generative-ai@0.1.3':
     optional: true
 
-  '@google/generative-ai@0.21.0': {}
+  '@google/generative-ai@0.24.0': {}
 
   '@graphql-typed-document-node/core@3.2.0(graphql@16.10.0)':
     dependencies: