Release 0.10.3 (#1898 )

Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: marcusschiesser <17126+marcusschiesser@users.noreply.github.com>
feat: support file content type in message content (#1894 )
2026-07-01 22:14:03 -04:00 · 2025-04-29 13:05:36 +07:00 · 2025-04-29 12:57:35 +07:00 · 2025-04-29 11:25:01 +07:00 · 2025-04-28 14:39:47 +07:00 · 2025-04-28 11:46:06 +07:00
251 changed files with 2206 additions and 11813 deletions
@@ -1,5 +1,18 @@
 # @llamaindex/doc

+## 0.2.15
+
+### Patch Changes
+
+- Updated dependencies [3ee8c83]
+  - @llamaindex/core@0.6.3
+  - llamaindex@0.10.3
+  - @llamaindex/openai@0.3.5
+  - @llamaindex/cloud@4.0.4
+  - @llamaindex/node-parser@2.0.3
+  - @llamaindex/readers@3.1.1
+  - @llamaindex/workflow@1.0.4
+
 ## 0.2.14

 ### Patch Changes
@@ -1,6 +1,6 @@
 {
  "name": "@llamaindex/doc",
-  "version": "0.2.14",
+  "version": "0.2.15",
  "private": true,
  "scripts": {
    "postinstall": "fumadocs-mdx",
@@ -11,8 +11,3 @@ export const CodeNodeParserDemo = dynamic(() =>
    (mod) => mod.CodeNodeParserDemo,
  ),
 );
-export const WorkflowStreamingDemo = dynamic(() =>
-  import("@/components/demo/workflow-streaming-ui").then(
-    (mod) => mod.WorkflowStreamingDemo,
-  ),
-);
@@ -1,152 +0,0 @@
-"use client";
-import FlowInput from "@/components/flow-input";
-import { Button } from "@/components/ui/button";
-import {
-  StartEvent,
-  StopEvent,
-  Workflow,
-  WorkflowEvent,
-} from "@llamaindex/workflow";
-import { ReactNode, startTransition, useState } from "react";
-import { StickToBottom, useStickToBottomContext } from "use-stick-to-bottom";
-
-class ComputeEvent extends WorkflowEvent<number> {
-  constructor(data: number) {
-    super(data);
-  }
-}
-
-class ComputeResultEvent extends WorkflowEvent<number> {
-  constructor(data: number) {
-    super(data);
-  }
-}
-
-type ContextData = {
-  sum: number;
-};
-
-const workflow = new Workflow<ContextData, number, number>();
-
-const max = 1000;
-const min = 100;
-
-workflow.addStep(
-  {
-    inputs: [StartEvent<number>],
-    outputs: [StopEvent<number>],
-  },
-  async (context, event) => {
-    const total = event.data;
-    for (let i = 0; i < total; i++) {
-      context.sendEvent(new ComputeEvent(i));
-    }
-    console.log("waiting");
-    const computeResults = await Promise.all(
-      Array.from({ length: total }).map(() =>
-        context.requireEvent(ComputeResultEvent),
-      ),
-    );
-    context.data.sum = computeResults.reduce(
-      (acc, result) => acc + result.data,
-      0,
-    );
-    console.log("stop");
-    return new StopEvent(context.data.sum);
-  },
-);
-
-workflow.addStep(
-  {
-    inputs: [ComputeEvent],
-    outputs: [ComputeResultEvent],
-  },
-  async (context, event) => {
-    await new Promise((resolve) =>
-      setTimeout(resolve, Math.floor(Math.random() * (max - min + 1) + min)),
-    );
-    return new ComputeResultEvent(event.data);
-  },
-);
-
-function ScrollToBottom() {
-  const { isAtBottom, scrollToBottom } = useStickToBottomContext();
-
-  return (
-    !isAtBottom && (
-      <button
-        className="i-ph-arrow-circle-down-fill absolute bottom-0 left-[50%] translate-x-[-50%] rounded-lg text-4xl"
-        onClick={() => scrollToBottom()}
-      />
-    )
-  );
-}
-
-export function WorkflowStreamingDemo() {
-  const [ui, setUI] = useState<ReactNode[]>([
-    <div key={0} className="bg-gray-100 dark:bg-gray-800">
-      Waiting for workflow to start
-    </div>,
-  ]);
-  const [total, setTotal] = useState<number>(10);
-
-  return (
-    <div className="flex w-full flex-col items-start gap-2">
-      <div className="flex flex-row items-center justify-center">
-        <div className="mr-2 text-lg">Compute total</div>{" "}
-        <FlowInput value={total} onChange={(value) => setTotal(value)} />
-      </div>
-      <Button
-        onClick={async () => {
-          startTransition(() => {
-            setUI([]);
-          });
-          const context = workflow.run(total, {
-            sum: 0,
-          });
-          let i = 0;
-          for await (const event of context) {
-            console.log(event);
-            if (event instanceof ComputeEvent) {
-              setUI((ui) => [
-                ...ui,
-                <div key={i++} className="bg-yellow-100 dark:bg-yellow-800">
-                  Computing task id: {event.data}
-                </div>,
-              ]);
-            } else if (event instanceof ComputeResultEvent) {
-              setUI((ui) => [
-                ...ui,
-                <div key={i++} className="bg-green-100 dark:bg-green-800">
-                  Computed task id: {event.data}
-                </div>,
-              ]);
-            } else if (event instanceof StartEvent) {
-              setUI((ui) => [
-                ...ui,
-                <div key={i++} className="bg-blue-100 dark:bg-blue-800">
-                  Started workflow with total {event.data}
-                </div>,
-              ]);
-            } else if (event instanceof StopEvent) {
-              setUI((ui) => [
-                ...ui,
-                <div key={i++} className="bg-red-100 dark:bg-red-800">
-                  Workflow stopped
-                </div>,
-              ]);
-            }
-          }
-        }}
-      >
-        Start Workflow
-      </Button>
-      <StickToBottom className="flex max-h-96 w-full flex-col gap-2 overflow-y-auto rounded-lg border border-gray-200 p-2">
-        <StickToBottom.Content className="flex flex-col gap-2">
-          {ui}
-        </StickToBottom.Content>
-        <ScrollToBottom />
-      </StickToBottom>
-    </div>
-  );
-}
@@ -1,5 +0,0 @@
-export type {
-  HandlerContext,
-  Workflow,
-  WorkflowContext,
-} from "@llamaindex/workflow";
@@ -1,5 +1,12 @@
 # @llamaindex/cloudflare-worker-agent-test

+## 0.0.157
+
+### Patch Changes
+
+- Updated dependencies [3ee8c83]
+  - llamaindex@0.10.3
+
 ## 0.0.156

 ### Patch Changes
@@ -1,6 +1,6 @@
 {
  "name": "@llamaindex/cloudflare-worker-agent-test",
-  "version": "0.0.156",
+  "version": "0.0.157",
  "type": "module",
  "private": true,
  "scripts": {
@@ -1,5 +1,11 @@
 # @llamaindex/llama-parse-browser-test

+## 0.0.59
+
+### Patch Changes
+
+- @llamaindex/cloud@4.0.4
+
 ## 0.0.58

 ### Patch Changes
@@ -1,7 +1,7 @@
 {
  "name": "@llamaindex/llama-parse-browser-test",
  "private": true,
-  "version": "0.0.58",
+  "version": "0.0.59",
  "type": "module",
  "scripts": {
    "dev": "vite",
@@ -10,8 +10,8 @@
  },
  "devDependencies": {
    "typescript": "^5.7.3",
-    "vite": "^5.4.16",
-    "vite-plugin-wasm": "^3.3.0"
+    "vite": "^6.3.3",
+    "vite-plugin-wasm": "^3.4.1"
  },
  "dependencies": {
    "@llamaindex/cloud": "workspace:*"
@@ -1,5 +1,12 @@
 # @llamaindex/next-agent-test

+## 0.1.157
+
+### Patch Changes
+
+- Updated dependencies [3ee8c83]
+  - llamaindex@0.10.3
+
 ## 0.1.156

 ### Patch Changes
@@ -1,6 +1,6 @@
 {
  "name": "@llamaindex/next-agent-test",
-  "version": "0.1.156",
+  "version": "0.1.157",
  "private": true,
  "scripts": {
    "dev": "next dev",
@@ -1,5 +1,12 @@
 # test-edge-runtime

+## 0.1.156
+
+### Patch Changes
+
+- Updated dependencies [3ee8c83]
+  - llamaindex@0.10.3
+
 ## 0.1.155

 ### Patch Changes
@@ -1,6 +1,6 @@
 {
  "name": "@llamaindex/nextjs-edge-runtime-test",
-  "version": "0.1.155",
+  "version": "0.1.156",
  "private": true,
  "scripts": {
    "dev": "next dev",
@@ -1,5 +1,14 @@
 # @llamaindex/next-node-runtime

+## 0.1.24
+
+### Patch Changes
+
+- Updated dependencies [3ee8c83]
+  - llamaindex@0.10.3
+  - @llamaindex/huggingface@0.1.7
+  - @llamaindex/readers@3.1.1
+
 ## 0.1.23

 ### Patch Changes
@@ -1,6 +1,6 @@
 {
  "name": "@llamaindex/next-node-runtime-test",
-  "version": "0.1.23",
+  "version": "0.1.24",
  "private": true,
  "scripts": {
    "dev": "next dev",
@@ -1,5 +1,12 @@
 # vite-import-llamaindex

+## 0.0.23
+
+### Patch Changes
+
+- Updated dependencies [3ee8c83]
+  - llamaindex@0.10.3
+
 ## 0.0.22

 ### Patch Changes
@@ -1,7 +1,7 @@
 {
  "name": "vite-import-llamaindex",
  "private": true,
-  "version": "0.0.22",
+  "version": "0.0.23",
  "type": "module",
  "scripts": {
    "build": "vite build",
@@ -16,7 +16,7 @@
    "@size-limit/preset-big-lib": "^11.1.6",
    "size-limit": "^11.1.6",
    "typescript": "^5.7.3",
-    "vite": "^5.4.16"
+    "vite": "^6.3.3"
  },
  "dependencies": {
    "llamaindex": "workspace:*"
@@ -1,5 +1,12 @@
 # @llamaindex/waku-query-engine-test

+## 0.0.157
+
+### Patch Changes
+
+- Updated dependencies [3ee8c83]
+  - llamaindex@0.10.3
+
 ## 0.0.156

 ### Patch Changes
@@ -1,6 +1,6 @@
 {
  "name": "@llamaindex/waku-query-engine-test",
-  "version": "0.0.156",
+  "version": "0.0.157",
  "type": "module",
  "private": true,
  "scripts": {
@@ -1,5 +1,55 @@
 # examples

+## 0.3.13
+
+### Patch Changes
+
+- Updated dependencies [3ee8c83]
+  - @llamaindex/core@0.6.3
+  - llamaindex@0.10.3
+  - @llamaindex/anthropic@0.3.4
+  - @llamaindex/google@0.2.5
+  - @llamaindex/openai@0.3.5
+  - @llamaindex/vercel@0.1.3
+  - @llamaindex/cloud@4.0.4
+  - @llamaindex/node-parser@2.0.3
+  - @llamaindex/assemblyai@0.1.2
+  - @llamaindex/clip@0.0.53
+  - @llamaindex/cohere@0.0.17
+  - @llamaindex/deepinfra@0.0.53
+  - @llamaindex/discord@0.1.2
+  - @llamaindex/huggingface@0.1.7
+  - @llamaindex/jinaai@0.0.13
+  - @llamaindex/mistral@0.1.3
+  - @llamaindex/mixedbread@0.0.17
+  - @llamaindex/notion@0.1.2
+  - @llamaindex/ollama@0.1.3
+  - @llamaindex/perplexity@0.0.10
+  - @llamaindex/portkey-ai@0.0.45
+  - @llamaindex/replicate@0.0.45
+  - @llamaindex/astra@0.0.17
+  - @llamaindex/azure@0.1.13
+  - @llamaindex/chroma@0.0.17
+  - @llamaindex/elastic-search@0.1.3
+  - @llamaindex/firestore@1.0.10
+  - @llamaindex/milvus@0.1.12
+  - @llamaindex/mongodb@0.0.18
+  - @llamaindex/pinecone@0.1.3
+  - @llamaindex/postgres@0.0.46
+  - @llamaindex/qdrant@0.1.12
+  - @llamaindex/supabase@0.1.2
+  - @llamaindex/upstash@0.0.17
+  - @llamaindex/weaviate@0.0.17
+  - @llamaindex/voyage-ai@1.0.9
+  - @llamaindex/readers@3.1.1
+  - @llamaindex/tools@0.0.8
+  - @llamaindex/workflow@1.0.4
+  - @llamaindex/deepseek@0.0.13
+  - @llamaindex/fireworks@0.0.13
+  - @llamaindex/groq@0.0.68
+  - @llamaindex/together@0.0.13
+  - @llamaindex/vllm@0.0.39
+
 ## 0.3.12

 ### Patch Changes
@@ -0,0 +1,39 @@
+import { Anthropic } from "@llamaindex/anthropic";
+import fs from "fs";
+
+// Note that: Anthropic only supports PDF files for now with limited models
+// See: https://docs.anthropic.com/en/docs/build-with-claude/pdf-support?q=pdf#supported-platforms-and-models
+
+async function main() {
+  if (!process.env.ANTHROPIC_API_KEY) {
+    throw new Error("Please set the ANTHROPIC_API_KEY environment variable.");
+  }
+
+  const llm = new Anthropic({
+    apiKey: process.env.ANTHROPIC_API_KEY,
+    model: "claude-3-7-sonnet",
+  });
+
+  const result = await llm.chat({
+    messages: [
+      {
+        role: "user",
+        content: [
+          {
+            type: "text",
+            text: "What's in this document? Describe it in detail.",
+          },
+          {
+            type: "file",
+            data: fs.readFileSync("./data/manga.pdf"),
+            mimeType: "application/pdf",
+          },
+        ],
+      },
+    ],
+  });
+
+  console.log(result.message);
+}
+
+void main().catch(console.error);
@@ -1,11 +1,12 @@
 import { Gemini, GEMINI_MODEL } from "@llamaindex/google";
+import fs from "fs";

 (async () => {
  if (!process.env.GOOGLE_API_KEY) {
    throw new Error("Please set the GOOGLE_API_KEY environment variable.");
  }
  const gemini = new Gemini({
-    model: GEMINI_MODEL.GEMINI_PRO,
+    model: GEMINI_MODEL.GEMINI_PRO_1_5,
  });
  const result = await gemini.chat({
    messages: [
@@ -18,4 +19,26 @@ import { Gemini, GEMINI_MODEL } from "@llamaindex/google";
    ],
  });
  console.log(result);
+
+  // chat with file
+  const resultWithFile = await gemini.chat({
+    messages: [
+      {
+        role: "user",
+        content: [
+          {
+            type: "text",
+            text: "What's in this document? Describe it in detail.",
+          },
+          {
+            type: "file",
+            data: fs.readFileSync("./data/manga.pdf"),
+            mimeType: "application/pdf",
+          },
+        ],
+      },
+    ],
+  });
+
+  console.log(resultWithFile);
 })();
@@ -11,7 +11,6 @@ const workflow = new Workflow<ContextData, string, string>();
 workflow.addStep(
  {
    inputs: [StartEvent<string>],
-    outputs: [StopEvent<string>],
  },
  async (context, startEvent) => {
    const input = startEvent.data;
@@ -1,7 +1,8 @@
 import { OpenAI, OpenAIEmbedding } from "@llamaindex/openai";
+import fs from "fs";

 (async () => {
-  const llm = new OpenAI({ model: "gpt-4.5-preview", temperature: 0.1 });
+  const llm = new OpenAI({ model: "gpt-4o" });

  // complete api
  const response1 = await llm.complete({ prompt: "How are you?" });
@@ -13,7 +14,51 @@ import { OpenAI, OpenAIEmbedding } from "@llamaindex/openai";
  });
  console.log(response2.message.content);

-  // embeddings
+  // chat with file
+  const response3 = await llm.chat({
+    messages: [
+      {
+        role: "user",
+        content: [
+          {
+            type: "text",
+            text: "What's in this document? Describe it in detail.",
+          },
+          {
+            type: "file",
+            data: fs.readFileSync("./data/manga.pdf"),
+            mimeType: "application/pdf",
+          },
+        ],
+      },
+    ],
+  });
+  console.log(response3.message.content);
+
+  // chat with image
+  const response4 = await llm.chat({
+    messages: [
+      {
+        role: "user",
+        content: [
+          {
+            type: "text",
+            text: "What's in this image? Describe it in detail.",
+          },
+          {
+            type: "image_url",
+            image_url: {
+              url: "https://storage.googleapis.com/cloud-samples-data/vision/face/faces.jpeg",
+            },
+          },
+        ],
+      },
+    ],
+  });
+
+  console.log("Single Image Analysis:", response4.message.content);
+
+  // // embeddings
  const embedModel = new OpenAIEmbedding();
  const texts = ["hello", "world"];
  const embeddings = await embedModel.getTextEmbeddingsBatch(texts);
@@ -0,0 +1,35 @@
+import { openaiResponses } from "@llamaindex/openai";
+import fs from "fs";
+
+async function main() {
+  if (!process.env.OPENAI_API_KEY) {
+    throw new Error("Please set the OPENAI_API_KEY environment variable.");
+  }
+
+  const llm = openaiResponses({
+    apiKey: process.env.OPENAI_API_KEY,
+  });
+
+  const result = await llm.chat({
+    messages: [
+      {
+        role: "user",
+        content: [
+          {
+            type: "text",
+            text: "What's in this document? Describe it in detail.",
+          },
+          {
+            type: "file",
+            data: fs.readFileSync("./data/manga.pdf"),
+            mimeType: "application/pdf",
+          },
+        ],
+      },
+    ],
+  });
+
+  console.log(result);
+}
+
+void main().catch(console.error);
@@ -1,6 +1,6 @@
 {
  "name": "@llamaindex/examples",
-  "version": "0.3.12",
+  "version": "0.3.13",
  "private": true,
  "scripts": {
    "lint": "eslint .",
@@ -11,60 +11,60 @@
    "@azure/cosmos": "^4.1.1",
    "@azure/identity": "^4.4.1",
    "@azure/search-documents": "^12.1.0",
-    "@llamaindex/anthropic": "^0.3.3",
-    "@llamaindex/astra": "^0.0.16",
-    "@llamaindex/azure": "^0.1.12",
-    "@llamaindex/chroma": "^0.0.16",
-    "@llamaindex/clip": "^0.0.52",
-    "@llamaindex/cloud": "^4.0.3",
-    "@llamaindex/cohere": "^0.0.16",
-    "@llamaindex/core": "^0.6.2",
-    "@llamaindex/deepinfra": "^0.0.52",
+    "@llamaindex/anthropic": "^0.3.4",
+    "@llamaindex/astra": "^0.0.17",
+    "@llamaindex/azure": "^0.1.13",
+    "@llamaindex/chroma": "^0.0.17",
+    "@llamaindex/clip": "^0.0.53",
+    "@llamaindex/cloud": "^4.0.4",
+    "@llamaindex/cohere": "^0.0.17",
+    "@llamaindex/core": "^0.6.3",
+    "@llamaindex/deepinfra": "^0.0.53",
    "@llamaindex/env": "^0.1.29",
-    "@llamaindex/firestore": "^1.0.9",
-    "@llamaindex/google": "^0.2.4",
-    "@llamaindex/groq": "^0.0.67",
-    "@llamaindex/huggingface": "^0.1.6",
-    "@llamaindex/milvus": "^0.1.11",
-    "@llamaindex/mistral": "^0.1.2",
-    "@llamaindex/mixedbread": "^0.0.16",
-    "@llamaindex/mongodb": "^0.0.17",
-    "@llamaindex/elastic-search": "^0.1.2",
-    "@llamaindex/node-parser": "^2.0.2",
-    "@llamaindex/ollama": "^0.1.2",
-    "@llamaindex/openai": "^0.3.4",
-    "@llamaindex/pinecone": "^0.1.2",
-    "@llamaindex/portkey-ai": "^0.0.44",
-    "@llamaindex/postgres": "^0.0.45",
-    "@llamaindex/qdrant": "^0.1.11",
-    "@llamaindex/readers": "^3.1.0",
-    "@llamaindex/replicate": "^0.0.44",
-    "@llamaindex/upstash": "^0.0.16",
-    "@llamaindex/vercel": "^0.1.2",
-    "@llamaindex/vllm": "^0.0.38",
-    "@llamaindex/voyage-ai": "^1.0.8",
-    "@llamaindex/weaviate": "^0.0.16",
-    "@llamaindex/workflow": "^1.0.3",
-    "@llamaindex/deepseek": "^0.0.12",
-    "@llamaindex/fireworks": "^0.0.12",
-    "@llamaindex/together": "^0.0.12",
-    "@llamaindex/jinaai": "^0.0.12",
-    "@llamaindex/perplexity": "^0.0.9",
+    "@llamaindex/firestore": "^1.0.10",
+    "@llamaindex/google": "^0.2.5",
+    "@llamaindex/groq": "^0.0.68",
+    "@llamaindex/huggingface": "^0.1.7",
+    "@llamaindex/milvus": "^0.1.12",
+    "@llamaindex/mistral": "^0.1.3",
+    "@llamaindex/mixedbread": "^0.0.17",
+    "@llamaindex/mongodb": "^0.0.18",
+    "@llamaindex/elastic-search": "^0.1.3",
+    "@llamaindex/node-parser": "^2.0.3",
+    "@llamaindex/ollama": "^0.1.3",
+    "@llamaindex/openai": "^0.3.5",
+    "@llamaindex/pinecone": "^0.1.3",
+    "@llamaindex/portkey-ai": "^0.0.45",
+    "@llamaindex/postgres": "^0.0.46",
+    "@llamaindex/qdrant": "^0.1.12",
+    "@llamaindex/readers": "^3.1.1",
+    "@llamaindex/replicate": "^0.0.45",
+    "@llamaindex/upstash": "^0.0.17",
+    "@llamaindex/vercel": "^0.1.3",
+    "@llamaindex/vllm": "^0.0.39",
+    "@llamaindex/voyage-ai": "^1.0.9",
+    "@llamaindex/weaviate": "^0.0.17",
+    "@llamaindex/workflow": "^1.0.4",
+    "@llamaindex/deepseek": "^0.0.13",
+    "@llamaindex/fireworks": "^0.0.13",
+    "@llamaindex/together": "^0.0.13",
+    "@llamaindex/jinaai": "^0.0.13",
+    "@llamaindex/perplexity": "^0.0.10",
    "@llamaindex/server": "^0.1.6",
-    "@llamaindex/supabase": "^0.1.1",
-    "@llamaindex/tools": "^0.0.7",
+    "@llamaindex/supabase": "^0.1.2",
+    "@llamaindex/tools": "^0.0.8",
    "@notionhq/client": "^2.2.15",
    "@pinecone-database/pinecone": "^4.0.0",
-    "@llamaindex/assemblyai": "^0.1.1",
-    "@llamaindex/discord": "^0.1.1",
-    "@llamaindex/notion": "^0.1.1",
+    "@llamaindex/assemblyai": "^0.1.2",
+    "@llamaindex/discord": "^0.1.2",
+    "@llamaindex/notion": "^0.1.2",
    "@vercel/postgres": "^0.10.0",
    "ai": "^4.0.0",
    "ajv": "^8.17.1",
    "commander": "^12.1.0",
    "dotenv": "^16.4.5",
    "js-tiktoken": "^1.0.14",
-    "llamaindex": "^0.10.2",
+    "llamaindex": "^0.10.3",
    "mongodb": "6.7.0",
    "postgres": "^3.4.4",
    "wikipedia": "^2.1.2",
@@ -1,5 +1,12 @@
 # @llamaindex/autotool

+## 7.0.3
+
+### Patch Changes
+
+- Updated dependencies [3ee8c83]
+  - llamaindex@0.10.3
+
 ## 7.0.2

 ### Patch Changes
@@ -1,5 +1,13 @@
 # @llamaindex/autotool-01-node-example

+## 0.0.104
+
+### Patch Changes
+
+- Updated dependencies [3ee8c83]
+  - llamaindex@0.10.3
+  - @llamaindex/autotool@7.0.3
+
 ## 0.0.103

 ### Patch Changes
@@ -13,5 +13,5 @@
  "scripts": {
    "start": "node --import tsx --import @llamaindex/autotool/node ./src/index.ts"
  },
-  "version": "0.0.103"
+  "version": "0.0.104"
 }
@@ -6,7 +6,7 @@
    "url": "git+https://github.com/run-llama/LlamaIndexTS.git",
    "directory": "packages/autotool"
  },
-  "version": "7.0.2",
+  "version": "7.0.3",
  "description": "auto transpile your JS function to LLM Agent compatible",
  "files": [
    "dist",
@@ -1,5 +1,12 @@
 # @llamaindex/cloud

+## 4.0.4
+
+### Patch Changes
+
+- Updated dependencies [3ee8c83]
+  - @llamaindex/core@0.6.3
+
 ## 4.0.3

 ### Patch Changes
@@ -1,6 +1,6 @@
 {
  "name": "@llamaindex/cloud",
-  "version": "4.0.3",
+  "version": "4.0.4",
  "type": "module",
  "license": "MIT",
  "scripts": {
@@ -1,5 +1,12 @@
 # @llamaindex/community

+## 0.0.97
+
+### Patch Changes
+
+- Updated dependencies [3ee8c83]
+  - @llamaindex/core@0.6.3
+
 ## 0.0.96

 ### Patch Changes
@@ -1,7 +1,7 @@
 {
  "name": "@llamaindex/community",
  "description": "Community package for LlamaIndexTS",
-  "version": "0.0.96",
+  "version": "0.0.97",
  "type": "module",
  "types": "dist/type/index.d.ts",
  "main": "dist/cjs/index.js",
@@ -1,5 +1,11 @@
 # @llamaindex/core

+## 0.6.3
+
+### Patch Changes
+
+- 3ee8c83: feat: support file content type in message content
+
 ## 0.6.2

 ### Patch Changes
@@ -1,7 +1,7 @@
 {
  "name": "@llamaindex/core",
  "type": "module",
-  "version": "0.6.2",
+  "version": "0.6.3",
  "description": "LlamaIndex Core Module",
  "exports": {
    "./agent": {
@@ -17,6 +17,7 @@ export type {
  LLMMetadata,
  MessageContent,
  MessageContentDetail,
+  MessageContentFileDetail,
  MessageContentImageDetail,
  MessageContentTextDetail,
  MessageType,
@@ -163,9 +163,16 @@ export type MessageContentImageDetail = {
  detail?: "high" | "low" | "auto";
 };

+export type MessageContentFileDetail = {
+  type: "file";
+  data: Buffer;
+  mimeType: string;
+};
+
 export type MessageContentDetail =
  | MessageContentTextDetail
-  | MessageContentImageDetail;
+  | MessageContentImageDetail
+  | MessageContentFileDetail;

 /**
 * Extended type for the content of a message that allows for multi-modal messages.
@@ -1,5 +1,12 @@
 # @llamaindex/experimental

+## 0.0.173
+
+### Patch Changes
+
+- Updated dependencies [3ee8c83]
+  - llamaindex@0.10.3
+
 ## 0.0.172

 ### Patch Changes
@@ -1,7 +1,7 @@
 {
  "name": "@llamaindex/experimental",
  "description": "Experimental package for LlamaIndexTS",
-  "version": "0.0.172",
+  "version": "0.0.173",
  "type": "module",
  "types": "dist/type/index.d.ts",
  "main": "dist/cjs/index.js",
@@ -1,5 +1,17 @@
 # llamaindex

+## 0.10.3
+
+### Patch Changes
+
+- 3ee8c83: feat: support file content type in message content
+- Updated dependencies [3ee8c83]
+  - @llamaindex/core@0.6.3
+  - @llamaindex/openai@0.3.5
+  - @llamaindex/cloud@4.0.4
+  - @llamaindex/node-parser@2.0.3
+  - @llamaindex/workflow@1.0.4
+
 ## 0.10.2

 ### Patch Changes
@@ -1,6 +1,6 @@
 {
  "name": "llamaindex",
-  "version": "0.10.2",
+  "version": "0.10.3",
  "license": "MIT",
  "type": "module",
  "keywords": [
@@ -25,7 +25,7 @@
    "@llamaindex/env": "workspace:*",
    "@llamaindex/node-parser": "workspace:*",
    "@llamaindex/openai": "workspace:*",
-    "@llamaindex/workflow": "workspace:*",
+    "@llamaindex/workflow": "1.0.4",
    "@types/lodash": "^4.17.7",
    "@types/node": "^22.9.0",
    "ajv": "^8.17.1",
@@ -51,7 +51,7 @@ export class QueryEngineTool implements BaseTool<QueryEngineParam> {
    const response = await this.queryEngine.query({ query });

    if (!this.includeSourceNodes) {
-      return { content: response.message.content };
+      return { content: response.message.content } as unknown as JSONValue;
    }

    return {
@@ -1,5 +1,12 @@
 # @llamaindex/node-parser

+## 2.0.3
+
+### Patch Changes
+
+- Updated dependencies [3ee8c83]
+  - @llamaindex/core@0.6.3
+
 ## 2.0.2

 ### Patch Changes
@@ -1,6 +1,6 @@
 {
  "name": "@llamaindex/node-parser",
-  "version": "2.0.2",
+  "version": "2.0.3",
  "description": "Node parser for LlamaIndex",
  "type": "module",
  "exports": {
@@ -1,5 +1,13 @@
 # @llamaindex/anthropic

+## 0.3.4
+
+### Patch Changes
+
+- 3ee8c83: feat: support file content type in message content
+- Updated dependencies [3ee8c83]
+  - @llamaindex/core@0.6.3
+
 ## 0.3.3

 ### Patch Changes
@@ -1,7 +1,7 @@
 {
  "name": "@llamaindex/anthropic",
  "description": "Anthropic Adapter for LlamaIndex",
-  "version": "0.3.3",
+  "version": "0.3.4",
  "type": "module",
  "main": "./dist/index.cjs",
  "module": "./dist/index.js",
@@ -319,6 +319,24 @@ export class Anthropic extends ToolCallLLM<
              text: content.text,
            };
          }
+
+          if (content.type === "file") {
+            if (content.mimeType !== "application/pdf") {
+              throw new Error(
+                "Only supports mimeType `application/pdf` for file content.",
+              );
+            }
+
+            return {
+              type: "document" as const,
+              source: {
+                type: "base64" as const,
+                media_type: content.mimeType,
+                data: content.data.toString("base64"),
+              },
+            };
+          }
+
          return {
            type: "image" as const,
            source: {
@@ -164,6 +164,67 @@ describe("Message Formatting", () => {

      expect(anthropic.formatMessages(inputMessages)).toEqual(expectedOutput);
    });
+
+    test("Anthropic handles PDF file content", () => {
+      const anthropic = new Anthropic();
+      const pdfBuffer = Buffer.from("test PDF content");
+      const inputMessages: ChatMessage[] = [
+        {
+          content: [
+            {
+              type: "text",
+              text: "Here's a PDF document:",
+            },
+            {
+              type: "file",
+              mimeType: "application/pdf",
+              data: pdfBuffer,
+            },
+          ],
+          role: "user",
+        },
+      ];
+      const expectedOutput: MessageParam[] = [
+        {
+          role: "user",
+          content: [
+            {
+              type: "text",
+              text: "Here's a PDF document:",
+            },
+            {
+              type: "document",
+              source: {
+                type: "base64",
+                media_type: "application/pdf",
+                data: pdfBuffer.toString("base64"),
+              },
+            },
+          ],
+        },
+      ];
+
+      expect(anthropic.formatMessages(inputMessages)).toEqual(expectedOutput);
+    });
+
+    test("Anthropic throws error for non-PDF files", () => {
+      const anthropic = new Anthropic();
+      const docxBuffer = Buffer.from("fake docx content");
+      const inputMessages: ChatMessage[] = [
+        {
+          content: [
+            {
+              type: "file",
+              mimeType: "application/docx",
+              data: docxBuffer,
+            },
+          ],
+          role: "user",
+        },
+      ];
+
+      expect(() => anthropic.formatMessages(inputMessages)).toThrowError();
+    });
  });

  describe("Tool Message Formatting", () => {
@@ -1,5 +1,12 @@
 # @llamaindex/assemblyai

+## 0.1.2
+
+### Patch Changes
+
+- Updated dependencies [3ee8c83]
+  - @llamaindex/core@0.6.3
+
 ## 0.1.1

 ### Patch Changes
@@ -1,7 +1,7 @@
 {
  "name": "@llamaindex/assemblyai",
  "description": "AssemblyAI Reader for LlamaIndex",
-  "version": "0.1.1",
+  "version": "0.1.2",
  "type": "module",
  "types": "dist/index.d.ts",
  "main": "dist/index.cjs",
@@ -1,5 +1,13 @@
 # @llamaindex/clip

+## 0.0.53
+
+### Patch Changes
+
+- Updated dependencies [3ee8c83]
+  - @llamaindex/core@0.6.3
+  - @llamaindex/openai@0.3.5
+
 ## 0.0.52

 ### Patch Changes
@@ -1,7 +1,7 @@
 {
  "name": "@llamaindex/clip",
  "description": "Clip Embedding Adapter for LlamaIndex",
-  "version": "0.0.52",
+  "version": "0.0.53",
  "type": "module",
  "types": "dist/index.d.ts",
  "main": "dist/index.cjs",
@@ -1,5 +1,12 @@
 # @llamaindex/cohere

+## 0.0.17
+
+### Patch Changes
+
+- Updated dependencies [3ee8c83]
+  - @llamaindex/core@0.6.3
+
 ## 0.0.16

 ### Patch Changes
@@ -1,7 +1,7 @@
 {
  "name": "@llamaindex/cohere",
  "description": "Cohere Adapter for LlamaIndex",
-  "version": "0.0.16",
+  "version": "0.0.17",
  "type": "module",
  "main": "./dist/index.cjs",
  "module": "./dist/index.js",
@@ -1,5 +1,13 @@
 # @llamaindex/deepinfra

+## 0.0.53
+
+### Patch Changes
+
+- Updated dependencies [3ee8c83]
+  - @llamaindex/core@0.6.3
+  - @llamaindex/openai@0.3.5
+
 ## 0.0.52

 ### Patch Changes
@@ -1,7 +1,7 @@
 {
  "name": "@llamaindex/deepinfra",
  "description": "Deepinfra Adapter for LlamaIndex",
-  "version": "0.0.52",
+  "version": "0.0.53",
  "type": "module",
  "main": "./dist/index.cjs",
  "module": "./dist/index.js",
@@ -1,5 +1,12 @@
 # @llamaindex/deepseek

+## 0.0.13
+
+### Patch Changes
+
+- Updated dependencies [3ee8c83]
+  - @llamaindex/openai@0.3.5
+
 ## 0.0.12

 ### Patch Changes
@@ -1,7 +1,7 @@
 {
  "name": "@llamaindex/deepseek",
  "description": "DeepSeek Adapter for LlamaIndex",
-  "version": "0.0.12",
+  "version": "0.0.13",
  "type": "module",
  "main": "./dist/index.cjs",
  "module": "./dist/index.js",
@@ -1,5 +1,12 @@
 # @llamaindex/discord

+## 0.1.2
+
+### Patch Changes
+
+- Updated dependencies [3ee8c83]
+  - @llamaindex/core@0.6.3
+
 ## 0.1.1

 ### Patch Changes
@@ -1,7 +1,7 @@
 {
  "name": "@llamaindex/discord",
  "description": "Discord Reader for LlamaIndex",
-  "version": "0.1.1",
+  "version": "0.1.2",
  "type": "module",
  "types": "dist/index.d.ts",
  "main": "dist/index.cjs",
@@ -1,5 +1,12 @@
 # @llamaindex/fireworks

+## 0.0.13
+
+### Patch Changes
+
+- Updated dependencies [3ee8c83]
+  - @llamaindex/openai@0.3.5
+
 ## 0.0.12

 ### Patch Changes
@@ -1,7 +1,7 @@
 {
  "name": "@llamaindex/fireworks",
  "description": "Fireworks Adapter for LlamaIndex",
-  "version": "0.0.12",
+  "version": "0.0.13",
  "type": "module",
  "main": "./dist/index.cjs",
  "module": "./dist/index.js",
@@ -1,5 +1,13 @@
 # @llamaindex/google

+## 0.2.5
+
+### Patch Changes
+
+- 3ee8c83: feat: support file content type in message content
+- Updated dependencies [3ee8c83]
+  - @llamaindex/core@0.6.3
+
 ## 0.2.4

 ### Patch Changes
@@ -1,7 +1,7 @@
 {
  "name": "@llamaindex/google",
  "description": "Google Adapter for LlamaIndex",
-  "version": "0.2.4",
+  "version": "0.2.5",
  "type": "module",
  "main": "./dist/index.cjs",
  "module": "./dist/index.js",
@@ -32,7 +32,7 @@
  "dependencies": {
    "@google-cloud/vertexai": "1.9.0",
    "@google/genai": "^0.4.0",
-    "@google/generative-ai": "0.21.0",
+    "@google/generative-ai": "0.24.0",
    "@llamaindex/core": "workspace:*",
    "@llamaindex/env": "workspace:*"
  }
@@ -5,10 +5,12 @@ import {
  type FunctionCall,
  type ModelParams as GoogleModelParams,
  type RequestOptions as GoogleRequestOptions,
+  type StartChatParams as GoogleStartChatParams,
  type GenerateContentStreamResult as GoogleStreamGenerateContentResult,
  type SafetySetting,
 } from "@google/generative-ai";

+import type { StartChatParams as VertexStartChatParams } from "@google-cloud/vertexai";
 import { wrapLLMEvent } from "@llamaindex/core/decorator";
 import type {
  CompletionResponse,
@@ -96,6 +98,8 @@ export type GeminiConfig = Partial<typeof DEFAULT_GEMINI_PARAMS> & {
  safetySettings?: SafetySetting[];
 };

+type StartChatParams = GoogleStartChatParams & VertexStartChatParams;
+
 /**
 * Gemini Session to manage the connection to the Gemini API
 */
@@ -254,13 +258,13 @@ export class Gemini extends ToolCallLLM<GeminiAdditionalChatOptions> {
    };
  }

-  private createStartChatParams(
+  private async createStartChatParams(
    params: GeminiChatParamsNonStreaming | GeminiChatParamsStreaming,
  ) {
-    const context = getChatContext(params);
+    const context = await getChatContext(params);
    const common = {
      history: context.history,
-      safetySettings: this.safetySettings,
+      safetySettings: this.safetySettings as SafetySetting[],
    };

    return params.tools?.length
@@ -282,12 +286,14 @@ export class Gemini extends ToolCallLLM<GeminiAdditionalChatOptions> {
  protected async nonStreamChat(
    params: GeminiChatParamsNonStreaming,
  ): Promise<GeminiChatNonStreamResponse> {
-    const context = getChatContext(params);
+    const context = await getChatContext(params);
    const client = this.session.getGenerativeModel(
      this.metadata,
      this.#requestOptions,
    );
-    const chat = client.startChat(this.createStartChatParams(params));
+    const chat = client.startChat(
+      (await this.createStartChatParams(params)) as StartChatParams,
+    );
    const { response } = await chat.sendMessage(context.message);
    const topCandidate = response.candidates![0]!;

@@ -311,12 +317,14 @@ export class Gemini extends ToolCallLLM<GeminiAdditionalChatOptions> {
  protected async *streamChat(
    params: GeminiChatParamsStreaming,
  ): GeminiChatStreamResponse {
-    const context = getChatContext(params);
+    const context = await getChatContext(params);
    const client = this.session.getGenerativeModel(
      this.metadata,
      this.#requestOptions,
    );
-    const chat = client.startChat(this.createStartChatParams(params));
+    const chat = client.startChat(
+      (await this.createStartChatParams(params)) as StartChatParams,
+    );
    const result = await chat.sendMessageStream(context.message);
    yield* this.session.getChatStream(result);
  }
@@ -350,7 +358,7 @@ export class Gemini extends ToolCallLLM<GeminiAdditionalChatOptions> {
    if (stream) {
      const result = await client.generateContentStream(
        getPartsText(
-          GeminiHelper.messageContentToGeminiParts({ content: prompt }),
+          await GeminiHelper.messageContentToGeminiParts({ content: prompt }),
        ),
      );
      return this.session.getCompletionStream(result);
@@ -358,7 +366,7 @@ export class Gemini extends ToolCallLLM<GeminiAdditionalChatOptions> {

    const result = await client.generateContent(
      getPartsText(
-        GeminiHelper.messageContentToGeminiParts({ content: prompt }),
+        await GeminiHelper.messageContentToGeminiParts({ content: prompt }),
      ),
    );
    return {
@@ -68,8 +68,8 @@ export enum GEMINI_MODEL {
  GEMINI_PRO_1_5_LATEST = "gemini-1.5-pro-002",
  GEMINI_PRO_1_5_FLASH_LATEST = "gemini-1.5-flash-002",
  GEMINI_2_0_FLASH_EXPERIMENTAL = "gemini-2.0-flash-exp",
-  GEMINI_2_0_FLASH = "gemini-2.0-flash-001",
-  GEMINI_2_0_FLASH_LITE = "gemini-2.0-flash-lite-001",
+  GEMINI_2_0_FLASH = "gemini-2.0-flash",
+  GEMINI_2_0_FLASH_LITE = "gemini-2.0-flash-lite",
  GEMINI_2_0_FLASH_LITE_PREVIEW = "gemini-2.0-flash-lite-preview-02-05",
  GEMINI_2_0_FLASH_THINKING_EXP = "gemini-2.0-flash-thinking-exp-01-21",
  GEMINI_2_0_PRO_EXPERIMENTAL = "gemini-2.0-pro-exp-02-05",
@@ -8,15 +8,18 @@ import {
 } from "@google/generative-ai";

 import { type GenerateContentResponse } from "@google-cloud/vertexai";
+import { FileState, GoogleAIFileManager } from "@google/generative-ai/server";
 import type {
  BaseTool,
  ChatMessage,
+  MessageContentFileDetail,
  MessageContentImageDetail,
  MessageContentTextDetail,
  MessageType,
  ToolCallLLMMessageOptions,
 } from "@llamaindex/core/llms";
 import { extractDataUrlComponents } from "@llamaindex/core/utils";
+import { getEnv } from "@llamaindex/env";
 import type {
  ChatContext,
  FileDataPart,
@@ -126,9 +129,9 @@ export const cleanParts = (
  };
 };

-export const getChatContext = (
+export const getChatContext = async (
  params: GeminiChatParamsStreaming | GeminiChatParamsNonStreaming,
-): ChatContext => {
+): Promise<ChatContext> => {
  // Gemini doesn't allow:
  // 1. Consecutive messages from the same role
  // 2. Parts that have empty text
@@ -145,8 +148,10 @@ export const getChatContext = (
    {} as Record<string, string>,
  );
  const messages = GeminiHelper.mergeNeighboringSameRoleMessages(
-    params.messages.map((message) =>
-      GeminiHelper.chatMessageToGemini(message, fnMap),
+    await Promise.all(
+      params.messages.map((message) =>
+        GeminiHelper.chatMessageToGemini(message, fnMap),
+      ),
    ),
  ).map(cleanParts);

@@ -226,13 +231,13 @@ export class GeminiHelper {
      );
  }

-  public static messageContentToGeminiParts({
+  public static async messageContentToGeminiParts({
    content,
    options = undefined,
    fnMap = undefined,
  }: Pick<ChatMessage<ToolCallLLMMessageOptions>, "content" | "options"> & {
    fnMap?: Record<string, string>;
-  }): Part[] {
+  }): Promise<Part[]> {
    if (options && "toolResult" in options) {
      if (!fnMap) throw Error("fnMap must be set");
      const name = fnMap[options.toolResult.id];
@@ -276,9 +281,53 @@ export class GeminiHelper {
      (i) => i.type === "text",
    ) as MessageContentTextDetail[];
    parts.push(...textContents.map((t) => ({ text: t.text })));
+
+    const fileContents = content.filter(
+      (i) => i.type === "file",
+    ) as MessageContentFileDetail[];
+
+    if (fileContents.length > 0) {
+      for (const file of fileContents) {
+        const uploadResponse = await GeminiHelper.uploadFile(
+          file.data,
+          file.mimeType,
+        );
+        parts.push({
+          fileData: {
+            mimeType: uploadResponse.file.mimeType,
+            fileUri: uploadResponse.file.uri,
+          },
+        });
+      }
+    }
+
    return parts;
  }

+  // Upload a file for AI processing
+  public static async uploadFile(
+    data: string | Buffer, // file name or buffer
+    mimeType: string, // eg. application/pdf
+    interval = 5_000, // time to refetch upload status
+  ) {
+    const fileManager = new GoogleAIFileManager(getEnv("GOOGLE_API_KEY")!);
+
+    const uploadResponse = await fileManager.uploadFile(data, { mimeType });
+
+    let file = await fileManager.getFile(uploadResponse.file.name);
+
+    while (file.state === FileState.PROCESSING) {
+      await new Promise((resolve) => setTimeout(resolve, interval));
+      file = await fileManager.getFile(uploadResponse.file.name);
+    }
+
+    if (file.state === FileState.FAILED) {
+      throw new Error("Failed to upload file");
+    }
+
+    return uploadResponse;
+  }
+
  public static getGeminiMessageRole(
    message: ChatMessage<ToolCallLLMMessageOptions>,
  ): GeminiMessageRole {
@@ -290,13 +339,16 @@ export class GeminiHelper {
    ];
  }

-  public static chatMessageToGemini(
+  public static async chatMessageToGemini(
    message: ChatMessage<ToolCallLLMMessageOptions>,
    fnMap: Record<string, string>, // mapping of fn call id to fn call name
-  ): GeminiMessageContent {
+  ): Promise<GeminiMessageContent> {
    return {
      role: GeminiHelper.getGeminiMessageRole(message),
-      parts: GeminiHelper.messageContentToGeminiParts({ ...message, fnMap }),
+      parts: await GeminiHelper.messageContentToGeminiParts({
+        ...message,
+        fnMap,
+      }),
    };
  }
 }
@@ -1,5 +1,6 @@
 import {
  type GenerateContentResponse,
+  type SafetySetting,
  VertexAI,
  GenerativeModel as VertexGenerativeModel,
  GenerativeModelPreview as VertexGenerativeModelPreview,
@@ -62,12 +63,12 @@ export class GeminiVertexSession implements IGeminiSession {
    const safetySettings = metadata.safetySettings ?? DEFAULT_SAFETY_SETTINGS;
    if (this.preview) {
      return this.vertex.preview.getGenerativeModel({
-        safetySettings,
+        safetySettings: safetySettings as SafetySetting[],
        ...metadata,
      });
    }
    return this.vertex.getGenerativeModel({
-      safetySettings,
+      safetySettings: safetySettings as SafetySetting[],
      ...metadata,
    });
  }
@@ -1,5 +1,12 @@
 # @llamaindex/groq

+## 0.0.68
+
+### Patch Changes
+
+- Updated dependencies [3ee8c83]
+  - @llamaindex/openai@0.3.5
+
 ## 0.0.67

 ### Patch Changes
@@ -1,7 +1,7 @@
 {
  "name": "@llamaindex/groq",
  "description": "Groq Adapter for LlamaIndex",
-  "version": "0.0.67",
+  "version": "0.0.68",
  "type": "module",
  "main": "./dist/index.cjs",
  "module": "./dist/index.js",
@@ -1,5 +1,13 @@
 # @llamaindex/huggingface

+## 0.1.7
+
+### Patch Changes
+
+- Updated dependencies [3ee8c83]
+  - @llamaindex/core@0.6.3
+  - @llamaindex/openai@0.3.5
+
 ## 0.1.6

 ### Patch Changes
@@ -1,7 +1,7 @@
 {
  "name": "@llamaindex/huggingface",
  "description": "Huggingface Adapter for LlamaIndex",
-  "version": "0.1.6",
+  "version": "0.1.7",
  "type": "module",
  "types": "dist/index.d.ts",
  "main": "dist/index.cjs",
@@ -1,5 +1,13 @@
 # @llamaindex/jinaai

+## 0.0.13
+
+### Patch Changes
+
+- Updated dependencies [3ee8c83]
+  - @llamaindex/core@0.6.3
+  - @llamaindex/openai@0.3.5
+
 ## 0.0.12

 ### Patch Changes
@@ -1,7 +1,7 @@
 {
  "name": "@llamaindex/jinaai",
  "description": "JinaAI Adapter for LlamaIndex",
-  "version": "0.0.12",
+  "version": "0.0.13",
  "type": "module",
  "main": "./dist/index.cjs",
  "module": "./dist/index.js",
@@ -1,5 +1,12 @@
 # @llamaindex/mistral

+## 0.1.3
+
+### Patch Changes
+
+- Updated dependencies [3ee8c83]
+  - @llamaindex/core@0.6.3
+
 ## 0.1.2

 ### Patch Changes
@@ -1,7 +1,7 @@
 {
  "name": "@llamaindex/mistral",
  "description": "Mistral Adapter for LlamaIndex",
-  "version": "0.1.2",
+  "version": "0.1.3",
  "type": "module",
  "main": "./dist/index.cjs",
  "module": "./dist/index.js",
@@ -1,5 +1,12 @@
 # @llamaindex/mixedbread

+## 0.0.17
+
+### Patch Changes
+
+- Updated dependencies [3ee8c83]
+  - @llamaindex/core@0.6.3
+
 ## 0.0.16

 ### Patch Changes
@@ -1,7 +1,7 @@
 {
  "name": "@llamaindex/mixedbread",
  "description": "Mixedbread Adapter for LlamaIndex",
-  "version": "0.0.16",
+  "version": "0.0.17",
  "type": "module",
  "main": "./dist/index.cjs",
  "module": "./dist/index.js",
@@ -1,5 +1,12 @@
 # @llamaindex/notion

+## 0.1.2
+
+### Patch Changes
+
+- Updated dependencies [3ee8c83]
+  - @llamaindex/core@0.6.3
+
 ## 0.1.1

 ### Patch Changes
@@ -1,7 +1,7 @@
 {
  "name": "@llamaindex/notion",
  "description": "Notion Reader for LlamaIndex",
-  "version": "0.1.1",
+  "version": "0.1.2",
  "type": "module",
  "types": "dist/index.d.ts",
  "main": "dist/index.cjs",
@@ -1,5 +1,12 @@
 # @llamaindex/ollama

+## 0.1.3
+
+### Patch Changes
+
+- Updated dependencies [3ee8c83]
+  - @llamaindex/core@0.6.3
+
 ## 0.1.2

 ### Patch Changes
@@ -1,7 +1,7 @@
 {
  "name": "@llamaindex/ollama",
  "description": "Ollama Adapter for LlamaIndex",
-  "version": "0.1.2",
+  "version": "0.1.3",
  "type": "module",
  "main": "./dist/index.cjs",
  "module": "./dist/index.js",
@@ -1,5 +1,13 @@
 # @llamaindex/openai

+## 0.3.5
+
+### Patch Changes
+
+- 3ee8c83: feat: support file content type in message content
+- Updated dependencies [3ee8c83]
+  - @llamaindex/core@0.6.3
+
 ## 0.3.4

 ### Patch Changes
@@ -1,7 +1,7 @@
 {
  "name": "@llamaindex/openai",
  "description": "OpenAI Adapter for LlamaIndex",
-  "version": "0.3.4",
+  "version": "0.3.5",
  "type": "module",
  "main": "./dist/index.cjs",
  "module": "./dist/index.js",
@@ -1,5 +1,6 @@
 import { wrapEventCaller, wrapLLMEvent } from "@llamaindex/core/decorator";
 import {
+  ToolCallLLM,
  type BaseTool,
  type ChatMessage,
  type ChatResponse,
@@ -9,7 +10,6 @@ import {
  type LLMMetadata,
  type MessageType,
  type PartialToolCall,
-  ToolCallLLM,
  type ToolCallLLMMessageOptions,
 } from "@llamaindex/core/llms";
 import { extractText } from "@llamaindex/core/utils";
@@ -24,6 +24,7 @@ import { zodResponseFormat } from "openai/helpers/zod";
 import type { ChatModel } from "openai/resources/chat/chat";
 import type {
  ChatCompletionAssistantMessageParam,
+  ChatCompletionContentPart,
  ChatCompletionMessageToolCall,
  ChatCompletionRole,
  ChatCompletionSystemMessageParam,
@@ -205,9 +206,29 @@ export class OpenAI extends ToolCallLLM<OpenAIAdditionalChatOptions> {
          }),
        } satisfies ChatCompletionAssistantMessageParam;
      } else if (message.role === "user") {
+        if (typeof message.content === "string") {
+          return { role: "user", content: message.content };
+        }
+
        return {
          role: "user",
-          content: message.content,
+          content: message.content.map((item, index) => {
+            if (item.type === "file") {
+              if (item.mimeType !== "application/pdf") {
+                throw new Error("Only PDF files are supported");
+              }
+              return {
+                type: "file",
+                file: {
+                  file_data: `data:${item.mimeType};base64,${item.data.toString("base64")}`,
+                  filename: `part-${index}.pdf`,
+                },
+              } satisfies ChatCompletionContentPart.File;
+            }
+
+            // keep it as is for other types
+            return item;
+          }),
        } satisfies ChatCompletionUserMessageParam;
      }

@@ -682,7 +682,7 @@ export class OpenAIResponses extends ToolCallLLM<OpenAIResponsesChatOptions> {
      return content;
    }

-    return content.map((item) => {
+    return content.map((item, index) => {
      if (item.type === "text") {
        return {
          type: "input_text",
@@ -696,6 +696,19 @@ export class OpenAIResponses extends ToolCallLLM<OpenAIResponsesChatOptions> {
          detail: item.detail || "auto",
        };
      }
+      if (item.type === "file") {
+        if (item.mimeType !== "application/pdf") {
+          throw new Error(
+            "Only supports mimeType `application/pdf` for file content.",
+          );
+        }
+
+        return {
+          type: "input_file",
+          filename: `part-${index}.pdf`,
+          file_data: `data:${item.mimeType};base64,${item.data.toString("base64")}`,
+        };
+      }
      throw new Error("Unsupported content type");
    });
  }
@@ -227,9 +227,17 @@ export type ResponsesMessageContentImageDetail = {
  image_url: string;
  detail: "high" | "low" | "auto";
 };
+
+export type ResponsesMessageContentFileDetail = {
+  type: "input_file";
+  filename: string;
+  file_data: string;
+};
+
 export type ResponsesMessageContentDetail =
  | ResponsesMessageContentTextDetail
-  | ResponsesMessageContentImageDetail;
+  | ResponsesMessageContentImageDetail
+  | ResponsesMessageContentFileDetail;

 export type ResponseMessageContent = string | ResponsesMessageContentDetail[];

@@ -1,3 +1,4 @@
+import { ChatMessage, ToolCallLLMMessageOptions } from "@llamaindex/core/llms";
 import { describe, expect, it } from "vitest";
 import { z } from "zod";
 import { OpenAI } from "../src/llm";
@@ -44,3 +45,189 @@ describe("OpenAI Chat Tests", () => {
    });
  });
 });
+
+describe("OpenAI Static Methods", () => {
+  describe("toOpenAIMessage", () => {
+    it("should convert simple text messages", () => {
+      const messages: ChatMessage<ToolCallLLMMessageOptions>[] = [
+        {
+          role: "user",
+          content: "Hello world",
+        },
+        {
+          role: "assistant",
+          content: "Hi there",
+        },
+        {
+          role: "system",
+          content: "You are a helpful assistant",
+        },
+      ];
+
+      const result = OpenAI.toOpenAIMessage(messages);
+      expect(result).toEqual([
+        {
+          role: "user",
+          content: "Hello world",
+        },
+        {
+          role: "assistant",
+          content: "Hi there",
+        },
+        {
+          role: "system",
+          content: "You are a helpful assistant",
+        },
+      ]);
+    });
+
+    it("should convert tool result messages", () => {
+      const messages: ChatMessage<ToolCallLLMMessageOptions>[] = [
+        {
+          role: "assistant",
+          content: "Weather result",
+          options: {
+            toolResult: {
+              id: "weather-123",
+            },
+          },
+        },
+      ];
+
+      const result = OpenAI.toOpenAIMessage(messages);
+      expect(result).toEqual([
+        {
+          role: "tool",
+          content: "Weather result",
+          tool_call_id: "weather-123",
+        },
+      ]);
+    });
+
+    it("should convert tool call messages", () => {
+      const messages: ChatMessage<ToolCallLLMMessageOptions>[] = [
+        {
+          role: "assistant",
+          content: "Let me check the weather",
+          options: {
+            toolCall: [
+              {
+                id: "weather-123",
+                name: "get_weather",
+                input: { location: "London" },
+              },
+            ],
+          },
+        },
+      ];
+
+      const result = OpenAI.toOpenAIMessage(messages);
+      expect(result).toEqual([
+        {
+          role: "assistant",
+          content: "Let me check the weather",
+          tool_calls: [
+            {
+              id: "weather-123",
+              type: "function",
+              function: {
+                name: "get_weather",
+                arguments: JSON.stringify({ location: "London" }),
+              },
+            },
+          ],
+        },
+      ]);
+    });
+
+    it("should convert user messages with file content", () => {
+      const pdfBuffer = Buffer.from("test PDF content");
+      const messages: ChatMessage<ToolCallLLMMessageOptions>[] = [
+        {
+          role: "user",
+          content: [
+            {
+              type: "file",
+              mimeType: "application/pdf",
+              data: pdfBuffer,
+            },
+          ],
+        },
+      ];
+
+      const result = OpenAI.toOpenAIMessage(messages);
+      expect(result).toEqual([
+        {
+          role: "user",
+          content: [
+            {
+              type: "file",
+              file: {
+                file_data: `data:application/pdf;base64,${pdfBuffer.toString("base64")}`,
+                filename: "part-0.pdf",
+              },
+            },
+          ],
+        },
+      ]);
+    });
+
+    it("should convert user messages with mixed content", () => {
+      const pdfBuffer = Buffer.from("test PDF content");
+      const messages: ChatMessage<ToolCallLLMMessageOptions>[] = [
+        {
+          role: "user",
+          content: [
+            {
+              type: "text",
+              text: "Here's a PDF file:",
+            },
+            {
+              type: "file",
+              mimeType: "application/pdf",
+              data: pdfBuffer,
+            },
+          ],
+        },
+      ];
+
+      const result = OpenAI.toOpenAIMessage(messages);
+      expect(result).toEqual([
+        {
+          role: "user",
+          content: [
+            {
+              type: "text",
+              text: "Here's a PDF file:",
+            },
+            {
+              type: "file",
+              file: {
+                file_data: `data:application/pdf;base64,${pdfBuffer.toString("base64")}`,
+                filename: "part-1.pdf",
+              },
+            },
+          ],
+        },
+      ]);
+    });
+
+    it("should throw error for non-PDF files", () => {
+      const fileBuffer = Buffer.from("fake file content");
+      const messages: ChatMessage<ToolCallLLMMessageOptions>[] = [
+        {
+          role: "user",
+          content: [
+            {
+              type: "file",
+              mimeType: "text/csv",
+              data: fileBuffer,
+            },
+          ],
+        },
+      ];
+
+      expect(() => OpenAI.toOpenAIMessage(messages)).toThrowError();
+    });
+  });
+});
@@ -181,6 +181,36 @@ describe("OpenAIResponses Unit Tests", () => {
        },
      ]);
    });
+
+    it("should process file content with PDF type", () => {
+      const pdfBuffer = Buffer.from("test PDF content");
+      const content = [
+        {
+          type: "file",
+          mimeType: "application/pdf",
+          data: pdfBuffer,
+        },
+      ];
+      // @ts-expect-error accessing private method
+      const result = llm.processMessageContent(content);
+      expect(result[0]).toEqual({
+        type: "input_file",
+        filename: "part-0.pdf",
+        file_data: `data:application/pdf;base64,${pdfBuffer.toString("base64")}`,
+      });
+    });
+
+    it("should throw error for non-PDF file types", () => {
+      const content = [
+        {
+          type: "file",
+          mimeType: "image/jpeg",
+          data: Buffer.from("test image content"),
+        },
+      ];
+      // @ts-expect-error accessing private method
+      expect(() => llm.processMessageContent(content)).toThrowError();
+    });
  });

  describe("isResponseCreatedEvent", () => {
@@ -1,5 +1,13 @@
 # @llamaindex/perplexity

+## 0.0.10
+
+### Patch Changes
+
+- Updated dependencies [3ee8c83]
+  - @llamaindex/core@0.6.3
+  - @llamaindex/openai@0.3.5
+
 ## 0.0.9

 ### Patch Changes
@@ -1,7 +1,7 @@
 {
  "name": "@llamaindex/perplexity",
  "description": "Perplexity Adapter for LlamaIndex",
-  "version": "0.0.9",
+  "version": "0.0.10",
  "type": "module",
  "main": "./dist/index.cjs",
  "module": "./dist/index.js",
@@ -1,5 +1,12 @@
 # @llamaindex/portkey-ai

+## 0.0.45
+
+### Patch Changes
+
+- Updated dependencies [3ee8c83]
+  - @llamaindex/core@0.6.3
+
 ## 0.0.44

 ### Patch Changes
@@ -1,7 +1,7 @@
 {
  "name": "@llamaindex/portkey-ai",
  "description": "Portkey Adapter for LlamaIndex",
-  "version": "0.0.44",
+  "version": "0.0.45",
  "type": "module",
  "main": "./dist/index.cjs",
  "module": "./dist/index.js",
@@ -1,5 +1,12 @@
 # @llamaindex/replicate

+## 0.0.45
+
+### Patch Changes
+
+- Updated dependencies [3ee8c83]
+  - @llamaindex/core@0.6.3
+
 ## 0.0.44

 ### Patch Changes
@@ -1,7 +1,7 @@
 {
  "name": "@llamaindex/replicate",
  "description": "Replicate Adapter for LlamaIndex",
-  "version": "0.0.44",
+  "version": "0.0.45",
  "type": "module",
  "main": "./dist/index.cjs",
  "module": "./dist/index.js",
@@ -1,5 +1,12 @@
 # @llamaindex/astra

+## 0.0.17
+
+### Patch Changes
+
+- Updated dependencies [3ee8c83]
+  - @llamaindex/core@0.6.3
+
 ## 0.0.16

 ### Patch Changes
@@ -1,7 +1,7 @@
 {
  "name": "@llamaindex/astra",
  "description": "Astra Storage for LlamaIndex",
-  "version": "0.0.16",
+  "version": "0.0.17",
  "type": "module",
  "main": "./dist/index.cjs",
  "module": "./dist/index.js",
@@ -1,5 +1,12 @@
 # @llamaindex/azure

+## 0.1.13
+
+### Patch Changes
+
+- Updated dependencies [3ee8c83]
+  - @llamaindex/core@0.6.3
+
 ## 0.1.12

 ### Patch Changes
@@ -1,7 +1,7 @@
 {
  "name": "@llamaindex/azure",
  "description": "Azure Storage for LlamaIndex",
-  "version": "0.1.12",
+  "version": "0.1.13",
  "type": "module",
  "main": "./dist/index.cjs",
  "module": "./dist/index.js",
--- a/Show More
+++ b/Show More
Author	SHA1	Message	Date
github-actions[bot]	fa66c9ca8e	Release 0.10.3 (#1898 ) Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: marcusschiesser <17126+marcusschiesser@users.noreply.github.com>	2025-04-29 13:05:36 +07:00
Thuc Pham	3ee8c83200	feat: support file content type in message content (#1894 )	2025-04-29 12:57:35 +07:00
Peter Goldstein	e919bab568	Update Gemini Flash and Gemini Flash Lite model keys to exclude patch version (#1897 )	2025-04-29 11:25:01 +07:00
Thuc Pham	d28b6b7c4f	chore: move server package code to create-llama (#1893 )	2025-04-28 14:39:47 +07:00
Marcus Schiesser	1c7a262ff7	chore: stop workflow update (#1892 )	2025-04-28 11:46:06 +07:00
Alex Yang	5a1838cc91	fix: remove workflow streaming demo (#1891 )	2025-04-24 15:44:55 -07:00
Alex Yang	b9805f4899	fix: migrate to llamaflow (#1889 )	2025-04-24 15:17:02 -07:00