Release 0.10.5 (#1922 )

Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: marcusschiesser <17126+marcusschiesser@users.noreply.github.com>
feat: Support for google live api (#1905 )
2026-07-02 20:13:52 -04:00 · 2025-05-09 14:30:39 +07:00 · 2025-05-09 14:20:40 +07:00 · 2025-05-08 13:19:59 +07:00
150 changed files with 1758 additions and 247 deletions
@@ -1,5 +1,18 @@
 # @llamaindex/doc

+## 0.2.17
+
+### Patch Changes
+
+- Updated dependencies [9b2e25a]
+  - @llamaindex/openai@0.3.6
+  - @llamaindex/core@0.6.4
+  - llamaindex@0.10.5
+  - @llamaindex/cloud@4.0.6
+  - @llamaindex/node-parser@2.0.4
+  - @llamaindex/readers@3.1.2
+  - @llamaindex/workflow@1.1.1
+
 ## 0.2.16

 ### Patch Changes
@@ -1,6 +1,6 @@
 {
  "name": "@llamaindex/doc",
-  "version": "0.2.16",
+  "version": "0.2.17",
  "private": true,
  "scripts": {
    "postinstall": "fumadocs-mdx",
@@ -1,5 +1,11 @@
 # @llamaindex/cloudflare-worker-agent-test

+## 0.0.159
+
+### Patch Changes
+
+- llamaindex@0.10.5
+
 ## 0.0.158

 ### Patch Changes
@@ -1,6 +1,6 @@
 {
  "name": "@llamaindex/cloudflare-worker-agent-test",
-  "version": "0.0.158",
+  "version": "0.0.159",
  "type": "module",
  "private": true,
  "scripts": {
@@ -1,5 +1,11 @@
 # @llamaindex/llama-parse-browser-test

+## 0.0.61
+
+### Patch Changes
+
+- @llamaindex/cloud@4.0.6
+
 ## 0.0.60

 ### Patch Changes
@@ -1,7 +1,7 @@
 {
  "name": "@llamaindex/llama-parse-browser-test",
  "private": true,
-  "version": "0.0.60",
+  "version": "0.0.61",
  "type": "module",
  "scripts": {
    "dev": "vite",
@@ -1,5 +1,11 @@
 # @llamaindex/next-agent-test

+## 0.1.159
+
+### Patch Changes
+
+- llamaindex@0.10.5
+
 ## 0.1.158

 ### Patch Changes
@@ -1,6 +1,6 @@
 {
  "name": "@llamaindex/next-agent-test",
-  "version": "0.1.158",
+  "version": "0.1.159",
  "private": true,
  "scripts": {
    "dev": "next dev",
@@ -1,5 +1,11 @@
 # test-edge-runtime

+## 0.1.158
+
+### Patch Changes
+
+- llamaindex@0.10.5
+
 ## 0.1.157

 ### Patch Changes
@@ -1,6 +1,6 @@
 {
  "name": "@llamaindex/nextjs-edge-runtime-test",
-  "version": "0.1.157",
+  "version": "0.1.158",
  "private": true,
  "scripts": {
    "dev": "next dev",
@@ -1,5 +1,13 @@
 # @llamaindex/next-node-runtime

+## 0.1.26
+
+### Patch Changes
+
+- llamaindex@0.10.5
+- @llamaindex/huggingface@0.1.8
+- @llamaindex/readers@3.1.2
+
 ## 0.1.25

 ### Patch Changes
@@ -1,6 +1,6 @@
 {
  "name": "@llamaindex/next-node-runtime-test",
-  "version": "0.1.25",
+  "version": "0.1.26",
  "private": true,
  "scripts": {
    "dev": "next dev",
@@ -1,5 +1,11 @@
 # vite-import-llamaindex

+## 0.0.25
+
+### Patch Changes
+
+- llamaindex@0.10.5
+
 ## 0.0.24

 ### Patch Changes
@@ -1,7 +1,7 @@
 {
  "name": "vite-import-llamaindex",
  "private": true,
-  "version": "0.0.24",
+  "version": "0.0.25",
  "type": "module",
  "scripts": {
    "build": "vite build",
@@ -1,5 +1,13 @@
 # @llamaindex/waku-query-engine-test

+## 0.0.159
+
+### Patch Changes
+
+- Updated dependencies [9b2e25a]
+  - @llamaindex/env@0.1.30
+  - llamaindex@0.10.5
+
 ## 0.0.158

 ### Patch Changes
@@ -1,6 +1,6 @@
 {
  "name": "@llamaindex/waku-query-engine-test",
-  "version": "0.0.158",
+  "version": "0.0.159",
  "type": "module",
  "private": true,
  "scripts": {
@@ -1,5 +1,59 @@
 # examples

+## 0.3.14
+
+### Patch Changes
+
+- 9b2e25a: Use Uint8Array instead of Buffer for file type messages (works with non-NodeJS)
+- 206b491: Add support for google live api
+- Updated dependencies [9b2e25a]
+- Updated dependencies [206b491]
+  - @llamaindex/anthropic@0.3.5
+  - @llamaindex/google@0.3.0
+  - @llamaindex/openai@0.3.6
+  - @llamaindex/core@0.6.4
+  - @llamaindex/env@0.1.30
+  - llamaindex@0.10.5
+  - @llamaindex/clip@0.0.54
+  - @llamaindex/deepinfra@0.0.54
+  - @llamaindex/deepseek@0.0.14
+  - @llamaindex/fireworks@0.0.14
+  - @llamaindex/groq@0.0.69
+  - @llamaindex/huggingface@0.1.8
+  - @llamaindex/jinaai@0.0.14
+  - @llamaindex/perplexity@0.0.11
+  - @llamaindex/azure@0.1.14
+  - @llamaindex/elastic-search@0.1.4
+  - @llamaindex/milvus@0.1.13
+  - @llamaindex/qdrant@0.1.13
+  - @llamaindex/supabase@0.1.3
+  - @llamaindex/together@0.0.14
+  - @llamaindex/vllm@0.0.40
+  - @llamaindex/cloud@4.0.6
+  - @llamaindex/node-parser@2.0.4
+  - @llamaindex/assemblyai@0.1.3
+  - @llamaindex/cohere@0.0.18
+  - @llamaindex/discord@0.1.3
+  - @llamaindex/mistral@0.1.4
+  - @llamaindex/mixedbread@0.0.18
+  - @llamaindex/notion@0.1.3
+  - @llamaindex/ollama@0.1.4
+  - @llamaindex/portkey-ai@0.0.46
+  - @llamaindex/replicate@0.0.46
+  - @llamaindex/astra@0.0.18
+  - @llamaindex/chroma@0.0.18
+  - @llamaindex/firestore@1.0.11
+  - @llamaindex/mongodb@0.0.19
+  - @llamaindex/pinecone@0.1.4
+  - @llamaindex/postgres@0.0.47
+  - @llamaindex/upstash@0.0.18
+  - @llamaindex/weaviate@0.0.18
+  - @llamaindex/vercel@0.1.4
+  - @llamaindex/voyage-ai@1.0.10
+  - @llamaindex/readers@3.1.2
+  - @llamaindex/tools@0.0.9
+  - @llamaindex/workflow@1.1.1
+
 ## 0.3.13

 ### Patch Changes
@@ -25,7 +25,7 @@ async function main() {
          },
          {
            type: "file",
-            data: fs.readFileSync("./data/manga.pdf"),
+            data: Uint8Array.from(fs.readFileSync("./data/manga.pdf")),
            mimeType: "application/pdf",
          },
        ],
@@ -32,7 +32,7 @@ import fs from "fs";
          },
          {
            type: "file",
-            data: fs.readFileSync("./data/manga.pdf"),
+            data: Uint8Array.from(fs.readFileSync("./data/manga.pdf")),
            mimeType: "application/pdf",
          },
        ],
@@ -0,0 +1,104 @@
+import { ModalityType } from "@llamaindex/core/schema";
+import { tool } from "@llamaindex/core/tools";
+import { gemini, GEMINI_MODEL } from "@llamaindex/google";
+
+import { liveEvents } from "llamaindex";
+import { z } from "zod";
+
+const weatherTool = tool({
+  name: "weather",
+  description: "Get the weather",
+  parameters: z.object({
+    location: z.string({
+      description: "The location to get the weather for",
+    }),
+  }),
+  execute: ({ location }) => {
+    return `The weather in ${location} is rainy`;
+  },
+});
+
+const divideNumbers = tool({
+  name: "divideNumbers",
+  description: "Use this function to divide two numbers",
+  parameters: z.object({
+    a: z.number().describe("The dividend a to divide"),
+    b: z.number().describe("The divisor b to divide by"),
+  }),
+  execute: ({ a, b }) => `${a / b}`,
+});
+
+async function main() {
+  const apiKey = process.env.GOOGLE_API_KEY;
+  if (!apiKey) {
+    console.error(
+      "Please set GOOGLE_API_KEY in your environment variables or .env file",
+    );
+    process.exit(1);
+  }
+
+  console.log("🚀 Initializing Gemini Live API with tools example...");
+
+  const llm = gemini({
+    apiKey: apiKey,
+    model: GEMINI_MODEL.GEMINI_2_0_FLASH_LIVE, // Must use a live-compatible model
+  });
+
+  console.log("📡 Connecting to Gemini Live session...");
+
+  // Connect to a live session with tools
+  const session = await llm.live.connect({
+    // Specify response modalities (text response is required for tools)
+    responseModality: [ModalityType.TEXT],
+    // Register our tools with the session
+    tools: [weatherTool, divideNumbers],
+    // Optional system instruction
+    systemInstruction:
+      "You are a helpful assistant that can use tools. When answering questions about weather or divide numbers, always use the appropriate tool.",
+  });
+
+  (async () => {
+    console.log("🎧 Listening for events...");
+
+    for await (const event of session.streamEvents()) {
+      if (liveEvents.open.include(event)) {
+        console.log("✅ Connected to Gemini Live session");
+
+        console.log(
+          "💬 Sending message: 'What's the weather in San Francisco and what is 100 / 2?'",
+        );
+        session.sendMessage({
+          content: "What's the weather in San Francisco and what is 100 / 2?",
+          role: "user",
+        });
+      } else if (liveEvents.text.include(event)) {
+        process.stdout.write(event.text);
+      } else if (liveEvents.error.include(event)) {
+        console.error("❌ Error:", event.error);
+      } else if (liveEvents.close.include(event)) {
+        console.log("👋 Session closed");
+        process.exit(0);
+      } else if (liveEvents.setupComplete.include(event)) {
+        console.log("🔧 Setup complete");
+      }
+    }
+  })();
+
+  process.on("SIGINT", async () => {
+    console.log("\n👋 Interrupted by user. Closing session...");
+    await session.disconnect();
+    process.exit(0);
+  });
+
+  // Timeout after 2 minutes if no interaction
+  setTimeout(async () => {
+    console.log("\n⏱️ Session timeout. Closing session...");
+    await session.disconnect();
+    process.exit(0);
+  }, 120000);
+}
+
+main().catch((error) => {
+  console.error("❌ Fatal error:", error);
+  process.exit(1);
+});
@@ -0,0 +1,221 @@
+import { fs } from "@llamaindex/env";
+import { gemini, GEMINI_MODEL, GeminiLiveSession } from "@llamaindex/google";
+import { liveEvents } from "llamaindex";
+
+import path from "path";
+
+function createWavHeader(
+  sampleRate = 16000,
+  bitsPerSample = 16,
+  channels = 1,
+  dataLength: number,
+) {
+  const buffer = Buffer.alloc(44);
+
+  // RIFF chunk descriptor
+  buffer.write("RIFF", 0);
+  buffer.writeUInt32LE(36 + dataLength, 4); // File size - 8
+  buffer.write("WAVE", 8);
+
+  // fmt sub-chunk
+  buffer.write("fmt ", 12);
+  buffer.writeUInt32LE(16, 16); // Subchunk1Size (16 for PCM)
+  buffer.writeUInt16LE(1, 20); // AudioFormat (1 for PCM)
+  buffer.writeUInt16LE(channels, 22); // NumChannels
+  buffer.writeUInt32LE(sampleRate, 24); // SampleRate
+  buffer.writeUInt32LE((sampleRate * channels * bitsPerSample) / 8, 28); // ByteRate
+  buffer.writeUInt16LE((channels * bitsPerSample) / 8, 32); // BlockAlign
+  buffer.writeUInt16LE(bitsPerSample, 34); // BitsPerSample
+
+  // data sub-chunk
+  buffer.write("data", 36);
+  buffer.writeUInt32LE(dataLength, 40); // Subchunk2Size
+
+  return buffer;
+}
+
+async function saveWavFile(
+  audioChunks: Buffer[],
+  filePath: string,
+  sampleRate = 16000,
+  bitsPerSample = 16,
+  channels = 1,
+): Promise<void> {
+  if (audioChunks.length === 0) {
+    throw new Error("No audio data to save");
+  }
+
+  try {
+    const combinedAudioData = Buffer.concat(audioChunks);
+    console.log(`Total audio data: ${combinedAudioData.length} bytes`);
+
+    const wavHeader = createWavHeader(
+      sampleRate,
+      bitsPerSample,
+      channels,
+      combinedAudioData.length,
+    );
+    const wavFile = Buffer.concat([wavHeader, combinedAudioData]);
+
+    await fs.writeFile(filePath, wavFile);
+    console.log(`💾 Saved audio to ${filePath}`);
+    return;
+  } catch (error) {
+    console.error("❌ Error saving audio file:", error);
+    throw error;
+  }
+}
+
+async function main() {
+  const apiKey = process.env.GOOGLE_API_KEY;
+  if (!apiKey) {
+    console.error(
+      "Please set GOOGLE_API_KEY in your environment variables or .env file",
+    );
+    process.exit(1);
+  }
+
+  console.log("🚀 Initializing Gemini Live API example...");
+
+  const llm = gemini({
+    model: GEMINI_MODEL.GEMINI_2_0_FLASH_LIVE,
+    voiceName: "Zephyr",
+  });
+
+  console.log("📡 Connecting to Gemini Live session...");
+
+  const session = await llm.live.connect();
+
+  let isRunning = true;
+
+  const audioChunks: Buffer[] = [];
+  let audioResponse = false;
+
+  (async () => {
+    try {
+      console.log("🎧 Listening for events...");
+
+      for await (const event of session.streamEvents()) {
+        if (liveEvents.open.include(event)) {
+          console.log("✅ Connected to Gemini Live session");
+
+          console.log(
+            "💬 Sending text message: 'Say something about you for 10 seconds'",
+          );
+          session.sendMessage({
+            content: "Say something about you for 10 seconds",
+            role: "user",
+          });
+
+          setTimeout(() => {
+            sendPcmAudioFile(session);
+          }, 3000);
+        } else if (liveEvents.setupComplete.include(event)) {
+          console.log("✅ Setup complete");
+        } else if (liveEvents.text.include(event)) {
+          process.stdout.write(event.text);
+        } else if (liveEvents.audio.include(event)) {
+          console.log("\n🔊 Received audio chunk");
+          audioResponse = true;
+
+          try {
+            const chunk = Buffer.from(event.data as string, "base64");
+            audioChunks.push(chunk);
+            console.log(`Received audio chunk: ${chunk.length} bytes`);
+          } catch (error) {
+            console.error("❌ Error processing audio chunk:", error);
+          }
+        } else if (liveEvents.error.include(event)) {
+          console.error("❌ Error:", event.error);
+        } else if (liveEvents.close.include(event)) {
+          console.log("👋 Session closed");
+
+          if (audioResponse && audioChunks.length > 0) {
+            try {
+              await saveWavFile(audioChunks, "gemini-response.wav");
+            } catch (error) {
+              console.error("❌ Error saving final audio file:", error);
+            }
+          }
+
+          isRunning = false;
+          break;
+        }
+      }
+    } catch (error) {
+      console.error("❌ Error processing stream:", error);
+    }
+  })();
+
+  async function sendPcmAudioFile(session: GeminiLiveSession) {
+    try {
+      console.log("🎤 Reading PCM audio file...");
+
+      const filePath = path.join(__dirname, "hello_are_you_there.pcm");
+      console.log(`Reading file from: ${filePath}`);
+
+      const audioBuffer = await fs.readFile(filePath);
+
+      const base64Audio = audioBuffer.toString("base64");
+
+      session.sendMessage({
+        content: [
+          {
+            type: "audio",
+            data: base64Audio,
+            mimeType: "audio/pcm;rate=16000",
+          },
+        ],
+        role: "user",
+      });
+
+      console.log("🎤 PCM audio file sent! Waiting for response...");
+    } catch (error) {
+      console.error("❌ Error sending audio file:", error);
+    }
+  }
+
+  setTimeout(async () => {
+    console.log("\n⏱️ Time's up! Closing session...");
+
+    if (audioResponse && audioChunks.length > 0) {
+      try {
+        await saveWavFile(audioChunks, "gemini-response.wav");
+      } catch (error) {
+        console.error("❌ Error saving final audio file:", error);
+      }
+    }
+
+    await session.disconnect();
+    isRunning = false;
+  }, 60000);
+
+  process.on("SIGINT", async () => {
+    console.log("\n👋 Interrupted by user. Closing session...");
+
+    if (audioResponse && audioChunks.length > 0) {
+      try {
+        await saveWavFile(audioChunks, "gemini-response.wav");
+      } catch (error) {
+        console.error("❌ Error saving final audio file:", error);
+      }
+    }
+
+    await session.disconnect();
+    isRunning = false;
+  });
+
+  const waitForClose = () => {
+    if (isRunning) {
+      setTimeout(waitForClose, 1000);
+    } else {
+      process.exit(0);
+    }
+  };
+  waitForClose();
+}
+
+main().catch((error) => {
+  console.error("❌ Fatal error:", error);
+  process.exit(1);
+});
@@ -37,8 +37,10 @@ async function main() {
  const index = await VectorStoreIndex.init({
    storageContext,
  });
-  // topK for text is 0 and for image 1 => we only retrieve one image and no text based on the query
-  const retriever = index.asRetriever({ topK: { TEXT: 0, IMAGE: 1 } });
+  // topK for text is 0, for image 1, for audio 0  => we only retrieve one image and no text based on the query
+  const retriever = index.asRetriever({
+    topK: { TEXT: 0, IMAGE: 1, AUDIO: 0 },
+  });
  // NOTE: we set the contextRole to "user" (default is "system"). The reason is that GPT-4 does not support
  // images in a system message
  const chatEngine = new ContextChatEngine({ retriever, contextRole: "user" });
@@ -30,7 +30,7 @@ async function main() {

  const queryEngine = index.asQueryEngine({
    responseSynthesizer: getResponseSynthesizer("multi_modal"),
-    retriever: index.asRetriever({ topK: { TEXT: 3, IMAGE: 1 } }),
+    retriever: index.asRetriever({ topK: { TEXT: 3, IMAGE: 1, AUDIO: 0 } }),
  });
  const stream = await queryEngine.query({
    query: "Tell me more about Vincent van Gogh's famous paintings",
@@ -12,7 +12,9 @@ async function main() {
    nodes: [],
    storageContext,
  });
-  const retriever = index.asRetriever({ topK: { TEXT: 1, IMAGE: 3 } });
+  const retriever = index.asRetriever({
+    topK: { TEXT: 1, IMAGE: 3, AUDIO: 0 },
+  });
  const results = await retriever.retrieve({
    query: "what are Vincent van Gogh's famous paintings",
  });
@@ -26,7 +26,7 @@ import fs from "fs";
          },
          {
            type: "file",
-            data: fs.readFileSync("./data/manga.pdf"),
+            data: new Uint8Array(fs.readFileSync("./data/manga.pdf")),
            mimeType: "application/pdf",
          },
        ],
@@ -21,7 +21,7 @@ async function main() {
          },
          {
            type: "file",
-            data: fs.readFileSync("./data/manga.pdf"),
+            data: Uint8Array.from(fs.readFileSync("./data/manga.pdf")),
            mimeType: "application/pdf",
          },
        ],
@@ -1,6 +1,6 @@
 {
  "name": "@llamaindex/examples",
-  "version": "0.3.13",
+  "version": "0.3.14",
  "private": true,
  "scripts": {
    "lint": "eslint .",
@@ -11,59 +11,59 @@
    "@azure/cosmos": "^4.1.1",
    "@azure/identity": "^4.4.1",
    "@azure/search-documents": "^12.1.0",
-    "@llamaindex/anthropic": "^0.3.4",
-    "@llamaindex/astra": "^0.0.17",
-    "@llamaindex/azure": "^0.1.13",
-    "@llamaindex/chroma": "^0.0.17",
-    "@llamaindex/clip": "^0.0.53",
-    "@llamaindex/cloud": "^4.0.4",
-    "@llamaindex/cohere": "^0.0.17",
-    "@llamaindex/core": "^0.6.3",
-    "@llamaindex/deepinfra": "^0.0.53",
-    "@llamaindex/env": "^0.1.29",
-    "@llamaindex/firestore": "^1.0.10",
-    "@llamaindex/google": "^0.2.5",
-    "@llamaindex/groq": "^0.0.68",
-    "@llamaindex/huggingface": "^0.1.7",
-    "@llamaindex/milvus": "^0.1.12",
-    "@llamaindex/mistral": "^0.1.3",
-    "@llamaindex/mixedbread": "^0.0.17",
-    "@llamaindex/mongodb": "^0.0.18",
-    "@llamaindex/elastic-search": "^0.1.3",
-    "@llamaindex/node-parser": "^2.0.3",
-    "@llamaindex/ollama": "^0.1.3",
-    "@llamaindex/openai": "^0.3.5",
-    "@llamaindex/pinecone": "^0.1.3",
-    "@llamaindex/portkey-ai": "^0.0.45",
-    "@llamaindex/postgres": "^0.0.46",
-    "@llamaindex/qdrant": "^0.1.12",
-    "@llamaindex/readers": "^3.1.1",
-    "@llamaindex/replicate": "^0.0.45",
-    "@llamaindex/upstash": "^0.0.17",
-    "@llamaindex/vercel": "^0.1.3",
-    "@llamaindex/vllm": "^0.0.39",
-    "@llamaindex/voyage-ai": "^1.0.9",
-    "@llamaindex/weaviate": "^0.0.17",
-    "@llamaindex/workflow": "^1.0.4",
-    "@llamaindex/deepseek": "^0.0.13",
-    "@llamaindex/fireworks": "^0.0.13",
-    "@llamaindex/together": "^0.0.13",
-    "@llamaindex/jinaai": "^0.0.13",
-    "@llamaindex/perplexity": "^0.0.10",
-    "@llamaindex/supabase": "^0.1.2",
-    "@llamaindex/tools": "^0.0.8",
+    "@llamaindex/anthropic": "^0.3.5",
+    "@llamaindex/astra": "^0.0.18",
+    "@llamaindex/azure": "^0.1.14",
+    "@llamaindex/chroma": "^0.0.18",
+    "@llamaindex/clip": "^0.0.54",
+    "@llamaindex/cloud": "^4.0.6",
+    "@llamaindex/cohere": "^0.0.18",
+    "@llamaindex/core": "^0.6.4",
+    "@llamaindex/deepinfra": "^0.0.54",
+    "@llamaindex/env": "^0.1.30",
+    "@llamaindex/firestore": "^1.0.11",
+    "@llamaindex/google": "^0.3.0",
+    "@llamaindex/groq": "^0.0.69",
+    "@llamaindex/huggingface": "^0.1.8",
+    "@llamaindex/milvus": "^0.1.13",
+    "@llamaindex/mistral": "^0.1.4",
+    "@llamaindex/mixedbread": "^0.0.18",
+    "@llamaindex/mongodb": "^0.0.19",
+    "@llamaindex/elastic-search": "^0.1.4",
+    "@llamaindex/node-parser": "^2.0.4",
+    "@llamaindex/ollama": "^0.1.4",
+    "@llamaindex/openai": "^0.3.6",
+    "@llamaindex/pinecone": "^0.1.4",
+    "@llamaindex/portkey-ai": "^0.0.46",
+    "@llamaindex/postgres": "^0.0.47",
+    "@llamaindex/qdrant": "^0.1.13",
+    "@llamaindex/readers": "^3.1.2",
+    "@llamaindex/replicate": "^0.0.46",
+    "@llamaindex/upstash": "^0.0.18",
+    "@llamaindex/vercel": "^0.1.4",
+    "@llamaindex/vllm": "^0.0.40",
+    "@llamaindex/voyage-ai": "^1.0.10",
+    "@llamaindex/weaviate": "^0.0.18",
+    "@llamaindex/workflow": "^1.1.1",
+    "@llamaindex/deepseek": "^0.0.14",
+    "@llamaindex/fireworks": "^0.0.14",
+    "@llamaindex/together": "^0.0.14",
+    "@llamaindex/jinaai": "^0.0.14",
+    "@llamaindex/perplexity": "^0.0.11",
+    "@llamaindex/supabase": "^0.1.3",
+    "@llamaindex/tools": "^0.0.9",
    "@notionhq/client": "^2.2.15",
    "@pinecone-database/pinecone": "^4.0.0",
-    "@llamaindex/assemblyai": "^0.1.2",
-    "@llamaindex/discord": "^0.1.2",
-    "@llamaindex/notion": "^0.1.2",
+    "@llamaindex/assemblyai": "^0.1.3",
+    "@llamaindex/discord": "^0.1.3",
+    "@llamaindex/notion": "^0.1.3",
    "@vercel/postgres": "^0.10.0",
    "ai": "^4.0.0",
    "ajv": "^8.17.1",
    "commander": "^12.1.0",
    "dotenv": "^16.4.5",
    "js-tiktoken": "^1.0.14",
-    "llamaindex": "^0.10.3",
+    "llamaindex": "^0.10.5",
    "mongodb": "6.7.0",
    "postgres": "^3.4.4",
    "wikipedia": "^2.1.2",
@@ -1,5 +1,11 @@
 # @llamaindex/autotool

+## 7.0.5
+
+### Patch Changes
+
+- llamaindex@0.10.5
+
 ## 7.0.4

 ### Patch Changes
@@ -1,5 +1,12 @@
 # @llamaindex/autotool-01-node-example

+## 0.0.106
+
+### Patch Changes
+
+- llamaindex@0.10.5
+- @llamaindex/autotool@7.0.5
+
 ## 0.0.105

 ### Patch Changes
@@ -13,5 +13,5 @@
  "scripts": {
    "start": "node --import tsx --import @llamaindex/autotool/node ./src/index.ts"
  },
-  "version": "0.0.105"
+  "version": "0.0.106"
 }
@@ -6,7 +6,7 @@
    "url": "git+https://github.com/run-llama/LlamaIndexTS.git",
    "directory": "packages/autotool"
  },
-  "version": "7.0.4",
+  "version": "7.0.5",
  "description": "auto transpile your JS function to LLM Agent compatible",
  "files": [
    "dist",
@@ -1,5 +1,13 @@
 # @llamaindex/cloud

+## 4.0.6
+
+### Patch Changes
+
+- Updated dependencies [9b2e25a]
+  - @llamaindex/core@0.6.4
+  - @llamaindex/env@0.1.30
+
 ## 4.0.5

 ### Patch Changes
@@ -1,6 +1,6 @@
 {
  "name": "@llamaindex/cloud",
-  "version": "4.0.5",
+  "version": "4.0.6",
  "type": "module",
  "license": "MIT",
  "scripts": {
@@ -1,5 +1,13 @@
 # @llamaindex/community

+## 0.0.98
+
+### Patch Changes
+
+- Updated dependencies [9b2e25a]
+  - @llamaindex/core@0.6.4
+  - @llamaindex/env@0.1.30
+
 ## 0.0.97

 ### Patch Changes
@@ -1,7 +1,7 @@
 {
  "name": "@llamaindex/community",
  "description": "Community package for LlamaIndexTS",
-  "version": "0.0.97",
+  "version": "0.0.98",
  "type": "module",
  "types": "dist/type/index.d.ts",
  "main": "dist/cjs/index.js",
@@ -1,5 +1,13 @@
 # @llamaindex/core

+## 0.6.4
+
+### Patch Changes
+
+- 9b2e25a: Use Uint8Array instead of Buffer for file type messages (works with non-NodeJS)
+- Updated dependencies [9b2e25a]
+  - @llamaindex/env@0.1.30
+
 ## 0.6.3

 ### Patch Changes
@@ -1,7 +1,7 @@
 {
  "name": "@llamaindex/core",
  "type": "module",
-  "version": "0.6.3",
+  "version": "0.6.4",
  "description": "LlamaIndex Core Module",
  "exports": {
    "./agent": {
@@ -1,4 +1,5 @@
 export { BaseLLM, ToolCallLLM } from "./base";
+export { LiveLLM, LiveLLMSession, liveEvents, type LiveEvent } from "./live";
 export type {
  BaseTool,
  BaseToolWithCall,
@@ -15,11 +16,15 @@ export type {
  LLMCompletionParamsNonStreaming,
  LLMCompletionParamsStreaming,
  LLMMetadata,
+  LiveConnectConfig,
  MessageContent,
+  MessageContentAudioDetail,
  MessageContentDetail,
  MessageContentFileDetail,
+  MessageContentImageDataDetail,
  MessageContentImageDetail,
  MessageContentTextDetail,
+  MessageContentVideoDetail,
  MessageType,
  PartialToolCall,
  TextChatMessage,
@@ -0,0 +1,87 @@
+import type {
+  ChatMessage,
+  LiveConnectConfig,
+  MessageContentAudioDetail,
+  MessageContentTextDetail,
+} from "./type";
+
+export type OpenEvent = { type: "open" };
+
+export type AudioEvent = MessageContentAudioDetail;
+
+export type TextEvent = MessageContentTextDetail;
+
+export type ErrorEvent = { type: "error"; error: unknown };
+
+export type CloseEvent = { type: "close" };
+
+export type SetupCompleteEvent = { type: "setupComplete" };
+
+export type LiveEvent =
+  | OpenEvent
+  | AudioEvent
+  | TextEvent
+  | ErrorEvent
+  | CloseEvent
+  | SetupCompleteEvent;
+
+export const liveEvents = {
+  open: { include: (e: LiveEvent): e is OpenEvent => e.type === "open" },
+  audio: {
+    include: (e: LiveEvent): e is AudioEvent => e.type === "audio",
+  },
+  text: { include: (e: LiveEvent): e is TextEvent => e.type === "text" },
+  error: {
+    include: (e: LiveEvent): e is ErrorEvent => e.type === "error",
+  },
+  close: {
+    include: (e: LiveEvent): e is CloseEvent => e.type === "close",
+  },
+  setupComplete: {
+    include: (e: LiveEvent): e is SetupCompleteEvent =>
+      e.type === "setupComplete",
+  },
+};
+
+export abstract class LiveLLMSession {
+  protected eventQueue: LiveEvent[] = [];
+  protected eventResolvers: ((value: LiveEvent) => void)[] = [];
+  protected closed = false;
+  abstract sendMessage(message: ChatMessage): void;
+  async *streamEvents(): AsyncIterable<LiveEvent> {
+    while (true) {
+      const event = await this.nextEvent();
+      if (event === undefined) {
+        break;
+      }
+      yield event;
+    }
+  }
+  abstract disconnect(): Promise<void>;
+
+  protected async nextEvent(): Promise<LiveEvent | undefined> {
+    if (this.eventQueue.length) {
+      return Promise.resolve(this.eventQueue.shift());
+    }
+
+    return new Promise((resolve) => {
+      this.eventResolvers.push(resolve);
+    });
+  }
+
+  //Uses an async queue to send events to the client
+  // if the consumer is waiting for an event, it will be resolved immediately
+  // otherwise, the event will be queued up and sent when the consumer is ready
+  pushEventToQueue(event: LiveEvent) {
+    if (this.eventResolvers.length) {
+      //resolving the promise with the event
+      this.eventResolvers.shift()!(event);
+    } else {
+      this.eventQueue.push(event);
+    }
+  }
+}
+
+export abstract class LiveLLM {
+  abstract connect(config?: LiveConnectConfig): Promise<LiveLLMSession>;
+}
@@ -2,6 +2,7 @@ import type { Tokenizers } from "@llamaindex/env/tokenizers";
 import type { JSONSchemaType } from "ajv";
 import { z } from "zod";
 import type { JSONObject, JSONValue } from "../global";
+import type { ModalityType } from "../schema";
 /**
 * @internal
 */
@@ -163,15 +164,39 @@ export type MessageContentImageDetail = {
  detail?: "high" | "low" | "auto";
 };

+export type MessageContentAudioDetail = {
+  type: "audio";
+  //audio could be a base64 string as well
+  data: string | Uint8Array;
+  mimeType: string;
+};
+
+export type MessageContentVideoDetail = {
+  type: "video";
+  //video could be a base64 string as well
+  data: string | Uint8Array;
+  mimeType: string;
+};
+
+export type MessageContentImageDataDetail = {
+  type: "image";
+  //image could be a base64 string as well
+  data: string | Uint8Array;
+  mimeType: string;
+};
+
 export type MessageContentFileDetail = {
  type: "file";
-  data: Buffer;
+  data: Uint8Array;
  mimeType: string;
 };

 export type MessageContentDetail =
  | MessageContentTextDetail
  | MessageContentImageDetail
+  | MessageContentAudioDetail
+  | MessageContentVideoDetail
+  | MessageContentImageDataDetail
  | MessageContentFileDetail;

 /**
@@ -267,3 +292,9 @@ export type ToolOutput = {
  output: JSONValue;
  isError: boolean;
 };
+
+export interface LiveConnectConfig {
+  tools?: BaseTool[];
+  responseModality?: ModalityType[];
+  systemInstruction?: string;
+}
@@ -431,6 +431,7 @@ export interface NodeWithScore<T extends Metadata = Metadata> {
 export enum ModalityType {
  TEXT = "TEXT",
  IMAGE = "IMAGE",
+  AUDIO = "AUDIO",
 }

 type NodesByType = {
@@ -1,5 +1,11 @@
 # @llamaindex/env

+## 0.1.30
+
+### Patch Changes
+
+- 9b2e25a: Use Uint8Array instead of Buffer for file type messages (works with non-NodeJS)
+
 ## 0.1.29

 ### Patch Changes
@@ -1,7 +1,7 @@
 {
  "name": "@llamaindex/env",
  "description": "environment wrapper, supports all JS environment including node, deno, bun, edge runtime, and cloudflare worker",
-  "version": "0.1.29",
+  "version": "0.1.30",
  "type": "module",
  "types": "dist/index.d.ts",
  "module": "dist/index.js",
@@ -6,7 +6,8 @@
 import "./global-check.js";

 export * from "./als/index.web.js";
-export { consoleLogger, emptyLogger, type Logger } from "./logger/index.js";
+export * from "./logger/index.js";
+export * from "./utils/base64.js";
 export { NotSupportCurrentRuntimeClass } from "./utils/shared.js";
 export * from "./web-polyfill.js";
 if (typeof window === "undefined") {
@@ -5,6 +5,7 @@
 */

 export * from "./als/index.non-node.js";
-export { consoleLogger, emptyLogger, type Logger } from "./logger/index.js";
+export * from "./logger/index.js";
 export * from "./node-polyfill.js";
+export * from "./utils/base64.js";
 export { NotSupportCurrentRuntimeClass } from "./utils/shared.js";
@@ -37,7 +37,8 @@ export function createSHA256(): SHA256 {
 export const process = globalThis.process;

 export * from "./als/index.node.js";
-export { consoleLogger, emptyLogger, type Logger } from "./logger/index.js";
+export * from "./logger/index.js";
+export * from "./utils/base64.js";
 export { CustomEvent, getEnv, setEnvs } from "./utils/index.js";
 export { NotSupportCurrentRuntimeClass } from "./utils/shared.js";
 export {
@@ -11,9 +11,10 @@ export * from "./als/index.workerd.js";
 export { NotSupportCurrentRuntimeClass } from "./utils/shared.js";

 export * from "./node-polyfill.js";
+export * from "./utils/base64.js";

 export function getEnv(name: string): string | undefined {
  return INTERNAL_ENV[name];
 }

-export { consoleLogger, emptyLogger, type Logger } from "./logger/index.js";
+export * from "./logger/index.js";
@@ -0,0 +1,38 @@
+/**
+ * Converts a Uint8Array to a base64 string.
+ * For large arrays, it processes the data in chunks to avoid memory issues.
+ * Falls back to Buffer if available for better performance.
+ *
+ * @param bytes - The Uint8Array to convert
+ * @returns The base64 string representation
+ */
+export function uint8ArrayToBase64(bytes: Uint8Array): string {
+  // Use Buffer if available (Node.js environment)
+  if (typeof Buffer !== "undefined") {
+    return Buffer.from(bytes).toString("base64");
+  }
+
+  // For browsers and other environments without Buffer
+  // Process in chunks for large arrays to avoid memory issues
+  const CHUNK_SIZE = 32768; // 32KB chunks
+  let result = "";
+
+  // For small arrays, use the built-in btoa function directly
+  if (bytes.length < CHUNK_SIZE) {
+    const binary = Array.from(bytes)
+      .map((byte) => String.fromCharCode(byte))
+      .join("");
+    return globalThis.btoa(binary);
+  }
+
+  // For large arrays, process in chunks
+  for (let i = 0; i < bytes.length; i += CHUNK_SIZE) {
+    const chunk = bytes.subarray(i, i + CHUNK_SIZE);
+    const binary = Array.from(chunk)
+      .map((byte) => String.fromCharCode(byte))
+      .join("");
+    result += globalThis.btoa(binary);
+  }
+
+  return result;
+}
@@ -1,5 +1,11 @@
 # @llamaindex/experimental

+## 0.0.175
+
+### Patch Changes
+
+- llamaindex@0.10.5
+
 ## 0.0.174

 ### Patch Changes
@@ -1,7 +1,7 @@
 {
  "name": "@llamaindex/experimental",
  "description": "Experimental package for LlamaIndexTS",
-  "version": "0.0.174",
+  "version": "0.0.175",
  "type": "module",
  "types": "dist/type/index.d.ts",
  "main": "dist/cjs/index.js",
@@ -1,5 +1,17 @@
 # llamaindex

+## 0.10.5
+
+### Patch Changes
+
+- Updated dependencies [9b2e25a]
+  - @llamaindex/openai@0.3.6
+  - @llamaindex/core@0.6.4
+  - @llamaindex/env@0.1.30
+  - @llamaindex/cloud@4.0.6
+  - @llamaindex/node-parser@2.0.4
+  - @llamaindex/workflow@1.1.1
+
 ## 0.10.4

 ### Patch Changes
@@ -1,6 +1,6 @@
 {
  "name": "llamaindex",
-  "version": "0.10.4",
+  "version": "0.10.5",
  "license": "MIT",
  "type": "module",
  "keywords": [
@@ -397,6 +397,7 @@ export class VectorStoreIndex extends BaseIndex<IndexDict> {
 * VectorIndexRetriever retrieves nodes from a VectorIndex.
 */

+// TopKMap type now only includes TEXT and IMAGE modalities
 type TopKMap = { [P in ModalityType]: number };

 // eslint-disable-next-line @typescript-eslint/no-explicit-any
@@ -435,6 +436,7 @@ export class VectorIndexRetriever extends BaseRetriever {
            ? options.similarityTopK
            : DEFAULT_SIMILARITY_TOP_K,
        [ModalityType.IMAGE]: DEFAULT_SIMILARITY_TOP_K,
+        [ModalityType.AUDIO]: DEFAULT_SIMILARITY_TOP_K,
      };
    }
    this.filters = options.filters;
@@ -1,5 +1,13 @@
 # @llamaindex/node-parser

+## 2.0.4
+
+### Patch Changes
+
+- Updated dependencies [9b2e25a]
+  - @llamaindex/core@0.6.4
+  - @llamaindex/env@0.1.30
+
 ## 2.0.3

 ### Patch Changes
@@ -1,6 +1,6 @@
 {
  "name": "@llamaindex/node-parser",
-  "version": "2.0.3",
+  "version": "2.0.4",
  "description": "Node parser for LlamaIndex",
  "type": "module",
  "exports": {
@@ -1,5 +1,14 @@
 # @llamaindex/anthropic

+## 0.3.5
+
+### Patch Changes
+
+- 9b2e25a: Use Uint8Array instead of Buffer for file type messages (works with non-NodeJS)
+- Updated dependencies [9b2e25a]
+  - @llamaindex/core@0.6.4
+  - @llamaindex/env@0.1.30
+
 ## 0.3.4

 ### Patch Changes
@@ -1,7 +1,7 @@
 {
  "name": "@llamaindex/anthropic",
  "description": "Anthropic Adapter for LlamaIndex",
-  "version": "0.3.4",
+  "version": "0.3.5",
  "type": "module",
  "main": "./dist/index.cjs",
  "module": "./dist/index.js",
@@ -28,7 +28,7 @@ import type {
 } from "@llamaindex/core/llms";
 import { ToolCallLLM } from "@llamaindex/core/llms";
 import { extractText } from "@llamaindex/core/utils";
-import { getEnv } from "@llamaindex/env";
+import { getEnv, uint8ArrayToBase64 } from "@llamaindex/env";
 import { isDeepEqual } from "remeda";

 export class AnthropicSession {
@@ -332,24 +332,27 @@ export class Anthropic extends ToolCallLLM<
              source: {
                type: "base64" as const,
                media_type: content.mimeType,
-                data: content.data.toString("base64"),
+                data: uint8ArrayToBase64(content.data),
              },
            };
          }

-          return {
-            type: "image" as const,
-            source: {
-              type: "base64" as const,
-              media_type: `image/${content.image_url.url.substring(
-                "data:image/".length,
-                content.image_url.url.indexOf(";base64"),
-              )}` as "image/jpeg" | "image/png" | "image/gif" | "image/webp",
-              data: content.image_url.url.substring(
-                content.image_url.url.indexOf(",") + 1,
-              ),
-            },
-          };
+          if (content.type === "image_url") {
+            return {
+              type: "image" as const,
+              source: {
+                type: "base64" as const,
+                media_type: `image/${content.image_url.url.substring(
+                  "data:image/".length,
+                  content.image_url.url.indexOf(";base64"),
+                )}` as "image/jpeg" | "image/png" | "image/gif" | "image/webp",
+                data: content.image_url.url.substring(
+                  content.image_url.url.indexOf(",") + 1,
+                ),
+              },
+            };
+          }
+          throw new Error(`Unsupported content type: ${content.type}`);
        }),
      } satisfies MessageParam;
    });
@@ -1,5 +1,13 @@
 # @llamaindex/assemblyai

+## 0.1.3
+
+### Patch Changes
+
+- Updated dependencies [9b2e25a]
+  - @llamaindex/core@0.6.4
+  - @llamaindex/env@0.1.30
+
 ## 0.1.2

 ### Patch Changes
@@ -1,7 +1,7 @@
 {
  "name": "@llamaindex/assemblyai",
  "description": "AssemblyAI Reader for LlamaIndex",
-  "version": "0.1.2",
+  "version": "0.1.3",
  "type": "module",
  "types": "dist/index.d.ts",
  "main": "dist/index.cjs",
@@ -1,5 +1,14 @@
 # @llamaindex/clip

+## 0.0.54
+
+### Patch Changes
+
+- Updated dependencies [9b2e25a]
+  - @llamaindex/openai@0.3.6
+  - @llamaindex/core@0.6.4
+  - @llamaindex/env@0.1.30
+
 ## 0.0.53

 ### Patch Changes
@@ -1,7 +1,7 @@
 {
  "name": "@llamaindex/clip",
  "description": "Clip Embedding Adapter for LlamaIndex",
-  "version": "0.0.53",
+  "version": "0.0.54",
  "type": "module",
  "types": "dist/index.d.ts",
  "main": "dist/index.cjs",
@@ -1,5 +1,13 @@
 # @llamaindex/cohere

+## 0.0.18
+
+### Patch Changes
+
+- Updated dependencies [9b2e25a]
+  - @llamaindex/core@0.6.4
+  - @llamaindex/env@0.1.30
+
 ## 0.0.17

 ### Patch Changes
@@ -1,7 +1,7 @@
 {
  "name": "@llamaindex/cohere",
  "description": "Cohere Adapter for LlamaIndex",
-  "version": "0.0.17",
+  "version": "0.0.18",
  "type": "module",
  "main": "./dist/index.cjs",
  "module": "./dist/index.js",
@@ -1,5 +1,14 @@
 # @llamaindex/deepinfra

+## 0.0.54
+
+### Patch Changes
+
+- Updated dependencies [9b2e25a]
+  - @llamaindex/openai@0.3.6
+  - @llamaindex/core@0.6.4
+  - @llamaindex/env@0.1.30
+
 ## 0.0.53

 ### Patch Changes
@@ -1,7 +1,7 @@
 {
  "name": "@llamaindex/deepinfra",
  "description": "Deepinfra Adapter for LlamaIndex",
-  "version": "0.0.53",
+  "version": "0.0.54",
  "type": "module",
  "main": "./dist/index.cjs",
  "module": "./dist/index.js",
@@ -1,5 +1,13 @@
 # @llamaindex/deepseek

+## 0.0.14
+
+### Patch Changes
+
+- Updated dependencies [9b2e25a]
+  - @llamaindex/openai@0.3.6
+  - @llamaindex/env@0.1.30
+
 ## 0.0.13

 ### Patch Changes
@@ -1,7 +1,7 @@
 {
  "name": "@llamaindex/deepseek",
  "description": "DeepSeek Adapter for LlamaIndex",
-  "version": "0.0.13",
+  "version": "0.0.14",
  "type": "module",
  "main": "./dist/index.cjs",
  "module": "./dist/index.js",
@@ -1,5 +1,13 @@
 # @llamaindex/discord

+## 0.1.3
+
+### Patch Changes
+
+- Updated dependencies [9b2e25a]
+  - @llamaindex/core@0.6.4
+  - @llamaindex/env@0.1.30
+
 ## 0.1.2

 ### Patch Changes
@@ -1,7 +1,7 @@
 {
  "name": "@llamaindex/discord",
  "description": "Discord Reader for LlamaIndex",
-  "version": "0.1.2",
+  "version": "0.1.3",
  "type": "module",
  "types": "dist/index.d.ts",
  "main": "dist/index.cjs",
@@ -1,5 +1,13 @@
 # @llamaindex/fireworks

+## 0.0.14
+
+### Patch Changes
+
+- Updated dependencies [9b2e25a]
+  - @llamaindex/openai@0.3.6
+  - @llamaindex/env@0.1.30
+
 ## 0.0.13

 ### Patch Changes
@@ -1,7 +1,7 @@
 {
  "name": "@llamaindex/fireworks",
  "description": "Fireworks Adapter for LlamaIndex",
-  "version": "0.0.13",
+  "version": "0.0.14",
  "type": "module",
  "main": "./dist/index.cjs",
  "module": "./dist/index.js",
@@ -1,5 +1,18 @@
 # @llamaindex/google

+## 0.3.0
+
+### Minor Changes
+
+- 206b491: Add support for google live api
+
+### Patch Changes
+
+- 9b2e25a: Use Uint8Array instead of Buffer for file type messages (works with non-NodeJS)
+- Updated dependencies [9b2e25a]
+  - @llamaindex/core@0.6.4
+  - @llamaindex/env@0.1.30
+
 ## 0.2.6

 ### Patch Changes
@@ -1,7 +1,7 @@
 {
  "name": "@llamaindex/google",
  "description": "Google Adapter for LlamaIndex",
-  "version": "0.2.6",
+  "version": "0.3.0",
  "type": "module",
  "main": "./dist/index.cjs",
  "module": "./dist/index.js",
@@ -31,7 +31,7 @@
  },
  "dependencies": {
    "@google-cloud/vertexai": "1.9.0",
-    "@google/genai": "^0.4.0",
+    "@google/genai": "^0.12.0",
    "@google/generative-ai": "0.24.0",
    "@llamaindex/core": "workspace:*",
    "@llamaindex/env": "workspace:*"
@@ -23,6 +23,7 @@ import type {
 import { ToolCallLLM } from "@llamaindex/core/llms";
 import { streamConverter } from "@llamaindex/core/utils";
 import { getEnv, randomUUID } from "@llamaindex/env";
+import { GeminiLive } from "./live.js";
 import {
  GEMINI_BACKENDS,
  GEMINI_MODEL,
@@ -34,6 +35,7 @@ import {
  type GeminiMessageRole,
  type GeminiModelInfo,
  type GeminiSessionOptions,
+  type GeminiVoiceName,
  type GoogleGeminiSessionOptions,
  type IGeminiSession,
 } from "./types.js";
@@ -61,6 +63,7 @@ export const GEMINI_MODEL_INFO_MAP: Record<GEMINI_MODEL, GeminiModelInfo> = {
  [GEMINI_MODEL.GEMINI_2_0_FLASH]: { contextWindow: 10 ** 6 },
  [GEMINI_MODEL.GEMINI_2_0_FLASH_LITE_PREVIEW]: { contextWindow: 10 ** 6 },
  [GEMINI_MODEL.GEMINI_2_0_FLASH_LITE]: { contextWindow: 10 ** 6 },
+  [GEMINI_MODEL.GEMINI_2_0_FLASH_LIVE]: { contextWindow: 10 ** 6 },
  [GEMINI_MODEL.GEMINI_2_0_FLASH_THINKING_EXP]: { contextWindow: 32768 },
  [GEMINI_MODEL.GEMINI_2_0_PRO_EXPERIMENTAL]: { contextWindow: 2 * 10 ** 6 },
  [GEMINI_MODEL.GEMINI_2_5_PRO_PREVIEW]: { contextWindow: 10 ** 6 },
@@ -95,9 +98,11 @@ export const DEFAULT_GEMINI_PARAMS = {
 };

 export type GeminiConfig = Partial<typeof DEFAULT_GEMINI_PARAMS> & {
+  apiKey?: string;
  session?: IGeminiSession;
  requestOptions?: GoogleRequestOptions;
  safetySettings?: SafetySetting[];
+  voiceName?: GeminiVoiceName;
 };

 type StartChatParams = GoogleStartChatParams & VertexStartChatParams;
@@ -231,7 +236,9 @@ export class Gemini extends ToolCallLLM<GeminiAdditionalChatOptions> {
  #requestOptions?: GoogleRequestOptions | undefined;
  session: IGeminiSession;
  safetySettings: SafetySetting[];
-
+  apiKey?: string | undefined;
+  voiceName?: GeminiVoiceName | undefined;
+  private _live: GeminiLive | undefined;
  constructor(init?: GeminiConfig) {
    super();
    this.model = init?.model ?? GEMINI_MODEL.GEMINI_PRO;
@@ -241,12 +248,25 @@ export class Gemini extends ToolCallLLM<GeminiAdditionalChatOptions> {
    this.session = init?.session ?? GeminiSessionStore.get();
    this.#requestOptions = init?.requestOptions ?? undefined;
    this.safetySettings = init?.safetySettings ?? DEFAULT_SAFETY_SETTINGS;
+    this.apiKey = init?.apiKey ?? getEnv("GOOGLE_API_KEY");
+    this.voiceName = init?.voiceName ?? undefined;
  }

  get supportToolCall(): boolean {
    return SUPPORT_TOOL_CALL_MODELS.includes(this.model);
  }

+  get live(): GeminiLive {
+    if (!this._live) {
+      this._live = new GeminiLive({
+        apiKey: this.apiKey,
+        voiceName: this.voiceName,
+        model: this.model,
+      });
+    }
+    return this._live;
+  }
+
  get metadata(): LLMMetadata & { safetySettings: SafetySetting[] } {
    return {
      model: this.model,
@@ -1,12 +1,13 @@
 export * from "./base";
-export * from "./types";
-export * from "./utils";
-export * from "./vertex";

+export * from "./live";
 export {
  GoogleStudio,
  Modality,
  getGoogleStudioInlineData,
 } from "./studio/index.js";
+export * from "./types";
+export * from "./utils";
+export * from "./vertex";

 export * from "./GeminiEmbedding";
@@ -0,0 +1,361 @@
+import {
+  FunctionResponse,
+  GoogleGenAI,
+  Modality,
+  Session,
+  type FunctionCall,
+  type FunctionDeclaration,
+  type LiveConnectConfig as GoogleLiveConnectConfig,
+  type LiveServerMessage,
+} from "@google/genai";
+import {
+  LiveLLM,
+  LiveLLMSession,
+  type BaseTool,
+  type ChatMessage,
+  type LiveConnectConfig,
+  type MessageContentAudioDetail,
+  type MessageContentDetail,
+  type MessageContentImageDataDetail,
+  type MessageContentVideoDetail,
+} from "@llamaindex/core/llms";
+import { getEnv, uint8ArrayToBase64 } from "@llamaindex/env";
+import { GEMINI_MODEL, type GeminiVoiceName } from "./types";
+import {
+  mapBaseToolToGeminiLiveFunctionDeclaration,
+  mapResponseModalityToGeminiLiveResponseModality,
+} from "./utils";
+
+interface GeminiLiveConfig {
+  apiKey?: string | undefined;
+  voiceName?: GeminiVoiceName | undefined;
+  model?: GEMINI_MODEL | undefined;
+}
+
+export class GeminiLiveSession extends LiveLLMSession {
+  session: Session | undefined;
+  closed = false;
+
+  constructor() {
+    super();
+  }
+
+  private isTextEvent(event: LiveServerMessage): boolean {
+    return event.serverContent?.modelTurn?.parts?.[0]?.text !== undefined;
+  }
+
+  private isAudioEvent(event: LiveServerMessage): boolean {
+    return (
+      event.serverContent?.modelTurn?.parts?.[0]?.inlineData?.data !== undefined
+    );
+  }
+
+  private isToolCallEvent(event: LiveServerMessage): boolean {
+    return event.toolCall !== undefined;
+  }
+
+  private isSetupCompleteEvent(event: LiveServerMessage): boolean {
+    return event.setupComplete !== undefined;
+  }
+
+  private isTextMessage(content: MessageContentDetail) {
+    return content.type === "text";
+  }
+
+  private isAudioMessage(content: MessageContentDetail) {
+    return content.type === "audio";
+  }
+
+  private isImageMessage(content: MessageContentDetail) {
+    return content.type === "image";
+  }
+
+  private isVideoMessage(content: MessageContentDetail) {
+    return content.type === "video";
+  }
+
+  //for the tool call event, we need to return the response with function responses
+  private async handleToolCallEvent(
+    event: LiveServerMessage,
+    toolCalls: BaseTool[],
+  ) {
+    const eventToolCalls = event.toolCall?.functionCalls;
+
+    if (eventToolCalls) {
+      await this.sendToolCallResponses(eventToolCalls, toolCalls);
+    }
+  }
+
+  handleLiveEvents(event: LiveServerMessage, toolCalls: BaseTool[]) {
+    if (this.isTextEvent(event)) {
+      this.pushEventToQueue({
+        type: "text",
+        text: event.serverContent?.modelTurn?.parts?.[0]?.text || "",
+      });
+    }
+    if (this.isSetupCompleteEvent(event)) {
+      this.pushEventToQueue({
+        type: "setupComplete",
+      });
+    }
+    if (this.isAudioEvent(event)) {
+      this.pushEventToQueue({
+        type: "audio",
+        data:
+          event.serverContent?.modelTurn?.parts?.[0]?.inlineData?.data || "",
+        mimeType:
+          event.serverContent?.modelTurn?.parts?.[0]?.inlineData?.mimeType ||
+          "audio/wav",
+      });
+    }
+    if (this.isToolCallEvent(event)) {
+      this.handleToolCallEvent(event, toolCalls);
+    }
+  }
+
+  private executeToolCall(toolCall: FunctionCall, tool: BaseTool) {
+    return tool.call!(toolCall.args);
+  }
+
+  private storeToolCallResponse(
+    toolCall: FunctionCall,
+    response: unknown,
+    functionResponses: FunctionResponse[],
+  ) {
+    functionResponses.push({
+      id: toolCall.id || "",
+      name: toolCall.name || "",
+      response:
+        typeof response === "string"
+          ? { result: response }
+          : (response as Record<string, unknown>),
+    });
+  }
+
+  private async executeToolCallsAndStoreResponses(
+    eventToolCalls: FunctionCall[],
+    toolCalls: BaseTool[],
+  ) {
+    const functionResponses: FunctionResponse[] = [];
+
+    for (const toolCall of eventToolCalls) {
+      const tool = toolCalls.find((t) => t.metadata.name === toolCall.name);
+      if (tool && tool.call) {
+        const response = await this.executeToolCall(toolCall, tool);
+        this.storeToolCallResponse(toolCall, response, functionResponses);
+      }
+    }
+
+    return functionResponses;
+  }
+
+  //execute the tool call and send the response to the server
+  private async sendToolCallResponses(
+    eventToolCalls: FunctionCall[],
+    toolCalls: BaseTool[],
+  ) {
+    let functionResponses: FunctionResponse[] = [];
+
+    if (eventToolCalls) {
+      functionResponses = await this.executeToolCallsAndStoreResponses(
+        eventToolCalls,
+        toolCalls,
+      );
+      //send the function responses to the gemini
+      this.session?.sendToolResponse({
+        functionResponses,
+      });
+    }
+  }
+
+  private sendTextMessage(message: string, role?: string) {
+    this.session?.sendClientContent({
+      turns: [
+        {
+          parts: [{ text: message }],
+          ...(role ? { role } : {}),
+        },
+      ],
+    });
+  }
+
+  private sendAudioMessage(content: MessageContentAudioDetail, role?: string) {
+    if (typeof content.data === "string") {
+      this.session?.sendRealtimeInput({
+        audio: {
+          data: content.data,
+          mimeType: content.mimeType,
+        },
+      });
+    } else {
+      this.session?.sendRealtimeInput({
+        audio: {
+          data: uint8ArrayToBase64(content.data),
+          mimeType: content.mimeType,
+        },
+      });
+    }
+  }
+
+  private sendImageMessage(
+    content: MessageContentImageDataDetail,
+    role?: string,
+  ) {
+    if (typeof content.data === "string") {
+      this.session?.sendRealtimeInput({
+        media: {
+          data: content.data,
+          mimeType: content.mimeType,
+        },
+      });
+    } else {
+      this.session?.sendRealtimeInput({
+        media: {
+          data: uint8ArrayToBase64(content.data),
+          mimeType: content.mimeType,
+        },
+      });
+    }
+  }
+
+  private sendVideoMessage(content: MessageContentVideoDetail, role?: string) {
+    if (typeof content.data === "string") {
+      this.session?.sendRealtimeInput({
+        video: {
+          data: content.data,
+          mimeType: content.mimeType,
+        },
+      });
+    } else {
+      this.session?.sendRealtimeInput({
+        video: {
+          data: uint8ArrayToBase64(content.data),
+          mimeType: content.mimeType,
+        },
+      });
+    }
+  }
+
+  private handleUserInput(message: ChatMessage) {
+    const { content, role } = message;
+
+    if (!Array.isArray(content)) {
+      this.sendTextMessage(content, role);
+    } else {
+      for (const item of content) {
+        if (this.isTextMessage(item)) {
+          this.sendTextMessage(item.text, role);
+        } else if (this.isAudioMessage(item)) {
+          this.sendAudioMessage(item as MessageContentAudioDetail, role);
+        } else if (this.isImageMessage(item)) {
+          this.sendImageMessage(item as MessageContentImageDataDetail, role);
+        } else if (this.isVideoMessage(item)) {
+          this.sendVideoMessage(item as MessageContentVideoDetail, role);
+        }
+      }
+    }
+  }
+
+  sendMessage(message: ChatMessage) {
+    if (!this.session) {
+      throw new Error("Session not connected");
+    }
+    this.handleUserInput(message);
+  }
+
+  async disconnect() {
+    if (!this.session) {
+      throw new Error("Session not connected");
+    }
+    this.session.close();
+  }
+}
+export class GeminiLive extends LiveLLM {
+  private apiKey: string | undefined;
+  private client: GoogleGenAI;
+  voiceName?: GeminiVoiceName | undefined;
+  model: GEMINI_MODEL;
+
+  constructor(init?: GeminiLiveConfig) {
+    super();
+    this.apiKey = init?.apiKey ?? getEnv("GOOGLE_API_KEY");
+
+    if (!this.apiKey) {
+      throw new Error("GOOGLE_API_KEY is not set");
+    }
+
+    this.client = new GoogleGenAI({
+      apiKey: this.apiKey,
+    });
+    this.voiceName = init?.voiceName;
+    this.model = init?.model ?? GEMINI_MODEL.GEMINI_2_0_FLASH_LIVE;
+    /* Only 2.0 flash live is supported for live mode */
+    if (this.model !== GEMINI_MODEL.GEMINI_2_0_FLASH_LIVE) {
+      throw new Error("Only GEMINI_2_0_FLASH_LIVE is supported for live mode");
+    }
+  }
+
+  async connect(config?: LiveConnectConfig) {
+    const liveConfig: GoogleLiveConnectConfig = {
+      responseModalities: config?.responseModality
+        ? config.responseModality.map(
+            mapResponseModalityToGeminiLiveResponseModality,
+          )
+        : [Modality.AUDIO],
+    };
+
+    if (config?.tools) {
+      const tools = config.tools.map(
+        mapBaseToolToGeminiLiveFunctionDeclaration,
+      );
+      liveConfig.tools = [
+        {
+          functionDeclarations: tools as FunctionDeclaration[],
+        },
+      ];
+    }
+
+    if (config?.systemInstruction) {
+      liveConfig.systemInstruction = config.systemInstruction;
+    }
+
+    if (this.voiceName) {
+      liveConfig.speechConfig = {
+        voiceConfig: {
+          prebuiltVoiceConfig: {
+            voiceName: this.voiceName,
+          },
+        },
+      };
+    }
+
+    const geminiLiveSession = new GeminiLiveSession();
+
+    geminiLiveSession.session = await this.client.live.connect({
+      model: this.model,
+      config: {
+        ...liveConfig,
+      },
+
+      callbacks: {
+        onmessage: (event) => {
+          geminiLiveSession.handleLiveEvents(event, config?.tools || []);
+        },
+        onerror: (error) => {
+          geminiLiveSession.pushEventToQueue({
+            type: "error",
+            error: error.error,
+          });
+        },
+        onopen: () => {
+          geminiLiveSession.pushEventToQueue({ type: "open" });
+        },
+        onclose: () => {
+          geminiLiveSession.pushEventToQueue({ type: "close" });
+        },
+      },
+    });
+
+    return geminiLiveSession;
+  }
+}
@@ -1,6 +1,6 @@
 import type {
+  Content,
  ContentListUnion,
-  ContentUnion,
  Part,
  Schema,
  ToolListUnion,
@@ -95,42 +95,53 @@ export const mapChatMessagesToGoogleMessages = <
      Object.assign(functionNames, mapped);
    }
  });
-  return messages.flatMap((msg: T): ContentListUnion => {
+
+  // Transform messages to Google API format
+  const contents = messages.flatMap((msg: T) => {
    if (msg.options && "toolResult" in msg.options) {
-      return {
-        role: "user",
-        parts: [
-          {
-            functionResponse: {
-              name: functionNames[msg.options.toolResult.id] ?? "",
-              response: msg.options.toolResult,
+      return [
+        {
+          role: "user",
+          parts: [
+            {
+              functionResponse: {
+                name: functionNames[msg.options.toolResult.id] ?? "",
+                response: msg.options.toolResult,
+              },
            },
-          },
-        ],
-      };
+          ],
+        },
+      ];
    }

    if (msg.options && "toolCall" in msg.options) {
-      return {
-        role: "model",
-        parts: msg.options.toolCall.map((call) => ({
-          functionCall: {
-            name: call.name,
-            args: call.input as Record<string, unknown>,
-          },
-        })),
-      };
+      return [
+        {
+          role: "model",
+          parts: msg.options.toolCall.map((call) => ({
+            functionCall: {
+              name: call.name,
+              args: call.input as Record<string, unknown>,
+            },
+          })),
+        },
+      ];
    }
-    return mapMessageContentToMessageContentDetails(msg.content)
-      .map((detail: MessageContentDetail): ContentUnion | null => {
+
+    const mapped = mapMessageContentToMessageContentDetails(msg.content)
+      .map((detail: MessageContentDetail) => {
        const part = mapMessageContentDetailToGooglePart(detail);
        if (!part.text && !part.inlineData) return null;

        return {
          role: msg.role === "assistant" ? "model" : "user",
          parts: [part],
-        };
+        } as Content;
      })
-      .filter((content) => content) as ContentUnion;
+      .filter((content): content is Content => content !== null);
+
+    return mapped;
  });
+
+  return contents;
 };
@@ -73,6 +73,7 @@ export enum GEMINI_MODEL {
  GEMINI_2_0_FLASH_LITE_PREVIEW = "gemini-2.0-flash-lite-preview-02-05",
  GEMINI_2_0_FLASH_THINKING_EXP = "gemini-2.0-flash-thinking-exp-01-21",
  GEMINI_2_0_PRO_EXPERIMENTAL = "gemini-2.0-pro-exp-02-05",
+  GEMINI_2_0_FLASH_LIVE = "gemini-2.0-flash-live-001",
  GEMINI_2_5_PRO_PREVIEW = "gemini-2.5-pro-preview-03-25",
  GEMINI_2_5_PRO_PREVIEW_LATEST = "gemini-2.5-pro-preview-05-06",
  GEMINI_2_5_FLASH_PREVIEW = "gemini-2.5-flash-preview-04-17",
@@ -148,3 +149,24 @@ export interface IGeminiSession {
    response: EnhancedGenerateContentResponse | GenerateContentResponse,
  ): ToolCall[] | undefined;
 }
+
+export type GeminiLiveMessage = {
+  content: string | GeminiLiveMessageDetail;
+  role: "user" | "model";
+};
+
+export type GeminiLiveMessageDetail = {
+  type: "text" | "audio" | "image" | "video";
+  data: string;
+  mimeType: string;
+};
+
+export type GeminiVoiceName =
+  | "Puck"
+  | "Charon"
+  | "Fenrir"
+  | "Aoede"
+  | "Leda"
+  | "Kore"
+  | "Orus"
+  | "Zephyr";
@@ -1,3 +1,10 @@
+import { type GenerateContentResponse } from "@google-cloud/vertexai";
+import {
+  type FunctionDeclaration as LiveFunctionDeclaration,
+  Modality,
+  type Schema,
+  Type,
+} from "@google/genai";
 import {
  type FunctionCall,
  type Content as GeminiMessageContent,
@@ -6,8 +13,6 @@ import {
  type SafetySetting,
  SchemaType,
 } from "@google/generative-ai";
-
-import { type GenerateContentResponse } from "@google-cloud/vertexai";
 import { FileState, GoogleAIFileManager } from "@google/generative-ai/server";
 import type {
  BaseTool,
@@ -18,6 +23,7 @@ import type {
  MessageType,
  ToolCallLLMMessageOptions,
 } from "@llamaindex/core/llms";
+import { ModalityType } from "@llamaindex/core/schema";
 import { extractDataUrlComponents } from "@llamaindex/core/utils";
 import { getEnv } from "@llamaindex/env";
 import type {
@@ -180,6 +186,39 @@ export const mapBaseToolToGeminiFunctionDeclaration = (
  };
 };

+/**
+ * Maps a BaseTool to a Gemini Live Function Declaration format
+ * Used for converting LlamaIndex tools to be compatible with Gemini's live API function calling
+ *
+ * @param tool - The BaseTool to convert
+ * @returns A LiveFunctionDeclaration object that can be used with Gemini's live API
+ */
+export const mapBaseToolToGeminiLiveFunctionDeclaration = (
+  tool: BaseTool,
+): LiveFunctionDeclaration => {
+  const parameters: Schema = {
+    type: tool.metadata.parameters?.type.toLowerCase() as Type,
+    properties: tool.metadata.parameters?.properties,
+    description: tool.metadata.parameters?.description,
+    required: tool.metadata.parameters?.required,
+  };
+  return {
+    name: tool.metadata.name,
+    description: tool.metadata.description,
+    parameters,
+  };
+};
+
+export const mapResponseModalityToGeminiLiveResponseModality = (
+  responseModality: ModalityType,
+): Modality => {
+  return responseModality === ModalityType.TEXT
+    ? Modality.TEXT
+    : responseModality === ModalityType.AUDIO
+      ? Modality.AUDIO
+      : Modality.IMAGE;
+};
+
 /**
 * Helper class providing utility functions for Gemini
 */
@@ -289,7 +328,7 @@ export class GeminiHelper {
    if (fileContents.length > 0) {
      for (const file of fileContents) {
        const uploadResponse = await GeminiHelper.uploadFile(
-          file.data,
+          Buffer.from(file.data),
          file.mimeType,
        );
        parts.push({
@@ -1,5 +1,13 @@
 # @llamaindex/groq

+## 0.0.69
+
+### Patch Changes
+
+- Updated dependencies [9b2e25a]
+  - @llamaindex/openai@0.3.6
+  - @llamaindex/env@0.1.30
+
 ## 0.0.68

 ### Patch Changes
@@ -1,7 +1,7 @@
 {
  "name": "@llamaindex/groq",
  "description": "Groq Adapter for LlamaIndex",
-  "version": "0.0.68",
+  "version": "0.0.69",
  "type": "module",
  "main": "./dist/index.cjs",
  "module": "./dist/index.js",
@@ -1,5 +1,14 @@
 # @llamaindex/huggingface

+## 0.1.8
+
+### Patch Changes
+
+- Updated dependencies [9b2e25a]
+  - @llamaindex/openai@0.3.6
+  - @llamaindex/core@0.6.4
+  - @llamaindex/env@0.1.30
+
 ## 0.1.7

 ### Patch Changes
@@ -1,7 +1,7 @@
 {
  "name": "@llamaindex/huggingface",
  "description": "Huggingface Adapter for LlamaIndex",
-  "version": "0.1.7",
+  "version": "0.1.8",
  "type": "module",
  "types": "dist/index.d.ts",
  "main": "dist/index.cjs",
@@ -1,5 +1,14 @@
 # @llamaindex/jinaai

+## 0.0.14
+
+### Patch Changes
+
+- Updated dependencies [9b2e25a]
+  - @llamaindex/openai@0.3.6
+  - @llamaindex/core@0.6.4
+  - @llamaindex/env@0.1.30
+
 ## 0.0.13

 ### Patch Changes
@@ -1,7 +1,7 @@
 {
  "name": "@llamaindex/jinaai",
  "description": "JinaAI Adapter for LlamaIndex",
-  "version": "0.0.13",
+  "version": "0.0.14",
  "type": "module",
  "main": "./dist/index.cjs",
  "module": "./dist/index.js",
@@ -1,5 +1,13 @@
 # @llamaindex/mistral

+## 0.1.4
+
+### Patch Changes
+
+- Updated dependencies [9b2e25a]
+  - @llamaindex/core@0.6.4
+  - @llamaindex/env@0.1.30
+
 ## 0.1.3

 ### Patch Changes
@@ -1,7 +1,7 @@
 {
  "name": "@llamaindex/mistral",
  "description": "Mistral Adapter for LlamaIndex",
-  "version": "0.1.3",
+  "version": "0.1.4",
  "type": "module",
  "main": "./dist/index.cjs",
  "module": "./dist/index.js",
@@ -1,5 +1,13 @@
 # @llamaindex/mixedbread

+## 0.0.18
+
+### Patch Changes
+
+- Updated dependencies [9b2e25a]
+  - @llamaindex/core@0.6.4
+  - @llamaindex/env@0.1.30
+
 ## 0.0.17

 ### Patch Changes
@@ -1,7 +1,7 @@
 {
  "name": "@llamaindex/mixedbread",
  "description": "Mixedbread Adapter for LlamaIndex",
-  "version": "0.0.17",
+  "version": "0.0.18",
  "type": "module",
  "main": "./dist/index.cjs",
  "module": "./dist/index.js",
@@ -1,5 +1,13 @@
 # @llamaindex/notion

+## 0.1.3
+
+### Patch Changes
+
+- Updated dependencies [9b2e25a]
+  - @llamaindex/core@0.6.4
+  - @llamaindex/env@0.1.30
+
 ## 0.1.2

 ### Patch Changes
@@ -1,7 +1,7 @@
 {
  "name": "@llamaindex/notion",
  "description": "Notion Reader for LlamaIndex",
-  "version": "0.1.2",
+  "version": "0.1.3",
  "type": "module",
  "types": "dist/index.d.ts",
  "main": "dist/index.cjs",
@@ -1,5 +1,13 @@
 # @llamaindex/ollama

+## 0.1.4
+
+### Patch Changes
+
+- Updated dependencies [9b2e25a]
+  - @llamaindex/core@0.6.4
+  - @llamaindex/env@0.1.30
+
 ## 0.1.3

 ### Patch Changes
@@ -1,7 +1,7 @@
 {
  "name": "@llamaindex/ollama",
  "description": "Ollama Adapter for LlamaIndex",
-  "version": "0.1.3",
+  "version": "0.1.4",
  "type": "module",
  "main": "./dist/index.cjs",
  "module": "./dist/index.js",
@@ -1,5 +1,14 @@
 # @llamaindex/openai

+## 0.3.6
+
+### Patch Changes
+
+- 9b2e25a: Use Uint8Array instead of Buffer for file type messages (works with non-NodeJS)
+- Updated dependencies [9b2e25a]
+  - @llamaindex/core@0.6.4
+  - @llamaindex/env@0.1.30
+
 ## 0.3.5

 ### Patch Changes
@@ -1,7 +1,7 @@
 {
  "name": "@llamaindex/openai",
  "description": "OpenAI Adapter for LlamaIndex",
-  "version": "0.3.5",
+  "version": "0.3.6",
  "type": "module",
  "main": "./dist/index.cjs",
  "module": "./dist/index.js",
@@ -13,7 +13,7 @@ import {
  type ToolCallLLMMessageOptions,
 } from "@llamaindex/core/llms";
 import { extractText } from "@llamaindex/core/utils";
-import { getEnv } from "@llamaindex/env";
+import { getEnv, uint8ArrayToBase64 } from "@llamaindex/env";
 import { Tokenizers } from "@llamaindex/env/tokenizers";
 import type {
  AzureClientOptions,
@@ -213,22 +213,44 @@ export class OpenAI extends ToolCallLLM<OpenAIAdditionalChatOptions> {
        return {
          role: "user",
          content: message.content.map((item, index) => {
+            // Handle MessageContentMediaDetail (audio, video, image)
+            if (
+              "data" in item &&
+              "mimeType" in item &&
+              (item.type === "audio" ||
+                item.type === "video" ||
+                item.type === "image")
+            ) {
+              if (item.type === "audio" || item.type === "video") {
+                throw new Error("Audio and video are not supported");
+              }
+              // Convert image type to file format for OpenAI
+              return {
+                type: "file",
+                file: {
+                  file_data: `data:${item.mimeType};base64,${item.data}`,
+                  filename: `image-${index}.${item.mimeType.split("/")[1] || "png"}`,
+                },
+              } satisfies ChatCompletionContentPart.File;
+            }
+
            if (item.type === "file") {
              if (item.mimeType !== "application/pdf") {
                throw new Error("Only PDF files are supported");
              }
+              const base64Data = uint8ArrayToBase64(item.data);
              return {
                type: "file",
                file: {
-                  file_data: `data:${item.mimeType};base64,${item.data.toString("base64")}`,
+                  file_data: `data:${item.mimeType};base64,${base64Data}`,
                  filename: `part-${index}.pdf`,
                },
              } satisfies ChatCompletionContentPart.File;
            }

-            // keep it as is for other types
+            // Keep other types as is (text, image_url, etc.)
            return item;
-          }),
+          }) as ChatCompletionContentPart[],
        } satisfies ChatCompletionUserMessageParam;
      }

@@ -16,7 +16,7 @@ import {
 } from "@llamaindex/core/llms";
 import type { StoredValue } from "@llamaindex/core/schema";
 import { extractText } from "@llamaindex/core/utils";
-import { getEnv } from "@llamaindex/env";
+import { getEnv, uint8ArrayToBase64 } from "@llamaindex/env";
 import {
  OpenAI as OpenAILLM,
  type AzureClientOptions,
@@ -703,10 +703,11 @@ export class OpenAIResponses extends ToolCallLLM<OpenAIResponsesChatOptions> {
          );
        }

+        const base64Data = uint8ArrayToBase64(item.data);
        return {
          type: "input_file",
          filename: `part-${index}.pdf`,
-          file_data: `data:${item.mimeType};base64,${item.data.toString("base64")}`,
+          file_data: `data:${item.mimeType};base64,${base64Data}`,
        };
      }
      throw new Error("Unsupported content type");
@@ -205,7 +205,7 @@ describe("OpenAIResponses Unit Tests", () => {
        {
          type: "file",
          mimeType: "image/jpeg",
-          data: Buffer.from("test image content"),
+          data: Uint8Array.from(Buffer.from("test image content")),
        },
      ];
      // @ts-expect-error accessing private method
@@ -1,5 +1,14 @@
 # @llamaindex/perplexity

+## 0.0.11
+
+### Patch Changes
+
+- Updated dependencies [9b2e25a]
+  - @llamaindex/openai@0.3.6
+  - @llamaindex/core@0.6.4
+  - @llamaindex/env@0.1.30
+
 ## 0.0.10

 ### Patch Changes
--- a/Show More
+++ b/Show More
Author	SHA1	Message	Date
github-actions[bot]	4a18a2eb3d	Release 0.10.5 (#1922 ) Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: marcusschiesser <17126+marcusschiesser@users.noreply.github.com>	2025-05-09 14:30:39 +07:00
ANKIT VARSHNEY	206b491724	feat: Support for google live api (#1905 )	2025-05-09 14:20:40 +07:00
Marcus Schiesser	9b2e25a184	fix: Use Uint8Array instead of Buffer for file type messages (works w… (#1921 )	2025-05-08 13:19:59 +07:00