feat: Support for google live api (#1905)

.changeset/ten-rice-melt.md

@@ -0,0 +1,6 @@
+---
+"@llamaindex/google": minor
+"@llamaindex/examples": patch
+---
+
+Add support for google live api

examples/gemini/gemini-live-tool-call.ts

@@ -0,0 +1,104 @@
+import { ModalityType } from "@llamaindex/core/schema";
+import { tool } from "@llamaindex/core/tools";
+import { gemini, GEMINI_MODEL } from "@llamaindex/google";
+
+import { liveEvents } from "llamaindex";
+import { z } from "zod";
+
+const weatherTool = tool({
+  name: "weather",
+  description: "Get the weather",
+  parameters: z.object({
+    location: z.string({
+      description: "The location to get the weather for",
+    }),
+  }),
+  execute: ({ location }) => {
+    return `The weather in ${location} is rainy`;
+  },
+});
+
+const divideNumbers = tool({
+  name: "divideNumbers",
+  description: "Use this function to divide two numbers",
+  parameters: z.object({
+    a: z.number().describe("The dividend a to divide"),
+    b: z.number().describe("The divisor b to divide by"),
+  }),
+  execute: ({ a, b }) => `${a / b}`,
+});
+
+async function main() {
+  const apiKey = process.env.GOOGLE_API_KEY;
+  if (!apiKey) {
+    console.error(
+      "Please set GOOGLE_API_KEY in your environment variables or .env file",
+    );
+    process.exit(1);
+  }
+
+  console.log("🚀 Initializing Gemini Live API with tools example...");
+
+  const llm = gemini({
+    apiKey: apiKey,
+    model: GEMINI_MODEL.GEMINI_2_0_FLASH_LIVE, // Must use a live-compatible model
+  });
+
+  console.log("📡 Connecting to Gemini Live session...");
+
+  // Connect to a live session with tools
+  const session = await llm.live.connect({
+    // Specify response modalities (text response is required for tools)
+    responseModality: [ModalityType.TEXT],
+    // Register our tools with the session
+    tools: [weatherTool, divideNumbers],
+    // Optional system instruction
+    systemInstruction:
+      "You are a helpful assistant that can use tools. When answering questions about weather or divide numbers, always use the appropriate tool.",
+  });
+
+  (async () => {
+    console.log("🎧 Listening for events...");
+
+    for await (const event of session.streamEvents()) {
+      if (liveEvents.open.include(event)) {
+        console.log("✅ Connected to Gemini Live session");
+
+        console.log(
+          "💬 Sending message: 'What's the weather in San Francisco and what is 100 / 2?'",
+        );
+        session.sendMessage({
+          content: "What's the weather in San Francisco and what is 100 / 2?",
+          role: "user",
+        });
+      } else if (liveEvents.text.include(event)) {
+        process.stdout.write(event.text);
+      } else if (liveEvents.error.include(event)) {
+        console.error("❌ Error:", event.error);
+      } else if (liveEvents.close.include(event)) {
+        console.log("👋 Session closed");
+        process.exit(0);
+      } else if (liveEvents.setupComplete.include(event)) {
+        console.log("🔧 Setup complete");
+      }
+    }
+  })();
+
+  process.on("SIGINT", async () => {
+    console.log("\n👋 Interrupted by user. Closing session...");
+    await session.disconnect();
+    process.exit(0);
+  });
+
+  // Timeout after 2 minutes if no interaction
+  setTimeout(async () => {
+    console.log("\n⏱️ Session timeout. Closing session...");
+    await session.disconnect();
+    process.exit(0);
+  }, 120000);
+}
+
+main().catch((error) => {
+  console.error("❌ Fatal error:", error);
+  process.exit(1);
+});

examples/gemini/gemini-live.ts

@@ -0,0 +1,221 @@
+import { fs } from "@llamaindex/env";
+import { gemini, GEMINI_MODEL, GeminiLiveSession } from "@llamaindex/google";
+import { liveEvents } from "llamaindex";
+
+import path from "path";
+
+function createWavHeader(
+  sampleRate = 16000,
+  bitsPerSample = 16,
+  channels = 1,
+  dataLength: number,
+) {
+  const buffer = Buffer.alloc(44);
+
+  // RIFF chunk descriptor
+  buffer.write("RIFF", 0);
+  buffer.writeUInt32LE(36 + dataLength, 4); // File size - 8
+  buffer.write("WAVE", 8);
+
+  // fmt sub-chunk
+  buffer.write("fmt ", 12);
+  buffer.writeUInt32LE(16, 16); // Subchunk1Size (16 for PCM)
+  buffer.writeUInt16LE(1, 20); // AudioFormat (1 for PCM)
+  buffer.writeUInt16LE(channels, 22); // NumChannels
+  buffer.writeUInt32LE(sampleRate, 24); // SampleRate
+  buffer.writeUInt32LE((sampleRate * channels * bitsPerSample) / 8, 28); // ByteRate
+  buffer.writeUInt16LE((channels * bitsPerSample) / 8, 32); // BlockAlign
+  buffer.writeUInt16LE(bitsPerSample, 34); // BitsPerSample
+
+  // data sub-chunk
+  buffer.write("data", 36);
+  buffer.writeUInt32LE(dataLength, 40); // Subchunk2Size
+
+  return buffer;
+}
+
+async function saveWavFile(
+  audioChunks: Buffer[],
+  filePath: string,
+  sampleRate = 16000,
+  bitsPerSample = 16,
+  channels = 1,
+): Promise<void> {
+  if (audioChunks.length === 0) {
+    throw new Error("No audio data to save");
+  }
+
+  try {
+    const combinedAudioData = Buffer.concat(audioChunks);
+    console.log(`Total audio data: ${combinedAudioData.length} bytes`);
+
+    const wavHeader = createWavHeader(
+      sampleRate,
+      bitsPerSample,
+      channels,
+      combinedAudioData.length,
+    );
+    const wavFile = Buffer.concat([wavHeader, combinedAudioData]);
+
+    await fs.writeFile(filePath, wavFile);
+    console.log(`💾 Saved audio to ${filePath}`);
+    return;
+  } catch (error) {
+    console.error("❌ Error saving audio file:", error);
+    throw error;
+  }
+}
+
+async function main() {
+  const apiKey = process.env.GOOGLE_API_KEY;
+  if (!apiKey) {
+    console.error(
+      "Please set GOOGLE_API_KEY in your environment variables or .env file",
+    );
+    process.exit(1);
+  }
+
+  console.log("🚀 Initializing Gemini Live API example...");
+
+  const llm = gemini({
+    model: GEMINI_MODEL.GEMINI_2_0_FLASH_LIVE,
+    voiceName: "Zephyr",
+  });
+
+  console.log("📡 Connecting to Gemini Live session...");
+
+  const session = await llm.live.connect();
+
+  let isRunning = true;
+
+  const audioChunks: Buffer[] = [];
+  let audioResponse = false;
+
+  (async () => {
+    try {
+      console.log("🎧 Listening for events...");
+
+      for await (const event of session.streamEvents()) {
+        if (liveEvents.open.include(event)) {
+          console.log("✅ Connected to Gemini Live session");
+
+          console.log(
+            "💬 Sending text message: 'Say something about you for 10 seconds'",
+          );
+          session.sendMessage({
+            content: "Say something about you for 10 seconds",
+            role: "user",
+          });
+
+          setTimeout(() => {
+            sendPcmAudioFile(session);
+          }, 3000);
+        } else if (liveEvents.setupComplete.include(event)) {
+          console.log("✅ Setup complete");
+        } else if (liveEvents.text.include(event)) {
+          process.stdout.write(event.text);
+        } else if (liveEvents.audio.include(event)) {
+          console.log("\n🔊 Received audio chunk");
+          audioResponse = true;
+
+          try {
+            const chunk = Buffer.from(event.data as string, "base64");
+            audioChunks.push(chunk);
+            console.log(`Received audio chunk: ${chunk.length} bytes`);
+          } catch (error) {
+            console.error("❌ Error processing audio chunk:", error);
+          }
+        } else if (liveEvents.error.include(event)) {
+          console.error("❌ Error:", event.error);
+        } else if (liveEvents.close.include(event)) {
+          console.log("👋 Session closed");
+
+          if (audioResponse && audioChunks.length > 0) {
+            try {
+              await saveWavFile(audioChunks, "gemini-response.wav");
+            } catch (error) {
+              console.error("❌ Error saving final audio file:", error);
+            }
+          }
+
+          isRunning = false;
+          break;
+        }
+      }
+    } catch (error) {
+      console.error("❌ Error processing stream:", error);
+    }
+  })();
+
+  async function sendPcmAudioFile(session: GeminiLiveSession) {
+    try {
+      console.log("🎤 Reading PCM audio file...");
+
+      const filePath = path.join(__dirname, "hello_are_you_there.pcm");
+      console.log(`Reading file from: ${filePath}`);
+
+      const audioBuffer = await fs.readFile(filePath);
+
+      const base64Audio = audioBuffer.toString("base64");
+
+      session.sendMessage({
+        content: [
+          {
+            type: "audio",
+            data: base64Audio,
+            mimeType: "audio/pcm;rate=16000",
+          },
+        ],
+        role: "user",
+      });
+
+      console.log("🎤 PCM audio file sent! Waiting for response...");
+    } catch (error) {
+      console.error("❌ Error sending audio file:", error);
+    }
+  }
+
+  setTimeout(async () => {
+    console.log("\n⏱️ Time's up! Closing session...");
+
+    if (audioResponse && audioChunks.length > 0) {
+      try {
+        await saveWavFile(audioChunks, "gemini-response.wav");
+      } catch (error) {
+        console.error("❌ Error saving final audio file:", error);
+      }
+    }
+
+    await session.disconnect();
+    isRunning = false;
+  }, 60000);
+
+  process.on("SIGINT", async () => {
+    console.log("\n👋 Interrupted by user. Closing session...");
+
+    if (audioResponse && audioChunks.length > 0) {
+      try {
+        await saveWavFile(audioChunks, "gemini-response.wav");
+      } catch (error) {
+        console.error("❌ Error saving final audio file:", error);
+      }
+    }
+
+    await session.disconnect();
+    isRunning = false;
+  });
+
+  const waitForClose = () => {
+    if (isRunning) {
+      setTimeout(waitForClose, 1000);
+    } else {
+      process.exit(0);
+    }
+  };
+  waitForClose();
+}
+
+main().catch((error) => {
+  console.error("❌ Fatal error:", error);
+  process.exit(1);
+});

examples/multimodal/context.ts

@@ -37,8 +37,10 @@ async function main() {
   const index = await VectorStoreIndex.init({
     storageContext,
   });
-  // topK for text is 0 and for image 1 => we only retrieve one image and no text based on the query
-  const retriever = index.asRetriever({ topK: { TEXT: 0, IMAGE: 1 } });
+  // topK for text is 0, for image 1, for audio 0  => we only retrieve one image and no text based on the query
+  const retriever = index.asRetriever({
+    topK: { TEXT: 0, IMAGE: 1, AUDIO: 0 },
+  });
   // NOTE: we set the contextRole to "user" (default is "system"). The reason is that GPT-4 does not support
   // images in a system message
   const chatEngine = new ContextChatEngine({ retriever, contextRole: "user" });

examples/multimodal/rag.ts

@@ -30,7 +30,7 @@ async function main() {
 
   const queryEngine = index.asQueryEngine({
     responseSynthesizer: getResponseSynthesizer("multi_modal"),
-    retriever: index.asRetriever({ topK: { TEXT: 3, IMAGE: 1 } }),
+    retriever: index.asRetriever({ topK: { TEXT: 3, IMAGE: 1, AUDIO: 0 } }),
   });
   const stream = await queryEngine.query({
     query: "Tell me more about Vincent van Gogh's famous paintings",

examples/multimodal/retrieve.ts

@@ -12,7 +12,9 @@ async function main() {
     nodes: [],
     storageContext,
   });
-  const retriever = index.asRetriever({ topK: { TEXT: 1, IMAGE: 3 } });
+  const retriever = index.asRetriever({
+    topK: { TEXT: 1, IMAGE: 3, AUDIO: 0 },
+  });
   const results = await retriever.retrieve({
     query: "what are Vincent van Gogh's famous paintings",
   });

packages/core/src/llms/index.ts

@@ -1,4 +1,5 @@
 export { BaseLLM, ToolCallLLM } from "./base";
+export { LiveLLM, LiveLLMSession, liveEvents, type LiveEvent } from "./live";
 export type {
   BaseTool,
   BaseToolWithCall,
@@ -15,11 +16,15 @@ export type {
   LLMCompletionParamsNonStreaming,
   LLMCompletionParamsStreaming,
   LLMMetadata,
+  LiveConnectConfig,
   MessageContent,
+  MessageContentAudioDetail,
   MessageContentDetail,
   MessageContentFileDetail,
+  MessageContentImageDataDetail,
   MessageContentImageDetail,
   MessageContentTextDetail,
+  MessageContentVideoDetail,
   MessageType,
   PartialToolCall,
   TextChatMessage,

packages/core/src/llms/live.ts

@@ -0,0 +1,87 @@
+import type {
+  ChatMessage,
+  LiveConnectConfig,
+  MessageContentAudioDetail,
+  MessageContentTextDetail,
+} from "./type";
+
+export type OpenEvent = { type: "open" };
+
+export type AudioEvent = MessageContentAudioDetail;
+
+export type TextEvent = MessageContentTextDetail;
+
+export type ErrorEvent = { type: "error"; error: unknown };
+
+export type CloseEvent = { type: "close" };
+
+export type SetupCompleteEvent = { type: "setupComplete" };
+
+export type LiveEvent =
+  | OpenEvent
+  | AudioEvent
+  | TextEvent
+  | ErrorEvent
+  | CloseEvent
+  | SetupCompleteEvent;
+
+export const liveEvents = {
+  open: { include: (e: LiveEvent): e is OpenEvent => e.type === "open" },
+  audio: {
+    include: (e: LiveEvent): e is AudioEvent => e.type === "audio",
+  },
+  text: { include: (e: LiveEvent): e is TextEvent => e.type === "text" },
+  error: {
+    include: (e: LiveEvent): e is ErrorEvent => e.type === "error",
+  },
+  close: {
+    include: (e: LiveEvent): e is CloseEvent => e.type === "close",
+  },
+  setupComplete: {
+    include: (e: LiveEvent): e is SetupCompleteEvent =>
+      e.type === "setupComplete",
+  },
+};
+
+export abstract class LiveLLMSession {
+  protected eventQueue: LiveEvent[] = [];
+  protected eventResolvers: ((value: LiveEvent) => void)[] = [];
+  protected closed = false;
+  abstract sendMessage(message: ChatMessage): void;
+  async *streamEvents(): AsyncIterable<LiveEvent> {
+    while (true) {
+      const event = await this.nextEvent();
+      if (event === undefined) {
+        break;
+      }
+      yield event;
+    }
+  }
+  abstract disconnect(): Promise<void>;
+
+  protected async nextEvent(): Promise<LiveEvent | undefined> {
+    if (this.eventQueue.length) {
+      return Promise.resolve(this.eventQueue.shift());
+    }
+
+    return new Promise((resolve) => {
+      this.eventResolvers.push(resolve);
+    });
+  }
+
+  //Uses an async queue to send events to the client
+  // if the consumer is waiting for an event, it will be resolved immediately
+  // otherwise, the event will be queued up and sent when the consumer is ready
+  pushEventToQueue(event: LiveEvent) {
+    if (this.eventResolvers.length) {
+      //resolving the promise with the event
+      this.eventResolvers.shift()!(event);
+    } else {
+      this.eventQueue.push(event);
+    }
+  }
+}
+
+export abstract class LiveLLM {
+  abstract connect(config?: LiveConnectConfig): Promise<LiveLLMSession>;
+}

packages/core/src/llms/type.ts

@@ -2,6 +2,7 @@ import type { Tokenizers } from "@llamaindex/env/tokenizers";
 import type { JSONSchemaType } from "ajv";
 import { z } from "zod";
 import type { JSONObject, JSONValue } from "../global";
+import type { ModalityType } from "../schema";
 /**
  * @internal
  */
@@ -163,6 +164,27 @@ export type MessageContentImageDetail = {
   detail?: "high" | "low" | "auto";
 };
 
+export type MessageContentAudioDetail = {
+  type: "audio";
+  //audio could be a base64 string as well
+  data: string | Uint8Array;
+  mimeType: string;
+};
+
+export type MessageContentVideoDetail = {
+  type: "video";
+  //video could be a base64 string as well
+  data: string | Uint8Array;
+  mimeType: string;
+};
+
+export type MessageContentImageDataDetail = {
+  type: "image";
+  //image could be a base64 string as well
+  data: string | Uint8Array;
+  mimeType: string;
+};
+
 export type MessageContentFileDetail = {
   type: "file";
   data: Uint8Array;
@@ -172,6 +194,9 @@ export type MessageContentFileDetail = {
 export type MessageContentDetail =
   | MessageContentTextDetail
   | MessageContentImageDetail
+  | MessageContentAudioDetail
+  | MessageContentVideoDetail
+  | MessageContentImageDataDetail
   | MessageContentFileDetail;
 
 /**
@@ -267,3 +292,9 @@ export type ToolOutput = {
   output: JSONValue;
   isError: boolean;
 };
+
+export interface LiveConnectConfig {
+  tools?: BaseTool[];
+  responseModality?: ModalityType[];
+  systemInstruction?: string;
+}

packages/core/src/schema/node.ts

@@ -431,6 +431,7 @@ export interface NodeWithScore<T extends Metadata = Metadata> {
 export enum ModalityType {
   TEXT = "TEXT",
   IMAGE = "IMAGE",
+  AUDIO = "AUDIO",
 }
 
 type NodesByType = {

packages/llamaindex/src/indices/vectorStore/index.ts

@@ -397,6 +397,7 @@ export class VectorStoreIndex extends BaseIndex<IndexDict> {
  * VectorIndexRetriever retrieves nodes from a VectorIndex.
  */
 
+// TopKMap type now only includes TEXT and IMAGE modalities
 type TopKMap = { [P in ModalityType]: number };
 
 // eslint-disable-next-line @typescript-eslint/no-explicit-any
@@ -435,6 +436,7 @@ export class VectorIndexRetriever extends BaseRetriever {
             ? options.similarityTopK
             : DEFAULT_SIMILARITY_TOP_K,
         [ModalityType.IMAGE]: DEFAULT_SIMILARITY_TOP_K,
+        [ModalityType.AUDIO]: DEFAULT_SIMILARITY_TOP_K,
       };
     }
     this.filters = options.filters;

packages/providers/anthropic/src/llm.ts

@@ -337,19 +337,22 @@ export class Anthropic extends ToolCallLLM<
             };
           }
 
-          return {
-            type: "image" as const,
-            source: {
-              type: "base64" as const,
-              media_type: `image/${content.image_url.url.substring(
-                "data:image/".length,
-                content.image_url.url.indexOf(";base64"),
-              )}` as "image/jpeg" | "image/png" | "image/gif" | "image/webp",
-              data: content.image_url.url.substring(
-                content.image_url.url.indexOf(",") + 1,
-              ),
-            },
-          };
+          if (content.type === "image_url") {
+            return {
+              type: "image" as const,
+              source: {
+                type: "base64" as const,
+                media_type: `image/${content.image_url.url.substring(
+                  "data:image/".length,
+                  content.image_url.url.indexOf(";base64"),
+                )}` as "image/jpeg" | "image/png" | "image/gif" | "image/webp",
+                data: content.image_url.url.substring(
+                  content.image_url.url.indexOf(",") + 1,
+                ),
+              },
+            };
+          }
+          throw new Error(`Unsupported content type: ${content.type}`);
         }),
       } satisfies MessageParam;
     });

packages/providers/google/package.json

@@ -31,7 +31,7 @@
   },
   "dependencies": {
     "@google-cloud/vertexai": "1.9.0",
-    "@google/genai": "^0.4.0",
+    "@google/genai": "^0.12.0",
     "@google/generative-ai": "0.24.0",
     "@llamaindex/core": "workspace:*",
     "@llamaindex/env": "workspace:*"

packages/providers/google/src/base.ts

@@ -23,6 +23,7 @@ import type {
 import { ToolCallLLM } from "@llamaindex/core/llms";
 import { streamConverter } from "@llamaindex/core/utils";
 import { getEnv, randomUUID } from "@llamaindex/env";
+import { GeminiLive } from "./live.js";
 import {
   GEMINI_BACKENDS,
   GEMINI_MODEL,
@@ -34,6 +35,7 @@ import {
   type GeminiMessageRole,
   type GeminiModelInfo,
   type GeminiSessionOptions,
+  type GeminiVoiceName,
   type GoogleGeminiSessionOptions,
   type IGeminiSession,
 } from "./types.js";
@@ -61,6 +63,7 @@ export const GEMINI_MODEL_INFO_MAP: Record<GEMINI_MODEL, GeminiModelInfo> = {
   [GEMINI_MODEL.GEMINI_2_0_FLASH]: { contextWindow: 10 ** 6 },
   [GEMINI_MODEL.GEMINI_2_0_FLASH_LITE_PREVIEW]: { contextWindow: 10 ** 6 },
   [GEMINI_MODEL.GEMINI_2_0_FLASH_LITE]: { contextWindow: 10 ** 6 },
+  [GEMINI_MODEL.GEMINI_2_0_FLASH_LIVE]: { contextWindow: 10 ** 6 },
   [GEMINI_MODEL.GEMINI_2_0_FLASH_THINKING_EXP]: { contextWindow: 32768 },
   [GEMINI_MODEL.GEMINI_2_0_PRO_EXPERIMENTAL]: { contextWindow: 2 * 10 ** 6 },
   [GEMINI_MODEL.GEMINI_2_5_PRO_PREVIEW]: { contextWindow: 10 ** 6 },
@@ -95,9 +98,11 @@ export const DEFAULT_GEMINI_PARAMS = {
 };
 
 export type GeminiConfig = Partial<typeof DEFAULT_GEMINI_PARAMS> & {
+  apiKey?: string;
   session?: IGeminiSession;
   requestOptions?: GoogleRequestOptions;
   safetySettings?: SafetySetting[];
+  voiceName?: GeminiVoiceName;
 };
 
 type StartChatParams = GoogleStartChatParams & VertexStartChatParams;
@@ -231,7 +236,9 @@ export class Gemini extends ToolCallLLM<GeminiAdditionalChatOptions> {
   #requestOptions?: GoogleRequestOptions | undefined;
   session: IGeminiSession;
   safetySettings: SafetySetting[];
-
+  apiKey?: string | undefined;
+  voiceName?: GeminiVoiceName | undefined;
+  private _live: GeminiLive | undefined;
   constructor(init?: GeminiConfig) {
     super();
     this.model = init?.model ?? GEMINI_MODEL.GEMINI_PRO;
@@ -241,12 +248,25 @@ export class Gemini extends ToolCallLLM<GeminiAdditionalChatOptions> {
     this.session = init?.session ?? GeminiSessionStore.get();
     this.#requestOptions = init?.requestOptions ?? undefined;
     this.safetySettings = init?.safetySettings ?? DEFAULT_SAFETY_SETTINGS;
+    this.apiKey = init?.apiKey ?? getEnv("GOOGLE_API_KEY");
+    this.voiceName = init?.voiceName ?? undefined;
   }
 
   get supportToolCall(): boolean {
     return SUPPORT_TOOL_CALL_MODELS.includes(this.model);
   }
 
+  get live(): GeminiLive {
+    if (!this._live) {
+      this._live = new GeminiLive({
+        apiKey: this.apiKey,
+        voiceName: this.voiceName,
+        model: this.model,
+      });
+    }
+    return this._live;
+  }
+
   get metadata(): LLMMetadata & { safetySettings: SafetySetting[] } {
     return {
       model: this.model,

packages/providers/google/src/index.ts

@@ -1,12 +1,13 @@
 export * from "./base";
-export * from "./types";
-export * from "./utils";
-export * from "./vertex";
 
+export * from "./live";
 export {
   GoogleStudio,
   Modality,
   getGoogleStudioInlineData,
 } from "./studio/index.js";
+export * from "./types";
+export * from "./utils";
+export * from "./vertex";
 
 export * from "./GeminiEmbedding";

packages/providers/google/src/live.ts

@@ -0,0 +1,361 @@
+import {
+  FunctionResponse,
+  GoogleGenAI,
+  Modality,
+  Session,
+  type FunctionCall,
+  type FunctionDeclaration,
+  type LiveConnectConfig as GoogleLiveConnectConfig,
+  type LiveServerMessage,
+} from "@google/genai";
+import {
+  LiveLLM,
+  LiveLLMSession,
+  type BaseTool,
+  type ChatMessage,
+  type LiveConnectConfig,
+  type MessageContentAudioDetail,
+  type MessageContentDetail,
+  type MessageContentImageDataDetail,
+  type MessageContentVideoDetail,
+} from "@llamaindex/core/llms";
+import { getEnv, uint8ArrayToBase64 } from "@llamaindex/env";
+import { GEMINI_MODEL, type GeminiVoiceName } from "./types";
+import {
+  mapBaseToolToGeminiLiveFunctionDeclaration,
+  mapResponseModalityToGeminiLiveResponseModality,
+} from "./utils";
+
+interface GeminiLiveConfig {
+  apiKey?: string | undefined;
+  voiceName?: GeminiVoiceName | undefined;
+  model?: GEMINI_MODEL | undefined;
+}
+
+export class GeminiLiveSession extends LiveLLMSession {
+  session: Session | undefined;
+  closed = false;
+
+  constructor() {
+    super();
+  }
+
+  private isTextEvent(event: LiveServerMessage): boolean {
+    return event.serverContent?.modelTurn?.parts?.[0]?.text !== undefined;
+  }
+
+  private isAudioEvent(event: LiveServerMessage): boolean {
+    return (
+      event.serverContent?.modelTurn?.parts?.[0]?.inlineData?.data !== undefined
+    );
+  }
+
+  private isToolCallEvent(event: LiveServerMessage): boolean {
+    return event.toolCall !== undefined;
+  }
+
+  private isSetupCompleteEvent(event: LiveServerMessage): boolean {
+    return event.setupComplete !== undefined;
+  }
+
+  private isTextMessage(content: MessageContentDetail) {
+    return content.type === "text";
+  }
+
+  private isAudioMessage(content: MessageContentDetail) {
+    return content.type === "audio";
+  }
+
+  private isImageMessage(content: MessageContentDetail) {
+    return content.type === "image";
+  }
+
+  private isVideoMessage(content: MessageContentDetail) {
+    return content.type === "video";
+  }
+
+  //for the tool call event, we need to return the response with function responses
+  private async handleToolCallEvent(
+    event: LiveServerMessage,
+    toolCalls: BaseTool[],
+  ) {
+    const eventToolCalls = event.toolCall?.functionCalls;
+
+    if (eventToolCalls) {
+      await this.sendToolCallResponses(eventToolCalls, toolCalls);
+    }
+  }
+
+  handleLiveEvents(event: LiveServerMessage, toolCalls: BaseTool[]) {
+    if (this.isTextEvent(event)) {
+      this.pushEventToQueue({
+        type: "text",
+        text: event.serverContent?.modelTurn?.parts?.[0]?.text || "",
+      });
+    }
+    if (this.isSetupCompleteEvent(event)) {
+      this.pushEventToQueue({
+        type: "setupComplete",
+      });
+    }
+    if (this.isAudioEvent(event)) {
+      this.pushEventToQueue({
+        type: "audio",
+        data:
+          event.serverContent?.modelTurn?.parts?.[0]?.inlineData?.data || "",
+        mimeType:
+          event.serverContent?.modelTurn?.parts?.[0]?.inlineData?.mimeType ||
+          "audio/wav",
+      });
+    }
+    if (this.isToolCallEvent(event)) {
+      this.handleToolCallEvent(event, toolCalls);
+    }
+  }
+
+  private executeToolCall(toolCall: FunctionCall, tool: BaseTool) {
+    return tool.call!(toolCall.args);
+  }
+
+  private storeToolCallResponse(
+    toolCall: FunctionCall,
+    response: unknown,
+    functionResponses: FunctionResponse[],
+  ) {
+    functionResponses.push({
+      id: toolCall.id || "",
+      name: toolCall.name || "",
+      response:
+        typeof response === "string"
+          ? { result: response }
+          : (response as Record<string, unknown>),
+    });
+  }
+
+  private async executeToolCallsAndStoreResponses(
+    eventToolCalls: FunctionCall[],
+    toolCalls: BaseTool[],
+  ) {
+    const functionResponses: FunctionResponse[] = [];
+
+    for (const toolCall of eventToolCalls) {
+      const tool = toolCalls.find((t) => t.metadata.name === toolCall.name);
+      if (tool && tool.call) {
+        const response = await this.executeToolCall(toolCall, tool);
+        this.storeToolCallResponse(toolCall, response, functionResponses);
+      }
+    }
+
+    return functionResponses;
+  }
+
+  //execute the tool call and send the response to the server
+  private async sendToolCallResponses(
+    eventToolCalls: FunctionCall[],
+    toolCalls: BaseTool[],
+  ) {
+    let functionResponses: FunctionResponse[] = [];
+
+    if (eventToolCalls) {
+      functionResponses = await this.executeToolCallsAndStoreResponses(
+        eventToolCalls,
+        toolCalls,
+      );
+      //send the function responses to the gemini
+      this.session?.sendToolResponse({
+        functionResponses,
+      });
+    }
+  }
+
+  private sendTextMessage(message: string, role?: string) {
+    this.session?.sendClientContent({
+      turns: [
+        {
+          parts: [{ text: message }],
+          ...(role ? { role } : {}),
+        },
+      ],
+    });
+  }
+
+  private sendAudioMessage(content: MessageContentAudioDetail, role?: string) {
+    if (typeof content.data === "string") {
+      this.session?.sendRealtimeInput({
+        audio: {
+          data: content.data,
+          mimeType: content.mimeType,
+        },
+      });
+    } else {
+      this.session?.sendRealtimeInput({
+        audio: {
+          data: uint8ArrayToBase64(content.data),
+          mimeType: content.mimeType,
+        },
+      });
+    }
+  }
+
+  private sendImageMessage(
+    content: MessageContentImageDataDetail,
+    role?: string,
+  ) {
+    if (typeof content.data === "string") {
+      this.session?.sendRealtimeInput({
+        media: {
+          data: content.data,
+          mimeType: content.mimeType,
+        },
+      });
+    } else {
+      this.session?.sendRealtimeInput({
+        media: {
+          data: uint8ArrayToBase64(content.data),
+          mimeType: content.mimeType,
+        },
+      });
+    }
+  }
+
+  private sendVideoMessage(content: MessageContentVideoDetail, role?: string) {
+    if (typeof content.data === "string") {
+      this.session?.sendRealtimeInput({
+        video: {
+          data: content.data,
+          mimeType: content.mimeType,
+        },
+      });
+    } else {
+      this.session?.sendRealtimeInput({
+        video: {
+          data: uint8ArrayToBase64(content.data),
+          mimeType: content.mimeType,
+        },
+      });
+    }
+  }
+
+  private handleUserInput(message: ChatMessage) {
+    const { content, role } = message;
+
+    if (!Array.isArray(content)) {
+      this.sendTextMessage(content, role);
+    } else {
+      for (const item of content) {
+        if (this.isTextMessage(item)) {
+          this.sendTextMessage(item.text, role);
+        } else if (this.isAudioMessage(item)) {
+          this.sendAudioMessage(item as MessageContentAudioDetail, role);
+        } else if (this.isImageMessage(item)) {
+          this.sendImageMessage(item as MessageContentImageDataDetail, role);
+        } else if (this.isVideoMessage(item)) {
+          this.sendVideoMessage(item as MessageContentVideoDetail, role);
+        }
+      }
+    }
+  }
+
+  sendMessage(message: ChatMessage) {
+    if (!this.session) {
+      throw new Error("Session not connected");
+    }
+    this.handleUserInput(message);
+  }
+
+  async disconnect() {
+    if (!this.session) {
+      throw new Error("Session not connected");
+    }
+    this.session.close();
+  }
+}
+export class GeminiLive extends LiveLLM {
+  private apiKey: string | undefined;
+  private client: GoogleGenAI;
+  voiceName?: GeminiVoiceName | undefined;
+  model: GEMINI_MODEL;
+
+  constructor(init?: GeminiLiveConfig) {
+    super();
+    this.apiKey = init?.apiKey ?? getEnv("GOOGLE_API_KEY");
+
+    if (!this.apiKey) {
+      throw new Error("GOOGLE_API_KEY is not set");
+    }
+
+    this.client = new GoogleGenAI({
+      apiKey: this.apiKey,
+    });
+    this.voiceName = init?.voiceName;
+    this.model = init?.model ?? GEMINI_MODEL.GEMINI_2_0_FLASH_LIVE;
+    /* Only 2.0 flash live is supported for live mode */
+    if (this.model !== GEMINI_MODEL.GEMINI_2_0_FLASH_LIVE) {
+      throw new Error("Only GEMINI_2_0_FLASH_LIVE is supported for live mode");
+    }
+  }
+
+  async connect(config?: LiveConnectConfig) {
+    const liveConfig: GoogleLiveConnectConfig = {
+      responseModalities: config?.responseModality
+        ? config.responseModality.map(
+            mapResponseModalityToGeminiLiveResponseModality,
+          )
+        : [Modality.AUDIO],
+    };
+
+    if (config?.tools) {
+      const tools = config.tools.map(
+        mapBaseToolToGeminiLiveFunctionDeclaration,
+      );
+      liveConfig.tools = [
+        {
+          functionDeclarations: tools as FunctionDeclaration[],
+        },
+      ];
+    }
+
+    if (config?.systemInstruction) {
+      liveConfig.systemInstruction = config.systemInstruction;
+    }
+
+    if (this.voiceName) {
+      liveConfig.speechConfig = {
+        voiceConfig: {
+          prebuiltVoiceConfig: {
+            voiceName: this.voiceName,
+          },
+        },
+      };
+    }
+
+    const geminiLiveSession = new GeminiLiveSession();
+
+    geminiLiveSession.session = await this.client.live.connect({
+      model: this.model,
+      config: {
+        ...liveConfig,
+      },
+
+      callbacks: {
+        onmessage: (event) => {
+          geminiLiveSession.handleLiveEvents(event, config?.tools || []);
+        },
+        onerror: (error) => {
+          geminiLiveSession.pushEventToQueue({
+            type: "error",
+            error: error.error,
+          });
+        },
+        onopen: () => {
+          geminiLiveSession.pushEventToQueue({ type: "open" });
+        },
+        onclose: () => {
+          geminiLiveSession.pushEventToQueue({ type: "close" });
+        },
+      },
+    });
+
+    return geminiLiveSession;
+  }
+}

packages/providers/google/src/studio/utils.ts

@@ -1,6 +1,6 @@
 import type {
+  Content,
   ContentListUnion,
-  ContentUnion,
   Part,
   Schema,
   ToolListUnion,
@@ -95,42 +95,53 @@ export const mapChatMessagesToGoogleMessages = <
       Object.assign(functionNames, mapped);
     }
   });
-  return messages.flatMap((msg: T): ContentListUnion => {
+
+  // Transform messages to Google API format
+  const contents = messages.flatMap((msg: T) => {
     if (msg.options && "toolResult" in msg.options) {
-      return {
-        role: "user",
-        parts: [
-          {
-            functionResponse: {
-              name: functionNames[msg.options.toolResult.id] ?? "",
-              response: msg.options.toolResult,
+      return [
+        {
+          role: "user",
+          parts: [
+            {
+              functionResponse: {
+                name: functionNames[msg.options.toolResult.id] ?? "",
+                response: msg.options.toolResult,
+              },
             },
-          },
-        ],
-      };
+          ],
+        },
+      ];
     }
 
     if (msg.options && "toolCall" in msg.options) {
-      return {
-        role: "model",
-        parts: msg.options.toolCall.map((call) => ({
-          functionCall: {
-            name: call.name,
-            args: call.input as Record<string, unknown>,
-          },
-        })),
-      };
+      return [
+        {
+          role: "model",
+          parts: msg.options.toolCall.map((call) => ({
+            functionCall: {
+              name: call.name,
+              args: call.input as Record<string, unknown>,
+            },
+          })),
+        },
+      ];
     }
-    return mapMessageContentToMessageContentDetails(msg.content)
-      .map((detail: MessageContentDetail): ContentUnion | null => {
+
+    const mapped = mapMessageContentToMessageContentDetails(msg.content)
+      .map((detail: MessageContentDetail) => {
         const part = mapMessageContentDetailToGooglePart(detail);
         if (!part.text && !part.inlineData) return null;
 
         return {
           role: msg.role === "assistant" ? "model" : "user",
           parts: [part],
-        };
+        } as Content;
       })
-      .filter((content) => content) as ContentUnion;
+      .filter((content): content is Content => content !== null);
+
+    return mapped;
   });
+
+  return contents;
 };

packages/providers/google/src/types.ts

@@ -73,6 +73,7 @@ export enum GEMINI_MODEL {
   GEMINI_2_0_FLASH_LITE_PREVIEW = "gemini-2.0-flash-lite-preview-02-05",
   GEMINI_2_0_FLASH_THINKING_EXP = "gemini-2.0-flash-thinking-exp-01-21",
   GEMINI_2_0_PRO_EXPERIMENTAL = "gemini-2.0-pro-exp-02-05",
+  GEMINI_2_0_FLASH_LIVE = "gemini-2.0-flash-live-001",
   GEMINI_2_5_PRO_PREVIEW = "gemini-2.5-pro-preview-03-25",
   GEMINI_2_5_PRO_PREVIEW_LATEST = "gemini-2.5-pro-preview-05-06",
   GEMINI_2_5_FLASH_PREVIEW = "gemini-2.5-flash-preview-04-17",
@@ -148,3 +149,24 @@ export interface IGeminiSession {
     response: EnhancedGenerateContentResponse | GenerateContentResponse,
   ): ToolCall[] | undefined;
 }
+
+export type GeminiLiveMessage = {
+  content: string | GeminiLiveMessageDetail;
+  role: "user" | "model";
+};
+
+export type GeminiLiveMessageDetail = {
+  type: "text" | "audio" | "image" | "video";
+  data: string;
+  mimeType: string;
+};
+
+export type GeminiVoiceName =
+  | "Puck"
+  | "Charon"
+  | "Fenrir"
+  | "Aoede"
+  | "Leda"
+  | "Kore"
+  | "Orus"
+  | "Zephyr";

packages/providers/google/src/utils.ts

@@ -1,3 +1,10 @@
+import { type GenerateContentResponse } from "@google-cloud/vertexai";
+import {
+  type FunctionDeclaration as LiveFunctionDeclaration,
+  Modality,
+  type Schema,
+  Type,
+} from "@google/genai";
 import {
   type FunctionCall,
   type Content as GeminiMessageContent,
@@ -6,8 +13,6 @@ import {
   type SafetySetting,
   SchemaType,
 } from "@google/generative-ai";
-
-import { type GenerateContentResponse } from "@google-cloud/vertexai";
 import { FileState, GoogleAIFileManager } from "@google/generative-ai/server";
 import type {
   BaseTool,
@@ -18,6 +23,7 @@ import type {
   MessageType,
   ToolCallLLMMessageOptions,
 } from "@llamaindex/core/llms";
+import { ModalityType } from "@llamaindex/core/schema";
 import { extractDataUrlComponents } from "@llamaindex/core/utils";
 import { getEnv } from "@llamaindex/env";
 import type {
@@ -180,6 +186,39 @@ export const mapBaseToolToGeminiFunctionDeclaration = (
   };
 };
 
+/**
+ * Maps a BaseTool to a Gemini Live Function Declaration format
+ * Used for converting LlamaIndex tools to be compatible with Gemini's live API function calling
+ *
+ * @param tool - The BaseTool to convert
+ * @returns A LiveFunctionDeclaration object that can be used with Gemini's live API
+ */
+export const mapBaseToolToGeminiLiveFunctionDeclaration = (
+  tool: BaseTool,
+): LiveFunctionDeclaration => {
+  const parameters: Schema = {
+    type: tool.metadata.parameters?.type.toLowerCase() as Type,
+    properties: tool.metadata.parameters?.properties,
+    description: tool.metadata.parameters?.description,
+    required: tool.metadata.parameters?.required,
+  };
+  return {
+    name: tool.metadata.name,
+    description: tool.metadata.description,
+    parameters,
+  };
+};
+
+export const mapResponseModalityToGeminiLiveResponseModality = (
+  responseModality: ModalityType,
+): Modality => {
+  return responseModality === ModalityType.TEXT
+    ? Modality.TEXT
+    : responseModality === ModalityType.AUDIO
+      ? Modality.AUDIO
+      : Modality.IMAGE;
+};
+
 /**
  * Helper class providing utility functions for Gemini
  */

packages/providers/openai/src/llm.ts

@@ -213,6 +213,27 @@ export class OpenAI extends ToolCallLLM<OpenAIAdditionalChatOptions> {
         return {
           role: "user",
           content: message.content.map((item, index) => {
+            // Handle MessageContentMediaDetail (audio, video, image)
+            if (
+              "data" in item &&
+              "mimeType" in item &&
+              (item.type === "audio" ||
+                item.type === "video" ||
+                item.type === "image")
+            ) {
+              if (item.type === "audio" || item.type === "video") {
+                throw new Error("Audio and video are not supported");
+              }
+              // Convert image type to file format for OpenAI
+              return {
+                type: "file",
+                file: {
+                  file_data: `data:${item.mimeType};base64,${item.data}`,
+                  filename: `image-${index}.${item.mimeType.split("/")[1] || "png"}`,
+                },
+              } satisfies ChatCompletionContentPart.File;
+            }
+
             if (item.type === "file") {
               if (item.mimeType !== "application/pdf") {
                 throw new Error("Only PDF files are supported");
@@ -227,9 +248,9 @@ export class OpenAI extends ToolCallLLM<OpenAIAdditionalChatOptions> {
               } satisfies ChatCompletionContentPart.File;
             }
 
-            // keep it as is for other types
+            // Keep other types as is (text, image_url, etc.)
             return item;
-          }),
+          }) as ChatCompletionContentPart[],
         } satisfies ChatCompletionUserMessageParam;
       }
 

packages/providers/vercel/src/llm.ts

@@ -99,10 +99,15 @@ export class VercelLLM extends ToolCallLLM<VercelAdditionalChatOptions> {
                   if (contentDetail.type === "file") {
                     throw new Error("File content not supported yet");
                   }
-                  return {
-                    type: "text",
-                    text: contentDetail.text,
-                  } satisfies TextPart;
+                  if (contentDetail.type === "text") {
+                    return {
+                      type: "text",
+                      text: contentDetail.text,
+                    } satisfies TextPart;
+                  }
+                  throw new Error(
+                    `Unsupported content type: ${contentDetail.type}`,
+                  );
                 }),
         } satisfies CoreUserMessage;
       }

pnpm-lock.yaml

@@ -1201,8 +1201,8 @@ importers:
         specifier: 1.9.0
         version: 1.9.0
       '@google/genai':
-        specifier: ^0.4.0
-        version: 0.4.0(bufferutil@4.0.9)
+        specifier: ^0.12.0
+        version: 0.12.0(bufferutil@4.0.9)
       '@google/generative-ai':
         specifier: 0.24.0
         version: 0.24.0
@@ -3182,8 +3182,8 @@ packages:
     resolution: {integrity: sha512-8brlcJwFXI4fPuBtsDNQqCdWZmz8gV9jeEKOU0vc5H2SjehCQpXK/NwuSEr916zbhlBHtg/sU37qQQdgvh5BRA==}
     engines: {node: '>=18.0.0'}
 
-  '@google/genai@0.4.0':
-    resolution: {integrity: sha512-u9KHoIDbnUi6GpH6mtkZjdeVy3FXI0Hfvl5QWZyYPBttXWaJ13Q4OXE+8zynbHvvGh4XUaH5fBvzsuNLQqB+qQ==}
+  '@google/genai@0.12.0':
+    resolution: {integrity: sha512-SJtCHac+HPgmwELpJpPKbaV4rk397bS2D42XgFR2NBEARDKd/79RcaRUFFd55pYUJ+gfaz9Bv6KYoiz/P6eZKA==}
     engines: {node: '>=18.0.0'}
 
   '@google/generative-ai@0.1.3':
@@ -14960,10 +14960,12 @@ snapshots:
       - encoding
       - supports-color
 
-  '@google/genai@0.4.0(bufferutil@4.0.9)':
+  '@google/genai@0.12.0(bufferutil@4.0.9)':
     dependencies:
       google-auth-library: 9.15.1
       ws: 8.18.1(bufferutil@4.0.9)
+      zod: 3.24.2
+      zod-to-json-schema: 3.24.5(zod@3.24.2)
     transitivePeerDependencies:
       - bufferutil
       - encoding