Release 0.11.11 (#2044)

Co-authored-by: marcusschiesser <17126+marcusschiesser@users.noreply.github.com>

apps/next/CHANGELOG.md

@@ -1,5 +1,34 @@
 # @llamaindex/doc
 
+## 0.2.31
+
+### Patch Changes
+
+- Updated dependencies [7039e1a]
+- Updated dependencies [7039e1a]
+  - llamaindex@0.11.11
+  - @llamaindex/core@0.6.12
+  - @llamaindex/cloud@4.0.16
+  - @llamaindex/node-parser@2.0.12
+  - @llamaindex/openai@0.4.6
+  - @llamaindex/readers@3.1.11
+  - @llamaindex/workflow@1.1.12
+
+## 0.2.30
+
+### Patch Changes
+
+- Updated dependencies [f7ec293]
+  - @llamaindex/workflow@1.1.11
+  - llamaindex@0.11.10
+
+## 0.2.29
+
+### Patch Changes
+
+- Updated dependencies [c5846bd]
+  - @llamaindex/readers@3.1.10
+
 ## 0.2.28
 
 ### Patch Changes

apps/next/CLAUDE.md

@@ -111,7 +111,7 @@ Key build process:
 **Content Sources:**
 
 - Local MDX files in `src/content/docs/`
-- External docs from `@llama-flow/docs` package
+- External docs from `@llamaindex/workflow-docs` package
 - Generated API docs from TypeScript source
 
 ### Development Notes

apps/next/next.config.mjs

@@ -23,8 +23,8 @@ const config = {
         permanent: true,
       },
       {
-        source: "/docs/llamaflow/:path*.mdx",
-        destination: "/docs/llamaflow/:path*",
+        source: "/docs/workflows/:path*.mdx",
+        destination: "/docs/workflows/:path*",
         permanent: true,
       },
     ];

apps/next/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/doc",
-  "version": "0.2.28",
+  "version": "0.2.31",
   "private": true,
   "scripts": {
     "postinstall": "fumadocs-mdx",
@@ -15,7 +15,7 @@
   "dependencies": {
     "@huggingface/transformers": "^3.5.0",
     "@icons-pack/react-simple-icons": "^10.1.0",
-    "@llama-flow/docs": "0.0.8",
+    "@llamaindex/workflow-docs": "0.1.1",
     "@llamaindex/chat-ui-docs": "^0.0.5",
     "@llamaindex/cloud": "workspace:*",
     "@llamaindex/core": "workspace:*",
@@ -69,7 +69,7 @@
     "twoslash": "^0.3.1",
     "use-stick-to-bottom": "^1.0.42",
     "web-tree-sitter": "^0.24.4",
-    "zod": "^3.23.8"
+    "zod": "^3.25.67"
   },
   "devDependencies": {
     "@next/env": "^15.3.0",

apps/next/scripts/validate-links.mts

@@ -13,7 +13,7 @@ const INTERNAL_LINK_REGEX = /(?:(?:\]\(|\bhref=["'])\/docs\/([^")]+))/g;
 // This captures relative links like [text](./path) or ![alt](../images/image.png)
 const RELATIVE_LINK_REGEX = /(?:\]\()(?:\s*)(?:\.\.?)\//g;
 
-const ALLOWED_LINKS = ["/docs/llamaflow", "/docs/chat-ui"];
+const ALLOWED_LINKS = ["/docs/workflows", "/docs/chat-ui"];
 
 interface LinkValidationResult {
   file: string;

apps/next/source.config.ts

@@ -11,9 +11,9 @@ import remarkMath from "remark-math";
 export const docs = defineDocs({
   dir: [
     "./src/content/docs",
-    "./node_modules/@llama-flow/docs",
+    "./node_modules/@llamaindex/workflow-docs",
     "./node_modules/@llamaindex/chat-ui-docs",
-    // NOTE: When adding external docs (like chat-ui or llama-flow above),
+    // NOTE: When adding external docs (like chat-ui or workflow-docs above),
     // make sure to also update:
     // 1. scripts/validate-links.mts - add to ALLOWED_LINKS array
     // 2. next.config.mjs - add redirect for .mdx files

apps/next/src/content/docs/llamaindex/modules/agents/tool/index.mdx

@@ -74,12 +74,21 @@ const server = mcp({
   args: ["-y", "@modelcontextprotocol/server-filesystem", "."],
   verbose: true,
 });
-// or by SSE
+// or by StreamableHTTP transport
 const server = mcp({
   url: "http://localhost:8000/mcp",
   verbose: true,
 });
 
+// if your MCP server is not using StreamableHTTP transport, you can also use SSE transport
+// by setting useSSETransport to true.
+// See: https://modelcontextprotocol.io/docs/concepts/transports#server-sent-events-sse-deprecated
+const server = mcp({
+  url: "http://localhost:8000/mcp",
+  useSSETransport: true,
+  verbose: true,
+});
+
 // 3. Get tools from MCP server
 const tools = await server.tools();
 

apps/next/src/content/docs/llamaindex/modules/agents/workflows.mdx

@@ -9,10 +9,13 @@ Workflows are designed to be flexible and can be used to build agents, RAG flows
 To use workflows install this package:
 
 ```package-install
-npm i @llamaindex/workflow
+npm i @llamaindex/workflow-core
 ```
 
-This package is a stable, production-ready version of our [llama-flow](/docs/llamaflow) project. 
+This contains the core functionality for the workflow system. You can read more about the core concepts in the [workflow-core](/docs/workflows) section.
 
-While you can still reference the llama-flow documentation for detailed information about the underlying concepts, we recommend using the `@llamaindex/workflow` package for all new projects to ensure stability and long-term availability.
+In contrast, the `@llamaindex/workflow` package contains more utiltities, such as prebuilt agents.
 
+```package-install
+npm i @llamaindex/workflow
+```

apps/next/src/content/docs/llamaindex/modules/data/stores/vector_stores/supabase.mdx

@@ -28,11 +28,12 @@ embedding vector(1536)
 );
 ```
 
--- Create a function for similarity search
+-- Create a function for similarity search with filtering support
 ```sql
 create function match_documents (
 query_embedding vector(1536),
-match_count int
+match_count int,
+filter jsonb DEFAULT '{}'
 ) returns table (
 id uuid,
 content text,
@@ -52,6 +53,7 @@ metadata,
 embedding,
 1 - (embedding <=> query_embedding) as similarity
 from documents
+where metadata @> filter
 order by embedding <=> query_embedding
 limit match_count;
 end;
@@ -96,6 +98,7 @@ const index = await VectorStoreIndex.fromDocuments(documents, {
 ```ts
 const queryEngine = index.asQueryEngine();
 
+// Basic query without filters
 const response = await queryEngine.query({
   query: "What is in the document?",
 });
@@ -104,6 +107,32 @@ const response = await queryEngine.query({
 console.log(response.toString());
 ```
 
+## Query with filters
+
+You can filter documents based on metadata when querying:
+
+```ts
+import { FilterOperator, MetadataFilters } from "llamaindex";
+
+// Create a filter for documents with author = "Jane Smith"
+const filters: MetadataFilters = {
+  filters: [
+    {
+      key: "author",
+      value: "Jane Smith",
+      operator: FilterOperator.EQ,
+    },
+  ],
+};
+
+// Query with filters
+const filteredResponse = await vectorStore.query({
+  queryEmbedding: embedModel.getQueryEmbedding("What is vector search?"),
+  similarityTopK: 5,
+  filters,
+});
+```
+
 ## Full code
 
 ```ts

apps/next/src/content/docs/llamaindex/modules/models/llms/gemini.mdx

@@ -11,58 +11,130 @@ npm i llamaindex @llamaindex/google
 ## Usage
 
 ```ts
-import { Gemini, GEMINI_MODEL } from "@llamaindex/google";
+import { gemini, GEMINI_MODEL } from "@llamaindex/google";
 import { Settings } from "llamaindex";
 
-Settings.llm = new Gemini({
-  model: GEMINI_MODEL.GEMINI_PRO,
-});
-```
-
-## Usage with Proxy
-
-```ts
-import { Gemini, GEMINI_MODEL } from "@llamaindex/google";
-import { Settings } from "llamaindex";
-
-Settings.llm = new Gemini({
-  model: GEMINI_MODEL.GEMINI_PRO,
-  requestOptions: {
-    baseUrl: <YOUR_PROXY_URL>   // optional, but useful for custom endpoints
-  }
+Settings.llm = gemini({
+  model: GEMINI_MODEL.GEMINI_2_0_FLASH,
 });
 ```
 
 ### Usage with Vertex AI
 
-To use Gemini via Vertex AI you can use `GeminiVertexSession`.
-
-GeminiVertexSession accepts the env variables: `GOOGLE_VERTEX_LOCATION` and `GOOGLE_VERTEX_PROJECT`
+To use Gemini via Vertex AI, you can specify the vertex configuration:
 
 ```ts
-import { Gemini, GEMINI_MODEL, GeminiVertexSession } from "@llamaindex/google";
+import { gemini, GEMINI_MODEL } from "@llamaindex/google";
 
-const gemini = new Gemini({
-  model: GEMINI_MODEL.GEMINI_PRO,
-  session: new GeminiVertexSession({
-    location: "us-central1",      // optional if provided by GOOGLE_VERTEX_LOCATION env variable
-    project: "project1",          // optional if provided by GOOGLE_VERTEX_PROJECT env variable
-    googleAuthOptions: {...},     // optional, but useful for production. It accepts all values from `GoogleAuthOptions`
-  }),
+const llm = gemini({
+  model: GEMINI_MODEL.GEMINI_2_0_FLASH,
+  vertex: {
+    project: "your-cloud-project",    // required for Vertex AI
+    location: "us-central1",          // required for Vertex AI
+  },
 });
 ```
 
-[GoogleAuthOptions](https://github.com/googleapis/google-auth-library-nodejs/blob/main/src/auth/googleauth.ts)
-
 To authenticate for local development:
 
 ```bash
-npm i @google-cloud/vertexai
 gcloud auth application-default login
 ```
 
 To authenticate for production you'll have to use a [service account](https://cloud.google.com/docs/authentication/). `googleAuthOptions` has `credentials` which might be useful for you.
 
+## Multimodal Usage
+
+Gemini supports multimodal inputs including text, images, audio, and video:
+
+```ts
+import { gemini, GEMINI_MODEL } from "@llamaindex/google";
+import fs from "fs";
+
+const llm = gemini({ model: GEMINI_MODEL.GEMINI_2_0_FLASH });
+
+const result = await llm.chat({
+  messages: [
+    {
+      role: "user",
+      content: [
+        {
+          type: "text",
+          text: "What's in this image?",
+        },
+        {
+          type: "image",
+          data: fs.readFileSync("./image.jpg").toString("base64"),
+          mimeType: "image/jpeg",
+        },
+      ],
+    },
+  ],
+});
+```
+
+## Tool Calling
+
+Gemini supports function calling with tools:
+
+```ts
+import { gemini, GEMINI_MODEL } from "@llamaindex/google";
+import { tool } from "llamaindex";
+import { z } from "zod";
+
+const llm = gemini({ model: GEMINI_MODEL.GEMINI_2_0_FLASH });
+
+const result = await llm.chat({
+  messages: [
+    {
+      content: "What's the weather in Tokyo?",
+      role: "user",
+    },
+  ],
+  tools: [
+    tool({
+      name: "weather",
+      description: "Get the weather",
+      parameters: z.object({
+        location: z.string().describe("The location to get the weather for"),
+      }),
+      execute: ({ location }) => {
+        return `The weather in ${location} is sunny and hot`;
+      },
+    }),
+  ],
+});
+```
+
+## Live API (Real-time Conversations)
+
+For real-time audio/video conversations using [Gemini Live API](https://ai.google.dev/gemini-api/docs/live). 
+
+The Live API is running directly in the frontend. That's why you have to generate an ephemeral key first on the server side and pass it to the frontend.
+
+To use the Live API, make sure to pass `apiVersion: "v1alpha"` to the `httpOptions`.
+
+```ts
+import { gemini, GEMINI_MODEL } from "@llamaindex/google";
+
+// Server-side: Generate ephemeral key
+const serverLlm = gemini({
+  model: GEMINI_MODEL.GEMINI_2_0_FLASH_LIVE,
+  httpOptions: { apiVersion: "v1alpha" },
+});
+const ephemeralKey = await serverLlm.live.getEphemeralKey();
+
+// Client-side: Use ephemeral key for Live API
+const llm = gemini({
+  apiKey: ephemeralKey,
+  model: GEMINI_MODEL.GEMINI_2_0_FLASH_LIVE,
+  voiceName: "Zephyr",
+  httpOptions: { apiVersion: "v1alpha" },
+});
+
+const session = await llm.live.connect();
+```
+
 ## Load and index documents
 
 For this example, we will use a single document. In a real-world scenario, you would have multiple documents to index.
@@ -90,11 +162,11 @@ const results = await queryEngine.query({
 ## Full Example
 
 ```ts
-import { Gemini, GEMINI_MODEL } from "@llamaindex/google";
+import { gemini, GEMINI_MODEL } from "@llamaindex/google";
 import { Document, VectorStoreIndex, Settings } from "llamaindex";
 
-Settings.llm = new Gemini({
-  model: GEMINI_MODEL.GEMINI_PRO,
+Settings.llm = gemini({
+  model: GEMINI_MODEL.GEMINI_2_0_FLASH,
 });
 
 async function main() {
@@ -104,9 +176,7 @@ async function main() {
   const index = await VectorStoreIndex.fromDocuments([document]);
 
   // Create a query engine
-  const queryEngine = index.asQueryEngine({
-    retriever,
-  });
+  const queryEngine = index.asQueryEngine();
 
   const query = "What is the meaning of life?";
 

apps/next/src/content/docs/meta.json

@@ -1,3 +1,3 @@
 {
-  "pages": ["llamaindex", "api", "llamaflow", "chat-ui"]
+  "pages": ["llamaindex", "api", "workflows", "chat-ui"]
 }

apps/next/turbo.json

@@ -4,7 +4,7 @@
   "tasks": {
     "build": {
       "inputs": [
-        "node_modules/@llama-flow/docs/**",
+        "node_modules/@llamaindex/workflow-docs/**",
         "node_modules/@llamaindex/chat-ui-docs/**",
         "src/**/*.ts",
         "src/**/*.tsx",

e2e/examples/cloudflare-worker-agent/CHANGELOG.md

@@ -1,5 +1,18 @@
 # @llamaindex/cloudflare-worker-agent-test
 
+## 0.0.172
+
+### Patch Changes
+
+- Updated dependencies [7039e1a]
+  - llamaindex@0.11.11
+
+## 0.0.171
+
+### Patch Changes
+
+- llamaindex@0.11.10
+
 ## 0.0.170
 
 ### Patch Changes

e2e/examples/cloudflare-worker-agent/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/cloudflare-worker-agent-test",
-  "version": "0.0.170",
+  "version": "0.0.172",
   "type": "module",
   "private": true,
   "scripts": {

e2e/examples/llama-parse-browser/CHANGELOG.md

@@ -1,5 +1,11 @@
 # @llamaindex/llama-parse-browser-test
 
+## 0.0.71
+
+### Patch Changes
+
+- @llamaindex/cloud@4.0.16
+
 ## 0.0.70
 
 ### Patch Changes

e2e/examples/llama-parse-browser/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/llama-parse-browser-test",
   "private": true,
-  "version": "0.0.70",
+  "version": "0.0.71",
   "type": "module",
   "scripts": {
     "dev": "vite",

e2e/examples/nextjs-agent/CHANGELOG.md

@@ -1,5 +1,18 @@
 # @llamaindex/next-agent-test
 
+## 0.1.172
+
+### Patch Changes
+
+- Updated dependencies [7039e1a]
+  - llamaindex@0.11.11
+
+## 0.1.171
+
+### Patch Changes
+
+- llamaindex@0.11.10
+
 ## 0.1.170
 
 ### Patch Changes

e2e/examples/nextjs-agent/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/next-agent-test",
-  "version": "0.1.170",
+  "version": "0.1.172",
   "private": true,
   "scripts": {
     "dev": "next dev",

e2e/examples/nextjs-edge-runtime/CHANGELOG.md

@@ -1,5 +1,18 @@
 # test-edge-runtime
 
+## 0.1.171
+
+### Patch Changes
+
+- Updated dependencies [7039e1a]
+  - llamaindex@0.11.11
+
+## 0.1.170
+
+### Patch Changes
+
+- llamaindex@0.11.10
+
 ## 0.1.169
 
 ### Patch Changes

e2e/examples/nextjs-edge-runtime/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/nextjs-edge-runtime-test",
-  "version": "0.1.169",
+  "version": "0.1.171",
   "private": true,
   "scripts": {
     "dev": "next dev",

e2e/examples/nextjs-node-runtime/CHANGELOG.md

@@ -1,5 +1,27 @@
 # @llamaindex/next-node-runtime
 
+## 0.1.40
+
+### Patch Changes
+
+- Updated dependencies [7039e1a]
+  - llamaindex@0.11.11
+  - @llamaindex/huggingface@0.1.16
+  - @llamaindex/readers@3.1.11
+
+## 0.1.39
+
+### Patch Changes
+
+- llamaindex@0.11.10
+
+## 0.1.38
+
+### Patch Changes
+
+- Updated dependencies [c5846bd]
+  - @llamaindex/readers@3.1.10
+
 ## 0.1.37
 
 ### Patch Changes

e2e/examples/nextjs-node-runtime/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/next-node-runtime-test",
-  "version": "0.1.37",
+  "version": "0.1.40",
   "private": true,
   "scripts": {
     "dev": "next dev",

e2e/examples/vite-import-llamaindex/CHANGELOG.md

@@ -1,5 +1,18 @@
 # vite-import-llamaindex
 
+## 0.0.38
+
+### Patch Changes
+
+- Updated dependencies [7039e1a]
+  - llamaindex@0.11.11
+
+## 0.0.37
+
+### Patch Changes
+
+- llamaindex@0.11.10
+
 ## 0.0.36
 
 ### Patch Changes

e2e/examples/vite-import-llamaindex/package.json

@@ -1,7 +1,7 @@
 {
   "name": "vite-import-llamaindex",
   "private": true,
-  "version": "0.0.36",
+  "version": "0.0.38",
   "type": "module",
   "scripts": {
     "build": "vite build",

e2e/examples/waku-query-engine/CHANGELOG.md

@@ -1,5 +1,18 @@
 # @llamaindex/waku-query-engine-test
 
+## 0.0.172
+
+### Patch Changes
+
+- Updated dependencies [7039e1a]
+  - llamaindex@0.11.11
+
+## 0.0.171
+
+### Patch Changes
+
+- llamaindex@0.11.10
+
 ## 0.0.170
 
 ### Patch Changes

e2e/examples/waku-query-engine/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/waku-query-engine-test",
-  "version": "0.0.170",
+  "version": "0.0.172",
   "type": "module",
   "private": true,
   "scripts": {

e2e/npm/package.json

@@ -7,7 +7,7 @@
   "dependencies": {
     "@llamaindex/workflow": "1.1.1",
     "llamaindex": "0.10.5",
-    "zod": "^3.23.8"
+    "zod": "^3.25.67"
   },
   "devDependencies": {
     "tsx": "^4.19.1",

e2e/package.json

@@ -27,6 +27,6 @@
     "pg": "^8.12.0",
     "pgvector": "0.2.0",
     "tsx": "^4.19.3",
-    "zod": "^3.24.2"
+    "zod": "^3.25.67"
   }
 }

examples/CHANGELOG.md

@@ -1,5 +1,66 @@
 # examples
 
+## 0.3.25
+
+### Patch Changes
+
+- Updated dependencies [7039e1a]
+- Updated dependencies [7039e1a]
+  - llamaindex@0.11.11
+  - @llamaindex/core@0.6.12
+  - @llamaindex/google@0.3.11
+  - @llamaindex/cloud@4.0.16
+  - @llamaindex/node-parser@2.0.12
+  - @llamaindex/anthropic@0.3.14
+  - @llamaindex/assemblyai@0.1.11
+  - @llamaindex/clip@0.0.62
+  - @llamaindex/cohere@0.0.26
+  - @llamaindex/deepinfra@0.0.62
+  - @llamaindex/discord@0.1.11
+  - @llamaindex/huggingface@0.1.16
+  - @llamaindex/jinaai@0.0.22
+  - @llamaindex/mistral@0.1.12
+  - @llamaindex/mixedbread@0.0.26
+  - @llamaindex/notion@0.1.11
+  - @llamaindex/ollama@0.1.12
+  - @llamaindex/openai@0.4.6
+  - @llamaindex/perplexity@0.0.19
+  - @llamaindex/portkey-ai@0.0.54
+  - @llamaindex/replicate@0.0.54
+  - @llamaindex/astra@0.0.26
+  - @llamaindex/azure@0.1.23
+  - @llamaindex/chroma@0.0.26
+  - @llamaindex/elastic-search@0.1.12
+  - @llamaindex/firestore@1.0.19
+  - @llamaindex/milvus@0.1.21
+  - @llamaindex/mongodb@0.0.27
+  - @llamaindex/pinecone@0.1.12
+  - @llamaindex/postgres@0.0.55
+  - @llamaindex/qdrant@0.1.22
+  - @llamaindex/supabase@0.1.12
+  - @llamaindex/upstash@0.0.26
+  - @llamaindex/weaviate@0.0.27
+  - @llamaindex/vercel@0.1.12
+  - @llamaindex/voyage-ai@1.0.18
+  - @llamaindex/readers@3.1.11
+  - @llamaindex/tools@0.1.1
+  - @llamaindex/workflow@1.1.12
+  - @llamaindex/deepseek@0.0.22
+  - @llamaindex/fireworks@0.0.22
+  - @llamaindex/groq@0.0.77
+  - @llamaindex/together@0.0.22
+  - @llamaindex/vllm@0.0.48
+  - @llamaindex/xai@0.0.9
+
+## 0.3.24
+
+### Patch Changes
+
+- Updated dependencies [096bf2b]
+- Updated dependencies [c5846bd]
+  - @llamaindex/tools@0.1.0
+  - @llamaindex/readers@3.1.10
+
 ## 0.3.23
 
 ### Patch Changes

examples/agents/agent/mcp-tools.ts

@@ -6,15 +6,24 @@ async function main() {
   // Create an MCP server for filesystem tools
   const server = mcp({
     command: "npx",
-    args: ["-y", "@modelcontextprotocol/server-filesystem", "."],
+    args: ["-y", "@modelcontextprotocol/server-filesystem@latest", "."],
     verbose: true,
   });
-  // You can also connect to the MCP server using SSE
-  // See: https://modelcontextprotocol.io/docs/concepts/transports#server-sent-events-sse
+  //
+  // You can also connect to a remote MCP server using:
+  // 1. StreamableHTTP transport (recommended)
+  // See: https://modelcontextprotocol.io/docs/concepts/transports#streamable-http
   // const server = mcp({
   //   url: "http://localhost:8000/mcp",
   //   verbose: true,
   // });
+  // 2.Or using SSE transport (will be deprecated soon)
+  // See: https://modelcontextprotocol.io/docs/concepts/transports#server-sent-events-sse-deprecated
+  // const server = mcp({
+  //   url: "http://localhost:8000/mcp",
+  //   useSSETransport: true,
+  //   verbose: true,
+  // });
 
   try {
     // Create an agent that uses the MCP tools

examples/data/company.xml

@@ -0,0 +1,72 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<company name="MidSizeCorp" founded="2008">
+  <division name="Engineering" head="Dana White">
+    <department name="Frontend" lead="Alex Kim">
+      <team name="Web">
+        <employee id="E01">
+          <name>Jordan Lee</name>
+          <role>Lead Developer</role>
+          <projects>
+            <project code="PRJ101" status="active">
+              <title>User Portal</title>
+              <deadline>2025-08-01</deadline>
+              <tasks>
+                <task id="T1011">
+                  <description>Implement login page</description>
+                  <due>2025-05-10</due>
+                </task>
+                <task id="T1012">
+                  <description>Design dashboard</description>
+                  <due>2025-05-20</due>
+                </task>
+              </tasks>
+            </project>
+          </projects>
+        </employee>
+        <employee id="E02">
+          <name>Riley Chen</name>
+          <role>UI Designer</role>
+        </employee>
+      </team>
+      <team name="Mobile">
+        <employee id="E03">
+          <name>Sam Patel</name>
+          <role>iOS Developer</role>
+        </employee>
+      </team>
+    </department>
+    <department name="Backend" lead="Morgan Reed">
+      <team name="API">
+        <employee id="E04">
+          <name>Taylor Jones</name>
+          <role>API Engineer</role>
+        </employee>
+      </team>
+      <team name="Database">
+        <employee id="E05">
+          <name>Casey Nguyen</name>
+          <role>DB Administrator</role>
+        </employee>
+      </team>
+    </department>
+  </division>
+
+  <division name="Marketing" head="Pat Morgan">
+    <department name="Digital" lead="Alex Rivera">
+      <team name="Content">
+        <employee id="M01">
+          <name>Charlie Brooks</name>
+          <role>Content Strategist</role>
+        </employee>
+      </team>
+    </department>
+  </division>
+
+  <headquarters location="Chicago, USA">
+    <address>
+      <street>789 Lake Shore Drive</street>
+      <city>Chicago</city>
+      <zip>60601</zip>
+    </address>
+  </headquarters>
+</company>

examples/models/gemini/chat.ts

@@ -1,14 +1,16 @@
-import { Gemini, GEMINI_MODEL } from "@llamaindex/google";
+import { gemini, GEMINI_MODEL } from "@llamaindex/google";
 import fs from "fs";
+import { tool } from "llamaindex";
+import { z } from "zod";
 
 (async () => {
   if (!process.env.GOOGLE_API_KEY) {
     throw new Error("Please set the GOOGLE_API_KEY environment variable.");
   }
-  const gemini = new Gemini({
-    model: GEMINI_MODEL.GEMINI_PRO_1_5,
-  });
-  const result = await gemini.chat({
+  const llm = gemini({ model: GEMINI_MODEL.GEMINI_2_0_FLASH });
+
+  // normal chat
+  const result = await llm.chat({
     messages: [
       { content: "You want to talk in rhymes.", role: "system" },
       {
@@ -18,10 +20,10 @@ import fs from "fs";
       },
     ],
   });
-  console.log(result);
+  console.log("\n normal chat: \n", result);
 
   // chat with file
-  const resultWithFile = await gemini.chat({
+  const resultWithFile = await llm.chat({
     messages: [
       {
         role: "user",
@@ -39,6 +41,52 @@ import fs from "fs";
       },
     ],
   });
+  console.log("\n chat with file: \n", resultWithFile);
 
-  console.log(resultWithFile);
+  // chat with image base64
+  const resultWithImageFile = await llm.chat({
+    messages: [
+      {
+        role: "user",
+        content: [
+          {
+            type: "text",
+            text: "What's in this image?",
+          },
+          {
+            type: "image",
+            data: fs
+              .readFileSync("./multimodal/data/60.jpg")
+              .toString("base64"),
+            mimeType: "image/png",
+          },
+        ],
+      },
+    ],
+  });
+  console.log("\n chat with image base64: \n", resultWithImageFile);
+
+  // chat with tool
+  const resultWithTool = await llm.chat({
+    messages: [
+      {
+        content: "What's the weather in Tokyo?",
+        role: "user",
+      },
+    ],
+    tools: [
+      tool({
+        name: "weather",
+        description: "Get the weather",
+        parameters: z.object({
+          location: z.string().describe("The location to get the weather for"),
+        }),
+        execute: ({ location }) => {
+          console.log("weather", location);
+          return `The weather in ${location} is sunny and hot`;
+        },
+      }),
+    ],
+  });
+  console.log("\n chat with tool: \n", resultWithTool.message.options); // should have toolCall
 })();

examples/models/gemini/chatVertex.ts

@@ -1,11 +1,14 @@
-import { Gemini, GEMINI_MODEL, GeminiVertexSession } from "@llamaindex/google";
+import { gemini, GEMINI_MODEL } from "@llamaindex/google";
 
 (async () => {
-  const gemini = new Gemini({
-    model: GEMINI_MODEL.GEMINI_PRO,
-    session: new GeminiVertexSession(),
+  const llm = gemini({
+    model: GEMINI_MODEL.GEMINI_2_0_FLASH,
+    vertex: {
+      project: "your-cloud-project", // update to your cloud project
+      location: "us-central1",
+    },
   });
-  const result = await gemini.chat({
+  const result = await llm.chat({
     messages: [
       { content: "You want to talk in rhymes.", role: "system" },
       {

examples/models/gemini/live/complex.ts

@@ -16,9 +16,19 @@ async function main() {
 
   console.log("🚀 Initializing Gemini Live API example...");
 
+  // Server-side (token creation):
+  const serverllm = gemini({
+    model: GEMINI_MODEL.GEMINI_2_0_FLASH_LIVE,
+    httpOptions: { apiVersion: "v1alpha" }, // must use v1alpha to generate ephemeral key
+  });
+  const ephemeralKey = await serverllm.live.getEphemeralKey();
+
+  // Client-side (Live API connection):
   const llm = gemini({
+    apiKey: ephemeralKey, // use ephemeral key for client-side
     model: GEMINI_MODEL.GEMINI_2_0_FLASH_LIVE,
     voiceName: "Zephyr",
+    httpOptions: { apiVersion: "v1alpha" }, // must use v1alpha to init client with ephemeral key
   });
 
   console.log("📡 Connecting to Gemini Live session...");

examples/models/gemini/live/simple.ts

@@ -3,8 +3,18 @@ import { liveEvents } from "llamaindex";
 import { saveWavFile } from "./util";
 
 async function main() {
-  const llm = gemini({
+  // Server-side (token creation):
+  const serverllm = gemini({
     model: GEMINI_MODEL.GEMINI_2_0_FLASH_LIVE,
+    httpOptions: { apiVersion: "v1alpha" }, // must use v1alpha to generate ephemeral key
+  });
+  const ephemeralKey = await serverllm.live.getEphemeralKey();
+
+  // Client-side (Live API connection):
+  const llm = gemini({
+    apiKey: ephemeralKey, // use ephemeral key for client-side
+    model: GEMINI_MODEL.GEMINI_2_0_FLASH_LIVE,
+    httpOptions: { apiVersion: "v1alpha" }, // must use v1alpha to init client with ephemeral key
   });
 
   const session = await llm.live.connect();

examples/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/examples",
-  "version": "0.3.23",
+  "version": "0.3.25",
   "private": true,
   "scripts": {
     "lint": "eslint .",
@@ -11,51 +11,51 @@
     "@azure/cosmos": "^4.1.1",
     "@azure/identity": "^4.4.1",
     "@azure/search-documents": "^12.1.0",
-    "@llamaindex/anthropic": "^0.3.13",
-    "@llamaindex/assemblyai": "^0.1.10",
-    "@llamaindex/astra": "^0.0.25",
-    "@llamaindex/azure": "^0.1.22",
-    "@llamaindex/chroma": "^0.0.25",
-    "@llamaindex/clip": "^0.0.61",
-    "@llamaindex/cloud": "^4.0.15",
-    "@llamaindex/cohere": "^0.0.25",
-    "@llamaindex/core": "^0.6.11",
-    "@llamaindex/deepinfra": "^0.0.61",
-    "@llamaindex/deepseek": "^0.0.21",
-    "@llamaindex/discord": "^0.1.10",
-    "@llamaindex/elastic-search": "^0.1.11",
+    "@llamaindex/anthropic": "^0.3.14",
+    "@llamaindex/assemblyai": "^0.1.11",
+    "@llamaindex/astra": "^0.0.26",
+    "@llamaindex/azure": "^0.1.23",
+    "@llamaindex/chroma": "^0.0.26",
+    "@llamaindex/clip": "^0.0.62",
+    "@llamaindex/cloud": "^4.0.16",
+    "@llamaindex/cohere": "^0.0.26",
+    "@llamaindex/core": "^0.6.12",
+    "@llamaindex/deepinfra": "^0.0.62",
+    "@llamaindex/deepseek": "^0.0.22",
+    "@llamaindex/discord": "^0.1.11",
+    "@llamaindex/elastic-search": "^0.1.12",
     "@llamaindex/env": "^0.1.30",
-    "@llamaindex/firestore": "^1.0.18",
-    "@llamaindex/fireworks": "^0.0.21",
-    "@llamaindex/google": "^0.3.10",
-    "@llamaindex/groq": "^0.0.76",
-    "@llamaindex/huggingface": "^0.1.15",
-    "@llamaindex/jinaai": "^0.0.21",
-    "@llamaindex/milvus": "^0.1.20",
-    "@llamaindex/mistral": "^0.1.11",
-    "@llamaindex/mixedbread": "^0.0.25",
-    "@llamaindex/mongodb": "^0.0.26",
-    "@llamaindex/node-parser": "^2.0.11",
-    "@llamaindex/notion": "^0.1.10",
-    "@llamaindex/ollama": "^0.1.11",
-    "@llamaindex/openai": "^0.4.5",
-    "@llamaindex/perplexity": "^0.0.18",
-    "@llamaindex/pinecone": "^0.1.11",
-    "@llamaindex/portkey-ai": "^0.0.53",
-    "@llamaindex/postgres": "^0.0.54",
-    "@llamaindex/qdrant": "^0.1.21",
-    "@llamaindex/readers": "^3.1.9",
-    "@llamaindex/replicate": "^0.0.53",
-    "@llamaindex/supabase": "^0.1.10",
-    "@llamaindex/together": "^0.0.21",
-    "@llamaindex/tools": "^0.0.17",
-    "@llamaindex/upstash": "^0.0.25",
-    "@llamaindex/vercel": "^0.1.11",
-    "@llamaindex/vllm": "^0.0.47",
-    "@llamaindex/voyage-ai": "^1.0.17",
-    "@llamaindex/weaviate": "^0.0.26",
-    "@llamaindex/workflow": "^1.1.10",
-    "@llamaindex/xai": "workspace:^0.0.8",
+    "@llamaindex/firestore": "^1.0.19",
+    "@llamaindex/fireworks": "^0.0.22",
+    "@llamaindex/google": "^0.3.11",
+    "@llamaindex/groq": "^0.0.77",
+    "@llamaindex/huggingface": "^0.1.16",
+    "@llamaindex/jinaai": "^0.0.22",
+    "@llamaindex/milvus": "^0.1.21",
+    "@llamaindex/mistral": "^0.1.12",
+    "@llamaindex/mixedbread": "^0.0.26",
+    "@llamaindex/mongodb": "^0.0.27",
+    "@llamaindex/node-parser": "^2.0.12",
+    "@llamaindex/notion": "^0.1.11",
+    "@llamaindex/ollama": "^0.1.12",
+    "@llamaindex/openai": "^0.4.6",
+    "@llamaindex/perplexity": "^0.0.19",
+    "@llamaindex/pinecone": "^0.1.12",
+    "@llamaindex/portkey-ai": "^0.0.54",
+    "@llamaindex/postgres": "^0.0.55",
+    "@llamaindex/qdrant": "^0.1.22",
+    "@llamaindex/readers": "^3.1.11",
+    "@llamaindex/replicate": "^0.0.54",
+    "@llamaindex/supabase": "^0.1.12",
+    "@llamaindex/together": "^0.0.22",
+    "@llamaindex/tools": "^0.1.1",
+    "@llamaindex/upstash": "^0.0.26",
+    "@llamaindex/vercel": "^0.1.12",
+    "@llamaindex/vllm": "^0.0.48",
+    "@llamaindex/voyage-ai": "^1.0.18",
+    "@llamaindex/weaviate": "^0.0.27",
+    "@llamaindex/workflow": "^1.1.12",
+    "@llamaindex/xai": "workspace:^0.0.9",
     "@notionhq/client": "^2.2.15",
     "@pinecone-database/pinecone": "^4.0.0",
     "@vercel/postgres": "^0.10.0",
@@ -64,11 +64,11 @@
     "commander": "^12.1.0",
     "dotenv": "^16.4.5",
     "js-tiktoken": "^1.0.14",
-    "llamaindex": "^0.11.9",
+    "llamaindex": "^0.11.11",
     "mongodb": "6.7.0",
     "postgres": "^3.4.4",
     "wikipedia": "^2.1.2",
-    "zod": "^3.23.8"
+    "zod": "^3.25.67"
   },
   "devDependencies": {
     "@types/node": "^22.9.0",

examples/readers/package.json

@@ -15,11 +15,14 @@
     "start:llamaparse-json": "node --import tsx ./src/llamaparse-json.ts",
     "start:discord": "node --import tsx ./src/discord.ts",
     "start:json": "node --import tsx ./src/json.ts",
-    "start:obsidian": "node --import tsx ./src/obsidian.ts"
+    "start:obsidian": "node --import tsx ./src/obsidian.ts",
+    "start:xml": "node --import tsx ./src/xml.ts",
+    "start:excel": "node --import tsx ./src/excel.ts"
   },
   "dependencies": {
     "@llamaindex/cloud": "workspace:* || ^2.0.24",
     "@llamaindex/readers": "workspace:* || ^1.0.25",
+    "@llamaindex/excel": "workspace:*",
     "llamaindex": "workspace:* || ^0.8.37"
   },
   "devDependencies": {

examples/readers/src/excel.ts

@@ -0,0 +1,20 @@
+import { ExcelReader } from "@llamaindex/excel";
+
+async function main() {
+  // Load PDF
+  const reader = new ExcelReader({
+    sheetSpecifier: 0,
+    concatRows: true,
+    fieldSeparator: ",",
+    keyValueSeparator: ":",
+  });
+
+  const documents = await reader.loadData("../data/sample_excel_sheet.xls");
+
+  for (const doc of documents) {
+    console.log(doc.text);
+    console.log("----");
+  }
+}
+
+main().catch(console.error);

examples/readers/src/xml.ts

@@ -0,0 +1,16 @@
+import { XMLReader } from "@llamaindex/readers/xml";
+
+async function main() {
+  // Load PDF
+  const reader = new XMLReader({
+    splitLevel: 2,
+  });
+  const documents = await reader.loadData("../data/company.xml");
+
+  for (const doc of documents) {
+    console.log(doc.text);
+    console.log("----");
+  }
+}
+
+main().catch(console.error);

examples/storage/qdrantdb/with-gemini.ts

@@ -1,8 +1,4 @@
-import {
-  GEMINI_EMBEDDING_MODEL,
-  GeminiEmbedding,
-  GeminiSession,
-} from "@llamaindex/google";
+import { GEMINI_EMBEDDING_MODEL, GeminiEmbedding } from "@llamaindex/google";
 import { QdrantVectorStore } from "@llamaindex/qdrant";
 import {
   Document,
@@ -12,9 +8,6 @@ import {
 
 const embedding = new GeminiEmbedding({
   model: GEMINI_EMBEDDING_MODEL.EMBEDDING_001,
-  session: new GeminiSession({
-    apiKey: process.env.GEMINI_API_KEY,
-  }),
 });
 
 async function main() {

packages/autotool/CHANGELOG.md

@@ -1,5 +1,18 @@
 # @llamaindex/autotool
 
+## 8.0.11
+
+### Patch Changes
+
+- Updated dependencies [7039e1a]
+  - llamaindex@0.11.11
+
+## 8.0.10
+
+### Patch Changes
+
+- llamaindex@0.11.10
+
 ## 8.0.9
 
 ### Patch Changes

packages/autotool/examples/01_node/CHANGELOG.md

@@ -1,5 +1,20 @@
 # @llamaindex/autotool-01-node-example
 
+## 0.0.119
+
+### Patch Changes
+
+- Updated dependencies [7039e1a]
+  - llamaindex@0.11.11
+  - @llamaindex/autotool@8.0.11
+
+## 0.0.118
+
+### Patch Changes
+
+- llamaindex@0.11.10
+- @llamaindex/autotool@8.0.10
+
 ## 0.0.117
 
 ### Patch Changes

packages/autotool/examples/01_node/package.json

@@ -13,5 +13,5 @@
   "scripts": {
     "start": "node --import tsx --import @llamaindex/autotool/node ./src/index.ts"
   },
-  "version": "0.0.117"
+  "version": "0.0.119"
 }

packages/autotool/package.json

@@ -6,7 +6,7 @@
     "url": "git+https://github.com/run-llama/LlamaIndexTS.git",
     "directory": "packages/autotool"
   },
-  "version": "8.0.9",
+  "version": "8.0.11",
   "description": "auto transpile your JS function to LLM Agent compatible",
   "files": [
     "dist",

packages/cloud/CHANGELOG.md

@@ -1,5 +1,13 @@
 # @llamaindex/cloud
 
+## 4.0.16
+
+### Patch Changes
+
+- Updated dependencies [7039e1a]
+- Updated dependencies [7039e1a]
+  - @llamaindex/core@0.6.12
+
 ## 4.0.15
 
 ### Patch Changes

packages/cloud/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/cloud",
-  "version": "4.0.15",
+  "version": "4.0.16",
   "type": "module",
   "license": "MIT",
   "scripts": {
@@ -79,6 +79,6 @@
   },
   "dependencies": {
     "p-retry": "^6.2.1",
-    "zod": "^3.25.7"
+    "zod": "^3.25.67"
   }
 }

packages/core/CHANGELOG.md

@@ -1,5 +1,12 @@
 # @llamaindex/core
 
+## 0.6.12
+
+### Patch Changes
+
+- 7039e1a: Internal cleanup of base64 encoding
+- 7039e1a: chore: migrate to @google/genai SDK
+
 ## 0.6.11
 
 ### Patch Changes

packages/core/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/core",
   "type": "module",
-  "version": "0.6.11",
+  "version": "0.6.12",
   "description": "LlamaIndex Core Module",
   "exports": {
     "./agent": {
@@ -312,7 +312,7 @@
     "@llamaindex/env": "workspace:*",
     "@types/node": "^22.9.0",
     "magic-bytes.js": "^1.10.0",
-    "zod": "^3.23.8",
-    "zod-to-json-schema": "^3.23.3"
+    "zod": "^3.25.67",
+    "zod-to-json-schema": "^3.24.6"
   }
 }

packages/core/src/utils/encoding.ts

@@ -0,0 +1,103 @@
+import { filetypemime } from "magic-bytes.js";
+
+/**
+ * Converts a base64 string (without data: prefix) to a Uint8Array
+ * @param base64 - The base64 string without data: prefix
+ * @returns The Uint8Array
+ */
+export function base64ToUint8Array(base64: string): Uint8Array {
+  // Decode Base64 string
+  const binaryString = atob(base64);
+
+  // Convert binary string to Uint8Array
+  const bytes = new Uint8Array(binaryString.length);
+  for (let i = 0; i < binaryString.length; i++) {
+    bytes[i] = binaryString.charCodeAt(i);
+  }
+
+  return bytes;
+}
+
+/**
+ * Converts a Uint8Array to a base64 string.
+ * @param uint8Array The Uint8Array to convert.
+ * @returns The base64-encoded string.
+ */
+export function uint8ArrayToBase64(uint8Array: Uint8Array): string {
+  let binary = "";
+  for (let i = 0; i < uint8Array.byteLength; i++) {
+    // Asserts that the value is not undefined, for `noUncheckedIndexedAccess`
+    binary += String.fromCharCode(uint8Array[i]!);
+  }
+  return btoa(binary);
+}
+
+/**
+ * Extracts the MIME type from a data URL.
+ * @param dataUrl The data URL string.
+ * @returns The MIME type from the data URL.
+ * @throws An error if the data URL is malformed.
+ */
+export function getMimeTypeFromDataUrl(dataUrl: string): string {
+  if (!dataUrl.startsWith("data:")) {
+    throw new Error("Not a data URL");
+  }
+  const commaIndex = dataUrl.indexOf(",");
+  if (commaIndex === -1) {
+    throw new Error("Invalid data URL format");
+  }
+
+  const header = dataUrl.slice(0, commaIndex);
+  const semicolonIndex = header.indexOf(";base64");
+  if (semicolonIndex === -1) {
+    throw new Error("Invalid data URL format: missing base64 encoding");
+  }
+
+  return header.slice(5, semicolonIndex);
+}
+
+/**
+ * Convert base64 data to Blob
+ * @param base64 - The base64 string
+ * @param mimeType - The MIME type of the file
+ * @returns The Blob
+ */
+export function base64ToBlob(base64: string, mimeType?: string): Blob {
+  let extractedMimeType = mimeType;
+  let base64Data = base64;
+
+  // Extract mimeType from data URL if not provided
+  if (!mimeType && base64.startsWith("data:")) {
+    extractedMimeType = getMimeTypeFromDataUrl(base64);
+    base64Data = base64.slice(base64.indexOf(",") + 1);
+  } else if (!mimeType) {
+    throw new Error(
+      "No MIME type provided and base64 is not in data URL format",
+    );
+  } else {
+    // Extract base64 data from data URL if present
+    const commaIndex = base64.indexOf(",");
+    base64Data = commaIndex !== -1 ? base64.slice(commaIndex + 1) : base64;
+  }
+
+  if (!extractedMimeType) {
+    throw new Error("No MIME type found in base64 data");
+  }
+
+  // convert base64 to Uint8Array
+  const bytes = base64ToUint8Array(base64Data);
+
+  // Create Blob
+  return new Blob([bytes], { type: extractedMimeType });
+}
+
+export async function blobToDataUrl(input: Blob) {
+  const arrayBuffer = await input.arrayBuffer();
+  const uint8Array = new Uint8Array(arrayBuffer);
+  const mimes = filetypemime(uint8Array);
+  if (mimes.length < 1) {
+    throw new Error("Unsupported image type");
+  }
+  const base64 = uint8ArrayToBase64(uint8Array);
+  return `data:${mimes[0]};base64,${base64}`;
+}

packages/core/src/utils/index.ts

@@ -72,5 +72,6 @@ export {
 
 export { MockLLM } from "./mock";
 
+export * from "./encoding";
 export { objectEntries } from "./object-entries";
 export * from "./stream";

packages/core/src/utils/llms.ts

@@ -1,5 +1,4 @@
 import { fs } from "@llamaindex/env";
-import { filetypemime } from "magic-bytes.js";
 import type {
   ChatMessage,
   MessageContent,
@@ -9,6 +8,7 @@ import type {
 } from "../llms";
 import type { QueryType } from "../query-engine";
 import type { ImageType } from "../schema";
+import { blobToDataUrl } from "./encoding";
 
 /**
  * Extracts just the text whether from
@@ -110,15 +110,6 @@ export function toToolDescriptions(tools: ToolMetadata[]): string {
   return JSON.stringify(toolsObj, null, 4);
 }
 
-async function blobToDataUrl(input: Blob) {
-  const buffer = Buffer.from(await input.arrayBuffer());
-  const mimes = filetypemime(buffer);
-  if (mimes.length < 1) {
-    throw new Error("Unsupported image type");
-  }
-  return "data:" + mimes[0] + ";base64," + buffer.toString("base64");
-}
-
 export async function imageToDataUrl(
   input: ImageType | Uint8Array,
 ): Promise<string> {

packages/core/tests/encoding.test.ts

@@ -0,0 +1,99 @@
+import {
+  base64ToBlob,
+  base64ToUint8Array,
+  blobToDataUrl,
+  getMimeTypeFromDataUrl,
+  uint8ArrayToBase64,
+} from "@llamaindex/core/utils";
+import { describe, expect, it } from "vitest";
+
+const testString = "LlamaIndex";
+const testBase64 = "TGxhbWFJbmRleA=="; // btoa('LlamaIndex')
+const testUint8Array = new TextEncoder().encode(testString);
+
+const pngB64 =
+  "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNkYAAAAAYAAjCB0C8AAAAASUVORK5CYII=";
+const pngMime = "image/png";
+const pngDataUrl = `data:${pngMime};base64,${pngB64}`;
+const pngBinaryString = atob(pngB64);
+const pngBytes = new Uint8Array(pngBinaryString.length);
+for (let i = 0; i < pngBinaryString.length; i++) {
+  pngBytes[i] = pngBinaryString.charCodeAt(i);
+}
+
+describe("Encoding utils", () => {
+  describe("base64ToUint8Array", () => {
+    it("should correctly convert a base64 string to a Uint8Array", () => {
+      const result = base64ToUint8Array(testBase64);
+      expect(result).toBeInstanceOf(Uint8Array);
+      expect(result).toEqual(testUint8Array);
+    });
+  });
+
+  describe("uint8ArrayToBase64", () => {
+    it("should correctly convert a Uint8Array to a base64 string", () => {
+      const result = uint8ArrayToBase64(testUint8Array);
+      expect(result).toBe(testBase64);
+    });
+  });
+
+  describe("getMimeTypeFromDataUrl", () => {
+    it("should extract the correct MIME type from a data URL", () => {
+      const result = getMimeTypeFromDataUrl(pngDataUrl);
+      expect(result).toBe(pngMime);
+    });
+
+    it("should throw an error for non-data URLs", () => {
+      expect(() => getMimeTypeFromDataUrl("not a data url")).toThrow(
+        "Not a data URL",
+      );
+    });
+
+    it("should throw an error for malformed data URLs", () => {
+      expect(() => getMimeTypeFromDataUrl("data:image/pngbase64,abc")).toThrow(
+        "Invalid data URL format: missing base64 encoding",
+      );
+      expect(() => getMimeTypeFromDataUrl("data:image/png;base64")).toThrow(
+        "Invalid data URL format",
+      );
+    });
+  });
+
+  describe("base64ToBlob", () => {
+    it("should convert from a data URL string", async () => {
+      const blob = base64ToBlob(pngDataUrl);
+      expect(blob).toBeInstanceOf(Blob);
+      expect(blob.type).toBe(pngMime);
+      const arrayBuffer = await blob.arrayBuffer();
+      expect(new Uint8Array(arrayBuffer)).toEqual(pngBytes);
+    });
+
+    it("should convert from a base64 string with an explicit MIME type", async () => {
+      const blob = base64ToBlob(pngB64, pngMime);
+      expect(blob).toBeInstanceOf(Blob);
+      expect(blob.type).toBe(pngMime);
+      const arrayBuffer = await blob.arrayBuffer();
+      expect(new Uint8Array(arrayBuffer)).toEqual(pngBytes);
+    });
+
+    it("should prioritize the explicit MIME type if a data URL is provided", async () => {
+      const differentMime = "image/jpeg";
+      const blob = base64ToBlob(pngDataUrl, differentMime);
+      expect(blob.type).toBe(differentMime);
+    });
+
+    it("should throw an error if no MIME type can be determined", () => {
+      expect(() => base64ToBlob(pngB64)).toThrow(
+        "No MIME type provided and base64 is not in data URL format",
+      );
+    });
+  });
+
+  describe("blobToDataUrl", () => {
+    it("should correctly convert a blob to a data URL", async () => {
+      const blob = new Blob([pngBytes], { type: "image/png" });
+      const result = await blobToDataUrl(blob);
+      expect(result).toBe(pngDataUrl);
+    });
+  });
+});

packages/experimental/CHANGELOG.md

@@ -1,5 +1,18 @@
 # @llamaindex/experimental
 
+## 0.0.188
+
+### Patch Changes
+
+- Updated dependencies [7039e1a]
+  - llamaindex@0.11.11
+
+## 0.0.187
+
+### Patch Changes
+
+- llamaindex@0.11.10
+
 ## 0.0.186
 
 ### Patch Changes

packages/experimental/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/experimental",
   "description": "Experimental package for LlamaIndexTS",
-  "version": "0.0.186",
+  "version": "0.0.188",
   "type": "module",
   "types": "dist/type/index.d.ts",
   "main": "dist/cjs/index.js",

packages/llamaindex/CHANGELOG.md

@@ -1,5 +1,24 @@
 # llamaindex
 
+## 0.11.11
+
+### Patch Changes
+
+- 7039e1a: Internal cleanup of base64 encoding
+- Updated dependencies [7039e1a]
+- Updated dependencies [7039e1a]
+  - @llamaindex/core@0.6.12
+  - @llamaindex/cloud@4.0.16
+  - @llamaindex/node-parser@2.0.12
+  - @llamaindex/workflow@1.1.12
+
+## 0.11.10
+
+### Patch Changes
+
+- Updated dependencies [f7ec293]
+  - @llamaindex/workflow@1.1.11
+
 ## 0.11.9
 
 ### Patch Changes

packages/llamaindex/package.json

@@ -1,6 +1,6 @@
 {
   "name": "llamaindex",
-  "version": "0.11.9",
+  "version": "0.11.11",
   "license": "MIT",
   "type": "module",
   "keywords": [

packages/llamaindex/src/agent/react.ts

@@ -18,11 +18,11 @@ import type {
 } from "@llamaindex/core/llms";
 import {
   extractText,
+  isAsyncIterable,
   stringifyJSONToMessageContent,
 } from "@llamaindex/core/utils";
 import { randomUUID } from "@llamaindex/env";
 import { getReACTAgentSystemHeader } from "../internal/prompt/react.js";
-import { isAsyncIterable } from "../internal/utils.js";
 import { Settings } from "../Settings.js";
 
 export type ReACTAgentParams = AgentParamsBase<LLM>;

packages/llamaindex/src/index.edge.ts

@@ -63,7 +63,6 @@ export * from "./evaluation/index.js";
 export * from "./extractors/index.js";
 export * from "./indices/index.js";
 export * from "./ingestion/index.js";
-export { imageToDataUrl } from "./internal/utils.js";
 export * from "./node-parser.js";
 export * from "./objects/index.js";
 export * from "./OutputParser.js";

packages/llamaindex/src/internal/utils.ts

@@ -1,77 +0,0 @@
-import type { ImageType } from "@llamaindex/core/schema";
-import { fs } from "@llamaindex/env";
-import { filetypemime } from "magic-bytes.js";
-
-export const isAsyncIterable = (
-  obj: unknown,
-): obj is AsyncIterable<unknown> => {
-  return obj != null && typeof obj === "object" && Symbol.asyncIterator in obj;
-};
-
-export const isReadableStream = (obj: unknown): obj is ReadableStream => {
-  return obj instanceof ReadableStream;
-};
-
-export const isIterable = (obj: unknown): obj is Iterable<unknown> => {
-  return obj != null && typeof obj === "object" && Symbol.iterator in obj;
-};
-
-async function blobToDataUrl(input: Blob) {
-  const buffer = Buffer.from(await input.arrayBuffer());
-  const mimes = filetypemime(buffer);
-  if (mimes.length < 1) {
-    throw new Error("Unsupported image type");
-  }
-  return "data:" + mimes[0] + ";base64," + buffer.toString("base64");
-}
-
-export async function imageToString(input: ImageType): Promise<string> {
-  if (input instanceof Blob) {
-    // if the image is a Blob, convert it to a base64 data URL
-    return await blobToDataUrl(input);
-  } else if (typeof input === "string") {
-    return input;
-  } else if (input instanceof URL) {
-    return input.toString();
-  } else {
-    throw new Error(`Unsupported input type: ${typeof input}`);
-  }
-}
-
-export function stringToImage(input: string): ImageType {
-  if (input.startsWith("data:")) {
-    // if the input is a base64 data URL, convert it back to a Blob
-    const base64Data = input.split(",")[1]!;
-    const byteArray = Buffer.from(base64Data, "base64");
-    return new Blob([byteArray]);
-  } else if (input.startsWith("http://") || input.startsWith("https://")) {
-    return new URL(input);
-  } else {
-    return input;
-  }
-}
-
-export async function imageToDataUrl(
-  input: ImageType | Uint8Array,
-): Promise<string> {
-  // first ensure, that the input is a Blob
-  if (
-    (input instanceof URL && input.protocol === "file:") ||
-    typeof input === "string"
-  ) {
-    // string or file URL
-    const dataBuffer = await fs.readFile(
-      input instanceof URL ? input.pathname : input,
-    );
-    input = new Blob([dataBuffer]);
-  } else if (!(input instanceof Blob)) {
-    if (input instanceof URL) {
-      throw new Error(`Unsupported URL with protocol: ${input.protocol}`);
-    } else if (input instanceof Uint8Array) {
-      input = new Blob([input]); // convert Uint8Array to Blob
-    } else {
-      throw new Error(`Unsupported input type: ${typeof input}`);
-    }
-  }
-  return await blobToDataUrl(input);
-}

packages/llamaindex/tests/CHANGELOG.md

@@ -1,5 +1,11 @@
 # @llamaindex/core-test
 
+## 0.1.7
+
+### Patch Changes
+
+- @llamaindex/openai@0.4.6
+
 ## 0.1.6
 
 ### Patch Changes

packages/llamaindex/tests/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/llamaindex-test",
   "private": true,
-  "version": "0.1.6",
+  "version": "0.1.7",
   "type": "module",
   "scripts": {
     "test": "vitest run"

packages/node-parser/CHANGELOG.md

@@ -1,5 +1,13 @@
 # @llamaindex/node-parser
 
+## 2.0.12
+
+### Patch Changes
+
+- Updated dependencies [7039e1a]
+- Updated dependencies [7039e1a]
+  - @llamaindex/core@0.6.12
+
 ## 2.0.11
 
 ### Patch Changes

packages/node-parser/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/node-parser",
-  "version": "2.0.11",
+  "version": "2.0.12",
   "description": "Node parser for LlamaIndex",
   "type": "module",
   "exports": {

packages/providers/anthropic/CHANGELOG.md

@@ -1,5 +1,13 @@
 # @llamaindex/anthropic
 
+## 0.3.14
+
+### Patch Changes
+
+- Updated dependencies [7039e1a]
+- Updated dependencies [7039e1a]
+  - @llamaindex/core@0.6.12
+
 ## 0.3.13
 
 ### Patch Changes

packages/providers/anthropic/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/anthropic",
   "description": "Anthropic Adapter for LlamaIndex",
-  "version": "0.3.13",
+  "version": "0.3.14",
   "type": "module",
   "main": "./dist/index.cjs",
   "module": "./dist/index.js",

packages/providers/assemblyai/CHANGELOG.md

@@ -1,5 +1,13 @@
 # @llamaindex/assemblyai
 
+## 0.1.11
+
+### Patch Changes
+
+- Updated dependencies [7039e1a]
+- Updated dependencies [7039e1a]
+  - @llamaindex/core@0.6.12
+
 ## 0.1.10
 
 ### Patch Changes

packages/providers/assemblyai/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/assemblyai",
   "description": "AssemblyAI Reader for LlamaIndex",
-  "version": "0.1.10",
+  "version": "0.1.11",
   "type": "module",
   "types": "dist/index.d.ts",
   "main": "dist/index.cjs",

packages/providers/aws/CHANGELOG.md

@@ -1,5 +1,13 @@
 # @llamaindex/community
 
+## 0.0.107
+
+### Patch Changes
+
+- Updated dependencies [7039e1a]
+- Updated dependencies [7039e1a]
+  - @llamaindex/core@0.6.12
+
 ## 0.0.106
 
 ### Patch Changes

packages/providers/aws/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/aws",
   "description": "AWS package for LlamaIndexTS",
-  "version": "0.0.106",
+  "version": "0.0.107",
   "type": "module",
   "types": "dist/type/index.d.ts",
   "main": "dist/cjs/index.js",

packages/providers/clip/CHANGELOG.md

@@ -1,5 +1,14 @@
 # @llamaindex/clip
 
+## 0.0.62
+
+### Patch Changes
+
+- Updated dependencies [7039e1a]
+- Updated dependencies [7039e1a]
+  - @llamaindex/core@0.6.12
+  - @llamaindex/openai@0.4.6
+
 ## 0.0.61
 
 ### Patch Changes

packages/providers/clip/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/clip",
   "description": "Clip Embedding Adapter for LlamaIndex",
-  "version": "0.0.61",
+  "version": "0.0.62",
   "type": "module",
   "types": "dist/index.d.ts",
   "main": "dist/index.cjs",

packages/providers/cohere/CHANGELOG.md

@@ -1,5 +1,13 @@
 # @llamaindex/cohere
 
+## 0.0.26
+
+### Patch Changes
+
+- Updated dependencies [7039e1a]
+- Updated dependencies [7039e1a]
+  - @llamaindex/core@0.6.12
+
 ## 0.0.25
 
 ### Patch Changes

packages/providers/cohere/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/cohere",
   "description": "Cohere Adapter for LlamaIndex",
-  "version": "0.0.25",
+  "version": "0.0.26",
   "type": "module",
   "main": "./dist/index.cjs",
   "module": "./dist/index.js",

packages/providers/deepinfra/CHANGELOG.md

@@ -1,5 +1,14 @@
 # @llamaindex/deepinfra
 
+## 0.0.62
+
+### Patch Changes
+
+- Updated dependencies [7039e1a]
+- Updated dependencies [7039e1a]
+  - @llamaindex/core@0.6.12
+  - @llamaindex/openai@0.4.6
+
 ## 0.0.61
 
 ### Patch Changes

packages/providers/deepinfra/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/deepinfra",
   "description": "Deepinfra Adapter for LlamaIndex",
-  "version": "0.0.61",
+  "version": "0.0.62",
   "type": "module",
   "main": "./dist/index.cjs",
   "module": "./dist/index.js",

packages/providers/deepseek/CHANGELOG.md

@@ -1,5 +1,11 @@
 # @llamaindex/deepseek
 
+## 0.0.22
+
+### Patch Changes
+
+- @llamaindex/openai@0.4.6
+
 ## 0.0.21
 
 ### Patch Changes

packages/providers/deepseek/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/deepseek",
   "description": "DeepSeek Adapter for LlamaIndex",
-  "version": "0.0.21",
+  "version": "0.0.22",
   "type": "module",
   "main": "./dist/index.cjs",
   "module": "./dist/index.js",

packages/providers/discord/CHANGELOG.md

@@ -1,5 +1,13 @@
 # @llamaindex/discord
 
+## 0.1.11
+
+### Patch Changes
+
+- Updated dependencies [7039e1a]
+- Updated dependencies [7039e1a]
+  - @llamaindex/core@0.6.12
+
 ## 0.1.10
 
 ### Patch Changes

packages/providers/discord/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/discord",
   "description": "Discord Reader for LlamaIndex",
-  "version": "0.1.10",
+  "version": "0.1.11",
   "type": "module",
   "types": "dist/index.d.ts",
   "main": "dist/index.cjs",

packages/providers/excel/CHANGELOG.md

@@ -0,0 +1,15 @@
+# @llamaindex/excel
+
+## 0.1.12
+
+### Patch Changes
+
+- Updated dependencies [7039e1a]
+- Updated dependencies [7039e1a]
+  - @llamaindex/core@0.6.12
+
+## 0.1.11
+
+### Patch Changes
+
+- 49a5e0a: feat: add Excel Reader

packages/providers/excel/package.json

@@ -0,0 +1,36 @@
+{
+  "name": "@llamaindex/excel",
+  "description": "Excel Reader for LlamaIndex",
+  "version": "0.1.12",
+  "type": "module",
+  "types": "dist/index.d.ts",
+  "main": "dist/index.cjs",
+  "module": "dist/index.js",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "default": "./dist/index.js"
+    }
+  },
+  "files": [
+    "dist"
+  ],
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/run-llama/LlamaIndexTS.git",
+    "directory": "packages/providers/excel"
+  },
+  "scripts": {
+    "build": "bunchee",
+    "dev": "bunchee --watch"
+  },
+  "devDependencies": {
+    "@llamaindex/core": "workspace:*"
+  },
+  "peerDependencies": {
+    "@llamaindex/core": "workspace:*"
+  },
+  "dependencies": {
+    "xlsx": "^0.18.5"
+  }
+}

packages/providers/excel/src/index.ts

@@ -0,0 +1 @@
+export * from "./reader";

packages/providers/excel/src/reader.ts

@@ -0,0 +1,100 @@
+import { Document, FileReader } from "@llamaindex/core/schema";
+import * as XLSX from "xlsx";
+
+interface ExcelReaderOptions {
+  /** concatenate all rows into one document (default: true) */
+  concatRows?: boolean;
+  /** which sheet(s) to read; string name or zero-based index */
+  sheetSpecifier?: string | number | undefined;
+  /** what to put between each field (default: ", ") */
+  fieldSeparator?: string;
+  /** what to put between key and value (default: ":") */
+  keyValueSeparator?: string;
+}
+
+/**
+ * ExcelReader reads an XLSX file from a Uint8Array (or Buffer) and
+ * produces Document instances, either one per sheet (concatenated)
+ * or one per row per sheet.
+ */
+export class ExcelReader extends FileReader<Document> {
+  private concatRows: boolean;
+  private sheetSpecifier?: string | number | undefined;
+  private fieldSeparator: string;
+  private keyValueSeparator: string;
+
+  /**
+   * @param concatRows        If true, all rows in a sheet become one Document;
+   *                          otherwise one Document per row.
+   * @param sheetSpecifier    Name or zero-based index of the sheet to read;
+   *                          undefined = all sheets.
+   * @param fieldSeparator    Separator between fields, e.g. ", ".
+   * @param keyValueSeparator Separator between key & value, e.g. ": ".
+   */
+  constructor({
+    concatRows = true,
+    sheetSpecifier = undefined,
+    fieldSeparator = ", ",
+    keyValueSeparator = ":",
+  }: ExcelReaderOptions = {}) {
+    super();
+    this.concatRows = concatRows;
+    this.sheetSpecifier = sheetSpecifier;
+    this.fieldSeparator = fieldSeparator;
+    this.keyValueSeparator = keyValueSeparator;
+  }
+
+  async loadDataAsContent(content: Uint8Array): Promise<Document[]> {
+    // Parse workbook from raw bytes
+    const workbook = XLSX.read(content, { type: "array" });
+
+    // Choose which sheets to process
+    let sheets = workbook.SheetNames;
+    if (this.sheetSpecifier !== undefined) {
+      if (typeof this.sheetSpecifier === "number") {
+        const name = sheets[this.sheetSpecifier];
+        sheets = name ? [name] : [];
+      } else {
+        sheets = sheets.includes(this.sheetSpecifier)
+          ? [this.sheetSpecifier]
+          : [];
+      }
+    }
+
+    const docs: Document[] = [];
+
+    for (const name of sheets) {
+      const ws = workbook.Sheets[name];
+
+      if (!ws) {
+        continue; // Skip if worksheet is undefined
+      }
+
+      // Convert sheet to JSON rows, using headers from first row
+      const rows: Record<string, unknown>[] = XLSX.utils.sheet_to_json(ws, {
+        defval: "N/A",
+      });
+
+      const textLines: string[] = rows.map((r) =>
+        Object.entries(r)
+          .map(([h, v]) => `${h}${this.keyValueSeparator}${v}`)
+          .join(this.fieldSeparator),
+      );
+
+      if (this.concatRows) {
+        docs.push(
+          new Document({
+            text: textLines.join("\n"),
+            metadata: { sheetName: name },
+          }),
+        );
+      } else {
+        textLines.forEach((text) =>
+          docs.push(new Document({ text, metadata: { sheetName: name } })),
+        );
+      }
+    }
+
+    return docs;
+  }
+}

packages/providers/excel/tsconfig.json

@@ -0,0 +1,19 @@
+{
+  "extends": "../../../tsconfig.json",
+  "compilerOptions": {
+    "target": "ESNext",
+    "module": "ESNext",
+    "moduleResolution": "bundler",
+    "outDir": "./lib",
+    "tsBuildInfoFile": "./lib/.tsbuildinfo"
+  },
+  "include": ["./src"],
+  "references": [
+    {
+      "path": "../../core/tsconfig.json"
+    },
+    {
+      "path": "../../env/tsconfig.json"
+    }
+  ]
+}

packages/providers/fireworks/CHANGELOG.md

@@ -1,5 +1,11 @@
 # @llamaindex/fireworks
 
+## 0.0.22
+
+### Patch Changes
+
+- @llamaindex/openai@0.4.6
+
 ## 0.0.21
 
 ### Patch Changes

packages/providers/fireworks/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/fireworks",
   "description": "Fireworks Adapter for LlamaIndex",
-  "version": "0.0.21",
+  "version": "0.0.22",
   "type": "module",
   "main": "./dist/index.cjs",
   "module": "./dist/index.js",

packages/providers/google/CHANGELOG.md

@@ -1,5 +1,14 @@
 # @llamaindex/google
 
+## 0.3.11
+
+### Patch Changes
+
+- 7039e1a: chore: migrate to @google/genai SDK
+- Updated dependencies [7039e1a]
+- Updated dependencies [7039e1a]
+  - @llamaindex/core@0.6.12
+
 ## 0.3.10
 
 ### Patch Changes

packages/providers/google/CLAUDE.md

@@ -9,36 +9,29 @@ The `@llamaindex/google` package provides Google AI integrations for LlamaIndex.
 - Gemini LLM models (via Google AI Studio and Vertex AI)
 - Google Studio API integration
 - Gemini embeddings
-- Live conversation capabilities
+- Live conversation capabilities with ephemeral tokens
 - Tool calling and function declarations
 
 ## Architecture
 
-This package implements the provider pattern from `@llamaindex/core`, offering multiple Google AI backends:
+This package implements the provider pattern from `@llamaindex/core`, using the unified `@google/genai` SDK:
 
 ### Core Classes
 
 - **`Gemini`** (`src/base.ts`): Main LLM class extending `ToolCallLLM` for Google AI Studio API
 - **`GoogleStudio`** (`src/studio/index.ts`): Alternative LLM implementation using `@google/genai` SDK
-- **`GeminiVertexSession`** (`src/vertex.ts`): Vertex AI backend session management
 - **`GeminiEmbedding`** (`src/GeminiEmbedding.ts`): Embedding implementation for text vectorization
 - **`GeminiLive`** (`src/live.ts`): Real-time conversation capabilities
 
 ### Session Management
 
-The package uses a session store pattern:
-
-- `GeminiSession`: Manages Google AI Studio connections
-- `GeminiVertexSession`: Manages Vertex AI connections
-- `GeminiSessionStore`: Centralized session management and reuse
+The package now uses the unified `@google/genai` SDK for all Google AI operations, supporting both Google AI Studio and Vertex AI backends through a single interface.
 
 ## Dependencies
 
-This package depends on three Google AI SDKs:
+This package depends on the unified Google AI SDK:
 
-- `@google/generative-ai`: Google AI Studio API (requires API key)
-- `@google-cloud/vertexai`: Vertex AI API (requires GCP authentication)
-- `@google/genai`: Alternative Google Studio implementation
+- `@google/genai`: Unified Google AI SDK supporting both Google AI Studio and Vertex AI
 
 ## Environment Variables
 
@@ -48,9 +41,6 @@ This package depends on three Google AI SDKs:
 
 ### Vertex AI (GCP Authentication)
 
-- `GOOGLE_VERTEX_PROJECT`: GCP project ID
-- `GOOGLE_VERTEX_LOCATION`: GCP region (e.g., 'us-central1')
-
 For Vertex AI, authentication uses Application Default Credentials:
 
 ```bash
@@ -59,7 +49,7 @@ gcloud auth application-default login
 
 ## Model Support
 
-### Available Models (src/types.ts:58-80)
+### Available Models
 
 - Gemini Pro/Flash family (1.0, 1.5, 2.0, 2.5)
 - Context windows: 30K to 2M tokens depending on model
@@ -87,6 +77,7 @@ gcloud auth application-default login
 ### Live Conversations (src/live.ts)
 
 - Real-time audio/video conversations
+- Ephemeral token authentication for secure client-side usage
 - Voice synthesis with configurable voice names
 - Streaming response handling
 
@@ -95,36 +86,58 @@ gcloud auth application-default login
 ### Basic LLM Usage
 
 ```typescript
-import { Gemini, GEMINI_MODEL } from "@llamaindex/google";
+import { gemini, GEMINI_MODEL } from "@llamaindex/google";
 
-const llm = new Gemini({
-  model: GEMINI_MODEL.GEMINI_PRO_1_5,
+const llm = gemini({
+  model: GEMINI_MODEL.GEMINI_2_0_FLASH,
   temperature: 0.1,
-  apiKey: "your-api-key",
 });
 ```
 
 ### Vertex AI Usage
 
 ```typescript
-import { GeminiVertexSession } from "@llamaindex/google";
+import { gemini, GEMINI_MODEL } from "@llamaindex/google";
 
-const session = new GeminiVertexSession({
-  project: "your-project",
-  location: "us-central1",
+const llm = gemini({
+  model: GEMINI_MODEL.GEMINI_2_0_FLASH,
+  vertex: {
+    project: "your-cloud-project",
+    location: "us-central1",
+  },
 });
 ```
 
 ### Embeddings
 
 ```typescript
-import { GeminiEmbedding } from "@llamaindex/google";
+import { GeminiEmbedding, GEMINI_EMBEDDING_MODEL } from "@llamaindex/google";
 
 const embedding = new GeminiEmbedding({
   model: GEMINI_EMBEDDING_MODEL.TEXT_EMBEDDING_004,
 });
 ```
 
+### Live API with Ephemeral Tokens
+
+```typescript
+import { gemini, GEMINI_MODEL } from "@llamaindex/google";
+
+// Server-side: Generate ephemeral key
+const serverLlm = gemini({
+  model: GEMINI_MODEL.GEMINI_2_0_FLASH_LIVE,
+  httpOptions: { apiVersion: "v1alpha" },
+});
+const ephemeralKey = await serverLlm.live.getEphemeralKey();
+
+// Client-side: Use ephemeral key
+const llm = gemini({
+  apiKey: ephemeralKey,
+  model: GEMINI_MODEL.GEMINI_2_0_FLASH_LIVE,
+  httpOptions: { apiVersion: "v1alpha" },
+});
+```
+
 ## Development Notes
 
 - The package uses bunchee for building with dual CJS/ESM support
@@ -133,6 +146,7 @@ const embedding = new GeminiEmbedding({
 - Safety settings are configurable with defaults provided
 - Session management enables connection reuse across multiple requests
 - Error handling includes proper API key validation and environment checks
+- Ephemeral tokens enable secure client-side Live API usage
 
 ## Testing Considerations
 
@@ -147,13 +161,12 @@ const embedding = new GeminiEmbedding({
 ```
 src/
 ├── base.ts           # Main Gemini LLM implementation
-├── vertex.ts         # Vertex AI session management
 ├── studio/           # Google Studio API implementation
 │   ├── index.ts      # GoogleStudio LLM class
 │   └── utils.ts      # Studio-specific utilities
 ├── live.ts           # Live conversation capabilities
 ├── GeminiEmbedding.ts # Embedding implementation
-├── types.ts          # Type definitions and enums
+├── constants.ts      # Constants and enums
 ├── utils.ts          # Shared utilities and helpers
-└── index.ts          # Package exports
+└── index.ts          # Package exports and factory functions
 ```

packages/providers/google/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/google",
   "description": "Google Adapter for LlamaIndex",
-  "version": "0.3.10",
+  "version": "0.3.11",
   "type": "module",
   "main": "./dist/index.cjs",
   "module": "./dist/index.js",
@@ -38,8 +38,6 @@
     "@llamaindex/env": "workspace:*"
   },
   "dependencies": {
-    "@google-cloud/vertexai": "1.9.0",
-    "@google/genai": "^0.12.0",
-    "@google/generative-ai": "0.24.0"
+    "@google/genai": "^1.7.0"
   }
 }

packages/providers/google/src/GeminiEmbedding.ts

@@ -1,39 +1,53 @@
+import { GoogleGenAI, type GoogleGenAIOptions } from "@google/genai";
 import { BaseEmbedding } from "@llamaindex/core/embeddings";
-import { GeminiSession, GeminiSessionStore } from "./base.js";
-import { GEMINI_BACKENDS } from "./types.js";
+import { getEnv } from "@llamaindex/env";
 
 export enum GEMINI_EMBEDDING_MODEL {
   EMBEDDING_001 = "embedding-001",
   TEXT_EMBEDDING_004 = "text-embedding-004",
 }
 
+/**
+ * Configuration options for GeminiEmbedding.
+ */
+export type GeminiEmbeddingOptions = {
+  model?: GEMINI_EMBEDDING_MODEL;
+} & GoogleGenAIOptions;
+
 /**
  * GeminiEmbedding is an alias for Gemini that implements the BaseEmbedding interface.
- * Note: Vertex SDK currently does not support embeddings
  */
 export class GeminiEmbedding extends BaseEmbedding {
   model: GEMINI_EMBEDDING_MODEL;
-  session: GeminiSession;
+  ai: GoogleGenAI;
 
-  constructor(init?: Partial<GeminiEmbedding>) {
+  constructor(opts?: GeminiEmbeddingOptions) {
     super();
-    this.model = init?.model ?? GEMINI_EMBEDDING_MODEL.EMBEDDING_001;
-    this.session =
-      init?.session ??
-      (GeminiSessionStore.get({
-        backend: GEMINI_BACKENDS.GOOGLE,
-      }) as GeminiSession);
+
+    const apiKey = opts?.apiKey ?? getEnv("GOOGLE_API_KEY");
+    if (!apiKey) {
+      throw new Error("Set Google API Key in GOOGLE_API_KEY env variable");
+    }
+
+    this.ai = new GoogleGenAI({ ...opts, apiKey });
+    this.model = opts?.model ?? GEMINI_EMBEDDING_MODEL.EMBEDDING_001;
   }
 
-  private async getEmbedding(prompt: string): Promise<number[]> {
-    const client = this.session.getGenerativeModel({
+  async getTextEmbeddingsBatch(texts: string[]): Promise<number[][]> {
+    const result = await this.ai.models.embedContent({
       model: this.model,
+      contents: texts,
     });
-    const result = await client.embedContent(prompt);
-    return result.embedding.values;
+
+    return result.embeddings?.map((embedding) => embedding.values ?? []) ?? [];
   }
 
-  getTextEmbedding(text: string): Promise<number[]> {
-    return this.getEmbedding(text);
+  async getTextEmbedding(text: string): Promise<number[]> {
+    const result = await this.ai.models.embedContent({
+      model: this.model,
+      contents: text,
+    });
+
+    return result.embeddings?.[0]?.values ?? [];
   }
 }

packages/providers/google/src/base.ts

@@ -1,260 +1,128 @@
 import {
-  GoogleGenerativeAI,
-  GenerativeModel as GoogleGenerativeModel,
-  type EnhancedGenerateContentResponse,
+  GoogleGenAI,
   type FunctionCall,
-  type ModelParams as GoogleModelParams,
-  type RequestOptions as GoogleRequestOptions,
-  type StartChatParams as GoogleStartChatParams,
-  type GenerateContentStreamResult as GoogleStreamGenerateContentResult,
+  type Content as GeminiMessage,
+  type GenerateContentConfig,
+  type HttpOptions,
+  type Part,
   type SafetySetting,
-} from "@google/generative-ai";
-
-import type { StartChatParams as VertexStartChatParams } from "@google-cloud/vertexai";
+} from "@google/genai";
 import { wrapLLMEvent } from "@llamaindex/core/decorator";
 import type {
+  ChatMessage,
+  ChatResponse,
+  ChatResponseChunk,
   CompletionResponse,
+  LLMChatParamsNonStreaming,
+  LLMChatParamsStreaming,
   LLMCompletionParamsNonStreaming,
   LLMCompletionParamsStreaming,
   LLMMetadata,
-  ToolCall,
+  MessageContent,
   ToolCallLLMMessageOptions,
 } from "@llamaindex/core/llms";
 import { ToolCallLLM } from "@llamaindex/core/llms";
 import { streamConverter } from "@llamaindex/core/utils";
 import { getEnv, randomUUID } from "@llamaindex/env";
-import { GeminiLive } from "./live.js";
-import {
-  GEMINI_BACKENDS,
-  GEMINI_MODEL,
-  type GeminiAdditionalChatOptions,
-  type GeminiChatNonStreamResponse,
-  type GeminiChatParamsNonStreaming,
-  type GeminiChatParamsStreaming,
-  type GeminiChatStreamResponse,
-  type GeminiMessageRole,
-  type GeminiModelInfo,
-  type GeminiSessionOptions,
-  type GeminiVoiceName,
-  type GoogleGeminiSessionOptions,
-  type IGeminiSession,
-} from "./types.js";
+import { GEMINI_MESSAGE_ROLE, GEMINI_MODEL } from "./constants";
 import {
   DEFAULT_SAFETY_SETTINGS,
-  GeminiHelper,
-  getChatContext,
-  getPartsText,
+  GEMINI_MODEL_INFO_MAP,
+  ROLES_FROM_GEMINI,
+  ROLES_TO_GEMINI,
+  SUPPORT_TOOL_CALL_MODELS,
+} from "./constants.js";
+import { GeminiLive, type GeminiVoiceName } from "./live.js";
+import {
   mapBaseToolToGeminiFunctionDeclaration,
+  mergeNeighboringSameRoleMessages,
+  messageContentDetailToGeminiPart,
 } from "./utils.js";
 
-export const GEMINI_MODEL_INFO_MAP: Record<GEMINI_MODEL, GeminiModelInfo> = {
-  [GEMINI_MODEL.GEMINI_PRO]: { contextWindow: 30720 },
-  [GEMINI_MODEL.GEMINI_PRO_VISION]: { contextWindow: 12288 },
-  // multi-modal/multi turn
-  [GEMINI_MODEL.GEMINI_PRO_LATEST]: { contextWindow: 10 ** 6 },
-  [GEMINI_MODEL.GEMINI_PRO_FLASH_LATEST]: { contextWindow: 10 ** 6 },
-  [GEMINI_MODEL.GEMINI_PRO_1_5_PRO_PREVIEW]: { contextWindow: 10 ** 6 },
-  [GEMINI_MODEL.GEMINI_PRO_1_5_FLASH_PREVIEW]: { contextWindow: 10 ** 6 },
-  [GEMINI_MODEL.GEMINI_PRO_1_5]: { contextWindow: 2 * 10 ** 6 },
-  [GEMINI_MODEL.GEMINI_PRO_1_5_FLASH]: { contextWindow: 10 ** 6 },
-  [GEMINI_MODEL.GEMINI_PRO_1_5_LATEST]: { contextWindow: 2 * 10 ** 6 },
-  [GEMINI_MODEL.GEMINI_PRO_1_5_FLASH_LATEST]: { contextWindow: 10 ** 6 },
-  [GEMINI_MODEL.GEMINI_2_0_FLASH_EXPERIMENTAL]: { contextWindow: 10 ** 6 },
-  [GEMINI_MODEL.GEMINI_2_0_FLASH]: { contextWindow: 10 ** 6 },
-  [GEMINI_MODEL.GEMINI_2_0_FLASH_LITE_PREVIEW]: { contextWindow: 10 ** 6 },
-  [GEMINI_MODEL.GEMINI_2_0_FLASH_LITE]: { contextWindow: 10 ** 6 },
-  [GEMINI_MODEL.GEMINI_2_0_FLASH_LIVE]: { contextWindow: 10 ** 6 },
-  [GEMINI_MODEL.GEMINI_2_0_FLASH_THINKING_EXP]: { contextWindow: 32768 },
-  [GEMINI_MODEL.GEMINI_2_0_PRO_EXPERIMENTAL]: { contextWindow: 2 * 10 ** 6 },
-  [GEMINI_MODEL.GEMINI_2_5_PRO_PREVIEW]: { contextWindow: 10 ** 6 },
-  [GEMINI_MODEL.GEMINI_2_5_PRO_PREVIEW_LATEST]: { contextWindow: 10 ** 6 },
-  [GEMINI_MODEL.GEMINI_2_5_FLASH_PREVIEW]: { contextWindow: 10 ** 6 },
-};
+type GeminiAdditionalChatOptions = object;
 
-export const SUPPORT_TOOL_CALL_MODELS: GEMINI_MODEL[] = [
-  GEMINI_MODEL.GEMINI_PRO,
-  GEMINI_MODEL.GEMINI_PRO_VISION,
-  GEMINI_MODEL.GEMINI_PRO_1_5_PRO_PREVIEW,
-  GEMINI_MODEL.GEMINI_PRO_1_5_FLASH_PREVIEW,
-  GEMINI_MODEL.GEMINI_PRO_1_5,
-  GEMINI_MODEL.GEMINI_PRO_1_5_FLASH,
-  GEMINI_MODEL.GEMINI_PRO_LATEST,
-  GEMINI_MODEL.GEMINI_PRO_FLASH_LATEST,
-  GEMINI_MODEL.GEMINI_PRO_1_5_LATEST,
-  GEMINI_MODEL.GEMINI_PRO_1_5_FLASH_LATEST,
-  GEMINI_MODEL.GEMINI_2_0_FLASH_EXPERIMENTAL,
-  GEMINI_MODEL.GEMINI_2_0_FLASH,
-  GEMINI_MODEL.GEMINI_2_0_PRO_EXPERIMENTAL,
-  GEMINI_MODEL.GEMINI_2_5_PRO_PREVIEW,
-  GEMINI_MODEL.GEMINI_2_5_PRO_PREVIEW_LATEST,
-  GEMINI_MODEL.GEMINI_2_5_FLASH_PREVIEW,
-];
+type GeminiChatParamsStreaming = LLMChatParamsStreaming<
+  GeminiAdditionalChatOptions,
+  ToolCallLLMMessageOptions
+>;
 
-export const DEFAULT_GEMINI_PARAMS = {
-  model: GEMINI_MODEL.GEMINI_PRO,
-  temperature: 0.1,
-  topP: 1,
-  maxTokens: undefined,
-};
+type GeminiChatStreamResponse = AsyncIterable<
+  ChatResponseChunk<ToolCallLLMMessageOptions>
+>;
 
-export type GeminiConfig = Partial<typeof DEFAULT_GEMINI_PARAMS> & {
-  apiKey?: string;
-  session?: IGeminiSession;
-  requestOptions?: GoogleRequestOptions;
+type GeminiChatParamsNonStreaming = LLMChatParamsNonStreaming<
+  GeminiAdditionalChatOptions,
+  ToolCallLLMMessageOptions
+>;
+
+type GeminiChatNonStreamResponse = ChatResponse<ToolCallLLMMessageOptions>;
+
+export type GeminiConfig = {
+  model?: GEMINI_MODEL;
+  temperature?: number;
+  topP?: number;
+  maxTokens?: number;
   safetySettings?: SafetySetting[];
   voiceName?: GeminiVoiceName;
+  apiKey?: string;
+  httpOptions?: HttpOptions;
+  vertex?: {
+    project?: string;
+    location?: string;
+  };
 };
 
-type StartChatParams = GoogleStartChatParams & VertexStartChatParams;
-
-/**
- * Gemini Session to manage the connection to the Gemini API
- */
-export class GeminiSession implements IGeminiSession {
-  private gemini: GoogleGenerativeAI;
-
-  constructor(options: GoogleGeminiSessionOptions) {
-    if (!options.apiKey) {
-      options.apiKey = getEnv("GOOGLE_API_KEY")!;
-    }
-    if (!options.apiKey) {
-      throw new Error("Set Google API Key in GOOGLE_API_KEY env variable");
-    }
-    this.gemini = new GoogleGenerativeAI(options.apiKey);
-  }
-
-  getGenerativeModel(
-    metadata: GoogleModelParams,
-    requestOpts?: GoogleRequestOptions,
-  ): GoogleGenerativeModel {
-    return this.gemini.getGenerativeModel(
-      {
-        safetySettings: metadata.safetySettings ?? DEFAULT_SAFETY_SETTINGS,
-        ...metadata,
-      },
-      requestOpts,
-    );
-  }
-
-  getResponseText(response: EnhancedGenerateContentResponse): string {
-    return response.text();
-  }
-
-  getToolsFromResponse(
-    response: EnhancedGenerateContentResponse,
-  ): ToolCall[] | undefined {
-    return response.functionCalls()?.map(
-      (call: FunctionCall) =>
-        ({
-          name: call.name,
-          input: call.args,
-          id: randomUUID(),
-        }) as ToolCall,
-    );
-  }
-
-  async *getChatStream(
-    result: GoogleStreamGenerateContentResult,
-  ): GeminiChatStreamResponse {
-    yield* streamConverter(result.stream, (response) => {
-      const tools = this.getToolsFromResponse(response);
-      const options: ToolCallLLMMessageOptions = tools?.length
-        ? { toolCall: tools }
-        : {};
-      return {
-        delta: this.getResponseText(response),
-        raw: response,
-        options,
-      };
-    });
-  }
-
-  getCompletionStream(
-    result: GoogleStreamGenerateContentResult,
-  ): AsyncIterable<CompletionResponse> {
-    return streamConverter(result.stream, (response) => ({
-      text: this.getResponseText(response),
-      raw: response,
-    }));
-  }
-}
-
-/**
- * Gemini Session Store to manage the current Gemini sessions
- */
-export class GeminiSessionStore {
-  static sessions: Array<{
-    session: IGeminiSession;
-    options: GeminiSessionOptions;
-  }> = [];
-
-  private static getSessionId(options: GeminiSessionOptions): string {
-    if (options.backend === GEMINI_BACKENDS.GOOGLE) {
-      return options?.apiKey ?? "";
-    }
-    return "";
-  }
-  private static sessionMatched(
-    o1: GeminiSessionOptions,
-    o2: GeminiSessionOptions,
-  ): boolean {
-    // #TODO: check if the session is matched
-    // Q: should we check the requestOptions?
-    // A: wait for confirmation from author
-    return (
-      GeminiSessionStore.getSessionId(o1) ===
-      GeminiSessionStore.getSessionId(o2)
-    );
-  }
-
-  static get(
-    options: GeminiSessionOptions = { backend: GEMINI_BACKENDS.GOOGLE },
-  ): IGeminiSession {
-    let session = this.sessions.find((session) =>
-      this.sessionMatched(session.options, options),
-    )?.session;
-    if (session) return session;
-
-    if (options.backend === GEMINI_BACKENDS.VERTEX) {
-      throw Error("No Session");
-    } else {
-      session = new GeminiSession(options);
-    }
-    this.sessions.push({ session, options });
-    return session;
-  }
-}
-
 /**
  * ToolCallLLM for Gemini
  */
 export class Gemini extends ToolCallLLM<GeminiAdditionalChatOptions> {
+  private client: GoogleGenAI;
+
   model: GEMINI_MODEL;
   temperature: number;
   topP: number;
   maxTokens?: number | undefined;
-  #requestOptions?: GoogleRequestOptions | undefined;
-  session: IGeminiSession;
   safetySettings: SafetySetting[];
   apiKey?: string | undefined;
-  voiceName?: GeminiVoiceName | undefined;
+
   private _live: GeminiLive | undefined;
+
+  voiceName?: GeminiVoiceName | undefined;
+  httpOptions?: HttpOptions | undefined;
+
   constructor(init?: GeminiConfig) {
     super();
     this.model = init?.model ?? GEMINI_MODEL.GEMINI_PRO;
     this.temperature = init?.temperature ?? 0.1;
     this.topP = init?.topP ?? 1;
     this.maxTokens = init?.maxTokens ?? undefined;
-    this.#requestOptions = init?.requestOptions ?? undefined;
     this.safetySettings = init?.safetySettings ?? DEFAULT_SAFETY_SETTINGS;
+    this.voiceName = init?.voiceName;
+    this.httpOptions = init?.httpOptions;
     this.apiKey = init?.apiKey ?? getEnv("GOOGLE_API_KEY");
-    this.voiceName = init?.voiceName ?? undefined;
-    this.session =
-      init?.session ??
-      GeminiSessionStore.get({
-        apiKey: this.apiKey,
-        backend: this.apiKey ? GEMINI_BACKENDS.GOOGLE : GEMINI_BACKENDS.VERTEX,
-      });
+
+    if (!this.apiKey) {
+      throw new Error("Set Google API Key in GOOGLE_API_KEY env variable");
+    }
+
+    if (init?.vertex) {
+      // Project/location and API key are mutually exclusive in the client initializer for Vertex AI
+      // So that we either need to provide project/location or apiKey
+      // See: https://github.com/googleapis/js-genai/blob/58a369ed36cd05be652c2279dcc9634fffc9fc10/src/node/node_client.ts#L89-L93
+      if (init?.vertex.project && init?.vertex.location) {
+        // when using Vertex AI, if project and location are provided, use them to create the client
+        this.client = new GoogleGenAI({
+          vertexai: true,
+          project: init.vertex.project,
+          location: init.vertex.location,
+        });
+      } else {
+        // when using Vertex AI, if project and location are not provided, use apiKey to create the client
+        this.client = new GoogleGenAI({ vertexai: true, apiKey: this.apiKey });
+      }
+    } else {
+      this.client = new GoogleGenAI({ ...init, apiKey: this.apiKey });
+    }
   }
 
   get supportToolCall(): boolean {
@@ -267,6 +135,7 @@ export class Gemini extends ToolCallLLM<GeminiAdditionalChatOptions> {
         apiKey: this.apiKey,
         voiceName: this.voiceName,
         model: this.model,
+        httpOptions: this.httpOptions,
       });
     }
     return this._live;
@@ -285,77 +154,15 @@ export class Gemini extends ToolCallLLM<GeminiAdditionalChatOptions> {
     };
   }
 
-  private async createStartChatParams(
-    params: GeminiChatParamsNonStreaming | GeminiChatParamsStreaming,
-  ) {
-    const context = await getChatContext(params);
-    const common = {
-      history: context.history,
-      safetySettings: this.safetySettings as SafetySetting[],
-    };
-
-    return params.tools?.length
-      ? {
-          ...common,
-          // only if non-empty tools list
-          tools: [
-            {
-              functionDeclarations: params.tools.map(
-                mapBaseToolToGeminiFunctionDeclaration,
-              ),
-            },
-          ],
-          safetySettings: this.safetySettings,
-        }
-      : common;
-  }
-
-  protected async nonStreamChat(
-    params: GeminiChatParamsNonStreaming,
-  ): Promise<GeminiChatNonStreamResponse> {
-    const context = await getChatContext(params);
-    const client = this.session.getGenerativeModel(
-      this.metadata,
-      this.#requestOptions,
-    );
-    const chat = client.startChat(
-      (await this.createStartChatParams(params)) as StartChatParams,
-    );
-    const { response } = await chat.sendMessage(context.message);
-    const topCandidate = response.candidates![0]!;
-
-    const tools = this.session.getToolsFromResponse(response);
-    const options: ToolCallLLMMessageOptions = tools?.length
-      ? { toolCall: tools }
-      : {};
-
+  get generationConfig(): GenerateContentConfig {
     return {
-      raw: response,
-      message: {
-        content: this.session.getResponseText(response),
-        role: GeminiHelper.ROLES_FROM_GEMINI[
-          topCandidate.content.role as GeminiMessageRole
-        ],
-        options,
-      },
+      temperature: this.temperature,
+      topP: this.topP,
+      safetySettings: this.safetySettings,
+      ...(this.maxTokens ? { maxOutputTokens: this.maxTokens } : {}),
     };
   }
 
-  protected async *streamChat(
-    params: GeminiChatParamsStreaming,
-  ): GeminiChatStreamResponse {
-    const context = await getChatContext(params);
-    const client = this.session.getGenerativeModel(
-      this.metadata,
-      this.#requestOptions,
-    );
-    const chat = client.startChat(
-      (await this.createStartChatParams(params)) as StartChatParams,
-    );
-    const result = await chat.sendMessageStream(context.message);
-    yield* this.session.getChatStream(result);
-  }
-
   chat(params: GeminiChatParamsStreaming): Promise<GeminiChatStreamResponse>;
   chat(
     params: GeminiChatParamsNonStreaming,
@@ -377,29 +184,211 @@ export class Gemini extends ToolCallLLM<GeminiAdditionalChatOptions> {
   async complete(
     params: LLMCompletionParamsStreaming | LLMCompletionParamsNonStreaming,
   ): Promise<CompletionResponse | AsyncIterable<CompletionResponse>> {
-    const { prompt, stream } = params;
-    const client = this.session.getGenerativeModel(
-      this.metadata,
-      this.#requestOptions,
-    );
-    if (stream) {
-      const result = await client.generateContentStream(
-        getPartsText(
-          await GeminiHelper.messageContentToGeminiParts({ content: prompt }),
-        ),
-      );
-      return this.session.getCompletionStream(result);
+    if (params.stream) return this.streamGenerate(params);
+    return this.nonStreamGenerate(params);
+  }
+
+  private async nonStreamChat(
+    params: GeminiChatParamsNonStreaming,
+  ): Promise<GeminiChatNonStreamResponse> {
+    const config = this.prepareChatConfig(params);
+    const { message, history } = await this.prepareChatContext(params.messages);
+
+    const chat = this.client.chats.create({
+      model: this.model,
+      config,
+      history,
+    });
+
+    const response = await chat.sendMessage({ message });
+
+    const topCandidate = response.candidates?.[0];
+    const geminiRole = topCandidate?.content?.role ?? GEMINI_MESSAGE_ROLE.MODEL;
+    const toolCall = response.functionCalls?.map((call) => ({
+      id: call.id ?? randomUUID(),
+      name: call.name,
+      input: call.args,
+    }));
+
+    return {
+      message: {
+        content: response.text ?? "",
+        role: ROLES_FROM_GEMINI[geminiRole as GEMINI_MESSAGE_ROLE],
+        options: toolCall?.length ? { toolCall } : undefined,
+      },
+      raw: response,
+    };
+  }
+
+  private async streamChat(
+    params: GeminiChatParamsStreaming,
+  ): Promise<GeminiChatStreamResponse> {
+    const config = this.prepareChatConfig(params);
+    const { message, history } = await this.prepareChatContext(params.messages);
+
+    console.log("history", JSON.stringify(history, null, 2));
+    console.log("message", JSON.stringify(message, null, 2));
+
+    const chat = this.client.chats.create({
+      model: this.model,
+      config,
+      history,
+    });
+
+    const generator = await chat.sendMessageStream({ message });
+
+    return streamConverter(generator, (response) => {
+      const toolCall = response.functionCalls?.map((call) => ({
+        id: call.id ?? randomUUID(),
+        name: call.name,
+        input: call.args,
+      }));
+      return {
+        delta: response.text ?? "",
+        options: toolCall?.length ? { toolCall } : undefined,
+        raw: response,
+      };
+    });
+  }
+
+  private async streamGenerate(
+    params: LLMCompletionParamsStreaming,
+  ): Promise<AsyncIterable<CompletionResponse>> {
+    const { prompt: content } = params;
+
+    const contents = await this.messageContentToGeminiParts(content);
+
+    const generator = await this.client.models.generateContentStream({
+      model: this.model,
+      contents,
+      config: this.generationConfig,
+    });
+
+    return streamConverter(generator, (response) => ({
+      text: response.text ?? "",
+      raw: response,
+    }));
+  }
+
+  private async nonStreamGenerate(
+    params: LLMCompletionParamsNonStreaming,
+  ): Promise<CompletionResponse> {
+    const { prompt: content } = params;
+
+    const contents = await this.messageContentToGeminiParts(content);
+
+    const result = await this.client.models.generateContent({
+      model: this.model,
+      config: this.generationConfig,
+      contents,
+    });
+
+    return {
+      text: result.text ?? "",
+      raw: result,
+    };
+  }
+
+  private prepareChatConfig(
+    params: GeminiChatParamsStreaming | GeminiChatParamsNonStreaming,
+  ): GenerateContentConfig {
+    const config = { ...this.generationConfig };
+
+    const functionDeclarations =
+      params.tools?.map(mapBaseToolToGeminiFunctionDeclaration) ?? [];
+
+    if (functionDeclarations.length) {
+      config.tools = [{ functionDeclarations }];
     }
 
-    const result = await client.generateContent(
-      getPartsText(
-        await GeminiHelper.messageContentToGeminiParts({ content: prompt }),
+    return config;
+  }
+
+  /**
+   * Prepare chat history and last message for chatting
+   * @param messages - array of LlamaIndex ChatMessage
+   * @returns chat history and last message
+   */
+  private async prepareChatContext(
+    messages: ChatMessage<ToolCallLLMMessageOptions>[],
+  ): Promise<{
+    message: Part[]; // content of the last message to send
+    history: GeminiMessage[]; // history (not including the last message)
+  }> {
+    // map tool call id to tool name
+    const toolMap = messages.reduce(
+      (result, message) => {
+        const { options } = message;
+        if (options && "toolCall" in options) {
+          options.toolCall.forEach((call) => {
+            result[call.id] = call.name;
+          });
+        }
+        return result;
+      },
+      {} as Record<string, string>,
+    );
+
+    const geminiMessages = await Promise.all(
+      messages.map(async (m) => ({
+        role: ROLES_TO_GEMINI[m.role],
+        parts: await this.messageToGeminiParts(m, toolMap),
+      })),
+    );
+    const mergedMessages = mergeNeighboringSameRoleMessages(geminiMessages);
+
+    return {
+      history: mergedMessages.slice(0, -1),
+      message: mergedMessages[mergedMessages.length - 1]?.parts || [],
+    };
+  }
+
+  private async messageToGeminiParts(
+    message: ChatMessage<ToolCallLLMMessageOptions>,
+    toolMap: Record<string, string>,
+  ): Promise<Part[]> {
+    const { options, content } = message;
+
+    // if message has toolResult, return a gemini function response part
+    if (options && "toolResult" in options) {
+      const { id, result } = options.toolResult;
+
+      const name = toolMap[id];
+      if (!name) {
+        throw Error(`Could not find the tool name with id ${id}`);
+      }
+
+      return [{ functionResponse: { id, name, response: { result } } }];
+    }
+
+    // if message has toolCall, return a list of gemini function call parts
+    if (options && "toolCall" in options) {
+      return options.toolCall.map((call) => ({
+        functionCall: {
+          id: call.id,
+          name: call.name,
+          args: call.input,
+        } as FunctionCall,
+      }));
+    }
+
+    // otherwise, extract parts from message content
+    return await this.messageContentToGeminiParts(content);
+  }
+
+  private async messageContentToGeminiParts(
+    content: MessageContent,
+  ): Promise<Part[]> {
+    // if message content is a string, just return a gemini text part
+    if (typeof content === "string") return [{ text: content }];
+
+    // if message content is an array of content details, convert each to a gemini part
+    // also upload files if needed
+    return await Promise.all(
+      content.map((item) =>
+        messageContentDetailToGeminiPart(item, this.client),
       ),
     );
-    return {
-      text: this.session.getResponseText(result.response),
-      raw: result.response,
-    };
   }
 }
 

packages/providers/google/src/constants.ts

@@ -0,0 +1,125 @@
+import {
+  HarmBlockThreshold,
+  HarmCategory,
+  type SafetySetting,
+} from "@google/genai";
+import type { MessageType } from "@llamaindex/core/llms";
+
+export enum GEMINI_MODEL {
+  GEMINI_PRO = "gemini-pro",
+  GEMINI_PRO_VISION = "gemini-pro-vision",
+  GEMINI_PRO_LATEST = "gemini-1.5-pro-latest",
+  GEMINI_PRO_FLASH_LATEST = "gemini-1.5-flash-latest",
+  GEMINI_PRO_1_5_PRO_PREVIEW = "gemini-1.5-pro-preview-0514",
+  GEMINI_PRO_1_5_FLASH_PREVIEW = "gemini-1.5-flash-preview-0514",
+  GEMINI_PRO_1_5 = "gemini-1.5-pro-001",
+  GEMINI_PRO_1_5_FLASH = "gemini-1.5-flash-001",
+  // Note: should be switched to -latest suffix when google supports it
+  GEMINI_PRO_1_5_LATEST = "gemini-1.5-pro-002",
+  GEMINI_PRO_1_5_FLASH_LATEST = "gemini-1.5-flash-002",
+  GEMINI_2_0_FLASH_EXPERIMENTAL = "gemini-2.0-flash-exp",
+  GEMINI_2_0_FLASH = "gemini-2.0-flash-001",
+  GEMINI_2_0_FLASH_LITE = "gemini-2.0-flash-lite-001",
+  GEMINI_2_0_FLASH_LITE_PREVIEW = "gemini-2.0-flash-lite-preview-02-05",
+  GEMINI_2_0_FLASH_THINKING_EXP = "gemini-2.0-flash-thinking-exp-01-21",
+  GEMINI_2_0_PRO_EXPERIMENTAL = "gemini-2.0-pro-exp-02-05",
+  GEMINI_2_0_FLASH_LIVE = "gemini-2.0-flash-live-001",
+  GEMINI_2_5_PRO_PREVIEW = "gemini-2.5-pro-preview-03-25",
+  GEMINI_2_5_PRO_PREVIEW_LATEST = "gemini-2.5-pro-preview-06-05",
+  GEMINI_2_5_FLASH_PREVIEW = "gemini-2.5-flash-preview-05-20",
+}
+
+export const GEMINI_MODEL_INFO_MAP: Record<
+  GEMINI_MODEL,
+  { contextWindow: number }
+> = {
+  [GEMINI_MODEL.GEMINI_PRO]: { contextWindow: 30720 },
+  [GEMINI_MODEL.GEMINI_PRO_VISION]: { contextWindow: 12288 },
+  // multi-modal/multi turn
+  [GEMINI_MODEL.GEMINI_PRO_LATEST]: { contextWindow: 10 ** 6 },
+  [GEMINI_MODEL.GEMINI_PRO_FLASH_LATEST]: { contextWindow: 10 ** 6 },
+  [GEMINI_MODEL.GEMINI_PRO_1_5_PRO_PREVIEW]: { contextWindow: 10 ** 6 },
+  [GEMINI_MODEL.GEMINI_PRO_1_5_FLASH_PREVIEW]: { contextWindow: 10 ** 6 },
+  [GEMINI_MODEL.GEMINI_PRO_1_5]: { contextWindow: 2 * 10 ** 6 },
+  [GEMINI_MODEL.GEMINI_PRO_1_5_FLASH]: { contextWindow: 10 ** 6 },
+  [GEMINI_MODEL.GEMINI_PRO_1_5_LATEST]: { contextWindow: 2 * 10 ** 6 },
+  [GEMINI_MODEL.GEMINI_PRO_1_5_FLASH_LATEST]: { contextWindow: 10 ** 6 },
+  [GEMINI_MODEL.GEMINI_2_0_FLASH_EXPERIMENTAL]: { contextWindow: 10 ** 6 },
+  [GEMINI_MODEL.GEMINI_2_0_FLASH]: { contextWindow: 10 ** 6 },
+  [GEMINI_MODEL.GEMINI_2_0_FLASH_LITE_PREVIEW]: { contextWindow: 10 ** 6 },
+  [GEMINI_MODEL.GEMINI_2_0_FLASH_LITE]: { contextWindow: 10 ** 6 },
+  [GEMINI_MODEL.GEMINI_2_0_FLASH_LIVE]: { contextWindow: 10 ** 6 },
+  [GEMINI_MODEL.GEMINI_2_0_FLASH_THINKING_EXP]: { contextWindow: 32768 },
+  [GEMINI_MODEL.GEMINI_2_0_PRO_EXPERIMENTAL]: { contextWindow: 2 * 10 ** 6 },
+  [GEMINI_MODEL.GEMINI_2_5_PRO_PREVIEW]: { contextWindow: 10 ** 6 },
+  [GEMINI_MODEL.GEMINI_2_5_PRO_PREVIEW_LATEST]: { contextWindow: 10 ** 6 },
+  [GEMINI_MODEL.GEMINI_2_5_FLASH_PREVIEW]: { contextWindow: 10 ** 6 },
+};
+
+export const SUPPORT_TOOL_CALL_MODELS: GEMINI_MODEL[] = [
+  GEMINI_MODEL.GEMINI_PRO,
+  GEMINI_MODEL.GEMINI_PRO_VISION,
+  GEMINI_MODEL.GEMINI_PRO_1_5_PRO_PREVIEW,
+  GEMINI_MODEL.GEMINI_PRO_1_5_FLASH_PREVIEW,
+  GEMINI_MODEL.GEMINI_PRO_1_5,
+  GEMINI_MODEL.GEMINI_PRO_1_5_FLASH,
+  GEMINI_MODEL.GEMINI_PRO_LATEST,
+  GEMINI_MODEL.GEMINI_PRO_FLASH_LATEST,
+  GEMINI_MODEL.GEMINI_PRO_1_5_LATEST,
+  GEMINI_MODEL.GEMINI_PRO_1_5_FLASH_LATEST,
+  GEMINI_MODEL.GEMINI_2_0_FLASH_EXPERIMENTAL,
+  GEMINI_MODEL.GEMINI_2_0_FLASH,
+  GEMINI_MODEL.GEMINI_2_0_PRO_EXPERIMENTAL,
+  GEMINI_MODEL.GEMINI_2_5_PRO_PREVIEW,
+  GEMINI_MODEL.GEMINI_2_5_PRO_PREVIEW_LATEST,
+  GEMINI_MODEL.GEMINI_2_5_FLASH_PREVIEW,
+];
+
+export const DEFAULT_GEMINI_PARAMS = {
+  model: GEMINI_MODEL.GEMINI_PRO,
+  temperature: 0.1,
+  topP: 1,
+  maxTokens: undefined,
+};
+
+/**
+ * Safety settings to disable external filters
+ * Documentation: https://ai.google.dev/gemini-api/docs/safety-settings
+ */
+export const DEFAULT_SAFETY_SETTINGS: SafetySetting[] = [
+  {
+    category: HarmCategory.HARM_CATEGORY_DANGEROUS_CONTENT,
+    threshold: HarmBlockThreshold.BLOCK_NONE,
+  },
+  {
+    category: HarmCategory.HARM_CATEGORY_HARASSMENT,
+    threshold: HarmBlockThreshold.BLOCK_NONE,
+  },
+  {
+    category: HarmCategory.HARM_CATEGORY_HATE_SPEECH,
+    threshold: HarmBlockThreshold.BLOCK_NONE,
+  },
+  {
+    category: HarmCategory.HARM_CATEGORY_SEXUALLY_EXPLICIT,
+    threshold: HarmBlockThreshold.BLOCK_NONE,
+  },
+];
+
+export enum GEMINI_MESSAGE_ROLE {
+  USER = "user",
+  MODEL = "model",
+}
+
+// Gemini only has user and model roles. Put the rest in user role.
+export const ROLES_TO_GEMINI: Record<MessageType, GEMINI_MESSAGE_ROLE> = {
+  assistant: GEMINI_MESSAGE_ROLE.MODEL,
+  user: GEMINI_MESSAGE_ROLE.USER,
+  system: GEMINI_MESSAGE_ROLE.USER,
+  memory: GEMINI_MESSAGE_ROLE.USER,
+  developer: GEMINI_MESSAGE_ROLE.USER,
+};
+
+export const ROLES_FROM_GEMINI: Record<GEMINI_MESSAGE_ROLE, MessageType> = {
+  [GEMINI_MESSAGE_ROLE.MODEL]: "assistant",
+  [GEMINI_MESSAGE_ROLE.USER]: "user",
+};

packages/providers/google/src/index.ts

@@ -1,13 +1,12 @@
 export * from "./base";
 
+export * from "./constants";
 export * from "./live";
 export {
   GoogleStudio,
   Modality,
   getGoogleStudioInlineData,
 } from "./studio/index.js";
-export * from "./types";
 export * from "./utils";
-export * from "./vertex";
 
 export * from "./GeminiEmbedding";

packages/providers/google/src/live.ts

@@ -6,6 +6,7 @@ import {
   type FunctionCall,
   type FunctionDeclaration,
   type LiveConnectConfig as GoogleLiveConnectConfig,
+  type HttpOptions,
   type LiveServerMessage,
 } from "@google/genai";
 import {
@@ -16,17 +17,28 @@ import {
   type MessageSender,
 } from "@llamaindex/core/llms";
 import { getEnv } from "@llamaindex/env";
+import { GEMINI_MODEL } from "./constants";
 import { GeminiMessageSender } from "./message-sender";
-import { GEMINI_MODEL, type GeminiVoiceName } from "./types";
 import {
   mapBaseToolToGeminiLiveFunctionDeclaration,
   mapResponseModalityToGeminiLiveResponseModality,
 } from "./utils";
 
-interface GeminiLiveConfig {
+export type GeminiVoiceName =
+  | "Puck"
+  | "Charon"
+  | "Fenrir"
+  | "Aoede"
+  | "Leda"
+  | "Kore"
+  | "Orus"
+  | "Zephyr";
+
+export interface GeminiLiveConfig {
   apiKey?: string | undefined;
   voiceName?: GeminiVoiceName | undefined;
   model?: GEMINI_MODEL | undefined;
+  httpOptions?: HttpOptions | undefined;
 }
 
 export class GeminiLiveSession extends LiveLLMSession {
@@ -193,10 +205,12 @@ export class GeminiLive extends LiveLLM {
   private client: GoogleGenAI;
   voiceName?: GeminiVoiceName | undefined;
   model: GEMINI_MODEL;
+  httpOptions?: HttpOptions | undefined;
 
   constructor(init?: GeminiLiveConfig) {
     super();
     this.apiKey = init?.apiKey ?? getEnv("GOOGLE_API_KEY");
+    this.httpOptions = init?.httpOptions;
 
     if (!this.apiKey) {
       throw new Error("GOOGLE_API_KEY is not set");
@@ -204,6 +218,7 @@ export class GeminiLive extends LiveLLM {
 
     this.client = new GoogleGenAI({
       apiKey: this.apiKey,
+      ...(this.httpOptions ? { httpOptions: this.httpOptions } : {}),
     });
     this.voiceName = init?.voiceName;
     this.model = init?.model ?? GEMINI_MODEL.GEMINI_2_0_FLASH_LIVE;
@@ -213,8 +228,28 @@ export class GeminiLive extends LiveLLM {
     }
   }
 
-  async getEphemeralKey(): Promise<string | undefined> {
-    throw new Error("Ephemeral key is not supported for Gemini Live");
+  async getEphemeralKey(): Promise<string> {
+    if (this.httpOptions?.apiVersion !== "v1alpha") {
+      // see: https://github.com/googleapis/js-genai/issues/691#issuecomment-3002302279
+      throw new Error("Ephemeral key generation is only supported in v1alpha");
+    }
+
+    const token = await this.client.authTokens.create({
+      config: {
+        liveConnectConstraints: {
+          model: this.model,
+          config: {
+            responseModalities: [Modality.AUDIO],
+          },
+        },
+        httpOptions: this.httpOptions,
+      },
+    });
+    if (!token.name) {
+      throw new Error("Failed to generate ephemeral key");
+    }
+
+    return token.name;
   }
 
   async connect(config?: LiveConnectConfig) {

packages/providers/google/src/studio/index.ts

@@ -26,8 +26,11 @@ import { streamConverter } from "@llamaindex/core/utils";
 import { wrapLLMEvent } from "@llamaindex/core/decorator";
 import type { JSONObject } from "@llamaindex/core/global";
 import { randomUUID } from "@llamaindex/env";
-import { DEFAULT_GEMINI_PARAMS, SUPPORT_TOOL_CALL_MODELS } from "../base";
-import type { GEMINI_MODEL } from "../types";
+import {
+  DEFAULT_GEMINI_PARAMS,
+  GEMINI_MODEL,
+  SUPPORT_TOOL_CALL_MODELS,
+} from "../constants";
 import {
   mapChatMessagesToGoogleFunctions,
   mapChatMessagesToGoogleMessages,

packages/providers/google/src/types.ts

@@ -1,172 +0,0 @@
-import {
-  GenerativeModel as GoogleGenerativeModel,
-  type EnhancedGenerateContentResponse,
-  type Content as GeminiMessageContent,
-  type FileDataPart as GoogleFileDataPart,
-  type FunctionDeclaration as GoogleFunctionDeclaration,
-  type FunctionDeclarationSchema as GoogleFunctionDeclarationSchema,
-  type InlineDataPart as GoogleInlineFileDataPart,
-  type ModelParams as GoogleModelParams,
-  type Part as GooglePart,
-  type RequestOptions as GoogleRequestOptions,
-  type GenerateContentStreamResult as GoogleStreamGenerateContentResult,
-} from "@google/generative-ai";
-
-import {
-  GenerativeModel as VertexGenerativeModel,
-  GenerativeModelPreview as VertexGenerativeModelPreview,
-  type GenerateContentResponse,
-  type FileDataPart as VertexFileDataPart,
-  type FunctionDeclaration as VertexFunctionDeclaration,
-  type FunctionDeclarationSchema as VertexFunctionDeclarationSchema,
-  type VertexInit,
-  type InlineDataPart as VertexInlineFileDataPart,
-  type ModelParams as VertexModelParams,
-  type Part as VertexPart,
-  type StreamGenerateContentResult as VertexStreamGenerateContentResult,
-} from "@google-cloud/vertexai";
-
-import type {
-  ChatResponse,
-  ChatResponseChunk,
-  CompletionResponse,
-  LLMChatParamsNonStreaming,
-  LLMChatParamsStreaming,
-  ToolCall,
-  ToolCallLLMMessageOptions,
-} from "@llamaindex/core/llms";
-
-export { type RequestOptions as GoogleRequestOptions } from "@google/generative-ai";
-
-export enum GEMINI_BACKENDS {
-  GOOGLE = "google",
-  VERTEX = "vertex",
-}
-
-export type GoogleGeminiSessionOptions = {
-  apiKey?: string | undefined;
-};
-
-export type VertexGeminiSessionOptions = {
-  preview?: boolean;
-} & VertexInit;
-
-export type GeminiSessionOptions =
-  | (GoogleGeminiSessionOptions & { backend: GEMINI_BACKENDS.GOOGLE })
-  | (VertexGeminiSessionOptions & { backend: GEMINI_BACKENDS.VERTEX });
-
-export enum GEMINI_MODEL {
-  GEMINI_PRO = "gemini-pro",
-  GEMINI_PRO_VISION = "gemini-pro-vision",
-  GEMINI_PRO_LATEST = "gemini-1.5-pro-latest",
-  GEMINI_PRO_FLASH_LATEST = "gemini-1.5-flash-latest",
-  GEMINI_PRO_1_5_PRO_PREVIEW = "gemini-1.5-pro-preview-0514",
-  GEMINI_PRO_1_5_FLASH_PREVIEW = "gemini-1.5-flash-preview-0514",
-  GEMINI_PRO_1_5 = "gemini-1.5-pro-001",
-  GEMINI_PRO_1_5_FLASH = "gemini-1.5-flash-001",
-  // Note: should be switched to -latest suffix when google supports it
-  GEMINI_PRO_1_5_LATEST = "gemini-1.5-pro-002",
-  GEMINI_PRO_1_5_FLASH_LATEST = "gemini-1.5-flash-002",
-  GEMINI_2_0_FLASH_EXPERIMENTAL = "gemini-2.0-flash-exp",
-  GEMINI_2_0_FLASH = "gemini-2.0-flash-001",
-  GEMINI_2_0_FLASH_LITE = "gemini-2.0-flash-lite-001",
-  GEMINI_2_0_FLASH_LITE_PREVIEW = "gemini-2.0-flash-lite-preview-02-05",
-  GEMINI_2_0_FLASH_THINKING_EXP = "gemini-2.0-flash-thinking-exp-01-21",
-  GEMINI_2_0_PRO_EXPERIMENTAL = "gemini-2.0-pro-exp-02-05",
-  GEMINI_2_0_FLASH_LIVE = "gemini-2.0-flash-live-001",
-  GEMINI_2_5_PRO_PREVIEW = "gemini-2.5-pro-preview-03-25",
-  GEMINI_2_5_PRO_PREVIEW_LATEST = "gemini-2.5-pro-preview-06-05",
-  GEMINI_2_5_FLASH_PREVIEW = "gemini-2.5-flash-preview-05-20",
-}
-
-export interface GeminiModelInfo {
-  contextWindow: number;
-}
-
-export type Part = GooglePart | VertexPart;
-export type FileDataPart = GoogleFileDataPart | VertexFileDataPart;
-export type InlineDataPart =
-  | GoogleInlineFileDataPart
-  | VertexInlineFileDataPart;
-
-export type ModelParams = GoogleModelParams | VertexModelParams;
-
-export type FunctionDeclaration =
-  | VertexFunctionDeclaration
-  | GoogleFunctionDeclaration;
-
-export type FunctionDeclarationSchema =
-  | GoogleFunctionDeclarationSchema
-  | VertexFunctionDeclarationSchema;
-
-export type GenerativeModel =
-  | VertexGenerativeModelPreview
-  | VertexGenerativeModel
-  | GoogleGenerativeModel;
-
-export type ChatContext = { message: Part[]; history: GeminiMessageContent[] };
-
-export type GeminiMessageRole = "user" | "model" | "function";
-
-export type GeminiAdditionalChatOptions = object;
-
-export type GeminiChatParamsStreaming = LLMChatParamsStreaming<
-  GeminiAdditionalChatOptions,
-  ToolCallLLMMessageOptions
->;
-
-export type GeminiChatStreamResponse = AsyncIterable<
-  ChatResponseChunk<ToolCallLLMMessageOptions>
->;
-
-export type GeminiChatParamsNonStreaming = LLMChatParamsNonStreaming<
-  GeminiAdditionalChatOptions,
-  ToolCallLLMMessageOptions
->;
-
-export type GeminiChatNonStreamResponse =
-  ChatResponse<ToolCallLLMMessageOptions>;
-
-export interface IGeminiSession {
-  getGenerativeModel(
-    metadata: ModelParams,
-    requestOptions?: GoogleRequestOptions,
-  ): GenerativeModel;
-  getResponseText(
-    response: EnhancedGenerateContentResponse | GenerateContentResponse,
-  ): string;
-  getCompletionStream(
-    result:
-      | GoogleStreamGenerateContentResult
-      | VertexStreamGenerateContentResult,
-  ): AsyncIterable<CompletionResponse>;
-  getChatStream(
-    result:
-      | GoogleStreamGenerateContentResult
-      | VertexStreamGenerateContentResult,
-  ): GeminiChatStreamResponse;
-  getToolsFromResponse(
-    response: EnhancedGenerateContentResponse | GenerateContentResponse,
-  ): ToolCall[] | undefined;
-}
-
-export type GeminiLiveMessage = {
-  content: string | GeminiLiveMessageDetail;
-  role: "user" | "model";
-};
-
-export type GeminiLiveMessageDetail = {
-  type: "text" | "audio" | "image" | "video";
-  data: string;
-  mimeType: string;
-};
-
-export type GeminiVoiceName =
-  | "Puck"
-  | "Charon"
-  | "Fenrir"
-  | "Aoede"
-  | "Leda"
-  | "Kore"
-  | "Orus"
-  | "Zephyr";

packages/providers/google/src/utils.ts

@@ -1,188 +1,30 @@
-import { type GenerateContentResponse } from "@google-cloud/vertexai";
 import {
-  type FunctionDeclaration as LiveFunctionDeclaration,
+  createPartFromUri,
+  type FunctionDeclaration,
+  type Content as GeminiMessage,
+  GoogleGenAI,
   Modality,
+  type Part,
   type Schema,
   Type,
 } from "@google/genai";
-import {
-  type FunctionCall,
-  type Content as GeminiMessageContent,
-  HarmBlockThreshold,
-  HarmCategory,
-  type SafetySetting,
-  SchemaType,
-} from "@google/generative-ai";
-import { FileState, GoogleAIFileManager } from "@google/generative-ai/server";
-import type {
-  BaseTool,
-  ChatMessage,
-  MessageContentFileDetail,
-  MessageContentImageDetail,
-  MessageContentTextDetail,
-  MessageType,
-  ToolCallLLMMessageOptions,
-} from "@llamaindex/core/llms";
+import type { BaseTool, MessageContentDetail } from "@llamaindex/core/llms";
 import { ModalityType } from "@llamaindex/core/schema";
-import { extractDataUrlComponents } from "@llamaindex/core/utils";
-import { getEnv } from "@llamaindex/env";
-import type {
-  ChatContext,
-  FileDataPart,
-  FunctionDeclaration,
-  FunctionDeclarationSchema,
-  GeminiChatParamsNonStreaming,
-  GeminiChatParamsStreaming,
-  GeminiMessageRole,
-  InlineDataPart,
-  Part,
-} from "./types.js";
-
-const FILE_EXT_MIME_TYPES: { [key: string]: string } = {
-  png: "image/png",
-  jpeg: "image/jpeg",
-  jpg: "image/jpeg",
-  webp: "image/webp",
-  heic: "image/heic",
-  heif: "image/heif",
-};
-const ACCEPTED_IMAGE_MIME_TYPES = Object.values(FILE_EXT_MIME_TYPES);
-
-const getFileURLExtension = (url: string): string | null => {
-  const pathname = new URL(url).pathname;
-  const parts = pathname.split(".");
-  return parts.length > 1 ? parts.pop()?.toLowerCase() || null : null;
-};
-
-const getFileURLMimeType = (url: string): string | null => {
-  const ext = getFileURLExtension(url);
-  return ext ? FILE_EXT_MIME_TYPES[ext] || null : null;
-};
-
-const getImageParts = (
-  message: MessageContentImageDetail,
-): InlineDataPart | FileDataPart => {
-  if (message.image_url.url.startsWith("data:")) {
-    const { mimeType, base64: data } = extractDataUrlComponents(
-      message.image_url.url,
-    );
-    if (!mimeType || !ACCEPTED_IMAGE_MIME_TYPES.includes(mimeType)) {
-      throw new Error(
-        `Gemini only accepts the following mimeTypes: ${ACCEPTED_IMAGE_MIME_TYPES.join(
-          "\n",
-        )}`,
-      );
-    }
-    return {
-      inlineData: {
-        mimeType,
-        data,
-      },
-    };
-  }
-  const mimeType = getFileURLMimeType(message.image_url.url);
-  if (!mimeType || !ACCEPTED_IMAGE_MIME_TYPES.includes(mimeType)) {
-    throw new Error(
-      `Gemini only accepts the following mimeTypes: ${ACCEPTED_IMAGE_MIME_TYPES.join(
-        "\n",
-      )}`,
-    );
-  }
-  return {
-    fileData: { mimeType, fileUri: message.image_url.url },
-  };
-};
-
-export const getPartsText = (parts: Part[]): string => {
-  const textStrings = [];
-  if (parts.length) {
-    for (const part of parts) {
-      if (part.text) {
-        textStrings.push(part.text);
-      }
-    }
-  }
-  if (textStrings.length > 0) {
-    return textStrings.join("");
-  } else {
-    return "";
-  }
-};
-
-/**
- * Returns all text found in all parts of first candidate.
- */
-export const getText = (response: GenerateContentResponse): string => {
-  if (response.candidates?.[0]!.content?.parts) {
-    return getPartsText(response.candidates?.[0].content?.parts);
-  }
-  return "";
-};
-
-export const cleanParts = (
-  message: GeminiMessageContent,
-): GeminiMessageContent => {
-  return {
-    ...message,
-    parts: message.parts.filter(
-      (part) =>
-        part.text?.trim() ||
-        part.inlineData ||
-        part.fileData ||
-        part.functionCall ||
-        part.functionResponse,
-    ),
-  };
-};
-
-export const getChatContext = async (
-  params: GeminiChatParamsStreaming | GeminiChatParamsNonStreaming,
-): Promise<ChatContext> => {
-  // Gemini doesn't allow:
-  // 1. Consecutive messages from the same role
-  // 2. Parts that have empty text
-  const fnMap = params.messages.reduce(
-    (result, message) => {
-      if (message.options && "toolCall" in message.options) {
-        message.options.toolCall.forEach((call) => {
-          result[call.id] = call.name;
-        });
-      }
-
-      return result;
-    },
-    {} as Record<string, string>,
-  );
-  const messages = GeminiHelper.mergeNeighboringSameRoleMessages(
-    await Promise.all(
-      params.messages.map((message) =>
-        GeminiHelper.chatMessageToGemini(message, fnMap),
-      ),
-    ),
-  ).map(cleanParts);
-
-  const history = messages.slice(0, -1);
-  const message = messages[messages.length - 1]!.parts;
-  return {
-    history,
-    message,
-  };
-};
+import { base64ToBlob } from "@llamaindex/core/utils";
 
 export const mapBaseToolToGeminiFunctionDeclaration = (
   tool: BaseTool,
 ): FunctionDeclaration => {
-  const parameters: FunctionDeclarationSchema = {
-    type: tool.metadata.parameters?.type.toLowerCase() as SchemaType,
-    properties: tool.metadata.parameters?.properties,
-    description: tool.metadata.parameters?.description,
-    required: tool.metadata.parameters?.required,
-  };
-
+  const { name, description, parameters } = tool.metadata;
   return {
-    name: tool.metadata.name,
-    description: tool.metadata.description,
-    parameters,
+    name,
+    description,
+    parameters: {
+      type: parameters?.type.toLowerCase() as Type,
+      properties: parameters?.properties,
+      description: parameters?.description,
+      required: parameters?.required,
+    },
   };
 };
 
@@ -193,9 +35,9 @@ export const mapBaseToolToGeminiFunctionDeclaration = (
  * @param tool - The BaseTool to convert
  * @returns A LiveFunctionDeclaration object that can be used with Gemini's live API
  */
-export const mapBaseToolToGeminiLiveFunctionDeclaration = (
+export function mapBaseToolToGeminiLiveFunctionDeclaration(
   tool: BaseTool,
-): LiveFunctionDeclaration => {
+): FunctionDeclaration {
   const parameters: Schema = {
     type: tool.metadata.parameters?.type.toLowerCase() as Type,
     properties: tool.metadata.parameters?.properties,
@@ -207,233 +49,111 @@ export const mapBaseToolToGeminiLiveFunctionDeclaration = (
     description: tool.metadata.description,
     parameters,
   };
-};
+}
 
-export const mapResponseModalityToGeminiLiveResponseModality = (
+export function mapResponseModalityToGeminiLiveResponseModality(
   responseModality: ModalityType,
-): Modality => {
+): Modality {
   return responseModality === ModalityType.TEXT
     ? Modality.TEXT
     : responseModality === ModalityType.AUDIO
       ? Modality.AUDIO
       : Modality.IMAGE;
-};
+}
 
-/**
- * Helper class providing utility functions for Gemini
- */
-export class GeminiHelper {
-  // Gemini only has user and model roles. Put the rest in user role.
-  public static readonly ROLES_TO_GEMINI: Record<
-    Exclude<MessageType, "developer">,
-    GeminiMessageRole
-  > = {
-    user: "user",
-    system: "user",
-    assistant: "model",
-    memory: "user",
-  };
-
-  public static readonly ROLES_FROM_GEMINI: Record<
-    GeminiMessageRole,
-    MessageType
-  > = {
-    user: "user",
-    model: "assistant",
-    function: "user",
-  };
-
-  public static mergeNeighboringSameRoleMessages(
-    messages: GeminiMessageContent[],
-  ): GeminiMessageContent[] {
-    return messages
-      .map(cleanParts)
-      .filter((message) => message.parts.length)
-      .reduce(
-        (
-          result: GeminiMessageContent[],
-          current: GeminiMessageContent,
-          index: number,
-          original: GeminiMessageContent[],
-        ) => {
-          if (index > 0 && original[index - 1]!.role === current.role) {
-            result[result.length - 1]!.parts = [
-              ...result[result.length - 1]!.parts,
-              ...current.parts,
-            ];
-          } else {
-            result.push(current);
-          }
-          return result;
-        },
-        [],
-      );
-  }
-
-  public static async messageContentToGeminiParts({
-    content,
-    options = undefined,
-    fnMap = undefined,
-  }: Pick<ChatMessage<ToolCallLLMMessageOptions>, "content" | "options"> & {
-    fnMap?: Record<string, string>;
-  }): Promise<Part[]> {
-    if (options && "toolResult" in options) {
-      if (!fnMap) throw Error("fnMap must be set");
-      const name = fnMap[options.toolResult.id];
-      if (!name) {
-        throw Error(
-          `Could not find the name for fn call with id ${options.toolResult.id}`,
-        );
-      }
-
-      return [
-        {
-          functionResponse: {
-            name,
-            response: {
-              result: options.toolResult.result,
-            },
-          },
-        },
-      ];
-    }
-    if (options && "toolCall" in options) {
-      return options.toolCall.map((call) => ({
-        functionCall: {
-          name: call.name,
-          args: call.input,
-        } as FunctionCall,
-      }));
-    }
-    if (typeof content === "string") {
-      return [{ text: content }];
-    }
-
-    const parts: Part[] = [];
-    const imageContents = content.filter(
-      (i) => i.type === "image_url",
-    ) as MessageContentImageDetail[];
-
-    parts.push(...imageContents.map(getImageParts));
-
-    const textContents = content.filter(
-      (i) => i.type === "text",
-    ) as MessageContentTextDetail[];
-    parts.push(...textContents.map((t) => ({ text: t.text })));
-
-    const fileContents = content.filter(
-      (i) => i.type === "file",
-    ) as MessageContentFileDetail[];
-
-    if (fileContents.length > 0) {
-      for (const file of fileContents) {
-        const uploadResponse = await GeminiHelper.uploadFile(
-          Buffer.from(file.data),
-          file.mimeType,
-        );
-        parts.push({
-          fileData: {
-            mimeType: uploadResponse.file.mimeType,
-            fileUri: uploadResponse.file.uri,
-          },
-        });
-      }
-    }
-
-    return parts;
-  }
-
-  // Upload a file for AI processing
-  public static async uploadFile(
-    data: string | Buffer, // file name or buffer
-    mimeType: string, // eg. application/pdf
-    interval = 5_000, // time to refetch upload status
-  ) {
-    const fileManager = new GoogleAIFileManager(getEnv("GOOGLE_API_KEY")!);
-
-    const uploadResponse = await fileManager.uploadFile(data, { mimeType });
-
-    let file = await fileManager.getFile(uploadResponse.file.name);
-
-    while (file.state === FileState.PROCESSING) {
-      await new Promise((resolve) => setTimeout(resolve, interval));
-      file = await fileManager.getFile(uploadResponse.file.name);
-    }
-
-    if (file.state === FileState.FAILED) {
-      throw new Error("Failed to upload file");
-    }
-
-    return uploadResponse;
-  }
-
-  public static getGeminiMessageRole(
-    message: ChatMessage<ToolCallLLMMessageOptions>,
-  ): GeminiMessageRole {
-    if (message.options && "toolResult" in message.options) {
-      return "function";
-    }
-    return GeminiHelper.ROLES_TO_GEMINI[
-      message.role as Exclude<MessageType, "developer">
-    ];
-  }
-
-  public static async chatMessageToGemini(
-    message: ChatMessage<ToolCallLLMMessageOptions>,
-    fnMap: Record<string, string>, // mapping of fn call id to fn call name
-  ): Promise<GeminiMessageContent> {
-    return {
-      role: GeminiHelper.getGeminiMessageRole(message),
-      parts: await GeminiHelper.messageContentToGeminiParts({
-        ...message,
-        fnMap,
-      }),
-    };
-  }
+// Gemini doesn't allow consecutive messages from the same role, so we need to merge them
+export function mergeNeighboringSameRoleMessages(
+  messages: GeminiMessage[],
+): GeminiMessage[] {
+  return messages
+    .map(cleanParts)
+    .filter((message) => message.parts?.length)
+    .reduce(
+      (
+        result: GeminiMessage[],
+        current: GeminiMessage,
+        index: number,
+        original: GeminiMessage[],
+      ) => {
+        if (index > 0 && original[index - 1]!.role === current.role) {
+          result[result.length - 1]!.parts = [
+            ...(result[result.length - 1]?.parts || []),
+            ...(current.parts || []),
+          ];
+        } else {
+          result.push(current);
+        }
+        return result;
+      },
+      [],
+    );
 }
 
 /**
- * Returns functionCall of first candidate.
- * Taken from https://github.com/google-gemini/generative-ai-js/ to be used with
- * vertexai as that library doesn't include it
+ * Converts a MessageContentDetail object into a Google Gemini Part object.
+ *
+ * This function handles different content types appropriately for the Gemini API:
+ * - Text content: Directly converts to Gemini text part
+ * - Image URLs: Extracts MIME type and creates part from URI
+ * - File/media content: Uploads to Google servers first, then creates part from uploaded URI
+ *
+ * @param content - The content to be converted (text, image URL, or base64 file data)
+ * @param client - Google GenAI client
+ *
+ * @returns Promise that resolves to a Gemini-compatible Part object
+ *
+ * @throws {Error} When MIME type cannot be extracted from image URL
+ * @throws {Error} When file upload fails
+ * @throws {Error} When upload succeeds but URI or MIME type is missing from result
  */
-export function getFunctionCalls(
-  response: GenerateContentResponse,
-): FunctionCall[] | undefined {
-  const functionCalls: FunctionCall[] = [];
-  if (response.candidates?.[0]!.content?.parts) {
-    for (const part of response.candidates[0].content.parts) {
-      if (part.functionCall) {
-        functionCalls.push(part.functionCall);
-      }
-    }
+export async function messageContentDetailToGeminiPart(
+  content: MessageContentDetail,
+  client: GoogleGenAI,
+): Promise<Part> {
+  // for text, just return the gemini text part
+  if (content.type === "text") {
+    return { text: content.text };
   }
-  if (functionCalls.length > 0) {
-    return functionCalls;
-  } else {
-    return undefined;
+
+  // for image has url already, extract mime type and create part from uri
+  if (content.type === "image_url") {
+    throw new Error(
+      "URL-based images are not supported in Gemini, please use type='image' for base64-encoded images instead",
+    );
   }
+
+  // for the rest content types: image(base64), audio, video, file
+  // upload it first and then create part from uri
+  const result = await client.files.upload({
+    file: base64ToBlob(content.data, content.mimeType), // convert base64 to blob for upload
+    config: { mimeType: content.mimeType },
+  });
+
+  if (result.error) {
+    throw new Error(`Failed to upload file`);
+  }
+
+  if (!result.uri || !result.mimeType) {
+    throw new Error(
+      `File is uploaded successfully, but missing uri or mimeType. URI: ${result.uri}, MIME Type: ${result.mimeType}`,
+    );
+  }
+
+  return createPartFromUri(result.uri, result.mimeType);
 }
 
-/**
- * Safety settings to disable external filters
- * Documentation: https://ai.google.dev/gemini-api/docs/safety-settings
- */
-export const DEFAULT_SAFETY_SETTINGS: SafetySetting[] = [
-  {
-    category: HarmCategory.HARM_CATEGORY_DANGEROUS_CONTENT,
-    threshold: HarmBlockThreshold.BLOCK_NONE,
-  },
-  {
-    category: HarmCategory.HARM_CATEGORY_HARASSMENT,
-    threshold: HarmBlockThreshold.BLOCK_NONE,
-  },
-  {
-    category: HarmCategory.HARM_CATEGORY_HATE_SPEECH,
-    threshold: HarmBlockThreshold.BLOCK_NONE,
-  },
-  {
-    category: HarmCategory.HARM_CATEGORY_SEXUALLY_EXPLICIT,
-    threshold: HarmBlockThreshold.BLOCK_NONE,
-  },
-];
+// Gemini doesn't allow parts that have empty text, so we need to clean them
+function cleanParts(message: GeminiMessage): GeminiMessage {
+  return {
+    ...message,
+    parts:
+      message.parts?.filter(
+        (part) =>
+          part.text?.trim() ||
+          part.inlineData ||
+          part.fileData ||
+          part.functionCall ||
+          part.functionResponse,
+      ) || [],
+  };
+}

packages/providers/google/src/vertex.ts

@@ -1,117 +0,0 @@
-import {
-  type GenerateContentResponse,
-  type SafetySetting,
-  VertexAI,
-  GenerativeModel as VertexGenerativeModel,
-  GenerativeModelPreview as VertexGenerativeModelPreview,
-  type ModelParams as VertexModelParams,
-  type StreamGenerateContentResult as VertexStreamGenerateContentResult,
-} from "@google-cloud/vertexai";
-
-import type {
-  GeminiChatStreamResponse,
-  IGeminiSession,
-  VertexGeminiSessionOptions,
-} from "./types.js";
-
-import type { FunctionCall } from "@google/generative-ai";
-import type {
-  CompletionResponse,
-  ToolCall,
-  ToolCallLLMMessageOptions,
-} from "@llamaindex/core/llms";
-import { streamConverter } from "@llamaindex/core/utils";
-import { getEnv, randomUUID } from "@llamaindex/env";
-import { DEFAULT_SAFETY_SETTINGS, getFunctionCalls, getText } from "./utils.js";
-
-/* To use Google's Vertex AI backend, it doesn't use api key authentication.
- *
- * To authenticate for local development:
- *
- *     ```
- *     npm install @google-cloud/vertexai
- *     gcloud auth application-default login
- *     ```
- * For production the prefered method is via a service account, more
- * details: https://cloud.google.com/docs/authentication/
- *
- * */
-
-export class GeminiVertexSession implements IGeminiSession {
-  private vertex: VertexAI;
-  private preview: boolean = false;
-
-  constructor(options?: Partial<VertexGeminiSessionOptions>) {
-    const project = options?.project ?? getEnv("GOOGLE_VERTEX_PROJECT");
-    const location = options?.location ?? getEnv("GOOGLE_VERTEX_LOCATION");
-    if (!project || !location) {
-      throw new Error(
-        "Set Google Vertex project and location in GOOGLE_VERTEX_PROJECT and GOOGLE_VERTEX_LOCATION env variables",
-      );
-    }
-    this.vertex = new VertexAI({
-      ...options,
-      project,
-      location,
-    });
-    this.preview = options?.preview ?? false;
-  }
-
-  getGenerativeModel(
-    metadata: VertexModelParams,
-  ): VertexGenerativeModelPreview | VertexGenerativeModel {
-    const safetySettings = metadata.safetySettings ?? DEFAULT_SAFETY_SETTINGS;
-    if (this.preview) {
-      return this.vertex.preview.getGenerativeModel({
-        safetySettings: safetySettings as SafetySetting[],
-        ...metadata,
-      });
-    }
-    return this.vertex.getGenerativeModel({
-      safetySettings: safetySettings as SafetySetting[],
-      ...metadata,
-    });
-  }
-
-  getResponseText(response: GenerateContentResponse): string {
-    return getText(response);
-  }
-
-  getToolsFromResponse(
-    response: GenerateContentResponse,
-  ): ToolCall[] | undefined {
-    return getFunctionCalls(response)?.map(
-      (call: FunctionCall) =>
-        ({
-          name: call.name,
-          input: call.args,
-          id: randomUUID(),
-        }) as ToolCall,
-    );
-  }
-
-  async *getChatStream(
-    result: VertexStreamGenerateContentResult,
-  ): GeminiChatStreamResponse {
-    yield* streamConverter(result.stream, (response) => {
-      const tools = this.getToolsFromResponse(response);
-      const options: ToolCallLLMMessageOptions = tools?.length
-        ? { toolCall: tools }
-        : {};
-      return {
-        delta: this.getResponseText(response),
-        raw: response,
-        options,
-      };
-    });
-  }
-
-  getCompletionStream(
-    result: VertexStreamGenerateContentResult,
-  ): AsyncIterable<CompletionResponse> {
-    return streamConverter(result.stream, (response) => ({
-      text: this.getResponseText(response),
-      raw: response,
-    }));
-  }
-}

packages/providers/groq/CHANGELOG.md

@@ -1,5 +1,11 @@
 # @llamaindex/groq
 
+## 0.0.77
+
+### Patch Changes
+
+- @llamaindex/openai@0.4.6
+
 ## 0.0.76
 
 ### Patch Changes