Release 0.11.11 (#2044 )

Co-authored-by: marcusschiesser <17126+marcusschiesser@users.noreply.github.com>
chore: migrate to @google/genai SDK (#2038 )
2026-07-04 03:40:26 -04:00 · 2025-06-27 14:51:09 +07:00 · 2025-06-27 12:09:26 +07:00 · 2025-06-26 14:29:33 +07:00 · 2025-06-26 14:18:56 +07:00 · 2025-06-26 14:03:03 +07:00
227 changed files with 5927 additions and 3439 deletions
@@ -1,5 +0,0 @@
---
-"@llamaindex/core": patch
---
-
-Add more Acronyms on SentenceSplitter
@@ -1,5 +1,50 @@
 # @llamaindex/doc

+## 0.2.31
+
+### Patch Changes
+
+- Updated dependencies [7039e1a]
+- Updated dependencies [7039e1a]
+  - llamaindex@0.11.11
+  - @llamaindex/core@0.6.12
+  - @llamaindex/cloud@4.0.16
+  - @llamaindex/node-parser@2.0.12
+  - @llamaindex/openai@0.4.6
+  - @llamaindex/readers@3.1.11
+  - @llamaindex/workflow@1.1.12
+
+## 0.2.30
+
+### Patch Changes
+
+- Updated dependencies [f7ec293]
+  - @llamaindex/workflow@1.1.11
+  - llamaindex@0.11.10
+
+## 0.2.29
+
+### Patch Changes
+
+- Updated dependencies [c5846bd]
+  - @llamaindex/readers@3.1.10
+
+## 0.2.28
+
+### Patch Changes
+
+- Updated dependencies [a89e187]
+- Updated dependencies [62699b7]
+- Updated dependencies [c5b2691]
+- Updated dependencies [d8ac8d3]
+  - @llamaindex/core@0.6.11
+  - @llamaindex/openai@0.4.5
+  - @llamaindex/cloud@4.0.15
+  - llamaindex@0.11.9
+  - @llamaindex/node-parser@2.0.11
+  - @llamaindex/readers@3.1.9
+  - @llamaindex/workflow@1.1.10
+
 ## 0.2.27

 ### Patch Changes
@@ -111,7 +111,7 @@ Key build process:
 **Content Sources:**

 - Local MDX files in `src/content/docs/`
- External docs from `@llama-flow/docs` package
+- External docs from `@llamaindex/workflow-docs` package
 - Generated API docs from TypeScript source

 ### Development Notes
@@ -23,8 +23,8 @@ const config = {
        permanent: true,
      },
      {
-        source: "/docs/llamaflow/:path*.mdx",
-        destination: "/docs/llamaflow/:path*",
+        source: "/docs/workflows/:path*.mdx",
+        destination: "/docs/workflows/:path*",
        permanent: true,
      },
    ];
@@ -1,6 +1,6 @@
 {
  "name": "@llamaindex/doc",
-  "version": "0.2.27",
+  "version": "0.2.31",
  "private": true,
  "scripts": {
    "postinstall": "fumadocs-mdx",
@@ -15,7 +15,7 @@
  "dependencies": {
    "@huggingface/transformers": "^3.5.0",
    "@icons-pack/react-simple-icons": "^10.1.0",
-    "@llama-flow/docs": "0.0.8",
+    "@llamaindex/workflow-docs": "0.1.1",
    "@llamaindex/chat-ui-docs": "^0.0.5",
    "@llamaindex/cloud": "workspace:*",
    "@llamaindex/core": "workspace:*",
@@ -69,7 +69,7 @@
    "twoslash": "^0.3.1",
    "use-stick-to-bottom": "^1.0.42",
    "web-tree-sitter": "^0.24.4",
-    "zod": "^3.23.8"
+    "zod": "^3.25.67"
  },
  "devDependencies": {
    "@next/env": "^15.3.0",
@@ -13,7 +13,7 @@ const INTERNAL_LINK_REGEX = /(?:(?:\]\(|\bhref=["'])\/docs\/([^")]+))/g;
 // This captures relative links like [text](./path) or ![alt](../images/image.png)
 const RELATIVE_LINK_REGEX = /(?:\]\()(?:\s*)(?:\.\.?)\//g;

-const ALLOWED_LINKS = ["/docs/llamaflow", "/docs/chat-ui"];
+const ALLOWED_LINKS = ["/docs/workflows", "/docs/chat-ui"];

 interface LinkValidationResult {
  file: string;
@@ -11,9 +11,9 @@ import remarkMath from "remark-math";
 export const docs = defineDocs({
  dir: [
    "./src/content/docs",
-    "./node_modules/@llama-flow/docs",
+    "./node_modules/@llamaindex/workflow-docs",
    "./node_modules/@llamaindex/chat-ui-docs",
-    // NOTE: When adding external docs (like chat-ui or llama-flow above),
+    // NOTE: When adding external docs (like chat-ui or workflow-docs above),
    // make sure to also update:
    // 1. scripts/validate-links.mts - add to ALLOWED_LINKS array
    // 2. next.config.mjs - add redirect for .mdx files
@@ -74,12 +74,21 @@ const server = mcp({
  args: ["-y", "@modelcontextprotocol/server-filesystem", "."],
  verbose: true,
 });
-// or by SSE
+// or by StreamableHTTP transport
 const server = mcp({
  url: "http://localhost:8000/mcp",
  verbose: true,
 });

+// if your MCP server is not using StreamableHTTP transport, you can also use SSE transport
+// by setting useSSETransport to true.
+// See: https://modelcontextprotocol.io/docs/concepts/transports#server-sent-events-sse-deprecated
+const server = mcp({
+  url: "http://localhost:8000/mcp",
+  useSSETransport: true,
+  verbose: true,
+});
+
 // 3. Get tools from MCP server
 const tools = await server.tools();

@@ -9,10 +9,13 @@ Workflows are designed to be flexible and can be used to build agents, RAG flows
 To use workflows install this package:

 ```package-install
-npm i @llamaindex/workflow
+npm i @llamaindex/workflow-core
 ```

-This package is a stable, production-ready version of our [llama-flow](/docs/llamaflow) project. 
+This contains the core functionality for the workflow system. You can read more about the core concepts in the [workflow-core](/docs/workflows) section.

-While you can still reference the llama-flow documentation for detailed information about the underlying concepts, we recommend using the `@llamaindex/workflow` package for all new projects to ensure stability and long-term availability.
+In contrast, the `@llamaindex/workflow` package contains more utiltities, such as prebuilt agents.

+```package-install
+npm i @llamaindex/workflow
+```
@@ -28,11 +28,12 @@ embedding vector(1536)
 );
 ```

-- Create a function for similarity search
+-- Create a function for similarity search with filtering support
 ```sql
 create function match_documents (
 query_embedding vector(1536),
-match_count int
+match_count int,
+filter jsonb DEFAULT '{}'
 ) returns table (
 id uuid,
 content text,
@@ -52,6 +53,7 @@ metadata,
 embedding,
 1 - (embedding <=> query_embedding) as similarity
 from documents
+where metadata @> filter
 order by embedding <=> query_embedding
 limit match_count;
 end;
@@ -96,6 +98,7 @@ const index = await VectorStoreIndex.fromDocuments(documents, {
 ```ts
 const queryEngine = index.asQueryEngine();

+// Basic query without filters
 const response = await queryEngine.query({
  query: "What is in the document?",
 });
@@ -104,6 +107,32 @@ const response = await queryEngine.query({
 console.log(response.toString());
 ```

+## Query with filters
+
+You can filter documents based on metadata when querying:
+
+```ts
+import { FilterOperator, MetadataFilters } from "llamaindex";
+
+// Create a filter for documents with author = "Jane Smith"
+const filters: MetadataFilters = {
+  filters: [
+    {
+      key: "author",
+      value: "Jane Smith",
+      operator: FilterOperator.EQ,
+    },
+  ],
+};
+
+// Query with filters
+const filteredResponse = await vectorStore.query({
+  queryEmbedding: embedModel.getQueryEmbedding("What is vector search?"),
+  similarityTopK: 5,
+  filters,
+});
+```
+
 ## Full code

 ```ts
@@ -11,58 +11,130 @@ npm i llamaindex @llamaindex/google
 ## Usage

 ```ts
-import { Gemini, GEMINI_MODEL } from "@llamaindex/google";
+import { gemini, GEMINI_MODEL } from "@llamaindex/google";
 import { Settings } from "llamaindex";

-Settings.llm = new Gemini({
-  model: GEMINI_MODEL.GEMINI_PRO,
-});
-```
-
-## Usage with Proxy
-
-```ts
-import { Gemini, GEMINI_MODEL } from "@llamaindex/google";
-import { Settings } from "llamaindex";
-
-Settings.llm = new Gemini({
-  model: GEMINI_MODEL.GEMINI_PRO,
-  requestOptions: {
-    baseUrl: <YOUR_PROXY_URL>   // optional, but useful for custom endpoints
-  }
+Settings.llm = gemini({
+  model: GEMINI_MODEL.GEMINI_2_0_FLASH,
 });
 ```

 ### Usage with Vertex AI

-To use Gemini via Vertex AI you can use `GeminiVertexSession`.
-
-GeminiVertexSession accepts the env variables: `GOOGLE_VERTEX_LOCATION` and `GOOGLE_VERTEX_PROJECT`
+To use Gemini via Vertex AI, you can specify the vertex configuration:

 ```ts
-import { Gemini, GEMINI_MODEL, GeminiVertexSession } from "@llamaindex/google";
+import { gemini, GEMINI_MODEL } from "@llamaindex/google";

-const gemini = new Gemini({
-  model: GEMINI_MODEL.GEMINI_PRO,
-  session: new GeminiVertexSession({
-    location: "us-central1",      // optional if provided by GOOGLE_VERTEX_LOCATION env variable
-    project: "project1",          // optional if provided by GOOGLE_VERTEX_PROJECT env variable
-    googleAuthOptions: {...},     // optional, but useful for production. It accepts all values from `GoogleAuthOptions`
-  }),
+const llm = gemini({
+  model: GEMINI_MODEL.GEMINI_2_0_FLASH,
+  vertex: {
+    project: "your-cloud-project",    // required for Vertex AI
+    location: "us-central1",          // required for Vertex AI
+  },
 });
 ```

-[GoogleAuthOptions](https://github.com/googleapis/google-auth-library-nodejs/blob/main/src/auth/googleauth.ts)
-
 To authenticate for local development:

 ```bash
-npm i @google-cloud/vertexai
 gcloud auth application-default login
 ```

 To authenticate for production you'll have to use a [service account](https://cloud.google.com/docs/authentication/). `googleAuthOptions` has `credentials` which might be useful for you.

+## Multimodal Usage
+
+Gemini supports multimodal inputs including text, images, audio, and video:
+
+```ts
+import { gemini, GEMINI_MODEL } from "@llamaindex/google";
+import fs from "fs";
+
+const llm = gemini({ model: GEMINI_MODEL.GEMINI_2_0_FLASH });
+
+const result = await llm.chat({
+  messages: [
+    {
+      role: "user",
+      content: [
+        {
+          type: "text",
+          text: "What's in this image?",
+        },
+        {
+          type: "image",
+          data: fs.readFileSync("./image.jpg").toString("base64"),
+          mimeType: "image/jpeg",
+        },
+      ],
+    },
+  ],
+});
+```
+
+## Tool Calling
+
+Gemini supports function calling with tools:
+
+```ts
+import { gemini, GEMINI_MODEL } from "@llamaindex/google";
+import { tool } from "llamaindex";
+import { z } from "zod";
+
+const llm = gemini({ model: GEMINI_MODEL.GEMINI_2_0_FLASH });
+
+const result = await llm.chat({
+  messages: [
+    {
+      content: "What's the weather in Tokyo?",
+      role: "user",
+    },
+  ],
+  tools: [
+    tool({
+      name: "weather",
+      description: "Get the weather",
+      parameters: z.object({
+        location: z.string().describe("The location to get the weather for"),
+      }),
+      execute: ({ location }) => {
+        return `The weather in ${location} is sunny and hot`;
+      },
+    }),
+  ],
+});
+```
+
+## Live API (Real-time Conversations)
+
+For real-time audio/video conversations using [Gemini Live API](https://ai.google.dev/gemini-api/docs/live). 
+
+The Live API is running directly in the frontend. That's why you have to generate an ephemeral key first on the server side and pass it to the frontend.
+
+To use the Live API, make sure to pass `apiVersion: "v1alpha"` to the `httpOptions`.
+
+```ts
+import { gemini, GEMINI_MODEL } from "@llamaindex/google";
+
+// Server-side: Generate ephemeral key
+const serverLlm = gemini({
+  model: GEMINI_MODEL.GEMINI_2_0_FLASH_LIVE,
+  httpOptions: { apiVersion: "v1alpha" },
+});
+const ephemeralKey = await serverLlm.live.getEphemeralKey();
+
+// Client-side: Use ephemeral key for Live API
+const llm = gemini({
+  apiKey: ephemeralKey,
+  model: GEMINI_MODEL.GEMINI_2_0_FLASH_LIVE,
+  voiceName: "Zephyr",
+  httpOptions: { apiVersion: "v1alpha" },
+});
+
+const session = await llm.live.connect();
+```
+
 ## Load and index documents

 For this example, we will use a single document. In a real-world scenario, you would have multiple documents to index.
@@ -90,11 +162,11 @@ const results = await queryEngine.query({
 ## Full Example

 ```ts
-import { Gemini, GEMINI_MODEL } from "@llamaindex/google";
+import { gemini, GEMINI_MODEL } from "@llamaindex/google";
 import { Document, VectorStoreIndex, Settings } from "llamaindex";

-Settings.llm = new Gemini({
-  model: GEMINI_MODEL.GEMINI_PRO,
+Settings.llm = gemini({
+  model: GEMINI_MODEL.GEMINI_2_0_FLASH,
 });

 async function main() {
@@ -104,9 +176,7 @@ async function main() {
  const index = await VectorStoreIndex.fromDocuments([document]);

  // Create a query engine
-  const queryEngine = index.asQueryEngine({
-    retriever,
-  });
+  const queryEngine = index.asQueryEngine();

  const query = "What is the meaning of life?";

@@ -378,3 +378,186 @@ async function main() {
 ## API Reference

 - [OpenAI](/docs/api/classes/OpenAI)
+
+
+# OpenAI Live LLM
+
+The OpenAI Live LLM integration in LlamaIndex provides real-time chat capabilities with support for audio streaming and tool calling.
+
+## Basic Usage
+
+```typescript
+import { openai } from "@llamaindex/openai";
+import { tool, ModalityType } from "llamaindex";
+
+// Get the ephimeral key on the server 
+const serverllm = openai({
+  apiKey: "your-api-key", 
+  model: "gpt-4o-realtime-preview-2025-06-03",
+});
+
+// Get an ephemeral key 
+// Usually this code is run on the server and the ephemeral key is passed to the
+// client - the ephemeral key can be securely used on the client side
+const ephemeralKey = await serverllm.live.getEphemeralKey();
+
+// Create a client-side LLM instance with the ephemeral key
+const llm = openai({
+  apiKey: ephemeralKey,
+  model: "gpt-4o-realtime-preview-2025-06-03"
+});
+
+// Create a live sessionimport { tool } from "llamaindex";
+const session = await llm.live.connect({
+  systemInstruction: "You are a helpful assistant.",
+});
+
+// Send a message
+session.sendMessage({
+  content: "Hello!",
+  role: "user",
+});
+```
+
+## Tool Integration
+
+Tools are handled server-side, making it simple to pass them to the live session:
+
+```typescript
+// Define your tools
+const weatherTool = tool({
+  name: "weather",
+  description: "Get the weather for a location",
+  parameters: z.object({
+    location: z.string().describe("The location to get weather for"),
+  }),
+  execute: async ({ location }) => {
+    return `The weather in ${location} is sunny`;
+  },
+});
+
+// Create session with tools
+const session = await llm.live.connect({
+  systemInstruction: "You are a helpful assistant.",
+  tools: [weatherTool],
+});
+```
+
+## Audio Support
+
+For audio capabilities:
+
+```typescript
+// Get microphone access
+const userStream = await navigator.mediaDevices.getUserMedia({
+  audio: true,
+});
+
+// Create session with audio
+const session = await llm.live.connect({
+  audioConfig: {
+    stream: userStream,
+    onTrack: (remoteStream) => {
+      // Handle incoming audio
+      audioElement.srcObject = remoteStream;
+    },
+  },
+});
+```
+
+## Event Handling
+
+Listen to events from the session:
+
+```typescript
+for await (const event of session.streamEvents()) {
+  if (liveEvents.open.include(event)) {
+    // Connection established
+    console.log("Connected!");
+  } else if (liveEvents.text.include(event)) {
+    // Received text response
+    console.log("Assistant:", event.text);
+  }
+}
+```
+
+## Capabilities
+
+The OpenAI Live LLM supports:
+
+- Real-time text chat
+- Audio streaming (if configured)
+- Tool calling (server-side execution)
+- Ephemeral key generation for secure sessions
+
+## API Reference
+
+### LiveLLM Methods
+// Get an ephemeral key 
+// Usually this code is run on the server and the ephemeral key is passed to the
+// client - the ephemeral key can be securely used on the client side
+
+#### `connect(config?: LiveConnectConfig)`
+
+Creates a new live session.
+
+```typescript
+interface LiveConnectConfig {
+  systemInstruction?: string;
+  tools?: BaseTool[];
+  audioConfig?: AudioConfig;
+  responseModality?: ModalityType[];
+}
+```
+
+#### `getEphemeralKey()`
+
+Gets a temporary key for the session.
+
+### LiveLLMSession Methods
+
+#### `sendMessage(message: ChatMessage)`
+
+Sends a message to the assistant.
+
+```typescript
+interface ChatMessage {
+  content: string | MessageContentDetail[];
+  role: "user" | "assistant";
+}
+```
+
+#### `disconnect()`
+
+Closes the session and cleans up resources.
+
+## Error Handling
+
+```typescript
+try {
+  const session = await llm.live.connect();
+} catch (error) {
+  if (error instanceof Error) {
+    console.error("Connection failed:", error.message);
+  }
+}
+```
+
+## Best Practices
+
+1. **Tool Definition**
+
+   - Keep tool implementations server-side
+   - Use clear descriptions for tools
+   - Handle tool errors gracefully
+
+2. **Session Management**
+
+   - Always disconnect sessions when done
+   - Clean up audio resources
+   - Handle reconnection scenarios
+
+3. **Security**
+   - Use ephemeral keys for sessions
+   - Validate tool inputs
+   - Secure API key handling
@@ -1,3 +1,3 @@
 {
-  "pages": ["llamaindex", "api", "llamaflow", "chat-ui"]
+  "pages": ["llamaindex", "api", "workflows", "chat-ui"]
 }
@@ -4,7 +4,7 @@
  "tasks": {
    "build": {
      "inputs": [
-        "node_modules/@llama-flow/docs/**",
+        "node_modules/@llamaindex/workflow-docs/**",
        "node_modules/@llamaindex/chat-ui-docs/**",
        "src/**/*.ts",
        "src/**/*.tsx",
@@ -1,5 +1,24 @@
 # @llamaindex/cloudflare-worker-agent-test

+## 0.0.172
+
+### Patch Changes
+
+- Updated dependencies [7039e1a]
+  - llamaindex@0.11.11
+
+## 0.0.171
+
+### Patch Changes
+
+- llamaindex@0.11.10
+
+## 0.0.170
+
+### Patch Changes
+
+- llamaindex@0.11.9
+
 ## 0.0.169

 ### Patch Changes
@@ -1,6 +1,6 @@
 {
  "name": "@llamaindex/cloudflare-worker-agent-test",
-  "version": "0.0.169",
+  "version": "0.0.172",
  "type": "module",
  "private": true,
  "scripts": {
@@ -1,5 +1,17 @@
 # @llamaindex/llama-parse-browser-test

+## 0.0.71
+
+### Patch Changes
+
+- @llamaindex/cloud@4.0.16
+
+## 0.0.70
+
+### Patch Changes
+
+- @llamaindex/cloud@4.0.15
+
 ## 0.0.69

 ### Patch Changes
@@ -1,7 +1,7 @@
 {
  "name": "@llamaindex/llama-parse-browser-test",
  "private": true,
-  "version": "0.0.69",
+  "version": "0.0.71",
  "type": "module",
  "scripts": {
    "dev": "vite",
@@ -1,5 +1,24 @@
 # @llamaindex/next-agent-test

+## 0.1.172
+
+### Patch Changes
+
+- Updated dependencies [7039e1a]
+  - llamaindex@0.11.11
+
+## 0.1.171
+
+### Patch Changes
+
+- llamaindex@0.11.10
+
+## 0.1.170
+
+### Patch Changes
+
+- llamaindex@0.11.9
+
 ## 0.1.169

 ### Patch Changes
@@ -1,6 +1,6 @@
 {
  "name": "@llamaindex/next-agent-test",
-  "version": "0.1.169",
+  "version": "0.1.172",
  "private": true,
  "scripts": {
    "dev": "next dev",
@@ -1,5 +1,24 @@
 # test-edge-runtime

+## 0.1.171
+
+### Patch Changes
+
+- Updated dependencies [7039e1a]
+  - llamaindex@0.11.11
+
+## 0.1.170
+
+### Patch Changes
+
+- llamaindex@0.11.10
+
+## 0.1.169
+
+### Patch Changes
+
+- llamaindex@0.11.9
+
 ## 0.1.168

 ### Patch Changes
@@ -1,6 +1,6 @@
 {
  "name": "@llamaindex/nextjs-edge-runtime-test",
-  "version": "0.1.168",
+  "version": "0.1.171",
  "private": true,
  "scripts": {
    "dev": "next dev",
@@ -1,5 +1,35 @@
 # @llamaindex/next-node-runtime

+## 0.1.40
+
+### Patch Changes
+
+- Updated dependencies [7039e1a]
+  - llamaindex@0.11.11
+  - @llamaindex/huggingface@0.1.16
+  - @llamaindex/readers@3.1.11
+
+## 0.1.39
+
+### Patch Changes
+
+- llamaindex@0.11.10
+
+## 0.1.38
+
+### Patch Changes
+
+- Updated dependencies [c5846bd]
+  - @llamaindex/readers@3.1.10
+
+## 0.1.37
+
+### Patch Changes
+
+- llamaindex@0.11.9
+- @llamaindex/huggingface@0.1.15
+- @llamaindex/readers@3.1.9
+
 ## 0.1.36

 ### Patch Changes
@@ -1,6 +1,6 @@
 {
  "name": "@llamaindex/next-node-runtime-test",
-  "version": "0.1.36",
+  "version": "0.1.40",
  "private": true,
  "scripts": {
    "dev": "next dev",
@@ -1,5 +1,24 @@
 # vite-import-llamaindex

+## 0.0.38
+
+### Patch Changes
+
+- Updated dependencies [7039e1a]
+  - llamaindex@0.11.11
+
+## 0.0.37
+
+### Patch Changes
+
+- llamaindex@0.11.10
+
+## 0.0.36
+
+### Patch Changes
+
+- llamaindex@0.11.9
+
 ## 0.0.35

 ### Patch Changes
@@ -1,7 +1,7 @@
 {
  "name": "vite-import-llamaindex",
  "private": true,
-  "version": "0.0.35",
+  "version": "0.0.38",
  "type": "module",
  "scripts": {
    "build": "vite build",
@@ -1 +1,9 @@
-{"root":["./src/main.ts","./vite.config.ts"],"version":"5.7.3"}
+{
+  "root": [
+    "./src/main.ts",
+    "./vite.config.ts",
+    "./tsconfig.json"
+  ],
+  "errors": true,
+  "version": "5.7.3"
+}
@@ -1,5 +1,24 @@
 # @llamaindex/waku-query-engine-test

+## 0.0.172
+
+### Patch Changes
+
+- Updated dependencies [7039e1a]
+  - llamaindex@0.11.11
+
+## 0.0.171
+
+### Patch Changes
+
+- llamaindex@0.11.10
+
+## 0.0.170
+
+### Patch Changes
+
+- llamaindex@0.11.9
+
 ## 0.0.169

 ### Patch Changes
@@ -1,6 +1,6 @@
 {
  "name": "@llamaindex/waku-query-engine-test",
-  "version": "0.0.169",
+  "version": "0.0.172",
  "type": "module",
  "private": true,
  "scripts": {
@@ -7,7 +7,7 @@
  "dependencies": {
    "@llamaindex/workflow": "1.1.1",
    "llamaindex": "0.10.5",
-    "zod": "^3.23.8"
+    "zod": "^3.25.67"
  },
  "devDependencies": {
    "tsx": "^4.19.1",
@@ -27,6 +27,6 @@
    "pg": "^8.12.0",
    "pgvector": "0.2.0",
    "tsx": "^4.19.3",
-    "zod": "^3.24.2"
+    "zod": "^3.25.67"
  }
 }
@@ -1,5 +1,120 @@
 # examples

+## 0.3.25
+
+### Patch Changes
+
+- Updated dependencies [7039e1a]
+- Updated dependencies [7039e1a]
+  - llamaindex@0.11.11
+  - @llamaindex/core@0.6.12
+  - @llamaindex/google@0.3.11
+  - @llamaindex/cloud@4.0.16
+  - @llamaindex/node-parser@2.0.12
+  - @llamaindex/anthropic@0.3.14
+  - @llamaindex/assemblyai@0.1.11
+  - @llamaindex/clip@0.0.62
+  - @llamaindex/cohere@0.0.26
+  - @llamaindex/deepinfra@0.0.62
+  - @llamaindex/discord@0.1.11
+  - @llamaindex/huggingface@0.1.16
+  - @llamaindex/jinaai@0.0.22
+  - @llamaindex/mistral@0.1.12
+  - @llamaindex/mixedbread@0.0.26
+  - @llamaindex/notion@0.1.11
+  - @llamaindex/ollama@0.1.12
+  - @llamaindex/openai@0.4.6
+  - @llamaindex/perplexity@0.0.19
+  - @llamaindex/portkey-ai@0.0.54
+  - @llamaindex/replicate@0.0.54
+  - @llamaindex/astra@0.0.26
+  - @llamaindex/azure@0.1.23
+  - @llamaindex/chroma@0.0.26
+  - @llamaindex/elastic-search@0.1.12
+  - @llamaindex/firestore@1.0.19
+  - @llamaindex/milvus@0.1.21
+  - @llamaindex/mongodb@0.0.27
+  - @llamaindex/pinecone@0.1.12
+  - @llamaindex/postgres@0.0.55
+  - @llamaindex/qdrant@0.1.22
+  - @llamaindex/supabase@0.1.12
+  - @llamaindex/upstash@0.0.26
+  - @llamaindex/weaviate@0.0.27
+  - @llamaindex/vercel@0.1.12
+  - @llamaindex/voyage-ai@1.0.18
+  - @llamaindex/readers@3.1.11
+  - @llamaindex/tools@0.1.1
+  - @llamaindex/workflow@1.1.12
+  - @llamaindex/deepseek@0.0.22
+  - @llamaindex/fireworks@0.0.22
+  - @llamaindex/groq@0.0.77
+  - @llamaindex/together@0.0.22
+  - @llamaindex/vllm@0.0.48
+  - @llamaindex/xai@0.0.9
+
+## 0.3.24
+
+### Patch Changes
+
+- Updated dependencies [096bf2b]
+- Updated dependencies [c5846bd]
+  - @llamaindex/tools@0.1.0
+  - @llamaindex/readers@3.1.10
+
+## 0.3.23
+
+### Patch Changes
+
+- Updated dependencies [a89e187]
+- Updated dependencies [62699b7]
+- Updated dependencies [c5b2691]
+- Updated dependencies [d8ac8d3]
+  - @llamaindex/core@0.6.11
+  - @llamaindex/google@0.3.10
+  - @llamaindex/openai@0.4.5
+  - @llamaindex/cloud@4.0.15
+  - llamaindex@0.11.9
+  - @llamaindex/node-parser@2.0.11
+  - @llamaindex/anthropic@0.3.13
+  - @llamaindex/assemblyai@0.1.10
+  - @llamaindex/clip@0.0.61
+  - @llamaindex/cohere@0.0.25
+  - @llamaindex/deepinfra@0.0.61
+  - @llamaindex/discord@0.1.10
+  - @llamaindex/huggingface@0.1.15
+  - @llamaindex/jinaai@0.0.21
+  - @llamaindex/mistral@0.1.11
+  - @llamaindex/mixedbread@0.0.25
+  - @llamaindex/notion@0.1.10
+  - @llamaindex/ollama@0.1.11
+  - @llamaindex/perplexity@0.0.18
+  - @llamaindex/portkey-ai@0.0.53
+  - @llamaindex/replicate@0.0.53
+  - @llamaindex/astra@0.0.25
+  - @llamaindex/azure@0.1.22
+  - @llamaindex/chroma@0.0.25
+  - @llamaindex/elastic-search@0.1.11
+  - @llamaindex/firestore@1.0.18
+  - @llamaindex/milvus@0.1.20
+  - @llamaindex/mongodb@0.0.26
+  - @llamaindex/pinecone@0.1.11
+  - @llamaindex/postgres@0.0.54
+  - @llamaindex/qdrant@0.1.21
+  - @llamaindex/supabase@0.1.10
+  - @llamaindex/upstash@0.0.25
+  - @llamaindex/weaviate@0.0.26
+  - @llamaindex/vercel@0.1.11
+  - @llamaindex/voyage-ai@1.0.17
+  - @llamaindex/readers@3.1.9
+  - @llamaindex/tools@0.0.17
+  - @llamaindex/workflow@1.1.10
+  - @llamaindex/deepseek@0.0.21
+  - @llamaindex/fireworks@0.0.21
+  - @llamaindex/groq@0.0.76
+  - @llamaindex/together@0.0.21
+  - @llamaindex/vllm@0.0.47
+  - @llamaindex/xai@0.0.8
+
 ## 0.3.22

 ### Patch Changes
@@ -6,15 +6,24 @@ async function main() {
  // Create an MCP server for filesystem tools
  const server = mcp({
    command: "npx",
-    args: ["-y", "@modelcontextprotocol/server-filesystem", "."],
+    args: ["-y", "@modelcontextprotocol/server-filesystem@latest", "."],
    verbose: true,
  });
-  // You can also connect to the MCP server using SSE
-  // See: https://modelcontextprotocol.io/docs/concepts/transports#server-sent-events-sse
+  //
+  // You can also connect to a remote MCP server using:
+  // 1. StreamableHTTP transport (recommended)
+  // See: https://modelcontextprotocol.io/docs/concepts/transports#streamable-http
  // const server = mcp({
  //   url: "http://localhost:8000/mcp",
  //   verbose: true,
  // });
+  // 2.Or using SSE transport (will be deprecated soon)
+  // See: https://modelcontextprotocol.io/docs/concepts/transports#server-sent-events-sse-deprecated
+  // const server = mcp({
+  //   url: "http://localhost:8000/mcp",
+  //   useSSETransport: true,
+  //   verbose: true,
+  // });

  try {
    // Create an agent that uses the MCP tools
@@ -0,0 +1,72 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<company name="MidSizeCorp" founded="2008">
+  <division name="Engineering" head="Dana White">
+    <department name="Frontend" lead="Alex Kim">
+      <team name="Web">
+        <employee id="E01">
+          <name>Jordan Lee</name>
+          <role>Lead Developer</role>
+          <projects>
+            <project code="PRJ101" status="active">
+              <title>User Portal</title>
+              <deadline>2025-08-01</deadline>
+              <tasks>
+                <task id="T1011">
+                  <description>Implement login page</description>
+                  <due>2025-05-10</due>
+                </task>
+                <task id="T1012">
+                  <description>Design dashboard</description>
+                  <due>2025-05-20</due>
+                </task>
+              </tasks>
+            </project>
+          </projects>
+        </employee>
+        <employee id="E02">
+          <name>Riley Chen</name>
+          <role>UI Designer</role>
+        </employee>
+      </team>
+      <team name="Mobile">
+        <employee id="E03">
+          <name>Sam Patel</name>
+          <role>iOS Developer</role>
+        </employee>
+      </team>
+    </department>
+    <department name="Backend" lead="Morgan Reed">
+      <team name="API">
+        <employee id="E04">
+          <name>Taylor Jones</name>
+          <role>API Engineer</role>
+        </employee>
+      </team>
+      <team name="Database">
+        <employee id="E05">
+          <name>Casey Nguyen</name>
+          <role>DB Administrator</role>
+        </employee>
+      </team>
+    </department>
+  </division>
+
+  <division name="Marketing" head="Pat Morgan">
+    <department name="Digital" lead="Alex Rivera">
+      <team name="Content">
+        <employee id="M01">
+          <name>Charlie Brooks</name>
+          <role>Content Strategist</role>
+        </employee>
+      </team>
+    </department>
+  </division>
+
+  <headquarters location="Chicago, USA">
+    <address>
+      <street>789 Lake Shore Drive</street>
+      <city>Chicago</city>
+      <zip>60601</zip>
+    </address>
+  </headquarters>
+</company>
@@ -1,14 +1,16 @@
-import { Gemini, GEMINI_MODEL } from "@llamaindex/google";
+import { gemini, GEMINI_MODEL } from "@llamaindex/google";
 import fs from "fs";
+import { tool } from "llamaindex";
+import { z } from "zod";

 (async () => {
  if (!process.env.GOOGLE_API_KEY) {
    throw new Error("Please set the GOOGLE_API_KEY environment variable.");
  }
-  const gemini = new Gemini({
-    model: GEMINI_MODEL.GEMINI_PRO_1_5,
-  });
-  const result = await gemini.chat({
+  const llm = gemini({ model: GEMINI_MODEL.GEMINI_2_0_FLASH });
+
+  // normal chat
+  const result = await llm.chat({
    messages: [
      { content: "You want to talk in rhymes.", role: "system" },
      {
@@ -18,10 +20,10 @@ import fs from "fs";
      },
    ],
  });
-  console.log(result);
+  console.log("\n normal chat: \n", result);

  // chat with file
-  const resultWithFile = await gemini.chat({
+  const resultWithFile = await llm.chat({
    messages: [
      {
        role: "user",
@@ -39,6 +41,52 @@ import fs from "fs";
      },
    ],
  });
+  console.log("\n chat with file: \n", resultWithFile);

-  console.log(resultWithFile);
+  // chat with image base64
+  const resultWithImageFile = await llm.chat({
+    messages: [
+      {
+        role: "user",
+        content: [
+          {
+            type: "text",
+            text: "What's in this image?",
+          },
+          {
+            type: "image",
+            data: fs
+              .readFileSync("./multimodal/data/60.jpg")
+              .toString("base64"),
+            mimeType: "image/png",
+          },
+        ],
+      },
+    ],
+  });
+  console.log("\n chat with image base64: \n", resultWithImageFile);
+
+  // chat with tool
+  const resultWithTool = await llm.chat({
+    messages: [
+      {
+        content: "What's the weather in Tokyo?",
+        role: "user",
+      },
+    ],
+    tools: [
+      tool({
+        name: "weather",
+        description: "Get the weather",
+        parameters: z.object({
+          location: z.string().describe("The location to get the weather for"),
+        }),
+        execute: ({ location }) => {
+          console.log("weather", location);
+          return `The weather in ${location} is sunny and hot`;
+        },
+      }),
+    ],
+  });
+  console.log("\n chat with tool: \n", resultWithTool.message.options); // should have toolCall
 })();
@@ -1,11 +1,14 @@
-import { Gemini, GEMINI_MODEL, GeminiVertexSession } from "@llamaindex/google";
+import { gemini, GEMINI_MODEL } from "@llamaindex/google";

 (async () => {
-  const gemini = new Gemini({
-    model: GEMINI_MODEL.GEMINI_PRO,
-    session: new GeminiVertexSession(),
+  const llm = gemini({
+    model: GEMINI_MODEL.GEMINI_2_0_FLASH,
+    vertex: {
+      project: "your-cloud-project", // update to your cloud project
+      location: "us-central1",
+    },
  });
-  const result = await gemini.chat({
+  const result = await llm.chat({
    messages: [
      { content: "You want to talk in rhymes.", role: "system" },
      {
@@ -16,9 +16,19 @@ async function main() {

  console.log("🚀 Initializing Gemini Live API example...");

+  // Server-side (token creation):
+  const serverllm = gemini({
+    model: GEMINI_MODEL.GEMINI_2_0_FLASH_LIVE,
+    httpOptions: { apiVersion: "v1alpha" }, // must use v1alpha to generate ephemeral key
+  });
+  const ephemeralKey = await serverllm.live.getEphemeralKey();
+
+  // Client-side (Live API connection):
  const llm = gemini({
+    apiKey: ephemeralKey, // use ephemeral key for client-side
    model: GEMINI_MODEL.GEMINI_2_0_FLASH_LIVE,
    voiceName: "Zephyr",
+    httpOptions: { apiVersion: "v1alpha" }, // must use v1alpha to init client with ephemeral key
  });

  console.log("📡 Connecting to Gemini Live session...");
@@ -3,8 +3,18 @@ import { liveEvents } from "llamaindex";
 import { saveWavFile } from "./util";

 async function main() {
-  const llm = gemini({
+  // Server-side (token creation):
+  const serverllm = gemini({
    model: GEMINI_MODEL.GEMINI_2_0_FLASH_LIVE,
+    httpOptions: { apiVersion: "v1alpha" }, // must use v1alpha to generate ephemeral key
+  });
+  const ephemeralKey = await serverllm.live.getEphemeralKey();
+
+  // Client-side (Live API connection):
+  const llm = gemini({
+    apiKey: ephemeralKey, // use ephemeral key for client-side
+    model: GEMINI_MODEL.GEMINI_2_0_FLASH_LIVE,
+    httpOptions: { apiVersion: "v1alpha" }, // must use v1alpha to init client with ephemeral key
  });

  const session = await llm.live.connect();
@@ -23,10 +33,7 @@ async function main() {
        content: "Say something about you for 10 seconds",
        role: "user",
      });
-    } else if (
-      liveEvents.audio.include(event) &&
-      typeof event.data === "string"
-    ) {
+    } else if (liveEvents.audio.include(event)) {
      const chunk = Buffer.from(event.data, "base64");
      audioChunks.push(chunk);
      console.log(`Received audio chunk: ${chunk.length} bytes`);
@@ -0,0 +1,24 @@
+# Logs
+logs
+*.log
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+pnpm-debug.log*
+lerna-debug.log*
+
+node_modules
+dist
+dist-ssr
+*.local
+
+# Editor directories and files
+.vscode/*
+!.vscode/extensions.json
+.idea
+.DS_Store
+*.suo
+*.ntvs*
+*.njsproj
+*.sln
+*.sw?
@@ -0,0 +1,54 @@
+# OpenAI Realtime Chat with LlamaIndex
+
+This is a demo application showcasing real-time audio and text chat capabilities using OpenAI's GPT-4 with voice through LlamaIndex. The application demonstrates bidirectional audio communication and text chat with an AI assistant.
+
+## Features
+
+- Real-time voice communication with GPT-4
+- Text-based chat interface
+- WebRTC-based audio streaming
+- Bidirectional communication (both text and voice)
+- React + TypeScript implementation
+
+## Prerequisites
+
+- Node.js (v18 or higher)
+- OpenAI API key with access to GPT-4 voice models
+- Modern browser with WebRTC support
+
+## Getting Started
+
+1. Install dependencies:
+
+```bash
+pnpm install
+```
+
+2. Start the development server:
+
+```bash
+pnpm run dev
+```
+
+## Usage
+
+The application provides a simple interface where you can:
+
+- Start/Stop a chat session
+- Speak to the AI assistant through your microphone
+- Receive audio responses from the assistant
+- See text transcripts of the conversation
+
+## Technical Details
+
+This project uses:
+
+- LlamaIndex for AI interaction management
+- WebRTC for real-time audio streaming
+- React for the UI
+- Vite for development and building
+- TypeScript for type safety
+
+```
+
+```
@@ -0,0 +1,28 @@
+import js from "@eslint/js";
+import reactHooks from "eslint-plugin-react-hooks";
+import reactRefresh from "eslint-plugin-react-refresh";
+import globals from "globals";
+import tseslint from "typescript-eslint";
+
+export default tseslint.config(
+  { ignores: ["dist"] },
+  {
+    extends: [js.configs.recommended, ...tseslint.configs.recommended],
+    files: ["**/*.{ts,tsx}"],
+    languageOptions: {
+      ecmaVersion: 2020,
+      globals: globals.browser,
+    },
+    plugins: {
+      "react-hooks": reactHooks,
+      "react-refresh": reactRefresh,
+    },
+    rules: {
+      ...reactHooks.configs.recommended.rules,
+      "react-refresh/only-export-components": [
+        "warn",
+        { allowConstantExport: true },
+      ],
+    },
+  },
+);
@@ -0,0 +1,13 @@
+<!doctype html>
+<html lang="en">
+  <head>
+    <meta charset="UTF-8" />
+    <link rel="icon" type="image/svg+xml" href="/vite.svg" />
+    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+    <title>Vite + React + TS</title>
+  </head>
+  <body>
+    <div id="root"></div>
+    <script type="module" src="/src/main.tsx"></script>
+  </body>
+</html>
@@ -0,0 +1,29 @@
+{
+  "name": "open-ai-realtime",
+  "private": true,
+  "version": "0.0.0",
+  "type": "module",
+  "scripts": {
+    "dev": "vite",
+    "build": "tsc -b && vite build",
+    "lint": "eslint .",
+    "preview": "vite preview"
+  },
+  "dependencies": {
+    "react": "^19.1.0",
+    "react-dom": "^19.1.0"
+  },
+  "devDependencies": {
+    "@eslint/js": "^9.25.0",
+    "@types/react": "^19.1.2",
+    "@types/react-dom": "^19.1.2",
+    "@vitejs/plugin-react": "^4.5.2",
+    "eslint": "^9.25.0",
+    "eslint-plugin-react-hooks": "^5.2.0",
+    "eslint-plugin-react-refresh": "^0.4.19",
+    "globals": "^16.0.0",
+    "typescript": "~5.8.3",
+    "typescript-eslint": "^8.30.1",
+    "vite": "^6.3.5"
+  }
+}
@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" class="iconify iconify--logos" width="31.88" height="32" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 257"><defs><linearGradient id="IconifyId1813088fe1fbc01fb466" x1="-.828%" x2="57.636%" y1="7.652%" y2="78.411%"><stop offset="0%" stop-color="#41D1FF"></stop><stop offset="100%" stop-color="#BD34FE"></stop></linearGradient><linearGradient id="IconifyId1813088fe1fbc01fb467" x1="43.376%" x2="50.316%" y1="2.242%" y2="89.03%"><stop offset="0%" stop-color="#FFEA83"></stop><stop offset="8.333%" stop-color="#FFDD35"></stop><stop offset="100%" stop-color="#FFA800"></stop></linearGradient></defs><path fill="url(#IconifyId1813088fe1fbc01fb466)" d="M255.153 37.938L134.897 252.976c-2.483 4.44-8.862 4.466-11.382.048L.875 37.958c-2.746-4.814 1.371-10.646 6.827-9.67l120.385 21.517a6.537 6.537 0 0 0 2.322-.004l117.867-21.483c5.438-.991 9.574 4.796 6.877 9.62Z"></path><path fill="url(#IconifyId1813088fe1fbc01fb467)" d="M185.432.063L96.44 17.501a3.268 3.268 0 0 0-2.634 3.014l-5.474 92.456a3.268 3.268 0 0 0 3.997 3.378l24.777-5.718c2.318-.535 4.413 1.507 3.936 3.838l-7.361 36.047c-.495 2.426 1.782 4.5 4.151 3.78l15.304-4.649c2.372-.72 4.652 1.36 4.15 3.788l-11.698 56.621c-.732 3.542 3.979 5.473 5.943 2.437l1.313-2.028l72.516-144.72c1.215-2.423-.88-5.186-3.54-4.672l-25.505 4.922c-2.396.462-4.435-1.77-3.759-4.114l16.646-57.705c.677-2.35-1.37-4.583-3.769-4.113Z"></path></svg>
@@ -0,0 +1,183 @@
+import { openai } from "@llamaindex/openai";
+import { liveEvents, LiveLLMSession, ModalityType } from "llamaindex";
+import { useEffect, useRef, useState } from "react";
+
+const MicIcon = ({ isConnected }: { isConnected: boolean }) => (
+  <svg
+    xmlns="http://www.w3.org/2000/svg"
+    viewBox="0 0 24 24"
+    fill="currentColor"
+    strokeWidth="2"
+    strokeLinecap="round"
+    strokeLinejoin="round"
+  >
+    {isConnected ? (
+      <>
+        <path d="M12 1a3 3 0 0 0-3 3v8a3 3 0 0 0 6 0V4a3 3 0 0 0-3-3z" />
+        <path d="M19 10v2a7 7 0 0 1-14 0v-2" />
+        <line x1="12" y1="19" x2="12" y2="23" />
+        <line x1="8" y1="23" x2="16" y2="23" />
+      </>
+    ) : (
+      <>
+        <path d="M12 1a3 3 0 0 0-3 3v8a3 3 0 0 0 6 0V4a3 3 0 0 0-3-3z" />
+        <path d="M19 10v2a7 7 0 0 1-14 0v-2" />
+      </>
+    )}
+  </svg>
+);
+
+const WaveAnimation = () => (
+  <div className="wave-animation">
+    {[...Array(3)].map((_, i) => (
+      <div key={i} className="wave" style={{ animationDelay: `${i * 0.2}s` }} />
+    ))}
+  </div>
+);
+
+export const AudioChat = () => {
+  const [isConnected, setIsConnected] = useState(false);
+  const [messages, setMessages] = useState<
+    Array<{ role: string; content: string }>
+  >([]);
+  const [status, setStatus] = useState<string>("");
+  const audioRef = useRef<HTMLAudioElement>(null);
+  const sessionRef = useRef<LiveLLMSession | null>(null);
+  const [stream, setStream] = useState<MediaStream | null>(null);
+  const messagesEndRef = useRef<HTMLDivElement>(null);
+
+  const scrollToBottom = () => {
+    messagesEndRef.current?.scrollIntoView({ behavior: "smooth" });
+  };
+
+  useEffect(() => {
+    scrollToBottom();
+  }, [messages]);
+
+  useEffect(() => {
+    return () => {
+      if (stream) {
+        stream.getTracks().forEach((track) => track.stop());
+      }
+    };
+  }, [stream]);
+
+  const startChat = async () => {
+    try {
+      setStatus("Initializing microphone...");
+      const userStream = await navigator.mediaDevices.getUserMedia({
+        audio: true,
+      });
+      setStream(userStream);
+
+      setStatus("Connecting to AI...");
+      const apiKey = prompt("Please enter your OpenAI API key:");
+      if (!apiKey) {
+        throw new Error("API key is required");
+      }
+
+      // move this call to the server side for security reasons
+      // Do not store the API key in the frontend!
+      const serverllm = openai({
+        apiKey: apiKey,
+        model: "gpt-4o-realtime-preview-2025-06-03",
+      });
+
+      const tempKey = await serverllm.live.getEphemeralKey();
+
+      const llm = openai({
+        apiKey: tempKey,
+        model: "gpt-4o-realtime-preview-2025-06-03",
+      });
+      const session = await llm.live.connect({
+        systemInstruction: "You are a helpful assistant who speaks naturally.",
+        responseModality: [ModalityType.TEXT, ModalityType.AUDIO],
+        audioConfig: {
+          stream: userStream,
+          onTrack: (remoteStream) => {
+            if (audioRef.current && remoteStream) {
+              audioRef.current.srcObject = remoteStream;
+              audioRef.current.play().catch(console.error);
+            }
+          },
+        },
+      });
+
+      sessionRef.current = session;
+      setIsConnected(true);
+      setStatus("Connected! Listening...");
+
+      for await (const event of session.streamEvents()) {
+        if (liveEvents.open.include(event)) {
+          setMessages((prev) => [
+            ...prev,
+            {
+              role: "user",
+              content: "Hello, I'm ready to chat!",
+            },
+          ]);
+          session.sendMessage({
+            content: "Hello, I'm ready to chat!",
+            role: "user",
+          });
+        } else if (liveEvents.text.include(event)) {
+          setMessages((prev) => [
+            ...prev,
+            {
+              role: "assistant",
+              content: event.text,
+            },
+          ]);
+        }
+      }
+    } catch (error) {
+      console.error("Error starting chat:", error);
+      setStatus("Error connecting. Please try again.");
+      setIsConnected(false);
+    }
+  };
+
+  const stopChat = async () => {
+    setStatus("Disconnecting...");
+    if (sessionRef.current) {
+      await sessionRef.current.disconnect();
+      sessionRef.current = null;
+    }
+    if (stream) {
+      stream.getTracks().forEach((track) => track.stop());
+      setStream(null);
+    }
+    if (audioRef.current) {
+      audioRef.current.srcObject = null;
+    }
+    setIsConnected(false);
+    setStatus("");
+  };
+
+  return (
+    <div className="audio-chat-container">
+      <h1>AI Voice Chat</h1>
+      <div className="messages-container">
+        {messages.map((msg, idx) => (
+          <div key={idx} className={`message ${msg.role}`}>
+            {msg.content}
+          </div>
+        ))}
+        <div ref={messagesEndRef} />
+      </div>
+
+      <div className="controls">
+        {status && <div className="status-indicator">{status}</div>}
+        <button
+          className={`mic-button ${isConnected ? "connected" : ""}`}
+          onClick={isConnected ? stopChat : startChat}
+          title={isConnected ? "Stop Chat" : "Start Chat"}
+        >
+          <MicIcon isConnected={isConnected} />
+          {isConnected && <WaveAnimation />}
+        </button>
+        <audio ref={audioRef} style={{ display: "none" }} />
+      </div>
+    </div>
+  );
+};
@@ -0,0 +1,322 @@
+:root {
+  --primary-color: #646cff;
+  --secondary-color: #535bf2;
+  --background-dark: #1a1a1a;
+  --chat-bg: #242424;
+  --text-primary: #ffffff;
+  --text-secondary: #888888;
+  --success-color: #4caf50;
+  --error-color: #f44336;
+  --gradient-start: #4776e6;
+  --gradient-end: #8e54e9;
+}
+
+body {
+  background-color: var(--background-dark);
+  color: var(--text-primary);
+  margin: 0;
+  min-height: 100vh;
+  font-family:
+    -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, Oxygen, Ubuntu,
+    Cantarell, "Open Sans", "Helvetica Neue", sans-serif;
+}
+
+#root {
+  max-width: 1280px;
+  height: 100vh;
+  margin: 0 auto;
+  padding: 2rem;
+  display: flex;
+  align-items: center;
+  justify-content: center;
+}
+
+.logo {
+  height: 6em;
+  padding: 1.5em;
+  will-change: filter;
+  transition: filter 300ms;
+}
+.logo:hover {
+  filter: drop-shadow(0 0 2em #646cffaa);
+}
+.logo.react:hover {
+  filter: drop-shadow(0 0 2em #61dafbaa);
+}
+
+@keyframes logo-spin {
+  from {
+    transform: rotate(0deg);
+  }
+  to {
+    transform: rotate(360deg);
+  }
+}
+
+@media (prefers-reduced-motion: no-preference) {
+  a:nth-of-type(2) .logo {
+    animation: logo-spin infinite 20s linear;
+  }
+}
+
+.card {
+  padding: 2em;
+}
+
+.read-the-docs {
+  color: #888;
+}
+
+.audio-chat-container {
+  display: flex;
+  flex-direction: column;
+  gap: 2rem;
+  width: 100%;
+  max-width: 800px;
+  height: 80vh;
+  margin: 0 auto;
+  padding: 2rem;
+  background: var(--chat-bg);
+  border-radius: 24px;
+  box-shadow: 0 8px 32px rgba(0, 0, 0, 0.2);
+  position: relative;
+  overflow: hidden;
+}
+
+.audio-chat-container::before {
+  content: "";
+  position: absolute;
+  top: 0;
+  left: 0;
+  right: 0;
+  height: 4px;
+  background: linear-gradient(
+    to right,
+    var(--gradient-start),
+    var(--gradient-end)
+  );
+}
+
+.audio-chat-container h1 {
+  font-size: 2.5rem;
+  margin: 0;
+  background: linear-gradient(
+    to right,
+    var(--gradient-start),
+    var(--gradient-end)
+  );
+  -webkit-background-clip: text;
+  background-clip: text;
+  color: transparent;
+  text-align: center;
+}
+
+.messages-container {
+  display: flex;
+  flex-direction: column;
+  gap: 1rem;
+  flex: 1;
+  overflow-y: auto;
+  padding: 1rem;
+  border-radius: 16px;
+  background: rgba(255, 255, 255, 0.05);
+  backdrop-filter: blur(10px);
+  margin: 1rem 0;
+}
+
+.message {
+  padding: 1rem 1.5rem;
+  border-radius: 16px;
+  max-width: 80%;
+  text-align: left;
+  animation: messageSlide 0.3s ease-out;
+  box-shadow: 0 2px 4px rgba(0, 0, 0, 0.1);
+}
+
+@keyframes messageSlide {
+  from {
+    opacity: 0;
+    transform: translateY(20px);
+  }
+  to {
+    opacity: 1;
+    transform: translateY(0);
+  }
+}
+
+.message.user {
+  background: linear-gradient(
+    135deg,
+    var(--gradient-start),
+    var(--gradient-end)
+  );
+  align-self: flex-end;
+  margin-left: 20%;
+  color: white;
+}
+
+.message.assistant {
+  background: rgba(255, 255, 255, 0.1);
+  align-self: flex-start;
+  margin-right: 20%;
+  border: 1px solid rgba(255, 255, 255, 0.1);
+}
+
+.controls {
+  display: flex;
+  justify-content: center;
+  align-items: center;
+  padding: 2rem;
+  position: relative;
+}
+
+.mic-button {
+  width: 80px;
+  height: 80px;
+  border-radius: 50%;
+  border: none;
+  background: linear-gradient(
+    135deg,
+    var(--gradient-start),
+    var(--gradient-end)
+  );
+  color: white;
+  cursor: pointer;
+  transition: all 0.3s cubic-bezier(0.4, 0, 0.2, 1);
+  display: flex;
+  align-items: center;
+  justify-content: center;
+  box-shadow: 0 4px 15px rgba(0, 0, 0, 0.2);
+  position: relative;
+  overflow: hidden;
+}
+
+.mic-button::before {
+  content: "";
+  position: absolute;
+  top: 0;
+  left: 0;
+  right: 0;
+  bottom: 0;
+  background: linear-gradient(
+    135deg,
+    rgba(255, 255, 255, 0.1),
+    rgba(255, 255, 255, 0)
+  );
+  border-radius: 50%;
+  transition: transform 0.3s ease;
+}
+
+.mic-button:hover {
+  transform: scale(1.05);
+  box-shadow: 0 8px 25px rgba(0, 0, 0, 0.3);
+}
+
+.mic-button:hover::before {
+  transform: translateY(-100%);
+}
+
+.mic-button.connected {
+  background: var(--error-color);
+  animation: pulseError 2s infinite;
+}
+
+.mic-button svg {
+  width: 32px;
+  height: 32px;
+  filter: drop-shadow(0 2px 4px rgba(0, 0, 0, 0.2));
+  transition: transform 0.3s ease;
+}
+
+.mic-button:hover svg {
+  transform: scale(1.1);
+}
+
+@keyframes pulseError {
+  0% {
+    box-shadow: 0 0 0 0 rgba(244, 67, 54, 0.4);
+  }
+  70% {
+    box-shadow: 0 0 0 20px rgba(244, 67, 54, 0);
+  }
+  100% {
+    box-shadow: 0 0 0 0 rgba(244, 67, 54, 0);
+  }
+}
+
+/* Status indicator */
+.status-indicator {
+  position: absolute;
+  top: -30px;
+  left: 50%;
+  transform: translateX(-50%);
+  font-size: 0.9rem;
+  color: var(--text-secondary);
+  opacity: 0;
+  transition: opacity 0.3s ease;
+}
+
+.controls:hover .status-indicator {
+  opacity: 1;
+}
+
+/* Scrollbar styling */
+.messages-container::-webkit-scrollbar {
+  width: 8px;
+}
+
+.messages-container::-webkit-scrollbar-track {
+  background: rgba(255, 255, 255, 0.05);
+  border-radius: 4px;
+}
+
+.messages-container::-webkit-scrollbar-thumb {
+  background: linear-gradient(var(--gradient-start), var(--gradient-end));
+  border-radius: 4px;
+}
+
+.messages-container::-webkit-scrollbar-thumb:hover {
+  background: linear-gradient(var(--gradient-end), var(--gradient-start));
+}
+
+/* Wave Animation */
+.wave-animation {
+  position: absolute;
+  bottom: -15px;
+  left: 50%;
+  transform: translateX(-50%);
+  display: flex;
+  gap: 4px;
+}
+
+.wave {
+  width: 4px;
+  height: 15px;
+  background: currentColor;
+  border-radius: 2px;
+  animation: wave 0.5s ease-in-out infinite;
+}
+
+@keyframes wave {
+  0%,
+  100% {
+    transform: scaleY(0.5);
+  }
+  50% {
+    transform: scaleY(1.5);
+  }
+}
+
+/* Loading state */
+.mic-button.loading {
+  animation: rotate 1s linear infinite;
+}
+
+@keyframes rotate {
+  from {
+    transform: rotate(0deg);
+  }
+  to {
+    transform: rotate(360deg);
+  }
+}
@@ -0,0 +1,10 @@
+import { StrictMode } from "react";
+import { createRoot } from "react-dom/client";
+import { AudioChat } from "./audio-chat.tsx";
+import "./index.css";
+
+createRoot(document.getElementById("root")!).render(
+  <StrictMode>
+    <AudioChat />
+  </StrictMode>,
+);
@@ -0,0 +1 @@
+/// <reference types="vite/client" />
@@ -0,0 +1,26 @@
+{
+  "compilerOptions": {
+    "tsBuildInfoFile": "./node_modules/.tmp/tsconfig.app.tsbuildinfo",
+    "target": "ES2020",
+    "useDefineForClassFields": true,
+    "lib": ["ES2020", "DOM", "DOM.Iterable"],
+    "module": "ESNext",
+    "skipLibCheck": true,
+
+    /* Bundler mode */
+    "moduleResolution": "bundler",
+    "allowImportingTsExtensions": true,
+    "verbatimModuleSyntax": true,
+    "moduleDetection": "force",
+    "noEmit": true,
+    "jsx": "react-jsx",
+
+    /* Linting */
+    "strict": true,
+    "noUnusedLocals": true,
+    "noUnusedParameters": true,
+    "noFallthroughCasesInSwitch": true,
+    "noUncheckedSideEffectImports": true
+  },
+  "include": ["src"]
+}
@@ -0,0 +1,4 @@
+{
+  "files": [],
+  "references": [{ "path": "./tsconfig.app.json" }]
+}
@@ -0,0 +1,7 @@
+import react from "@vitejs/plugin-react";
+import { defineConfig } from "vite";
+
+// https://vite.dev/config/
+export default defineConfig({
+  plugins: [react()],
+});
@@ -1,6 +1,6 @@
 {
  "name": "@llamaindex/examples",
-  "version": "0.3.22",
+  "version": "0.3.25",
  "private": true,
  "scripts": {
    "lint": "eslint .",
@@ -11,51 +11,51 @@
    "@azure/cosmos": "^4.1.1",
    "@azure/identity": "^4.4.1",
    "@azure/search-documents": "^12.1.0",
-    "@llamaindex/anthropic": "^0.3.12",
-    "@llamaindex/assemblyai": "^0.1.9",
-    "@llamaindex/astra": "^0.0.24",
-    "@llamaindex/azure": "^0.1.21",
-    "@llamaindex/chroma": "^0.0.24",
-    "@llamaindex/clip": "^0.0.60",
-    "@llamaindex/cloud": "^4.0.14",
-    "@llamaindex/cohere": "^0.0.24",
-    "@llamaindex/core": "^0.6.10",
-    "@llamaindex/deepinfra": "^0.0.60",
-    "@llamaindex/deepseek": "^0.0.20",
-    "@llamaindex/discord": "^0.1.9",
-    "@llamaindex/elastic-search": "^0.1.10",
+    "@llamaindex/anthropic": "^0.3.14",
+    "@llamaindex/assemblyai": "^0.1.11",
+    "@llamaindex/astra": "^0.0.26",
+    "@llamaindex/azure": "^0.1.23",
+    "@llamaindex/chroma": "^0.0.26",
+    "@llamaindex/clip": "^0.0.62",
+    "@llamaindex/cloud": "^4.0.16",
+    "@llamaindex/cohere": "^0.0.26",
+    "@llamaindex/core": "^0.6.12",
+    "@llamaindex/deepinfra": "^0.0.62",
+    "@llamaindex/deepseek": "^0.0.22",
+    "@llamaindex/discord": "^0.1.11",
+    "@llamaindex/elastic-search": "^0.1.12",
    "@llamaindex/env": "^0.1.30",
-    "@llamaindex/firestore": "^1.0.17",
-    "@llamaindex/fireworks": "^0.0.20",
-    "@llamaindex/google": "^0.3.9",
-    "@llamaindex/groq": "^0.0.75",
-    "@llamaindex/huggingface": "^0.1.14",
-    "@llamaindex/jinaai": "^0.0.20",
-    "@llamaindex/milvus": "^0.1.19",
-    "@llamaindex/mistral": "^0.1.10",
-    "@llamaindex/mixedbread": "^0.0.24",
-    "@llamaindex/mongodb": "^0.0.25",
-    "@llamaindex/node-parser": "^2.0.10",
-    "@llamaindex/notion": "^0.1.9",
-    "@llamaindex/ollama": "^0.1.10",
-    "@llamaindex/openai": "^0.4.4",
-    "@llamaindex/perplexity": "^0.0.17",
-    "@llamaindex/pinecone": "^0.1.10",
-    "@llamaindex/portkey-ai": "^0.0.52",
-    "@llamaindex/postgres": "^0.0.53",
-    "@llamaindex/qdrant": "^0.1.20",
-    "@llamaindex/readers": "^3.1.8",
-    "@llamaindex/replicate": "^0.0.52",
-    "@llamaindex/supabase": "^0.1.9",
-    "@llamaindex/together": "^0.0.20",
-    "@llamaindex/tools": "^0.0.16",
-    "@llamaindex/upstash": "^0.0.24",
-    "@llamaindex/vercel": "^0.1.10",
-    "@llamaindex/vllm": "^0.0.46",
-    "@llamaindex/voyage-ai": "^1.0.16",
-    "@llamaindex/weaviate": "^0.0.25",
-    "@llamaindex/workflow": "^1.1.9",
-    "@llamaindex/xai": "workspace:^0.0.7",
+    "@llamaindex/firestore": "^1.0.19",
+    "@llamaindex/fireworks": "^0.0.22",
+    "@llamaindex/google": "^0.3.11",
+    "@llamaindex/groq": "^0.0.77",
+    "@llamaindex/huggingface": "^0.1.16",
+    "@llamaindex/jinaai": "^0.0.22",
+    "@llamaindex/milvus": "^0.1.21",
+    "@llamaindex/mistral": "^0.1.12",
+    "@llamaindex/mixedbread": "^0.0.26",
+    "@llamaindex/mongodb": "^0.0.27",
+    "@llamaindex/node-parser": "^2.0.12",
+    "@llamaindex/notion": "^0.1.11",
+    "@llamaindex/ollama": "^0.1.12",
+    "@llamaindex/openai": "^0.4.6",
+    "@llamaindex/perplexity": "^0.0.19",
+    "@llamaindex/pinecone": "^0.1.12",
+    "@llamaindex/portkey-ai": "^0.0.54",
+    "@llamaindex/postgres": "^0.0.55",
+    "@llamaindex/qdrant": "^0.1.22",
+    "@llamaindex/readers": "^3.1.11",
+    "@llamaindex/replicate": "^0.0.54",
+    "@llamaindex/supabase": "^0.1.12",
+    "@llamaindex/together": "^0.0.22",
+    "@llamaindex/tools": "^0.1.1",
+    "@llamaindex/upstash": "^0.0.26",
+    "@llamaindex/vercel": "^0.1.12",
+    "@llamaindex/vllm": "^0.0.48",
+    "@llamaindex/voyage-ai": "^1.0.18",
+    "@llamaindex/weaviate": "^0.0.27",
+    "@llamaindex/workflow": "^1.1.12",
+    "@llamaindex/xai": "workspace:^0.0.9",
    "@notionhq/client": "^2.2.15",
    "@pinecone-database/pinecone": "^4.0.0",
    "@vercel/postgres": "^0.10.0",
@@ -64,11 +64,11 @@
    "commander": "^12.1.0",
    "dotenv": "^16.4.5",
    "js-tiktoken": "^1.0.14",
-    "llamaindex": "^0.11.8",
+    "llamaindex": "^0.11.11",
    "mongodb": "6.7.0",
    "postgres": "^3.4.4",
    "wikipedia": "^2.1.2",
-    "zod": "^3.23.8"
+    "zod": "^3.25.67"
  },
  "devDependencies": {
    "@types/node": "^22.9.0",
@@ -1,3 +1,4 @@
+import { OpenAIEmbedding } from "@llamaindex/openai";
 import {
  Document,
  SentenceSplitter,
@@ -7,6 +8,10 @@ import {
 import { OldSentenceSplitter } from "./old-sentence-splitter";
 export const STORAGE_DIR = "./data";

+Settings.embedModel = new OpenAIEmbedding({
+  model: "text-embedding-3-small",
+});
+
 // Update node parser
 (async () => {
  // generate a document with a very long sentence (9000 words long)
@@ -15,11 +15,14 @@
    "start:llamaparse-json": "node --import tsx ./src/llamaparse-json.ts",
    "start:discord": "node --import tsx ./src/discord.ts",
    "start:json": "node --import tsx ./src/json.ts",
-    "start:obsidian": "node --import tsx ./src/obsidian.ts"
+    "start:obsidian": "node --import tsx ./src/obsidian.ts",
+    "start:xml": "node --import tsx ./src/xml.ts",
+    "start:excel": "node --import tsx ./src/excel.ts"
  },
  "dependencies": {
    "@llamaindex/cloud": "workspace:* || ^2.0.24",
    "@llamaindex/readers": "workspace:* || ^1.0.25",
+    "@llamaindex/excel": "workspace:*",
    "llamaindex": "workspace:* || ^0.8.37"
  },
  "devDependencies": {
@@ -0,0 +1,20 @@
+import { ExcelReader } from "@llamaindex/excel";
+
+async function main() {
+  // Load PDF
+  const reader = new ExcelReader({
+    sheetSpecifier: 0,
+    concatRows: true,
+    fieldSeparator: ",",
+    keyValueSeparator: ":",
+  });
+
+  const documents = await reader.loadData("../data/sample_excel_sheet.xls");
+
+  for (const doc of documents) {
+    console.log(doc.text);
+    console.log("----");
+  }
+}
+
+main().catch(console.error);
@@ -0,0 +1,16 @@
+import { XMLReader } from "@llamaindex/readers/xml";
+
+async function main() {
+  // Load PDF
+  const reader = new XMLReader({
+    splitLevel: 2,
+  });
+  const documents = await reader.loadData("../data/company.xml");
+
+  for (const doc of documents) {
+    console.log(doc.text);
+    console.log("----");
+  }
+}
+
+main().catch(console.error);
@@ -1,8 +1,4 @@
-import {
-  GEMINI_EMBEDDING_MODEL,
-  GeminiEmbedding,
-  GeminiSession,
-} from "@llamaindex/google";
+import { GEMINI_EMBEDDING_MODEL, GeminiEmbedding } from "@llamaindex/google";
 import { QdrantVectorStore } from "@llamaindex/qdrant";
 import {
  Document,
@@ -12,9 +8,6 @@ import {

 const embedding = new GeminiEmbedding({
  model: GEMINI_EMBEDDING_MODEL.EMBEDDING_001,
-  session: new GeminiSession({
-    apiKey: process.env.GEMINI_API_KEY,
-  }),
 });

 async function main() {
@@ -18,5 +18,11 @@
      "module": "commonjs"
    }
  },
-  "include": ["./**/*.ts"]
+  "include": ["./**/*.ts"],
+  "exclude": [
+    "node_modules",
+    "dist",
+    "models/openai/live/browser/open-ai-realtime",
+    "**/browser/**"
+  ]
 }
@@ -1,5 +1,24 @@
 # @llamaindex/autotool

+## 8.0.11
+
+### Patch Changes
+
+- Updated dependencies [7039e1a]
+  - llamaindex@0.11.11
+
+## 8.0.10
+
+### Patch Changes
+
+- llamaindex@0.11.10
+
+## 8.0.9
+
+### Patch Changes
+
+- llamaindex@0.11.9
+
 ## 8.0.8

 ### Patch Changes
@@ -1,5 +1,27 @@
 # @llamaindex/autotool-01-node-example

+## 0.0.119
+
+### Patch Changes
+
+- Updated dependencies [7039e1a]
+  - llamaindex@0.11.11
+  - @llamaindex/autotool@8.0.11
+
+## 0.0.118
+
+### Patch Changes
+
+- llamaindex@0.11.10
+- @llamaindex/autotool@8.0.10
+
+## 0.0.117
+
+### Patch Changes
+
+- llamaindex@0.11.9
+- @llamaindex/autotool@8.0.9
+
 ## 0.0.116

 ### Patch Changes
@@ -13,5 +13,5 @@
  "scripts": {
    "start": "node --import tsx --import @llamaindex/autotool/node ./src/index.ts"
  },
-  "version": "0.0.116"
+  "version": "0.0.119"
 }
@@ -6,7 +6,7 @@
    "url": "git+https://github.com/run-llama/LlamaIndexTS.git",
    "directory": "packages/autotool"
  },
-  "version": "8.0.8",
+  "version": "8.0.11",
  "description": "auto transpile your JS function to LLM Agent compatible",
  "files": [
    "dist",
@@ -1,5 +1,23 @@
 # @llamaindex/cloud

+## 4.0.16
+
+### Patch Changes
+
+- Updated dependencies [7039e1a]
+- Updated dependencies [7039e1a]
+  - @llamaindex/core@0.6.12
+
+## 4.0.15
+
+### Patch Changes
+
+- Updated dependencies [a89e187]
+- Updated dependencies [62699b7]
+- Updated dependencies [c5b2691]
+- Updated dependencies [d8ac8d3]
+  - @llamaindex/core@0.6.11
+
 ## 4.0.14

 ### Patch Changes
@@ -1,6 +1,6 @@
 {
  "name": "@llamaindex/cloud",
-  "version": "4.0.14",
+  "version": "4.0.16",
  "type": "module",
  "license": "MIT",
  "scripts": {
@@ -79,6 +79,6 @@
  },
  "dependencies": {
    "p-retry": "^6.2.1",
-    "zod": "^3.25.7"
+    "zod": "^3.25.67"
  }
 }
@@ -1,5 +1,21 @@
 # @llamaindex/core

+## 0.6.12
+
+### Patch Changes
+
+- 7039e1a: Internal cleanup of base64 encoding
+- 7039e1a: chore: migrate to @google/genai SDK
+
+## 0.6.11
+
+### Patch Changes
+
+- a89e187: Feat: added custom abbreviations to sentence splitter
+- 62699b7: Improve performance of sentence splitter
+- c5b2691: Add more Acronyms on SentenceSplitter
+- d8ac8d3: Feat: add support for openai realtime API
+
 ## 0.6.10

 ### Patch Changes
@@ -1,7 +1,7 @@
 {
  "name": "@llamaindex/core",
  "type": "module",
-  "version": "0.6.10",
+  "version": "0.6.12",
  "description": "LlamaIndex Core Module",
  "exports": {
    "./agent": {
@@ -312,7 +312,7 @@
    "@llamaindex/env": "workspace:*",
    "@types/node": "^22.9.0",
    "magic-bytes.js": "^1.10.0",
-    "zod": "^3.23.8",
-    "zod-to-json-schema": "^3.23.3"
+    "zod": "^3.25.67",
+    "zod-to-json-schema": "^3.24.6"
  }
 }
@@ -1,4 +1,5 @@
-import { extractText, streamConverter } from "../utils";
+import { extractText } from "../utils/llms";
+import { streamConverter } from "../utils/stream";
 import type {
  ChatResponse,
  ChatResponseChunk,
@@ -1,6 +1,9 @@
 export { BaseLLM, ToolCallLLM } from "./base";
-export { LiveLLM, LiveLLMSession, liveEvents, type LiveEvent } from "./live";
+export { LiveLLM, LiveLLMCapability, LiveLLMSession } from "./live/live";
+export { liveEvents, type LiveEvent } from "./live/live-types";
+export type { MessageSender } from "./live/sender";
 export type {
+  AudioConfig,
  BaseTool,
  BaseToolWithCall,
  ChatMessage,
@@ -1,9 +1,7 @@
 import type {
-  ChatMessage,
-  LiveConnectConfig,
  MessageContentAudioDetail,
  MessageContentTextDetail,
-} from "./type";
+} from "../type";

 export type OpenEvent = { type: "open" };

@@ -63,45 +61,3 @@ export const liveEvents = {
      e.type === "turnComplete",
  },
 };
-export abstract class LiveLLMSession {
-  protected eventQueue: LiveEvent[] = [];
-  protected eventResolvers: ((value: LiveEvent) => void)[] = [];
-  protected closed = false;
-  abstract sendMessage(message: ChatMessage): void;
-  async *streamEvents(): AsyncIterable<LiveEvent> {
-    while (true) {
-      const event = await this.nextEvent();
-      if (event === undefined) {
-        break;
-      }
-      yield event;
-    }
-  }
-  abstract disconnect(): Promise<void>;
-
-  protected async nextEvent(): Promise<LiveEvent | undefined> {
-    if (this.eventQueue.length) {
-      return Promise.resolve(this.eventQueue.shift());
-    }
-
-    return new Promise((resolve) => {
-      this.eventResolvers.push(resolve);
-    });
-  }
-
-  //Uses an async queue to send events to the client
-  // if the consumer is waiting for an event, it will be resolved immediately
-  // otherwise, the event will be queued up and sent when the consumer is ready
-  pushEventToQueue(event: LiveEvent) {
-    if (this.eventResolvers.length) {
-      //resolving the promise with the event
-      this.eventResolvers.shift()!(event);
-    } else {
-      this.eventQueue.push(event);
-    }
-  }
-}
-
-export abstract class LiveLLM {
-  abstract connect(config?: LiveConnectConfig): Promise<LiveLLMSession>;
-}
@@ -0,0 +1,124 @@
+import type {
+  ChatMessage,
+  LiveConnectConfig,
+  MessageContentAudioDetail,
+  MessageContentDetail,
+  MessageContentImageDataDetail,
+  MessageContentVideoDetail,
+} from "../type";
+import type { LiveEvent } from "./live-types";
+import type { MessageSender } from "./sender";
+
+export enum LiveLLMCapability {
+  EPHEMERAL_KEY = "ephemeral_key",
+  AUDIO_CONFIG = "audio_config",
+}
+
+export abstract class LiveLLMSession {
+  protected eventQueue: LiveEvent[] = [];
+  protected eventResolvers: ((value: LiveEvent) => void)[] = [];
+  closed = false;
+  abstract get messageSender(): MessageSender;
+
+  private isTextMessage(content: MessageContentDetail) {
+    return content.type === "text";
+  }
+
+  private isAudioMessage(
+    content: MessageContentDetail,
+  ): content is MessageContentAudioDetail {
+    return content.type === "audio";
+  }
+
+  private isImageMessage(
+    content: MessageContentDetail,
+  ): content is MessageContentImageDataDetail {
+    return content.type === "image";
+  }
+
+  private isVideoMessage(
+    content: MessageContentDetail,
+  ): content is MessageContentVideoDetail {
+    return content.type === "video";
+  }
+
+  sendMessage(message: ChatMessage) {
+    const { content, role } = message;
+    if (!Array.isArray(content)) {
+      this.messageSender.sendTextMessage(content, role);
+    } else {
+      for (const item of content) {
+        this.processMessage(item, role);
+      }
+    }
+  }
+
+  private processMessage(message: MessageContentDetail, role?: string) {
+    if (this.isTextMessage(message)) {
+      this.messageSender.sendTextMessage(message.text, role);
+    } else if (
+      this.isAudioMessage(message) &&
+      this.messageSender.sendAudioMessage
+    ) {
+      this.messageSender.sendAudioMessage(message, role);
+    } else if (
+      this.isImageMessage(message) &&
+      this.messageSender.sendImageMessage
+    ) {
+      this.messageSender.sendImageMessage(message, role);
+    } else if (
+      this.isVideoMessage(message) &&
+      this.messageSender.sendVideoMessage
+    ) {
+      this.messageSender.sendVideoMessage(message, role);
+    }
+  }
+
+  async *streamEvents(): AsyncIterable<LiveEvent> {
+    while (true) {
+      const event = await this.nextEvent();
+      if (event === undefined) {
+        break;
+      }
+      yield event;
+    }
+  }
+  abstract disconnect(): Promise<void>;
+
+  protected async nextEvent(): Promise<LiveEvent | undefined> {
+    if (this.eventQueue.length) {
+      return Promise.resolve(this.eventQueue.shift());
+    }
+
+    return new Promise((resolve) => {
+      this.eventResolvers.push(resolve);
+    });
+  }
+
+  //Uses an async queue to send events to the client
+  // if the consumer is waiting for an event, it will be resolved immediately
+  // otherwise, the event will be queued up and sent when the consumer is ready
+  pushEventToQueue(event: LiveEvent) {
+    if (this.eventResolvers.length) {
+      //resolving the promise with the event
+      this.eventResolvers.shift()!(event);
+    } else {
+      this.eventQueue.push(event);
+    }
+  }
+}
+
+export abstract class LiveLLM {
+  /**
+   * Set of capabilities supported by this implementation.
+   * Override in subclasses as needed.
+   */
+  capabilities: Set<LiveLLMCapability> = new Set();
+
+  abstract connect(config?: LiveConnectConfig): Promise<LiveLLMSession>;
+  abstract getEphemeralKey(): Promise<string | undefined>;
+
+  hasCapability(capability: LiveLLMCapability): boolean {
+    return this.capabilities.has(capability);
+  }
+}
@@ -0,0 +1,15 @@
+import type {
+  MessageContentAudioDetail,
+  MessageContentImageDataDetail,
+  MessageContentVideoDetail,
+} from "../type";
+
+export interface MessageSender {
+  sendTextMessage(message: string, role?: string): void;
+  sendAudioMessage?(content: MessageContentAudioDetail, role?: string): void;
+  sendImageMessage?(
+    content: MessageContentImageDataDetail,
+    role?: string,
+  ): void;
+  sendVideoMessage?(content: MessageContentVideoDetail, role?: string): void;
+}
@@ -290,8 +290,14 @@ export type ToolOutput = {
  isError: boolean;
 };

+export interface AudioConfig {
+  stream?: MediaStream;
+  onTrack?: (track: MediaStream | null) => void;
+}
+
 export interface LiveConnectConfig {
  tools?: BaseTool[];
  responseModality?: ModalityType[];
  systemInstruction?: string;
+  audioConfig?: AudioConfig;
 }
@@ -1,8 +1,7 @@
 import { Settings } from "../global";
-import type { ChatMessage, MessageContent, MessageType } from "../llms";
-import { SimpleChatStore, type BaseChatStore } from "../storage/chat-store";
+import type { ChatMessage } from "../llms";
+import { type BaseChatStore, SimpleChatStore } from "../storage/chat-store";
 import { extractText } from "../utils";
-import type { MemoryBlockContent } from "./types";

 export const DEFAULT_TOKEN_LIMIT_RATIO = 0.75;
 export const DEFAULT_CHAT_STORE_KEY = "chat_history";
@@ -84,70 +83,3 @@ export abstract class BaseChatStoreMemory<
    this.chatStore.deleteMessages(this.chatStoreKey);
  }
 }
-
-export abstract class BaseMemoryBlock<
-  AdditionalMessageOptions extends object = object,
-> {
-  protected priority: number;
-  protected content: MemoryBlockContent[];
-
-  constructor(priority: number, content: MemoryBlockContent[]) {
-    this.priority = priority;
-    this.content = content;
-  }
-
-  /**
-   * Retrieves all content from the memory block.
-   * @returns An array of content.
-   */
-  get(): MemoryBlockContent[] {
-    return this.content;
-  }
-
-  /**
-   * Adds a new content to the memory block.
-   * @param content The content to be added to the memory block.
-   */
-  add(content: MemoryBlockContent): void {
-    this.content.push(content);
-  }
-
-  /**
-   * Clears all content from the memory block.
-   */
-  clear(): void {
-    this.content = [];
-  }
-
-  /**
-   * Retrieves the priority of the memory block.
-   * @returns The priority of the memory block.
-   */
-  getPriority = (): number => this.priority;
-
-  /**
-   * Converts content to messages.
-   * @param content The content to be converted to messages.
-   * @returns An array of messages.
-   */
-  toMessages(): ChatMessage<AdditionalMessageOptions>[] {
-    return this.content.map((entry) => {
-      // If entry is a ChatMessage, return it
-      if (typeof entry === "object" && "content" in entry && "role" in entry) {
-        return entry as ChatMessage<AdditionalMessageOptions>;
-      }
-      // Else, create a new ChatMessage with the entry as content
-      return {
-        content: entry as MessageContent,
-        role: "system" as MessageType,
-        options: {} as AdditionalMessageOptions,
-      };
-    });
-  }
-}
-
-export class StaticMemoryBlock extends BaseMemoryBlock {
-  constructor(content: MemoryBlockContent[], priority: number) {
-    super(priority, content);
-  }
-}
@@ -1,55 +0,0 @@
-import { StaticMemoryBlock } from "./base";
-import { Memory } from "./memory";
-import type { MemoryBlockContent, MemoryOptions } from "./types";
-
-/**
- * Create a new enhanced memory instance with support for dual message formats,
- * snapshots, and memory blocks
- *
- * @example
- * ```typescript
- * // Basic usage
- * const memory = createMemory();
- *
- * // With configuration
- * const memory = createMemory({
- *   tokenLimit: 10_000,
- *   blocks: [
- *     staticMemoryBlock(["You are a helpful assistant named Claude."], 0),
- *   ],
- * });
- *
- * // Add messages in different formats
- * await memory.add({
- *   content: "Hello, how are you?",
- *   role: "user",
- * });
- *
- * // Get messages in Vercel AI format
- * const uiMessages = await memory.get({ type: "vercel" });
- *
- * // Save and restore memory state
- * const snapshot = memory.snapshot();
- * await memory.loadSnapshot(snapshot);
- * ```
- *
- * @param options Configuration options for the memory instance
- * @returns A new EnhancedMemory instance
- */
-export function createMemory(options: MemoryOptions = {}): Memory {
-  return new Memory(options);
-}
-
-/**
- * Create a new static memory block instance
- *
- * @param content The content of the memory block
- * @param priority The priority of the memory block
- * @returns A new StaticMemoryBlock instance
- */
-export function staticMemoryBlock(
-  content: MemoryBlockContent[],
-  priority: number,
-): StaticMemoryBlock {
-  return new StaticMemoryBlock(content, priority);
-}
@@ -1,16 +1,3 @@
-// Existing exports (backward compatibility)
-export { BaseChatStoreMemory, BaseMemory } from "./base";
+export { BaseMemory } from "./base";
 export { ChatMemoryBuffer } from "./chat-memory-buffer";
 export { ChatSummaryMemoryBuffer } from "./summary-memory";
-
-export { createMemory, staticMemoryBlock } from "./factory";
-export { Memory } from "./memory";
-export { MessageConverter } from "./message-converter";
-
-// Type exports
-export type {
-  GetMessageOptions,
-  MemoryOptions,
-  UIMessage,
-  UIPart,
-} from "./types";
@@ -1,140 +0,0 @@
-import { Settings, type JSONValue } from "../global";
-import type { ChatMessage } from "../llms";
-import { extractText } from "../utils";
-import { BaseMemoryBlock, StaticMemoryBlock } from "./base";
-import { MessageConverter } from "./message-converter";
-import type { GetMessageOptions, MemoryOptions, UIMessage } from "./types";
-import { serializeChatMessage, serializeMessageContent } from "./utils";
-
-const DEFAULT_TOKEN_LIMIT = 4096;
-
-export class Memory {
-  private blocks: BaseMemoryBlock[] = [];
-  private tokenLimit: number;
-
-  constructor(options: MemoryOptions = {}) {
-    this.blocks = options.blocks || [];
-    this.tokenLimit = options.tokenLimit || DEFAULT_TOKEN_LIMIT;
-  }
-
-  private async getAllMessages(): Promise<ChatMessage[]> {
-    // Order blocks by priority
-    const orderedBlocks = this.blocks.sort(
-      (a, b) => b.getPriority() - a.getPriority(),
-    );
-
-    // Get all messages
-    return orderedBlocks.map((block) => block.toMessages()).flat();
-  }
-
-  async getMessagesWithLimit(tokenLimit: number): Promise<ChatMessage[]> {
-    const messages = await this.getAllMessages();
-    return this.applyTokenLimit(messages, tokenLimit);
-  }
-
-  /**
-   * @returns Return a serialized snapshot of the memory in JSON format.
-   */
-  snapshot(): JSONValue {
-    return {
-      blocks: this.blocks.map((block) =>
-        block.get().map((message) => {
-          if (MessageConverter.isChatMessage(message)) {
-            return serializeChatMessage(message);
-          }
-          return serializeMessageContent(message);
-        }),
-      ),
-      metadata: {
-        tokenLimit: this.tokenLimit,
-      },
-    };
-  }
-
-  async loadSnapshot(snapshot: JSONValue): Promise<void> {}
-
-  async add(message: ChatMessage | UIMessage): Promise<void> {
-    let llamaMessage: ChatMessage;
-
-    if (MessageConverter.isUIMessage(message)) {
-      llamaMessage = MessageConverter.toLlamaIndexMessage(message);
-    } else if (MessageConverter.isChatMessage(message)) {
-      llamaMessage = message as ChatMessage;
-    } else {
-      throw new Error(
-        "Invalid message format. Expected ChatMessage or UIMessage.",
-      );
-    }
-
-    // Convert message to block
-    const block = new StaticMemoryBlock([llamaMessage.content], 0);
-
-    this.blocks.push(block);
-  }
-
-  async get(options?: GetMessageOptions): Promise<ChatMessage[] | UIMessage[]> {
-    const messages = await this.getAllMessages();
-
-    if (options?.type === "vercel") {
-      return messages.map((message) => MessageConverter.toUIMessage(message));
-    }
-
-    // Default to LlamaIndex format
-    return messages;
-  }
-
-  async getLLM(): Promise<ChatMessage[]> {
-    // Convert all blocks to messages
-    const allMessages = await this.getAllMessages();
-
-    // Apply token limit
-    return this.applyTokenLimit(allMessages, this.tokenLimit);
-  }
-
-  async clear(): Promise<void> {
-    this.blocks = [];
-  }
-
-  private applyTokenLimit(
-    messages: ChatMessage[],
-    tokenLimit: number,
-  ): ChatMessage[] {
-    if (messages.length === 0) {
-      return [];
-    }
-
-    let tokenCount = 0;
-    const result: ChatMessage[] = [];
-
-    // Process messages in reverse order (keep most recent)
-    for (let i = messages.length - 1; i >= 0; i--) {
-      const message = messages[i];
-      if (!message) continue;
-
-      const messageTokens = this.countMessagesToken([message]);
-
-      if (tokenCount + messageTokens <= tokenLimit) {
-        result.unshift(message);
-        tokenCount += messageTokens;
-      } else {
-        // If we can't fit any more messages, break
-        // But always try to include at least the most recent message
-        if (result.length === 0 && messageTokens <= tokenLimit) {
-          result.push(message);
-        }
-        break;
-      }
-    }
-
-    return result;
-  }
-
-  private countMessagesToken(messages: ChatMessage[]): number {
-    if (messages.length === 0) {
-      return 0;
-    }
-    const tokenizer = Settings.tokenizer;
-    const str = messages.map((m) => extractText(m.content)).join(" ");
-    return tokenizer.encode(str).length;
-  }
-}
@@ -1,243 +0,0 @@
-import { randomUUID } from "@llamaindex/env";
-import type {
-  ChatMessage,
-  MessageContent,
-  MessageContentDetail,
-} from "../llms";
-import { extractText } from "../utils";
-import type { UIMessage, UIPart } from "./types";
-
-/**
- * Utility class for converting between LlamaIndex ChatMessage and Vercel UI Message formats
- */
-export class MessageConverter {
-  /**
-   * Convert Vercel UI Message to LlamaIndex ChatMessage format
-   */
-  static toLlamaIndexMessage<AdditionalMessageOptions extends object = object>(
-    uiMessage: UIMessage,
-  ): ChatMessage<AdditionalMessageOptions> {
-    // Convert UI message role to MessageType
-    let role: ChatMessage["role"];
-    switch (uiMessage.role) {
-      case "system":
-        role = "system";
-        break;
-      case "user":
-        role = "user";
-        break;
-      case "assistant":
-        role = "assistant";
-        break;
-      case "data":
-        role = "system"; // Map data role to system
-        break;
-      default:
-        role = "user"; // Default fallback
-    }
-
-    // Convert parts to MessageContent
-    const content = this.convertPartsToMessageContent(uiMessage.parts);
-
-    return {
-      content: content || uiMessage.content,
-      role,
-      options: undefined as unknown as AdditionalMessageOptions,
-    };
-  }
-
-  /**
-   * Convert LlamaIndex ChatMessage to Vercel UI Message format
-   */
-  static toUIMessage(llamaMessage: ChatMessage): UIMessage {
-    const parts: UIPart[] = this.convertMessageContentToParts(
-      llamaMessage.content,
-    );
-
-    // Convert role to UI message role
-    let role: UIMessage["role"];
-    switch (llamaMessage.role) {
-      case "user":
-        role = "user";
-        break;
-      case "assistant":
-        role = "assistant";
-        break;
-      case "system":
-      case "memory":
-      case "developer":
-        role = "system";
-        break;
-      default:
-        role = "user";
-    }
-
-    return {
-      id: randomUUID(),
-      role,
-      content: extractText(llamaMessage.content),
-      parts,
-      createdAt: new Date(),
-      annotations: [],
-    };
-  }
-
-  /**
-   * Validate if object matches UIMessage structure
-   */
-  static isUIMessage(message: unknown): message is UIMessage {
-    if (!message || typeof message !== "object") {
-      return false;
-    }
-
-    const msg = message as Record<string, unknown>;
-
-    return (
-      typeof msg.id === "string" &&
-      typeof msg.role === "string" &&
-      ["system", "user", "assistant", "data"].includes(msg.role as string) &&
-      typeof msg.content === "string" &&
-      Array.isArray(msg.parts)
-    );
-  }
-
-  /**
-   * Validate if object matches ChatMessage structure
-   */
-  static isChatMessage(message: unknown): message is ChatMessage {
-    if (!message || typeof message !== "object") {
-      return false;
-    }
-
-    const msg = message as Record<string, unknown>;
-
-    return (
-      (typeof msg.content === "string" || Array.isArray(msg.content)) &&
-      typeof msg.role === "string" &&
-      ["user", "assistant", "system", "memory", "developer"].includes(
-        msg.role as string,
-      )
-    );
-  }
-
-  /**
-   * Convert UI parts to MessageContent
-   */
-  private static convertPartsToMessageContent(parts: UIPart[]): MessageContent {
-    if (parts.length === 0) {
-      return "";
-    }
-
-    const details: MessageContentDetail[] = [];
-
-    for (const part of parts) {
-      switch (part.type) {
-        case "text": {
-          details.push({
-            type: "text",
-            text: part.content || "",
-          });
-          break;
-        }
-        case "tool": {
-          const toolPart = part as unknown as UIPart;
-          details.push({
-            type: "text",
-            text: `Tool: ${part.data?.toolName}`,
-          });
-          break;
-        }
-        case "reasoning": {
-          const resultPart = part as unknown as UIPart;
-          details.push({
-            type: "text",
-            text: `Result: ${JSON.stringify(part.data?.result)}`,
-          });
-          break;
-        }
-        default:
-          // For other part types, convert to text
-          details.push({
-            type: "text",
-            text: JSON.stringify(part),
-          });
-          break;
-      }
-    }
-
-    // If only one text detail, return as string
-    if (details.length === 1 && details[0]?.type === "text") {
-      return details[0].text;
-    }
-
-    return details;
-  }
-
-  /**
-   * Convert MessageContent to UI parts
-   */
-  private static convertMessageContentToParts(
-    content: MessageContent,
-  ): UIPart[] {
-    if (typeof content === "string") {
-      return [
-        {
-          type: "text",
-          text: content,
-        } as UIPart,
-      ];
-    }
-
-    const parts: UIPart[] = [];
-
-    for (const detail of content) {
-      switch (detail.type) {
-        case "text":
-          parts.push({
-            type: "text",
-            text: detail.text,
-          } as UIPart);
-          break;
-        case "image_url":
-          // Convert image to text representation for UI
-          parts.push({
-            type: "text",
-            text: `[Image: ${detail.image_url.url}]`,
-          } as UIPart);
-          break;
-        case "audio":
-          parts.push({
-            type: "text",
-            text: `[Audio: ${detail.mimeType}]`,
-          } as UIPart);
-          break;
-        case "video":
-          parts.push({
-            type: "text",
-            text: `[Video: ${detail.mimeType}]`,
-          } as UIPart);
-          break;
-        case "image":
-          parts.push({
-            type: "text",
-            text: `[Image: ${detail.mimeType}]`,
-          } as UIPart);
-          break;
-        case "file":
-          parts.push({
-            type: "text",
-            text: `[File: ${detail.mimeType}]`,
-          } as UIPart);
-          break;
-        default:
-          // For unknown types, create a text representation
-          parts.push({
-            type: "text",
-            text: JSON.stringify(detail),
-          } as UIPart);
-      }
-    }
-
-    return parts;
-  }
-}
@@ -1,33 +0,0 @@
-import type { JSONValue } from "../global";
-import type { ChatMessage, MessageContent } from "../llms";
-import type { BaseMemoryBlock } from "./base";
-
-export type MemoryBlockContent = MessageContent | ChatMessage;
-
-export type MemoryOptions = {
-  blocks?: BaseMemoryBlock[];
-  tokenLimit?: number;
-};
-
-/**
- * Vercel AI SDK message types (avoid dependency)
- * These types mirror the Vercel AI SDK without requiring it as a dependency
- */
-export interface UIMessage {
-  id: string;
-  role: "system" | "user" | "assistant" | "data";
-  content: string;
-  createdAt?: Date;
-  annotations?: Array<JSONValue>;
-  parts: Array<UIPart>;
-}
-
-export interface UIPart {
-  type: "text" | "tool" | "reasoning" | "source" | "step";
-  content?: string;
-  data?: Record<string, unknown>; // TODO: Can expand this to be more specific later
-}
-
-export type GetMessageOptions = {
-  type?: "llamaindex" | "vercel";
-};
@@ -1,74 +0,0 @@
-import { type JSONValue } from "../global";
-import type { ChatMessage, MessageContent } from "../llms";
-
-/**
- * Serialize a message content to a JSON value.
- * @param message - The message content to serialize.
- * @returns The serialized message content.
- */
-export function serializeMessageContent(message: MessageContent): JSONValue {
-  if (typeof message === "string") {
-    return message;
-  }
-
-  // message is an array of MessageContentDetail
-  return message.map((detail) => {
-    const serialized: { [key: string]: JSONValue } = {
-      type: detail.type,
-    };
-
-    switch (detail.type) {
-      case "text":
-        serialized.text = detail.text;
-        break;
-      case "image_url":
-        serialized.image_url = {
-          url: detail.image_url.url,
-        };
-        if (detail.detail) {
-          (serialized.image_url as { [key: string]: JSONValue }).detail =
-            detail.detail;
-        }
-        break;
-      case "audio":
-        serialized.data = detail.data;
-        serialized.mimeType = detail.mimeType;
-        break;
-      case "video":
-        serialized.data = detail.data;
-        serialized.mimeType = detail.mimeType;
-        break;
-      case "image":
-        serialized.data = detail.data;
-        serialized.mimeType = detail.mimeType;
-        break;
-      case "file":
-        serialized.data = detail.data;
-        serialized.mimeType = detail.mimeType;
-        break;
-      default:
-        // For any unknown types, serialize all properties
-        Object.assign(serialized, detail);
-    }
-
-    return serialized;
-  });
-}
-
-/**
- * Serialize a chat message to a JSON value.
- * @param message - The chat message to serialize.
- * @returns The serialized chat message.
- */
-export function serializeChatMessage(message: ChatMessage): JSONValue {
-  const serialized: { [key: string]: JSONValue } = {
-    content: serializeMessageContent(message.content),
-    role: message.role,
-  };
-
-  if (message.options) {
-    serialized.options = message.options as JSONValue;
-  }
-
-  return serialized;
-}
@@ -0,0 +1,42 @@
+export const abbreviations = {
+  english: [
+    "i.e.",
+    "etc.",
+    "vs.",
+    "Inc.",
+    "A.S.A.P.",
+    "Mr.",
+    "Mrs.",
+    "Ms.",
+    "Dr.",
+    "Prof.",
+    "Sr.",
+    "Jr.",
+    "Tel.",
+    "a.m.",
+    "p.m.",
+    "Art.",
+  ],
+  spanish: [
+    "Sr.",
+    "Sres.",
+    "Srs.",
+    "Sra.",
+    "Sras.",
+    "Srta.",
+    "Srtas.",
+    "Dr.",
+    "Drs.",
+    "Dra.",
+    "Dras.",
+    "Prof.",
+    "Profs.",
+    "Profa.",
+    "Profas.",
+    "Ing.",
+    "Lic.",
+    "Arq.",
+    "Ab.",
+    "Abs.",
+  ],
+};
@@ -12,11 +12,7 @@ import {
  type TextSplitterFn,
 } from "./utils";

-type _Split = {
-  text: string;
-  isSentence: boolean;
-  tokenSize: number;
-};
+type _Split = [string, boolean, number]; // [text, isSentence, tokenSize]

 /**
 * Parse text with a preference for complete sentences.
@@ -42,6 +38,11 @@ export class SentenceSplitter extends MetadataAwareTextSplitter {
   * Backup regex for splitting into sentences.
   */
  secondaryChunkingRegex: string = "[^,.;。？！]+[,.;。？！]?";
+  /**
+   * Extra abbreviations to consider while splitting into sentences.
+   * For example, for contracts, you may want to consider "LLC." as an important abbreviation
+   */
+  extraAbbreviations: string[] | undefined = [];

  #chunkingTokenizerFn = splitBySentenceTokenizer();
  #splitFns: Set<TextSplitterFn> = new Set();
@@ -59,7 +60,11 @@ export class SentenceSplitter extends MetadataAwareTextSplitter {
      this.separator = parsedParams.separator;
      this.paragraphSeparator = parsedParams.paragraphSeparator;
      this.secondaryChunkingRegex = parsedParams.secondaryChunkingRegex;
+      this.extraAbbreviations = parsedParams.extraAbbreviations;
    }
+    this.#chunkingTokenizerFn = splitBySentenceTokenizer(
+      this.extraAbbreviations,
+    );
    this.#tokenizer = params?.tokenizer ?? Settings.tokenizer;
    this.#splitFns.add(splitBySep(this.paragraphSeparator));
    this.#splitFns.add(this.#chunkingTokenizerFn);
@@ -105,34 +110,26 @@ export class SentenceSplitter extends MetadataAwareTextSplitter {
    return chunks;
  }

-  #split(text: string, chunkSize: number): _Split[] {
-    const tokenSize = this.tokenSize(text);
-    if (tokenSize <= chunkSize) {
-      return [
-        {
-          text,
-          isSentence: true,
-          tokenSize,
-        },
-      ];
+  *#split(
+    text: string,
+    chunkSize: number,
+    tokenSize?: number,
+  ): Generator<_Split> {
+    const _tokenSize = tokenSize ?? this.tokenSize(text);
+    if (_tokenSize <= chunkSize) {
+      yield [text, true, _tokenSize];
+      return;
    }
    const [textSplitsByFns, isSentence] = this.#getSplitsByFns(text);
-    const textSplits: _Split[] = [];

    for (const textSplit of textSplitsByFns) {
-      const tokenSize = this.tokenSize(textSplit);
-      if (tokenSize <= chunkSize) {
-        textSplits.push({
-          text: textSplit,
-          isSentence,
-          tokenSize,
-        });
+      const textSplitTokenSize = this.tokenSize(textSplit);
+      if (textSplitTokenSize <= chunkSize) {
+        yield [textSplit, isSentence, textSplitTokenSize];
      } else {
-        const recursiveTextSplits = this.#split(textSplit, chunkSize);
-        textSplits.push(...recursiveTextSplits);
+        yield* this.#split(textSplit, chunkSize, textSplitTokenSize);
      }
    }
-    return textSplits;
  }

  #getSplitsByFns(text: string): [splits: string[], isSentence: boolean] {
@@ -151,7 +148,7 @@ export class SentenceSplitter extends MetadataAwareTextSplitter {
    return [[text], true];
  }

-  #merge(splits: _Split[], chunkSize: number): string[] {
+  #merge(splits: Iterable<_Split>, chunkSize: number): string[] {
    const chunks: string[] = [];
    let currentChunk: [string, number][] = [];
    let lastChunk: [string, number][] = [];
@@ -177,23 +174,26 @@ export class SentenceSplitter extends MetadataAwareTextSplitter {
      }
    };

-    while (splits.length > 0) {
-      const curSplit = splits[0]!;
-      if (curSplit.tokenSize > chunkSize) {
+    const splitsIterator = splits[Symbol.iterator]();
+    let currentSplitResult = splitsIterator.next();
+
+    while (!currentSplitResult.done) {
+      const [text, isSentence, tokenSize] = currentSplitResult.value;
+      if (tokenSize > chunkSize) {
        throw new Error("Single token exceeded chunk size");
      }
-      if (currentChunkLength + curSplit.tokenSize > chunkSize && !newChunk) {
+      if (currentChunkLength + tokenSize > chunkSize && !newChunk) {
        closeChunk();
      } else {
        if (
-          curSplit.isSentence ||
-          currentChunkLength + curSplit.tokenSize <= chunkSize ||
+          isSentence ||
+          currentChunkLength + tokenSize <= chunkSize ||
          newChunk
        ) {
-          currentChunkLength += curSplit.tokenSize;
-          currentChunk.push([curSplit.text, curSplit.tokenSize]);
-          splits.shift();
+          currentChunkLength += tokenSize;
+          currentChunk.push([text, tokenSize]);
          newChunk = false;
+          currentSplitResult = splitsIterator.next();
        } else {
          closeChunk();
        }
@@ -1,3 +1,4 @@
+import { abbreviations } from "./abbreviations";
 import type { TextSplitter } from "./base";
 import SentenceTokenizer from "./sentence_tokenizer";

@@ -34,53 +35,15 @@ export const splitByChar = (): TextSplitterFn => {
  return (text: string) => text.split("");
 };

-let sentenceTokenizer: SentenceTokenizer | null = null;
-
-export const splitBySentenceTokenizer = (): TextSplitterFn => {
-  if (!sentenceTokenizer) {
-    sentenceTokenizer = new SentenceTokenizer([
-      // TODO: This should be improved. Take a look at: https://github.com/run-llama/LlamaIndexTS/issues/2019
-      // English
-      "i.e.",
-      "etc.",
-      "vs.",
-      "Inc.",
-      "A.S.A.P.",
-      "Mr.",
-      "Mrs.",
-      "Ms.",
-      "Dr.",
-      "Prof.",
-      "Sr.",
-      "Jr.",
-      // Spanish
-      "Sr.",
-      "Sres.",
-      "Srs.",
-      "Sra.",
-      "Sras.",
-      "Srta.",
-      "Srtas.",
-      "Dr.",
-      "Drs.",
-      "Dra.",
-      "Dras.",
-      "Prof.",
-      "Profs.",
-      "Profa.",
-      "Profas.",
-      "Ing.",
-      "Lic.",
-      "Arq.",
-      "Ab.",
-      "Abs.",
-      "Tel.",
-      "a.m.",
-      "p.m.",
-      "Art.",
-    ]);
-  }
-  const tokenizer = sentenceTokenizer;
+export const splitBySentenceTokenizer = (
+  extraAbbreviations: string[] | undefined = [],
+): TextSplitterFn => {
+  const tokenizer = new SentenceTokenizer([
+    ...abbreviations.english,
+    ...abbreviations.spanish,
+    // Add the extra abbreviations provided by the user, e.g. for business-specific context
+    ...extraAbbreviations,
+  ]);
  return (text: string) => {
    try {
      return tokenizer.tokenize(text);
@@ -51,6 +51,13 @@ export const sentenceSplitterSchema = z
      })
      .optional()
      .default("[^,.;。？！]+[,.;。？！]?"),
+    extraAbbreviations: z
+      .array(z.string(), {
+        description:
+          "Extra abbreviations to consider while splitting into sentences. For example, for contracts, you may want to consider 'LLC.' as an important abbreviation",
+      })
+      .optional()
+      .default([]),
  })
  .refine(
    (data) => data.chunkOverlap < data.chunkSize,
@@ -0,0 +1,103 @@
+import { filetypemime } from "magic-bytes.js";
+
+/**
+ * Converts a base64 string (without data: prefix) to a Uint8Array
+ * @param base64 - The base64 string without data: prefix
+ * @returns The Uint8Array
+ */
+export function base64ToUint8Array(base64: string): Uint8Array {
+  // Decode Base64 string
+  const binaryString = atob(base64);
+
+  // Convert binary string to Uint8Array
+  const bytes = new Uint8Array(binaryString.length);
+  for (let i = 0; i < binaryString.length; i++) {
+    bytes[i] = binaryString.charCodeAt(i);
+  }
+
+  return bytes;
+}
+
+/**
+ * Converts a Uint8Array to a base64 string.
+ * @param uint8Array The Uint8Array to convert.
+ * @returns The base64-encoded string.
+ */
+export function uint8ArrayToBase64(uint8Array: Uint8Array): string {
+  let binary = "";
+  for (let i = 0; i < uint8Array.byteLength; i++) {
+    // Asserts that the value is not undefined, for `noUncheckedIndexedAccess`
+    binary += String.fromCharCode(uint8Array[i]!);
+  }
+  return btoa(binary);
+}
+
+/**
+ * Extracts the MIME type from a data URL.
+ * @param dataUrl The data URL string.
+ * @returns The MIME type from the data URL.
+ * @throws An error if the data URL is malformed.
+ */
+export function getMimeTypeFromDataUrl(dataUrl: string): string {
+  if (!dataUrl.startsWith("data:")) {
+    throw new Error("Not a data URL");
+  }
+  const commaIndex = dataUrl.indexOf(",");
+  if (commaIndex === -1) {
+    throw new Error("Invalid data URL format");
+  }
+
+  const header = dataUrl.slice(0, commaIndex);
+  const semicolonIndex = header.indexOf(";base64");
+  if (semicolonIndex === -1) {
+    throw new Error("Invalid data URL format: missing base64 encoding");
+  }
+
+  return header.slice(5, semicolonIndex);
+}
+
+/**
+ * Convert base64 data to Blob
+ * @param base64 - The base64 string
+ * @param mimeType - The MIME type of the file
+ * @returns The Blob
+ */
+export function base64ToBlob(base64: string, mimeType?: string): Blob {
+  let extractedMimeType = mimeType;
+  let base64Data = base64;
+
+  // Extract mimeType from data URL if not provided
+  if (!mimeType && base64.startsWith("data:")) {
+    extractedMimeType = getMimeTypeFromDataUrl(base64);
+    base64Data = base64.slice(base64.indexOf(",") + 1);
+  } else if (!mimeType) {
+    throw new Error(
+      "No MIME type provided and base64 is not in data URL format",
+    );
+  } else {
+    // Extract base64 data from data URL if present
+    const commaIndex = base64.indexOf(",");
+    base64Data = commaIndex !== -1 ? base64.slice(commaIndex + 1) : base64;
+  }
+
+  if (!extractedMimeType) {
+    throw new Error("No MIME type found in base64 data");
+  }
+
+  // convert base64 to Uint8Array
+  const bytes = base64ToUint8Array(base64Data);
+
+  // Create Blob
+  return new Blob([bytes], { type: extractedMimeType });
+}
+
+export async function blobToDataUrl(input: Blob) {
+  const arrayBuffer = await input.arrayBuffer();
+  const uint8Array = new Uint8Array(arrayBuffer);
+  const mimes = filetypemime(uint8Array);
+  if (mimes.length < 1) {
+    throw new Error("Unsupported image type");
+  }
+  const base64 = uint8ArrayToBase64(uint8Array);
+  return `data:${mimes[0]};base64,${base64}`;
+}
@@ -14,19 +14,6 @@ export const isIterable = (obj: unknown): obj is Iterable<unknown> => {
  return obj != null && typeof obj === "object" && Symbol.iterator in obj;
 };

-export async function* streamConverter<S, D>(
-  stream: AsyncIterable<S>,
-  converter: (s: S) => D | null,
-): AsyncIterable<D> {
-  for await (const data of stream) {
-    const newData = converter(data);
-    if (newData === null) {
-      return;
-    }
-    yield newData;
-  }
-}
-
 export async function* streamCallbacks<S>(
  stream: AsyncIterable<S>,
  callbacks: {
@@ -85,4 +72,6 @@ export {

 export { MockLLM } from "./mock";

+export * from "./encoding";
 export { objectEntries } from "./object-entries";
+export * from "./stream";
@@ -1,5 +1,4 @@
 import { fs } from "@llamaindex/env";
-import { filetypemime } from "magic-bytes.js";
 import type {
  ChatMessage,
  MessageContent,
@@ -9,6 +8,7 @@ import type {
 } from "../llms";
 import type { QueryType } from "../query-engine";
 import type { ImageType } from "../schema";
+import { blobToDataUrl } from "./encoding";

 /**
 * Extracts just the text whether from
@@ -110,15 +110,6 @@ export function toToolDescriptions(tools: ToolMetadata[]): string {
  return JSON.stringify(toolsObj, null, 4);
 }

-async function blobToDataUrl(input: Blob) {
-  const buffer = Buffer.from(await input.arrayBuffer());
-  const mimes = filetypemime(buffer);
-  if (mimes.length < 1) {
-    throw new Error("Unsupported image type");
-  }
-  return "data:" + mimes[0] + ";base64," + buffer.toString("base64");
-}
-
 export async function imageToDataUrl(
  input: ImageType | Uint8Array,
 ): Promise<string> {
@@ -0,0 +1,12 @@
+export async function* streamConverter<S, D>(
+  stream: AsyncIterable<S>,
+  converter: (s: S) => D | null,
+): AsyncIterable<D> {
+  for await (const data of stream) {
+    const newData = converter(data);
+    if (newData === null) {
+      return;
+    }
+    yield newData;
+  }
+}
@@ -0,0 +1,99 @@
+import {
+  base64ToBlob,
+  base64ToUint8Array,
+  blobToDataUrl,
+  getMimeTypeFromDataUrl,
+  uint8ArrayToBase64,
+} from "@llamaindex/core/utils";
+import { describe, expect, it } from "vitest";
+
+const testString = "LlamaIndex";
+const testBase64 = "TGxhbWFJbmRleA=="; // btoa('LlamaIndex')
+const testUint8Array = new TextEncoder().encode(testString);
+
+const pngB64 =
+  "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNkYAAAAAYAAjCB0C8AAAAASUVORK5CYII=";
+const pngMime = "image/png";
+const pngDataUrl = `data:${pngMime};base64,${pngB64}`;
+const pngBinaryString = atob(pngB64);
+const pngBytes = new Uint8Array(pngBinaryString.length);
+for (let i = 0; i < pngBinaryString.length; i++) {
+  pngBytes[i] = pngBinaryString.charCodeAt(i);
+}
+
+describe("Encoding utils", () => {
+  describe("base64ToUint8Array", () => {
+    it("should correctly convert a base64 string to a Uint8Array", () => {
+      const result = base64ToUint8Array(testBase64);
+      expect(result).toBeInstanceOf(Uint8Array);
+      expect(result).toEqual(testUint8Array);
+    });
+  });
+
+  describe("uint8ArrayToBase64", () => {
+    it("should correctly convert a Uint8Array to a base64 string", () => {
+      const result = uint8ArrayToBase64(testUint8Array);
+      expect(result).toBe(testBase64);
+    });
+  });
+
+  describe("getMimeTypeFromDataUrl", () => {
+    it("should extract the correct MIME type from a data URL", () => {
+      const result = getMimeTypeFromDataUrl(pngDataUrl);
+      expect(result).toBe(pngMime);
+    });
+
+    it("should throw an error for non-data URLs", () => {
+      expect(() => getMimeTypeFromDataUrl("not a data url")).toThrow(
+        "Not a data URL",
+      );
+    });
+
+    it("should throw an error for malformed data URLs", () => {
+      expect(() => getMimeTypeFromDataUrl("data:image/pngbase64,abc")).toThrow(
+        "Invalid data URL format: missing base64 encoding",
+      );
+      expect(() => getMimeTypeFromDataUrl("data:image/png;base64")).toThrow(
+        "Invalid data URL format",
+      );
+    });
+  });
+
+  describe("base64ToBlob", () => {
+    it("should convert from a data URL string", async () => {
+      const blob = base64ToBlob(pngDataUrl);
+      expect(blob).toBeInstanceOf(Blob);
+      expect(blob.type).toBe(pngMime);
+      const arrayBuffer = await blob.arrayBuffer();
+      expect(new Uint8Array(arrayBuffer)).toEqual(pngBytes);
+    });
+
+    it("should convert from a base64 string with an explicit MIME type", async () => {
+      const blob = base64ToBlob(pngB64, pngMime);
+      expect(blob).toBeInstanceOf(Blob);
+      expect(blob.type).toBe(pngMime);
+      const arrayBuffer = await blob.arrayBuffer();
+      expect(new Uint8Array(arrayBuffer)).toEqual(pngBytes);
+    });
+
+    it("should prioritize the explicit MIME type if a data URL is provided", async () => {
+      const differentMime = "image/jpeg";
+      const blob = base64ToBlob(pngDataUrl, differentMime);
+      expect(blob.type).toBe(differentMime);
+    });
+
+    it("should throw an error if no MIME type can be determined", () => {
+      expect(() => base64ToBlob(pngB64)).toThrow(
+        "No MIME type provided and base64 is not in data URL format",
+      );
+    });
+  });
+
+  describe("blobToDataUrl", () => {
+    it("should correctly convert a blob to a data URL", async () => {
+      const blob = new Blob([pngBytes], { type: "image/png" });
+      const result = await blobToDataUrl(blob);
+      expect(result).toBe(pngDataUrl);
+    });
+  });
+});
@@ -1,96 +0,0 @@
-import { Settings } from "@llamaindex/core/global";
-import type { ChatMessage } from "@llamaindex/core/llms";
-import { createMemory, staticMemoryBlock } from "@llamaindex/core/memory";
-import { beforeEach, describe, expect, test } from "vitest";
-
-describe("createMemory factory function", () => {
-  beforeEach(() => {
-    // Mock the Settings.llm
-    // eslint-disable-next-line @typescript-eslint/no-explicit-any
-    (Settings.llm as any) = {
-      metadata: {
-        contextWindow: 1000,
-        maxTokens: 100,
-      },
-    };
-  });
-
-  test("creates memory with default settings", () => {
-    const memory = createMemory();
-    expect(memory).toBeDefined();
-    expect(typeof memory.add).toBe("function");
-    expect(typeof memory.get).toBe("function");
-    expect(typeof memory.getLLM).toBe("function");
-    expect(typeof memory.clear).toBe("function");
-    expect(typeof memory.snapshot).toBe("function");
-    expect(typeof memory.loadSnapshot).toBe("function");
-  });
-
-  test("creates memory with custom token limit", () => {
-    const memory = createMemory({ tokenLimit: 5000 });
-    expect(memory).toBeDefined();
-  });
-
-  test("creates memory with memory blocks", () => {
-    const blocks = [staticMemoryBlock(["You are a helpful assistant."], 0)];
-    const memory = createMemory({ blocks });
-    expect(memory).toBeDefined();
-  });
-
-  test("supports basic operations", async () => {
-    const memory = createMemory({ tokenLimit: 1000 });
-
-    // Test add and get
-    const message: ChatMessage = { content: "Hello", role: "user" };
-    await memory.add(message);
-
-    const messages = await memory.get();
-    expect(messages).toHaveLength(1);
-    expect(messages[0]?.content).toBe("Hello");
-  });
-
-  // test("supports snapshot operations", async () => {
-  //   const memory = createMemory({ tokenLimit: 1000 });
-
-  //   await memory.add({ content: "Test message", role: "user" });
-
-  //   const snapshot = memory.snapshot();
-  //   expect(snapshot).toHaveProperty("blocks");
-  //   expect(snapshot).toHaveProperty("metadata");
-
-  //   const newMemory = createMemory();
-  //   await newMemory.loadSnapshot(snapshot);
-
-  //   const messages = await newMemory.get();
-  //   expect(messages).toHaveLength(1);
-  //   expect(messages[0]?.content).toBe("Test message");
-  // });
-
-  test("supports backward compatibility methods", async () => {
-    const memory = createMemory({ tokenLimit: 1000 });
-
-    // Test legacy methods exist and work
-    const message: ChatMessage = { content: "Hello", role: "user" };
-    memory.add(message);
-
-    const messages = await memory.get();
-    expect(messages).toHaveLength(1);
-    expect(messages[0]?.content).toBe("Hello");
-
-    await memory.clear();
-    const emptyMessages = await memory.get();
-    expect(emptyMessages).toHaveLength(0);
-  });
-
-  test("creates memory with different message format support", async () => {
-    const memory = createMemory({ tokenLimit: 1000 });
-
-    await memory.add({ content: "LlamaIndex message", role: "user" });
-
-    const llamaMessages = await memory.get({ type: "llamaindex" });
-    expect(llamaMessages).toHaveLength(1);
-
-    const uiMessages = await memory.get({ type: "vercel" });
-    expect(uiMessages).toHaveLength(1);
-  });
-});
@@ -1,238 +0,0 @@
-import { Settings } from "@llamaindex/core/global";
-import type { ChatMessage } from "@llamaindex/core/llms";
-import { beforeEach, describe, expect, test } from "vitest";
-
-import type { UIMessage, UIPart } from "@llamaindex/core/memory";
-import { Memory, staticMemoryBlock } from "@llamaindex/core/memory";
-
-describe("Memory", () => {
-  beforeEach(() => {
-    // Mock the Settings.llm
-    // eslint-disable-next-line @typescript-eslint/no-explicit-any
-    (Settings.llm as any) = {
-      metadata: {
-        contextWindow: 1000,
-        maxTokens: 100,
-      },
-    };
-  });
-
-  describe("constructor and initialization", () => {
-    test("creates with default options", () => {
-      const memory = new Memory();
-      expect(memory).toBeInstanceOf(Memory);
-    });
-
-    test("creates with custom token limit", () => {
-      const memory = new Memory({ tokenLimit: 5000 });
-      // Test through behavior rather than private property access
-      expect(memory).toBeInstanceOf(Memory);
-    });
-
-    test("creates with memory blocks", () => {
-      const blocks = [staticMemoryBlock([], 0)];
-      const memory = new Memory({ blocks });
-      // Test through behavior rather than private property access
-      expect(memory).toBeInstanceOf(Memory);
-    });
-  });
-
-  describe("new API methods", () => {
-    let memory: Memory;
-
-    beforeEach(() => {
-      memory = new Memory({ tokenLimit: 1000 });
-    });
-
-    describe("add method", () => {
-      test("adds ChatMessage successfully", async () => {
-        const message: ChatMessage = { content: "Hello", role: "user" };
-        await memory.add(message);
-
-        const messages = await memory.get();
-        expect(messages).toHaveLength(1);
-        expect(messages[0]?.content).toBe("Hello");
-      });
-
-      test("adds UIMessage successfully", async () => {
-        const uiMessage: UIMessage = {
-          id: "msg-1",
-          content: "Hello from UI",
-          role: "user",
-          parts: [{ type: "text", content: "Hello from UI" } as UIPart],
-          createdAt: new Date(),
-        };
-
-        await memory.add(uiMessage);
-        const messages = await memory.get();
-        expect(messages).toHaveLength(1);
-      });
-
-      test("adds multiple messages in sequence", async () => {
-        const messages = [
-          { content: "Hello", role: "user" as const },
-          { content: "Hi there!", role: "assistant" as const },
-          { content: "How are you?", role: "user" as const },
-        ];
-
-        for (const msg of messages) {
-          await memory.add(msg);
-        }
-
-        const storedMessages = await memory.get();
-        expect(storedMessages).toHaveLength(3);
-      });
-
-      test("throws error for invalid message format", async () => {
-        const invalidMessage = { content: "test" }; // missing role
-        await expect(
-          memory.add(invalidMessage as unknown as ChatMessage),
-        ).rejects.toThrow();
-      });
-    });
-
-    describe("get method", () => {
-      beforeEach(async () => {
-        await memory.add({ content: "Hello", role: "user" });
-        await memory.add({ content: "Hi there!", role: "assistant" });
-      });
-
-      test("returns messages in LlamaIndex format by default", async () => {
-        const messages = await memory.get();
-        expect(messages).toHaveLength(2);
-        expect(messages[0]).toHaveProperty("content");
-        expect(messages[0]).toHaveProperty("role");
-      });
-
-      test("returns messages in LlamaIndex format when specified", async () => {
-        const messages = await memory.get({ type: "llamaindex" });
-        expect(messages).toHaveLength(2);
-        expect(messages[0]).toHaveProperty("content");
-        expect(messages[0]).toHaveProperty("role");
-      });
-
-      test("returns messages in UI format when specified", async () => {
-        const messages = await memory.get({ type: "vercel" });
-        expect(messages).toHaveLength(2);
-        // UI messages should have id, parts, etc.
-        expect(messages[0]).toHaveProperty("id");
-        expect(messages[0]).toHaveProperty("parts");
-        expect(messages[0]).toHaveProperty("content");
-        expect(messages[0]).toHaveProperty("role");
-      });
-
-      test("handles empty memory", async () => {
-        const emptyMemory = new Memory();
-        const messages = await emptyMemory.get();
-        expect(messages).toHaveLength(0);
-      });
-    });
-
-    describe("getLLM method", () => {
-      test("returns token-limited messages", async () => {
-        const shortMemory = new Memory({ tokenLimit: 10 });
-
-        // Add messages that exceed token limit
-        await shortMemory.add({
-          content: "Very long message that exceeds token limit",
-          role: "user",
-        });
-        await shortMemory.add({
-          content: "Another long message",
-          role: "assistant",
-        });
-        await shortMemory.add({ content: "Short", role: "user" });
-
-        const llmMessages = await shortMemory.getLLM();
-        const allMessages = await shortMemory.get();
-
-        expect(llmMessages.length).toBeLessThanOrEqual(allMessages.length);
-      });
-
-      test("includes memory blocks in LLM messages", async () => {
-        const blocks = [staticMemoryBlock(["System instruction"], 0)];
-        const memoryWithBlocks = new Memory({
-          blocks,
-          tokenLimit: 1000,
-        });
-
-        await memoryWithBlocks.add({ content: "User message", role: "user" });
-
-        const llmMessages = await memoryWithBlocks.getLLM();
-        expect(llmMessages.length).toBeGreaterThan(1); // Should include block + user message
-      });
-
-      test("prioritizes recent messages when token limit is exceeded", async () => {
-        const memory = new Memory({ tokenLimit: 20 });
-
-        await memory.add({ content: "Old message", role: "user" });
-        await memory.add({ content: "Recent message", role: "user" });
-
-        const llmMessages = await memory.getLLM();
-        const lastMessage = llmMessages[llmMessages.length - 1];
-        expect(lastMessage?.content).toBe("Recent message");
-      });
-    });
-
-    describe("clear method", () => {
-      test("clears all messages", async () => {
-        await memory.add({ content: "Hello", role: "user" });
-        await memory.add({ content: "Hi there!", role: "assistant" });
-
-        let messages = await memory.get();
-        expect(messages).toHaveLength(2);
-
-        await memory.clear();
-        messages = await memory.get();
-        expect(messages).toHaveLength(0);
-      });
-    });
-  });
-
-  describe("memory blocks functionality", () => {
-    test("includes static blocks in LLM messages", async () => {
-      const blocks = [
-        staticMemoryBlock(["You are a helpful assistant."], 0),
-        staticMemoryBlock(["Always be polite."], 0),
-      ];
-
-      const memory = new Memory({ blocks, tokenLimit: 1000 });
-      await memory.add({ content: "Hello", role: "user" });
-
-      const llmMessages = await memory.getLLM();
-      expect(llmMessages.length).toBe(3);
-
-      // First messages should be from blocks (converted to system messages)
-      expect(llmMessages[0]?.content).toBe("You are a helpful assistant.");
-      expect(llmMessages[1]?.content).toBe("Always be polite.");
-      expect(llmMessages[2]?.content).toBe("Hello");
-    });
-  });
-
-  describe("error handling", () => {
-    let memory: Memory;
-
-    beforeEach(() => {
-      memory = new Memory({ tokenLimit: 1000 });
-    });
-
-    test("handles invalid message types gracefully", async () => {
-      const invalidMessage = { content: 123, role: "user" }; // invalid content type
-      await expect(
-        memory.add(invalidMessage as unknown as ChatMessage),
-      ).rejects.toThrow();
-    });
-
-    test("handles token limit edge cases", async () => {
-      const verySmallMemory = new Memory({ tokenLimit: 1 });
-      await verySmallMemory.add({
-        content: "This message is too long for the token limit",
-        role: "user",
-      });
-
-      const llmMessages = await verySmallMemory.getLLM();
-      // Should handle gracefully, possibly returning empty array
-      expect(Array.isArray(llmMessages)).toBe(true);
-    });
-  });
-});
@@ -1,99 +0,0 @@
-import type { ChatMessage } from "@llamaindex/core/llms";
-import type { UIMessage } from "@llamaindex/core/memory";
-import { MessageConverter } from "@llamaindex/core/memory";
-import { describe, expect, test } from "vitest";
-
-describe("MessageConverter", () => {
-  describe("toUIMessage", () => {
-    test("converts simple ChatMessage to UIMessage", () => {
-      const chatMessage: ChatMessage = {
-        content: "Hello, how are you?",
-        role: "user",
-      };
-
-      const uiMessage = MessageConverter.toUIMessage(chatMessage);
-
-      expect(uiMessage).toHaveProperty("id");
-      expect(uiMessage.role).toBe("user");
-      expect(uiMessage.content).toBe("Hello, how are you?");
-      expect(uiMessage).toHaveProperty("parts");
-      expect(uiMessage.parts).toHaveLength(1);
-    });
-
-    test("generates unique IDs for each conversion", () => {
-      const chatMessage: ChatMessage = {
-        content: "Test message",
-        role: "user",
-      };
-
-      const uiMessage1 = MessageConverter.toUIMessage(chatMessage);
-      const uiMessage2 = MessageConverter.toUIMessage(chatMessage);
-
-      expect(uiMessage1.id).not.toBe(uiMessage2.id);
-    });
-  });
-
-  describe("toLlamaIndexMessage", () => {
-    test("converts simple UIMessage to ChatMessage", () => {
-      const uiMessage: UIMessage = {
-        id: "msg-123",
-        content: "Hello, how are you?",
-        role: "user",
-        parts: [],
-      };
-
-      const chatMessage = MessageConverter.toLlamaIndexMessage(uiMessage);
-
-      expect(chatMessage.content).toBe("Hello, how are you?");
-      expect(chatMessage.role).toBe("user");
-    });
-  });
-
-  describe("type guards", () => {
-    test("isChatMessage identifies valid ChatMessage", () => {
-      const validChatMessage = {
-        content: "Hello",
-        role: "user",
-      };
-
-      expect(MessageConverter.isChatMessage(validChatMessage)).toBe(true);
-    });
-
-    test("isChatMessage rejects invalid objects", () => {
-      const invalidObjects = [
-        null,
-        undefined,
-        { content: "Hello" }, // missing role
-        { role: "user" }, // missing content
-      ];
-
-      for (const obj of invalidObjects) {
-        expect(MessageConverter.isChatMessage(obj)).toBe(false);
-      }
-    });
-
-    test("isUIMessage identifies valid UIMessage", () => {
-      const validUIMessage = {
-        id: "msg-123",
-        content: "Hello",
-        role: "user",
-        parts: [],
-      };
-
-      expect(MessageConverter.isUIMessage(validUIMessage)).toBe(true);
-    });
-
-    test("isUIMessage rejects invalid objects", () => {
-      const invalidObjects = [
-        null,
-        undefined,
-        { content: "Hello", role: "user" }, // missing id and parts
-        { id: "123", role: "user" }, // missing content and parts
-      ];
-
-      for (const obj of invalidObjects) {
-        expect(MessageConverter.isUIMessage(obj)).toBe(false);
-      }
-    });
-  });
-});
@@ -75,6 +75,22 @@ describe("sentence splitter", () => {
    expect(splits).toEqual(["This is a sentence. This is another sentence."]);
  });

+  test("overall split long text", () => {
+    const sentenceSplitter = new SentenceSplitter({
+      chunkSize: 10,
+      chunkOverlap: 0,
+    });
+    const splits = sentenceSplitter.splitText(
+      "The first short sentence. The first long long long sentence. The second short sentence. The second long long long sentence.",
+    );
+    expect(splits).toEqual([
+      "The first short sentence.",
+      "The first long long long sentence.",
+      "The second short sentence.",
+      "The second long long long sentence.",
+    ]);
+  });
+
  test("doesn't split decimals", () => {
    const sentenceSplitter = new SentenceSplitter({
      chunkSize: 5,
@@ -91,6 +107,39 @@ describe("sentence splitter", () => {
    ]);
  });

+  test("doesn't split basic abbreviations", () => {
+    const sentenceSplitter = new SentenceSplitter({
+      chunkSize: 15,
+      chunkOverlap: 0,
+    });
+    const splits = sentenceSplitter.splitText(
+      "This is a sentence of Broda Noel. This is the sentence of Sr. Broda Noel. This is a sentence of somebody else",
+    );
+
+    expect(splits).toEqual([
+      "This is a sentence of Broda Noel.",
+      "This is the sentence of Sr. Broda Noel.",
+      "This is a sentence of somebody else",
+    ]);
+  });
+
+  test("doesn't split extra abbreviations", () => {
+    const sentenceSplitter = new SentenceSplitter({
+      chunkSize: 10,
+      chunkOverlap: 0,
+      extraAbbreviations: ["S.A."],
+    });
+    const splits = sentenceSplitter.splitText(
+      "This is a sentence. The S.A. Broda Company. This is another sentence",
+    );
+
+    expect(splits).toEqual([
+      "This is a sentence.",
+      "The S.A. Broda Company.",
+      "This is another sentence",
+    ]);
+  });
+
  test("splits cjk", () => {
    const sentenceSplitter = new SentenceSplitter({
      chunkSize: 30,
@@ -1,5 +1,24 @@
 # @llamaindex/experimental

+## 0.0.188
+
+### Patch Changes
+
+- Updated dependencies [7039e1a]
+  - llamaindex@0.11.11
+
+## 0.0.187
+
+### Patch Changes
+
+- llamaindex@0.11.10
+
+## 0.0.186
+
+### Patch Changes
+
+- llamaindex@0.11.9
+
 ## 0.0.185

 ### Patch Changes
@@ -1,7 +1,7 @@
 {
  "name": "@llamaindex/experimental",
  "description": "Experimental package for LlamaIndexTS",
-  "version": "0.0.185",
+  "version": "0.0.188",
  "type": "module",
  "types": "dist/type/index.d.ts",
  "main": "dist/cjs/index.js",
@@ -1,5 +1,37 @@
 # llamaindex

+## 0.11.11
+
+### Patch Changes
+
+- 7039e1a: Internal cleanup of base64 encoding
+- Updated dependencies [7039e1a]
+- Updated dependencies [7039e1a]
+  - @llamaindex/core@0.6.12
+  - @llamaindex/cloud@4.0.16
+  - @llamaindex/node-parser@2.0.12
+  - @llamaindex/workflow@1.1.12
+
+## 0.11.10
+
+### Patch Changes
+
+- Updated dependencies [f7ec293]
+  - @llamaindex/workflow@1.1.11
+
+## 0.11.9
+
+### Patch Changes
+
+- Updated dependencies [a89e187]
+- Updated dependencies [62699b7]
+- Updated dependencies [c5b2691]
+- Updated dependencies [d8ac8d3]
+  - @llamaindex/core@0.6.11
+  - @llamaindex/cloud@4.0.15
+  - @llamaindex/node-parser@2.0.11
+  - @llamaindex/workflow@1.1.10
+
 ## 0.11.8

 ### Patch Changes
@@ -1,6 +1,6 @@
 {
  "name": "llamaindex",
-  "version": "0.11.8",
+  "version": "0.11.11",
  "license": "MIT",
  "type": "module",
  "keywords": [
@@ -18,11 +18,11 @@ import type {
 } from "@llamaindex/core/llms";
 import {
  extractText,
+  isAsyncIterable,
  stringifyJSONToMessageContent,
 } from "@llamaindex/core/utils";
 import { randomUUID } from "@llamaindex/env";
 import { getReACTAgentSystemHeader } from "../internal/prompt/react.js";
-import { isAsyncIterable } from "../internal/utils.js";
 import { Settings } from "../Settings.js";

 export type ReACTAgentParams = AgentParamsBase<LLM>;
@@ -63,7 +63,6 @@ export * from "./evaluation/index.js";
 export * from "./extractors/index.js";
 export * from "./indices/index.js";
 export * from "./ingestion/index.js";
-export { imageToDataUrl } from "./internal/utils.js";
 export * from "./node-parser.js";
 export * from "./objects/index.js";
 export * from "./OutputParser.js";
@@ -1,77 +0,0 @@
-import type { ImageType } from "@llamaindex/core/schema";
-import { fs } from "@llamaindex/env";
-import { filetypemime } from "magic-bytes.js";
-
-export const isAsyncIterable = (
-  obj: unknown,
-): obj is AsyncIterable<unknown> => {
-  return obj != null && typeof obj === "object" && Symbol.asyncIterator in obj;
-};
-
-export const isReadableStream = (obj: unknown): obj is ReadableStream => {
-  return obj instanceof ReadableStream;
-};
-
-export const isIterable = (obj: unknown): obj is Iterable<unknown> => {
-  return obj != null && typeof obj === "object" && Symbol.iterator in obj;
-};
-
-async function blobToDataUrl(input: Blob) {
-  const buffer = Buffer.from(await input.arrayBuffer());
-  const mimes = filetypemime(buffer);
-  if (mimes.length < 1) {
-    throw new Error("Unsupported image type");
-  }
-  return "data:" + mimes[0] + ";base64," + buffer.toString("base64");
-}
-
-export async function imageToString(input: ImageType): Promise<string> {
-  if (input instanceof Blob) {
-    // if the image is a Blob, convert it to a base64 data URL
-    return await blobToDataUrl(input);
-  } else if (typeof input === "string") {
-    return input;
-  } else if (input instanceof URL) {
-    return input.toString();
-  } else {
-    throw new Error(`Unsupported input type: ${typeof input}`);
-  }
-}
-
-export function stringToImage(input: string): ImageType {
-  if (input.startsWith("data:")) {
-    // if the input is a base64 data URL, convert it back to a Blob
-    const base64Data = input.split(",")[1]!;
-    const byteArray = Buffer.from(base64Data, "base64");
-    return new Blob([byteArray]);
-  } else if (input.startsWith("http://") || input.startsWith("https://")) {
-    return new URL(input);
-  } else {
-    return input;
-  }
-}
-
-export async function imageToDataUrl(
-  input: ImageType | Uint8Array,
-): Promise<string> {
-  // first ensure, that the input is a Blob
-  if (
-    (input instanceof URL && input.protocol === "file:") ||
-    typeof input === "string"
-  ) {
-    // string or file URL
-    const dataBuffer = await fs.readFile(
-      input instanceof URL ? input.pathname : input,
-    );
-    input = new Blob([dataBuffer]);
-  } else if (!(input instanceof Blob)) {
-    if (input instanceof URL) {
-      throw new Error(`Unsupported URL with protocol: ${input.protocol}`);
-    } else if (input instanceof Uint8Array) {
-      input = new Blob([input]); // convert Uint8Array to Blob
-    } else {
-      throw new Error(`Unsupported input type: ${typeof input}`);
-    }
-  }
-  return await blobToDataUrl(input);
-}
--- a/Show More
+++ b/Show More
Author	SHA1	Message	Date
github-actions[bot]	a50acf634c	Release 0.11.11 (#2044 ) Co-authored-by: marcusschiesser <17126+marcusschiesser@users.noreply.github.com>	2025-06-27 14:51:09 +07:00
Thuc Pham	7039e1a214	chore: migrate to @google/genai SDK (#2038 ) Co-authored-by: Marcus Schiesser <mail@marcusschiesser.de>	2025-06-27 12:09:26 +07:00
github-actions[bot]	785d010cd3	Release 0.11.10 (#2037 ) Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>	2025-06-26 14:29:33 +07:00
Marcus Schiesser	b878032131	fix release step	2025-06-26 14:18:56 +07:00
Marcus Schiesser	f7ec293a0f	chore: Update workflow-core (#2042 )	2025-06-26 14:03:03 +07:00
jerinthomascarmel	49a5e0a8cf	feat(readers): add ExcelReader for parsing Excel files (run-llama#1959) (#2033 ) Co-authored-by: Marcus Schiesser <mail@marcusschiesser.de> Co-authored-by: leehuwuj <leehuwuj@gmail.com>	2025-06-26 11:15:19 +07:00
Logan	118924799a	Rename llama-flow -> workflows in docs (#2040 )	2025-06-25 15:52:04 -07:00
allen	ec8f673dae	support filter to supabase vector search (#2036 )	2025-06-25 16:17:54 +07:00
github-actions[bot]	85039a5360	Release @llamaindex/tools@0.1.0 (#2034 )	2025-06-24 12:32:24 +07:00
Marcus Schiesser	d7305edb53	fix changesets	2025-06-24 12:26:09 +07:00
Huu Le	096bf2bda1	feat: Add support for StreamableHTTP MCP Client (#2032 )	2025-06-24 11:40:34 +07:00
jerinthomascarmel	c5846bd7dc	feat(readers): add XMLReader for parsing XML files (#1846 ) (#2031 ) Co-authored-by: Marcus Schiesser <marcus.schiesser@googlemail.com>	2025-06-24 10:46:32 +07:00
github-actions[bot]	97bbce6e13	Release 0.11.9 (#2023 ) Co-authored-by: marcusschiesser <17126+marcusschiesser@users.noreply.github.com>	2025-06-20 12:28:01 +07:00
Marcus Schiesser	62699b7497	chore: improve performance of sentence splitter (#2030 )	2025-06-20 12:16:24 +07:00
Broda Noel	a89e187796	Add `extraAbbreviations` on sentence-splitter (#2029 ) Co-authored-by: Marcus Schiesser <mail@marcusschiesser.de>	2025-06-20 11:27:06 +07:00
ANKIT VARSHNEY	d8ac8d385d	feat: add openai realtime api (#2006 ) Co-authored-by: Marcus Schiesser <mail@marcusschiesser.de>	2025-06-20 10:22:04 +07:00
				`@@ -0,0 +1 @@`
				<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" class="iconify iconify--logos" width="31.88" height="32" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 257"><defs><linearGradient id="IconifyId1813088fe1fbc01fb466" x1="-.828%" x2="57.636%" y1="7.652%" y2="78.411%"><stop offset="0%" stop-color="#41D1FF"></stop><stop offset="100%" stop-color="#BD34FE"></stop></linearGradient><linearGradient id="IconifyId1813088fe1fbc01fb467" x1="43.376%" x2="50.316%" y1="2.242%" y2="89.03%"><stop offset="0%" stop-color="#FFEA83"></stop><stop offset="8.333%" stop-color="#FFDD35"></stop><stop offset="100%" stop-color="#FFA800"></stop></linearGradient></defs><path fill="url(#IconifyId1813088fe1fbc01fb466)" d="M255.153 37.938L134.897 252.976c-2.483 4.44-8.862 4.466-11.382.048L.875 37.958c-2.746-4.814 1.371-10.646 6.827-9.67l120.385 21.517a6.537 6.537 0 0 0 2.322-.004l117.867-21.483c5.438-.991 9.574 4.796 6.877 9.62Z"></path><path fill="url(#IconifyId1813088fe1fbc01fb467)" d="M185.432.063L96.44 17.501a3.268 3.268 0 0 0-2.634 3.014l-5.474 92.456a3.268 3.268 0 0 0 3.997 3.378l24.777-5.718c2.318-.535 4.413 1.507 3.936 3.838l-7.361 36.047c-.495 2.426 1.782 4.5 4.151 3.78l15.304-4.649c2.372-.72 4.652 1.36 4.15 3.788l-11.698 56.621c-.732 3.542 3.979 5.473 5.943 2.437l1.313-2.028l72.516-144.72c1.215-2.423-.88-5.186-3.54-4.672l-25.505 4.922c-2.396.462-4.435-1.77-3.759-4.114l16.646-57.705c.677-2.35-1.37-4.583-3.769-4.113Z"></path></svg>