Release @llamaindex/tools@0.1.0 (#2034 )

fix changesets
feat: Add support for StreamableHTTP MCP Client (#2032 )
2026-07-01 22:14:03 -04:00 · 2025-06-24 12:32:24 +07:00 · 2025-06-24 12:26:09 +07:00 · 2025-06-24 11:40:34 +07:00 · 2025-06-24 10:46:32 +07:00 · 2025-06-20 12:28:01 +07:00
171 changed files with 3625 additions and 884 deletions
@@ -1,5 +1,28 @@
 # @llamaindex/doc

+## 0.2.29
+
+### Patch Changes
+
+- Updated dependencies [c5846bd]
+  - @llamaindex/readers@3.1.10
+
+## 0.2.28
+
+### Patch Changes
+
+- Updated dependencies [a89e187]
+- Updated dependencies [62699b7]
+- Updated dependencies [c5b2691]
+- Updated dependencies [d8ac8d3]
+  - @llamaindex/core@0.6.11
+  - @llamaindex/openai@0.4.5
+  - @llamaindex/cloud@4.0.15
+  - llamaindex@0.11.9
+  - @llamaindex/node-parser@2.0.11
+  - @llamaindex/readers@3.1.9
+  - @llamaindex/workflow@1.1.10
+
 ## 0.2.27

 ### Patch Changes
@@ -1,6 +1,6 @@
 {
  "name": "@llamaindex/doc",
-  "version": "0.2.27",
+  "version": "0.2.29",
  "private": true,
  "scripts": {
    "postinstall": "fumadocs-mdx",
@@ -74,12 +74,21 @@ const server = mcp({
  args: ["-y", "@modelcontextprotocol/server-filesystem", "."],
  verbose: true,
 });
-// or by SSE
+// or by StreamableHTTP transport
 const server = mcp({
  url: "http://localhost:8000/mcp",
  verbose: true,
 });

+// if your MCP server is not using StreamableHTTP transport, you can also use SSE transport
+// by setting useSSETransport to true.
+// See: https://modelcontextprotocol.io/docs/concepts/transports#server-sent-events-sse-deprecated
+const server = mcp({
+  url: "http://localhost:8000/mcp",
+  useSSETransport: true,
+  verbose: true,
+});
+
 // 3. Get tools from MCP server
 const tools = await server.tools();

@@ -378,3 +378,186 @@ async function main() {
 ## API Reference

 - [OpenAI](/docs/api/classes/OpenAI)
+
+
+# OpenAI Live LLM
+
+The OpenAI Live LLM integration in LlamaIndex provides real-time chat capabilities with support for audio streaming and tool calling.
+
+## Basic Usage
+
+```typescript
+import { openai } from "@llamaindex/openai";
+import { tool, ModalityType } from "llamaindex";
+
+// Get the ephimeral key on the server 
+const serverllm = openai({
+  apiKey: "your-api-key", 
+  model: "gpt-4o-realtime-preview-2025-06-03",
+});
+
+// Get an ephemeral key 
+// Usually this code is run on the server and the ephemeral key is passed to the
+// client - the ephemeral key can be securely used on the client side
+const ephemeralKey = await serverllm.live.getEphemeralKey();
+
+// Create a client-side LLM instance with the ephemeral key
+const llm = openai({
+  apiKey: ephemeralKey,
+  model: "gpt-4o-realtime-preview-2025-06-03"
+});
+
+// Create a live sessionimport { tool } from "llamaindex";
+const session = await llm.live.connect({
+  systemInstruction: "You are a helpful assistant.",
+});
+
+// Send a message
+session.sendMessage({
+  content: "Hello!",
+  role: "user",
+});
+```
+
+## Tool Integration
+
+Tools are handled server-side, making it simple to pass them to the live session:
+
+```typescript
+// Define your tools
+const weatherTool = tool({
+  name: "weather",
+  description: "Get the weather for a location",
+  parameters: z.object({
+    location: z.string().describe("The location to get weather for"),
+  }),
+  execute: async ({ location }) => {
+    return `The weather in ${location} is sunny`;
+  },
+});
+
+// Create session with tools
+const session = await llm.live.connect({
+  systemInstruction: "You are a helpful assistant.",
+  tools: [weatherTool],
+});
+```
+
+## Audio Support
+
+For audio capabilities:
+
+```typescript
+// Get microphone access
+const userStream = await navigator.mediaDevices.getUserMedia({
+  audio: true,
+});
+
+// Create session with audio
+const session = await llm.live.connect({
+  audioConfig: {
+    stream: userStream,
+    onTrack: (remoteStream) => {
+      // Handle incoming audio
+      audioElement.srcObject = remoteStream;
+    },
+  },
+});
+```
+
+## Event Handling
+
+Listen to events from the session:
+
+```typescript
+for await (const event of session.streamEvents()) {
+  if (liveEvents.open.include(event)) {
+    // Connection established
+    console.log("Connected!");
+  } else if (liveEvents.text.include(event)) {
+    // Received text response
+    console.log("Assistant:", event.text);
+  }
+}
+```
+
+## Capabilities
+
+The OpenAI Live LLM supports:
+
+- Real-time text chat
+- Audio streaming (if configured)
+- Tool calling (server-side execution)
+- Ephemeral key generation for secure sessions
+
+## API Reference
+
+### LiveLLM Methods
+// Get an ephemeral key 
+// Usually this code is run on the server and the ephemeral key is passed to the
+// client - the ephemeral key can be securely used on the client side
+
+#### `connect(config?: LiveConnectConfig)`
+
+Creates a new live session.
+
+```typescript
+interface LiveConnectConfig {
+  systemInstruction?: string;
+  tools?: BaseTool[];
+  audioConfig?: AudioConfig;
+  responseModality?: ModalityType[];
+}
+```
+
+#### `getEphemeralKey()`
+
+Gets a temporary key for the session.
+
+### LiveLLMSession Methods
+
+#### `sendMessage(message: ChatMessage)`
+
+Sends a message to the assistant.
+
+```typescript
+interface ChatMessage {
+  content: string | MessageContentDetail[];
+  role: "user" | "assistant";
+}
+```
+
+#### `disconnect()`
+
+Closes the session and cleans up resources.
+
+## Error Handling
+
+```typescript
+try {
+  const session = await llm.live.connect();
+} catch (error) {
+  if (error instanceof Error) {
+    console.error("Connection failed:", error.message);
+  }
+}
+```
+
+## Best Practices
+
+1. **Tool Definition**
+
+   - Keep tool implementations server-side
+   - Use clear descriptions for tools
+   - Handle tool errors gracefully
+
+2. **Session Management**
+
+   - Always disconnect sessions when done
+   - Clean up audio resources
+   - Handle reconnection scenarios
+
+3. **Security**
+   - Use ephemeral keys for sessions
+   - Validate tool inputs
+   - Secure API key handling
@@ -1,5 +1,11 @@
 # @llamaindex/cloudflare-worker-agent-test

+## 0.0.170
+
+### Patch Changes
+
+- llamaindex@0.11.9
+
 ## 0.0.169

 ### Patch Changes
@@ -1,6 +1,6 @@
 {
  "name": "@llamaindex/cloudflare-worker-agent-test",
-  "version": "0.0.169",
+  "version": "0.0.170",
  "type": "module",
  "private": true,
  "scripts": {
@@ -1,5 +1,11 @@
 # @llamaindex/llama-parse-browser-test

+## 0.0.70
+
+### Patch Changes
+
+- @llamaindex/cloud@4.0.15
+
 ## 0.0.69

 ### Patch Changes
@@ -1,7 +1,7 @@
 {
  "name": "@llamaindex/llama-parse-browser-test",
  "private": true,
-  "version": "0.0.69",
+  "version": "0.0.70",
  "type": "module",
  "scripts": {
    "dev": "vite",
@@ -1,5 +1,11 @@
 # @llamaindex/next-agent-test

+## 0.1.170
+
+### Patch Changes
+
+- llamaindex@0.11.9
+
 ## 0.1.169

 ### Patch Changes
@@ -1,6 +1,6 @@
 {
  "name": "@llamaindex/next-agent-test",
-  "version": "0.1.169",
+  "version": "0.1.170",
  "private": true,
  "scripts": {
    "dev": "next dev",
@@ -1,5 +1,11 @@
 # test-edge-runtime

+## 0.1.169
+
+### Patch Changes
+
+- llamaindex@0.11.9
+
 ## 0.1.168

 ### Patch Changes
@@ -1,6 +1,6 @@
 {
  "name": "@llamaindex/nextjs-edge-runtime-test",
-  "version": "0.1.168",
+  "version": "0.1.169",
  "private": true,
  "scripts": {
    "dev": "next dev",
@@ -1,5 +1,20 @@
 # @llamaindex/next-node-runtime

+## 0.1.38
+
+### Patch Changes
+
+- Updated dependencies [c5846bd]
+  - @llamaindex/readers@3.1.10
+
+## 0.1.37
+
+### Patch Changes
+
+- llamaindex@0.11.9
+- @llamaindex/huggingface@0.1.15
+- @llamaindex/readers@3.1.9
+
 ## 0.1.36

 ### Patch Changes
@@ -1,6 +1,6 @@
 {
  "name": "@llamaindex/next-node-runtime-test",
-  "version": "0.1.36",
+  "version": "0.1.38",
  "private": true,
  "scripts": {
    "dev": "next dev",
@@ -1,5 +1,11 @@
 # vite-import-llamaindex

+## 0.0.36
+
+### Patch Changes
+
+- llamaindex@0.11.9
+
 ## 0.0.35

 ### Patch Changes
@@ -1,7 +1,7 @@
 {
  "name": "vite-import-llamaindex",
  "private": true,
-  "version": "0.0.35",
+  "version": "0.0.36",
  "type": "module",
  "scripts": {
    "build": "vite build",
@@ -1 +1,9 @@
-{"root":["./src/main.ts","./vite.config.ts"],"version":"5.7.3"}
+{
+  "root": [
+    "./src/main.ts",
+    "./vite.config.ts",
+    "./tsconfig.json"
+  ],
+  "errors": true,
+  "version": "5.7.3"
+}
@@ -1,5 +1,11 @@
 # @llamaindex/waku-query-engine-test

+## 0.0.170
+
+### Patch Changes
+
+- llamaindex@0.11.9
+
 ## 0.0.169

 ### Patch Changes
@@ -1,6 +1,6 @@
 {
  "name": "@llamaindex/waku-query-engine-test",
-  "version": "0.0.169",
+  "version": "0.0.170",
  "type": "module",
  "private": true,
  "scripts": {
@@ -1,5 +1,68 @@
 # examples

+## 0.3.24
+
+### Patch Changes
+
+- Updated dependencies [096bf2b]
+- Updated dependencies [c5846bd]
+  - @llamaindex/tools@0.1.0
+  - @llamaindex/readers@3.1.10
+
+## 0.3.23
+
+### Patch Changes
+
+- Updated dependencies [a89e187]
+- Updated dependencies [62699b7]
+- Updated dependencies [c5b2691]
+- Updated dependencies [d8ac8d3]
+  - @llamaindex/core@0.6.11
+  - @llamaindex/google@0.3.10
+  - @llamaindex/openai@0.4.5
+  - @llamaindex/cloud@4.0.15
+  - llamaindex@0.11.9
+  - @llamaindex/node-parser@2.0.11
+  - @llamaindex/anthropic@0.3.13
+  - @llamaindex/assemblyai@0.1.10
+  - @llamaindex/clip@0.0.61
+  - @llamaindex/cohere@0.0.25
+  - @llamaindex/deepinfra@0.0.61
+  - @llamaindex/discord@0.1.10
+  - @llamaindex/huggingface@0.1.15
+  - @llamaindex/jinaai@0.0.21
+  - @llamaindex/mistral@0.1.11
+  - @llamaindex/mixedbread@0.0.25
+  - @llamaindex/notion@0.1.10
+  - @llamaindex/ollama@0.1.11
+  - @llamaindex/perplexity@0.0.18
+  - @llamaindex/portkey-ai@0.0.53
+  - @llamaindex/replicate@0.0.53
+  - @llamaindex/astra@0.0.25
+  - @llamaindex/azure@0.1.22
+  - @llamaindex/chroma@0.0.25
+  - @llamaindex/elastic-search@0.1.11
+  - @llamaindex/firestore@1.0.18
+  - @llamaindex/milvus@0.1.20
+  - @llamaindex/mongodb@0.0.26
+  - @llamaindex/pinecone@0.1.11
+  - @llamaindex/postgres@0.0.54
+  - @llamaindex/qdrant@0.1.21
+  - @llamaindex/supabase@0.1.10
+  - @llamaindex/upstash@0.0.25
+  - @llamaindex/weaviate@0.0.26
+  - @llamaindex/vercel@0.1.11
+  - @llamaindex/voyage-ai@1.0.17
+  - @llamaindex/readers@3.1.9
+  - @llamaindex/tools@0.0.17
+  - @llamaindex/workflow@1.1.10
+  - @llamaindex/deepseek@0.0.21
+  - @llamaindex/fireworks@0.0.21
+  - @llamaindex/groq@0.0.76
+  - @llamaindex/together@0.0.21
+  - @llamaindex/vllm@0.0.47
+  - @llamaindex/xai@0.0.8
+
 ## 0.3.22

 ### Patch Changes
@@ -1,4 +1,3 @@
-import { tool } from "@llamaindex/core/tools";
 import { openai } from "@llamaindex/openai";
 import {
  agent,
@@ -7,6 +6,7 @@ import {
  multiAgent,
 } from "@llamaindex/workflow";
 import fs from "fs";
+import { tool } from "llamaindex";
 import os from "os";
 import { z } from "zod";

@@ -6,15 +6,24 @@ async function main() {
  // Create an MCP server for filesystem tools
  const server = mcp({
    command: "npx",
-    args: ["-y", "@modelcontextprotocol/server-filesystem", "."],
+    args: ["-y", "@modelcontextprotocol/server-filesystem@latest", "."],
    verbose: true,
  });
-  // You can also connect to the MCP server using SSE
-  // See: https://modelcontextprotocol.io/docs/concepts/transports#server-sent-events-sse
+  //
+  // You can also connect to a remote MCP server using:
+  // 1. StreamableHTTP transport (recommended)
+  // See: https://modelcontextprotocol.io/docs/concepts/transports#streamable-http
  // const server = mcp({
  //   url: "http://localhost:8000/mcp",
  //   verbose: true,
  // });
+  // 2.Or using SSE transport (will be deprecated soon)
+  // See: https://modelcontextprotocol.io/docs/concepts/transports#server-sent-events-sse-deprecated
+  // const server = mcp({
+  //   url: "http://localhost:8000/mcp",
+  //   useSSETransport: true,
+  //   verbose: true,
+  // });

  try {
    // Create an agent that uses the MCP tools
@@ -1,6 +1,6 @@
-import { Settings } from "@llamaindex/core/global";
 import { openai } from "@llamaindex/openai";
 import { agentHandler, createWorkflow, zodEvent } from "@llamaindex/workflow";
+import { Settings } from "llamaindex";
 import { z } from "zod";

 // Create LLM instance
@@ -0,0 +1,72 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<company name="MidSizeCorp" founded="2008">
+  <division name="Engineering" head="Dana White">
+    <department name="Frontend" lead="Alex Kim">
+      <team name="Web">
+        <employee id="E01">
+          <name>Jordan Lee</name>
+          <role>Lead Developer</role>
+          <projects>
+            <project code="PRJ101" status="active">
+              <title>User Portal</title>
+              <deadline>2025-08-01</deadline>
+              <tasks>
+                <task id="T1011">
+                  <description>Implement login page</description>
+                  <due>2025-05-10</due>
+                </task>
+                <task id="T1012">
+                  <description>Design dashboard</description>
+                  <due>2025-05-20</due>
+                </task>
+              </tasks>
+            </project>
+          </projects>
+        </employee>
+        <employee id="E02">
+          <name>Riley Chen</name>
+          <role>UI Designer</role>
+        </employee>
+      </team>
+      <team name="Mobile">
+        <employee id="E03">
+          <name>Sam Patel</name>
+          <role>iOS Developer</role>
+        </employee>
+      </team>
+    </department>
+    <department name="Backend" lead="Morgan Reed">
+      <team name="API">
+        <employee id="E04">
+          <name>Taylor Jones</name>
+          <role>API Engineer</role>
+        </employee>
+      </team>
+      <team name="Database">
+        <employee id="E05">
+          <name>Casey Nguyen</name>
+          <role>DB Administrator</role>
+        </employee>
+      </team>
+    </department>
+  </division>
+
+  <division name="Marketing" head="Pat Morgan">
+    <department name="Digital" lead="Alex Rivera">
+      <team name="Content">
+        <employee id="M01">
+          <name>Charlie Brooks</name>
+          <role>Content Strategist</role>
+        </employee>
+      </team>
+    </department>
+  </division>
+
+  <headquarters location="Chicago, USA">
+    <address>
+      <street>789 Lake Shore Drive</street>
+      <city>Chicago</city>
+      <zip>60601</zip>
+    </address>
+  </headquarters>
+</company>
@@ -23,10 +23,7 @@ async function main() {
        content: "Say something about you for 10 seconds",
        role: "user",
      });
-    } else if (
-      liveEvents.audio.include(event) &&
-      typeof event.data === "string"
-    ) {
+    } else if (liveEvents.audio.include(event)) {
      const chunk = Buffer.from(event.data, "base64");
      audioChunks.push(chunk);
      console.log(`Received audio chunk: ${chunk.length} bytes`);
@@ -1,6 +1,5 @@
-import { ModalityType } from "@llamaindex/core/schema";
-import { tool } from "@llamaindex/core/tools";
 import { gemini, GEMINI_MODEL } from "@llamaindex/google";
+import { ModalityType, tool } from "llamaindex";

 import { liveEvents } from "llamaindex";
 import { z } from "zod";
@@ -0,0 +1,24 @@
+# Logs
+logs
+*.log
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+pnpm-debug.log*
+lerna-debug.log*
+
+node_modules
+dist
+dist-ssr
+*.local
+
+# Editor directories and files
+.vscode/*
+!.vscode/extensions.json
+.idea
+.DS_Store
+*.suo
+*.ntvs*
+*.njsproj
+*.sln
+*.sw?
@@ -0,0 +1,54 @@
+# OpenAI Realtime Chat with LlamaIndex
+
+This is a demo application showcasing real-time audio and text chat capabilities using OpenAI's GPT-4 with voice through LlamaIndex. The application demonstrates bidirectional audio communication and text chat with an AI assistant.
+
+## Features
+
+- Real-time voice communication with GPT-4
+- Text-based chat interface
+- WebRTC-based audio streaming
+- Bidirectional communication (both text and voice)
+- React + TypeScript implementation
+
+## Prerequisites
+
+- Node.js (v18 or higher)
+- OpenAI API key with access to GPT-4 voice models
+- Modern browser with WebRTC support
+
+## Getting Started
+
+1. Install dependencies:
+
+```bash
+pnpm install
+```
+
+2. Start the development server:
+
+```bash
+pnpm run dev
+```
+
+## Usage
+
+The application provides a simple interface where you can:
+
+- Start/Stop a chat session
+- Speak to the AI assistant through your microphone
+- Receive audio responses from the assistant
+- See text transcripts of the conversation
+
+## Technical Details
+
+This project uses:
+
+- LlamaIndex for AI interaction management
+- WebRTC for real-time audio streaming
+- React for the UI
+- Vite for development and building
+- TypeScript for type safety
+
+```
+
+```
@@ -0,0 +1,28 @@
+import js from "@eslint/js";
+import reactHooks from "eslint-plugin-react-hooks";
+import reactRefresh from "eslint-plugin-react-refresh";
+import globals from "globals";
+import tseslint from "typescript-eslint";
+
+export default tseslint.config(
+  { ignores: ["dist"] },
+  {
+    extends: [js.configs.recommended, ...tseslint.configs.recommended],
+    files: ["**/*.{ts,tsx}"],
+    languageOptions: {
+      ecmaVersion: 2020,
+      globals: globals.browser,
+    },
+    plugins: {
+      "react-hooks": reactHooks,
+      "react-refresh": reactRefresh,
+    },
+    rules: {
+      ...reactHooks.configs.recommended.rules,
+      "react-refresh/only-export-components": [
+        "warn",
+        { allowConstantExport: true },
+      ],
+    },
+  },
+);
@@ -0,0 +1,13 @@
+<!doctype html>
+<html lang="en">
+  <head>
+    <meta charset="UTF-8" />
+    <link rel="icon" type="image/svg+xml" href="/vite.svg" />
+    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+    <title>Vite + React + TS</title>
+  </head>
+  <body>
+    <div id="root"></div>
+    <script type="module" src="/src/main.tsx"></script>
+  </body>
+</html>
@@ -0,0 +1,29 @@
+{
+  "name": "open-ai-realtime",
+  "private": true,
+  "version": "0.0.0",
+  "type": "module",
+  "scripts": {
+    "dev": "vite",
+    "build": "tsc -b && vite build",
+    "lint": "eslint .",
+    "preview": "vite preview"
+  },
+  "dependencies": {
+    "react": "^19.1.0",
+    "react-dom": "^19.1.0"
+  },
+  "devDependencies": {
+    "@eslint/js": "^9.25.0",
+    "@types/react": "^19.1.2",
+    "@types/react-dom": "^19.1.2",
+    "@vitejs/plugin-react": "^4.5.2",
+    "eslint": "^9.25.0",
+    "eslint-plugin-react-hooks": "^5.2.0",
+    "eslint-plugin-react-refresh": "^0.4.19",
+    "globals": "^16.0.0",
+    "typescript": "~5.8.3",
+    "typescript-eslint": "^8.30.1",
+    "vite": "^6.3.5"
+  }
+}
@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" class="iconify iconify--logos" width="31.88" height="32" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 257"><defs><linearGradient id="IconifyId1813088fe1fbc01fb466" x1="-.828%" x2="57.636%" y1="7.652%" y2="78.411%"><stop offset="0%" stop-color="#41D1FF"></stop><stop offset="100%" stop-color="#BD34FE"></stop></linearGradient><linearGradient id="IconifyId1813088fe1fbc01fb467" x1="43.376%" x2="50.316%" y1="2.242%" y2="89.03%"><stop offset="0%" stop-color="#FFEA83"></stop><stop offset="8.333%" stop-color="#FFDD35"></stop><stop offset="100%" stop-color="#FFA800"></stop></linearGradient></defs><path fill="url(#IconifyId1813088fe1fbc01fb466)" d="M255.153 37.938L134.897 252.976c-2.483 4.44-8.862 4.466-11.382.048L.875 37.958c-2.746-4.814 1.371-10.646 6.827-9.67l120.385 21.517a6.537 6.537 0 0 0 2.322-.004l117.867-21.483c5.438-.991 9.574 4.796 6.877 9.62Z"></path><path fill="url(#IconifyId1813088fe1fbc01fb467)" d="M185.432.063L96.44 17.501a3.268 3.268 0 0 0-2.634 3.014l-5.474 92.456a3.268 3.268 0 0 0 3.997 3.378l24.777-5.718c2.318-.535 4.413 1.507 3.936 3.838l-7.361 36.047c-.495 2.426 1.782 4.5 4.151 3.78l15.304-4.649c2.372-.72 4.652 1.36 4.15 3.788l-11.698 56.621c-.732 3.542 3.979 5.473 5.943 2.437l1.313-2.028l72.516-144.72c1.215-2.423-.88-5.186-3.54-4.672l-25.505 4.922c-2.396.462-4.435-1.77-3.759-4.114l16.646-57.705c.677-2.35-1.37-4.583-3.769-4.113Z"></path></svg>
@@ -0,0 +1,183 @@
+import { openai } from "@llamaindex/openai";
+import { liveEvents, LiveLLMSession, ModalityType } from "llamaindex";
+import { useEffect, useRef, useState } from "react";
+
+const MicIcon = ({ isConnected }: { isConnected: boolean }) => (
+  <svg
+    xmlns="http://www.w3.org/2000/svg"
+    viewBox="0 0 24 24"
+    fill="currentColor"
+    strokeWidth="2"
+    strokeLinecap="round"
+    strokeLinejoin="round"
+  >
+    {isConnected ? (
+      <>
+        <path d="M12 1a3 3 0 0 0-3 3v8a3 3 0 0 0 6 0V4a3 3 0 0 0-3-3z" />
+        <path d="M19 10v2a7 7 0 0 1-14 0v-2" />
+        <line x1="12" y1="19" x2="12" y2="23" />
+        <line x1="8" y1="23" x2="16" y2="23" />
+      </>
+    ) : (
+      <>
+        <path d="M12 1a3 3 0 0 0-3 3v8a3 3 0 0 0 6 0V4a3 3 0 0 0-3-3z" />
+        <path d="M19 10v2a7 7 0 0 1-14 0v-2" />
+      </>
+    )}
+  </svg>
+);
+
+const WaveAnimation = () => (
+  <div className="wave-animation">
+    {[...Array(3)].map((_, i) => (
+      <div key={i} className="wave" style={{ animationDelay: `${i * 0.2}s` }} />
+    ))}
+  </div>
+);
+
+export const AudioChat = () => {
+  const [isConnected, setIsConnected] = useState(false);
+  const [messages, setMessages] = useState<
+    Array<{ role: string; content: string }>
+  >([]);
+  const [status, setStatus] = useState<string>("");
+  const audioRef = useRef<HTMLAudioElement>(null);
+  const sessionRef = useRef<LiveLLMSession | null>(null);
+  const [stream, setStream] = useState<MediaStream | null>(null);
+  const messagesEndRef = useRef<HTMLDivElement>(null);
+
+  const scrollToBottom = () => {
+    messagesEndRef.current?.scrollIntoView({ behavior: "smooth" });
+  };
+
+  useEffect(() => {
+    scrollToBottom();
+  }, [messages]);
+
+  useEffect(() => {
+    return () => {
+      if (stream) {
+        stream.getTracks().forEach((track) => track.stop());
+      }
+    };
+  }, [stream]);
+
+  const startChat = async () => {
+    try {
+      setStatus("Initializing microphone...");
+      const userStream = await navigator.mediaDevices.getUserMedia({
+        audio: true,
+      });
+      setStream(userStream);
+
+      setStatus("Connecting to AI...");
+      const apiKey = prompt("Please enter your OpenAI API key:");
+      if (!apiKey) {
+        throw new Error("API key is required");
+      }
+
+      // move this call to the server side for security reasons
+      // Do not store the API key in the frontend!
+      const serverllm = openai({
+        apiKey: apiKey,
+        model: "gpt-4o-realtime-preview-2025-06-03",
+      });
+
+      const tempKey = await serverllm.live.getEphemeralKey();
+
+      const llm = openai({
+        apiKey: tempKey,
+        model: "gpt-4o-realtime-preview-2025-06-03",
+      });
+      const session = await llm.live.connect({
+        systemInstruction: "You are a helpful assistant who speaks naturally.",
+        responseModality: [ModalityType.TEXT, ModalityType.AUDIO],
+        audioConfig: {
+          stream: userStream,
+          onTrack: (remoteStream) => {
+            if (audioRef.current && remoteStream) {
+              audioRef.current.srcObject = remoteStream;
+              audioRef.current.play().catch(console.error);
+            }
+          },
+        },
+      });
+
+      sessionRef.current = session;
+      setIsConnected(true);
+      setStatus("Connected! Listening...");
+
+      for await (const event of session.streamEvents()) {
+        if (liveEvents.open.include(event)) {
+          setMessages((prev) => [
+            ...prev,
+            {
+              role: "user",
+              content: "Hello, I'm ready to chat!",
+            },
+          ]);
+          session.sendMessage({
+            content: "Hello, I'm ready to chat!",
+            role: "user",
+          });
+        } else if (liveEvents.text.include(event)) {
+          setMessages((prev) => [
+            ...prev,
+            {
+              role: "assistant",
+              content: event.text,
+            },
+          ]);
+        }
+      }
+    } catch (error) {
+      console.error("Error starting chat:", error);
+      setStatus("Error connecting. Please try again.");
+      setIsConnected(false);
+    }
+  };
+
+  const stopChat = async () => {
+    setStatus("Disconnecting...");
+    if (sessionRef.current) {
+      await sessionRef.current.disconnect();
+      sessionRef.current = null;
+    }
+    if (stream) {
+      stream.getTracks().forEach((track) => track.stop());
+      setStream(null);
+    }
+    if (audioRef.current) {
+      audioRef.current.srcObject = null;
+    }
+    setIsConnected(false);
+    setStatus("");
+  };
+
+  return (
+    <div className="audio-chat-container">
+      <h1>AI Voice Chat</h1>
+      <div className="messages-container">
+        {messages.map((msg, idx) => (
+          <div key={idx} className={`message ${msg.role}`}>
+            {msg.content}
+          </div>
+        ))}
+        <div ref={messagesEndRef} />
+      </div>
+
+      <div className="controls">
+        {status && <div className="status-indicator">{status}</div>}
+        <button
+          className={`mic-button ${isConnected ? "connected" : ""}`}
+          onClick={isConnected ? stopChat : startChat}
+          title={isConnected ? "Stop Chat" : "Start Chat"}
+        >
+          <MicIcon isConnected={isConnected} />
+          {isConnected && <WaveAnimation />}
+        </button>
+        <audio ref={audioRef} style={{ display: "none" }} />
+      </div>
+    </div>
+  );
+};
@@ -0,0 +1,322 @@
+:root {
+  --primary-color: #646cff;
+  --secondary-color: #535bf2;
+  --background-dark: #1a1a1a;
+  --chat-bg: #242424;
+  --text-primary: #ffffff;
+  --text-secondary: #888888;
+  --success-color: #4caf50;
+  --error-color: #f44336;
+  --gradient-start: #4776e6;
+  --gradient-end: #8e54e9;
+}
+
+body {
+  background-color: var(--background-dark);
+  color: var(--text-primary);
+  margin: 0;
+  min-height: 100vh;
+  font-family:
+    -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, Oxygen, Ubuntu,
+    Cantarell, "Open Sans", "Helvetica Neue", sans-serif;
+}
+
+#root {
+  max-width: 1280px;
+  height: 100vh;
+  margin: 0 auto;
+  padding: 2rem;
+  display: flex;
+  align-items: center;
+  justify-content: center;
+}
+
+.logo {
+  height: 6em;
+  padding: 1.5em;
+  will-change: filter;
+  transition: filter 300ms;
+}
+.logo:hover {
+  filter: drop-shadow(0 0 2em #646cffaa);
+}
+.logo.react:hover {
+  filter: drop-shadow(0 0 2em #61dafbaa);
+}
+
+@keyframes logo-spin {
+  from {
+    transform: rotate(0deg);
+  }
+  to {
+    transform: rotate(360deg);
+  }
+}
+
+@media (prefers-reduced-motion: no-preference) {
+  a:nth-of-type(2) .logo {
+    animation: logo-spin infinite 20s linear;
+  }
+}
+
+.card {
+  padding: 2em;
+}
+
+.read-the-docs {
+  color: #888;
+}
+
+.audio-chat-container {
+  display: flex;
+  flex-direction: column;
+  gap: 2rem;
+  width: 100%;
+  max-width: 800px;
+  height: 80vh;
+  margin: 0 auto;
+  padding: 2rem;
+  background: var(--chat-bg);
+  border-radius: 24px;
+  box-shadow: 0 8px 32px rgba(0, 0, 0, 0.2);
+  position: relative;
+  overflow: hidden;
+}
+
+.audio-chat-container::before {
+  content: "";
+  position: absolute;
+  top: 0;
+  left: 0;
+  right: 0;
+  height: 4px;
+  background: linear-gradient(
+    to right,
+    var(--gradient-start),
+    var(--gradient-end)
+  );
+}
+
+.audio-chat-container h1 {
+  font-size: 2.5rem;
+  margin: 0;
+  background: linear-gradient(
+    to right,
+    var(--gradient-start),
+    var(--gradient-end)
+  );
+  -webkit-background-clip: text;
+  background-clip: text;
+  color: transparent;
+  text-align: center;
+}
+
+.messages-container {
+  display: flex;
+  flex-direction: column;
+  gap: 1rem;
+  flex: 1;
+  overflow-y: auto;
+  padding: 1rem;
+  border-radius: 16px;
+  background: rgba(255, 255, 255, 0.05);
+  backdrop-filter: blur(10px);
+  margin: 1rem 0;
+}
+
+.message {
+  padding: 1rem 1.5rem;
+  border-radius: 16px;
+  max-width: 80%;
+  text-align: left;
+  animation: messageSlide 0.3s ease-out;
+  box-shadow: 0 2px 4px rgba(0, 0, 0, 0.1);
+}
+
+@keyframes messageSlide {
+  from {
+    opacity: 0;
+    transform: translateY(20px);
+  }
+  to {
+    opacity: 1;
+    transform: translateY(0);
+  }
+}
+
+.message.user {
+  background: linear-gradient(
+    135deg,
+    var(--gradient-start),
+    var(--gradient-end)
+  );
+  align-self: flex-end;
+  margin-left: 20%;
+  color: white;
+}
+
+.message.assistant {
+  background: rgba(255, 255, 255, 0.1);
+  align-self: flex-start;
+  margin-right: 20%;
+  border: 1px solid rgba(255, 255, 255, 0.1);
+}
+
+.controls {
+  display: flex;
+  justify-content: center;
+  align-items: center;
+  padding: 2rem;
+  position: relative;
+}
+
+.mic-button {
+  width: 80px;
+  height: 80px;
+  border-radius: 50%;
+  border: none;
+  background: linear-gradient(
+    135deg,
+    var(--gradient-start),
+    var(--gradient-end)
+  );
+  color: white;
+  cursor: pointer;
+  transition: all 0.3s cubic-bezier(0.4, 0, 0.2, 1);
+  display: flex;
+  align-items: center;
+  justify-content: center;
+  box-shadow: 0 4px 15px rgba(0, 0, 0, 0.2);
+  position: relative;
+  overflow: hidden;
+}
+
+.mic-button::before {
+  content: "";
+  position: absolute;
+  top: 0;
+  left: 0;
+  right: 0;
+  bottom: 0;
+  background: linear-gradient(
+    135deg,
+    rgba(255, 255, 255, 0.1),
+    rgba(255, 255, 255, 0)
+  );
+  border-radius: 50%;
+  transition: transform 0.3s ease;
+}
+
+.mic-button:hover {
+  transform: scale(1.05);
+  box-shadow: 0 8px 25px rgba(0, 0, 0, 0.3);
+}
+
+.mic-button:hover::before {
+  transform: translateY(-100%);
+}
+
+.mic-button.connected {
+  background: var(--error-color);
+  animation: pulseError 2s infinite;
+}
+
+.mic-button svg {
+  width: 32px;
+  height: 32px;
+  filter: drop-shadow(0 2px 4px rgba(0, 0, 0, 0.2));
+  transition: transform 0.3s ease;
+}
+
+.mic-button:hover svg {
+  transform: scale(1.1);
+}
+
+@keyframes pulseError {
+  0% {
+    box-shadow: 0 0 0 0 rgba(244, 67, 54, 0.4);
+  }
+  70% {
+    box-shadow: 0 0 0 20px rgba(244, 67, 54, 0);
+  }
+  100% {
+    box-shadow: 0 0 0 0 rgba(244, 67, 54, 0);
+  }
+}
+
+/* Status indicator */
+.status-indicator {
+  position: absolute;
+  top: -30px;
+  left: 50%;
+  transform: translateX(-50%);
+  font-size: 0.9rem;
+  color: var(--text-secondary);
+  opacity: 0;
+  transition: opacity 0.3s ease;
+}
+
+.controls:hover .status-indicator {
+  opacity: 1;
+}
+
+/* Scrollbar styling */
+.messages-container::-webkit-scrollbar {
+  width: 8px;
+}
+
+.messages-container::-webkit-scrollbar-track {
+  background: rgba(255, 255, 255, 0.05);
+  border-radius: 4px;
+}
+
+.messages-container::-webkit-scrollbar-thumb {
+  background: linear-gradient(var(--gradient-start), var(--gradient-end));
+  border-radius: 4px;
+}
+
+.messages-container::-webkit-scrollbar-thumb:hover {
+  background: linear-gradient(var(--gradient-end), var(--gradient-start));
+}
+
+/* Wave Animation */
+.wave-animation {
+  position: absolute;
+  bottom: -15px;
+  left: 50%;
+  transform: translateX(-50%);
+  display: flex;
+  gap: 4px;
+}
+
+.wave {
+  width: 4px;
+  height: 15px;
+  background: currentColor;
+  border-radius: 2px;
+  animation: wave 0.5s ease-in-out infinite;
+}
+
+@keyframes wave {
+  0%,
+  100% {
+    transform: scaleY(0.5);
+  }
+  50% {
+    transform: scaleY(1.5);
+  }
+}
+
+/* Loading state */
+.mic-button.loading {
+  animation: rotate 1s linear infinite;
+}
+
+@keyframes rotate {
+  from {
+    transform: rotate(0deg);
+  }
+  to {
+    transform: rotate(360deg);
+  }
+}
@@ -0,0 +1,10 @@
+import { StrictMode } from "react";
+import { createRoot } from "react-dom/client";
+import { AudioChat } from "./audio-chat.tsx";
+import "./index.css";
+
+createRoot(document.getElementById("root")!).render(
+  <StrictMode>
+    <AudioChat />
+  </StrictMode>,
+);
@@ -0,0 +1 @@
+/// <reference types="vite/client" />
@@ -0,0 +1,26 @@
+{
+  "compilerOptions": {
+    "tsBuildInfoFile": "./node_modules/.tmp/tsconfig.app.tsbuildinfo",
+    "target": "ES2020",
+    "useDefineForClassFields": true,
+    "lib": ["ES2020", "DOM", "DOM.Iterable"],
+    "module": "ESNext",
+    "skipLibCheck": true,
+
+    /* Bundler mode */
+    "moduleResolution": "bundler",
+    "allowImportingTsExtensions": true,
+    "verbatimModuleSyntax": true,
+    "moduleDetection": "force",
+    "noEmit": true,
+    "jsx": "react-jsx",
+
+    /* Linting */
+    "strict": true,
+    "noUnusedLocals": true,
+    "noUnusedParameters": true,
+    "noFallthroughCasesInSwitch": true,
+    "noUncheckedSideEffectImports": true
+  },
+  "include": ["src"]
+}
@@ -0,0 +1,4 @@
+{
+  "files": [],
+  "references": [{ "path": "./tsconfig.app.json" }]
+}
@@ -0,0 +1,7 @@
+import react from "@vitejs/plugin-react";
+import { defineConfig } from "vite";
+
+// https://vite.dev/config/
+export default defineConfig({
+  plugins: [react()],
+});
@@ -1,6 +1,6 @@
 {
  "name": "@llamaindex/examples",
-  "version": "0.3.22",
+  "version": "0.3.24",
  "private": true,
  "scripts": {
    "lint": "eslint .",
@@ -11,51 +11,51 @@
    "@azure/cosmos": "^4.1.1",
    "@azure/identity": "^4.4.1",
    "@azure/search-documents": "^12.1.0",
-    "@llamaindex/anthropic": "^0.3.12",
-    "@llamaindex/assemblyai": "^0.1.9",
-    "@llamaindex/astra": "^0.0.24",
-    "@llamaindex/azure": "^0.1.21",
-    "@llamaindex/chroma": "^0.0.24",
-    "@llamaindex/clip": "^0.0.60",
-    "@llamaindex/cloud": "^4.0.14",
-    "@llamaindex/cohere": "^0.0.24",
-    "@llamaindex/core": "^0.6.10",
-    "@llamaindex/deepinfra": "^0.0.60",
-    "@llamaindex/deepseek": "^0.0.20",
-    "@llamaindex/discord": "^0.1.9",
-    "@llamaindex/elastic-search": "^0.1.10",
+    "@llamaindex/anthropic": "^0.3.13",
+    "@llamaindex/assemblyai": "^0.1.10",
+    "@llamaindex/astra": "^0.0.25",
+    "@llamaindex/azure": "^0.1.22",
+    "@llamaindex/chroma": "^0.0.25",
+    "@llamaindex/clip": "^0.0.61",
+    "@llamaindex/cloud": "^4.0.15",
+    "@llamaindex/cohere": "^0.0.25",
+    "@llamaindex/core": "^0.6.11",
+    "@llamaindex/deepinfra": "^0.0.61",
+    "@llamaindex/deepseek": "^0.0.21",
+    "@llamaindex/discord": "^0.1.10",
+    "@llamaindex/elastic-search": "^0.1.11",
    "@llamaindex/env": "^0.1.30",
-    "@llamaindex/firestore": "^1.0.17",
-    "@llamaindex/fireworks": "^0.0.20",
-    "@llamaindex/google": "^0.3.9",
-    "@llamaindex/groq": "^0.0.75",
-    "@llamaindex/huggingface": "^0.1.14",
-    "@llamaindex/jinaai": "^0.0.20",
-    "@llamaindex/milvus": "^0.1.19",
-    "@llamaindex/mistral": "^0.1.10",
-    "@llamaindex/mixedbread": "^0.0.24",
-    "@llamaindex/mongodb": "^0.0.25",
-    "@llamaindex/node-parser": "^2.0.10",
-    "@llamaindex/notion": "^0.1.9",
-    "@llamaindex/ollama": "^0.1.10",
-    "@llamaindex/openai": "^0.4.4",
-    "@llamaindex/perplexity": "^0.0.17",
-    "@llamaindex/pinecone": "^0.1.10",
-    "@llamaindex/portkey-ai": "^0.0.52",
-    "@llamaindex/postgres": "^0.0.53",
-    "@llamaindex/qdrant": "^0.1.20",
-    "@llamaindex/readers": "^3.1.8",
-    "@llamaindex/replicate": "^0.0.52",
-    "@llamaindex/supabase": "^0.1.9",
-    "@llamaindex/together": "^0.0.20",
-    "@llamaindex/tools": "^0.0.16",
-    "@llamaindex/upstash": "^0.0.24",
-    "@llamaindex/vercel": "^0.1.10",
-    "@llamaindex/vllm": "^0.0.46",
-    "@llamaindex/voyage-ai": "^1.0.16",
-    "@llamaindex/weaviate": "^0.0.25",
-    "@llamaindex/workflow": "^1.1.9",
-    "@llamaindex/xai": "workspace:^0.0.7",
+    "@llamaindex/firestore": "^1.0.18",
+    "@llamaindex/fireworks": "^0.0.21",
+    "@llamaindex/google": "^0.3.10",
+    "@llamaindex/groq": "^0.0.76",
+    "@llamaindex/huggingface": "^0.1.15",
+    "@llamaindex/jinaai": "^0.0.21",
+    "@llamaindex/milvus": "^0.1.20",
+    "@llamaindex/mistral": "^0.1.11",
+    "@llamaindex/mixedbread": "^0.0.25",
+    "@llamaindex/mongodb": "^0.0.26",
+    "@llamaindex/node-parser": "^2.0.11",
+    "@llamaindex/notion": "^0.1.10",
+    "@llamaindex/ollama": "^0.1.11",
+    "@llamaindex/openai": "^0.4.5",
+    "@llamaindex/perplexity": "^0.0.18",
+    "@llamaindex/pinecone": "^0.1.11",
+    "@llamaindex/portkey-ai": "^0.0.53",
+    "@llamaindex/postgres": "^0.0.54",
+    "@llamaindex/qdrant": "^0.1.21",
+    "@llamaindex/readers": "^3.1.10",
+    "@llamaindex/replicate": "^0.0.53",
+    "@llamaindex/supabase": "^0.1.10",
+    "@llamaindex/together": "^0.0.21",
+    "@llamaindex/tools": "^0.1.0",
+    "@llamaindex/upstash": "^0.0.25",
+    "@llamaindex/vercel": "^0.1.11",
+    "@llamaindex/vllm": "^0.0.47",
+    "@llamaindex/voyage-ai": "^1.0.17",
+    "@llamaindex/weaviate": "^0.0.26",
+    "@llamaindex/workflow": "^1.1.10",
+    "@llamaindex/xai": "workspace:^0.0.8",
    "@notionhq/client": "^2.2.15",
    "@pinecone-database/pinecone": "^4.0.0",
    "@vercel/postgres": "^0.10.0",
@@ -64,7 +64,7 @@
    "commander": "^12.1.0",
    "dotenv": "^16.4.5",
    "js-tiktoken": "^1.0.14",
-    "llamaindex": "^0.11.8",
+    "llamaindex": "^0.11.9",
    "mongodb": "6.7.0",
    "postgres": "^3.4.4",
    "wikipedia": "^2.1.2",
@@ -1,3 +1,4 @@
+import { OpenAIEmbedding } from "@llamaindex/openai";
 import {
  Document,
  SentenceSplitter,
@@ -7,6 +8,10 @@ import {
 import { OldSentenceSplitter } from "./old-sentence-splitter";
 export const STORAGE_DIR = "./data";

+Settings.embedModel = new OpenAIEmbedding({
+  model: "text-embedding-3-small",
+});
+
 // Update node parser
 (async () => {
  // generate a document with a very long sentence (9000 words long)
@@ -15,7 +15,8 @@
    "start:llamaparse-json": "node --import tsx ./src/llamaparse-json.ts",
    "start:discord": "node --import tsx ./src/discord.ts",
    "start:json": "node --import tsx ./src/json.ts",
-    "start:obsidian": "node --import tsx ./src/obsidian.ts"
+    "start:obsidian": "node --import tsx ./src/obsidian.ts",
+    "start:xml": "node --import tsx ./src/xml.ts"
  },
  "dependencies": {
    "@llamaindex/cloud": "workspace:* || ^2.0.24",
@@ -0,0 +1,16 @@
+import { XMLReader } from "@llamaindex/readers/xml";
+
+async function main() {
+  // Load PDF
+  const reader = new XMLReader({
+    splitLevel: 2,
+  });
+  const documents = await reader.loadData("../data/company.xml");
+
+  for (const doc of documents) {
+    console.log(doc.text);
+    console.log("----");
+  }
+}
+
+main().catch(console.error);
@@ -18,5 +18,11 @@
      "module": "commonjs"
    }
  },
-  "include": ["./**/*.ts"]
+  "include": ["./**/*.ts"],
+  "exclude": [
+    "node_modules",
+    "dist",
+    "models/openai/live/browser/open-ai-realtime",
+    "**/browser/**"
+  ]
 }
@@ -1,5 +1,11 @@
 # @llamaindex/autotool

+## 8.0.9
+
+### Patch Changes
+
+- llamaindex@0.11.9
+
 ## 8.0.8

 ### Patch Changes
@@ -1,5 +1,12 @@
 # @llamaindex/autotool-01-node-example

+## 0.0.117
+
+### Patch Changes
+
+- llamaindex@0.11.9
+- @llamaindex/autotool@8.0.9
+
 ## 0.0.116

 ### Patch Changes
@@ -13,5 +13,5 @@
  "scripts": {
    "start": "node --import tsx --import @llamaindex/autotool/node ./src/index.ts"
  },
-  "version": "0.0.116"
+  "version": "0.0.117"
 }
@@ -6,7 +6,7 @@
    "url": "git+https://github.com/run-llama/LlamaIndexTS.git",
    "directory": "packages/autotool"
  },
-  "version": "8.0.8",
+  "version": "8.0.9",
  "description": "auto transpile your JS function to LLM Agent compatible",
  "files": [
    "dist",
@@ -1,5 +1,15 @@
 # @llamaindex/cloud

+## 4.0.15
+
+### Patch Changes
+
+- Updated dependencies [a89e187]
+- Updated dependencies [62699b7]
+- Updated dependencies [c5b2691]
+- Updated dependencies [d8ac8d3]
+  - @llamaindex/core@0.6.11
+
 ## 4.0.14

 ### Patch Changes
@@ -1,6 +1,6 @@
 {
  "name": "@llamaindex/cloud",
-  "version": "4.0.14",
+  "version": "4.0.15",
  "type": "module",
  "license": "MIT",
  "scripts": {
@@ -1,5 +1,14 @@
 # @llamaindex/core

+## 0.6.11
+
+### Patch Changes
+
+- a89e187: Feat: added custom abbreviations to sentence splitter
+- 62699b7: Improve performance of sentence splitter
+- c5b2691: Add more Acronyms on SentenceSplitter
+- d8ac8d3: Feat: add support for openai realtime API
+
 ## 0.6.10

 ### Patch Changes
@@ -1,7 +1,7 @@
 {
  "name": "@llamaindex/core",
  "type": "module",
-  "version": "0.6.10",
+  "version": "0.6.11",
  "description": "LlamaIndex Core Module",
  "exports": {
    "./agent": {
@@ -1,4 +1,5 @@
-import { extractText, streamConverter } from "../utils";
+import { extractText } from "../utils/llms";
+import { streamConverter } from "../utils/stream";
 import type {
  ChatResponse,
  ChatResponseChunk,
@@ -1,6 +1,9 @@
 export { BaseLLM, ToolCallLLM } from "./base";
-export { LiveLLM, LiveLLMSession, liveEvents, type LiveEvent } from "./live";
+export { LiveLLM, LiveLLMCapability, LiveLLMSession } from "./live/live";
+export { liveEvents, type LiveEvent } from "./live/live-types";
+export type { MessageSender } from "./live/sender";
 export type {
+  AudioConfig,
  BaseTool,
  BaseToolWithCall,
  ChatMessage,
@@ -1,9 +1,7 @@
 import type {
-  ChatMessage,
-  LiveConnectConfig,
  MessageContentAudioDetail,
  MessageContentTextDetail,
-} from "./type";
+} from "../type";

 export type OpenEvent = { type: "open" };

@@ -63,45 +61,3 @@ export const liveEvents = {
      e.type === "turnComplete",
  },
 };
-export abstract class LiveLLMSession {
-  protected eventQueue: LiveEvent[] = [];
-  protected eventResolvers: ((value: LiveEvent) => void)[] = [];
-  protected closed = false;
-  abstract sendMessage(message: ChatMessage): void;
-  async *streamEvents(): AsyncIterable<LiveEvent> {
-    while (true) {
-      const event = await this.nextEvent();
-      if (event === undefined) {
-        break;
-      }
-      yield event;
-    }
-  }
-  abstract disconnect(): Promise<void>;
-
-  protected async nextEvent(): Promise<LiveEvent | undefined> {
-    if (this.eventQueue.length) {
-      return Promise.resolve(this.eventQueue.shift());
-    }
-
-    return new Promise((resolve) => {
-      this.eventResolvers.push(resolve);
-    });
-  }
-
-  //Uses an async queue to send events to the client
-  // if the consumer is waiting for an event, it will be resolved immediately
-  // otherwise, the event will be queued up and sent when the consumer is ready
-  pushEventToQueue(event: LiveEvent) {
-    if (this.eventResolvers.length) {
-      //resolving the promise with the event
-      this.eventResolvers.shift()!(event);
-    } else {
-      this.eventQueue.push(event);
-    }
-  }
-}
-
-export abstract class LiveLLM {
-  abstract connect(config?: LiveConnectConfig): Promise<LiveLLMSession>;
-}
@@ -0,0 +1,124 @@
+import type {
+  ChatMessage,
+  LiveConnectConfig,
+  MessageContentAudioDetail,
+  MessageContentDetail,
+  MessageContentImageDataDetail,
+  MessageContentVideoDetail,
+} from "../type";
+import type { LiveEvent } from "./live-types";
+import type { MessageSender } from "./sender";
+
+export enum LiveLLMCapability {
+  EPHEMERAL_KEY = "ephemeral_key",
+  AUDIO_CONFIG = "audio_config",
+}
+
+export abstract class LiveLLMSession {
+  protected eventQueue: LiveEvent[] = [];
+  protected eventResolvers: ((value: LiveEvent) => void)[] = [];
+  closed = false;
+  abstract get messageSender(): MessageSender;
+
+  private isTextMessage(content: MessageContentDetail) {
+    return content.type === "text";
+  }
+
+  private isAudioMessage(
+    content: MessageContentDetail,
+  ): content is MessageContentAudioDetail {
+    return content.type === "audio";
+  }
+
+  private isImageMessage(
+    content: MessageContentDetail,
+  ): content is MessageContentImageDataDetail {
+    return content.type === "image";
+  }
+
+  private isVideoMessage(
+    content: MessageContentDetail,
+  ): content is MessageContentVideoDetail {
+    return content.type === "video";
+  }
+
+  sendMessage(message: ChatMessage) {
+    const { content, role } = message;
+    if (!Array.isArray(content)) {
+      this.messageSender.sendTextMessage(content, role);
+    } else {
+      for (const item of content) {
+        this.processMessage(item, role);
+      }
+    }
+  }
+
+  private processMessage(message: MessageContentDetail, role?: string) {
+    if (this.isTextMessage(message)) {
+      this.messageSender.sendTextMessage(message.text, role);
+    } else if (
+      this.isAudioMessage(message) &&
+      this.messageSender.sendAudioMessage
+    ) {
+      this.messageSender.sendAudioMessage(message, role);
+    } else if (
+      this.isImageMessage(message) &&
+      this.messageSender.sendImageMessage
+    ) {
+      this.messageSender.sendImageMessage(message, role);
+    } else if (
+      this.isVideoMessage(message) &&
+      this.messageSender.sendVideoMessage
+    ) {
+      this.messageSender.sendVideoMessage(message, role);
+    }
+  }
+
+  async *streamEvents(): AsyncIterable<LiveEvent> {
+    while (true) {
+      const event = await this.nextEvent();
+      if (event === undefined) {
+        break;
+      }
+      yield event;
+    }
+  }
+  abstract disconnect(): Promise<void>;
+
+  protected async nextEvent(): Promise<LiveEvent | undefined> {
+    if (this.eventQueue.length) {
+      return Promise.resolve(this.eventQueue.shift());
+    }
+
+    return new Promise((resolve) => {
+      this.eventResolvers.push(resolve);
+    });
+  }
+
+  //Uses an async queue to send events to the client
+  // if the consumer is waiting for an event, it will be resolved immediately
+  // otherwise, the event will be queued up and sent when the consumer is ready
+  pushEventToQueue(event: LiveEvent) {
+    if (this.eventResolvers.length) {
+      //resolving the promise with the event
+      this.eventResolvers.shift()!(event);
+    } else {
+      this.eventQueue.push(event);
+    }
+  }
+}
+
+export abstract class LiveLLM {
+  /**
+   * Set of capabilities supported by this implementation.
+   * Override in subclasses as needed.
+   */
+  capabilities: Set<LiveLLMCapability> = new Set();
+
+  abstract connect(config?: LiveConnectConfig): Promise<LiveLLMSession>;
+  abstract getEphemeralKey(): Promise<string | undefined>;
+
+  hasCapability(capability: LiveLLMCapability): boolean {
+    return this.capabilities.has(capability);
+  }
+}
@@ -0,0 +1,15 @@
+import type {
+  MessageContentAudioDetail,
+  MessageContentImageDataDetail,
+  MessageContentVideoDetail,
+} from "../type";
+
+export interface MessageSender {
+  sendTextMessage(message: string, role?: string): void;
+  sendAudioMessage?(content: MessageContentAudioDetail, role?: string): void;
+  sendImageMessage?(
+    content: MessageContentImageDataDetail,
+    role?: string,
+  ): void;
+  sendVideoMessage?(content: MessageContentVideoDetail, role?: string): void;
+}
@@ -290,8 +290,14 @@ export type ToolOutput = {
  isError: boolean;
 };

+export interface AudioConfig {
+  stream?: MediaStream;
+  onTrack?: (track: MediaStream | null) => void;
+}
+
 export interface LiveConnectConfig {
  tools?: BaseTool[];
  responseModality?: ModalityType[];
  systemInstruction?: string;
+  audioConfig?: AudioConfig;
 }
@@ -0,0 +1,42 @@
+export const abbreviations = {
+  english: [
+    "i.e.",
+    "etc.",
+    "vs.",
+    "Inc.",
+    "A.S.A.P.",
+    "Mr.",
+    "Mrs.",
+    "Ms.",
+    "Dr.",
+    "Prof.",
+    "Sr.",
+    "Jr.",
+    "Tel.",
+    "a.m.",
+    "p.m.",
+    "Art.",
+  ],
+  spanish: [
+    "Sr.",
+    "Sres.",
+    "Srs.",
+    "Sra.",
+    "Sras.",
+    "Srta.",
+    "Srtas.",
+    "Dr.",
+    "Drs.",
+    "Dra.",
+    "Dras.",
+    "Prof.",
+    "Profs.",
+    "Profa.",
+    "Profas.",
+    "Ing.",
+    "Lic.",
+    "Arq.",
+    "Ab.",
+    "Abs.",
+  ],
+};
@@ -12,11 +12,7 @@ import {
  type TextSplitterFn,
 } from "./utils";

-type _Split = {
-  text: string;
-  isSentence: boolean;
-  tokenSize: number;
-};
+type _Split = [string, boolean, number]; // [text, isSentence, tokenSize]

 /**
 * Parse text with a preference for complete sentences.
@@ -42,6 +38,11 @@ export class SentenceSplitter extends MetadataAwareTextSplitter {
   * Backup regex for splitting into sentences.
   */
  secondaryChunkingRegex: string = "[^,.;。？！]+[,.;。？！]?";
+  /**
+   * Extra abbreviations to consider while splitting into sentences.
+   * For example, for contracts, you may want to consider "LLC." as an important abbreviation
+   */
+  extraAbbreviations: string[] | undefined = [];

  #chunkingTokenizerFn = splitBySentenceTokenizer();
  #splitFns: Set<TextSplitterFn> = new Set();
@@ -59,7 +60,11 @@ export class SentenceSplitter extends MetadataAwareTextSplitter {
      this.separator = parsedParams.separator;
      this.paragraphSeparator = parsedParams.paragraphSeparator;
      this.secondaryChunkingRegex = parsedParams.secondaryChunkingRegex;
+      this.extraAbbreviations = parsedParams.extraAbbreviations;
    }
+    this.#chunkingTokenizerFn = splitBySentenceTokenizer(
+      this.extraAbbreviations,
+    );
    this.#tokenizer = params?.tokenizer ?? Settings.tokenizer;
    this.#splitFns.add(splitBySep(this.paragraphSeparator));
    this.#splitFns.add(this.#chunkingTokenizerFn);
@@ -105,34 +110,26 @@ export class SentenceSplitter extends MetadataAwareTextSplitter {
    return chunks;
  }

-  #split(text: string, chunkSize: number): _Split[] {
-    const tokenSize = this.tokenSize(text);
-    if (tokenSize <= chunkSize) {
-      return [
-        {
-          text,
-          isSentence: true,
-          tokenSize,
-        },
-      ];
+  *#split(
+    text: string,
+    chunkSize: number,
+    tokenSize?: number,
+  ): Generator<_Split> {
+    const _tokenSize = tokenSize ?? this.tokenSize(text);
+    if (_tokenSize <= chunkSize) {
+      yield [text, true, _tokenSize];
+      return;
    }
    const [textSplitsByFns, isSentence] = this.#getSplitsByFns(text);
-    const textSplits: _Split[] = [];

    for (const textSplit of textSplitsByFns) {
-      const tokenSize = this.tokenSize(textSplit);
-      if (tokenSize <= chunkSize) {
-        textSplits.push({
-          text: textSplit,
-          isSentence,
-          tokenSize,
-        });
+      const textSplitTokenSize = this.tokenSize(textSplit);
+      if (textSplitTokenSize <= chunkSize) {
+        yield [textSplit, isSentence, textSplitTokenSize];
      } else {
-        const recursiveTextSplits = this.#split(textSplit, chunkSize);
-        textSplits.push(...recursiveTextSplits);
+        yield* this.#split(textSplit, chunkSize, textSplitTokenSize);
      }
    }
-    return textSplits;
  }

  #getSplitsByFns(text: string): [splits: string[], isSentence: boolean] {
@@ -151,7 +148,7 @@ export class SentenceSplitter extends MetadataAwareTextSplitter {
    return [[text], true];
  }

-  #merge(splits: _Split[], chunkSize: number): string[] {
+  #merge(splits: Iterable<_Split>, chunkSize: number): string[] {
    const chunks: string[] = [];
    let currentChunk: [string, number][] = [];
    let lastChunk: [string, number][] = [];
@@ -177,23 +174,26 @@ export class SentenceSplitter extends MetadataAwareTextSplitter {
      }
    };

-    while (splits.length > 0) {
-      const curSplit = splits[0]!;
-      if (curSplit.tokenSize > chunkSize) {
+    const splitsIterator = splits[Symbol.iterator]();
+    let currentSplitResult = splitsIterator.next();
+
+    while (!currentSplitResult.done) {
+      const [text, isSentence, tokenSize] = currentSplitResult.value;
+      if (tokenSize > chunkSize) {
        throw new Error("Single token exceeded chunk size");
      }
-      if (currentChunkLength + curSplit.tokenSize > chunkSize && !newChunk) {
+      if (currentChunkLength + tokenSize > chunkSize && !newChunk) {
        closeChunk();
      } else {
        if (
-          curSplit.isSentence ||
-          currentChunkLength + curSplit.tokenSize <= chunkSize ||
+          isSentence ||
+          currentChunkLength + tokenSize <= chunkSize ||
          newChunk
        ) {
-          currentChunkLength += curSplit.tokenSize;
-          currentChunk.push([curSplit.text, curSplit.tokenSize]);
-          splits.shift();
+          currentChunkLength += tokenSize;
+          currentChunk.push([text, tokenSize]);
          newChunk = false;
+          currentSplitResult = splitsIterator.next();
        } else {
          closeChunk();
        }
@@ -1,3 +1,4 @@
+import { abbreviations } from "./abbreviations";
 import type { TextSplitter } from "./base";
 import SentenceTokenizer from "./sentence_tokenizer";

@@ -34,19 +35,15 @@ export const splitByChar = (): TextSplitterFn => {
  return (text: string) => text.split("");
 };

-let sentenceTokenizer: SentenceTokenizer | null = null;
-
-export const splitBySentenceTokenizer = (): TextSplitterFn => {
-  if (!sentenceTokenizer) {
-    sentenceTokenizer = new SentenceTokenizer([
-      "i.e.",
-      "etc.",
-      "vs.",
-      "Inc.",
-      "A.S.A.P.",
-    ]);
-  }
-  const tokenizer = sentenceTokenizer;
+export const splitBySentenceTokenizer = (
+  extraAbbreviations: string[] | undefined = [],
+): TextSplitterFn => {
+  const tokenizer = new SentenceTokenizer([
+    ...abbreviations.english,
+    ...abbreviations.spanish,
+    // Add the extra abbreviations provided by the user, e.g. for business-specific context
+    ...extraAbbreviations,
+  ]);
  return (text: string) => {
    try {
      return tokenizer.tokenize(text);
@@ -51,6 +51,13 @@ export const sentenceSplitterSchema = z
      })
      .optional()
      .default("[^,.;。？！]+[,.;。？！]?"),
+    extraAbbreviations: z
+      .array(z.string(), {
+        description:
+          "Extra abbreviations to consider while splitting into sentences. For example, for contracts, you may want to consider 'LLC.' as an important abbreviation",
+      })
+      .optional()
+      .default([]),
  })
  .refine(
    (data) => data.chunkOverlap < data.chunkSize,
@@ -14,19 +14,6 @@ export const isIterable = (obj: unknown): obj is Iterable<unknown> => {
  return obj != null && typeof obj === "object" && Symbol.iterator in obj;
 };

-export async function* streamConverter<S, D>(
-  stream: AsyncIterable<S>,
-  converter: (s: S) => D | null,
-): AsyncIterable<D> {
-  for await (const data of stream) {
-    const newData = converter(data);
-    if (newData === null) {
-      return;
-    }
-    yield newData;
-  }
-}
-
 export async function* streamCallbacks<S>(
  stream: AsyncIterable<S>,
  callbacks: {
@@ -86,3 +73,4 @@ export {
 export { MockLLM } from "./mock";

 export { objectEntries } from "./object-entries";
+export * from "./stream";
@@ -0,0 +1,12 @@
+export async function* streamConverter<S, D>(
+  stream: AsyncIterable<S>,
+  converter: (s: S) => D | null,
+): AsyncIterable<D> {
+  for await (const data of stream) {
+    const newData = converter(data);
+    if (newData === null) {
+      return;
+    }
+    yield newData;
+  }
+}
@@ -75,6 +75,22 @@ describe("sentence splitter", () => {
    expect(splits).toEqual(["This is a sentence. This is another sentence."]);
  });

+  test("overall split long text", () => {
+    const sentenceSplitter = new SentenceSplitter({
+      chunkSize: 10,
+      chunkOverlap: 0,
+    });
+    const splits = sentenceSplitter.splitText(
+      "The first short sentence. The first long long long sentence. The second short sentence. The second long long long sentence.",
+    );
+    expect(splits).toEqual([
+      "The first short sentence.",
+      "The first long long long sentence.",
+      "The second short sentence.",
+      "The second long long long sentence.",
+    ]);
+  });
+
  test("doesn't split decimals", () => {
    const sentenceSplitter = new SentenceSplitter({
      chunkSize: 5,
@@ -91,6 +107,39 @@ describe("sentence splitter", () => {
    ]);
  });

+  test("doesn't split basic abbreviations", () => {
+    const sentenceSplitter = new SentenceSplitter({
+      chunkSize: 15,
+      chunkOverlap: 0,
+    });
+    const splits = sentenceSplitter.splitText(
+      "This is a sentence of Broda Noel. This is the sentence of Sr. Broda Noel. This is a sentence of somebody else",
+    );
+
+    expect(splits).toEqual([
+      "This is a sentence of Broda Noel.",
+      "This is the sentence of Sr. Broda Noel.",
+      "This is a sentence of somebody else",
+    ]);
+  });
+
+  test("doesn't split extra abbreviations", () => {
+    const sentenceSplitter = new SentenceSplitter({
+      chunkSize: 10,
+      chunkOverlap: 0,
+      extraAbbreviations: ["S.A."],
+    });
+    const splits = sentenceSplitter.splitText(
+      "This is a sentence. The S.A. Broda Company. This is another sentence",
+    );
+
+    expect(splits).toEqual([
+      "This is a sentence.",
+      "The S.A. Broda Company.",
+      "This is another sentence",
+    ]);
+  });
+
  test("splits cjk", () => {
    const sentenceSplitter = new SentenceSplitter({
      chunkSize: 30,
@@ -1,5 +1,11 @@
 # @llamaindex/experimental

+## 0.0.186
+
+### Patch Changes
+
+- llamaindex@0.11.9
+
 ## 0.0.185

 ### Patch Changes
@@ -1,7 +1,7 @@
 {
  "name": "@llamaindex/experimental",
  "description": "Experimental package for LlamaIndexTS",
-  "version": "0.0.185",
+  "version": "0.0.186",
  "type": "module",
  "types": "dist/type/index.d.ts",
  "main": "dist/cjs/index.js",
@@ -1,5 +1,18 @@
 # llamaindex

+## 0.11.9
+
+### Patch Changes
+
+- Updated dependencies [a89e187]
+- Updated dependencies [62699b7]
+- Updated dependencies [c5b2691]
+- Updated dependencies [d8ac8d3]
+  - @llamaindex/core@0.6.11
+  - @llamaindex/cloud@4.0.15
+  - @llamaindex/node-parser@2.0.11
+  - @llamaindex/workflow@1.1.10
+
 ## 0.11.8

 ### Patch Changes
@@ -1,6 +1,6 @@
 {
  "name": "llamaindex",
-  "version": "0.11.8",
+  "version": "0.11.9",
  "license": "MIT",
  "type": "module",
  "keywords": [
@@ -1,5 +1,12 @@
 # @llamaindex/core-test

+## 0.1.6
+
+### Patch Changes
+
+- Updated dependencies [d8ac8d3]
+  - @llamaindex/openai@0.4.5
+
 ## 0.1.5

 ### Patch Changes
@@ -1,7 +1,7 @@
 {
  "name": "@llamaindex/llamaindex-test",
  "private": true,
-  "version": "0.1.5",
+  "version": "0.1.6",
  "type": "module",
  "scripts": {
    "test": "vitest run"
@@ -1,5 +1,15 @@
 # @llamaindex/node-parser

+## 2.0.11
+
+### Patch Changes
+
+- Updated dependencies [a89e187]
+- Updated dependencies [62699b7]
+- Updated dependencies [c5b2691]
+- Updated dependencies [d8ac8d3]
+  - @llamaindex/core@0.6.11
+
 ## 2.0.10

 ### Patch Changes
@@ -1,6 +1,6 @@
 {
  "name": "@llamaindex/node-parser",
-  "version": "2.0.10",
+  "version": "2.0.11",
  "description": "Node parser for LlamaIndex",
  "type": "module",
  "exports": {
@@ -1,5 +1,15 @@
 # @llamaindex/anthropic

+## 0.3.13
+
+### Patch Changes
+
+- Updated dependencies [a89e187]
+- Updated dependencies [62699b7]
+- Updated dependencies [c5b2691]
+- Updated dependencies [d8ac8d3]
+  - @llamaindex/core@0.6.11
+
 ## 0.3.12

 ### Patch Changes
@@ -1,7 +1,7 @@
 {
  "name": "@llamaindex/anthropic",
  "description": "Anthropic Adapter for LlamaIndex",
-  "version": "0.3.12",
+  "version": "0.3.13",
  "type": "module",
  "main": "./dist/index.cjs",
  "module": "./dist/index.js",
@@ -1,5 +1,15 @@
 # @llamaindex/assemblyai

+## 0.1.10
+
+### Patch Changes
+
+- Updated dependencies [a89e187]
+- Updated dependencies [62699b7]
+- Updated dependencies [c5b2691]
+- Updated dependencies [d8ac8d3]
+  - @llamaindex/core@0.6.11
+
 ## 0.1.9

 ### Patch Changes
@@ -1,7 +1,7 @@
 {
  "name": "@llamaindex/assemblyai",
  "description": "AssemblyAI Reader for LlamaIndex",
-  "version": "0.1.9",
+  "version": "0.1.10",
  "type": "module",
  "types": "dist/index.d.ts",
  "main": "dist/index.cjs",
@@ -1,5 +1,15 @@
 # @llamaindex/community

+## 0.0.106
+
+### Patch Changes
+
+- Updated dependencies [a89e187]
+- Updated dependencies [62699b7]
+- Updated dependencies [c5b2691]
+- Updated dependencies [d8ac8d3]
+  - @llamaindex/core@0.6.11
+
 ## 0.0.105

 ### Patch Changes
@@ -1,7 +1,7 @@
 {
  "name": "@llamaindex/aws",
  "description": "AWS package for LlamaIndexTS",
-  "version": "0.0.105",
+  "version": "0.0.106",
  "type": "module",
  "types": "dist/type/index.d.ts",
  "main": "dist/cjs/index.js",
@@ -1,5 +1,16 @@
 # @llamaindex/clip

+## 0.0.61
+
+### Patch Changes
+
+- Updated dependencies [a89e187]
+- Updated dependencies [62699b7]
+- Updated dependencies [c5b2691]
+- Updated dependencies [d8ac8d3]
+  - @llamaindex/core@0.6.11
+  - @llamaindex/openai@0.4.5
+
 ## 0.0.60

 ### Patch Changes
@@ -1,7 +1,7 @@
 {
  "name": "@llamaindex/clip",
  "description": "Clip Embedding Adapter for LlamaIndex",
-  "version": "0.0.60",
+  "version": "0.0.61",
  "type": "module",
  "types": "dist/index.d.ts",
  "main": "dist/index.cjs",
@@ -1,5 +1,15 @@
 # @llamaindex/cohere

+## 0.0.25
+
+### Patch Changes
+
+- Updated dependencies [a89e187]
+- Updated dependencies [62699b7]
+- Updated dependencies [c5b2691]
+- Updated dependencies [d8ac8d3]
+  - @llamaindex/core@0.6.11
+
 ## 0.0.24

 ### Patch Changes
@@ -1,7 +1,7 @@
 {
  "name": "@llamaindex/cohere",
  "description": "Cohere Adapter for LlamaIndex",
-  "version": "0.0.24",
+  "version": "0.0.25",
  "type": "module",
  "main": "./dist/index.cjs",
  "module": "./dist/index.js",
@@ -1,5 +1,16 @@
 # @llamaindex/deepinfra

+## 0.0.61
+
+### Patch Changes
+
+- Updated dependencies [a89e187]
+- Updated dependencies [62699b7]
+- Updated dependencies [c5b2691]
+- Updated dependencies [d8ac8d3]
+  - @llamaindex/core@0.6.11
+  - @llamaindex/openai@0.4.5
+
 ## 0.0.60

 ### Patch Changes
@@ -1,7 +1,7 @@
 {
  "name": "@llamaindex/deepinfra",
  "description": "Deepinfra Adapter for LlamaIndex",
-  "version": "0.0.60",
+  "version": "0.0.61",
  "type": "module",
  "main": "./dist/index.cjs",
  "module": "./dist/index.js",
@@ -1,5 +1,12 @@
 # @llamaindex/deepseek

+## 0.0.21
+
+### Patch Changes
+
+- Updated dependencies [d8ac8d3]
+  - @llamaindex/openai@0.4.5
+
 ## 0.0.20

 ### Patch Changes
@@ -1,7 +1,7 @@
 {
  "name": "@llamaindex/deepseek",
  "description": "DeepSeek Adapter for LlamaIndex",
-  "version": "0.0.20",
+  "version": "0.0.21",
  "type": "module",
  "main": "./dist/index.cjs",
  "module": "./dist/index.js",
@@ -1,5 +1,15 @@
 # @llamaindex/discord

+## 0.1.10
+
+### Patch Changes
+
+- Updated dependencies [a89e187]
+- Updated dependencies [62699b7]
+- Updated dependencies [c5b2691]
+- Updated dependencies [d8ac8d3]
+  - @llamaindex/core@0.6.11
+
 ## 0.1.9

 ### Patch Changes
@@ -1,7 +1,7 @@
 {
  "name": "@llamaindex/discord",
  "description": "Discord Reader for LlamaIndex",
-  "version": "0.1.9",
+  "version": "0.1.10",
  "type": "module",
  "types": "dist/index.d.ts",
  "main": "dist/index.cjs",
@@ -1,5 +1,12 @@
 # @llamaindex/fireworks

+## 0.0.21
+
+### Patch Changes
+
+- Updated dependencies [d8ac8d3]
+  - @llamaindex/openai@0.4.5
+
 ## 0.0.20

 ### Patch Changes
@@ -1,7 +1,7 @@
 {
  "name": "@llamaindex/fireworks",
  "description": "Fireworks Adapter for LlamaIndex",
-  "version": "0.0.20",
+  "version": "0.0.21",
  "type": "module",
  "main": "./dist/index.cjs",
  "module": "./dist/index.js",
@@ -1,5 +1,16 @@
 # @llamaindex/google

+## 0.3.10
+
+### Patch Changes
+
+- d8ac8d3: Feat: add support for openai realtime API
+- Updated dependencies [a89e187]
+- Updated dependencies [62699b7]
+- Updated dependencies [c5b2691]
+- Updated dependencies [d8ac8d3]
+  - @llamaindex/core@0.6.11
+
 ## 0.3.9

 ### Patch Changes
@@ -1,7 +1,7 @@
 {
  "name": "@llamaindex/google",
  "description": "Google Adapter for LlamaIndex",
-  "version": "0.3.9",
+  "version": "0.3.10",
  "type": "module",
  "main": "./dist/index.cjs",
  "module": "./dist/index.js",
@@ -12,14 +12,11 @@ import {
  LiveLLM,
  LiveLLMSession,
  type BaseTool,
-  type ChatMessage,
  type LiveConnectConfig,
-  type MessageContentAudioDetail,
-  type MessageContentDetail,
-  type MessageContentImageDataDetail,
-  type MessageContentVideoDetail,
+  type MessageSender,
 } from "@llamaindex/core/llms";
-import { getEnv, uint8ArrayToBase64 } from "@llamaindex/env";
+import { getEnv } from "@llamaindex/env";
+import { GeminiMessageSender } from "./message-sender";
 import { GEMINI_MODEL, type GeminiVoiceName } from "./types";
 import {
  mapBaseToolToGeminiLiveFunctionDeclaration,
@@ -40,6 +37,10 @@ export class GeminiLiveSession extends LiveLLMSession {
    super();
  }

+  get messageSender(): MessageSender {
+    return new GeminiMessageSender(this);
+  }
+
  private isInterruptedEvent(event: LiveServerMessage): boolean {
    return event.serverContent?.interrupted === true;
  }
@@ -70,22 +71,6 @@ export class GeminiLiveSession extends LiveLLMSession {
    return event.setupComplete !== undefined;
  }

-  private isTextMessage(content: MessageContentDetail) {
-    return content.type === "text";
-  }
-
-  private isAudioMessage(content: MessageContentDetail) {
-    return content.type === "audio";
-  }
-
-  private isImageMessage(content: MessageContentDetail) {
-    return content.type === "image";
-  }
-
-  private isVideoMessage(content: MessageContentDetail) {
-    return content.type === "video";
-  }
-
  //for the tool call event, we need to return the response with function responses
  private async handleToolCallEvent(
    event: LiveServerMessage,
@@ -195,105 +180,11 @@ export class GeminiLiveSession extends LiveLLMSession {
    }
  }

-  private sendTextMessage(message: string, role?: string) {
-    this.session?.sendClientContent({
-      turns: [
-        {
-          parts: [{ text: message }],
-          ...(role ? { role } : {}),
-        },
-      ],
-    });
-  }
-
-  private sendAudioMessage(content: MessageContentAudioDetail, role?: string) {
-    if (typeof content.data === "string") {
-      this.session?.sendRealtimeInput({
-        audio: {
-          data: content.data,
-          mimeType: content.mimeType,
-        },
-      });
-    } else {
-      this.session?.sendRealtimeInput({
-        audio: {
-          data: uint8ArrayToBase64(content.data),
-          mimeType: content.mimeType,
-        },
-      });
-    }
-  }
-
-  private sendImageMessage(
-    content: MessageContentImageDataDetail,
-    role?: string,
-  ) {
-    if (typeof content.data === "string") {
-      this.session?.sendRealtimeInput({
-        media: {
-          data: content.data,
-          mimeType: content.mimeType,
-        },
-      });
-    } else {
-      this.session?.sendRealtimeInput({
-        media: {
-          data: uint8ArrayToBase64(content.data),
-          mimeType: content.mimeType,
-        },
-      });
-    }
-  }
-
-  private sendVideoMessage(content: MessageContentVideoDetail, role?: string) {
-    if (typeof content.data === "string") {
-      this.session?.sendRealtimeInput({
-        video: {
-          data: content.data,
-          mimeType: content.mimeType,
-        },
-      });
-    } else {
-      this.session?.sendRealtimeInput({
-        video: {
-          data: uint8ArrayToBase64(content.data),
-          mimeType: content.mimeType,
-        },
-      });
-    }
-  }
-
-  private handleUserInput(message: ChatMessage) {
-    const { content, role } = message;
-
-    if (!Array.isArray(content)) {
-      this.sendTextMessage(content, role);
-    } else {
-      for (const item of content) {
-        if (this.isTextMessage(item)) {
-          this.sendTextMessage(item.text, role);
-        } else if (this.isAudioMessage(item)) {
-          this.sendAudioMessage(item as MessageContentAudioDetail, role);
-        } else if (this.isImageMessage(item)) {
-          this.sendImageMessage(item as MessageContentImageDataDetail, role);
-        } else if (this.isVideoMessage(item)) {
-          this.sendVideoMessage(item as MessageContentVideoDetail, role);
-        }
-      }
-    }
-  }
-
-  sendMessage(message: ChatMessage) {
-    if (!this.session) {
-      throw new Error("Session not connected");
-    }
-    this.handleUserInput(message);
-  }
-
  async disconnect() {
    if (!this.session) {
      throw new Error("Session not connected");
    }
+
    this.session.close();
  }
 }
@@ -322,6 +213,10 @@ export class GeminiLive extends LiveLLM {
    }
  }

+  async getEphemeralKey(): Promise<string | undefined> {
+    throw new Error("Ephemeral key is not supported for Gemini Live");
+  }
+
  async connect(config?: LiveConnectConfig) {
    const liveConfig: GoogleLiveConnectConfig = {
      responseModalities: config?.responseModality
@@ -356,6 +251,12 @@ export class GeminiLive extends LiveLLM {
      };
    }

+    if (config?.audioConfig) {
+      throw new Error(
+        "Audio config is not supported for Gemini Live, directly send and recieve audio events instead",
+      );
+    }
+
    const geminiLiveSession = new GeminiLiveSession();

    geminiLiveSession.session = await this.client.live.connect({
@@ -0,0 +1,53 @@
+import type {
+  MessageContentAudioDetail,
+  MessageContentImageDataDetail,
+  MessageContentVideoDetail,
+  MessageSender,
+} from "@llamaindex/core/llms";
+import type { GeminiLiveSession } from "./live";
+
+export class GeminiMessageSender implements MessageSender {
+  private geminiSession: GeminiLiveSession;
+
+  constructor(session: GeminiLiveSession) {
+    this.geminiSession = session;
+  }
+
+  sendTextMessage(message: string, role?: string) {
+    this.geminiSession.session?.sendClientContent({
+      turns: [
+        {
+          parts: [{ text: message }],
+          ...(role ? { role } : {}),
+        },
+      ],
+    });
+  }
+
+  sendAudioMessage(content: MessageContentAudioDetail, role?: string) {
+    this.geminiSession.session?.sendRealtimeInput({
+      audio: {
+        data: content.data,
+        mimeType: content.mimeType,
+      },
+    });
+  }
+
+  sendImageMessage(content: MessageContentImageDataDetail, role?: string) {
+    this.geminiSession.session?.sendRealtimeInput({
+      media: {
+        data: content.data,
+        mimeType: content.mimeType,
+      },
+    });
+  }
+
+  sendVideoMessage(content: MessageContentVideoDetail, role?: string) {
+    this.geminiSession.session?.sendRealtimeInput({
+      video: {
+        data: content.data,
+        mimeType: content.mimeType,
+      },
+    });
+  }
+}
@@ -1,5 +1,12 @@
 # @llamaindex/groq

+## 0.0.76
+
+### Patch Changes
+
+- Updated dependencies [d8ac8d3]
+  - @llamaindex/openai@0.4.5
+
 ## 0.0.75

 ### Patch Changes
@@ -1,7 +1,7 @@
 {
  "name": "@llamaindex/groq",
  "description": "Groq Adapter for LlamaIndex",
-  "version": "0.0.75",
+  "version": "0.0.76",
  "type": "module",
  "main": "./dist/index.cjs",
  "module": "./dist/index.js",
@@ -1,5 +1,16 @@
 # @llamaindex/huggingface

+## 0.1.15
+
+### Patch Changes
+
+- Updated dependencies [a89e187]
+- Updated dependencies [62699b7]
+- Updated dependencies [c5b2691]
+- Updated dependencies [d8ac8d3]
+  - @llamaindex/core@0.6.11
+  - @llamaindex/openai@0.4.5
+
 ## 0.1.14

 ### Patch Changes
@@ -1,7 +1,7 @@
 {
  "name": "@llamaindex/huggingface",
  "description": "Huggingface Adapter for LlamaIndex",
-  "version": "0.1.14",
+  "version": "0.1.15",
  "type": "module",
  "types": "dist/index.d.ts",
  "main": "dist/index.cjs",
@@ -1,5 +1,16 @@
 # @llamaindex/jinaai

+## 0.0.21
+
+### Patch Changes
+
+- Updated dependencies [a89e187]
+- Updated dependencies [62699b7]
+- Updated dependencies [c5b2691]
+- Updated dependencies [d8ac8d3]
+  - @llamaindex/core@0.6.11
+  - @llamaindex/openai@0.4.5
+
 ## 0.0.20

 ### Patch Changes
--- a/Show More
+++ b/Show More
Author	SHA1	Message	Date
github-actions[bot]	85039a5360	Release @llamaindex/tools@0.1.0 (#2034 )	2025-06-24 12:32:24 +07:00
Marcus Schiesser	d7305edb53	fix changesets	2025-06-24 12:26:09 +07:00
Huu Le	096bf2bda1	feat: Add support for StreamableHTTP MCP Client (#2032 )	2025-06-24 11:40:34 +07:00
jerinthomascarmel	c5846bd7dc	feat(readers): add XMLReader for parsing XML files (#1846 ) (#2031 ) Co-authored-by: Marcus Schiesser <marcus.schiesser@googlemail.com>	2025-06-24 10:46:32 +07:00
github-actions[bot]	97bbce6e13	Release 0.11.9 (#2023 ) Co-authored-by: marcusschiesser <17126+marcusschiesser@users.noreply.github.com>	2025-06-20 12:28:01 +07:00
Marcus Schiesser	62699b7497	chore: improve performance of sentence splitter (#2030 )	2025-06-20 12:16:24 +07:00
Broda Noel	a89e187796	Add `extraAbbreviations` on sentence-splitter (#2029 ) Co-authored-by: Marcus Schiesser <mail@marcusschiesser.de>	2025-06-20 11:27:06 +07:00
ANKIT VARSHNEY	d8ac8d385d	feat: add openai realtime api (#2006 ) Co-authored-by: Marcus Schiesser <mail@marcusschiesser.de>	2025-06-20 10:22:04 +07:00
Marcus Schiesser	a6cef9c6be	chore: no core in examples (#2024 )	2025-06-18 09:39:32 +07:00
Broda Noel	c5b2691302	Add more Acronyms on SentenceSplitter (#2022 ) Co-authored-by: Marcus Schiesser <marcus.schiesser@googlemail.com>	2025-06-17 10:43:36 +07:00
				`@@ -0,0 +1 @@`
				<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" class="iconify iconify--logos" width="31.88" height="32" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 257"><defs><linearGradient id="IconifyId1813088fe1fbc01fb466" x1="-.828%" x2="57.636%" y1="7.652%" y2="78.411%"><stop offset="0%" stop-color="#41D1FF"></stop><stop offset="100%" stop-color="#BD34FE"></stop></linearGradient><linearGradient id="IconifyId1813088fe1fbc01fb467" x1="43.376%" x2="50.316%" y1="2.242%" y2="89.03%"><stop offset="0%" stop-color="#FFEA83"></stop><stop offset="8.333%" stop-color="#FFDD35"></stop><stop offset="100%" stop-color="#FFA800"></stop></linearGradient></defs><path fill="url(#IconifyId1813088fe1fbc01fb466)" d="M255.153 37.938L134.897 252.976c-2.483 4.44-8.862 4.466-11.382.048L.875 37.958c-2.746-4.814 1.371-10.646 6.827-9.67l120.385 21.517a6.537 6.537 0 0 0 2.322-.004l117.867-21.483c5.438-.991 9.574 4.796 6.877 9.62Z"></path><path fill="url(#IconifyId1813088fe1fbc01fb467)" d="M185.432.063L96.44 17.501a3.268 3.268 0 0 0-2.634 3.014l-5.474 92.456a3.268 3.268 0 0 0 3.997 3.378l24.777-5.718c2.318-.535 4.413 1.507 3.936 3.838l-7.361 36.047c-.495 2.426 1.782 4.5 4.151 3.78l15.304-4.649c2.372-.72 4.652 1.36 4.15 3.788l-11.698 56.621c-.732 3.542 3.979 5.473 5.943 2.437l1.313-2.028l72.516-144.72c1.215-2.423-.88-5.186-3.54-4.672l-25.505 4.922c-2.396.462-4.435-1.77-3.759-4.114l16.646-57.705c.677-2.35-1.37-4.583-3.769-4.113Z"></path></svg>