Release @llamaindex/tools@0.1.2 (#2048 )

chore: export type MCPClientOptions (#2047 )
Co-authored-by: Marcus Schiesser <marcus.schiesser@googlemail.com>
2026-07-01 22:14:03 -04:00 · 2025-06-30 11:40:54 +07:00 · 2025-06-28 10:55:07 +07:00 · 2025-06-27 14:51:09 +07:00 · 2025-06-27 12:09:26 +07:00 · 2025-06-26 14:29:33 +07:00
230 changed files with 8189 additions and 2504 deletions
@@ -1,5 +1,83 @@
 # @llamaindex/doc

+## 0.2.31
+
+### Patch Changes
+
+- Updated dependencies [7039e1a]
+- Updated dependencies [7039e1a]
+  - llamaindex@0.11.11
+  - @llamaindex/core@0.6.12
+  - @llamaindex/cloud@4.0.16
+  - @llamaindex/node-parser@2.0.12
+  - @llamaindex/openai@0.4.6
+  - @llamaindex/readers@3.1.11
+  - @llamaindex/workflow@1.1.12
+
+## 0.2.30
+
+### Patch Changes
+
+- Updated dependencies [f7ec293]
+  - @llamaindex/workflow@1.1.11
+  - llamaindex@0.11.10
+
+## 0.2.29
+
+### Patch Changes
+
+- Updated dependencies [c5846bd]
+  - @llamaindex/readers@3.1.10
+
+## 0.2.28
+
+### Patch Changes
+
+- Updated dependencies [a89e187]
+- Updated dependencies [62699b7]
+- Updated dependencies [c5b2691]
+- Updated dependencies [d8ac8d3]
+  - @llamaindex/core@0.6.11
+  - @llamaindex/openai@0.4.5
+  - @llamaindex/cloud@4.0.15
+  - llamaindex@0.11.9
+  - @llamaindex/node-parser@2.0.11
+  - @llamaindex/readers@3.1.9
+  - @llamaindex/workflow@1.1.10
+
+## 0.2.27
+
+### Patch Changes
+
+- 8a51c16: Add natural language agent page
+- Updated dependencies [8a51c16]
+- Updated dependencies [1b5af14]
+  - @llamaindex/workflow@1.1.9
+  - @llamaindex/core@0.6.10
+  - llamaindex@0.11.8
+  - @llamaindex/cloud@4.0.14
+  - @llamaindex/node-parser@2.0.10
+  - @llamaindex/openai@0.4.4
+  - @llamaindex/readers@3.1.8
+
+## 0.2.26
+
+### Patch Changes
+
+- a4d394f: fix: correct SimpleDirectoryReader import path in documentation example
+- Updated dependencies [dbd857f]
+- Updated dependencies [3c857f4]
+  - @llamaindex/workflow@1.1.8
+  - llamaindex@0.11.7
+
+## 0.2.25
+
+### Patch Changes
+
+- Updated dependencies [40161fe]
+  - @llamaindex/workflow@1.1.7
+  - llamaindex@0.11.6
+
 ## 0.2.24

 ### Patch Changes
@@ -111,7 +111,7 @@ Key build process:
 **Content Sources:**

 - Local MDX files in `src/content/docs/`
- External docs from `@llama-flow/docs` package
+- External docs from `@llamaindex/workflow-docs` package
 - Generated API docs from TypeScript source

 ### Development Notes
@@ -15,6 +15,20 @@ const config = {
    "twoslash",
    "typescript",
  ],
+  async redirects() {
+    return [
+      {
+        source: "/docs/chat-ui/:path*.mdx",
+        destination: "/docs/chat-ui/:path*",
+        permanent: true,
+      },
+      {
+        source: "/docs/workflows/:path*.mdx",
+        destination: "/docs/workflows/:path*",
+        permanent: true,
+      },
+    ];
+  },
  turbopack: {
    resolveAlias: {
      fs: { browser: "./fallback.js" },
@@ -1,6 +1,6 @@
 {
  "name": "@llamaindex/doc",
-  "version": "0.2.24",
+  "version": "0.2.31",
  "private": true,
  "scripts": {
    "postinstall": "fumadocs-mdx",
@@ -15,8 +15,8 @@
  "dependencies": {
    "@huggingface/transformers": "^3.5.0",
    "@icons-pack/react-simple-icons": "^10.1.0",
-    "@llama-flow/docs": "0.0.8",
-    "@llamaindex/chat-ui-docs": "0.0.3",
+    "@llamaindex/workflow-docs": "0.1.1",
+    "@llamaindex/chat-ui-docs": "^0.0.5",
    "@llamaindex/cloud": "workspace:*",
    "@llamaindex/core": "workspace:*",
    "@llamaindex/node-parser": "workspace:*",
@@ -39,13 +39,13 @@
    "clsx": "2.1.1",
    "foxact": "^0.2.41",
    "framer-motion": "^11.11.17",
-    "fumadocs-core": "^15.2.7",
+    "fumadocs-core": "^15.5.0",
    "fumadocs-docgen": "^2.0.0",
-    "fumadocs-mdx": "^11.6.0",
-    "fumadocs-openapi": "^8.0.1",
-    "fumadocs-twoslash": "^3.1.1",
-    "fumadocs-typescript": "^4.0.2",
-    "fumadocs-ui": "^15.2.7",
+    "fumadocs-mdx": "^11.6.6",
+    "fumadocs-openapi": "^9.0.5",
+    "fumadocs-twoslash": "^3.1.3",
+    "fumadocs-typescript": "^4.0.5",
+    "fumadocs-ui": "^15.5.0",
    "hast-util-to-jsx-runtime": "^2.3.2",
    "llamaindex": "workspace:*",
    "lucide-react": "^0.460.0",
@@ -69,7 +69,7 @@
    "twoslash": "^0.3.1",
    "use-stick-to-bottom": "^1.0.42",
    "web-tree-sitter": "^0.24.4",
-    "zod": "^3.23.8"
+    "zod": "^3.25.67"
  },
  "devDependencies": {
    "@next/env": "^15.3.0",
@@ -13,7 +13,7 @@ const INTERNAL_LINK_REGEX = /(?:(?:\]\(|\bhref=["'])\/docs\/([^")]+))/g;
 // This captures relative links like [text](./path) or ![alt](../images/image.png)
 const RELATIVE_LINK_REGEX = /(?:\]\()(?:\s*)(?:\.\.?)\//g;

-const ALLOWED_LINKS = ["/docs/llamaflow", "/docs/chat-ui"];
+const ALLOWED_LINKS = ["/docs/workflows", "/docs/chat-ui"];

 interface LinkValidationResult {
  file: string;
@@ -11,8 +11,13 @@ import remarkMath from "remark-math";
 export const docs = defineDocs({
  dir: [
    "./src/content/docs",
-    "./node_modules/@llama-flow/docs",
+    "./node_modules/@llamaindex/workflow-docs",
    "./node_modules/@llamaindex/chat-ui-docs",
+    // NOTE: When adding external docs (like chat-ui or workflow-docs above),
+    // make sure to also update:
+    // 1. scripts/validate-links.mts - add to ALLOWED_LINKS array
+    // 2. next.config.mjs - add redirect for .mdx files
+    // 3. src/content/docs/meta.json - add to pages array
  ],
  docs: {
    async: true,
@@ -113,7 +113,8 @@ export default function HomePage() {
          description="Truly powerful retrieval-augmented generation applications use agentic techniques, and LlamaIndex.TS makes it easy to build them."
        >
          <CodeBlock
-            code={`import { SimpleDirectoryReader, VectorStoreIndex } from "llamaindex";
+            code={`import { VectorStoreIndex } from "llamaindex";
+import { SimpleDirectoryReader } from "@llamaindex/readers/directory";
 import { openai } from "@llamaindex/openai";
 import { agent } from "@llamaindex/workflow";

@@ -1,4 +1,4 @@
 {
  "title": "Agents",
-  "pages": ["tool", "agent_workflow", "workflows"]
+  "pages": ["tool", "agent_workflow", "workflows", "natural_language_workflow"]
 }
@@ -0,0 +1,103 @@
+---
+title: Define workflows using natural language  
+---
+
+When working with Workflows, you have to write code to handle an event in the workflow. 
+Often, the logic of the handler is not too complex so that it can be expressed using natural language and executed by an LLM.  
+Besides the instructions, we just need the expected result event of the step, possible tool calls and optionally other events that can be emitted.  
+
+## Usage
+
+Let's take an example of a workflow that generates a joke, gets a critique for it, and then improves it.
+
+### Define the events
+
+First, we define the events for our workflow. We need one for writing the joke, one for critiquing it, and one for the final result:
+
+```typescript
+import { z } from "zod";
+import { zodEvent } from "@llamaindex/workflow";
+
+const writeJokeSchema = z.object({
+  description: z
+    .string()
+    .describe("The topic to write a joke or describe the joke to improve."),
+  writtenJoke: z.optional(z.string()).describe("The written joke."),
+  retriedTimes: z
+    .number()
+    .default(0)
+    .describe(
+      "The retried times for writing the joke. Always increase this from the input retriedTimes.",
+    ),
+});
+
+const critiqueSchema = z.object({
+  joke: z.string().describe("The joke to critique"),
+  retriedTimes: z.number().describe("The retried times for writing the joke."),
+});
+
+const finalResultSchema = z.object({
+  joke: z.string().describe("The joke to critique"),
+  critique: z.string().describe("The critique of the joke"),
+});
+
+const writeJokeEvent = zodEvent(writeJokeSchema, {
+  debugLabel: "writeJokeEvent",
+}); 
+const critiqueEvent = zodEvent(critiqueSchema, {
+  debugLabel: "critiqueEvent",
+}); 
+const finalResultEvent = zodEvent(finalResultSchema, {
+  debugLabel: "finalResultEvent",
+}); 
+```
+
+Note that your natural language workflows the events need to be created by the `zodEvent` function passing the zod schema as an argument. The agent needs the schema of the event data to correctly generate events.  
+Also, we need a `debugLabel` so the LLM can identify the event to emit in the workflow.
+
+### Define the workflow
+
+As usual you first create the workflow:
+
+```typescript
+import { agentHandler, createWorkflow } from "@llamaindex/workflow";
+
+const jokeFlow = createWorkflow();
+```
+
+Then you need to handle the events. For the handlers, instead of code, you're now going to use natural language by calling the `agentHandler` function.
+
+It only requires two parameters:
+- `instructions`: A prompt to guide the agent how to handle the steps.
+- `results`: The output events that the agent should return after handling the step.
+
+Then you will have a simple code to handle the step:
+
+```typescript
+jokeFlow.handle(
+  [writeJokeEvent],
+  agentHandler({
+    instructions: `You are a joke writer. You are given a topic and you need to write a joke about it.`,
+    results: [critiqueEvent],
+  }),
+);
+
+jokeFlow.handle(
+  [critiqueEvent],
+  agentHandler({
+    instructions: `
+You are given a joke and you need to critique it. Follow the following guidelines:
+1. You have maximum 3 times to improve the joke.
+2. If the joke is not good, increase the retriedTimes, describe how to improve the joke and send a writeJokeEvent.
+3. If the joke is good, trigger the finalResultEvent event.
+`,
+    results: [writeJokeEvent, finalResultEvent],
+  }),
+);
+```
+
+For advanced usage, you can add more functionality to `agentHandler` by using these parameters:
+- `events`: A list of additional events that the agent can emit to the workflow. E.g., your agent can emit a `uiEvent` to update the UI during the execution.
+- `tools`: A list of tools that the agent can use to handle the step. E.g., your agent can use a `search` tool to search the web.
+
+You can find more code examples in the [examples](https://github.com/run-llama/LlamaIndexTS/tree/main/examples/agents/natural) folder.
@@ -74,12 +74,21 @@ const server = mcp({
  args: ["-y", "@modelcontextprotocol/server-filesystem", "."],
  verbose: true,
 });
-// or by SSE
+// or by StreamableHTTP transport
 const server = mcp({
  url: "http://localhost:8000/mcp",
  verbose: true,
 });

+// if your MCP server is not using StreamableHTTP transport, you can also use SSE transport
+// by setting useSSETransport to true.
+// See: https://modelcontextprotocol.io/docs/concepts/transports#server-sent-events-sse-deprecated
+const server = mcp({
+  url: "http://localhost:8000/mcp",
+  useSSETransport: true,
+  verbose: true,
+});
+
 // 3. Get tools from MCP server
 const tools = await server.tools();

@@ -9,10 +9,13 @@ Workflows are designed to be flexible and can be used to build agents, RAG flows
 To use workflows install this package:

 ```package-install
-npm i @llamaindex/workflow
+npm i @llamaindex/workflow-core
 ```

-This package is a stable, production-ready version of our [llama-flow](/docs/llamaflow) project. 
+This contains the core functionality for the workflow system. You can read more about the core concepts in the [workflow-core](/docs/workflows) section.

-While you can still reference the llama-flow documentation for detailed information about the underlying concepts, we recommend using the `@llamaindex/workflow` package for all new projects to ensure stability and long-term availability.
+In contrast, the `@llamaindex/workflow` package contains more utiltities, such as prebuilt agents.

+```package-install
+npm i @llamaindex/workflow
+```
@@ -28,11 +28,12 @@ embedding vector(1536)
 );
 ```

-- Create a function for similarity search
+-- Create a function for similarity search with filtering support
 ```sql
 create function match_documents (
 query_embedding vector(1536),
-match_count int
+match_count int,
+filter jsonb DEFAULT '{}'
 ) returns table (
 id uuid,
 content text,
@@ -52,6 +53,7 @@ metadata,
 embedding,
 1 - (embedding <=> query_embedding) as similarity
 from documents
+where metadata @> filter
 order by embedding <=> query_embedding
 limit match_count;
 end;
@@ -96,6 +98,7 @@ const index = await VectorStoreIndex.fromDocuments(documents, {
 ```ts
 const queryEngine = index.asQueryEngine();

+// Basic query without filters
 const response = await queryEngine.query({
  query: "What is in the document?",
 });
@@ -104,6 +107,32 @@ const response = await queryEngine.query({
 console.log(response.toString());
 ```

+## Query with filters
+
+You can filter documents based on metadata when querying:
+
+```ts
+import { FilterOperator, MetadataFilters } from "llamaindex";
+
+// Create a filter for documents with author = "Jane Smith"
+const filters: MetadataFilters = {
+  filters: [
+    {
+      key: "author",
+      value: "Jane Smith",
+      operator: FilterOperator.EQ,
+    },
+  ],
+};
+
+// Query with filters
+const filteredResponse = await vectorStore.query({
+  queryEmbedding: embedModel.getQueryEmbedding("What is vector search?"),
+  similarityTopK: 5,
+  filters,
+});
+```
+
 ## Full code

 ```ts
@@ -11,58 +11,130 @@ npm i llamaindex @llamaindex/google
 ## Usage

 ```ts
-import { Gemini, GEMINI_MODEL } from "@llamaindex/google";
+import { gemini, GEMINI_MODEL } from "@llamaindex/google";
 import { Settings } from "llamaindex";

-Settings.llm = new Gemini({
-  model: GEMINI_MODEL.GEMINI_PRO,
-});
-```
-
-## Usage with Proxy
-
-```ts
-import { Gemini, GEMINI_MODEL } from "@llamaindex/google";
-import { Settings } from "llamaindex";
-
-Settings.llm = new Gemini({
-  model: GEMINI_MODEL.GEMINI_PRO,
-  requestOptions: {
-    baseUrl: <YOUR_PROXY_URL>   // optional, but useful for custom endpoints
-  }
+Settings.llm = gemini({
+  model: GEMINI_MODEL.GEMINI_2_0_FLASH,
 });
 ```

 ### Usage with Vertex AI

-To use Gemini via Vertex AI you can use `GeminiVertexSession`.
-
-GeminiVertexSession accepts the env variables: `GOOGLE_VERTEX_LOCATION` and `GOOGLE_VERTEX_PROJECT`
+To use Gemini via Vertex AI, you can specify the vertex configuration:

 ```ts
-import { Gemini, GEMINI_MODEL, GeminiVertexSession } from "@llamaindex/google";
+import { gemini, GEMINI_MODEL } from "@llamaindex/google";

-const gemini = new Gemini({
-  model: GEMINI_MODEL.GEMINI_PRO,
-  session: new GeminiVertexSession({
-    location: "us-central1",      // optional if provided by GOOGLE_VERTEX_LOCATION env variable
-    project: "project1",          // optional if provided by GOOGLE_VERTEX_PROJECT env variable
-    googleAuthOptions: {...},     // optional, but useful for production. It accepts all values from `GoogleAuthOptions`
-  }),
+const llm = gemini({
+  model: GEMINI_MODEL.GEMINI_2_0_FLASH,
+  vertex: {
+    project: "your-cloud-project",    // required for Vertex AI
+    location: "us-central1",          // required for Vertex AI
+  },
 });
 ```

-[GoogleAuthOptions](https://github.com/googleapis/google-auth-library-nodejs/blob/main/src/auth/googleauth.ts)
-
 To authenticate for local development:

 ```bash
-npm i @google-cloud/vertexai
 gcloud auth application-default login
 ```

 To authenticate for production you'll have to use a [service account](https://cloud.google.com/docs/authentication/). `googleAuthOptions` has `credentials` which might be useful for you.

+## Multimodal Usage
+
+Gemini supports multimodal inputs including text, images, audio, and video:
+
+```ts
+import { gemini, GEMINI_MODEL } from "@llamaindex/google";
+import fs from "fs";
+
+const llm = gemini({ model: GEMINI_MODEL.GEMINI_2_0_FLASH });
+
+const result = await llm.chat({
+  messages: [
+    {
+      role: "user",
+      content: [
+        {
+          type: "text",
+          text: "What's in this image?",
+        },
+        {
+          type: "image",
+          data: fs.readFileSync("./image.jpg").toString("base64"),
+          mimeType: "image/jpeg",
+        },
+      ],
+    },
+  ],
+});
+```
+
+## Tool Calling
+
+Gemini supports function calling with tools:
+
+```ts
+import { gemini, GEMINI_MODEL } from "@llamaindex/google";
+import { tool } from "llamaindex";
+import { z } from "zod";
+
+const llm = gemini({ model: GEMINI_MODEL.GEMINI_2_0_FLASH });
+
+const result = await llm.chat({
+  messages: [
+    {
+      content: "What's the weather in Tokyo?",
+      role: "user",
+    },
+  ],
+  tools: [
+    tool({
+      name: "weather",
+      description: "Get the weather",
+      parameters: z.object({
+        location: z.string().describe("The location to get the weather for"),
+      }),
+      execute: ({ location }) => {
+        return `The weather in ${location} is sunny and hot`;
+      },
+    }),
+  ],
+});
+```
+
+## Live API (Real-time Conversations)
+
+For real-time audio/video conversations using [Gemini Live API](https://ai.google.dev/gemini-api/docs/live). 
+
+The Live API is running directly in the frontend. That's why you have to generate an ephemeral key first on the server side and pass it to the frontend.
+
+To use the Live API, make sure to pass `apiVersion: "v1alpha"` to the `httpOptions`.
+
+```ts
+import { gemini, GEMINI_MODEL } from "@llamaindex/google";
+
+// Server-side: Generate ephemeral key
+const serverLlm = gemini({
+  model: GEMINI_MODEL.GEMINI_2_0_FLASH_LIVE,
+  httpOptions: { apiVersion: "v1alpha" },
+});
+const ephemeralKey = await serverLlm.live.getEphemeralKey();
+
+// Client-side: Use ephemeral key for Live API
+const llm = gemini({
+  apiKey: ephemeralKey,
+  model: GEMINI_MODEL.GEMINI_2_0_FLASH_LIVE,
+  voiceName: "Zephyr",
+  httpOptions: { apiVersion: "v1alpha" },
+});
+
+const session = await llm.live.connect();
+```
+
 ## Load and index documents

 For this example, we will use a single document. In a real-world scenario, you would have multiple documents to index.
@@ -90,11 +162,11 @@ const results = await queryEngine.query({
 ## Full Example

 ```ts
-import { Gemini, GEMINI_MODEL } from "@llamaindex/google";
+import { gemini, GEMINI_MODEL } from "@llamaindex/google";
 import { Document, VectorStoreIndex, Settings } from "llamaindex";

-Settings.llm = new Gemini({
-  model: GEMINI_MODEL.GEMINI_PRO,
+Settings.llm = gemini({
+  model: GEMINI_MODEL.GEMINI_2_0_FLASH,
 });

 async function main() {
@@ -104,9 +176,7 @@ async function main() {
  const index = await VectorStoreIndex.fromDocuments([document]);

  // Create a query engine
-  const queryEngine = index.asQueryEngine({
-    retriever,
-  });
+  const queryEngine = index.asQueryEngine();

  const query = "What is the meaning of life?";

@@ -378,3 +378,186 @@ async function main() {
 ## API Reference

 - [OpenAI](/docs/api/classes/OpenAI)
+
+
+# OpenAI Live LLM
+
+The OpenAI Live LLM integration in LlamaIndex provides real-time chat capabilities with support for audio streaming and tool calling.
+
+## Basic Usage
+
+```typescript
+import { openai } from "@llamaindex/openai";
+import { tool, ModalityType } from "llamaindex";
+
+// Get the ephimeral key on the server 
+const serverllm = openai({
+  apiKey: "your-api-key", 
+  model: "gpt-4o-realtime-preview-2025-06-03",
+});
+
+// Get an ephemeral key 
+// Usually this code is run on the server and the ephemeral key is passed to the
+// client - the ephemeral key can be securely used on the client side
+const ephemeralKey = await serverllm.live.getEphemeralKey();
+
+// Create a client-side LLM instance with the ephemeral key
+const llm = openai({
+  apiKey: ephemeralKey,
+  model: "gpt-4o-realtime-preview-2025-06-03"
+});
+
+// Create a live sessionimport { tool } from "llamaindex";
+const session = await llm.live.connect({
+  systemInstruction: "You are a helpful assistant.",
+});
+
+// Send a message
+session.sendMessage({
+  content: "Hello!",
+  role: "user",
+});
+```
+
+## Tool Integration
+
+Tools are handled server-side, making it simple to pass them to the live session:
+
+```typescript
+// Define your tools
+const weatherTool = tool({
+  name: "weather",
+  description: "Get the weather for a location",
+  parameters: z.object({
+    location: z.string().describe("The location to get weather for"),
+  }),
+  execute: async ({ location }) => {
+    return `The weather in ${location} is sunny`;
+  },
+});
+
+// Create session with tools
+const session = await llm.live.connect({
+  systemInstruction: "You are a helpful assistant.",
+  tools: [weatherTool],
+});
+```
+
+## Audio Support
+
+For audio capabilities:
+
+```typescript
+// Get microphone access
+const userStream = await navigator.mediaDevices.getUserMedia({
+  audio: true,
+});
+
+// Create session with audio
+const session = await llm.live.connect({
+  audioConfig: {
+    stream: userStream,
+    onTrack: (remoteStream) => {
+      // Handle incoming audio
+      audioElement.srcObject = remoteStream;
+    },
+  },
+});
+```
+
+## Event Handling
+
+Listen to events from the session:
+
+```typescript
+for await (const event of session.streamEvents()) {
+  if (liveEvents.open.include(event)) {
+    // Connection established
+    console.log("Connected!");
+  } else if (liveEvents.text.include(event)) {
+    // Received text response
+    console.log("Assistant:", event.text);
+  }
+}
+```
+
+## Capabilities
+
+The OpenAI Live LLM supports:
+
+- Real-time text chat
+- Audio streaming (if configured)
+- Tool calling (server-side execution)
+- Ephemeral key generation for secure sessions
+
+## API Reference
+
+### LiveLLM Methods
+// Get an ephemeral key 
+// Usually this code is run on the server and the ephemeral key is passed to the
+// client - the ephemeral key can be securely used on the client side
+
+#### `connect(config?: LiveConnectConfig)`
+
+Creates a new live session.
+
+```typescript
+interface LiveConnectConfig {
+  systemInstruction?: string;
+  tools?: BaseTool[];
+  audioConfig?: AudioConfig;
+  responseModality?: ModalityType[];
+}
+```
+
+#### `getEphemeralKey()`
+
+Gets a temporary key for the session.
+
+### LiveLLMSession Methods
+
+#### `sendMessage(message: ChatMessage)`
+
+Sends a message to the assistant.
+
+```typescript
+interface ChatMessage {
+  content: string | MessageContentDetail[];
+  role: "user" | "assistant";
+}
+```
+
+#### `disconnect()`
+
+Closes the session and cleans up resources.
+
+## Error Handling
+
+```typescript
+try {
+  const session = await llm.live.connect();
+} catch (error) {
+  if (error instanceof Error) {
+    console.error("Connection failed:", error.message);
+  }
+}
+```
+
+## Best Practices
+
+1. **Tool Definition**
+
+   - Keep tool implementations server-side
+   - Use clear descriptions for tools
+   - Handle tool errors gracefully
+
+2. **Session Management**
+
+   - Always disconnect sessions when done
+   - Clean up audio resources
+   - Handle reconnection scenarios
+
+3. **Security**
+   - Use ephemeral keys for sessions
+   - Validate tool inputs
+   - Secure API key handling
@@ -1,3 +1,3 @@
 {
-  "pages": ["llamaindex", "api", "llamaflow", "chat-ui"]
+  "pages": ["llamaindex", "api", "workflows", "chat-ui"]
 }
@@ -4,7 +4,7 @@
  "tasks": {
    "build": {
      "inputs": [
-        "node_modules/@llama-flow/docs/**",
+        "node_modules/@llamaindex/workflow-docs/**",
        "node_modules/@llamaindex/chat-ui-docs/**",
        "src/**/*.ts",
        "src/**/*.tsx",
@@ -1,5 +1,43 @@
 # @llamaindex/cloudflare-worker-agent-test

+## 0.0.172
+
+### Patch Changes
+
+- Updated dependencies [7039e1a]
+  - llamaindex@0.11.11
+
+## 0.0.171
+
+### Patch Changes
+
+- llamaindex@0.11.10
+
+## 0.0.170
+
+### Patch Changes
+
+- llamaindex@0.11.9
+
+## 0.0.169
+
+### Patch Changes
+
+- llamaindex@0.11.8
+
+## 0.0.168
+
+### Patch Changes
+
+- Updated dependencies [3c857f4]
+  - llamaindex@0.11.7
+
+## 0.0.167
+
+### Patch Changes
+
+- llamaindex@0.11.6
+
 ## 0.0.166

 ### Patch Changes
@@ -1,6 +1,6 @@
 {
  "name": "@llamaindex/cloudflare-worker-agent-test",
-  "version": "0.0.166",
+  "version": "0.0.172",
  "type": "module",
  "private": true,
  "scripts": {
@@ -1,5 +1,23 @@
 # @llamaindex/llama-parse-browser-test

+## 0.0.71
+
+### Patch Changes
+
+- @llamaindex/cloud@4.0.16
+
+## 0.0.70
+
+### Patch Changes
+
+- @llamaindex/cloud@4.0.15
+
+## 0.0.69
+
+### Patch Changes
+
+- @llamaindex/cloud@4.0.14
+
 ## 0.0.68

 ### Patch Changes
@@ -1,7 +1,7 @@
 {
  "name": "@llamaindex/llama-parse-browser-test",
  "private": true,
-  "version": "0.0.68",
+  "version": "0.0.71",
  "type": "module",
  "scripts": {
    "dev": "vite",
@@ -1,5 +1,43 @@
 # @llamaindex/next-agent-test

+## 0.1.172
+
+### Patch Changes
+
+- Updated dependencies [7039e1a]
+  - llamaindex@0.11.11
+
+## 0.1.171
+
+### Patch Changes
+
+- llamaindex@0.11.10
+
+## 0.1.170
+
+### Patch Changes
+
+- llamaindex@0.11.9
+
+## 0.1.169
+
+### Patch Changes
+
+- llamaindex@0.11.8
+
+## 0.1.168
+
+### Patch Changes
+
+- Updated dependencies [3c857f4]
+  - llamaindex@0.11.7
+
+## 0.1.167
+
+### Patch Changes
+
+- llamaindex@0.11.6
+
 ## 0.1.166

 ### Patch Changes
@@ -1,6 +1,6 @@
 {
  "name": "@llamaindex/next-agent-test",
-  "version": "0.1.166",
+  "version": "0.1.172",
  "private": true,
  "scripts": {
    "dev": "next dev",
@@ -1,5 +1,43 @@
 # test-edge-runtime

+## 0.1.171
+
+### Patch Changes
+
+- Updated dependencies [7039e1a]
+  - llamaindex@0.11.11
+
+## 0.1.170
+
+### Patch Changes
+
+- llamaindex@0.11.10
+
+## 0.1.169
+
+### Patch Changes
+
+- llamaindex@0.11.9
+
+## 0.1.168
+
+### Patch Changes
+
+- llamaindex@0.11.8
+
+## 0.1.167
+
+### Patch Changes
+
+- Updated dependencies [3c857f4]
+  - llamaindex@0.11.7
+
+## 0.1.166
+
+### Patch Changes
+
+- llamaindex@0.11.6
+
 ## 0.1.165

 ### Patch Changes
@@ -1,6 +1,6 @@
 {
  "name": "@llamaindex/nextjs-edge-runtime-test",
-  "version": "0.1.165",
+  "version": "0.1.171",
  "private": true,
  "scripts": {
    "dev": "next dev",
@@ -1,5 +1,56 @@
 # @llamaindex/next-node-runtime

+## 0.1.40
+
+### Patch Changes
+
+- Updated dependencies [7039e1a]
+  - llamaindex@0.11.11
+  - @llamaindex/huggingface@0.1.16
+  - @llamaindex/readers@3.1.11
+
+## 0.1.39
+
+### Patch Changes
+
+- llamaindex@0.11.10
+
+## 0.1.38
+
+### Patch Changes
+
+- Updated dependencies [c5846bd]
+  - @llamaindex/readers@3.1.10
+
+## 0.1.37
+
+### Patch Changes
+
+- llamaindex@0.11.9
+- @llamaindex/huggingface@0.1.15
+- @llamaindex/readers@3.1.9
+
+## 0.1.36
+
+### Patch Changes
+
+- llamaindex@0.11.8
+- @llamaindex/huggingface@0.1.14
+- @llamaindex/readers@3.1.8
+
+## 0.1.35
+
+### Patch Changes
+
+- Updated dependencies [3c857f4]
+  - llamaindex@0.11.7
+
+## 0.1.34
+
+### Patch Changes
+
+- llamaindex@0.11.6
+
 ## 0.1.33

 ### Patch Changes
@@ -1,6 +1,6 @@
 {
  "name": "@llamaindex/next-node-runtime-test",
-  "version": "0.1.33",
+  "version": "0.1.40",
  "private": true,
  "scripts": {
    "dev": "next dev",
@@ -1,5 +1,43 @@
 # vite-import-llamaindex

+## 0.0.38
+
+### Patch Changes
+
+- Updated dependencies [7039e1a]
+  - llamaindex@0.11.11
+
+## 0.0.37
+
+### Patch Changes
+
+- llamaindex@0.11.10
+
+## 0.0.36
+
+### Patch Changes
+
+- llamaindex@0.11.9
+
+## 0.0.35
+
+### Patch Changes
+
+- llamaindex@0.11.8
+
+## 0.0.34
+
+### Patch Changes
+
+- Updated dependencies [3c857f4]
+  - llamaindex@0.11.7
+
+## 0.0.33
+
+### Patch Changes
+
+- llamaindex@0.11.6
+
 ## 0.0.32

 ### Patch Changes
@@ -1,7 +1,7 @@
 {
  "name": "vite-import-llamaindex",
  "private": true,
-  "version": "0.0.32",
+  "version": "0.0.38",
  "type": "module",
  "scripts": {
    "build": "vite build",
@@ -1 +1,9 @@
-{"root":["./src/main.ts","./vite.config.ts"],"version":"5.7.3"}
+{
+  "root": [
+    "./src/main.ts",
+    "./vite.config.ts",
+    "./tsconfig.json"
+  ],
+  "errors": true,
+  "version": "5.7.3"
+}
@@ -1,5 +1,43 @@
 # @llamaindex/waku-query-engine-test

+## 0.0.172
+
+### Patch Changes
+
+- Updated dependencies [7039e1a]
+  - llamaindex@0.11.11
+
+## 0.0.171
+
+### Patch Changes
+
+- llamaindex@0.11.10
+
+## 0.0.170
+
+### Patch Changes
+
+- llamaindex@0.11.9
+
+## 0.0.169
+
+### Patch Changes
+
+- llamaindex@0.11.8
+
+## 0.0.168
+
+### Patch Changes
+
+- Updated dependencies [3c857f4]
+  - llamaindex@0.11.7
+
+## 0.0.167
+
+### Patch Changes
+
+- llamaindex@0.11.6
+
 ## 0.0.166

 ### Patch Changes
@@ -1,6 +1,6 @@
 {
  "name": "@llamaindex/waku-query-engine-test",
-  "version": "0.0.166",
+  "version": "0.0.172",
  "type": "module",
  "private": true,
  "scripts": {
@@ -7,7 +7,7 @@
  "dependencies": {
    "@llamaindex/workflow": "1.1.1",
    "llamaindex": "0.10.5",
-    "zod": "^3.23.8"
+    "zod": "^3.25.67"
  },
  "devDependencies": {
    "tsx": "^4.19.1",
@@ -27,6 +27,6 @@
    "pg": "^8.12.0",
    "pgvector": "0.2.0",
    "tsx": "^4.19.3",
-    "zod": "^3.24.2"
+    "zod": "^3.25.67"
  }
 }
@@ -1,5 +1,189 @@
 # examples

+## 0.3.25
+
+### Patch Changes
+
+- Updated dependencies [7039e1a]
+- Updated dependencies [7039e1a]
+  - llamaindex@0.11.11
+  - @llamaindex/core@0.6.12
+  - @llamaindex/google@0.3.11
+  - @llamaindex/cloud@4.0.16
+  - @llamaindex/node-parser@2.0.12
+  - @llamaindex/anthropic@0.3.14
+  - @llamaindex/assemblyai@0.1.11
+  - @llamaindex/clip@0.0.62
+  - @llamaindex/cohere@0.0.26
+  - @llamaindex/deepinfra@0.0.62
+  - @llamaindex/discord@0.1.11
+  - @llamaindex/huggingface@0.1.16
+  - @llamaindex/jinaai@0.0.22
+  - @llamaindex/mistral@0.1.12
+  - @llamaindex/mixedbread@0.0.26
+  - @llamaindex/notion@0.1.11
+  - @llamaindex/ollama@0.1.12
+  - @llamaindex/openai@0.4.6
+  - @llamaindex/perplexity@0.0.19
+  - @llamaindex/portkey-ai@0.0.54
+  - @llamaindex/replicate@0.0.54
+  - @llamaindex/astra@0.0.26
+  - @llamaindex/azure@0.1.23
+  - @llamaindex/chroma@0.0.26
+  - @llamaindex/elastic-search@0.1.12
+  - @llamaindex/firestore@1.0.19
+  - @llamaindex/milvus@0.1.21
+  - @llamaindex/mongodb@0.0.27
+  - @llamaindex/pinecone@0.1.12
+  - @llamaindex/postgres@0.0.55
+  - @llamaindex/qdrant@0.1.22
+  - @llamaindex/supabase@0.1.12
+  - @llamaindex/upstash@0.0.26
+  - @llamaindex/weaviate@0.0.27
+  - @llamaindex/vercel@0.1.12
+  - @llamaindex/voyage-ai@1.0.18
+  - @llamaindex/readers@3.1.11
+  - @llamaindex/tools@0.1.1
+  - @llamaindex/workflow@1.1.12
+  - @llamaindex/deepseek@0.0.22
+  - @llamaindex/fireworks@0.0.22
+  - @llamaindex/groq@0.0.77
+  - @llamaindex/together@0.0.22
+  - @llamaindex/vllm@0.0.48
+  - @llamaindex/xai@0.0.9
+
+## 0.3.24
+
+### Patch Changes
+
+- Updated dependencies [096bf2b]
+- Updated dependencies [c5846bd]
+  - @llamaindex/tools@0.1.0
+  - @llamaindex/readers@3.1.10
+
+## 0.3.23
+
+### Patch Changes
+
+- Updated dependencies [a89e187]
+- Updated dependencies [62699b7]
+- Updated dependencies [c5b2691]
+- Updated dependencies [d8ac8d3]
+  - @llamaindex/core@0.6.11
+  - @llamaindex/google@0.3.10
+  - @llamaindex/openai@0.4.5
+  - @llamaindex/cloud@4.0.15
+  - llamaindex@0.11.9
+  - @llamaindex/node-parser@2.0.11
+  - @llamaindex/anthropic@0.3.13
+  - @llamaindex/assemblyai@0.1.10
+  - @llamaindex/clip@0.0.61
+  - @llamaindex/cohere@0.0.25
+  - @llamaindex/deepinfra@0.0.61
+  - @llamaindex/discord@0.1.10
+  - @llamaindex/huggingface@0.1.15
+  - @llamaindex/jinaai@0.0.21
+  - @llamaindex/mistral@0.1.11
+  - @llamaindex/mixedbread@0.0.25
+  - @llamaindex/notion@0.1.10
+  - @llamaindex/ollama@0.1.11
+  - @llamaindex/perplexity@0.0.18
+  - @llamaindex/portkey-ai@0.0.53
+  - @llamaindex/replicate@0.0.53
+  - @llamaindex/astra@0.0.25
+  - @llamaindex/azure@0.1.22
+  - @llamaindex/chroma@0.0.25
+  - @llamaindex/elastic-search@0.1.11
+  - @llamaindex/firestore@1.0.18
+  - @llamaindex/milvus@0.1.20
+  - @llamaindex/mongodb@0.0.26
+  - @llamaindex/pinecone@0.1.11
+  - @llamaindex/postgres@0.0.54
+  - @llamaindex/qdrant@0.1.21
+  - @llamaindex/supabase@0.1.10
+  - @llamaindex/upstash@0.0.25
+  - @llamaindex/weaviate@0.0.26
+  - @llamaindex/vercel@0.1.11
+  - @llamaindex/voyage-ai@1.0.17
+  - @llamaindex/readers@3.1.9
+  - @llamaindex/tools@0.0.17
+  - @llamaindex/workflow@1.1.10
+  - @llamaindex/deepseek@0.0.21
+  - @llamaindex/fireworks@0.0.21
+  - @llamaindex/groq@0.0.76
+  - @llamaindex/together@0.0.21
+  - @llamaindex/vllm@0.0.47
+  - @llamaindex/xai@0.0.8
+
+## 0.3.22
+
+### Patch Changes
+
+- Updated dependencies [8a51c16]
+- Updated dependencies [1b5af14]
+  - @llamaindex/workflow@1.1.9
+  - @llamaindex/core@0.6.10
+  - llamaindex@0.11.8
+  - @llamaindex/cloud@4.0.14
+  - @llamaindex/node-parser@2.0.10
+  - @llamaindex/anthropic@0.3.12
+  - @llamaindex/assemblyai@0.1.9
+  - @llamaindex/clip@0.0.60
+  - @llamaindex/cohere@0.0.24
+  - @llamaindex/deepinfra@0.0.60
+  - @llamaindex/discord@0.1.9
+  - @llamaindex/google@0.3.9
+  - @llamaindex/huggingface@0.1.14
+  - @llamaindex/jinaai@0.0.20
+  - @llamaindex/mistral@0.1.10
+  - @llamaindex/mixedbread@0.0.24
+  - @llamaindex/notion@0.1.9
+  - @llamaindex/ollama@0.1.10
+  - @llamaindex/openai@0.4.4
+  - @llamaindex/perplexity@0.0.17
+  - @llamaindex/portkey-ai@0.0.52
+  - @llamaindex/replicate@0.0.52
+  - @llamaindex/astra@0.0.24
+  - @llamaindex/azure@0.1.21
+  - @llamaindex/chroma@0.0.24
+  - @llamaindex/elastic-search@0.1.10
+  - @llamaindex/firestore@1.0.17
+  - @llamaindex/milvus@0.1.19
+  - @llamaindex/mongodb@0.0.25
+  - @llamaindex/pinecone@0.1.10
+  - @llamaindex/postgres@0.0.53
+  - @llamaindex/qdrant@0.1.20
+  - @llamaindex/supabase@0.1.9
+  - @llamaindex/upstash@0.0.24
+  - @llamaindex/weaviate@0.0.25
+  - @llamaindex/vercel@0.1.10
+  - @llamaindex/voyage-ai@1.0.16
+  - @llamaindex/readers@3.1.8
+  - @llamaindex/tools@0.0.16
+  - @llamaindex/deepseek@0.0.20
+  - @llamaindex/fireworks@0.0.20
+  - @llamaindex/groq@0.0.75
+  - @llamaindex/together@0.0.20
+  - @llamaindex/vllm@0.0.46
+  - @llamaindex/xai@0.0.7
+
+## 0.3.21
+
+### Patch Changes
+
+- Updated dependencies [dbd857f]
+- Updated dependencies [3c857f4]
+  - @llamaindex/workflow@1.1.8
+  - llamaindex@0.11.7
+  - @llamaindex/tools@0.0.15
+
+## 0.3.20
+
+### Patch Changes
+
+- Updated dependencies [e7484ef]
+  - @llamaindex/weaviate@0.0.24
+
 ## 0.3.19

 ### Patch Changes
@@ -1,4 +1,3 @@
-import { tool } from "@llamaindex/core/tools";
 import { openai } from "@llamaindex/openai";
 import {
  agent,
@@ -7,6 +6,7 @@ import {
  multiAgent,
 } from "@llamaindex/workflow";
 import fs from "fs";
+import { tool } from "llamaindex";
 import os from "os";
 import { z } from "zod";

@@ -6,15 +6,24 @@ async function main() {
  // Create an MCP server for filesystem tools
  const server = mcp({
    command: "npx",
-    args: ["-y", "@modelcontextprotocol/server-filesystem", "."],
+    args: ["-y", "@modelcontextprotocol/server-filesystem@latest", "."],
    verbose: true,
  });
-  // You can also connect to the MCP server using SSE
-  // See: https://modelcontextprotocol.io/docs/concepts/transports#server-sent-events-sse
+  //
+  // You can also connect to a remote MCP server using:
+  // 1. StreamableHTTP transport (recommended)
+  // See: https://modelcontextprotocol.io/docs/concepts/transports#streamable-http
  // const server = mcp({
  //   url: "http://localhost:8000/mcp",
  //   verbose: true,
  // });
+  // 2.Or using SSE transport (will be deprecated soon)
+  // See: https://modelcontextprotocol.io/docs/concepts/transports#server-sent-events-sse-deprecated
+  // const server = mcp({
+  //   url: "http://localhost:8000/mcp",
+  //   useSSETransport: true,
+  //   verbose: true,
+  // });

  try {
    // Create an agent that uses the MCP tools
@@ -0,0 +1,130 @@
+import { ToolCallLLM } from "llamaindex";
+
+import {
+  agentHandler,
+  createWorkflow,
+  workflowEvent,
+  zodEvent,
+} from "@llamaindex/workflow";
+
+import { openai } from "@llamaindex/openai";
+import { z } from "zod";
+
+// ===== 1. Define events =====
+// An event to trigger the workflow
+const planEvent = workflowEvent<{ topic: string }>();
+
+// Generate artifact event
+const ArtifactRequirementSchema = z.object({
+  type: z.literal("markdown"),
+  title: z.string().describe("The title of the artifact."),
+  requirement: z
+    .string()
+    .describe("The requirement for the artifact generation."),
+});
+
+const generateArtifactEvent = zodEvent(ArtifactRequirementSchema, {
+  debugLabel: "generateArtifactEvent",
+});
+
+// Artifact output event
+const ArtifactSchema = z.object({
+  type: z.literal("artifact"),
+  data: z.object({
+    type: z.literal("document"),
+    data: z.object({
+      title: z.string().describe("The title of the data."),
+      content: z.string().describe("The content of the data."),
+      type: z.enum(["markdown", "html"]).describe("The type of the data."),
+    }),
+  }),
+});
+const outputArtifactEvent = zodEvent(ArtifactSchema, {
+  debugLabel: "outputArtifactEvent",
+});
+
+// Events for updating UI
+// assume that we have a UI that can render different states of the workflow
+// and update the UI based on the state and the requirement
+export const UIEventSchema = z.object({
+  type: z.literal("ui_event"),
+  data: z.object({
+    state: z
+      .enum(["plan", "generate", "completed"])
+      .describe("The current state of the workflow."),
+    requirement: z
+      .string()
+      .optional()
+      .describe(
+        "An optional requirement creating or updating a document, if applicable.",
+      ),
+  }),
+});
+const uiEvent = zodEvent(UIEventSchema, { debugLabel: "uiEvent" });
+
+// ===== 2. Define workflow with agents using natural language =====
+// We have a document artifact workflow that made up of 2 steps:
+// 1. Generate requirement for the document
+// 2. Generate document content based on the requirement
+export function createDocumentArtifactWorkflow(llm: ToolCallLLM) {
+  const workflow = createWorkflow();
+
+  // Generate requirement for the document
+  workflow.handle(
+    [planEvent],
+    agentHandler({
+      instructions: `
+Your task is to analyze the request and provide requirements for document generation or update.
+1. Send an uiEvent with the \`plan\` to show UI what you are going to do.
+2. Analyze the conversation history and the user's request carefully to determine the completed tasks and the next steps.
+3. Return the generateArtifactEvent with the requirement for the next step of the document generation or update.
+`,
+      results: [generateArtifactEvent],
+      events: [uiEvent],
+      llm,
+    }),
+  );
+
+  // Generate document content based on the requirement
+  workflow.handle(
+    [generateArtifactEvent],
+    agentHandler({
+      instructions: `
+You are a skilled technical writer who can assist users with documentation.
+Your task is to generate document content based on the requirement and update the UI state.
+
+Here are the steps to handle this task:
+1. First, send an uiEvent with the \`generate\` state and the requirement you received from the input.
+2. Next, start generating the content based on the requirement then send an uiEvent with the \`completed\` state to update the state.
+3. Finally, return the outputArtifactEvent with the document values.
+`,
+      results: [outputArtifactEvent],
+      events: [uiEvent],
+      llm,
+    }),
+  );
+
+  return workflow;
+}
+
+async function main() {
+  const llm = openai({ model: "gpt-4.1-mini" });
+  const workflow = createDocumentArtifactWorkflow(llm);
+  const { stream, sendEvent } = workflow.createContext();
+
+  // Ask the workflow to generate a document about `llama`
+  sendEvent(planEvent.with({ topic: "llama" }));
+
+  await stream.until(outputArtifactEvent).forEach((event) => {
+    if (planEvent.include(event)) {
+      console.log("Starting workflow: ", event.data);
+    }
+    if (uiEvent.include(event)) {
+      console.log("UI event: ", event.data);
+    } else if (outputArtifactEvent.include(event)) {
+      console.log("Output artifact event: ", event.data);
+    }
+  });
+}
+
+main();
@@ -0,0 +1,93 @@
+import { openai } from "@llamaindex/openai";
+import { agentHandler, createWorkflow, zodEvent } from "@llamaindex/workflow";
+import { Settings } from "llamaindex";
+import { z } from "zod";
+
+// Create LLM instance
+const llm = openai({ model: "gpt-4.1-mini" });
+Settings.llm = llm;
+
+// Define our workflow events
+const writeJokeSchema = z.object({
+  description: z
+    .string()
+    .describe("The topic to write a joke or describe the joke to improve."),
+  writtenJoke: z.optional(z.string()).describe("The written joke."),
+  retriedTimes: z
+    .number()
+    .default(0)
+    .describe(
+      "The retried times for writing the joke. Always increase this from the input retriedTimes.",
+    ),
+});
+
+const critiqueSchema = z.object({
+  joke: z.string().describe("The joke to critique"),
+  retriedTimes: z.number().describe("The retried times for writing the joke."),
+});
+
+const finalResultSchema = z.object({
+  joke: z.string().describe("The joke to critique"),
+  critique: z.string().describe("The critique of the joke"),
+});
+
+const writeJokeEvent = zodEvent(writeJokeSchema, {
+  debugLabel: "writeJokeEvent",
+}); // Input topic for writing a joke
+const critiqueEvent = zodEvent(critiqueSchema, {
+  debugLabel: "critiqueEvent",
+}); // Ask for critique of the joke
+const finalResultEvent = zodEvent(finalResultSchema, {
+  debugLabel: "finalResultEvent",
+}); // Final result
+
+// Create our workflow
+const jokeFlow = createWorkflow();
+
+// Define handlers for each step
+// This step always write a joke based on the description
+jokeFlow.handle(
+  [writeJokeEvent],
+  agentHandler({
+    instructions: `You are a joke writer. You are given a topic and you need to write a joke about it.`,
+    results: [critiqueEvent],
+  }),
+);
+
+// This step critiques the joke and asks the writer to improve the joke or send a final result event for stopping.
+jokeFlow.handle(
+  [critiqueEvent],
+  agentHandler({
+    instructions: `
+You are given a joke and you need to critique it. Follow the following guidelines:
+1. You have maximum 3 times to improve the joke.
+2. If the joke is not good, increase the retriedTimes, describe how to improve the joke and send a writeJokeEvent.
+3. If the joke is good, trigger the finalResultEvent event.
+`,
+    results: [writeJokeEvent, finalResultEvent],
+  }),
+);
+
+// Usage
+async function main() {
+  const { stream, sendEvent } = jokeFlow.createContext();
+  sendEvent(writeJokeEvent.with({ description: "write a joke about llama" }));
+
+  await stream.until(finalResultEvent).forEach((event) => {
+    if (writeJokeEvent.include(event)) {
+      console.log(
+        "Triggering write joke: ",
+        JSON.stringify(event.data, null, 2),
+      );
+    } else if (critiqueEvent.include(event)) {
+      console.log("Written joke:  ", JSON.stringify(event.data, null, 2));
+    } else if (finalResultEvent.include(event)) {
+      console.log("Output: ", JSON.stringify(event.data, null, 2));
+    } else {
+      console.log("Unknown event: ", JSON.stringify(event.data, null, 2));
+    }
+  });
+  console.log("Done");
+}
+
+main().catch(console.error);
@@ -0,0 +1,72 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<company name="MidSizeCorp" founded="2008">
+  <division name="Engineering" head="Dana White">
+    <department name="Frontend" lead="Alex Kim">
+      <team name="Web">
+        <employee id="E01">
+          <name>Jordan Lee</name>
+          <role>Lead Developer</role>
+          <projects>
+            <project code="PRJ101" status="active">
+              <title>User Portal</title>
+              <deadline>2025-08-01</deadline>
+              <tasks>
+                <task id="T1011">
+                  <description>Implement login page</description>
+                  <due>2025-05-10</due>
+                </task>
+                <task id="T1012">
+                  <description>Design dashboard</description>
+                  <due>2025-05-20</due>
+                </task>
+              </tasks>
+            </project>
+          </projects>
+        </employee>
+        <employee id="E02">
+          <name>Riley Chen</name>
+          <role>UI Designer</role>
+        </employee>
+      </team>
+      <team name="Mobile">
+        <employee id="E03">
+          <name>Sam Patel</name>
+          <role>iOS Developer</role>
+        </employee>
+      </team>
+    </department>
+    <department name="Backend" lead="Morgan Reed">
+      <team name="API">
+        <employee id="E04">
+          <name>Taylor Jones</name>
+          <role>API Engineer</role>
+        </employee>
+      </team>
+      <team name="Database">
+        <employee id="E05">
+          <name>Casey Nguyen</name>
+          <role>DB Administrator</role>
+        </employee>
+      </team>
+    </department>
+  </division>
+
+  <division name="Marketing" head="Pat Morgan">
+    <department name="Digital" lead="Alex Rivera">
+      <team name="Content">
+        <employee id="M01">
+          <name>Charlie Brooks</name>
+          <role>Content Strategist</role>
+        </employee>
+      </team>
+    </department>
+  </division>
+
+  <headquarters location="Chicago, USA">
+    <address>
+      <street>789 Lake Shore Drive</street>
+      <city>Chicago</city>
+      <zip>60601</zip>
+    </address>
+  </headquarters>
+</company>
@@ -1,14 +1,16 @@
-import { Gemini, GEMINI_MODEL } from "@llamaindex/google";
+import { gemini, GEMINI_MODEL } from "@llamaindex/google";
 import fs from "fs";
+import { tool } from "llamaindex";
+import { z } from "zod";

 (async () => {
  if (!process.env.GOOGLE_API_KEY) {
    throw new Error("Please set the GOOGLE_API_KEY environment variable.");
  }
-  const gemini = new Gemini({
-    model: GEMINI_MODEL.GEMINI_PRO_1_5,
-  });
-  const result = await gemini.chat({
+  const llm = gemini({ model: GEMINI_MODEL.GEMINI_2_0_FLASH });
+
+  // normal chat
+  const result = await llm.chat({
    messages: [
      { content: "You want to talk in rhymes.", role: "system" },
      {
@@ -18,10 +20,10 @@ import fs from "fs";
      },
    ],
  });
-  console.log(result);
+  console.log("\n normal chat: \n", result);

  // chat with file
-  const resultWithFile = await gemini.chat({
+  const resultWithFile = await llm.chat({
    messages: [
      {
        role: "user",
@@ -39,6 +41,52 @@ import fs from "fs";
      },
    ],
  });
+  console.log("\n chat with file: \n", resultWithFile);

-  console.log(resultWithFile);
+  // chat with image base64
+  const resultWithImageFile = await llm.chat({
+    messages: [
+      {
+        role: "user",
+        content: [
+          {
+            type: "text",
+            text: "What's in this image?",
+          },
+          {
+            type: "image",
+            data: fs
+              .readFileSync("./multimodal/data/60.jpg")
+              .toString("base64"),
+            mimeType: "image/png",
+          },
+        ],
+      },
+    ],
+  });
+  console.log("\n chat with image base64: \n", resultWithImageFile);
+
+  // chat with tool
+  const resultWithTool = await llm.chat({
+    messages: [
+      {
+        content: "What's the weather in Tokyo?",
+        role: "user",
+      },
+    ],
+    tools: [
+      tool({
+        name: "weather",
+        description: "Get the weather",
+        parameters: z.object({
+          location: z.string().describe("The location to get the weather for"),
+        }),
+        execute: ({ location }) => {
+          console.log("weather", location);
+          return `The weather in ${location} is sunny and hot`;
+        },
+      }),
+    ],
+  });
+  console.log("\n chat with tool: \n", resultWithTool.message.options); // should have toolCall
 })();
@@ -1,11 +1,14 @@
-import { Gemini, GEMINI_MODEL, GeminiVertexSession } from "@llamaindex/google";
+import { gemini, GEMINI_MODEL } from "@llamaindex/google";

 (async () => {
-  const gemini = new Gemini({
-    model: GEMINI_MODEL.GEMINI_PRO,
-    session: new GeminiVertexSession(),
+  const llm = gemini({
+    model: GEMINI_MODEL.GEMINI_2_0_FLASH,
+    vertex: {
+      project: "your-cloud-project", // update to your cloud project
+      location: "us-central1",
+    },
  });
-  const result = await gemini.chat({
+  const result = await llm.chat({
    messages: [
      { content: "You want to talk in rhymes.", role: "system" },
      {
@@ -16,9 +16,19 @@ async function main() {

  console.log("🚀 Initializing Gemini Live API example...");

+  // Server-side (token creation):
+  const serverllm = gemini({
+    model: GEMINI_MODEL.GEMINI_2_0_FLASH_LIVE,
+    httpOptions: { apiVersion: "v1alpha" }, // must use v1alpha to generate ephemeral key
+  });
+  const ephemeralKey = await serverllm.live.getEphemeralKey();
+
+  // Client-side (Live API connection):
  const llm = gemini({
+    apiKey: ephemeralKey, // use ephemeral key for client-side
    model: GEMINI_MODEL.GEMINI_2_0_FLASH_LIVE,
    voiceName: "Zephyr",
+    httpOptions: { apiVersion: "v1alpha" }, // must use v1alpha to init client with ephemeral key
  });

  console.log("📡 Connecting to Gemini Live session...");
@@ -3,8 +3,18 @@ import { liveEvents } from "llamaindex";
 import { saveWavFile } from "./util";

 async function main() {
-  const llm = gemini({
+  // Server-side (token creation):
+  const serverllm = gemini({
    model: GEMINI_MODEL.GEMINI_2_0_FLASH_LIVE,
+    httpOptions: { apiVersion: "v1alpha" }, // must use v1alpha to generate ephemeral key
+  });
+  const ephemeralKey = await serverllm.live.getEphemeralKey();
+
+  // Client-side (Live API connection):
+  const llm = gemini({
+    apiKey: ephemeralKey, // use ephemeral key for client-side
+    model: GEMINI_MODEL.GEMINI_2_0_FLASH_LIVE,
+    httpOptions: { apiVersion: "v1alpha" }, // must use v1alpha to init client with ephemeral key
  });

  const session = await llm.live.connect();
@@ -23,10 +33,7 @@ async function main() {
        content: "Say something about you for 10 seconds",
        role: "user",
      });
-    } else if (
-      liveEvents.audio.include(event) &&
-      typeof event.data === "string"
-    ) {
+    } else if (liveEvents.audio.include(event)) {
      const chunk = Buffer.from(event.data, "base64");
      audioChunks.push(chunk);
      console.log(`Received audio chunk: ${chunk.length} bytes`);
@@ -1,6 +1,5 @@
-import { ModalityType } from "@llamaindex/core/schema";
-import { tool } from "@llamaindex/core/tools";
 import { gemini, GEMINI_MODEL } from "@llamaindex/google";
+import { ModalityType, tool } from "llamaindex";

 import { liveEvents } from "llamaindex";
 import { z } from "zod";
@@ -0,0 +1,24 @@
+# Logs
+logs
+*.log
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+pnpm-debug.log*
+lerna-debug.log*
+
+node_modules
+dist
+dist-ssr
+*.local
+
+# Editor directories and files
+.vscode/*
+!.vscode/extensions.json
+.idea
+.DS_Store
+*.suo
+*.ntvs*
+*.njsproj
+*.sln
+*.sw?
@@ -0,0 +1,54 @@
+# OpenAI Realtime Chat with LlamaIndex
+
+This is a demo application showcasing real-time audio and text chat capabilities using OpenAI's GPT-4 with voice through LlamaIndex. The application demonstrates bidirectional audio communication and text chat with an AI assistant.
+
+## Features
+
+- Real-time voice communication with GPT-4
+- Text-based chat interface
+- WebRTC-based audio streaming
+- Bidirectional communication (both text and voice)
+- React + TypeScript implementation
+
+## Prerequisites
+
+- Node.js (v18 or higher)
+- OpenAI API key with access to GPT-4 voice models
+- Modern browser with WebRTC support
+
+## Getting Started
+
+1. Install dependencies:
+
+```bash
+pnpm install
+```
+
+2. Start the development server:
+
+```bash
+pnpm run dev
+```
+
+## Usage
+
+The application provides a simple interface where you can:
+
+- Start/Stop a chat session
+- Speak to the AI assistant through your microphone
+- Receive audio responses from the assistant
+- See text transcripts of the conversation
+
+## Technical Details
+
+This project uses:
+
+- LlamaIndex for AI interaction management
+- WebRTC for real-time audio streaming
+- React for the UI
+- Vite for development and building
+- TypeScript for type safety
+
+```
+
+```
@@ -0,0 +1,28 @@
+import js from "@eslint/js";
+import reactHooks from "eslint-plugin-react-hooks";
+import reactRefresh from "eslint-plugin-react-refresh";
+import globals from "globals";
+import tseslint from "typescript-eslint";
+
+export default tseslint.config(
+  { ignores: ["dist"] },
+  {
+    extends: [js.configs.recommended, ...tseslint.configs.recommended],
+    files: ["**/*.{ts,tsx}"],
+    languageOptions: {
+      ecmaVersion: 2020,
+      globals: globals.browser,
+    },
+    plugins: {
+      "react-hooks": reactHooks,
+      "react-refresh": reactRefresh,
+    },
+    rules: {
+      ...reactHooks.configs.recommended.rules,
+      "react-refresh/only-export-components": [
+        "warn",
+        { allowConstantExport: true },
+      ],
+    },
+  },
+);
@@ -0,0 +1,13 @@
+<!doctype html>
+<html lang="en">
+  <head>
+    <meta charset="UTF-8" />
+    <link rel="icon" type="image/svg+xml" href="/vite.svg" />
+    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+    <title>Vite + React + TS</title>
+  </head>
+  <body>
+    <div id="root"></div>
+    <script type="module" src="/src/main.tsx"></script>
+  </body>
+</html>
@@ -0,0 +1,29 @@
+{
+  "name": "open-ai-realtime",
+  "private": true,
+  "version": "0.0.0",
+  "type": "module",
+  "scripts": {
+    "dev": "vite",
+    "build": "tsc -b && vite build",
+    "lint": "eslint .",
+    "preview": "vite preview"
+  },
+  "dependencies": {
+    "react": "^19.1.0",
+    "react-dom": "^19.1.0"
+  },
+  "devDependencies": {
+    "@eslint/js": "^9.25.0",
+    "@types/react": "^19.1.2",
+    "@types/react-dom": "^19.1.2",
+    "@vitejs/plugin-react": "^4.5.2",
+    "eslint": "^9.25.0",
+    "eslint-plugin-react-hooks": "^5.2.0",
+    "eslint-plugin-react-refresh": "^0.4.19",
+    "globals": "^16.0.0",
+    "typescript": "~5.8.3",
+    "typescript-eslint": "^8.30.1",
+    "vite": "^6.3.5"
+  }
+}
@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" class="iconify iconify--logos" width="31.88" height="32" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 257"><defs><linearGradient id="IconifyId1813088fe1fbc01fb466" x1="-.828%" x2="57.636%" y1="7.652%" y2="78.411%"><stop offset="0%" stop-color="#41D1FF"></stop><stop offset="100%" stop-color="#BD34FE"></stop></linearGradient><linearGradient id="IconifyId1813088fe1fbc01fb467" x1="43.376%" x2="50.316%" y1="2.242%" y2="89.03%"><stop offset="0%" stop-color="#FFEA83"></stop><stop offset="8.333%" stop-color="#FFDD35"></stop><stop offset="100%" stop-color="#FFA800"></stop></linearGradient></defs><path fill="url(#IconifyId1813088fe1fbc01fb466)" d="M255.153 37.938L134.897 252.976c-2.483 4.44-8.862 4.466-11.382.048L.875 37.958c-2.746-4.814 1.371-10.646 6.827-9.67l120.385 21.517a6.537 6.537 0 0 0 2.322-.004l117.867-21.483c5.438-.991 9.574 4.796 6.877 9.62Z"></path><path fill="url(#IconifyId1813088fe1fbc01fb467)" d="M185.432.063L96.44 17.501a3.268 3.268 0 0 0-2.634 3.014l-5.474 92.456a3.268 3.268 0 0 0 3.997 3.378l24.777-5.718c2.318-.535 4.413 1.507 3.936 3.838l-7.361 36.047c-.495 2.426 1.782 4.5 4.151 3.78l15.304-4.649c2.372-.72 4.652 1.36 4.15 3.788l-11.698 56.621c-.732 3.542 3.979 5.473 5.943 2.437l1.313-2.028l72.516-144.72c1.215-2.423-.88-5.186-3.54-4.672l-25.505 4.922c-2.396.462-4.435-1.77-3.759-4.114l16.646-57.705c.677-2.35-1.37-4.583-3.769-4.113Z"></path></svg>
@@ -0,0 +1,183 @@
+import { openai } from "@llamaindex/openai";
+import { liveEvents, LiveLLMSession, ModalityType } from "llamaindex";
+import { useEffect, useRef, useState } from "react";
+
+const MicIcon = ({ isConnected }: { isConnected: boolean }) => (
+  <svg
+    xmlns="http://www.w3.org/2000/svg"
+    viewBox="0 0 24 24"
+    fill="currentColor"
+    strokeWidth="2"
+    strokeLinecap="round"
+    strokeLinejoin="round"
+  >
+    {isConnected ? (
+      <>
+        <path d="M12 1a3 3 0 0 0-3 3v8a3 3 0 0 0 6 0V4a3 3 0 0 0-3-3z" />
+        <path d="M19 10v2a7 7 0 0 1-14 0v-2" />
+        <line x1="12" y1="19" x2="12" y2="23" />
+        <line x1="8" y1="23" x2="16" y2="23" />
+      </>
+    ) : (
+      <>
+        <path d="M12 1a3 3 0 0 0-3 3v8a3 3 0 0 0 6 0V4a3 3 0 0 0-3-3z" />
+        <path d="M19 10v2a7 7 0 0 1-14 0v-2" />
+      </>
+    )}
+  </svg>
+);
+
+const WaveAnimation = () => (
+  <div className="wave-animation">
+    {[...Array(3)].map((_, i) => (
+      <div key={i} className="wave" style={{ animationDelay: `${i * 0.2}s` }} />
+    ))}
+  </div>
+);
+
+export const AudioChat = () => {
+  const [isConnected, setIsConnected] = useState(false);
+  const [messages, setMessages] = useState<
+    Array<{ role: string; content: string }>
+  >([]);
+  const [status, setStatus] = useState<string>("");
+  const audioRef = useRef<HTMLAudioElement>(null);
+  const sessionRef = useRef<LiveLLMSession | null>(null);
+  const [stream, setStream] = useState<MediaStream | null>(null);
+  const messagesEndRef = useRef<HTMLDivElement>(null);
+
+  const scrollToBottom = () => {
+    messagesEndRef.current?.scrollIntoView({ behavior: "smooth" });
+  };
+
+  useEffect(() => {
+    scrollToBottom();
+  }, [messages]);
+
+  useEffect(() => {
+    return () => {
+      if (stream) {
+        stream.getTracks().forEach((track) => track.stop());
+      }
+    };
+  }, [stream]);
+
+  const startChat = async () => {
+    try {
+      setStatus("Initializing microphone...");
+      const userStream = await navigator.mediaDevices.getUserMedia({
+        audio: true,
+      });
+      setStream(userStream);
+
+      setStatus("Connecting to AI...");
+      const apiKey = prompt("Please enter your OpenAI API key:");
+      if (!apiKey) {
+        throw new Error("API key is required");
+      }
+
+      // move this call to the server side for security reasons
+      // Do not store the API key in the frontend!
+      const serverllm = openai({
+        apiKey: apiKey,
+        model: "gpt-4o-realtime-preview-2025-06-03",
+      });
+
+      const tempKey = await serverllm.live.getEphemeralKey();
+
+      const llm = openai({
+        apiKey: tempKey,
+        model: "gpt-4o-realtime-preview-2025-06-03",
+      });
+      const session = await llm.live.connect({
+        systemInstruction: "You are a helpful assistant who speaks naturally.",
+        responseModality: [ModalityType.TEXT, ModalityType.AUDIO],
+        audioConfig: {
+          stream: userStream,
+          onTrack: (remoteStream) => {
+            if (audioRef.current && remoteStream) {
+              audioRef.current.srcObject = remoteStream;
+              audioRef.current.play().catch(console.error);
+            }
+          },
+        },
+      });
+
+      sessionRef.current = session;
+      setIsConnected(true);
+      setStatus("Connected! Listening...");
+
+      for await (const event of session.streamEvents()) {
+        if (liveEvents.open.include(event)) {
+          setMessages((prev) => [
+            ...prev,
+            {
+              role: "user",
+              content: "Hello, I'm ready to chat!",
+            },
+          ]);
+          session.sendMessage({
+            content: "Hello, I'm ready to chat!",
+            role: "user",
+          });
+        } else if (liveEvents.text.include(event)) {
+          setMessages((prev) => [
+            ...prev,
+            {
+              role: "assistant",
+              content: event.text,
+            },
+          ]);
+        }
+      }
+    } catch (error) {
+      console.error("Error starting chat:", error);
+      setStatus("Error connecting. Please try again.");
+      setIsConnected(false);
+    }
+  };
+
+  const stopChat = async () => {
+    setStatus("Disconnecting...");
+    if (sessionRef.current) {
+      await sessionRef.current.disconnect();
+      sessionRef.current = null;
+    }
+    if (stream) {
+      stream.getTracks().forEach((track) => track.stop());
+      setStream(null);
+    }
+    if (audioRef.current) {
+      audioRef.current.srcObject = null;
+    }
+    setIsConnected(false);
+    setStatus("");
+  };
+
+  return (
+    <div className="audio-chat-container">
+      <h1>AI Voice Chat</h1>
+      <div className="messages-container">
+        {messages.map((msg, idx) => (
+          <div key={idx} className={`message ${msg.role}`}>
+            {msg.content}
+          </div>
+        ))}
+        <div ref={messagesEndRef} />
+      </div>
+
+      <div className="controls">
+        {status && <div className="status-indicator">{status}</div>}
+        <button
+          className={`mic-button ${isConnected ? "connected" : ""}`}
+          onClick={isConnected ? stopChat : startChat}
+          title={isConnected ? "Stop Chat" : "Start Chat"}
+        >
+          <MicIcon isConnected={isConnected} />
+          {isConnected && <WaveAnimation />}
+        </button>
+        <audio ref={audioRef} style={{ display: "none" }} />
+      </div>
+    </div>
+  );
+};
@@ -0,0 +1,322 @@
+:root {
+  --primary-color: #646cff;
+  --secondary-color: #535bf2;
+  --background-dark: #1a1a1a;
+  --chat-bg: #242424;
+  --text-primary: #ffffff;
+  --text-secondary: #888888;
+  --success-color: #4caf50;
+  --error-color: #f44336;
+  --gradient-start: #4776e6;
+  --gradient-end: #8e54e9;
+}
+
+body {
+  background-color: var(--background-dark);
+  color: var(--text-primary);
+  margin: 0;
+  min-height: 100vh;
+  font-family:
+    -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, Oxygen, Ubuntu,
+    Cantarell, "Open Sans", "Helvetica Neue", sans-serif;
+}
+
+#root {
+  max-width: 1280px;
+  height: 100vh;
+  margin: 0 auto;
+  padding: 2rem;
+  display: flex;
+  align-items: center;
+  justify-content: center;
+}
+
+.logo {
+  height: 6em;
+  padding: 1.5em;
+  will-change: filter;
+  transition: filter 300ms;
+}
+.logo:hover {
+  filter: drop-shadow(0 0 2em #646cffaa);
+}
+.logo.react:hover {
+  filter: drop-shadow(0 0 2em #61dafbaa);
+}
+
+@keyframes logo-spin {
+  from {
+    transform: rotate(0deg);
+  }
+  to {
+    transform: rotate(360deg);
+  }
+}
+
+@media (prefers-reduced-motion: no-preference) {
+  a:nth-of-type(2) .logo {
+    animation: logo-spin infinite 20s linear;
+  }
+}
+
+.card {
+  padding: 2em;
+}
+
+.read-the-docs {
+  color: #888;
+}
+
+.audio-chat-container {
+  display: flex;
+  flex-direction: column;
+  gap: 2rem;
+  width: 100%;
+  max-width: 800px;
+  height: 80vh;
+  margin: 0 auto;
+  padding: 2rem;
+  background: var(--chat-bg);
+  border-radius: 24px;
+  box-shadow: 0 8px 32px rgba(0, 0, 0, 0.2);
+  position: relative;
+  overflow: hidden;
+}
+
+.audio-chat-container::before {
+  content: "";
+  position: absolute;
+  top: 0;
+  left: 0;
+  right: 0;
+  height: 4px;
+  background: linear-gradient(
+    to right,
+    var(--gradient-start),
+    var(--gradient-end)
+  );
+}
+
+.audio-chat-container h1 {
+  font-size: 2.5rem;
+  margin: 0;
+  background: linear-gradient(
+    to right,
+    var(--gradient-start),
+    var(--gradient-end)
+  );
+  -webkit-background-clip: text;
+  background-clip: text;
+  color: transparent;
+  text-align: center;
+}
+
+.messages-container {
+  display: flex;
+  flex-direction: column;
+  gap: 1rem;
+  flex: 1;
+  overflow-y: auto;
+  padding: 1rem;
+  border-radius: 16px;
+  background: rgba(255, 255, 255, 0.05);
+  backdrop-filter: blur(10px);
+  margin: 1rem 0;
+}
+
+.message {
+  padding: 1rem 1.5rem;
+  border-radius: 16px;
+  max-width: 80%;
+  text-align: left;
+  animation: messageSlide 0.3s ease-out;
+  box-shadow: 0 2px 4px rgba(0, 0, 0, 0.1);
+}
+
+@keyframes messageSlide {
+  from {
+    opacity: 0;
+    transform: translateY(20px);
+  }
+  to {
+    opacity: 1;
+    transform: translateY(0);
+  }
+}
+
+.message.user {
+  background: linear-gradient(
+    135deg,
+    var(--gradient-start),
+    var(--gradient-end)
+  );
+  align-self: flex-end;
+  margin-left: 20%;
+  color: white;
+}
+
+.message.assistant {
+  background: rgba(255, 255, 255, 0.1);
+  align-self: flex-start;
+  margin-right: 20%;
+  border: 1px solid rgba(255, 255, 255, 0.1);
+}
+
+.controls {
+  display: flex;
+  justify-content: center;
+  align-items: center;
+  padding: 2rem;
+  position: relative;
+}
+
+.mic-button {
+  width: 80px;
+  height: 80px;
+  border-radius: 50%;
+  border: none;
+  background: linear-gradient(
+    135deg,
+    var(--gradient-start),
+    var(--gradient-end)
+  );
+  color: white;
+  cursor: pointer;
+  transition: all 0.3s cubic-bezier(0.4, 0, 0.2, 1);
+  display: flex;
+  align-items: center;
+  justify-content: center;
+  box-shadow: 0 4px 15px rgba(0, 0, 0, 0.2);
+  position: relative;
+  overflow: hidden;
+}
+
+.mic-button::before {
+  content: "";
+  position: absolute;
+  top: 0;
+  left: 0;
+  right: 0;
+  bottom: 0;
+  background: linear-gradient(
+    135deg,
+    rgba(255, 255, 255, 0.1),
+    rgba(255, 255, 255, 0)
+  );
+  border-radius: 50%;
+  transition: transform 0.3s ease;
+}
+
+.mic-button:hover {
+  transform: scale(1.05);
+  box-shadow: 0 8px 25px rgba(0, 0, 0, 0.3);
+}
+
+.mic-button:hover::before {
+  transform: translateY(-100%);
+}
+
+.mic-button.connected {
+  background: var(--error-color);
+  animation: pulseError 2s infinite;
+}
+
+.mic-button svg {
+  width: 32px;
+  height: 32px;
+  filter: drop-shadow(0 2px 4px rgba(0, 0, 0, 0.2));
+  transition: transform 0.3s ease;
+}
+
+.mic-button:hover svg {
+  transform: scale(1.1);
+}
+
+@keyframes pulseError {
+  0% {
+    box-shadow: 0 0 0 0 rgba(244, 67, 54, 0.4);
+  }
+  70% {
+    box-shadow: 0 0 0 20px rgba(244, 67, 54, 0);
+  }
+  100% {
+    box-shadow: 0 0 0 0 rgba(244, 67, 54, 0);
+  }
+}
+
+/* Status indicator */
+.status-indicator {
+  position: absolute;
+  top: -30px;
+  left: 50%;
+  transform: translateX(-50%);
+  font-size: 0.9rem;
+  color: var(--text-secondary);
+  opacity: 0;
+  transition: opacity 0.3s ease;
+}
+
+.controls:hover .status-indicator {
+  opacity: 1;
+}
+
+/* Scrollbar styling */
+.messages-container::-webkit-scrollbar {
+  width: 8px;
+}
+
+.messages-container::-webkit-scrollbar-track {
+  background: rgba(255, 255, 255, 0.05);
+  border-radius: 4px;
+}
+
+.messages-container::-webkit-scrollbar-thumb {
+  background: linear-gradient(var(--gradient-start), var(--gradient-end));
+  border-radius: 4px;
+}
+
+.messages-container::-webkit-scrollbar-thumb:hover {
+  background: linear-gradient(var(--gradient-end), var(--gradient-start));
+}
+
+/* Wave Animation */
+.wave-animation {
+  position: absolute;
+  bottom: -15px;
+  left: 50%;
+  transform: translateX(-50%);
+  display: flex;
+  gap: 4px;
+}
+
+.wave {
+  width: 4px;
+  height: 15px;
+  background: currentColor;
+  border-radius: 2px;
+  animation: wave 0.5s ease-in-out infinite;
+}
+
+@keyframes wave {
+  0%,
+  100% {
+    transform: scaleY(0.5);
+  }
+  50% {
+    transform: scaleY(1.5);
+  }
+}
+
+/* Loading state */
+.mic-button.loading {
+  animation: rotate 1s linear infinite;
+}
+
+@keyframes rotate {
+  from {
+    transform: rotate(0deg);
+  }
+  to {
+    transform: rotate(360deg);
+  }
+}
@@ -0,0 +1,10 @@
+import { StrictMode } from "react";
+import { createRoot } from "react-dom/client";
+import { AudioChat } from "./audio-chat.tsx";
+import "./index.css";
+
+createRoot(document.getElementById("root")!).render(
+  <StrictMode>
+    <AudioChat />
+  </StrictMode>,
+);
@@ -0,0 +1 @@
+/// <reference types="vite/client" />
@@ -0,0 +1,26 @@
+{
+  "compilerOptions": {
+    "tsBuildInfoFile": "./node_modules/.tmp/tsconfig.app.tsbuildinfo",
+    "target": "ES2020",
+    "useDefineForClassFields": true,
+    "lib": ["ES2020", "DOM", "DOM.Iterable"],
+    "module": "ESNext",
+    "skipLibCheck": true,
+
+    /* Bundler mode */
+    "moduleResolution": "bundler",
+    "allowImportingTsExtensions": true,
+    "verbatimModuleSyntax": true,
+    "moduleDetection": "force",
+    "noEmit": true,
+    "jsx": "react-jsx",
+
+    /* Linting */
+    "strict": true,
+    "noUnusedLocals": true,
+    "noUnusedParameters": true,
+    "noFallthroughCasesInSwitch": true,
+    "noUncheckedSideEffectImports": true
+  },
+  "include": ["src"]
+}
@@ -0,0 +1,4 @@
+{
+  "files": [],
+  "references": [{ "path": "./tsconfig.app.json" }]
+}
@@ -0,0 +1,7 @@
+import react from "@vitejs/plugin-react";
+import { defineConfig } from "vite";
+
+// https://vite.dev/config/
+export default defineConfig({
+  plugins: [react()],
+});
@@ -1,6 +1,6 @@
 {
  "name": "@llamaindex/examples",
-  "version": "0.3.19",
+  "version": "0.3.25",
  "private": true,
  "scripts": {
    "lint": "eslint .",
@@ -11,51 +11,51 @@
    "@azure/cosmos": "^4.1.1",
    "@azure/identity": "^4.4.1",
    "@azure/search-documents": "^12.1.0",
-    "@llamaindex/anthropic": "^0.3.11",
-    "@llamaindex/assemblyai": "^0.1.8",
-    "@llamaindex/astra": "^0.0.23",
-    "@llamaindex/azure": "^0.1.20",
-    "@llamaindex/chroma": "^0.0.23",
-    "@llamaindex/clip": "^0.0.59",
-    "@llamaindex/cloud": "^4.0.13",
-    "@llamaindex/cohere": "^0.0.23",
-    "@llamaindex/core": "^0.6.9",
-    "@llamaindex/deepinfra": "^0.0.59",
-    "@llamaindex/deepseek": "^0.0.19",
-    "@llamaindex/discord": "^0.1.8",
-    "@llamaindex/elastic-search": "^0.1.9",
+    "@llamaindex/anthropic": "^0.3.14",
+    "@llamaindex/assemblyai": "^0.1.11",
+    "@llamaindex/astra": "^0.0.26",
+    "@llamaindex/azure": "^0.1.23",
+    "@llamaindex/chroma": "^0.0.26",
+    "@llamaindex/clip": "^0.0.62",
+    "@llamaindex/cloud": "^4.0.16",
+    "@llamaindex/cohere": "^0.0.26",
+    "@llamaindex/core": "^0.6.12",
+    "@llamaindex/deepinfra": "^0.0.62",
+    "@llamaindex/deepseek": "^0.0.22",
+    "@llamaindex/discord": "^0.1.11",
+    "@llamaindex/elastic-search": "^0.1.12",
    "@llamaindex/env": "^0.1.30",
-    "@llamaindex/firestore": "^1.0.16",
-    "@llamaindex/fireworks": "^0.0.19",
-    "@llamaindex/google": "^0.3.6",
-    "@llamaindex/groq": "^0.0.74",
-    "@llamaindex/huggingface": "^0.1.13",
-    "@llamaindex/jinaai": "^0.0.19",
-    "@llamaindex/milvus": "^0.1.18",
-    "@llamaindex/mistral": "^0.1.9",
-    "@llamaindex/mixedbread": "^0.0.23",
-    "@llamaindex/mongodb": "^0.0.24",
-    "@llamaindex/node-parser": "^2.0.9",
-    "@llamaindex/notion": "^0.1.8",
-    "@llamaindex/ollama": "^0.1.9",
-    "@llamaindex/openai": "^0.4.3",
-    "@llamaindex/perplexity": "^0.0.16",
-    "@llamaindex/pinecone": "^0.1.9",
-    "@llamaindex/portkey-ai": "^0.0.51",
-    "@llamaindex/postgres": "^0.0.52",
-    "@llamaindex/qdrant": "^0.1.19",
-    "@llamaindex/readers": "^3.1.7",
-    "@llamaindex/replicate": "^0.0.51",
-    "@llamaindex/supabase": "^0.1.8",
-    "@llamaindex/together": "^0.0.19",
-    "@llamaindex/tools": "^0.0.14",
-    "@llamaindex/upstash": "^0.0.23",
-    "@llamaindex/vercel": "^0.1.9",
-    "@llamaindex/vllm": "^0.0.45",
-    "@llamaindex/voyage-ai": "^1.0.15",
-    "@llamaindex/weaviate": "^0.0.23",
-    "@llamaindex/workflow": "^1.1.6",
-    "@llamaindex/xai": "workspace:^0.0.6",
+    "@llamaindex/firestore": "^1.0.19",
+    "@llamaindex/fireworks": "^0.0.22",
+    "@llamaindex/google": "^0.3.11",
+    "@llamaindex/groq": "^0.0.77",
+    "@llamaindex/huggingface": "^0.1.16",
+    "@llamaindex/jinaai": "^0.0.22",
+    "@llamaindex/milvus": "^0.1.21",
+    "@llamaindex/mistral": "^0.1.12",
+    "@llamaindex/mixedbread": "^0.0.26",
+    "@llamaindex/mongodb": "^0.0.27",
+    "@llamaindex/node-parser": "^2.0.12",
+    "@llamaindex/notion": "^0.1.11",
+    "@llamaindex/ollama": "^0.1.12",
+    "@llamaindex/openai": "^0.4.6",
+    "@llamaindex/perplexity": "^0.0.19",
+    "@llamaindex/pinecone": "^0.1.12",
+    "@llamaindex/portkey-ai": "^0.0.54",
+    "@llamaindex/postgres": "^0.0.55",
+    "@llamaindex/qdrant": "^0.1.22",
+    "@llamaindex/readers": "^3.1.11",
+    "@llamaindex/replicate": "^0.0.54",
+    "@llamaindex/supabase": "^0.1.12",
+    "@llamaindex/together": "^0.0.22",
+    "@llamaindex/tools": "^0.1.1",
+    "@llamaindex/upstash": "^0.0.26",
+    "@llamaindex/vercel": "^0.1.12",
+    "@llamaindex/vllm": "^0.0.48",
+    "@llamaindex/voyage-ai": "^1.0.18",
+    "@llamaindex/weaviate": "^0.0.27",
+    "@llamaindex/workflow": "^1.1.12",
+    "@llamaindex/xai": "workspace:^0.0.9",
    "@notionhq/client": "^2.2.15",
    "@pinecone-database/pinecone": "^4.0.0",
    "@vercel/postgres": "^0.10.0",
@@ -64,11 +64,11 @@
    "commander": "^12.1.0",
    "dotenv": "^16.4.5",
    "js-tiktoken": "^1.0.14",
-    "llamaindex": "^0.11.5",
+    "llamaindex": "^0.11.11",
    "mongodb": "6.7.0",
    "postgres": "^3.4.4",
    "wikipedia": "^2.1.2",
-    "zod": "^3.23.8"
+    "zod": "^3.25.67"
  },
  "devDependencies": {
    "@types/node": "^22.9.0",
@@ -1,3 +1,4 @@
+import { OpenAIEmbedding } from "@llamaindex/openai";
 import {
  Document,
  SentenceSplitter,
@@ -7,6 +8,10 @@ import {
 import { OldSentenceSplitter } from "./old-sentence-splitter";
 export const STORAGE_DIR = "./data";

+Settings.embedModel = new OpenAIEmbedding({
+  model: "text-embedding-3-small",
+});
+
 // Update node parser
 (async () => {
  // generate a document with a very long sentence (9000 words long)
@@ -15,11 +15,14 @@
    "start:llamaparse-json": "node --import tsx ./src/llamaparse-json.ts",
    "start:discord": "node --import tsx ./src/discord.ts",
    "start:json": "node --import tsx ./src/json.ts",
-    "start:obsidian": "node --import tsx ./src/obsidian.ts"
+    "start:obsidian": "node --import tsx ./src/obsidian.ts",
+    "start:xml": "node --import tsx ./src/xml.ts",
+    "start:excel": "node --import tsx ./src/excel.ts"
  },
  "dependencies": {
    "@llamaindex/cloud": "workspace:* || ^2.0.24",
    "@llamaindex/readers": "workspace:* || ^1.0.25",
+    "@llamaindex/excel": "workspace:*",
    "llamaindex": "workspace:* || ^0.8.37"
  },
  "devDependencies": {
@@ -0,0 +1,20 @@
+import { ExcelReader } from "@llamaindex/excel";
+
+async function main() {
+  // Load PDF
+  const reader = new ExcelReader({
+    sheetSpecifier: 0,
+    concatRows: true,
+    fieldSeparator: ",",
+    keyValueSeparator: ":",
+  });
+
+  const documents = await reader.loadData("../data/sample_excel_sheet.xls");
+
+  for (const doc of documents) {
+    console.log(doc.text);
+    console.log("----");
+  }
+}
+
+main().catch(console.error);
@@ -0,0 +1,16 @@
+import { XMLReader } from "@llamaindex/readers/xml";
+
+async function main() {
+  // Load PDF
+  const reader = new XMLReader({
+    splitLevel: 2,
+  });
+  const documents = await reader.loadData("../data/company.xml");
+
+  for (const doc of documents) {
+    console.log(doc.text);
+    console.log("----");
+  }
+}
+
+main().catch(console.error);
@@ -1,8 +1,4 @@
-import {
-  GEMINI_EMBEDDING_MODEL,
-  GeminiEmbedding,
-  GeminiSession,
-} from "@llamaindex/google";
+import { GEMINI_EMBEDDING_MODEL, GeminiEmbedding } from "@llamaindex/google";
 import { QdrantVectorStore } from "@llamaindex/qdrant";
 import {
  Document,
@@ -12,9 +8,6 @@ import {

 const embedding = new GeminiEmbedding({
  model: GEMINI_EMBEDDING_MODEL.EMBEDDING_001,
-  session: new GeminiSession({
-    apiKey: process.env.GEMINI_API_KEY,
-  }),
 });

 async function main() {
@@ -18,5 +18,11 @@
      "module": "commonjs"
    }
  },
-  "include": ["./**/*.ts"]
+  "include": ["./**/*.ts"],
+  "exclude": [
+    "node_modules",
+    "dist",
+    "models/openai/live/browser/open-ai-realtime",
+    "**/browser/**"
+  ]
 }
@@ -1,5 +1,43 @@
 # @llamaindex/autotool

+## 8.0.11
+
+### Patch Changes
+
+- Updated dependencies [7039e1a]
+  - llamaindex@0.11.11
+
+## 8.0.10
+
+### Patch Changes
+
+- llamaindex@0.11.10
+
+## 8.0.9
+
+### Patch Changes
+
+- llamaindex@0.11.9
+
+## 8.0.8
+
+### Patch Changes
+
+- llamaindex@0.11.8
+
+## 8.0.7
+
+### Patch Changes
+
+- Updated dependencies [3c857f4]
+  - llamaindex@0.11.7
+
+## 8.0.6
+
+### Patch Changes
+
+- llamaindex@0.11.6
+
 ## 8.0.5

 ### Patch Changes
@@ -1,5 +1,49 @@
 # @llamaindex/autotool-01-node-example

+## 0.0.119
+
+### Patch Changes
+
+- Updated dependencies [7039e1a]
+  - llamaindex@0.11.11
+  - @llamaindex/autotool@8.0.11
+
+## 0.0.118
+
+### Patch Changes
+
+- llamaindex@0.11.10
+- @llamaindex/autotool@8.0.10
+
+## 0.0.117
+
+### Patch Changes
+
+- llamaindex@0.11.9
+- @llamaindex/autotool@8.0.9
+
+## 0.0.116
+
+### Patch Changes
+
+- llamaindex@0.11.8
+- @llamaindex/autotool@8.0.8
+
+## 0.0.115
+
+### Patch Changes
+
+- Updated dependencies [3c857f4]
+  - llamaindex@0.11.7
+  - @llamaindex/autotool@8.0.7
+
+## 0.0.114
+
+### Patch Changes
+
+- llamaindex@0.11.6
+- @llamaindex/autotool@8.0.6
+
 ## 0.0.113

 ### Patch Changes
@@ -13,5 +13,5 @@
  "scripts": {
    "start": "node --import tsx --import @llamaindex/autotool/node ./src/index.ts"
  },
-  "version": "0.0.113"
+  "version": "0.0.119"
 }
@@ -6,7 +6,7 @@
    "url": "git+https://github.com/run-llama/LlamaIndexTS.git",
    "directory": "packages/autotool"
  },
-  "version": "8.0.5",
+  "version": "8.0.11",
  "description": "auto transpile your JS function to LLM Agent compatible",
  "files": [
    "dist",
@@ -1,5 +1,30 @@
 # @llamaindex/cloud

+## 4.0.16
+
+### Patch Changes
+
+- Updated dependencies [7039e1a]
+- Updated dependencies [7039e1a]
+  - @llamaindex/core@0.6.12
+
+## 4.0.15
+
+### Patch Changes
+
+- Updated dependencies [a89e187]
+- Updated dependencies [62699b7]
+- Updated dependencies [c5b2691]
+- Updated dependencies [d8ac8d3]
+  - @llamaindex/core@0.6.11
+
+## 4.0.14
+
+### Patch Changes
+
+- Updated dependencies [1b5af14]
+  - @llamaindex/core@0.6.10
+
 ## 4.0.13

 ### Patch Changes
@@ -1,6 +1,6 @@
 {
  "name": "@llamaindex/cloud",
-  "version": "4.0.13",
+  "version": "4.0.16",
  "type": "module",
  "license": "MIT",
  "scripts": {
@@ -79,6 +79,6 @@
  },
  "dependencies": {
    "p-retry": "^6.2.1",
-    "zod": "^3.25.7"
+    "zod": "^3.25.67"
  }
 }
@@ -1,5 +1,27 @@
 # @llamaindex/core

+## 0.6.12
+
+### Patch Changes
+
+- 7039e1a: Internal cleanup of base64 encoding
+- 7039e1a: chore: migrate to @google/genai SDK
+
+## 0.6.11
+
+### Patch Changes
+
+- a89e187: Feat: added custom abbreviations to sentence splitter
+- 62699b7: Improve performance of sentence splitter
+- c5b2691: Add more Acronyms on SentenceSplitter
+- d8ac8d3: Feat: add support for openai realtime API
+
+## 0.6.10
+
+### Patch Changes
+
+- 1b5af14: fix: jsonToNode for image nodes
+
 ## 0.6.9

 ### Patch Changes
@@ -1,7 +1,7 @@
 {
  "name": "@llamaindex/core",
  "type": "module",
-  "version": "0.6.9",
+  "version": "0.6.12",
  "description": "LlamaIndex Core Module",
  "exports": {
    "./agent": {
@@ -312,7 +312,7 @@
    "@llamaindex/env": "workspace:*",
    "@types/node": "^22.9.0",
    "magic-bytes.js": "^1.10.0",
-    "zod": "^3.23.8",
-    "zod-to-json-schema": "^3.23.3"
+    "zod": "^3.25.67",
+    "zod-to-json-schema": "^3.24.6"
  }
 }
@@ -1,4 +1,5 @@
-import { extractText, streamConverter } from "../utils";
+import { extractText } from "../utils/llms";
+import { streamConverter } from "../utils/stream";
 import type {
  ChatResponse,
  ChatResponseChunk,
@@ -1,6 +1,9 @@
 export { BaseLLM, ToolCallLLM } from "./base";
-export { LiveLLM, LiveLLMSession, liveEvents, type LiveEvent } from "./live";
+export { LiveLLM, LiveLLMCapability, LiveLLMSession } from "./live/live";
+export { liveEvents, type LiveEvent } from "./live/live-types";
+export type { MessageSender } from "./live/sender";
 export type {
+  AudioConfig,
  BaseTool,
  BaseToolWithCall,
  ChatMessage,
@@ -1,9 +1,7 @@
 import type {
-  ChatMessage,
-  LiveConnectConfig,
  MessageContentAudioDetail,
  MessageContentTextDetail,
-} from "./type";
+} from "../type";

 export type OpenEvent = { type: "open" };

@@ -63,45 +61,3 @@ export const liveEvents = {
      e.type === "turnComplete",
  },
 };
-export abstract class LiveLLMSession {
-  protected eventQueue: LiveEvent[] = [];
-  protected eventResolvers: ((value: LiveEvent) => void)[] = [];
-  protected closed = false;
-  abstract sendMessage(message: ChatMessage): void;
-  async *streamEvents(): AsyncIterable<LiveEvent> {
-    while (true) {
-      const event = await this.nextEvent();
-      if (event === undefined) {
-        break;
-      }
-      yield event;
-    }
-  }
-  abstract disconnect(): Promise<void>;
-
-  protected async nextEvent(): Promise<LiveEvent | undefined> {
-    if (this.eventQueue.length) {
-      return Promise.resolve(this.eventQueue.shift());
-    }
-
-    return new Promise((resolve) => {
-      this.eventResolvers.push(resolve);
-    });
-  }
-
-  //Uses an async queue to send events to the client
-  // if the consumer is waiting for an event, it will be resolved immediately
-  // otherwise, the event will be queued up and sent when the consumer is ready
-  pushEventToQueue(event: LiveEvent) {
-    if (this.eventResolvers.length) {
-      //resolving the promise with the event
-      this.eventResolvers.shift()!(event);
-    } else {
-      this.eventQueue.push(event);
-    }
-  }
-}
-
-export abstract class LiveLLM {
-  abstract connect(config?: LiveConnectConfig): Promise<LiveLLMSession>;
-}
@@ -0,0 +1,124 @@
+import type {
+  ChatMessage,
+  LiveConnectConfig,
+  MessageContentAudioDetail,
+  MessageContentDetail,
+  MessageContentImageDataDetail,
+  MessageContentVideoDetail,
+} from "../type";
+import type { LiveEvent } from "./live-types";
+import type { MessageSender } from "./sender";
+
+export enum LiveLLMCapability {
+  EPHEMERAL_KEY = "ephemeral_key",
+  AUDIO_CONFIG = "audio_config",
+}
+
+export abstract class LiveLLMSession {
+  protected eventQueue: LiveEvent[] = [];
+  protected eventResolvers: ((value: LiveEvent) => void)[] = [];
+  closed = false;
+  abstract get messageSender(): MessageSender;
+
+  private isTextMessage(content: MessageContentDetail) {
+    return content.type === "text";
+  }
+
+  private isAudioMessage(
+    content: MessageContentDetail,
+  ): content is MessageContentAudioDetail {
+    return content.type === "audio";
+  }
+
+  private isImageMessage(
+    content: MessageContentDetail,
+  ): content is MessageContentImageDataDetail {
+    return content.type === "image";
+  }
+
+  private isVideoMessage(
+    content: MessageContentDetail,
+  ): content is MessageContentVideoDetail {
+    return content.type === "video";
+  }
+
+  sendMessage(message: ChatMessage) {
+    const { content, role } = message;
+    if (!Array.isArray(content)) {
+      this.messageSender.sendTextMessage(content, role);
+    } else {
+      for (const item of content) {
+        this.processMessage(item, role);
+      }
+    }
+  }
+
+  private processMessage(message: MessageContentDetail, role?: string) {
+    if (this.isTextMessage(message)) {
+      this.messageSender.sendTextMessage(message.text, role);
+    } else if (
+      this.isAudioMessage(message) &&
+      this.messageSender.sendAudioMessage
+    ) {
+      this.messageSender.sendAudioMessage(message, role);
+    } else if (
+      this.isImageMessage(message) &&
+      this.messageSender.sendImageMessage
+    ) {
+      this.messageSender.sendImageMessage(message, role);
+    } else if (
+      this.isVideoMessage(message) &&
+      this.messageSender.sendVideoMessage
+    ) {
+      this.messageSender.sendVideoMessage(message, role);
+    }
+  }
+
+  async *streamEvents(): AsyncIterable<LiveEvent> {
+    while (true) {
+      const event = await this.nextEvent();
+      if (event === undefined) {
+        break;
+      }
+      yield event;
+    }
+  }
+  abstract disconnect(): Promise<void>;
+
+  protected async nextEvent(): Promise<LiveEvent | undefined> {
+    if (this.eventQueue.length) {
+      return Promise.resolve(this.eventQueue.shift());
+    }
+
+    return new Promise((resolve) => {
+      this.eventResolvers.push(resolve);
+    });
+  }
+
+  //Uses an async queue to send events to the client
+  // if the consumer is waiting for an event, it will be resolved immediately
+  // otherwise, the event will be queued up and sent when the consumer is ready
+  pushEventToQueue(event: LiveEvent) {
+    if (this.eventResolvers.length) {
+      //resolving the promise with the event
+      this.eventResolvers.shift()!(event);
+    } else {
+      this.eventQueue.push(event);
+    }
+  }
+}
+
+export abstract class LiveLLM {
+  /**
+   * Set of capabilities supported by this implementation.
+   * Override in subclasses as needed.
+   */
+  capabilities: Set<LiveLLMCapability> = new Set();
+
+  abstract connect(config?: LiveConnectConfig): Promise<LiveLLMSession>;
+  abstract getEphemeralKey(): Promise<string | undefined>;
+
+  hasCapability(capability: LiveLLMCapability): boolean {
+    return this.capabilities.has(capability);
+  }
+}
@@ -0,0 +1,15 @@
+import type {
+  MessageContentAudioDetail,
+  MessageContentImageDataDetail,
+  MessageContentVideoDetail,
+} from "../type";
+
+export interface MessageSender {
+  sendTextMessage(message: string, role?: string): void;
+  sendAudioMessage?(content: MessageContentAudioDetail, role?: string): void;
+  sendImageMessage?(
+    content: MessageContentImageDataDetail,
+    role?: string,
+  ): void;
+  sendVideoMessage?(content: MessageContentVideoDetail, role?: string): void;
+}
@@ -290,8 +290,14 @@ export type ToolOutput = {
  isError: boolean;
 };

+export interface AudioConfig {
+  stream?: MediaStream;
+  onTrack?: (track: MediaStream | null) => void;
+}
+
 export interface LiveConnectConfig {
  tools?: BaseTool[];
  responseModality?: ModalityType[];
  systemInstruction?: string;
+  audioConfig?: AudioConfig;
 }
@@ -0,0 +1,42 @@
+export const abbreviations = {
+  english: [
+    "i.e.",
+    "etc.",
+    "vs.",
+    "Inc.",
+    "A.S.A.P.",
+    "Mr.",
+    "Mrs.",
+    "Ms.",
+    "Dr.",
+    "Prof.",
+    "Sr.",
+    "Jr.",
+    "Tel.",
+    "a.m.",
+    "p.m.",
+    "Art.",
+  ],
+  spanish: [
+    "Sr.",
+    "Sres.",
+    "Srs.",
+    "Sra.",
+    "Sras.",
+    "Srta.",
+    "Srtas.",
+    "Dr.",
+    "Drs.",
+    "Dra.",
+    "Dras.",
+    "Prof.",
+    "Profs.",
+    "Profa.",
+    "Profas.",
+    "Ing.",
+    "Lic.",
+    "Arq.",
+    "Ab.",
+    "Abs.",
+  ],
+};
@@ -12,11 +12,7 @@ import {
  type TextSplitterFn,
 } from "./utils";

-type _Split = {
-  text: string;
-  isSentence: boolean;
-  tokenSize: number;
-};
+type _Split = [string, boolean, number]; // [text, isSentence, tokenSize]

 /**
 * Parse text with a preference for complete sentences.
@@ -42,6 +38,11 @@ export class SentenceSplitter extends MetadataAwareTextSplitter {
   * Backup regex for splitting into sentences.
   */
  secondaryChunkingRegex: string = "[^,.;。？！]+[,.;。？！]?";
+  /**
+   * Extra abbreviations to consider while splitting into sentences.
+   * For example, for contracts, you may want to consider "LLC." as an important abbreviation
+   */
+  extraAbbreviations: string[] | undefined = [];

  #chunkingTokenizerFn = splitBySentenceTokenizer();
  #splitFns: Set<TextSplitterFn> = new Set();
@@ -59,7 +60,11 @@ export class SentenceSplitter extends MetadataAwareTextSplitter {
      this.separator = parsedParams.separator;
      this.paragraphSeparator = parsedParams.paragraphSeparator;
      this.secondaryChunkingRegex = parsedParams.secondaryChunkingRegex;
+      this.extraAbbreviations = parsedParams.extraAbbreviations;
    }
+    this.#chunkingTokenizerFn = splitBySentenceTokenizer(
+      this.extraAbbreviations,
+    );
    this.#tokenizer = params?.tokenizer ?? Settings.tokenizer;
    this.#splitFns.add(splitBySep(this.paragraphSeparator));
    this.#splitFns.add(this.#chunkingTokenizerFn);
@@ -105,34 +110,26 @@ export class SentenceSplitter extends MetadataAwareTextSplitter {
    return chunks;
  }

-  #split(text: string, chunkSize: number): _Split[] {
-    const tokenSize = this.tokenSize(text);
-    if (tokenSize <= chunkSize) {
-      return [
-        {
-          text,
-          isSentence: true,
-          tokenSize,
-        },
-      ];
+  *#split(
+    text: string,
+    chunkSize: number,
+    tokenSize?: number,
+  ): Generator<_Split> {
+    const _tokenSize = tokenSize ?? this.tokenSize(text);
+    if (_tokenSize <= chunkSize) {
+      yield [text, true, _tokenSize];
+      return;
    }
    const [textSplitsByFns, isSentence] = this.#getSplitsByFns(text);
-    const textSplits: _Split[] = [];

    for (const textSplit of textSplitsByFns) {
-      const tokenSize = this.tokenSize(textSplit);
-      if (tokenSize <= chunkSize) {
-        textSplits.push({
-          text: textSplit,
-          isSentence,
-          tokenSize,
-        });
+      const textSplitTokenSize = this.tokenSize(textSplit);
+      if (textSplitTokenSize <= chunkSize) {
+        yield [textSplit, isSentence, textSplitTokenSize];
      } else {
-        const recursiveTextSplits = this.#split(textSplit, chunkSize);
-        textSplits.push(...recursiveTextSplits);
+        yield* this.#split(textSplit, chunkSize, textSplitTokenSize);
      }
    }
-    return textSplits;
  }

  #getSplitsByFns(text: string): [splits: string[], isSentence: boolean] {
@@ -151,7 +148,7 @@ export class SentenceSplitter extends MetadataAwareTextSplitter {
    return [[text], true];
  }

-  #merge(splits: _Split[], chunkSize: number): string[] {
+  #merge(splits: Iterable<_Split>, chunkSize: number): string[] {
    const chunks: string[] = [];
    let currentChunk: [string, number][] = [];
    let lastChunk: [string, number][] = [];
@@ -177,23 +174,26 @@ export class SentenceSplitter extends MetadataAwareTextSplitter {
      }
    };

-    while (splits.length > 0) {
-      const curSplit = splits[0]!;
-      if (curSplit.tokenSize > chunkSize) {
+    const splitsIterator = splits[Symbol.iterator]();
+    let currentSplitResult = splitsIterator.next();
+
+    while (!currentSplitResult.done) {
+      const [text, isSentence, tokenSize] = currentSplitResult.value;
+      if (tokenSize > chunkSize) {
        throw new Error("Single token exceeded chunk size");
      }
-      if (currentChunkLength + curSplit.tokenSize > chunkSize && !newChunk) {
+      if (currentChunkLength + tokenSize > chunkSize && !newChunk) {
        closeChunk();
      } else {
        if (
-          curSplit.isSentence ||
-          currentChunkLength + curSplit.tokenSize <= chunkSize ||
+          isSentence ||
+          currentChunkLength + tokenSize <= chunkSize ||
          newChunk
        ) {
-          currentChunkLength += curSplit.tokenSize;
-          currentChunk.push([curSplit.text, curSplit.tokenSize]);
-          splits.shift();
+          currentChunkLength += tokenSize;
+          currentChunk.push([text, tokenSize]);
          newChunk = false;
+          currentSplitResult = splitsIterator.next();
        } else {
          closeChunk();
        }
@@ -1,3 +1,4 @@
+import { abbreviations } from "./abbreviations";
 import type { TextSplitter } from "./base";
 import SentenceTokenizer from "./sentence_tokenizer";

@@ -34,19 +35,15 @@ export const splitByChar = (): TextSplitterFn => {
  return (text: string) => text.split("");
 };

-let sentenceTokenizer: SentenceTokenizer | null = null;
-
-export const splitBySentenceTokenizer = (): TextSplitterFn => {
-  if (!sentenceTokenizer) {
-    sentenceTokenizer = new SentenceTokenizer([
-      "i.e.",
-      "etc.",
-      "vs.",
-      "Inc.",
-      "A.S.A.P.",
-    ]);
-  }
-  const tokenizer = sentenceTokenizer;
+export const splitBySentenceTokenizer = (
+  extraAbbreviations: string[] | undefined = [],
+): TextSplitterFn => {
+  const tokenizer = new SentenceTokenizer([
+    ...abbreviations.english,
+    ...abbreviations.spanish,
+    // Add the extra abbreviations provided by the user, e.g. for business-specific context
+    ...extraAbbreviations,
+  ]);
  return (text: string) => {
    try {
      return tokenizer.tokenize(text);
@@ -347,6 +347,8 @@ export function jsonToNode(json: any, type?: ObjectType) {
      return new Document(json);
    case ObjectType.IMAGE_DOCUMENT:
      return new ImageDocument(json);
+    case ObjectType.IMAGE:
+      return new ImageNode(json);
    default:
      throw new Error(`Invalid node type: ${nodeType}`);
  }
@@ -51,6 +51,13 @@ export const sentenceSplitterSchema = z
      })
      .optional()
      .default("[^,.;。？！]+[,.;。？！]?"),
+    extraAbbreviations: z
+      .array(z.string(), {
+        description:
+          "Extra abbreviations to consider while splitting into sentences. For example, for contracts, you may want to consider 'LLC.' as an important abbreviation",
+      })
+      .optional()
+      .default([]),
  })
  .refine(
    (data) => data.chunkOverlap < data.chunkSize,
@@ -0,0 +1,103 @@
+import { filetypemime } from "magic-bytes.js";
+
+/**
+ * Converts a base64 string (without data: prefix) to a Uint8Array
+ * @param base64 - The base64 string without data: prefix
+ * @returns The Uint8Array
+ */
+export function base64ToUint8Array(base64: string): Uint8Array {
+  // Decode Base64 string
+  const binaryString = atob(base64);
+
+  // Convert binary string to Uint8Array
+  const bytes = new Uint8Array(binaryString.length);
+  for (let i = 0; i < binaryString.length; i++) {
+    bytes[i] = binaryString.charCodeAt(i);
+  }
+
+  return bytes;
+}
+
+/**
+ * Converts a Uint8Array to a base64 string.
+ * @param uint8Array The Uint8Array to convert.
+ * @returns The base64-encoded string.
+ */
+export function uint8ArrayToBase64(uint8Array: Uint8Array): string {
+  let binary = "";
+  for (let i = 0; i < uint8Array.byteLength; i++) {
+    // Asserts that the value is not undefined, for `noUncheckedIndexedAccess`
+    binary += String.fromCharCode(uint8Array[i]!);
+  }
+  return btoa(binary);
+}
+
+/**
+ * Extracts the MIME type from a data URL.
+ * @param dataUrl The data URL string.
+ * @returns The MIME type from the data URL.
+ * @throws An error if the data URL is malformed.
+ */
+export function getMimeTypeFromDataUrl(dataUrl: string): string {
+  if (!dataUrl.startsWith("data:")) {
+    throw new Error("Not a data URL");
+  }
+  const commaIndex = dataUrl.indexOf(",");
+  if (commaIndex === -1) {
+    throw new Error("Invalid data URL format");
+  }
+
+  const header = dataUrl.slice(0, commaIndex);
+  const semicolonIndex = header.indexOf(";base64");
+  if (semicolonIndex === -1) {
+    throw new Error("Invalid data URL format: missing base64 encoding");
+  }
+
+  return header.slice(5, semicolonIndex);
+}
+
+/**
+ * Convert base64 data to Blob
+ * @param base64 - The base64 string
+ * @param mimeType - The MIME type of the file
+ * @returns The Blob
+ */
+export function base64ToBlob(base64: string, mimeType?: string): Blob {
+  let extractedMimeType = mimeType;
+  let base64Data = base64;
+
+  // Extract mimeType from data URL if not provided
+  if (!mimeType && base64.startsWith("data:")) {
+    extractedMimeType = getMimeTypeFromDataUrl(base64);
+    base64Data = base64.slice(base64.indexOf(",") + 1);
+  } else if (!mimeType) {
+    throw new Error(
+      "No MIME type provided and base64 is not in data URL format",
+    );
+  } else {
+    // Extract base64 data from data URL if present
+    const commaIndex = base64.indexOf(",");
+    base64Data = commaIndex !== -1 ? base64.slice(commaIndex + 1) : base64;
+  }
+
+  if (!extractedMimeType) {
+    throw new Error("No MIME type found in base64 data");
+  }
+
+  // convert base64 to Uint8Array
+  const bytes = base64ToUint8Array(base64Data);
+
+  // Create Blob
+  return new Blob([bytes], { type: extractedMimeType });
+}
+
+export async function blobToDataUrl(input: Blob) {
+  const arrayBuffer = await input.arrayBuffer();
+  const uint8Array = new Uint8Array(arrayBuffer);
+  const mimes = filetypemime(uint8Array);
+  if (mimes.length < 1) {
+    throw new Error("Unsupported image type");
+  }
+  const base64 = uint8ArrayToBase64(uint8Array);
+  return `data:${mimes[0]};base64,${base64}`;
+}
@@ -14,19 +14,6 @@ export const isIterable = (obj: unknown): obj is Iterable<unknown> => {
  return obj != null && typeof obj === "object" && Symbol.iterator in obj;
 };

-export async function* streamConverter<S, D>(
-  stream: AsyncIterable<S>,
-  converter: (s: S) => D | null,
-): AsyncIterable<D> {
-  for await (const data of stream) {
-    const newData = converter(data);
-    if (newData === null) {
-      return;
-    }
-    yield newData;
-  }
-}
-
 export async function* streamCallbacks<S>(
  stream: AsyncIterable<S>,
  callbacks: {
@@ -85,4 +72,6 @@ export {

 export { MockLLM } from "./mock";

+export * from "./encoding";
 export { objectEntries } from "./object-entries";
+export * from "./stream";
@@ -1,5 +1,4 @@
 import { fs } from "@llamaindex/env";
-import { filetypemime } from "magic-bytes.js";
 import type {
  ChatMessage,
  MessageContent,
@@ -9,6 +8,7 @@ import type {
 } from "../llms";
 import type { QueryType } from "../query-engine";
 import type { ImageType } from "../schema";
+import { blobToDataUrl } from "./encoding";

 /**
 * Extracts just the text whether from
@@ -110,15 +110,6 @@ export function toToolDescriptions(tools: ToolMetadata[]): string {
  return JSON.stringify(toolsObj, null, 4);
 }

-async function blobToDataUrl(input: Blob) {
-  const buffer = Buffer.from(await input.arrayBuffer());
-  const mimes = filetypemime(buffer);
-  if (mimes.length < 1) {
-    throw new Error("Unsupported image type");
-  }
-  return "data:" + mimes[0] + ";base64," + buffer.toString("base64");
-}
-
 export async function imageToDataUrl(
  input: ImageType | Uint8Array,
 ): Promise<string> {
@@ -0,0 +1,12 @@
+export async function* streamConverter<S, D>(
+  stream: AsyncIterable<S>,
+  converter: (s: S) => D | null,
+): AsyncIterable<D> {
+  for await (const data of stream) {
+    const newData = converter(data);
+    if (newData === null) {
+      return;
+    }
+    yield newData;
+  }
+}
@@ -0,0 +1,99 @@
+import {
+  base64ToBlob,
+  base64ToUint8Array,
+  blobToDataUrl,
+  getMimeTypeFromDataUrl,
+  uint8ArrayToBase64,
+} from "@llamaindex/core/utils";
+import { describe, expect, it } from "vitest";
+
+const testString = "LlamaIndex";
+const testBase64 = "TGxhbWFJbmRleA=="; // btoa('LlamaIndex')
+const testUint8Array = new TextEncoder().encode(testString);
+
+const pngB64 =
+  "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNkYAAAAAYAAjCB0C8AAAAASUVORK5CYII=";
+const pngMime = "image/png";
+const pngDataUrl = `data:${pngMime};base64,${pngB64}`;
+const pngBinaryString = atob(pngB64);
+const pngBytes = new Uint8Array(pngBinaryString.length);
+for (let i = 0; i < pngBinaryString.length; i++) {
+  pngBytes[i] = pngBinaryString.charCodeAt(i);
+}
+
+describe("Encoding utils", () => {
+  describe("base64ToUint8Array", () => {
+    it("should correctly convert a base64 string to a Uint8Array", () => {
+      const result = base64ToUint8Array(testBase64);
+      expect(result).toBeInstanceOf(Uint8Array);
+      expect(result).toEqual(testUint8Array);
+    });
+  });
+
+  describe("uint8ArrayToBase64", () => {
+    it("should correctly convert a Uint8Array to a base64 string", () => {
+      const result = uint8ArrayToBase64(testUint8Array);
+      expect(result).toBe(testBase64);
+    });
+  });
+
+  describe("getMimeTypeFromDataUrl", () => {
+    it("should extract the correct MIME type from a data URL", () => {
+      const result = getMimeTypeFromDataUrl(pngDataUrl);
+      expect(result).toBe(pngMime);
+    });
+
+    it("should throw an error for non-data URLs", () => {
+      expect(() => getMimeTypeFromDataUrl("not a data url")).toThrow(
+        "Not a data URL",
+      );
+    });
+
+    it("should throw an error for malformed data URLs", () => {
+      expect(() => getMimeTypeFromDataUrl("data:image/pngbase64,abc")).toThrow(
+        "Invalid data URL format: missing base64 encoding",
+      );
+      expect(() => getMimeTypeFromDataUrl("data:image/png;base64")).toThrow(
+        "Invalid data URL format",
+      );
+    });
+  });
+
+  describe("base64ToBlob", () => {
+    it("should convert from a data URL string", async () => {
+      const blob = base64ToBlob(pngDataUrl);
+      expect(blob).toBeInstanceOf(Blob);
+      expect(blob.type).toBe(pngMime);
+      const arrayBuffer = await blob.arrayBuffer();
+      expect(new Uint8Array(arrayBuffer)).toEqual(pngBytes);
+    });
+
+    it("should convert from a base64 string with an explicit MIME type", async () => {
+      const blob = base64ToBlob(pngB64, pngMime);
+      expect(blob).toBeInstanceOf(Blob);
+      expect(blob.type).toBe(pngMime);
+      const arrayBuffer = await blob.arrayBuffer();
+      expect(new Uint8Array(arrayBuffer)).toEqual(pngBytes);
+    });
+
+    it("should prioritize the explicit MIME type if a data URL is provided", async () => {
+      const differentMime = "image/jpeg";
+      const blob = base64ToBlob(pngDataUrl, differentMime);
+      expect(blob.type).toBe(differentMime);
+    });
+
+    it("should throw an error if no MIME type can be determined", () => {
+      expect(() => base64ToBlob(pngB64)).toThrow(
+        "No MIME type provided and base64 is not in data URL format",
+      );
+    });
+  });
+
+  describe("blobToDataUrl", () => {
+    it("should correctly convert a blob to a data URL", async () => {
+      const blob = new Blob([pngBytes], { type: "image/png" });
+      const result = await blobToDataUrl(blob);
+      expect(result).toBe(pngDataUrl);
+    });
+  });
+});
@@ -75,6 +75,22 @@ describe("sentence splitter", () => {
    expect(splits).toEqual(["This is a sentence. This is another sentence."]);
  });

+  test("overall split long text", () => {
+    const sentenceSplitter = new SentenceSplitter({
+      chunkSize: 10,
+      chunkOverlap: 0,
+    });
+    const splits = sentenceSplitter.splitText(
+      "The first short sentence. The first long long long sentence. The second short sentence. The second long long long sentence.",
+    );
+    expect(splits).toEqual([
+      "The first short sentence.",
+      "The first long long long sentence.",
+      "The second short sentence.",
+      "The second long long long sentence.",
+    ]);
+  });
+
  test("doesn't split decimals", () => {
    const sentenceSplitter = new SentenceSplitter({
      chunkSize: 5,
@@ -91,6 +107,39 @@ describe("sentence splitter", () => {
    ]);
  });

+  test("doesn't split basic abbreviations", () => {
+    const sentenceSplitter = new SentenceSplitter({
+      chunkSize: 15,
+      chunkOverlap: 0,
+    });
+    const splits = sentenceSplitter.splitText(
+      "This is a sentence of Broda Noel. This is the sentence of Sr. Broda Noel. This is a sentence of somebody else",
+    );
+
+    expect(splits).toEqual([
+      "This is a sentence of Broda Noel.",
+      "This is the sentence of Sr. Broda Noel.",
+      "This is a sentence of somebody else",
+    ]);
+  });
+
+  test("doesn't split extra abbreviations", () => {
+    const sentenceSplitter = new SentenceSplitter({
+      chunkSize: 10,
+      chunkOverlap: 0,
+      extraAbbreviations: ["S.A."],
+    });
+    const splits = sentenceSplitter.splitText(
+      "This is a sentence. The S.A. Broda Company. This is another sentence",
+    );
+
+    expect(splits).toEqual([
+      "This is a sentence.",
+      "The S.A. Broda Company.",
+      "This is another sentence",
+    ]);
+  });
+
  test("splits cjk", () => {
    const sentenceSplitter = new SentenceSplitter({
      chunkSize: 30,
@@ -1,5 +1,43 @@
 # @llamaindex/experimental

+## 0.0.188
+
+### Patch Changes
+
+- Updated dependencies [7039e1a]
+  - llamaindex@0.11.11
+
+## 0.0.187
+
+### Patch Changes
+
+- llamaindex@0.11.10
+
+## 0.0.186
+
+### Patch Changes
+
+- llamaindex@0.11.9
+
+## 0.0.185
+
+### Patch Changes
+
+- llamaindex@0.11.8
+
+## 0.0.184
+
+### Patch Changes
+
+- Updated dependencies [3c857f4]
+  - llamaindex@0.11.7
+
+## 0.0.183
+
+### Patch Changes
+
+- llamaindex@0.11.6
+
 ## 0.0.182

 ### Patch Changes
@@ -1,7 +1,7 @@
 {
  "name": "@llamaindex/experimental",
  "description": "Experimental package for LlamaIndexTS",
-  "version": "0.0.182",
+  "version": "0.0.188",
  "type": "module",
  "types": "dist/type/index.d.ts",
  "main": "dist/cjs/index.js",
@@ -1,5 +1,63 @@
 # llamaindex

+## 0.11.11
+
+### Patch Changes
+
+- 7039e1a: Internal cleanup of base64 encoding
+- Updated dependencies [7039e1a]
+- Updated dependencies [7039e1a]
+  - @llamaindex/core@0.6.12
+  - @llamaindex/cloud@4.0.16
+  - @llamaindex/node-parser@2.0.12
+  - @llamaindex/workflow@1.1.12
+
+## 0.11.10
+
+### Patch Changes
+
+- Updated dependencies [f7ec293]
+  - @llamaindex/workflow@1.1.11
+
+## 0.11.9
+
+### Patch Changes
+
+- Updated dependencies [a89e187]
+- Updated dependencies [62699b7]
+- Updated dependencies [c5b2691]
+- Updated dependencies [d8ac8d3]
+  - @llamaindex/core@0.6.11
+  - @llamaindex/cloud@4.0.15
+  - @llamaindex/node-parser@2.0.11
+  - @llamaindex/workflow@1.1.10
+
+## 0.11.8
+
+### Patch Changes
+
+- Updated dependencies [8a51c16]
+- Updated dependencies [1b5af14]
+  - @llamaindex/workflow@1.1.9
+  - @llamaindex/core@0.6.10
+  - @llamaindex/cloud@4.0.14
+  - @llamaindex/node-parser@2.0.10
+
+## 0.11.7
+
+### Patch Changes
+
+- 3c857f4: chore: move ajv to dev deps
+- Updated dependencies [dbd857f]
+  - @llamaindex/workflow@1.1.8
+
+## 0.11.6
+
+### Patch Changes
+
+- Updated dependencies [40161fe]
+  - @llamaindex/workflow@1.1.7
+
 ## 0.11.5

 ### Patch Changes
@@ -1,6 +1,6 @@
 {
  "name": "llamaindex",
-  "version": "0.11.5",
+  "version": "0.11.11",
  "license": "MIT",
  "type": "module",
  "keywords": [
@@ -24,14 +24,14 @@
    "@llamaindex/core": "workspace:*",
    "@llamaindex/env": "workspace:*",
    "@llamaindex/node-parser": "workspace:*",
-    "@llamaindex/workflow": "1.1.6",
+    "@llamaindex/workflow": "workspace:*",
    "@types/lodash": "^4.17.7",
    "@types/node": "^22.9.0",
-    "ajv": "^8.17.1",
    "lodash": "^4.17.21",
    "magic-bytes.js": "^1.10.0"
  },
  "devDependencies": {
+    "ajv": "^8.17.1",
    "@types/node": "^22.9.0"
  },
  "engines": {
@@ -18,11 +18,11 @@ import type {
 } from "@llamaindex/core/llms";
 import {
  extractText,
+  isAsyncIterable,
  stringifyJSONToMessageContent,
 } from "@llamaindex/core/utils";
 import { randomUUID } from "@llamaindex/env";
 import { getReACTAgentSystemHeader } from "../internal/prompt/react.js";
-import { isAsyncIterable } from "../internal/utils.js";
 import { Settings } from "../Settings.js";

 export type ReACTAgentParams = AgentParamsBase<LLM>;
@@ -63,7 +63,6 @@ export * from "./evaluation/index.js";
 export * from "./extractors/index.js";
 export * from "./indices/index.js";
 export * from "./ingestion/index.js";
-export { imageToDataUrl } from "./internal/utils.js";
 export * from "./node-parser.js";
 export * from "./objects/index.js";
 export * from "./OutputParser.js";
@@ -1,77 +0,0 @@
-import type { ImageType } from "@llamaindex/core/schema";
-import { fs } from "@llamaindex/env";
-import { filetypemime } from "magic-bytes.js";
-
-export const isAsyncIterable = (
-  obj: unknown,
-): obj is AsyncIterable<unknown> => {
-  return obj != null && typeof obj === "object" && Symbol.asyncIterator in obj;
-};
-
-export const isReadableStream = (obj: unknown): obj is ReadableStream => {
-  return obj instanceof ReadableStream;
-};
-
-export const isIterable = (obj: unknown): obj is Iterable<unknown> => {
-  return obj != null && typeof obj === "object" && Symbol.iterator in obj;
-};
-
-async function blobToDataUrl(input: Blob) {
-  const buffer = Buffer.from(await input.arrayBuffer());
-  const mimes = filetypemime(buffer);
-  if (mimes.length < 1) {
-    throw new Error("Unsupported image type");
-  }
-  return "data:" + mimes[0] + ";base64," + buffer.toString("base64");
-}
-
-export async function imageToString(input: ImageType): Promise<string> {
-  if (input instanceof Blob) {
-    // if the image is a Blob, convert it to a base64 data URL
-    return await blobToDataUrl(input);
-  } else if (typeof input === "string") {
-    return input;
-  } else if (input instanceof URL) {
-    return input.toString();
-  } else {
-    throw new Error(`Unsupported input type: ${typeof input}`);
-  }
-}
-
-export function stringToImage(input: string): ImageType {
-  if (input.startsWith("data:")) {
-    // if the input is a base64 data URL, convert it back to a Blob
-    const base64Data = input.split(",")[1]!;
-    const byteArray = Buffer.from(base64Data, "base64");
-    return new Blob([byteArray]);
-  } else if (input.startsWith("http://") || input.startsWith("https://")) {
-    return new URL(input);
-  } else {
-    return input;
-  }
-}
-
-export async function imageToDataUrl(
-  input: ImageType | Uint8Array,
-): Promise<string> {
-  // first ensure, that the input is a Blob
-  if (
-    (input instanceof URL && input.protocol === "file:") ||
-    typeof input === "string"
-  ) {
-    // string or file URL
-    const dataBuffer = await fs.readFile(
-      input instanceof URL ? input.pathname : input,
-    );
-    input = new Blob([dataBuffer]);
-  } else if (!(input instanceof Blob)) {
-    if (input instanceof URL) {
-      throw new Error(`Unsupported URL with protocol: ${input.protocol}`);
-    } else if (input instanceof Uint8Array) {
-      input = new Blob([input]); // convert Uint8Array to Blob
-    } else {
-      throw new Error(`Unsupported input type: ${typeof input}`);
-    }
-  }
-  return await blobToDataUrl(input);
-}
@@ -1,5 +1,24 @@
 # @llamaindex/core-test

+## 0.1.7
+
+### Patch Changes
+
+- @llamaindex/openai@0.4.6
+
+## 0.1.6
+
+### Patch Changes
+
+- Updated dependencies [d8ac8d3]
+  - @llamaindex/openai@0.4.5
+
+## 0.1.5
+
+### Patch Changes
+
+- @llamaindex/openai@0.4.4
+
 ## 0.1.4

 ### Patch Changes
@@ -1,7 +1,7 @@
 {
  "name": "@llamaindex/llamaindex-test",
  "private": true,
-  "version": "0.1.4",
+  "version": "0.1.7",
  "type": "module",
  "scripts": {
    "test": "vitest run"
@@ -1,5 +1,30 @@
 # @llamaindex/node-parser

+## 2.0.12
+
+### Patch Changes
+
+- Updated dependencies [7039e1a]
+- Updated dependencies [7039e1a]
+  - @llamaindex/core@0.6.12
+
+## 2.0.11
+
+### Patch Changes
+
+- Updated dependencies [a89e187]
+- Updated dependencies [62699b7]
+- Updated dependencies [c5b2691]
+- Updated dependencies [d8ac8d3]
+  - @llamaindex/core@0.6.11
+
+## 2.0.10
+
+### Patch Changes
+
+- Updated dependencies [1b5af14]
+  - @llamaindex/core@0.6.10
+
 ## 2.0.9

 ### Patch Changes
--- a/Show More
+++ b/Show More
Author	SHA1	Message	Date
github-actions[bot]	7e8efc6284	Release @llamaindex/tools@0.1.2 (#2048 )	2025-06-30 11:40:54 +07:00
Wassim Chegham	0fcf65126d	chore: export type MCPClientOptions (#2047 ) Co-authored-by: Marcus Schiesser <marcus.schiesser@googlemail.com>	2025-06-28 10:55:07 +07:00
github-actions[bot]	a50acf634c	Release 0.11.11 (#2044 ) Co-authored-by: marcusschiesser <17126+marcusschiesser@users.noreply.github.com>	2025-06-27 14:51:09 +07:00
Thuc Pham	7039e1a214	chore: migrate to @google/genai SDK (#2038 ) Co-authored-by: Marcus Schiesser <mail@marcusschiesser.de>	2025-06-27 12:09:26 +07:00
github-actions[bot]	785d010cd3	Release 0.11.10 (#2037 ) Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>	2025-06-26 14:29:33 +07:00
Marcus Schiesser	b878032131	fix release step	2025-06-26 14:18:56 +07:00
Marcus Schiesser	f7ec293a0f	chore: Update workflow-core (#2042 )	2025-06-26 14:03:03 +07:00
jerinthomascarmel	49a5e0a8cf	feat(readers): add ExcelReader for parsing Excel files (run-llama#1959) (#2033 ) Co-authored-by: Marcus Schiesser <mail@marcusschiesser.de> Co-authored-by: leehuwuj <leehuwuj@gmail.com>	2025-06-26 11:15:19 +07:00
Logan	118924799a	Rename llama-flow -> workflows in docs (#2040 )	2025-06-25 15:52:04 -07:00
allen	ec8f673dae	support filter to supabase vector search (#2036 )	2025-06-25 16:17:54 +07:00
github-actions[bot]	85039a5360	Release @llamaindex/tools@0.1.0 (#2034 )	2025-06-24 12:32:24 +07:00
Marcus Schiesser	d7305edb53	fix changesets	2025-06-24 12:26:09 +07:00
Huu Le	096bf2bda1	feat: Add support for StreamableHTTP MCP Client (#2032 )	2025-06-24 11:40:34 +07:00
jerinthomascarmel	c5846bd7dc	feat(readers): add XMLReader for parsing XML files (#1846 ) (#2031 ) Co-authored-by: Marcus Schiesser <marcus.schiesser@googlemail.com>	2025-06-24 10:46:32 +07:00
github-actions[bot]	97bbce6e13	Release 0.11.9 (#2023 ) Co-authored-by: marcusschiesser <17126+marcusschiesser@users.noreply.github.com>	2025-06-20 12:28:01 +07:00
Marcus Schiesser	62699b7497	chore: improve performance of sentence splitter (#2030 )	2025-06-20 12:16:24 +07:00
Broda Noel	a89e187796	Add `extraAbbreviations` on sentence-splitter (#2029 ) Co-authored-by: Marcus Schiesser <mail@marcusschiesser.de>	2025-06-20 11:27:06 +07:00
ANKIT VARSHNEY	d8ac8d385d	feat: add openai realtime api (#2006 ) Co-authored-by: Marcus Schiesser <mail@marcusschiesser.de>	2025-06-20 10:22:04 +07:00
Marcus Schiesser	a6cef9c6be	chore: no core in examples (#2024 )	2025-06-18 09:39:32 +07:00
Broda Noel	c5b2691302	Add more Acronyms on SentenceSplitter (#2022 ) Co-authored-by: Marcus Schiesser <marcus.schiesser@googlemail.com>	2025-06-17 10:43:36 +07:00
github-actions[bot]	8122c7245e	Release 0.11.8 (#2018 ) Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: marcusschiesser <17126+marcusschiesser@users.noreply.github.com>	2025-06-12 16:20:58 +07:00
Huu Le	8a51c167f8	feat: use agent to handle a workflow step (#2014 ) Co-authored-by: Marcus Schiesser <mail@marcusschiesser.de>	2025-06-12 16:06:13 +07:00
Marcus Schiesser	1b5af1402d	fix: jsonToNode for image nodes (#2017 )	2025-06-12 11:59:05 +07:00
github-actions[bot]	fffe93fac8	Release 0.11.7 (#2013 ) Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: marcusschiesser <17126+marcusschiesser@users.noreply.github.com>	2025-06-12 10:34:24 +07:00
Marcus Schiesser	dbd857f6b5	chore: add changeset	2025-06-11 16:20:32 +07:00
정물결	a4d394f727	fix: correct SimpleDirectoryReader import path (#2011 )	2025-06-10 12:43:01 +07:00
Marcus Schiesser	3c857f4132	chore: move ajv to dev deps (#2012 )	2025-06-10 12:20:54 +07:00
Thuc Pham	36cfb93eb2	feat: export snapshot apis from llama-flow (#2009 )	2025-06-10 11:56:33 +07:00
github-actions[bot]	ab4762f026	Release (#2005 ) Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>	2025-06-06 14:45:39 +07:00
Peter Goldstein	56763dc57d	Update to the latest Gemini 2.5 Pro Preview key (#2004 )	2025-06-06 11:25:41 +07:00
github-actions[bot]	5375fdd704	Release (#2003 ) Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: marcusschiesser <17126+marcusschiesser@users.noreply.github.com>	2025-06-05 09:57:35 +07:00
Marcus Schiesser	e7484efca5	feat: weaviate: Add metadata sanitization before adding node. Add err… (#2001 )	2025-06-04 11:48:18 +07:00
Marcus Schiesser	c958a1645a	docs: update chat-ui (#2002 )	2025-06-03 17:01:07 +07:00
github-actions[bot]	0140a257c4	Release 0.11.6 (#1999 ) Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: marcusschiesser <17126+marcusschiesser@users.noreply.github.com>	2025-06-02 18:03:31 +07:00
GhosT	40161fe8d2	chore: Bump @llama-flow/core package version (#1998 ) Co-authored-by: Marcus Schiesser <marcus.schiesser@googlemail.com>	2025-06-02 17:28:47 +07:00
github-actions[bot]	d883fe7351	Release @llamaindex/google@0.3.7 (#1994 ) Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>	2025-05-31 14:04:14 +07:00
Parham Saidi	2bc6914784	fix: ignore empty parts for gemini which confuses agent (#1993 )	2025-05-30 22:47:21 +07:00
				`@@ -0,0 +1 @@`
				<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" class="iconify iconify--logos" width="31.88" height="32" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 257"><defs><linearGradient id="IconifyId1813088fe1fbc01fb466" x1="-.828%" x2="57.636%" y1="7.652%" y2="78.411%"><stop offset="0%" stop-color="#41D1FF"></stop><stop offset="100%" stop-color="#BD34FE"></stop></linearGradient><linearGradient id="IconifyId1813088fe1fbc01fb467" x1="43.376%" x2="50.316%" y1="2.242%" y2="89.03%"><stop offset="0%" stop-color="#FFEA83"></stop><stop offset="8.333%" stop-color="#FFDD35"></stop><stop offset="100%" stop-color="#FFA800"></stop></linearGradient></defs><path fill="url(#IconifyId1813088fe1fbc01fb466)" d="M255.153 37.938L134.897 252.976c-2.483 4.44-8.862 4.466-11.382.048L.875 37.958c-2.746-4.814 1.371-10.646 6.827-9.67l120.385 21.517a6.537 6.537 0 0 0 2.322-.004l117.867-21.483c5.438-.991 9.574 4.796 6.877 9.62Z"></path><path fill="url(#IconifyId1813088fe1fbc01fb467)" d="M185.432.063L96.44 17.501a3.268 3.268 0 0 0-2.634 3.014l-5.474 92.456a3.268 3.268 0 0 0 3.997 3.378l24.777-5.718c2.318-.535 4.413 1.507 3.936 3.838l-7.361 36.047c-.495 2.426 1.782 4.5 4.151 3.78l15.304-4.649c2.372-.72 4.652 1.36 4.15 3.788l-11.698 56.621c-.732 3.542 3.979 5.473 5.943 2.437l1.313-2.028l72.516-144.72c1.215-2.423-.88-5.186-3.54-4.672l-25.505 4.922c-2.396.462-4.435-1.77-3.759-4.114l16.646-57.705c.677-2.35-1.37-4.583-3.769-4.113Z"></path></svg>