fix: type

fix: missing fields
chore: bump sdk version
2026-07-16 07:14:29 -04:00 · 2025-06-30 11:33:46 -07:00 · 2025-06-30 11:16:27 -07:00 · 2025-06-30 10:44:27 -07:00 · 2025-06-30 17:32:04 +07:00 · 2025-06-30 13:41:13 +07:00
197 changed files with 4855 additions and 1898 deletions
@@ -0,0 +1,5 @@
+---
+"@llamaindex/core": patch
+---
+
+Fix: split sentences must not trim whitespaces
@@ -0,0 +1,6 @@
+---
+"llamaindex": patch
+"@llamaindex/core": patch
+---
+
+Fix logging for fromPersistPath
@@ -0,0 +1,5 @@
+---
+"@llamaindex/cloud": patch
+---
+
+chore: bump sdk version
@@ -1,5 +1,34 @@
 # @llamaindex/doc

+## 0.2.31
+
+### Patch Changes
+
+- Updated dependencies [7039e1a]
+- Updated dependencies [7039e1a]
+  - llamaindex@0.11.11
+  - @llamaindex/core@0.6.12
+  - @llamaindex/cloud@4.0.16
+  - @llamaindex/node-parser@2.0.12
+  - @llamaindex/openai@0.4.6
+  - @llamaindex/readers@3.1.11
+  - @llamaindex/workflow@1.1.12
+
+## 0.2.30
+
+### Patch Changes
+
+- Updated dependencies [f7ec293]
+  - @llamaindex/workflow@1.1.11
+  - llamaindex@0.11.10
+
+## 0.2.29
+
+### Patch Changes
+
+- Updated dependencies [c5846bd]
+  - @llamaindex/readers@3.1.10
+
 ## 0.2.28

 ### Patch Changes
@@ -111,7 +111,7 @@ Key build process:
 **Content Sources:**

 - Local MDX files in `src/content/docs/`
- External docs from `@llama-flow/docs` package
+- External docs from `@llamaindex/workflow-docs` package
 - Generated API docs from TypeScript source

 ### Development Notes
@@ -23,8 +23,8 @@ const config = {
        permanent: true,
      },
      {
-        source: "/docs/llamaflow/:path*.mdx",
-        destination: "/docs/llamaflow/:path*",
+        source: "/docs/workflows/:path*.mdx",
+        destination: "/docs/workflows/:path*",
        permanent: true,
      },
    ];
@@ -1,6 +1,6 @@
 {
  "name": "@llamaindex/doc",
-  "version": "0.2.28",
+  "version": "0.2.31",
  "private": true,
  "scripts": {
    "postinstall": "fumadocs-mdx",
@@ -15,7 +15,7 @@
  "dependencies": {
    "@huggingface/transformers": "^3.5.0",
    "@icons-pack/react-simple-icons": "^10.1.0",
-    "@llama-flow/docs": "0.0.8",
+    "@llamaindex/workflow-docs": "0.1.1",
    "@llamaindex/chat-ui-docs": "^0.0.5",
    "@llamaindex/cloud": "workspace:*",
    "@llamaindex/core": "workspace:*",
@@ -69,7 +69,7 @@
    "twoslash": "^0.3.1",
    "use-stick-to-bottom": "^1.0.42",
    "web-tree-sitter": "^0.24.4",
-    "zod": "^3.23.8"
+    "zod": "^3.25.67"
  },
  "devDependencies": {
    "@next/env": "^15.3.0",
@@ -13,7 +13,7 @@ const INTERNAL_LINK_REGEX = /(?:(?:\]\(|\bhref=["'])\/docs\/([^")]+))/g;
 // This captures relative links like [text](./path) or ![alt](../images/image.png)
 const RELATIVE_LINK_REGEX = /(?:\]\()(?:\s*)(?:\.\.?)\//g;

-const ALLOWED_LINKS = ["/docs/llamaflow", "/docs/chat-ui"];
+const ALLOWED_LINKS = ["/docs/workflows", "/docs/chat-ui"];

 interface LinkValidationResult {
  file: string;
@@ -11,9 +11,9 @@ import remarkMath from "remark-math";
 export const docs = defineDocs({
  dir: [
    "./src/content/docs",
-    "./node_modules/@llama-flow/docs",
+    "./node_modules/@llamaindex/workflow-docs",
    "./node_modules/@llamaindex/chat-ui-docs",
-    // NOTE: When adding external docs (like chat-ui or llama-flow above),
+    // NOTE: When adding external docs (like chat-ui or workflow-docs above),
    // make sure to also update:
    // 1. scripts/validate-links.mts - add to ALLOWED_LINKS array
    // 2. next.config.mjs - add redirect for .mdx files
@@ -74,12 +74,21 @@ const server = mcp({
  args: ["-y", "@modelcontextprotocol/server-filesystem", "."],
  verbose: true,
 });
-// or by SSE
+// or by StreamableHTTP transport
 const server = mcp({
  url: "http://localhost:8000/mcp",
  verbose: true,
 });

+// if your MCP server is not using StreamableHTTP transport, you can also use SSE transport
+// by setting useSSETransport to true.
+// See: https://modelcontextprotocol.io/docs/concepts/transports#server-sent-events-sse-deprecated
+const server = mcp({
+  url: "http://localhost:8000/mcp",
+  useSSETransport: true,
+  verbose: true,
+});
+
 // 3. Get tools from MCP server
 const tools = await server.tools();

@@ -9,10 +9,13 @@ Workflows are designed to be flexible and can be used to build agents, RAG flows
 To use workflows install this package:

 ```package-install
-npm i @llamaindex/workflow
+npm i @llamaindex/workflow-core
 ```

-This package is a stable, production-ready version of our [llama-flow](/docs/llamaflow) project. 
+This contains the core functionality for the workflow system. You can read more about the core concepts in the [workflow-core](/docs/workflows) section.

-While you can still reference the llama-flow documentation for detailed information about the underlying concepts, we recommend using the `@llamaindex/workflow` package for all new projects to ensure stability and long-term availability.
+In contrast, the `@llamaindex/workflow` package contains more utiltities, such as prebuilt agents.

+```package-install
+npm i @llamaindex/workflow
+```
@@ -28,11 +28,12 @@ embedding vector(1536)
 );
 ```

-- Create a function for similarity search
+-- Create a function for similarity search with filtering support
 ```sql
 create function match_documents (
 query_embedding vector(1536),
-match_count int
+match_count int,
+filter jsonb DEFAULT '{}'
 ) returns table (
 id uuid,
 content text,
@@ -52,6 +53,7 @@ metadata,
 embedding,
 1 - (embedding <=> query_embedding) as similarity
 from documents
+where metadata @> filter
 order by embedding <=> query_embedding
 limit match_count;
 end;
@@ -96,6 +98,7 @@ const index = await VectorStoreIndex.fromDocuments(documents, {
 ```ts
 const queryEngine = index.asQueryEngine();

+// Basic query without filters
 const response = await queryEngine.query({
  query: "What is in the document?",
 });
@@ -104,6 +107,32 @@ const response = await queryEngine.query({
 console.log(response.toString());
 ```

+## Query with filters
+
+You can filter documents based on metadata when querying:
+
+```ts
+import { FilterOperator, MetadataFilters } from "llamaindex";
+
+// Create a filter for documents with author = "Jane Smith"
+const filters: MetadataFilters = {
+  filters: [
+    {
+      key: "author",
+      value: "Jane Smith",
+      operator: FilterOperator.EQ,
+    },
+  ],
+};
+
+// Query with filters
+const filteredResponse = await vectorStore.query({
+  queryEmbedding: embedModel.getQueryEmbedding("What is vector search?"),
+  similarityTopK: 5,
+  filters,
+});
+```
+
 ## Full code

 ```ts
@@ -11,58 +11,130 @@ npm i llamaindex @llamaindex/google
 ## Usage

 ```ts
-import { Gemini, GEMINI_MODEL } from "@llamaindex/google";
+import { gemini, GEMINI_MODEL } from "@llamaindex/google";
 import { Settings } from "llamaindex";

-Settings.llm = new Gemini({
-  model: GEMINI_MODEL.GEMINI_PRO,
-});
-```
-
-## Usage with Proxy
-
-```ts
-import { Gemini, GEMINI_MODEL } from "@llamaindex/google";
-import { Settings } from "llamaindex";
-
-Settings.llm = new Gemini({
-  model: GEMINI_MODEL.GEMINI_PRO,
-  requestOptions: {
-    baseUrl: <YOUR_PROXY_URL>   // optional, but useful for custom endpoints
-  }
+Settings.llm = gemini({
+  model: GEMINI_MODEL.GEMINI_2_0_FLASH,
 });
 ```

 ### Usage with Vertex AI

-To use Gemini via Vertex AI you can use `GeminiVertexSession`.
-
-GeminiVertexSession accepts the env variables: `GOOGLE_VERTEX_LOCATION` and `GOOGLE_VERTEX_PROJECT`
+To use Gemini via Vertex AI, you can specify the vertex configuration:

 ```ts
-import { Gemini, GEMINI_MODEL, GeminiVertexSession } from "@llamaindex/google";
+import { gemini, GEMINI_MODEL } from "@llamaindex/google";

-const gemini = new Gemini({
-  model: GEMINI_MODEL.GEMINI_PRO,
-  session: new GeminiVertexSession({
-    location: "us-central1",      // optional if provided by GOOGLE_VERTEX_LOCATION env variable
-    project: "project1",          // optional if provided by GOOGLE_VERTEX_PROJECT env variable
-    googleAuthOptions: {...},     // optional, but useful for production. It accepts all values from `GoogleAuthOptions`
-  }),
+const llm = gemini({
+  model: GEMINI_MODEL.GEMINI_2_0_FLASH,
+  vertex: {
+    project: "your-cloud-project",    // required for Vertex AI
+    location: "us-central1",          // required for Vertex AI
+  },
 });
 ```

-[GoogleAuthOptions](https://github.com/googleapis/google-auth-library-nodejs/blob/main/src/auth/googleauth.ts)
-
 To authenticate for local development:

 ```bash
-npm i @google-cloud/vertexai
 gcloud auth application-default login
 ```

 To authenticate for production you'll have to use a [service account](https://cloud.google.com/docs/authentication/). `googleAuthOptions` has `credentials` which might be useful for you.

+## Multimodal Usage
+
+Gemini supports multimodal inputs including text, images, audio, and video:
+
+```ts
+import { gemini, GEMINI_MODEL } from "@llamaindex/google";
+import fs from "fs";
+
+const llm = gemini({ model: GEMINI_MODEL.GEMINI_2_0_FLASH });
+
+const result = await llm.chat({
+  messages: [
+    {
+      role: "user",
+      content: [
+        {
+          type: "text",
+          text: "What's in this image?",
+        },
+        {
+          type: "image",
+          data: fs.readFileSync("./image.jpg").toString("base64"),
+          mimeType: "image/jpeg",
+        },
+      ],
+    },
+  ],
+});
+```
+
+## Tool Calling
+
+Gemini supports function calling with tools:
+
+```ts
+import { gemini, GEMINI_MODEL } from "@llamaindex/google";
+import { tool } from "llamaindex";
+import { z } from "zod";
+
+const llm = gemini({ model: GEMINI_MODEL.GEMINI_2_0_FLASH });
+
+const result = await llm.chat({
+  messages: [
+    {
+      content: "What's the weather in Tokyo?",
+      role: "user",
+    },
+  ],
+  tools: [
+    tool({
+      name: "weather",
+      description: "Get the weather",
+      parameters: z.object({
+        location: z.string().describe("The location to get the weather for"),
+      }),
+      execute: ({ location }) => {
+        return `The weather in ${location} is sunny and hot`;
+      },
+    }),
+  ],
+});
+```
+
+## Live API (Real-time Conversations)
+
+For real-time audio/video conversations using [Gemini Live API](https://ai.google.dev/gemini-api/docs/live). 
+
+The Live API is running directly in the frontend. That's why you have to generate an ephemeral key first on the server side and pass it to the frontend.
+
+To use the Live API, make sure to pass `apiVersion: "v1alpha"` to the `httpOptions`.
+
+```ts
+import { gemini, GEMINI_MODEL } from "@llamaindex/google";
+
+// Server-side: Generate ephemeral key
+const serverLlm = gemini({
+  model: GEMINI_MODEL.GEMINI_2_0_FLASH_LIVE,
+  httpOptions: { apiVersion: "v1alpha" },
+});
+const ephemeralKey = await serverLlm.live.getEphemeralKey();
+
+// Client-side: Use ephemeral key for Live API
+const llm = gemini({
+  apiKey: ephemeralKey,
+  model: GEMINI_MODEL.GEMINI_2_0_FLASH_LIVE,
+  voiceName: "Zephyr",
+  httpOptions: { apiVersion: "v1alpha" },
+});
+
+const session = await llm.live.connect();
+```
+
 ## Load and index documents

 For this example, we will use a single document. In a real-world scenario, you would have multiple documents to index.
@@ -90,11 +162,11 @@ const results = await queryEngine.query({
 ## Full Example

 ```ts
-import { Gemini, GEMINI_MODEL } from "@llamaindex/google";
+import { gemini, GEMINI_MODEL } from "@llamaindex/google";
 import { Document, VectorStoreIndex, Settings } from "llamaindex";

-Settings.llm = new Gemini({
-  model: GEMINI_MODEL.GEMINI_PRO,
+Settings.llm = gemini({
+  model: GEMINI_MODEL.GEMINI_2_0_FLASH,
 });

 async function main() {
@@ -104,9 +176,7 @@ async function main() {
  const index = await VectorStoreIndex.fromDocuments([document]);

  // Create a query engine
-  const queryEngine = index.asQueryEngine({
-    retriever,
-  });
+  const queryEngine = index.asQueryEngine();

  const query = "What is the meaning of life?";

@@ -1,3 +1,3 @@
 {
-  "pages": ["llamaindex", "api", "llamaflow", "chat-ui"]
+  "pages": ["llamaindex", "api", "workflows", "chat-ui"]
 }
@@ -4,7 +4,7 @@
  "tasks": {
    "build": {
      "inputs": [
-        "node_modules/@llama-flow/docs/**",
+        "node_modules/@llamaindex/workflow-docs/**",
        "node_modules/@llamaindex/chat-ui-docs/**",
        "src/**/*.ts",
        "src/**/*.tsx",
@@ -1,5 +1,18 @@
 # @llamaindex/cloudflare-worker-agent-test

+## 0.0.172
+
+### Patch Changes
+
+- Updated dependencies [7039e1a]
+  - llamaindex@0.11.11
+
+## 0.0.171
+
+### Patch Changes
+
+- llamaindex@0.11.10
+
 ## 0.0.170

 ### Patch Changes
@@ -1,6 +1,6 @@
 {
  "name": "@llamaindex/cloudflare-worker-agent-test",
-  "version": "0.0.170",
+  "version": "0.0.172",
  "type": "module",
  "private": true,
  "scripts": {
@@ -1,5 +1,11 @@
 # @llamaindex/llama-parse-browser-test

+## 0.0.71
+
+### Patch Changes
+
+- @llamaindex/cloud@4.0.16
+
 ## 0.0.70

 ### Patch Changes
@@ -1,7 +1,7 @@
 {
  "name": "@llamaindex/llama-parse-browser-test",
  "private": true,
-  "version": "0.0.70",
+  "version": "0.0.71",
  "type": "module",
  "scripts": {
    "dev": "vite",
@@ -1,5 +1,18 @@
 # @llamaindex/next-agent-test

+## 0.1.172
+
+### Patch Changes
+
+- Updated dependencies [7039e1a]
+  - llamaindex@0.11.11
+
+## 0.1.171
+
+### Patch Changes
+
+- llamaindex@0.11.10
+
 ## 0.1.170

 ### Patch Changes
@@ -1,6 +1,6 @@
 {
  "name": "@llamaindex/next-agent-test",
-  "version": "0.1.170",
+  "version": "0.1.172",
  "private": true,
  "scripts": {
    "dev": "next dev",
@@ -1,5 +1,18 @@
 # test-edge-runtime

+## 0.1.171
+
+### Patch Changes
+
+- Updated dependencies [7039e1a]
+  - llamaindex@0.11.11
+
+## 0.1.170
+
+### Patch Changes
+
+- llamaindex@0.11.10
+
 ## 0.1.169

 ### Patch Changes
@@ -1,6 +1,6 @@
 {
  "name": "@llamaindex/nextjs-edge-runtime-test",
-  "version": "0.1.169",
+  "version": "0.1.171",
  "private": true,
  "scripts": {
    "dev": "next dev",
@@ -1,5 +1,27 @@
 # @llamaindex/next-node-runtime

+## 0.1.40
+
+### Patch Changes
+
+- Updated dependencies [7039e1a]
+  - llamaindex@0.11.11
+  - @llamaindex/huggingface@0.1.16
+  - @llamaindex/readers@3.1.11
+
+## 0.1.39
+
+### Patch Changes
+
+- llamaindex@0.11.10
+
+## 0.1.38
+
+### Patch Changes
+
+- Updated dependencies [c5846bd]
+  - @llamaindex/readers@3.1.10
+
 ## 0.1.37

 ### Patch Changes
@@ -1,6 +1,6 @@
 {
  "name": "@llamaindex/next-node-runtime-test",
-  "version": "0.1.37",
+  "version": "0.1.40",
  "private": true,
  "scripts": {
    "dev": "next dev",
@@ -1,5 +1,18 @@
 # vite-import-llamaindex

+## 0.0.38
+
+### Patch Changes
+
+- Updated dependencies [7039e1a]
+  - llamaindex@0.11.11
+
+## 0.0.37
+
+### Patch Changes
+
+- llamaindex@0.11.10
+
 ## 0.0.36

 ### Patch Changes
@@ -1,7 +1,7 @@
 {
  "name": "vite-import-llamaindex",
  "private": true,
-  "version": "0.0.36",
+  "version": "0.0.38",
  "type": "module",
  "scripts": {
    "build": "vite build",
@@ -1,5 +1,18 @@
 # @llamaindex/waku-query-engine-test

+## 0.0.172
+
+### Patch Changes
+
+- Updated dependencies [7039e1a]
+  - llamaindex@0.11.11
+
+## 0.0.171
+
+### Patch Changes
+
+- llamaindex@0.11.10
+
 ## 0.0.170

 ### Patch Changes
@@ -1,6 +1,6 @@
 {
  "name": "@llamaindex/waku-query-engine-test",
-  "version": "0.0.170",
+  "version": "0.0.172",
  "type": "module",
  "private": true,
  "scripts": {
@@ -7,7 +7,7 @@
  "dependencies": {
    "@llamaindex/workflow": "1.1.1",
    "llamaindex": "0.10.5",
-    "zod": "^3.23.8"
+    "zod": "^3.25.67"
  },
  "devDependencies": {
    "tsx": "^4.19.1",
@@ -27,6 +27,6 @@
    "pg": "^8.12.0",
    "pgvector": "0.2.0",
    "tsx": "^4.19.3",
-    "zod": "^3.24.2"
+    "zod": "^3.25.67"
  }
 }
@@ -1,5 +1,66 @@
 # examples

+## 0.3.25
+
+### Patch Changes
+
+- Updated dependencies [7039e1a]
+- Updated dependencies [7039e1a]
+  - llamaindex@0.11.11
+  - @llamaindex/core@0.6.12
+  - @llamaindex/google@0.3.11
+  - @llamaindex/cloud@4.0.16
+  - @llamaindex/node-parser@2.0.12
+  - @llamaindex/anthropic@0.3.14
+  - @llamaindex/assemblyai@0.1.11
+  - @llamaindex/clip@0.0.62
+  - @llamaindex/cohere@0.0.26
+  - @llamaindex/deepinfra@0.0.62
+  - @llamaindex/discord@0.1.11
+  - @llamaindex/huggingface@0.1.16
+  - @llamaindex/jinaai@0.0.22
+  - @llamaindex/mistral@0.1.12
+  - @llamaindex/mixedbread@0.0.26
+  - @llamaindex/notion@0.1.11
+  - @llamaindex/ollama@0.1.12
+  - @llamaindex/openai@0.4.6
+  - @llamaindex/perplexity@0.0.19
+  - @llamaindex/portkey-ai@0.0.54
+  - @llamaindex/replicate@0.0.54
+  - @llamaindex/astra@0.0.26
+  - @llamaindex/azure@0.1.23
+  - @llamaindex/chroma@0.0.26
+  - @llamaindex/elastic-search@0.1.12
+  - @llamaindex/firestore@1.0.19
+  - @llamaindex/milvus@0.1.21
+  - @llamaindex/mongodb@0.0.27
+  - @llamaindex/pinecone@0.1.12
+  - @llamaindex/postgres@0.0.55
+  - @llamaindex/qdrant@0.1.22
+  - @llamaindex/supabase@0.1.12
+  - @llamaindex/upstash@0.0.26
+  - @llamaindex/weaviate@0.0.27
+  - @llamaindex/vercel@0.1.12
+  - @llamaindex/voyage-ai@1.0.18
+  - @llamaindex/readers@3.1.11
+  - @llamaindex/tools@0.1.1
+  - @llamaindex/workflow@1.1.12
+  - @llamaindex/deepseek@0.0.22
+  - @llamaindex/fireworks@0.0.22
+  - @llamaindex/groq@0.0.77
+  - @llamaindex/together@0.0.22
+  - @llamaindex/vllm@0.0.48
+  - @llamaindex/xai@0.0.9
+
+## 0.3.24
+
+### Patch Changes
+
+- Updated dependencies [096bf2b]
+- Updated dependencies [c5846bd]
+  - @llamaindex/tools@0.1.0
+  - @llamaindex/readers@3.1.10
+
 ## 0.3.23

 ### Patch Changes
@@ -6,15 +6,24 @@ async function main() {
  // Create an MCP server for filesystem tools
  const server = mcp({
    command: "npx",
-    args: ["-y", "@modelcontextprotocol/server-filesystem", "."],
+    args: ["-y", "@modelcontextprotocol/server-filesystem@latest", "."],
    verbose: true,
  });
-  // You can also connect to the MCP server using SSE
-  // See: https://modelcontextprotocol.io/docs/concepts/transports#server-sent-events-sse
+  //
+  // You can also connect to a remote MCP server using:
+  // 1. StreamableHTTP transport (recommended)
+  // See: https://modelcontextprotocol.io/docs/concepts/transports#streamable-http
  // const server = mcp({
  //   url: "http://localhost:8000/mcp",
  //   verbose: true,
  // });
+  // 2.Or using SSE transport (will be deprecated soon)
+  // See: https://modelcontextprotocol.io/docs/concepts/transports#server-sent-events-sse-deprecated
+  // const server = mcp({
+  //   url: "http://localhost:8000/mcp",
+  //   useSSETransport: true,
+  //   verbose: true,
+  // });

  try {
    // Create an agent that uses the MCP tools
@@ -0,0 +1,72 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<company name="MidSizeCorp" founded="2008">
+  <division name="Engineering" head="Dana White">
+    <department name="Frontend" lead="Alex Kim">
+      <team name="Web">
+        <employee id="E01">
+          <name>Jordan Lee</name>
+          <role>Lead Developer</role>
+          <projects>
+            <project code="PRJ101" status="active">
+              <title>User Portal</title>
+              <deadline>2025-08-01</deadline>
+              <tasks>
+                <task id="T1011">
+                  <description>Implement login page</description>
+                  <due>2025-05-10</due>
+                </task>
+                <task id="T1012">
+                  <description>Design dashboard</description>
+                  <due>2025-05-20</due>
+                </task>
+              </tasks>
+            </project>
+          </projects>
+        </employee>
+        <employee id="E02">
+          <name>Riley Chen</name>
+          <role>UI Designer</role>
+        </employee>
+      </team>
+      <team name="Mobile">
+        <employee id="E03">
+          <name>Sam Patel</name>
+          <role>iOS Developer</role>
+        </employee>
+      </team>
+    </department>
+    <department name="Backend" lead="Morgan Reed">
+      <team name="API">
+        <employee id="E04">
+          <name>Taylor Jones</name>
+          <role>API Engineer</role>
+        </employee>
+      </team>
+      <team name="Database">
+        <employee id="E05">
+          <name>Casey Nguyen</name>
+          <role>DB Administrator</role>
+        </employee>
+      </team>
+    </department>
+  </division>
+
+  <division name="Marketing" head="Pat Morgan">
+    <department name="Digital" lead="Alex Rivera">
+      <team name="Content">
+        <employee id="M01">
+          <name>Charlie Brooks</name>
+          <role>Content Strategist</role>
+        </employee>
+      </team>
+    </department>
+  </division>
+
+  <headquarters location="Chicago, USA">
+    <address>
+      <street>789 Lake Shore Drive</street>
+      <city>Chicago</city>
+      <zip>60601</zip>
+    </address>
+  </headquarters>
+</company>
@@ -1,14 +1,16 @@
-import { Gemini, GEMINI_MODEL } from "@llamaindex/google";
+import { gemini, GEMINI_MODEL } from "@llamaindex/google";
 import fs from "fs";
+import { tool } from "llamaindex";
+import { z } from "zod";

 (async () => {
  if (!process.env.GOOGLE_API_KEY) {
    throw new Error("Please set the GOOGLE_API_KEY environment variable.");
  }
-  const gemini = new Gemini({
-    model: GEMINI_MODEL.GEMINI_PRO_1_5,
-  });
-  const result = await gemini.chat({
+  const llm = gemini({ model: GEMINI_MODEL.GEMINI_2_0_FLASH });
+
+  // normal chat
+  const result = await llm.chat({
    messages: [
      { content: "You want to talk in rhymes.", role: "system" },
      {
@@ -18,10 +20,10 @@ import fs from "fs";
      },
    ],
  });
-  console.log(result);
+  console.log("\n normal chat: \n", result);

  // chat with file
-  const resultWithFile = await gemini.chat({
+  const resultWithFile = await llm.chat({
    messages: [
      {
        role: "user",
@@ -39,6 +41,52 @@ import fs from "fs";
      },
    ],
  });
+  console.log("\n chat with file: \n", resultWithFile);

-  console.log(resultWithFile);
+  // chat with image base64
+  const resultWithImageFile = await llm.chat({
+    messages: [
+      {
+        role: "user",
+        content: [
+          {
+            type: "text",
+            text: "What's in this image?",
+          },
+          {
+            type: "image",
+            data: fs
+              .readFileSync("./multimodal/data/60.jpg")
+              .toString("base64"),
+            mimeType: "image/png",
+          },
+        ],
+      },
+    ],
+  });
+  console.log("\n chat with image base64: \n", resultWithImageFile);
+
+  // chat with tool
+  const resultWithTool = await llm.chat({
+    messages: [
+      {
+        content: "What's the weather in Tokyo?",
+        role: "user",
+      },
+    ],
+    tools: [
+      tool({
+        name: "weather",
+        description: "Get the weather",
+        parameters: z.object({
+          location: z.string().describe("The location to get the weather for"),
+        }),
+        execute: ({ location }) => {
+          console.log("weather", location);
+          return `The weather in ${location} is sunny and hot`;
+        },
+      }),
+    ],
+  });
+  console.log("\n chat with tool: \n", resultWithTool.message.options); // should have toolCall
 })();
@@ -1,11 +1,14 @@
-import { Gemini, GEMINI_MODEL, GeminiVertexSession } from "@llamaindex/google";
+import { gemini, GEMINI_MODEL } from "@llamaindex/google";

 (async () => {
-  const gemini = new Gemini({
-    model: GEMINI_MODEL.GEMINI_PRO,
-    session: new GeminiVertexSession(),
+  const llm = gemini({
+    model: GEMINI_MODEL.GEMINI_2_0_FLASH,
+    vertex: {
+      project: "your-cloud-project", // update to your cloud project
+      location: "us-central1",
+    },
  });
-  const result = await gemini.chat({
+  const result = await llm.chat({
    messages: [
      { content: "You want to talk in rhymes.", role: "system" },
      {
@@ -16,9 +16,19 @@ async function main() {

  console.log("🚀 Initializing Gemini Live API example...");

+  // Server-side (token creation):
+  const serverllm = gemini({
+    model: GEMINI_MODEL.GEMINI_2_0_FLASH_LIVE,
+    httpOptions: { apiVersion: "v1alpha" }, // must use v1alpha to generate ephemeral key
+  });
+  const ephemeralKey = await serverllm.live.getEphemeralKey();
+
+  // Client-side (Live API connection):
  const llm = gemini({
+    apiKey: ephemeralKey, // use ephemeral key for client-side
    model: GEMINI_MODEL.GEMINI_2_0_FLASH_LIVE,
    voiceName: "Zephyr",
+    httpOptions: { apiVersion: "v1alpha" }, // must use v1alpha to init client with ephemeral key
  });

  console.log("📡 Connecting to Gemini Live session...");
@@ -3,8 +3,18 @@ import { liveEvents } from "llamaindex";
 import { saveWavFile } from "./util";

 async function main() {
-  const llm = gemini({
+  // Server-side (token creation):
+  const serverllm = gemini({
    model: GEMINI_MODEL.GEMINI_2_0_FLASH_LIVE,
+    httpOptions: { apiVersion: "v1alpha" }, // must use v1alpha to generate ephemeral key
+  });
+  const ephemeralKey = await serverllm.live.getEphemeralKey();
+
+  // Client-side (Live API connection):
+  const llm = gemini({
+    apiKey: ephemeralKey, // use ephemeral key for client-side
+    model: GEMINI_MODEL.GEMINI_2_0_FLASH_LIVE,
+    httpOptions: { apiVersion: "v1alpha" }, // must use v1alpha to init client with ephemeral key
  });

  const session = await llm.live.connect();
@@ -1,6 +1,6 @@
 {
  "name": "@llamaindex/examples",
-  "version": "0.3.23",
+  "version": "0.3.25",
  "private": true,
  "scripts": {
    "lint": "eslint .",
@@ -11,51 +11,51 @@
    "@azure/cosmos": "^4.1.1",
    "@azure/identity": "^4.4.1",
    "@azure/search-documents": "^12.1.0",
-    "@llamaindex/anthropic": "^0.3.13",
-    "@llamaindex/assemblyai": "^0.1.10",
-    "@llamaindex/astra": "^0.0.25",
-    "@llamaindex/azure": "^0.1.22",
-    "@llamaindex/chroma": "^0.0.25",
-    "@llamaindex/clip": "^0.0.61",
-    "@llamaindex/cloud": "^4.0.15",
-    "@llamaindex/cohere": "^0.0.25",
-    "@llamaindex/core": "^0.6.11",
-    "@llamaindex/deepinfra": "^0.0.61",
-    "@llamaindex/deepseek": "^0.0.21",
-    "@llamaindex/discord": "^0.1.10",
-    "@llamaindex/elastic-search": "^0.1.11",
+    "@llamaindex/anthropic": "^0.3.14",
+    "@llamaindex/assemblyai": "^0.1.11",
+    "@llamaindex/astra": "^0.0.26",
+    "@llamaindex/azure": "^0.1.23",
+    "@llamaindex/chroma": "^0.0.26",
+    "@llamaindex/clip": "^0.0.62",
+    "@llamaindex/cloud": "^4.0.16",
+    "@llamaindex/cohere": "^0.0.26",
+    "@llamaindex/core": "^0.6.12",
+    "@llamaindex/deepinfra": "^0.0.62",
+    "@llamaindex/deepseek": "^0.0.22",
+    "@llamaindex/discord": "^0.1.11",
+    "@llamaindex/elastic-search": "^0.1.12",
    "@llamaindex/env": "^0.1.30",
-    "@llamaindex/firestore": "^1.0.18",
-    "@llamaindex/fireworks": "^0.0.21",
-    "@llamaindex/google": "^0.3.10",
-    "@llamaindex/groq": "^0.0.76",
-    "@llamaindex/huggingface": "^0.1.15",
-    "@llamaindex/jinaai": "^0.0.21",
-    "@llamaindex/milvus": "^0.1.20",
-    "@llamaindex/mistral": "^0.1.11",
-    "@llamaindex/mixedbread": "^0.0.25",
-    "@llamaindex/mongodb": "^0.0.26",
-    "@llamaindex/node-parser": "^2.0.11",
-    "@llamaindex/notion": "^0.1.10",
-    "@llamaindex/ollama": "^0.1.11",
-    "@llamaindex/openai": "^0.4.5",
-    "@llamaindex/perplexity": "^0.0.18",
-    "@llamaindex/pinecone": "^0.1.11",
-    "@llamaindex/portkey-ai": "^0.0.53",
-    "@llamaindex/postgres": "^0.0.54",
-    "@llamaindex/qdrant": "^0.1.21",
-    "@llamaindex/readers": "^3.1.9",
-    "@llamaindex/replicate": "^0.0.53",
-    "@llamaindex/supabase": "^0.1.10",
-    "@llamaindex/together": "^0.0.21",
-    "@llamaindex/tools": "^0.0.17",
-    "@llamaindex/upstash": "^0.0.25",
-    "@llamaindex/vercel": "^0.1.11",
-    "@llamaindex/vllm": "^0.0.47",
-    "@llamaindex/voyage-ai": "^1.0.17",
-    "@llamaindex/weaviate": "^0.0.26",
-    "@llamaindex/workflow": "^1.1.10",
-    "@llamaindex/xai": "workspace:^0.0.8",
+    "@llamaindex/firestore": "^1.0.19",
+    "@llamaindex/fireworks": "^0.0.22",
+    "@llamaindex/google": "^0.3.11",
+    "@llamaindex/groq": "^0.0.77",
+    "@llamaindex/huggingface": "^0.1.16",
+    "@llamaindex/jinaai": "^0.0.22",
+    "@llamaindex/milvus": "^0.1.21",
+    "@llamaindex/mistral": "^0.1.12",
+    "@llamaindex/mixedbread": "^0.0.26",
+    "@llamaindex/mongodb": "^0.0.27",
+    "@llamaindex/node-parser": "^2.0.12",
+    "@llamaindex/notion": "^0.1.11",
+    "@llamaindex/ollama": "^0.1.12",
+    "@llamaindex/openai": "^0.4.6",
+    "@llamaindex/perplexity": "^0.0.19",
+    "@llamaindex/pinecone": "^0.1.12",
+    "@llamaindex/portkey-ai": "^0.0.54",
+    "@llamaindex/postgres": "^0.0.55",
+    "@llamaindex/qdrant": "^0.1.22",
+    "@llamaindex/readers": "^3.1.11",
+    "@llamaindex/replicate": "^0.0.54",
+    "@llamaindex/supabase": "^0.1.12",
+    "@llamaindex/together": "^0.0.22",
+    "@llamaindex/tools": "^0.1.1",
+    "@llamaindex/upstash": "^0.0.26",
+    "@llamaindex/vercel": "^0.1.12",
+    "@llamaindex/vllm": "^0.0.48",
+    "@llamaindex/voyage-ai": "^1.0.18",
+    "@llamaindex/weaviate": "^0.0.27",
+    "@llamaindex/workflow": "^1.1.12",
+    "@llamaindex/xai": "workspace:^0.0.9",
    "@notionhq/client": "^2.2.15",
    "@pinecone-database/pinecone": "^4.0.0",
    "@vercel/postgres": "^0.10.0",
@@ -64,11 +64,11 @@
    "commander": "^12.1.0",
    "dotenv": "^16.4.5",
    "js-tiktoken": "^1.0.14",
-    "llamaindex": "^0.11.9",
+    "llamaindex": "^0.11.11",
    "mongodb": "6.7.0",
    "postgres": "^3.4.4",
    "wikipedia": "^2.1.2",
-    "zod": "^3.23.8"
+    "zod": "^3.25.67"
  },
  "devDependencies": {
    "@types/node": "^22.9.0",
@@ -15,11 +15,14 @@
    "start:llamaparse-json": "node --import tsx ./src/llamaparse-json.ts",
    "start:discord": "node --import tsx ./src/discord.ts",
    "start:json": "node --import tsx ./src/json.ts",
-    "start:obsidian": "node --import tsx ./src/obsidian.ts"
+    "start:obsidian": "node --import tsx ./src/obsidian.ts",
+    "start:xml": "node --import tsx ./src/xml.ts",
+    "start:excel": "node --import tsx ./src/excel.ts"
  },
  "dependencies": {
    "@llamaindex/cloud": "workspace:* || ^2.0.24",
    "@llamaindex/readers": "workspace:* || ^1.0.25",
+    "@llamaindex/excel": "workspace:*",
    "llamaindex": "workspace:* || ^0.8.37"
  },
  "devDependencies": {
@@ -0,0 +1,20 @@
+import { ExcelReader } from "@llamaindex/excel";
+
+async function main() {
+  // Load PDF
+  const reader = new ExcelReader({
+    sheetSpecifier: 0,
+    concatRows: true,
+    fieldSeparator: ",",
+    keyValueSeparator: ":",
+  });
+
+  const documents = await reader.loadData("../data/sample_excel_sheet.xls");
+
+  for (const doc of documents) {
+    console.log(doc.text);
+    console.log("----");
+  }
+}
+
+main().catch(console.error);
@@ -0,0 +1,16 @@
+import { XMLReader } from "@llamaindex/readers/xml";
+
+async function main() {
+  // Load PDF
+  const reader = new XMLReader({
+    splitLevel: 2,
+  });
+  const documents = await reader.loadData("../data/company.xml");
+
+  for (const doc of documents) {
+    console.log(doc.text);
+    console.log("----");
+  }
+}
+
+main().catch(console.error);
@@ -1,8 +1,4 @@
-import {
-  GEMINI_EMBEDDING_MODEL,
-  GeminiEmbedding,
-  GeminiSession,
-} from "@llamaindex/google";
+import { GEMINI_EMBEDDING_MODEL, GeminiEmbedding } from "@llamaindex/google";
 import { QdrantVectorStore } from "@llamaindex/qdrant";
 import {
  Document,
@@ -12,9 +8,6 @@ import {

 const embedding = new GeminiEmbedding({
  model: GEMINI_EMBEDDING_MODEL.EMBEDDING_001,
-  session: new GeminiSession({
-    apiKey: process.env.GEMINI_API_KEY,
-  }),
 });

 async function main() {
@@ -1,5 +1,18 @@
 # @llamaindex/autotool

+## 8.0.11
+
+### Patch Changes
+
+- Updated dependencies [7039e1a]
+  - llamaindex@0.11.11
+
+## 8.0.10
+
+### Patch Changes
+
+- llamaindex@0.11.10
+
 ## 8.0.9

 ### Patch Changes
@@ -1,5 +1,20 @@
 # @llamaindex/autotool-01-node-example

+## 0.0.119
+
+### Patch Changes
+
+- Updated dependencies [7039e1a]
+  - llamaindex@0.11.11
+  - @llamaindex/autotool@8.0.11
+
+## 0.0.118
+
+### Patch Changes
+
+- llamaindex@0.11.10
+- @llamaindex/autotool@8.0.10
+
 ## 0.0.117

 ### Patch Changes
@@ -13,5 +13,5 @@
  "scripts": {
    "start": "node --import tsx --import @llamaindex/autotool/node ./src/index.ts"
  },
-  "version": "0.0.117"
+  "version": "0.0.119"
 }
@@ -6,7 +6,7 @@
    "url": "git+https://github.com/run-llama/LlamaIndexTS.git",
    "directory": "packages/autotool"
  },
-  "version": "8.0.9",
+  "version": "8.0.11",
  "description": "auto transpile your JS function to LLM Agent compatible",
  "files": [
    "dist",
@@ -1,5 +1,13 @@
 # @llamaindex/cloud

+## 4.0.16
+
+### Patch Changes
+
+- Updated dependencies [7039e1a]
+- Updated dependencies [7039e1a]
+  - @llamaindex/core@0.6.12
+
 ## 4.0.15

 ### Patch Changes
@@ -1,6 +1,6 @@
 {
  "name": "@llamaindex/cloud",
-  "version": "4.0.15",
+  "version": "4.0.16",
  "type": "module",
  "license": "MIT",
  "scripts": {
@@ -66,19 +66,19 @@
    "directory": "packages/cloud"
  },
  "devDependencies": {
-    "@hey-api/client-fetch": "^0.10.1",
-    "@hey-api/openapi-ts": "^0.67.5",
-    "@llama-flow/core": "^0.4.1",
+    "@hey-api/client-fetch": "^0.13.1",
+    "@hey-api/openapi-ts": "^0.77.0",
    "@llamaindex/core": "workspace:*",
-    "@llamaindex/env": "workspace:*"
+    "@llamaindex/env": "workspace:*",
+    "@llamaindex/workflow-core": "^1.0.0"
  },
  "peerDependencies": {
-    "@llama-flow/core": "^0.4.1",
    "@llamaindex/core": "workspace:*",
-    "@llamaindex/env": "workspace:*"
+    "@llamaindex/env": "workspace:*",
+    "@llamaindex/workflow-core": "^1.0.0"
  },
  "dependencies": {
    "p-retry": "^6.2.1",
-    "zod": "^3.25.7"
+    "zod": "^3.25.67"
  }
 }
@@ -1,25 +1,27 @@
-import { workflowEvent } from "@llama-flow/core";
-import { zodEvent } from "@llama-flow/core/util/zod";
+import { workflowEvent, type WorkflowEvent } from "@llamaindex/workflow-core";
+import { zodEvent } from "@llamaindex/workflow-core/util/zod";
 import { z } from "zod";
 import { parseFormSchema } from "./schema";

-export const uploadEvent = zodEvent(
-  parseFormSchema.merge(
-    z.object({
-      file: z
-        .string()
-        .or(z.instanceof(File))
-        .or(z.instanceof(Blob))
-        .or(z.instanceof(Uint8Array))
-        .optional()
-        .describe("input"),
-    }),
-  ),
-  {
-    debugLabel: "upload",
-    uniqueId: "52099967-34a8-419b-8950-c859eab60145",
-  },
+const uploadSchema = parseFormSchema.merge(
+  z.object({
+    file: z
+      .string()
+      .or(z.instanceof(File))
+      .or(z.instanceof(Blob))
+      .or(z.instanceof(Uint8Array))
+      .optional()
+      .describe("input"),
+  }),
 );
+
+export const uploadEvent: WorkflowEvent<
+  z.infer<typeof uploadSchema>,
+  "upload"
+> = zodEvent(uploadSchema, {
+  debugLabel: "upload",
+  uniqueId: "52099967-34a8-419b-8950-c859eab60145",
+});
 export const checkStatusEvent = workflowEvent<string>({
  debugLabel: "check-status",
  uniqueId: "462157fc-1ded-4ba7-9bc4-3e870395bd20",
@@ -1,9 +1,11 @@
-import { createClient, createConfig } from "@hey-api/client-fetch";
-import { createWorkflow, type InferWorkflowEventData } from "@llama-flow/core";
-import { createStatefulMiddleware } from "@llama-flow/core/middleware/state";
-import { withTraceEvents } from "@llama-flow/core/middleware/trace-events";
-import { pRetryHandler } from "@llama-flow/core/util/p-retry";
 import { fs, getEnv, path } from "@llamaindex/env";
+import {
+  createWorkflow,
+  type InferWorkflowEventData,
+} from "@llamaindex/workflow-core";
+import { createStatefulMiddleware } from "@llamaindex/workflow-core/middleware/state";
+import { withTraceEvents } from "@llamaindex/workflow-core/middleware/trace-events";
+import { pRetryHandler } from "@llamaindex/workflow-core/util/p-retry";
 import {
  type BodyUploadFileApiV1ParsingUploadPost,
  getJobApiV1ParsingJobJobIdGet,
@@ -13,6 +15,7 @@ import {
  type StatusEnum,
  uploadFileApiV1ParsingUploadPost,
 } from "./client";
+import { createClient, createConfig } from "./client/client";
 import {
  checkStatusEvent,
  checkStatusSuccessEvent,
@@ -1,5 +1,4 @@
 /* eslint-disable @typescript-eslint/no-explicit-any */
-import { type Client, createClient, createConfig } from "@hey-api/client-fetch";
 import { Document, FileReader } from "@llamaindex/core/schema";
 import { fs, getEnv, path } from "@llamaindex/env";
 import {
@@ -14,6 +13,7 @@ import {
  getJobTextResultApiV1ParsingJobJobIdResultTextGet,
  uploadFileApiV1ParsingUploadPost,
 } from "./api";
+import { type Client, createClient, createConfig } from "./client/client";
 import { sleep } from "./utils";

 export type Language = ParserLanguages;
@@ -171,6 +171,14 @@ export class LlamaParseReader extends FileReader {
  replace_failed_page_with_error_message_suffix?: string | undefined;
  save_images?: boolean | undefined;
  preset?: string | undefined;
+  high_res_ocr?: boolean | undefined;
+  outlined_table_extraction?: boolean | undefined;
+  hide_headers?: boolean | undefined;
+  hide_footers?: boolean | undefined;
+  page_header_prefix?: string | undefined;
+  page_header_suffix?: string | undefined;
+  page_footer_prefix?: string | undefined;
+  page_footer_suffix?: string | undefined;

  constructor(
    params: Partial<Omit<LlamaParseReader, "language" | "apiKey">> & {
@@ -352,6 +360,14 @@ export class LlamaParseReader extends FileReader {
        this.replace_failed_page_with_error_message_suffix,
      save_images: this.save_images,
      preset: this.preset,
+      high_res_ocr: this.high_res_ocr,
+      outlined_table_extraction: this.outlined_table_extraction,
+      hide_headers: this.hide_headers,
+      hide_footers: this.hide_footers,
+      page_header_prefix: this.page_header_prefix,
+      page_header_suffix: this.page_header_suffix,
+      page_footer_prefix: this.page_footer_prefix,
+      page_footer_suffix: this.page_footer_suffix,
    } satisfies {
      [Key in keyof BodyUploadFileApiParsingUploadPost]-?:
        | BodyUploadFileApiParsingUploadPost[Key]
@@ -1,5 +1,12 @@
 # @llamaindex/core

+## 0.6.12
+
+### Patch Changes
+
+- 7039e1a: Internal cleanup of base64 encoding
+- 7039e1a: chore: migrate to @google/genai SDK
+
 ## 0.6.11

 ### Patch Changes
@@ -1,7 +1,7 @@
 {
  "name": "@llamaindex/core",
  "type": "module",
-  "version": "0.6.11",
+  "version": "0.6.12",
  "description": "LlamaIndex Core Module",
  "exports": {
    "./agent": {
@@ -312,7 +312,7 @@
    "@llamaindex/env": "workspace:*",
    "@types/node": "^22.9.0",
    "magic-bytes.js": "^1.10.0",
-    "zod": "^3.23.8",
-    "zod-to-json-schema": "^3.23.3"
+    "zod": "^3.25.67",
+    "zod-to-json-schema": "^3.24.6"
  }
 }
@@ -17,7 +17,7 @@ export class SentenceWindowNodeParser extends NodeParser<TextNode[]> {
  windowSize: number;
  windowMetadataKey: string;
  originalTextMetadataKey: string;
-  sentenceSplitter: TextSplitterFn = splitBySentenceTokenizer();
+  sentenceSplitter: TextSplitterFn = splitBySentenceTokenizer([], true);
  idGenerator: () => string = () => randomUUID();

  constructor(params?: z.input<typeof sentenceWindowNodeParserSchema>) {
@@ -1,5 +1,5 @@
 declare class SentenceTokenizer {
-  constructor(abbreviations?: string[]);
+  constructor(abbreviations?: string[], trimSentences?: boolean);
  tokenize(text: string): string[];
 }

@@ -1,3 +1,24 @@
+/*
+Copyright (c) 2024, Hugo W.L. ter Doest
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
 var __getOwnPropNames = Object.getOwnPropertyNames;
 var __commonJS = (cb, mod) =>
  function __require() {
@@ -30,32 +51,47 @@ var require_tokenizer = __commonJS({
 // lib/natural/tokenizers/sentence_tokenizer.js
 var require_sentence_tokenizer = __commonJS({
  "lib/natural/tokenizers/sentence_tokenizer.js"(exports, module) {
-    var Tokenizer = require_tokenizer();
-    var NUM = "NUMBER";
-    var DELIM = "DELIM";
-    var URI = "URI";
-    var ABBREV = "ABBREV";
-    var DEBUG = false;
+    const Tokenizer = require_tokenizer();
+
+    // Strings that will be used to create placeholders
+    const NUM = "NUMBER";
+    const DELIM = "DELIM";
+    const URI = "URI";
+    const ABBREV = "ABBREV";
+
+    const DEBUG = false;
+
    function generateUniqueCode(base, index) {
+      // Surround the placeholder with {{}} to prevent shorter numbers to be recognized
+      // in larger numbers
      return `{{${base}_${index}}}`;
    }
+
    function escapeRegExp(string) {
      return string.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
    }
-    var SentenceTokenizer = class extends Tokenizer {
-      constructor(abbreviations) {
+
+    class SentenceTokenizer extends Tokenizer {
+      constructor(abbreviations, trimSentences) {
        super();
        if (abbreviations) {
          this.abbreviations = abbreviations;
        } else {
          this.abbreviations = [];
        }
+        if (trimSentences === undefined) {
+          this.trimSentences = true;
+        } else {
+          this.trimSentences = trimSentences;
+        }
        this.replacementMap = null;
        this.replacementCounter = 0;
      }
+
      replaceUrisWithPlaceholders(text) {
        const urlPattern =
          /(https?:\/\/\S+|www\.\S+|ftp:\/\/\S+|(mailto:)?[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}|file:\/\/\S+)/gi;
+
        const modifiedText = text.replace(urlPattern, (match) => {
          const placeholder = generateUniqueCode(
            URI,
@@ -64,8 +100,10 @@ var require_sentence_tokenizer = __commonJS({
          this.replacementMap.set(placeholder, match);
          return placeholder;
        });
+
        return modifiedText;
      }
+
      replaceAbbreviations(text) {
        if (this.abbreviations.length === 0) {
          return text;
@@ -79,9 +117,14 @@ var require_sentence_tokenizer = __commonJS({
          this.replacementMap.set(code, match);
          return code;
        });
+
        return replacedText;
      }
+
      replaceDelimitersWithPlaceholders(text) {
+        // Regular expression for sentence delimiters optionally followed by a bracket or quote
+        // Multiple delimiters with spaces in between are allowed
+        // The expression makes sure that the sentence delimiter group ends with a sentence delimiter
        const delimiterPattern = /([.?!… ]*)([.?!…])(["'”’)}\]]?)/g;
        const modifiedText = text.replace(
          delimiterPattern,
@@ -94,32 +137,42 @@ var require_sentence_tokenizer = __commonJS({
            return placeholder;
          },
        );
+
        return modifiedText;
      }
+
      splitOnPlaceholders(text, placeholders) {
        if (this.delimiterMap.size === 0) {
          return [text];
        }
+
        const keys = Array.from(this.delimiterMap.keys());
        const pattern = new RegExp(`(${keys.map(escapeRegExp).join("|")})`);
        const parts = text.split(pattern);
+
        const sentences = [];
        for (let i = 0; i < parts.length; i += 2) {
          const sentence = parts[i];
          const placeholder = parts[i + 1] || "";
          sentences.push(sentence + placeholder);
        }
+
        return sentences;
      }
+
      replaceNumbersWithCode(text) {
+        // Regular expression to match numbers, including decimal points and commas
        const numberPattern = /\b\d{1,3}(?:,\d{3})*(?:\.\d+)?\b/g;
+
        const replacedText = text.replace(numberPattern, (match) => {
          const code = generateUniqueCode(NUM, this.replacementCounter++);
          this.replacementMap.set(code, match);
          return code;
        });
+
        return replacedText;
      }
+
      revertReplacements(text) {
        let originalText = text;
        for (const [
@@ -129,16 +182,20 @@ var require_sentence_tokenizer = __commonJS({
          const pattern = new RegExp(escapeRegExp(placeholder), "g");
          originalText = originalText.replace(pattern, replacement);
        }
+
        return originalText;
      }
+
      revertDelimiters(text) {
        let originalText = text;
        for (const [placeholder, replacement] of this.delimiterMap.entries()) {
          const pattern = new RegExp(escapeRegExp(placeholder), "g");
          originalText = originalText.replace(pattern, replacement);
        }
+
        return originalText;
      }
+
      tokenize(text) {
        this.replacementCounter = 0;
        this.replacementMap = /* @__PURE__ */ new Map();
@@ -148,32 +205,43 @@ var require_sentence_tokenizer = __commonJS({
            "---Start of sentence tokenization-----------------------",
          );
        DEBUG && console.log("Original input: >>>" + text + "<<<");
+        // Replace abbreviations
        const result1 = this.replaceAbbreviations(text);
        DEBUG &&
          console.log(
            "Phase 1: replacing abbreviations: " + JSON.stringify(result1),
          );
+
+        // Replace URIs
        const result2 = this.replaceUrisWithPlaceholders(result1);
        DEBUG &&
          console.log("Phase 2: replacing URIs: " + JSON.stringify(result2));
+
+        // Replace delimiters followed by optional quotes, brackets, and braces
        const result3 = this.replaceNumbersWithCode(result2);
        DEBUG &&
          console.log(
            "Phase 3: replacing numbers with placeholders: " +
              JSON.stringify(result3),
          );
+
+        // Replace delimiters followed by optional quotes, brackets, and braces
        const result4 = this.replaceDelimitersWithPlaceholders(result3);
        DEBUG &&
          console.log(
            "Phase 4: replacing delimiters with placeholders: " +
              JSON.stringify(result4),
          );
+
+        // Split on placeholders for sentence delimiters
        const sentences = this.splitOnPlaceholders(result4);
        DEBUG &&
          console.log(
            "Phase 5: splitting into sentences on placeholders: " +
              JSON.stringify(sentences),
          );
+
+        // Replace back all abbreviations, URIs, and delimiters
        const newSentences = sentences.map((s) => {
          const s1 = this.revertReplacements(s);
          return this.revertDelimiters(s1);
@@ -183,13 +251,17 @@ var require_sentence_tokenizer = __commonJS({
            "Phase 6: replacing back abbreviations, URIs, numbers and delimiters: " +
              JSON.stringify(newSentences),
          );
+
        const trimmedSentences = this.trim(newSentences);
        DEBUG &&
          console.log(
            "Phase 7: trimming array of empty sentences: " +
              JSON.stringify(trimmedSentences),
          );
-        const trimmedSentences2 = trimmedSentences.map((sent) => sent.trim());
+
+        const trimmedSentences2 = trimmedSentences.map((sent) =>
+          this.trimSentences ? sent.trim() : sent,
+        );
        DEBUG &&
          console.log(
            "Phase 8: trimming sentences from surrounding whitespace: " +
@@ -213,9 +285,10 @@ var require_sentence_tokenizer = __commonJS({
          console.log(
            "---------------------------------------------------------",
          );
+
        return trimmedSentences2;
      }
-    };
+    }
    module.exports = SentenceTokenizer;
  },
 });
@@ -37,13 +37,17 @@ export const splitByChar = (): TextSplitterFn => {

 export const splitBySentenceTokenizer = (
  extraAbbreviations: string[] | undefined = [],
+  trimSentences: boolean = false,
 ): TextSplitterFn => {
-  const tokenizer = new SentenceTokenizer([
-    ...abbreviations.english,
-    ...abbreviations.spanish,
-    // Add the extra abbreviations provided by the user, e.g. for business-specific context
-    ...extraAbbreviations,
-  ]);
+  const tokenizer = new SentenceTokenizer(
+    [
+      ...abbreviations.english,
+      ...abbreviations.spanish,
+      // Add the extra abbreviations provided by the user, e.g. for business-specific context
+      ...extraAbbreviations,
+    ],
+    trimSentences,
+  );
  return (text: string) => {
    try {
      return tokenizer.tokenize(text);
@@ -101,17 +101,21 @@ export class SimpleKVStore extends BaseKVStore {
  static async fromPersistPath(persistPath: string): Promise<SimpleKVStore> {
    const dirPath = path.dirname(persistPath);
    if (!(await exists(dirPath))) {
-      await fs.mkdir(dirPath);
+      await fs.mkdir(dirPath, { recursive: true });
    }

    let data: DataType = {};
-    try {
-      const fileData = await fs.readFile(persistPath);
-      data = JSON.parse(fileData.toString());
-    } catch (e) {
-      console.error(
-        `No valid data found at path: ${persistPath} starting new store.`,
-      );
+    if (!(await exists(persistPath))) {
+      console.info(`Starting new store from path: ${persistPath}`);
+    } else {
+      try {
+        const fileData = await fs.readFile(persistPath);
+        data = JSON.parse(fileData.toString());
+      } catch (e) {
+        throw new Error(`Failed to load data from path: ${persistPath}`, {
+          cause: e,
+        });
+      }
    }

    const store = new SimpleKVStore(data);
@@ -0,0 +1,103 @@
+import { filetypemime } from "magic-bytes.js";
+
+/**
+ * Converts a base64 string (without data: prefix) to a Uint8Array
+ * @param base64 - The base64 string without data: prefix
+ * @returns The Uint8Array
+ */
+export function base64ToUint8Array(base64: string): Uint8Array {
+  // Decode Base64 string
+  const binaryString = atob(base64);
+
+  // Convert binary string to Uint8Array
+  const bytes = new Uint8Array(binaryString.length);
+  for (let i = 0; i < binaryString.length; i++) {
+    bytes[i] = binaryString.charCodeAt(i);
+  }
+
+  return bytes;
+}
+
+/**
+ * Converts a Uint8Array to a base64 string.
+ * @param uint8Array The Uint8Array to convert.
+ * @returns The base64-encoded string.
+ */
+export function uint8ArrayToBase64(uint8Array: Uint8Array): string {
+  let binary = "";
+  for (let i = 0; i < uint8Array.byteLength; i++) {
+    // Asserts that the value is not undefined, for `noUncheckedIndexedAccess`
+    binary += String.fromCharCode(uint8Array[i]!);
+  }
+  return btoa(binary);
+}
+
+/**
+ * Extracts the MIME type from a data URL.
+ * @param dataUrl The data URL string.
+ * @returns The MIME type from the data URL.
+ * @throws An error if the data URL is malformed.
+ */
+export function getMimeTypeFromDataUrl(dataUrl: string): string {
+  if (!dataUrl.startsWith("data:")) {
+    throw new Error("Not a data URL");
+  }
+  const commaIndex = dataUrl.indexOf(",");
+  if (commaIndex === -1) {
+    throw new Error("Invalid data URL format");
+  }
+
+  const header = dataUrl.slice(0, commaIndex);
+  const semicolonIndex = header.indexOf(";base64");
+  if (semicolonIndex === -1) {
+    throw new Error("Invalid data URL format: missing base64 encoding");
+  }
+
+  return header.slice(5, semicolonIndex);
+}
+
+/**
+ * Convert base64 data to Blob
+ * @param base64 - The base64 string
+ * @param mimeType - The MIME type of the file
+ * @returns The Blob
+ */
+export function base64ToBlob(base64: string, mimeType?: string): Blob {
+  let extractedMimeType = mimeType;
+  let base64Data = base64;
+
+  // Extract mimeType from data URL if not provided
+  if (!mimeType && base64.startsWith("data:")) {
+    extractedMimeType = getMimeTypeFromDataUrl(base64);
+    base64Data = base64.slice(base64.indexOf(",") + 1);
+  } else if (!mimeType) {
+    throw new Error(
+      "No MIME type provided and base64 is not in data URL format",
+    );
+  } else {
+    // Extract base64 data from data URL if present
+    const commaIndex = base64.indexOf(",");
+    base64Data = commaIndex !== -1 ? base64.slice(commaIndex + 1) : base64;
+  }
+
+  if (!extractedMimeType) {
+    throw new Error("No MIME type found in base64 data");
+  }
+
+  // convert base64 to Uint8Array
+  const bytes = base64ToUint8Array(base64Data);
+
+  // Create Blob
+  return new Blob([bytes], { type: extractedMimeType });
+}
+
+export async function blobToDataUrl(input: Blob) {
+  const arrayBuffer = await input.arrayBuffer();
+  const uint8Array = new Uint8Array(arrayBuffer);
+  const mimes = filetypemime(uint8Array);
+  if (mimes.length < 1) {
+    throw new Error("Unsupported image type");
+  }
+  const base64 = uint8ArrayToBase64(uint8Array);
+  return `data:${mimes[0]};base64,${base64}`;
+}
@@ -72,5 +72,6 @@ export {

 export { MockLLM } from "./mock";

+export * from "./encoding";
 export { objectEntries } from "./object-entries";
 export * from "./stream";
@@ -1,5 +1,4 @@
 import { fs } from "@llamaindex/env";
-import { filetypemime } from "magic-bytes.js";
 import type {
  ChatMessage,
  MessageContent,
@@ -9,6 +8,7 @@ import type {
 } from "../llms";
 import type { QueryType } from "../query-engine";
 import type { ImageType } from "../schema";
+import { blobToDataUrl } from "./encoding";

 /**
 * Extracts just the text whether from
@@ -110,15 +110,6 @@ export function toToolDescriptions(tools: ToolMetadata[]): string {
  return JSON.stringify(toolsObj, null, 4);
 }

-async function blobToDataUrl(input: Blob) {
-  const buffer = Buffer.from(await input.arrayBuffer());
-  const mimes = filetypemime(buffer);
-  if (mimes.length < 1) {
-    throw new Error("Unsupported image type");
-  }
-  return "data:" + mimes[0] + ";base64," + buffer.toString("base64");
-}
-
 export async function imageToDataUrl(
  input: ImageType | Uint8Array,
 ): Promise<string> {
@@ -0,0 +1,99 @@
+import {
+  base64ToBlob,
+  base64ToUint8Array,
+  blobToDataUrl,
+  getMimeTypeFromDataUrl,
+  uint8ArrayToBase64,
+} from "@llamaindex/core/utils";
+import { describe, expect, it } from "vitest";
+
+const testString = "LlamaIndex";
+const testBase64 = "TGxhbWFJbmRleA=="; // btoa('LlamaIndex')
+const testUint8Array = new TextEncoder().encode(testString);
+
+const pngB64 =
+  "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNkYAAAAAYAAjCB0C8AAAAASUVORK5CYII=";
+const pngMime = "image/png";
+const pngDataUrl = `data:${pngMime};base64,${pngB64}`;
+const pngBinaryString = atob(pngB64);
+const pngBytes = new Uint8Array(pngBinaryString.length);
+for (let i = 0; i < pngBinaryString.length; i++) {
+  pngBytes[i] = pngBinaryString.charCodeAt(i);
+}
+
+describe("Encoding utils", () => {
+  describe("base64ToUint8Array", () => {
+    it("should correctly convert a base64 string to a Uint8Array", () => {
+      const result = base64ToUint8Array(testBase64);
+      expect(result).toBeInstanceOf(Uint8Array);
+      expect(result).toEqual(testUint8Array);
+    });
+  });
+
+  describe("uint8ArrayToBase64", () => {
+    it("should correctly convert a Uint8Array to a base64 string", () => {
+      const result = uint8ArrayToBase64(testUint8Array);
+      expect(result).toBe(testBase64);
+    });
+  });
+
+  describe("getMimeTypeFromDataUrl", () => {
+    it("should extract the correct MIME type from a data URL", () => {
+      const result = getMimeTypeFromDataUrl(pngDataUrl);
+      expect(result).toBe(pngMime);
+    });
+
+    it("should throw an error for non-data URLs", () => {
+      expect(() => getMimeTypeFromDataUrl("not a data url")).toThrow(
+        "Not a data URL",
+      );
+    });
+
+    it("should throw an error for malformed data URLs", () => {
+      expect(() => getMimeTypeFromDataUrl("data:image/pngbase64,abc")).toThrow(
+        "Invalid data URL format: missing base64 encoding",
+      );
+      expect(() => getMimeTypeFromDataUrl("data:image/png;base64")).toThrow(
+        "Invalid data URL format",
+      );
+    });
+  });
+
+  describe("base64ToBlob", () => {
+    it("should convert from a data URL string", async () => {
+      const blob = base64ToBlob(pngDataUrl);
+      expect(blob).toBeInstanceOf(Blob);
+      expect(blob.type).toBe(pngMime);
+      const arrayBuffer = await blob.arrayBuffer();
+      expect(new Uint8Array(arrayBuffer)).toEqual(pngBytes);
+    });
+
+    it("should convert from a base64 string with an explicit MIME type", async () => {
+      const blob = base64ToBlob(pngB64, pngMime);
+      expect(blob).toBeInstanceOf(Blob);
+      expect(blob.type).toBe(pngMime);
+      const arrayBuffer = await blob.arrayBuffer();
+      expect(new Uint8Array(arrayBuffer)).toEqual(pngBytes);
+    });
+
+    it("should prioritize the explicit MIME type if a data URL is provided", async () => {
+      const differentMime = "image/jpeg";
+      const blob = base64ToBlob(pngDataUrl, differentMime);
+      expect(blob.type).toBe(differentMime);
+    });
+
+    it("should throw an error if no MIME type can be determined", () => {
+      expect(() => base64ToBlob(pngB64)).toThrow(
+        "No MIME type provided and base64 is not in data URL format",
+      );
+    });
+  });
+
+  describe("blobToDataUrl", () => {
+    it("should correctly convert a blob to a data URL", async () => {
+      const blob = new Blob([pngBytes], { type: "image/png" });
+      const result = await blobToDataUrl(blob);
+      expect(result).toBe(pngDataUrl);
+    });
+  });
+});
@@ -3,6 +3,7 @@ import {
  splitBySentenceTokenizer,
 } from "@llamaindex/core/node-parser";
 import { Document } from "@llamaindex/core/schema";
+import { Tokenizers, tokenizers } from "@llamaindex/env/tokenizers";
 import { describe, expect, test } from "vitest";

 describe("sentence splitter", () => {
@@ -75,20 +76,26 @@ describe("sentence splitter", () => {
    expect(splits).toEqual(["This is a sentence. This is another sentence."]);
  });

-  test("overall split long text", () => {
+  test("keep the space, if there's no split", () => {
+    const text = "The short sentence. The long long long long sentence.";
    const sentenceSplitter = new SentenceSplitter({
-      chunkSize: 10,
+      chunkSize: 1024,
      chunkOverlap: 0,
    });
-    const splits = sentenceSplitter.splitText(
-      "The first short sentence. The first long long long sentence. The second short sentence. The second long long long sentence.",
-    );
-    expect(splits).toEqual([
-      "The first short sentence.",
-      "The first long long long sentence.",
-      "The second short sentence.",
-      "The second long long long sentence.",
-    ]);
+    const splits = sentenceSplitter.splitText(text);
+    expect(splits).toEqual([text]);
+  });
+
+  test("split at tokenizer limit", () => {
+    const tokenizer = tokenizers.tokenizer(Tokenizers.CL100K_BASE);
+    const text = "The short sentence. The long long long long sentence.";
+    const sentenceSplitter = new SentenceSplitter({
+      tokenizer,
+      chunkSize: tokenizer.encode(text).length,
+      chunkOverlap: 0,
+    });
+    const splits = sentenceSplitter.splitText(text + " " + text);
+    expect(splits).toEqual([text, text]);
  });

  test("doesn't split decimals", () => {
@@ -1,5 +1,18 @@
 # @llamaindex/experimental

+## 0.0.188
+
+### Patch Changes
+
+- Updated dependencies [7039e1a]
+  - llamaindex@0.11.11
+
+## 0.0.187
+
+### Patch Changes
+
+- llamaindex@0.11.10
+
 ## 0.0.186

 ### Patch Changes
@@ -1,7 +1,7 @@
 {
  "name": "@llamaindex/experimental",
  "description": "Experimental package for LlamaIndexTS",
-  "version": "0.0.186",
+  "version": "0.0.188",
  "type": "module",
  "types": "dist/type/index.d.ts",
  "main": "dist/cjs/index.js",
@@ -1,5 +1,24 @@
 # llamaindex

+## 0.11.11
+
+### Patch Changes
+
+- 7039e1a: Internal cleanup of base64 encoding
+- Updated dependencies [7039e1a]
+- Updated dependencies [7039e1a]
+  - @llamaindex/core@0.6.12
+  - @llamaindex/cloud@4.0.16
+  - @llamaindex/node-parser@2.0.12
+  - @llamaindex/workflow@1.1.12
+
+## 0.11.10
+
+### Patch Changes
+
+- Updated dependencies [f7ec293]
+  - @llamaindex/workflow@1.1.11
+
 ## 0.11.9

 ### Patch Changes
@@ -1,6 +1,6 @@
 {
  "name": "llamaindex",
-  "version": "0.11.9",
+  "version": "0.11.11",
  "license": "MIT",
  "type": "module",
  "keywords": [
@@ -18,11 +18,11 @@ import type {
 } from "@llamaindex/core/llms";
 import {
  extractText,
+  isAsyncIterable,
  stringifyJSONToMessageContent,
 } from "@llamaindex/core/utils";
 import { randomUUID } from "@llamaindex/env";
 import { getReACTAgentSystemHeader } from "../internal/prompt/react.js";
-import { isAsyncIterable } from "../internal/utils.js";
 import { Settings } from "../Settings.js";

 export type ReACTAgentParams = AgentParamsBase<LLM>;
@@ -63,7 +63,6 @@ export * from "./evaluation/index.js";
 export * from "./extractors/index.js";
 export * from "./indices/index.js";
 export * from "./ingestion/index.js";
-export { imageToDataUrl } from "./internal/utils.js";
 export * from "./node-parser.js";
 export * from "./objects/index.js";
 export * from "./OutputParser.js";
@@ -1,77 +0,0 @@
-import type { ImageType } from "@llamaindex/core/schema";
-import { fs } from "@llamaindex/env";
-import { filetypemime } from "magic-bytes.js";
-
-export const isAsyncIterable = (
-  obj: unknown,
-): obj is AsyncIterable<unknown> => {
-  return obj != null && typeof obj === "object" && Symbol.asyncIterator in obj;
-};
-
-export const isReadableStream = (obj: unknown): obj is ReadableStream => {
-  return obj instanceof ReadableStream;
-};
-
-export const isIterable = (obj: unknown): obj is Iterable<unknown> => {
-  return obj != null && typeof obj === "object" && Symbol.iterator in obj;
-};
-
-async function blobToDataUrl(input: Blob) {
-  const buffer = Buffer.from(await input.arrayBuffer());
-  const mimes = filetypemime(buffer);
-  if (mimes.length < 1) {
-    throw new Error("Unsupported image type");
-  }
-  return "data:" + mimes[0] + ";base64," + buffer.toString("base64");
-}
-
-export async function imageToString(input: ImageType): Promise<string> {
-  if (input instanceof Blob) {
-    // if the image is a Blob, convert it to a base64 data URL
-    return await blobToDataUrl(input);
-  } else if (typeof input === "string") {
-    return input;
-  } else if (input instanceof URL) {
-    return input.toString();
-  } else {
-    throw new Error(`Unsupported input type: ${typeof input}`);
-  }
-}
-
-export function stringToImage(input: string): ImageType {
-  if (input.startsWith("data:")) {
-    // if the input is a base64 data URL, convert it back to a Blob
-    const base64Data = input.split(",")[1]!;
-    const byteArray = Buffer.from(base64Data, "base64");
-    return new Blob([byteArray]);
-  } else if (input.startsWith("http://") || input.startsWith("https://")) {
-    return new URL(input);
-  } else {
-    return input;
-  }
-}
-
-export async function imageToDataUrl(
-  input: ImageType | Uint8Array,
-): Promise<string> {
-  // first ensure, that the input is a Blob
-  if (
-    (input instanceof URL && input.protocol === "file:") ||
-    typeof input === "string"
-  ) {
-    // string or file URL
-    const dataBuffer = await fs.readFile(
-      input instanceof URL ? input.pathname : input,
-    );
-    input = new Blob([dataBuffer]);
-  } else if (!(input instanceof Blob)) {
-    if (input instanceof URL) {
-      throw new Error(`Unsupported URL with protocol: ${input.protocol}`);
-    } else if (input instanceof Uint8Array) {
-      input = new Blob([input]); // convert Uint8Array to Blob
-    } else {
-      throw new Error(`Unsupported input type: ${typeof input}`);
-    }
-  }
-  return await blobToDataUrl(input);
-}
@@ -280,18 +280,17 @@ export class SimpleVectorStore extends BaseVectorStore {
    }

    let dataDict: Record<string, unknown> = {};
-    try {
-      const fileData = await fs.readFile(persistPath);
-      dataDict = JSON.parse(fileData.toString());
-    } catch (e) {
-      console.error(
-        `No valid data found at path: ${persistPath} starting new store.`,
-      );
-      // persist empty data, to ignore this error in the future
-      await SimpleVectorStore.persistData(
-        persistPath,
-        new SimpleVectorStoreData(),
-      );
+    if (!(await exists(persistPath))) {
+      console.info(`Starting new store from path: ${persistPath}`);
+    } else {
+      try {
+        const fileData = await fs.readFile(persistPath);
+        dataDict = JSON.parse(fileData.toString());
+      } catch (e) {
+        throw new Error(`Failed to load data from path: ${persistPath}`, {
+          cause: e,
+        });
+      }
    }

    const data = new SimpleVectorStoreData();
@@ -1,5 +1,11 @@
 # @llamaindex/core-test

+## 0.1.7
+
+### Patch Changes
+
+- @llamaindex/openai@0.4.6
+
 ## 0.1.6

 ### Patch Changes
@@ -1,43 +0,0 @@
-import { OpenAIEmbedding } from "@llamaindex/openai";
-import {
-  Settings,
-  storageContextFromDefaults,
-  type StorageContext,
-} from "llamaindex";
-import { existsSync, rmSync } from "node:fs";
-import { mkdtemp } from "node:fs/promises";
-import { tmpdir } from "node:os";
-import { join } from "node:path";
-import {
-  afterAll,
-  beforeAll,
-  describe,
-  expect,
-  test,
-  vi,
-  vitest,
-} from "vitest";
-const testDir = await mkdtemp(join(tmpdir(), "test-"));
-vitest.spyOn(console, "error");
-
-describe("StorageContext", () => {
-  let storageContext: StorageContext;
-
-  beforeAll(async () => {
-    Settings.embedModel = new OpenAIEmbedding();
-    storageContext = await storageContextFromDefaults({
-      persistDir: testDir,
-    });
-  });
-
-  test("initializes", async () => {
-    vi.mocked(console.error).mockImplementation(() => {}); // silence console.error
-
-    expect(existsSync(testDir)).toBe(true);
-    expect(storageContext).toBeDefined();
-  });
-
-  afterAll(() => {
-    rmSync(testDir, { recursive: true });
-  });
-});
@@ -1,7 +1,7 @@
 {
  "name": "@llamaindex/llamaindex-test",
  "private": true,
-  "version": "0.1.6",
+  "version": "0.1.7",
  "type": "module",
  "scripts": {
    "test": "vitest run"
@@ -0,0 +1,130 @@
+import { OpenAIEmbedding } from "@llamaindex/openai";
+import {
+  Document,
+  Settings,
+  SimpleDocumentStore,
+  SimpleIndexStore,
+  SimpleVectorStore,
+  storageContextFromDefaults,
+  VectorStoreIndex,
+} from "llamaindex";
+import { existsSync, rmSync, writeFileSync } from "node:fs";
+import { access, mkdtemp } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import {
+  afterEach,
+  beforeAll,
+  beforeEach,
+  describe,
+  expect,
+  test,
+  vi,
+} from "vitest";
+import { mockEmbeddingModel } from "../utility/mockOpenAI.js";
+
+describe("StorageContext", () => {
+  let testDir: string;
+
+  beforeAll(async () => {
+    const embedModel = new OpenAIEmbedding();
+    mockEmbeddingModel(embedModel);
+    Settings.embedModel = embedModel;
+  });
+
+  beforeEach(async () => {
+    testDir = await mkdtemp(join(tmpdir(), "test-"));
+  });
+
+  test("initializes", async () => {
+    const storageContext = await storageContextFromDefaults({
+      persistDir: testDir,
+    });
+
+    expect(existsSync(testDir)).toBe(true);
+    expect(storageContext).toBeDefined();
+  });
+
+  test("persists and loads", async () => {
+    const doc = new Document({ text: "test document" });
+    const consoleInfoSpy = vi
+      .spyOn(console, "info")
+      .mockImplementation(() => {});
+
+    // storage context from individual stores
+    const storageContext = await storageContextFromDefaults({
+      docStore: await SimpleDocumentStore.fromPersistDir(testDir),
+      vectorStore: await SimpleVectorStore.fromPersistDir(testDir),
+      indexStore: await SimpleIndexStore.fromPersistDir(testDir),
+    });
+
+    const index = await VectorStoreIndex.fromDocuments([doc], {
+      storageContext,
+    });
+    expect(consoleInfoSpy).toHaveBeenCalledTimes(3);
+    expect(consoleInfoSpy).toHaveBeenCalledWith(
+      expect.stringContaining("Starting new store"),
+    );
+    expect(index).toBeDefined();
+
+    const retriever = index.asRetriever();
+    const result = await retriever.retrieve("test");
+    expect(result).toBeDefined();
+    expect(result.length).toBe(1);
+
+    // Check that the test data files exist
+    await expectTestDataFilesExist(testDir);
+
+    consoleInfoSpy.mockClear();
+
+    // Now, load it again. Since data was persisted, we should not see the error.
+    const newStorageContext = await storageContextFromDefaults({
+      docStore: await SimpleDocumentStore.fromPersistDir(testDir),
+      vectorStore: await SimpleVectorStore.fromPersistDir(testDir),
+      indexStore: await SimpleIndexStore.fromPersistDir(testDir),
+    });
+
+    const loadedIndex = await VectorStoreIndex.init({
+      storageContext: newStorageContext,
+    });
+    const loadedRetriever = loadedIndex.asRetriever();
+    const loadedResult = await loadedRetriever.retrieve("test");
+    expect(loadedResult).toBeDefined();
+    expect(loadedResult.length).toBe(1);
+
+    await expectTestDataFilesExist(testDir);
+
+    expect(consoleInfoSpy).not.toHaveBeenCalled();
+
+    consoleInfoSpy.mockRestore();
+  });
+
+  test("throws error on corrupted data", async () => {
+    // test SimpleKVStore
+    const docStorePath = join(testDir, "doc_store.json");
+    writeFileSync(docStorePath, "corrupted data");
+    await expect(SimpleDocumentStore.fromPersistDir(testDir)).rejects.toThrow(
+      /Failed to load data from path/,
+    );
+    // test SimpleVectorStore
+    const vectorStorePath = join(testDir, "vector_store.json");
+    writeFileSync(vectorStorePath, "corrupted data");
+    await expect(SimpleVectorStore.fromPersistDir(testDir)).rejects.toThrow(
+      /Failed to load data from path/,
+    );
+  });
+
+  afterEach(() => {
+    rmSync(testDir, { recursive: true, force: true });
+  });
+});
+
+async function expectTestDataFilesExist(testDir: string) {
+  const docStorePath = `${testDir}/doc_store.json`;
+  const vectorStorePath = `${testDir}/vector_store.json`;
+  const indexStorePath = `${testDir}/index_store.json`;
+
+  expect(await access(docStorePath)).toBeUndefined();
+  expect(await access(vectorStorePath)).toBeUndefined();
+  expect(await access(indexStorePath)).toBeUndefined();
+}
@@ -1,5 +1,13 @@
 # @llamaindex/node-parser

+## 2.0.12
+
+### Patch Changes
+
+- Updated dependencies [7039e1a]
+- Updated dependencies [7039e1a]
+  - @llamaindex/core@0.6.12
+
 ## 2.0.11

 ### Patch Changes
@@ -1,6 +1,6 @@
 {
  "name": "@llamaindex/node-parser",
-  "version": "2.0.11",
+  "version": "2.0.12",
  "description": "Node parser for LlamaIndex",
  "type": "module",
  "exports": {
@@ -1,5 +1,13 @@
 # @llamaindex/anthropic

+## 0.3.14
+
+### Patch Changes
+
+- Updated dependencies [7039e1a]
+- Updated dependencies [7039e1a]
+  - @llamaindex/core@0.6.12
+
 ## 0.3.13

 ### Patch Changes
@@ -1,7 +1,7 @@
 {
  "name": "@llamaindex/anthropic",
  "description": "Anthropic Adapter for LlamaIndex",
-  "version": "0.3.13",
+  "version": "0.3.14",
  "type": "module",
  "main": "./dist/index.cjs",
  "module": "./dist/index.js",
@@ -1,5 +1,13 @@
 # @llamaindex/assemblyai

+## 0.1.11
+
+### Patch Changes
+
+- Updated dependencies [7039e1a]
+- Updated dependencies [7039e1a]
+  - @llamaindex/core@0.6.12
+
 ## 0.1.10

 ### Patch Changes
@@ -1,7 +1,7 @@
 {
  "name": "@llamaindex/assemblyai",
  "description": "AssemblyAI Reader for LlamaIndex",
-  "version": "0.1.10",
+  "version": "0.1.11",
  "type": "module",
  "types": "dist/index.d.ts",
  "main": "dist/index.cjs",
@@ -1,5 +1,13 @@
 # @llamaindex/community

+## 0.0.107
+
+### Patch Changes
+
+- Updated dependencies [7039e1a]
+- Updated dependencies [7039e1a]
+  - @llamaindex/core@0.6.12
+
 ## 0.0.106

 ### Patch Changes
@@ -1,7 +1,7 @@
 {
  "name": "@llamaindex/aws",
  "description": "AWS package for LlamaIndexTS",
-  "version": "0.0.106",
+  "version": "0.0.107",
  "type": "module",
  "types": "dist/type/index.d.ts",
  "main": "dist/cjs/index.js",
@@ -1,5 +1,14 @@
 # @llamaindex/clip

+## 0.0.62
+
+### Patch Changes
+
+- Updated dependencies [7039e1a]
+- Updated dependencies [7039e1a]
+  - @llamaindex/core@0.6.12
+  - @llamaindex/openai@0.4.6
+
 ## 0.0.61

 ### Patch Changes
@@ -1,7 +1,7 @@
 {
  "name": "@llamaindex/clip",
  "description": "Clip Embedding Adapter for LlamaIndex",
-  "version": "0.0.61",
+  "version": "0.0.62",
  "type": "module",
  "types": "dist/index.d.ts",
  "main": "dist/index.cjs",
@@ -1,5 +1,13 @@
 # @llamaindex/cohere

+## 0.0.26
+
+### Patch Changes
+
+- Updated dependencies [7039e1a]
+- Updated dependencies [7039e1a]
+  - @llamaindex/core@0.6.12
+
 ## 0.0.25

 ### Patch Changes
@@ -1,7 +1,7 @@
 {
  "name": "@llamaindex/cohere",
  "description": "Cohere Adapter for LlamaIndex",
-  "version": "0.0.25",
+  "version": "0.0.26",
  "type": "module",
  "main": "./dist/index.cjs",
  "module": "./dist/index.js",
@@ -1,5 +1,14 @@
 # @llamaindex/deepinfra

+## 0.0.62
+
+### Patch Changes
+
+- Updated dependencies [7039e1a]
+- Updated dependencies [7039e1a]
+  - @llamaindex/core@0.6.12
+  - @llamaindex/openai@0.4.6
+
 ## 0.0.61

 ### Patch Changes
@@ -1,7 +1,7 @@
 {
  "name": "@llamaindex/deepinfra",
  "description": "Deepinfra Adapter for LlamaIndex",
-  "version": "0.0.61",
+  "version": "0.0.62",
  "type": "module",
  "main": "./dist/index.cjs",
  "module": "./dist/index.js",
@@ -1,5 +1,11 @@
 # @llamaindex/deepseek

+## 0.0.22
+
+### Patch Changes
+
+- @llamaindex/openai@0.4.6
+
 ## 0.0.21

 ### Patch Changes
@@ -1,7 +1,7 @@
 {
  "name": "@llamaindex/deepseek",
  "description": "DeepSeek Adapter for LlamaIndex",
-  "version": "0.0.21",
+  "version": "0.0.22",
  "type": "module",
  "main": "./dist/index.cjs",
  "module": "./dist/index.js",
@@ -1,5 +1,13 @@
 # @llamaindex/discord

+## 0.1.11
+
+### Patch Changes
+
+- Updated dependencies [7039e1a]
+- Updated dependencies [7039e1a]
+  - @llamaindex/core@0.6.12
+
 ## 0.1.10

 ### Patch Changes
@@ -1,7 +1,7 @@
 {
  "name": "@llamaindex/discord",
  "description": "Discord Reader for LlamaIndex",
-  "version": "0.1.10",
+  "version": "0.1.11",
  "type": "module",
  "types": "dist/index.d.ts",
  "main": "dist/index.cjs",
@@ -0,0 +1,15 @@
+# @llamaindex/excel
+
+## 0.1.12
+
+### Patch Changes
+
+- Updated dependencies [7039e1a]
+- Updated dependencies [7039e1a]
+  - @llamaindex/core@0.6.12
+
+## 0.1.11
+
+### Patch Changes
+
+- 49a5e0a: feat: add Excel Reader
@@ -0,0 +1,36 @@
+{
+  "name": "@llamaindex/excel",
+  "description": "Excel Reader for LlamaIndex",
+  "version": "0.1.12",
+  "type": "module",
+  "types": "dist/index.d.ts",
+  "main": "dist/index.cjs",
+  "module": "dist/index.js",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "default": "./dist/index.js"
+    }
+  },
+  "files": [
+    "dist"
+  ],
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/run-llama/LlamaIndexTS.git",
+    "directory": "packages/providers/excel"
+  },
+  "scripts": {
+    "build": "bunchee",
+    "dev": "bunchee --watch"
+  },
+  "devDependencies": {
+    "@llamaindex/core": "workspace:*"
+  },
+  "peerDependencies": {
+    "@llamaindex/core": "workspace:*"
+  },
+  "dependencies": {
+    "xlsx": "^0.18.5"
+  }
+}
@@ -0,0 +1 @@
+export * from "./reader";
@@ -0,0 +1,100 @@
+import { Document, FileReader } from "@llamaindex/core/schema";
+import * as XLSX from "xlsx";
+
+interface ExcelReaderOptions {
+  /** concatenate all rows into one document (default: true) */
+  concatRows?: boolean;
+  /** which sheet(s) to read; string name or zero-based index */
+  sheetSpecifier?: string | number | undefined;
+  /** what to put between each field (default: ", ") */
+  fieldSeparator?: string;
+  /** what to put between key and value (default: ":") */
+  keyValueSeparator?: string;
+}
+
+/**
+ * ExcelReader reads an XLSX file from a Uint8Array (or Buffer) and
+ * produces Document instances, either one per sheet (concatenated)
+ * or one per row per sheet.
+ */
+export class ExcelReader extends FileReader<Document> {
+  private concatRows: boolean;
+  private sheetSpecifier?: string | number | undefined;
+  private fieldSeparator: string;
+  private keyValueSeparator: string;
+
+  /**
+   * @param concatRows        If true, all rows in a sheet become one Document;
+   *                          otherwise one Document per row.
+   * @param sheetSpecifier    Name or zero-based index of the sheet to read;
+   *                          undefined = all sheets.
+   * @param fieldSeparator    Separator between fields, e.g. ", ".
+   * @param keyValueSeparator Separator between key & value, e.g. ": ".
+   */
+  constructor({
+    concatRows = true,
+    sheetSpecifier = undefined,
+    fieldSeparator = ", ",
+    keyValueSeparator = ":",
+  }: ExcelReaderOptions = {}) {
+    super();
+    this.concatRows = concatRows;
+    this.sheetSpecifier = sheetSpecifier;
+    this.fieldSeparator = fieldSeparator;
+    this.keyValueSeparator = keyValueSeparator;
+  }
+
+  async loadDataAsContent(content: Uint8Array): Promise<Document[]> {
+    // Parse workbook from raw bytes
+    const workbook = XLSX.read(content, { type: "array" });
+
+    // Choose which sheets to process
+    let sheets = workbook.SheetNames;
+    if (this.sheetSpecifier !== undefined) {
+      if (typeof this.sheetSpecifier === "number") {
+        const name = sheets[this.sheetSpecifier];
+        sheets = name ? [name] : [];
+      } else {
+        sheets = sheets.includes(this.sheetSpecifier)
+          ? [this.sheetSpecifier]
+          : [];
+      }
+    }
+
+    const docs: Document[] = [];
+
+    for (const name of sheets) {
+      const ws = workbook.Sheets[name];
+
+      if (!ws) {
+        continue; // Skip if worksheet is undefined
+      }
+
+      // Convert sheet to JSON rows, using headers from first row
+      const rows: Record<string, unknown>[] = XLSX.utils.sheet_to_json(ws, {
+        defval: "N/A",
+      });
+
+      const textLines: string[] = rows.map((r) =>
+        Object.entries(r)
+          .map(([h, v]) => `${h}${this.keyValueSeparator}${v}`)
+          .join(this.fieldSeparator),
+      );
+
+      if (this.concatRows) {
+        docs.push(
+          new Document({
+            text: textLines.join("\n"),
+            metadata: { sheetName: name },
+          }),
+        );
+      } else {
+        textLines.forEach((text) =>
+          docs.push(new Document({ text, metadata: { sheetName: name } })),
+        );
+      }
+    }
+
+    return docs;
+  }
+}
--- a/Show More
+++ b/Show More
Author	SHA1	Message	Date
Alex Yang	f5b51c1def	fix: type	2025-06-30 11:33:46 -07:00
Alex Yang	606c303526	fix: missing fields	2025-06-30 11:16:27 -07:00
Alex Yang	8713364914	chore: bump sdk version	2025-06-30 10:44:27 -07:00
Marcus Schiesser	0fcc92f632	fix: sentence splitter must not trim whitespaces (#2046 )	2025-06-30 17:32:04 +07:00
Marcus Schiesser	515a8b9111	fix: error logging for fromPersistPath (#2049 )	2025-06-30 13:41:13 +07:00
github-actions[bot]	7e8efc6284	Release @llamaindex/tools@0.1.2 (#2048 )	2025-06-30 11:40:54 +07:00
Wassim Chegham	0fcf65126d	chore: export type MCPClientOptions (#2047 ) Co-authored-by: Marcus Schiesser <marcus.schiesser@googlemail.com>	2025-06-28 10:55:07 +07:00
github-actions[bot]	a50acf634c	Release 0.11.11 (#2044 ) Co-authored-by: marcusschiesser <17126+marcusschiesser@users.noreply.github.com>	2025-06-27 14:51:09 +07:00
Thuc Pham	7039e1a214	chore: migrate to @google/genai SDK (#2038 ) Co-authored-by: Marcus Schiesser <mail@marcusschiesser.de>	2025-06-27 12:09:26 +07:00
github-actions[bot]	785d010cd3	Release 0.11.10 (#2037 ) Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>	2025-06-26 14:29:33 +07:00
Marcus Schiesser	b878032131	fix release step	2025-06-26 14:18:56 +07:00
Marcus Schiesser	f7ec293a0f	chore: Update workflow-core (#2042 )	2025-06-26 14:03:03 +07:00
jerinthomascarmel	49a5e0a8cf	feat(readers): add ExcelReader for parsing Excel files (run-llama#1959) (#2033 ) Co-authored-by: Marcus Schiesser <mail@marcusschiesser.de> Co-authored-by: leehuwuj <leehuwuj@gmail.com>	2025-06-26 11:15:19 +07:00
Logan	118924799a	Rename llama-flow -> workflows in docs (#2040 )	2025-06-25 15:52:04 -07:00
allen	ec8f673dae	support filter to supabase vector search (#2036 )	2025-06-25 16:17:54 +07:00
github-actions[bot]	85039a5360	Release @llamaindex/tools@0.1.0 (#2034 )	2025-06-24 12:32:24 +07:00
Marcus Schiesser	d7305edb53	fix changesets	2025-06-24 12:26:09 +07:00
Huu Le	096bf2bda1	feat: Add support for StreamableHTTP MCP Client (#2032 )	2025-06-24 11:40:34 +07:00
jerinthomascarmel	c5846bd7dc	feat(readers): add XMLReader for parsing XML files (#1846 ) (#2031 ) Co-authored-by: Marcus Schiesser <marcus.schiesser@googlemail.com>	2025-06-24 10:46:32 +07:00