fix: type

Co-authored-by: Marcus Schiesser <marcus.schiesser@googlemail.com>

.changeset/new-chairs-camp.md

@@ -0,0 +1,5 @@
+---
+"@llamaindex/core": patch
+---
+
+Fix: split sentences must not trim whitespaces

.changeset/ninety-boats-cheer.md

@@ -0,0 +1,6 @@
+---
+"llamaindex": patch
+"@llamaindex/core": patch
+---
+
+Fix logging for fromPersistPath

.changeset/quick-lizards-wash.md

@@ -0,0 +1,5 @@
+---
+"@llamaindex/cloud": patch
+---
+
+chore: bump sdk version

packages/cloud/package.json

@@ -66,16 +66,16 @@
     "directory": "packages/cloud"
   },
   "devDependencies": {
-    "@hey-api/client-fetch": "^0.10.1",
-    "@hey-api/openapi-ts": "^0.67.5",
-    "@llama-flow/core": "^0.4.1",
+    "@hey-api/client-fetch": "^0.13.1",
+    "@hey-api/openapi-ts": "^0.77.0",
     "@llamaindex/core": "workspace:*",
-    "@llamaindex/env": "workspace:*"
+    "@llamaindex/env": "workspace:*",
+    "@llamaindex/workflow-core": "^1.0.0"
   },
   "peerDependencies": {
-    "@llama-flow/core": "^0.4.1",
     "@llamaindex/core": "workspace:*",
-    "@llamaindex/env": "workspace:*"
+    "@llamaindex/env": "workspace:*",
+    "@llamaindex/workflow-core": "^1.0.0"
   },
   "dependencies": {
     "p-retry": "^6.2.1",

packages/cloud/src/events.ts

@@ -1,25 +1,27 @@
-import { workflowEvent } from "@llama-flow/core";
-import { zodEvent } from "@llama-flow/core/util/zod";
+import { workflowEvent, type WorkflowEvent } from "@llamaindex/workflow-core";
+import { zodEvent } from "@llamaindex/workflow-core/util/zod";
 import { z } from "zod";
 import { parseFormSchema } from "./schema";
 
-export const uploadEvent = zodEvent(
-  parseFormSchema.merge(
-    z.object({
-      file: z
-        .string()
-        .or(z.instanceof(File))
-        .or(z.instanceof(Blob))
-        .or(z.instanceof(Uint8Array))
-        .optional()
-        .describe("input"),
-    }),
-  ),
-  {
-    debugLabel: "upload",
-    uniqueId: "52099967-34a8-419b-8950-c859eab60145",
-  },
+const uploadSchema = parseFormSchema.merge(
+  z.object({
+    file: z
+      .string()
+      .or(z.instanceof(File))
+      .or(z.instanceof(Blob))
+      .or(z.instanceof(Uint8Array))
+      .optional()
+      .describe("input"),
+  }),
 );
+
+export const uploadEvent: WorkflowEvent<
+  z.infer<typeof uploadSchema>,
+  "upload"
+> = zodEvent(uploadSchema, {
+  debugLabel: "upload",
+  uniqueId: "52099967-34a8-419b-8950-c859eab60145",
+});
 export const checkStatusEvent = workflowEvent<string>({
   debugLabel: "check-status",
   uniqueId: "462157fc-1ded-4ba7-9bc4-3e870395bd20",

packages/cloud/src/parse.ts

@@ -1,9 +1,11 @@
-import { createClient, createConfig } from "@hey-api/client-fetch";
-import { createWorkflow, type InferWorkflowEventData } from "@llama-flow/core";
-import { createStatefulMiddleware } from "@llama-flow/core/middleware/state";
-import { withTraceEvents } from "@llama-flow/core/middleware/trace-events";
-import { pRetryHandler } from "@llama-flow/core/util/p-retry";
 import { fs, getEnv, path } from "@llamaindex/env";
+import {
+  createWorkflow,
+  type InferWorkflowEventData,
+} from "@llamaindex/workflow-core";
+import { createStatefulMiddleware } from "@llamaindex/workflow-core/middleware/state";
+import { withTraceEvents } from "@llamaindex/workflow-core/middleware/trace-events";
+import { pRetryHandler } from "@llamaindex/workflow-core/util/p-retry";
 import {
   type BodyUploadFileApiV1ParsingUploadPost,
   getJobApiV1ParsingJobJobIdGet,
@@ -13,6 +15,7 @@ import {
   type StatusEnum,
   uploadFileApiV1ParsingUploadPost,
 } from "./client";
+import { createClient, createConfig } from "./client/client";
 import {
   checkStatusEvent,
   checkStatusSuccessEvent,

packages/cloud/src/reader.ts

@@ -1,5 +1,4 @@
 /* eslint-disable @typescript-eslint/no-explicit-any */
-import { type Client, createClient, createConfig } from "@hey-api/client-fetch";
 import { Document, FileReader } from "@llamaindex/core/schema";
 import { fs, getEnv, path } from "@llamaindex/env";
 import {
@@ -14,6 +13,7 @@ import {
   getJobTextResultApiV1ParsingJobJobIdResultTextGet,
   uploadFileApiV1ParsingUploadPost,
 } from "./api";
+import { type Client, createClient, createConfig } from "./client/client";
 import { sleep } from "./utils";
 
 export type Language = ParserLanguages;
@@ -171,6 +171,14 @@ export class LlamaParseReader extends FileReader {
   replace_failed_page_with_error_message_suffix?: string | undefined;
   save_images?: boolean | undefined;
   preset?: string | undefined;
+  high_res_ocr?: boolean | undefined;
+  outlined_table_extraction?: boolean | undefined;
+  hide_headers?: boolean | undefined;
+  hide_footers?: boolean | undefined;
+  page_header_prefix?: string | undefined;
+  page_header_suffix?: string | undefined;
+  page_footer_prefix?: string | undefined;
+  page_footer_suffix?: string | undefined;
 
   constructor(
     params: Partial<Omit<LlamaParseReader, "language" | "apiKey">> & {
@@ -352,6 +360,14 @@ export class LlamaParseReader extends FileReader {
         this.replace_failed_page_with_error_message_suffix,
       save_images: this.save_images,
       preset: this.preset,
+      high_res_ocr: this.high_res_ocr,
+      outlined_table_extraction: this.outlined_table_extraction,
+      hide_headers: this.hide_headers,
+      hide_footers: this.hide_footers,
+      page_header_prefix: this.page_header_prefix,
+      page_header_suffix: this.page_header_suffix,
+      page_footer_prefix: this.page_footer_prefix,
+      page_footer_suffix: this.page_footer_suffix,
     } satisfies {
       [Key in keyof BodyUploadFileApiParsingUploadPost]-?:
         | BodyUploadFileApiParsingUploadPost[Key]

packages/core/src/node-parser/sentence-window.ts

@@ -17,7 +17,7 @@ export class SentenceWindowNodeParser extends NodeParser<TextNode[]> {
   windowSize: number;
   windowMetadataKey: string;
   originalTextMetadataKey: string;
-  sentenceSplitter: TextSplitterFn = splitBySentenceTokenizer();
+  sentenceSplitter: TextSplitterFn = splitBySentenceTokenizer([], true);
   idGenerator: () => string = () => randomUUID();
 
   constructor(params?: z.input<typeof sentenceWindowNodeParserSchema>) {

packages/core/src/node-parser/sentence_tokenizer.d.ts

@@ -1,5 +1,5 @@
 declare class SentenceTokenizer {
-  constructor(abbreviations?: string[]);
+  constructor(abbreviations?: string[], trimSentences?: boolean);
   tokenize(text: string): string[];
 }
 

packages/core/src/node-parser/sentence_tokenizer.js

@@ -1,3 +1,24 @@
+/*
+Copyright (c) 2024, Hugo W.L. ter Doest
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
 var __getOwnPropNames = Object.getOwnPropertyNames;
 var __commonJS = (cb, mod) =>
   function __require() {
@@ -30,32 +51,47 @@ var require_tokenizer = __commonJS({
 // lib/natural/tokenizers/sentence_tokenizer.js
 var require_sentence_tokenizer = __commonJS({
   "lib/natural/tokenizers/sentence_tokenizer.js"(exports, module) {
-    var Tokenizer = require_tokenizer();
-    var NUM = "NUMBER";
-    var DELIM = "DELIM";
-    var URI = "URI";
-    var ABBREV = "ABBREV";
-    var DEBUG = false;
+    const Tokenizer = require_tokenizer();
+
+    // Strings that will be used to create placeholders
+    const NUM = "NUMBER";
+    const DELIM = "DELIM";
+    const URI = "URI";
+    const ABBREV = "ABBREV";
+
+    const DEBUG = false;
+
     function generateUniqueCode(base, index) {
+      // Surround the placeholder with {{}} to prevent shorter numbers to be recognized
+      // in larger numbers
       return `{{${base}_${index}}}`;
     }
+
     function escapeRegExp(string) {
       return string.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
     }
-    var SentenceTokenizer = class extends Tokenizer {
-      constructor(abbreviations) {
+
+    class SentenceTokenizer extends Tokenizer {
+      constructor(abbreviations, trimSentences) {
         super();
         if (abbreviations) {
           this.abbreviations = abbreviations;
         } else {
           this.abbreviations = [];
         }
+        if (trimSentences === undefined) {
+          this.trimSentences = true;
+        } else {
+          this.trimSentences = trimSentences;
+        }
         this.replacementMap = null;
         this.replacementCounter = 0;
       }
+
       replaceUrisWithPlaceholders(text) {
         const urlPattern =
           /(https?:\/\/\S+|www\.\S+|ftp:\/\/\S+|(mailto:)?[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}|file:\/\/\S+)/gi;
+
         const modifiedText = text.replace(urlPattern, (match) => {
           const placeholder = generateUniqueCode(
             URI,
@@ -64,8 +100,10 @@ var require_sentence_tokenizer = __commonJS({
           this.replacementMap.set(placeholder, match);
           return placeholder;
         });
+
         return modifiedText;
       }
+
       replaceAbbreviations(text) {
         if (this.abbreviations.length === 0) {
           return text;
@@ -79,9 +117,14 @@ var require_sentence_tokenizer = __commonJS({
           this.replacementMap.set(code, match);
           return code;
         });
+
         return replacedText;
       }
+
       replaceDelimitersWithPlaceholders(text) {
+        // Regular expression for sentence delimiters optionally followed by a bracket or quote
+        // Multiple delimiters with spaces in between are allowed
+        // The expression makes sure that the sentence delimiter group ends with a sentence delimiter
         const delimiterPattern = /([.?!… ]*)([.?!…])(["'”’)}\]]?)/g;
         const modifiedText = text.replace(
           delimiterPattern,
@@ -94,32 +137,42 @@ var require_sentence_tokenizer = __commonJS({
             return placeholder;
           },
         );
+
         return modifiedText;
       }
+
       splitOnPlaceholders(text, placeholders) {
         if (this.delimiterMap.size === 0) {
           return [text];
         }
+
         const keys = Array.from(this.delimiterMap.keys());
         const pattern = new RegExp(`(${keys.map(escapeRegExp).join("|")})`);
         const parts = text.split(pattern);
+
         const sentences = [];
         for (let i = 0; i < parts.length; i += 2) {
           const sentence = parts[i];
           const placeholder = parts[i + 1] || "";
           sentences.push(sentence + placeholder);
         }
+
         return sentences;
       }
+
       replaceNumbersWithCode(text) {
+        // Regular expression to match numbers, including decimal points and commas
         const numberPattern = /\b\d{1,3}(?:,\d{3})*(?:\.\d+)?\b/g;
+
         const replacedText = text.replace(numberPattern, (match) => {
           const code = generateUniqueCode(NUM, this.replacementCounter++);
           this.replacementMap.set(code, match);
           return code;
         });
+
         return replacedText;
       }
+
       revertReplacements(text) {
         let originalText = text;
         for (const [
@@ -129,16 +182,20 @@ var require_sentence_tokenizer = __commonJS({
           const pattern = new RegExp(escapeRegExp(placeholder), "g");
           originalText = originalText.replace(pattern, replacement);
         }
+
         return originalText;
       }
+
       revertDelimiters(text) {
         let originalText = text;
         for (const [placeholder, replacement] of this.delimiterMap.entries()) {
           const pattern = new RegExp(escapeRegExp(placeholder), "g");
           originalText = originalText.replace(pattern, replacement);
         }
+
         return originalText;
       }
+
       tokenize(text) {
         this.replacementCounter = 0;
         this.replacementMap = /* @__PURE__ */ new Map();
@@ -148,32 +205,43 @@ var require_sentence_tokenizer = __commonJS({
             "---Start of sentence tokenization-----------------------",
           );
         DEBUG && console.log("Original input: >>>" + text + "<<<");
+        // Replace abbreviations
         const result1 = this.replaceAbbreviations(text);
         DEBUG &&
           console.log(
             "Phase 1: replacing abbreviations: " + JSON.stringify(result1),
           );
+
+        // Replace URIs
         const result2 = this.replaceUrisWithPlaceholders(result1);
         DEBUG &&
           console.log("Phase 2: replacing URIs: " + JSON.stringify(result2));
+
+        // Replace delimiters followed by optional quotes, brackets, and braces
         const result3 = this.replaceNumbersWithCode(result2);
         DEBUG &&
           console.log(
             "Phase 3: replacing numbers with placeholders: " +
               JSON.stringify(result3),
           );
+
+        // Replace delimiters followed by optional quotes, brackets, and braces
         const result4 = this.replaceDelimitersWithPlaceholders(result3);
         DEBUG &&
           console.log(
             "Phase 4: replacing delimiters with placeholders: " +
               JSON.stringify(result4),
           );
+
+        // Split on placeholders for sentence delimiters
         const sentences = this.splitOnPlaceholders(result4);
         DEBUG &&
           console.log(
             "Phase 5: splitting into sentences on placeholders: " +
               JSON.stringify(sentences),
           );
+
+        // Replace back all abbreviations, URIs, and delimiters
         const newSentences = sentences.map((s) => {
           const s1 = this.revertReplacements(s);
           return this.revertDelimiters(s1);
@@ -183,13 +251,17 @@ var require_sentence_tokenizer = __commonJS({
             "Phase 6: replacing back abbreviations, URIs, numbers and delimiters: " +
               JSON.stringify(newSentences),
           );
+
         const trimmedSentences = this.trim(newSentences);
         DEBUG &&
           console.log(
             "Phase 7: trimming array of empty sentences: " +
               JSON.stringify(trimmedSentences),
           );
-        const trimmedSentences2 = trimmedSentences.map((sent) => sent.trim());
+
+        const trimmedSentences2 = trimmedSentences.map((sent) =>
+          this.trimSentences ? sent.trim() : sent,
+        );
         DEBUG &&
           console.log(
             "Phase 8: trimming sentences from surrounding whitespace: " +
@@ -213,9 +285,10 @@ var require_sentence_tokenizer = __commonJS({
           console.log(
             "---------------------------------------------------------",
           );
+
         return trimmedSentences2;
       }
-    };
+    }
     module.exports = SentenceTokenizer;
   },
 });

packages/core/src/node-parser/utils.ts

@@ -37,13 +37,17 @@ export const splitByChar = (): TextSplitterFn => {
 
 export const splitBySentenceTokenizer = (
   extraAbbreviations: string[] | undefined = [],
+  trimSentences: boolean = false,
 ): TextSplitterFn => {
-  const tokenizer = new SentenceTokenizer([
-    ...abbreviations.english,
-    ...abbreviations.spanish,
-    // Add the extra abbreviations provided by the user, e.g. for business-specific context
-    ...extraAbbreviations,
-  ]);
+  const tokenizer = new SentenceTokenizer(
+    [
+      ...abbreviations.english,
+      ...abbreviations.spanish,
+      // Add the extra abbreviations provided by the user, e.g. for business-specific context
+      ...extraAbbreviations,
+    ],
+    trimSentences,
+  );
   return (text: string) => {
     try {
       return tokenizer.tokenize(text);

packages/core/src/storage/kv-store/index.ts

@@ -101,17 +101,21 @@ export class SimpleKVStore extends BaseKVStore {
   static async fromPersistPath(persistPath: string): Promise<SimpleKVStore> {
     const dirPath = path.dirname(persistPath);
     if (!(await exists(dirPath))) {
-      await fs.mkdir(dirPath);
+      await fs.mkdir(dirPath, { recursive: true });
     }
 
     let data: DataType = {};
-    try {
-      const fileData = await fs.readFile(persistPath);
-      data = JSON.parse(fileData.toString());
-    } catch (e) {
-      console.error(
-        `No valid data found at path: ${persistPath} starting new store.`,
-      );
+    if (!(await exists(persistPath))) {
+      console.info(`Starting new store from path: ${persistPath}`);
+    } else {
+      try {
+        const fileData = await fs.readFile(persistPath);
+        data = JSON.parse(fileData.toString());
+      } catch (e) {
+        throw new Error(`Failed to load data from path: ${persistPath}`, {
+          cause: e,
+        });
+      }
     }
 
     const store = new SimpleKVStore(data);

packages/core/tests/node-parser/text-splitter.test.ts

@@ -3,6 +3,7 @@ import {
   splitBySentenceTokenizer,
 } from "@llamaindex/core/node-parser";
 import { Document } from "@llamaindex/core/schema";
+import { Tokenizers, tokenizers } from "@llamaindex/env/tokenizers";
 import { describe, expect, test } from "vitest";
 
 describe("sentence splitter", () => {
@@ -75,20 +76,26 @@ describe("sentence splitter", () => {
     expect(splits).toEqual(["This is a sentence. This is another sentence."]);
   });
 
-  test("overall split long text", () => {
+  test("keep the space, if there's no split", () => {
+    const text = "The short sentence. The long long long long sentence.";
     const sentenceSplitter = new SentenceSplitter({
-      chunkSize: 10,
+      chunkSize: 1024,
       chunkOverlap: 0,
     });
-    const splits = sentenceSplitter.splitText(
-      "The first short sentence. The first long long long sentence. The second short sentence. The second long long long sentence.",
-    );
-    expect(splits).toEqual([
-      "The first short sentence.",
-      "The first long long long sentence.",
-      "The second short sentence.",
-      "The second long long long sentence.",
-    ]);
+    const splits = sentenceSplitter.splitText(text);
+    expect(splits).toEqual([text]);
+  });
+
+  test("split at tokenizer limit", () => {
+    const tokenizer = tokenizers.tokenizer(Tokenizers.CL100K_BASE);
+    const text = "The short sentence. The long long long long sentence.";
+    const sentenceSplitter = new SentenceSplitter({
+      tokenizer,
+      chunkSize: tokenizer.encode(text).length,
+      chunkOverlap: 0,
+    });
+    const splits = sentenceSplitter.splitText(text + " " + text);
+    expect(splits).toEqual([text, text]);
   });
 
   test("doesn't split decimals", () => {

packages/llamaindex/src/vector-store/SimpleVectorStore.ts

@@ -280,18 +280,17 @@ export class SimpleVectorStore extends BaseVectorStore {
     }
 
     let dataDict: Record<string, unknown> = {};
-    try {
-      const fileData = await fs.readFile(persistPath);
-      dataDict = JSON.parse(fileData.toString());
-    } catch (e) {
-      console.error(
-        `No valid data found at path: ${persistPath} starting new store.`,
-      );
-      // persist empty data, to ignore this error in the future
-      await SimpleVectorStore.persistData(
-        persistPath,
-        new SimpleVectorStoreData(),
-      );
+    if (!(await exists(persistPath))) {
+      console.info(`Starting new store from path: ${persistPath}`);
+    } else {
+      try {
+        const fileData = await fs.readFile(persistPath);
+        dataDict = JSON.parse(fileData.toString());
+      } catch (e) {
+        throw new Error(`Failed to load data from path: ${persistPath}`, {
+          cause: e,
+        });
+      }
     }
 
     const data = new SimpleVectorStoreData();

packages/llamaindex/tests/StorageContext.test.ts

@@ -1,43 +0,0 @@
-import { OpenAIEmbedding } from "@llamaindex/openai";
-import {
-  Settings,
-  storageContextFromDefaults,
-  type StorageContext,
-} from "llamaindex";
-import { existsSync, rmSync } from "node:fs";
-import { mkdtemp } from "node:fs/promises";
-import { tmpdir } from "node:os";
-import { join } from "node:path";
-import {
-  afterAll,
-  beforeAll,
-  describe,
-  expect,
-  test,
-  vi,
-  vitest,
-} from "vitest";
-const testDir = await mkdtemp(join(tmpdir(), "test-"));
-vitest.spyOn(console, "error");
-
-describe("StorageContext", () => {
-  let storageContext: StorageContext;
-
-  beforeAll(async () => {
-    Settings.embedModel = new OpenAIEmbedding();
-    storageContext = await storageContextFromDefaults({
-      persistDir: testDir,
-    });
-  });
-
-  test("initializes", async () => {
-    vi.mocked(console.error).mockImplementation(() => {}); // silence console.error
-
-    expect(existsSync(testDir)).toBe(true);
-    expect(storageContext).toBeDefined();
-  });
-
-  afterAll(() => {
-    rmSync(testDir, { recursive: true });
-  });
-});

packages/llamaindex/tests/storage/StorageContext.test.ts

@@ -0,0 +1,130 @@
+import { OpenAIEmbedding } from "@llamaindex/openai";
+import {
+  Document,
+  Settings,
+  SimpleDocumentStore,
+  SimpleIndexStore,
+  SimpleVectorStore,
+  storageContextFromDefaults,
+  VectorStoreIndex,
+} from "llamaindex";
+import { existsSync, rmSync, writeFileSync } from "node:fs";
+import { access, mkdtemp } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import {
+  afterEach,
+  beforeAll,
+  beforeEach,
+  describe,
+  expect,
+  test,
+  vi,
+} from "vitest";
+import { mockEmbeddingModel } from "../utility/mockOpenAI.js";
+
+describe("StorageContext", () => {
+  let testDir: string;
+
+  beforeAll(async () => {
+    const embedModel = new OpenAIEmbedding();
+    mockEmbeddingModel(embedModel);
+    Settings.embedModel = embedModel;
+  });
+
+  beforeEach(async () => {
+    testDir = await mkdtemp(join(tmpdir(), "test-"));
+  });
+
+  test("initializes", async () => {
+    const storageContext = await storageContextFromDefaults({
+      persistDir: testDir,
+    });
+
+    expect(existsSync(testDir)).toBe(true);
+    expect(storageContext).toBeDefined();
+  });
+
+  test("persists and loads", async () => {
+    const doc = new Document({ text: "test document" });
+    const consoleInfoSpy = vi
+      .spyOn(console, "info")
+      .mockImplementation(() => {});
+
+    // storage context from individual stores
+    const storageContext = await storageContextFromDefaults({
+      docStore: await SimpleDocumentStore.fromPersistDir(testDir),
+      vectorStore: await SimpleVectorStore.fromPersistDir(testDir),
+      indexStore: await SimpleIndexStore.fromPersistDir(testDir),
+    });
+
+    const index = await VectorStoreIndex.fromDocuments([doc], {
+      storageContext,
+    });
+    expect(consoleInfoSpy).toHaveBeenCalledTimes(3);
+    expect(consoleInfoSpy).toHaveBeenCalledWith(
+      expect.stringContaining("Starting new store"),
+    );
+    expect(index).toBeDefined();
+
+    const retriever = index.asRetriever();
+    const result = await retriever.retrieve("test");
+    expect(result).toBeDefined();
+    expect(result.length).toBe(1);
+
+    // Check that the test data files exist
+    await expectTestDataFilesExist(testDir);
+
+    consoleInfoSpy.mockClear();
+
+    // Now, load it again. Since data was persisted, we should not see the error.
+    const newStorageContext = await storageContextFromDefaults({
+      docStore: await SimpleDocumentStore.fromPersistDir(testDir),
+      vectorStore: await SimpleVectorStore.fromPersistDir(testDir),
+      indexStore: await SimpleIndexStore.fromPersistDir(testDir),
+    });
+
+    const loadedIndex = await VectorStoreIndex.init({
+      storageContext: newStorageContext,
+    });
+    const loadedRetriever = loadedIndex.asRetriever();
+    const loadedResult = await loadedRetriever.retrieve("test");
+    expect(loadedResult).toBeDefined();
+    expect(loadedResult.length).toBe(1);
+
+    await expectTestDataFilesExist(testDir);
+
+    expect(consoleInfoSpy).not.toHaveBeenCalled();
+
+    consoleInfoSpy.mockRestore();
+  });
+
+  test("throws error on corrupted data", async () => {
+    // test SimpleKVStore
+    const docStorePath = join(testDir, "doc_store.json");
+    writeFileSync(docStorePath, "corrupted data");
+    await expect(SimpleDocumentStore.fromPersistDir(testDir)).rejects.toThrow(
+      /Failed to load data from path/,
+    );
+    // test SimpleVectorStore
+    const vectorStorePath = join(testDir, "vector_store.json");
+    writeFileSync(vectorStorePath, "corrupted data");
+    await expect(SimpleVectorStore.fromPersistDir(testDir)).rejects.toThrow(
+      /Failed to load data from path/,
+    );
+  });
+
+  afterEach(() => {
+    rmSync(testDir, { recursive: true, force: true });
+  });
+});
+
+async function expectTestDataFilesExist(testDir: string) {
+  const docStorePath = `${testDir}/doc_store.json`;
+  const vectorStorePath = `${testDir}/vector_store.json`;
+  const indexStorePath = `${testDir}/index_store.json`;
+
+  expect(await access(docStorePath)).toBeUndefined();
+  expect(await access(vectorStorePath)).toBeUndefined();
+  expect(await access(indexStorePath)).toBeUndefined();
+}

packages/tools/CHANGELOG.md

@@ -1,5 +1,11 @@
 # @llamaindex/tools
 
+## 0.1.2
+
+### Patch Changes
+
+- 0fcf651: chore: export type MCPClientOptions
+
 ## 0.1.1
 
 ### Patch Changes

packages/tools/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/tools",
   "description": "LlamaIndex Tools",
-  "version": "0.1.1",
+  "version": "0.1.2",
   "type": "module",
   "main": "./dist/index.cjs",
   "module": "./dist/index.js",

packages/tools/src/tools/mcp.ts

@@ -59,7 +59,8 @@ type SSEMCPClientOptions = SSEClientTransportOptions & URLMCPOptions;
 type StreamableHTTPMCPClientOptions = StreamableHTTPClientTransportOptions &
   URLMCPOptions;
 
-type MCPClientOptions =
+// Export MCPClientOptions type for external consumers to use
+export type MCPClientOptions =
   | StdioMCPClientOptions
   | SSEMCPClientOptions
   | StreamableHTTPMCPClientOptions;

pnpm-lock.yaml

@@ -944,20 +944,20 @@ importers:
         version: 3.25.67
     devDependencies:
       '@hey-api/client-fetch':
-        specifier: ^0.10.1
-        version: 0.10.1(@hey-api/openapi-ts@0.67.5(typescript@5.8.3))
+        specifier: ^0.13.1
+        version: 0.13.1(@hey-api/openapi-ts@0.77.0(typescript@5.8.3))
       '@hey-api/openapi-ts':
-        specifier: ^0.67.5
-        version: 0.67.5(typescript@5.8.3)
-      '@llama-flow/core':
-        specifier: ^0.4.1
-        version: 0.4.1(@modelcontextprotocol/sdk@1.13.0)(hono@4.7.7)(next@15.3.2(@opentelemetry/api@1.9.0)(react-dom@19.1.0(react@19.1.0))(react@19.1.0))(p-retry@6.2.1)(zod@3.25.67)
+        specifier: ^0.77.0
+        version: 0.77.0(typescript@5.8.3)
       '@llamaindex/core':
         specifier: workspace:*
         version: link:../core
       '@llamaindex/env':
         specifier: workspace:*
         version: link:../env
+      '@llamaindex/workflow-core':
+        specifier: ^1.0.0
+        version: 1.0.0(@modelcontextprotocol/sdk@1.13.0)(hono@4.7.7)(next@15.3.2(@opentelemetry/api@1.9.0)(react-dom@19.1.0(react@19.1.0))(react@19.1.0))(p-retry@6.2.1)(zod@3.25.67)
 
   packages/core:
     dependencies:
@@ -3606,8 +3606,8 @@ packages:
     peerDependencies:
       vue: ^3.2.0
 
-  '@hey-api/client-fetch@0.10.1':
-    resolution: {integrity: sha512-C1XZEnzvOIdXppvMcnO8/V/RpcORxA4rh+5qjuMcItkV++hv7aBz7tSLd0z+bSLFUwttec077WT/nPS+oO4BiA==}
+  '@hey-api/client-fetch@0.13.1':
+    resolution: {integrity: sha512-29jBRYNdxVGlx5oewFgOrkulZckpIpBIRHth3uHFn1PrL2ucMy52FvWOY3U3dVx2go1Z3kUmMi6lr07iOpUqqA==}
     deprecated: Starting with v0.73.0, this package is bundled directly inside @hey-api/openapi-ts.
     peerDependencies:
       '@hey-api/openapi-ts': < 2
@@ -3616,8 +3616,8 @@ packages:
     resolution: {integrity: sha512-yktiFZoWPtEW8QKS65eqKwA5MTKp88CyiL8q72WynrBs/73SAaxlSWlA2zW/DZlywZ5hX1OYzrCC0wFdvO9c2w==}
     engines: {node: '>= 16'}
 
-  '@hey-api/openapi-ts@0.67.5':
-    resolution: {integrity: sha512-fZd+3im0038rnK7W0mAfLTrdygCR/k6zq7XfYZj/wu0/3GvIg/aGlOwMUtZinRJBH21W+GoH0MJoce3BagVXsw==}
+  '@hey-api/openapi-ts@0.77.0':
+    resolution: {integrity: sha512-HAJbd8QfxeBPZ788Ghiw7bzzzTKxW+wW+34foleEztyZJnRV20barvevu8YAK1BtyiIGIpEtAfoqO8KUj4VuBw==}
     engines: {node: ^18.18.0 || ^20.9.0 || >=22.10.0}
     hasBin: true
     peerDependencies:
@@ -4010,29 +4010,6 @@ packages:
   '@lezer/yaml@1.0.3':
     resolution: {integrity: sha512-GuBLekbw9jDBDhGur82nuwkxKQ+a3W5H0GfaAthDXcAu+XdpS43VlnxA9E9hllkpSP5ellRDKjLLj7Lu9Wr6xA==}
 
-  '@llama-flow/core@0.4.1':
-    resolution: {integrity: sha512-xHhJMRmY16C1pYPWIonmLWPkkjTGuj1iVQCTXOM6sXajQ3r0+mEVERQCjPqf48tvX0K+szbdgxjg6wx+KwVqcg==}
-    peerDependencies:
-      '@modelcontextprotocol/sdk': ^1.7.0
-      hono: ^4.7.4
-      next: ^15.2.2
-      p-retry: ^6.2.1
-      rxjs: ^7.8.2
-      zod: ^3.24.2
-    peerDependenciesMeta:
-      '@modelcontextprotocol/sdk':
-        optional: true
-      hono:
-        optional: true
-      next:
-        optional: true
-      p-retry:
-        optional: true
-      rxjs:
-        optional: true
-      zod:
-        optional: true
-
   '@llamaindex/chat-ui-docs@0.0.5':
     resolution: {integrity: sha512-kG8QjnCxsvFXC8I+Xes64Bb7H6DEkoUAAsIzLsD28Ppfalgb7brPnR6jCoJsJkajKiFmnJUwrsNz/s39HZ8SrQ==}
 
@@ -10963,6 +10940,10 @@ packages:
     resolution: {integrity: sha512-zy1wx4+P3PfhXSEPJNtZmJXfhkkIaxU1VauWIrDZw1O7uJRDRJtKr9n3Ic4NgbA16KyOxOXO2ng9gYwCdXuSXA==}
     engines: {node: '>=18'}
 
+  open@10.1.2:
+    resolution: {integrity: sha512-cxN6aIDPz6rm8hbebcP7vrQNhvRcveZoJU72Y7vskh4oIm+BZwBECnx5nTmrlres1Qapvx27Qo1Auukpf8PKXw==}
+    engines: {node: '>=18'}
+
   openai@4.102.0:
     resolution: {integrity: sha512-CWk15CMhPSHNZnjz+6rwVYV551xaC8CwOd7/zxImrC1btEo37dX/Ii5tBKWfqqxqyzpJ6p3Y4bICzzKhW03WhQ==}
     hasBin: true
@@ -16376,9 +16357,9 @@ snapshots:
       '@tanstack/vue-virtual': 3.13.6(vue@3.5.13(typescript@5.7.3))
       vue: 3.5.13(typescript@5.7.3)
 
-  '@hey-api/client-fetch@0.10.1(@hey-api/openapi-ts@0.67.5(typescript@5.8.3))':
+  '@hey-api/client-fetch@0.13.1(@hey-api/openapi-ts@0.77.0(typescript@5.8.3))':
     dependencies:
-      '@hey-api/openapi-ts': 0.67.5(typescript@5.8.3)
+      '@hey-api/openapi-ts': 0.77.0(typescript@5.8.3)
 
   '@hey-api/json-schema-ref-parser@1.0.6':
     dependencies:
@@ -16387,12 +16368,15 @@ snapshots:
       js-yaml: 4.1.0
       lodash: 4.17.21
 
-  '@hey-api/openapi-ts@0.67.5(typescript@5.8.3)':
+  '@hey-api/openapi-ts@0.77.0(typescript@5.8.3)':
     dependencies:
       '@hey-api/json-schema-ref-parser': 1.0.6
+      ansi-colors: 4.1.3
       c12: 2.0.1
+      color-support: 1.1.3
       commander: 13.0.0
       handlebars: 4.7.8
+      open: 10.1.2
       typescript: 5.8.3
     transitivePeerDependencies:
       - magicast
@@ -16765,14 +16749,6 @@ snapshots:
       '@lezer/highlight': 1.2.1
       '@lezer/lr': 1.4.2
 
-  '@llama-flow/core@0.4.1(@modelcontextprotocol/sdk@1.13.0)(hono@4.7.7)(next@15.3.2(@opentelemetry/api@1.9.0)(react-dom@19.1.0(react@19.1.0))(react@19.1.0))(p-retry@6.2.1)(zod@3.25.67)':
-    optionalDependencies:
-      '@modelcontextprotocol/sdk': 1.13.0
-      hono: 4.7.7
-      next: 15.3.2(@opentelemetry/api@1.9.0)(react-dom@19.1.0(react@19.1.0))(react@19.1.0)
-      p-retry: 6.2.1
-      zod: 3.25.67
-
   '@llamaindex/chat-ui-docs@0.0.5': {}
 
   '@llamaindex/workflow-core@1.0.0(@modelcontextprotocol/sdk@1.13.0)(hono@4.7.7)(next@15.3.2(@opentelemetry/api@1.9.0)(react-dom@19.1.0(react@19.1.0))(react@19.1.0))(p-retry@6.2.1)(zod@3.25.67)':
@@ -21161,8 +21137,7 @@ snapshots:
       color-name: 1.1.4
       simple-swizzle: 0.2.2
 
-  color-support@1.1.3:
-    optional: true
+  color-support@1.1.3: {}
 
   color@3.2.1:
     dependencies:
@@ -21893,7 +21868,7 @@ snapshots:
       '@typescript-eslint/parser': 6.21.0(eslint@9.22.0(jiti@2.4.2))(typescript@5.7.3)
       eslint: 9.22.0(jiti@2.4.2)
       eslint-import-resolver-node: 0.3.9
-      eslint-import-resolver-typescript: 3.7.0(eslint-plugin-import@2.31.0(@typescript-eslint/parser@6.21.0(eslint@9.22.0(jiti@2.4.2))(typescript@5.7.3))(eslint@9.22.0(jiti@2.4.2)))(eslint@9.22.0(jiti@2.4.2))
+      eslint-import-resolver-typescript: 3.7.0(eslint-plugin-import@2.31.0)(eslint@9.22.0(jiti@2.4.2))
       eslint-plugin-import: 2.31.0(@typescript-eslint/parser@6.21.0(eslint@9.22.0(jiti@2.4.2))(typescript@5.7.3))(eslint-import-resolver-typescript@3.7.0)(eslint@9.22.0(jiti@2.4.2))
       eslint-plugin-jsx-a11y: 6.10.2(eslint@9.22.0(jiti@2.4.2))
       eslint-plugin-react: 7.37.2(eslint@9.22.0(jiti@2.4.2))
@@ -21923,22 +21898,6 @@ snapshots:
     transitivePeerDependencies:
       - supports-color
 
-  eslint-import-resolver-typescript@3.7.0(eslint-plugin-import@2.31.0(@typescript-eslint/parser@6.21.0(eslint@9.22.0(jiti@2.4.2))(typescript@5.7.3))(eslint@9.22.0(jiti@2.4.2)))(eslint@9.22.0(jiti@2.4.2)):
-    dependencies:
-      '@nolyfill/is-core-module': 1.0.39
-      debug: 4.4.0
-      enhanced-resolve: 5.18.1
-      eslint: 9.22.0(jiti@2.4.2)
-      fast-glob: 3.3.3
-      get-tsconfig: 4.10.0
-      is-bun-module: 1.3.0
-      is-glob: 4.0.3
-      stable-hash: 0.0.4
-    optionalDependencies:
-      eslint-plugin-import: 2.31.0(@typescript-eslint/parser@6.21.0(eslint@9.22.0(jiti@2.4.2))(typescript@5.7.3))(eslint-import-resolver-typescript@3.7.0)(eslint@9.22.0(jiti@2.4.2))
-    transitivePeerDependencies:
-      - supports-color
-
   eslint-import-resolver-typescript@3.7.0(eslint-plugin-import@2.31.0)(eslint@9.16.0(jiti@2.4.2)):
     dependencies:
       '@nolyfill/is-core-module': 1.0.39
@@ -21955,14 +21914,30 @@ snapshots:
     transitivePeerDependencies:
       - supports-color
 
-  eslint-module-utils@2.12.0(@typescript-eslint/parser@6.21.0(eslint@9.22.0(jiti@2.4.2))(typescript@5.7.3))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.7.0(eslint-plugin-import@2.31.0(@typescript-eslint/parser@6.21.0(eslint@9.22.0(jiti@2.4.2))(typescript@5.7.3))(eslint@9.22.0(jiti@2.4.2)))(eslint@9.22.0(jiti@2.4.2)))(eslint@9.22.0(jiti@2.4.2)):
+  eslint-import-resolver-typescript@3.7.0(eslint-plugin-import@2.31.0)(eslint@9.22.0(jiti@2.4.2)):
+    dependencies:
+      '@nolyfill/is-core-module': 1.0.39
+      debug: 4.4.0
+      enhanced-resolve: 5.18.1
+      eslint: 9.22.0(jiti@2.4.2)
+      fast-glob: 3.3.3
+      get-tsconfig: 4.10.0
+      is-bun-module: 1.3.0
+      is-glob: 4.0.3
+      stable-hash: 0.0.4
+    optionalDependencies:
+      eslint-plugin-import: 2.31.0(@typescript-eslint/parser@6.21.0(eslint@9.22.0(jiti@2.4.2))(typescript@5.7.3))(eslint-import-resolver-typescript@3.7.0)(eslint@9.22.0(jiti@2.4.2))
+    transitivePeerDependencies:
+      - supports-color
+
+  eslint-module-utils@2.12.0(@typescript-eslint/parser@6.21.0(eslint@9.22.0(jiti@2.4.2))(typescript@5.7.3))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.7.0)(eslint@9.22.0(jiti@2.4.2)):
     dependencies:
       debug: 3.2.7
     optionalDependencies:
       '@typescript-eslint/parser': 6.21.0(eslint@9.22.0(jiti@2.4.2))(typescript@5.7.3)
       eslint: 9.22.0(jiti@2.4.2)
       eslint-import-resolver-node: 0.3.9
-      eslint-import-resolver-typescript: 3.7.0(eslint-plugin-import@2.31.0(@typescript-eslint/parser@6.21.0(eslint@9.22.0(jiti@2.4.2))(typescript@5.7.3))(eslint@9.22.0(jiti@2.4.2)))(eslint@9.22.0(jiti@2.4.2))
+      eslint-import-resolver-typescript: 3.7.0(eslint-plugin-import@2.31.0)(eslint@9.22.0(jiti@2.4.2))
     transitivePeerDependencies:
       - supports-color
 
@@ -21988,7 +21963,7 @@ snapshots:
       doctrine: 2.1.0
       eslint: 9.22.0(jiti@2.4.2)
       eslint-import-resolver-node: 0.3.9
-      eslint-module-utils: 2.12.0(@typescript-eslint/parser@6.21.0(eslint@9.22.0(jiti@2.4.2))(typescript@5.7.3))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.7.0(eslint-plugin-import@2.31.0(@typescript-eslint/parser@6.21.0(eslint@9.22.0(jiti@2.4.2))(typescript@5.7.3))(eslint@9.22.0(jiti@2.4.2)))(eslint@9.22.0(jiti@2.4.2)))(eslint@9.22.0(jiti@2.4.2))
+      eslint-module-utils: 2.12.0(@typescript-eslint/parser@6.21.0(eslint@9.22.0(jiti@2.4.2))(typescript@5.7.3))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.7.0)(eslint@9.22.0(jiti@2.4.2))
       hasown: 2.0.2
       is-core-module: 2.16.1
       is-glob: 4.0.3
@@ -24958,7 +24933,7 @@ snapshots:
 
   mlly@1.7.4:
     dependencies:
-      acorn: 8.14.1
+      acorn: 8.15.0
       pathe: 2.0.3
       pkg-types: 1.3.1
       ufo: 1.6.1
@@ -25577,6 +25552,13 @@ snapshots:
       is-inside-container: 1.0.0
       is-wsl: 3.1.0
 
+  open@10.1.2:
+    dependencies:
+      default-browser: 5.2.1
+      define-lazy-prop: 3.0.0
+      is-inside-container: 1.0.0
+      is-wsl: 3.1.0
+
   openai@4.102.0(ws@8.18.1(bufferutil@4.0.9))(zod@3.25.67):
     dependencies:
       '@types/node': 18.19.86