Release 0.5.9 (#1070)

Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>

apps/docs/CHANGELOG.md

@@ -1,5 +1,27 @@
 # docs
 
+## 0.0.50
+
+### Patch Changes
+
+- Updated dependencies [15962b3]
+  - llamaindex@0.5.9
+
+## 0.0.49
+
+### Patch Changes
+
+- Updated dependencies [3d5ba08]
+- Updated dependencies [d917cdc]
+  - llamaindex@0.5.8
+
+## 0.0.48
+
+### Patch Changes
+
+- Updated dependencies [ec59acd]
+  - llamaindex@0.5.7
+
 ## 0.0.47
 
 ### Patch Changes

apps/docs/docs/modules/llms/available_llms/bedrock.md

@@ -29,6 +29,9 @@ META_LLAMA2_13B_CHAT = "meta.llama2-13b-chat-v1";
 META_LLAMA2_70B_CHAT = "meta.llama2-70b-chat-v1";
 META_LLAMA3_8B_INSTRUCT = "meta.llama3-8b-instruct-v1:0";
 META_LLAMA3_70B_INSTRUCT = "meta.llama3-70b-instruct-v1:0";
+META_LLAMA3_1_8B_INSTRUCT = "meta.llama3-1-8b-instruct-v1:0"; // available on us-west-2
+META_LLAMA3_1_70B_INSTRUCT = "meta.llama3-1-70b-instruct-v1:0"; // available on us-west-2
+META_LLAMA3_1_405B_INSTRUCT = "meta.llama3-1-405b-instruct-v1:0"; // preview only, available on us-west-2
 ```
 
 Sonnet, Haiku and Opus are multimodal, image_url only supports base64 data url format, e.g. `data:image/jpeg;base64,SGVsbG8sIFdvcmxkIQ==`

apps/docs/package.json

@@ -1,6 +1,6 @@
 {
   "name": "docs",
-  "version": "0.0.47",
+  "version": "0.0.50",
   "private": true,
   "scripts": {
     "docusaurus": "docusaurus",

examples/longText.ts

@@ -1,17 +1,16 @@
 import {
   Document,
+  SentenceSplitter,
   Settings,
-  SimpleNodeParser,
   VectorStoreIndex,
 } from "llamaindex";
 
 export const STORAGE_DIR = "./data";
 
 // Update node parser
-Settings.nodeParser = new SimpleNodeParser({
+Settings.nodeParser = new SentenceSplitter({
   chunkSize: 512,
   chunkOverlap: 20,
-  splitLongSentences: true,
 });
 (async () => {
   // generate a document with a very long sentence (9000 words long)

examples/readers/package.json

@@ -11,6 +11,7 @@
     "start:pdf": "node --import tsx ./src/pdf.ts",
     "start:llamaparse": "node --import tsx ./src/llamaparse.ts",
     "start:notion": "node --import tsx ./src/notion.ts",
+    "start:assemblyai": "node --import tsx ./src/assemblyai.ts",
     "start:llamaparse-dir": "node --import tsx ./src/simple-directory-reader-with-llamaparse.ts",
     "start:llamaparse-json": "node --import tsx ./src/llamaparse-json.ts",
     "start:discord": "node --import tsx ./src/discord.ts"

examples/split.ts

@@ -8,7 +8,7 @@ async function main() {
 
   const textSplitter = new SentenceSplitter();
 
-  const chunks = textSplitter.splitTextWithOverlaps(essay);
+  const chunks = textSplitter.splitText(essay);
 
   console.log(chunks);
 }

packages/autotool/examples/02_nextjs/CHANGELOG.md

@@ -1,5 +1,30 @@
 # @llamaindex/autotool-02-next-example
 
+## 0.1.34
+
+### Patch Changes
+
+- Updated dependencies [15962b3]
+  - llamaindex@0.5.9
+  - @llamaindex/autotool@2.0.0
+
+## 0.1.33
+
+### Patch Changes
+
+- Updated dependencies [3d5ba08]
+- Updated dependencies [d917cdc]
+  - llamaindex@0.5.8
+  - @llamaindex/autotool@2.0.0
+
+## 0.1.32
+
+### Patch Changes
+
+- Updated dependencies [ec59acd]
+  - llamaindex@0.5.7
+  - @llamaindex/autotool@2.0.0
+
 ## 0.1.31
 
 ### Patch Changes

packages/autotool/examples/02_nextjs/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/autotool-02-next-example",
   "private": true,
-  "version": "0.1.31",
+  "version": "0.1.34",
   "scripts": {
     "dev": "next dev",
     "build": "next build",

packages/autotool/package.json

@@ -51,7 +51,7 @@
     "unplugin": "^1.10.1"
   },
   "peerDependencies": {
-    "llamaindex": "^0.5.6",
+    "llamaindex": "^0.5.9",
     "openai": "^4",
     "typescript": "^4"
   },

packages/community/CHANGELOG.md

@@ -1,5 +1,13 @@
 # @llamaindex/community
 
+## 0.0.24
+
+### Patch Changes
+
+- 3d9a802: feat: added llama 3.1
+- Updated dependencies [15962b3]
+  - @llamaindex/core@0.1.4
+
 ## 0.0.23
 
 ### Patch Changes

packages/community/README.md

@@ -5,7 +5,7 @@
 ## Current Features:
 
 - Bedrock support for the Anthropic Claude Models [usage](https://ts.llamaindex.ai/modules/llms/available_llms/bedrock)
-- Bedrock support for the Meta LLama 2 and 3 Models [usage](https://ts.llamaindex.ai/modules/llms/available_llms/bedrock)
+- Bedrock support for the Meta LLama 2, 3 and 3.1 Models [usage](https://ts.llamaindex.ai/modules/llms/available_llms/bedrock)
 
 ## LICENSE
 

packages/community/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/community",
   "description": "Community package for LlamaIndexTS",
-  "version": "0.0.23",
+  "version": "0.0.24",
   "type": "module",
   "types": "dist/type/index.d.ts",
   "main": "dist/cjs/index.js",

packages/community/src/llm/bedrock/base.ts

@@ -60,6 +60,9 @@ export enum BEDROCK_MODELS {
   META_LLAMA2_70B_CHAT = "meta.llama2-70b-chat-v1",
   META_LLAMA3_8B_INSTRUCT = "meta.llama3-8b-instruct-v1:0",
   META_LLAMA3_70B_INSTRUCT = "meta.llama3-70b-instruct-v1:0",
+  META_LLAMA3_1_8B_INSTRUCT = "meta.llama3-1-8b-instruct-v1:0",
+  META_LLAMA3_1_70B_INSTRUCT = "meta.llama3-1-70b-instruct-v1:0",
+  META_LLAMA3_1_405B_INSTRUCT = "meta.llama3-1-405b-instruct-v1:0",
   MISTRAL_7B_INSTRUCT = "mistral.mistral-7b-instruct-v0:2",
   MISTRAL_MIXTRAL_7B_INSTRUCT = "mistral.mixtral-8x7b-instruct-v0:1",
   MISTRAL_MIXTRAL_LARGE_2402 = "mistral.mistral-large-2402-v1:0",
@@ -94,6 +97,9 @@ const CHAT_ONLY_MODELS = {
   [BEDROCK_MODELS.META_LLAMA2_70B_CHAT]: 4096,
   [BEDROCK_MODELS.META_LLAMA3_8B_INSTRUCT]: 8192,
   [BEDROCK_MODELS.META_LLAMA3_70B_INSTRUCT]: 8192,
+  [BEDROCK_MODELS.META_LLAMA3_1_8B_INSTRUCT]: 128000,
+  [BEDROCK_MODELS.META_LLAMA3_1_70B_INSTRUCT]: 128000,
+  [BEDROCK_MODELS.META_LLAMA3_1_405B_INSTRUCT]: 128000,
   [BEDROCK_MODELS.MISTRAL_7B_INSTRUCT]: 32000,
   [BEDROCK_MODELS.MISTRAL_MIXTRAL_7B_INSTRUCT]: 32000,
   [BEDROCK_MODELS.MISTRAL_MIXTRAL_LARGE_2402]: 32000,
@@ -121,6 +127,9 @@ export const STREAMING_MODELS = new Set([
   BEDROCK_MODELS.META_LLAMA2_70B_CHAT,
   BEDROCK_MODELS.META_LLAMA3_8B_INSTRUCT,
   BEDROCK_MODELS.META_LLAMA3_70B_INSTRUCT,
+  BEDROCK_MODELS.META_LLAMA3_1_8B_INSTRUCT,
+  BEDROCK_MODELS.META_LLAMA3_1_70B_INSTRUCT,
+  BEDROCK_MODELS.META_LLAMA3_1_405B_INSTRUCT,
   BEDROCK_MODELS.MISTRAL_7B_INSTRUCT,
   BEDROCK_MODELS.MISTRAL_MIXTRAL_7B_INSTRUCT,
   BEDROCK_MODELS.MISTRAL_MIXTRAL_LARGE_2402,
@@ -160,6 +169,9 @@ export const BEDROCK_MODEL_MAX_TOKENS: Partial<Record<BEDROCK_MODELS, number>> =
     [BEDROCK_MODELS.META_LLAMA2_70B_CHAT]: 2048,
     [BEDROCK_MODELS.META_LLAMA3_8B_INSTRUCT]: 2048,
     [BEDROCK_MODELS.META_LLAMA3_70B_INSTRUCT]: 2048,
+    [BEDROCK_MODELS.META_LLAMA3_1_8B_INSTRUCT]: 2048,
+    [BEDROCK_MODELS.META_LLAMA3_1_70B_INSTRUCT]: 2048,
+    [BEDROCK_MODELS.META_LLAMA3_1_405B_INSTRUCT]: 2048,
   };
 
 const DEFAULT_BEDROCK_PARAMS = {

packages/core/CHANGELOG.md

@@ -1,5 +1,16 @@
 # @llamaindex/core
 
+## 0.1.4
+
+### Patch Changes
+
+- 15962b3: feat: node parser refactor
+
+  Align the text splitter logic with Python; it has almost the same logic as Python; Zod checks for input and better error messages and event system.
+
+  This change will not be considered a breaking change since it doesn't have a significant output difference from the last version,
+  but some edge cases will change, like the page separator and parameter for the constructor.
+
 ## 0.1.3
 
 ### Patch Changes

packages/core/package.json

@@ -1,9 +1,23 @@
 {
   "name": "@llamaindex/core",
   "type": "module",
-  "version": "0.1.3",
+  "version": "0.1.4",
   "description": "LlamaIndex Core Module",
   "exports": {
+    "./node-parser": {
+      "require": {
+        "types": "./dist/node-parser/index.d.cts",
+        "default": "./dist/node-parser/index.cjs"
+      },
+      "import": {
+        "types": "./dist/node-parser/index.d.ts",
+        "default": "./dist/node-parser/index.js"
+      },
+      "default": {
+        "types": "./dist/node-parser/index.d.ts",
+        "default": "./dist/node-parser/index.js"
+      }
+    },
     "./query-engine": {
       "require": {
         "types": "./dist/query-engine/index.d.cts",
@@ -117,7 +131,8 @@
   },
   "devDependencies": {
     "ajv": "^8.16.0",
-    "bunchee": "5.3.0-beta.0"
+    "bunchee": "5.3.0-beta.0",
+    "natural": "^7.1.0"
   },
   "dependencies": {
     "@llamaindex/env": "workspace:*",

packages/core/src/embeddings/base.ts

@@ -20,9 +20,7 @@ export type BaseEmbeddingOptions = {
   logProgress?: boolean;
 };
 
-export abstract class BaseEmbedding
-  implements TransformComponent<BaseEmbeddingOptions>
-{
+export abstract class BaseEmbedding implements TransformComponent {
   embedBatchSize = DEFAULT_EMBED_BATCH_SIZE;
   embedInfo?: EmbeddingInfo;
 

packages/core/src/global/settings.ts

@@ -1,3 +1,4 @@
+import type { Tokenizer } from "@llamaindex/env";
 import {
   type CallbackManager,
   getCallbackManager,
@@ -9,8 +10,22 @@ import {
   setChunkSize,
   withChunkSize,
 } from "./settings/chunk-size";
+import {
+  getTokenizer,
+  setTokenizer,
+  withTokenizer,
+} from "./settings/tokenizer";
 
 export const Settings = {
+  get tokenizer() {
+    return getTokenizer();
+  },
+  set tokenizer(tokenizer) {
+    setTokenizer(tokenizer);
+  },
+  withTokenizer<Result>(tokenizer: Tokenizer, fn: () => Result): Result {
+    return withTokenizer(tokenizer, fn);
+  },
   get chunkSize(): number | undefined {
     return getChunkSize();
   },

packages/core/src/global/settings/callback-manager.ts

@@ -6,6 +6,7 @@ import type {
   ToolCall,
   ToolOutput,
 } from "../../llms";
+import { TextNode } from "../../schema";
 import { EventCaller, getEventCaller } from "../../utils/event-caller";
 import type { UUID } from "../type";
 
@@ -33,12 +34,32 @@ export type LLMStreamEvent = {
   chunk: ChatResponseChunk;
 };
 
+export type ChunkingStartEvent = {
+  text: string[];
+};
+
+export type ChunkingEndEvent = {
+  chunks: string[];
+};
+
+export type NodeParsingStartEvent = {
+  documents: TextNode[];
+};
+
+export type NodeParsingEndEvent = {
+  nodes: TextNode[];
+};
+
 export interface LlamaIndexEventMaps {
   "llm-start": LLMStartEvent;
   "llm-end": LLMEndEvent;
   "llm-tool-call": LLMToolCallEvent;
   "llm-tool-result": LLMToolResultEvent;
   "llm-stream": LLMStreamEvent;
+  "chunking-start": ChunkingStartEvent;
+  "chunking-end": ChunkingEndEvent;
+  "node-parsing-start": NodeParsingStartEvent;
+  "node-parsing-end": NodeParsingEndEvent;
 }
 
 export class LlamaIndexCustomEvent<T = any> extends CustomEvent<T> {
@@ -116,14 +137,10 @@ export const globalCallbackManager = new CallbackManager();
 const callbackManagerAsyncLocalStorage =
   new AsyncLocalStorage<CallbackManager>();
 
-let currentCallbackManager: CallbackManager | null = null;
+let currentCallbackManager: CallbackManager = globalCallbackManager;
 
 export function getCallbackManager(): CallbackManager {
-  return (
-    callbackManagerAsyncLocalStorage.getStore() ??
-    currentCallbackManager ??
-    globalCallbackManager
-  );
+  return callbackManagerAsyncLocalStorage.getStore() ?? currentCallbackManager;
 }
 
 export function setCallbackManager(callbackManager: CallbackManager) {

packages/core/src/global/settings/chunk-size.ts

@@ -1,9 +1,9 @@
 import { AsyncLocalStorage } from "@llamaindex/env";
 
 const chunkSizeAsyncLocalStorage = new AsyncLocalStorage<number | undefined>();
-let globalChunkSize: number | null = null;
+let globalChunkSize: number = 1024;
 
-export function getChunkSize(): number | undefined {
+export function getChunkSize(): number {
   return globalChunkSize ?? chunkSizeAsyncLocalStorage.getStore();
 }
 

packages/core/src/global/settings/tokenizer.ts

@@ -0,0 +1,21 @@
+import { AsyncLocalStorage, type Tokenizer, tokenizers } from "@llamaindex/env";
+
+const chunkSizeAsyncLocalStorage = new AsyncLocalStorage<Tokenizer>();
+let globalTokenizer: Tokenizer = tokenizers.tokenizer();
+
+export function getTokenizer(): Tokenizer {
+  return globalTokenizer ?? chunkSizeAsyncLocalStorage.getStore();
+}
+
+export function setTokenizer(tokenizer: Tokenizer | undefined) {
+  if (tokenizer !== undefined) {
+    globalTokenizer = tokenizer;
+  }
+}
+
+export function withTokenizer<Result>(
+  tokenizer: Tokenizer,
+  fn: () => Result,
+): Result {
+  return chunkSizeAsyncLocalStorage.run(tokenizer, fn);
+}

packages/core/src/node-parser/base.ts

@@ -0,0 +1,147 @@
+import { Settings } from "../global";
+import {
+  BaseNode,
+  buildNodeFromSplits,
+  MetadataMode,
+  NodeRelationship,
+  TextNode,
+  type TransformComponent,
+} from "../schema";
+
+export abstract class NodeParser implements TransformComponent {
+  includeMetadata: boolean = true;
+  includePrevNextRel: boolean = true;
+
+  protected postProcessParsedNodes(
+    nodes: TextNode[],
+    parentDocMap: Map<string, TextNode>,
+  ): TextNode[] {
+    nodes.forEach((node, i) => {
+      const parentDoc = parentDocMap.get(node.sourceNode?.nodeId || "");
+
+      if (parentDoc) {
+        const startCharIdx = parentDoc.text.indexOf(
+          node.getContent(MetadataMode.NONE),
+        );
+        if (startCharIdx >= 0) {
+          node.startCharIdx = startCharIdx;
+          node.endCharIdx =
+            startCharIdx + node.getContent(MetadataMode.NONE).length;
+        }
+        if (this.includeMetadata && node.metadata && parentDoc.metadata) {
+          node.metadata = { ...node.metadata, ...parentDoc.metadata };
+        }
+      }
+
+      if (this.includePrevNextRel && node.sourceNode) {
+        const previousNode = i > 0 ? nodes[i - 1] : null;
+        const nextNode = i < nodes.length - 1 ? nodes[i + 1] : null;
+
+        if (
+          previousNode &&
+          previousNode.sourceNode &&
+          previousNode.sourceNode.nodeId === node.sourceNode.nodeId
+        ) {
+          node.relationships = {
+            ...node.relationships,
+            [NodeRelationship.PREVIOUS]: previousNode.asRelatedNodeInfo(),
+          };
+        }
+
+        if (
+          nextNode &&
+          nextNode.sourceNode &&
+          nextNode.sourceNode.nodeId === node.sourceNode.nodeId
+        ) {
+          node.relationships = {
+            ...node.relationships,
+            [NodeRelationship.NEXT]: nextNode.asRelatedNodeInfo(),
+          };
+        }
+      }
+    });
+
+    return nodes;
+  }
+
+  protected abstract parseNodes(
+    documents: TextNode[],
+    showProgress?: boolean,
+  ): TextNode[];
+
+  public getNodesFromDocuments(documents: TextNode[]): TextNode[] {
+    const docsId: Map<string, TextNode> = new Map(
+      documents.map((doc) => [doc.id_, doc]),
+    );
+    const callbackManager = Settings.callbackManager;
+
+    callbackManager.dispatchEvent("node-parsing-start", {
+      documents,
+    });
+
+    const nodes = this.postProcessParsedNodes(
+      this.parseNodes(documents),
+      docsId,
+    );
+
+    callbackManager.dispatchEvent("node-parsing-end", {
+      nodes,
+    });
+
+    return nodes;
+  }
+
+  async transform(nodes: BaseNode[], options?: {}): Promise<BaseNode[]> {
+    return this.getNodesFromDocuments(nodes as TextNode[]);
+  }
+}
+
+export abstract class TextSplitter extends NodeParser {
+  abstract splitText(text: string): string[];
+
+  public splitTexts(texts: string[]): string[] {
+    return texts.flatMap((text) => this.splitText(text));
+  }
+
+  protected parseNodes(nodes: TextNode[]): TextNode[] {
+    return nodes.reduce<TextNode[]>((allNodes, node) => {
+      const splits = this.splitText(node.getContent(MetadataMode.ALL));
+      const nodes = buildNodeFromSplits(splits, node);
+      return allNodes.concat(nodes);
+    }, []);
+  }
+}
+
+export abstract class MetadataAwareTextSplitter extends TextSplitter {
+  abstract splitTextMetadataAware(text: string, metadata: string): string[];
+
+  splitTextsMetadataAware(texts: string[], metadata: string[]): string[] {
+    if (texts.length !== metadata.length) {
+      throw new TypeError("`texts` and `metadata` must have the same length");
+    }
+    return texts.flatMap((text, i) =>
+      this.splitTextMetadataAware(text, metadata[i]),
+    );
+  }
+
+  protected getMetadataString(node: TextNode): string {
+    const embedStr = node.getMetadataStr(MetadataMode.EMBED);
+    const llmStr = node.getMetadataStr(MetadataMode.LLM);
+    if (embedStr.length > llmStr.length) {
+      return embedStr;
+    } else {
+      return llmStr;
+    }
+  }
+
+  protected parseNodes(nodes: TextNode[]): TextNode[] {
+    return nodes.reduce<TextNode[]>((allNodes, node) => {
+      const metadataStr = this.getMetadataString(node);
+      const splits = this.splitTextMetadataAware(
+        node.getContent(MetadataMode.ALL),
+        metadataStr,
+      );
+      return allNodes.concat(buildNodeFromSplits(splits, node));
+    }, []);
+  }
+}

packages/core/src/node-parser/index.ts

@@ -0,0 +1,24 @@
+/**
+ * Current logic is based on the following implementation:
+ * @link @link https://github.com/run-llama/llama_index/blob/cc0ea90e7e72b8e4f5069aac981d56bb1d568323/llama-index-core/llama_index/core/node_parser
+ */
+import { SentenceSplitter } from "./sentence-splitter";
+
+/**
+ * @deprecated Use `SentenceSplitter` instead
+ */
+export const SimpleNodeParser = SentenceSplitter;
+
+export { MetadataAwareTextSplitter, NodeParser, TextSplitter } from "./base";
+export { MarkdownNodeParser } from "./markdown";
+export { SentenceSplitter } from "./sentence-splitter";
+export { SentenceWindowNodeParser } from "./sentence-window";
+export type { SplitterParams } from "./type";
+export {
+  splitByChar,
+  splitByPhraseRegex,
+  splitByRegex,
+  splitBySentenceTokenizer,
+  splitBySep,
+} from "./utils";
+export type { TextSplitterFn } from "./utils";

packages/core/src/node-parser/markdown.ts

@@ -0,0 +1,87 @@
+import {
+  buildNodeFromSplits,
+  type Metadata,
+  MetadataMode,
+  TextNode,
+} from "../schema";
+import { NodeParser } from "./base";
+
+export class MarkdownNodeParser extends NodeParser {
+  override parseNodes(nodes: TextNode[], showProgress?: boolean): TextNode[] {
+    return nodes.reduce<TextNode[]>((allNodes, node) => {
+      const markdownNodes = this.getNodesFromNode(node);
+      return allNodes.concat(markdownNodes);
+    }, []);
+  }
+
+  protected getNodesFromNode(node: TextNode): TextNode[] {
+    const text = node.getContent(MetadataMode.NONE);
+    const markdownNodes: TextNode[] = [];
+    const lines = text.split("\n");
+    let metadata: { [key: string]: string } = {};
+    let codeBlock = false;
+    let currentSection = "";
+
+    for (const line of lines) {
+      if (line.trim().startsWith("```")) {
+        codeBlock = !codeBlock;
+      }
+      const headerMatch = /^(#+)\s(.*)/.exec(line);
+      if (headerMatch && !codeBlock) {
+        if (currentSection !== "") {
+          markdownNodes.push(
+            this.buildNodeFromSplit(currentSection.trim(), node, metadata),
+          );
+        }
+        metadata = this.updateMetadata(
+          metadata,
+          headerMatch[2],
+          headerMatch[1].trim().length,
+        );
+        currentSection = `${headerMatch[2]}\n`;
+      } else {
+        currentSection += line + "\n";
+      }
+    }
+
+    if (currentSection !== "") {
+      markdownNodes.push(
+        this.buildNodeFromSplit(currentSection.trim(), node, metadata),
+      );
+    }
+
+    return markdownNodes;
+  }
+
+  private updateMetadata(
+    headersMetadata: { [key: string]: string },
+    newHeader: string,
+    newHeaderLevel: number,
+  ): { [key: string]: string } {
+    const updatedHeaders: { [key: string]: string } = {};
+
+    for (let i = 1; i < newHeaderLevel; i++) {
+      const key = `Header_${i}`;
+      if (key in headersMetadata) {
+        updatedHeaders[key] = headersMetadata[key];
+      }
+    }
+
+    updatedHeaders[`Header_${newHeaderLevel}`] = newHeader;
+    return updatedHeaders;
+  }
+
+  private buildNodeFromSplit(
+    textSplit: string,
+    node: TextNode,
+    metadata: Metadata,
+  ): TextNode {
+    const newNode = buildNodeFromSplits([textSplit], node, undefined)[0];
+
+    if (this.includeMetadata) {
+      newNode.metadata = { ...newNode.metadata, ...metadata };
+    }
+
+    return newNode;
+  }
+}

packages/core/src/node-parser/sentence-splitter.ts

@@ -0,0 +1,226 @@
+import type { Tokenizer } from "@llamaindex/env";
+import { z } from "zod";
+import { Settings } from "../global";
+import { sentenceSplitterSchema } from "../schema";
+import { MetadataAwareTextSplitter } from "./base";
+import type { SplitterParams } from "./type";
+import {
+  splitByChar,
+  splitByRegex,
+  splitBySentenceTokenizer,
+  splitBySep,
+  type TextSplitterFn,
+} from "./utils";
+
+type _Split = {
+  text: string;
+  isSentence: boolean;
+  tokenSize: number;
+};
+
+/**
+ * Parse text with a preference for complete sentences.
+ */
+export class SentenceSplitter extends MetadataAwareTextSplitter {
+  /**
+   * The token chunk size for each chunk.
+   */
+  chunkSize: number = 1024;
+  /**
+   * The token overlap of each chunk when splitting.
+   */
+  chunkOverlap: number = 200;
+  /**
+   * Default separator for splitting into words
+   */
+  separator: string = " ";
+  /**
+   * Separator between paragraphs.
+   */
+  paragraphSeparator: string = "\n\n\n";
+  /**
+   * Backup regex for splitting into sentences.
+   */
+  secondaryChunkingRegex: string = "[^,.;。？！]+[,.;。？！]?";
+
+  #chunkingTokenizerFn = splitBySentenceTokenizer();
+  #splitFns: Set<TextSplitterFn> = new Set();
+  #subSentenceSplitFns: Set<TextSplitterFn> = new Set();
+  #tokenizer: Tokenizer;
+
+  constructor(
+    params?: z.input<typeof sentenceSplitterSchema> & SplitterParams,
+  ) {
+    super();
+    if (params) {
+      const parsedParams = sentenceSplitterSchema.parse(params);
+      this.chunkSize = parsedParams.chunkSize;
+      this.chunkOverlap = parsedParams.chunkOverlap;
+      this.separator = parsedParams.separator;
+      this.paragraphSeparator = parsedParams.paragraphSeparator;
+      this.secondaryChunkingRegex = parsedParams.secondaryChunkingRegex;
+    }
+    this.#tokenizer = params?.tokenizer ?? Settings.tokenizer;
+    this.#splitFns.add(splitBySep(this.paragraphSeparator));
+    this.#splitFns.add(this.#chunkingTokenizerFn);
+
+    this.#subSentenceSplitFns.add(splitByRegex(this.secondaryChunkingRegex));
+    this.#subSentenceSplitFns.add(splitBySep(this.separator));
+    this.#subSentenceSplitFns.add(splitByChar());
+  }
+
+  splitTextMetadataAware(text: string, metadata: string): string[] {
+    const metadataLength = this.tokenSize(metadata);
+    const effectiveChunkSize = this.chunkSize - metadataLength;
+    if (effectiveChunkSize <= 0) {
+      throw new Error(
+        `Metadata length (${metadataLength}) is longer than chunk size (${this.chunkSize}). Consider increasing the chunk size or decreasing the size of your metadata to avoid this.`,
+      );
+    } else if (effectiveChunkSize < 50) {
+      console.log(
+        `Metadata length (${metadataLength}) is close to chunk size (${this.chunkSize}). Resulting chunks are less than 50 tokens. Consider increasing the chunk size or decreasing the size of your metadata to avoid this.`,
+      );
+    }
+    return this._splitText(text, effectiveChunkSize);
+  }
+
+  splitText(text: string): string[] {
+    return this._splitText(text, this.chunkSize);
+  }
+
+  _splitText(text: string, chunkSize: number): string[] {
+    if (text === "") return [text];
+
+    const callbackManager = Settings.callbackManager;
+
+    callbackManager.dispatchEvent("chunking-start", {
+      text: [text],
+    });
+    const splits = this.#split(text, chunkSize);
+    const chunks = this.#merge(splits, chunkSize);
+
+    callbackManager.dispatchEvent("chunking-end", {
+      chunks,
+    });
+    return chunks;
+  }
+
+  #split(text: string, chunkSize: number): _Split[] {
+    const tokenSize = this.tokenSize(text);
+    if (tokenSize <= chunkSize) {
+      return [
+        {
+          text,
+          isSentence: true,
+          tokenSize,
+        },
+      ];
+    }
+    const [textSplitsByFns, isSentence] = this.#getSplitsByFns(text);
+    const textSplits: _Split[] = [];
+
+    for (const textSplit of textSplitsByFns) {
+      const tokenSize = this.tokenSize(textSplit);
+      if (tokenSize <= chunkSize) {
+        textSplits.push({
+          text: textSplit,
+          isSentence,
+          tokenSize,
+        });
+      } else {
+        const recursiveTextSplits = this.#split(textSplit, chunkSize);
+        textSplits.push(...recursiveTextSplits);
+      }
+    }
+    return textSplits;
+  }
+
+  #getSplitsByFns(text: string): [splits: string[], isSentence: boolean] {
+    for (const splitFn of this.#splitFns) {
+      const splits = splitFn(text);
+      if (splits.length > 1) {
+        return [splits, true];
+      }
+    }
+    for (const splitFn of this.#subSentenceSplitFns) {
+      const splits = splitFn(text);
+      if (splits.length > 1) {
+        return [splits, false];
+      }
+    }
+    return [[text], true];
+  }
+
+  #merge(splits: _Split[], chunkSize: number): string[] {
+    const chunks: string[] = [];
+    let currentChunk: [string, number][] = [];
+    let lastChunk: [string, number][] = [];
+    let currentChunkLength = 0;
+    let newChunk = true;
+
+    const closeChunk = (): void => {
+      chunks.push(currentChunk.map(([text]) => text).join(""));
+      lastChunk = currentChunk;
+      currentChunk = [];
+      currentChunkLength = 0;
+      newChunk = true;
+
+      let lastIndex = lastChunk.length - 1;
+      while (
+        lastIndex >= 0 &&
+        currentChunkLength + lastChunk[lastIndex][1] <= this.chunkOverlap
+      ) {
+        const [text, length] = lastChunk[lastIndex];
+        currentChunkLength += length;
+        currentChunk.unshift([text, length]);
+        lastIndex -= 1;
+      }
+    };
+
+    while (splits.length > 0) {
+      const curSplit = splits[0];
+      if (curSplit.tokenSize > chunkSize) {
+        throw new Error("Single token exceeded chunk size");
+      }
+      if (currentChunkLength + curSplit.tokenSize > chunkSize && !newChunk) {
+        closeChunk();
+      } else {
+        if (
+          curSplit.isSentence ||
+          currentChunkLength + curSplit.tokenSize <= chunkSize ||
+          newChunk
+        ) {
+          currentChunkLength += curSplit.tokenSize;
+          currentChunk.push([curSplit.text, curSplit.tokenSize]);
+          splits.shift();
+          newChunk = false;
+        } else {
+          closeChunk();
+        }
+      }
+    }
+
+    // Handle the last chunk
+    if (!newChunk) {
+      chunks.push(currentChunk.map(([text]) => text).join(""));
+    }
+
+    return this.#postprocessChunks(chunks);
+  }
+
+  /**
+   * Remove whitespace only chunks and remove leading and trailing whitespace.
+   */
+  #postprocessChunks(chunks: string[]): string[] {
+    const newChunks: string[] = [];
+    for (const chunk of chunks) {
+      const trimmedChunk = chunk.trim();
+      if (trimmedChunk !== "") {
+        newChunks.push(trimmedChunk);
+      }
+    }
+    return newChunks;
+  }
+
+  tokenSize = (text: string) => this.#tokenizer.encode(text).length;
+}

packages/core/src/node-parser/sentence-tokenizer-parser.d.ts

@@ -0,0 +1,5 @@
+declare class SentenceTokenizer {
+  tokenize(text: string): string[];
+}
+
+export { SentenceTokenizer as default };

packages/core/src/node-parser/sentence-window.ts

@@ -0,0 +1,85 @@
+import { randomUUID } from "@llamaindex/env";
+import { z } from "zod";
+import {
+  buildNodeFromSplits,
+  Document,
+  sentenceWindowNodeParserSchema,
+  TextNode,
+} from "../schema";
+import { NodeParser } from "./base";
+import { splitBySentenceTokenizer, type TextSplitterFn } from "./utils";
+
+export class SentenceWindowNodeParser extends NodeParser {
+  static DEFAULT_WINDOW_SIZE = 3;
+  static DEFAULT_WINDOW_METADATA_KEY = "window";
+  static DEFAULT_ORIGINAL_TEXT_METADATA_KEY = "originalText";
+
+  windowSize: number;
+  windowMetadataKey: string;
+  originalTextMetadataKey: string;
+  sentenceSplitter: TextSplitterFn = splitBySentenceTokenizer();
+  idGenerator: () => string = () => randomUUID();
+
+  constructor(params?: z.input<typeof sentenceWindowNodeParserSchema>) {
+    super();
+    if (params) {
+      const parsedParams = sentenceWindowNodeParserSchema.parse(params);
+      this.windowSize = parsedParams.windowSize;
+      this.windowMetadataKey = parsedParams.windowMetadataKey;
+      this.originalTextMetadataKey = parsedParams.originalTextMetadataKey;
+    } else {
+      this.windowSize = SentenceWindowNodeParser.DEFAULT_WINDOW_SIZE;
+      this.windowMetadataKey =
+        SentenceWindowNodeParser.DEFAULT_WINDOW_METADATA_KEY;
+      this.originalTextMetadataKey =
+        SentenceWindowNodeParser.DEFAULT_ORIGINAL_TEXT_METADATA_KEY;
+    }
+  }
+
+  override parseNodes(nodes: TextNode[], showProgress?: boolean): TextNode[] {
+    return nodes.reduce<TextNode[]>((allNodes, node) => {
+      const nodes = this.buildWindowNodesFromDocuments([node]);
+      return allNodes.concat(nodes);
+    }, []);
+  }
+
+  buildWindowNodesFromDocuments(documents: Document[]): TextNode[] {
+    const allNodes: TextNode[] = [];
+
+    for (const doc of documents) {
+      const text = doc.text;
+      const textSplits = this.sentenceSplitter(text);
+      const nodes = buildNodeFromSplits(
+        textSplits,
+        doc,
+        undefined,
+        this.idGenerator,
+      );
+
+      nodes.forEach((node, i) => {
+        const windowNodes = nodes.slice(
+          Math.max(0, i - this.windowSize),
+          Math.min(i + this.windowSize + 1, nodes.length),
+        );
+
+        node.metadata[this.windowMetadataKey] = windowNodes
+          .map((n) => n.text)
+          .join(" ");
+        node.metadata[this.originalTextMetadataKey] = node.text;
+
+        node.excludedEmbedMetadataKeys.push(
+          this.windowMetadataKey,
+          this.originalTextMetadataKey,
+        );
+        node.excludedLlmMetadataKeys.push(
+          this.windowMetadataKey,
+          this.originalTextMetadataKey,
+        );
+      });
+
+      allNodes.push(...nodes);
+    }
+
+    return allNodes;
+  }
+}

packages/core/src/node-parser/type.ts

@@ -0,0 +1,5 @@
+import type { Tokenizer } from "@llamaindex/env";
+
+export type SplitterParams = {
+  tokenizer?: Tokenizer;
+};

packages/core/src/node-parser/utils.ts

@@ -0,0 +1,53 @@
+import type { TextSplitter } from "./base";
+import SentenceTokenizerNew from "./sentence-tokenizer-parser.js";
+
+export type TextSplitterFn = (text: string) => string[];
+
+const truncateText = (text: string, textSplitter: TextSplitter): string => {
+  const chunks = textSplitter.splitText(text);
+  return chunks[0];
+};
+
+const splitTextKeepSeparator = (text: string, separator: string): string[] => {
+  const parts = text.split(separator);
+  const result = parts.map((part, index) =>
+    index > 0 ? separator + part : part,
+  );
+  return result.filter((s) => s);
+};
+
+export const splitBySep = (
+  sep: string,
+  keepSep: boolean = true,
+): TextSplitterFn => {
+  if (keepSep) {
+    return (text: string) => splitTextKeepSeparator(text, sep);
+  } else {
+    return (text: string) => text.split(sep);
+  }
+};
+
+export const splitByChar = (): TextSplitterFn => {
+  return (text: string) => text.split("");
+};
+
+let sentenceTokenizer: SentenceTokenizerNew | null = null;
+
+export const splitBySentenceTokenizer = (): TextSplitterFn => {
+  if (!sentenceTokenizer) {
+    sentenceTokenizer = new SentenceTokenizerNew();
+  }
+  const tokenizer = sentenceTokenizer;
+  return (text: string) => {
+    return tokenizer.tokenize(text);
+  };
+};
+
+export const splitByRegex = (regex: string): TextSplitterFn => {
+  return (text: string) => text.match(new RegExp(regex, "g")) || [];
+};
+
+export const splitByPhraseRegex = (): TextSplitterFn => {
+  const regex = "[^,.;]+[,.;]?";
+  return splitByRegex(regex);
+};

packages/core/src/schema/node.ts

@@ -450,3 +450,48 @@ export function splitNodesByType(nodes: BaseNode[]): NodesByType {
   }
   return result;
 }
+
+export function buildNodeFromSplits(
+  textSplits: string[],
+  doc: BaseNode,
+  refDoc: BaseNode = doc,
+  idGenerator: (idx: number, refDoc: BaseNode) => string = () => randomUUID(),
+) {
+  const nodes: TextNode[] = [];
+  const relationships = {
+    [NodeRelationship.SOURCE]: refDoc.asRelatedNodeInfo(),
+  };
+
+  textSplits.forEach((textChunk, i) => {
+    if (doc instanceof ImageDocument) {
+      const imageNode = new ImageNode({
+        id_: idGenerator(i, doc),
+        text: textChunk,
+        image: doc.image,
+        embedding: doc.embedding,
+        excludedEmbedMetadataKeys: [...doc.excludedEmbedMetadataKeys],
+        excludedLlmMetadataKeys: [...doc.excludedLlmMetadataKeys],
+        metadataSeparator: doc.metadataSeparator,
+        textTemplate: doc.textTemplate,
+        relationships: { ...relationships },
+      });
+      nodes.push(imageNode);
+    } else if (doc instanceof Document || doc instanceof TextNode) {
+      const node = new TextNode({
+        id_: idGenerator(i, doc),
+        text: textChunk,
+        embedding: doc.embedding,
+        excludedEmbedMetadataKeys: [...doc.excludedEmbedMetadataKeys],
+        excludedLlmMetadataKeys: [...doc.excludedLlmMetadataKeys],
+        metadataSeparator: doc.metadataSeparator,
+        textTemplate: doc.textTemplate,
+        relationships: { ...relationships },
+      });
+      nodes.push(node);
+    } else {
+      throw new Error(`Unknown document type: ${doc.type}`);
+    }
+  });
+
+  return nodes;
+}

packages/core/src/schema/type.ts

@@ -1,5 +1,8 @@
 import type { BaseNode } from "./node";
 
-export interface TransformComponent<Options extends Record<string, unknown>> {
-  transform(nodes: BaseNode[], options?: Options): Promise<BaseNode[]>;
+export interface TransformComponent {
+  transform<Options extends Record<string, unknown>>(
+    nodes: BaseNode[],
+    options?: Options,
+  ): Promise<BaseNode[]>;
 }

packages/core/src/schema/zod.ts

@@ -1,4 +1,5 @@
 import { z } from "zod";
+import { Settings } from "../global";
 
 export const anyFunctionSchema = z.function(z.tuple([]).rest(z.any()), z.any());
 
@@ -16,3 +17,62 @@ export const baseToolSchema = z.object({
 export const baseToolWithCallSchema = baseToolSchema.extend({
   call: z.function(),
 });
+
+export const sentenceSplitterSchema = z
+  .object({
+    chunkSize: z
+      .number({
+        description: "The token chunk size for each chunk.",
+      })
+      .gt(0)
+      .optional()
+      .default(() => Settings.chunkSize ?? 1024),
+    chunkOverlap: z
+      .number({
+        description: "The token overlap of each chunk when splitting.",
+      })
+      .gte(0)
+      .optional()
+      .default(200),
+    separator: z
+      .string({
+        description: "Default separator for splitting into words",
+      })
+      .default(" "),
+    paragraphSeparator: z
+      .string({
+        description: "Separator between paragraphs.",
+      })
+      .optional()
+      .default("\n\n\n"),
+    secondaryChunkingRegex: z
+      .string({
+        description: "Backup regex for splitting into sentences.",
+      })
+      .optional()
+      .default("[^,.;。？！]+[,.;。？！]?"),
+  })
+  .refine(
+    (data) => data.chunkOverlap < data.chunkSize,
+    "Chunk overlap must be less than chunk size.",
+  );
+
+export const sentenceWindowNodeParserSchema = z.object({
+  windowSize: z
+    .number({
+      description:
+        "The number of sentences on each side of a sentence to capture.",
+    })
+    .gt(0)
+    .default(3),
+  windowMetadataKey: z
+    .string({
+      description: "The metadata key to store the sentence window under.",
+    })
+    .default("window"),
+  originalTextMetadataKey: z
+    .string({
+      description: "The metadata key to store the original sentence in.",
+    })
+    .default("originalText"),
+});

packages/core/src/utils/wrap-llm-event.ts

@@ -1,5 +1,5 @@
 import { AsyncLocalStorage, randomUUID } from "@llamaindex/env";
-import { getCallbackManager } from "../global/settings/callback-manager";
+import { Settings } from "../global";
 import type { ChatResponse, ChatResponseChunk, LLM, LLMChat } from "../llms";
 
 export function wrapLLMEvent<
@@ -21,7 +21,7 @@ export function wrapLLMEvent<
     LLMChat<AdditionalChatOptions, AdditionalMessageOptions>["chat"]
   > {
     const id = randomUUID();
-    getCallbackManager().dispatchEvent("llm-start", {
+    Settings.callbackManager.dispatchEvent("llm-start", {
       id,
       messages: params[0].messages,
     });
@@ -55,7 +55,7 @@ export function wrapLLMEvent<
               ...chunk.options,
             };
           }
-          getCallbackManager().dispatchEvent("llm-stream", {
+          Settings.callbackManager.dispatchEvent("llm-stream", {
             id,
             chunk,
           });
@@ -63,14 +63,14 @@ export function wrapLLMEvent<
           yield chunk;
         }
         snapshot(() => {
-          getCallbackManager().dispatchEvent("llm-end", {
+          Settings.callbackManager.dispatchEvent("llm-end", {
             id,
             response: finalResponse,
           });
         });
       };
     } else {
-      getCallbackManager().dispatchEvent("llm-end", {
+      Settings.callbackManager.dispatchEvent("llm-end", {
         id,
         response,
       });

packages/core/tests/node-parser/markdown.test.ts

@@ -1,5 +1,5 @@
+import { MarkdownNodeParser } from "@llamaindex/core/node-parser";
 import { Document, MetadataMode } from "@llamaindex/core/schema";
-import { MarkdownNodeParser } from "llamaindex/nodeParsers/index";
 import { describe, expect, test } from "vitest";
 
 describe("MarkdownNodeParser", () => {
@@ -19,8 +19,8 @@ Header 2 content
     ]);
 
     expect(splits.length).toBe(2);
-    expect(splits[0].metadata).toEqual({ "Header 1": "Main Header" });
-    expect(splits[1].metadata).toEqual({ "Header 1": "Header 2" });
+    expect(splits[0].metadata).toEqual({ Header_1: "Main Header" });
+    expect(splits[1].metadata).toEqual({ Header_1: "Header 2" });
     expect(splits[0].getContent(MetadataMode.NONE)).toStrictEqual(
       "Main Header\n\nHeader 1 content",
     );
@@ -89,16 +89,16 @@ Content
       }),
     ]);
     expect(splits.length).toBe(4);
-    expect(splits[0].metadata).toEqual({ "Header 1": "Main Header" });
+    expect(splits[0].metadata).toEqual({ Header_1: "Main Header" });
     expect(splits[1].metadata).toEqual({
-      "Header 1": "Main Header",
-      "Header 2": "Sub-header",
+      Header_1: "Main Header",
+      Header_2: "Sub-header",
     });
     expect(splits[2].metadata).toEqual({
-      "Header 1": "Main Header",
-      "Header 2": "Sub-header",
-      "Header 3": "Sub-sub header",
+      Header_1: "Main Header",
+      Header_2: "Sub-header",
+      Header_3: "Sub-sub header",
     });
-    expect(splits[3].metadata).toEqual({ "Header 1": "New title" });
+    expect(splits[3].metadata).toEqual({ Header_1: "New title" });
   });
 });

packages/core/tests/node-parser/sentence-spiller.test.ts

@@ -1,12 +1,13 @@
+import { SentenceSplitter } from "@llamaindex/core/node-parser";
 import { Document } from "@llamaindex/core/schema";
-import { SimpleNodeParser } from "llamaindex/nodeParsers/index";
+import { tokenizers } from "@llamaindex/env";
 import { beforeEach, describe, expect, test } from "vitest";
 
-describe("SimpleNodeParser", () => {
-  let simpleNodeParser: SimpleNodeParser;
+describe("SentenceSplitter", () => {
+  let sentenceSplitter: SentenceSplitter;
 
   beforeEach(() => {
-    simpleNodeParser = new SimpleNodeParser({
+    sentenceSplitter = new SentenceSplitter({
       chunkSize: 1024,
       chunkOverlap: 20,
     });
@@ -19,7 +20,7 @@ describe("SimpleNodeParser", () => {
       excludedLlmMetadataKeys: ["animals"],
       excludedEmbedMetadataKeys: ["animals"],
     });
-    const result = simpleNodeParser.getNodesFromDocuments([doc]);
+    const result = sentenceSplitter.getNodesFromDocuments([doc]);
     expect(result.length).toEqual(1);
     const node = result[0];
     // check not the same object
@@ -37,4 +38,15 @@ describe("SimpleNodeParser", () => {
     // check relationship
     expect(node.sourceNode?.nodeId).toBe(doc.id_);
   });
+
+  test("split long text", async () => {
+    const longSentence = "is ".repeat(9000) + ".";
+    const document = new Document({ text: longSentence, id_: "1" });
+    const result = sentenceSplitter.getNodesFromDocuments([document]);
+    expect(result.length).toEqual(9);
+    result.forEach((node) => {
+      const { length } = tokenizers.tokenizer().encode(node.text);
+      expect(length).toBeLessThanOrEqual(1024);
+    });
+  });
 });

packages/core/tests/node-parser/sentence-window.test.ts

@@ -1,8 +1,5 @@
+import { SentenceWindowNodeParser } from "@llamaindex/core/node-parser";
 import { Document, MetadataMode } from "@llamaindex/core/schema";
-import {
-  DEFAULT_WINDOW_METADATA_KEY,
-  SentenceWindowNodeParser,
-} from "llamaindex/nodeParsers/index";
 import { describe, expect, test } from "vitest";
 
 describe("Tests for the SentenceWindowNodeParser class", () => {
@@ -11,7 +8,7 @@ describe("Tests for the SentenceWindowNodeParser class", () => {
     expect(sentenceWindowNodeParser).toBeDefined();
   });
   test("testing the getNodesFromDocuments method", () => {
-    const sentenceWindowNodeParser = SentenceWindowNodeParser.fromDefaults({
+    const sentenceWindowNodeParser = new SentenceWindowNodeParser({
       windowSize: 1,
     });
     const doc = new Document({ text: "Hello. Cat Mouse. Dog." });
@@ -25,7 +22,9 @@ describe("Tests for the SentenceWindowNodeParser class", () => {
       "Dog.",
     ]);
     expect(
-      resultingNodes.map((n) => n.metadata[DEFAULT_WINDOW_METADATA_KEY]),
+      resultingNodes.map(
+        (n) => n.metadata[SentenceWindowNodeParser.DEFAULT_WINDOW_METADATA_KEY],
+      ),
     ).toEqual([
       "Hello. Cat Mouse.",
       "Hello. Cat Mouse. Dog.",

packages/core/tests/node-parser/text-splitter.test.ts

@@ -1,7 +1,4 @@
-import {
-  SentenceSplitter,
-  cjkSentenceTokenizer,
-} from "llamaindex/TextSplitter";
+import { SentenceSplitter } from "@llamaindex/core/node-parser";
 import { describe, expect, test } from "vitest";
 
 describe("SentenceSplitter", () => {
@@ -10,14 +7,16 @@ describe("SentenceSplitter", () => {
     expect(sentenceSplitter).toBeDefined();
   });
 
+  test("chunk size should less than chunk", async () => {});
+
   test("splits paragraphs w/o effective chunk size", () => {
     const sentenceSplitter = new SentenceSplitter({
-      paragraphSeparator: "\n\n\n",
+      chunkSize: 9,
+      chunkOverlap: 0,
     });
     // generate the same line as above but correct syntax errors
-    const splits = sentenceSplitter.getParagraphSplits(
+    const splits = sentenceSplitter.splitText(
       "This is a paragraph.\n\n\nThis is another paragraph.",
-      undefined,
     );
     expect(splits).toEqual([
       "This is a paragraph.",
@@ -30,9 +29,8 @@ describe("SentenceSplitter", () => {
       paragraphSeparator: "\n",
     });
     // generate the same line as above but correct syntax errors
-    const splits = sentenceSplitter.getParagraphSplits(
+    const splits = sentenceSplitter.splitText(
       "This is a paragraph.\nThis is another paragraph.",
-      1000,
     );
     expect(splits).toEqual([
       "This is a paragraph.\nThis is another paragraph.",
@@ -40,10 +38,12 @@ describe("SentenceSplitter", () => {
   });
 
   test("splits sentences", () => {
-    const sentenceSplitter = new SentenceSplitter();
-    const splits = sentenceSplitter.getSentenceSplits(
+    const sentenceSplitter = new SentenceSplitter({
+      chunkSize: 9,
+      chunkOverlap: 0,
+    });
+    const splits = sentenceSplitter.splitText(
       "This is a sentence. This is another sentence.",
-      undefined,
     );
     expect(splits).toEqual([
       "This is a sentence.",
@@ -89,9 +89,10 @@ describe("SentenceSplitter", () => {
 
   test("splits cjk", () => {
     const sentenceSplitter = new SentenceSplitter({
-      chunkSize: 12,
+      chunkSize: 30,
       chunkOverlap: 0,
-      chunkingTokenizerFn: cjkSentenceTokenizer,
+      secondaryChunkingRegex:
+        '.*?([﹒﹔﹖﹗．；。！？]["’”」』]{0,2}|：(?=["‘“「『]{1,2}|$))',
     });
 
     const splits = sentenceSplitter.splitText(

packages/experimental/CHANGELOG.md

@@ -1,5 +1,27 @@
 # @llamaindex/experimental
 
+## 0.0.59
+
+### Patch Changes
+
+- Updated dependencies [15962b3]
+  - llamaindex@0.5.9
+
+## 0.0.58
+
+### Patch Changes
+
+- Updated dependencies [3d5ba08]
+- Updated dependencies [d917cdc]
+  - llamaindex@0.5.8
+
+## 0.0.57
+
+### Patch Changes
+
+- Updated dependencies [ec59acd]
+  - llamaindex@0.5.7
+
 ## 0.0.56
 
 ### Patch Changes

packages/experimental/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/experimental",
   "description": "Experimental package for LlamaIndexTS",
-  "version": "0.0.56",
+  "version": "0.0.59",
   "type": "module",
   "types": "dist/type/index.d.ts",
   "main": "dist/cjs/index.js",

packages/llamaindex/CHANGELOG.md

@@ -1,5 +1,32 @@
 # llamaindex
 
+## 0.5.9
+
+### Patch Changes
+
+- 15962b3: feat: node parser refactor
+
+  Align the text splitter logic with Python; it has almost the same logic as Python; Zod checks for input and better error messages and event system.
+
+  This change will not be considered a breaking change since it doesn't have a significant output difference from the last version,
+  but some edge cases will change, like the page separator and parameter for the constructor.
+
+- Updated dependencies [15962b3]
+  - @llamaindex/core@0.1.4
+
+## 0.5.8
+
+### Patch Changes
+
+- 3d5ba08: fix: update user agent in AssemblyAI
+- d917cdc: Add azure interpreter tool to tool factory
+
+## 0.5.7
+
+### Patch Changes
+
+- ec59acd: fix: bundling issue with pnpm
+
 ## 0.5.6
 
 ### Patch Changes

packages/llamaindex/e2e/examples/cloudflare-worker-agent/CHANGELOG.md

@@ -1,5 +1,27 @@
 # @llamaindex/cloudflare-worker-agent-test
 
+## 0.0.43
+
+### Patch Changes
+
+- Updated dependencies [15962b3]
+  - llamaindex@0.5.9
+
+## 0.0.42
+
+### Patch Changes
+
+- Updated dependencies [3d5ba08]
+- Updated dependencies [d917cdc]
+  - llamaindex@0.5.8
+
+## 0.0.41
+
+### Patch Changes
+
+- Updated dependencies [ec59acd]
+  - llamaindex@0.5.7
+
 ## 0.0.40
 
 ### Patch Changes

packages/llamaindex/e2e/examples/cloudflare-worker-agent/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/cloudflare-worker-agent-test",
-  "version": "0.0.40",
+  "version": "0.0.43",
   "type": "module",
   "private": true,
   "scripts": {

packages/llamaindex/e2e/examples/nextjs-agent/CHANGELOG.md

@@ -1,5 +1,27 @@
 # @llamaindex/next-agent-test
 
+## 0.1.43
+
+### Patch Changes
+
+- Updated dependencies [15962b3]
+  - llamaindex@0.5.9
+
+## 0.1.42
+
+### Patch Changes
+
+- Updated dependencies [3d5ba08]
+- Updated dependencies [d917cdc]
+  - llamaindex@0.5.8
+
+## 0.1.41
+
+### Patch Changes
+
+- Updated dependencies [ec59acd]
+  - llamaindex@0.5.7
+
 ## 0.1.40
 
 ### Patch Changes

packages/llamaindex/e2e/examples/nextjs-agent/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/next-agent-test",
-  "version": "0.1.40",
+  "version": "0.1.43",
   "private": true,
   "scripts": {
     "dev": "next dev",

packages/llamaindex/e2e/examples/nextjs-edge-runtime/CHANGELOG.md

@@ -1,5 +1,27 @@
 # test-edge-runtime
 
+## 0.1.42
+
+### Patch Changes
+
+- Updated dependencies [15962b3]
+  - llamaindex@0.5.9
+
+## 0.1.41
+
+### Patch Changes
+
+- Updated dependencies [3d5ba08]
+- Updated dependencies [d917cdc]
+  - llamaindex@0.5.8
+
+## 0.1.40
+
+### Patch Changes
+
+- Updated dependencies [ec59acd]
+  - llamaindex@0.5.7
+
 ## 0.1.39
 
 ### Patch Changes

packages/llamaindex/e2e/examples/nextjs-edge-runtime/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/nextjs-edge-runtime-test",
-  "version": "0.1.39",
+  "version": "0.1.42",
   "private": true,
   "scripts": {
     "dev": "next dev",

packages/llamaindex/e2e/examples/nextjs-node-runtime/CHANGELOG.md

@@ -1,5 +1,27 @@
 # @llamaindex/next-node-runtime
 
+## 0.0.24
+
+### Patch Changes
+
+- Updated dependencies [15962b3]
+  - llamaindex@0.5.9
+
+## 0.0.23
+
+### Patch Changes
+
+- Updated dependencies [3d5ba08]
+- Updated dependencies [d917cdc]
+  - llamaindex@0.5.8
+
+## 0.0.22
+
+### Patch Changes
+
+- Updated dependencies [ec59acd]
+  - llamaindex@0.5.7
+
 ## 0.0.21
 
 ### Patch Changes

packages/llamaindex/e2e/examples/nextjs-node-runtime/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/next-node-runtime-test",
-  "version": "0.0.21",
+  "version": "0.0.24",
   "private": true,
   "scripts": {
     "dev": "next dev",

packages/llamaindex/e2e/examples/waku-query-engine/CHANGELOG.md

@@ -1,5 +1,27 @@
 # @llamaindex/waku-query-engine-test
 
+## 0.0.43
+
+### Patch Changes
+
+- Updated dependencies [15962b3]
+  - llamaindex@0.5.9
+
+## 0.0.42
+
+### Patch Changes
+
+- Updated dependencies [3d5ba08]
+- Updated dependencies [d917cdc]
+  - llamaindex@0.5.8
+
+## 0.0.41
+
+### Patch Changes
+
+- Updated dependencies [ec59acd]
+  - llamaindex@0.5.7
+
 ## 0.0.40
 
 ### Patch Changes

packages/llamaindex/e2e/examples/waku-query-engine/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/waku-query-engine-test",
-  "version": "0.0.40",
+  "version": "0.0.43",
   "type": "module",
   "private": true,
   "scripts": {

packages/llamaindex/e2e/node/openai.e2e.ts

@@ -7,8 +7,8 @@ import {
   OpenAI,
   OpenAIAgent,
   QueryEngineTool,
+  SentenceSplitter,
   Settings,
-  SimpleNodeParser,
   SimpleToolNodeMapping,
   SubQuestionQueryEngine,
   SummaryIndex,
@@ -124,7 +124,7 @@ await test("agent with object retriever", async (t) => {
   const alexInfoText = await readFile(alexInfoPath, "utf-8");
   const alexDocument = new Document({ text: alexInfoText, id_: alexInfoPath });
 
-  const nodes = new SimpleNodeParser({
+  const nodes = new SentenceSplitter({
     chunkSize: 200,
     chunkOverlap: 20,
   }).getNodesFromDocuments([alexDocument]);

packages/llamaindex/e2e/node/snapshot/agent.snap

@@ -22,7 +22,7 @@
           "options": {
             "toolCall": [
               {
-                "id": "call_sH6QfjsymHW7JFl68j8AY6xg",
+                "id": "call_8kF02T5eJKwUL5hCGF8upWgn",
                 "name": "Weather",
                 "input": "{\"location\":\"San Francisco\"}"
               }
@@ -30,13 +30,13 @@
           }
         },
         {
-          "role": "user",
           "content": "35 degrees and sunny in San Francisco",
+          "role": "user",
           "options": {
             "toolResult": {
               "result": "35 degrees and sunny in San Francisco",
               "isError": false,
-              "id": "call_sH6QfjsymHW7JFl68j8AY6xg"
+              "id": "call_8kF02T5eJKwUL5hCGF8upWgn"
             }
           }
         }
@@ -64,7 +64,7 @@
           "options": {
             "toolCall": [
               {
-                "id": "call_V7zs8cyDT5FqJhjwBqcCydgA",
+                "id": "call_xsgmMFgliEDiOmZuLaBjUiXE",
                 "name": "unique_id",
                 "input": "{\"firstName\":\"Alex\",\"lastName\":\"Yang\"}"
               }
@@ -72,13 +72,13 @@
           }
         },
         {
-          "role": "user",
           "content": "123456789",
+          "role": "user",
           "options": {
             "toolResult": {
               "result": "123456789",
               "isError": false,
-              "id": "call_V7zs8cyDT5FqJhjwBqcCydgA"
+              "id": "call_xsgmMFgliEDiOmZuLaBjUiXE"
             }
           }
         }
@@ -106,7 +106,7 @@
           "options": {
             "toolCall": [
               {
-                "id": "call_BrlGGU6GDGWr0hrwXt9qZKyt",
+                "id": "call_OTMrLMikpT37PLm6KOIG5LCF",
                 "name": "sumNumbers",
                 "input": "{\"a\":1,\"b\":1}"
               }
@@ -114,13 +114,13 @@
           }
         },
         {
-          "role": "user",
           "content": "2",
+          "role": "user",
           "options": {
             "toolResult": {
               "result": "2",
               "isError": false,
-              "id": "call_BrlGGU6GDGWr0hrwXt9qZKyt"
+              "id": "call_OTMrLMikpT37PLm6KOIG5LCF"
             }
           }
         }
@@ -138,7 +138,7 @@
           "options": {
             "toolCall": [
               {
-                "id": "call_sH6QfjsymHW7JFl68j8AY6xg",
+                "id": "call_8kF02T5eJKwUL5hCGF8upWgn",
                 "name": "Weather",
                 "input": "{\"location\":\"San Francisco\"}"
               }
@@ -168,7 +168,7 @@
           "options": {
             "toolCall": [
               {
-                "id": "call_V7zs8cyDT5FqJhjwBqcCydgA",
+                "id": "call_xsgmMFgliEDiOmZuLaBjUiXE",
                 "name": "unique_id",
                 "input": "{\"firstName\":\"Alex\",\"lastName\":\"Yang\"}"
               }
@@ -198,7 +198,7 @@
           "options": {
             "toolCall": [
               {
-                "id": "call_BrlGGU6GDGWr0hrwXt9qZKyt",
+                "id": "call_OTMrLMikpT37PLm6KOIG5LCF",
                 "name": "sumNumbers",
                 "input": "{\"a\":1,\"b\":1}"
               }

packages/llamaindex/e2e/node/snapshot/agent_stream.snap

@@ -23,7 +23,7 @@
             "toolCall": [
               {
                 "name": "divideNumbers",
-                "id": "call_V4daSdWk9QeYeSMKLBisNbSf",
+                "id": "call_XgB0tixgYDhGXXSgdY19XDmt",
                 "input": {
                   "a": 16,
                   "b": 2
@@ -31,7 +31,7 @@
               },
               {
                 "name": "sumNumbers",
-                "id": "call_n2OlBxlaoeMIMVeU9DeDfiPX",
+                "id": "call_nXP1Rc85Ntdv5XcI6FBWmB6d",
                 "input": "{\"a\": 8, \"b\": 20}"
               }
             ]
@@ -44,7 +44,7 @@
             "toolResult": {
               "result": "8",
               "isError": false,
-              "id": "call_V4daSdWk9QeYeSMKLBisNbSf"
+              "id": "call_XgB0tixgYDhGXXSgdY19XDmt"
             }
           }
         },
@@ -55,7 +55,7 @@
             "toolResult": {
               "result": "28",
               "isError": false,
-              "id": "call_n2OlBxlaoeMIMVeU9DeDfiPX"
+              "id": "call_nXP1Rc85Ntdv5XcI6FBWmB6d"
             }
           }
         }
@@ -74,7 +74,7 @@
             "toolCall": [
               {
                 "name": "sumNumbers",
-                "id": "call_n2OlBxlaoeMIMVeU9DeDfiPX",
+                "id": "call_nXP1Rc85Ntdv5XcI6FBWmB6d",
                 "input": "{\"a\": 8, \"b\": 20}"
               }
             ]
@@ -87,7 +87,7 @@
       "response": {
         "raw": null,
         "message": {
-          "content": "The result of dividing 16 by 2 is 8. When you add 20 to 8, the total is 28.",
+          "content": "After dividing 16 by 2, we get 8. Adding 20 to 8 gives us 28.",
           "role": "assistant",
           "options": {}
         }
@@ -103,7 +103,7 @@
           "toolCall": [
             {
               "name": "divideNumbers",
-              "id": "call_V4daSdWk9QeYeSMKLBisNbSf",
+              "id": "call_XgB0tixgYDhGXXSgdY19XDmt",
               "input": "{\"a\": 16, \"b\": 2}"
             }
           ]
@@ -119,7 +119,7 @@
           "toolCall": [
             {
               "name": "divideNumbers",
-              "id": "call_V4daSdWk9QeYeSMKLBisNbSf",
+              "id": "call_XgB0tixgYDhGXXSgdY19XDmt",
               "input": "{\"a\": 16, \"b\": 2}"
             }
           ]
@@ -135,7 +135,7 @@
           "toolCall": [
             {
               "name": "divideNumbers",
-              "id": "call_V4daSdWk9QeYeSMKLBisNbSf",
+              "id": "call_XgB0tixgYDhGXXSgdY19XDmt",
               "input": "{\"a\": 16, \"b\": 2}"
             }
           ]
@@ -151,7 +151,7 @@
           "toolCall": [
             {
               "name": "divideNumbers",
-              "id": "call_V4daSdWk9QeYeSMKLBisNbSf",
+              "id": "call_XgB0tixgYDhGXXSgdY19XDmt",
               "input": "{\"a\": 16, \"b\": 2}"
             }
           ]
@@ -167,7 +167,7 @@
           "toolCall": [
             {
               "name": "divideNumbers",
-              "id": "call_V4daSdWk9QeYeSMKLBisNbSf",
+              "id": "call_XgB0tixgYDhGXXSgdY19XDmt",
               "input": "{\"a\": 16, \"b\": 2}"
             }
           ]
@@ -183,7 +183,7 @@
           "toolCall": [
             {
               "name": "divideNumbers",
-              "id": "call_V4daSdWk9QeYeSMKLBisNbSf",
+              "id": "call_XgB0tixgYDhGXXSgdY19XDmt",
               "input": {
                 "a": 16,
                 "b": 2
@@ -202,7 +202,7 @@
           "toolCall": [
             {
               "name": "sumNumbers",
-              "id": "call_n2OlBxlaoeMIMVeU9DeDfiPX",
+              "id": "call_nXP1Rc85Ntdv5XcI6FBWmB6d",
               "input": "{\"a\": 8, \"b\": 20}"
             }
           ]
@@ -218,7 +218,7 @@
           "toolCall": [
             {
               "name": "sumNumbers",
-              "id": "call_n2OlBxlaoeMIMVeU9DeDfiPX",
+              "id": "call_nXP1Rc85Ntdv5XcI6FBWmB6d",
               "input": "{\"a\": 8, \"b\": 20}"
             }
           ]
@@ -234,7 +234,7 @@
           "toolCall": [
             {
               "name": "sumNumbers",
-              "id": "call_n2OlBxlaoeMIMVeU9DeDfiPX",
+              "id": "call_nXP1Rc85Ntdv5XcI6FBWmB6d",
               "input": "{\"a\": 8, \"b\": 20}"
             }
           ]
@@ -250,7 +250,7 @@
           "toolCall": [
             {
               "name": "sumNumbers",
-              "id": "call_n2OlBxlaoeMIMVeU9DeDfiPX",
+              "id": "call_nXP1Rc85Ntdv5XcI6FBWmB6d",
               "input": "{\"a\": 8, \"b\": 20}"
             }
           ]
@@ -263,23 +263,7 @@
       "chunk": {
         "raw": null,
         "options": {},
-        "delta": "The"
-      }
-    },
-    {
-      "id": "PRESERVE_1",
-      "chunk": {
-        "raw": null,
-        "options": {},
-        "delta": " result"
-      }
-    },
-    {
-      "id": "PRESERVE_1",
-      "chunk": {
-        "raw": null,
-        "options": {},
-        "delta": " of"
+        "delta": "After"
       }
     },
     {
@@ -335,7 +319,23 @@
       "chunk": {
         "raw": null,
         "options": {},
-        "delta": " is"
+        "delta": ","
+      }
+    },
+    {
+      "id": "PRESERVE_1",
+      "chunk": {
+        "raw": null,
+        "options": {},
+        "delta": " we"
+      }
+    },
+    {
+      "id": "PRESERVE_1",
+      "chunk": {
+        "raw": null,
+        "options": {},
+        "delta": " get"
       }
     },
     {
@@ -367,23 +367,7 @@
       "chunk": {
         "raw": null,
         "options": {},
-        "delta": " When"
-      }
-    },
-    {
-      "id": "PRESERVE_1",
-      "chunk": {
-        "raw": null,
-        "options": {},
-        "delta": " you"
-      }
-    },
-    {
-      "id": "PRESERVE_1",
-      "chunk": {
-        "raw": null,
-        "options": {},
-        "delta": " add"
+        "delta": " Adding"
       }
     },
     {
@@ -431,7 +415,7 @@
       "chunk": {
         "raw": null,
         "options": {},
-        "delta": ","
+        "delta": " gives"
       }
     },
     {
@@ -439,23 +423,7 @@
       "chunk": {
         "raw": null,
         "options": {},
-        "delta": " the"
-      }
-    },
-    {
-      "id": "PRESERVE_1",
-      "chunk": {
-        "raw": null,
-        "options": {},
-        "delta": " total"
-      }
-    },
-    {
-      "id": "PRESERVE_1",
-      "chunk": {
-        "raw": null,
-        "options": {},
-        "delta": " is"
+        "delta": " us"
       }
     },
     {

packages/llamaindex/e2e/node/snapshot/agent_with_object_function_call.snap

@@ -22,7 +22,7 @@
           "options": {
             "toolCall": [
               {
-                "id": "call_uERMumWlJLTO2GW93X6C2W3N",
+                "id": "call_FNnPEPNeSDmdDj7b5x4LIxBG",
                 "name": "get_weather",
                 "input": "{\"location\":\"San Francisco\"}"
               }
@@ -30,8 +30,8 @@
           }
         },
         {
-          "role": "user",
           "content": "{\n  location: San Francisco,\n  temperature: 72,\n  weather: cloudy,\n  rain_prediction: 0.89\n}",
+          "role": "user",
           "options": {
             "toolResult": {
               "result": {
@@ -41,7 +41,7 @@
                 "rain_prediction": 0.89
               },
               "isError": false,
-              "id": "call_uERMumWlJLTO2GW93X6C2W3N"
+              "id": "call_FNnPEPNeSDmdDj7b5x4LIxBG"
             }
           }
         }
@@ -59,7 +59,7 @@
           "options": {
             "toolCall": [
               {
-                "id": "call_uERMumWlJLTO2GW93X6C2W3N",
+                "id": "call_FNnPEPNeSDmdDj7b5x4LIxBG",
                 "name": "get_weather",
                 "input": "{\"location\":\"San Francisco\"}"
               }

packages/llamaindex/e2e/node/snapshot/agent_with_object_retriever.snap

@@ -17,7 +17,7 @@
       "id": "PRESERVE_1",
       "messages": [
         {
-          "content": "Context information is below.\n---------------------\nAlex is a male. What's very important, Alex is not in the Brazil.\n---------------------\nGiven the context information and not prior knowledge, answer the query.\nQuery: Alex\nAnswer:",
+          "content": "Context information is below.\n---------------------\nAlex is a male.\nWhat's very important, Alex is not in the Brazil.\n---------------------\nGiven the context information and not prior knowledge, answer the query.\nQuery: Alex\nAnswer:",
           "role": "user"
         }
       ]
@@ -26,7 +26,7 @@
       "id": "PRESERVE_2",
       "messages": [
         {
-          "content": "Context information is below.\n---------------------\nAlex is a male. What's very important, Alex is not in the Brazil.\n---------------------\nGiven the context information and not prior knowledge, answer the query.\nQuery: Brazil\nAnswer:",
+          "content": "Context information is below.\n---------------------\nAlex is a male.\nWhat's very important, Alex is not in the Brazil.\n---------------------\nGiven the context information and not prior knowledge, answer the query.\nQuery: Brazil\nAnswer:",
           "role": "user"
         }
       ]
@@ -48,12 +48,12 @@
           "options": {
             "toolCall": [
               {
-                "id": "call_vv1XW3xv4j2us5sZOtzCU2lL",
+                "id": "call_xkMkcEJYa2vVFUpg1Ng3UZTK",
                 "name": "summary_tool",
                 "input": "{\"query\": \"Alex\"}"
               },
               {
-                "id": "call_V36LMHbwUJkEa20A3GoA4wIr",
+                "id": "call_BYhP9Coo4NIBJDFaEKRA5qSl",
                 "name": "summary_tool",
                 "input": "{\"query\": \"Brazil\"}"
               }
@@ -61,24 +61,24 @@
           }
         },
         {
-          "role": "user",
           "content": "Alex is not in Brazil.",
+          "role": "user",
           "options": {
             "toolResult": {
               "result": "Alex is not in Brazil.",
               "isError": false,
-              "id": "call_vv1XW3xv4j2us5sZOtzCU2lL"
+              "id": "call_xkMkcEJYa2vVFUpg1Ng3UZTK"
             }
           }
         },
         {
-          "role": "user",
           "content": "Alex is not in Brazil.",
+          "role": "user",
           "options": {
             "toolResult": {
               "result": "Alex is not in Brazil.",
               "isError": false,
-              "id": "call_V36LMHbwUJkEa20A3GoA4wIr"
+              "id": "call_BYhP9Coo4NIBJDFaEKRA5qSl"
             }
           }
         }
@@ -96,12 +96,12 @@
           "options": {
             "toolCall": [
               {
-                "id": "call_vv1XW3xv4j2us5sZOtzCU2lL",
+                "id": "call_xkMkcEJYa2vVFUpg1Ng3UZTK",
                 "name": "summary_tool",
                 "input": "{\"query\": \"Alex\"}"
               },
               {
-                "id": "call_V36LMHbwUJkEa20A3GoA4wIr",
+                "id": "call_BYhP9Coo4NIBJDFaEKRA5qSl",
                 "name": "summary_tool",
                 "input": "{\"query\": \"Brazil\"}"
               }

packages/llamaindex/e2e/node/snapshot/anthropic-agent-multiple-chat.snap

@@ -20,7 +20,7 @@
           "content": [
             {
               "type": "text",
-              "text": "<thinking>\nThe user has not asked a question that requires using any of the available tools. They have simply requested that I respond with the word \"Yes\".\n</thinking>\n\nYes"
+              "text": "<thinking>\nThe user has simply asked me to respond with the word \"Yes\". This is a very straightforward request that does not require the use of any tools. I have all the information I need to directly provide the response the user has requested.\n</thinking>\n\nYes"
             }
           ],
           "role": "assistant",
@@ -43,7 +43,7 @@
           "content": [
             {
               "type": "text",
-              "text": "<thinking>\nThe user has not asked a question that requires using any of the available tools. They have simply requested that I respond with the word \"Yes\".\n</thinking>\n\nYes"
+              "text": "<thinking>\nThe user has simply asked me to respond with the word \"Yes\". This is a very straightforward request that does not require the use of any tools. I have all the information I need to directly provide the response the user has requested.\n</thinking>\n\nYes"
             }
           ],
           "role": "assistant",
@@ -57,7 +57,7 @@
           "content": [
             {
               "type": "text",
-              "text": "<thinking>\nThe user has not asked a question that requires using any of the available tools. They have simply requested that I respond with the word \"No\".\n</thinking>\n\nNo"
+              "text": "<thinking>\nThe user has now asked me to respond with the word \"No\" instead of \"Yes\". Once again, this is a very simple and direct request. No additional information or tool usage is needed. I can immediately provide the response the user has requested by simply replying with the word \"No\".\n</thinking>\n\nNo"
             }
           ],
           "role": "assistant",
@@ -80,7 +80,7 @@
           "content": [
             {
               "type": "text",
-              "text": "<thinking>\nThe user has not asked a question that requires using any of the available tools. They have simply requested that I respond with the word \"Yes\".\n</thinking>\n\nYes"
+              "text": "<thinking>\nThe user has simply asked me to respond with the word \"Yes\". This is a very straightforward request that does not require the use of any tools. I have all the information I need to directly provide the response the user has requested.\n</thinking>\n\nYes"
             }
           ],
           "role": "assistant",
@@ -94,7 +94,7 @@
           "content": [
             {
               "type": "text",
-              "text": "<thinking>\nThe user has not asked a question that requires using any of the available tools. They have simply requested that I respond with the word \"No\".\n</thinking>\n\nNo"
+              "text": "<thinking>\nThe user has now asked me to respond with the word \"No\" instead of \"Yes\". Once again, this is a very simple and direct request. No additional information or tool usage is needed. I can immediately provide the response the user has requested by simply replying with the word \"No\".\n</thinking>\n\nNo"
             }
           ],
           "role": "assistant",
@@ -108,7 +108,7 @@
           "content": [
             {
               "type": "text",
-              "text": "<thinking>\nThe user has not asked a question that requires using any of the available tools. They have simply requested that I respond with the word \"Maybe\".\n</thinking>\n\nMaybe"
+              "text": "<thinking>\nThe user has made another simple request, this time asking me to respond with the word \"Maybe\". Just like the previous requests for \"Yes\" and \"No\", I have all the necessary information to fulfill this request without needing to use any tools or gather additional details. I can provide the requested response by replying with only the word \"Maybe\".\n</thinking>\n\nMaybe"
             }
           ],
           "role": "assistant",
@@ -131,7 +131,7 @@
           "content": [
             {
               "type": "text",
-              "text": "<thinking>\nThe user has not asked a question that requires using any of the available tools. They have simply requested that I respond with the word \"Yes\".\n</thinking>\n\nYes"
+              "text": "<thinking>\nThe user has simply asked me to respond with the word \"Yes\". This is a very straightforward request that does not require the use of any tools. I have all the information I need to directly provide the response the user has requested.\n</thinking>\n\nYes"
             }
           ],
           "role": "assistant",
@@ -145,7 +145,7 @@
           "content": [
             {
               "type": "text",
-              "text": "<thinking>\nThe user has not asked a question that requires using any of the available tools. They have simply requested that I respond with the word \"No\".\n</thinking>\n\nNo"
+              "text": "<thinking>\nThe user has now asked me to respond with the word \"No\" instead of \"Yes\". Once again, this is a very simple and direct request. No additional information or tool usage is needed. I can immediately provide the response the user has requested by simply replying with the word \"No\".\n</thinking>\n\nNo"
             }
           ],
           "role": "assistant",
@@ -159,7 +159,7 @@
           "content": [
             {
               "type": "text",
-              "text": "<thinking>\nThe user has not asked a question that requires using any of the available tools. They have simply requested that I respond with the word \"Maybe\".\n</thinking>\n\nMaybe"
+              "text": "<thinking>\nThe user has made another simple request, this time asking me to respond with the word \"Maybe\". Just like the previous requests for \"Yes\" and \"No\", I have all the necessary information to fulfill this request without needing to use any tools or gather additional details. I can provide the requested response by replying with only the word \"Maybe\".\n</thinking>\n\nMaybe"
             }
           ],
           "role": "assistant",
@@ -173,14 +173,14 @@
           "content": [
             {
               "type": "text",
-              "text": "<thinking>\nThe user has asked for the weather in a specific city, San Francisco. The relevant tool to answer this is the getWeather function.\n\nLooking at the required parameters for getWeather:\ncity (string): The user directly provided the city \"San Francisco\"\n\nSince the required \"city\" parameter has been provided, we can proceed with the getWeather function call.\n</thinking>"
+              "text": "<thinking>\nThe user has asked for the current weather in San Francisco. To answer this, I will need to use the getWeather tool.\n\nLooking at the parameters for getWeather, I see it requires a single parameter:\n- city (string): The city to get the weather for \n\nThe user has directly provided the value for the city parameter in their request - they want the weather for San Francisco.\n\nI have all the required information to make the getWeather tool call, so I will proceed with calling the tool with \"San Francisco\" as the city parameter value.\n</thinking>"
             }
           ],
           "role": "assistant",
           "options": {
             "toolCall": [
               {
-                "id": "toolu_017MxwaxaLYkmt4dpP5HKzFe",
+                "id": "toolu_01Q5tWdx8CTrqUaQty8C1PD5",
                 "name": "getWeather",
                 "input": {
                   "city": "San Francisco"
@@ -196,7 +196,7 @@
             "toolResult": {
               "result": "The weather in San Francisco is 72 degrees",
               "isError": false,
-              "id": "toolu_017MxwaxaLYkmt4dpP5HKzFe"
+              "id": "toolu_01Q5tWdx8CTrqUaQty8C1PD5"
             }
           }
         }
@@ -213,7 +213,7 @@
           "content": [
             {
               "type": "text",
-              "text": "<thinking>\nThe user has not asked a question that requires using any of the available tools. They have simply requested that I respond with the word \"Yes\".\n</thinking>\n\nYes"
+              "text": "<thinking>\nThe user has simply asked me to respond with the word \"Yes\". This is a very straightforward request that does not require the use of any tools. I have all the information I need to directly provide the response the user has requested.\n</thinking>\n\nYes"
             }
           ],
           "role": "assistant",
@@ -227,7 +227,7 @@
           "content": [
             {
               "type": "text",
-              "text": "<thinking>\nThe user has not asked a question that requires using any of the available tools. They have simply requested that I respond with the word \"No\".\n</thinking>\n\nNo"
+              "text": "<thinking>\nThe user has now asked me to respond with the word \"No\" instead of \"Yes\". Once again, this is a very simple and direct request. No additional information or tool usage is needed. I can immediately provide the response the user has requested by simply replying with the word \"No\".\n</thinking>\n\nNo"
             }
           ],
           "role": "assistant",
@@ -241,7 +241,7 @@
           "content": [
             {
               "type": "text",
-              "text": "<thinking>\nThe user has not asked a question that requires using any of the available tools. They have simply requested that I respond with the word \"Maybe\".\n</thinking>\n\nMaybe"
+              "text": "<thinking>\nThe user has made another simple request, this time asking me to respond with the word \"Maybe\". Just like the previous requests for \"Yes\" and \"No\", I have all the necessary information to fulfill this request without needing to use any tools or gather additional details. I can provide the requested response by replying with only the word \"Maybe\".\n</thinking>\n\nMaybe"
             }
           ],
           "role": "assistant",
@@ -255,14 +255,14 @@
           "content": [
             {
               "type": "text",
-              "text": "<thinking>\nThe user has asked for the weather in a specific city, San Francisco. The relevant tool to answer this is the getWeather function.\n\nLooking at the required parameters for getWeather:\ncity (string): The user directly provided the city \"San Francisco\"\n\nSince the required \"city\" parameter has been provided, we can proceed with the getWeather function call.\n</thinking>"
+              "text": "<thinking>\nThe user has asked for the current weather in San Francisco. To answer this, I will need to use the getWeather tool.\n\nLooking at the parameters for getWeather, I see it requires a single parameter:\n- city (string): The city to get the weather for \n\nThe user has directly provided the value for the city parameter in their request - they want the weather for San Francisco.\n\nI have all the required information to make the getWeather tool call, so I will proceed with calling the tool with \"San Francisco\" as the city parameter value.\n</thinking>"
             }
           ],
           "role": "assistant",
           "options": {
             "toolCall": [
               {
-                "id": "toolu_017MxwaxaLYkmt4dpP5HKzFe",
+                "id": "toolu_01Q5tWdx8CTrqUaQty8C1PD5",
                 "name": "getWeather",
                 "input": {
                   "city": "San Francisco"
@@ -278,7 +278,7 @@
             "toolResult": {
               "result": "The weather in San Francisco is 72 degrees",
               "isError": false,
-              "id": "toolu_017MxwaxaLYkmt4dpP5HKzFe"
+              "id": "toolu_01Q5tWdx8CTrqUaQty8C1PD5"
             }
           }
         },
@@ -286,7 +286,7 @@
           "content": [
             {
               "type": "text",
-              "text": "The current weather in San Francisco is 72 degrees."
+              "text": "Based on the result from calling the getWeather tool, the current weather in San Francisco is 72 degrees."
             }
           ],
           "role": "assistant",
@@ -309,7 +309,7 @@
           "content": [
             {
               "type": "text",
-              "text": "<thinking>\nThe user has not asked a question that requires using any of the available tools. They have simply requested that I respond with the word \"Yes\".\n</thinking>\n\nYes"
+              "text": "<thinking>\nThe user has simply asked me to respond with the word \"Yes\". This is a very straightforward request that does not require the use of any tools. I have all the information I need to directly provide the response the user has requested.\n</thinking>\n\nYes"
             }
           ],
           "role": "assistant",
@@ -323,7 +323,7 @@
           "content": [
             {
               "type": "text",
-              "text": "<thinking>\nThe user has not asked a question that requires using any of the available tools. They have simply requested that I respond with the word \"No\".\n</thinking>\n\nNo"
+              "text": "<thinking>\nThe user has now asked me to respond with the word \"No\" instead of \"Yes\". Once again, this is a very simple and direct request. No additional information or tool usage is needed. I can immediately provide the response the user has requested by simply replying with the word \"No\".\n</thinking>\n\nNo"
             }
           ],
           "role": "assistant",
@@ -337,7 +337,7 @@
           "content": [
             {
               "type": "text",
-              "text": "<thinking>\nThe user has not asked a question that requires using any of the available tools. They have simply requested that I respond with the word \"Maybe\".\n</thinking>\n\nMaybe"
+              "text": "<thinking>\nThe user has made another simple request, this time asking me to respond with the word \"Maybe\". Just like the previous requests for \"Yes\" and \"No\", I have all the necessary information to fulfill this request without needing to use any tools or gather additional details. I can provide the requested response by replying with only the word \"Maybe\".\n</thinking>\n\nMaybe"
             }
           ],
           "role": "assistant",
@@ -351,14 +351,14 @@
           "content": [
             {
               "type": "text",
-              "text": "<thinking>\nThe user has asked for the weather in a specific city, San Francisco. The relevant tool to answer this is the getWeather function.\n\nLooking at the required parameters for getWeather:\ncity (string): The user directly provided the city \"San Francisco\"\n\nSince the required \"city\" parameter has been provided, we can proceed with the getWeather function call.\n</thinking>"
+              "text": "<thinking>\nThe user has asked for the current weather in San Francisco. To answer this, I will need to use the getWeather tool.\n\nLooking at the parameters for getWeather, I see it requires a single parameter:\n- city (string): The city to get the weather for \n\nThe user has directly provided the value for the city parameter in their request - they want the weather for San Francisco.\n\nI have all the required information to make the getWeather tool call, so I will proceed with calling the tool with \"San Francisco\" as the city parameter value.\n</thinking>"
             }
           ],
           "role": "assistant",
           "options": {
             "toolCall": [
               {
-                "id": "toolu_017MxwaxaLYkmt4dpP5HKzFe",
+                "id": "toolu_01Q5tWdx8CTrqUaQty8C1PD5",
                 "name": "getWeather",
                 "input": {
                   "city": "San Francisco"
@@ -374,7 +374,7 @@
             "toolResult": {
               "result": "The weather in San Francisco is 72 degrees",
               "isError": false,
-              "id": "toolu_017MxwaxaLYkmt4dpP5HKzFe"
+              "id": "toolu_01Q5tWdx8CTrqUaQty8C1PD5"
             }
           }
         },
@@ -382,7 +382,7 @@
           "content": [
             {
               "type": "text",
-              "text": "The current weather in San Francisco is 72 degrees."
+              "text": "Based on the result from calling the getWeather tool, the current weather in San Francisco is 72 degrees."
             }
           ],
           "role": "assistant",
@@ -396,14 +396,14 @@
           "content": [
             {
               "type": "text",
-              "text": "<thinking>\nThe user has asked for the weather in a specific city, Shanghai. The relevant tool to answer this is the getWeather function.\n\nLooking at the required parameters for getWeather:\ncity (string): The user directly provided the city \"Shanghai\"\n\nSince the required \"city\" parameter has been provided, we can proceed with the getWeather function call.\n</thinking>"
+              "text": "<thinking>\nThe user has now asked for the current weather in Shanghai. This is very similar to the previous request for San Francisco's weather. I will once again need to use the getWeather tool to answer this.\n\nThe getWeather tool requires the following parameter:\n- city (string): The city to get the weather for\n\nThe user has provided the city they want the weather for directly in their request - in this case, the city is Shanghai.\n\nSince I have the value for the required city parameter, I can go ahead and call the getWeather tool, passing in \"Shanghai\" as the city.\n</thinking>"
             }
           ],
           "role": "assistant",
           "options": {
             "toolCall": [
               {
-                "id": "toolu_013ZnATGKhvPkt2jjxdFheDK",
+                "id": "toolu_01HxDqJpm33QxDXAExTxm6zu",
                 "name": "getWeather",
                 "input": {
                   "city": "Shanghai"
@@ -419,7 +419,7 @@
             "toolResult": {
               "result": "The weather in Shanghai is 72 degrees",
               "isError": false,
-              "id": "toolu_013ZnATGKhvPkt2jjxdFheDK"
+              "id": "toolu_01HxDqJpm33QxDXAExTxm6zu"
             }
           }
         }
@@ -435,7 +435,7 @@
           "content": [
             {
               "type": "text",
-              "text": "<thinking>\nThe user has not asked a question that requires using any of the available tools. They have simply requested that I respond with the word \"Yes\".\n</thinking>\n\nYes"
+              "text": "<thinking>\nThe user has simply asked me to respond with the word \"Yes\". This is a very straightforward request that does not require the use of any tools. I have all the information I need to directly provide the response the user has requested.\n</thinking>\n\nYes"
             }
           ],
           "role": "assistant",
@@ -451,7 +451,7 @@
           "content": [
             {
               "type": "text",
-              "text": "<thinking>\nThe user has not asked a question that requires using any of the available tools. They have simply requested that I respond with the word \"No\".\n</thinking>\n\nNo"
+              "text": "<thinking>\nThe user has now asked me to respond with the word \"No\" instead of \"Yes\". Once again, this is a very simple and direct request. No additional information or tool usage is needed. I can immediately provide the response the user has requested by simply replying with the word \"No\".\n</thinking>\n\nNo"
             }
           ],
           "role": "assistant",
@@ -467,7 +467,7 @@
           "content": [
             {
               "type": "text",
-              "text": "<thinking>\nThe user has not asked a question that requires using any of the available tools. They have simply requested that I respond with the word \"Maybe\".\n</thinking>\n\nMaybe"
+              "text": "<thinking>\nThe user has made another simple request, this time asking me to respond with the word \"Maybe\". Just like the previous requests for \"Yes\" and \"No\", I have all the necessary information to fulfill this request without needing to use any tools or gather additional details. I can provide the requested response by replying with only the word \"Maybe\".\n</thinking>\n\nMaybe"
             }
           ],
           "role": "assistant",
@@ -483,14 +483,14 @@
           "content": [
             {
               "type": "text",
-              "text": "<thinking>\nThe user has asked for the weather in a specific city, San Francisco. The relevant tool to answer this is the getWeather function.\n\nLooking at the required parameters for getWeather:\ncity (string): The user directly provided the city \"San Francisco\"\n\nSince the required \"city\" parameter has been provided, we can proceed with the getWeather function call.\n</thinking>"
+              "text": "<thinking>\nThe user has asked for the current weather in San Francisco. To answer this, I will need to use the getWeather tool.\n\nLooking at the parameters for getWeather, I see it requires a single parameter:\n- city (string): The city to get the weather for \n\nThe user has directly provided the value for the city parameter in their request - they want the weather for San Francisco.\n\nI have all the required information to make the getWeather tool call, so I will proceed with calling the tool with \"San Francisco\" as the city parameter value.\n</thinking>"
             }
           ],
           "role": "assistant",
           "options": {
             "toolCall": [
               {
-                "id": "toolu_017MxwaxaLYkmt4dpP5HKzFe",
+                "id": "toolu_01Q5tWdx8CTrqUaQty8C1PD5",
                 "name": "getWeather",
                 "input": {
                   "city": "San Francisco"
@@ -509,7 +509,7 @@
           "content": [
             {
               "type": "text",
-              "text": "The current weather in San Francisco is 72 degrees."
+              "text": "Based on the result from calling the getWeather tool, the current weather in San Francisco is 72 degrees."
             }
           ],
           "role": "assistant",
@@ -525,14 +525,14 @@
           "content": [
             {
               "type": "text",
-              "text": "<thinking>\nThe user has asked for the weather in a specific city, Shanghai. The relevant tool to answer this is the getWeather function.\n\nLooking at the required parameters for getWeather:\ncity (string): The user directly provided the city \"Shanghai\"\n\nSince the required \"city\" parameter has been provided, we can proceed with the getWeather function call.\n</thinking>"
+              "text": "<thinking>\nThe user has now asked for the current weather in Shanghai. This is very similar to the previous request for San Francisco's weather. I will once again need to use the getWeather tool to answer this.\n\nThe getWeather tool requires the following parameter:\n- city (string): The city to get the weather for\n\nThe user has provided the city they want the weather for directly in their request - in this case, the city is Shanghai.\n\nSince I have the value for the required city parameter, I can go ahead and call the getWeather tool, passing in \"Shanghai\" as the city.\n</thinking>"
             }
           ],
           "role": "assistant",
           "options": {
             "toolCall": [
               {
-                "id": "toolu_013ZnATGKhvPkt2jjxdFheDK",
+                "id": "toolu_01HxDqJpm33QxDXAExTxm6zu",
                 "name": "getWeather",
                 "input": {
                   "city": "Shanghai"
@@ -551,7 +551,7 @@
           "content": [
             {
               "type": "text",
-              "text": "The current weather in Shanghai is 72 degrees."
+              "text": "The getWeather tool indicates that the current weather in Shanghai is 72 degrees."
             }
           ],
           "role": "assistant",

packages/llamaindex/e2e/node/snapshot/anthropic-agent.snap

@@ -20,14 +20,14 @@
           "content": [
             {
               "type": "text",
-              "text": "<thinking>\nThe user has asked for the weather in a specific location, San Francisco. The Weather tool is the relevant function to answer this request, as it returns weather information for a given location.\n\nThe Weather tool requires a single parameter:\n- location (string, required): The user has directly provided the location as \"San Francisco\".\n\nSince the required location parameter has been provided, we have enough information to call the Weather tool.\n</thinking>"
+              "text": "<thinking>\nThe Weather tool is relevant to answer this question, as it can provide weather information for a specified location.\n\nThe Weather tool requires a \"location\" parameter. The user has directly provided the location of \"San Francisco\" in their request.\n\nSince the required \"location\" parameter has been provided, we can proceed with calling the Weather tool to get the weather information for San Francisco. No other tools are needed.\n</thinking>"
             }
           ],
           "role": "assistant",
           "options": {
             "toolCall": [
               {
-                "id": "toolu_011YcKkPygw2woJZrVWRRMwH",
+                "id": "toolu_01Y6KpbYCFw4XGtvyXEmeTrB",
                 "name": "Weather",
                 "input": {
                   "location": "San Francisco"
@@ -43,7 +43,7 @@
             "toolResult": {
               "result": "35 degrees and sunny in San Francisco",
               "isError": false,
-              "id": "toolu_011YcKkPygw2woJZrVWRRMwH"
+              "id": "toolu_01Y6KpbYCFw4XGtvyXEmeTrB"
             }
           }
         }
@@ -69,14 +69,14 @@
           "content": [
             {
               "type": "text",
-              "text": "<thinking>\nThe unique_id function takes firstName and lastName as required parameters. The user has provided their first name (Alex) and last name (Yang) in the request, so we have all the necessary information to call the function.\n</thinking>"
+              "text": "<thinking>\nThe unique_id tool looks relevant for answering this request, as it can provide a unique identifier for a user given their first and last name. \nThe tool requires two parameters:\n- firstName (string)\n- lastName (string)\nLooking at the user's request, they have provided both their first name (Alex) and last name (Yang). So we have all the required parameters to call the unique_id tool.\n</thinking>"
             }
           ],
           "role": "assistant",
           "options": {
             "toolCall": [
               {
-                "id": "toolu_012ZLUq5SWsghkXUjsEXsB1f",
+                "id": "toolu_01Q7wYKoNtpGAphTbRSv51fE",
                 "name": "unique_id",
                 "input": {
                   "firstName": "Alex",
@@ -93,7 +93,7 @@
             "toolResult": {
               "result": "123456789",
               "isError": false,
-              "id": "toolu_012ZLUq5SWsghkXUjsEXsB1f"
+              "id": "toolu_01Q7wYKoNtpGAphTbRSv51fE"
             }
           }
         }
@@ -119,14 +119,14 @@
           "content": [
             {
               "type": "text",
-              "text": "<thinking>\nThe user is asking to sum the numbers 1 and 1. The relevant tool is the sumNumbers function, which takes two number parameters a and b.\nThe user has directly provided the values for the required parameters:\na = 1 \nb = 1\nSince all the required parameters are provided, we can proceed with calling the function.\n</thinking>"
+              "text": "<thinking>\nThe user is asking to sum the numbers 1 and 1. The sumNumbers tool is directly relevant for this request.\n\nLooking at the required parameters for sumNumbers:\na (number): The user provided the value 1 for this\nb (number): The user also provided the value 1 for this\n\nSince the user has directly provided values for all the required parameters, we can proceed with calling the sumNumbers tool without needing any additional information from the user.\n</thinking>"
             }
           ],
           "role": "assistant",
           "options": {
             "toolCall": [
               {
-                "id": "toolu_01SQD2XkaXkDNLQ2xaFzmguG",
+                "id": "toolu_01AxEUd1s1UmhPc6Y91LBSmw",
                 "name": "sumNumbers",
                 "input": {
                   "a": 1,
@@ -143,7 +143,7 @@
             "toolResult": {
               "result": "2",
               "isError": false,
-              "id": "toolu_01SQD2XkaXkDNLQ2xaFzmguG"
+              "id": "toolu_01AxEUd1s1UmhPc6Y91LBSmw"
             }
           }
         }
@@ -159,14 +159,14 @@
           "content": [
             {
               "type": "text",
-              "text": "<thinking>\nThe user has asked for the weather in a specific location, San Francisco. The Weather tool is the relevant function to answer this request, as it returns weather information for a given location.\n\nThe Weather tool requires a single parameter:\n- location (string, required): The user has directly provided the location as \"San Francisco\".\n\nSince the required location parameter has been provided, we have enough information to call the Weather tool.\n</thinking>"
+              "text": "<thinking>\nThe Weather tool is relevant to answer this question, as it can provide weather information for a specified location.\n\nThe Weather tool requires a \"location\" parameter. The user has directly provided the location of \"San Francisco\" in their request.\n\nSince the required \"location\" parameter has been provided, we can proceed with calling the Weather tool to get the weather information for San Francisco. No other tools are needed.\n</thinking>"
             }
           ],
           "role": "assistant",
           "options": {
             "toolCall": [
               {
-                "id": "toolu_011YcKkPygw2woJZrVWRRMwH",
+                "id": "toolu_01Y6KpbYCFw4XGtvyXEmeTrB",
                 "name": "Weather",
                 "input": {
                   "location": "San Francisco"
@@ -201,14 +201,14 @@
           "content": [
             {
               "type": "text",
-              "text": "<thinking>\nThe unique_id function takes firstName and lastName as required parameters. The user has provided their first name (Alex) and last name (Yang) in the request, so we have all the necessary information to call the function.\n</thinking>"
+              "text": "<thinking>\nThe unique_id tool looks relevant for answering this request, as it can provide a unique identifier for a user given their first and last name. \nThe tool requires two parameters:\n- firstName (string)\n- lastName (string)\nLooking at the user's request, they have provided both their first name (Alex) and last name (Yang). So we have all the required parameters to call the unique_id tool.\n</thinking>"
             }
           ],
           "role": "assistant",
           "options": {
             "toolCall": [
               {
-                "id": "toolu_012ZLUq5SWsghkXUjsEXsB1f",
+                "id": "toolu_01Q7wYKoNtpGAphTbRSv51fE",
                 "name": "unique_id",
                 "input": {
                   "firstName": "Alex",
@@ -244,14 +244,14 @@
           "content": [
             {
               "type": "text",
-              "text": "<thinking>\nThe user is asking to sum the numbers 1 and 1. The relevant tool is the sumNumbers function, which takes two number parameters a and b.\nThe user has directly provided the values for the required parameters:\na = 1 \nb = 1\nSince all the required parameters are provided, we can proceed with calling the function.\n</thinking>"
+              "text": "<thinking>\nThe user is asking to sum the numbers 1 and 1. The sumNumbers tool is directly relevant for this request.\n\nLooking at the required parameters for sumNumbers:\na (number): The user provided the value 1 for this\nb (number): The user also provided the value 1 for this\n\nSince the user has directly provided values for all the required parameters, we can proceed with calling the sumNumbers tool without needing any additional information from the user.\n</thinking>"
             }
           ],
           "role": "assistant",
           "options": {
             "toolCall": [
               {
-                "id": "toolu_01SQD2XkaXkDNLQ2xaFzmguG",
+                "id": "toolu_01AxEUd1s1UmhPc6Y91LBSmw",
                 "name": "sumNumbers",
                 "input": {
                   "a": 1,
@@ -271,7 +271,7 @@
           "content": [
             {
               "type": "text",
-              "text": "So 1 + 1 = 2."
+              "text": "So 1 + 1 = 2"
             }
           ],
           "role": "assistant",

packages/llamaindex/e2e/node/snapshot/gpt-4-turbo.snap

@@ -22,7 +22,7 @@
           "options": {
             "toolCall": [
               {
-                "id": "call_Xa2Kxa2zUE073mnougPWzRlh",
+                "id": "call_UMsqyh51lvDjy2JvMKFoyai3",
                 "name": "Weather",
                 "input": "{\"location\":\"San Jose\"}"
               }
@@ -30,13 +30,13 @@
           }
         },
         {
-          "role": "user",
           "content": "45 degrees and sunny in San Jose",
+          "role": "user",
           "options": {
             "toolResult": {
               "result": "45 degrees and sunny in San Jose",
               "isError": false,
-              "id": "call_Xa2Kxa2zUE073mnougPWzRlh"
+              "id": "call_UMsqyh51lvDjy2JvMKFoyai3"
             }
           }
         }
@@ -54,7 +54,7 @@
           "options": {
             "toolCall": [
               {
-                "id": "call_Xa2Kxa2zUE073mnougPWzRlh",
+                "id": "call_UMsqyh51lvDjy2JvMKFoyai3",
                 "name": "Weather",
                 "input": "{\"location\":\"San Jose\"}"
               }

packages/llamaindex/e2e/node/snapshot/llm-anthropic.snap

@@ -49,7 +49,7 @@
       "id": "PRESERVE_1",
       "chunk": {
         "raw": null,
-        "delta": "Hello",
+        "delta": "Hello! How can",
         "options": {}
       }
     },
@@ -57,55 +57,7 @@
       "id": "PRESERVE_1",
       "chunk": {
         "raw": null,
-        "delta": "!",
-        "options": {}
-      }
-    },
-    {
-      "id": "PRESERVE_1",
-      "chunk": {
-        "raw": null,
-        "delta": " How",
-        "options": {}
-      }
-    },
-    {
-      "id": "PRESERVE_1",
-      "chunk": {
-        "raw": null,
-        "delta": " can",
-        "options": {}
-      }
-    },
-    {
-      "id": "PRESERVE_1",
-      "chunk": {
-        "raw": null,
-        "delta": " I",
-        "options": {}
-      }
-    },
-    {
-      "id": "PRESERVE_1",
-      "chunk": {
-        "raw": null,
-        "delta": " assist",
-        "options": {}
-      }
-    },
-    {
-      "id": "PRESERVE_1",
-      "chunk": {
-        "raw": null,
-        "delta": " you",
-        "options": {}
-      }
-    },
-    {
-      "id": "PRESERVE_1",
-      "chunk": {
-        "raw": null,
-        "delta": " today",
+        "delta": " I assist you today",
         "options": {}
       }
     },

packages/llamaindex/e2e/node/snapshot/openai_agent_system_prompt.snap

@@ -30,7 +30,7 @@
           "options": {
             "toolCall": [
               {
-                "id": "call_1SlugFuJ7rhwsmXd5aRnJhPe",
+                "id": "call_i3rlYaBlvhTf60phYJh8irvZ",
                 "name": "getWeather",
                 "input": "{\"city\":\"San Francisco\"}"
               }
@@ -38,13 +38,13 @@
           }
         },
         {
-          "role": "user",
           "content": "The weather in San Francisco is 72 degrees",
+          "role": "user",
           "options": {
             "toolResult": {
               "result": "The weather in San Francisco is 72 degrees",
               "isError": false,
-              "id": "call_1SlugFuJ7rhwsmXd5aRnJhPe"
+              "id": "call_i3rlYaBlvhTf60phYJh8irvZ"
             }
           }
         }
@@ -62,7 +62,7 @@
           "options": {
             "toolCall": [
               {
-                "id": "call_1SlugFuJ7rhwsmXd5aRnJhPe",
+                "id": "call_i3rlYaBlvhTf60phYJh8irvZ",
                 "name": "getWeather",
                 "input": "{\"city\":\"San Francisco\"}"
               }

packages/llamaindex/e2e/node/snapshot/queryEngine_subquestion.snap

@@ -13,7 +13,7 @@
       "id": "PRESERVE_1",
       "messages": [
         {
-          "content": "Context information is below.\n---------------------\nBill Gates stole from Apple. Steve Jobs stole from Xerox.\n---------------------\nGiven the context information and not prior knowledge, answer the query.\nQuery: What is Bill Gates' idea\nAnswer:",
+          "content": "Context information is below.\n---------------------\nBill Gates stole from Apple.\n Steve Jobs stole from Xerox.\n---------------------\nGiven the context information and not prior knowledge, answer the query.\nQuery: What idea did Bill Gates get?\nAnswer:",
           "role": "user"
         }
       ]
@@ -22,7 +22,7 @@
       "id": "PRESERVE_2",
       "messages": [
         {
-          "content": "Context information is below.\n---------------------\nSub question: What is Bill Gates' idea\nResponse: Bill Gates' idea was to steal from Apple.\n---------------------\nGiven the context information and not prior knowledge, answer the query.\nQuery: What did Bill Gates steal from?\nAnswer:",
+          "content": "Context information is below.\n---------------------\nSub question: What idea did Bill Gates get?\nResponse: Bill Gates got the idea from Apple.\n---------------------\nGiven the context information and not prior knowledge, answer the query.\nQuery: What did Bill Gates steal from?\nAnswer:",
           "role": "user"
         }
       ]
@@ -34,7 +34,7 @@
       "response": {
         "raw": null,
         "message": {
-          "content": "```json\n[\n    {\n        \"subQuestion\": \"What is Bill Gates' idea\",\n        \"toolName\": \"bill_gates_idea\"\n    }\n]\n```",
+          "content": "```json\n[\n    {\n        \"subQuestion\": \"What idea did Bill Gates get?\",\n        \"toolName\": \"bill_gates_idea\"\n    }\n]\n```",
           "role": "assistant",
           "options": {}
         }
@@ -45,7 +45,7 @@
       "response": {
         "raw": null,
         "message": {
-          "content": "Bill Gates' idea was to steal from Apple.",
+          "content": "Bill Gates got the idea from Apple.",
           "role": "assistant",
           "options": {}
         }
@@ -56,7 +56,7 @@
       "response": {
         "raw": null,
         "message": {
-          "content": "Bill Gates stole from Apple.",
+          "content": "Bill Gates stole the idea from Apple.",
           "role": "assistant",
           "options": {}
         }

packages/llamaindex/e2e/node/snapshot/react-agent-stream.snap

@@ -41,7 +41,7 @@
       "response": {
         "raw": null,
         "message": {
-          "content": "Thought: I need to use a tool to help me answer the question. \nAction: getWeather\nAction Input: {\"city\": \"San Francisco\"}",
+          "content": "Thought: I need to use a tool to help me answer the question.\nAction: getWeather\nAction Input: {\"city\": \"San Francisco\"}",
           "role": "assistant",
           "options": {}
         }
@@ -177,15 +177,7 @@
       "chunk": {
         "raw": null,
         "options": {},
-        "delta": "."
-      }
-    },
-    {
-      "id": "PRESERVE_0",
-      "chunk": {
-        "raw": null,
-        "options": {},
-        "delta": " \n"
+        "delta": ".\n"
       }
     },
     {

packages/llamaindex/e2e/node/utils.ts

@@ -42,7 +42,7 @@ export async function mockLLMEvent(
     newLLMCompleteMockStorage.llmEventStart.push({
       ...event.detail,
       // @ts-expect-error id is not UUID, but it is fine for testing
-      id: idMap.get(event.detail.payload.id)!,
+      id: idMap.get(event.detail.id)!,
     });
   }
 
@@ -50,7 +50,7 @@ export async function mockLLMEvent(
     newLLMCompleteMockStorage.llmEventEnd.push({
       ...event.detail,
       // @ts-expect-error id is not UUID, but it is fine for testing
-      id: idMap.get(event.detail.payload.id)!,
+      id: idMap.get(event.detail.id)!,
       response: {
         ...event.detail.response,
         // hide raw object since it might too big
@@ -63,7 +63,7 @@ export async function mockLLMEvent(
     newLLMCompleteMockStorage.llmEventStream.push({
       ...event.detail,
       // @ts-expect-error id is not UUID, but it is fine for testing
-      id: idMap.get(event.detail.payload.id)!,
+      id: idMap.get(event.detail.id)!,
       chunk: {
         ...event.detail.chunk,
         // hide raw object since it might too big

packages/llamaindex/package.json

@@ -1,6 +1,6 @@
 {
   "name": "llamaindex",
-  "version": "0.5.6",
+  "version": "0.5.9",
   "license": "MIT",
   "type": "module",
   "keywords": [

packages/llamaindex/src/PromptHelper.ts

@@ -1,6 +1,6 @@
-import { tokenizers, type Tokenizer } from "@llamaindex/env";
+import { SentenceSplitter } from "@llamaindex/core/node-parser";
+import { type Tokenizer, tokenizers } from "@llamaindex/env";
 import type { SimplePrompt } from "./Prompt.js";
-import { SentenceSplitter } from "./TextSplitter.js";
 import {
   DEFAULT_CHUNK_OVERLAP_RATIO,
   DEFAULT_CONTEXT_WINDOW,
@@ -107,8 +107,7 @@ export class PromptHelper {
       throw new Error("Got 0 as available chunk size");
     }
     const chunkOverlap = this.chunkOverlapRatio * chunkSize;
-    const textSplitter = new SentenceSplitter({ chunkSize, chunkOverlap });
-    return textSplitter;
+    return new SentenceSplitter({ chunkSize, chunkOverlap });
   }
 
   /**

packages/llamaindex/src/ServiceContext.ts

@@ -1,10 +1,12 @@
 import type { BaseEmbedding } from "@llamaindex/core/embeddings";
 import type { LLM } from "@llamaindex/core/llms";
+import {
+  type NodeParser,
+  SentenceSplitter,
+} from "@llamaindex/core/node-parser";
 import { PromptHelper } from "./PromptHelper.js";
 import { OpenAIEmbedding } from "./embeddings/OpenAIEmbedding.js";
 import { OpenAI } from "./llm/openai.js";
-import { SimpleNodeParser } from "./nodeParsers/SimpleNodeParser.js";
-import type { NodeParser } from "./nodeParsers/types.js";
 
 /**
  * The ServiceContext is a collection of components that are used in different parts of the application.
@@ -33,7 +35,7 @@ export function serviceContextFromDefaults(options?: ServiceContextOptions) {
     embedModel: options?.embedModel ?? new OpenAIEmbedding(),
     nodeParser:
       options?.nodeParser ??
-      new SimpleNodeParser({
+      new SentenceSplitter({
         chunkSize: options?.chunkSize,
         chunkOverlap: options?.chunkOverlap,
       }),

packages/llamaindex/src/Settings.ts

@@ -5,10 +5,13 @@ import {
 import { OpenAI } from "./llm/openai.js";
 
 import { PromptHelper } from "./PromptHelper.js";
-import { SimpleNodeParser } from "./nodeParsers/SimpleNodeParser.js";
 
 import type { BaseEmbedding } from "@llamaindex/core/embeddings";
 import type { LLM } from "@llamaindex/core/llms";
+import {
+  type NodeParser,
+  SentenceSplitter,
+} from "@llamaindex/core/node-parser";
 import { AsyncLocalStorage, getEnv } from "@llamaindex/env";
 import type { ServiceContext } from "./ServiceContext.js";
 import {
@@ -16,7 +19,6 @@ import {
   setEmbeddedModel,
   withEmbeddedModel,
 } from "./internal/settings/EmbedModel.js";
-import type { NodeParser } from "./nodeParsers/types.js";
 
 export type PromptConfig = {
   llm?: string;
@@ -108,7 +110,7 @@ class GlobalSettings implements Config {
 
   get nodeParser(): NodeParser {
     if (this.#nodeParser === null) {
-      this.#nodeParser = new SimpleNodeParser({
+      this.#nodeParser = new SentenceSplitter({
         chunkSize: this.chunkSize,
         chunkOverlap: this.chunkOverlap,
       });

packages/llamaindex/src/TextSplitter.ts

@@ -1,309 +0,0 @@
-import { EOL, tokenizers, type Tokenizer } from "@llamaindex/env";
-// GitHub translated
-import { DEFAULT_CHUNK_OVERLAP, DEFAULT_CHUNK_SIZE } from "./constants.js";
-
-class TextSplit {
-  textChunk: string;
-  numCharOverlap: number | undefined;
-
-  constructor(
-    textChunk: string,
-    numCharOverlap: number | undefined = undefined,
-  ) {
-    this.textChunk = textChunk;
-    this.numCharOverlap = numCharOverlap;
-  }
-}
-
-type SplitRep = { text: string; numTokens: number };
-
-const defaultregex = /[.?!][\])'"`’”]*(?:\s|$)/g;
-export const defaultSentenceTokenizer = (text: string): string[] => {
-  const slist = [];
-  const iter = text.matchAll(defaultregex);
-  let lastIdx = 0;
-  for (const match of iter) {
-    slist.push(text.slice(lastIdx, match.index! + 1));
-    lastIdx = match.index! + 1;
-  }
-  slist.push(text.slice(lastIdx));
-  return slist.filter((s) => s.length > 0);
-};
-
-// Refs: https://github.com/fxsjy/jieba/issues/575#issuecomment-359637511
-const resentencesp =
-  /([﹒﹔﹖﹗．；。！？]["’”」』]{0,2}|：(?=["‘“「『]{1,2}|$))/;
-/**
- * Tokenizes sentences. Suitable for Chinese, Japanese, and Korean. Use instead of `defaultSentenceTokenizer`.
- * @param text
- * @returns string[]
- */
-export function cjkSentenceTokenizer(sentence: string): string[] {
-  const slist = [];
-  const parts = sentence.split(resentencesp);
-
-  for (let i = 0; i < parts.length; i++) {
-    const part = parts[i];
-    if (resentencesp.test(part) && slist.length > 0) {
-      slist[slist.length - 1] += part;
-    } else if (part) {
-      slist.push(part);
-    }
-  }
-
-  return slist.filter((s) => s.length > 0);
-}
-
-export const defaultParagraphSeparator = EOL + EOL + EOL;
-
-// In theory there's also Mac style \r only, but it's pre-OSX and I don't think
-// many documents will use it.
-
-/**
- * SentenceSplitter is our default text splitter that supports splitting into sentences, paragraphs, or fixed length chunks with overlap.
- *
- * One of the advantages of SentenceSplitter is that even in the fixed length chunks it will try to keep sentences together.
- */
-export class SentenceSplitter {
-  public chunkSize: number;
-  public chunkOverlap: number;
-
-  private tokenizer: Tokenizer;
-  private paragraphSeparator: string;
-  private chunkingTokenizerFn: (text: string) => string[];
-  private splitLongSentences: boolean;
-
-  constructor(options?: {
-    chunkSize?: number;
-    chunkOverlap?: number;
-    tokenizer?: Tokenizer;
-    paragraphSeparator?: string;
-    chunkingTokenizerFn?: (text: string) => string[];
-    splitLongSentences?: boolean;
-  }) {
-    const {
-      chunkSize = DEFAULT_CHUNK_SIZE,
-      chunkOverlap = DEFAULT_CHUNK_OVERLAP,
-      tokenizer = null,
-      paragraphSeparator = defaultParagraphSeparator,
-      chunkingTokenizerFn,
-      splitLongSentences = false,
-    } = options ?? {};
-
-    if (chunkOverlap > chunkSize) {
-      throw new Error(
-        `Got a larger chunk overlap (${chunkOverlap}) than chunk size (${chunkSize}), should be smaller.`,
-      );
-    }
-    this.chunkSize = chunkSize;
-    this.chunkOverlap = chunkOverlap;
-
-    this.tokenizer = tokenizer ?? tokenizers.tokenizer();
-
-    this.paragraphSeparator = paragraphSeparator;
-    this.chunkingTokenizerFn = chunkingTokenizerFn ?? defaultSentenceTokenizer;
-    this.splitLongSentences = splitLongSentences;
-  }
-
-  private getEffectiveChunkSize(extraInfoStr?: string): number {
-    // get "effective" chunk size by removing the metadata
-    let effectiveChunkSize;
-    if (extraInfoStr != undefined) {
-      const numExtraTokens =
-        this.tokenizer.encode(`${extraInfoStr}\n\n`).length + 1;
-      effectiveChunkSize = this.chunkSize - numExtraTokens;
-      if (effectiveChunkSize <= 0) {
-        throw new Error(
-          "Effective chunk size is non positive after considering extra_info",
-        );
-      }
-    } else {
-      effectiveChunkSize = this.chunkSize;
-    }
-    return effectiveChunkSize;
-  }
-
-  getParagraphSplits(text: string, effectiveChunkSize?: number): string[] {
-    // get paragraph splits
-    const paragraphSplits: string[] = text.split(this.paragraphSeparator);
-    let idx = 0;
-    if (effectiveChunkSize == undefined) {
-      return paragraphSplits;
-    }
-
-    // merge paragraphs that are too small
-    while (idx < paragraphSplits.length) {
-      if (
-        idx < paragraphSplits.length - 1 &&
-        paragraphSplits[idx].length < effectiveChunkSize
-      ) {
-        paragraphSplits[idx] = [
-          paragraphSplits[idx],
-          paragraphSplits[idx + 1],
-        ].join(this.paragraphSeparator);
-        paragraphSplits.splice(idx + 1, 1);
-      } else {
-        idx += 1;
-      }
-    }
-    return paragraphSplits;
-  }
-
-  getSentenceSplits(text: string, effectiveChunkSize?: number): string[] {
-    const paragraphSplits = this.getParagraphSplits(text, effectiveChunkSize);
-    // Next we split the text using the chunk tokenizer fn/
-    const splits = [];
-    for (const parText of paragraphSplits) {
-      const sentenceSplits = this.chunkingTokenizerFn(parText);
-
-      if (!sentenceSplits) {
-        continue;
-      }
-
-      for (const sentence_split of sentenceSplits) {
-        splits.push(sentence_split.trim());
-      }
-    }
-    return splits;
-  }
-
-  /**
-   * Splits sentences into chunks if necessary.
-   *
-   * This isn't great behavior because it can split down the middle of a
-   * word or in non-English split down the middle of a Unicode codepoint
-   * so the splitting is turned off by default. If you need it, please
-   * set the splitLongSentences option to true.
-   * @param sentenceSplits
-   * @param effectiveChunkSize
-   * @returns
-   */
-  private processSentenceSplits(
-    sentenceSplits: string[],
-    effectiveChunkSize: number,
-  ): SplitRep[] {
-    if (!this.splitLongSentences) {
-      return sentenceSplits.map((split) => ({
-        text: split,
-        numTokens: this.tokenizer.encode(split).length,
-      }));
-    }
-
-    const newSplits: SplitRep[] = [];
-    for (const split of sentenceSplits) {
-      const splitTokens = this.tokenizer.encode(split);
-      const splitLen = splitTokens.length;
-      if (splitLen <= effectiveChunkSize) {
-        newSplits.push({ text: split, numTokens: splitLen });
-      } else {
-        for (let i = 0; i < splitLen; i += effectiveChunkSize) {
-          const cur_split = this.tokenizer.decode(
-            splitTokens.slice(i, i + effectiveChunkSize),
-          );
-          newSplits.push({ text: cur_split, numTokens: effectiveChunkSize });
-        }
-      }
-    }
-    return newSplits;
-  }
-
-  combineTextSplits(
-    newSentenceSplits: SplitRep[],
-    effectiveChunkSize: number,
-  ): TextSplit[] {
-    // go through sentence splits, combine to chunks that are within the chunk size
-
-    // docs represents final list of text chunks
-    const docs: TextSplit[] = [];
-    // curChunkSentences represents the current list of sentence splits (that)
-    // will be merged into a chunk
-    let curChunkSentences: SplitRep[] = [];
-    let curChunkTokens = 0;
-
-    for (let i = 0; i < newSentenceSplits.length; i++) {
-      // if adding newSentenceSplits[i] to curDocBuffer would exceed effectiveChunkSize,
-      // then we need to add the current curDocBuffer to docs
-      if (
-        curChunkTokens + newSentenceSplits[i].numTokens >
-        effectiveChunkSize
-      ) {
-        if (curChunkSentences.length > 0) {
-          // push curent doc list to docs
-          docs.push(
-            new TextSplit(
-              curChunkSentences
-                .map((sentence) => sentence.text)
-                .join(" ")
-                .trim(),
-            ),
-          );
-        }
-
-        const lastChunkSentences = curChunkSentences;
-
-        // reset docs list
-        curChunkTokens = 0;
-        curChunkSentences = [];
-
-        // add the last sentences from the last chunk until we've hit the overlap
-        // do it in reverse order
-        for (let j = lastChunkSentences.length - 1; j >= 0; j--) {
-          if (
-            curChunkTokens + lastChunkSentences[j].numTokens >
-            this.chunkOverlap
-          ) {
-            break;
-          }
-          curChunkSentences.unshift(lastChunkSentences[j]);
-          curChunkTokens += lastChunkSentences[j].numTokens + 1;
-        }
-      }
-
-      curChunkSentences.push(newSentenceSplits[i]);
-      curChunkTokens += newSentenceSplits[i].numTokens + 1;
-    }
-    docs.push(
-      new TextSplit(
-        curChunkSentences
-          .map((sentence) => sentence.text)
-          .join(" ")
-          .trim(),
-      ),
-    );
-    return docs;
-  }
-
-  splitTextWithOverlaps(text: string, extraInfoStr?: string): TextSplit[] {
-    // Split incoming text and return chunks with overlap size.
-    // Has a preference for complete sentences, phrases, and minimal overlap.
-
-    // here is the typescript code (skip callback manager)
-    if (text == "") {
-      return [];
-    }
-
-    const effectiveChunkSize = this.getEffectiveChunkSize(extraInfoStr);
-    const sentenceSplits = this.getSentenceSplits(text, effectiveChunkSize);
-
-    // Check if any sentences exceed the chunk size. If they don't,
-    // force split by tokenizer
-    const newSentenceSplits = this.processSentenceSplits(
-      sentenceSplits,
-      effectiveChunkSize,
-    );
-
-    // combine sentence splits into chunks of text that can then be returned
-    const combinedTextSplits = this.combineTextSplits(
-      newSentenceSplits,
-      effectiveChunkSize,
-    );
-
-    return combinedTextSplits;
-  }
-
-  splitText(text: string, extraInfoStr?: string): string[] {
-    const text_splits = this.splitTextWithOverlaps(text);
-    const chunks = text_splits.map((text_split) => text_split.textChunk);
-    return chunks;
-  }
-}

packages/llamaindex/src/cloud/LlamaCloudIndex.ts

@@ -11,10 +11,10 @@ import type { CloudConstructorParams } from "./constants.js";
 import { getAppBaseUrl, initService } from "./utils.js";
 
 import { PipelinesService, ProjectsService } from "@llamaindex/cloud/api";
+import { SentenceSplitter } from "@llamaindex/core/node-parser";
 import { getEnv } from "@llamaindex/env";
 import { Settings } from "../Settings.js";
 import { OpenAIEmbedding } from "../embeddings/OpenAIEmbedding.js";
-import { SimpleNodeParser } from "../nodeParsers/SimpleNodeParser.js";
 
 export class LlamaCloudIndex {
   params: CloudConstructorParams;
@@ -147,13 +147,13 @@ export class LlamaCloudIndex {
   static async fromDocuments(
     params: {
       documents: Document[];
-      transformations?: TransformComponent<any>[];
+      transformations?: TransformComponent[];
       verbose?: boolean;
     } & CloudConstructorParams,
   ): Promise<LlamaCloudIndex> {
     initService(params);
-    const defaultTransformations: TransformComponent<any>[] = [
-      new SimpleNodeParser(),
+    const defaultTransformations: TransformComponent[] = [
+      new SentenceSplitter(),
       new OpenAIEmbedding({
         apiKey: getEnv("OPENAI_API_KEY"),
       }),

packages/llamaindex/src/cloud/config.ts

@@ -3,26 +3,26 @@ import type {
   PipelineCreate,
   PipelineType,
 } from "@llamaindex/cloud/api";
+import { SentenceSplitter } from "@llamaindex/core/node-parser";
 import { BaseNode, type TransformComponent } from "@llamaindex/core/schema";
 import { OpenAIEmbedding } from "../embeddings/OpenAIEmbedding.js";
-import { SimpleNodeParser } from "../nodeParsers/SimpleNodeParser.js";
 
 export type GetPipelineCreateParams = {
   pipelineName: string;
   pipelineType: PipelineType;
-  transformations?: TransformComponent<any>[];
+  transformations?: TransformComponent[];
   inputNodes?: BaseNode[];
 };
 
 function getTransformationConfig(
-  transformation: TransformComponent<any>,
+  transformation: TransformComponent,
 ): ConfiguredTransformationItem {
-  if (transformation instanceof SimpleNodeParser) {
+  if (transformation instanceof SentenceSplitter) {
     return {
       configurable_transformation_type: "SENTENCE_AWARE_NODE_PARSER",
       component: {
-        chunk_size: transformation.textSplitter.chunkSize, // TODO: set to public in SentenceSplitter
-        chunk_overlap: transformation.textSplitter.chunkOverlap, // TODO: set to public in SentenceSplitter
+        chunk_size: transformation.chunkSize, // TODO: set to public in SentenceSplitter
+        chunk_overlap: transformation.chunkOverlap, // TODO: set to public in SentenceSplitter
         include_metadata: transformation.includeMetadata,
         include_prev_next_rel: transformation.includePrevNextRel,
       },

packages/llamaindex/src/extractors/types.ts

@@ -5,7 +5,7 @@ import { defaultNodeTextTemplate } from "./prompts.js";
 /*
  * Abstract class for all extractors.
  */
-export abstract class BaseExtractor implements TransformComponent<any> {
+export abstract class BaseExtractor implements TransformComponent {
   isTextNodeOnly: boolean = true;
   showProgress: boolean = true;
   metadataMode: MetadataMode = MetadataMode.ALL;

packages/llamaindex/src/index.edge.ts

@@ -50,6 +50,5 @@ export * from "./ServiceContext.js";
 export { Settings } from "./Settings.js";
 export * from "./storage/StorageContext.js";
 export * from "./synthesizers/index.js";
-export * from "./TextSplitter.js";
 export * from "./tools/index.js";
 export * from "./types.js";

packages/llamaindex/src/ingestion/IngestionCache.ts

@@ -5,7 +5,7 @@ import { docToJson, jsonToDoc } from "../storage/docStore/utils.js";
 import { SimpleKVStore } from "../storage/kvStore/SimpleKVStore.js";
 import type { BaseKVStore } from "../storage/kvStore/types.js";
 
-const transformToJSON = (obj: TransformComponent<any>) => {
+const transformToJSON = (obj: TransformComponent) => {
   const seen: any[] = [];
 
   const replacer = (key: string, value: any) => {
@@ -26,7 +26,7 @@ const transformToJSON = (obj: TransformComponent<any>) => {
 
 export function getTransformationHash(
   nodes: BaseNode[],
-  transform: TransformComponent<any>,
+  transform: TransformComponent,
 ) {
   const nodesStr: string = nodes
     .map((node) => node.getContent(MetadataMode.ALL))

packages/llamaindex/src/ingestion/IngestionPipeline.ts

@@ -26,12 +26,12 @@ type IngestionRunArgs = {
 type TransformRunArgs = {
   inPlace?: boolean;
   cache?: IngestionCache;
-  docStoreStrategy?: TransformComponent<any>;
+  docStoreStrategy?: TransformComponent;
 };
 
 export async function runTransformations(
   nodesToRun: BaseNode[],
-  transformations: TransformComponent<any>[],
+  transformations: TransformComponent[],
   transformOptions: any = {},
   { inPlace = true, cache, docStoreStrategy }: TransformRunArgs = {},
 ): Promise<BaseNode[]> {
@@ -60,7 +60,7 @@ export async function runTransformations(
 }
 
 export class IngestionPipeline {
-  transformations: TransformComponent<any>[] = [];
+  transformations: TransformComponent[] = [];
   documents?: Document[];
   reader?: BaseReader;
   vectorStore?: VectorStore;
@@ -70,7 +70,7 @@ export class IngestionPipeline {
   cache?: IngestionCache;
   disableCache: boolean = false;
 
-  private _docStoreStrategy?: TransformComponent<any>;
+  private _docStoreStrategy?: TransformComponent;
 
   constructor(init?: Partial<IngestionPipeline>) {
     Object.assign(this, init);

packages/llamaindex/src/ingestion/strategies/DuplicatesStrategy.ts

@@ -4,7 +4,7 @@ import type { BaseDocumentStore } from "../../storage/docStore/types.js";
 /**
  * Handle doc store duplicates by checking all hashes.
  */
-export class DuplicatesStrategy implements TransformComponent<any> {
+export class DuplicatesStrategy implements TransformComponent {
   private docStore: BaseDocumentStore;
 
   constructor(docStore: BaseDocumentStore) {

packages/llamaindex/src/ingestion/strategies/UpsertsAndDeleteStrategy.ts

@@ -7,7 +7,7 @@ import { classify } from "./classify.js";
  * Handle docstore upserts by checking hashes and ids.
  * Identify missing docs and delete them from docstore and vector store
  */
-export class UpsertsAndDeleteStrategy implements TransformComponent<any> {
+export class UpsertsAndDeleteStrategy implements TransformComponent {
   protected docStore: BaseDocumentStore;
   protected vectorStores?: VectorStore[];
 

packages/llamaindex/src/ingestion/strategies/UpsertsStrategy.ts

@@ -6,7 +6,7 @@ import { classify } from "./classify.js";
 /**
  * Handles doc store upserts by checking hashes and ids.
  */
-export class UpsertsStrategy implements TransformComponent<any> {
+export class UpsertsStrategy implements TransformComponent {
   protected docStore: BaseDocumentStore;
   protected vectorStores?: VectorStore[];
 

packages/llamaindex/src/ingestion/strategies/index.ts

@@ -19,7 +19,7 @@ export enum DocStoreStrategy {
   NONE = "none", // no-op strategy
 }
 
-class NoOpStrategy implements TransformComponent<any> {
+class NoOpStrategy implements TransformComponent {
   async transform(nodes: any[]): Promise<any[]> {
     return nodes;
   }
@@ -29,7 +29,7 @@ export function createDocStoreStrategy(
   docStoreStrategy: DocStoreStrategy,
   docStore?: BaseDocumentStore,
   vectorStores: VectorStore[] = [],
-): TransformComponent<any> {
+): TransformComponent {
   if (docStoreStrategy === DocStoreStrategy.NONE) {
     return new NoOpStrategy();
   }

packages/llamaindex/src/next.ts

@@ -32,6 +32,12 @@ export default function withLlamaIndex(config: any) {
       "@google-cloud/vertexai": false,
       "groq-sdk": false,
     };
+    // Following lines will fix issues with onnxruntime-node when using pnpm
+    // See: https://github.com/vercel/next.js/issues/43433
+    webpackConfig.externals.push({
+      "onnxruntime-node": "commonjs onnxruntime-node",
+      sharp: "commonjs sharp",
+    });
     return webpackConfig;
   };
   return config;

packages/llamaindex/src/nodeParsers/MarkdownNodeParser.ts

@@ -1,109 +0,0 @@
-import type { BaseNode, Metadata } from "@llamaindex/core/schema";
-import { MetadataMode, TextNode } from "@llamaindex/core/schema";
-import type { NodeParser } from "./types.js";
-
-export class MarkdownNodeParser implements NodeParser {
-  includeMetadata: boolean;
-  includePrevNextRel: boolean;
-
-  constructor(init?: {
-    includeMetadata?: boolean;
-    includePrevNextRel?: boolean;
-  }) {
-    this.includeMetadata = init?.includeMetadata ?? true;
-    this.includePrevNextRel = init?.includePrevNextRel ?? true;
-  }
-
-  async transform(nodes: BaseNode[], _options?: any): Promise<BaseNode[]> {
-    return this.getNodesFromDocuments(nodes);
-  }
-
-  static fromDefaults(init?: {
-    includeMetadata?: boolean;
-    includePrevNextRel?: boolean;
-  }): MarkdownNodeParser {
-    return new MarkdownNodeParser(init);
-  }
-
-  buildNodeFromSplit(
-    textSplit: string,
-    node: BaseNode<Metadata>,
-    metadata: Metadata,
-  ): BaseNode<Metadata> {
-    const newNode = new TextNode({
-      text: textSplit,
-      relationships: {
-        PARENT: [
-          {
-            ...node,
-            nodeId: node.id_,
-          },
-        ],
-      },
-      metadata: this.includeMetadata ? metadata : {},
-    });
-    return newNode;
-  }
-
-  updateMetadata(
-    headersMetadata: Metadata,
-    newHeader: string,
-    newHeaderLevel: number,
-  ): Metadata {
-    const updatedHeaders: Metadata = {};
-    for (let i = 1; i < newHeaderLevel; i++) {
-      const key = `Header ${i}`;
-      if (key in headersMetadata) {
-        updatedHeaders[key] = headersMetadata[key];
-      }
-    }
-    updatedHeaders[`Header ${newHeaderLevel}`] = newHeader;
-    return updatedHeaders;
-  }
-
-  getNodesFromNode(node: BaseNode<Metadata>): BaseNode<Metadata>[] {
-    const text = node.getContent(MetadataMode.NONE);
-    const markdownNodes: BaseNode<Metadata>[] = [];
-    const lines = text.split("\n");
-    let metadata: Metadata = {};
-    let codeBlock = false;
-    let currentSection = "";
-
-    for (const line of lines) {
-      if (line.startsWith("```")) {
-        codeBlock = !codeBlock;
-      }
-      const headerMatch = line.match(/^(#+)\s(.*)/);
-      if (headerMatch && !codeBlock) {
-        if (currentSection !== "") {
-          markdownNodes.push(
-            this.buildNodeFromSplit(currentSection.trim(), node, metadata),
-          );
-        }
-        metadata = this.updateMetadata(
-          metadata,
-          headerMatch[2],
-          headerMatch[1].length,
-        );
-        currentSection = `${headerMatch[2]}\n`;
-      } else {
-        currentSection += line + "\n";
-      }
-    }
-
-    markdownNodes.push(
-      this.buildNodeFromSplit(currentSection.trim(), node, metadata),
-    );
-
-    return markdownNodes;
-  }
-
-  getNodesFromDocuments(documents: BaseNode<Metadata>[]): BaseNode<Metadata>[] {
-    let allNodes: BaseNode<Metadata>[] = [];
-    for (const node of documents) {
-      const nodes = this.getNodesFromNode(node);
-      allNodes = allNodes.concat(nodes);
-    }
-    return allNodes;
-  }
-}

packages/llamaindex/src/nodeParsers/SentenceWindowNodeParser.ts

@@ -1,89 +0,0 @@
-import type { BaseNode } from "@llamaindex/core/schema";
-import { SentenceSplitter } from "../TextSplitter.js";
-import type { NodeParser } from "./types.js";
-import { getNodesFromDocument } from "./utils.js";
-
-export const DEFAULT_WINDOW_SIZE = 3;
-export const DEFAULT_WINDOW_METADATA_KEY = "window";
-export const DEFAULT_OG_TEXT_METADATA_KEY = "original_text";
-
-export class SentenceWindowNodeParser implements NodeParser {
-  /**
-   * The text splitter to use.
-   */
-  textSplitter: SentenceSplitter;
-  /**
-   * The number of sentences on each side of a sentence to capture.
-   */
-  windowSize: number = DEFAULT_WINDOW_SIZE;
-  /**
-   * The metadata key to store the sentence window under.
-   */
-  windowMetadataKey: string = DEFAULT_WINDOW_METADATA_KEY;
-  /**
-   * The metadata key to store the original sentence in.
-   */
-  originalTextMetadataKey: string = DEFAULT_OG_TEXT_METADATA_KEY;
-  /**
-   * Whether to include metadata in the nodes.
-   */
-  includeMetadata: boolean = true;
-  /**
-   * Whether to include previous and next relationships in the nodes.
-   */
-  includePrevNextRel: boolean = true;
-
-  constructor(init?: Partial<SentenceWindowNodeParser>) {
-    Object.assign(this, init);
-    this.textSplitter = init?.textSplitter ?? new SentenceSplitter();
-  }
-
-  static fromDefaults(
-    init?: Partial<SentenceWindowNodeParser>,
-  ): SentenceWindowNodeParser {
-    return new SentenceWindowNodeParser(init);
-  }
-
-  async transform(nodes: BaseNode[], _options?: any): Promise<BaseNode[]> {
-    return this.getNodesFromDocuments(nodes);
-  }
-
-  getNodesFromDocuments(documents: BaseNode[]) {
-    return documents
-      .map((document) => this.buildWindowNodesFromDocument(document))
-      .flat();
-  }
-
-  protected buildWindowNodesFromDocument(doc: BaseNode): BaseNode[] {
-    const nodes = getNodesFromDocument(
-      doc,
-      this.textSplitter.getSentenceSplits.bind(this.textSplitter),
-      this.includeMetadata,
-      this.includePrevNextRel,
-    );
-
-    for (let i = 0; i < nodes.length; i++) {
-      const node = nodes[i];
-      const windowNodes = nodes.slice(
-        Math.max(0, i - this.windowSize),
-        Math.min(i + this.windowSize + 1, nodes.length),
-      );
-
-      node.metadata[this.windowMetadataKey] = windowNodes
-        .map((n) => n.getText())
-        .join(" ");
-      node.metadata[this.originalTextMetadataKey] = node.getText();
-
-      node.excludedEmbedMetadataKeys.push(
-        this.windowMetadataKey,
-        this.originalTextMetadataKey,
-      );
-      node.excludedLlmMetadataKeys.push(
-        this.windowMetadataKey,
-        this.originalTextMetadataKey,
-      );
-    }
-
-    return nodes;
-  }
-}

packages/llamaindex/src/nodeParsers/SimpleNodeParser.ts

@@ -1,72 +0,0 @@
-import type { BaseNode } from "@llamaindex/core/schema";
-import { SentenceSplitter } from "../TextSplitter.js";
-import { DEFAULT_CHUNK_OVERLAP, DEFAULT_CHUNK_SIZE } from "../constants.js";
-import type { NodeParser } from "./types.js";
-import { getNodesFromDocument } from "./utils.js";
-
-/**
- * SimpleNodeParser is the default NodeParser. It splits documents into TextNodes using a splitter, by default SentenceSplitter
- */
-export class SimpleNodeParser implements NodeParser {
-  /**
-   * The text splitter to use.
-   */
-  textSplitter: SentenceSplitter;
-  /**
-   * Whether to include metadata in the nodes.
-   */
-  includeMetadata: boolean;
-  /**
-   * Whether to include previous and next relationships in the nodes.
-   */
-  includePrevNextRel: boolean;
-
-  constructor(init?: {
-    textSplitter?: SentenceSplitter;
-    includeMetadata?: boolean;
-    includePrevNextRel?: boolean;
-    chunkSize?: number;
-    chunkOverlap?: number;
-    splitLongSentences?: boolean;
-  }) {
-    this.textSplitter =
-      init?.textSplitter ??
-      new SentenceSplitter({
-        chunkSize: init?.chunkSize ?? DEFAULT_CHUNK_SIZE,
-        chunkOverlap: init?.chunkOverlap ?? DEFAULT_CHUNK_OVERLAP,
-        splitLongSentences: init?.splitLongSentences ?? false,
-      });
-    this.includeMetadata = init?.includeMetadata ?? true;
-    this.includePrevNextRel = init?.includePrevNextRel ?? true;
-  }
-
-  async transform(nodes: BaseNode[], _options?: any): Promise<BaseNode[]> {
-    return this.getNodesFromDocuments(nodes);
-  }
-
-  static fromDefaults(init?: {
-    chunkSize?: number;
-    chunkOverlap?: number;
-    includeMetadata?: boolean;
-    includePrevNextRel?: boolean;
-  }): SimpleNodeParser {
-    return new SimpleNodeParser(init);
-  }
-
-  /**
-   * Generate Node objects from documents
-   * @param documents
-   */
-  getNodesFromDocuments(documents: BaseNode[]) {
-    return documents
-      .map((document) =>
-        getNodesFromDocument(
-          document,
-          this.textSplitter.splitText.bind(this.textSplitter),
-          this.includeMetadata,
-          this.includePrevNextRel,
-        ),
-      )
-      .flat();
-  }
-}

packages/llamaindex/src/nodeParsers/index.ts

@@ -1,4 +1 @@
-export * from "./MarkdownNodeParser.js";
-export * from "./SentenceWindowNodeParser.js";
-export * from "./SimpleNodeParser.js";
-export * from "./types.js";
+export * from "@llamaindex/core/node-parser";

packages/llamaindex/src/nodeParsers/types.ts

@@ -1,13 +0,0 @@
-import type { BaseNode, TransformComponent } from "@llamaindex/core/schema";
-
-/**
- * A NodeParser generates Nodes from Documents
- */
-export interface NodeParser extends TransformComponent<any> {
-  /**
-   * Generates an array of nodes from an array of documents.
-   * @param documents - The documents to generate nodes from.
-   * @returns An array of nodes.
-   */
-  getNodesFromDocuments(documents: BaseNode[]): BaseNode[];
-}

packages/llamaindex/src/nodeParsers/utils.ts

@@ -1,79 +0,0 @@
-import type { BaseNode } from "@llamaindex/core/schema";
-import {
-  Document,
-  ImageDocument,
-  NodeRelationship,
-  TextNode,
-} from "@llamaindex/core/schema";
-import _ from "lodash";
-
-type TextSplitter = (s: string) => string[];
-
-/**
- * Splits the text of a document into smaller parts.
- * @param document - The document to split.
- * @param textSplitter - The text splitter to use.
- * @returns An array of text splits.
- */
-function getTextSplitsFromDocument(
-  document: Document,
-  textSplitter: TextSplitter,
-) {
-  const text = document.getText();
-  return textSplitter(text);
-}
-
-/**
- * Generates an array of nodes from a document.
- * @param doc
- * @param textSplitter - The text splitter to use.
- * @param includeMetadata - Whether to include metadata in the nodes.
- * @param includePrevNextRel - Whether to include previous and next relationships in the nodes.
- * @returns An array of nodes.
- */
-export function getNodesFromDocument(
-  doc: BaseNode,
-  textSplitter: TextSplitter,
-  includeMetadata: boolean = true,
-  includePrevNextRel: boolean = true,
-): TextNode[] {
-  if (doc instanceof ImageDocument) {
-    // TODO: use text splitter on text of image documents
-    return [doc];
-  }
-  if (!(doc instanceof Document)) {
-    throw new Error("Expected either an Image Document or Document");
-  }
-  const document = doc as Document;
-  const nodes: TextNode[] = [];
-
-  const textSplits = getTextSplitsFromDocument(document, textSplitter);
-
-  textSplits.forEach((textSplit) => {
-    const node = new TextNode({
-      text: textSplit,
-      metadata: includeMetadata ? _.cloneDeep(document.metadata) : {},
-      excludedEmbedMetadataKeys: _.cloneDeep(
-        document.excludedEmbedMetadataKeys,
-      ),
-      excludedLlmMetadataKeys: _.cloneDeep(document.excludedLlmMetadataKeys),
-    });
-    node.relationships[NodeRelationship.SOURCE] = document.asRelatedNodeInfo();
-    nodes.push(node);
-  });
-
-  if (includePrevNextRel) {
-    nodes.forEach((node, index) => {
-      if (index > 0) {
-        node.relationships[NodeRelationship.PREVIOUS] =
-          nodes[index - 1].asRelatedNodeInfo();
-      }
-      if (index < nodes.length - 1) {
-        node.relationships[NodeRelationship.NEXT] =
-          nodes[index + 1].asRelatedNodeInfo();
-      }
-    });
-  }
-
-  return nodes;
-}

packages/llamaindex/src/readers/AssemblyAIReader.ts

@@ -11,7 +11,14 @@ import { AssemblyAI } from "assemblyai";
 import type { BaseReader } from "./type.js";
 
 type AssemblyAIOptions = Partial<BaseServiceParams>;
-
+const defaultOptions = {
+  userAgent: {
+    integration: {
+      name: "LlamaIndexTS",
+      version: "1.0.1",
+    },
+  },
+};
 /**
  * Base class for AssemblyAI Readers.
  */
@@ -37,7 +44,10 @@ abstract class AssemblyAIReader implements BaseReader {
       );
     }
 
-    this.client = new AssemblyAI(options as BaseServiceParams);
+    this.client = new AssemblyAI({
+      ...defaultOptions,
+      ...options,
+    } as BaseServiceParams);
   }
 
   abstract loadData(params: TranscribeParams | string): Promise<Document[]>;

packages/llamaindex/src/tools/AzureDynamicSessionTool.node.ts

@@ -177,7 +177,7 @@ export class AzureDynamicSessionTool
   /**
    * The endpoint of the Azure pool management service.
    * This is where the tool will send requests to interact with the session pool.
-   * If not provided, the tool will use the value of the `AZURE_CONTAINER_APP_SESSION_POOL_MANAGEMENT_ENDPOINT` environment variable.
+   * If not provided, the tool will use the value of the `AZURE_POOL_MANAGEMENT_ENDPOINT` environment variable.
    */
   private poolManagementEndpoint: string;
 
@@ -191,14 +191,12 @@ export class AzureDynamicSessionTool
     this.sessionId = params?.sessionId || randomUUID();
     this.poolManagementEndpoint =
       params?.poolManagementEndpoint ||
-      (getEnv("AZURE_CONTAINER_APP_SESSION_POOL_MANAGEMENT_ENDPOINT") ?? "");
+      (getEnv("AZURE_POOL_MANAGEMENT_ENDPOINT") ?? "");
     this.azureADTokenProvider =
       params?.azureADTokenProvider ?? getAzureADTokenProvider();
 
     if (!this.poolManagementEndpoint) {
-      throw new Error(
-        "AZURE_CONTAINER_APP_SESSION_POOL_MANAGEMENT_ENDPOINT must be defined.",
-      );
+      throw new Error("AZURE_POOL_MANAGEMENT_ENDPOINT must be defined.");
     }
   }
 

packages/llamaindex/src/tools/ToolsFactory.ts

@@ -1,12 +1,18 @@
-import { WikipediaTool } from "./WikipediaTool.js";
+import {
+  AzureDynamicSessionTool,
+  type AzureDynamicSessionToolParams,
+} from "./AzureDynamicSessionTool.node.js";
+import { WikipediaTool, type WikipediaToolParams } from "./WikipediaTool.js";
 
 export namespace ToolsFactory {
   type ToolsMap = {
     [Tools.Wikipedia]: typeof WikipediaTool;
+    [Tools.AzureCodeInterpreter]: typeof AzureDynamicSessionTool;
   };
 
   export enum Tools {
     Wikipedia = "wikipedia.WikipediaToolSpec",
+    AzureCodeInterpreter = "azure_code_interpreter.AzureCodeInterpreterToolSpec",
   }
 
   export async function createTool<Tool extends Tools>(
@@ -14,7 +20,15 @@ export namespace ToolsFactory {
     ...params: ConstructorParameters<ToolsMap[Tool]>
   ): Promise<InstanceType<ToolsMap[Tool]>> {
     if (key === Tools.Wikipedia) {
-      return new WikipediaTool(...params) as InstanceType<ToolsMap[Tool]>;
+      return new WikipediaTool(
+        ...(params as WikipediaToolParams[]),
+      ) as InstanceType<ToolsMap[Tool]>;
+    }
+
+    if (key === Tools.AzureCodeInterpreter) {
+      return new AzureDynamicSessionTool(
+        ...(params as AzureDynamicSessionToolParams[]),
+      ) as InstanceType<ToolsMap[Tool]>;
     }
 
     throw new Error(

packages/llamaindex/tests/ingestion/IngestionCache.test.ts

@@ -28,7 +28,7 @@ describe("IngestionCache", () => {
 });
 
 describe("getTransformationHash", () => {
-  let nodes: BaseNode[], transform: TransformComponent<any>;
+  let nodes: BaseNode[], transform: TransformComponent;
 
   beforeAll(() => {
     nodes = [new TextNode({ text: "some text", id_: "some id" })];

pnpm-lock.yaml

@@ -378,6 +378,9 @@ importers:
       bunchee:
         specifier: 5.3.0-beta.0
         version: 5.3.0-beta.0(typescript@5.5.3)
+      natural:
+        specifier: ^7.1.0
+        version: 7.1.0(@aws-sdk/credential-providers@3.613.0)
 
   packages/core/tests:
     devDependencies:
@@ -3111,6 +3114,35 @@ packages:
       '@types/react':
         optional: true
 
+  '@redis/bloom@1.2.0':
+    resolution: {integrity: sha512-HG2DFjYKbpNmVXsa0keLHp/3leGJz1mjh09f2RLGGLQZzSHpkmZWuwJbAvo3QcRY8p80m5+ZdXZdYOSBLlp7Cg==}
+    peerDependencies:
+      '@redis/client': ^1.0.0
+
+  '@redis/client@1.5.17':
+    resolution: {integrity: sha512-IPvU9A31qRCZ7lds/x+ksuK/UMndd0EASveAvCvEtFFKIZjZ+m/a4a0L7S28KEWoR5ka8526hlSghDo4Hrc2Hg==}
+    engines: {node: '>=14'}
+
+  '@redis/graph@1.1.1':
+    resolution: {integrity: sha512-FEMTcTHZozZciLRl6GiiIB4zGm5z5F3F6a6FZCyrfxdKOhFlGkiAqlexWMBzCi4DcRoyiOsuLfW+cjlGWyExOw==}
+    peerDependencies:
+      '@redis/client': ^1.0.0
+
+  '@redis/json@1.0.6':
+    resolution: {integrity: sha512-rcZO3bfQbm2zPRpqo82XbW8zg4G/w4W3tI7X8Mqleq9goQjAGLL7q/1n1ZX4dXEAmORVZ4s1+uKLaUOg7LrUhw==}
+    peerDependencies:
+      '@redis/client': ^1.0.0
+
+  '@redis/search@1.1.6':
+    resolution: {integrity: sha512-mZXCxbTYKBQ3M2lZnEddwEAks0Kc7nauire8q20oA0oA/LoA+E/b5Y5KZn232ztPb1FkIGqo12vh3Lf+Vw5iTw==}
+    peerDependencies:
+      '@redis/client': ^1.0.0
+
+  '@redis/time-series@1.0.5':
+    resolution: {integrity: sha512-IFjIgTusQym2B5IZJG3XKr5llka7ey84fw/NOYqESP5WUfQs9zz1ww/9+qoz4ka/S6KcGBodzlCeZ5UImKbscg==}
+    peerDependencies:
+      '@redis/client': ^1.0.0
+
   '@rollup/plugin-commonjs@25.0.8':
     resolution: {integrity: sha512-ZEZWTK5n6Qde0to4vS9Mr5x/0UZoqCxPVR9KRUjU4kA2sO7GEUn1fop0DAwpO6z0Nw/kJON9bDmSxdWxO/TT1A==}
     engines: {node: '>=14.0.0'}
@@ -4279,6 +4311,12 @@ packages:
     resolution: {integrity: sha512-4B/qKCfeE/ODUaAUpSwfzazo5x29WD4r3vXiWsB7I2mSDAihwEqKO+g8GELZUQSSAo5e1XTYh3ZVfLyxBc12nA==}
     engines: {node: '>= 10.0.0'}
 
+  afinn-165-financialmarketnews@3.0.0:
+    resolution: {integrity: sha512-0g9A1S3ZomFIGDTzZ0t6xmv4AuokBvBmpes8htiyHpH7N4xDmvSQL6UxL/Zcs2ypRb3VwgCscaD8Q3zEawKYhw==}
+
+  afinn-165@1.0.4:
+    resolution: {integrity: sha512-7+Wlx3BImrK0HiG6y3lU4xX7SpBPSSu8T9iguPMlaueRFxjbYwAQrp9lqZUuFikqKbd/en8lVREILvP2J80uJA==}
+
   agent-base@6.0.2:
     resolution: {integrity: sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==}
     engines: {node: '>= 6.0.0'}
@@ -4398,6 +4436,10 @@ packages:
   app-module-path@2.2.0:
     resolution: {integrity: sha512-gkco+qxENJV+8vFcDiiFhuoSvRXb2a/QPqpSoWhVz829VNJfOTnELbBmPmNKFxf3xdNnw4DWCkzkDaavcX/1YQ==}
 
+  apparatus@0.0.10:
+    resolution: {integrity: sha512-KLy/ugo33KZA7nugtQ7O0E1c8kQ52N3IvD/XgIh4w/Nr28ypfkwDfA67F1ev4N1m5D+BOk1+b2dEJDfpj/VvZg==}
+    engines: {node: '>=0.2.6'}
+
   aproba@2.0.0:
     resolution: {integrity: sha512-lYe4Gx7QT+MKGbDsA+Z+he/Wtef0BiwDOlK/XkBrdfsh9J/jPPXbX0tE9x9cl27Tmu5gg3QUbUrQYa/y+KOHPQ==}
 
@@ -4496,6 +4538,9 @@ packages:
     resolution: {integrity: sha512-ISvCdHdlTDlH5IpxQJIex7BWBywFWgjJSVdwst+/iQCoEYnyOaQ95+X1JGshuBjGp6nxKUy1jMgE3zPqN7fQdg==}
     hasBin: true
 
+  async@2.6.4:
+    resolution: {integrity: sha512-mzo5dfJYwAn29PeiJ0zvwTo04zj8HDJj0Mn8TD7sno7q12prdbnasKJHhkm2c1LgrhlJ0teaea8860oxi51mGA==}
+
   async@3.2.5:
     resolution: {integrity: sha512-baNZyqaaLhyLVKm/DlvdW051MSgO6b8eVfIezl9E5PqWxFgzLm/wQntEW4zOytVburDEr0JlALEpdOFwvErLsg==}
 
@@ -4582,6 +4627,10 @@ packages:
   base64-js@1.5.1:
     resolution: {integrity: sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==}
 
+  basic-auth@2.0.1:
+    resolution: {integrity: sha512-NF+epuEdnUYVlGuhaxbbq+dvJttwLnGY+YixlXlME5KpQ5W3CnXA5cVTneY3SPbPDRkcjMbifrwmFYcClgOZeg==}
+    engines: {node: '>= 0.8'}
+
   batch@0.6.1:
     resolution: {integrity: sha512-x+VAiMRL6UPkx+kudNvxTl6hB2XNNCG2r+7wixVfIYwu/2HKRXimwQyaumLjMveWvT2Hkd/cAJw+QBMfJ/EKVw==}
 
@@ -4949,6 +4998,10 @@ packages:
     resolution: {integrity: sha512-eYm0QWBtUrBWZWG0d386OGAw16Z995PiOVo2B7bjWSbHedGl5e0ZWaq65kOGgUSNesEIDkB9ISbTg/JK9dhCZA==}
     engines: {node: '>=6'}
 
+  cluster-key-slot@1.1.2:
+    resolution: {integrity: sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA==}
+    engines: {node: '>=0.10.0'}
+
   code-red@1.0.4:
     resolution: {integrity: sha512-7qJWqItLA8/VPVlKJlFXU+NBlo/qyfs39aJcuMT/2ere32ZqvF5OSxgdM5xOfJJ7O429gg2HM47y8v9P+9wrNw==}
 
@@ -5133,6 +5186,10 @@ packages:
   core-util-is@1.0.3:
     resolution: {integrity: sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ==}
 
+  corser@2.0.1:
+    resolution: {integrity: sha512-utCYNzRSQIZNPIcGZdQc92UVJYAhtGAteCFg0yRaFm8f0P+CPtyGyHXJcGXnffjCybUCEx3FQ2G7U3/o9eIkVQ==}
+    engines: {node: '>= 0.4.0'}
+
   cosmiconfig@6.0.0:
     resolution: {integrity: sha512-xb3ZL6+L8b9JLLCx3ZdoZy4+2ECphCMo2PwqgP1tlfVq6M6YReyzBJtvWWtbDSpNr9hn96pkCiZqUcFEc+54Qg==}
     engines: {node: '>=8'}
@@ -6517,6 +6574,10 @@ packages:
   hpack.js@2.1.6:
     resolution: {integrity: sha512-zJxVehUdMGIKsRaNt7apO2Gqp0BdqW5yaiGHXXmbpvxgBYVZnAql+BJb4RO5ad2MgpbZKn5G6nMnegrH1FcNYQ==}
 
+  html-encoding-sniffer@3.0.0:
+    resolution: {integrity: sha512-oWv4T4yJ52iKrufjnyZPkrN0CH3QnrUqdB6In1g5Fe1mia8GmF36gnfNySxoZtxD5+NmYw1EElVXiBk93UeskA==}
+    engines: {node: '>=12'}
+
   html-entities@2.5.2:
     resolution: {integrity: sha512-K//PSRMQk4FZ78Kyau+mZurHn3FH0Vwr+H36eE0rPbeYkRRi9YxceYPhuN60UwWorxyKHhqoAJl2OFKa4BVtaA==}
 
@@ -6595,6 +6656,11 @@ packages:
     resolution: {integrity: sha512-7mz/721AbnJwIVbnaSv1Cz3Am0ZLT/UBwkC92VlxhXv/k/BBQfM2fXElQNC27BVGr0uwUpplYPQM9LnaBMR5NQ==}
     engines: {node: '>=8.0.0'}
 
+  http-server@14.1.1:
+    resolution: {integrity: sha512-+cbxadF40UXd9T01zUHgA+rlo2Bg1Srer4+B4NwIHdaGxAGGv59nYRnGGDJ9LBk7alpS0US+J+bLLdQOOkJq4A==}
+    engines: {node: '>=12'}
+    hasBin: true
+
   http2-wrapper@1.0.3:
     resolution: {integrity: sha512-V+23sDMr12Wnz7iTcDeJr3O6AIxlnvT/bmaAAAP/Xda35C90p9599p0F1eHR/N1KILWSoWVAiOMFjBBXaXSMxg==}
     engines: {node: '>=10.19.0'}
@@ -7172,6 +7238,10 @@ packages:
   jws@4.0.0:
     resolution: {integrity: sha512-KDncfTmOZoOMTFG4mBlG0qUIOlc03fmzH+ru6RgYVZhPkyiy/92Owlt/8UEN+a4TXR1FQetfIpJE8ApdvdVxTg==}
 
+  kareem@2.6.3:
+    resolution: {integrity: sha512-C3iHfuGUXK2u8/ipq9LfjFfXFxAZMQJJq7vLS45r3D9Y2xQ/m4S8zaR4zMLFWh9AsNPXmcFfUDhTEO8UIC/V6Q==}
+    engines: {node: '>=12.0.0'}
+
   keyv@4.5.4:
     resolution: {integrity: sha512-oxVHkHR/EJf2CNXnWxRLW6mg7JyCCUcG0DtEGmL2ctUo1PNTin1PUil+r/+4r5MpVgC/fn1kjsx7mjSujKqIpw==}
 
@@ -7512,6 +7582,10 @@ packages:
     resolution: {integrity: sha512-UERzLsxzllchadvbPs5aolHh65ISpKpM+ccLbOJ8/vvpBKmAWf+la7dXFy7Mr0ySHbdHrFv5kGFCUHHe6GFEmw==}
     engines: {node: '>= 4.0.0'}
 
+  memjs@1.3.2:
+    resolution: {integrity: sha512-qUEg2g8vxPe+zPn09KidjIStHPtoBO8Cttm8bgJFWWabbsjQ9Av9Ky+6UcvKx6ue0LLb/LEhtcyQpRyKfzeXcg==}
+    engines: {node: '>=0.10.0'}
+
   memory-pager@1.5.0:
     resolution: {integrity: sha512-ZS4Bp4r/Zoeq6+NLJpP+0Zzm0pR8whtGPf1XExKLJBAczGMnSi3It14OiNCStjQjM6NU1okjQGSxgEZN8eBYKg==}
 
@@ -7761,6 +7835,10 @@ packages:
   mkdirp-classic@0.5.3:
     resolution: {integrity: sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==}
 
+  mkdirp@0.5.6:
+    resolution: {integrity: sha512-FP+p8RB8OWpF3YZBCrP5gtADmtXApB5AMLn+vdyA+PyxCjrCs00mjyUozssO33cwDeT3wNGdLxJ5M//YqtHAJw==}
+    hasBin: true
+
   mkdirp@1.0.4:
     resolution: {integrity: sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw==}
     engines: {node: '>=10'}
@@ -7782,6 +7860,33 @@ packages:
   mongodb-connection-string-url@3.0.1:
     resolution: {integrity: sha512-XqMGwRX0Lgn05TDB4PyG2h2kKO/FfWJyCzYQbIhXUxz7ETt0I/FqHjUeqj37irJ+Dl1ZtU82uYyj14u2XsZKfg==}
 
+  mongodb@6.7.0:
+    resolution: {integrity: sha512-TMKyHdtMcO0fYBNORiYdmM25ijsHs+Njs963r4Tro4OQZzqYigAzYQouwWRg4OIaiLRUEGUh/1UAcH5lxdSLIA==}
+    engines: {node: '>=16.20.1'}
+    peerDependencies:
+      '@aws-sdk/credential-providers': ^3.188.0
+      '@mongodb-js/zstd': ^1.1.0
+      gcp-metadata: ^5.2.0
+      kerberos: ^2.0.1
+      mongodb-client-encryption: '>=6.0.0 <7'
+      snappy: ^7.2.2
+      socks: ^2.7.1
+    peerDependenciesMeta:
+      '@aws-sdk/credential-providers':
+        optional: true
+      '@mongodb-js/zstd':
+        optional: true
+      gcp-metadata:
+        optional: true
+      kerberos:
+        optional: true
+      mongodb-client-encryption:
+        optional: true
+      snappy:
+        optional: true
+      socks:
+        optional: true
+
   mongodb@6.8.0:
     resolution: {integrity: sha512-HGQ9NWDle5WvwMnrvUxsFYPd3JEbqD3RgABHBQRuoCEND0qzhsd0iH5ypHsf1eJ+sXmvmyKpP+FLOKY8Il7jMw==}
     engines: {node: '>=16.20.1'}
@@ -7809,6 +7914,18 @@ packages:
       socks:
         optional: true
 
+  mongoose@8.5.1:
+    resolution: {integrity: sha512-OhVcwVl91A1G6+XpjDcpkGP7l7ikZkxa0DylX7NT/lcEqAjggzSdqDxb48A+xsDxqNAr0ntSJ1yiE3+KJTOd5Q==}
+    engines: {node: '>=16.20.1'}
+
+  mpath@0.9.0:
+    resolution: {integrity: sha512-ikJRQTk8hw5DEoFVxHG1Gn9T/xcjtdnOKIU1JTmGjZZlg9LST2mBLmcX3/ICIbgJydT2GOc15RnNy5mHmzfSew==}
+    engines: {node: '>=4.0.0'}
+
+  mquery@5.0.0:
+    resolution: {integrity: sha512-iQMncpmEK8R8ncT8HJGsGc9Dsp8xcgYMVSbs5jgnm1lFHTZqMJTUWTDx1LBO8+mK3tPNZWFLBghQEIOULSTHZg==}
+    engines: {node: '>=14.0.0'}
+
   mri@1.2.0:
     resolution: {integrity: sha512-tzzskb3bG8LvYGFF/mDTpq3jpI6Q9wc3LEmBaghu+DdCssd1FakN7Bc0hVNmEyGq1bq3RgfkCb3cmQLpNPOroA==}
     engines: {node: '>=4'}
@@ -7856,6 +7973,10 @@ packages:
   natural-compare@1.4.0:
     resolution: {integrity: sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw==}
 
+  natural@7.1.0:
+    resolution: {integrity: sha512-GBhiRgF0VUX+zPWahBVir1ajARQDZF1Fe6UpQORNzyQT57JQ2KLKYvubecvjIYh/uDaociusmySeRh+WL5OdxQ==}
+    engines: {node: '>=0.4.10'}
+
   negotiator@0.6.3:
     resolution: {integrity: sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg==}
     engines: {node: '>= 0.6'}
@@ -8424,6 +8545,10 @@ packages:
     resolution: {integrity: sha512-Nc3IT5yHzflTfbjgqWcCPpo7DaKy4FnpB0l/zCAW0Tc7jxAiuqSxHasntB3D7887LSrA93kDJ9IXovxJYxyLCA==}
     engines: {node: '>=4'}
 
+  portfinder@1.0.32:
+    resolution: {integrity: sha512-on2ZJVVDXRADWE6jnQaX0ioEylzgBpQk8r55NE4wjXW1ZxO+BgDlY6DXwj20i0V8eB4SenDQ00WEaxfiIQPcxg==}
+    engines: {node: '>= 0.12.0'}
+
   portkey-ai@0.1.16:
     resolution: {integrity: sha512-EY4FRp6PZSD75Q1o1qc08DfPNTG9FnkUPN3Z1/lEvaq9iFpSO5UekcagUZaKSVhao311qjBjns+kF0rS9ht7iA==}
 
@@ -9101,6 +9226,9 @@ packages:
     resolution: {integrity: sha512-8HrF5ZsXk5FAH9dgsx3BlUer73nIhuj+9OrQwEbLTPOBzGkL1lsFCR01am+v+0m2Cmbs1nP12hLDl5FA7EszKA==}
     engines: {node: '>=6.0.0'}
 
+  redis@4.6.15:
+    resolution: {integrity: sha512-2NtuOpMW3tnYzBw6S8mbXSX7RPzvVFCA2wFJq9oErushO2UeBkxObk+uvo7gv7n0rhWeOj/IzrHO8TjcFlRSOg==}
+
   reflect.getprototypeof@1.0.6:
     resolution: {integrity: sha512-fmfw4XgoDke3kdI6h4xcUz1dG8uaiv5q9gcEwLS4Pnth2kxT+GZ7YehS1JTMGBQmtV7Y4GFGbs2re2NqhdozUg==}
     engines: {node: '>= 0.4'}
@@ -9371,6 +9499,9 @@ packages:
     resolution: {integrity: sha512-vfD3pmTzGpufjScBh50YHKzEu2lxBWhVEHsNGoEXmCmn2hKGfeNLYMzCJpe8cD7gqX7TJluOVpBkAequ6dgMmA==}
     engines: {node: '>=4'}
 
+  secure-compare@3.0.1:
+    resolution: {integrity: sha512-AckIIV90rPDcBcglUwXPF3kg0P0qmPsPXAj6BBEENQE1p5yA1xfmDJzfi1Tappj37Pv2mVbKpL3Z1T+Nn7k1Qw==}
+
   secure-json-parse@2.7.0:
     resolution: {integrity: sha512-6aU+Rwsezw7VR8/nyvKTx8QpWH9FrcYiXXlqC4z5d5XQBDRqtbfsRjnwGyqbi3gddNtWHuEk9OANUotL26qKUw==}
 
@@ -9486,6 +9617,9 @@ packages:
     resolution: {integrity: sha512-fDW/EZ6Q9RiO8eFG8Hj+7u/oW+XrPTIChwCOM2+th2A6OblDtYYIpve9m+KvI9Z4C9qSEXlaGR6bTEYHReuglA==}
     engines: {node: '>= 0.4'}
 
+  sift@17.1.3:
+    resolution: {integrity: sha512-Rtlj66/b0ICeFzYTuNvX/EF1igRbbnGSvEyT79McoZa/DeGhMyC5pWKOEsZKnpkqtSeovd5FL/bjHWC3CIIvCQ==}
+
   siginfo@2.0.0:
     resolution: {integrity: sha512-ybx0WO1/8bSBLEWXZvEd7gMW3Sn3JFlW3TvX1nREbDLRNQNaeNN8WK0meBwPdAaOI7TtRRRJn/Es1zhrrCHu7g==}
 
@@ -9656,6 +9790,10 @@ packages:
     resolution: {integrity: sha512-KXDYZ9dszj6bzvnEMRYvxgeTHU74QBFL54XKtP3nyMuJ81CFYtABZ3bAzL2EdFUaEwJOBOgENyFj3R7oTzDyyw==}
     engines: {node: '>=4', npm: '>=6'}
 
+  stopwords-iso@1.1.0:
+    resolution: {integrity: sha512-I6GPS/E0zyieHehMRPQcqkiBMJKGgLta+1hREixhoLPqEA0AlVFiC43dl8uPpmkkeRdDMzYRWFWk5/l9x7nmNg==}
+    engines: {node: '>=0.10.0'}
+
   stream-to-array@2.3.0:
     resolution: {integrity: sha512-UsZtOYEn4tWU2RGLOXr/o/xjRBftZRlG3dEWoaHr8j4GuypJ3isitGbVyjQKAuMu+xbiop8q224TjiZWc4XTZA==}
 
@@ -9868,6 +10006,10 @@ packages:
     peerDependencies:
       vue: '>=3.2.26 < 4'
 
+  sylvester@0.0.12:
+    resolution: {integrity: sha512-SzRP5LQ6Ts2G5NyAa/jg16s8e3R7rfdFjizy1zeoecYWw+nGL+YA1xZvW/+iJmidBGSdLkuvdwTYEyJEb+EiUw==}
+    engines: {node: '>=0.2.6'}
+
   tailwind-merge@2.4.0:
     resolution: {integrity: sha512-49AwoOQNKdqKPd9CViyH5wJoSKsCDjUlzL8DxuGp3P1FsGY36NJDAa18jLZcaHAUUuTj+JB8IAo8zWgBNvBF7A==}
 
@@ -10240,6 +10382,10 @@ packages:
   unified@11.0.5:
     resolution: {integrity: sha512-xKvGhPWw3k84Qjh8bI3ZeJjqnyadK+GEFtazSfZv/rKeTkTjOJho6mFqh2SM96iIcZokxiOpg78GazTSg8+KHA==}
 
+  union@0.5.0:
+    resolution: {integrity: sha512-N6uOhuW6zO95P3Mel2I2zMsbsanvvtgn6jVqJv4vbVcz/JN0OkL9suomjQGmWtxJQXOCqUJvquc1sMeNz/IwlA==}
+    engines: {node: '>= 0.8.0'}
+
   unique-string@3.0.0:
     resolution: {integrity: sha512-VGXBUVwxKMBUznyffQweQABPRRW1vHZAbadFZud4pLFAqRGvv/96vafgjWFqzourzr8YonlQiPgH0YCJfawoGQ==}
     engines: {node: '>=12'}
@@ -10592,6 +10738,10 @@ packages:
     resolution: {integrity: sha512-OqedPIGOfsDlo31UNwYbCFMSaO9m9G/0faIHj5/dZFDMFqPTcx6UwqyOy3COEaEOg/9VsGIpdqn62W5KhoKSpg==}
     engines: {node: '>=0.8.0'}
 
+  whatwg-encoding@2.0.0:
+    resolution: {integrity: sha512-p41ogyeMUrw3jWclHWTQg1k05DSVXPLcVxRTYsXUk+ZooOCZLcoYgPZ/HL/D/N+uQPOtcp1me1WhBEaX02mhWg==}
+    engines: {node: '>=12'}
+
   whatwg-fetch@3.6.20:
     resolution: {integrity: sha512-EqhiFU6daOA8kpjOWTL0olhVOF3i7OrFzSYiGsEMB8GcXS+RrzauAERX65xMeNWVqxA6HXH2m69Z9LaKKdisfg==}
 
@@ -10664,6 +10814,10 @@ packages:
     resolution: {integrity: sha512-BN22B5eaMMI9UMtjrGd5g5eCYPpCPDUy0FJXbYsaT5zYxjFOckS53SQDE3pWkVoWpHXVb3BrYcEN4Twa55B5cA==}
     engines: {node: '>=0.10.0'}
 
+  wordnet-db@3.1.14:
+    resolution: {integrity: sha512-zVyFsvE+mq9MCmwXUWHIcpfbrHHClZWZiVOzKSxNJruIcFn2RbY55zkhiAMMxM8zCVSmtNiViq8FsAZSFpMYag==}
+    engines: {node: '>=0.6.0'}
+
   wordwrap@1.0.0:
     resolution: {integrity: sha512-gvVzJFlPycKc5dZN4yPkP8w7Dc37BtP1yczEneOb4uq34pXZcvrtRTmWV8W+Ume+XCxKgbjM+nevkyFPMybd4Q==}
 
@@ -14130,6 +14284,32 @@ snapshots:
     optionalDependencies:
       '@types/react': 18.3.3
 
+  '@redis/bloom@1.2.0(@redis/client@1.5.17)':
+    dependencies:
+      '@redis/client': 1.5.17
+
+  '@redis/client@1.5.17':
+    dependencies:
+      cluster-key-slot: 1.1.2
+      generic-pool: 3.9.0
+      yallist: 4.0.0
+
+  '@redis/graph@1.1.1(@redis/client@1.5.17)':
+    dependencies:
+      '@redis/client': 1.5.17
+
+  '@redis/json@1.0.6(@redis/client@1.5.17)':
+    dependencies:
+      '@redis/client': 1.5.17
+
+  '@redis/search@1.1.6(@redis/client@1.5.17)':
+    dependencies:
+      '@redis/client': 1.5.17
+
+  '@redis/time-series@1.0.5(@redis/client@1.5.17)':
+    dependencies:
+      '@redis/client': 1.5.17
+
   '@rollup/plugin-commonjs@25.0.8(rollup@4.18.1)':
     dependencies:
       '@rollup/pluginutils': 5.1.0(rollup@4.18.1)
@@ -15523,6 +15703,10 @@ snapshots:
 
   address@1.2.2: {}
 
+  afinn-165-financialmarketnews@3.0.0: {}
+
+  afinn-165@1.0.4: {}
+
   agent-base@6.0.2:
     dependencies:
       debug: 4.3.5
@@ -15657,6 +15841,10 @@ snapshots:
 
   app-module-path@2.2.0: {}
 
+  apparatus@0.0.10:
+    dependencies:
+      sylvester: 0.0.12
+
   aproba@2.0.0:
     optional: true
 
@@ -15786,6 +15974,10 @@ snapshots:
 
   astring@1.8.6: {}
 
+  async@2.6.4:
+    dependencies:
+      lodash: 4.17.21
+
   async@3.2.5: {}
 
   asynckit@0.4.0: {}
@@ -15890,6 +16082,10 @@ snapshots:
 
   base64-js@1.5.1: {}
 
+  basic-auth@2.0.1:
+    dependencies:
+      safe-buffer: 5.1.2
+
   batch@0.6.1: {}
 
   better-path-resolve@1.0.0:
@@ -16315,6 +16511,8 @@ snapshots:
 
   clsx@2.1.1: {}
 
+  cluster-key-slot@1.1.2: {}
+
   code-red@1.0.4:
     dependencies:
       '@jridgewell/sourcemap-codec': 1.5.0
@@ -16507,6 +16705,8 @@ snapshots:
 
   core-util-is@1.0.3: {}
 
+  corser@2.0.1: {}
+
   cosmiconfig@6.0.0:
     dependencies:
       '@types/parse-json': 4.0.2
@@ -18428,6 +18628,10 @@ snapshots:
       readable-stream: 2.3.8
       wbuf: 1.7.3
 
+  html-encoding-sniffer@3.0.0:
+    dependencies:
+      whatwg-encoding: 2.0.0
+
   html-entities@2.5.2: {}
 
   html-escaper@2.0.2: {}
@@ -18530,6 +18734,25 @@ snapshots:
     transitivePeerDependencies:
       - debug
 
+  http-server@14.1.1:
+    dependencies:
+      basic-auth: 2.0.1
+      chalk: 4.1.2
+      corser: 2.0.1
+      he: 1.2.0
+      html-encoding-sniffer: 3.0.0
+      http-proxy: 1.18.1
+      mime: 1.6.0
+      minimist: 1.2.8
+      opener: 1.5.2
+      portfinder: 1.0.32
+      secure-compare: 3.0.1
+      union: 0.5.0
+      url-join: 4.0.1
+    transitivePeerDependencies:
+      - debug
+      - supports-color
+
   http2-wrapper@1.0.3:
     dependencies:
       quick-lru: 5.1.1
@@ -19061,6 +19284,8 @@ snapshots:
       jwa: 2.0.0
       safe-buffer: 5.2.1
 
+  kareem@2.6.3: {}
+
   keyv@4.5.4:
     dependencies:
       json-buffer: 3.0.1
@@ -19554,6 +19779,8 @@ snapshots:
     dependencies:
       fs-monkey: 1.0.6
 
+  memjs@1.3.2: {}
+
   memory-pager@1.5.0: {}
 
   merge-descriptors@1.0.1: {}
@@ -19957,6 +20184,10 @@ snapshots:
 
   mkdirp-classic@0.5.3: {}
 
+  mkdirp@0.5.6:
+    dependencies:
+      minimist: 1.2.8
+
   mkdirp@1.0.4: {}
 
   mlly@1.7.1:
@@ -19983,6 +20214,14 @@ snapshots:
       '@types/whatwg-url': 11.0.5
       whatwg-url: 13.0.0
 
+  mongodb@6.7.0(@aws-sdk/credential-providers@3.613.0):
+    dependencies:
+      '@mongodb-js/saslprep': 1.1.7
+      bson: 6.8.0
+      mongodb-connection-string-url: 3.0.1
+    optionalDependencies:
+      '@aws-sdk/credential-providers': 3.613.0(@aws-sdk/client-sso-oidc@3.613.0(@aws-sdk/client-sts@3.613.0))
+
   mongodb@6.8.0(@aws-sdk/credential-providers@3.613.0(@aws-sdk/client-sso-oidc@3.613.0(@aws-sdk/client-sts@3.613.0))):
     dependencies:
       '@mongodb-js/saslprep': 1.1.7
@@ -19991,6 +20230,33 @@ snapshots:
     optionalDependencies:
       '@aws-sdk/credential-providers': 3.613.0(@aws-sdk/client-sso-oidc@3.613.0(@aws-sdk/client-sts@3.613.0))
 
+  mongoose@8.5.1(@aws-sdk/credential-providers@3.613.0):
+    dependencies:
+      bson: 6.8.0
+      kareem: 2.6.3
+      mongodb: 6.7.0(@aws-sdk/credential-providers@3.613.0)
+      mpath: 0.9.0
+      mquery: 5.0.0
+      ms: 2.1.3
+      sift: 17.1.3
+    transitivePeerDependencies:
+      - '@aws-sdk/credential-providers'
+      - '@mongodb-js/zstd'
+      - gcp-metadata
+      - kerberos
+      - mongodb-client-encryption
+      - snappy
+      - socks
+      - supports-color
+
+  mpath@0.9.0: {}
+
+  mquery@5.0.0:
+    dependencies:
+      debug: 4.3.5
+    transitivePeerDependencies:
+      - supports-color
+
   mri@1.2.0: {}
 
   mrmime@2.0.0: {}
@@ -20025,6 +20291,35 @@ snapshots:
 
   natural-compare@1.4.0: {}
 
+  natural@7.1.0(@aws-sdk/credential-providers@3.613.0):
+    dependencies:
+      afinn-165: 1.0.4
+      afinn-165-financialmarketnews: 3.0.0
+      apparatus: 0.0.10
+      dotenv: 16.4.5
+      http-server: 14.1.1
+      memjs: 1.3.2
+      mongoose: 8.5.1(@aws-sdk/credential-providers@3.613.0)
+      pg: 8.12.0
+      redis: 4.6.15
+      safe-stable-stringify: 2.4.3
+      stopwords-iso: 1.1.0
+      sylvester: 0.0.12
+      underscore: 1.13.6
+      uuid: 9.0.1
+      wordnet-db: 3.1.14
+    transitivePeerDependencies:
+      - '@aws-sdk/credential-providers'
+      - '@mongodb-js/zstd'
+      - debug
+      - gcp-metadata
+      - kerberos
+      - mongodb-client-encryption
+      - pg-native
+      - snappy
+      - socks
+      - supports-color
+
   negotiator@0.6.3: {}
 
   neo-async@2.6.2: {}
@@ -20660,6 +20955,14 @@ snapshots:
 
   pluralize@8.0.0: {}
 
+  portfinder@1.0.32:
+    dependencies:
+      async: 2.6.4
+      debug: 3.2.7
+      mkdirp: 0.5.6
+    transitivePeerDependencies:
+      - supports-color
+
   portkey-ai@0.1.16:
     dependencies:
       agentkeepalive: 4.5.0
@@ -21379,6 +21682,15 @@ snapshots:
     dependencies:
       minimatch: 3.1.2
 
+  redis@4.6.15:
+    dependencies:
+      '@redis/bloom': 1.2.0(@redis/client@1.5.17)
+      '@redis/client': 1.5.17
+      '@redis/graph': 1.1.1(@redis/client@1.5.17)
+      '@redis/json': 1.0.6(@redis/client@1.5.17)
+      '@redis/search': 1.1.6(@redis/client@1.5.17)
+      '@redis/time-series': 1.0.5(@redis/client@1.5.17)
+
   reflect.getprototypeof@1.0.6:
     dependencies:
       call-bind: 1.0.7
@@ -21725,6 +22037,8 @@ snapshots:
       extend-shallow: 2.0.1
       kind-of: 6.0.3
 
+  secure-compare@3.0.1: {}
+
   secure-json-parse@2.7.0: {}
 
   select-hose@2.0.0: {}
@@ -21903,6 +22217,8 @@ snapshots:
       get-intrinsic: 1.2.4
       object-inspect: 1.13.2
 
+  sift@17.1.3: {}
+
   siginfo@2.0.0: {}
 
   signal-exit@3.0.7: {}
@@ -22074,6 +22390,8 @@ snapshots:
 
   stoppable@1.1.0: {}
 
+  stopwords-iso@1.1.0: {}
+
   stream-to-array@2.3.0:
     dependencies:
       any-promise: 1.3.0
@@ -22317,6 +22635,8 @@ snapshots:
     dependencies:
       vue: 3.4.31(typescript@5.5.3)
 
+  sylvester@0.0.12: {}
+
   tailwind-merge@2.4.0: {}
 
   tailwindcss@3.4.4:
@@ -22713,6 +23033,10 @@ snapshots:
       trough: 2.2.0
       vfile: 6.0.1
 
+  union@0.5.0:
+    dependencies:
+      qs: 6.11.2
+
   unique-string@3.0.0:
     dependencies:
       crypto-random-string: 4.0.0
@@ -23186,6 +23510,10 @@ snapshots:
 
   websocket-extensions@0.1.4: {}
 
+  whatwg-encoding@2.0.0:
+    dependencies:
+      iconv-lite: 0.6.3
+
   whatwg-fetch@3.6.20: {}
 
   whatwg-url@13.0.0:
@@ -23296,6 +23624,8 @@ snapshots:
 
   word-wrap@1.2.5: {}
 
+  wordnet-db@3.1.14: {}
+
   wordwrap@1.0.0: {}
 
   workerd@1.20240701.0: