release v0.2.9

Co-authored-by: Yi Ding <yi.s.ding@gmail.com>

.eslintrc.cjs

@@ -0,0 +1,76 @@
+module.exports = {
+  root: true,
+  extends: [
+    "turbo",
+    "prettier",
+    "plugin:@typescript-eslint/recommended-type-checked-only",
+  ],
+  parserOptions: {
+    project: true,
+    __tsconfigRootDir: __dirname,
+  },
+  settings: {
+    react: {
+      version: "999.999.999",
+    },
+  },
+  rules: {
+    "max-params": ["error", 4],
+    "prefer-const": "error",
+    "@typescript-eslint/no-floating-promises": [
+      "error",
+      {
+        ignoreIIFE: true,
+      },
+    ],
+    "@typescript-eslint/await-thenable": "off",
+    "@typescript-eslint/ban-ts-comment": "off",
+    "@typescript-eslint/ban-types": "off",
+    "no-array-constructor": "off",
+    "@typescript-eslint/no-array-constructor": "off",
+    "@typescript-eslint/no-base-to-string": "off",
+    "@typescript-eslint/no-duplicate-enum-values": "off",
+    "@typescript-eslint/no-duplicate-type-constituents": "off",
+    "@typescript-eslint/no-explicit-any": "off",
+    "@typescript-eslint/no-extra-non-null-assertion": "off",
+    "@typescript-eslint/no-for-in-array": "off",
+    "no-implied-eval": "off",
+    "@typescript-eslint/no-implied-eval": "off",
+    "no-loss-of-precision": "off",
+    "@typescript-eslint/no-loss-of-precision": "off",
+    "@typescript-eslint/no-misused-new": "off",
+    "@typescript-eslint/no-misused-promises": "off",
+    "@typescript-eslint/no-namespace": "off",
+    "@typescript-eslint/no-non-null-asserted-optional-chain": "off",
+    "@typescript-eslint/no-redundant-type-constituents": "off",
+    "@typescript-eslint/no-this-alias": "off",
+    "@typescript-eslint/no-unnecessary-type-assertion": "off",
+    "@typescript-eslint/no-unnecessary-type-constraint": "off",
+    "@typescript-eslint/no-unsafe-argument": "off",
+    "@typescript-eslint/no-unsafe-assignment": "off",
+    "@typescript-eslint/no-unsafe-call": "off",
+    "@typescript-eslint/no-unsafe-declaration-merging": "off",
+    "@typescript-eslint/no-unsafe-enum-comparison": "off",
+    "@typescript-eslint/no-unsafe-member-access": "off",
+    "@typescript-eslint/no-unsafe-return": "off",
+    "no-unused-vars": "off",
+    "@typescript-eslint/no-unused-vars": "off",
+    "@typescript-eslint/no-var-requires": "off",
+    "@typescript-eslint/prefer-as-const": "off",
+    "require-await": "off",
+    "@typescript-eslint/require-await": "off",
+    "@typescript-eslint/restrict-plus-operands": "off",
+    "@typescript-eslint/restrict-template-expressions": "off",
+    "@typescript-eslint/triple-slash-reference": "off",
+    "@typescript-eslint/unbound-method": "off",
+  },
+  overrides: [
+    {
+      files: ["examples/**/*.ts"],
+      rules: {
+        "turbo/no-undeclared-env-vars": "off",
+      },
+    },
+  ],
+  ignorePatterns: ["dist/", "lib/"],
+};

.eslintrc.js

@@ -1,23 +0,0 @@
-module.exports = {
-  root: true,
-  // This tells ESLint to load the config from the package `eslint-config-custom`
-  extends: ["custom"],
-  settings: {
-    next: {
-      rootDir: ["apps/*/"],
-    },
-  },
-  rules: {
-    "max-params": ["error", 4],
-    "prefer-const": "error",
-  },
-  overrides: [
-    {
-      files: ["examples/**/*.ts"],
-      rules: {
-        "turbo/no-undeclared-env-vars": "off",
-      },
-    },
-  ],
-  ignorePatterns: ["dist/", "lib/"],
-};

.github/workflows/lint_on_push_or_pull.yml

@@ -14,8 +14,6 @@ jobs:
     steps:
       - uses: actions/checkout@v4
       - uses: pnpm/action-setup@v2
-        with:
-          version: latest
       - name: Setup Node.js
         uses: actions/setup-node@v4
         with:

.github/workflows/test.yml

@@ -1,9 +1,44 @@
 name: Run Tests
 
-on: [push, pull_request]
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - main
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
 
 jobs:
+  e2e:
+    strategy:
+      fail-fast: false
+      matrix:
+        node-version: [18.x, 20.x, 21.x]
+    name: E2E on Node.js ${{ matrix.node-version }}
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: pnpm/action-setup@v2
+      - name: Setup Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version-file: ".nvmrc"
+          cache: "pnpm"
+      - name: Install dependencies
+        run: pnpm install
+      - name: Run E2E Tests
+        run: pnpm run e2e
+
   test:
+    strategy:
+      fail-fast: false
+      matrix:
+        node-version: [18.x, 20.x, 21.x]
+    name: Test on Node.js ${{ matrix.node-version }}
     runs-on: ubuntu-latest
 
     steps:
@@ -12,7 +47,7 @@ jobs:
       - name: Setup Node.js
         uses: actions/setup-node@v4
         with:
-          node-version-file: ".nvmrc"
+          node-version: ${{ matrix.node-version }}
           cache: "pnpm"
       - name: Install dependencies
         run: pnpm install
@@ -33,6 +68,9 @@ jobs:
         run: pnpm install
       - name: Build
         run: pnpm run build --filter llamaindex
+      - name: Use Build For Examples
+        run: pnpm link ../packages/core/
+        working-directory: ./examples
       - name: Run Type Check
         run: pnpm run type-check
       - name: Run Circular Dependency Check

.vscode/launch.json

@@ -10,8 +10,9 @@
       "name": "Debug Example",
       "skipFiles": ["<node_internals>/**"],
       "runtimeExecutable": "pnpm",
+      "console": "integratedTerminal",
       "cwd": "${workspaceFolder}/examples",
-      "runtimeArgs": ["ts-node", "${fileBasename}"]
+      "runtimeArgs": ["npx", "tsx", "${file}"]
     }
   ]
 }

README.md

@@ -154,7 +154,7 @@ If you need any of those classes, you have to import them instead directly. Here
 import { PineconeVectorStore } from "@llamaindex/edge/storage/vectorStore/PineconeVectorStore";
 ```
 
-As the `PDFReader` is not with the Edge runtime, here's how to use the `SimpleDirectoryReader` with the `LlamaParseReader` to load PDFs:
+As the `PDFReader` is not working with the Edge runtime, here's how to use the `SimpleDirectoryReader` with the `LlamaParseReader` to load PDFs:
 
 ```typescript
 import { SimpleDirectoryReader } from "@llamaindex/edge/readers/SimpleDirectoryReader";

apps/docs/docs/examples/agent.mdx

@@ -67,7 +67,6 @@ async function main() {
   // Create an OpenAIAgent with the function tools
   const agent = new OpenAIAgent({
     tools: [sumFunctionTool, divideFunctionTool],
-    verbose: true,
   });
 
   // Chat with the agent

apps/docs/docs/modules/agent/multi_document_agent.mdx

@@ -221,7 +221,6 @@ for (const title of wikiTitles) {
   const agent = new OpenAIAgent({
     tools: queryEngineTools,
     llm,
-    verbose: true,
   });
 
   documentAgents[title] = agent;
@@ -282,7 +281,6 @@ const objectIndex = await ObjectIndex.fromObjects(
 const topAgent = new OpenAIAgent({
   toolRetriever: await objectIndex.asRetriever({}),
   llm,
-  verbose: true,
   prefixMessages: [
     {
       content:

apps/docs/docs/modules/agent/openai.mdx

@@ -88,7 +88,6 @@ Now we can create an OpenAIAgent with the function tools.
 ```ts
 const agent = new OpenAIAgent({
   tools: [sumFunctionTool, divideFunctionTool],
-  verbose: true,
 });
 ```
 
@@ -169,7 +168,6 @@ async function main() {
   // Create an OpenAIAgent with the function tools
   const agent = new OpenAIAgent({
     tools: [sumFunctionTool, divideFunctionTool],
-    verbose: true,
   });
 
   // Chat with the agent

apps/docs/docs/modules/agent/query_engine_tool.mdx

@@ -64,7 +64,6 @@ const queryEngineTool = new QueryEngineTool({
 
 const agent = new OpenAIAgent({
   tools: [queryEngineTool],
-  verbose: true,
 });
 ```
 
@@ -114,7 +113,6 @@ async function main() {
   // Create an OpenAIAgent with the function tools
   const agent = new OpenAIAgent({
     tools: [queryEngineTool],
-    verbose: true,
   });
 
   // Chat with the agent

apps/docs/docs/modules/agent/react_agent.mdx

@@ -90,7 +90,6 @@ Now we can create an OpenAIAgent with the function tools.
 ```ts
 const agent = new ReActAgent({
   tools: [sumFunctionTool, divideFunctionTool],
-  verbose: true,
 });
 ```
 
@@ -185,7 +184,6 @@ async function main() {
   // Create an OpenAIAgent with the function tools
   const agent = new OpenAIAgent({
     tools: [sumFunctionTool, divideFunctionTool],
-    verbose: true,
   });
 
   // Chat with the agent

apps/docs/docs/modules/llms/available_llms/llama2.md

@@ -3,7 +3,7 @@
 ## Usage
 
 ```ts
-import { Ollama, Settings } from "llamaindex";
+import { Ollama, Settings, DeuceChatStrategy } from "llamaindex";
 
 Settings.llm = new LlamaDeuce({ chatStrategy: DeuceChatStrategy.META });
 ```
@@ -11,7 +11,12 @@ Settings.llm = new LlamaDeuce({ chatStrategy: DeuceChatStrategy.META });
 ## Usage with Replication
 
 ```ts
-import { Ollama, ReplicateSession, Settings } from "llamaindex";
+import {
+  Ollama,
+  ReplicateSession,
+  Settings,
+  DeuceChatStrategy,
+} from "llamaindex";
 
 const replicateSession = new ReplicateSession({
   replicateKey,
@@ -48,7 +53,13 @@ const results = await queryEngine.query({
 ## Full Example
 
 ```ts
-import { LlamaDeuce, Document, VectorStoreIndex, Settings } from "llamaindex";
+import {
+  LlamaDeuce,
+  Document,
+  VectorStoreIndex,
+  Settings,
+  DeuceChatStrategy,
+} from "llamaindex";
 
 // Use the LlamaDeuce LLM
 Settings.llm = new LlamaDeuce({ chatStrategy: DeuceChatStrategy.META });

apps/docs/docs/observability/index.md

@@ -14,6 +14,9 @@ Configure a variable once, and you'll be able to do things like the following:
 
 Each provider has similarities and differences. Take a look below for the full set of guides for each one!
 
+- [OpenLLMetry](#openllmetry)
+- [Langtrace](#langtrace)
+
 ## OpenLLMetry
 
 [OpenLLMetry](https://github.com/traceloop/openllmetry-js) is an open-source project based on OpenTelemetry for tracing and monitoring
@@ -33,3 +36,29 @@ traceloop.initialize({
   disableBatch: true,
 });
 ```
+
+## Langtrace
+
+Enhance your observability with Langtrace, a robust open-source tool supports OpenTelemetry and is designed to trace, evaluate, and manage LLM applications seamlessly. Langtrace integrates directly with LlamaIndex, offering detailed, real-time insights into performance metrics such as accuracy, evaluations, and latency.
+
+#### Install
+
+- Self-host or sign-up and generate an API key using [Langtrace](https://www.langtrace.ai) Cloud
+
+```bash
+npm install @langtrase/typescript-sdk
+```
+
+#### Initialize
+
+```js
+import * as Langtrace from "@langtrase/typescript-sdk";
+Langtrace.init({ api_key: "<YOUR_API_KEY>" });
+```
+
+Features:
+
+- OpenTelemetry compliant, ensuring broad compatibility with observability platforms.
+- Provides comprehensive logs and detailed traces of all components.
+- Real-time monitoring of accuracy, evaluations, usage, costs, and latency.
+- For more configuration options and details, visit [Langtrace Docs](https://docs.langtrace.ai/introduction).

apps/docs/docs/recipes/cost-analysis.mdx

@@ -6,7 +6,7 @@ This page shows how to track LLM cost using APIs.
 
 The callback manager is a class that manages the callback functions.
 
-You can register `llm-start`, and `llm-end` callbacks to the callback manager for tracking the cost.
+You can register `llm-start`, `llm-end`, and `llm-stream` callbacks to the callback manager for tracking the cost.
 
 import CodeBlock from "@theme/CodeBlock";
 import CodeSource from "!raw-loader!../../../../examples/recipes/cost-analysis";

apps/docs/package.json

@@ -15,12 +15,12 @@
     "typecheck": "tsc"
   },
   "dependencies": {
-    "@docusaurus/core": "^3.2.0",
-    "@docusaurus/remark-plugin-npm2yarn": "^3.2.0",
+    "@docusaurus/core": "^3.2.1",
+    "@docusaurus/remark-plugin-npm2yarn": "^3.2.1",
     "@llamaindex/examples": "workspace:*",
-    "@mdx-js/react": "^3.0.0",
+    "@mdx-js/react": "^3.0.1",
     "clsx": "^2.1.0",
-    "postcss": "^8.4.33",
+    "postcss": "^8.4.38",
     "prism-react-renderer": "^2.3.1",
     "raw-loader": "^4.0.2",
     "react": "^18.2.0",
@@ -28,15 +28,15 @@
   },
   "devDependencies": {
     "@docusaurus/module-type-aliases": "3.2.0",
-    "@docusaurus/preset-classic": "^3.2.0",
-    "@docusaurus/theme-classic": "^3.2.0",
-    "@docusaurus/types": "^3.2.0",
+    "@docusaurus/preset-classic": "^3.2.1",
+    "@docusaurus/theme-classic": "^3.2.1",
+    "@docusaurus/types": "^3.2.1",
     "@tsconfig/docusaurus": "^2.0.3",
-    "@types/node": "^18.19.10",
+    "@types/node": "^20.12.7",
     "docusaurus-plugin-typedoc": "^0.22.0",
-    "typedoc": "^0.25.12",
+    "typedoc": "^0.25.13",
     "typedoc-plugin-markdown": "^3.17.1",
-    "typescript": "^5.4.3"
+    "typescript": "^5.4.4"
   },
   "browserslist": {
     "production": [

examples/agent/multi_document_agent.ts

@@ -86,7 +86,6 @@ async function main() {
     const agent = new OpenAIAgent({
       tools: queryEngineTools,
       llm: new OpenAI({ model: "gpt-4" }),
-      verbose: true,
     });
 
     documentAgents[title] = agent;
@@ -126,7 +125,6 @@ async function main() {
   const topAgent = new OpenAIAgent({
     toolRetriever: await objectIndex.asRetriever({}),
     llm: new OpenAI({ model: "gpt-4" }),
-    verbose: true,
     prefixMessages: [
       {
         content:
@@ -145,4 +143,4 @@ async function main() {
   });
 }
 
-main();
+void main();

examples/agent/openai.ts

@@ -1,13 +1,13 @@
 import { FunctionTool, OpenAIAgent } from "llamaindex";
 
 // Define a function to sum two numbers
-function sumNumbers({ a, b }: { a: number; b: number }): number {
-  return a + b;
+function sumNumbers({ a, b }: { a: number; b: number }) {
+  return `${a + b}`;
 }
 
 // Define a function to divide two numbers
-function divideNumbers({ a, b }: { a: number; b: number }): number {
-  return a / b;
+function divideNumbers({ a, b }: { a: number; b: number }) {
+  return `${a / b}`;
 }
 
 // Define the parameters of the sum function as a JSON schema
@@ -24,7 +24,7 @@ const sumJSON = {
     },
   },
   required: ["a", "b"],
-};
+} as const;
 
 const divideJSON = {
   type: "object",
@@ -39,7 +39,7 @@ const divideJSON = {
     },
   },
   required: ["a", "b"],
-};
+} as const;
 
 async function main() {
   // Create a function tool from the sum function
@@ -59,7 +59,6 @@ async function main() {
   // Create an OpenAIAgent with the function tools
   const agent = new OpenAIAgent({
     tools: [functionTool, functionTool2],
-    verbose: true,
   });
 
   // Chat with the agent
@@ -71,6 +70,6 @@ async function main() {
   console.log(String(response));
 }
 
-main().then(() => {
+void main().then(() => {
   console.log("Done");
 });

examples/agent/query_openai_agent.ts

@@ -29,7 +29,6 @@ async function main() {
   // Create an OpenAIAgent with the function tools
   const agent = new OpenAIAgent({
     tools: [queryEngineTool],
-    verbose: true,
   });
 
   // Chat with the agent
@@ -41,6 +40,6 @@ async function main() {
   console.log(String(response));
 }
 
-main().then(() => {
+void main().then(() => {
   console.log("Done");
 });

examples/agent/react_agent.ts

@@ -1,13 +1,13 @@
 import { Anthropic, FunctionTool, ReActAgent } from "llamaindex";
 
 // Define a function to sum two numbers
-function sumNumbers({ a, b }: { a: number; b: number }): number {
-  return a + b;
+function sumNumbers({ a, b }: { a: number; b: number }) {
+  return `${a + b}`;
 }
 
 // Define a function to divide two numbers
-function divideNumbers({ a, b }: { a: number; b: number }): number {
-  return a / b;
+function divideNumbers({ a, b }: { a: number; b: number }) {
+  return `${a / b}`;
 }
 
 // Define the parameters of the sum function as a JSON schema
@@ -24,7 +24,7 @@ const sumJSON = {
     },
   },
   required: ["a", "b"],
-};
+} as const;
 
 const divideJSON = {
   type: "object",
@@ -39,7 +39,7 @@ const divideJSON = {
     },
   },
   required: ["a", "b"],
-};
+} as const;
 
 async function main() {
   // Create a function tool from the sum function
@@ -65,7 +65,6 @@ async function main() {
   const agent = new ReActAgent({
     llm: anthropic,
     tools: [functionTool, functionTool2],
-    verbose: true,
   });
 
   // Chat with the agent
@@ -77,6 +76,6 @@ async function main() {
   console.log(String(response));
 }
 
-main().then(() => {
+void main().then(() => {
   console.log("Done");
 });

examples/agent/step_wise_openai.ts

@@ -1,13 +1,13 @@
 import { FunctionTool, OpenAIAgent } from "llamaindex";
 
 // Define a function to sum two numbers
-function sumNumbers({ a, b }: { a: number; b: number }): number {
-  return a + b;
+function sumNumbers({ a, b }: { a: number; b: number }) {
+  return `${a + b}`;
 }
 
 // Define a function to divide two numbers
-function divideNumbers({ a, b }: { a: number; b: number }): number {
-  return a / b;
+function divideNumbers({ a, b }: { a: number; b: number }) {
+  return `${a / b}`;
 }
 
 // Define the parameters of the sum function as a JSON schema
@@ -24,22 +24,22 @@ const sumJSON = {
     },
   },
   required: ["a", "b"],
-};
+} as const;
 
 const divideJSON = {
   type: "object",
   properties: {
     a: {
       type: "number",
-      description: "The dividend a to divide",
+      description: "The dividend",
     },
     b: {
       type: "number",
-      description: "The divisor b to divide by",
+      description: "The divisor",
     },
   },
   required: ["a", "b"],
-};
+} as const;
 
 async function main() {
   // Create a function tool from the sum function
@@ -59,7 +59,6 @@ async function main() {
   // Create an OpenAIAgent with the function tools
   const agent = new OpenAIAgent({
     tools: [functionTool, functionTool2],
-    verbose: true,
   });
 
   // Create a task to sum and divide numbers
@@ -90,6 +89,6 @@ async function main() {
   }
 }
 
-main().then(() => {
+void main().then(() => {
   console.log("Done");
 });

examples/agent/step_wise_query_tool.ts

@@ -29,7 +29,6 @@ async function main() {
   // Create an OpenAIAgent with the function tools
   const agent = new OpenAIAgent({
     tools: [queryEngineTool],
-    verbose: true,
   });
 
   const task = agent.createTask("What was his salary?");
@@ -59,6 +58,6 @@ async function main() {
   }
 }
 
-main().then(() => {
+void main().then(() => {
   console.log("Done");
 });

examples/agent/step_wise_react.ts

@@ -1,13 +1,13 @@
 import { FunctionTool, ReActAgent } from "llamaindex";
 
 // Define a function to sum two numbers
-function sumNumbers({ a, b }: { a: number; b: number }): number {
-  return a + b;
+function sumNumbers({ a, b }: { a: number; b: number }) {
+  return `${a + b}`;
 }
 
 // Define a function to divide two numbers
-function divideNumbers({ a, b }: { a: number; b: number }): number {
-  return a / b;
+function divideNumbers({ a, b }: { a: number; b: number }) {
+  return `${a / b}`;
 }
 
 // Define the parameters of the sum function as a JSON schema
@@ -24,7 +24,7 @@ const sumJSON = {
     },
   },
   required: ["a", "b"],
-};
+} as const;
 
 const divideJSON = {
   type: "object",
@@ -39,7 +39,7 @@ const divideJSON = {
     },
   },
   required: ["a", "b"],
-};
+} as const;
 
 async function main() {
   // Create a function tool from the sum function
@@ -59,7 +59,6 @@ async function main() {
   // Create an OpenAIAgent with the function tools
   const agent = new ReActAgent({
     tools: [functionTool, functionTool2],
-    verbose: true,
   });
 
   const task = agent.createTask("Divide 16 by 2 then add 20");
@@ -85,6 +84,6 @@ async function main() {
   }
 }
 
-main().then(() => {
+void main().then(() => {
   console.log("Done");
 });

examples/agent/stream_openai_agent.ts

@@ -1,13 +1,13 @@
 import { FunctionTool, OpenAIAgent } from "llamaindex";
 
 // Define a function to sum two numbers
-function sumNumbers({ a, b }: { a: number; b: number }): number {
-  return a + b;
+function sumNumbers({ a, b }: { a: number; b: number }) {
+  return `${a + b}`;
 }
 
 // Define a function to divide two numbers
-function divideNumbers({ a, b }: { a: number; b: number }): number {
-  return a / b;
+function divideNumbers({ a, b }: { a: number; b: number }) {
+  return `${a / b}`;
 }
 
 // Define the parameters of the sum function as a JSON schema
@@ -24,7 +24,7 @@ const sumJSON = {
     },
   },
   required: ["a", "b"],
-};
+} as const;
 
 const divideJSON = {
   type: "object",
@@ -39,18 +39,18 @@ const divideJSON = {
     },
   },
   required: ["a", "b"],
-};
+} as const;
 
 async function main() {
   // Create a function tool from the sum function
-  const functionTool = new FunctionTool(sumNumbers, {
+  const functionTool = FunctionTool.from(sumNumbers, {
     name: "sumNumbers",
     description: "Use this function to sum two numbers",
     parameters: sumJSON,
   });
 
   // Create a function tool from the divide function
-  const functionTool2 = new FunctionTool(divideNumbers, {
+  const functionTool2 = FunctionTool.from(divideNumbers, {
     name: "divideNumbers",
     description: "Use this function to divide two numbers",
     parameters: divideJSON,
@@ -59,7 +59,6 @@ async function main() {
   // Create an OpenAIAgent with the function tools
   const agent = new OpenAIAgent({
     tools: [functionTool, functionTool2],
-    verbose: false,
   });
 
   const stream = await agent.chat({
@@ -72,6 +71,6 @@ async function main() {
   }
 }
 
-main().then(() => {
+void main().then(() => {
   console.log("\nDone");
 });

examples/agent/wiki.ts

@@ -0,0 +1,27 @@
+import { OpenAI, OpenAIAgent, WikipediaTool } from "llamaindex";
+
+async function main() {
+  const llm = new OpenAI({ model: "gpt-4-turbo" });
+  const wikiTool = new WikipediaTool();
+
+  // Create an OpenAIAgent with the Wikipedia tool
+  const agent = new OpenAIAgent({
+    llm,
+    tools: [wikiTool],
+  });
+
+  // Chat with the agent
+  const response = await agent.chat({
+    message: "Who was Goethe?",
+    stream: true,
+  });
+
+  for await (const chunk of response.response) {
+    process.stdout.write(chunk.response);
+  }
+}
+
+(async function () {
+  await main();
+  console.log("\nDone");
+})();

examples/agent/wikipedia-tool.ts

@@ -1,23 +0,0 @@
-import { OpenAIAgent, WikipediaTool } from "llamaindex";
-
-async function main() {
-  const wikipediaTool = new WikipediaTool();
-
-  // Create an OpenAIAgent with the function tools
-  const agent = new OpenAIAgent({
-    tools: [wikipediaTool],
-    verbose: true,
-  });
-
-  // Chat with the agent
-  const response = await agent.chat({
-    message: "Where is Ho Chi Minh City?",
-  });
-
-  // Print the response
-  console.log(response);
-}
-
-main().then(() => {
-  console.log("Done");
-});

examples/anthropic/agent.ts

@@ -0,0 +1,43 @@
+import { FunctionTool, Settings, WikipediaTool } from "llamaindex";
+import { AnthropicAgent } from "llamaindex/agent/anthropic";
+
+Settings.callbackManager.on("llm-tool-call", (event) => {
+  console.log("llm-tool-call", event.detail.payload.toolCall);
+});
+
+const agent = new AnthropicAgent({
+  tools: [
+    FunctionTool.from<{ location: string }>(
+      (query) => {
+        return `The weather in ${query.location} is sunny`;
+      },
+      {
+        name: "weather",
+        description: "Get the weather",
+        parameters: {
+          type: "object",
+          properties: {
+            location: {
+              type: "string",
+              description: "The location to get the weather for",
+            },
+          },
+          required: ["location"],
+        },
+      },
+    ),
+    new WikipediaTool(),
+  ],
+});
+
+async function main() {
+  // https://docs.anthropic.com/claude/docs/tool-use#tool-use-best-practices-and-limitations
+  const { response } = await agent.chat({
+    message:
+      "What is the weather in New York? What's the history of New York from Wikipedia in 3 sentences?",
+  });
+
+  console.log(response);
+}
+
+void main();

examples/astradb/example.ts

@@ -55,4 +55,4 @@ async function main() {
   }
 }
 
-main();
+void main();

examples/astradb/load.ts

@@ -27,4 +27,4 @@ async function main() {
   }
 }
 
-main();
+void main();

examples/astradb/query.ts

@@ -23,4 +23,4 @@ async function main() {
   }
 }
 
-main();
+void main();

examples/chatHistory.ts

@@ -1,7 +1,18 @@
 import { stdin as input, stdout as output } from "node:process";
 import readline from "node:readline/promises";
 
-import { OpenAI, SimpleChatEngine, SummaryChatHistory } from "llamaindex";
+import {
+  OpenAI,
+  Settings,
+  SimpleChatEngine,
+  SummaryChatHistory,
+} from "llamaindex";
+
+if (process.env.NODE_ENV === "development") {
+  Settings.callbackManager.on("llm-end", (event) => {
+    console.log("callers chain", event.reason?.computedCallers);
+  });
+}
 
 async function main() {
   // Set maxTokens to 75% of the context window size of 4096

examples/chromadb/preFilters.ts

@@ -54,4 +54,4 @@ async function main() {
   }
 }
 
-main();
+void main();

examples/chromadb/test.ts

@@ -37,4 +37,4 @@ async function main() {
   }
 }
 
-main();
+void main();

examples/cloud/from_documents.ts

@@ -0,0 +1,44 @@
+import fs from "node:fs/promises";
+
+import { stdin as input, stdout as output } from "node:process";
+
+import readline from "node:readline/promises";
+
+import { Document, LlamaCloudIndex } from "llamaindex";
+
+async function main() {
+  const path = "node_modules/llamaindex/examples/abramov.txt";
+
+  const essay = await fs.readFile(path, "utf-8");
+
+  // Create Document object with essay
+  const document = new Document({ text: essay, id_: path });
+
+  const index = await LlamaCloudIndex.fromDocuments({
+    documents: [document],
+    name: "test",
+    projectName: "default",
+    apiKey: process.env.LLAMA_CLOUD_API_KEY,
+    baseUrl: process.env.LLAMA_CLOUD_BASE_URL,
+  });
+
+  const queryEngine = index.asQueryEngine({
+    denseSimilarityTopK: 5,
+  });
+
+  const rl = readline.createInterface({ input, output });
+
+  while (true) {
+    const query = await rl.question("Query: ");
+    const stream = await queryEngine.query({
+      query,
+      stream: true,
+    });
+    console.log();
+    for await (const chunk of stream) {
+      process.stdout.write(chunk.response);
+    }
+  }
+}
+
+main().catch(console.error);

examples/evaluation/correctness.ts

@@ -22,4 +22,4 @@ However, general relativity, published in 1915, extended these ideas to include
   console.log(result);
 }
 
-main();
+void main();

examples/evaluation/faithfulness.ts

@@ -36,4 +36,4 @@ async function main() {
   console.log(result);
 }
 
-main();
+void main();

examples/evaluation/relevancy.ts

@@ -37,4 +37,4 @@ async function main() {
   console.log(result);
 }
 
-main();
+void main();

examples/jsonExtract.ts

@@ -36,9 +36,7 @@ async function main() {
     ],
   });
 
-  const json = JSON.parse(response.message.content);
-
-  console.log(json);
+  console.log(response.message.content);
 }
 
 main().catch(console.error);

examples/milvus/load.ts

@@ -23,4 +23,4 @@ async function main() {
   }
 }
 
-main();
+void main();

examples/milvus/query.ts

@@ -22,4 +22,4 @@ async function main() {
   }
 }
 
-main();
+void main();

examples/mongo.ts

@@ -61,4 +61,4 @@ async function main() {
   }
 }
 
-main();
+void main();

examples/mongodb/1_import.ts

@@ -31,4 +31,4 @@ async function importJsonToMongo() {
 }
 
 // Run the import function
-importJsonToMongo();
+void importJsonToMongo();

examples/mongodb/3_query.ts

@@ -27,4 +27,4 @@ async function query() {
   await client.close();
 }
 
-query();
+void query();

examples/multimodal/clip.ts

@@ -30,4 +30,4 @@ async function main() {
   console.log(`Similarity between "${text2}" and the image is ${sim2}`);
 }
 
-main();
+void main();

examples/multimodal/load.ts

@@ -4,6 +4,7 @@ import {
   VectorStoreIndex,
   storageContextFromDefaults,
 } from "llamaindex";
+import { DocStoreStrategy } from "llamaindex/ingestion/strategies/index";
 
 import * as path from "path";
 
@@ -31,6 +32,7 @@ async function generateDatasource() {
     });
     await VectorStoreIndex.fromDocuments(documents, {
       storageContext,
+      docStoreStrategy: DocStoreStrategy.NONE,
     });
   });
   console.log(`Storage successfully generated in ${ms / 1000}s.`);

examples/multimodal/rag.ts

@@ -13,7 +13,7 @@ Settings.chunkSize = 512;
 Settings.chunkOverlap = 20;
 
 // Update llm
-Settings.llm = new OpenAI({ model: "gpt-4-vision-preview", maxTokens: 512 });
+Settings.llm = new OpenAI({ model: "gpt-4-turbo", maxTokens: 512 });
 
 // Update callbackManager
 Settings.callbackManager = new CallbackManager({

examples/nodeParser/MarkdownNodeParser.ts

@@ -21,4 +21,4 @@ Sub-header content
   console.log(splits);
 }
 
-main();
+void main();

examples/package.json

@@ -5,21 +5,22 @@
   "dependencies": {
     "@aws-crypto/sha256-js": "^5.2.0",
     "@datastax/astra-db-ts": "^0.1.4",
-    "@notionhq/client": "^2.2.14",
+    "@notionhq/client": "^2.2.15",
     "@pinecone-database/pinecone": "^1.1.3",
     "@zilliz/milvus2-sdk-node": "^2.3.5",
     "chromadb": "^1.8.1",
     "commander": "^11.1.0",
-    "dotenv": "^16.4.1",
+    "dotenv": "^16.4.5",
     "js-tiktoken": "^1.0.10",
     "llamaindex": "latest",
-    "mongodb": "^6.2.0",
+    "mongodb": "^6.5.0",
     "pathe": "^1.1.2"
   },
   "devDependencies": {
-    "@types/node": "^18.19.10",
+    "@types/node": "^20.12.7",
     "ts-node": "^10.9.2",
-    "typescript": "^5.4.3"
+    "tsx": "^4.7.2",
+    "typescript": "^5.4.5"
   },
   "scripts": {
     "lint": "eslint ."

examples/pg-vector-store/load-docs.ts

@@ -32,7 +32,7 @@ async function main(args: any) {
   console.log(`Found ${count} files`);
 
   console.log(`Importing contents from ${count} files in ${sourceDir}`);
-  var fileName = "";
+  const fileName = "";
   try {
     // Passing callback fn to the ctor here
     // will enable looging to console.
@@ -42,7 +42,7 @@ async function main(args: any) {
 
     const pgvs = new PGVectorStore();
     pgvs.setCollection(sourceDir);
-    pgvs.clearCollection();
+    await pgvs.clearCollection();
 
     const ctx = await storageContextFromDefaults({ vectorStore: pgvs });
 
@@ -65,4 +65,4 @@ async function main(args: any) {
   process.exit(0);
 }
 
-main(process.argv).catch((err) => console.error(err));
+void main(process.argv).catch((err) => console.error(err));

examples/pinecone-vector-store/load-docs.ts

@@ -32,7 +32,7 @@ async function main(args: any) {
   console.log(`Found ${count} files`);
 
   console.log(`Importing contents from ${count} files in ${sourceDir}`);
-  var fileName = "";
+  const fileName = "";
   try {
     // Passing callback fn to the ctor here
     // will enable looging to console.
@@ -63,4 +63,4 @@ async function main(args: any) {
   process.exit(0);
 }
 
-main(process.argv).catch((err) => console.error(err));
+void main(process.argv).catch((err) => console.error(err));

examples/prompts/promptMixin.ts

@@ -45,4 +45,4 @@ async function main() {
   await queryEngine.query({ query });
 }
 
-main();
+void main();

examples/qdrantdb/preFilters.ts

@@ -79,4 +79,4 @@ async function main() {
   }
 }
 
-main();
+void main();

examples/readers/package.json

@@ -15,7 +15,7 @@
     "llamaindex": "latest"
   },
   "devDependencies": {
-    "@types/node": "^20.11.14",
+    "@types/node": "^20.12.7",
     "ts-node": "^10.9.2",
     "typescript": "^5.4.3"
   }

examples/readers/src/docx.ts

@@ -20,4 +20,4 @@ async function main() {
   console.log(`Test query > ${SAMPLE_QUERY}:\n`, response.toString());
 }
 
-main();
+void main();

examples/readers/src/markdown.ts

@@ -20,4 +20,4 @@ async function main() {
   console.log(`Test query > ${SAMPLE_QUERY}:\n`, response.toString());
 }
 
-main();
+void main();

examples/recipes/cost-analysis.ts

@@ -1,11 +1,13 @@
 import { encodingForModel } from "js-tiktoken";
 import { OpenAI } from "llamaindex";
 import { Settings } from "llamaindex/Settings";
+import { extractText } from "llamaindex/llm/utils";
 
 const encoding = encodingForModel("gpt-4-0125-preview");
 
 const llm = new OpenAI({
-  model: "gpt-4-0125-preview",
+  // currently is "gpt-4-turbo-2024-04-09"
+  model: "gpt-4-turbo",
 });
 
 let tokenCount = 0;
@@ -13,25 +15,32 @@ let tokenCount = 0;
 Settings.callbackManager.on("llm-start", (event) => {
   const { messages } = event.detail.payload;
   tokenCount += messages.reduce((count, message) => {
-    return count + encoding.encode(message.content).length;
+    return count + encoding.encode(extractText(message.content)).length;
   }, 0);
   console.log("Token count:", tokenCount);
   // https://openai.com/pricing
   // $10.00 / 1M tokens
-  console.log(`Price: $${(tokenCount / 1_000_000) * 10}`);
-});
-Settings.callbackManager.on("llm-end", (event) => {
-  const { response } = event.detail.payload;
-  tokenCount += encoding.encode(response.message.content).length;
-  console.log("Token count:", tokenCount);
-  // https://openai.com/pricing
-  // $30.00 / 1M tokens
-  console.log(`Price: $${(tokenCount / 1_000_000) * 30}`);
+  console.log(`Total Price: $${(tokenCount / 1_000_000) * 10}`);
 });
 
-const question = "Hello, how are you?";
+Settings.callbackManager.on("llm-stream", (event) => {
+  const { chunk } = event.detail.payload;
+  const { delta } = chunk;
+  tokenCount += encoding.encode(extractText(delta)).length;
+  if (tokenCount > 20) {
+    // This is just an example, you can set your own limit or handle it differently
+    throw new Error("Token limit exceeded!");
+  }
+});
+Settings.callbackManager.on("llm-end", () => {
+  // https://openai.com/pricing
+  // $30.00 / 1M tokens
+  console.log(`Total Price: $${(tokenCount / 1_000_000) * 30}`);
+});
+
+const question = "Hello, how are you? Please response about 50 tokens.";
 console.log("Question:", question);
-llm
+void llm
   .chat({
     stream: true,
     messages: [

examples/routerQueryEngine.ts

@@ -65,4 +65,4 @@ async function main() {
   });
 }
 
-main().then(() => console.log("Done"));
+void main().then(() => console.log("Done"));

examples/split.ts

@@ -13,4 +13,4 @@ async function main() {
   console.log(chunks);
 }
 
-main();
+void main();

examples/toolsStream.ts

@@ -0,0 +1,48 @@
+import { OpenAI } from "llamaindex";
+
+async function main() {
+  const llm = new OpenAI({ model: "gpt-4-turbo" });
+  const args: Parameters<typeof llm.chat>[0] = {
+    additionalChatOptions: {
+      tool_choice: "auto",
+    },
+    messages: [
+      {
+        content: "Who was Goethe?",
+        role: "user",
+      },
+    ],
+    tools: [
+      {
+        metadata: {
+          name: "wikipedia_tool",
+          description: "A tool that uses a query engine to search Wikipedia.",
+          parameters: {
+            type: "object",
+            properties: {
+              query: {
+                type: "string",
+                description: "The query to search for",
+              },
+            },
+            required: ["query"],
+          },
+        },
+      },
+    ],
+  };
+
+  const stream = await llm.chat({ ...args, stream: true });
+  for await (const chunk of stream) {
+    process.stdout.write(chunk.delta);
+    if (chunk.options && "toolCall" in chunk.options) {
+      console.log("Tool call:");
+      console.log(chunk.options.toolCall);
+    }
+  }
+}
+
+(async function () {
+  await main();
+  console.log("Done");
+})();

examples/vectorIndex.ts

@@ -1,6 +1,11 @@
 import fs from "node:fs/promises";
 
-import { Document, VectorStoreIndex } from "llamaindex";
+import {
+  Document,
+  MetadataMode,
+  NodeWithScore,
+  VectorStoreIndex,
+} from "llamaindex";
 
 async function main() {
   // Load essay from abramov.txt in Node
@@ -16,12 +21,20 @@ async function main() {
 
   // Query the index
   const queryEngine = index.asQueryEngine();
-  const response = await queryEngine.query({
+  const { response, sourceNodes } = await queryEngine.query({
     query: "What did the author do in college?",
   });
 
-  // Output response
-  console.log(response.toString());
+  // Output response with sources
+  console.log(response);
+
+  if (sourceNodes) {
+    sourceNodes.forEach((source: NodeWithScore, index: number) => {
+      console.log(
+        `\n${index}: Score: ${source.score} - ${source.node.getContent(MetadataMode.NONE).substring(0, 50)}...\n`,
+      );
+    });
+  }
 }
 
 main().catch(console.error);

examples/vision.ts

@@ -1,7 +1,7 @@
 import { OpenAI } from "llamaindex";
 
 (async () => {
-  const llm = new OpenAI({ model: "gpt-4-vision-preview", temperature: 0.1 });
+  const llm = new OpenAI({ model: "gpt-4-turbo", temperature: 0.1 });
 
   // complete api
   const response1 = await llm.complete({ prompt: "How are you?" });

package.json

@@ -9,31 +9,36 @@
     "format:write": "prettier --ignore-unknown --write .",
     "lint": "turbo run lint",
     "prepare": "husky",
+    "e2e": "turbo run e2e",
     "test": "turbo run test",
     "type-check": "tsc -b --diagnostics",
     "release": "pnpm run check-minor-version && pnpm run build:release && changeset publish",
     "release-snapshot": "pnpm run check-minor-version && pnpm run build:release && changeset publish --tag snapshot",
     "check-minor-version": "node ./scripts/check-minor-version",
-    "update-version": "node ./scripts/update-version",
-    "new-version": "pnpm run build:release && changeset version && pnpm run check-minor-version && pnpm run update-version",
-    "new-snapshot": "pnpm run build:release && changeset version --snapshot && pnpm run update-version"
+    "new-version": "changeset version && pnpm run check-minor-version && pnpm run build:release",
+    "new-snapshot": "pnpm run build:release && changeset version --snapshot"
   },
   "devDependencies": {
     "@changesets/cli": "^2.27.1",
-    "eslint": "^8.56.0",
-    "eslint-config-custom": "workspace:*",
-    "husky": "^9.0.10",
+    "@typescript-eslint/eslint-plugin": "^7.7.0",
+    "eslint": "^8.57.0",
+    "eslint-config-next": "^13.5.6",
+    "eslint-config-prettier": "^8.10.0",
+    "eslint-config-turbo": "^1.13.2",
+    "eslint-plugin-react": "7.28.0",
+    "husky": "^9.0.11",
     "lint-staged": "^15.2.2",
     "prettier": "^3.2.5",
     "prettier-plugin-organize-imports": "^3.2.4",
-    "turbo": "^1.12.3",
-    "typescript": "^5.4.3"
+    "turbo": "^1.13.2",
+    "typescript": "^5.4.5"
   },
-  "packageManager": "pnpm@8.15.1",
+  "packageManager": "pnpm@9.0.1+sha256.46d50ee2afecb42b185ebbd662dc7bdd52ef5be56bf035bb615cab81a75345df",
   "pnpm": {
     "overrides": {
       "trim": "1.0.1",
-      "@babel/traverse": "7.23.2"
+      "@babel/traverse": "7.23.2",
+      "protobufjs": "7.2.6"
     }
   },
   "lint-staged": {

packages/core/CHANGELOG.md

@@ -1,5 +1,57 @@
 # llamaindex
 
+## 0.2.9
+
+### Patch Changes
+
+- 76c3fd6: Add score to source nodes response
+- 208282d: feat: init anthropic agent
+
+  remove the `tool` | `function` type in `MessageType`. Replace with `assistant` instead.
+  This is because these two types are only available for `OpenAI`.
+  Since `OpenAI` deprecates the function type, we support the Claude 3 tool call.
+
+## 0.2.8
+
+### Patch Changes
+
+- Add ToolsFactory to generate agent tools
+
+## 0.2.7
+
+### Patch Changes
+
+- 96f8f40: fix: agent stream
+- Updated dependencies
+  - @llamaindex/env@0.0.7
+
+## 0.2.6
+
+### Patch Changes
+
+- a3b4409: Fix agent streaming with new OpenAI models
+
+## 0.2.5
+
+### Patch Changes
+
+- 7d56cdf: Allow OpenAIAgent to be called without tools
+
+## 0.2.4
+
+### Patch Changes
+
+- 3bc77f7: gpt-4-turbo GA
+- 8d2b21e: Mistral 0.1.3
+
+## 0.2.3
+
+### Patch Changes
+
+- f0704ec: Support streaming for OpenAI agent (and OpenAI tool calls)
+- Removed 'parentEvent' - Use 'event.reason?.computedCallers' instead
+- 3cbfa98: Added LlamaCloudIndex.fromDocuments
+
 ## 0.2.2
 
 ### Patch Changes

packages/core/e2e/.gitignore

@@ -0,0 +1 @@
+logs

packages/core/e2e/README.md

@@ -0,0 +1,38 @@
+# LlamaIndexTS Core E2E Tests
+
+## Overview
+
+We are using Node.js Test Runner to run E2E tests for LlamaIndexTS Core.
+
+It supports the following features:
+
+- Run tests in parallel
+- Pure Node.js Environment
+- Switch between mock and real LLM API
+- Customizable logics
+
+## Usage
+
+- Run with mock register:
+
+```shell
+node --import tsx --import ./mock-register.js --test ./node/basic.e2e.ts
+```
+
+- Run without mock register:
+
+```shell
+node --import tsx --test ./node/basic.e2e.ts
+```
+
+- Run with specific test:
+
+```shell
+node --import tsx --import ./mock-register.js --test-name-pattern=agent --test ./node/basic.e2e.ts
+```
+
+- Run with debug logs:
+
+```shell
+CONSOLA_LEVEL=5 node --import tsx --import ./mock-register.js --test-name-pattern=agent --test ./node/basic.e2e.ts
+```

packages/core/e2e/fixtures/embeddings/OpenAIEmbedding.ts

@@ -0,0 +1,34 @@
+import { BaseNode, SimilarityType, type BaseEmbedding } from "llamaindex";
+
+export class OpenAIEmbedding implements BaseEmbedding {
+  embedBatchSize = 512;
+
+  async getQueryEmbedding(query: string) {
+    return [0];
+  }
+
+  async getTextEmbedding(text: string) {
+    return [0];
+  }
+
+  async getTextEmbeddings(texts: string[]) {
+    return [[0]];
+  }
+
+  async getTextEmbeddingsBatch(texts: string[]) {
+    return [[0]];
+  }
+
+  similarity(
+    embedding1: number[],
+    embedding2: number[],
+    mode?: SimilarityType,
+  ) {
+    return 1;
+  }
+
+  async transform(nodes: BaseNode[], _options?: any): Promise<BaseNode[]> {
+    nodes.forEach((node) => (node.embedding = [0]));
+    return nodes;
+  }
+}

packages/core/e2e/fixtures/llm/anthropic.ts

@@ -0,0 +1,3 @@
+import { OpenAI } from "./open_ai.js";
+
+export class Anthropic extends OpenAI {}

packages/core/e2e/fixtures/llm/open_ai.ts

@@ -0,0 +1,105 @@
+import type {
+  ChatResponse,
+  ChatResponseChunk,
+  CompletionResponse,
+  LLM,
+  LLMChatParamsNonStreaming,
+  LLMChatParamsStreaming,
+  LLMCompletionParamsNonStreaming,
+  LLMCompletionParamsStreaming,
+} from "llamaindex/llm/types";
+import { extractText } from "llamaindex/llm/utils";
+import { deepStrictEqual, strictEqual } from "node:assert";
+import { llmCompleteMockStorage } from "../../node/utils.js";
+
+export function getOpenAISession() {
+  return {};
+}
+
+export function isFunctionCallingModel() {
+  return true;
+}
+
+export class OpenAI implements LLM {
+  supportToolCall = true;
+  get metadata() {
+    return {
+      model: "mock-model",
+      temperature: 0.1,
+      topP: 1,
+      contextWindow: 2048,
+      tokenizer: undefined,
+      isFunctionCallingModel: true,
+    };
+  }
+  chat(
+    params: LLMChatParamsStreaming<Record<string, unknown>>,
+  ): Promise<AsyncIterable<ChatResponseChunk>>;
+  chat(
+    params: LLMChatParamsNonStreaming<Record<string, unknown>>,
+  ): Promise<ChatResponse>;
+  chat(
+    params:
+      | LLMChatParamsStreaming<Record<string, unknown>>
+      | LLMChatParamsNonStreaming<Record<string, unknown>>,
+  ): unknown {
+    if (llmCompleteMockStorage.llmEventStart.length > 0) {
+      const chatMessage =
+        llmCompleteMockStorage.llmEventStart.shift()!["messages"];
+      strictEqual(chatMessage.length, params.messages.length);
+      for (let i = 0; i < chatMessage.length; i++) {
+        strictEqual(chatMessage[i].role, params.messages[i].role);
+        deepStrictEqual(chatMessage[i].content, params.messages[i].content);
+      }
+
+      if (llmCompleteMockStorage.llmEventEnd.length > 0) {
+        const { id, response } = llmCompleteMockStorage.llmEventEnd.shift()!;
+        if (params.stream) {
+          return {
+            [Symbol.asyncIterator]: async function* () {
+              while (true) {
+                const idx = llmCompleteMockStorage.llmEventStream.findIndex(
+                  (e) => e.id === id,
+                );
+                if (idx === -1) {
+                  break;
+                }
+                const chunk = llmCompleteMockStorage.llmEventStream[idx].chunk;
+                llmCompleteMockStorage.llmEventStream.splice(idx, 1);
+                yield chunk;
+              }
+            },
+          };
+        } else {
+          return response;
+        }
+      }
+    }
+    throw new Error("Method not implemented.");
+  }
+  complete(
+    params: LLMCompletionParamsStreaming,
+  ): Promise<AsyncIterable<CompletionResponse>>;
+  complete(
+    params: LLMCompletionParamsNonStreaming,
+  ): Promise<CompletionResponse>;
+  async complete(
+    params: LLMCompletionParamsStreaming | LLMCompletionParamsNonStreaming,
+  ): Promise<AsyncIterable<CompletionResponse> | CompletionResponse> {
+    if (llmCompleteMockStorage.llmEventStart.length > 0) {
+      const chatMessage =
+        llmCompleteMockStorage.llmEventStart.shift()!["messages"];
+      strictEqual(chatMessage.length, 1);
+      strictEqual(chatMessage[0].role, "user");
+      strictEqual(chatMessage[0].content, params.prompt);
+    }
+    if (llmCompleteMockStorage.llmEventEnd.length > 0) {
+      const response = llmCompleteMockStorage.llmEventEnd.shift()!["response"];
+      return {
+        raw: response,
+        text: extractText(response.message.content),
+      } satisfies CompletionResponse;
+    }
+    throw new Error("Method not implemented.");
+  }
+}

packages/core/e2e/mock-module.js

@@ -0,0 +1,36 @@
+/**
+ * This script will replace the resolved module with the corresponding fixture file.
+ */
+import { stat } from "node:fs/promises";
+import { join, relative } from "node:path";
+import { fileURLToPath, pathToFileURL } from "node:url";
+const packageDistDir = fileURLToPath(new URL("../dist", import.meta.url));
+const fixturesDir = fileURLToPath(new URL("./fixtures", import.meta.url));
+
+export async function resolve(specifier, context, nextResolve) {
+  const result = await nextResolve(specifier, context);
+  if (result.format === "builtin" || result.url.startsWith("node:")) {
+    return result;
+  }
+  const targetUrl = fileURLToPath(result.url).replace(/\.js$/, ".ts");
+  const relativePath = relative(packageDistDir, targetUrl);
+  if (relativePath.startsWith(".") || relativePath.startsWith("/")) {
+    return result;
+  }
+  const url = pathToFileURL(join(fixturesDir, relativePath)).toString();
+  const exist = await stat(fileURLToPath(url))
+    .then((stat) => stat.isFile())
+    .catch((err) => {
+      if (err.code === "ENOENT") {
+        return false;
+      }
+      throw err;
+    });
+  if (!exist) {
+    return result;
+  }
+  return {
+    url,
+    format: "module",
+  };
+}

packages/core/e2e/mock-register.js

@@ -0,0 +1,3 @@
+import { register } from "node:module";
+
+register("./mock-module.js", import.meta.url);

packages/core/e2e/node/claude.e2e.ts

@@ -0,0 +1,129 @@
+import { consola } from "consola";
+import { Anthropic, FunctionTool, Settings, type LLM } from "llamaindex";
+import { AnthropicAgent } from "llamaindex/agent/anthropic";
+import { ok } from "node:assert";
+import { beforeEach, test } from "node:test";
+import { sumNumbersTool } from "./fixtures/tools.js";
+import { mockLLMEvent } from "./utils.js";
+
+let llm: LLM;
+beforeEach(async () => {
+  Settings.llm = new Anthropic({
+    model: "claude-3-opus",
+  });
+  llm = Settings.llm;
+});
+
+await test("anthropic llm", async (t) => {
+  await mockLLMEvent(t, "llm-anthropic");
+  await t.test("llm.chat", async () => {
+    const response = await llm.chat({
+      messages: [
+        {
+          content: "Hello",
+          role: "user",
+          options: {},
+        },
+      ],
+    });
+    consola.debug("response:", response);
+    ok(typeof response.message.content === "string");
+  });
+
+  await t.test("stream llm.chat", async () => {
+    const iter = await llm.chat({
+      stream: true,
+      messages: [
+        {
+          content: "hello",
+          role: "user",
+        },
+      ],
+    });
+    for await (const chunk of iter) {
+      consola.debug("chunk:", chunk);
+      ok(typeof chunk.delta === "string");
+    }
+  });
+});
+
+await test("anthropic agent", async (t) => {
+  await mockLLMEvent(t, "anthropic-agent");
+  await t.test("chat", async () => {
+    const agent = new AnthropicAgent({
+      tools: [
+        {
+          call: async () => {
+            return "35 degrees and sunny in San Francisco";
+          },
+          metadata: {
+            name: "Weather",
+            description: "Get the weather",
+            parameters: {
+              type: "object",
+              properties: {
+                location: { type: "string" },
+              },
+              required: ["location"],
+            },
+          },
+        },
+      ],
+    });
+    const result = await agent.chat({
+      message: "What is the weather in San Francisco?",
+    });
+    consola.debug("response:", result.response);
+    ok(typeof result.response === "string");
+    ok(result.response.includes("35"));
+  });
+
+  await t.test("async function", async () => {
+    const uniqueId = "123456789";
+    const showUniqueId = FunctionTool.from<{
+      firstName: string;
+      lastName: string;
+    }>(
+      async ({ firstName, lastName }) => {
+        ok(typeof firstName === "string");
+        ok(typeof lastName === "string");
+        const fullName = firstName + lastName;
+        ok(fullName.toLowerCase().includes("alex"));
+        ok(fullName.toLowerCase().includes("yang"));
+        return uniqueId;
+      },
+      {
+        name: "unique_id",
+        description: "show user unique id",
+        parameters: {
+          type: "object",
+          properties: {
+            firstName: { type: "string" },
+            lastName: { type: "string" },
+          },
+          required: ["firstName", "lastName"],
+        },
+      },
+    );
+    const agent = new AnthropicAgent({
+      tools: [showUniqueId],
+    });
+    const { response } = await agent.chat({
+      message: "My name is Alex Yang. What is my unique id?",
+    });
+    consola.debug("response:", response);
+    ok(response.includes(uniqueId));
+  });
+
+  await t.test("sum numbers", async () => {
+    const openaiAgent = new AnthropicAgent({
+      tools: [sumNumbersTool],
+    });
+
+    const response = await openaiAgent.chat({
+      message: "how much is 1 + 1?",
+    });
+
+    ok(response.response.includes("2"));
+  });
+});

packages/core/e2e/node/fixtures/tools.ts

@@ -0,0 +1,47 @@
+import { FunctionTool } from "llamaindex";
+
+function sumNumbers({ a, b }: { a: number; b: number }) {
+  return `${a + b}`;
+}
+
+function divideNumbers({ a, b }: { a: number; b: number }) {
+  return `${a / b}`;
+}
+
+export const sumNumbersTool = FunctionTool.from(sumNumbers, {
+  name: "sumNumbers",
+  description: "Use this function to sum two numbers",
+  parameters: {
+    type: "object",
+    properties: {
+      a: {
+        type: "number",
+        description: "The first number",
+      },
+      b: {
+        type: "number",
+        description: "The second number",
+      },
+    },
+    required: ["a", "b"],
+  },
+});
+
+export const divideNumbersTool = FunctionTool.from(divideNumbers, {
+  name: "divideNumbers",
+  description: "Use this function to divide two numbers",
+  parameters: {
+    type: "object",
+    properties: {
+      a: {
+        type: "number",
+        description: "The first number",
+      },
+      b: {
+        type: "number",
+        description: "The second number",
+      },
+    },
+    required: ["a", "b"],
+  },
+});

packages/core/e2e/node/openai.e2e.ts

@@ -0,0 +1,233 @@
+import { consola } from "consola";
+import {
+  Document,
+  FunctionTool,
+  OpenAI,
+  OpenAIAgent,
+  QueryEngineTool,
+  Settings,
+  SubQuestionQueryEngine,
+  VectorStoreIndex,
+  type LLM,
+} from "llamaindex";
+import { ok, strictEqual } from "node:assert";
+import { beforeEach, test } from "node:test";
+import { divideNumbersTool, sumNumbersTool } from "./fixtures/tools.js";
+import { mockLLMEvent } from "./utils.js";
+
+let llm: LLM;
+beforeEach(async () => {
+  Settings.llm = new OpenAI({
+    model: "gpt-3.5-turbo",
+  });
+  llm = Settings.llm;
+});
+
+await test("openai llm", async (t) => {
+  await mockLLMEvent(t, "llm");
+  await t.test("llm.chat", async () => {
+    const response = await llm.chat({
+      messages: [
+        {
+          content: "Hello",
+          role: "user",
+        },
+      ],
+    });
+    consola.debug("response:", response);
+    ok(typeof response.message.content === "string");
+  });
+
+  await t.test("stream llm.chat", async () => {
+    const iter = await llm.chat({
+      stream: true,
+      messages: [
+        {
+          content: "hello",
+          role: "user",
+        },
+      ],
+    });
+    for await (const chunk of iter) {
+      consola.debug("chunk:", chunk);
+      ok(typeof chunk.delta === "string");
+    }
+  });
+});
+
+await test("gpt-4-turbo", async (t) => {
+  const llm = new OpenAI({ model: "gpt-4-turbo" });
+  Settings.llm = llm;
+  await mockLLMEvent(t, "gpt-4-turbo");
+  await t.test("agent", async () => {
+    const agent = new OpenAIAgent({
+      llm,
+      tools: [
+        {
+          call: async () => {
+            return "45 degrees and sunny in San Jose";
+          },
+          metadata: {
+            name: "Weather",
+            description: "Get the weather",
+            parameters: {
+              type: "object",
+              properties: {
+                location: { type: "string" },
+              },
+              required: ["location"],
+            },
+          },
+        },
+      ],
+    });
+    const { response } = await agent.chat({
+      message: "What is the weather in San Jose?",
+    });
+    consola.debug("response:", response);
+    ok(typeof response === "string");
+    ok(response.includes("45"));
+  });
+});
+
+await test("agent", async (t) => {
+  await mockLLMEvent(t, "agent");
+  await t.test("chat", async () => {
+    const agent = new OpenAIAgent({
+      tools: [
+        {
+          call: async () => {
+            return "35 degrees and sunny in San Francisco";
+          },
+          metadata: {
+            name: "Weather",
+            description: "Get the weather",
+            parameters: {
+              type: "object",
+              properties: {
+                location: { type: "string" },
+              },
+              required: ["location"],
+            },
+          },
+        },
+      ],
+    });
+    const result = await agent.chat({
+      message: "What is the weather in San Francisco?",
+    });
+    consola.debug("response:", result.response);
+    ok(typeof result.response === "string");
+    ok(result.response.includes("35"));
+  });
+
+  await t.test("async function", async () => {
+    const uniqueId = "123456789";
+    const showUniqueId = FunctionTool.from<{
+      firstName: string;
+      lastName: string;
+    }>(
+      async ({ firstName, lastName }) => {
+        ok(typeof firstName === "string");
+        ok(typeof lastName === "string");
+        const fullName = firstName + lastName;
+        ok(fullName.toLowerCase().includes("alex"));
+        ok(fullName.toLowerCase().includes("yang"));
+        return uniqueId;
+      },
+      {
+        name: "unique_id",
+        description: "show user unique id",
+        parameters: {
+          type: "object",
+          properties: {
+            firstName: { type: "string" },
+            lastName: { type: "string" },
+          },
+          required: ["firstName", "lastName"],
+        },
+      },
+    );
+    const agent = new OpenAIAgent({
+      tools: [showUniqueId],
+    });
+    const { response } = await agent.chat({
+      message: "My name is Alex Yang. What is my unique id?",
+    });
+    consola.debug("response:", response);
+    ok(response.includes(uniqueId));
+  });
+
+  await t.test("sum numbers", async () => {
+    const openaiAgent = new OpenAIAgent({
+      tools: [sumNumbersTool],
+    });
+
+    const response = await openaiAgent.chat({
+      message: "how much is 1 + 1?",
+    });
+
+    ok(response.response.includes("2"));
+  });
+});
+
+await test("agent stream", async (t) => {
+  await mockLLMEvent(t, "agent_stream");
+  await t.test("sum numbers stream", async (t) => {
+    const fn = t.mock.fn(() => {});
+    Settings.callbackManager.on("llm-tool-call", fn);
+
+    const agent = new OpenAIAgent({
+      tools: [sumNumbersTool, divideNumbersTool],
+    });
+
+    const { response } = await agent.chat({
+      message: "Divide 16 by 2 then add 20",
+      stream: true,
+    });
+
+    let message = "";
+
+    for await (const chunk of response) {
+      message += chunk.response;
+    }
+
+    strictEqual(fn.mock.callCount(), 2);
+    ok(message.includes("28"));
+    Settings.callbackManager.off("llm-tool-call", fn);
+  });
+});
+
+await test("queryEngine", async (t) => {
+  await mockLLMEvent(t, "queryEngine_subquestion");
+  await t.test("subquestion", async () => {
+    const fn = t.mock.fn(() => {});
+    Settings.callbackManager.on("llm-tool-call", fn);
+    const document = new Document({
+      text: "Bill Gates stole from Apple.\n Steve Jobs stole from Xerox.",
+    });
+    const index = await VectorStoreIndex.fromDocuments([document]);
+
+    const queryEngineTools = [
+      new QueryEngineTool({
+        queryEngine: index.asQueryEngine(),
+        metadata: {
+          name: "bill_gates_idea",
+          description: "Get what Bill Gates idea from.",
+        },
+      }),
+    ];
+
+    const queryEngine = SubQuestionQueryEngine.fromDefaults({
+      queryEngineTools,
+    });
+
+    const { response } = await queryEngine.query({
+      query: "What did Bill Gates steal from?",
+    });
+
+    ok(response.includes("Apple"));
+    strictEqual(fn.mock.callCount(), 0);
+    Settings.callbackManager.off("llm-tool-call", fn);
+  });
+});

packages/core/e2e/node/snapshot/agent.snap

@@ -0,0 +1,370 @@
+{
+  "llmEventStart": [
+    {
+      "id": "PRESERVE_0",
+      "messages": [
+        {
+          "content": "What is the weather in San Francisco?",
+          "role": "user"
+        }
+      ]
+    },
+    {
+      "id": "PRESERVE_1",
+      "messages": [
+        {
+          "content": "What is the weather in San Francisco?",
+          "role": "user"
+        },
+        {
+          "content": "",
+          "role": "assistant",
+          "options": {
+            "toolCall": {
+              "id": "HIDDEN",
+              "name": "Weather",
+              "input": "{\"location\":\"San Francisco\"}"
+            }
+          }
+        },
+        {
+          "content": "35 degrees and sunny in San Francisco",
+          "role": "user",
+          "options": {
+            "toolResult": {
+              "id": "HIDDEN",
+              "isError": false
+            }
+          }
+        }
+      ]
+    },
+    {
+      "id": "PRESERVE_2",
+      "messages": [
+        {
+          "content": "My name is Alex Yang. What is my unique id?",
+          "role": "user"
+        }
+      ]
+    },
+    {
+      "id": "PRESERVE_3",
+      "messages": [
+        {
+          "content": "My name is Alex Yang. What is my unique id?",
+          "role": "user"
+        },
+        {
+          "content": "",
+          "role": "assistant",
+          "options": {
+            "toolCall": {
+              "id": "HIDDEN",
+              "name": "unique_id",
+              "input": "{\"firstName\":\"Alex\",\"lastName\":\"Yang\"}"
+            }
+          }
+        },
+        {
+          "content": "123456789",
+          "role": "user",
+          "options": {
+            "toolResult": {
+              "id": "HIDDEN",
+              "isError": false
+            }
+          }
+        }
+      ]
+    },
+    {
+      "id": "PRESERVE_4",
+      "messages": [
+        {
+          "content": "how much is 1 + 1?",
+          "role": "user"
+        }
+      ]
+    },
+    {
+      "id": "PRESERVE_5",
+      "messages": [
+        {
+          "content": "how much is 1 + 1?",
+          "role": "user"
+        },
+        {
+          "content": "",
+          "role": "assistant",
+          "options": {
+            "toolCall": {
+              "id": "HIDDEN",
+              "name": "sumNumbers",
+              "input": "{\"a\":1,\"b\":1}"
+            }
+          }
+        },
+        {
+          "content": "2",
+          "role": "user",
+          "options": {
+            "toolResult": {
+              "id": "HIDDEN",
+              "isError": false
+            }
+          }
+        }
+      ]
+    }
+  ],
+  "llmEventEnd": [
+    {
+      "id": "PRESERVE_0",
+      "response": {
+        "raw": {
+          "id": "HIDDEN",
+          "object": "chat.completion",
+          "created": 114514,
+          "model": "gpt-3.5-turbo-0125",
+          "choices": [
+            {
+              "index": 0,
+              "message": {
+                "role": "assistant",
+                "content": null,
+                "tool_calls": [
+                  {
+                    "id": "HIDDEN",
+                    "type": "function",
+                    "function": {
+                      "name": "Weather",
+                      "arguments": "{\"location\":\"San Francisco\"}"
+                    }
+                  }
+                ]
+              },
+              "logprobs": null,
+              "finish_reason": "tool_calls"
+            }
+          ],
+          "usage": {
+            "prompt_tokens": 49,
+            "completion_tokens": 15,
+            "total_tokens": 64
+          },
+          "system_fingerprint": "HIDDEN"
+        },
+        "message": {
+          "content": "",
+          "role": "assistant",
+          "options": {
+            "toolCall": {
+              "id": "HIDDEN",
+              "name": "Weather",
+              "input": "{\"location\":\"San Francisco\"}"
+            }
+          }
+        }
+      }
+    },
+    {
+      "id": "PRESERVE_1",
+      "response": {
+        "raw": {
+          "id": "HIDDEN",
+          "object": "chat.completion",
+          "created": 114514,
+          "model": "gpt-3.5-turbo-0125",
+          "choices": [
+            {
+              "index": 0,
+              "message": {
+                "role": "assistant",
+                "content": "The weather in San Francisco is currently 35 degrees and sunny."
+              },
+              "logprobs": null,
+              "finish_reason": "stop"
+            }
+          ],
+          "usage": {
+            "prompt_tokens": 78,
+            "completion_tokens": 14,
+            "total_tokens": 92
+          },
+          "system_fingerprint": "HIDDEN"
+        },
+        "message": {
+          "content": "The weather in San Francisco is currently 35 degrees and sunny.",
+          "role": "assistant",
+          "options": {}
+        }
+      }
+    },
+    {
+      "id": "PRESERVE_2",
+      "response": {
+        "raw": {
+          "id": "HIDDEN",
+          "object": "chat.completion",
+          "created": 114514,
+          "model": "gpt-3.5-turbo-0125",
+          "choices": [
+            {
+              "index": 0,
+              "message": {
+                "role": "assistant",
+                "content": null,
+                "tool_calls": [
+                  {
+                    "id": "HIDDEN",
+                    "type": "function",
+                    "function": {
+                      "name": "unique_id",
+                      "arguments": "{\"firstName\":\"Alex\",\"lastName\":\"Yang\"}"
+                    }
+                  }
+                ]
+              },
+              "logprobs": null,
+              "finish_reason": "tool_calls"
+            }
+          ],
+          "usage": {
+            "prompt_tokens": 59,
+            "completion_tokens": 18,
+            "total_tokens": 77
+          },
+          "system_fingerprint": "HIDDEN"
+        },
+        "message": {
+          "content": "",
+          "role": "assistant",
+          "options": {
+            "toolCall": {
+              "id": "HIDDEN",
+              "name": "unique_id",
+              "input": "{\"firstName\":\"Alex\",\"lastName\":\"Yang\"}"
+            }
+          }
+        }
+      }
+    },
+    {
+      "id": "PRESERVE_3",
+      "response": {
+        "raw": {
+          "id": "HIDDEN",
+          "object": "chat.completion",
+          "created": 114514,
+          "model": "gpt-3.5-turbo-0125",
+          "choices": [
+            {
+              "index": 0,
+              "message": {
+                "role": "assistant",
+                "content": "Your unique id is 123456789."
+              },
+              "logprobs": null,
+              "finish_reason": "stop"
+            }
+          ],
+          "usage": {
+            "prompt_tokens": 88,
+            "completion_tokens": 10,
+            "total_tokens": 98
+          },
+          "system_fingerprint": "HIDDEN"
+        },
+        "message": {
+          "content": "Your unique id is 123456789.",
+          "role": "assistant",
+          "options": {}
+        }
+      }
+    },
+    {
+      "id": "PRESERVE_4",
+      "response": {
+        "raw": {
+          "id": "HIDDEN",
+          "object": "chat.completion",
+          "created": 114514,
+          "model": "gpt-3.5-turbo-0125",
+          "choices": [
+            {
+              "index": 0,
+              "message": {
+                "role": "assistant",
+                "content": null,
+                "tool_calls": [
+                  {
+                    "id": "HIDDEN",
+                    "type": "function",
+                    "function": {
+                      "name": "sumNumbers",
+                      "arguments": "{\"a\":1,\"b\":1}"
+                    }
+                  }
+                ]
+              },
+              "logprobs": null,
+              "finish_reason": "tool_calls"
+            }
+          ],
+          "usage": {
+            "prompt_tokens": 70,
+            "completion_tokens": 18,
+            "total_tokens": 88
+          },
+          "system_fingerprint": "HIDDEN"
+        },
+        "message": {
+          "content": "",
+          "role": "assistant",
+          "options": {
+            "toolCall": {
+              "id": "HIDDEN",
+              "name": "sumNumbers",
+              "input": "{\"a\":1,\"b\":1}"
+            }
+          }
+        }
+      }
+    },
+    {
+      "id": "PRESERVE_5",
+      "response": {
+        "raw": {
+          "id": "HIDDEN",
+          "object": "chat.completion",
+          "created": 114514,
+          "model": "gpt-3.5-turbo-0125",
+          "choices": [
+            {
+              "index": 0,
+              "message": {
+                "role": "assistant",
+                "content": "1 + 1 is equal to 2."
+              },
+              "logprobs": null,
+              "finish_reason": "stop"
+            }
+          ],
+          "usage": {
+            "prompt_tokens": 97,
+            "completion_tokens": 11,
+            "total_tokens": 108
+          },
+          "system_fingerprint": "HIDDEN"
+        },
+        "message": {
+          "content": "1 + 1 is equal to 2.",
+          "role": "assistant",
+          "options": {}
+        }
+      }
+    }
+  ],
+  "llmEventStream": []
+}

packages/core/e2e/node/snapshot/anthropic-agent.snap

@@ -0,0 +1,403 @@
+{
+  "llmEventStart": [
+    {
+      "id": "PRESERVE_0",
+      "messages": [
+        {
+          "role": "user",
+          "content": "What is the weather in San Francisco?",
+          "options": {}
+        }
+      ]
+    },
+    {
+      "id": "PRESERVE_1",
+      "messages": [
+        {
+          "role": "user",
+          "content": "What is the weather in San Francisco?",
+          "options": {}
+        },
+        {
+          "content": [
+            {
+              "type": "text",
+              "text": "<thinking>\nThe user is asking for the weather in a specific location, San Francisco. The Weather function is the relevant tool to answer this request, as it returns weather information for a given location.\n\nThe Weather function has one required parameter:\n- location (string): The user has directly provided the location of \"San Francisco\"\n\nSince the required location parameter has been provided by the user, we have all the necessary information to call the Weather function.\n</thinking>"
+            }
+          ],
+          "role": "assistant",
+          "options": {
+            "toolCall": {
+              "id": "HIDDEN",
+              "name": "Weather",
+              "input": {
+                "location": "San Francisco"
+              }
+            }
+          }
+        },
+        {
+          "content": "35 degrees and sunny in San Francisco",
+          "role": "user",
+          "options": {
+            "toolResult": {
+              "isError": false,
+              "id": "HIDDEN"
+            }
+          }
+        }
+      ]
+    },
+    {
+      "id": "PRESERVE_2",
+      "messages": [
+        {
+          "role": "user",
+          "content": "My name is Alex Yang. What is my unique id?",
+          "options": {}
+        }
+      ]
+    },
+    {
+      "id": "PRESERVE_3",
+      "messages": [
+        {
+          "role": "user",
+          "content": "My name is Alex Yang. What is my unique id?",
+          "options": {}
+        },
+        {
+          "content": [
+            {
+              "type": "text",
+              "text": "<thinking>\nThe unique_id function is the relevant tool to answer the user's request for their unique ID. It requires two parameters:\nfirstName: The user provided their first name, which is \"Alex\"\nlastName: The user also provided their last name, \"Yang\"\nSince the user has provided all the required parameters, we can proceed with calling the unique_id function.\n</thinking>"
+            }
+          ],
+          "role": "assistant",
+          "options": {
+            "toolCall": {
+              "id": "HIDDEN",
+              "name": "unique_id",
+              "input": {
+                "firstName": "Alex",
+                "lastName": "Yang"
+              }
+            }
+          }
+        },
+        {
+          "content": "123456789",
+          "role": "user",
+          "options": {
+            "toolResult": {
+              "isError": false,
+              "id": "HIDDEN"
+            }
+          }
+        }
+      ]
+    },
+    {
+      "id": "PRESERVE_4",
+      "messages": [
+        {
+          "role": "user",
+          "content": "how much is 1 + 1?",
+          "options": {}
+        }
+      ]
+    },
+    {
+      "id": "PRESERVE_5",
+      "messages": [
+        {
+          "role": "user",
+          "content": "how much is 1 + 1?",
+          "options": {}
+        },
+        {
+          "content": [
+            {
+              "type": "text",
+              "text": "<thinking>\nThe user is asking to sum the numbers 1 and 1. The relevant tool to use is the sumNumbers function, which takes two number parameters a and b.\nThe user has directly provided the values for the parameters:\na = 1 \nb = 1\nSince all the required parameters have been provided, we can proceed with calling the function.\n</thinking>"
+            }
+          ],
+          "role": "assistant",
+          "options": {
+            "toolCall": {
+              "id": "HIDDEN",
+              "name": "sumNumbers",
+              "input": {
+                "a": 1,
+                "b": 1
+              }
+            }
+          }
+        },
+        {
+          "content": "2",
+          "role": "user",
+          "options": {
+            "toolResult": {
+              "isError": false,
+              "id": "HIDDEN"
+            }
+          }
+        }
+      ]
+    }
+  ],
+  "llmEventEnd": [
+    {
+      "id": "PRESERVE_0",
+      "response": {
+        "raw": {
+          "id": "HIDDEN",
+          "type": "message",
+          "role": "assistant",
+          "model": "claude-3-opus-20240229",
+          "stop_sequence": null,
+          "usage": {
+            "input_tokens": 462,
+            "output_tokens": 147
+          },
+          "content": [
+            {
+              "type": "text",
+              "text": "<thinking>\nThe user is asking for the weather in a specific location, San Francisco. The Weather function is the relevant tool to answer this request, as it returns weather information for a given location.\n\nThe Weather function has one required parameter:\n- location (string): The user has directly provided the location of \"San Francisco\"\n\nSince the required location parameter has been provided by the user, we have all the necessary information to call the Weather function.\n</thinking>"
+            },
+            {
+              "type": "tool_use",
+              "id": "HIDDEN",
+              "name": "Weather",
+              "input": {
+                "location": "San Francisco"
+              }
+            }
+          ],
+          "stop_reason": "tool_use"
+        },
+        "message": {
+          "content": [
+            {
+              "type": "text",
+              "text": "<thinking>\nThe user is asking for the weather in a specific location, San Francisco. The Weather function is the relevant tool to answer this request, as it returns weather information for a given location.\n\nThe Weather function has one required parameter:\n- location (string): The user has directly provided the location of \"San Francisco\"\n\nSince the required location parameter has been provided by the user, we have all the necessary information to call the Weather function.\n</thinking>"
+            }
+          ],
+          "role": "assistant",
+          "options": {
+            "toolCall": {
+              "id": "HIDDEN",
+              "name": "Weather",
+              "input": {
+                "location": "San Francisco"
+              }
+            }
+          }
+        }
+      }
+    },
+    {
+      "id": "PRESERVE_1",
+      "response": {
+        "raw": {
+          "id": "HIDDEN",
+          "type": "message",
+          "role": "assistant",
+          "model": "claude-3-opus-20240229",
+          "stop_sequence": null,
+          "usage": {
+            "input_tokens": 628,
+            "output_tokens": 18
+          },
+          "content": [
+            {
+              "type": "text",
+              "text": "The current weather in San Francisco is 35 degrees and sunny."
+            }
+          ],
+          "stop_reason": "end_turn"
+        },
+        "message": {
+          "content": [
+            {
+              "type": "text",
+              "text": "The current weather in San Francisco is 35 degrees and sunny."
+            }
+          ],
+          "role": "assistant",
+          "options": {}
+        }
+      }
+    },
+    {
+      "id": "PRESERVE_2",
+      "response": {
+        "raw": {
+          "id": "HIDDEN",
+          "type": "message",
+          "role": "assistant",
+          "model": "claude-3-opus-20240229",
+          "stop_sequence": null,
+          "usage": {
+            "input_tokens": 482,
+            "output_tokens": 152
+          },
+          "content": [
+            {
+              "type": "text",
+              "text": "<thinking>\nThe unique_id function is the relevant tool to answer the user's request for their unique ID. It requires two parameters:\nfirstName: The user provided their first name, which is \"Alex\"\nlastName: The user also provided their last name, \"Yang\"\nSince the user has provided all the required parameters, we can proceed with calling the unique_id function.\n</thinking>"
+            },
+            {
+              "type": "tool_use",
+              "id": "HIDDEN",
+              "name": "unique_id",
+              "input": {
+                "firstName": "Alex",
+                "lastName": "Yang"
+              }
+            }
+          ],
+          "stop_reason": "tool_use"
+        },
+        "message": {
+          "content": [
+            {
+              "type": "text",
+              "text": "<thinking>\nThe unique_id function is the relevant tool to answer the user's request for their unique ID. It requires two parameters:\nfirstName: The user provided their first name, which is \"Alex\"\nlastName: The user also provided their last name, \"Yang\"\nSince the user has provided all the required parameters, we can proceed with calling the unique_id function.\n</thinking>"
+            }
+          ],
+          "role": "assistant",
+          "options": {
+            "toolCall": {
+              "id": "HIDDEN",
+              "name": "unique_id",
+              "input": {
+                "firstName": "Alex",
+                "lastName": "Yang"
+              }
+            }
+          }
+        }
+      }
+    },
+    {
+      "id": "PRESERVE_3",
+      "response": {
+        "raw": {
+          "id": "HIDDEN",
+          "type": "message",
+          "role": "assistant",
+          "model": "claude-3-opus-20240229",
+          "stop_sequence": null,
+          "usage": {
+            "input_tokens": 648,
+            "output_tokens": 13
+          },
+          "content": [
+            {
+              "type": "text",
+              "text": "Your unique ID is 123456789."
+            }
+          ],
+          "stop_reason": "end_turn"
+        },
+        "message": {
+          "content": [
+            {
+              "type": "text",
+              "text": "Your unique ID is 123456789."
+            }
+          ],
+          "role": "assistant",
+          "options": {}
+        }
+      }
+    },
+    {
+      "id": "PRESERVE_4",
+      "response": {
+        "raw": {
+          "id": "HIDDEN",
+          "type": "message",
+          "role": "assistant",
+          "model": "claude-3-opus-20240229",
+          "stop_sequence": null,
+          "usage": {
+            "input_tokens": 498,
+            "output_tokens": 151
+          },
+          "content": [
+            {
+              "type": "text",
+              "text": "<thinking>\nThe user is asking to sum the numbers 1 and 1. The relevant tool to use is the sumNumbers function, which takes two number parameters a and b.\nThe user has directly provided the values for the parameters:\na = 1 \nb = 1\nSince all the required parameters have been provided, we can proceed with calling the function.\n</thinking>"
+            },
+            {
+              "type": "tool_use",
+              "id": "HIDDEN",
+              "name": "sumNumbers",
+              "input": {
+                "a": 1,
+                "b": 1
+              }
+            }
+          ],
+          "stop_reason": "tool_use"
+        },
+        "message": {
+          "content": [
+            {
+              "type": "text",
+              "text": "<thinking>\nThe user is asking to sum the numbers 1 and 1. The relevant tool to use is the sumNumbers function, which takes two number parameters a and b.\nThe user has directly provided the values for the parameters:\na = 1 \nb = 1\nSince all the required parameters have been provided, we can proceed with calling the function.\n</thinking>"
+            }
+          ],
+          "role": "assistant",
+          "options": {
+            "toolCall": {
+              "id": "HIDDEN",
+              "name": "sumNumbers",
+              "input": {
+                "a": 1,
+                "b": 1
+              }
+            }
+          }
+        }
+      }
+    },
+    {
+      "id": "PRESERVE_5",
+      "response": {
+        "raw": {
+          "id": "HIDDEN",
+          "type": "message",
+          "role": "assistant",
+          "model": "claude-3-opus-20240229",
+          "stop_sequence": null,
+          "usage": {
+            "input_tokens": 661,
+            "output_tokens": 16
+          },
+          "content": [
+            {
+              "type": "text",
+              "text": "So 1 + 1 = 2."
+            }
+          ],
+          "stop_reason": "end_turn"
+        },
+        "message": {
+          "content": [
+            {
+              "type": "text",
+              "text": "So 1 + 1 = 2."
+            }
+          ],
+          "role": "assistant",
+          "options": {}
+        }
+      }
+    }
+  ],
+  "llmEventStream": []
+}

packages/core/e2e/node/snapshot/gpt-4-turbo.snap

@@ -0,0 +1,128 @@
+{
+  "llmEventStart": [
+    {
+      "id": "PRESERVE_0",
+      "messages": [
+        {
+          "content": "What is the weather in San Jose?",
+          "role": "user"
+        }
+      ]
+    },
+    {
+      "id": "PRESERVE_1",
+      "messages": [
+        {
+          "content": "What is the weather in San Jose?",
+          "role": "user"
+        },
+        {
+          "content": "",
+          "role": "assistant",
+          "options": {
+            "toolCall": {
+              "id": "HIDDEN",
+              "name": "Weather",
+              "input": "{\"location\":\"San Jose\"}"
+            }
+          }
+        },
+        {
+          "content": "45 degrees and sunny in San Jose",
+          "role": "user",
+          "options": {
+            "toolResult": {
+              "id": "HIDDEN",
+              "isError": false
+            }
+          }
+        }
+      ]
+    }
+  ],
+  "llmEventEnd": [
+    {
+      "id": "PRESERVE_0",
+      "response": {
+        "raw": {
+          "id": "HIDDEN",
+          "object": "chat.completion",
+          "created": 114514,
+          "model": "gpt-3.5-turbo-0125",
+          "choices": [
+            {
+              "index": 0,
+              "message": {
+                "role": "assistant",
+                "content": null,
+                "tool_calls": [
+                  {
+                    "id": "HIDDEN",
+                    "type": "function",
+                    "function": {
+                      "name": "Weather",
+                      "arguments": "{\"location\":\"San Jose\"}"
+                    }
+                  }
+                ]
+              },
+              "logprobs": null,
+              "finish_reason": "tool_calls"
+            }
+          ],
+          "usage": {
+            "prompt_tokens": 49,
+            "completion_tokens": 15,
+            "total_tokens": 64
+          },
+          "system_fingerprint": "HIDDEN"
+        },
+        "message": {
+          "content": "",
+          "role": "assistant",
+          "options": {
+            "toolCall": {
+              "id": "HIDDEN",
+              "name": "Weather",
+              "input": "{\"location\":\"San Jose\"}"
+            }
+          }
+        }
+      }
+    },
+    {
+      "id": "PRESERVE_1",
+      "response": {
+        "raw": {
+          "id": "HIDDEN",
+          "object": "chat.completion",
+          "created": 114514,
+          "model": "gpt-3.5-turbo-0125",
+          "choices": [
+            {
+              "index": 0,
+              "message": {
+                "role": "assistant",
+                "content": "The weather in San Jose is currently 45 degrees and sunny."
+              },
+              "logprobs": null,
+              "finish_reason": "stop"
+            }
+          ],
+          "usage": {
+            "prompt_tokens": 78,
+            "completion_tokens": 14,
+            "total_tokens": 92
+          },
+          "system_fingerprint": "HIDDEN"
+        },
+        "message": {
+          "content": "The weather in San Jose is currently 45 degrees and sunny.",
+          "role": "assistant",
+          "options": {}
+        }
+      }
+    }
+  ],
+  "llmEventStream": []
+}

packages/core/e2e/node/snapshot/llm-anthropic.snap

@@ -0,0 +1,310 @@
+{
+  "llmEventStart": [
+    {
+      "id": "PRESERVE_0",
+      "messages": [
+        {
+          "content": "Hello",
+          "role": "user",
+          "options": {}
+        }
+      ]
+    },
+    {
+      "id": "PRESERVE_1",
+      "messages": [
+        {
+          "content": "hello",
+          "role": "user"
+        }
+      ]
+    }
+  ],
+  "llmEventEnd": [
+    {
+      "id": "PRESERVE_0",
+      "response": {
+        "raw": {
+          "id": "HIDDEN",
+          "type": "message",
+          "role": "assistant",
+          "model": "claude-3-opus-20240229",
+          "stop_sequence": null,
+          "usage": {
+            "input_tokens": 8,
+            "output_tokens": 12
+          },
+          "content": [
+            {
+              "type": "text",
+              "text": "Hello! How can I assist you today?"
+            }
+          ],
+          "stop_reason": "end_turn"
+        },
+        "message": {
+          "content": "Hello! How can I assist you today?",
+          "role": "assistant",
+          "options": {}
+        }
+      }
+    },
+    {
+      "id": "PRESERVE_1",
+      "response": {
+        "raw": [
+          {
+            "raw": {
+              "type": "content_block_delta",
+              "index": 0,
+              "delta": {
+                "type": "text_delta",
+                "text": "Hello"
+              }
+            },
+            "delta": "Hello",
+            "options": {}
+          },
+          {
+            "raw": {
+              "type": "content_block_delta",
+              "index": 0,
+              "delta": {
+                "type": "text_delta",
+                "text": "!"
+              }
+            },
+            "delta": "!",
+            "options": {}
+          },
+          {
+            "raw": {
+              "type": "content_block_delta",
+              "index": 0,
+              "delta": {
+                "type": "text_delta",
+                "text": " How"
+              }
+            },
+            "delta": " How",
+            "options": {}
+          },
+          {
+            "raw": {
+              "type": "content_block_delta",
+              "index": 0,
+              "delta": {
+                "type": "text_delta",
+                "text": " can"
+              }
+            },
+            "delta": " can",
+            "options": {}
+          },
+          {
+            "raw": {
+              "type": "content_block_delta",
+              "index": 0,
+              "delta": {
+                "type": "text_delta",
+                "text": " I"
+              }
+            },
+            "delta": " I",
+            "options": {}
+          },
+          {
+            "raw": {
+              "type": "content_block_delta",
+              "index": 0,
+              "delta": {
+                "type": "text_delta",
+                "text": " assist"
+              }
+            },
+            "delta": " assist",
+            "options": {}
+          },
+          {
+            "raw": {
+              "type": "content_block_delta",
+              "index": 0,
+              "delta": {
+                "type": "text_delta",
+                "text": " you"
+              }
+            },
+            "delta": " you",
+            "options": {}
+          },
+          {
+            "raw": {
+              "type": "content_block_delta",
+              "index": 0,
+              "delta": {
+                "type": "text_delta",
+                "text": " today"
+              }
+            },
+            "delta": " today",
+            "options": {}
+          },
+          {
+            "raw": {
+              "type": "content_block_delta",
+              "index": 0,
+              "delta": {
+                "type": "text_delta",
+                "text": "?"
+              }
+            },
+            "delta": "?",
+            "options": {}
+          }
+        ],
+        "message": {
+          "content": "Hello! How can I assist you today?",
+          "role": "assistant",
+          "options": {}
+        }
+      }
+    }
+  ],
+  "llmEventStream": [
+    {
+      "id": "PRESERVE_1",
+      "chunk": {
+        "raw": {
+          "type": "content_block_delta",
+          "index": 0,
+          "delta": {
+            "type": "text_delta",
+            "text": "Hello"
+          }
+        },
+        "delta": "Hello",
+        "options": {}
+      }
+    },
+    {
+      "id": "PRESERVE_1",
+      "chunk": {
+        "raw": {
+          "type": "content_block_delta",
+          "index": 0,
+          "delta": {
+            "type": "text_delta",
+            "text": "!"
+          }
+        },
+        "delta": "!",
+        "options": {}
+      }
+    },
+    {
+      "id": "PRESERVE_1",
+      "chunk": {
+        "raw": {
+          "type": "content_block_delta",
+          "index": 0,
+          "delta": {
+            "type": "text_delta",
+            "text": " How"
+          }
+        },
+        "delta": " How",
+        "options": {}
+      }
+    },
+    {
+      "id": "PRESERVE_1",
+      "chunk": {
+        "raw": {
+          "type": "content_block_delta",
+          "index": 0,
+          "delta": {
+            "type": "text_delta",
+            "text": " can"
+          }
+        },
+        "delta": " can",
+        "options": {}
+      }
+    },
+    {
+      "id": "PRESERVE_1",
+      "chunk": {
+        "raw": {
+          "type": "content_block_delta",
+          "index": 0,
+          "delta": {
+            "type": "text_delta",
+            "text": " I"
+          }
+        },
+        "delta": " I",
+        "options": {}
+      }
+    },
+    {
+      "id": "PRESERVE_1",
+      "chunk": {
+        "raw": {
+          "type": "content_block_delta",
+          "index": 0,
+          "delta": {
+            "type": "text_delta",
+            "text": " assist"
+          }
+        },
+        "delta": " assist",
+        "options": {}
+      }
+    },
+    {
+      "id": "PRESERVE_1",
+      "chunk": {
+        "raw": {
+          "type": "content_block_delta",
+          "index": 0,
+          "delta": {
+            "type": "text_delta",
+            "text": " you"
+          }
+        },
+        "delta": " you",
+        "options": {}
+      }
+    },
+    {
+      "id": "PRESERVE_1",
+      "chunk": {
+        "raw": {
+          "type": "content_block_delta",
+          "index": 0,
+          "delta": {
+            "type": "text_delta",
+            "text": " today"
+          }
+        },
+        "delta": " today",
+        "options": {}
+      }
+    },
+    {
+      "id": "PRESERVE_1",
+      "chunk": {
+        "raw": {
+          "type": "content_block_delta",
+          "index": 0,
+          "delta": {
+            "type": "text_delta",
+            "text": "?"
+          }
+        },
+        "delta": "?",
+        "options": {}
+      }
+    }
+  ]
+}

packages/core/e2e/node/snapshot/llm.snap

@@ -0,0 +1,476 @@
+{
+  "llmEventStart": [
+    {
+      "id": "PRESERVE_0",
+      "messages": [
+        {
+          "content": "Hello",
+          "role": "user"
+        }
+      ]
+    },
+    {
+      "id": "PRESERVE_1",
+      "messages": [
+        {
+          "content": "hello",
+          "role": "user"
+        }
+      ]
+    }
+  ],
+  "llmEventEnd": [
+    {
+      "id": "PRESERVE_0",
+      "response": {
+        "raw": {
+          "id": "HIDDEN",
+          "object": "chat.completion",
+          "created": 114514,
+          "model": "gpt-3.5-turbo-0125",
+          "choices": [
+            {
+              "index": 0,
+              "message": {
+                "role": "assistant",
+                "content": "Hello! How can I assist you today?"
+              },
+              "logprobs": null,
+              "finish_reason": "stop"
+            }
+          ],
+          "usage": {
+            "prompt_tokens": 8,
+            "completion_tokens": 9,
+            "total_tokens": 17
+          },
+          "system_fingerprint": "HIDDEN"
+        },
+        "message": {
+          "content": "Hello! How can I assist you today?",
+          "role": "assistant",
+          "options": {}
+        }
+      }
+    },
+    {
+      "id": "PRESERVE_1",
+      "response": {
+        "raw": [
+          {
+            "raw": {
+              "id": "HIDDEN",
+              "object": "chat.completion.chunk",
+              "created": 114514,
+              "model": "gpt-3.5-turbo-0125",
+              "system_fingerprint": "HIDDEN",
+              "choices": [
+                {
+                  "index": 0,
+                  "delta": {
+                    "content": "Hello"
+                  },
+                  "logprobs": null,
+                  "finish_reason": null
+                }
+              ]
+            },
+            "options": {},
+            "delta": "Hello"
+          },
+          {
+            "raw": {
+              "id": "HIDDEN",
+              "object": "chat.completion.chunk",
+              "created": 114514,
+              "model": "gpt-3.5-turbo-0125",
+              "system_fingerprint": "HIDDEN",
+              "choices": [
+                {
+                  "index": 0,
+                  "delta": {
+                    "content": "!"
+                  },
+                  "logprobs": null,
+                  "finish_reason": null
+                }
+              ]
+            },
+            "options": {},
+            "delta": "!"
+          },
+          {
+            "raw": {
+              "id": "HIDDEN",
+              "object": "chat.completion.chunk",
+              "created": 114514,
+              "model": "gpt-3.5-turbo-0125",
+              "system_fingerprint": "HIDDEN",
+              "choices": [
+                {
+                  "index": 0,
+                  "delta": {
+                    "content": " How"
+                  },
+                  "logprobs": null,
+                  "finish_reason": null
+                }
+              ]
+            },
+            "options": {},
+            "delta": " How"
+          },
+          {
+            "raw": {
+              "id": "HIDDEN",
+              "object": "chat.completion.chunk",
+              "created": 114514,
+              "model": "gpt-3.5-turbo-0125",
+              "system_fingerprint": "HIDDEN",
+              "choices": [
+                {
+                  "index": 0,
+                  "delta": {
+                    "content": " can"
+                  },
+                  "logprobs": null,
+                  "finish_reason": null
+                }
+              ]
+            },
+            "options": {},
+            "delta": " can"
+          },
+          {
+            "raw": {
+              "id": "HIDDEN",
+              "object": "chat.completion.chunk",
+              "created": 114514,
+              "model": "gpt-3.5-turbo-0125",
+              "system_fingerprint": "HIDDEN",
+              "choices": [
+                {
+                  "index": 0,
+                  "delta": {
+                    "content": " I"
+                  },
+                  "logprobs": null,
+                  "finish_reason": null
+                }
+              ]
+            },
+            "options": {},
+            "delta": " I"
+          },
+          {
+            "raw": {
+              "id": "HIDDEN",
+              "object": "chat.completion.chunk",
+              "created": 114514,
+              "model": "gpt-3.5-turbo-0125",
+              "system_fingerprint": "HIDDEN",
+              "choices": [
+                {
+                  "index": 0,
+                  "delta": {
+                    "content": " assist"
+                  },
+                  "logprobs": null,
+                  "finish_reason": null
+                }
+              ]
+            },
+            "options": {},
+            "delta": " assist"
+          },
+          {
+            "raw": {
+              "id": "HIDDEN",
+              "object": "chat.completion.chunk",
+              "created": 114514,
+              "model": "gpt-3.5-turbo-0125",
+              "system_fingerprint": "HIDDEN",
+              "choices": [
+                {
+                  "index": 0,
+                  "delta": {
+                    "content": " you"
+                  },
+                  "logprobs": null,
+                  "finish_reason": null
+                }
+              ]
+            },
+            "options": {},
+            "delta": " you"
+          },
+          {
+            "raw": {
+              "id": "HIDDEN",
+              "object": "chat.completion.chunk",
+              "created": 114514,
+              "model": "gpt-3.5-turbo-0125",
+              "system_fingerprint": "HIDDEN",
+              "choices": [
+                {
+                  "index": 0,
+                  "delta": {
+                    "content": " today"
+                  },
+                  "logprobs": null,
+                  "finish_reason": null
+                }
+              ]
+            },
+            "options": {},
+            "delta": " today"
+          },
+          {
+            "raw": {
+              "id": "HIDDEN",
+              "object": "chat.completion.chunk",
+              "created": 114514,
+              "model": "gpt-3.5-turbo-0125",
+              "system_fingerprint": "HIDDEN",
+              "choices": [
+                {
+                  "index": 0,
+                  "delta": {
+                    "content": "?"
+                  },
+                  "logprobs": null,
+                  "finish_reason": null
+                }
+              ]
+            },
+            "options": {},
+            "delta": "?"
+          }
+        ],
+        "message": {
+          "content": "Hello! How can I assist you today?",
+          "role": "assistant",
+          "options": {}
+        }
+      }
+    }
+  ],
+  "llmEventStream": [
+    {
+      "id": "PRESERVE_1",
+      "chunk": {
+        "raw": {
+          "id": "HIDDEN",
+          "object": "chat.completion.chunk",
+          "created": 114514,
+          "model": "gpt-3.5-turbo-0125",
+          "system_fingerprint": "HIDDEN",
+          "choices": [
+            {
+              "index": 0,
+              "delta": {
+                "content": "Hello"
+              },
+              "logprobs": null,
+              "finish_reason": null
+            }
+          ]
+        },
+        "options": {},
+        "delta": "Hello"
+      }
+    },
+    {
+      "id": "PRESERVE_1",
+      "chunk": {
+        "raw": {
+          "id": "HIDDEN",
+          "object": "chat.completion.chunk",
+          "created": 114514,
+          "model": "gpt-3.5-turbo-0125",
+          "system_fingerprint": "HIDDEN",
+          "choices": [
+            {
+              "index": 0,
+              "delta": {
+                "content": "!"
+              },
+              "logprobs": null,
+              "finish_reason": null
+            }
+          ]
+        },
+        "options": {},
+        "delta": "!"
+      }
+    },
+    {
+      "id": "PRESERVE_1",
+      "chunk": {
+        "raw": {
+          "id": "HIDDEN",
+          "object": "chat.completion.chunk",
+          "created": 114514,
+          "model": "gpt-3.5-turbo-0125",
+          "system_fingerprint": "HIDDEN",
+          "choices": [
+            {
+              "index": 0,
+              "delta": {
+                "content": " How"
+              },
+              "logprobs": null,
+              "finish_reason": null
+            }
+          ]
+        },
+        "options": {},
+        "delta": " How"
+      }
+    },
+    {
+      "id": "PRESERVE_1",
+      "chunk": {
+        "raw": {
+          "id": "HIDDEN",
+          "object": "chat.completion.chunk",
+          "created": 114514,
+          "model": "gpt-3.5-turbo-0125",
+          "system_fingerprint": "HIDDEN",
+          "choices": [
+            {
+              "index": 0,
+              "delta": {
+                "content": " can"
+              },
+              "logprobs": null,
+              "finish_reason": null
+            }
+          ]
+        },
+        "options": {},
+        "delta": " can"
+      }
+    },
+    {
+      "id": "PRESERVE_1",
+      "chunk": {
+        "raw": {
+          "id": "HIDDEN",
+          "object": "chat.completion.chunk",
+          "created": 114514,
+          "model": "gpt-3.5-turbo-0125",
+          "system_fingerprint": "HIDDEN",
+          "choices": [
+            {
+              "index": 0,
+              "delta": {
+                "content": " I"
+              },
+              "logprobs": null,
+              "finish_reason": null
+            }
+          ]
+        },
+        "options": {},
+        "delta": " I"
+      }
+    },
+    {
+      "id": "PRESERVE_1",
+      "chunk": {
+        "raw": {
+          "id": "HIDDEN",
+          "object": "chat.completion.chunk",
+          "created": 114514,
+          "model": "gpt-3.5-turbo-0125",
+          "system_fingerprint": "HIDDEN",
+          "choices": [
+            {
+              "index": 0,
+              "delta": {
+                "content": " assist"
+              },
+              "logprobs": null,
+              "finish_reason": null
+            }
+          ]
+        },
+        "options": {},
+        "delta": " assist"
+      }
+    },
+    {
+      "id": "PRESERVE_1",
+      "chunk": {
+        "raw": {
+          "id": "HIDDEN",
+          "object": "chat.completion.chunk",
+          "created": 114514,
+          "model": "gpt-3.5-turbo-0125",
+          "system_fingerprint": "HIDDEN",
+          "choices": [
+            {
+              "index": 0,
+              "delta": {
+                "content": " you"
+              },
+              "logprobs": null,
+              "finish_reason": null
+            }
+          ]
+        },
+        "options": {},
+        "delta": " you"
+      }
+    },
+    {
+      "id": "PRESERVE_1",
+      "chunk": {
+        "raw": {
+          "id": "HIDDEN",
+          "object": "chat.completion.chunk",
+          "created": 114514,
+          "model": "gpt-3.5-turbo-0125",
+          "system_fingerprint": "HIDDEN",
+          "choices": [
+            {
+              "index": 0,
+              "delta": {
+                "content": " today"
+              },
+              "logprobs": null,
+              "finish_reason": null
+            }
+          ]
+        },
+        "options": {},
+        "delta": " today"
+      }
+    },
+    {
+      "id": "PRESERVE_1",
+      "chunk": {
+        "raw": {
+          "id": "HIDDEN",
+          "object": "chat.completion.chunk",
+          "created": 114514,
+          "model": "gpt-3.5-turbo-0125",
+          "system_fingerprint": "HIDDEN",
+          "choices": [
+            {
+              "index": 0,
+              "delta": {
+                "content": "?"
+              },
+              "logprobs": null,
+              "finish_reason": null
+            }
+          ]
+        },
+        "options": {},
+        "delta": "?"
+      }
+    }
+  ]
+}

packages/core/e2e/node/snapshot/queryEngine_subquestion.snap

@@ -0,0 +1,133 @@
+{
+  "llmEventStart": [
+    {
+      "id": "PRESERVE_0",
+      "messages": [
+        {
+          "content": "Given a user question, and a list of tools, output a list of relevant sub-questions that when composed can help answer the full user question:\n\n# Example 1\n<Tools>\n```json\n{\n    \"uber_10k\": \"Provides information about Uber financials for year 2021\",\n    \"lyft_10k\": \"Provides information about Lyft financials for year 2021\"\n}\n```\n\n<User Question>\nCompare and contrast the revenue growth and EBITDA of Uber and Lyft for year 2021\n\n<Output>\n```json\n[\n    {\n        \"subQuestion\": \"What is the revenue growth of Uber\",\n        \"toolName\": \"uber_10k\"\n    },\n    {\n        \"subQuestion\": \"What is the EBITDA of Uber\",\n        \"toolName\": \"uber_10k\"\n    },\n    {\n        \"subQuestion\": \"What is the revenue growth of Lyft\",\n        \"toolName\": \"lyft_10k\"\n    },\n    {\n        \"subQuestion\": \"What is the EBITDA of Lyft\",\n        \"toolName\": \"lyft_10k\"\n    }\n]\n```\n\n# Example 2\n<Tools>\n```json\n{\n    \"bill_gates_idea\": \"Get what Bill Gates idea from.\"\n}\n```\n\n<User Question>\nWhat did Bill Gates steal from?\n\n<Output>\n",
+          "role": "user"
+        }
+      ]
+    },
+    {
+      "id": "PRESERVE_1",
+      "messages": [
+        {
+          "content": "Context information is below.\n---------------------\nBill Gates stole from Apple. Steve Jobs stole from Xerox.\n---------------------\nGiven the context information and not prior knowledge, answer the query.\nQuery: What is Bill Gates' idea\nAnswer:",
+          "role": "user"
+        }
+      ]
+    },
+    {
+      "id": "PRESERVE_2",
+      "messages": [
+        {
+          "content": "Context information is below.\n---------------------\nSub question: What is Bill Gates' idea\nResponse: Bill Gates' idea was to steal from Apple.\n---------------------\nGiven the context information and not prior knowledge, answer the query.\nQuery: What did Bill Gates steal from?\nAnswer:",
+          "role": "user"
+        }
+      ]
+    }
+  ],
+  "llmEventEnd": [
+    {
+      "id": "PRESERVE_0",
+      "response": {
+        "raw": {
+          "id": "HIDDEN",
+          "object": "chat.completion",
+          "created": 114514,
+          "model": "gpt-3.5-turbo-0125",
+          "choices": [
+            {
+              "index": 0,
+              "message": {
+                "role": "assistant",
+                "content": "```json\n[\n    {\n        \"subQuestion\": \"What is Bill Gates' idea\",\n        \"toolName\": \"bill_gates_idea\"\n    }\n]\n```"
+              },
+              "logprobs": null,
+              "finish_reason": "stop"
+            }
+          ],
+          "usage": {
+            "prompt_tokens": 290,
+            "completion_tokens": 35,
+            "total_tokens": 325
+          },
+          "system_fingerprint": "HIDDEN"
+        },
+        "message": {
+          "content": "```json\n[\n    {\n        \"subQuestion\": \"What is Bill Gates' idea\",\n        \"toolName\": \"bill_gates_idea\"\n    }\n]\n```",
+          "role": "assistant",
+          "options": {}
+        }
+      }
+    },
+    {
+      "id": "PRESERVE_1",
+      "response": {
+        "raw": {
+          "id": "HIDDEN",
+          "object": "chat.completion",
+          "created": 114514,
+          "model": "gpt-3.5-turbo-0125",
+          "choices": [
+            {
+              "index": 0,
+              "message": {
+                "role": "assistant",
+                "content": "Bill Gates' idea was to steal from Apple."
+              },
+              "logprobs": null,
+              "finish_reason": "stop"
+            }
+          ],
+          "usage": {
+            "prompt_tokens": 53,
+            "completion_tokens": 10,
+            "total_tokens": 63
+          },
+          "system_fingerprint": "HIDDEN"
+        },
+        "message": {
+          "content": "Bill Gates' idea was to steal from Apple.",
+          "role": "assistant",
+          "options": {}
+        }
+      }
+    },
+    {
+      "id": "PRESERVE_2",
+      "response": {
+        "raw": {
+          "id": "HIDDEN",
+          "object": "chat.completion",
+          "created": 114514,
+          "model": "gpt-3.5-turbo-0125",
+          "choices": [
+            {
+              "index": 0,
+              "message": {
+                "role": "assistant",
+                "content": "Bill Gates stole from Apple."
+              },
+              "logprobs": null,
+              "finish_reason": "stop"
+            }
+          ],
+          "usage": {
+            "prompt_tokens": 62,
+            "completion_tokens": 6,
+            "total_tokens": 68
+          },
+          "system_fingerprint": "HIDDEN"
+        },
+        "message": {
+          "content": "Bill Gates stole from Apple.",
+          "role": "assistant",
+          "options": {}
+        }
+      }
+    }
+  ],
+  "llmEventStream": []
+}

packages/core/e2e/node/utils.ts

@@ -0,0 +1,139 @@
+/* eslint-disable turbo/no-undeclared-env-vars */
+import {
+  Settings,
+  type LLMEndEvent,
+  type LLMStartEvent,
+  type LLMStreamEvent,
+} from "llamaindex";
+import { readFile, writeFile } from "node:fs/promises";
+import { join } from "node:path";
+import { type test } from "node:test";
+import { fileURLToPath } from "node:url";
+
+type MockStorage = {
+  llmEventStart: LLMStartEvent["detail"]["payload"][];
+  llmEventEnd: LLMEndEvent["detail"]["payload"][];
+  llmEventStream: LLMStreamEvent["detail"]["payload"][];
+};
+
+export const llmCompleteMockStorage: MockStorage = {
+  llmEventStart: [],
+  llmEventEnd: [],
+  llmEventStream: [],
+};
+
+export const testRootDir = fileURLToPath(new URL(".", import.meta.url));
+
+export async function mockLLMEvent(
+  t: Parameters<NonNullable<Parameters<typeof test>[0]>>[0],
+  snapshotName: string,
+) {
+  const idMap = new Map<string, string>();
+  let counter = 0;
+  const newLLMCompleteMockStorage: MockStorage = {
+    llmEventStart: [],
+    llmEventEnd: [],
+    llmEventStream: [],
+  };
+
+  function captureLLMStart(event: LLMStartEvent) {
+    idMap.set(event.detail.payload.id, `PRESERVE_${counter++}`);
+    newLLMCompleteMockStorage.llmEventStart.push({
+      ...event.detail.payload,
+      // @ts-expect-error id is not UUID, but it is fine for testing
+      id: idMap.get(event.detail.payload.id)!,
+    });
+  }
+
+  function captureLLMEnd(event: LLMEndEvent) {
+    newLLMCompleteMockStorage.llmEventEnd.push({
+      ...event.detail.payload,
+      // @ts-expect-error id is not UUID, but it is fine for testing
+      id: idMap.get(event.detail.payload.id)!,
+    });
+  }
+
+  function captureLLMStream(event: LLMStreamEvent) {
+    newLLMCompleteMockStorage.llmEventStream.push({
+      ...event.detail.payload,
+      // @ts-expect-error id is not UUID, but it is fine for testing
+      id: idMap.get(event.detail.payload.id)!,
+    });
+  }
+
+  await readFile(join(testRootDir, "snapshot", `${snapshotName}.snap`), {
+    encoding: "utf-8",
+  })
+    .then((data) => {
+      const result = JSON.parse(data) as MockStorage;
+      result["llmEventEnd"].forEach((event) => {
+        llmCompleteMockStorage.llmEventEnd.push(event);
+      });
+      result["llmEventStart"].forEach((event) => {
+        llmCompleteMockStorage.llmEventStart.push(event);
+      });
+      result["llmEventStream"].forEach((event) => {
+        llmCompleteMockStorage.llmEventStream.push(event);
+      });
+    })
+    .catch((error) => {
+      if (error.code === "ENOENT") {
+        console.warn("Snapshot file not found, will create a new one");
+        return;
+      }
+    });
+  Settings.callbackManager.on("llm-start", captureLLMStart);
+  Settings.callbackManager.on("llm-end", captureLLMEnd);
+  Settings.callbackManager.on("llm-stream", captureLLMStream);
+
+  t.after(async () => {
+    Settings.callbackManager.off("llm-stream", captureLLMStream);
+    Settings.callbackManager.off("llm-end", captureLLMEnd);
+    Settings.callbackManager.off("llm-start", captureLLMStart);
+    // eslint-disable-next-line turbo/no-undeclared-env-vars
+    if (process.env.UPDATE_SNAPSHOT === "1") {
+      const data = JSON.stringify(newLLMCompleteMockStorage, null, 2)
+        .replace(/"id": "(?!PRESERVE_).*"/g, '"id": "HIDDEN"')
+        .replace(/"created": \d+/g, `"created": 114514`)
+        .replace(
+          /"system_fingerprint": ".*"/g,
+          '"system_fingerprint": "HIDDEN"',
+        )
+        .replace(/"tool_call_id": ".*"/g, '"tool_call_id": "HIDDEN"');
+      await writeFile(
+        join(testRootDir, "snapshot", `${snapshotName}.snap`),
+        data,
+      );
+      return;
+    }
+    if (
+      newLLMCompleteMockStorage.llmEventEnd.length !==
+      llmCompleteMockStorage.llmEventEnd.length
+    ) {
+      throw new Error("New LLMEndEvent does not match, please update snapshot");
+    }
+    if (
+      newLLMCompleteMockStorage.llmEventStart.length !==
+      llmCompleteMockStorage.llmEventStart.length
+    ) {
+      throw new Error(
+        "New LLMStartEvent does not match, please update snapshot",
+      );
+    }
+
+    if (
+      newLLMCompleteMockStorage.llmEventStream.length !==
+      llmCompleteMockStorage.llmEventStream.length
+    ) {
+      throw new Error(
+        "New LLMStreamEvent does not match, please update snapshot",
+      );
+    }
+  });
+  // cleanup
+  t.after(() => {
+    llmCompleteMockStorage.llmEventEnd = [];
+    llmCompleteMockStorage.llmEventStart = [];
+    llmCompleteMockStorage.llmEventStream = [];
+  });
+}

packages/core/e2e/package.json

@@ -0,0 +1,17 @@
+{
+  "name": "@llamaindex/core-e2e",
+  "private": true,
+  "version": "0.0.2",
+  "type": "module",
+  "scripts": {
+    "e2e": "node --import tsx --import ./mock-register.js --test ./node/*.e2e.ts",
+    "e2e:nomock": "node --import tsx --test ./node/*.e2e.ts",
+    "e2e:updatesnap": "UPDATE_SNAPSHOT=1 node --import tsx --test ./node/*.e2e.ts"
+  },
+  "devDependencies": {
+    "@faker-js/faker": "^8.4.1",
+    "consola": "^3.2.3",
+    "llamaindex": "workspace:*",
+    "tsx": "^4.7.2"
+  }
+}

packages/core/e2e/tsconfig.json

@@ -0,0 +1,23 @@
+{
+  "extends": "../../../tsconfig.json",
+  "compilerOptions": {
+    "outDir": "./lib",
+    "module": "node16",
+    "moduleResolution": "node16",
+    "target": "ESNext"
+  },
+  "include": [
+    "./**/*.ts",
+    "./mock-module.js",
+    "./mock-register.js",
+    "./fixtures"
+  ],
+  "references": [
+    {
+      "path": "../../core/tsconfig.json"
+    },
+    {
+      "path": "../../env/tsconfig.json"
+    }
+  ]
+}

packages/core/jsr.json

@@ -1,8 +1,8 @@
 {
   "name": "@llamaindex/core",
-  "version": "0.1.21",
+  "version": "0.2.3",
   "exports": "./src/index.ts",
   "imports": {
-    "@llamaindex/env": "jsr:@llamaindex/env@0.0.5"
+    "@llamaindex/env": "jsr:@llamaindex/env@0.0.6"
   }
 }

packages/core/package.json

@@ -1,56 +1,57 @@
 {
   "name": "llamaindex",
-  "version": "0.2.2",
+  "version": "0.2.9",
   "expectedMinorVersion": "2",
   "license": "MIT",
   "type": "module",
   "dependencies": {
-    "@anthropic-ai/sdk": "^0.18.0",
+    "@anthropic-ai/sdk": "^0.20.4",
     "@aws-crypto/sha256-js": "^5.2.0",
     "@datastax/astra-db-ts": "^0.1.4",
-    "@grpc/grpc-js": "^1.10.2",
-    "@llamaindex/cloud": "0.0.4",
+    "@grpc/grpc-js": "^1.10.6",
+    "@llamaindex/cloud": "0.0.5",
     "@llamaindex/env": "workspace:*",
-    "@mistralai/mistralai": "^0.0.10",
-    "@notionhq/client": "^2.2.14",
-    "@pinecone-database/pinecone": "^2.0.1",
-    "@qdrant/js-client-rest": "^1.7.0",
-    "@types/lodash": "^4.14.202",
-    "@types/node": "^18.19.14",
+    "@mistralai/mistralai": "^0.1.3",
+    "@notionhq/client": "^2.2.15",
+    "@pinecone-database/pinecone": "^2.2.0",
+    "@qdrant/js-client-rest": "^1.8.2",
+    "@types/lodash": "^4.17.0",
+    "@types/node": "^20.12.7",
     "@types/papaparse": "^5.3.14",
-    "@types/pg": "^8.11.0",
-    "@xenova/transformers": "^2.15.0",
+    "@types/pg": "^8.11.5",
+    "@xenova/transformers": "^2.16.1",
     "@zilliz/milvus2-sdk-node": "^2.3.5",
-    "assemblyai": "^4.2.2",
+    "ajv": "^8.12.0",
+    "assemblyai": "^4.3.4",
     "chromadb": "~1.7.3",
-    "cohere-ai": "^7.7.5",
+    "cohere-ai": "^7.9.3",
     "js-tiktoken": "^1.0.10",
     "lodash": "^4.17.21",
     "magic-bytes.js": "^1.10.0",
-    "mammoth": "^1.6.0",
+    "mammoth": "^1.7.1",
     "md-utils-ts": "^2.0.0",
-    "mongodb": "^6.3.0",
+    "mongodb": "^6.5.0",
     "notion-md-crawler": "^0.0.2",
-    "openai": "^4.26.1",
+    "openai": "^4.33.0",
     "papaparse": "^5.4.1",
     "pathe": "^1.1.2",
     "pdf2json": "^3.0.5",
-    "pg": "^8.11.3",
-    "pgvector": "^0.1.7",
+    "pg": "^8.11.5",
+    "pgvector": "^0.1.8",
     "portkey-ai": "^0.1.16",
     "rake-modified": "^1.0.8",
     "replicate": "^0.25.2",
-    "string-strip-html": "^13.4.6",
-    "wink-nlp": "^1.14.3",
-    "wikipedia": "^2.1.2"
+    "string-strip-html": "^13.4.8",
+    "wikipedia": "^2.1.2",
+    "wink-nlp": "^1.14.3"
   },
   "devDependencies": {
-    "@swc/cli": "^0.3.9",
-    "@swc/core": "^1.4.2",
+    "@swc/cli": "^0.3.12",
+    "@swc/core": "^1.4.13",
     "concurrently": "^8.2.2",
-    "glob": "^10.3.10",
+    "glob": "^10.3.12",
     "madge": "^6.1.0",
-    "typescript": "^5.3.3"
+    "typescript": "^5.4.5"
   },
   "engines": {
     "node": ">=18.0.0"

packages/core/src/ChatHistory.ts

@@ -1,26 +1,29 @@
 import { globalsHelper } from "./GlobalsHelper.js";
 import type { SummaryPrompt } from "./Prompt.js";
 import { defaultSummaryPrompt, messagesToHistoryStr } from "./Prompt.js";
-import { OpenAI } from "./llm/LLM.js";
+import { OpenAI } from "./llm/open_ai.js";
 import type { ChatMessage, LLM, MessageType } from "./llm/types.js";
+import { extractText } from "./llm/utils.js";
 
 /**
  * A ChatHistory is used to keep the state of back and forth chat messages
  */
-export abstract class ChatHistory {
-  abstract get messages(): ChatMessage[];
+export abstract class ChatHistory<
+  AdditionalMessageOptions extends object = object,
+> {
+  abstract get messages(): ChatMessage<AdditionalMessageOptions>[];
   /**
    * Adds a message to the chat history.
    * @param message
    */
-  abstract addMessage(message: ChatMessage): void;
+  abstract addMessage(message: ChatMessage<AdditionalMessageOptions>): void;
 
   /**
    * Returns the messages that should be used as input to the LLM.
    */
   abstract requestMessages(
-    transientMessages?: ChatMessage[],
-  ): Promise<ChatMessage[]>;
+    transientMessages?: ChatMessage<AdditionalMessageOptions>[],
+  ): Promise<ChatMessage<AdditionalMessageOptions>[]>;
 
   /**
    * Resets the chat history so that it's empty.
@@ -30,7 +33,7 @@ export abstract class ChatHistory {
   /**
    * Returns the new messages since the last call to this function (or since calling the constructor)
    */
-  abstract newMessages(): ChatMessage[];
+  abstract newMessages(): ChatMessage<AdditionalMessageOptions>[];
 }
 
 export class SimpleChatHistory extends ChatHistory {
@@ -107,6 +110,7 @@ export class SummaryChatHistory extends ChatHistory {
             context: messagesToHistoryStr(messagesToSummarize),
           }),
           role: "user" as MessageType,
+          options: {},
         },
       ];
       // remove oldest message until the chat history is short enough for the context window
@@ -115,7 +119,9 @@ export class SummaryChatHistory extends ChatHistory {
       this.tokenizer(promptMessages[0].content).length > this.tokensToSummarize
     );
 
-    const response = await this.llm.chat({ messages: promptMessages });
+    const response = await this.llm.chat({
+      messages: promptMessages,
+    });
     return { content: response.message.content, role: "memory" };
   }
 
@@ -188,7 +194,8 @@ export class SummaryChatHistory extends ChatHistory {
 
     // get tokens of current request messages and the transient messages
     const tokens = requestMessages.reduce(
-      (count, message) => count + this.tokenizer(message.content).length,
+      (count, message) =>
+        count + this.tokenizer(extractText(message.content)).length,
       0,
     );
     if (tokens > this.tokensToSummarize) {

packages/core/src/GlobalsHelper.ts

@@ -1,12 +1,5 @@
 import { encodingForModel } from "js-tiktoken";
 
-import { randomUUID } from "@llamaindex/env";
-import type {
-  Event,
-  EventTag,
-  EventType,
-} from "./callbacks/CallbackManager.js";
-
 export enum Tokenizers {
   CL100K_BASE = "cl100k_base",
 }
@@ -51,39 +44,6 @@ class GlobalsHelper {
 
     return this.defaultTokenizer!.decode.bind(this.defaultTokenizer);
   }
-
-  /**
-   * @deprecated createEvent will be removed in the future,
-   *  please use `new CustomEvent(eventType, { detail: payload })` instead.
-   *
-   *  Also, `parentEvent` will not be used in the future,
-   *    use `AsyncLocalStorage` to track parent events instead.
-   *    @example - Usage of `AsyncLocalStorage`:
-   *    let id = 0;
-   *    const asyncLocalStorage = new AsyncLocalStorage<number>();
-   *    asyncLocalStorage.run(++id, async () => {
-   *      setTimeout(() => {
-   *        console.log('parent event id:', asyncLocalStorage.getStore()); // 1
-   *      }, 1000)
-   *    });
-   */
-  createEvent({
-    parentEvent,
-    type,
-    tags,
-  }: {
-    parentEvent?: Event;
-    type: EventType;
-    tags?: EventTag[];
-  }): Event {
-    return {
-      id: randomUUID(),
-      type,
-      // inherit parent tags if tags not set
-      tags: tags || parentEvent?.tags,
-      parentId: parentEvent?.id,
-    };
-  }
 }
 
 export const globalsHelper = new GlobalsHelper();

packages/core/src/QuestionGenerator.ts

@@ -5,7 +5,7 @@ import type {
   BaseQuestionGenerator,
   SubQuestion,
 } from "./engines/query/types.js";
-import { OpenAI } from "./llm/LLM.js";
+import { OpenAI } from "./llm/open_ai.js";
 import type { LLM } from "./llm/types.js";
 import { PromptMixin } from "./prompts/index.js";
 import type {

packages/core/src/Response.ts

@@ -1,14 +1,14 @@
-import type { BaseNode } from "./Node.js";
+import type { NodeWithScore } from "./Node.js";
 
 /**
  * Response is the output of a LLM
  */
 export class Response {
   response: string;
-  sourceNodes?: BaseNode[];
+  sourceNodes?: NodeWithScore[];
   metadata: Record<string, unknown> = {};
 
-  constructor(response: string, sourceNodes?: BaseNode[]) {
+  constructor(response: string, sourceNodes?: NodeWithScore[]) {
     this.response = response;
     this.sourceNodes = sourceNodes || [];
   }

packages/core/src/Retriever.ts

@@ -1,13 +1,8 @@
-import type { Event } from "./callbacks/CallbackManager.js";
 import type { NodeWithScore } from "./Node.js";
 import type { ServiceContext } from "./ServiceContext.js";
 
 export type RetrieveParams = {
   query: string;
-  /**
-   * @deprecated will be removed in the next major version
-   */
-  parentEvent?: Event;
   preFilters?: unknown;
 };
 

packages/core/src/ServiceContext.ts

@@ -1,7 +1,7 @@
 import { PromptHelper } from "./PromptHelper.js";
 import { OpenAIEmbedding } from "./embeddings/OpenAIEmbedding.js";
 import type { BaseEmbedding } from "./embeddings/types.js";
-import { OpenAI } from "./llm/LLM.js";
+import { OpenAI } from "./llm/open_ai.js";
 import type { LLM } from "./llm/types.js";
 import { SimpleNodeParser } from "./nodeParsers/SimpleNodeParser.js";
 import type { NodeParser } from "./nodeParsers/types.js";

packages/core/src/Settings.ts

@@ -1,11 +1,11 @@
 import { CallbackManager } from "./callbacks/CallbackManager.js";
 import { OpenAIEmbedding } from "./embeddings/OpenAIEmbedding.js";
-import { OpenAI } from "./llm/LLM.js";
+import { OpenAI } from "./llm/open_ai.js";
 
 import { PromptHelper } from "./PromptHelper.js";
 import { SimpleNodeParser } from "./nodeParsers/SimpleNodeParser.js";
 
-import { AsyncLocalStorage } from "@llamaindex/env";
+import { AsyncLocalStorage, getEnv } from "@llamaindex/env";
 import type { ServiceContext } from "./ServiceContext.js";
 import type { BaseEmbedding } from "./embeddings/types.js";
 import {
@@ -52,6 +52,15 @@ class GlobalSettings implements Config {
   #chunkOverlapAsyncLocalStorage = new AsyncLocalStorage<number>();
   #promptAsyncLocalStorage = new AsyncLocalStorage<PromptConfig>();
 
+  get debug() {
+    const debug = getEnv("DEBUG");
+    return (
+      getEnv("NODE_ENV") === "development" &&
+      Boolean(debug) &&
+      debug?.includes("llamaindex")
+    );
+  }
+
   get llm(): LLM {
     if (this.#llm === null) {
       this.#llm = new OpenAI();

packages/core/src/TextSplitter.ts

@@ -66,8 +66,9 @@ export const defaultParagraphSeparator = EOL + EOL + EOL;
  * One of the advantages of SentenceSplitter is that even in the fixed length chunks it will try to keep sentences together.
  */
 export class SentenceSplitter {
-  private chunkSize: number;
-  private chunkOverlap: number;
+  public chunkSize: number;
+  public chunkOverlap: number;
+
   private tokenizer: any;
   private tokenizerDecoder: any;
   private paragraphSeparator: string;

packages/core/src/agent/anthropic.ts

@@ -0,0 +1,170 @@
+import { Settings } from "../Settings.js";
+import {
+  AgentChatResponse,
+  type ChatEngineParamsNonStreaming,
+} from "../engines/chat/index.js";
+import { wrapEventCaller } from "../internal/context/EventCaller.js";
+import { getCallbackManager } from "../internal/settings/CallbackManager.js";
+import { prettifyError } from "../internal/utils.js";
+import { Anthropic } from "../llm/anthropic.js";
+import type {
+  ChatMessage,
+  ChatResponse,
+  ToolCallLLMMessageOptions,
+} from "../llm/index.js";
+import { extractText } from "../llm/utils.js";
+import { ObjectRetriever } from "../objects/index.js";
+import type { BaseToolWithCall } from "../types.js";
+
+const MAX_TOOL_CALLS = 10;
+
+type AnthropicParamsBase = {
+  llm?: Anthropic;
+  chatHistory?: ChatMessage<ToolCallLLMMessageOptions>[];
+};
+
+type AnthropicParamsWithTools = AnthropicParamsBase & {
+  tools: BaseToolWithCall[];
+};
+
+type AnthropicParamsWithToolRetriever = AnthropicParamsBase & {
+  toolRetriever: ObjectRetriever<BaseToolWithCall>;
+};
+
+export type AnthropicAgentParams =
+  | AnthropicParamsWithTools
+  | AnthropicParamsWithToolRetriever;
+
+type AgentContext = {
+  toolCalls: number;
+  llm: Anthropic;
+  tools: BaseToolWithCall[];
+  messages: ChatMessage<ToolCallLLMMessageOptions>[];
+  shouldContinue: (context: AgentContext) => boolean;
+};
+
+type TaskResult = {
+  response: ChatResponse<ToolCallLLMMessageOptions>;
+  chatHistory: ChatMessage<ToolCallLLMMessageOptions>[];
+};
+
+async function task(
+  context: AgentContext,
+  input: ChatMessage<ToolCallLLMMessageOptions>,
+): Promise<TaskResult> {
+  const { llm, tools, messages } = context;
+  const nextMessages: ChatMessage<ToolCallLLMMessageOptions>[] = [
+    ...messages,
+    input,
+  ];
+  const response = await llm.chat({
+    stream: false,
+    tools,
+    messages: nextMessages,
+  });
+  const options = response.message.options ?? {};
+  if ("toolCall" in options) {
+    const { toolCall } = options;
+    const { input, name, id } = toolCall;
+    const targetTool = tools.find((tool) => tool.metadata.name === name);
+    let output: string;
+    let isError = true;
+    if (!context.shouldContinue(context)) {
+      output = "Error: Tool call limit reached";
+    } else if (!targetTool) {
+      output = `Error: Tool ${name} not found`;
+    } else {
+      try {
+        getCallbackManager().dispatchEvent("llm-tool-call", {
+          payload: {
+            toolCall: {
+              name,
+              input,
+            },
+          },
+        });
+        output = await targetTool.call(input);
+        isError = false;
+      } catch (error: unknown) {
+        output = prettifyError(error);
+      }
+    }
+    return task(
+      {
+        ...context,
+        toolCalls: context.toolCalls + 1,
+        messages: [...nextMessages, response.message],
+      },
+      {
+        content: output,
+        role: "user",
+        options: {
+          toolResult: {
+            isError,
+            id,
+          },
+        },
+      },
+    );
+  } else {
+    return { response, chatHistory: [...nextMessages, response.message] };
+  }
+}
+
+export class AnthropicAgent {
+  readonly #llm: Anthropic;
+  readonly #tools:
+    | BaseToolWithCall[]
+    | ((query: string) => Promise<BaseToolWithCall[]>) = [];
+  #chatHistory: ChatMessage<ToolCallLLMMessageOptions>[] = [];
+
+  constructor(params: AnthropicAgentParams) {
+    this.#llm =
+      params.llm ?? Settings.llm instanceof Anthropic
+        ? (Settings.llm as Anthropic)
+        : new Anthropic();
+    if ("tools" in params) {
+      this.#tools = params.tools;
+    } else if ("toolRetriever" in params) {
+      this.#tools = params.toolRetriever.retrieve.bind(params.toolRetriever);
+    }
+    if (Array.isArray(params.chatHistory)) {
+      this.#chatHistory = params.chatHistory;
+    }
+  }
+
+  static shouldContinue(context: AgentContext): boolean {
+    return context.toolCalls < MAX_TOOL_CALLS;
+  }
+
+  public reset(): void {
+    this.#chatHistory = [];
+  }
+
+  getTools(query: string): Promise<BaseToolWithCall[]> | BaseToolWithCall[] {
+    return typeof this.#tools === "function" ? this.#tools(query) : this.#tools;
+  }
+
+  @wrapEventCaller
+  async chat(
+    params: ChatEngineParamsNonStreaming,
+  ): Promise<Promise<AgentChatResponse>> {
+    const { chatHistory, response } = await task(
+      {
+        llm: this.#llm,
+        tools: await this.getTools(extractText(params.message)),
+        toolCalls: 0,
+        messages: [...this.#chatHistory],
+        // do we need this?
+        shouldContinue: AnthropicAgent.shouldContinue,
+      },
+      {
+        role: "user",
+        content: params.message,
+        options: {},
+      },
+    );
+    this.#chatHistory = [...chatHistory];
+    return new AgentChatResponse(extractText(response.message.content));
+  }
+}

packages/core/src/agent/index.ts

@@ -1,3 +1,5 @@
+// Not exporting the AnthropicAgent because it is not ready to ship yet.
+// export { AnthropicAgent, type AnthropicAgentParams } from "./anthropic.js";
 export * from "./openai/base.js";
 export * from "./openai/worker.js";
 export * from "./react/base.js";

packages/core/src/agent/openai/base.ts

@@ -1,5 +1,7 @@
+import { Settings } from "../../Settings.js";
 import type { ChatMessage } from "../../llm/index.js";
 import { OpenAI } from "../../llm/index.js";
+import type { BaseMemory } from "../../memory/types.js";
 import type { ObjectRetriever } from "../../objects/base.js";
 import type { BaseTool } from "../../types.js";
 import { AgentRunner } from "../runner/base.js";
@@ -8,12 +10,11 @@ import { OpenAIAgentWorker } from "./worker.js";
 type OpenAIAgentParams = {
   tools?: BaseTool[];
   llm?: OpenAI;
-  memory?: any;
+  memory?: BaseMemory;
   prefixMessages?: ChatMessage[];
-  verbose?: boolean;
   maxFunctionCalls?: number;
   defaultToolChoice?: string;
-  toolRetriever?: ObjectRetriever;
+  toolRetriever?: ObjectRetriever<BaseTool>;
   systemPrompt?: string;
 };
 
@@ -28,13 +29,19 @@ export class OpenAIAgent extends AgentRunner {
     llm,
     memory,
     prefixMessages,
-    verbose,
     maxFunctionCalls = 5,
     defaultToolChoice = "auto",
     toolRetriever,
     systemPrompt,
   }: OpenAIAgentParams) {
-    llm = llm ?? new OpenAI({ model: "gpt-3.5-turbo-0613" });
+    if (!llm) {
+      if (Settings.llm instanceof OpenAI) {
+        llm = Settings.llm;
+      } else {
+        console.warn("No OpenAI model provided, creating a new one");
+        llm = new OpenAI({ model: "gpt-3.5-turbo-0613" });
+      }
+    }
 
     if (systemPrompt) {
       if (prefixMessages) {
@@ -49,7 +56,7 @@ export class OpenAIAgent extends AgentRunner {
       ];
     }
 
-    if (!llm?.metadata.isFunctionCallingModel) {
+    if (!llm?.supportToolCall) {
       throw new Error("LLM model must be a function-calling model");
     }
 
@@ -59,13 +66,14 @@ export class OpenAIAgent extends AgentRunner {
       prefixMessages,
       maxFunctionCalls,
       toolRetriever,
-      verbose,
     });
 
     super({
       agentWorker: stepEngine,
+      llm,
       memory,
       defaultToolChoice,
+      // @ts-expect-error 2322
       chatHistory: prefixMessages,
     });
   }

packages/core/src/agent/openai/utils.ts

@@ -1,27 +0,0 @@
-import type { ToolMetadata } from "../../types.js";
-
-export type OpenAIFunction = {
-  type: "function";
-  function: ToolMetadata;
-};
-
-type OpenAiTool = {
-  name: string;
-  description: string;
-  parameters: ToolMetadata["parameters"];
-};
-
-export const toOpenAiTool = ({
-  name,
-  description,
-  parameters,
-}: OpenAiTool): OpenAIFunction => {
-  return {
-    type: "function",
-    function: {
-      name: name,
-      description: description,
-      parameters,
-    },
-  };
-};

packages/core/src/agent/openai/worker.ts

@@ -1,17 +1,23 @@
-import { randomUUID } from "@llamaindex/env";
+import { pipeline, randomUUID } from "@llamaindex/env";
+import type { ChatCompletionToolChoiceOption } from "openai/resources/chat/completions";
 import { Response } from "../../Response.js";
+import { Settings } from "../../Settings.js";
 import {
   AgentChatResponse,
   ChatResponseMode,
   StreamingAgentChatResponse,
 } from "../../engines/chat/types.js";
-import type {
-  ChatMessage,
-  ChatResponse,
-  ChatResponseChunk,
+import {
+  OpenAI,
+  isFunctionCallingModel,
+  type ChatMessage,
+  type ChatResponseChunk,
+  type LLMChatParamsBase,
+  type OpenAIAdditionalChatOptions,
+  type ToolCallLLMMessageOptions,
+  type ToolCallOptions,
 } from "../../llm/index.js";
-import { OpenAI } from "../../llm/index.js";
-import { streamConverter, streamReducer } from "../../llm/utils.js";
+import { extractText } from "../../llm/utils.js";
 import { ChatMemoryBuffer } from "../../memory/ChatMemoryBuffer.js";
 import type { ObjectRetriever } from "../../objects/base.js";
 import type { ToolOutput } from "../../tools/types.js";
@@ -20,52 +26,40 @@ import type { BaseTool } from "../../types.js";
 import type { AgentWorker, Task } from "../types.js";
 import { TaskStep, TaskStepOutput } from "../types.js";
 import { addUserStepToMemory, getFunctionByName } from "../utils.js";
-import type { OpenAIToolCall } from "./types/chat.js";
-import { toOpenAiTool } from "./utils.js";
 
-const DEFAULT_MAX_FUNCTION_CALLS = 5;
-
-/**
- * Call function.
- * @param tools: tools
- * @param toolCall: tool call
- * @param verbose: verbose
- * @returns: void
- */
 async function callFunction(
   tools: BaseTool[],
-  toolCall: OpenAIToolCall,
-  verbose: boolean = false,
-): Promise<[ChatMessage, ToolOutput]> {
-  const id_ = toolCall.id;
-  const functionCall = toolCall.function;
-  const name = toolCall.function.name;
-  const argumentsStr = toolCall.function.arguments;
+  toolCall: ToolCallOptions["toolCall"],
+): Promise<[ChatMessage<ToolCallLLMMessageOptions>, ToolOutput]> {
+  const id = toolCall.id;
+  const name = toolCall.name;
+  const input = toolCall.input;
 
-  if (verbose) {
+  if (Settings.debug) {
     console.log("=== Calling Function ===");
-    console.log(`Calling function: ${name} with args: ${argumentsStr}`);
+    console.log(`Calling function: ${name} with args: ${input}`);
   }
 
   const tool = getFunctionByName(tools, name);
-  const argumentDict = JSON.parse(argumentsStr);
 
   // Call tool
   // Use default error message
-  const output = await callToolWithErrorHandling(tool, argumentDict, null);
+  const output = await callToolWithErrorHandling(tool, input);
 
-  if (verbose) {
+  if (Settings.debug) {
     console.log(`Got output ${output}`);
     console.log("==========================");
   }
 
   return [
     {
-      content: String(output),
-      role: "tool",
-      additionalKwargs: {
-        name,
-        tool_call_id: id_,
+      content: `${output}`,
+      role: "user",
+      options: {
+        toolResult: {
+          id,
+          isError: false,
+        },
       },
     },
     output,
@@ -76,9 +70,8 @@ type OpenAIAgentWorkerParams = {
   tools?: BaseTool[];
   llm?: OpenAI;
   prefixMessages?: ChatMessage[];
-  verbose?: boolean;
   maxFunctionCalls?: number;
-  toolRetriever?: ObjectRetriever;
+  toolRetriever?: ObjectRetriever<BaseTool>;
 };
 
 type CallFunctionOutput = {
@@ -86,33 +79,32 @@ type CallFunctionOutput = {
   toolOutput: ToolOutput;
 };
 
-/**
- * OpenAI agent worker.
- * This class is responsible for running the agent.
- */
-export class OpenAIAgentWorker implements AgentWorker {
+export class OpenAIAgentWorker
+  implements AgentWorker<LLMChatParamsBase<OpenAIAdditionalChatOptions>>
+{
   private llm: OpenAI;
-  private verbose: boolean;
-  private maxFunctionCalls: number;
+  private maxFunctionCalls: number = 5;
 
   public prefixMessages: ChatMessage[];
 
   private _getTools: (input: string) => Promise<BaseTool[]>;
 
-  /**
-   * Initialize.
-   */
   constructor({
     tools = [],
     llm,
     prefixMessages,
-    verbose,
-    maxFunctionCalls = DEFAULT_MAX_FUNCTION_CALLS,
+    maxFunctionCalls,
     toolRetriever,
   }: OpenAIAgentWorkerParams) {
-    this.llm = llm ?? new OpenAI({ model: "gpt-3.5-turbo-0613" });
-    this.verbose = verbose || false;
-    this.maxFunctionCalls = maxFunctionCalls;
+    this.llm =
+      llm ?? isFunctionCallingModel(Settings.llm)
+        ? (Settings.llm as OpenAI)
+        : new OpenAI({
+            model: "gpt-3.5-turbo-0613",
+          });
+    if (maxFunctionCalls) {
+      this.maxFunctionCalls = maxFunctionCalls;
+    }
     this.prefixMessages = prefixMessages || [];
 
     if (tools.length > 0 && toolRetriever) {
@@ -120,6 +112,7 @@ export class OpenAIAgentWorker implements AgentWorker {
     } else if (tools.length > 0) {
       this._getTools = async () => tools;
     } else if (toolRetriever) {
+      // fixme: this won't work, type mismatch
       this._getTools = async (message: string) =>
         toolRetriever.retrieve(message);
     } else {
@@ -127,12 +120,7 @@ export class OpenAIAgentWorker implements AgentWorker {
     }
   }
 
-  /**
-   * Get all messages.
-   * @param task: task
-   * @returns: messages
-   */
-  public getAllMessages(task: Task): ChatMessage[] {
+  public getAllMessages(task: Task): ChatMessage<ToolCallLLMMessageOptions>[] {
     return [
       ...this.prefixMessages,
       ...task.memory.get(),
@@ -140,133 +128,158 @@ export class OpenAIAgentWorker implements AgentWorker {
     ];
   }
 
-  /**
-   * Get latest tool calls.
-   * @param task: task
-   * @returns: tool calls
-   */
-  public getLatestToolCalls(task: Task): OpenAIToolCall[] | null {
+  public getLatestToolCall(task: Task): ToolCallOptions["toolCall"] | null {
     const chatHistory: ChatMessage[] = task.extraState.newMemory.getAll();
 
     if (chatHistory.length === 0) {
       return null;
     }
 
-    return chatHistory[chatHistory.length - 1].additionalKwargs?.toolCalls;
+    // @ts-expect-error fixme
+    return chatHistory[chatHistory.length - 1].options?.toolCall;
   }
 
-  /**
-   *
-   * @param task
-   * @param openaiTools
-   * @param toolChoice
-   * @returns
-   */
-  private _getLlmChatKwargs(
+  private _getLlmChatParams(
     task: Task,
-    openaiTools: { [key: string]: any }[],
-    toolChoice: string | { [key: string]: any } = "auto",
-  ): { [key: string]: any } {
-    const llmChatKwargs: { [key: string]: any } = {
+    tools: BaseTool[],
+    toolChoice: ChatCompletionToolChoiceOption = "auto",
+  ): LLMChatParamsBase<OpenAIAdditionalChatOptions, ToolCallLLMMessageOptions> {
+    const llmChatParams = {
       messages: this.getAllMessages(task),
-    };
+      tools: undefined as BaseTool[] | undefined,
+      additionalChatOptions: {} as OpenAIAdditionalChatOptions,
+    } satisfies LLMChatParamsBase<
+      OpenAIAdditionalChatOptions,
+      ToolCallLLMMessageOptions
+    >;
 
-    if (openaiTools.length > 0) {
-      llmChatKwargs.tools = openaiTools;
-      llmChatKwargs.toolChoice = toolChoice;
+    if (tools.length > 0) {
+      llmChatParams.tools = tools;
+      llmChatParams.additionalChatOptions.tool_choice = toolChoice;
     }
 
-    return llmChatKwargs;
+    return llmChatParams;
   }
 
-  /**
-   * Process message.
-   * @param task: task
-   * @param chatResponse: chat response
-   * @returns: agent chat response
-   */
   private _processMessage(
     task: Task,
-    chatResponse: ChatResponse,
+    aiMessage: ChatMessage,
   ): AgentChatResponse {
-    const aiMessage = chatResponse.message;
     task.extraState.newMemory.put(aiMessage);
 
-    return new AgentChatResponse(aiMessage.content, task.extraState.sources);
+    return new AgentChatResponse(
+      extractText(aiMessage.content),
+      task.extraState.sources,
+    );
   }
 
   private async _getStreamAiResponse(
     task: Task,
-    llmChatKwargs: any,
-  ): Promise<StreamingAgentChatResponse> {
+    llmChatParams: LLMChatParamsBase<
+      OpenAIAdditionalChatOptions,
+      ToolCallLLMMessageOptions
+    >,
+  ): Promise<StreamingAgentChatResponse | AgentChatResponse> {
     const stream = await this.llm.chat({
       stream: true,
-      ...llmChatKwargs,
+      ...llmChatParams,
     });
 
-    const iterator = streamConverter(
-      streamReducer({
-        stream,
-        initialValue: "",
-        reducer: (accumulator, part) => (accumulator += part.delta),
-        finished: (accumulator) => {
-          task.extraState.newMemory.put({
-            content: accumulator,
-            role: "assistant",
-          });
-        },
-      }),
-      (r: ChatResponseChunk) => new Response(r.delta),
+    const responseChunkStream = new ReadableStream<
+      ChatResponseChunk<ToolCallLLMMessageOptions>
+    >({
+      async start(controller) {
+        for await (const chunk of stream) {
+          controller.enqueue(chunk);
+        }
+        controller.close();
+      },
+    });
+    const [pipStream, finalStream] = responseChunkStream.tee();
+    const reader = pipStream.getReader();
+    const { value } = await reader.read();
+    reader.releaseLock();
+    if (value === undefined) {
+      throw new Error("first chunk value is undefined, this should not happen");
+    }
+    // check if first chunk has tool calls, if so, this is a function call
+    // otherwise, it's a regular message
+    const hasToolCall: boolean = !!(
+      value.options && "toolCall" in value.options
     );
 
-    return new StreamingAgentChatResponse(iterator, task.extraState.sources);
+    if (hasToolCall) {
+      return this._processMessage(task, {
+        content: await pipeline(finalStream, async (iterator) => {
+          let content = "";
+          for await (const value of iterator) {
+            content += value.delta;
+          }
+          return content;
+        }),
+        role: "assistant",
+        options: value.options,
+      });
+    } else {
+      const [responseStream, chunkStream] = finalStream.tee();
+      let content = "";
+      return new StreamingAgentChatResponse(
+        responseStream.pipeThrough<Response>({
+          readable: new ReadableStream({
+            async start(controller) {
+              for await (const chunk of chunkStream) {
+                controller.enqueue(new Response(chunk.delta));
+              }
+              controller.close();
+            },
+          }),
+          writable: new WritableStream({
+            write(chunk) {
+              content += chunk.delta;
+            },
+            close() {
+              task.extraState.newMemory.put({
+                content,
+                role: "assistant",
+              });
+            },
+          }),
+        }),
+        task.extraState.sources,
+      );
+    }
   }
 
-  /**
-   * Get agent response.
-   * @param task: task
-   * @param mode: mode
-   * @param llmChatKwargs: llm chat kwargs
-   * @returns: agent chat response
-   */
   private async _getAgentResponse(
     task: Task,
     mode: ChatResponseMode,
-    llmChatKwargs: any,
+    llmChatParams: LLMChatParamsBase<
+      OpenAIAdditionalChatOptions,
+      ToolCallLLMMessageOptions
+    >,
   ): Promise<AgentChatResponse | StreamingAgentChatResponse> {
     if (mode === ChatResponseMode.WAIT) {
-      const chatResponse = (await this.llm.chat({
+      const chatResponse = await this.llm.chat({
         stream: false,
-        ...llmChatKwargs,
-      })) as unknown as ChatResponse;
+        ...llmChatParams,
+      });
 
-      return this._processMessage(task, chatResponse) as AgentChatResponse;
+      return this._processMessage(
+        task,
+        chatResponse.message,
+      ) as AgentChatResponse;
     } else if (mode === ChatResponseMode.STREAM) {
-      return this._getStreamAiResponse(task, llmChatKwargs);
+      return this._getStreamAiResponse(task, llmChatParams);
     }
 
     throw new Error("Invalid mode");
   }
 
-  /**
-   * Call function.
-   * @param tools: tools
-   * @param toolCall: tool call
-   * @param memory: memory
-   * @param sources: sources
-   * @returns: void
-   */
   async callFunction(
     tools: BaseTool[],
-    toolCall: OpenAIToolCall,
+    toolCall: ToolCallOptions["toolCall"],
   ): Promise<CallFunctionOutput> {
-    const functionCall = toolCall.function;
-
-    if (!functionCall) {
-      throw new Error("Invalid tool_call object");
-    }
-
-    const functionMessage = await callFunction(tools, toolCall, this.verbose);
+    const functionMessage = await callFunction(tools, toolCall);
 
     const message = functionMessage[0];
     const toolOutput = functionMessage[1];
@@ -277,16 +290,12 @@ export class OpenAIAgentWorker implements AgentWorker {
     };
   }
 
-  /**
-   * Initialize step.
-   * @param task: task
-   * @param kwargs: kwargs
-   * @returns: task step
-   */
-  initializeStep(task: Task, kwargs?: any): TaskStep {
+  initializeStep(task: Task): TaskStep {
     const sources: ToolOutput[] = [];
 
-    const newMemory = new ChatMemoryBuffer();
+    const newMemory = new ChatMemoryBuffer({
+      tokenLimit: task.memory.tokenLimit,
+    });
 
     const taskState = {
       sources,
@@ -302,32 +311,17 @@ export class OpenAIAgentWorker implements AgentWorker {
     return new TaskStep(task.taskId, randomUUID(), task.input);
   }
 
-  /**
-   * Should continue.
-   * @param toolCalls: tool calls
-   * @param nFunctionCalls: number of function calls
-   * @returns: boolean
-   */
   private _shouldContinue(
-    toolCalls: OpenAIToolCall[] | null,
+    toolCall: ToolCallOptions["toolCall"] | null,
     nFunctionCalls: number,
-  ): boolean {
+  ): toolCall is ToolCallOptions["toolCall"] {
     if (nFunctionCalls > this.maxFunctionCalls) {
       return false;
     }
 
-    if (toolCalls?.length === 0) {
-      return false;
-    }
-
-    return true;
+    return !!toolCall;
   }
 
-  /**
-   * Get tools.
-   * @param input: input
-   * @returns: tools
-   */
   async getTools(input: string): Promise<BaseTool[]> {
     return this._getTools(input);
   }
@@ -336,53 +330,41 @@ export class OpenAIAgentWorker implements AgentWorker {
     step: TaskStep,
     task: Task,
     mode: ChatResponseMode = ChatResponseMode.WAIT,
-    toolChoice: string | { [key: string]: any } = "auto",
+    toolChoice: ChatCompletionToolChoiceOption = "auto",
   ): Promise<TaskStepOutput> {
     const tools = await this.getTools(task.input);
 
     if (step.input) {
-      addUserStepToMemory(step, task.extraState.newMemory, this.verbose);
+      addUserStepToMemory(step, task.extraState.newMemory);
     }
 
-    const openaiTools = tools.map((tool) =>
-      toOpenAiTool({
-        name: tool.metadata.name,
-        description: tool.metadata.description,
-        parameters: tool.metadata.parameters,
-      }),
-    );
-
-    const llmChatKwargs = this._getLlmChatKwargs(task, openaiTools, toolChoice);
+    const llmChatParams = this._getLlmChatParams(task, tools, toolChoice);
 
     const agentChatResponse = await this._getAgentResponse(
       task,
       mode,
-      llmChatKwargs,
+      llmChatParams,
     );
 
-    const latestToolCalls = this.getLatestToolCalls(task) || [];
+    const latestToolCall = this.getLatestToolCall(task) ?? null;
 
     let isDone: boolean;
-    let newSteps: TaskStep[] = [];
+    let newSteps: TaskStep[];
 
-    if (
-      !this._shouldContinue(latestToolCalls, task.extraState.nFunctionCalls)
-    ) {
+    if (!this._shouldContinue(latestToolCall, task.extraState.nFunctionCalls)) {
       isDone = true;
       newSteps = [];
     } else {
       isDone = false;
-      for (const toolCall of latestToolCalls) {
-        const { message, toolOutput } = await this.callFunction(
-          tools,
-          toolCall,
-        );
+      const { message, toolOutput } = await this.callFunction(
+        tools,
+        latestToolCall,
+      );
 
-        task.extraState.sources.push(toolOutput);
-        task.extraState.newMemory.put(message);
+      task.extraState.sources.push(toolOutput);
+      task.extraState.newMemory.put(message);
 
-        task.extraState.nFunctionCalls += 1;
-      }
+      task.extraState.nFunctionCalls += 1;
 
       newSteps = [step.getNextStep(randomUUID(), undefined)];
     }
@@ -390,45 +372,25 @@ export class OpenAIAgentWorker implements AgentWorker {
     return new TaskStepOutput(agentChatResponse, step, newSteps, isDone);
   }
 
-  /**
-   * Run step.
-   * @param step: step
-   * @param task: task
-   * @param kwargs: kwargs
-   * @returns: task step output
-   */
   async runStep(
     step: TaskStep,
     task: Task,
-    kwargs?: any,
+    chatParams: LLMChatParamsBase<OpenAIAdditionalChatOptions>,
   ): Promise<TaskStepOutput> {
-    const toolChoice = kwargs?.toolChoice || "auto";
+    const toolChoice = chatParams?.additionalChatOptions?.tool_choice ?? "auto";
     return this._runStep(step, task, ChatResponseMode.WAIT, toolChoice);
   }
 
-  /**
-   * Stream step.
-   * @param step: step
-   * @param task: task
-   * @param kwargs: kwargs
-   * @returns: task step output
-   */
   async streamStep(
     step: TaskStep,
     task: Task,
-    kwargs?: any,
+    chatParams: LLMChatParamsBase<OpenAIAdditionalChatOptions>,
   ): Promise<TaskStepOutput> {
-    const toolChoice = kwargs?.toolChoice || "auto";
+    const toolChoice = chatParams?.additionalChatOptions?.tool_choice ?? "auto";
     return this._runStep(step, task, ChatResponseMode.STREAM, toolChoice);
   }
 
-  /**
-   * Finalize task.
-   * @param task: task
-   * @param kwargs: kwargs
-   * @returns: void
-   */
-  finalizeTask(task: Task, kwargs?: any): void {
+  finalizeTask(task: Task): void {
     task.memory.set(task.memory.get().concat(task.extraState.newMemory.get()));
     task.extraState.newMemory.reset();
   }

packages/core/src/agent/react/base.ts

@@ -1,4 +1,5 @@
 import type { ChatMessage, LLM } from "../../llm/index.js";
+import type { BaseMemory } from "../../memory/types.js";
 import type { ObjectRetriever } from "../../objects/base.js";
 import type { BaseTool } from "../../types.js";
 import { AgentRunner } from "../runner/base.js";
@@ -7,12 +8,11 @@ import { ReActAgentWorker } from "./worker.js";
 type ReActAgentParams = {
   tools: BaseTool[];
   llm?: LLM;
-  memory?: any;
+  memory?: BaseMemory;
   prefixMessages?: ChatMessage[];
-  verbose?: boolean;
   maxInteractions?: number;
   defaultToolChoice?: string;
-  toolRetriever?: ObjectRetriever;
+  toolRetriever?: ObjectRetriever<BaseTool>;
 };
 
 /**
@@ -26,7 +26,6 @@ export class ReActAgent extends AgentRunner {
     llm,
     memory,
     prefixMessages,
-    verbose,
     maxInteractions = 10,
     defaultToolChoice = "auto",
     toolRetriever,
@@ -36,13 +35,13 @@ export class ReActAgent extends AgentRunner {
       llm,
       maxInteractions,
       toolRetriever,
-      verbose,
     });
 
     super({
       agentWorker: stepEngine,
       memory,
       defaultToolChoice,
+      // @ts-expect-error 2322
       chatHistory: prefixMessages,
     });
   }

packages/core/src/agent/react/types.ts

@@ -1,4 +1,5 @@
 import type { ChatMessage } from "../../llm/index.js";
+import { extractText } from "../../llm/utils.js";
 
 export interface BaseReasoningStep {
   getContent(): string;
@@ -51,10 +52,12 @@ export abstract class BaseOutputParser {
   formatMessages(messages: ChatMessage[]): ChatMessage[] {
     if (messages) {
       if (messages[0].role === "system") {
-        messages[0].content = this.format(messages[0].content || "");
+        messages[0].content = this.format(
+          extractText(messages[0].content) || "",
+        );
       } else {
         messages[messages.length - 1].content = this.format(
-          messages[messages.length - 1].content || "",
+          extractText(messages[messages.length - 1].content) || "",
         );
       }
     }

packages/core/src/agent/react/worker.ts

@@ -1,7 +1,10 @@
 import { randomUUID } from "@llamaindex/env";
+import type { ChatMessage } from "cohere-ai/api";
+import { Settings } from "../../Settings.js";
 import { AgentChatResponse } from "../../engines/chat/index.js";
-import type { ChatResponse, LLM } from "../../llm/index.js";
-import { OpenAI } from "../../llm/index.js";
+import { getCallbackManager } from "../../internal/settings/CallbackManager.js";
+import { type ChatResponse, type LLM } from "../../llm/index.js";
+import { extractText } from "../../llm/utils.js";
 import { ChatMemoryBuffer } from "../../memory/ChatMemoryBuffer.js";
 import type { ObjectRetriever } from "../../objects/base.js";
 import { ToolOutput } from "../../tools/index.js";
@@ -16,32 +19,24 @@ import {
   ObservationReasoningStep,
   ResponseReasoningStep,
 } from "./types.js";
+
 type ReActAgentWorkerParams = {
   tools: BaseTool[];
   llm?: LLM;
   maxInteractions?: number;
   reactChatFormatter?: ReActChatFormatter | undefined;
   outputParser?: ReActOutputParser | undefined;
-  verbose?: boolean | undefined;
-  toolRetriever?: ObjectRetriever | undefined;
+  toolRetriever?: ObjectRetriever<BaseTool> | undefined;
 };
 
-/**
- *
- * @param step
- * @param memory
- * @param currentReasoning
- * @param verbose
- */
 function addUserStepToReasoning(
   step: TaskStep,
   memory: ChatMemoryBuffer,
   currentReasoning: BaseReasoningStep[],
-  verbose: boolean = false,
 ): void {
   if (step.stepState.isFirst) {
     memory.put({
-      content: step.input,
+      content: step.input ?? "",
       role: "user",
     });
     step.stepState.isFirst = false;
@@ -50,18 +45,22 @@ function addUserStepToReasoning(
       observation: step.input ?? undefined,
     });
     currentReasoning.push(reasoningStep);
-    if (verbose) {
+    if (Settings.debug) {
       console.log(`Added user message to memory: ${step.input}`);
     }
   }
 }
 
+type ChatParams = {
+  messages: ChatMessage[];
+  tools?: BaseTool[];
+};
+
 /**
  * ReAct agent worker.
  */
-export class ReActAgentWorker implements AgentWorker {
+export class ReActAgentWorker implements AgentWorker<ChatParams> {
   llm: LLM;
-  verbose: boolean;
 
   maxInteractions: number = 10;
   reactChatFormatter: ReActChatFormatter;
@@ -75,15 +74,13 @@ export class ReActAgentWorker implements AgentWorker {
     maxInteractions,
     reactChatFormatter,
     outputParser,
-    verbose,
     toolRetriever,
   }: ReActAgentWorkerParams) {
-    this.llm = llm ?? new OpenAI({ model: "gpt-3.5-turbo-0613" });
+    this.llm = llm ?? Settings.llm;
 
     this.maxInteractions = maxInteractions ?? 10;
     this.reactChatFormatter = reactChatFormatter ?? new ReActChatFormatter();
     this.outputParser = outputParser ?? new ReActOutputParser();
-    this.verbose = verbose || false;
 
     if (tools.length > 0 && toolRetriever) {
       throw new Error("Cannot specify both tools and tool_retriever");
@@ -97,16 +94,12 @@ export class ReActAgentWorker implements AgentWorker {
     }
   }
 
-  /**
-   * Initialize a task step.
-   * @param task - task
-   * @param kwargs - keyword arguments
-   * @returns - task step
-   */
-  initializeStep(task: Task, kwargs?: any): TaskStep {
+  initializeStep(task: Task): TaskStep {
     const sources: ToolOutput[] = [];
     const currentReasoning: BaseReasoningStep[] = [];
-    const newMemory = new ChatMemoryBuffer();
+    const newMemory = new ChatMemoryBuffer({
+      tokenLimit: task.memory.tokenLimit,
+    });
 
     const taskState = {
       sources,
@@ -124,12 +117,6 @@ export class ReActAgentWorker implements AgentWorker {
     });
   }
 
-  /**
-   * Extract reasoning step from chat response.
-   * @param output - chat response
-   * @param isStreaming - whether the chat response is streaming
-   * @returns - [message content, reasoning steps, is done]
-   */
   extractReasoningStep(
     output: ChatResponse,
     isStreaming: boolean,
@@ -145,21 +132,21 @@ export class ReActAgentWorker implements AgentWorker {
 
     try {
       reasoningStep = this.outputParser.parse(
-        messageContent,
+        extractText(messageContent),
         isStreaming,
       ) as ActionReasoningStep;
     } catch (e) {
       throw new Error(`Could not parse output: ${e}`);
     }
 
-    if (this.verbose) {
+    if (Settings.debug) {
       console.log(`${reasoningStep.getContent()}\n`);
     }
 
     currentReasoning.push(reasoningStep);
 
     if (reasoningStep.isDone()) {
-      return [messageContent, currentReasoning, true];
+      return [extractText(messageContent), currentReasoning, true];
     }
 
     const actionReasoningStep = new ActionReasoningStep({
@@ -172,17 +159,9 @@ export class ReActAgentWorker implements AgentWorker {
       throw new Error(`Expected ActionReasoningStep, got ${reasoningStep}`);
     }
 
-    return [messageContent, currentReasoning, false];
+    return [extractText(messageContent), currentReasoning, false];
   }
 
-  /**
-   * Process actions.
-   * @param task - task
-   * @param tools - tools
-   * @param output - chat response
-   * @param isStreaming - whether the chat response is streaming
-   * @returns - [reasoning steps, is done]
-   */
   async _processActions(
     task: Task,
     tools: BaseTool[],
@@ -216,7 +195,15 @@ export class ReActAgentWorker implements AgentWorker {
 
     const tool = toolsDict[actionReasoningStep.action];
 
-    const toolOutput = await tool?.call?.(actionReasoningStep.actionInput);
+    getCallbackManager().dispatchEvent("llm-tool-call", {
+      payload: {
+        toolCall: {
+          name: tool.metadata.name,
+          input: JSON.stringify(actionReasoningStep.actionInput),
+        },
+      },
+    });
+    const toolOutput = await tool.call!(actionReasoningStep.actionInput);
 
     task.extraState.sources.push(
       new ToolOutput(
@@ -233,19 +220,13 @@ export class ReActAgentWorker implements AgentWorker {
 
     currentReasoning.push(observationStep);
 
-    if (this.verbose) {
+    if (Settings.debug) {
       console.log(`${observationStep.getContent()}`);
     }
 
     return [currentReasoning, false];
   }
 
-  /**
-   * Get response.
-   * @param currentReasoning - current reasoning steps
-   * @param sources - tool outputs
-   * @returns - agent chat response
-   */
   _getResponse(
     currentReasoning: BaseReasoningStep[],
     sources: ToolOutput[],
@@ -269,13 +250,6 @@ export class ReActAgentWorker implements AgentWorker {
     return new AgentChatResponse(responseStr, sources);
   }
 
-  /**
-   * Get task step response.
-   * @param agentResponse - agent chat response
-   * @param step - task step
-   * @param isDone - whether the task is done
-   * @returns - task step output
-   */
   _getTaskStepResponse(
     agentResponse: AgentChatResponse,
     step: TaskStep,
@@ -292,24 +266,12 @@ export class ReActAgentWorker implements AgentWorker {
     return new TaskStepOutput(agentResponse, step, newSteps, isDone);
   }
 
-  /**
-   * Run a task step.
-   * @param step - task step
-   * @param task - task
-   * @param kwargs - keyword arguments
-   * @returns - task step output
-   */
-  async _runStep(
-    step: TaskStep,
-    task: Task,
-    kwargs?: any,
-  ): Promise<TaskStepOutput> {
+  async _runStep(step: TaskStep, task: Task): Promise<TaskStepOutput> {
     if (step.input) {
       addUserStepToReasoning(
         step,
         task.extraState.newMemory,
         task.extraState.currentReasoning,
-        this.verbose,
       );
     }
 
@@ -348,42 +310,15 @@ export class ReActAgentWorker implements AgentWorker {
     return this._getTaskStepResponse(agentResponse, step, isDone);
   }
 
-  /**
-   * Run a task step.
-   * @param step - task step
-   * @param task - task
-   * @param kwargs - keyword arguments
-   * @returns - task step output
-   */
-  async runStep(
-    step: TaskStep,
-    task: Task,
-    kwargs?: any,
-  ): Promise<TaskStepOutput> {
+  async runStep(step: TaskStep, task: Task): Promise<TaskStepOutput> {
     return await this._runStep(step, task);
   }
 
-  /**
-   * Run a task step.
-   * @param step - task step
-   * @param task - task
-   * @param kwargs - keyword arguments
-   * @returns - task step output
-   */
-  streamStep(
-    step: TaskStep,
-    task: Task,
-    kwargs?: any,
-  ): Promise<TaskStepOutput> {
+  streamStep(): Promise<TaskStepOutput> {
     throw new Error("Method not implemented.");
   }
 
-  /**
-   * Finalize a task.
-   * @param task - task
-   * @param kwargs - keyword arguments
-   */
-  finalizeTask(task: Task, kwargs?: any): void {
+  finalizeTask(task: Task): void {
     task.memory.set(task.memory.get() + task.extraState.newMemory.get());
     task.extraState.newMemory.reset();
   }