feat: improve base tool type (#709)

Co-authored-by: Alex Yang <himself65@outlook.com>

.changeset/nasty-buttons-confess.md

@@ -0,0 +1,5 @@
+---
+"llamaindex": patch
+---
+
+Fix agent streaming with new OpenAI models

.github/workflows/test.yml

@@ -1,6 +1,12 @@
 name: Run Tests
 
-on: [push, pull_request]
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - main
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
@@ -8,6 +14,11 @@ concurrency:
 
 jobs:
   e2e:
+    strategy:
+      fail-fast: false
+      matrix:
+        node-version: [18.x, 20.x, 21.x]
+    name: E2E on Node.js ${{ matrix.node-version }}
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4

apps/docs/docs/recipes/cost-analysis.mdx

@@ -6,7 +6,7 @@ This page shows how to track LLM cost using APIs.
 
 The callback manager is a class that manages the callback functions.
 
-You can register `llm-start`, and `llm-end` callbacks to the callback manager for tracking the cost.
+You can register `llm-start`, `llm-end`, and `llm-stream` callbacks to the callback manager for tracking the cost.
 
 import CodeBlock from "@theme/CodeBlock";
 import CodeSource from "!raw-loader!../../../../examples/recipes/cost-analysis";

examples/agent/openai.ts

@@ -1,13 +1,13 @@
 import { FunctionTool, OpenAIAgent } from "llamaindex";
 
 // Define a function to sum two numbers
-function sumNumbers({ a, b }: { a: number; b: number }): number {
-  return a + b;
+function sumNumbers({ a, b }: { a: number; b: number }) {
+  return `${a + b}`;
 }
 
 // Define a function to divide two numbers
-function divideNumbers({ a, b }: { a: number; b: number }): number {
-  return a / b;
+function divideNumbers({ a, b }: { a: number; b: number }) {
+  return `${a / b}`;
 }
 
 // Define the parameters of the sum function as a JSON schema
@@ -24,7 +24,7 @@ const sumJSON = {
     },
   },
   required: ["a", "b"],
-};
+} as const;
 
 const divideJSON = {
   type: "object",
@@ -39,7 +39,7 @@ const divideJSON = {
     },
   },
   required: ["a", "b"],
-};
+} as const;
 
 async function main() {
   // Create a function tool from the sum function

examples/agent/react_agent.ts

@@ -1,13 +1,13 @@
 import { Anthropic, FunctionTool, ReActAgent } from "llamaindex";
 
 // Define a function to sum two numbers
-function sumNumbers({ a, b }: { a: number; b: number }): number {
-  return a + b;
+function sumNumbers({ a, b }: { a: number; b: number }) {
+  return `${a + b}`;
 }
 
 // Define a function to divide two numbers
-function divideNumbers({ a, b }: { a: number; b: number }): number {
-  return a / b;
+function divideNumbers({ a, b }: { a: number; b: number }) {
+  return `${a / b}`;
 }
 
 // Define the parameters of the sum function as a JSON schema
@@ -24,7 +24,7 @@ const sumJSON = {
     },
   },
   required: ["a", "b"],
-};
+} as const;
 
 const divideJSON = {
   type: "object",
@@ -39,7 +39,7 @@ const divideJSON = {
     },
   },
   required: ["a", "b"],
-};
+} as const;
 
 async function main() {
   // Create a function tool from the sum function

examples/agent/step_wise_openai.ts

@@ -1,13 +1,13 @@
 import { FunctionTool, OpenAIAgent } from "llamaindex";
 
 // Define a function to sum two numbers
-function sumNumbers({ a, b }: { a: number; b: number }): number {
-  return a + b;
+function sumNumbers({ a, b }: { a: number; b: number }) {
+  return `${a + b}`;
 }
 
 // Define a function to divide two numbers
-function divideNumbers({ a, b }: { a: number; b: number }): number {
-  return a / b;
+function divideNumbers({ a, b }: { a: number; b: number }) {
+  return `${a / b}`;
 }
 
 // Define the parameters of the sum function as a JSON schema
@@ -24,22 +24,22 @@ const sumJSON = {
     },
   },
   required: ["a", "b"],
-};
+} as const;
 
 const divideJSON = {
   type: "object",
   properties: {
     a: {
       type: "number",
-      description: "The dividend a to divide",
+      description: "The dividend",
     },
     b: {
       type: "number",
-      description: "The divisor b to divide by",
+      description: "The divisor",
     },
   },
   required: ["a", "b"],
-};
+} as const;
 
 async function main() {
   // Create a function tool from the sum function

examples/agent/step_wise_react.ts

@@ -1,13 +1,13 @@
 import { FunctionTool, ReActAgent } from "llamaindex";
 
 // Define a function to sum two numbers
-function sumNumbers({ a, b }: { a: number; b: number }): number {
-  return a + b;
+function sumNumbers({ a, b }: { a: number; b: number }) {
+  return `${a + b}`;
 }
 
 // Define a function to divide two numbers
-function divideNumbers({ a, b }: { a: number; b: number }): number {
-  return a / b;
+function divideNumbers({ a, b }: { a: number; b: number }) {
+  return `${a / b}`;
 }
 
 // Define the parameters of the sum function as a JSON schema
@@ -24,7 +24,7 @@ const sumJSON = {
     },
   },
   required: ["a", "b"],
-};
+} as const;
 
 const divideJSON = {
   type: "object",
@@ -39,7 +39,7 @@ const divideJSON = {
     },
   },
   required: ["a", "b"],
-};
+} as const;
 
 async function main() {
   // Create a function tool from the sum function

examples/agent/stream_openai_agent.ts

@@ -1,13 +1,13 @@
 import { FunctionTool, OpenAIAgent } from "llamaindex";
 
 // Define a function to sum two numbers
-function sumNumbers({ a, b }: { a: number; b: number }): number {
-  return a + b;
+function sumNumbers({ a, b }: { a: number; b: number }) {
+  return `${a + b}`;
 }
 
 // Define a function to divide two numbers
-function divideNumbers({ a, b }: { a: number; b: number }): number {
-  return a / b;
+function divideNumbers({ a, b }: { a: number; b: number }) {
+  return `${a / b}`;
 }
 
 // Define the parameters of the sum function as a JSON schema
@@ -24,7 +24,7 @@ const sumJSON = {
     },
   },
   required: ["a", "b"],
-};
+} as const;
 
 const divideJSON = {
   type: "object",
@@ -39,18 +39,18 @@ const divideJSON = {
     },
   },
   required: ["a", "b"],
-};
+} as const;
 
 async function main() {
   // Create a function tool from the sum function
-  const functionTool = new FunctionTool(sumNumbers, {
+  const functionTool = FunctionTool.from(sumNumbers, {
     name: "sumNumbers",
     description: "Use this function to sum two numbers",
     parameters: sumJSON,
   });
 
   // Create a function tool from the divide function
-  const functionTool2 = new FunctionTool(divideNumbers, {
+  const functionTool2 = FunctionTool.from(divideNumbers, {
     name: "divideNumbers",
     description: "Use this function to divide two numbers",
     parameters: divideJSON,

examples/package.json

@@ -5,7 +5,7 @@
   "dependencies": {
     "@aws-crypto/sha256-js": "^5.2.0",
     "@datastax/astra-db-ts": "^0.1.4",
-    "@notionhq/client": "^2.2.14",
+    "@notionhq/client": "^2.2.15",
     "@pinecone-database/pinecone": "^1.1.3",
     "@zilliz/milvus2-sdk-node": "^2.3.5",
     "chromadb": "^1.8.1",
@@ -19,7 +19,8 @@
   "devDependencies": {
     "@types/node": "^18.19.31",
     "ts-node": "^10.9.2",
-    "typescript": "^5.4.4"
+    "tsx": "^4.7.2",
+    "typescript": "^5.4.5"
   },
   "scripts": {
     "lint": "eslint ."

examples/recipes/cost-analysis.ts

@@ -6,7 +6,8 @@ import { extractText } from "llamaindex/llm/utils";
 const encoding = encodingForModel("gpt-4-0125-preview");
 
 const llm = new OpenAI({
-  model: "gpt-4-0125-preview",
+  // currently is "gpt-4-turbo-2024-04-09"
+  model: "gpt-4-turbo",
 });
 
 let tokenCount = 0;
@@ -19,18 +20,25 @@ Settings.callbackManager.on("llm-start", (event) => {
   console.log("Token count:", tokenCount);
   // https://openai.com/pricing
   // $10.00 / 1M tokens
-  console.log(`Price: $${(tokenCount / 1_000_000) * 10}`);
-});
-Settings.callbackManager.on("llm-end", (event) => {
-  const { response } = event.detail.payload;
-  tokenCount += encoding.encode(extractText(response.message.content)).length;
-  console.log("Token count:", tokenCount);
-  // https://openai.com/pricing
-  // $30.00 / 1M tokens
-  console.log(`Price: $${(tokenCount / 1_000_000) * 30}`);
+  console.log(`Total Price: $${(tokenCount / 1_000_000) * 10}`);
 });
 
-const question = "Hello, how are you?";
+Settings.callbackManager.on("llm-stream", (event) => {
+  const { chunk } = event.detail.payload;
+  const { delta } = chunk;
+  tokenCount += encoding.encode(extractText(delta)).length;
+  if (tokenCount > 20) {
+    // This is just an example, you can set your own limit or handle it differently
+    throw new Error("Token limit exceeded!");
+  }
+});
+Settings.callbackManager.on("llm-end", () => {
+  // https://openai.com/pricing
+  // $30.00 / 1M tokens
+  console.log(`Total Price: $${(tokenCount / 1_000_000) * 30}`);
+});
+
+const question = "Hello, how are you? Please response about 50 tokens.";
 console.log("Question:", question);
 void llm
   .chat({

examples/toolsStream.ts

@@ -2,7 +2,6 @@ import { OpenAI } from "llamaindex";
 
 async function main() {
   const llm = new OpenAI({ model: "gpt-4-turbo" });
-
   const args: Parameters<typeof llm.chat>[0] = {
     additionalChatOptions: {
       tool_choice: "auto",

package.json

@@ -27,13 +27,14 @@
     "prettier": "^3.2.5",
     "prettier-plugin-organize-imports": "^3.2.4",
     "turbo": "^1.13.2",
-    "typescript": "^5.4.4"
+    "typescript": "^5.4.5"
   },
   "packageManager": "pnpm@8.15.6+sha256.01c01eeb990e379b31ef19c03e9d06a14afa5250b82e81303f88721c99ff2e6f",
   "pnpm": {
     "overrides": {
       "trim": "1.0.1",
-      "@babel/traverse": "7.23.2"
+      "@babel/traverse": "7.23.2",
+      "protobufjs": "7.2.6"
     }
   },
   "lint-staged": {

packages/core/e2e/fixtures/embeddings/OpenAIEmbedding.ts

@@ -0,0 +1,33 @@
+import { BaseNode, SimilarityType, type BaseEmbedding } from "llamaindex";
+
+export class OpenAIEmbedding implements BaseEmbedding {
+  embedBatchSize = 512;
+
+  async getQueryEmbedding(query: string) {
+    return [0];
+  }
+
+  async getTextEmbedding(text: string) {
+    return [0];
+  }
+
+  async getTextEmbeddings(texts: string[]) {
+    return [[0]];
+  }
+
+  async getTextEmbeddingsBatch(texts: string[]) {
+    return [[0]];
+  }
+
+  similarity(
+    embedding1: number[],
+    embedding2: number[],
+    mode?: SimilarityType,
+  ) {
+    return 1;
+  }
+
+  async transform(nodes: BaseNode[], _options?: any): Promise<BaseNode[]> {
+    return nodes;
+  }
+}

packages/core/e2e/fixtures/llm/open_ai.ts

@@ -1,4 +1,3 @@
-import { faker } from "@faker-js/faker";
 import type {
   ChatResponse,
   ChatResponseChunk,
@@ -9,6 +8,9 @@ import type {
   LLMCompletionParamsNonStreaming,
   LLMCompletionParamsStreaming,
 } from "llamaindex/llm/types";
+import { extractText } from "llamaindex/llm/utils";
+import { strictEqual } from "node:assert";
+import { llmCompleteMockStorage } from "../../node/utils.js";
 
 export function getOpenAISession() {
   return {};
@@ -40,24 +42,31 @@ export class OpenAI implements LLM {
       | LLMChatParamsStreaming<Record<string, unknown>>
       | LLMChatParamsNonStreaming<Record<string, unknown>>,
   ): unknown {
-    if (params.stream) {
-      return {
-        [Symbol.asyncIterator]: async function* () {
-          yield {
-            delta: faker.word.words(),
-          } satisfies ChatResponseChunk;
-        },
-      };
+    if (llmCompleteMockStorage.llmEventStart.length > 0) {
+      const chatMessage =
+        llmCompleteMockStorage.llmEventStart.shift()!["messages"];
+      strictEqual(chatMessage.length, params.messages.length);
+      for (let i = 0; i < chatMessage.length; i++) {
+        strictEqual(chatMessage[i].role, params.messages[i].role);
+        strictEqual(chatMessage[i].content, params.messages[i].content);
+      }
+
+      if (llmCompleteMockStorage.llmEventEnd.length > 0) {
+        const { id, response } = llmCompleteMockStorage.llmEventEnd.shift()!;
+        if (params.stream) {
+          return {
+            [Symbol.asyncIterator]: async function* () {
+              while (llmCompleteMockStorage.llmEventStream.at(-1)?.id === id) {
+                yield llmCompleteMockStorage.llmEventStream.shift()!["chunk"];
+              }
+            },
+          };
+        } else {
+          return response;
+        }
+      }
     }
-    return {
-      get raw(): never {
-        throw new Error("not implemented");
-      },
-      message: {
-        content: faker.lorem.paragraph(),
-        role: "assistant",
-      },
-    } satisfies ChatResponse;
+    throw new Error("Method not implemented.");
   }
   complete(
     params: LLMCompletionParamsStreaming,
@@ -65,7 +74,23 @@ export class OpenAI implements LLM {
   complete(
     params: LLMCompletionParamsNonStreaming,
   ): Promise<CompletionResponse>;
-  async complete(params: unknown): Promise<unknown> {
+  async complete(
+    params: LLMCompletionParamsStreaming | LLMCompletionParamsNonStreaming,
+  ): Promise<AsyncIterable<CompletionResponse> | CompletionResponse> {
+    if (llmCompleteMockStorage.llmEventStart.length > 0) {
+      const chatMessage =
+        llmCompleteMockStorage.llmEventStart.shift()!["messages"];
+      strictEqual(chatMessage.length, 1);
+      strictEqual(chatMessage[0].role, "user");
+      strictEqual(chatMessage[0].content, params.prompt);
+    }
+    if (llmCompleteMockStorage.llmEventEnd.length > 0) {
+      const response = llmCompleteMockStorage.llmEventEnd.shift()!["response"];
+      return {
+        raw: response,
+        text: extractText(response.message.content),
+      } satisfies CompletionResponse;
+    }
     throw new Error("Method not implemented.");
   }
 }

packages/core/e2e/node/basic.e2e.ts

@@ -1,139 +0,0 @@
-/* eslint-disable @typescript-eslint/no-floating-promises */
-import { consola } from "consola";
-import {
-  OpenAI,
-  OpenAIAgent,
-  Settings,
-  type LLM,
-  type LLMEndEvent,
-  type LLMStartEvent,
-} from "llamaindex";
-import { ok } from "node:assert";
-import type { WriteStream } from "node:fs";
-import { createWriteStream } from "node:fs";
-import { mkdir } from "node:fs/promises";
-import { join } from "node:path";
-import { after, before, beforeEach, describe, test } from "node:test";
-import { inspect } from "node:util";
-
-let llm: LLM;
-let fsStream: WriteStream;
-before(async () => {
-  const logUrl = new URL(
-    join(
-      "..",
-      "logs",
-      `basic.e2e.${new Date().toISOString().replace(/:/g, "-").replace(/\./g, "-")}.log`,
-    ),
-    import.meta.url,
-  );
-  await mkdir(new URL(".", logUrl), { recursive: true });
-  fsStream = createWriteStream(logUrl, {
-    encoding: "utf-8",
-  });
-});
-
-after(() => {
-  fsStream.end();
-});
-
-beforeEach((s) => {
-  fsStream.write("start: " + s.name + "\n");
-});
-
-const llmEventStartHandler = (event: LLMStartEvent) => {
-  const { payload } = event.detail;
-  fsStream.write(
-    "llmEventStart: " +
-      inspect(payload, {
-        depth: Infinity,
-      }) +
-      "\n",
-  );
-};
-
-const llmEventEndHandler = (event: LLMEndEvent) => {
-  const { payload } = event.detail;
-  fsStream.write(
-    "llmEventEnd: " +
-      inspect(payload, {
-        depth: Infinity,
-      }) +
-      "\n",
-  );
-};
-
-before(() => {
-  Settings.llm = new OpenAI({
-    model: "gpt-3.5-turbo",
-  });
-  llm = Settings.llm;
-  Settings.callbackManager.on("llm-start", llmEventStartHandler);
-  Settings.callbackManager.on("llm-end", llmEventEndHandler);
-});
-
-after(() => {
-  Settings.callbackManager.off("llm-start", llmEventStartHandler);
-  Settings.callbackManager.off("llm-end", llmEventEndHandler);
-});
-
-describe("llm", () => {
-  test("llm.chat", async () => {
-    const response = await llm.chat({
-      messages: [
-        {
-          content: "Hello",
-          role: "user",
-        },
-      ],
-    });
-    consola.debug("response:", response);
-    ok(typeof response.message.content === "string");
-  });
-
-  test("stream llm.chat", async () => {
-    const iter = await llm.chat({
-      stream: true,
-      messages: [
-        {
-          content: "hello",
-          role: "user",
-        },
-      ],
-    });
-    for await (const chunk of iter) {
-      consola.debug("chunk:", chunk);
-      ok(typeof chunk.delta === "string");
-    }
-  });
-});
-
-describe("agent", () => {
-  test("agent.chat", async () => {
-    const agent = new OpenAIAgent({
-      tools: [
-        {
-          call: async () => {
-            return "35 degrees and sunny in San Francisco";
-          },
-          metadata: {
-            name: "Weather",
-            description: "Get the weather",
-            parameters: {
-              type: "object",
-              properties: {
-                location: { type: "string" },
-              },
-              required: ["location"],
-            },
-          },
-        },
-      ],
-    });
-    const result = await agent.chat({
-      message: "What is the weather in San Francisco?",
-    });
-    consola.debug("response:", result.response);
-    ok(typeof result.response === "string");
-  });
-});

packages/core/e2e/node/openai.e2e.ts

@@ -0,0 +1,223 @@
+import { consola } from "consola";
+import {
+  Document,
+  FunctionTool,
+  OpenAI,
+  OpenAIAgent,
+  QueryEngineTool,
+  Settings,
+  SubQuestionQueryEngine,
+  VectorStoreIndex,
+  type LLM,
+} from "llamaindex";
+import { ok } from "node:assert";
+import { beforeEach, test } from "node:test";
+import { mockLLMEvent } from "./utils.js";
+
+let llm: LLM;
+beforeEach(async () => {
+  Settings.llm = new OpenAI({
+    model: "gpt-3.5-turbo",
+  });
+  llm = Settings.llm;
+});
+
+await test("llm", async (t) => {
+  await mockLLMEvent(t, "llm");
+  await t.test("llm.chat", async () => {
+    const response = await llm.chat({
+      messages: [
+        {
+          content: "Hello",
+          role: "user",
+        },
+      ],
+    });
+    consola.debug("response:", response);
+    ok(typeof response.message.content === "string");
+  });
+
+  await t.test("stream llm.chat", async () => {
+    const iter = await llm.chat({
+      stream: true,
+      messages: [
+        {
+          content: "hello",
+          role: "user",
+        },
+      ],
+    });
+    for await (const chunk of iter) {
+      consola.debug("chunk:", chunk);
+      ok(typeof chunk.delta === "string");
+    }
+  });
+});
+
+await test("gpt-4-turbo", async (t) => {
+  const llm = new OpenAI({ model: "gpt-4-turbo" });
+  Settings.llm = llm;
+  await mockLLMEvent(t, "gpt-4-turbo");
+  await t.test("agent", async () => {
+    const agent = new OpenAIAgent({
+      llm,
+      tools: [
+        {
+          call: async () => {
+            return "45 degrees and sunny in San Jose";
+          },
+          metadata: {
+            name: "Weather",
+            description: "Get the weather",
+            parameters: {
+              type: "object",
+              properties: {
+                location: { type: "string" },
+              },
+              required: ["location"],
+            },
+          },
+        },
+      ],
+    });
+    const { response } = await agent.chat({
+      message: "What is the weather in San Jose?",
+    });
+    consola.debug("response:", response);
+    ok(typeof response === "string");
+    ok(response.includes("45"));
+  });
+});
+
+await test("agent", async (t) => {
+  await mockLLMEvent(t, "agent");
+  await t.test("chat", async () => {
+    const agent = new OpenAIAgent({
+      tools: [
+        {
+          call: async () => {
+            return "35 degrees and sunny in San Francisco";
+          },
+          metadata: {
+            name: "Weather",
+            description: "Get the weather",
+            parameters: {
+              type: "object",
+              properties: {
+                location: { type: "string" },
+              },
+              required: ["location"],
+            },
+          },
+        },
+      ],
+    });
+    const result = await agent.chat({
+      message: "What is the weather in San Francisco?",
+    });
+    consola.debug("response:", result.response);
+    ok(typeof result.response === "string");
+    ok(result.response.includes("35"));
+  });
+
+  await t.test("async function", async () => {
+    const uniqueId = "123456789";
+    const showUniqueId = FunctionTool.from<{
+      firstName: string;
+      lastName: string;
+    }>(
+      async ({ firstName, lastName }) => {
+        ok(typeof firstName === "string");
+        ok(typeof lastName === "string");
+        const fullName = firstName + lastName;
+        ok(fullName.toLowerCase().includes("alex"));
+        ok(fullName.toLowerCase().includes("yang"));
+        return uniqueId;
+      },
+      {
+        name: "unique_id",
+        description: "show user unique id",
+        parameters: {
+          type: "object",
+          properties: {
+            firstName: { type: "string" },
+            lastName: { type: "string" },
+          },
+          required: ["firstName", "lastName"],
+        },
+      },
+    );
+    const agent = new OpenAIAgent({
+      tools: [showUniqueId],
+    });
+    const { response } = await agent.chat({
+      message: "My name is Alex Yang. What is my unique id?",
+    });
+    consola.debug("response:", response);
+    ok(response.includes(uniqueId));
+  });
+
+  await t.test("sum numbers", async () => {
+    function sumNumbers({ a, b }: { a: number; b: number }): string {
+      return `${a + b}`;
+    }
+    const sumFunctionTool = new FunctionTool(sumNumbers, {
+      name: "sumNumbers",
+      description: "Use this function to sum two numbers",
+      parameters: {
+        type: "object",
+        properties: {
+          a: {
+            type: "number",
+            description: "The first number",
+          },
+          b: {
+            type: "number",
+            description: "The second number",
+          },
+        },
+        required: ["a", "b"],
+      },
+    });
+
+    const openaiAgent = new OpenAIAgent({
+      tools: [sumFunctionTool],
+    });
+
+    const response = await openaiAgent.chat({
+      message: "how much is 1 + 1?",
+    });
+
+    ok(response.response.includes("2"));
+  });
+});
+
+await test("queryEngine", async (t) => {
+  await mockLLMEvent(t, "queryEngine_subquestion");
+  await t.test("subquestion", async () => {
+    const document = new Document({
+      text: "Bill Gates stole from Apple.\n Steve Jobs stole from Xerox.",
+    });
+    const index = await VectorStoreIndex.fromDocuments([document]);
+
+    const queryEngineTools = [
+      new QueryEngineTool({
+        queryEngine: index.asQueryEngine(),
+        metadata: {
+          name: "bill_gates_idea",
+          description: "Get what Bill Gates idea from.",
+        },
+      }),
+    ];
+
+    const queryEngine = SubQuestionQueryEngine.fromDefaults({
+      queryEngineTools,
+    });
+
+    const { response } = await queryEngine.query({
+      query: "What did Bill Gates steal from?",
+    });
+
+    ok(response.includes("Apple"));
+  });
+});

packages/core/e2e/node/snapshot/agent.snap

@@ -0,0 +1,394 @@
+{
+  "llmEventStart": [
+    {
+      "id": "HIDDEN",
+      "messages": [
+        {
+          "content": "What is the weather in San Francisco?",
+          "role": "user"
+        }
+      ]
+    },
+    {
+      "id": "HIDDEN",
+      "messages": [
+        {
+          "content": "What is the weather in San Francisco?",
+          "role": "user"
+        },
+        {
+          "content": "",
+          "role": "assistant",
+          "options": {
+            "toolCalls": [
+              {
+                "id": "HIDDEN",
+                "type": "function",
+                "function": {
+                  "name": "Weather",
+                  "arguments": "{\"location\":\"San Francisco\"}"
+                }
+              }
+            ]
+          }
+        },
+        {
+          "content": "35 degrees and sunny in San Francisco",
+          "role": "tool",
+          "options": {
+            "name": "Weather",
+            "tool_call_id": "HIDDEN"
+          }
+        }
+      ]
+    },
+    {
+      "id": "HIDDEN",
+      "messages": [
+        {
+          "content": "My name is Alex Yang. What is my unique id?",
+          "role": "user"
+        }
+      ]
+    },
+    {
+      "id": "HIDDEN",
+      "messages": [
+        {
+          "content": "My name is Alex Yang. What is my unique id?",
+          "role": "user"
+        },
+        {
+          "content": "",
+          "role": "assistant",
+          "options": {
+            "toolCalls": [
+              {
+                "id": "HIDDEN",
+                "type": "function",
+                "function": {
+                  "name": "unique_id",
+                  "arguments": "{\"firstName\":\"Alex\",\"lastName\":\"Yang\"}"
+                }
+              }
+            ]
+          }
+        },
+        {
+          "content": "123456789",
+          "role": "tool",
+          "options": {
+            "name": "unique_id",
+            "tool_call_id": "HIDDEN"
+          }
+        }
+      ]
+    },
+    {
+      "id": "HIDDEN",
+      "messages": [
+        {
+          "content": "how much is 1 + 1?",
+          "role": "user"
+        }
+      ]
+    },
+    {
+      "id": "HIDDEN",
+      "messages": [
+        {
+          "content": "how much is 1 + 1?",
+          "role": "user"
+        },
+        {
+          "content": "",
+          "role": "assistant",
+          "options": {
+            "toolCalls": [
+              {
+                "id": "HIDDEN",
+                "type": "function",
+                "function": {
+                  "name": "sumNumbers",
+                  "arguments": "{\"a\":1,\"b\":1}"
+                }
+              }
+            ]
+          }
+        },
+        {
+          "content": "2",
+          "role": "tool",
+          "options": {
+            "name": "sumNumbers",
+            "tool_call_id": "HIDDEN"
+          }
+        }
+      ]
+    }
+  ],
+  "llmEventEnd": [
+    {
+      "id": "HIDDEN",
+      "response": {
+        "raw": {
+          "id": "HIDDEN",
+          "object": "chat.completion",
+          "created": 114514,
+          "model": "gpt-3.5-turbo-0125",
+          "choices": [
+            {
+              "index": 0,
+              "message": {
+                "role": "assistant",
+                "content": null,
+                "tool_calls": [
+                  {
+                    "id": "HIDDEN",
+                    "type": "function",
+                    "function": {
+                      "name": "Weather",
+                      "arguments": "{\"location\":\"San Francisco\"}"
+                    }
+                  }
+                ]
+              },
+              "logprobs": null,
+              "finish_reason": "tool_calls"
+            }
+          ],
+          "usage": {
+            "prompt_tokens": 49,
+            "completion_tokens": 15,
+            "total_tokens": 64
+          },
+          "system_fingerprint": "HIDDEN"
+        },
+        "message": {
+          "content": "",
+          "role": "assistant",
+          "options": {
+            "toolCalls": [
+              {
+                "id": "HIDDEN",
+                "type": "function",
+                "function": {
+                  "name": "Weather",
+                  "arguments": "{\"location\":\"San Francisco\"}"
+                }
+              }
+            ]
+          }
+        }
+      }
+    },
+    {
+      "id": "HIDDEN",
+      "response": {
+        "raw": {
+          "id": "HIDDEN",
+          "object": "chat.completion",
+          "created": 114514,
+          "model": "gpt-3.5-turbo-0125",
+          "choices": [
+            {
+              "index": 0,
+              "message": {
+                "role": "assistant",
+                "content": "The weather in San Francisco is currently 35 degrees and sunny."
+              },
+              "logprobs": null,
+              "finish_reason": "stop"
+            }
+          ],
+          "usage": {
+            "prompt_tokens": 78,
+            "completion_tokens": 14,
+            "total_tokens": 92
+          },
+          "system_fingerprint": "HIDDEN"
+        },
+        "message": {
+          "content": "The weather in San Francisco is currently 35 degrees and sunny.",
+          "role": "assistant",
+          "options": {}
+        }
+      }
+    },
+    {
+      "id": "HIDDEN",
+      "response": {
+        "raw": {
+          "id": "HIDDEN",
+          "object": "chat.completion",
+          "created": 114514,
+          "model": "gpt-3.5-turbo-0125",
+          "choices": [
+            {
+              "index": 0,
+              "message": {
+                "role": "assistant",
+                "content": null,
+                "tool_calls": [
+                  {
+                    "id": "HIDDEN",
+                    "type": "function",
+                    "function": {
+                      "name": "unique_id",
+                      "arguments": "{\"firstName\":\"Alex\",\"lastName\":\"Yang\"}"
+                    }
+                  }
+                ]
+              },
+              "logprobs": null,
+              "finish_reason": "tool_calls"
+            }
+          ],
+          "usage": {
+            "prompt_tokens": 59,
+            "completion_tokens": 18,
+            "total_tokens": 77
+          },
+          "system_fingerprint": "HIDDEN"
+        },
+        "message": {
+          "content": "",
+          "role": "assistant",
+          "options": {
+            "toolCalls": [
+              {
+                "id": "HIDDEN",
+                "type": "function",
+                "function": {
+                  "name": "unique_id",
+                  "arguments": "{\"firstName\":\"Alex\",\"lastName\":\"Yang\"}"
+                }
+              }
+            ]
+          }
+        }
+      }
+    },
+    {
+      "id": "HIDDEN",
+      "response": {
+        "raw": {
+          "id": "HIDDEN",
+          "object": "chat.completion",
+          "created": 114514,
+          "model": "gpt-3.5-turbo-0125",
+          "choices": [
+            {
+              "index": 0,
+              "message": {
+                "role": "assistant",
+                "content": "Your unique id is 123456789."
+              },
+              "logprobs": null,
+              "finish_reason": "stop"
+            }
+          ],
+          "usage": {
+            "prompt_tokens": 88,
+            "completion_tokens": 10,
+            "total_tokens": 98
+          },
+          "system_fingerprint": "HIDDEN"
+        },
+        "message": {
+          "content": "Your unique id is 123456789.",
+          "role": "assistant",
+          "options": {}
+        }
+      }
+    },
+    {
+      "id": "HIDDEN",
+      "response": {
+        "raw": {
+          "id": "HIDDEN",
+          "object": "chat.completion",
+          "created": 114514,
+          "model": "gpt-3.5-turbo-0125",
+          "choices": [
+            {
+              "index": 0,
+              "message": {
+                "role": "assistant",
+                "content": null,
+                "tool_calls": [
+                  {
+                    "id": "HIDDEN",
+                    "type": "function",
+                    "function": {
+                      "name": "sumNumbers",
+                      "arguments": "{\"a\":1,\"b\":1}"
+                    }
+                  }
+                ]
+              },
+              "logprobs": null,
+              "finish_reason": "tool_calls"
+            }
+          ],
+          "usage": {
+            "prompt_tokens": 70,
+            "completion_tokens": 18,
+            "total_tokens": 88
+          },
+          "system_fingerprint": "HIDDEN"
+        },
+        "message": {
+          "content": "",
+          "role": "assistant",
+          "options": {
+            "toolCalls": [
+              {
+                "id": "HIDDEN",
+                "type": "function",
+                "function": {
+                  "name": "sumNumbers",
+                  "arguments": "{\"a\":1,\"b\":1}"
+                }
+              }
+            ]
+          }
+        }
+      }
+    },
+    {
+      "id": "HIDDEN",
+      "response": {
+        "raw": {
+          "id": "HIDDEN",
+          "object": "chat.completion",
+          "created": 114514,
+          "model": "gpt-3.5-turbo-0125",
+          "choices": [
+            {
+              "index": 0,
+              "message": {
+                "role": "assistant",
+                "content": "1 + 1 is equal to 2."
+              },
+              "logprobs": null,
+              "finish_reason": "stop"
+            }
+          ],
+          "usage": {
+            "prompt_tokens": 97,
+            "completion_tokens": 11,
+            "total_tokens": 108
+          },
+          "system_fingerprint": "HIDDEN"
+        },
+        "message": {
+          "content": "1 + 1 is equal to 2.",
+          "role": "assistant",
+          "options": {}
+        }
+      }
+    }
+  ],
+  "llmEventStream": []
+}

packages/core/e2e/node/snapshot/gpt-4-turbo.snap

@@ -0,0 +1,136 @@
+{
+  "llmEventStart": [
+    {
+      "id": "HIDDEN",
+      "messages": [
+        {
+          "content": "What is the weather in San Jose?",
+          "role": "user"
+        }
+      ]
+    },
+    {
+      "id": "HIDDEN",
+      "messages": [
+        {
+          "content": "What is the weather in San Jose?",
+          "role": "user"
+        },
+        {
+          "content": "",
+          "role": "assistant",
+          "options": {
+            "toolCalls": [
+              {
+                "id": "HIDDEN",
+                "type": "function",
+                "function": {
+                  "name": "Weather",
+                  "arguments": "{\"location\":\"San Jose\"}"
+                }
+              }
+            ]
+          }
+        },
+        {
+          "content": "45 degrees and sunny in San Jose",
+          "role": "tool",
+          "options": {
+            "name": "Weather",
+            "tool_call_id": "HIDDEN"
+          }
+        }
+      ]
+    }
+  ],
+  "llmEventEnd": [
+    {
+      "id": "HIDDEN",
+      "response": {
+        "raw": {
+          "id": "HIDDEN",
+          "object": "chat.completion",
+          "created": 114514,
+          "model": "gpt-3.5-turbo-0125",
+          "choices": [
+            {
+              "index": 0,
+              "message": {
+                "role": "assistant",
+                "content": null,
+                "tool_calls": [
+                  {
+                    "id": "HIDDEN",
+                    "type": "function",
+                    "function": {
+                      "name": "Weather",
+                      "arguments": "{\"location\":\"San Jose\"}"
+                    }
+                  }
+                ]
+              },
+              "logprobs": null,
+              "finish_reason": "tool_calls"
+            }
+          ],
+          "usage": {
+            "prompt_tokens": 49,
+            "completion_tokens": 15,
+            "total_tokens": 64
+          },
+          "system_fingerprint": "HIDDEN"
+        },
+        "message": {
+          "content": "",
+          "role": "assistant",
+          "options": {
+            "toolCalls": [
+              {
+                "id": "HIDDEN",
+                "type": "function",
+                "function": {
+                  "name": "Weather",
+                  "arguments": "{\"location\":\"San Jose\"}"
+                }
+              }
+            ]
+          }
+        }
+      }
+    },
+    {
+      "id": "HIDDEN",
+      "response": {
+        "raw": {
+          "id": "HIDDEN",
+          "object": "chat.completion",
+          "created": 114514,
+          "model": "gpt-3.5-turbo-0125",
+          "choices": [
+            {
+              "index": 0,
+              "message": {
+                "role": "assistant",
+                "content": "The weather in San Jose is 45 degrees and sunny."
+              },
+              "logprobs": null,
+              "finish_reason": "stop"
+            }
+          ],
+          "usage": {
+            "prompt_tokens": 78,
+            "completion_tokens": 13,
+            "total_tokens": 91
+          },
+          "system_fingerprint": "HIDDEN"
+        },
+        "message": {
+          "content": "The weather in San Jose is 45 degrees and sunny.",
+          "role": "assistant",
+          "options": {}
+        }
+      }
+    }
+  ],
+  "llmEventStream": []
+}

packages/core/e2e/node/snapshot/llm.snap

@@ -0,0 +1,476 @@
+{
+  "llmEventStart": [
+    {
+      "id": "HIDDEN",
+      "messages": [
+        {
+          "content": "Hello",
+          "role": "user"
+        }
+      ]
+    },
+    {
+      "id": "HIDDEN",
+      "messages": [
+        {
+          "content": "hello",
+          "role": "user"
+        }
+      ]
+    }
+  ],
+  "llmEventEnd": [
+    {
+      "id": "HIDDEN",
+      "response": {
+        "raw": {
+          "id": "HIDDEN",
+          "object": "chat.completion",
+          "created": 114514,
+          "model": "gpt-3.5-turbo-0125",
+          "choices": [
+            {
+              "index": 0,
+              "message": {
+                "role": "assistant",
+                "content": "Hello! How can I assist you today?"
+              },
+              "logprobs": null,
+              "finish_reason": "stop"
+            }
+          ],
+          "usage": {
+            "prompt_tokens": 8,
+            "completion_tokens": 9,
+            "total_tokens": 17
+          },
+          "system_fingerprint": "HIDDEN"
+        },
+        "message": {
+          "content": "Hello! How can I assist you today?",
+          "role": "assistant",
+          "options": {}
+        }
+      }
+    },
+    {
+      "id": "HIDDEN",
+      "response": {
+        "raw": [
+          {
+            "raw": {
+              "id": "HIDDEN",
+              "object": "chat.completion.chunk",
+              "created": 114514,
+              "model": "gpt-3.5-turbo-0125",
+              "system_fingerprint": "HIDDEN",
+              "choices": [
+                {
+                  "index": 0,
+                  "delta": {
+                    "content": "Hello"
+                  },
+                  "logprobs": null,
+                  "finish_reason": null
+                }
+              ]
+            },
+            "options": {},
+            "delta": "Hello"
+          },
+          {
+            "raw": {
+              "id": "HIDDEN",
+              "object": "chat.completion.chunk",
+              "created": 114514,
+              "model": "gpt-3.5-turbo-0125",
+              "system_fingerprint": "HIDDEN",
+              "choices": [
+                {
+                  "index": 0,
+                  "delta": {
+                    "content": "!"
+                  },
+                  "logprobs": null,
+                  "finish_reason": null
+                }
+              ]
+            },
+            "options": {},
+            "delta": "!"
+          },
+          {
+            "raw": {
+              "id": "HIDDEN",
+              "object": "chat.completion.chunk",
+              "created": 114514,
+              "model": "gpt-3.5-turbo-0125",
+              "system_fingerprint": "HIDDEN",
+              "choices": [
+                {
+                  "index": 0,
+                  "delta": {
+                    "content": " How"
+                  },
+                  "logprobs": null,
+                  "finish_reason": null
+                }
+              ]
+            },
+            "options": {},
+            "delta": " How"
+          },
+          {
+            "raw": {
+              "id": "HIDDEN",
+              "object": "chat.completion.chunk",
+              "created": 114514,
+              "model": "gpt-3.5-turbo-0125",
+              "system_fingerprint": "HIDDEN",
+              "choices": [
+                {
+                  "index": 0,
+                  "delta": {
+                    "content": " can"
+                  },
+                  "logprobs": null,
+                  "finish_reason": null
+                }
+              ]
+            },
+            "options": {},
+            "delta": " can"
+          },
+          {
+            "raw": {
+              "id": "HIDDEN",
+              "object": "chat.completion.chunk",
+              "created": 114514,
+              "model": "gpt-3.5-turbo-0125",
+              "system_fingerprint": "HIDDEN",
+              "choices": [
+                {
+                  "index": 0,
+                  "delta": {
+                    "content": " I"
+                  },
+                  "logprobs": null,
+                  "finish_reason": null
+                }
+              ]
+            },
+            "options": {},
+            "delta": " I"
+          },
+          {
+            "raw": {
+              "id": "HIDDEN",
+              "object": "chat.completion.chunk",
+              "created": 114514,
+              "model": "gpt-3.5-turbo-0125",
+              "system_fingerprint": "HIDDEN",
+              "choices": [
+                {
+                  "index": 0,
+                  "delta": {
+                    "content": " assist"
+                  },
+                  "logprobs": null,
+                  "finish_reason": null
+                }
+              ]
+            },
+            "options": {},
+            "delta": " assist"
+          },
+          {
+            "raw": {
+              "id": "HIDDEN",
+              "object": "chat.completion.chunk",
+              "created": 114514,
+              "model": "gpt-3.5-turbo-0125",
+              "system_fingerprint": "HIDDEN",
+              "choices": [
+                {
+                  "index": 0,
+                  "delta": {
+                    "content": " you"
+                  },
+                  "logprobs": null,
+                  "finish_reason": null
+                }
+              ]
+            },
+            "options": {},
+            "delta": " you"
+          },
+          {
+            "raw": {
+              "id": "HIDDEN",
+              "object": "chat.completion.chunk",
+              "created": 114514,
+              "model": "gpt-3.5-turbo-0125",
+              "system_fingerprint": "HIDDEN",
+              "choices": [
+                {
+                  "index": 0,
+                  "delta": {
+                    "content": " today"
+                  },
+                  "logprobs": null,
+                  "finish_reason": null
+                }
+              ]
+            },
+            "options": {},
+            "delta": " today"
+          },
+          {
+            "raw": {
+              "id": "HIDDEN",
+              "object": "chat.completion.chunk",
+              "created": 114514,
+              "model": "gpt-3.5-turbo-0125",
+              "system_fingerprint": "HIDDEN",
+              "choices": [
+                {
+                  "index": 0,
+                  "delta": {
+                    "content": "?"
+                  },
+                  "logprobs": null,
+                  "finish_reason": null
+                }
+              ]
+            },
+            "options": {},
+            "delta": "?"
+          }
+        ],
+        "message": {
+          "content": "Hello! How can I assist you today?",
+          "role": "assistant",
+          "options": {}
+        }
+      }
+    }
+  ],
+  "llmEventStream": [
+    {
+      "id": "HIDDEN",
+      "chunk": {
+        "raw": {
+          "id": "HIDDEN",
+          "object": "chat.completion.chunk",
+          "created": 114514,
+          "model": "gpt-3.5-turbo-0125",
+          "system_fingerprint": "HIDDEN",
+          "choices": [
+            {
+              "index": 0,
+              "delta": {
+                "content": "Hello"
+              },
+              "logprobs": null,
+              "finish_reason": null
+            }
+          ]
+        },
+        "options": {},
+        "delta": "Hello"
+      }
+    },
+    {
+      "id": "HIDDEN",
+      "chunk": {
+        "raw": {
+          "id": "HIDDEN",
+          "object": "chat.completion.chunk",
+          "created": 114514,
+          "model": "gpt-3.5-turbo-0125",
+          "system_fingerprint": "HIDDEN",
+          "choices": [
+            {
+              "index": 0,
+              "delta": {
+                "content": "!"
+              },
+              "logprobs": null,
+              "finish_reason": null
+            }
+          ]
+        },
+        "options": {},
+        "delta": "!"
+      }
+    },
+    {
+      "id": "HIDDEN",
+      "chunk": {
+        "raw": {
+          "id": "HIDDEN",
+          "object": "chat.completion.chunk",
+          "created": 114514,
+          "model": "gpt-3.5-turbo-0125",
+          "system_fingerprint": "HIDDEN",
+          "choices": [
+            {
+              "index": 0,
+              "delta": {
+                "content": " How"
+              },
+              "logprobs": null,
+              "finish_reason": null
+            }
+          ]
+        },
+        "options": {},
+        "delta": " How"
+      }
+    },
+    {
+      "id": "HIDDEN",
+      "chunk": {
+        "raw": {
+          "id": "HIDDEN",
+          "object": "chat.completion.chunk",
+          "created": 114514,
+          "model": "gpt-3.5-turbo-0125",
+          "system_fingerprint": "HIDDEN",
+          "choices": [
+            {
+              "index": 0,
+              "delta": {
+                "content": " can"
+              },
+              "logprobs": null,
+              "finish_reason": null
+            }
+          ]
+        },
+        "options": {},
+        "delta": " can"
+      }
+    },
+    {
+      "id": "HIDDEN",
+      "chunk": {
+        "raw": {
+          "id": "HIDDEN",
+          "object": "chat.completion.chunk",
+          "created": 114514,
+          "model": "gpt-3.5-turbo-0125",
+          "system_fingerprint": "HIDDEN",
+          "choices": [
+            {
+              "index": 0,
+              "delta": {
+                "content": " I"
+              },
+              "logprobs": null,
+              "finish_reason": null
+            }
+          ]
+        },
+        "options": {},
+        "delta": " I"
+      }
+    },
+    {
+      "id": "HIDDEN",
+      "chunk": {
+        "raw": {
+          "id": "HIDDEN",
+          "object": "chat.completion.chunk",
+          "created": 114514,
+          "model": "gpt-3.5-turbo-0125",
+          "system_fingerprint": "HIDDEN",
+          "choices": [
+            {
+              "index": 0,
+              "delta": {
+                "content": " assist"
+              },
+              "logprobs": null,
+              "finish_reason": null
+            }
+          ]
+        },
+        "options": {},
+        "delta": " assist"
+      }
+    },
+    {
+      "id": "HIDDEN",
+      "chunk": {
+        "raw": {
+          "id": "HIDDEN",
+          "object": "chat.completion.chunk",
+          "created": 114514,
+          "model": "gpt-3.5-turbo-0125",
+          "system_fingerprint": "HIDDEN",
+          "choices": [
+            {
+              "index": 0,
+              "delta": {
+                "content": " you"
+              },
+              "logprobs": null,
+              "finish_reason": null
+            }
+          ]
+        },
+        "options": {},
+        "delta": " you"
+      }
+    },
+    {
+      "id": "HIDDEN",
+      "chunk": {
+        "raw": {
+          "id": "HIDDEN",
+          "object": "chat.completion.chunk",
+          "created": 114514,
+          "model": "gpt-3.5-turbo-0125",
+          "system_fingerprint": "HIDDEN",
+          "choices": [
+            {
+              "index": 0,
+              "delta": {
+                "content": " today"
+              },
+              "logprobs": null,
+              "finish_reason": null
+            }
+          ]
+        },
+        "options": {},
+        "delta": " today"
+      }
+    },
+    {
+      "id": "HIDDEN",
+      "chunk": {
+        "raw": {
+          "id": "HIDDEN",
+          "object": "chat.completion.chunk",
+          "created": 114514,
+          "model": "gpt-3.5-turbo-0125",
+          "system_fingerprint": "HIDDEN",
+          "choices": [
+            {
+              "index": 0,
+              "delta": {
+                "content": "?"
+              },
+              "logprobs": null,
+              "finish_reason": null
+            }
+          ]
+        },
+        "options": {},
+        "delta": "?"
+      }
+    }
+  ]
+}

packages/core/e2e/node/snapshot/queryEngine_subquestion.snap

@@ -0,0 +1,133 @@
+{
+  "llmEventStart": [
+    {
+      "id": "HIDDEN",
+      "messages": [
+        {
+          "content": "Given a user question, and a list of tools, output a list of relevant sub-questions that when composed can help answer the full user question:\n\n# Example 1\n<Tools>\n```json\n{\n    \"uber_10k\": \"Provides information about Uber financials for year 2021\",\n    \"lyft_10k\": \"Provides information about Lyft financials for year 2021\"\n}\n```\n\n<User Question>\nCompare and contrast the revenue growth and EBITDA of Uber and Lyft for year 2021\n\n<Output>\n```json\n[\n    {\n        \"subQuestion\": \"What is the revenue growth of Uber\",\n        \"toolName\": \"uber_10k\"\n    },\n    {\n        \"subQuestion\": \"What is the EBITDA of Uber\",\n        \"toolName\": \"uber_10k\"\n    },\n    {\n        \"subQuestion\": \"What is the revenue growth of Lyft\",\n        \"toolName\": \"lyft_10k\"\n    },\n    {\n        \"subQuestion\": \"What is the EBITDA of Lyft\",\n        \"toolName\": \"lyft_10k\"\n    }\n]\n```\n\n# Example 2\n<Tools>\n```json\n{\n    \"bill_gates_idea\": \"Get what Bill Gates idea from.\"\n}\n```\n\n<User Question>\nWhat did Bill Gates steal from?\n\n<Output>\n",
+          "role": "user"
+        }
+      ]
+    },
+    {
+      "id": "HIDDEN",
+      "messages": [
+        {
+          "content": "Context information is below.\n---------------------\nBill Gates stole from Apple. Steve Jobs stole from Xerox.\n---------------------\nGiven the context information and not prior knowledge, answer the query.\nQuery: What is Bill Gates' idea\nAnswer:",
+          "role": "user"
+        }
+      ]
+    },
+    {
+      "id": "HIDDEN",
+      "messages": [
+        {
+          "content": "Context information is below.\n---------------------\nSub question: What is Bill Gates' idea\nResponse: Bill Gates' idea was to steal from Apple.\n---------------------\nGiven the context information and not prior knowledge, answer the query.\nQuery: What did Bill Gates steal from?\nAnswer:",
+          "role": "user"
+        }
+      ]
+    }
+  ],
+  "llmEventEnd": [
+    {
+      "id": "HIDDEN",
+      "response": {
+        "raw": {
+          "id": "HIDDEN",
+          "object": "chat.completion",
+          "created": 114514,
+          "model": "gpt-3.5-turbo-0125",
+          "choices": [
+            {
+              "index": 0,
+              "message": {
+                "role": "assistant",
+                "content": "```json\n[\n    {\n        \"subQuestion\": \"What is Bill Gates' idea\",\n        \"toolName\": \"bill_gates_idea\"\n    }\n]\n```"
+              },
+              "logprobs": null,
+              "finish_reason": "stop"
+            }
+          ],
+          "usage": {
+            "prompt_tokens": 290,
+            "completion_tokens": 35,
+            "total_tokens": 325
+          },
+          "system_fingerprint": "HIDDEN"
+        },
+        "message": {
+          "content": "```json\n[\n    {\n        \"subQuestion\": \"What is Bill Gates' idea\",\n        \"toolName\": \"bill_gates_idea\"\n    }\n]\n```",
+          "role": "assistant",
+          "options": {}
+        }
+      }
+    },
+    {
+      "id": "HIDDEN",
+      "response": {
+        "raw": {
+          "id": "HIDDEN",
+          "object": "chat.completion",
+          "created": 114514,
+          "model": "gpt-3.5-turbo-0125",
+          "choices": [
+            {
+              "index": 0,
+              "message": {
+                "role": "assistant",
+                "content": "Bill Gates' idea was to steal from Apple."
+              },
+              "logprobs": null,
+              "finish_reason": "stop"
+            }
+          ],
+          "usage": {
+            "prompt_tokens": 53,
+            "completion_tokens": 10,
+            "total_tokens": 63
+          },
+          "system_fingerprint": "HIDDEN"
+        },
+        "message": {
+          "content": "Bill Gates' idea was to steal from Apple.",
+          "role": "assistant",
+          "options": {}
+        }
+      }
+    },
+    {
+      "id": "HIDDEN",
+      "response": {
+        "raw": {
+          "id": "HIDDEN",
+          "object": "chat.completion",
+          "created": 114514,
+          "model": "gpt-3.5-turbo-0125",
+          "choices": [
+            {
+              "index": 0,
+              "message": {
+                "role": "assistant",
+                "content": "Bill Gates stole from Apple."
+              },
+              "logprobs": null,
+              "finish_reason": "stop"
+            }
+          ],
+          "usage": {
+            "prompt_tokens": 62,
+            "completion_tokens": 6,
+            "total_tokens": 68
+          },
+          "system_fingerprint": "HIDDEN"
+        },
+        "message": {
+          "content": "Bill Gates stole from Apple.",
+          "role": "assistant",
+          "options": {}
+        }
+      }
+    }
+  ],
+  "llmEventStream": []
+}

packages/core/e2e/node/utils.ts

@@ -0,0 +1,123 @@
+import {
+  Settings,
+  type LLMEndEvent,
+  type LLMStartEvent,
+  type LLMStreamEvent,
+} from "llamaindex";
+import { readFile, writeFile } from "node:fs/promises";
+import { join } from "node:path";
+import { type test } from "node:test";
+import { fileURLToPath } from "node:url";
+
+type MockStorage = {
+  llmEventStart: LLMStartEvent["detail"]["payload"][];
+  llmEventEnd: LLMEndEvent["detail"]["payload"][];
+  llmEventStream: LLMStreamEvent["detail"]["payload"][];
+};
+
+export const llmCompleteMockStorage: MockStorage = {
+  llmEventStart: [],
+  llmEventEnd: [],
+  llmEventStream: [],
+};
+
+export const testRootDir = fileURLToPath(new URL(".", import.meta.url));
+
+export async function mockLLMEvent(
+  t: Parameters<NonNullable<Parameters<typeof test>[0]>>[0],
+  snapshotName: string,
+) {
+  const newLLMCompleteMockStorage: MockStorage = {
+    llmEventStart: [],
+    llmEventEnd: [],
+    llmEventStream: [],
+  };
+
+  function captureLLMStart(event: LLMStartEvent) {
+    newLLMCompleteMockStorage.llmEventStart.push(event.detail.payload);
+  }
+
+  function captureLLMEnd(event: LLMEndEvent) {
+    newLLMCompleteMockStorage.llmEventEnd.push(event.detail.payload);
+  }
+
+  function captureLLMStream(event: LLMStreamEvent) {
+    newLLMCompleteMockStorage.llmEventStream.push(event.detail.payload);
+  }
+
+  await readFile(join(testRootDir, "snapshot", `${snapshotName}.snap`), {
+    encoding: "utf-8",
+  })
+    .then((data) => {
+      const result = JSON.parse(data) as MockStorage;
+      result["llmEventEnd"].forEach((event) => {
+        llmCompleteMockStorage.llmEventEnd.push(event);
+      });
+      result["llmEventStart"].forEach((event) => {
+        llmCompleteMockStorage.llmEventStart.push(event);
+      });
+      result["llmEventStream"].forEach((event) => {
+        llmCompleteMockStorage.llmEventStream.push(event);
+      });
+    })
+    .catch((error) => {
+      if (error.code === "ENOENT") {
+        console.warn("Snapshot file not found, will create a new one");
+        return;
+      }
+    });
+  Settings.callbackManager.on("llm-start", captureLLMStart);
+  Settings.callbackManager.on("llm-end", captureLLMEnd);
+  Settings.callbackManager.on("llm-stream", captureLLMStream);
+
+  t.after(async () => {
+    Settings.callbackManager.off("llm-stream", captureLLMStream);
+    Settings.callbackManager.off("llm-end", captureLLMEnd);
+    Settings.callbackManager.off("llm-start", captureLLMStart);
+    // eslint-disable-next-line turbo/no-undeclared-env-vars
+    if (process.env.UPDATE_SNAPSHOT === "1") {
+      const data = JSON.stringify(newLLMCompleteMockStorage, null, 2)
+        .replace(/"id": ".*"/g, `"id": "HIDDEN"`)
+        .replace(/"created": \d+/g, `"created": 114514`)
+        .replace(
+          /"system_fingerprint": ".*"/g,
+          '"system_fingerprint": "HIDDEN"',
+        )
+        .replace(/"tool_call_id": ".*"/g, '"tool_call_id": "HIDDEN"');
+      await writeFile(
+        join(testRootDir, "snapshot", `${snapshotName}.snap`),
+        data,
+      );
+      return;
+    }
+    if (
+      newLLMCompleteMockStorage.llmEventEnd.length !==
+      llmCompleteMockStorage.llmEventEnd.length
+    ) {
+      throw new Error("New LLMEndEvent does not match, please update snapshot");
+    }
+    if (
+      newLLMCompleteMockStorage.llmEventStart.length !==
+      llmCompleteMockStorage.llmEventStart.length
+    ) {
+      throw new Error(
+        "New LLMStartEvent does not match, please update snapshot",
+      );
+    }
+
+    if (
+      newLLMCompleteMockStorage.llmEventStream.length !==
+      llmCompleteMockStorage.llmEventStream.length
+    ) {
+      throw new Error(
+        "New LLMStreamEvent does not match, please update snapshot",
+      );
+    }
+  });
+  // cleanup
+  t.after(() => {
+    llmCompleteMockStorage.llmEventEnd = [];
+    llmCompleteMockStorage.llmEventStart = [];
+    llmCompleteMockStorage.llmEventStream = [];
+  });
+}

packages/core/package.json

@@ -12,7 +12,7 @@
     "@llamaindex/cloud": "0.0.5",
     "@llamaindex/env": "workspace:*",
     "@mistralai/mistralai": "^0.1.3",
-    "@notionhq/client": "^2.2.14",
+    "@notionhq/client": "^2.2.15",
     "@pinecone-database/pinecone": "^2.2.0",
     "@qdrant/js-client-rest": "^1.8.2",
     "@types/lodash": "^4.17.0",
@@ -21,9 +21,10 @@
     "@types/pg": "^8.11.5",
     "@xenova/transformers": "^2.16.1",
     "@zilliz/milvus2-sdk-node": "^2.3.5",
+    "ajv": "^8.12.0",
     "assemblyai": "^4.3.4",
     "chromadb": "~1.7.3",
-    "cohere-ai": "^7.9.2",
+    "cohere-ai": "^7.9.3",
     "js-tiktoken": "^1.0.10",
     "lodash": "^4.17.21",
     "magic-bytes.js": "^1.10.0",
@@ -50,7 +51,7 @@
     "concurrently": "^8.2.2",
     "glob": "^10.3.12",
     "madge": "^6.1.0",
-    "typescript": "^5.4.4"
+    "typescript": "^5.4.5"
   },
   "engines": {
     "node": ">=18.0.0"

packages/core/src/agent/openai/worker.ts

@@ -1,4 +1,4 @@
-import { randomUUID } from "@llamaindex/env";
+import { pipeline, randomUUID } from "@llamaindex/env";
 import type { ChatCompletionToolChoiceOption } from "openai/resources/chat/completions";
 import { Response } from "../../Response.js";
 import { Settings } from "../../Settings.js";
@@ -14,12 +14,9 @@ import {
   type ChatResponseChunk,
   type LLMChatParamsBase,
   type OpenAIAdditionalChatOptions,
+  type OpenAIAdditionalMessageOptions,
 } from "../../llm/index.js";
-import {
-  extractText,
-  streamConverter,
-  streamReducer,
-} from "../../llm/utils.js";
+import { extractText } from "../../llm/utils.js";
 import { ChatMemoryBuffer } from "../../memory/ChatMemoryBuffer.js";
 import type { ObjectRetriever } from "../../objects/base.js";
 import type { ToolOutput } from "../../tools/types.js";
@@ -49,7 +46,7 @@ async function callFunction(
 
   // Call tool
   // Use default error message
-  const output = await callToolWithErrorHandling(tool, argumentDict, null);
+  const output = await callToolWithErrorHandling(tool, argumentDict);
 
   if (Settings.debug) {
     console.log(`Got output ${output}`);
@@ -181,40 +178,70 @@ export class OpenAIAgentWorker
       stream: true,
       ...llmChatParams,
     });
-    // read first chunk from stream to find out if we need to call tools
-    const iterator = stream[Symbol.asyncIterator]();
-    let { value } = await iterator.next();
-    let content = value.delta;
-    const hasToolCalls = value.options?.toolCalls.length > 0;
+
+    const responseChunkStream = new ReadableStream<
+      ChatResponseChunk<OpenAIAdditionalMessageOptions>
+    >({
+      async start(controller) {
+        for await (const chunk of stream) {
+          controller.enqueue(chunk);
+        }
+      },
+    });
+    const [pipStream, finalStream] = responseChunkStream.tee();
+    const { value } = await pipStream.getReader().read();
+    if (value === undefined) {
+      throw new Error("first chunk value is undefined, this should not happen");
+    }
+    // check if first chunk has tool calls, if so, this is a function call
+    // otherwise, it's a regular message
+    const hasToolCalls: boolean =
+      !!value.options?.toolCalls?.length &&
+      value.options?.toolCalls?.length > 0;
 
     if (hasToolCalls) {
-      // consume stream until we have all the tool calls and return a non-streamed response
-      for await (value of stream) {
-        content += value.delta;
-      }
       return this._processMessage(task, {
-        content,
+        content: await pipeline(finalStream, async (iterator) => {
+          let content = "";
+          for await (const value of iterator) {
+            content += value.delta;
+          }
+          return content;
+        }),
         role: "assistant",
         options: value.options,
       });
-    }
-
-    const newStream = streamConverter.bind(this)(
-      streamReducer({
-        stream,
-        initialValue: content,
-        reducer: (accumulator, part) => (accumulator += part.delta),
-        finished: (accumulator) => {
-          task.extraState.newMemory.put({
-            content: accumulator,
-            role: "assistant",
-          });
+    } else {
+      let content = "";
+      return pipeline(
+        finalStream.pipeThrough<Response>({
+          readable: new ReadableStream({
+            async start(controller) {
+              for await (const chunk of finalStream) {
+                controller.enqueue(new Response(chunk.delta));
+              }
+            },
+          }),
+          writable: new WritableStream({
+            write(chunk) {
+              content += chunk.delta;
+            },
+            close() {
+              task.extraState.newMemory.put({
+                content,
+                role: "assistant",
+              });
+            },
+          }),
+        }),
+        async (iterator: AsyncIterable<Response>) => {
+          return new StreamingAgentChatResponse(
+            iterator,
+            task.extraState.sources,
+          );
         },
-      }),
-      (r: ChatResponseChunk) => new Response(r.delta),
-    );
-
-    return new StreamingAgentChatResponse(newStream, task.extraState.sources);
+      );
+    }
   }
 
   private async _getAgentResponse(

packages/core/src/agent/react/worker.ts

@@ -194,7 +194,7 @@ export class ReActAgentWorker implements AgentWorker<ChatParams> {
 
     const tool = toolsDict[actionReasoningStep.action];
 
-    const toolOutput = await tool?.call?.(actionReasoningStep.actionInput);
+    const toolOutput = await tool.call!(actionReasoningStep.actionInput);
 
     task.extraState.sources.push(
       new ToolOutput(

packages/core/src/callbacks/CallbackManager.ts

@@ -5,7 +5,11 @@ import {
   EventCaller,
   getEventCaller,
 } from "../internal/context/EventCaller.js";
-import type { LLMEndEvent, LLMStartEvent } from "../llm/types.js";
+import type {
+  LLMEndEvent,
+  LLMStartEvent,
+  LLMStreamEvent,
+} from "../llm/types.js";
 
 export class LlamaIndexCustomEvent<T = any> extends CustomEvent<T> {
   reason: EventCaller | null;
@@ -44,6 +48,7 @@ export interface LlamaIndexEventMaps {
   stream: CustomEvent<StreamCallbackResponse>;
   "llm-start": LLMStartEvent;
   "llm-end": LLMEndEvent;
+  "llm-stream": LLMStreamEvent;
 }
 
 //#region @deprecated remove in the next major version

packages/core/src/llm/LLM.ts

@@ -366,7 +366,7 @@ export class Portkey extends BaseLLM {
 
       idx_counter++;
 
-      yield { delta: part.choices[0].delta?.content ?? "" };
+      yield { raw: part, delta: part.choices[0].delta?.content ?? "" };
     }
     return;
   }

packages/core/src/llm/anthropic.ts

@@ -212,7 +212,10 @@ export class Anthropic extends BaseLLM {
       if (typeof content !== "string") continue;
 
       idx_counter++;
-      yield { delta: content };
+      yield {
+        raw: part,
+        delta: content,
+      };
     }
     return;
   }

packages/core/src/llm/base.ts

@@ -41,6 +41,7 @@ export abstract class BaseLLM<
       });
       return streamConverter(stream, (chunk) => {
         return {
+          raw: null,
           text: chunk.delta,
         };
       });

packages/core/src/llm/mistral.ts

@@ -137,6 +137,7 @@ export class MistralAI extends BaseLLM {
       idx_counter++;
 
       yield {
+        raw: part,
         delta: part.choices[0].delta.content ?? "",
       };
     }

packages/core/src/llm/ollama.ts

@@ -14,11 +14,13 @@ import type {
 
 const messageAccessor = (data: any): ChatResponseChunk => {
   return {
+    raw: data,
     delta: data.message.content,
   };
 };
+
 const completionAccessor = (data: any): CompletionResponse => {
-  return { text: data.response };
+  return { text: data.response, raw: null };
 };
 
 // https://github.com/jmorganca/ollama

packages/core/src/llm/open_ai.ts

@@ -391,6 +391,8 @@ export class OpenAI extends BaseLLM<
     for await (const part of stream) {
       if (!part.choices.length) continue;
       const choice = part.choices[0];
+      // skip parts that don't have any content
+      if (!(choice.delta.content || choice.delta.tool_calls)) continue;
       updateToolCalls(toolCalls, choice.delta.tool_calls);
 
       const isDone: boolean = choice.finish_reason !== null;
@@ -402,6 +404,7 @@ export class OpenAI extends BaseLLM<
       });
 
       yield {
+        raw: part,
         // add tool calls to final chunk
         options: toolCalls.length > 0 ? { toolCalls: toolCalls } : {},
         delta: choice.delta.content ?? "",
@@ -444,8 +447,11 @@ function updateToolCalls(
     return toolCall;
   }
   if (toolCallDeltas) {
-    toolCallDeltas?.forEach((toolCall, i) => {
-      toolCalls[i] = augmentToolCall(toolCalls[i], toolCall);
+    toolCallDeltas?.forEach((toolCall) => {
+      toolCalls[toolCall.index] = augmentToolCall(
+        toolCalls[toolCall.index],
+        toolCall,
+      );
     });
   }
 }

packages/core/src/llm/types.ts

@@ -22,6 +22,13 @@ export type LLMEndEvent = LLMBaseEvent<
     response: ChatResponse;
   }
 >;
+export type LLMStreamEvent = LLMBaseEvent<
+  "llm-stream",
+  {
+    id: UUID;
+    chunk: ChatResponseChunk;
+  }
+>;
 
 /**
  * @internal
@@ -126,8 +133,10 @@ export interface ChatResponse<
   message: ChatMessage<AdditionalMessageOptions>;
   /**
    * Raw response from the LLM
+   *
+   * If LLM response an iterable of chunks, this will be an array of those chunks
    */
-  raw: object;
+  raw: object | null;
 }
 
 export type ChatResponseChunk<
@@ -138,17 +147,24 @@ export type ChatResponseChunk<
 > =
   AdditionalMessageOptions extends Record<string, unknown>
     ? {
+        raw: object | null;
         delta: string;
         options?: AdditionalMessageOptions;
       }
     : {
+        raw: object | null;
         delta: string;
         options: AdditionalMessageOptions;
       };
 
 export interface CompletionResponse {
   text: string;
-  raw?: Record<string, any>;
+  /**
+   * Raw response from the LLM
+   *
+   * It's possible that this is `null` if the LLM response an iterable of chunks
+   */
+  raw: object | null;
 }
 
 export type LLMMetadata = {

packages/core/src/llm/utils.ts

@@ -2,6 +2,7 @@ import { AsyncLocalStorage, randomUUID } from "@llamaindex/env";
 import { getCallbackManager } from "../internal/settings/CallbackManager.js";
 import type {
   ChatResponse,
+  ChatResponseChunk,
   LLM,
   LLMChat,
   MessageContent,
@@ -83,13 +84,14 @@ export function wrapLLMEvent(
         [Symbol.asyncIterator]: response[Symbol.asyncIterator].bind(response),
       };
       response[Symbol.asyncIterator] = async function* () {
-        const finalResponse: ChatResponse = {
-          raw: response,
+        const finalResponse = {
+          raw: [] as ChatResponseChunk[],
           message: {
             content: "",
             role: "assistant",
+            options: {},
           },
-        };
+        } satisfies ChatResponse;
         let firstOne = false;
         for await (const chunk of originalAsyncIterator) {
           if (!firstOne) {
@@ -98,6 +100,19 @@ export function wrapLLMEvent(
           } else {
             finalResponse.message.content += chunk.delta;
           }
+          if (chunk.options) {
+            finalResponse.message.options = {
+              ...finalResponse.message.options,
+              ...chunk.options,
+            };
+          }
+          getCallbackManager().dispatchEvent("llm-stream", {
+            payload: {
+              id,
+              chunk,
+            },
+          });
+          finalResponse.raw.push(chunk);
           yield chunk;
         }
         snapshot(() => {

packages/core/src/tools/QueryEngineTool.ts

@@ -1,18 +1,11 @@
+import type { JSONSchemaType } from "ajv";
 import type { BaseQueryEngine, BaseTool, ToolMetadata } from "../types.js";
 
-export type QueryEngineToolParams = {
-  queryEngine: BaseQueryEngine;
-  metadata: ToolMetadata;
-};
-
-type QueryEngineCallParams = {
-  query: string;
-};
-
 const DEFAULT_NAME = "query_engine_tool";
 const DEFAULT_DESCRIPTION =
   "Useful for running a natural language query against a knowledge base and get back a natural language response.";
-const DEFAULT_PARAMETERS = {
+
+const DEFAULT_PARAMETERS: JSONSchemaType<QueryEngineParam> = {
   type: "object",
   properties: {
     query: {
@@ -23,9 +16,18 @@ const DEFAULT_PARAMETERS = {
   required: ["query"],
 };
 
-export class QueryEngineTool implements BaseTool {
+export type QueryEngineToolParams = {
+  queryEngine: BaseQueryEngine;
+  metadata: ToolMetadata<JSONSchemaType<QueryEngineParam>>;
+};
+
+export type QueryEngineParam = {
+  query: string;
+};
+
+export class QueryEngineTool implements BaseTool<QueryEngineParam> {
   private queryEngine: BaseQueryEngine;
-  metadata: ToolMetadata;
+  metadata: ToolMetadata<JSONSchemaType<QueryEngineParam>>;
 
   constructor({ queryEngine, metadata }: QueryEngineToolParams) {
     this.queryEngine = queryEngine;
@@ -36,18 +38,8 @@ export class QueryEngineTool implements BaseTool {
     };
   }
 
-  async call(...args: QueryEngineCallParams[]): Promise<any> {
-    let queryStr: string;
-
-    if (args && args.length > 0) {
-      queryStr = String(args[0].query);
-    } else {
-      throw new Error(
-        "Cannot call query engine without specifying `input` parameter.",
-      );
-    }
-
-    const response = await this.queryEngine.query({ query: queryStr });
+  async call({ query }: QueryEngineParam) {
+    const response = await this.queryEngine.query({ query });
 
     return response.response;
   }

packages/core/src/tools/ToolFactory.ts

@@ -1,33 +0,0 @@
-import type { BaseTool } from "../types.js";
-import { WikipediaTool } from "./WikipediaTool.js";
-
-enum Tools {
-  Wikipedia = "wikipedia.WikipediaToolSpec",
-}
-
-type ToolConfig = { [key in Tools]: Record<string, any> };
-
-export class ToolFactory {
-  private static async createTool(
-    key: Tools,
-    options: Record<string, any>,
-  ): Promise<BaseTool> {
-    if (key === Tools.Wikipedia) {
-      const tool = new WikipediaTool();
-      return tool;
-    }
-
-    throw new Error(
-      `Sorry! Tool ${key} is not supported yet. Options: ${options}`,
-    );
-  }
-
-  public static async createTools(config: ToolConfig): Promise<BaseTool[]> {
-    const tools: BaseTool[] = [];
-    for (const [key, value] of Object.entries(config as ToolConfig)) {
-      const tool = await ToolFactory.createTool(key as Tools, value);
-      tools.push(tool);
-    }
-    return tools;
-  }
-}

packages/core/src/tools/WikipediaTool.ts

@@ -1,16 +1,17 @@
+import type { JSONSchemaType } from "ajv";
 import { default as wiki } from "wikipedia";
 import type { BaseTool, ToolMetadata } from "../types.js";
 
-export type WikipediaToolParams = {
-  metadata?: ToolMetadata;
-};
-
-type WikipediaCallParams = {
+type WikipediaParameter = {
   query: string;
   lang?: string;
 };
 
-const DEFAULT_META_DATA: ToolMetadata = {
+export type WikipediaToolParams = {
+  metadata?: ToolMetadata<JSONSchemaType<WikipediaParameter>>;
+};
+
+const DEFAULT_META_DATA: ToolMetadata<JSONSchemaType<WikipediaParameter>> = {
   name: "wikipedia_tool",
   description: "A tool that uses a query engine to search Wikipedia.",
   parameters: {
@@ -20,14 +21,19 @@ const DEFAULT_META_DATA: ToolMetadata = {
         type: "string",
         description: "The query to search for",
       },
+      lang: {
+        type: "string",
+        description: "The language to search in",
+        nullable: true,
+      },
     },
     required: ["query"],
   },
 };
 
-export class WikipediaTool implements BaseTool {
+export class WikipediaTool implements BaseTool<WikipediaParameter> {
   private readonly DEFAULT_LANG = "en";
-  metadata: ToolMetadata;
+  metadata: ToolMetadata<JSONSchemaType<WikipediaParameter>>;
 
   constructor(params?: WikipediaToolParams) {
     this.metadata = params?.metadata || DEFAULT_META_DATA;
@@ -46,7 +52,7 @@ export class WikipediaTool implements BaseTool {
   async call({
     query,
     lang = this.DEFAULT_LANG,
-  }: WikipediaCallParams): Promise<string> {
+  }: WikipediaParameter): Promise<string> {
     const searchResult = await wiki.default.search(query);
     if (searchResult.results.length === 0) return "No search results.";
     return await this.loadData(searchResult.results[0].title, lang);

packages/core/src/tools/functionTool.ts

@@ -1,32 +1,30 @@
+import type { JSONSchemaType } from "ajv";
 import type { BaseTool, ToolMetadata } from "../types.js";
 
-type Metadata = {
-  name: string;
-  description: string;
-  parameters: ToolMetadata["parameters"];
-};
+export class FunctionTool<T, R extends string | Promise<string>>
+  implements BaseTool<T>
+{
+  constructor(
+    private readonly _fn: (input: T) => R,
+    private readonly _metadata: ToolMetadata<JSONSchemaType<T>>,
+  ) {}
 
-export class FunctionTool<T = any> implements BaseTool {
-  private _fn: (...args: any[]) => any;
-  private _metadata: ToolMetadata;
-
-  constructor(fn: (...args: any[]) => any, metadata: Metadata) {
-    this._fn = fn;
-    this._metadata = metadata as ToolMetadata;
+  static from<T>(
+    fn: (input: T) => string | Promise<string>,
+    schema: ToolMetadata<JSONSchemaType<T>>,
+  ): FunctionTool<T, string | Promise<string>>;
+  static from<T, R extends string | Promise<string>>(
+    fn: (input: T) => R,
+    schema: ToolMetadata<JSONSchemaType<T>>,
+  ): FunctionTool<T, R> {
+    return new FunctionTool(fn, schema);
   }
 
-  static fromDefaults<T = any>(
-    fn: (...args: any[]) => any,
-    metadata?: Metadata,
-  ): FunctionTool<T> {
-    return new FunctionTool(fn, metadata!);
+  get metadata(): BaseTool<T>["metadata"] {
+    return this._metadata as BaseTool<T>["metadata"];
   }
 
-  get metadata(): ToolMetadata {
-    return this._metadata;
-  }
-
-  async call(...args: any[]): Promise<any> {
-    return this._fn(...args);
+  call(input: T) {
+    return this._fn(input);
   }
 }

packages/core/src/tools/index.ts

@@ -1,5 +1,4 @@
 export * from "./QueryEngineTool.js";
-export * from "./ToolFactory.js";
 export * from "./WikipediaTool.js";
 export * from "./functionTool.js";
 export * from "./types.js";

packages/core/src/tools/utils.ts

@@ -1,30 +1,24 @@
 import type { BaseTool } from "../types.js";
 import { ToolOutput } from "./types.js";
 
-/**
- * Call tool with error handling.
- * @param tool: tool
- * @param inputDict: input dict
- * @param errorMessage: error message
- * @param raiseError: raise error
- * @returns: tool output
- */
 export async function callToolWithErrorHandling(
   tool: BaseTool,
   inputDict: { [key: string]: any },
-  errorMessage: string | null = null,
-  raiseError: boolean = false,
 ): Promise<ToolOutput> {
+  if (!tool.call) {
+    return new ToolOutput(
+      "Error: Tool does not have a call function.",
+      tool.metadata.name,
+      { kwargs: inputDict },
+      null,
+    );
+  }
   try {
-    const value = await tool.call?.(inputDict);
+    const value = await tool.call(inputDict);
     return new ToolOutput(value, tool.metadata.name, inputDict, value);
   } catch (e) {
-    if (raiseError) {
-      throw e;
-    }
-    errorMessage = errorMessage || `Error: ${e}`;
     return new ToolOutput(
-      errorMessage,
+      `Error: ${e}`,
       tool.metadata.name,
       { kwargs: inputDict },
       e,

packages/core/src/types.ts

@@ -1,6 +1,7 @@
 /**
  * Top level types to avoid circular dependencies
  */
+import { type JSONSchemaType } from "ajv";
 import type { Response } from "./Response.js";
 
 /**
@@ -30,14 +31,46 @@ export interface BaseQueryEngine {
   query(params: QueryEngineParamsNonStreaming): Promise<Response>;
 }
 
+type Known =
+  | { [key: string]: Known }
+  | [Known, ...Known[]]
+  | Known[]
+  | number
+  | string
+  | boolean
+  | null;
+
+export type ToolMetadata<
+  Parameters extends Record<string, unknown> = Record<string, unknown>,
+> = {
+  description: string;
+  name: string;
+  /**
+   * OpenAI uses JSON Schema to describe the parameters that a tool can take.
+   * @link https://json-schema.org/understanding-json-schema
+   */
+  parameters?: Parameters;
+};
+
 /**
  * Simple Tool interface. Likely to change.
  */
-export interface BaseTool {
-  call?: (...args: any[]) => any;
-  metadata: ToolMetadata;
+export interface BaseTool<Input = any> {
+  /**
+   * This could be undefined if the implementation is not provided,
+   *  which might be the case when communicating with a llm.
+   *
+   * @return string - the output of the tool, should be string in any case for LLM input.
+   */
+  call?: (input: Input) => string | Promise<string>;
+  metadata: // if user input any, we cannot check the schema
+  Input extends Known ? ToolMetadata<JSONSchemaType<Input>> : ToolMetadata;
 }
 
+export type ToolWithCall<Input = unknown> = Omit<BaseTool<Input>, "call"> & {
+  call: NonNullable<Pick<BaseTool<Input>, "call">["call"]>;
+};
+
 /**
  * An OutputParser is used to extract structured data from the raw output of the LLM.
  */
@@ -55,19 +88,6 @@ export interface StructuredOutput<T> {
   parsedOutput: T;
 }
 
-export type ToolParameters = {
-  type: string | "object";
-  properties: Record<string, { type: string; description?: string }>;
-  required?: string[];
-};
-
-export interface ToolMetadata {
-  description: string;
-  name: string;
-  parameters?: ToolParameters;
-  argsKwargs?: Record<string, any>;
-}
-
 export type ToolMetadataOnlyDescription = Pick<ToolMetadata, "description">;
 
 export class QueryBundle {

packages/core/tests/agent/OpenAIAgent.test.ts

@@ -1,61 +0,0 @@
-import { Settings } from "llamaindex";
-import { OpenAIAgent } from "llamaindex/agent/index";
-import { OpenAI } from "llamaindex/llm/index";
-import { FunctionTool } from "llamaindex/tools/index";
-import { beforeEach, describe, expect, it } from "vitest";
-import { mockLlmToolCallGeneration } from "../utility/mockOpenAI.js";
-
-// Define a function to sum two numbers
-function sumNumbers({ a, b }: { a: number; b: number }): number {
-  return a + b;
-}
-
-const sumJSON = {
-  type: "object",
-  properties: {
-    a: {
-      type: "number",
-      description: "The first number",
-    },
-    b: {
-      type: "number",
-      description: "The second number",
-    },
-  },
-  required: ["a", "b"],
-};
-
-describe("OpenAIAgent", () => {
-  let openaiAgent: OpenAIAgent;
-
-  beforeEach(() => {
-    const languageModel = new OpenAI({
-      model: "gpt-3.5-turbo",
-    });
-
-    Settings.llm = languageModel;
-
-    mockLlmToolCallGeneration({
-      languageModel,
-    });
-
-    const sumFunctionTool = new FunctionTool(sumNumbers, {
-      name: "sumNumbers",
-      description: "Use this function to sum two numbers",
-      parameters: sumJSON,
-    });
-
-    openaiAgent = new OpenAIAgent({
-      tools: [sumFunctionTool],
-      llm: languageModel,
-    });
-  });
-
-  it("should be able to chat with agent", async () => {
-    const response = await openaiAgent.chat({
-      message: "how much is 1 + 1?",
-    });
-
-    expect(String(response)).toEqual("The sum is 2");
-  });
-});

packages/core/tests/objects/ObjectIndex.test.ts

@@ -17,7 +17,7 @@ describe("ObjectIndex", () => {
   });
 
   test("test_object_with_tools", async () => {
-    const tool1 = new FunctionTool((x: any) => x, {
+    const tool1 = new FunctionTool(({ x }: { x: string }) => x, {
       name: "test_tool",
       description: "test tool",
       parameters: {
@@ -27,10 +27,11 @@ describe("ObjectIndex", () => {
             type: "string",
           },
         },
+        required: ["x"],
       },
     });
 
-    const tool2 = new FunctionTool((x: any) => x, {
+    const tool2 = new FunctionTool(({ x }: { x: string }) => x, {
       name: "test_tool_2",
       description: "test tool 2",
       parameters: {
@@ -40,6 +41,7 @@ describe("ObjectIndex", () => {
             type: "string",
           },
         },
+        required: ["x"],
       },
     });
 
@@ -62,7 +64,7 @@ describe("ObjectIndex", () => {
   });
 
   test("add a new object", async () => {
-    const tool1 = new FunctionTool((x: any) => x, {
+    const tool1 = new FunctionTool(({ x }: { x: string }) => x, {
       name: "test_tool",
       description: "test tool",
       parameters: {
@@ -72,10 +74,11 @@ describe("ObjectIndex", () => {
             type: "string",
           },
         },
+        required: ["x"],
       },
     });
 
-    const tool2 = new FunctionTool((x: any) => x, {
+    const tool2 = new FunctionTool(({ x }: { x: string }) => x, {
       name: "test_tool_2",
       description: "test tool 2",
       parameters: {
@@ -85,6 +88,7 @@ describe("ObjectIndex", () => {
             type: "string",
           },
         },
+        required: ["x"],
       },
     });
 

packages/core/tests/tools/Tools.test.ts

@@ -1,46 +0,0 @@
-import { FunctionTool, ToolOutput } from "llamaindex/tools/index";
-import { callToolWithErrorHandling } from "llamaindex/tools/utils";
-import { describe, expect, it } from "vitest";
-
-function sumNumbers({ a, b }: { a: number; b: number }): number {
-  return a + b;
-}
-
-const sumJSON = {
-  type: "object",
-  properties: {
-    a: {
-      type: "number",
-      description: "The first number",
-    },
-    b: {
-      type: "number",
-      description: "The second number",
-    },
-  },
-  required: ["a", "b"],
-};
-
-describe("Tools", () => {
-  it("should be able to call a tool with a common JSON", async () => {
-    const tool = new FunctionTool(sumNumbers, {
-      name: "sumNumbers",
-      description: "Use this function to sum two numbers",
-      parameters: sumJSON,
-    });
-
-    const response = await callToolWithErrorHandling(tool, {
-      a: 1,
-      b: 2,
-    });
-
-    expect(response).toEqual(
-      new ToolOutput(
-        response.content,
-        tool.metadata.name,
-        { a: 1, b: 2 },
-        response.content,
-      ),
-    );
-  });
-});

packages/edge/package.json

@@ -11,7 +11,7 @@
     "@llamaindex/cloud": "0.0.5",
     "@llamaindex/env": "workspace:*",
     "@mistralai/mistralai": "^0.1.3",
-    "@notionhq/client": "^2.2.14",
+    "@notionhq/client": "^2.2.15",
     "@pinecone-database/pinecone": "^2.2.0",
     "@qdrant/js-client-rest": "^1.8.2",
     "@types/lodash": "^4.17.0",
@@ -20,9 +20,10 @@
     "@types/pg": "^8.11.5",
     "@xenova/transformers": "^2.16.1",
     "@zilliz/milvus2-sdk-node": "^2.3.5",
+    "ajv": "^8.12.0",
     "assemblyai": "^4.3.4",
     "chromadb": "~1.7.3",
-    "cohere-ai": "^7.9.2",
+    "cohere-ai": "^7.9.3",
     "js-tiktoken": "^1.0.10",
     "lodash": "^4.17.21",
     "magic-bytes.js": "^1.10.0",

packages/env/package.json

@@ -56,8 +56,9 @@
     "@aws-crypto/sha256-js": "^5.2.0",
     "@swc/cli": "^0.3.9",
     "@swc/core": "^1.4.2",
+    "concurrently": "^8.2.2",
     "pathe": "^1.1.2",
-    "concurrently": "^8.2.2"
+    "readable-stream": "^4.5.2"
   },
   "dependencies": {
     "@types/lodash": "^4.14.202",
@@ -66,6 +67,7 @@
   },
   "peerDependencies": {
     "@aws-crypto/sha256-js": "^5.2.0",
-    "pathe": "^1.1.2"
+    "pathe": "^1.1.2",
+    "readable-stream": "^4.5.2"
   }
 }

packages/env/src/index.polyfill.ts

@@ -1,8 +1,10 @@
 import { Sha256 } from "@aws-crypto/sha256-js";
 import pathe from "pathe";
 import { InMemoryFileSystem, type CompleteFileSystem } from "./type.js";
+// @ts-expect-error
+import { pipeline } from "readable-stream";
 
-export { pathe as path };
+export { pathe as path, pipeline };
 
 export interface SHA256 {
   update(data: string | Uint8Array): void;

packages/env/src/index.ts

@@ -3,6 +3,7 @@ import { createHash, randomUUID } from "node:crypto";
 import fs from "node:fs/promises";
 import { EOL } from "node:os";
 import path from "node:path";
+import { pipeline } from "node:stream/promises";
 import type { SHA256 } from "./index.polyfill.js";
 import type { CompleteFileSystem } from "./type.js";
 
@@ -36,4 +37,4 @@ export const defaultFS: CompleteFileSystem = {
 
 export type * from "./type.js";
 export { AsyncLocalStorage, CustomEvent, getEnv } from "./utils.js";
-export { EOL, ok, path, randomUUID };
+export { EOL, ok, path, pipeline, randomUUID };

pnpm-lock.yaml

@@ -7,6 +7,7 @@ settings:
 overrides:
   trim: 1.0.1
   '@babel/traverse': 7.23.2
+  protobufjs: 7.2.6
 
 importers:
 
@@ -32,13 +33,13 @@ importers:
         version: 3.2.5
       prettier-plugin-organize-imports:
         specifier: ^3.2.4
-        version: 3.2.4(prettier@3.2.5)(typescript@5.4.4)
+        version: 3.2.4(prettier@3.2.5)(typescript@5.4.5)
       turbo:
         specifier: ^1.13.2
         version: 1.13.2
       typescript:
-        specifier: ^5.4.4
-        version: 5.4.4
+        specifier: ^5.4.5
+        version: 5.4.5
 
   apps/docs:
     dependencies:
@@ -78,7 +79,7 @@ importers:
         version: 3.2.0(react-dom@18.2.0)(react@18.2.0)
       '@docusaurus/preset-classic':
         specifier: ^3.2.1
-        version: 3.2.1(@algolia/client-search@4.23.2)(@types/react@18.2.75)(eslint@8.57.0)(react-dom@18.2.0)(react@18.2.0)(search-insights@2.13.0)(typescript@5.4.4)
+        version: 3.2.1(@algolia/client-search@4.23.3)(@types/react@18.2.75)(eslint@8.57.0)(react-dom@18.2.0)(react@18.2.0)(search-insights@2.13.0)(typescript@5.4.4)
       '@docusaurus/theme-classic':
         specifier: ^3.2.1
         version: 3.2.1(@types/react@18.2.75)(eslint@8.57.0)(react-dom@18.2.0)(react@18.2.0)(typescript@5.4.4)
@@ -113,8 +114,8 @@ importers:
         specifier: ^0.1.4
         version: 0.1.4
       '@notionhq/client':
-        specifier: ^2.2.14
-        version: 2.2.14
+        specifier: ^2.2.15
+        version: 2.2.15
       '@pinecone-database/pinecone':
         specifier: ^1.1.3
         version: 1.1.3
@@ -148,10 +149,13 @@ importers:
         version: 18.19.31
       ts-node:
         specifier: ^10.9.2
-        version: 10.9.2(@types/node@18.19.31)(typescript@5.4.4)
+        version: 10.9.2(@types/node@18.19.31)(typescript@5.4.5)
+      tsx:
+        specifier: ^4.7.2
+        version: 4.7.2
       typescript:
-        specifier: ^5.4.4
-        version: 5.4.4
+        specifier: ^5.4.5
+        version: 5.4.5
 
   examples/readers:
     dependencies:
@@ -193,14 +197,14 @@ importers:
         specifier: ^0.1.3
         version: 0.1.3
       '@notionhq/client':
-        specifier: ^2.2.14
-        version: 2.2.14
+        specifier: ^2.2.15
+        version: 2.2.15
       '@pinecone-database/pinecone':
         specifier: ^2.2.0
         version: 2.2.0
       '@qdrant/js-client-rest':
         specifier: ^1.8.2
-        version: 1.8.2(typescript@5.4.4)
+        version: 1.8.2(typescript@5.4.5)
       '@types/lodash':
         specifier: ^4.17.0
         version: 4.17.0
@@ -219,15 +223,18 @@ importers:
       '@zilliz/milvus2-sdk-node':
         specifier: ^2.3.5
         version: 2.3.5
+      ajv:
+        specifier: ^8.12.0
+        version: 8.12.0
       assemblyai:
         specifier: ^4.3.4
         version: 4.3.4
       chromadb:
         specifier: ~1.7.3
-        version: 1.7.3(cohere-ai@7.9.2)(openai@4.33.0)
+        version: 1.7.3(cohere-ai@7.9.3)(openai@4.33.0)
       cohere-ai:
-        specifier: ^7.9.2
-        version: 7.9.2
+        specifier: ^7.9.3
+        version: 7.9.3
       js-tiktoken:
         specifier: ^1.0.10
         version: 1.0.10
@@ -300,10 +307,10 @@ importers:
         version: 10.3.12
       madge:
         specifier: ^6.1.0
-        version: 6.1.0(typescript@5.4.4)
+        version: 6.1.0(typescript@5.4.5)
       typescript:
-        specifier: ^5.4.4
-        version: 5.4.4
+        specifier: ^5.4.5
+        version: 5.4.5
 
   packages/core/e2e:
     devDependencies:
@@ -353,14 +360,14 @@ importers:
         specifier: ^0.1.3
         version: 0.1.3
       '@notionhq/client':
-        specifier: ^2.2.14
-        version: 2.2.14
+        specifier: ^2.2.15
+        version: 2.2.15
       '@pinecone-database/pinecone':
         specifier: ^2.2.0
         version: 2.2.0
       '@qdrant/js-client-rest':
         specifier: ^1.8.2
-        version: 1.8.2(typescript@5.4.4)
+        version: 1.8.2(typescript@5.4.5)
       '@types/lodash':
         specifier: ^4.17.0
         version: 4.17.0
@@ -379,15 +386,18 @@ importers:
       '@zilliz/milvus2-sdk-node':
         specifier: ^2.3.5
         version: 2.3.5
+      ajv:
+        specifier: ^8.12.0
+        version: 8.12.0
       assemblyai:
         specifier: ^4.3.4
         version: 4.3.4
       chromadb:
         specifier: ~1.7.3
-        version: 1.7.3(cohere-ai@7.9.2)(openai@4.33.0)
+        version: 1.7.3(cohere-ai@7.9.3)(openai@4.33.0)
       cohere-ai:
-        specifier: ^7.9.2
-        version: 7.9.2
+        specifier: ^7.9.3
+        version: 7.9.3
       js-tiktoken:
         specifier: ^1.0.10
         version: 1.0.10
@@ -501,15 +511,18 @@ importers:
       pathe:
         specifier: ^1.1.2
         version: 1.1.2
+      readable-stream:
+        specifier: ^4.5.2
+        version: 4.5.2
 
   packages/eslint-config-custom:
     dependencies:
       '@typescript-eslint/eslint-plugin':
         specifier: ^7.5.0
-        version: 7.5.0(@typescript-eslint/parser@7.6.0)(eslint@8.57.0)(typescript@5.4.4)
+        version: 7.5.0(@typescript-eslint/parser@7.6.0)(eslint@8.57.0)(typescript@5.4.5)
       eslint-config-next:
         specifier: ^13.5.6
-        version: 13.5.6(eslint@8.57.0)(typescript@5.4.4)
+        version: 13.5.6(eslint@8.57.0)(typescript@5.4.5)
       eslint-config-prettier:
         specifier: ^8.10.0
         version: 8.10.0(eslint@8.57.0)
@@ -591,47 +604,47 @@ packages:
     resolution: {integrity: sha512-1Yjs2SvM8TflER/OD3cOjhWWOZb58A2t7wpE2S9XfBYTiIl+XFhQG2bjy4Pu1I+EAlCNUzRDYDdFwFYUKvXcIA==}
     engines: {node: '>=0.10.0'}
 
-  /@algolia/autocomplete-core@1.9.3(@algolia/client-search@4.23.2)(algoliasearch@4.23.2)(search-insights@2.13.0):
+  /@algolia/autocomplete-core@1.9.3(@algolia/client-search@4.23.3)(algoliasearch@4.23.2)(search-insights@2.13.0):
     resolution: {integrity: sha512-009HdfugtGCdC4JdXUbVJClA0q0zh24yyePn+KUGk3rP7j8FEe/m5Yo/z65gn6nP/cM39PxpzqKrL7A6fP6PPw==}
     dependencies:
-      '@algolia/autocomplete-plugin-algolia-insights': 1.9.3(@algolia/client-search@4.23.2)(algoliasearch@4.23.2)(search-insights@2.13.0)
-      '@algolia/autocomplete-shared': 1.9.3(@algolia/client-search@4.23.2)(algoliasearch@4.23.2)
+      '@algolia/autocomplete-plugin-algolia-insights': 1.9.3(@algolia/client-search@4.23.3)(algoliasearch@4.23.2)(search-insights@2.13.0)
+      '@algolia/autocomplete-shared': 1.9.3(@algolia/client-search@4.23.3)(algoliasearch@4.23.2)
     transitivePeerDependencies:
       - '@algolia/client-search'
       - algoliasearch
       - search-insights
     dev: true
 
-  /@algolia/autocomplete-plugin-algolia-insights@1.9.3(@algolia/client-search@4.23.2)(algoliasearch@4.23.2)(search-insights@2.13.0):
+  /@algolia/autocomplete-plugin-algolia-insights@1.9.3(@algolia/client-search@4.23.3)(algoliasearch@4.23.2)(search-insights@2.13.0):
     resolution: {integrity: sha512-a/yTUkcO/Vyy+JffmAnTWbr4/90cLzw+CC3bRbhnULr/EM0fGNvM13oQQ14f2moLMcVDyAx/leczLlAOovhSZg==}
     peerDependencies:
       search-insights: '>= 1 < 3'
     dependencies:
-      '@algolia/autocomplete-shared': 1.9.3(@algolia/client-search@4.23.2)(algoliasearch@4.23.2)
+      '@algolia/autocomplete-shared': 1.9.3(@algolia/client-search@4.23.3)(algoliasearch@4.23.2)
       search-insights: 2.13.0
     transitivePeerDependencies:
       - '@algolia/client-search'
       - algoliasearch
     dev: true
 
-  /@algolia/autocomplete-preset-algolia@1.9.3(@algolia/client-search@4.23.2)(algoliasearch@4.23.2):
+  /@algolia/autocomplete-preset-algolia@1.9.3(@algolia/client-search@4.23.3)(algoliasearch@4.23.2):
     resolution: {integrity: sha512-d4qlt6YmrLMYy95n5TB52wtNDr6EgAIPH81dvvvW8UmuWRgxEtY0NJiPwl/h95JtG2vmRM804M0DSwMCNZlzRA==}
     peerDependencies:
       '@algolia/client-search': '>= 4.9.1 < 6'
       algoliasearch: '>= 4.9.1 < 6'
     dependencies:
-      '@algolia/autocomplete-shared': 1.9.3(@algolia/client-search@4.23.2)(algoliasearch@4.23.2)
-      '@algolia/client-search': 4.23.2
+      '@algolia/autocomplete-shared': 1.9.3(@algolia/client-search@4.23.3)(algoliasearch@4.23.2)
+      '@algolia/client-search': 4.23.3
       algoliasearch: 4.23.2
     dev: true
 
-  /@algolia/autocomplete-shared@1.9.3(@algolia/client-search@4.23.2)(algoliasearch@4.23.2):
+  /@algolia/autocomplete-shared@1.9.3(@algolia/client-search@4.23.3)(algoliasearch@4.23.2):
     resolution: {integrity: sha512-Wnm9E4Ye6Rl6sTTqjoymD+l8DjSTHsHboVRYrKgEt8Q7UHm9nYbqhN/i0fhUYA3OAEH7WA8x3jfpnmJm3rKvaQ==}
     peerDependencies:
       '@algolia/client-search': '>= 4.9.1 < 6'
       algoliasearch: '>= 4.9.1 < 6'
     dependencies:
-      '@algolia/client-search': 4.23.2
+      '@algolia/client-search': 4.23.3
       algoliasearch: 4.23.2
     dev: true
 
@@ -645,6 +658,10 @@ packages:
     resolution: {integrity: sha512-OUK/6mqr6CQWxzl/QY0/mwhlGvS6fMtvEPyn/7AHUx96NjqDA4X4+Ju7aXFQKh+m3jW9VPB0B9xvEQgyAnRPNw==}
     dev: true
 
+  /@algolia/cache-common@4.23.3:
+    resolution: {integrity: sha512-h9XcNI6lxYStaw32pHpB1TMm0RuxphF+Ik4o7tcQiodEdpKK+wKufY6QXtba7t3k8eseirEMVB83uFFF3Nu54A==}
+    dev: true
+
   /@algolia/cache-in-memory@4.23.2:
     resolution: {integrity: sha512-rfbi/SnhEa3MmlqQvgYz/9NNJ156NkU6xFxjbxBtLWnHbpj+qnlMoKd+amoiacHRITpajg6zYbLM9dnaD3Bczw==}
     dependencies:
@@ -675,6 +692,13 @@ packages:
       '@algolia/transporter': 4.23.2
     dev: true
 
+  /@algolia/client-common@4.23.3:
+    resolution: {integrity: sha512-l6EiPxdAlg8CYhroqS5ybfIczsGUIAC47slLPOMDeKSVXYG1n0qGiz4RjAHLw2aD0xzh2EXZ7aRguPfz7UKDKw==}
+    dependencies:
+      '@algolia/requester-common': 4.23.3
+      '@algolia/transporter': 4.23.3
+    dev: true
+
   /@algolia/client-personalization@4.23.2:
     resolution: {integrity: sha512-vwPsgnCGhUcHhhQG5IM27z8q7dWrN9itjdvgA6uKf2e9r7vB+WXt4OocK0CeoYQt3OGEAExryzsB8DWqdMK5wg==}
     dependencies:
@@ -691,6 +715,14 @@ packages:
       '@algolia/transporter': 4.23.2
     dev: true
 
+  /@algolia/client-search@4.23.3:
+    resolution: {integrity: sha512-P4VAKFHqU0wx9O+q29Q8YVuaowaZ5EM77rxfmGnkHUJggh28useXQdopokgwMeYw2XUht49WX5RcTQ40rZIabw==}
+    dependencies:
+      '@algolia/client-common': 4.23.3
+      '@algolia/requester-common': 4.23.3
+      '@algolia/transporter': 4.23.3
+    dev: true
+
   /@algolia/events@4.0.1:
     resolution: {integrity: sha512-FQzvOCgoFXAbf5Y6mYozw2aj5KCJoA3m4heImceldzPSMbdyS4atVjJzXKMsfX3wnZTFYwkkt8/z8UesLHlSBQ==}
     dev: true
@@ -699,6 +731,10 @@ packages:
     resolution: {integrity: sha512-jGM49Q7626cXZ7qRAWXn0jDlzvoA1FvN4rKTi1g0hxKsTTSReyYk0i1ADWjChDPl3Q+nSDhJuosM2bBUAay7xw==}
     dev: true
 
+  /@algolia/logger-common@4.23.3:
+    resolution: {integrity: sha512-y9kBtmJwiZ9ZZ+1Ek66P0M68mHQzKRxkW5kAAXYN/rdzgDN0d2COsViEFufxJ0pb45K4FRcfC7+33YB4BLrZ+g==}
+    dev: true
+
   /@algolia/logger-console@4.23.2:
     resolution: {integrity: sha512-oo+lnxxEmlhTBTFZ3fGz1O8PJ+G+8FiAoMY2Qo3Q4w23xocQev6KqDTA1JQAGPDxAewNA2VBwWOsVXeXFjrI/Q==}
     dependencies:
@@ -731,6 +767,10 @@ packages:
     resolution: {integrity: sha512-3EfpBS0Hri0lGDB5H/BocLt7Vkop0bTTLVUBB844HH6tVycwShmsV6bDR7yXbQvFP1uNpgePRD3cdBCjeHmk6Q==}
     dev: true
 
+  /@algolia/requester-common@4.23.3:
+    resolution: {integrity: sha512-xloIdr/bedtYEGcXCiF2muajyvRhwop4cMZo+K2qzNht0CMzlRkm8YsDdj5IaBhshqfgmBb3rTg4sL4/PpvLYw==}
+    dev: true
+
   /@algolia/requester-node-http@4.23.2:
     resolution: {integrity: sha512-SVzgkZM/malo+2SB0NWDXpnT7nO5IZwuDTaaH6SjLeOHcya1o56LSWXk+3F3rNLz2GVH+I/rpYKiqmHhSOjerw==}
     dependencies:
@@ -745,6 +785,14 @@ packages:
       '@algolia/requester-common': 4.23.2
     dev: true
 
+  /@algolia/transporter@4.23.3:
+    resolution: {integrity: sha512-Wjl5gttqnf/gQKJA+dafnD0Y6Yw97yvfY8R9h0dQltX1GXTgNs1zWgvtWW0tHl1EgMdhAyw189uWiZMnL3QebQ==}
+    dependencies:
+      '@algolia/cache-common': 4.23.3
+      '@algolia/logger-common': 4.23.3
+      '@algolia/requester-common': 4.23.3
+    dev: true
+
   /@ampproject/remapping@2.3.0:
     resolution: {integrity: sha512-30iZtAPgz+LTIYoeivqYo853f02jBYSd5uGnGpkFV0M3xOt9aN73erkgYAmZU43x4VfqcnLxW9Kpg3R5LC4YYw==}
     engines: {node: '>=6.0.0'}
@@ -2252,7 +2300,7 @@ packages:
     resolution: {integrity: sha512-+sbxb71sWre+PwDK7X2T8+bhS6clcVMLwBPznX45Qu6opJcgRjAp7gYSDzVFp187J+feSj5dNBN1mJoi6ckkUQ==}
     dev: true
 
-  /@docsearch/react@3.6.0(@algolia/client-search@4.23.2)(@types/react@18.2.75)(react-dom@18.2.0)(react@18.2.0)(search-insights@2.13.0):
+  /@docsearch/react@3.6.0(@algolia/client-search@4.23.3)(@types/react@18.2.75)(react-dom@18.2.0)(react@18.2.0)(search-insights@2.13.0):
     resolution: {integrity: sha512-HUFut4ztcVNmqy9gp/wxNbC7pTOHhgVVkHVGCACTuLhUKUhKAF9KYHJtMiLUJxEqiFLQiuri1fWF8zqwM/cu1w==}
     peerDependencies:
       '@types/react': '>= 16.8.0 < 19.0.0'
@@ -2269,8 +2317,8 @@ packages:
       search-insights:
         optional: true
     dependencies:
-      '@algolia/autocomplete-core': 1.9.3(@algolia/client-search@4.23.2)(algoliasearch@4.23.2)(search-insights@2.13.0)
-      '@algolia/autocomplete-preset-algolia': 1.9.3(@algolia/client-search@4.23.2)(algoliasearch@4.23.2)
+      '@algolia/autocomplete-core': 1.9.3(@algolia/client-search@4.23.3)(algoliasearch@4.23.2)(search-insights@2.13.0)
+      '@algolia/autocomplete-preset-algolia': 1.9.3(@algolia/client-search@4.23.3)(algoliasearch@4.23.2)
       '@docsearch/css': 3.6.0
       '@types/react': 18.2.75
       algoliasearch: 4.23.2
@@ -2778,7 +2826,7 @@ packages:
       - webpack-cli
     dev: true
 
-  /@docusaurus/preset-classic@3.2.1(@algolia/client-search@4.23.2)(@types/react@18.2.75)(eslint@8.57.0)(react-dom@18.2.0)(react@18.2.0)(search-insights@2.13.0)(typescript@5.4.4):
+  /@docusaurus/preset-classic@3.2.1(@algolia/client-search@4.23.3)(@types/react@18.2.75)(eslint@8.57.0)(react-dom@18.2.0)(react@18.2.0)(search-insights@2.13.0)(typescript@5.4.4):
     resolution: {integrity: sha512-E3OHSmttpEBcSMhfPBq3EJMBxZBM01W1rnaCUTXy9EHvkmB5AwgTfW1PwGAybPAX579ntE03R+2zmXdizWfKnQ==}
     engines: {node: '>=18.0'}
     peerDependencies:
@@ -2796,7 +2844,7 @@ packages:
       '@docusaurus/plugin-sitemap': 3.2.1(eslint@8.57.0)(react-dom@18.2.0)(react@18.2.0)(typescript@5.4.4)
       '@docusaurus/theme-classic': 3.2.1(@types/react@18.2.75)(eslint@8.57.0)(react-dom@18.2.0)(react@18.2.0)(typescript@5.4.4)
       '@docusaurus/theme-common': 3.2.1(@docusaurus/types@3.2.1)(eslint@8.57.0)(react-dom@18.2.0)(react@18.2.0)(typescript@5.4.4)
-      '@docusaurus/theme-search-algolia': 3.2.1(@algolia/client-search@4.23.2)(@docusaurus/types@3.2.1)(@types/react@18.2.75)(eslint@8.57.0)(react-dom@18.2.0)(react@18.2.0)(search-insights@2.13.0)(typescript@5.4.4)
+      '@docusaurus/theme-search-algolia': 3.2.1(@algolia/client-search@4.23.3)(@docusaurus/types@3.2.1)(@types/react@18.2.75)(eslint@8.57.0)(react-dom@18.2.0)(react@18.2.0)(search-insights@2.13.0)(typescript@5.4.4)
       '@docusaurus/types': 3.2.1(react-dom@18.2.0)(react@18.2.0)
       react: 18.2.0
       react-dom: 18.2.0(react@18.2.0)
@@ -2942,14 +2990,14 @@ packages:
       - webpack-cli
     dev: true
 
-  /@docusaurus/theme-search-algolia@3.2.1(@algolia/client-search@4.23.2)(@docusaurus/types@3.2.1)(@types/react@18.2.75)(eslint@8.57.0)(react-dom@18.2.0)(react@18.2.0)(search-insights@2.13.0)(typescript@5.4.4):
+  /@docusaurus/theme-search-algolia@3.2.1(@algolia/client-search@4.23.3)(@docusaurus/types@3.2.1)(@types/react@18.2.75)(eslint@8.57.0)(react-dom@18.2.0)(react@18.2.0)(search-insights@2.13.0)(typescript@5.4.4):
     resolution: {integrity: sha512-bzhCrpyXBXzeydNUH83II2akvFEGfhsNTPPWsk5N7e+odgQCQwoHhcF+2qILbQXjaoZ6B3c48hrvkyCpeyqGHw==}
     engines: {node: '>=18.0'}
     peerDependencies:
       react: ^18.0.0
       react-dom: ^18.0.0
     dependencies:
-      '@docsearch/react': 3.6.0(@algolia/client-search@4.23.2)(@types/react@18.2.75)(react-dom@18.2.0)(react@18.2.0)(search-insights@2.13.0)
+      '@docsearch/react': 3.6.0(@algolia/client-search@4.23.3)(@types/react@18.2.75)(react-dom@18.2.0)(react@18.2.0)(search-insights@2.13.0)
       '@docusaurus/core': 3.2.1(@docusaurus/types@3.2.1)(eslint@8.57.0)(react-dom@18.2.0)(react@18.2.0)(typescript@5.4.4)
       '@docusaurus/logger': 3.2.1
       '@docusaurus/plugin-content-docs': 3.2.1(eslint@8.57.0)(react-dom@18.2.0)(react@18.2.0)(typescript@5.4.4)
@@ -3398,7 +3446,7 @@ packages:
       '@types/long': 4.0.2
       lodash.camelcase: 4.3.0
       long: 4.0.0
-      protobufjs: 7.2.4
+      protobufjs: 7.2.6
       yargs: 17.7.2
     dev: false
 
@@ -3806,8 +3854,8 @@ packages:
       '@nodelib/fs.scandir': 2.1.5
       fastq: 1.17.1
 
-  /@notionhq/client@2.2.14:
-    resolution: {integrity: sha512-oqUefZtCiJPCX+74A1Os9OVTef3fSnVWe2eVQtU1HJSD+nsfxfhwvDKnzJTh2Tw1ZHKLxpieHB/nzGdY+Uo12A==}
+  /@notionhq/client@2.2.15:
+    resolution: {integrity: sha512-XhdSY/4B1D34tSco/GION+23GMjaS9S2zszcqYkMHo8RcWInymF6L1x+Gk7EmHdrSxNFva2WM8orhC4BwQCwgw==}
     engines: {node: '>=12'}
     dependencies:
       '@types/node-fetch': 2.6.11
@@ -3907,7 +3955,7 @@ packages:
     resolution: {integrity: sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw==}
     dev: false
 
-  /@qdrant/js-client-rest@1.8.2(typescript@5.4.4):
+  /@qdrant/js-client-rest@1.8.2(typescript@5.4.5):
     resolution: {integrity: sha512-BCGC4YRcqjRxXVo500CxjhluPpGO0XpOwojauT8675Duv24YTlkhvDRmc1c9k/df2+yH/typtkecK3VOi3CD7A==}
     engines: {node: '>=18.0.0', pnpm: '>=8'}
     peerDependencies:
@@ -3915,7 +3963,7 @@ packages:
     dependencies:
       '@qdrant/openapi-typescript-fetch': 1.2.6
       '@sevinf/maybe': 0.5.0
-      typescript: 5.4.4
+      typescript: 5.4.5
       undici: 5.28.4
     dev: false
 
@@ -4947,7 +4995,7 @@ packages:
     dependencies:
       '@types/yargs-parser': 21.0.3
 
-  /@typescript-eslint/eslint-plugin@7.5.0(@typescript-eslint/parser@7.6.0)(eslint@8.57.0)(typescript@5.4.4):
+  /@typescript-eslint/eslint-plugin@7.5.0(@typescript-eslint/parser@7.6.0)(eslint@8.57.0)(typescript@5.4.5):
     resolution: {integrity: sha512-HpqNTH8Du34nLxbKgVMGljZMG0rJd2O9ecvr2QLYp+7512ty1j42KnsFwspPXg1Vh8an9YImf6CokUBltisZFQ==}
     engines: {node: ^18.18.0 || >=20.0.0}
     peerDependencies:
@@ -4959,10 +5007,10 @@ packages:
         optional: true
     dependencies:
       '@eslint-community/regexpp': 4.10.0
-      '@typescript-eslint/parser': 7.6.0(eslint@8.57.0)(typescript@5.4.4)
+      '@typescript-eslint/parser': 7.6.0(eslint@8.57.0)(typescript@5.4.5)
       '@typescript-eslint/scope-manager': 7.5.0
-      '@typescript-eslint/type-utils': 7.5.0(eslint@8.57.0)(typescript@5.4.4)
-      '@typescript-eslint/utils': 7.5.0(eslint@8.57.0)(typescript@5.4.4)
+      '@typescript-eslint/type-utils': 7.5.0(eslint@8.57.0)(typescript@5.4.5)
+      '@typescript-eslint/utils': 7.5.0(eslint@8.57.0)(typescript@5.4.5)
       '@typescript-eslint/visitor-keys': 7.5.0
       debug: 4.3.4
       eslint: 8.57.0
@@ -4970,13 +5018,13 @@ packages:
       ignore: 5.3.1
       natural-compare: 1.4.0
       semver: 7.6.0
-      ts-api-utils: 1.0.3(typescript@5.4.4)
-      typescript: 5.4.4
+      ts-api-utils: 1.0.3(typescript@5.4.5)
+      typescript: 5.4.5
     transitivePeerDependencies:
       - supports-color
     dev: false
 
-  /@typescript-eslint/parser@6.19.1(eslint@8.57.0)(typescript@5.4.4):
+  /@typescript-eslint/parser@6.19.1(eslint@8.57.0)(typescript@5.4.5):
     resolution: {integrity: sha512-WEfX22ziAh6pRE9jnbkkLGp/4RhTpffr2ZK5bJ18M8mIfA8A+k97U9ZyaXCEJRlmMHh7R9MJZWXp/r73DzINVQ==}
     engines: {node: ^16.0.0 || >=18.0.0}
     peerDependencies:
@@ -4988,16 +5036,16 @@ packages:
     dependencies:
       '@typescript-eslint/scope-manager': 6.19.1
       '@typescript-eslint/types': 6.19.1
-      '@typescript-eslint/typescript-estree': 6.19.1(typescript@5.4.4)
+      '@typescript-eslint/typescript-estree': 6.19.1(typescript@5.4.5)
       '@typescript-eslint/visitor-keys': 6.19.1
       debug: 4.3.4
       eslint: 8.57.0
-      typescript: 5.4.4
+      typescript: 5.4.5
     transitivePeerDependencies:
       - supports-color
     dev: false
 
-  /@typescript-eslint/parser@7.6.0(eslint@8.57.0)(typescript@5.4.4):
+  /@typescript-eslint/parser@7.6.0(eslint@8.57.0)(typescript@5.4.5):
     resolution: {integrity: sha512-usPMPHcwX3ZoPWnBnhhorc14NJw9J4HpSXQX4urF2TPKG0au0XhJoZyX62fmvdHONUkmyUe74Hzm1//XA+BoYg==}
     engines: {node: ^18.18.0 || >=20.0.0}
     peerDependencies:
@@ -5009,11 +5057,11 @@ packages:
     dependencies:
       '@typescript-eslint/scope-manager': 7.6.0
       '@typescript-eslint/types': 7.6.0
-      '@typescript-eslint/typescript-estree': 7.6.0(typescript@5.4.4)
+      '@typescript-eslint/typescript-estree': 7.6.0(typescript@5.4.5)
       '@typescript-eslint/visitor-keys': 7.6.0
       debug: 4.3.4
       eslint: 8.57.0
-      typescript: 5.4.4
+      typescript: 5.4.5
     transitivePeerDependencies:
       - supports-color
     dev: false
@@ -5042,7 +5090,7 @@ packages:
       '@typescript-eslint/visitor-keys': 7.6.0
     dev: false
 
-  /@typescript-eslint/type-utils@7.5.0(eslint@8.57.0)(typescript@5.4.4):
+  /@typescript-eslint/type-utils@7.5.0(eslint@8.57.0)(typescript@5.4.5):
     resolution: {integrity: sha512-A021Rj33+G8mx2Dqh0nMO9GyjjIBK3MqgVgZ2qlKf6CJy51wY/lkkFqq3TqqnH34XyAHUkq27IjlUkWlQRpLHw==}
     engines: {node: ^18.18.0 || >=20.0.0}
     peerDependencies:
@@ -5052,12 +5100,12 @@ packages:
       typescript:
         optional: true
     dependencies:
-      '@typescript-eslint/typescript-estree': 7.5.0(typescript@5.4.4)
-      '@typescript-eslint/utils': 7.5.0(eslint@8.57.0)(typescript@5.4.4)
+      '@typescript-eslint/typescript-estree': 7.5.0(typescript@5.4.5)
+      '@typescript-eslint/utils': 7.5.0(eslint@8.57.0)(typescript@5.4.5)
       debug: 4.3.4
       eslint: 8.57.0
-      ts-api-utils: 1.0.3(typescript@5.4.4)
-      typescript: 5.4.4
+      ts-api-utils: 1.0.3(typescript@5.4.5)
+      typescript: 5.4.5
     transitivePeerDependencies:
       - supports-color
     dev: false
@@ -5129,7 +5177,7 @@ packages:
       - supports-color
     dev: true
 
-  /@typescript-eslint/typescript-estree@6.19.1(typescript@5.4.4):
+  /@typescript-eslint/typescript-estree@6.19.1(typescript@5.4.5):
     resolution: {integrity: sha512-aFdAxuhzBFRWhy+H20nYu19+Km+gFfwNO4TEqyszkMcgBDYQjmPJ61erHxuT2ESJXhlhrO7I5EFIlZ+qGR8oVA==}
     engines: {node: ^16.0.0 || >=18.0.0}
     peerDependencies:
@@ -5145,13 +5193,13 @@ packages:
       is-glob: 4.0.3
       minimatch: 9.0.3
       semver: 7.6.0
-      ts-api-utils: 1.0.3(typescript@5.4.4)
-      typescript: 5.4.4
+      ts-api-utils: 1.0.3(typescript@5.4.5)
+      typescript: 5.4.5
     transitivePeerDependencies:
       - supports-color
     dev: false
 
-  /@typescript-eslint/typescript-estree@7.5.0(typescript@5.4.4):
+  /@typescript-eslint/typescript-estree@7.5.0(typescript@5.4.5):
     resolution: {integrity: sha512-YklQQfe0Rv2PZEueLTUffiQGKQneiIEKKnfIqPIOxgM9lKSZFCjT5Ad4VqRKj/U4+kQE3fa8YQpskViL7WjdPQ==}
     engines: {node: ^18.18.0 || >=20.0.0}
     peerDependencies:
@@ -5167,13 +5215,13 @@ packages:
       is-glob: 4.0.3
       minimatch: 9.0.3
       semver: 7.6.0
-      ts-api-utils: 1.0.3(typescript@5.4.4)
-      typescript: 5.4.4
+      ts-api-utils: 1.0.3(typescript@5.4.5)
+      typescript: 5.4.5
     transitivePeerDependencies:
       - supports-color
     dev: false
 
-  /@typescript-eslint/typescript-estree@7.6.0(typescript@5.4.4):
+  /@typescript-eslint/typescript-estree@7.6.0(typescript@5.4.5):
     resolution: {integrity: sha512-+7Y/GP9VuYibecrCQWSKgl3GvUM5cILRttpWtnAu8GNL9j11e4tbuGZmZjJ8ejnKYyBRb2ddGQ3rEFCq3QjMJw==}
     engines: {node: ^18.18.0 || >=20.0.0}
     peerDependencies:
@@ -5189,13 +5237,13 @@ packages:
       is-glob: 4.0.3
       minimatch: 9.0.4
       semver: 7.6.0
-      ts-api-utils: 1.3.0(typescript@5.4.4)
-      typescript: 5.4.4
+      ts-api-utils: 1.3.0(typescript@5.4.5)
+      typescript: 5.4.5
     transitivePeerDependencies:
       - supports-color
     dev: false
 
-  /@typescript-eslint/utils@7.5.0(eslint@8.57.0)(typescript@5.4.4):
+  /@typescript-eslint/utils@7.5.0(eslint@8.57.0)(typescript@5.4.5):
     resolution: {integrity: sha512-3vZl9u0R+/FLQcpy2EHyRGNqAS/ofJ3Ji8aebilfJe+fobK8+LbIFmrHciLVDxjDoONmufDcnVSF38KwMEOjzw==}
     engines: {node: ^18.18.0 || >=20.0.0}
     peerDependencies:
@@ -5206,7 +5254,7 @@ packages:
       '@types/semver': 7.5.6
       '@typescript-eslint/scope-manager': 7.5.0
       '@typescript-eslint/types': 7.5.0
-      '@typescript-eslint/typescript-estree': 7.5.0(typescript@5.4.4)
+      '@typescript-eslint/typescript-estree': 7.5.0(typescript@5.4.5)
       eslint: 8.57.0
       semver: 7.6.0
     transitivePeerDependencies:
@@ -5415,7 +5463,7 @@ packages:
       '@grpc/proto-loader': 0.7.7
       dayjs: 1.11.10
       lru-cache: 9.1.2
-      protobufjs: 7.2.4
+      protobufjs: 7.2.6
       winston: 3.13.0
     dev: false
 
@@ -5424,7 +5472,6 @@ packages:
     engines: {node: '>=6.5'}
     dependencies:
       event-target-shim: 5.0.1
-    dev: false
 
   /accepts@1.3.8:
     resolution: {integrity: sha512-PYAthTa2m2VKxuvSD3DPC/Gy+U+sOA1LAuT8mkmRuvw+NACSaeXEQ+NHcVF7rONl6qcaxV3Uuemwawk+7+SJLw==}
@@ -6104,8 +6151,6 @@ packages:
     dependencies:
       base64-js: 1.5.1
       ieee754: 1.2.1
-    dev: false
-    optional: true
 
   /busboy@1.6.0:
     resolution: {integrity: sha512-8SFQbg/0hQ9xy3UNTB0YEnsNBbWfhf7RtnzpL7TkBiTBRfrQ9Fxcnz7VJsleJpyp6rVLvXiuORqjlHi5q+PYuA==}
@@ -6340,7 +6385,7 @@ packages:
     resolution: {integrity: sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==}
     dev: false
 
-  /chromadb@1.7.3(cohere-ai@7.9.2)(openai@4.33.0):
+  /chromadb@1.7.3(cohere-ai@7.9.3)(openai@4.33.0):
     resolution: {integrity: sha512-3GgvQjpqgk5C89x5EuTDaXKbfrdqYDJ5UVyLQ3ZmwxnpetNc+HhRDGjkvXa5KSvpQ3lmKoyDoqnN4tZepfFkbw==}
     engines: {node: '>=14.17.0'}
     peerDependencies:
@@ -6356,7 +6401,7 @@ packages:
         optional: true
     dependencies:
       cliui: 8.0.1
-      cohere-ai: 7.9.2
+      cohere-ai: 7.9.3
       isomorphic-fetch: 3.0.0
       openai: 4.33.0
     transitivePeerDependencies:
@@ -6490,8 +6535,8 @@ packages:
       rfdc: 1.3.1
     dev: false
 
-  /cohere-ai@7.9.2:
-    resolution: {integrity: sha512-E9lCKUdEJSL0JAINwxqT+Mb8w8gB9i/G6CXHGhPx9DSFG5cgr+HyPJHAxy/D/M56XGG2pCuBYw6MGuQuzZs8qA==}
+  /cohere-ai@7.9.3:
+    resolution: {integrity: sha512-DcZ6B9Fa1yaFFkZ3+HO/aE4R97m08cZs/ww7YpfEcTatAljznmnXtaFKRMP57olLMGKmYA/oHpkUqh/Vt4kvFA==}
     dependencies:
       form-data: 4.0.0
       js-base64: 3.7.2
@@ -7840,7 +7885,7 @@ packages:
       source-map: 0.6.1
     dev: true
 
-  /eslint-config-next@13.5.6(eslint@8.57.0)(typescript@5.4.4):
+  /eslint-config-next@13.5.6(eslint@8.57.0)(typescript@5.4.5):
     resolution: {integrity: sha512-o8pQsUHTo9aHqJ2YiZDym5gQAMRf7O2HndHo/JZeY7TDD+W4hk6Ma8Vw54RHiBeb7OWWO5dPirQB+Is/aVQ7Kg==}
     peerDependencies:
       eslint: ^7.23.0 || ^8.0.0
@@ -7851,7 +7896,7 @@ packages:
     dependencies:
       '@next/eslint-plugin-next': 13.5.6
       '@rushstack/eslint-patch': 1.7.2
-      '@typescript-eslint/parser': 6.19.1(eslint@8.57.0)(typescript@5.4.4)
+      '@typescript-eslint/parser': 6.19.1(eslint@8.57.0)(typescript@5.4.5)
       eslint: 8.57.0
       eslint-import-resolver-node: 0.3.9
       eslint-import-resolver-typescript: 3.6.1(@typescript-eslint/parser@6.19.1)(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.29.1)(eslint@8.57.0)
@@ -7859,7 +7904,7 @@ packages:
       eslint-plugin-jsx-a11y: 6.8.0(eslint@8.57.0)
       eslint-plugin-react: 7.33.2(eslint@8.57.0)
       eslint-plugin-react-hooks: 4.6.0(eslint@8.57.0)
-      typescript: 5.4.4
+      typescript: 5.4.5
     transitivePeerDependencies:
       - eslint-import-resolver-webpack
       - supports-color
@@ -7937,7 +7982,7 @@ packages:
       eslint-import-resolver-webpack:
         optional: true
     dependencies:
-      '@typescript-eslint/parser': 6.19.1(eslint@8.57.0)(typescript@5.4.4)
+      '@typescript-eslint/parser': 6.19.1(eslint@8.57.0)(typescript@5.4.5)
       debug: 3.2.7
       eslint: 8.57.0
       eslint-import-resolver-node: 0.3.9
@@ -7967,7 +8012,7 @@ packages:
       eslint-import-resolver-webpack:
         optional: true
     dependencies:
-      '@typescript-eslint/parser': 7.6.0(eslint@8.57.0)(typescript@5.4.4)
+      '@typescript-eslint/parser': 7.6.0(eslint@8.57.0)(typescript@5.4.5)
       debug: 3.2.7
       eslint: 8.57.0
       eslint-import-resolver-node: 0.3.9
@@ -7985,7 +8030,7 @@ packages:
       '@typescript-eslint/parser':
         optional: true
     dependencies:
-      '@typescript-eslint/parser': 7.6.0(eslint@8.57.0)(typescript@5.4.4)
+      '@typescript-eslint/parser': 7.6.0(eslint@8.57.0)(typescript@5.4.5)
       array-includes: 3.1.7
       array.prototype.findlastindex: 1.2.3
       array.prototype.flat: 1.3.2
@@ -8271,7 +8316,6 @@ packages:
   /event-target-shim@5.0.1:
     resolution: {integrity: sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==}
     engines: {node: '>=6'}
-    dev: false
 
   /eventemitter3@4.0.7:
     resolution: {integrity: sha512-8guHBZCwKnFhYdHr2ysuRWErTwhoN2X8XELRlrRwpmfeY2jjuUN4taQMsULKUVo1K4DvZl+0pgfyoysHxvmvEw==}
@@ -8914,6 +8958,13 @@ packages:
     resolution: {integrity: sha512-wuMsz4leaj5hbGgg4IvDU0bqJagpftG5l5cXIAvo8uZrqn0NJqwtfupTN00VnkQJPcIRrxYrm1Ue24btpCha2A==}
     dependencies:
       resolve-pkg-maps: 1.0.0
+    dev: false
+
+  /get-tsconfig@4.7.3:
+    resolution: {integrity: sha512-ZvkrzoUA0PQZM6fy6+/Hce561s+faD1rsNwhnO5FelNjyy7EMGJ3Rz1AQ8GYDWjhRs/7dBLOEJvhK8MiEJOAFg==}
+    dependencies:
+      resolve-pkg-maps: 1.0.0
+    dev: true
 
   /github-from-package@0.0.0:
     resolution: {integrity: sha512-SyHy3T1v2NUXn29OsWdxmK6RwHD+vkj3v8en8AOBZ1wBQ/hCAQ5bAQTD02kW4W9tUp/3Qh6J8r9EvntiyCmOOw==}
@@ -10466,7 +10517,7 @@ packages:
     resolution: {integrity: sha512-zTU3DaZaF3Rt9rhN3uBMGQD3dD2/vFQqnvZCDv4dl5iOzq2IZQqTxu90r4E5J+nP70J3ilqVCrbho2eWaeW8Ow==}
     dev: true
 
-  /madge@6.1.0(typescript@5.4.4):
+  /madge@6.1.0(typescript@5.4.5):
     resolution: {integrity: sha512-irWhT5RpFOc6lkzGHKLihonCVgM0YtfNUh4IrFeW3EqHpnt/JHUG3z26j8PeJEktCGB4tmGOOOJi1Rl/ACWucQ==}
     engines: {node: '>=14'}
     hasBin: true
@@ -10497,7 +10548,7 @@ packages:
       rc: 1.2.8
       stream-to-array: 2.3.0
       ts-graphviz: 1.8.2
-      typescript: 5.4.4
+      typescript: 5.4.5
       walkdir: 0.4.1
     transitivePeerDependencies:
       - supports-color
@@ -11595,7 +11646,7 @@ packages:
   /notion-md-crawler@0.0.2:
     resolution: {integrity: sha512-lE3/DFMrg7GSbl1sBfDuLVLyxw+yjdarPVm1JGfQ6eONEbNGgO+BdZxpwwZQ1uYeEJurAXMXb/AXT8GKYjKAyg==}
     dependencies:
-      '@notionhq/client': 2.2.14
+      '@notionhq/client': 2.2.15
       md-utils-ts: 2.0.0
     transitivePeerDependencies:
       - encoding
@@ -11782,7 +11833,7 @@ packages:
   /onnx-proto@4.0.4:
     resolution: {integrity: sha512-aldMOB3HRoo6q/phyB6QRQxSt895HNNw82BNyZ2CMh4bjeKv7g/c+VpAFtJuEMVfYLMbRx61hbuqnKceLeDcDA==}
     dependencies:
-      protobufjs: 6.11.4
+      protobufjs: 7.2.6
     dev: false
 
   /onnxruntime-common@1.14.0:
@@ -12847,7 +12898,7 @@ packages:
     resolution: {integrity: sha512-vkcDPrRZo1QZLbn5RLGPpg/WmIQ65qoWWhcGKf/b5eplkkarX0m9z8ppCat4mlOqUsWpyNuYgO3VRyrYHSzX5g==}
     engines: {node: '>= 0.8.0'}
 
-  /prettier-plugin-organize-imports@3.2.4(prettier@3.2.5)(typescript@5.4.4):
+  /prettier-plugin-organize-imports@3.2.4(prettier@3.2.5)(typescript@5.4.5):
     resolution: {integrity: sha512-6m8WBhIp0dfwu0SkgfOxJqh+HpdyfqSSLfKKRZSFbDuEQXDDndb8fTpRWkUrX/uBenkex3MgnVk0J3b3Y5byog==}
     peerDependencies:
       '@volar/vue-language-plugin-pug': ^1.0.4
@@ -12861,7 +12912,7 @@ packages:
         optional: true
     dependencies:
       prettier: 3.2.5
-      typescript: 5.4.4
+      typescript: 5.4.5
     dev: true
 
   /prettier@2.8.8:
@@ -12923,8 +12974,6 @@ packages:
     resolution: {integrity: sha512-cdGef/drWFoydD1JsMzuFf8100nZl+GT+yacc2bEced5f9Rjk4z+WtFUTBu9PhOi9j/jfmBPu0mMEY4wIdAF8A==}
     engines: {node: '>= 0.6.0'}
     requiresBuild: true
-    dev: false
-    optional: true
 
   /prompts@2.4.2:
     resolution: {integrity: sha512-NxNv/kLguCA7p3jE8oL2aEBsrJWgAakBpgmgK6lpPWV+WuOmY6r2/zbAVnP+T8bQlA0nzHXSJSJW0Hq7ylaD2Q==}
@@ -12946,45 +12995,6 @@ packages:
   /proto-list@1.2.4:
     resolution: {integrity: sha512-vtK/94akxsTMhe0/cbfpR+syPuszcuwhqVjJq26CuNDgFGj682oRBXOP5MJpv2r7JtE8MsiepGIqvvOTBwn2vA==}
 
-  /protobufjs@6.11.4:
-    resolution: {integrity: sha512-5kQWPaJHi1WoCpjTGszzQ32PG2F4+wRY6BmAT4Vfw56Q2FZ4YZzK20xUYQH4YkfehY1e6QSICrJquM6xXZNcrw==}
-    hasBin: true
-    requiresBuild: true
-    dependencies:
-      '@protobufjs/aspromise': 1.1.2
-      '@protobufjs/base64': 1.1.2
-      '@protobufjs/codegen': 2.0.4
-      '@protobufjs/eventemitter': 1.1.0
-      '@protobufjs/fetch': 1.1.0
-      '@protobufjs/float': 1.0.2
-      '@protobufjs/inquire': 1.1.0
-      '@protobufjs/path': 1.1.2
-      '@protobufjs/pool': 1.1.0
-      '@protobufjs/utf8': 1.1.0
-      '@types/long': 4.0.2
-      '@types/node': 18.19.31
-      long: 4.0.0
-    dev: false
-
-  /protobufjs@7.2.4:
-    resolution: {integrity: sha512-AT+RJgD2sH8phPmCf7OUZR8xGdcJRga4+1cOaXJ64hvcSkVhNcRHOwIxUatPH15+nj59WAGTDv3LSGZPEQbJaQ==}
-    engines: {node: '>=12.0.0'}
-    requiresBuild: true
-    dependencies:
-      '@protobufjs/aspromise': 1.1.2
-      '@protobufjs/base64': 1.1.2
-      '@protobufjs/codegen': 2.0.4
-      '@protobufjs/eventemitter': 1.1.0
-      '@protobufjs/fetch': 1.1.0
-      '@protobufjs/float': 1.0.2
-      '@protobufjs/inquire': 1.1.0
-      '@protobufjs/path': 1.1.2
-      '@protobufjs/pool': 1.1.0
-      '@protobufjs/utf8': 1.1.0
-      '@types/node': 18.19.31
-      long: 5.2.3
-    dev: false
-
   /protobufjs@7.2.6:
     resolution: {integrity: sha512-dgJaEDDL6x8ASUZ1YqWciTRrdOuYNzoOf27oHNfdyvKqHr5i0FV7FSLU+aIeFjyFgVxrpTOtQUi0BLLBymZaBw==}
     engines: {node: '>=12.0.0'}
@@ -13378,8 +13388,6 @@ packages:
       events: 3.3.0
       process: 0.11.10
       string_decoder: 1.3.0
-    dev: false
-    optional: true
 
   /readable-web-to-node-stream@3.0.2:
     resolution: {integrity: sha512-ePeK6cc1EcKLEhJFt/AebMCLL+GgSKhuygrZ/GLaKZYEecIgIECf4UaUuaByiGtzckwR4ain9VzUh95T1exYGw==}
@@ -14882,22 +14890,22 @@ packages:
   /trough@2.2.0:
     resolution: {integrity: sha512-tmMpK00BjZiUyVyvrBK7knerNgmgvcV/KLVyuma/SC+TQN167GrMRciANTz09+k3zW8L8t60jWO1GpfkZdjTaw==}
 
-  /ts-api-utils@1.0.3(typescript@5.4.4):
+  /ts-api-utils@1.0.3(typescript@5.4.5):
     resolution: {integrity: sha512-wNMeqtMz5NtwpT/UZGY5alT+VoKdSsOOP/kqHFcUW1P/VRhH2wJ48+DN2WwUliNbQ976ETwDL0Ifd2VVvgonvg==}
     engines: {node: '>=16.13.0'}
     peerDependencies:
       typescript: '>=4.2.0'
     dependencies:
-      typescript: 5.4.4
+      typescript: 5.4.5
     dev: false
 
-  /ts-api-utils@1.3.0(typescript@5.4.4):
+  /ts-api-utils@1.3.0(typescript@5.4.5):
     resolution: {integrity: sha512-UQMIo7pb8WRomKR1/+MFVLTroIvDVtMX3K6OUir8ynLyzB8Jeriont2bTAtmNPa1ekAgN7YPDyf6V+ygrdU+eQ==}
     engines: {node: '>=16'}
     peerDependencies:
       typescript: '>=4.2.0'
     dependencies:
-      typescript: 5.4.4
+      typescript: 5.4.5
     dev: false
 
   /ts-graphviz@1.8.2:
@@ -14905,7 +14913,7 @@ packages:
     engines: {node: '>=14.16'}
     dev: true
 
-  /ts-node@10.9.2(@types/node@18.19.31)(typescript@5.4.4):
+  /ts-node@10.9.2(@types/node@18.19.31)(typescript@5.4.5):
     resolution: {integrity: sha512-f0FFpIdcHgn8zcPSbf1dRevwt047YMnaiJM3u2w2RewrB+fob/zePZcrOyQoLMMO7aBIddLcQIEK5dYjkLnGrQ==}
     hasBin: true
     peerDependencies:
@@ -14931,7 +14939,7 @@ packages:
       create-require: 1.1.1
       diff: 4.0.2
       make-error: 1.3.6
-      typescript: 5.4.4
+      typescript: 5.4.5
       v8-compile-cache-lib: 3.0.1
       yn: 3.1.1
     dev: true
@@ -15008,7 +15016,7 @@ packages:
     hasBin: true
     dependencies:
       esbuild: 0.19.12
-      get-tsconfig: 4.7.2
+      get-tsconfig: 4.7.3
     optionalDependencies:
       fsevents: 2.3.3
     dev: true
@@ -15236,6 +15244,11 @@ packages:
     engines: {node: '>=14.17'}
     hasBin: true
 
+  /typescript@5.4.5:
+    resolution: {integrity: sha512-vcI4UpRgg81oIRUFwR0WSIHKt11nJ7SAVlYNIu+QpqeyXP+gpQJy/Z4+F0aGxSE4MqwjyXvW/TzgkLAx2AGHwQ==}
+    engines: {node: '>=14.17'}
+    hasBin: true
+
   /ufo@1.4.0:
     resolution: {integrity: sha512-Hhy+BhRBleFjpJ2vchUNN40qgkh0366FWJGqVLYBHev0vpHTrXSA0ryT+74UiW6KWsldNurQMKGqCm1M2zBciQ==}
     dev: true