release v0.2.9

Co-authored-by: Yi Ding <yi.s.ding@gmail.com>

.eslintrc.cjs

@@ -1,12 +1,22 @@
 module.exports = {
+  root: true,
   extends: [
-    "next",
     "turbo",
     "prettier",
     "plugin:@typescript-eslint/recommended-type-checked-only",
   ],
+  parserOptions: {
+    project: true,
+    __tsconfigRootDir: __dirname,
+  },
+  settings: {
+    react: {
+      version: "999.999.999",
+    },
+  },
   rules: {
-    "@next/next/no-html-link-for-pages": "off",
+    "max-params": ["error", 4],
+    "prefer-const": "error",
     "@typescript-eslint/no-floating-promises": [
       "error",
       {
@@ -54,22 +64,13 @@ module.exports = {
     "@typescript-eslint/triple-slash-reference": "off",
     "@typescript-eslint/unbound-method": "off",
   },
-  // NOTE I think because we've temporarily removed all of the NextJS stuff
-  // from the turborepo not having next in the devDeps causes an error on only
-  // clean clones of the repo
-  // Not sure if this is a missing dependency in the package.json or just my not
-  // understanding how turborepo is supposed to work.
-  // Anyways, planning to add back a Next.JS example soon
-  parserOptions: {
-    babelOptions: {
-      presets: [require.resolve("next/babel")],
+  overrides: [
+    {
+      files: ["examples/**/*.ts"],
+      rules: {
+        "turbo/no-undeclared-env-vars": "off",
+      },
     },
-    project: true,
-    __tsconfigRootDir: __dirname,
-  },
-  settings: {
-    react: {
-      version: "999.999.999",
-    },
-  },
+  ],
+  ignorePatterns: ["dist/", "lib/"],
 };

.eslintrc.js

@@ -1,23 +0,0 @@
-module.exports = {
-  root: true,
-  // This tells ESLint to load the config from the package `eslint-config-custom`
-  extends: ["custom"],
-  settings: {
-    next: {
-      rootDir: ["apps/*/"],
-    },
-  },
-  rules: {
-    "max-params": ["error", 4],
-    "prefer-const": "error",
-  },
-  overrides: [
-    {
-      files: ["examples/**/*.ts"],
-      rules: {
-        "turbo/no-undeclared-env-vars": "off",
-      },
-    },
-  ],
-  ignorePatterns: ["dist/", "lib/"],
-};

.github/workflows/lint_on_push_or_pull.yml

@@ -14,8 +14,6 @@ jobs:
     steps:
       - uses: actions/checkout@v4
       - uses: pnpm/action-setup@v2
-        with:
-          version: latest
       - name: Setup Node.js
         uses: actions/setup-node@v4
         with:

.github/workflows/test.yml

@@ -68,6 +68,9 @@ jobs:
         run: pnpm install
       - name: Build
         run: pnpm run build --filter llamaindex
+      - name: Use Build For Examples
+        run: pnpm link ../packages/core/
+        working-directory: ./examples
       - name: Run Type Check
         run: pnpm run type-check
       - name: Run Circular Dependency Check

README.md

@@ -154,7 +154,7 @@ If you need any of those classes, you have to import them instead directly. Here
 import { PineconeVectorStore } from "@llamaindex/edge/storage/vectorStore/PineconeVectorStore";
 ```
 
-As the `PDFReader` is not with the Edge runtime, here's how to use the `SimpleDirectoryReader` with the `LlamaParseReader` to load PDFs:
+As the `PDFReader` is not working with the Edge runtime, here's how to use the `SimpleDirectoryReader` with the `LlamaParseReader` to load PDFs:
 
 ```typescript
 import { SimpleDirectoryReader } from "@llamaindex/edge/readers/SimpleDirectoryReader";

apps/docs/docs/examples/agent.mdx

@@ -67,7 +67,6 @@ async function main() {
   // Create an OpenAIAgent with the function tools
   const agent = new OpenAIAgent({
     tools: [sumFunctionTool, divideFunctionTool],
-    verbose: true,
   });
 
   // Chat with the agent

apps/docs/docs/modules/agent/multi_document_agent.mdx

@@ -221,7 +221,6 @@ for (const title of wikiTitles) {
   const agent = new OpenAIAgent({
     tools: queryEngineTools,
     llm,
-    verbose: true,
   });
 
   documentAgents[title] = agent;
@@ -282,7 +281,6 @@ const objectIndex = await ObjectIndex.fromObjects(
 const topAgent = new OpenAIAgent({
   toolRetriever: await objectIndex.asRetriever({}),
   llm,
-  verbose: true,
   prefixMessages: [
     {
       content:

apps/docs/docs/modules/agent/openai.mdx

@@ -88,7 +88,6 @@ Now we can create an OpenAIAgent with the function tools.
 ```ts
 const agent = new OpenAIAgent({
   tools: [sumFunctionTool, divideFunctionTool],
-  verbose: true,
 });
 ```
 
@@ -169,7 +168,6 @@ async function main() {
   // Create an OpenAIAgent with the function tools
   const agent = new OpenAIAgent({
     tools: [sumFunctionTool, divideFunctionTool],
-    verbose: true,
   });
 
   // Chat with the agent

apps/docs/docs/modules/agent/query_engine_tool.mdx

@@ -64,7 +64,6 @@ const queryEngineTool = new QueryEngineTool({
 
 const agent = new OpenAIAgent({
   tools: [queryEngineTool],
-  verbose: true,
 });
 ```
 
@@ -114,7 +113,6 @@ async function main() {
   // Create an OpenAIAgent with the function tools
   const agent = new OpenAIAgent({
     tools: [queryEngineTool],
-    verbose: true,
   });
 
   // Chat with the agent

apps/docs/docs/modules/agent/react_agent.mdx

@@ -90,7 +90,6 @@ Now we can create an OpenAIAgent with the function tools.
 ```ts
 const agent = new ReActAgent({
   tools: [sumFunctionTool, divideFunctionTool],
-  verbose: true,
 });
 ```
 
@@ -185,7 +184,6 @@ async function main() {
   // Create an OpenAIAgent with the function tools
   const agent = new OpenAIAgent({
     tools: [sumFunctionTool, divideFunctionTool],
-    verbose: true,
   });
 
   // Chat with the agent

apps/docs/docs/modules/llms/available_llms/llama2.md

@@ -3,7 +3,7 @@
 ## Usage
 
 ```ts
-import { Ollama, Settings } from "llamaindex";
+import { Ollama, Settings, DeuceChatStrategy } from "llamaindex";
 
 Settings.llm = new LlamaDeuce({ chatStrategy: DeuceChatStrategy.META });
 ```
@@ -11,7 +11,12 @@ Settings.llm = new LlamaDeuce({ chatStrategy: DeuceChatStrategy.META });
 ## Usage with Replication
 
 ```ts
-import { Ollama, ReplicateSession, Settings } from "llamaindex";
+import {
+  Ollama,
+  ReplicateSession,
+  Settings,
+  DeuceChatStrategy,
+} from "llamaindex";
 
 const replicateSession = new ReplicateSession({
   replicateKey,
@@ -48,7 +53,13 @@ const results = await queryEngine.query({
 ## Full Example
 
 ```ts
-import { LlamaDeuce, Document, VectorStoreIndex, Settings } from "llamaindex";
+import {
+  LlamaDeuce,
+  Document,
+  VectorStoreIndex,
+  Settings,
+  DeuceChatStrategy,
+} from "llamaindex";
 
 // Use the LlamaDeuce LLM
 Settings.llm = new LlamaDeuce({ chatStrategy: DeuceChatStrategy.META });

apps/docs/docs/observability/index.md

@@ -14,6 +14,9 @@ Configure a variable once, and you'll be able to do things like the following:
 
 Each provider has similarities and differences. Take a look below for the full set of guides for each one!
 
+- [OpenLLMetry](#openllmetry)
+- [Langtrace](#langtrace)
+
 ## OpenLLMetry
 
 [OpenLLMetry](https://github.com/traceloop/openllmetry-js) is an open-source project based on OpenTelemetry for tracing and monitoring
@@ -33,3 +36,29 @@ traceloop.initialize({
   disableBatch: true,
 });
 ```
+
+## Langtrace
+
+Enhance your observability with Langtrace, a robust open-source tool supports OpenTelemetry and is designed to trace, evaluate, and manage LLM applications seamlessly. Langtrace integrates directly with LlamaIndex, offering detailed, real-time insights into performance metrics such as accuracy, evaluations, and latency.
+
+#### Install
+
+- Self-host or sign-up and generate an API key using [Langtrace](https://www.langtrace.ai) Cloud
+
+```bash
+npm install @langtrase/typescript-sdk
+```
+
+#### Initialize
+
+```js
+import * as Langtrace from "@langtrase/typescript-sdk";
+Langtrace.init({ api_key: "<YOUR_API_KEY>" });
+```
+
+Features:
+
+- OpenTelemetry compliant, ensuring broad compatibility with observability platforms.
+- Provides comprehensive logs and detailed traces of all components.
+- Real-time monitoring of accuracy, evaluations, usage, costs, and latency.
+- For more configuration options and details, visit [Langtrace Docs](https://docs.langtrace.ai/introduction).

examples/anthropic/agent.ts

@@ -0,0 +1,43 @@
+import { FunctionTool, Settings, WikipediaTool } from "llamaindex";
+import { AnthropicAgent } from "llamaindex/agent/anthropic";
+
+Settings.callbackManager.on("llm-tool-call", (event) => {
+  console.log("llm-tool-call", event.detail.payload.toolCall);
+});
+
+const agent = new AnthropicAgent({
+  tools: [
+    FunctionTool.from<{ location: string }>(
+      (query) => {
+        return `The weather in ${query.location} is sunny`;
+      },
+      {
+        name: "weather",
+        description: "Get the weather",
+        parameters: {
+          type: "object",
+          properties: {
+            location: {
+              type: "string",
+              description: "The location to get the weather for",
+            },
+          },
+          required: ["location"],
+        },
+      },
+    ),
+    new WikipediaTool(),
+  ],
+});
+
+async function main() {
+  // https://docs.anthropic.com/claude/docs/tool-use#tool-use-best-practices-and-limitations
+  const { response } = await agent.chat({
+    message:
+      "What is the weather in New York? What's the history of New York from Wikipedia in 3 sentences?",
+  });
+
+  console.log(response);
+}
+
+void main();

examples/multimodal/load.ts

@@ -4,6 +4,7 @@ import {
   VectorStoreIndex,
   storageContextFromDefaults,
 } from "llamaindex";
+import { DocStoreStrategy } from "llamaindex/ingestion/strategies/index";
 
 import * as path from "path";
 
@@ -31,6 +32,7 @@ async function generateDatasource() {
     });
     await VectorStoreIndex.fromDocuments(documents, {
       storageContext,
+      docStoreStrategy: DocStoreStrategy.NONE,
     });
   });
   console.log(`Storage successfully generated in ${ms / 1000}s.`);

examples/toolsStream.ts

@@ -35,7 +35,10 @@ async function main() {
   const stream = await llm.chat({ ...args, stream: true });
   for await (const chunk of stream) {
     process.stdout.write(chunk.delta);
-    console.log(chunk.options?.toolCalls?.[0]);
+    if (chunk.options && "toolCall" in chunk.options) {
+      console.log("Tool call:");
+      console.log(chunk.options.toolCall);
+    }
   }
 }
 

examples/vectorIndex.ts

@@ -1,6 +1,11 @@
 import fs from "node:fs/promises";
 
-import { Document, VectorStoreIndex } from "llamaindex";
+import {
+  Document,
+  MetadataMode,
+  NodeWithScore,
+  VectorStoreIndex,
+} from "llamaindex";
 
 async function main() {
   // Load essay from abramov.txt in Node
@@ -16,12 +21,20 @@ async function main() {
 
   // Query the index
   const queryEngine = index.asQueryEngine();
-  const response = await queryEngine.query({
+  const { response, sourceNodes } = await queryEngine.query({
     query: "What did the author do in college?",
   });
 
-  // Output response
-  console.log(response.toString());
+  // Output response with sources
+  console.log(response);
+
+  if (sourceNodes) {
+    sourceNodes.forEach((source: NodeWithScore, index: number) => {
+      console.log(
+        `\n${index}: Score: ${source.score} - ${source.node.getContent(MetadataMode.NONE).substring(0, 50)}...\n`,
+      );
+    });
+  }
 }
 
 main().catch(console.error);

package.json

@@ -15,13 +15,17 @@
     "release": "pnpm run check-minor-version && pnpm run build:release && changeset publish",
     "release-snapshot": "pnpm run check-minor-version && pnpm run build:release && changeset publish --tag snapshot",
     "check-minor-version": "node ./scripts/check-minor-version",
-    "new-version": "pnpm run build:release && changeset version && pnpm run check-minor-version",
+    "new-version": "changeset version && pnpm run check-minor-version && pnpm run build:release",
     "new-snapshot": "pnpm run build:release && changeset version --snapshot"
   },
   "devDependencies": {
     "@changesets/cli": "^2.27.1",
+    "@typescript-eslint/eslint-plugin": "^7.7.0",
     "eslint": "^8.57.0",
-    "eslint-config-custom": "workspace:*",
+    "eslint-config-next": "^13.5.6",
+    "eslint-config-prettier": "^8.10.0",
+    "eslint-config-turbo": "^1.13.2",
+    "eslint-plugin-react": "7.28.0",
     "husky": "^9.0.11",
     "lint-staged": "^15.2.2",
     "prettier": "^3.2.5",
@@ -29,7 +33,7 @@
     "turbo": "^1.13.2",
     "typescript": "^5.4.5"
   },
-  "packageManager": "pnpm@8.15.6+sha256.01c01eeb990e379b31ef19c03e9d06a14afa5250b82e81303f88721c99ff2e6f",
+  "packageManager": "pnpm@9.0.1+sha256.46d50ee2afecb42b185ebbd662dc7bdd52ef5be56bf035bb615cab81a75345df",
   "pnpm": {
     "overrides": {
       "trim": "1.0.1",

packages/core/CHANGELOG.md

@@ -1,5 +1,16 @@
 # llamaindex
 
+## 0.2.9
+
+### Patch Changes
+
+- 76c3fd6: Add score to source nodes response
+- 208282d: feat: init anthropic agent
+
+  remove the `tool` | `function` type in `MessageType`. Replace with `assistant` instead.
+  This is because these two types are only available for `OpenAI`.
+  Since `OpenAI` deprecates the function type, we support the Claude 3 tool call.
+
 ## 0.2.8
 
 ### Patch Changes

packages/core/e2e/fixtures/embeddings/OpenAIEmbedding.ts

@@ -28,6 +28,7 @@ export class OpenAIEmbedding implements BaseEmbedding {
   }
 
   async transform(nodes: BaseNode[], _options?: any): Promise<BaseNode[]> {
+    nodes.forEach((node) => (node.embedding = [0]));
     return nodes;
   }
 }

packages/core/e2e/fixtures/llm/anthropic.ts

@@ -0,0 +1,3 @@
+import { OpenAI } from "./open_ai.js";
+
+export class Anthropic extends OpenAI {}

packages/core/e2e/fixtures/llm/open_ai.ts

@@ -9,7 +9,7 @@ import type {
   LLMCompletionParamsStreaming,
 } from "llamaindex/llm/types";
 import { extractText } from "llamaindex/llm/utils";
-import { strictEqual } from "node:assert";
+import { deepStrictEqual, strictEqual } from "node:assert";
 import { llmCompleteMockStorage } from "../../node/utils.js";
 
 export function getOpenAISession() {
@@ -21,6 +21,7 @@ export function isFunctionCallingModel() {
 }
 
 export class OpenAI implements LLM {
+  supportToolCall = true;
   get metadata() {
     return {
       model: "mock-model",
@@ -48,7 +49,7 @@ export class OpenAI implements LLM {
       strictEqual(chatMessage.length, params.messages.length);
       for (let i = 0; i < chatMessage.length; i++) {
         strictEqual(chatMessage[i].role, params.messages[i].role);
-        strictEqual(chatMessage[i].content, params.messages[i].content);
+        deepStrictEqual(chatMessage[i].content, params.messages[i].content);
       }
 
       if (llmCompleteMockStorage.llmEventEnd.length > 0) {

packages/core/e2e/node/claude.e2e.ts

@@ -0,0 +1,129 @@
+import { consola } from "consola";
+import { Anthropic, FunctionTool, Settings, type LLM } from "llamaindex";
+import { AnthropicAgent } from "llamaindex/agent/anthropic";
+import { ok } from "node:assert";
+import { beforeEach, test } from "node:test";
+import { sumNumbersTool } from "./fixtures/tools.js";
+import { mockLLMEvent } from "./utils.js";
+
+let llm: LLM;
+beforeEach(async () => {
+  Settings.llm = new Anthropic({
+    model: "claude-3-opus",
+  });
+  llm = Settings.llm;
+});
+
+await test("anthropic llm", async (t) => {
+  await mockLLMEvent(t, "llm-anthropic");
+  await t.test("llm.chat", async () => {
+    const response = await llm.chat({
+      messages: [
+        {
+          content: "Hello",
+          role: "user",
+          options: {},
+        },
+      ],
+    });
+    consola.debug("response:", response);
+    ok(typeof response.message.content === "string");
+  });
+
+  await t.test("stream llm.chat", async () => {
+    const iter = await llm.chat({
+      stream: true,
+      messages: [
+        {
+          content: "hello",
+          role: "user",
+        },
+      ],
+    });
+    for await (const chunk of iter) {
+      consola.debug("chunk:", chunk);
+      ok(typeof chunk.delta === "string");
+    }
+  });
+});
+
+await test("anthropic agent", async (t) => {
+  await mockLLMEvent(t, "anthropic-agent");
+  await t.test("chat", async () => {
+    const agent = new AnthropicAgent({
+      tools: [
+        {
+          call: async () => {
+            return "35 degrees and sunny in San Francisco";
+          },
+          metadata: {
+            name: "Weather",
+            description: "Get the weather",
+            parameters: {
+              type: "object",
+              properties: {
+                location: { type: "string" },
+              },
+              required: ["location"],
+            },
+          },
+        },
+      ],
+    });
+    const result = await agent.chat({
+      message: "What is the weather in San Francisco?",
+    });
+    consola.debug("response:", result.response);
+    ok(typeof result.response === "string");
+    ok(result.response.includes("35"));
+  });
+
+  await t.test("async function", async () => {
+    const uniqueId = "123456789";
+    const showUniqueId = FunctionTool.from<{
+      firstName: string;
+      lastName: string;
+    }>(
+      async ({ firstName, lastName }) => {
+        ok(typeof firstName === "string");
+        ok(typeof lastName === "string");
+        const fullName = firstName + lastName;
+        ok(fullName.toLowerCase().includes("alex"));
+        ok(fullName.toLowerCase().includes("yang"));
+        return uniqueId;
+      },
+      {
+        name: "unique_id",
+        description: "show user unique id",
+        parameters: {
+          type: "object",
+          properties: {
+            firstName: { type: "string" },
+            lastName: { type: "string" },
+          },
+          required: ["firstName", "lastName"],
+        },
+      },
+    );
+    const agent = new AnthropicAgent({
+      tools: [showUniqueId],
+    });
+    const { response } = await agent.chat({
+      message: "My name is Alex Yang. What is my unique id?",
+    });
+    consola.debug("response:", response);
+    ok(response.includes(uniqueId));
+  });
+
+  await t.test("sum numbers", async () => {
+    const openaiAgent = new AnthropicAgent({
+      tools: [sumNumbersTool],
+    });
+
+    const response = await openaiAgent.chat({
+      message: "how much is 1 + 1?",
+    });
+
+    ok(response.response.includes("2"));
+  });
+});

packages/core/e2e/node/fixtures/tools.ts

@@ -0,0 +1,47 @@
+import { FunctionTool } from "llamaindex";
+
+function sumNumbers({ a, b }: { a: number; b: number }) {
+  return `${a + b}`;
+}
+
+function divideNumbers({ a, b }: { a: number; b: number }) {
+  return `${a / b}`;
+}
+
+export const sumNumbersTool = FunctionTool.from(sumNumbers, {
+  name: "sumNumbers",
+  description: "Use this function to sum two numbers",
+  parameters: {
+    type: "object",
+    properties: {
+      a: {
+        type: "number",
+        description: "The first number",
+      },
+      b: {
+        type: "number",
+        description: "The second number",
+      },
+    },
+    required: ["a", "b"],
+  },
+});
+
+export const divideNumbersTool = FunctionTool.from(divideNumbers, {
+  name: "divideNumbers",
+  description: "Use this function to divide two numbers",
+  parameters: {
+    type: "object",
+    properties: {
+      a: {
+        type: "number",
+        description: "The first number",
+      },
+      b: {
+        type: "number",
+        description: "The second number",
+      },
+    },
+    required: ["a", "b"],
+  },
+});

packages/core/e2e/node/openai.e2e.ts

@@ -10,8 +10,9 @@ import {
   VectorStoreIndex,
   type LLM,
 } from "llamaindex";
-import { ok } from "node:assert";
+import { ok, strictEqual } from "node:assert";
 import { beforeEach, test } from "node:test";
+import { divideNumbersTool, sumNumbersTool } from "./fixtures/tools.js";
 import { mockLLMEvent } from "./utils.js";
 
 let llm: LLM;
@@ -22,15 +23,7 @@ beforeEach(async () => {
   llm = Settings.llm;
 });
 
-function sumNumbers({ a, b }: { a: number; b: number }) {
-  return `${a + b}`;
-}
-
-function divideNumbers({ a, b }: { a: number; b: number }) {
-  return `${a / b}`;
-}
-
-await test("llm", async (t) => {
+await test("openai llm", async (t) => {
   await mockLLMEvent(t, "llm");
   await t.test("llm.chat", async () => {
     const response = await llm.chat({
@@ -166,27 +159,8 @@ await test("agent", async (t) => {
   });
 
   await t.test("sum numbers", async () => {
-    const sumFunctionTool = new FunctionTool(sumNumbers, {
-      name: "sumNumbers",
-      description: "Use this function to sum two numbers",
-      parameters: {
-        type: "object",
-        properties: {
-          a: {
-            type: "number",
-            description: "The first number",
-          },
-          b: {
-            type: "number",
-            description: "The second number",
-          },
-        },
-        required: ["a", "b"],
-      },
-    });
-
     const openaiAgent = new OpenAIAgent({
-      tools: [sumFunctionTool],
+      tools: [sumNumbersTool],
     });
 
     const response = await openaiAgent.chat({
@@ -199,51 +173,12 @@ await test("agent", async (t) => {
 
 await test("agent stream", async (t) => {
   await mockLLMEvent(t, "agent_stream");
-  await t.test("sum numbers stream", async () => {
-    const sumJSON = {
-      type: "object",
-      properties: {
-        a: {
-          type: "number",
-          description: "The first number",
-        },
-        b: {
-          type: "number",
-          description: "The second number",
-        },
-      },
-      required: ["a", "b"],
-    } as const;
-
-    const divideJSON = {
-      type: "object",
-      properties: {
-        a: {
-          type: "number",
-          description: "The dividend",
-        },
-        b: {
-          type: "number",
-          description: "The divisor",
-        },
-      },
-      required: ["a", "b"],
-    } as const;
-
-    const functionTool = FunctionTool.from(sumNumbers, {
-      name: "sumNumbers",
-      description: "Use this function to sum two numbers",
-      parameters: sumJSON,
-    });
-
-    const functionTool2 = FunctionTool.from(divideNumbers, {
-      name: "divideNumbers",
-      description: "Use this function to divide two numbers",
-      parameters: divideJSON,
-    });
+  await t.test("sum numbers stream", async (t) => {
+    const fn = t.mock.fn(() => {});
+    Settings.callbackManager.on("llm-tool-call", fn);
 
     const agent = new OpenAIAgent({
-      tools: [functionTool, functionTool2],
+      tools: [sumNumbersTool, divideNumbersTool],
     });
 
     const { response } = await agent.chat({
@@ -257,13 +192,17 @@ await test("agent stream", async (t) => {
       message += chunk.response;
     }
 
+    strictEqual(fn.mock.callCount(), 2);
     ok(message.includes("28"));
+    Settings.callbackManager.off("llm-tool-call", fn);
   });
 });
 
 await test("queryEngine", async (t) => {
   await mockLLMEvent(t, "queryEngine_subquestion");
   await t.test("subquestion", async () => {
+    const fn = t.mock.fn(() => {});
+    Settings.callbackManager.on("llm-tool-call", fn);
     const document = new Document({
       text: "Bill Gates stole from Apple.\n Steve Jobs stole from Xerox.",
     });
@@ -288,5 +227,7 @@ await test("queryEngine", async (t) => {
     });
 
     ok(response.includes("Apple"));
+    strictEqual(fn.mock.callCount(), 0);
+    Settings.callbackManager.off("llm-tool-call", fn);
   });
 });

packages/core/e2e/node/snapshot/agent.snap

@@ -20,24 +20,21 @@
           "content": "",
           "role": "assistant",
           "options": {
-            "toolCalls": [
-              {
-                "id": "HIDDEN",
-                "type": "function",
-                "function": {
-                  "name": "Weather",
-                  "arguments": "{\"location\":\"San Francisco\"}"
-                }
-              }
-            ]
+            "toolCall": {
+              "id": "HIDDEN",
+              "name": "Weather",
+              "input": "{\"location\":\"San Francisco\"}"
+            }
           }
         },
         {
           "content": "35 degrees and sunny in San Francisco",
-          "role": "tool",
+          "role": "user",
           "options": {
-            "name": "Weather",
-            "tool_call_id": "HIDDEN"
+            "toolResult": {
+              "id": "HIDDEN",
+              "isError": false
+            }
           }
         }
       ]
@@ -62,24 +59,21 @@
           "content": "",
           "role": "assistant",
           "options": {
-            "toolCalls": [
-              {
-                "id": "HIDDEN",
-                "type": "function",
-                "function": {
-                  "name": "unique_id",
-                  "arguments": "{\"firstName\":\"Alex\",\"lastName\":\"Yang\"}"
-                }
-              }
-            ]
+            "toolCall": {
+              "id": "HIDDEN",
+              "name": "unique_id",
+              "input": "{\"firstName\":\"Alex\",\"lastName\":\"Yang\"}"
+            }
           }
         },
         {
           "content": "123456789",
-          "role": "tool",
+          "role": "user",
           "options": {
-            "name": "unique_id",
-            "tool_call_id": "HIDDEN"
+            "toolResult": {
+              "id": "HIDDEN",
+              "isError": false
+            }
           }
         }
       ]
@@ -104,24 +98,21 @@
           "content": "",
           "role": "assistant",
           "options": {
-            "toolCalls": [
-              {
-                "id": "HIDDEN",
-                "type": "function",
-                "function": {
-                  "name": "sumNumbers",
-                  "arguments": "{\"a\":1,\"b\":1}"
-                }
-              }
-            ]
+            "toolCall": {
+              "id": "HIDDEN",
+              "name": "sumNumbers",
+              "input": "{\"a\":1,\"b\":1}"
+            }
           }
         },
         {
           "content": "2",
-          "role": "tool",
+          "role": "user",
           "options": {
-            "name": "sumNumbers",
-            "tool_call_id": "HIDDEN"
+            "toolResult": {
+              "id": "HIDDEN",
+              "isError": false
+            }
           }
         }
       ]
@@ -168,16 +159,11 @@
           "content": "",
           "role": "assistant",
           "options": {
-            "toolCalls": [
-              {
-                "id": "HIDDEN",
-                "type": "function",
-                "function": {
-                  "name": "Weather",
-                  "arguments": "{\"location\":\"San Francisco\"}"
-                }
-              }
-            ]
+            "toolCall": {
+              "id": "HIDDEN",
+              "name": "Weather",
+              "input": "{\"location\":\"San Francisco\"}"
+            }
           }
         }
       }
@@ -255,16 +241,11 @@
           "content": "",
           "role": "assistant",
           "options": {
-            "toolCalls": [
-              {
-                "id": "HIDDEN",
-                "type": "function",
-                "function": {
-                  "name": "unique_id",
-                  "arguments": "{\"firstName\":\"Alex\",\"lastName\":\"Yang\"}"
-                }
-              }
-            ]
+            "toolCall": {
+              "id": "HIDDEN",
+              "name": "unique_id",
+              "input": "{\"firstName\":\"Alex\",\"lastName\":\"Yang\"}"
+            }
           }
         }
       }
@@ -342,16 +323,11 @@
           "content": "",
           "role": "assistant",
           "options": {
-            "toolCalls": [
-              {
-                "id": "HIDDEN",
-                "type": "function",
-                "function": {
-                  "name": "sumNumbers",
-                  "arguments": "{\"a\":1,\"b\":1}"
-                }
-              }
-            ]
+            "toolCall": {
+              "id": "HIDDEN",
+              "name": "sumNumbers",
+              "input": "{\"a\":1,\"b\":1}"
+            }
           }
         }
       }
@@ -369,7 +345,7 @@
               "index": 0,
               "message": {
                 "role": "assistant",
-                "content": "The sum of 1 + 1 is 2."
+                "content": "1 + 1 is equal to 2."
               },
               "logprobs": null,
               "finish_reason": "stop"
@@ -377,13 +353,13 @@
           ],
           "usage": {
             "prompt_tokens": 97,
-            "completion_tokens": 13,
-            "total_tokens": 110
+            "completion_tokens": 11,
+            "total_tokens": 108
           },
           "system_fingerprint": "HIDDEN"
         },
         "message": {
-          "content": "The sum of 1 + 1 is 2.",
+          "content": "1 + 1 is equal to 2.",
           "role": "assistant",
           "options": {}
         }

packages/core/e2e/node/snapshot/anthropic-agent.snap

@@ -0,0 +1,403 @@
+{
+  "llmEventStart": [
+    {
+      "id": "PRESERVE_0",
+      "messages": [
+        {
+          "role": "user",
+          "content": "What is the weather in San Francisco?",
+          "options": {}
+        }
+      ]
+    },
+    {
+      "id": "PRESERVE_1",
+      "messages": [
+        {
+          "role": "user",
+          "content": "What is the weather in San Francisco?",
+          "options": {}
+        },
+        {
+          "content": [
+            {
+              "type": "text",
+              "text": "<thinking>\nThe user is asking for the weather in a specific location, San Francisco. The Weather function is the relevant tool to answer this request, as it returns weather information for a given location.\n\nThe Weather function has one required parameter:\n- location (string): The user has directly provided the location of \"San Francisco\"\n\nSince the required location parameter has been provided by the user, we have all the necessary information to call the Weather function.\n</thinking>"
+            }
+          ],
+          "role": "assistant",
+          "options": {
+            "toolCall": {
+              "id": "HIDDEN",
+              "name": "Weather",
+              "input": {
+                "location": "San Francisco"
+              }
+            }
+          }
+        },
+        {
+          "content": "35 degrees and sunny in San Francisco",
+          "role": "user",
+          "options": {
+            "toolResult": {
+              "isError": false,
+              "id": "HIDDEN"
+            }
+          }
+        }
+      ]
+    },
+    {
+      "id": "PRESERVE_2",
+      "messages": [
+        {
+          "role": "user",
+          "content": "My name is Alex Yang. What is my unique id?",
+          "options": {}
+        }
+      ]
+    },
+    {
+      "id": "PRESERVE_3",
+      "messages": [
+        {
+          "role": "user",
+          "content": "My name is Alex Yang. What is my unique id?",
+          "options": {}
+        },
+        {
+          "content": [
+            {
+              "type": "text",
+              "text": "<thinking>\nThe unique_id function is the relevant tool to answer the user's request for their unique ID. It requires two parameters:\nfirstName: The user provided their first name, which is \"Alex\"\nlastName: The user also provided their last name, \"Yang\"\nSince the user has provided all the required parameters, we can proceed with calling the unique_id function.\n</thinking>"
+            }
+          ],
+          "role": "assistant",
+          "options": {
+            "toolCall": {
+              "id": "HIDDEN",
+              "name": "unique_id",
+              "input": {
+                "firstName": "Alex",
+                "lastName": "Yang"
+              }
+            }
+          }
+        },
+        {
+          "content": "123456789",
+          "role": "user",
+          "options": {
+            "toolResult": {
+              "isError": false,
+              "id": "HIDDEN"
+            }
+          }
+        }
+      ]
+    },
+    {
+      "id": "PRESERVE_4",
+      "messages": [
+        {
+          "role": "user",
+          "content": "how much is 1 + 1?",
+          "options": {}
+        }
+      ]
+    },
+    {
+      "id": "PRESERVE_5",
+      "messages": [
+        {
+          "role": "user",
+          "content": "how much is 1 + 1?",
+          "options": {}
+        },
+        {
+          "content": [
+            {
+              "type": "text",
+              "text": "<thinking>\nThe user is asking to sum the numbers 1 and 1. The relevant tool to use is the sumNumbers function, which takes two number parameters a and b.\nThe user has directly provided the values for the parameters:\na = 1 \nb = 1\nSince all the required parameters have been provided, we can proceed with calling the function.\n</thinking>"
+            }
+          ],
+          "role": "assistant",
+          "options": {
+            "toolCall": {
+              "id": "HIDDEN",
+              "name": "sumNumbers",
+              "input": {
+                "a": 1,
+                "b": 1
+              }
+            }
+          }
+        },
+        {
+          "content": "2",
+          "role": "user",
+          "options": {
+            "toolResult": {
+              "isError": false,
+              "id": "HIDDEN"
+            }
+          }
+        }
+      ]
+    }
+  ],
+  "llmEventEnd": [
+    {
+      "id": "PRESERVE_0",
+      "response": {
+        "raw": {
+          "id": "HIDDEN",
+          "type": "message",
+          "role": "assistant",
+          "model": "claude-3-opus-20240229",
+          "stop_sequence": null,
+          "usage": {
+            "input_tokens": 462,
+            "output_tokens": 147
+          },
+          "content": [
+            {
+              "type": "text",
+              "text": "<thinking>\nThe user is asking for the weather in a specific location, San Francisco. The Weather function is the relevant tool to answer this request, as it returns weather information for a given location.\n\nThe Weather function has one required parameter:\n- location (string): The user has directly provided the location of \"San Francisco\"\n\nSince the required location parameter has been provided by the user, we have all the necessary information to call the Weather function.\n</thinking>"
+            },
+            {
+              "type": "tool_use",
+              "id": "HIDDEN",
+              "name": "Weather",
+              "input": {
+                "location": "San Francisco"
+              }
+            }
+          ],
+          "stop_reason": "tool_use"
+        },
+        "message": {
+          "content": [
+            {
+              "type": "text",
+              "text": "<thinking>\nThe user is asking for the weather in a specific location, San Francisco. The Weather function is the relevant tool to answer this request, as it returns weather information for a given location.\n\nThe Weather function has one required parameter:\n- location (string): The user has directly provided the location of \"San Francisco\"\n\nSince the required location parameter has been provided by the user, we have all the necessary information to call the Weather function.\n</thinking>"
+            }
+          ],
+          "role": "assistant",
+          "options": {
+            "toolCall": {
+              "id": "HIDDEN",
+              "name": "Weather",
+              "input": {
+                "location": "San Francisco"
+              }
+            }
+          }
+        }
+      }
+    },
+    {
+      "id": "PRESERVE_1",
+      "response": {
+        "raw": {
+          "id": "HIDDEN",
+          "type": "message",
+          "role": "assistant",
+          "model": "claude-3-opus-20240229",
+          "stop_sequence": null,
+          "usage": {
+            "input_tokens": 628,
+            "output_tokens": 18
+          },
+          "content": [
+            {
+              "type": "text",
+              "text": "The current weather in San Francisco is 35 degrees and sunny."
+            }
+          ],
+          "stop_reason": "end_turn"
+        },
+        "message": {
+          "content": [
+            {
+              "type": "text",
+              "text": "The current weather in San Francisco is 35 degrees and sunny."
+            }
+          ],
+          "role": "assistant",
+          "options": {}
+        }
+      }
+    },
+    {
+      "id": "PRESERVE_2",
+      "response": {
+        "raw": {
+          "id": "HIDDEN",
+          "type": "message",
+          "role": "assistant",
+          "model": "claude-3-opus-20240229",
+          "stop_sequence": null,
+          "usage": {
+            "input_tokens": 482,
+            "output_tokens": 152
+          },
+          "content": [
+            {
+              "type": "text",
+              "text": "<thinking>\nThe unique_id function is the relevant tool to answer the user's request for their unique ID. It requires two parameters:\nfirstName: The user provided their first name, which is \"Alex\"\nlastName: The user also provided their last name, \"Yang\"\nSince the user has provided all the required parameters, we can proceed with calling the unique_id function.\n</thinking>"
+            },
+            {
+              "type": "tool_use",
+              "id": "HIDDEN",
+              "name": "unique_id",
+              "input": {
+                "firstName": "Alex",
+                "lastName": "Yang"
+              }
+            }
+          ],
+          "stop_reason": "tool_use"
+        },
+        "message": {
+          "content": [
+            {
+              "type": "text",
+              "text": "<thinking>\nThe unique_id function is the relevant tool to answer the user's request for their unique ID. It requires two parameters:\nfirstName: The user provided their first name, which is \"Alex\"\nlastName: The user also provided their last name, \"Yang\"\nSince the user has provided all the required parameters, we can proceed with calling the unique_id function.\n</thinking>"
+            }
+          ],
+          "role": "assistant",
+          "options": {
+            "toolCall": {
+              "id": "HIDDEN",
+              "name": "unique_id",
+              "input": {
+                "firstName": "Alex",
+                "lastName": "Yang"
+              }
+            }
+          }
+        }
+      }
+    },
+    {
+      "id": "PRESERVE_3",
+      "response": {
+        "raw": {
+          "id": "HIDDEN",
+          "type": "message",
+          "role": "assistant",
+          "model": "claude-3-opus-20240229",
+          "stop_sequence": null,
+          "usage": {
+            "input_tokens": 648,
+            "output_tokens": 13
+          },
+          "content": [
+            {
+              "type": "text",
+              "text": "Your unique ID is 123456789."
+            }
+          ],
+          "stop_reason": "end_turn"
+        },
+        "message": {
+          "content": [
+            {
+              "type": "text",
+              "text": "Your unique ID is 123456789."
+            }
+          ],
+          "role": "assistant",
+          "options": {}
+        }
+      }
+    },
+    {
+      "id": "PRESERVE_4",
+      "response": {
+        "raw": {
+          "id": "HIDDEN",
+          "type": "message",
+          "role": "assistant",
+          "model": "claude-3-opus-20240229",
+          "stop_sequence": null,
+          "usage": {
+            "input_tokens": 498,
+            "output_tokens": 151
+          },
+          "content": [
+            {
+              "type": "text",
+              "text": "<thinking>\nThe user is asking to sum the numbers 1 and 1. The relevant tool to use is the sumNumbers function, which takes two number parameters a and b.\nThe user has directly provided the values for the parameters:\na = 1 \nb = 1\nSince all the required parameters have been provided, we can proceed with calling the function.\n</thinking>"
+            },
+            {
+              "type": "tool_use",
+              "id": "HIDDEN",
+              "name": "sumNumbers",
+              "input": {
+                "a": 1,
+                "b": 1
+              }
+            }
+          ],
+          "stop_reason": "tool_use"
+        },
+        "message": {
+          "content": [
+            {
+              "type": "text",
+              "text": "<thinking>\nThe user is asking to sum the numbers 1 and 1. The relevant tool to use is the sumNumbers function, which takes two number parameters a and b.\nThe user has directly provided the values for the parameters:\na = 1 \nb = 1\nSince all the required parameters have been provided, we can proceed with calling the function.\n</thinking>"
+            }
+          ],
+          "role": "assistant",
+          "options": {
+            "toolCall": {
+              "id": "HIDDEN",
+              "name": "sumNumbers",
+              "input": {
+                "a": 1,
+                "b": 1
+              }
+            }
+          }
+        }
+      }
+    },
+    {
+      "id": "PRESERVE_5",
+      "response": {
+        "raw": {
+          "id": "HIDDEN",
+          "type": "message",
+          "role": "assistant",
+          "model": "claude-3-opus-20240229",
+          "stop_sequence": null,
+          "usage": {
+            "input_tokens": 661,
+            "output_tokens": 16
+          },
+          "content": [
+            {
+              "type": "text",
+              "text": "So 1 + 1 = 2."
+            }
+          ],
+          "stop_reason": "end_turn"
+        },
+        "message": {
+          "content": [
+            {
+              "type": "text",
+              "text": "So 1 + 1 = 2."
+            }
+          ],
+          "role": "assistant",
+          "options": {}
+        }
+      }
+    }
+  ],
+  "llmEventStream": []
+}

packages/core/e2e/node/snapshot/gpt-4-turbo.snap

@@ -20,24 +20,21 @@
           "content": "",
           "role": "assistant",
           "options": {
-            "toolCalls": [
-              {
-                "id": "HIDDEN",
-                "type": "function",
-                "function": {
-                  "name": "Weather",
-                  "arguments": "{\"location\":\"San Jose\"}"
-                }
-              }
-            ]
+            "toolCall": {
+              "id": "HIDDEN",
+              "name": "Weather",
+              "input": "{\"location\":\"San Jose\"}"
+            }
           }
         },
         {
           "content": "45 degrees and sunny in San Jose",
-          "role": "tool",
+          "role": "user",
           "options": {
-            "name": "Weather",
-            "tool_call_id": "HIDDEN"
+            "toolResult": {
+              "id": "HIDDEN",
+              "isError": false
+            }
           }
         }
       ]
@@ -84,16 +81,11 @@
           "content": "",
           "role": "assistant",
           "options": {
-            "toolCalls": [
-              {
-                "id": "HIDDEN",
-                "type": "function",
-                "function": {
-                  "name": "Weather",
-                  "arguments": "{\"location\":\"San Jose\"}"
-                }
-              }
-            ]
+            "toolCall": {
+              "id": "HIDDEN",
+              "name": "Weather",
+              "input": "{\"location\":\"San Jose\"}"
+            }
           }
         }
       }

packages/core/e2e/node/snapshot/llm-anthropic.snap

@@ -0,0 +1,310 @@
+{
+  "llmEventStart": [
+    {
+      "id": "PRESERVE_0",
+      "messages": [
+        {
+          "content": "Hello",
+          "role": "user",
+          "options": {}
+        }
+      ]
+    },
+    {
+      "id": "PRESERVE_1",
+      "messages": [
+        {
+          "content": "hello",
+          "role": "user"
+        }
+      ]
+    }
+  ],
+  "llmEventEnd": [
+    {
+      "id": "PRESERVE_0",
+      "response": {
+        "raw": {
+          "id": "HIDDEN",
+          "type": "message",
+          "role": "assistant",
+          "model": "claude-3-opus-20240229",
+          "stop_sequence": null,
+          "usage": {
+            "input_tokens": 8,
+            "output_tokens": 12
+          },
+          "content": [
+            {
+              "type": "text",
+              "text": "Hello! How can I assist you today?"
+            }
+          ],
+          "stop_reason": "end_turn"
+        },
+        "message": {
+          "content": "Hello! How can I assist you today?",
+          "role": "assistant",
+          "options": {}
+        }
+      }
+    },
+    {
+      "id": "PRESERVE_1",
+      "response": {
+        "raw": [
+          {
+            "raw": {
+              "type": "content_block_delta",
+              "index": 0,
+              "delta": {
+                "type": "text_delta",
+                "text": "Hello"
+              }
+            },
+            "delta": "Hello",
+            "options": {}
+          },
+          {
+            "raw": {
+              "type": "content_block_delta",
+              "index": 0,
+              "delta": {
+                "type": "text_delta",
+                "text": "!"
+              }
+            },
+            "delta": "!",
+            "options": {}
+          },
+          {
+            "raw": {
+              "type": "content_block_delta",
+              "index": 0,
+              "delta": {
+                "type": "text_delta",
+                "text": " How"
+              }
+            },
+            "delta": " How",
+            "options": {}
+          },
+          {
+            "raw": {
+              "type": "content_block_delta",
+              "index": 0,
+              "delta": {
+                "type": "text_delta",
+                "text": " can"
+              }
+            },
+            "delta": " can",
+            "options": {}
+          },
+          {
+            "raw": {
+              "type": "content_block_delta",
+              "index": 0,
+              "delta": {
+                "type": "text_delta",
+                "text": " I"
+              }
+            },
+            "delta": " I",
+            "options": {}
+          },
+          {
+            "raw": {
+              "type": "content_block_delta",
+              "index": 0,
+              "delta": {
+                "type": "text_delta",
+                "text": " assist"
+              }
+            },
+            "delta": " assist",
+            "options": {}
+          },
+          {
+            "raw": {
+              "type": "content_block_delta",
+              "index": 0,
+              "delta": {
+                "type": "text_delta",
+                "text": " you"
+              }
+            },
+            "delta": " you",
+            "options": {}
+          },
+          {
+            "raw": {
+              "type": "content_block_delta",
+              "index": 0,
+              "delta": {
+                "type": "text_delta",
+                "text": " today"
+              }
+            },
+            "delta": " today",
+            "options": {}
+          },
+          {
+            "raw": {
+              "type": "content_block_delta",
+              "index": 0,
+              "delta": {
+                "type": "text_delta",
+                "text": "?"
+              }
+            },
+            "delta": "?",
+            "options": {}
+          }
+        ],
+        "message": {
+          "content": "Hello! How can I assist you today?",
+          "role": "assistant",
+          "options": {}
+        }
+      }
+    }
+  ],
+  "llmEventStream": [
+    {
+      "id": "PRESERVE_1",
+      "chunk": {
+        "raw": {
+          "type": "content_block_delta",
+          "index": 0,
+          "delta": {
+            "type": "text_delta",
+            "text": "Hello"
+          }
+        },
+        "delta": "Hello",
+        "options": {}
+      }
+    },
+    {
+      "id": "PRESERVE_1",
+      "chunk": {
+        "raw": {
+          "type": "content_block_delta",
+          "index": 0,
+          "delta": {
+            "type": "text_delta",
+            "text": "!"
+          }
+        },
+        "delta": "!",
+        "options": {}
+      }
+    },
+    {
+      "id": "PRESERVE_1",
+      "chunk": {
+        "raw": {
+          "type": "content_block_delta",
+          "index": 0,
+          "delta": {
+            "type": "text_delta",
+            "text": " How"
+          }
+        },
+        "delta": " How",
+        "options": {}
+      }
+    },
+    {
+      "id": "PRESERVE_1",
+      "chunk": {
+        "raw": {
+          "type": "content_block_delta",
+          "index": 0,
+          "delta": {
+            "type": "text_delta",
+            "text": " can"
+          }
+        },
+        "delta": " can",
+        "options": {}
+      }
+    },
+    {
+      "id": "PRESERVE_1",
+      "chunk": {
+        "raw": {
+          "type": "content_block_delta",
+          "index": 0,
+          "delta": {
+            "type": "text_delta",
+            "text": " I"
+          }
+        },
+        "delta": " I",
+        "options": {}
+      }
+    },
+    {
+      "id": "PRESERVE_1",
+      "chunk": {
+        "raw": {
+          "type": "content_block_delta",
+          "index": 0,
+          "delta": {
+            "type": "text_delta",
+            "text": " assist"
+          }
+        },
+        "delta": " assist",
+        "options": {}
+      }
+    },
+    {
+      "id": "PRESERVE_1",
+      "chunk": {
+        "raw": {
+          "type": "content_block_delta",
+          "index": 0,
+          "delta": {
+            "type": "text_delta",
+            "text": " you"
+          }
+        },
+        "delta": " you",
+        "options": {}
+      }
+    },
+    {
+      "id": "PRESERVE_1",
+      "chunk": {
+        "raw": {
+          "type": "content_block_delta",
+          "index": 0,
+          "delta": {
+            "type": "text_delta",
+            "text": " today"
+          }
+        },
+        "delta": " today",
+        "options": {}
+      }
+    },
+    {
+      "id": "PRESERVE_1",
+      "chunk": {
+        "raw": {
+          "type": "content_block_delta",
+          "index": 0,
+          "delta": {
+            "type": "text_delta",
+            "text": "?"
+          }
+        },
+        "delta": "?",
+        "options": {}
+      }
+    }
+  ]
+}

packages/core/e2e/node/utils.ts

@@ -1,3 +1,4 @@
+/* eslint-disable turbo/no-undeclared-env-vars */
 import {
   Settings,
   type LLMEndEvent,

packages/core/package.json

@@ -1,11 +1,11 @@
 {
   "name": "llamaindex",
-  "version": "0.2.8",
+  "version": "0.2.9",
   "expectedMinorVersion": "2",
   "license": "MIT",
   "type": "module",
   "dependencies": {
-    "@anthropic-ai/sdk": "^0.18.0",
+    "@anthropic-ai/sdk": "^0.20.4",
     "@aws-crypto/sha256-js": "^5.2.0",
     "@datastax/astra-db-ts": "^0.1.4",
     "@grpc/grpc-js": "^1.10.6",

packages/core/src/ChatHistory.ts

@@ -8,20 +8,22 @@ import { extractText } from "./llm/utils.js";
 /**
  * A ChatHistory is used to keep the state of back and forth chat messages
  */
-export abstract class ChatHistory {
-  abstract get messages(): ChatMessage[];
+export abstract class ChatHistory<
+  AdditionalMessageOptions extends object = object,
+> {
+  abstract get messages(): ChatMessage<AdditionalMessageOptions>[];
   /**
    * Adds a message to the chat history.
    * @param message
    */
-  abstract addMessage(message: ChatMessage): void;
+  abstract addMessage(message: ChatMessage<AdditionalMessageOptions>): void;
 
   /**
    * Returns the messages that should be used as input to the LLM.
    */
   abstract requestMessages(
-    transientMessages?: ChatMessage[],
-  ): Promise<ChatMessage[]>;
+    transientMessages?: ChatMessage<AdditionalMessageOptions>[],
+  ): Promise<ChatMessage<AdditionalMessageOptions>[]>;
 
   /**
    * Resets the chat history so that it's empty.
@@ -31,7 +33,7 @@ export abstract class ChatHistory {
   /**
    * Returns the new messages since the last call to this function (or since calling the constructor)
    */
-  abstract newMessages(): ChatMessage[];
+  abstract newMessages(): ChatMessage<AdditionalMessageOptions>[];
 }
 
 export class SimpleChatHistory extends ChatHistory {
@@ -108,6 +110,7 @@ export class SummaryChatHistory extends ChatHistory {
             context: messagesToHistoryStr(messagesToSummarize),
           }),
           role: "user" as MessageType,
+          options: {},
         },
       ];
       // remove oldest message until the chat history is short enough for the context window
@@ -116,7 +119,9 @@ export class SummaryChatHistory extends ChatHistory {
       this.tokenizer(promptMessages[0].content).length > this.tokensToSummarize
     );
 
-    const response = await this.llm.chat({ messages: promptMessages });
+    const response = await this.llm.chat({
+      messages: promptMessages,
+    });
     return { content: response.message.content, role: "memory" };
   }
 

packages/core/src/Response.ts

@@ -1,14 +1,14 @@
-import type { BaseNode } from "./Node.js";
+import type { NodeWithScore } from "./Node.js";
 
 /**
  * Response is the output of a LLM
  */
 export class Response {
   response: string;
-  sourceNodes?: BaseNode[];
+  sourceNodes?: NodeWithScore[];
   metadata: Record<string, unknown> = {};
 
-  constructor(response: string, sourceNodes?: BaseNode[]) {
+  constructor(response: string, sourceNodes?: NodeWithScore[]) {
     this.response = response;
     this.sourceNodes = sourceNodes || [];
   }

packages/core/src/agent/anthropic.ts

@@ -0,0 +1,170 @@
+import { Settings } from "../Settings.js";
+import {
+  AgentChatResponse,
+  type ChatEngineParamsNonStreaming,
+} from "../engines/chat/index.js";
+import { wrapEventCaller } from "../internal/context/EventCaller.js";
+import { getCallbackManager } from "../internal/settings/CallbackManager.js";
+import { prettifyError } from "../internal/utils.js";
+import { Anthropic } from "../llm/anthropic.js";
+import type {
+  ChatMessage,
+  ChatResponse,
+  ToolCallLLMMessageOptions,
+} from "../llm/index.js";
+import { extractText } from "../llm/utils.js";
+import { ObjectRetriever } from "../objects/index.js";
+import type { BaseToolWithCall } from "../types.js";
+
+const MAX_TOOL_CALLS = 10;
+
+type AnthropicParamsBase = {
+  llm?: Anthropic;
+  chatHistory?: ChatMessage<ToolCallLLMMessageOptions>[];
+};
+
+type AnthropicParamsWithTools = AnthropicParamsBase & {
+  tools: BaseToolWithCall[];
+};
+
+type AnthropicParamsWithToolRetriever = AnthropicParamsBase & {
+  toolRetriever: ObjectRetriever<BaseToolWithCall>;
+};
+
+export type AnthropicAgentParams =
+  | AnthropicParamsWithTools
+  | AnthropicParamsWithToolRetriever;
+
+type AgentContext = {
+  toolCalls: number;
+  llm: Anthropic;
+  tools: BaseToolWithCall[];
+  messages: ChatMessage<ToolCallLLMMessageOptions>[];
+  shouldContinue: (context: AgentContext) => boolean;
+};
+
+type TaskResult = {
+  response: ChatResponse<ToolCallLLMMessageOptions>;
+  chatHistory: ChatMessage<ToolCallLLMMessageOptions>[];
+};
+
+async function task(
+  context: AgentContext,
+  input: ChatMessage<ToolCallLLMMessageOptions>,
+): Promise<TaskResult> {
+  const { llm, tools, messages } = context;
+  const nextMessages: ChatMessage<ToolCallLLMMessageOptions>[] = [
+    ...messages,
+    input,
+  ];
+  const response = await llm.chat({
+    stream: false,
+    tools,
+    messages: nextMessages,
+  });
+  const options = response.message.options ?? {};
+  if ("toolCall" in options) {
+    const { toolCall } = options;
+    const { input, name, id } = toolCall;
+    const targetTool = tools.find((tool) => tool.metadata.name === name);
+    let output: string;
+    let isError = true;
+    if (!context.shouldContinue(context)) {
+      output = "Error: Tool call limit reached";
+    } else if (!targetTool) {
+      output = `Error: Tool ${name} not found`;
+    } else {
+      try {
+        getCallbackManager().dispatchEvent("llm-tool-call", {
+          payload: {
+            toolCall: {
+              name,
+              input,
+            },
+          },
+        });
+        output = await targetTool.call(input);
+        isError = false;
+      } catch (error: unknown) {
+        output = prettifyError(error);
+      }
+    }
+    return task(
+      {
+        ...context,
+        toolCalls: context.toolCalls + 1,
+        messages: [...nextMessages, response.message],
+      },
+      {
+        content: output,
+        role: "user",
+        options: {
+          toolResult: {
+            isError,
+            id,
+          },
+        },
+      },
+    );
+  } else {
+    return { response, chatHistory: [...nextMessages, response.message] };
+  }
+}
+
+export class AnthropicAgent {
+  readonly #llm: Anthropic;
+  readonly #tools:
+    | BaseToolWithCall[]
+    | ((query: string) => Promise<BaseToolWithCall[]>) = [];
+  #chatHistory: ChatMessage<ToolCallLLMMessageOptions>[] = [];
+
+  constructor(params: AnthropicAgentParams) {
+    this.#llm =
+      params.llm ?? Settings.llm instanceof Anthropic
+        ? (Settings.llm as Anthropic)
+        : new Anthropic();
+    if ("tools" in params) {
+      this.#tools = params.tools;
+    } else if ("toolRetriever" in params) {
+      this.#tools = params.toolRetriever.retrieve.bind(params.toolRetriever);
+    }
+    if (Array.isArray(params.chatHistory)) {
+      this.#chatHistory = params.chatHistory;
+    }
+  }
+
+  static shouldContinue(context: AgentContext): boolean {
+    return context.toolCalls < MAX_TOOL_CALLS;
+  }
+
+  public reset(): void {
+    this.#chatHistory = [];
+  }
+
+  getTools(query: string): Promise<BaseToolWithCall[]> | BaseToolWithCall[] {
+    return typeof this.#tools === "function" ? this.#tools(query) : this.#tools;
+  }
+
+  @wrapEventCaller
+  async chat(
+    params: ChatEngineParamsNonStreaming,
+  ): Promise<Promise<AgentChatResponse>> {
+    const { chatHistory, response } = await task(
+      {
+        llm: this.#llm,
+        tools: await this.getTools(extractText(params.message)),
+        toolCalls: 0,
+        messages: [...this.#chatHistory],
+        // do we need this?
+        shouldContinue: AnthropicAgent.shouldContinue,
+      },
+      {
+        role: "user",
+        content: params.message,
+        options: {},
+      },
+    );
+    this.#chatHistory = [...chatHistory];
+    return new AgentChatResponse(extractText(response.message.content));
+  }
+}

packages/core/src/agent/index.ts

@@ -1,3 +1,5 @@
+// Not exporting the AnthropicAgent because it is not ready to ship yet.
+// export { AnthropicAgent, type AnthropicAgentParams } from "./anthropic.js";
 export * from "./openai/base.js";
 export * from "./openai/worker.js";
 export * from "./react/base.js";

packages/core/src/agent/openai/base.ts

@@ -14,7 +14,7 @@ type OpenAIAgentParams = {
   prefixMessages?: ChatMessage[];
   maxFunctionCalls?: number;
   defaultToolChoice?: string;
-  toolRetriever?: ObjectRetriever;
+  toolRetriever?: ObjectRetriever<BaseTool>;
   systemPrompt?: string;
 };
 
@@ -56,7 +56,7 @@ export class OpenAIAgent extends AgentRunner {
       ];
     }
 
-    if (!llm?.metadata.isFunctionCallingModel) {
+    if (!llm?.supportToolCall) {
       throw new Error("LLM model must be a function-calling model");
     }
 
@@ -73,6 +73,7 @@ export class OpenAIAgent extends AgentRunner {
       llm,
       memory,
       defaultToolChoice,
+      // @ts-expect-error 2322
       chatHistory: prefixMessages,
     });
   }

packages/core/src/agent/openai/worker.ts

@@ -14,7 +14,8 @@ import {
   type ChatResponseChunk,
   type LLMChatParamsBase,
   type OpenAIAdditionalChatOptions,
-  type OpenAIAdditionalMessageOptions,
+  type ToolCallLLMMessageOptions,
+  type ToolCallOptions,
 } from "../../llm/index.js";
 import { extractText } from "../../llm/utils.js";
 import { ChatMemoryBuffer } from "../../memory/ChatMemoryBuffer.js";
@@ -25,28 +26,25 @@ import type { BaseTool } from "../../types.js";
 import type { AgentWorker, Task } from "../types.js";
 import { TaskStep, TaskStepOutput } from "../types.js";
 import { addUserStepToMemory, getFunctionByName } from "../utils.js";
-import type { OpenAIToolCall } from "./types/chat.js";
 
 async function callFunction(
   tools: BaseTool[],
-  toolCall: OpenAIToolCall,
-): Promise<[ChatMessage, ToolOutput]> {
-  const id_ = toolCall.id;
-  const functionCall = toolCall.function;
-  const name = toolCall.function.name;
-  const argumentsStr = toolCall.function.arguments;
+  toolCall: ToolCallOptions["toolCall"],
+): Promise<[ChatMessage<ToolCallLLMMessageOptions>, ToolOutput]> {
+  const id = toolCall.id;
+  const name = toolCall.name;
+  const input = toolCall.input;
 
   if (Settings.debug) {
     console.log("=== Calling Function ===");
-    console.log(`Calling function: ${name} with args: ${argumentsStr}`);
+    console.log(`Calling function: ${name} with args: ${input}`);
   }
 
   const tool = getFunctionByName(tools, name);
-  const argumentDict = JSON.parse(argumentsStr);
 
   // Call tool
   // Use default error message
-  const output = await callToolWithErrorHandling(tool, argumentDict);
+  const output = await callToolWithErrorHandling(tool, input);
 
   if (Settings.debug) {
     console.log(`Got output ${output}`);
@@ -56,10 +54,12 @@ async function callFunction(
   return [
     {
       content: `${output}`,
-      role: "tool",
+      role: "user",
       options: {
-        name,
-        tool_call_id: id_,
+        toolResult: {
+          id,
+          isError: false,
+        },
       },
     },
     output,
@@ -71,7 +71,7 @@ type OpenAIAgentWorkerParams = {
   llm?: OpenAI;
   prefixMessages?: ChatMessage[];
   maxFunctionCalls?: number;
-  toolRetriever?: ObjectRetriever;
+  toolRetriever?: ObjectRetriever<BaseTool>;
 };
 
 type CallFunctionOutput = {
@@ -107,9 +107,9 @@ export class OpenAIAgentWorker
     }
     this.prefixMessages = prefixMessages || [];
 
-    if (Array.isArray(tools) && tools.length > 0 && toolRetriever) {
+    if (tools.length > 0 && toolRetriever) {
       throw new Error("Cannot specify both tools and tool_retriever");
-    } else if (Array.isArray(tools)) {
+    } else if (tools.length > 0) {
       this._getTools = async () => tools;
     } else if (toolRetriever) {
       // fixme: this won't work, type mismatch
@@ -120,7 +120,7 @@ export class OpenAIAgentWorker
     }
   }
 
-  public getAllMessages(task: Task): ChatMessage[] {
+  public getAllMessages(task: Task): ChatMessage<ToolCallLLMMessageOptions>[] {
     return [
       ...this.prefixMessages,
       ...task.memory.get(),
@@ -128,30 +128,33 @@ export class OpenAIAgentWorker
     ];
   }
 
-  public getLatestToolCalls(task: Task): OpenAIToolCall[] | null {
+  public getLatestToolCall(task: Task): ToolCallOptions["toolCall"] | null {
     const chatHistory: ChatMessage[] = task.extraState.newMemory.getAll();
 
     if (chatHistory.length === 0) {
       return null;
     }
 
-    // fixme
-    return chatHistory[chatHistory.length - 1].options?.toolCalls as any;
+    // @ts-expect-error fixme
+    return chatHistory[chatHistory.length - 1].options?.toolCall;
   }
 
   private _getLlmChatParams(
     task: Task,
-    openaiTools: BaseTool[],
+    tools: BaseTool[],
     toolChoice: ChatCompletionToolChoiceOption = "auto",
-  ): LLMChatParamsBase<OpenAIAdditionalChatOptions> {
+  ): LLMChatParamsBase<OpenAIAdditionalChatOptions, ToolCallLLMMessageOptions> {
     const llmChatParams = {
       messages: this.getAllMessages(task),
       tools: undefined as BaseTool[] | undefined,
       additionalChatOptions: {} as OpenAIAdditionalChatOptions,
-    } satisfies LLMChatParamsBase<OpenAIAdditionalChatOptions>;
+    } satisfies LLMChatParamsBase<
+      OpenAIAdditionalChatOptions,
+      ToolCallLLMMessageOptions
+    >;
 
-    if (openaiTools.length > 0) {
-      llmChatParams.tools = openaiTools;
+    if (tools.length > 0) {
+      llmChatParams.tools = tools;
       llmChatParams.additionalChatOptions.tool_choice = toolChoice;
     }
 
@@ -172,7 +175,10 @@ export class OpenAIAgentWorker
 
   private async _getStreamAiResponse(
     task: Task,
-    llmChatParams: LLMChatParamsBase<OpenAIAdditionalChatOptions>,
+    llmChatParams: LLMChatParamsBase<
+      OpenAIAdditionalChatOptions,
+      ToolCallLLMMessageOptions
+    >,
   ): Promise<StreamingAgentChatResponse | AgentChatResponse> {
     const stream = await this.llm.chat({
       stream: true,
@@ -180,7 +186,7 @@ export class OpenAIAgentWorker
     });
 
     const responseChunkStream = new ReadableStream<
-      ChatResponseChunk<OpenAIAdditionalMessageOptions>
+      ChatResponseChunk<ToolCallLLMMessageOptions>
     >({
       async start(controller) {
         for await (const chunk of stream) {
@@ -198,11 +204,11 @@ export class OpenAIAgentWorker
     }
     // check if first chunk has tool calls, if so, this is a function call
     // otherwise, it's a regular message
-    const hasToolCalls: boolean =
-      !!value.options?.toolCalls?.length &&
-      value.options?.toolCalls?.length > 0;
+    const hasToolCall: boolean = !!(
+      value.options && "toolCall" in value.options
+    );
 
-    if (hasToolCalls) {
+    if (hasToolCall) {
       return this._processMessage(task, {
         content: await pipeline(finalStream, async (iterator) => {
           let content = "";
@@ -247,7 +253,10 @@ export class OpenAIAgentWorker
   private async _getAgentResponse(
     task: Task,
     mode: ChatResponseMode,
-    llmChatParams: LLMChatParamsBase<OpenAIAdditionalChatOptions>,
+    llmChatParams: LLMChatParamsBase<
+      OpenAIAdditionalChatOptions,
+      ToolCallLLMMessageOptions
+    >,
   ): Promise<AgentChatResponse | StreamingAgentChatResponse> {
     if (mode === ChatResponseMode.WAIT) {
       const chatResponse = await this.llm.chat({
@@ -268,14 +277,8 @@ export class OpenAIAgentWorker
 
   async callFunction(
     tools: BaseTool[],
-    toolCall: OpenAIToolCall,
+    toolCall: ToolCallOptions["toolCall"],
   ): Promise<CallFunctionOutput> {
-    const functionCall = toolCall.function;
-
-    if (!functionCall) {
-      throw new Error("Invalid tool_call object");
-    }
-
     const functionMessage = await callFunction(tools, toolCall);
 
     const message = functionMessage[0];
@@ -309,18 +312,14 @@ export class OpenAIAgentWorker
   }
 
   private _shouldContinue(
-    toolCalls: OpenAIToolCall[] | null,
+    toolCall: ToolCallOptions["toolCall"] | null,
     nFunctionCalls: number,
-  ): boolean {
+  ): toolCall is ToolCallOptions["toolCall"] {
     if (nFunctionCalls > this.maxFunctionCalls) {
       return false;
     }
 
-    if (toolCalls?.length === 0) {
-      return false;
-    }
-
-    return true;
+    return !!toolCall;
   }
 
   async getTools(input: string): Promise<BaseTool[]> {
@@ -347,29 +346,25 @@ export class OpenAIAgentWorker
       llmChatParams,
     );
 
-    const latestToolCalls = this.getLatestToolCalls(task) || [];
+    const latestToolCall = this.getLatestToolCall(task) ?? null;
 
     let isDone: boolean;
-    let newSteps: TaskStep[] = [];
+    let newSteps: TaskStep[];
 
-    if (
-      !this._shouldContinue(latestToolCalls, task.extraState.nFunctionCalls)
-    ) {
+    if (!this._shouldContinue(latestToolCall, task.extraState.nFunctionCalls)) {
       isDone = true;
       newSteps = [];
     } else {
       isDone = false;
-      for (const toolCall of latestToolCalls) {
-        const { message, toolOutput } = await this.callFunction(
-          tools,
-          toolCall,
-        );
+      const { message, toolOutput } = await this.callFunction(
+        tools,
+        latestToolCall,
+      );
 
-        task.extraState.sources.push(toolOutput);
-        task.extraState.newMemory.put(message);
+      task.extraState.sources.push(toolOutput);
+      task.extraState.newMemory.put(message);
 
-        task.extraState.nFunctionCalls += 1;
-      }
+      task.extraState.nFunctionCalls += 1;
 
       newSteps = [step.getNextStep(randomUUID(), undefined)];
     }

packages/core/src/agent/react/base.ts

@@ -12,7 +12,7 @@ type ReActAgentParams = {
   prefixMessages?: ChatMessage[];
   maxInteractions?: number;
   defaultToolChoice?: string;
-  toolRetriever?: ObjectRetriever;
+  toolRetriever?: ObjectRetriever<BaseTool>;
 };
 
 /**
@@ -41,6 +41,7 @@ export class ReActAgent extends AgentRunner {
       agentWorker: stepEngine,
       memory,
       defaultToolChoice,
+      // @ts-expect-error 2322
       chatHistory: prefixMessages,
     });
   }

packages/core/src/agent/react/worker.ts

@@ -2,6 +2,7 @@ import { randomUUID } from "@llamaindex/env";
 import type { ChatMessage } from "cohere-ai/api";
 import { Settings } from "../../Settings.js";
 import { AgentChatResponse } from "../../engines/chat/index.js";
+import { getCallbackManager } from "../../internal/settings/CallbackManager.js";
 import { type ChatResponse, type LLM } from "../../llm/index.js";
 import { extractText } from "../../llm/utils.js";
 import { ChatMemoryBuffer } from "../../memory/ChatMemoryBuffer.js";
@@ -25,7 +26,7 @@ type ReActAgentWorkerParams = {
   maxInteractions?: number;
   reactChatFormatter?: ReActChatFormatter | undefined;
   outputParser?: ReActOutputParser | undefined;
-  toolRetriever?: ObjectRetriever | undefined;
+  toolRetriever?: ObjectRetriever<BaseTool> | undefined;
 };
 
 function addUserStepToReasoning(
@@ -194,6 +195,14 @@ export class ReActAgentWorker implements AgentWorker<ChatParams> {
 
     const tool = toolsDict[actionReasoningStep.action];
 
+    getCallbackManager().dispatchEvent("llm-tool-call", {
+      payload: {
+        toolCall: {
+          name: tool.metadata.name,
+          input: JSON.stringify(actionReasoningStep.actionInput),
+        },
+      },
+    });
     const toolOutput = await tool.call!(actionReasoningStep.actionInput);
 
     task.extraState.sources.push(

packages/core/src/agent/runner/base.ts

@@ -1,11 +1,12 @@
 import { randomUUID } from "@llamaindex/env";
+import type { ChatHistory } from "../../ChatHistory.js";
 import type { ChatEngineAgentParams } from "../../engines/chat/index.js";
 import {
   AgentChatResponse,
   ChatResponseMode,
   StreamingAgentChatResponse,
 } from "../../engines/chat/index.js";
-import type { ChatMessage, LLM } from "../../llm/index.js";
+import type { LLM } from "../../llm/index.js";
 import { ChatMemoryBuffer } from "../../memory/ChatMemoryBuffer.js";
 import type { BaseMemory } from "../../memory/types.js";
 import type { AgentWorker, TaskStepOutput } from "../types.js";
@@ -30,7 +31,7 @@ const validateStepFromArgs = (
 
 type AgentRunnerParams = {
   agentWorker: AgentWorker;
-  chatHistory?: ChatMessage[];
+  chatHistory?: ChatHistory;
   state?: AgentState;
   memory?: BaseMemory;
   llm?: LLM;

packages/core/src/callbacks/CallbackManager.ts

@@ -9,6 +9,7 @@ import type {
   LLMEndEvent,
   LLMStartEvent,
   LLMStreamEvent,
+  LLMToolCallEvent,
 } from "../llm/types.js";
 
 export class LlamaIndexCustomEvent<T = any> extends CustomEvent<T> {
@@ -48,6 +49,7 @@ export interface LlamaIndexEventMaps {
   stream: CustomEvent<StreamCallbackResponse>;
   "llm-start": LLMStartEvent;
   "llm-end": LLMEndEvent;
+  "llm-tool-call": LLMToolCallEvent;
   "llm-stream": LLMStreamEvent;
 }
 
@@ -203,8 +205,10 @@ export class CallbackManager implements CallbackManagerMethods {
     if (!handlers) {
       return;
     }
-    handlers.forEach((handler) =>
-      handler(LlamaIndexCustomEvent.fromEvent(event, detail)),
-    );
+    queueMicrotask(() => {
+      handlers.forEach((handler) =>
+        handler(LlamaIndexCustomEvent.fromEvent(event, detail)),
+      );
+    });
   }
 }

packages/core/src/embeddings/MultiModalEmbedding.ts

@@ -1,5 +1,10 @@
-import type { ImageType } from "../Node.js";
-import { BaseEmbedding } from "./types.js";
+import {
+  MetadataMode,
+  splitNodesByType,
+  type BaseNode,
+  type ImageType,
+} from "../Node.js";
+import { BaseEmbedding, batchEmbeddings } from "./types.js";
 
 /*
  * Base class for Multi Modal embeddings.
@@ -8,9 +13,39 @@ import { BaseEmbedding } from "./types.js";
 export abstract class MultiModalEmbedding extends BaseEmbedding {
   abstract getImageEmbedding(images: ImageType): Promise<number[]>;
 
+  /**
+   * Optionally override this method to retrieve multiple image embeddings in a single request
+   * @param texts
+   */
   async getImageEmbeddings(images: ImageType[]): Promise<number[][]> {
     return Promise.all(
       images.map((imgFilePath) => this.getImageEmbedding(imgFilePath)),
     );
   }
+
+  async transform(nodes: BaseNode[], _options?: any): Promise<BaseNode[]> {
+    const { imageNodes, textNodes } = splitNodesByType(nodes);
+
+    const embeddings = await batchEmbeddings(
+      textNodes.map((node) => node.getContent(MetadataMode.EMBED)),
+      this.getTextEmbeddings.bind(this),
+      this.embedBatchSize,
+      _options,
+    );
+    for (let i = 0; i < textNodes.length; i++) {
+      textNodes[i].embedding = embeddings[i];
+    }
+
+    const imageEmbeddings = await batchEmbeddings(
+      imageNodes.map((n) => n.image),
+      this.getImageEmbeddings.bind(this),
+      this.embedBatchSize,
+      _options,
+    );
+    for (let i = 0; i < imageNodes.length; i++) {
+      imageNodes[i].embedding = imageEmbeddings[i];
+    }
+
+    return nodes;
+  }
 }

packages/core/src/embeddings/types.ts

@@ -5,6 +5,8 @@ import { SimilarityType, similarity } from "./utils.js";
 
 const DEFAULT_EMBED_BATCH_SIZE = 10;
 
+type EmbedFunc<T> = (values: T[]) => Promise<Array<number[]>>;
+
 export abstract class BaseEmbedding implements TransformComponent {
   embedBatchSize = DEFAULT_EMBED_BATCH_SIZE;
 
@@ -45,35 +47,18 @@ export abstract class BaseEmbedding implements TransformComponent {
       logProgress?: boolean;
     },
   ): Promise<Array<number[]>> {
-    const resultEmbeddings: Array<number[]> = [];
-    const chunkSize = this.embedBatchSize;
-
-    const queue: string[] = texts;
-
-    const curBatch: string[] = [];
-
-    for (let i = 0; i < queue.length; i++) {
-      curBatch.push(queue[i]);
-      if (i == queue.length - 1 || curBatch.length == chunkSize) {
-        const embeddings = await this.getTextEmbeddings(curBatch);
-
-        resultEmbeddings.push(...embeddings);
-
-        if (options?.logProgress) {
-          console.log(`getting embedding progress: ${i} / ${queue.length}`);
-        }
-
-        curBatch.length = 0;
-      }
-    }
-
-    return resultEmbeddings;
+    return await batchEmbeddings(
+      texts,
+      this.getTextEmbeddings.bind(this),
+      this.embedBatchSize,
+      options,
+    );
   }
 
   async transform(nodes: BaseNode[], _options?: any): Promise<BaseNode[]> {
     const texts = nodes.map((node) => node.getContent(MetadataMode.EMBED));
 
-    const embeddings = await this.getTextEmbeddingsBatch(texts);
+    const embeddings = await this.getTextEmbeddingsBatch(texts, _options);
 
     for (let i = 0; i < nodes.length; i++) {
       nodes[i].embedding = embeddings[i];
@@ -82,3 +67,35 @@ export abstract class BaseEmbedding implements TransformComponent {
     return nodes;
   }
 }
+
+export async function batchEmbeddings<T>(
+  values: T[],
+  embedFunc: EmbedFunc<T>,
+  chunkSize: number,
+  options?: {
+    logProgress?: boolean;
+  },
+): Promise<Array<number[]>> {
+  const resultEmbeddings: Array<number[]> = [];
+
+  const queue: T[] = values;
+
+  const curBatch: T[] = [];
+
+  for (let i = 0; i < queue.length; i++) {
+    curBatch.push(queue[i]);
+    if (i == queue.length - 1 || curBatch.length == chunkSize) {
+      const embeddings = await embedFunc(curBatch);
+
+      resultEmbeddings.push(...embeddings);
+
+      if (options?.logProgress) {
+        console.log(`getting embedding progress: ${i} / ${queue.length}`);
+      }
+
+      curBatch.length = 0;
+    }
+  }
+
+  return resultEmbeddings;
+}

packages/core/src/engines/chat/ContextChatEngine.ts

@@ -113,10 +113,9 @@ export class ContextChatEngine extends PromptMixin implements ChatEngine {
     });
     const textOnly = extractText(message);
     const context = await this.contextGenerator.generate(textOnly);
-    const nodes = context.nodes.map((r) => r.node);
     const messages = await chatHistory.requestMessages(
       context ? [context.message] : undefined,
     );
-    return { nodes, messages };
+    return { nodes: context.nodes, messages };
   }
 }

packages/core/src/engines/chat/types.ts

@@ -1,5 +1,5 @@
 import type { ChatHistory } from "../../ChatHistory.js";
-import type { BaseNode, NodeWithScore } from "../../Node.js";
+import type { NodeWithScore } from "../../Node.js";
 import type { Response } from "../../Response.js";
 import type { ChatMessage } from "../../llm/index.js";
 import type { MessageContent } from "../../llm/types.js";
@@ -66,12 +66,12 @@ export enum ChatResponseMode {
 export class AgentChatResponse {
   response: string;
   sources: ToolOutput[];
-  sourceNodes?: BaseNode[];
+  sourceNodes?: NodeWithScore[];
 
   constructor(
     response: string,
     sources?: ToolOutput[],
-    sourceNodes?: BaseNode[],
+    sourceNodes?: NodeWithScore[],
   ) {
     this.response = response;
     this.sources = sources || [];
@@ -91,12 +91,12 @@ export class StreamingAgentChatResponse {
   response: AsyncIterable<Response>;
 
   sources: ToolOutput[];
-  sourceNodes?: BaseNode[];
+  sourceNodes?: NodeWithScore[];
 
   constructor(
     response: AsyncIterable<Response>,
     sources?: ToolOutput[],
-    sourceNodes?: BaseNode[],
+    sourceNodes?: NodeWithScore[],
   ) {
     this.response = response;
     this.sources = sources ?? [];

packages/core/src/engines/query/RouterQueryEngine.ts

@@ -1,4 +1,4 @@
-import type { BaseNode } from "../../Node.js";
+import type { NodeWithScore } from "../../Node.js";
 import { Response } from "../../Response.js";
 import type { ServiceContext } from "../../ServiceContext.js";
 import { llmFromSettingsOrContext } from "../../Settings.js";
@@ -33,7 +33,7 @@ async function combineResponses(
   }
 
   const responseStrs: string[] = [];
-  const sourceNodes: BaseNode[] = [];
+  const sourceNodes: NodeWithScore[] = [];
 
   for (const response of responses) {
     if (response?.sourceNodes) {

packages/core/src/evaluation/Correctness.ts

@@ -111,7 +111,7 @@ export class CorrectnessEvaluator extends PromptMixin implements BaseEvaluator {
 
     if (response) {
       for (const node of response.sourceNodes || []) {
-        contexts.push(node.getContent(MetadataMode.ALL));
+        contexts.push(node.node.getContent(MetadataMode.ALL));
       }
     }
 

packages/core/src/evaluation/Faithfulness.ts

@@ -137,7 +137,7 @@ export class FaithfulnessEvaluator
 
     if (response) {
       for (const node of response.sourceNodes || []) {
-        contexts.push(node.getContent(MetadataMode.ALL));
+        contexts.push(node.node.getContent(MetadataMode.ALL));
       }
     }
 

packages/core/src/evaluation/Relevancy.ts

@@ -126,7 +126,7 @@ export class RelevancyEvaluator extends PromptMixin implements BaseEvaluator {
 
     if (response) {
       for (const node of response.sourceNodes || []) {
-        contexts.push(node.getContent(MetadataMode.ALL));
+        contexts.push(node.node.getContent(MetadataMode.ALL));
       }
     }
 

packages/core/src/indices/vectorStore/index.ts

@@ -4,12 +4,7 @@ import type {
   Metadata,
   NodeWithScore,
 } from "../../Node.js";
-import {
-  ImageNode,
-  MetadataMode,
-  ObjectType,
-  splitNodesByType,
-} from "../../Node.js";
+import { ImageNode, ObjectType, splitNodesByType } from "../../Node.js";
 import type { BaseRetriever, RetrieveParams } from "../../Retriever.js";
 import type { ServiceContext } from "../../ServiceContext.js";
 import {
@@ -179,14 +174,21 @@ export class VectorStoreIndex extends BaseIndex<IndexDict> {
     nodes: BaseNode[],
     options?: { logProgress?: boolean },
   ): Promise<BaseNode[]> {
-    const texts = nodes.map((node) => node.getContent(MetadataMode.EMBED));
-    const embeddings = await this.embedModel.getTextEmbeddingsBatch(texts, {
+    const { imageNodes, textNodes } = splitNodesByType(nodes);
+    if (imageNodes.length > 0) {
+      if (!this.imageEmbedModel) {
+        throw new Error(
+          "Cannot calculate image nodes embedding without 'imageEmbedModel' set",
+        );
+      }
+      await this.imageEmbedModel.transform(imageNodes, {
+        logProgress: options?.logProgress,
+      });
+    }
+    await this.embedModel.transform(textNodes, {
       logProgress: options?.logProgress,
     });
-    return nodes.map((node, i) => {
-      node.embedding = embeddings[i];
-      return node;
-    });
+    return nodes;
   }
 
   /**
@@ -324,25 +326,15 @@ export class VectorStoreIndex extends BaseIndex<IndexDict> {
     if (!nodes || nodes.length === 0) {
       return;
     }
+    nodes = await this.getNodeEmbeddingResults(nodes, options);
     const { imageNodes, textNodes } = splitNodesByType(nodes);
     if (imageNodes.length > 0) {
       if (!this.imageVectorStore) {
         throw new Error("Cannot insert image nodes without image vector store");
       }
-      const imageNodesWithEmbedding = await this.getImageNodeEmbeddingResults(
-        imageNodes,
-        options,
-      );
-      await this.insertNodesToStore(
-        this.imageVectorStore,
-        imageNodesWithEmbedding,
-      );
+      await this.insertNodesToStore(this.imageVectorStore, imageNodes);
     }
-    const embeddingResults = await this.getNodeEmbeddingResults(
-      textNodes,
-      options,
-    );
-    await this.insertNodesToStore(this.vectorStore, embeddingResults);
+    await this.insertNodesToStore(this.vectorStore, textNodes);
     await this.indexStore.addIndexStruct(this.indexStruct);
   }
 
@@ -378,35 +370,6 @@ export class VectorStoreIndex extends BaseIndex<IndexDict> {
       await this.indexStore.addIndexStruct(this.indexStruct);
     }
   }
-
-  /**
-   * Calculates the embeddings for the given image nodes.
-   *
-   * @param nodes - An array of ImageNode objects representing the nodes for which embeddings are to be calculated.
-   * @param {Object} [options] - An optional object containing additional parameters.
-   *   @param {boolean} [options.logProgress] - A boolean indicating whether to log progress to the console (useful for debugging).
-   */
-  async getImageNodeEmbeddingResults(
-    nodes: ImageNode[],
-    options?: { logProgress?: boolean },
-  ): Promise<ImageNode[]> {
-    if (!this.imageEmbedModel) {
-      return [];
-    }
-
-    const nodesWithEmbeddings: ImageNode[] = [];
-
-    for (let i = 0; i < nodes.length; ++i) {
-      const node = nodes[i];
-      if (options?.logProgress) {
-        console.log(`Getting embedding for node ${i + 1}/${nodes.length}`);
-      }
-      node.embedding = await this.imageEmbedModel.getImageEmbedding(node.image);
-      nodesWithEmbeddings.push(node);
-    }
-
-    return nodesWithEmbeddings;
-  }
 }
 
 /**

packages/core/src/internal/utils.ts

@@ -5,3 +5,14 @@ export const isAsyncGenerator = (obj: unknown): obj is AsyncGenerator => {
 export const isGenerator = (obj: unknown): obj is Generator => {
   return obj != null && typeof obj === "object" && Symbol.iterator in obj;
 };
+
+/**
+ * Prettify an error for AI to read
+ */
+export function prettifyError(error: unknown): string {
+  if (error instanceof Error) {
+    return `Error(${error.name}): ${error.message}`;
+  } else {
+    return `${error}`;
+  }
+}

packages/core/src/llm/LLM.ts

@@ -1,373 +0,0 @@
-import { type StreamCallbackResponse } from "../callbacks/CallbackManager.js";
-
-import type { LLMOptions } from "portkey-ai";
-import { getCallbackManager } from "../internal/settings/CallbackManager.js";
-import { BaseLLM } from "./base.js";
-import type { PortkeySession } from "./portkey.js";
-import { getPortkeySession } from "./portkey.js";
-import { ReplicateSession } from "./replicate_ai.js";
-import type {
-  ChatMessage,
-  ChatResponse,
-  ChatResponseChunk,
-  LLMChatParamsNonStreaming,
-  LLMChatParamsStreaming,
-  LLMMetadata,
-  MessageType,
-} from "./types.js";
-import { extractText, wrapLLMEvent } from "./utils.js";
-
-export const ALL_AVAILABLE_LLAMADEUCE_MODELS = {
-  "Llama-2-70b-chat-old": {
-    contextWindow: 4096,
-    replicateApi:
-      "replicate/llama70b-v2-chat:e951f18578850b652510200860fc4ea62b3b16fac280f83ff32282f87bbd2e48",
-    //^ Previous 70b model. This is also actually 4 bit, although not exllama.
-  },
-  "Llama-2-70b-chat-4bit": {
-    contextWindow: 4096,
-    replicateApi:
-      "meta/llama-2-70b-chat:02e509c789964a7ea8736978a43525956ef40397be9033abf9fd2badfe68c9e3",
-    //^ Model is based off of exllama 4bit.
-  },
-  "Llama-2-13b-chat-old": {
-    contextWindow: 4096,
-    replicateApi:
-      "a16z-infra/llama13b-v2-chat:df7690f1994d94e96ad9d568eac121aecf50684a0b0963b25a41cc40061269e5",
-  },
-  //^ Last known good 13b non-quantized model. In future versions they add the SYS and INST tags themselves
-  "Llama-2-13b-chat-4bit": {
-    contextWindow: 4096,
-    replicateApi:
-      "meta/llama-2-13b-chat:f4e2de70d66816a838a89eeeb621910adffb0dd0baba3976c96980970978018d",
-  },
-  "Llama-2-7b-chat-old": {
-    contextWindow: 4096,
-    replicateApi:
-      "a16z-infra/llama7b-v2-chat:4f0a4744c7295c024a1de15e1a63c880d3da035fa1f49bfd344fe076074c8eea",
-    //^ Last (somewhat) known good 7b non-quantized model. In future versions they add the SYS and INST
-    // tags themselves
-    // https://github.com/replicate/cog-llama-template/commit/fa5ce83912cf82fc2b9c01a4e9dc9bff6f2ef137
-    // Problem is that they fix the max_new_tokens issue in the same commit. :-(
-  },
-  "Llama-2-7b-chat-4bit": {
-    contextWindow: 4096,
-    replicateApi:
-      "meta/llama-2-7b-chat:13c3cdee13ee059ab779f0291d29054dab00a47dad8261375654de5540165fb0",
-  },
-};
-
-export enum DeuceChatStrategy {
-  A16Z = "a16z",
-  META = "meta",
-  METAWBOS = "metawbos",
-  //^ This is not exactly right because SentencePiece puts the BOS and EOS token IDs in after tokenization
-  // Unfortunately any string only API won't support these properly.
-  REPLICATE4BIT = "replicate4bit",
-  //^ To satisfy Replicate's 4 bit models' requirements where they also insert some INST tags
-  REPLICATE4BITWNEWLINES = "replicate4bitwnewlines",
-  //^ Replicate's documentation recommends using newlines: https://replicate.com/blog/how-to-prompt-llama
-}
-
-/**
- * Llama2 LLM implementation
- */
-export class LlamaDeuce extends BaseLLM {
-  model: keyof typeof ALL_AVAILABLE_LLAMADEUCE_MODELS;
-  chatStrategy: DeuceChatStrategy;
-  temperature: number;
-  topP: number;
-  maxTokens?: number;
-  replicateSession: ReplicateSession;
-
-  constructor(init?: Partial<LlamaDeuce>) {
-    super();
-    this.model = init?.model ?? "Llama-2-70b-chat-4bit";
-    this.chatStrategy =
-      init?.chatStrategy ??
-      (this.model.endsWith("4bit")
-        ? DeuceChatStrategy.REPLICATE4BITWNEWLINES // With the newer Replicate models they do the system message themselves.
-        : DeuceChatStrategy.METAWBOS); // With BOS and EOS seems to work best, although they all have problems past a certain point
-    this.temperature = init?.temperature ?? 0.1; // minimum temperature is 0.01 for Replicate endpoint
-    this.topP = init?.topP ?? 1;
-    this.maxTokens =
-      init?.maxTokens ??
-      ALL_AVAILABLE_LLAMADEUCE_MODELS[this.model].contextWindow; // For Replicate, the default is 500 tokens which is too low.
-    this.replicateSession = init?.replicateSession ?? new ReplicateSession();
-  }
-
-  get metadata() {
-    return {
-      model: this.model,
-      temperature: this.temperature,
-      topP: this.topP,
-      maxTokens: this.maxTokens,
-      contextWindow: ALL_AVAILABLE_LLAMADEUCE_MODELS[this.model].contextWindow,
-      tokenizer: undefined,
-    };
-  }
-
-  mapMessagesToPrompt(messages: ChatMessage[]) {
-    if (this.chatStrategy === DeuceChatStrategy.A16Z) {
-      return this.mapMessagesToPromptA16Z(messages);
-    } else if (this.chatStrategy === DeuceChatStrategy.META) {
-      return this.mapMessagesToPromptMeta(messages);
-    } else if (this.chatStrategy === DeuceChatStrategy.METAWBOS) {
-      return this.mapMessagesToPromptMeta(messages, { withBos: true });
-    } else if (this.chatStrategy === DeuceChatStrategy.REPLICATE4BIT) {
-      return this.mapMessagesToPromptMeta(messages, {
-        replicate4Bit: true,
-        withNewlines: true,
-      });
-    } else if (this.chatStrategy === DeuceChatStrategy.REPLICATE4BITWNEWLINES) {
-      return this.mapMessagesToPromptMeta(messages, {
-        replicate4Bit: true,
-        withNewlines: true,
-      });
-    } else {
-      return this.mapMessagesToPromptMeta(messages);
-    }
-  }
-
-  mapMessagesToPromptA16Z(messages: ChatMessage[]) {
-    return {
-      prompt:
-        messages.reduce((acc, message) => {
-          return (
-            (acc && `${acc}\n\n`) +
-            `${this.mapMessageTypeA16Z(message.role)}${message.content}`
-          );
-        }, "") + "\n\nAssistant:",
-      //^ Here we're differing from A16Z by omitting the space. Generally spaces at the end of prompts decrease performance due to tokenization
-      systemPrompt: undefined,
-    };
-  }
-
-  mapMessageTypeA16Z(messageType: MessageType): string {
-    switch (messageType) {
-      case "user":
-        return "User: ";
-      case "assistant":
-        return "Assistant: ";
-      case "system":
-        return "";
-      default:
-        throw new Error("Unsupported LlamaDeuce message type");
-    }
-  }
-
-  mapMessagesToPromptMeta(
-    messages: ChatMessage[],
-    opts?: {
-      withBos?: boolean;
-      replicate4Bit?: boolean;
-      withNewlines?: boolean;
-    },
-  ) {
-    const {
-      withBos = false,
-      replicate4Bit = false,
-      withNewlines = false,
-    } = opts ?? {};
-    const DEFAULT_SYSTEM_PROMPT = `You are a helpful, respectful and honest assistant. Always answer as helpfully as possible, while being safe. Your answers should not include any harmful, unethical, racist, sexist, toxic, dangerous, or illegal content. Please ensure that your responses are socially unbiased and positive in nature.
-
-If a question does not make any sense, or is not factually coherent, explain why instead of answering something not correct. If you don't know the answer to a question, please don't share false information.`;
-
-    const B_SYS = "<<SYS>>\n";
-    const E_SYS = "\n<</SYS>>\n\n";
-    const B_INST = "[INST]";
-    const E_INST = "[/INST]";
-    const BOS = "<s>";
-    const EOS = "</s>";
-
-    if (messages.length === 0) {
-      return { prompt: "", systemPrompt: undefined };
-    }
-
-    messages = [...messages]; // so we can use shift without mutating the original array
-
-    let systemPrompt = undefined;
-    if (messages[0].role === "system") {
-      const systemMessage = messages.shift()!;
-
-      if (replicate4Bit) {
-        systemPrompt = systemMessage.content;
-      } else {
-        const systemStr = `${B_SYS}${systemMessage.content}${E_SYS}`;
-
-        // TS Bug: https://github.com/microsoft/TypeScript/issues/9998
-        // @ts-ignore
-        if (messages[0].role !== "user") {
-          throw new Error(
-            "LlamaDeuce: if there is a system message, the second message must be a user message.",
-          );
-        }
-
-        const userContent = messages[0].content;
-
-        messages[0].content = `${systemStr}${userContent}`;
-      }
-    } else {
-      if (!replicate4Bit) {
-        messages[0].content = `${B_SYS}${DEFAULT_SYSTEM_PROMPT}${E_SYS}${messages[0].content}`;
-      }
-    }
-
-    return {
-      prompt: messages.reduce((acc, message, index) => {
-        const content = extractText(message.content);
-        if (index % 2 === 0) {
-          return (
-            `${acc}${withBos ? BOS : ""}${B_INST} ${content.trim()} ${E_INST}` +
-            (withNewlines ? "\n" : "")
-          );
-        } else {
-          return (
-            `${acc} ${content.trim()}` +
-            (withNewlines ? "\n" : " ") +
-            (withBos ? EOS : "")
-          ); // Yes, the EOS comes after the space. This is not a mistake.
-        }
-      }, ""),
-      systemPrompt,
-    };
-  }
-
-  chat(
-    params: LLMChatParamsStreaming,
-  ): Promise<AsyncIterable<ChatResponseChunk>>;
-  chat(params: LLMChatParamsNonStreaming): Promise<ChatResponse>;
-  @wrapLLMEvent
-  async chat(
-    params: LLMChatParamsNonStreaming | LLMChatParamsStreaming,
-  ): Promise<ChatResponse | AsyncIterable<ChatResponseChunk>> {
-    const { messages, stream } = params;
-    const api = ALL_AVAILABLE_LLAMADEUCE_MODELS[this.model]
-      .replicateApi as `${string}/${string}:${string}`;
-
-    const { prompt, systemPrompt } = this.mapMessagesToPrompt(messages);
-
-    const replicateOptions: any = {
-      input: {
-        prompt,
-        system_prompt: systemPrompt,
-        temperature: this.temperature,
-        top_p: this.topP,
-      },
-    };
-
-    if (this.model.endsWith("4bit")) {
-      replicateOptions.input.max_new_tokens = this.maxTokens;
-    } else {
-      replicateOptions.input.max_length = this.maxTokens;
-    }
-
-    //TODO: Add streaming for this
-    if (stream) {
-      throw new Error("Streaming not supported for LlamaDeuce");
-    }
-
-    //Non-streaming
-    const response = await this.replicateSession.replicate.run(
-      api,
-      replicateOptions,
-    );
-    return {
-      raw: response,
-      message: {
-        content: (response as Array<string>).join("").trimStart(),
-        //^ We need to do this because Replicate returns a list of strings (for streaming functionality which is not exposed by the run function)
-        role: "assistant",
-      },
-    };
-  }
-}
-
-export class Portkey extends BaseLLM {
-  apiKey?: string = undefined;
-  baseURL?: string = undefined;
-  mode?: string = undefined;
-  llms?: [LLMOptions] | null = undefined;
-  session: PortkeySession;
-
-  constructor(init?: Partial<Portkey>) {
-    super();
-    this.apiKey = init?.apiKey;
-    this.baseURL = init?.baseURL;
-    this.mode = init?.mode;
-    this.llms = init?.llms;
-    this.session = getPortkeySession({
-      apiKey: this.apiKey,
-      baseURL: this.baseURL,
-      llms: this.llms,
-      mode: this.mode,
-    });
-  }
-
-  get metadata(): LLMMetadata {
-    throw new Error("metadata not implemented for Portkey");
-  }
-
-  chat(
-    params: LLMChatParamsStreaming,
-  ): Promise<AsyncIterable<ChatResponseChunk>>;
-  chat(params: LLMChatParamsNonStreaming): Promise<ChatResponse>;
-  @wrapLLMEvent
-  async chat(
-    params: LLMChatParamsNonStreaming | LLMChatParamsStreaming,
-  ): Promise<ChatResponse | AsyncIterable<ChatResponseChunk>> {
-    const { messages, stream, additionalChatOptions } = params;
-    if (stream) {
-      return this.streamChat(messages, additionalChatOptions);
-    } else {
-      const bodyParams = additionalChatOptions || {};
-      const response = await this.session.portkey.chatCompletions.create({
-        messages: messages.map((message) => ({
-          content: extractText(message.content),
-          role: message.role,
-        })),
-        ...bodyParams,
-      });
-
-      const content = response.choices[0].message?.content ?? "";
-      const role = response.choices[0].message?.role || "assistant";
-      return { raw: response, message: { content, role: role as MessageType } };
-    }
-  }
-
-  async *streamChat(
-    messages: ChatMessage[],
-    params?: Record<string, any>,
-  ): AsyncIterable<ChatResponseChunk> {
-    const chunkStream = await this.session.portkey.chatCompletions.create({
-      messages: messages.map((message) => ({
-        content: extractText(message.content),
-        role: message.role,
-      })),
-      ...params,
-      stream: true,
-    });
-
-    //Indices
-    let idx_counter: number = 0;
-    for await (const part of chunkStream) {
-      //Increment
-      part.choices[0].index = idx_counter;
-      const is_done: boolean =
-        part.choices[0].finish_reason === "stop" ? true : false;
-      //onLLMStream Callback
-
-      const stream_callback: StreamCallbackResponse = {
-        index: idx_counter,
-        isDone: is_done,
-        // token: part,
-      };
-      getCallbackManager().dispatchEvent("stream", stream_callback);
-
-      idx_counter++;
-
-      yield { raw: part, delta: part.choices[0].delta?.content ?? "" };
-    }
-    return;
-  }
-}

packages/core/src/llm/anthropic.ts

@@ -1,15 +1,30 @@
 import type { ClientOptions } from "@anthropic-ai/sdk";
 import { Anthropic as SDKAnthropic } from "@anthropic-ai/sdk";
+import type {
+  Tool,
+  ToolResultBlockParam,
+  ToolUseBlock,
+  ToolUseBlockParam,
+  ToolsBetaContentBlock,
+  ToolsBetaMessageParam,
+} from "@anthropic-ai/sdk/resources/beta/tools/messages";
+import type {
+  TextBlock,
+  TextBlockParam,
+} from "@anthropic-ai/sdk/resources/index";
+import type { MessageParam } from "@anthropic-ai/sdk/resources/messages";
 import { getEnv } from "@llamaindex/env";
+import _ from "lodash";
+import type { BaseTool } from "../types.js";
+import { ToolCallLLM } from "./base.js";
 import type {
   ChatMessage,
   ChatResponse,
   ChatResponseChunk,
   LLMChatParamsNonStreaming,
   LLMChatParamsStreaming,
-} from "llamaindex";
-import _ from "lodash";
-import { BaseLLM } from "./base.js";
+  ToolCallLLMMessageOptions,
+} from "./types.js";
 import { extractText, wrapLLMEvent } from "./utils.js";
 
 export class AnthropicSession {
@@ -81,7 +96,9 @@ const AVAILABLE_ANTHROPIC_MODELS_WITHOUT_DATE: { [key: string]: string } = {
   "claude-3-haiku": "claude-3-haiku-20240307",
 } as { [key in keyof typeof ALL_AVAILABLE_ANTHROPIC_MODELS]: string };
 
-export class Anthropic extends BaseLLM {
+export type AnthropicAdditionalChatOptions = {};
+
+export class Anthropic extends ToolCallLLM<AnthropicAdditionalChatOptions> {
   // Per completion Anthropic params
   model: keyof typeof ALL_AVAILABLE_ANTHROPIC_MODELS;
   temperature: number;
@@ -113,6 +130,10 @@ export class Anthropic extends BaseLLM {
       });
   }
 
+  get supportToolCall() {
+    return this.model.startsWith("claude-3");
+  }
+
   get metadata() {
     return {
       model: this.model,
@@ -131,30 +152,93 @@ export class Anthropic extends BaseLLM {
     return model;
   };
 
-  formatMessages(messages: ChatMessage[]) {
-    return messages.map((message) => {
+  formatMessages<Beta = false>(
+    messages: ChatMessage<ToolCallLLMMessageOptions>[],
+  ): Beta extends true ? ToolsBetaMessageParam[] : MessageParam[] {
+    return messages.map<any>((message) => {
       if (message.role !== "user" && message.role !== "assistant") {
         throw new Error("Unsupported Anthropic role");
       }
+      const options = message.options ?? {};
+      if ("toolResult" in options) {
+        const { id, isError } = options.toolResult;
+        return {
+          role: "user",
+          content: [
+            {
+              type: "tool_result",
+              is_error: isError,
+              content: [
+                {
+                  type: "text",
+                  text: extractText(message.content),
+                },
+              ],
+              tool_use_id: id,
+            },
+          ] satisfies ToolResultBlockParam[],
+        } satisfies ToolsBetaMessageParam;
+      } else if ("toolCall" in options) {
+        const aiThinkingText = extractText(message.content);
+        return {
+          role: "assistant",
+          content: [
+            // this could be empty when you call two tools in one query
+            ...(aiThinkingText.trim()
+              ? [
+                  {
+                    type: "text",
+                    text: aiThinkingText,
+                  } satisfies TextBlockParam,
+                ]
+              : []),
+            {
+              type: "tool_use",
+              id: options.toolCall.id,
+              name: options.toolCall.name,
+              input: options.toolCall.input,
+            } satisfies ToolUseBlockParam,
+          ] satisfies ToolsBetaContentBlock[],
+        } satisfies ToolsBetaMessageParam;
+      }
 
       return {
         content: extractText(message.content),
         role: message.role,
-      };
+      } satisfies MessageParam;
     });
   }
 
   chat(
-    params: LLMChatParamsStreaming,
-  ): Promise<AsyncIterable<ChatResponseChunk>>;
-  chat(params: LLMChatParamsNonStreaming): Promise<ChatResponse>;
+    params: LLMChatParamsStreaming<
+      AnthropicAdditionalChatOptions,
+      ToolCallLLMMessageOptions
+    >,
+  ): Promise<AsyncIterable<ChatResponseChunk<ToolCallLLMMessageOptions>>>;
+  chat(
+    params: LLMChatParamsNonStreaming<
+      AnthropicAdditionalChatOptions,
+      ToolCallLLMMessageOptions
+    >,
+  ): Promise<ChatResponse<ToolCallLLMMessageOptions>>;
   @wrapLLMEvent
   async chat(
-    params: LLMChatParamsNonStreaming | LLMChatParamsStreaming,
-  ): Promise<ChatResponse | AsyncIterable<ChatResponseChunk>> {
+    params:
+      | LLMChatParamsNonStreaming<
+          AnthropicAdditionalChatOptions,
+          ToolCallLLMMessageOptions
+        >
+      | LLMChatParamsStreaming<
+          AnthropicAdditionalChatOptions,
+          ToolCallLLMMessageOptions
+        >,
+  ): Promise<
+    | ChatResponse<ToolCallLLMMessageOptions>
+    | AsyncIterable<ChatResponseChunk<ToolCallLLMMessageOptions>>
+  > {
     let { messages } = params;
 
-    const { stream } = params;
+    const { stream, tools } = params;
 
     let systemPrompt: string | null = null;
 
@@ -169,34 +253,80 @@ export class Anthropic extends BaseLLM {
       messages = messages.filter((message) => message.role !== "system");
     }
 
-    //Streaming
+    // case: Streaming
     if (stream) {
+      if (tools) {
+        console.error("Tools are not supported in streaming mode");
+      }
       return this.streamChat(messages, systemPrompt);
     }
+    // case: Non-streaming
+    const anthropic = this.session.anthropic;
 
-    //Non-streaming
-    const response = await this.session.anthropic.messages.create({
-      model: this.getModelName(this.model),
-      messages: this.formatMessages(messages),
-      max_tokens: this.maxTokens ?? 4096,
-      temperature: this.temperature,
-      top_p: this.topP,
-      ...(systemPrompt && { system: systemPrompt }),
-    });
+    if (tools) {
+      const response = await anthropic.beta.tools.messages.create({
+        messages: this.formatMessages<true>(messages),
+        tools: tools.map(Anthropic.toTool),
+        model: this.getModelName(this.model),
+        temperature: this.temperature,
+        max_tokens: this.maxTokens ?? 4096,
+        top_p: this.topP,
+        ...(systemPrompt && { system: systemPrompt }),
+      });
 
-    return {
-      raw: response,
-      message: { content: response.content[0].text, role: "assistant" },
-    };
+      const toolUseBlock = response.content.find(
+        (content): content is ToolUseBlock => content.type === "tool_use",
+      );
+
+      return {
+        raw: response,
+        message: {
+          content: response.content
+            .filter((content): content is TextBlock => content.type === "text")
+            .map((content) => ({
+              type: "text",
+              text: content.text,
+            })),
+          role: "assistant",
+          options: toolUseBlock
+            ? {
+                toolCall: {
+                  id: toolUseBlock.id,
+                  name: toolUseBlock.name,
+                  input: toolUseBlock.input,
+                },
+              }
+            : {},
+        },
+      };
+    } else {
+      const response = await anthropic.messages.create({
+        model: this.getModelName(this.model),
+        messages: this.formatMessages(messages),
+        max_tokens: this.maxTokens ?? 4096,
+        temperature: this.temperature,
+        top_p: this.topP,
+        ...(systemPrompt && { system: systemPrompt }),
+      });
+
+      return {
+        raw: response,
+        message: {
+          content: response.content[0].text,
+          role: "assistant",
+          options: {},
+        },
+      };
+    }
   }
 
   protected async *streamChat(
-    messages: ChatMessage[],
+    messages: ChatMessage<ToolCallLLMMessageOptions>[],
     systemPrompt?: string | null,
-  ): AsyncIterable<ChatResponseChunk> {
+  ): AsyncIterable<ChatResponseChunk<ToolCallLLMMessageOptions>> {
     const stream = await this.session.anthropic.messages.create({
       model: this.getModelName(this.model),
-      messages: this.formatMessages(messages),
+      messages: this.formatMessages<false>(messages),
       max_tokens: this.maxTokens ?? 4096,
       temperature: this.temperature,
       top_p: this.topP,
@@ -215,8 +345,24 @@ export class Anthropic extends BaseLLM {
       yield {
         raw: part,
         delta: content,
+        options: {},
       };
     }
     return;
   }
+
+  static toTool(tool: BaseTool): Tool {
+    if (tool.metadata.parameters?.type !== "object") {
+      throw new TypeError("Tool parameters must be an object");
+    }
+    return {
+      input_schema: {
+        type: "object",
+        properties: tool.metadata.parameters.properties,
+        required: tool.metadata.parameters.required,
+      },
+      name: tool.metadata.name,
+      description: tool.metadata.description,
+    };
+  }
 }

packages/core/src/llm/base.ts

@@ -8,18 +8,13 @@ import type {
   LLMCompletionParamsNonStreaming,
   LLMCompletionParamsStreaming,
   LLMMetadata,
+  ToolCallLLMMessageOptions,
 } from "./types.js";
 import { extractText, streamConverter } from "./utils.js";
 
 export abstract class BaseLLM<
-  AdditionalChatOptions extends Record<string, unknown> = Record<
-    string,
-    unknown
-  >,
-  AdditionalMessageOptions extends Record<string, unknown> = Record<
-    string,
-    unknown
-  >,
+  AdditionalChatOptions extends object = object,
+  AdditionalMessageOptions extends object = object,
 > implements LLM<AdditionalChatOptions>
 {
   abstract metadata: LLMMetadata;
@@ -56,9 +51,21 @@ export abstract class BaseLLM<
   }
 
   abstract chat(
-    params: LLMChatParamsStreaming<AdditionalChatOptions>,
+    params: LLMChatParamsStreaming<
+      AdditionalChatOptions,
+      AdditionalMessageOptions
+    >,
   ): Promise<AsyncIterable<ChatResponseChunk>>;
   abstract chat(
-    params: LLMChatParamsNonStreaming<AdditionalChatOptions>,
+    params: LLMChatParamsNonStreaming<
+      AdditionalChatOptions,
+      AdditionalMessageOptions
+    >,
   ): Promise<ChatResponse<AdditionalMessageOptions>>;
 }
+
+export abstract class ToolCallLLM<
+  AdditionalChatOptions extends object = object,
+> extends BaseLLM<AdditionalChatOptions, ToolCallLLMMessageOptions> {
+  abstract supportToolCall: boolean;
+}

packages/core/src/llm/index.ts

@@ -1,4 +1,3 @@
-export * from "./LLM.js";
 export { Anthropic } from "./anthropic.js";
 export { FireworksLLM } from "./fireworks.js";
 export { Groq } from "./groq.js";
@@ -9,5 +8,12 @@ export {
 } from "./mistral.js";
 export { Ollama } from "./ollama.js";
 export * from "./open_ai.js";
+export { Portkey } from "./portkey.js";
+export * from "./replicate_ai.js";
+// Note: The type aliases for replicate are to simplify usage for Llama 2 (we're using replicate for Llama 2 support)
+export {
+  ReplicateChatStrategy as DeuceChatStrategy,
+  ReplicateLLM as LlamaDeuce,
+} from "./replicate_ai.js";
 export { TogetherLLM } from "./together.js";
 export * from "./types.js";

packages/core/src/llm/open_ai.ts

@@ -9,11 +9,11 @@ import { OpenAI as OrigOpenAI } from "openai";
 
 import type {
   ChatCompletionAssistantMessageParam,
-  ChatCompletionFunctionMessageParam,
   ChatCompletionMessageToolCall,
   ChatCompletionRole,
   ChatCompletionSystemMessageParam,
   ChatCompletionTool,
+  ChatCompletionToolMessageParam,
   ChatCompletionUserMessageParam,
 } from "openai/resources/chat/completions";
 import type { ChatCompletionMessageParam } from "openai/resources/index.js";
@@ -28,7 +28,7 @@ import {
   getAzureModel,
   shouldUseAzure,
 } from "./azure.js";
-import { BaseLLM } from "./base.js";
+import { ToolCallLLM } from "./base.js";
 import type {
   ChatMessage,
   ChatResponse,
@@ -37,8 +37,9 @@ import type {
   LLMChatParamsNonStreaming,
   LLMChatParamsStreaming,
   LLMMetadata,
-  MessageToolCall,
   MessageType,
+  ToolCallLLMMessageOptions,
+  ToolCallOptions,
 } from "./types.js";
 import { extractText, wrapLLMEvent } from "./utils.js";
 
@@ -143,14 +144,7 @@ export function isFunctionCallingModel(llm: LLM): llm is OpenAI {
   return isChatModel && !isOld;
 }
 
-export type OpenAIAdditionalMetadata = {
-  isFunctionCallingModel: boolean;
-};
-
-export type OpenAIAdditionalMessageOptions = {
-  functionName?: string;
-  toolCalls?: ChatCompletionMessageToolCall[];
-};
+export type OpenAIAdditionalMetadata = {};
 
 export type OpenAIAdditionalChatOptions = Omit<
   Partial<OpenAILLM.Chat.ChatCompletionCreateParams>,
@@ -164,10 +158,7 @@ export type OpenAIAdditionalChatOptions = Omit<
   | "toolChoice"
 >;
 
-export class OpenAI extends BaseLLM<
-  OpenAIAdditionalChatOptions,
-  OpenAIAdditionalMessageOptions
-> {
+export class OpenAI extends ToolCallLLM<OpenAIAdditionalChatOptions> {
   // Per completion OpenAI params
   model: keyof typeof ALL_AVAILABLE_OPENAI_MODELS | string;
   temperature: number;
@@ -238,6 +229,10 @@ export class OpenAI extends BaseLLM<
     }
   }
 
+  get supportToolCall() {
+    return isFunctionCallingModel(this);
+  }
+
   get metadata(): LLMMetadata & OpenAIAdditionalMetadata {
     const contextWindow =
       ALL_AVAILABLE_OPENAI_MODELS[
@@ -250,7 +245,6 @@ export class OpenAI extends BaseLLM<
       maxTokens: this.maxTokens,
       contextWindow,
       tokenizer: Tokenizers.CL100K_BASE,
-      isFunctionCallingModel: isFunctionCallingModel(this),
     };
   }
 
@@ -262,47 +256,46 @@ export class OpenAI extends BaseLLM<
         return "assistant";
       case "system":
         return "system";
-      case "function":
-        return "function";
-      case "tool":
-        return "tool";
       default:
         return "user";
     }
   }
 
   static toOpenAIMessage(
-    messages: ChatMessage<OpenAIAdditionalMessageOptions>[],
+    messages: ChatMessage<ToolCallLLMMessageOptions>[],
   ): ChatCompletionMessageParam[] {
     return messages.map((message) => {
-      const options: OpenAIAdditionalMessageOptions = message.options ?? {};
-      if (message.role === "user") {
+      const options = message.options ?? {};
+      if ("toolResult" in options) {
+        return {
+          tool_call_id: options.toolResult.id,
+          role: "tool",
+          content: extractText(message.content),
+        } satisfies ChatCompletionToolMessageParam;
+      } else if ("toolCall" in options) {
+        return {
+          role: "assistant",
+          content: extractText(message.content),
+          tool_calls: [
+            {
+              id: options.toolCall.id,
+              type: "function",
+              function: {
+                name: options.toolCall.name,
+                arguments:
+                  typeof options.toolCall.input === "string"
+                    ? options.toolCall.input
+                    : JSON.stringify(options.toolCall.input),
+              },
+            },
+          ],
+        } satisfies ChatCompletionAssistantMessageParam;
+      } else if (message.role === "user") {
         return {
           role: "user",
           content: message.content,
         } satisfies ChatCompletionUserMessageParam;
       }
-      if (typeof message.content !== "string") {
-        console.warn("Message content is not a string");
-      }
-      if (message.role === "function") {
-        if (!options.functionName) {
-          console.warn("Function message does not have a name");
-        }
-        return {
-          role: "function",
-          name: options.functionName ?? "UNKNOWN",
-          content: extractText(message.content),
-          // todo: remove this since this is deprecated in the OpenAI API
-        } satisfies ChatCompletionFunctionMessageParam;
-      }
-      if (message.role === "assistant") {
-        return {
-          role: "assistant",
-          content: extractText(message.content),
-          tool_calls: options.toolCalls,
-        } satisfies ChatCompletionAssistantMessageParam;
-      }
 
       const response:
         | ChatCompletionSystemMessageParam
@@ -312,27 +305,38 @@ export class OpenAI extends BaseLLM<
         role: OpenAI.toOpenAIRole(message.role) as never,
         // fixme: should not extract text, but assert content is string
         content: extractText(message.content),
-        ...options,
       };
       return response;
     });
   }
 
   chat(
-    params: LLMChatParamsStreaming<OpenAIAdditionalChatOptions>,
-  ): Promise<AsyncIterable<ChatResponseChunk<OpenAIAdditionalMessageOptions>>>;
+    params: LLMChatParamsStreaming<
+      OpenAIAdditionalChatOptions,
+      ToolCallLLMMessageOptions
+    >,
+  ): Promise<AsyncIterable<ChatResponseChunk<ToolCallLLMMessageOptions>>>;
   chat(
-    params: LLMChatParamsNonStreaming<OpenAIAdditionalChatOptions>,
-  ): Promise<ChatResponse<OpenAIAdditionalMessageOptions>>;
+    params: LLMChatParamsNonStreaming<
+      OpenAIAdditionalChatOptions,
+      ToolCallLLMMessageOptions
+    >,
+  ): Promise<ChatResponse<ToolCallLLMMessageOptions>>;
   @wrapEventCaller
   @wrapLLMEvent
   async chat(
     params:
-      | LLMChatParamsNonStreaming<OpenAIAdditionalChatOptions>
-      | LLMChatParamsStreaming<OpenAIAdditionalChatOptions>,
+      | LLMChatParamsNonStreaming<
+          OpenAIAdditionalChatOptions,
+          ToolCallLLMMessageOptions
+        >
+      | LLMChatParamsStreaming<
+          OpenAIAdditionalChatOptions,
+          ToolCallLLMMessageOptions
+        >,
   ): Promise<
-    | ChatResponse<OpenAIAdditionalMessageOptions>
-    | AsyncIterable<ChatResponseChunk<OpenAIAdditionalMessageOptions>>
+    | ChatResponse<ToolCallLLMMessageOptions>
+    | AsyncIterable<ChatResponseChunk<ToolCallLLMMessageOptions>>
   > {
     const { messages, stream, tools, additionalChatOptions } = params;
     const baseRequestParams: OpenAILLM.Chat.ChatCompletionCreateParams = {
@@ -358,18 +362,21 @@ export class OpenAI extends BaseLLM<
 
     const content = response.choices[0].message?.content ?? "";
 
-    const options: OpenAIAdditionalMessageOptions = {};
-
-    if (response.choices[0].message?.tool_calls) {
-      options.toolCalls = response.choices[0].message.tool_calls;
-    }
-
     return {
       raw: response,
       message: {
         content,
         role: response.choices[0].message.role,
-        options,
+        options: response.choices[0].message?.tool_calls
+          ? {
+              toolCall: {
+                id: response.choices[0].message.tool_calls[0].id,
+                name: response.choices[0].message.tool_calls[0].function.name,
+                input:
+                  response.choices[0].message.tool_calls[0].function.arguments,
+              },
+            }
+          : {},
       },
     };
   }
@@ -377,7 +384,7 @@ export class OpenAI extends BaseLLM<
   @wrapEventCaller
   protected async *streamChat(
     baseRequestParams: OpenAILLM.Chat.ChatCompletionCreateParams,
-  ): AsyncIterable<ChatResponseChunk<OpenAIAdditionalMessageOptions>> {
+  ): AsyncIterable<ChatResponseChunk<ToolCallLLMMessageOptions>> {
     const stream: AsyncIterable<OpenAILLM.Chat.ChatCompletionChunk> =
       await this.session.openai.chat.completions.create({
         ...baseRequestParams,
@@ -387,13 +394,26 @@ export class OpenAI extends BaseLLM<
     // TODO: add callback to streamConverter and use streamConverter here
     //Indices
     let idxCounter: number = 0;
-    const toolCalls: MessageToolCall[] = [];
+    let toolCallOptions: ToolCallOptions | null = null;
     for await (const part of stream) {
       if (!part.choices.length) continue;
       const choice = part.choices[0];
       // skip parts that don't have any content
       if (!(choice.delta.content || choice.delta.tool_calls)) continue;
-      updateToolCalls(toolCalls, choice.delta.tool_calls);
+      if (choice.delta.tool_calls?.[0].id) {
+        toolCallOptions = {
+          toolCall: {
+            name: choice.delta.tool_calls[0].function!.name!,
+            id: choice.delta.tool_calls[0].id,
+            input: choice.delta.tool_calls[0].function!.arguments!,
+          },
+        };
+      } else {
+        if (choice.delta.tool_calls?.[0].function?.arguments) {
+          toolCallOptions!.toolCall.input +=
+            choice.delta.tool_calls[0].function.arguments;
+        }
+      }
 
       const isDone: boolean = choice.finish_reason !== null;
 
@@ -405,8 +425,7 @@ export class OpenAI extends BaseLLM<
 
       yield {
         raw: part,
-        // add tool calls to final chunk
-        options: toolCalls.length > 0 ? { toolCalls: toolCalls } : {},
+        options: toolCallOptions ? toolCallOptions : {},
         delta: choice.delta.content ?? "",
       };
     }
@@ -424,34 +443,3 @@ export class OpenAI extends BaseLLM<
     };
   }
 }
-
-function updateToolCalls(
-  toolCalls: MessageToolCall[],
-  toolCallDeltas?: OpenAILLM.Chat.Completions.ChatCompletionChunk.Choice.Delta.ToolCall[],
-) {
-  function augmentToolCall(
-    toolCall?: MessageToolCall,
-    toolCallDelta?: OpenAILLM.Chat.Completions.ChatCompletionChunk.Choice.Delta.ToolCall,
-  ) {
-    toolCall =
-      toolCall ??
-      ({ function: { name: "", arguments: "" } } as MessageToolCall);
-    toolCall.id = toolCall.id ?? toolCallDelta?.id;
-    toolCall.type = toolCall.type ?? toolCallDelta?.type;
-    if (toolCallDelta?.function?.arguments) {
-      toolCall.function.arguments += toolCallDelta.function.arguments;
-    }
-    if (toolCallDelta?.function?.name) {
-      toolCall.function.name += toolCallDelta.function.name;
-    }
-    return toolCall;
-  }
-  if (toolCallDeltas) {
-    toolCallDeltas?.forEach((toolCall) => {
-      toolCalls[toolCall.index] = augmentToolCall(
-        toolCalls[toolCall.index],
-        toolCall,
-      );
-    });
-  }
-}

packages/core/src/llm/portkey.ts

@@ -1,7 +1,20 @@
 import { getEnv } from "@llamaindex/env";
 import _ from "lodash";
 import type { LLMOptions } from "portkey-ai";
-import { Portkey } from "portkey-ai";
+import { Portkey as OrigPortKey } from "portkey-ai";
+import { type StreamCallbackResponse } from "../callbacks/CallbackManager.js";
+import { getCallbackManager } from "../internal/settings/CallbackManager.js";
+import { BaseLLM } from "./base.js";
+import type {
+  ChatMessage,
+  ChatResponse,
+  ChatResponseChunk,
+  LLMChatParamsNonStreaming,
+  LLMChatParamsStreaming,
+  LLMMetadata,
+  MessageType,
+} from "./types.js";
+import { extractText, wrapLLMEvent } from "./utils.js";
 
 interface PortkeyOptions {
   apiKey?: string;
@@ -11,7 +24,7 @@ interface PortkeyOptions {
 }
 
 export class PortkeySession {
-  portkey: Portkey;
+  portkey: OrigPortKey;
 
   constructor(options: PortkeyOptions = {}) {
     if (!options.apiKey) {
@@ -22,13 +35,13 @@ export class PortkeySession {
       options.baseURL = getEnv("PORTKEY_BASE_URL") ?? "https://api.portkey.ai";
     }
 
-    this.portkey = new Portkey({});
+    this.portkey = new OrigPortKey({});
     this.portkey.llms = [{}];
     if (!options.apiKey) {
       throw new Error("Set Portkey ApiKey in PORTKEY_API_KEY env variable");
     }
 
-    this.portkey = new Portkey(options);
+    this.portkey = new OrigPortKey(options);
   }
 }
 
@@ -54,3 +67,92 @@ export function getPortkeySession(options: PortkeyOptions = {}) {
   }
   return session;
 }
+
+export class Portkey extends BaseLLM {
+  apiKey?: string = undefined;
+  baseURL?: string = undefined;
+  mode?: string = undefined;
+  llms?: [LLMOptions] | null = undefined;
+  session: PortkeySession;
+
+  constructor(init?: Partial<Portkey>) {
+    super();
+    this.apiKey = init?.apiKey;
+    this.baseURL = init?.baseURL;
+    this.mode = init?.mode;
+    this.llms = init?.llms;
+    this.session = getPortkeySession({
+      apiKey: this.apiKey,
+      baseURL: this.baseURL,
+      llms: this.llms,
+      mode: this.mode,
+    });
+  }
+
+  get metadata(): LLMMetadata {
+    throw new Error("metadata not implemented for Portkey");
+  }
+
+  chat(
+    params: LLMChatParamsStreaming,
+  ): Promise<AsyncIterable<ChatResponseChunk>>;
+  chat(params: LLMChatParamsNonStreaming): Promise<ChatResponse>;
+  @wrapLLMEvent
+  async chat(
+    params: LLMChatParamsNonStreaming | LLMChatParamsStreaming,
+  ): Promise<ChatResponse | AsyncIterable<ChatResponseChunk>> {
+    const { messages, stream, additionalChatOptions } = params;
+    if (stream) {
+      return this.streamChat(messages, additionalChatOptions);
+    } else {
+      const bodyParams = additionalChatOptions || {};
+      const response = await this.session.portkey.chatCompletions.create({
+        messages: messages.map((message) => ({
+          content: extractText(message.content),
+          role: message.role,
+        })),
+        ...bodyParams,
+      });
+
+      const content = response.choices[0].message?.content ?? "";
+      const role = response.choices[0].message?.role || "assistant";
+      return { raw: response, message: { content, role: role as MessageType } };
+    }
+  }
+
+  async *streamChat(
+    messages: ChatMessage[],
+    params?: Record<string, any>,
+  ): AsyncIterable<ChatResponseChunk> {
+    const chunkStream = await this.session.portkey.chatCompletions.create({
+      messages: messages.map((message) => ({
+        content: extractText(message.content),
+        role: message.role,
+      })),
+      ...params,
+      stream: true,
+    });
+
+    //Indices
+    let idx_counter: number = 0;
+    for await (const part of chunkStream) {
+      //Increment
+      part.choices[0].index = idx_counter;
+      const is_done: boolean =
+        part.choices[0].finish_reason === "stop" ? true : false;
+      //onLLMStream Callback
+
+      const stream_callback: StreamCallbackResponse = {
+        index: idx_counter,
+        isDone: is_done,
+        // token: part,
+      };
+      getCallbackManager().dispatchEvent("stream", stream_callback);
+
+      idx_counter++;
+
+      yield { raw: part, delta: part.choices[0].delta?.content ?? "" };
+    }
+    return;
+  }
+}

packages/core/src/llm/replicate_ai.ts

@@ -1,5 +1,15 @@
 import { getEnv } from "@llamaindex/env";
 import Replicate from "replicate";
+import { BaseLLM } from "./base.js";
+import type {
+  ChatMessage,
+  ChatResponse,
+  ChatResponseChunk,
+  LLMChatParamsNonStreaming,
+  LLMChatParamsStreaming,
+  MessageType,
+} from "./types.js";
+import { extractText, wrapLLMEvent } from "./utils.js";
 
 export class ReplicateSession {
   replicateKey: string | null = null;
@@ -20,12 +30,271 @@ export class ReplicateSession {
   }
 }
 
-let defaultReplicateSession: ReplicateSession | null = null;
+export const ALL_AVAILABLE_REPLICATE_MODELS = {
+  // TODO: add more models from replicate
+  "Llama-2-70b-chat-old": {
+    contextWindow: 4096,
+    replicateApi:
+      "replicate/llama70b-v2-chat:e951f18578850b652510200860fc4ea62b3b16fac280f83ff32282f87bbd2e48",
+    //^ Previous 70b model. This is also actually 4 bit, although not exllama.
+  },
+  "Llama-2-70b-chat-4bit": {
+    contextWindow: 4096,
+    replicateApi:
+      "meta/llama-2-70b-chat:02e509c789964a7ea8736978a43525956ef40397be9033abf9fd2badfe68c9e3",
+    //^ Model is based off of exllama 4bit.
+  },
+  "Llama-2-13b-chat-old": {
+    contextWindow: 4096,
+    replicateApi:
+      "a16z-infra/llama13b-v2-chat:df7690f1994d94e96ad9d568eac121aecf50684a0b0963b25a41cc40061269e5",
+  },
+  //^ Last known good 13b non-quantized model. In future versions they add the SYS and INST tags themselves
+  "Llama-2-13b-chat-4bit": {
+    contextWindow: 4096,
+    replicateApi:
+      "meta/llama-2-13b-chat:f4e2de70d66816a838a89eeeb621910adffb0dd0baba3976c96980970978018d",
+  },
+  "Llama-2-7b-chat-old": {
+    contextWindow: 4096,
+    replicateApi:
+      "a16z-infra/llama7b-v2-chat:4f0a4744c7295c024a1de15e1a63c880d3da035fa1f49bfd344fe076074c8eea",
+    //^ Last (somewhat) known good 7b non-quantized model. In future versions they add the SYS and INST
+    // tags themselves
+    // https://github.com/replicate/cog-llama-template/commit/fa5ce83912cf82fc2b9c01a4e9dc9bff6f2ef137
+    // Problem is that they fix the max_new_tokens issue in the same commit. :-(
+  },
+  "Llama-2-7b-chat-4bit": {
+    contextWindow: 4096,
+    replicateApi:
+      "meta/llama-2-7b-chat:13c3cdee13ee059ab779f0291d29054dab00a47dad8261375654de5540165fb0",
+  },
+};
 
-export function getReplicateSession(replicateKey: string | null = null) {
-  if (!defaultReplicateSession) {
-    defaultReplicateSession = new ReplicateSession(replicateKey);
+export enum ReplicateChatStrategy {
+  A16Z = "a16z",
+  META = "meta",
+  METAWBOS = "metawbos",
+  //^ This is not exactly right because SentencePiece puts the BOS and EOS token IDs in after tokenization
+  // Unfortunately any string only API won't support these properly.
+  REPLICATE4BIT = "replicate4bit",
+  //^ To satisfy Replicate's 4 bit models' requirements where they also insert some INST tags
+  REPLICATE4BITWNEWLINES = "replicate4bitwnewlines",
+  //^ Replicate's documentation recommends using newlines: https://replicate.com/blog/how-to-prompt-llama
+}
+
+/**
+ * Replicate LLM implementation used
+ */
+export class ReplicateLLM extends BaseLLM {
+  model: keyof typeof ALL_AVAILABLE_REPLICATE_MODELS;
+  chatStrategy: ReplicateChatStrategy;
+  temperature: number;
+  topP: number;
+  maxTokens?: number;
+  replicateSession: ReplicateSession;
+
+  constructor(init?: Partial<ReplicateLLM>) {
+    super();
+    this.model = init?.model ?? "Llama-2-70b-chat-4bit";
+    this.chatStrategy =
+      init?.chatStrategy ??
+      (this.model.endsWith("4bit")
+        ? ReplicateChatStrategy.REPLICATE4BITWNEWLINES // With the newer Replicate models they do the system message themselves.
+        : ReplicateChatStrategy.METAWBOS); // With BOS and EOS seems to work best, although they all have problems past a certain point
+    this.temperature = init?.temperature ?? 0.1; // minimum temperature is 0.01 for Replicate endpoint
+    this.topP = init?.topP ?? 1;
+    this.maxTokens =
+      init?.maxTokens ??
+      ALL_AVAILABLE_REPLICATE_MODELS[this.model].contextWindow; // For Replicate, the default is 500 tokens which is too low.
+    this.replicateSession = init?.replicateSession ?? new ReplicateSession();
   }
 
-  return defaultReplicateSession;
+  get metadata() {
+    return {
+      model: this.model,
+      temperature: this.temperature,
+      topP: this.topP,
+      maxTokens: this.maxTokens,
+      contextWindow: ALL_AVAILABLE_REPLICATE_MODELS[this.model].contextWindow,
+      tokenizer: undefined,
+    };
+  }
+
+  mapMessagesToPrompt(messages: ChatMessage[]) {
+    if (this.chatStrategy === ReplicateChatStrategy.A16Z) {
+      return this.mapMessagesToPromptA16Z(messages);
+    } else if (this.chatStrategy === ReplicateChatStrategy.META) {
+      return this.mapMessagesToPromptMeta(messages);
+    } else if (this.chatStrategy === ReplicateChatStrategy.METAWBOS) {
+      return this.mapMessagesToPromptMeta(messages, { withBos: true });
+    } else if (this.chatStrategy === ReplicateChatStrategy.REPLICATE4BIT) {
+      return this.mapMessagesToPromptMeta(messages, {
+        replicate4Bit: true,
+        withNewlines: true,
+      });
+    } else if (
+      this.chatStrategy === ReplicateChatStrategy.REPLICATE4BITWNEWLINES
+    ) {
+      return this.mapMessagesToPromptMeta(messages, {
+        replicate4Bit: true,
+        withNewlines: true,
+      });
+    } else {
+      return this.mapMessagesToPromptMeta(messages);
+    }
+  }
+
+  mapMessagesToPromptA16Z(messages: ChatMessage[]) {
+    return {
+      prompt:
+        messages.reduce((acc, message) => {
+          return (
+            (acc && `${acc}\n\n`) +
+            `${this.mapMessageTypeA16Z(message.role)}${message.content}`
+          );
+        }, "") + "\n\nAssistant:",
+      //^ Here we're differing from A16Z by omitting the space. Generally spaces at the end of prompts decrease performance due to tokenization
+      systemPrompt: undefined,
+    };
+  }
+
+  mapMessageTypeA16Z(messageType: MessageType): string {
+    switch (messageType) {
+      case "user":
+        return "User: ";
+      case "assistant":
+        return "Assistant: ";
+      case "system":
+        return "";
+      default:
+        throw new Error("Unsupported ReplicateLLM message type");
+    }
+  }
+
+  mapMessagesToPromptMeta(
+    messages: ChatMessage[],
+    opts?: {
+      withBos?: boolean;
+      replicate4Bit?: boolean;
+      withNewlines?: boolean;
+    },
+  ) {
+    const {
+      withBos = false,
+      replicate4Bit = false,
+      withNewlines = false,
+    } = opts ?? {};
+    const DEFAULT_SYSTEM_PROMPT = `You are a helpful, respectful and honest assistant. Always answer as helpfully as possible, while being safe. Your answers should not include any harmful, unethical, racist, sexist, toxic, dangerous, or illegal content. Please ensure that your responses are socially unbiased and positive in nature.
+
+If a question does not make any sense, or is not factually coherent, explain why instead of answering something not correct. If you don't know the answer to a question, please don't share false information.`;
+
+    const B_SYS = "<<SYS>>\n";
+    const E_SYS = "\n<</SYS>>\n\n";
+    const B_INST = "[INST]";
+    const E_INST = "[/INST]";
+    const BOS = "<s>";
+    const EOS = "</s>";
+
+    if (messages.length === 0) {
+      return { prompt: "", systemPrompt: undefined };
+    }
+
+    messages = [...messages]; // so we can use shift without mutating the original array
+
+    let systemPrompt = undefined;
+    if (messages[0].role === "system") {
+      const systemMessage = messages.shift()!;
+
+      if (replicate4Bit) {
+        systemPrompt = systemMessage.content;
+      } else {
+        const systemStr = `${B_SYS}${systemMessage.content}${E_SYS}`;
+
+        // TS Bug: https://github.com/microsoft/TypeScript/issues/9998
+        // @ts-ignore
+        if (messages[0].role !== "user") {
+          throw new Error(
+            "ReplicateLLM: if there is a system message, the second message must be a user message.",
+          );
+        }
+
+        const userContent = messages[0].content;
+
+        messages[0].content = `${systemStr}${userContent}`;
+      }
+    } else {
+      if (!replicate4Bit) {
+        messages[0].content = `${B_SYS}${DEFAULT_SYSTEM_PROMPT}${E_SYS}${messages[0].content}`;
+      }
+    }
+
+    return {
+      prompt: messages.reduce((acc, message, index) => {
+        const content = extractText(message.content);
+        if (index % 2 === 0) {
+          return (
+            `${acc}${withBos ? BOS : ""}${B_INST} ${content.trim()} ${E_INST}` +
+            (withNewlines ? "\n" : "")
+          );
+        } else {
+          return (
+            `${acc} ${content.trim()}` +
+            (withNewlines ? "\n" : " ") +
+            (withBos ? EOS : "")
+          ); // Yes, the EOS comes after the space. This is not a mistake.
+        }
+      }, ""),
+      systemPrompt,
+    };
+  }
+
+  chat(
+    params: LLMChatParamsStreaming,
+  ): Promise<AsyncIterable<ChatResponseChunk>>;
+  chat(params: LLMChatParamsNonStreaming): Promise<ChatResponse>;
+  @wrapLLMEvent
+  async chat(
+    params: LLMChatParamsNonStreaming | LLMChatParamsStreaming,
+  ): Promise<ChatResponse | AsyncIterable<ChatResponseChunk>> {
+    const { messages, stream } = params;
+    const api = ALL_AVAILABLE_REPLICATE_MODELS[this.model]
+      .replicateApi as `${string}/${string}:${string}`;
+
+    const { prompt, systemPrompt } = this.mapMessagesToPrompt(messages);
+
+    const replicateOptions: any = {
+      input: {
+        prompt,
+        system_prompt: systemPrompt,
+        temperature: this.temperature,
+        top_p: this.topP,
+      },
+    };
+
+    if (this.model.endsWith("4bit")) {
+      replicateOptions.input.max_new_tokens = this.maxTokens;
+    } else {
+      replicateOptions.input.max_length = this.maxTokens;
+    }
+
+    //TODO: Add streaming for this
+    if (stream) {
+      throw new Error("Streaming not supported for ReplicateLLM");
+    }
+
+    //Non-streaming
+    const response = await this.replicateSession.replicate.run(
+      api,
+      replicateOptions,
+    );
+    return {
+      raw: response,
+      message: {
+        content: (response as Array<string>).join("").trimStart(),
+        //^ We need to do this because Replicate returns a list of strings (for streaming functionality which is not exposed by the run function)
+        role: "assistant",
+      },
+    };
+  }
 }

packages/core/src/llm/types.ts

@@ -1,54 +1,42 @@
 import type { Tokenizers } from "../GlobalsHelper.js";
 import type { BaseTool, UUID } from "../types.js";
 
-type LLMBaseEvent<
-  Type extends string,
-  Payload extends Record<string, unknown>,
-> = CustomEvent<{
+type LLMBaseEvent<Payload extends Record<string, unknown>> = CustomEvent<{
   payload: Payload;
 }>;
 
-export type LLMStartEvent = LLMBaseEvent<
-  "llm-start",
-  {
-    id: UUID;
-    messages: ChatMessage[];
-  }
->;
-export type LLMEndEvent = LLMBaseEvent<
-  "llm-end",
-  {
-    id: UUID;
-    response: ChatResponse;
-  }
->;
-export type LLMStreamEvent = LLMBaseEvent<
-  "llm-stream",
-  {
-    id: UUID;
-    chunk: ChatResponseChunk;
-  }
->;
+export type LLMStartEvent = LLMBaseEvent<{
+  id: UUID;
+  messages: ChatMessage[];
+}>;
+export type LLMToolCallEvent = LLMBaseEvent<{
+  // fixme: id is missing in the context
+  // id: UUID;
+  toolCall: Omit<ToolCallOptions["toolCall"], "id">;
+}>;
+export type LLMEndEvent = LLMBaseEvent<{
+  id: UUID;
+  response: ChatResponse;
+}>;
+export type LLMStreamEvent = LLMBaseEvent<{
+  id: UUID;
+  chunk: ChatResponseChunk;
+}>;
 
 /**
  * @internal
  */
 export interface LLMChat<
-  AdditionalChatOptions extends Record<string, unknown> = Record<
-    string,
-    unknown
-  >,
-  AdditionalMessageOptions extends Record<string, unknown> = Record<
-    string,
-    unknown
-  >,
+  AdditionalChatOptions extends object = object,
+  AdditionalMessageOptions extends object = object,
 > {
   chat(
     params:
       | LLMChatParamsStreaming<AdditionalChatOptions>
       | LLMChatParamsNonStreaming<AdditionalChatOptions>,
   ): Promise<
-    ChatResponse<AdditionalMessageOptions> | AsyncIterable<ChatResponseChunk>
+    | ChatResponse<AdditionalMessageOptions>
+    | AsyncIterable<ChatResponseChunk<AdditionalMessageOptions>>
   >;
 }
 
@@ -56,24 +44,24 @@ export interface LLMChat<
  * Unified language model interface
  */
 export interface LLM<
-  AdditionalChatOptions extends Record<string, unknown> = Record<
-    string,
-    unknown
-  >,
-  AdditionalMessageOptions extends Record<string, unknown> = Record<
-    string,
-    unknown
-  >,
+  AdditionalChatOptions extends object = object,
+  AdditionalMessageOptions extends object = object,
 > extends LLMChat<AdditionalChatOptions> {
   metadata: LLMMetadata;
   /**
    * Get a chat response from the LLM
    */
   chat(
-    params: LLMChatParamsStreaming<AdditionalChatOptions>,
+    params: LLMChatParamsStreaming<
+      AdditionalChatOptions,
+      AdditionalMessageOptions
+    >,
   ): Promise<AsyncIterable<ChatResponseChunk>>;
   chat(
-    params: LLMChatParamsNonStreaming<AdditionalChatOptions>,
+    params: LLMChatParamsNonStreaming<
+      AdditionalChatOptions,
+      AdditionalMessageOptions
+    >,
   ): Promise<ChatResponse<AdditionalMessageOptions>>;
 
   /**
@@ -87,48 +75,16 @@ export interface LLM<
   ): Promise<CompletionResponse>;
 }
 
-// todo: remove "generic", "function", "memory";
-export type MessageType =
-  | "user"
-  | "assistant"
-  | "system"
-  /**
-   * @deprecated
-   */
-  | "generic"
-  /**
-   * @deprecated
-   */
-  | "function"
-  /**
-   * @deprecated
-   */
-  | "memory"
-  | "tool";
+export type MessageType = "user" | "assistant" | "system" | "memory";
 
-export type ChatMessage<
-  AdditionalMessageOptions extends Record<string, unknown> = Record<
-    string,
-    unknown
-  >,
-> =
-  AdditionalMessageOptions extends Record<string, unknown>
-    ? {
-        content: MessageContent;
-        role: MessageType;
-        options?: AdditionalMessageOptions;
-      }
-    : {
-        content: MessageContent;
-        role: MessageType;
-        options: AdditionalMessageOptions;
-      };
+export type ChatMessage<AdditionalMessageOptions extends object = object> = {
+  content: MessageContent;
+  role: MessageType;
+  options?: undefined | AdditionalMessageOptions;
+};
 
 export interface ChatResponse<
-  AdditionalMessageOptions extends Record<string, unknown> = Record<
-    string,
-    unknown
-  >,
+  AdditionalMessageOptions extends object = object,
 > {
   message: ChatMessage<AdditionalMessageOptions>;
   /**
@@ -140,22 +96,12 @@ export interface ChatResponse<
 }
 
 export type ChatResponseChunk<
-  AdditionalMessageOptions extends Record<string, unknown> = Record<
-    string,
-    unknown
-  >,
-> =
-  AdditionalMessageOptions extends Record<string, unknown>
-    ? {
-        raw: object | null;
-        delta: string;
-        options?: AdditionalMessageOptions;
-      }
-    : {
-        raw: object | null;
-        delta: string;
-        options: AdditionalMessageOptions;
-      };
+  AdditionalMessageOptions extends object = object,
+> = {
+  raw: object | null;
+  delta: string;
+  options?: undefined | AdditionalMessageOptions;
+};
 
 export interface CompletionResponse {
   text: string;
@@ -177,36 +123,25 @@ export type LLMMetadata = {
 };
 
 export interface LLMChatParamsBase<
-  AdditionalChatOptions extends Record<string, unknown> = Record<
-    string,
-    unknown
-  >,
-  AdditionalMessageOptions extends Record<string, unknown> = Record<
-    string,
-    unknown
-  >,
+  AdditionalChatOptions extends object = object,
+  AdditionalMessageOptions extends object = object,
 > {
   messages: ChatMessage<AdditionalMessageOptions>[];
   additionalChatOptions?: AdditionalChatOptions;
   tools?: BaseTool[];
-  additionalKwargs?: Record<string, unknown>;
 }
 
 export interface LLMChatParamsStreaming<
-  AdditionalChatOptions extends Record<string, unknown> = Record<
-    string,
-    unknown
-  >,
-> extends LLMChatParamsBase<AdditionalChatOptions> {
+  AdditionalChatOptions extends object = object,
+  AdditionalMessageOptions extends object = object,
+> extends LLMChatParamsBase<AdditionalChatOptions, AdditionalMessageOptions> {
   stream: true;
 }
 
 export interface LLMChatParamsNonStreaming<
-  AdditionalChatOptions extends Record<string, unknown> = Record<
-    string,
-    unknown
-  >,
-> extends LLMChatParamsBase<AdditionalChatOptions> {
+  AdditionalChatOptions extends object = object,
+  AdditionalMessageOptions extends object = object,
+> extends LLMChatParamsBase<AdditionalChatOptions, AdditionalMessageOptions> {
   stream?: false;
 }
 
@@ -242,13 +177,29 @@ export type MessageContentDetail =
  */
 export type MessageContent = string | MessageContentDetail[];
 
-interface Function {
-  arguments: string;
+export type ToolCall = {
   name: string;
-}
-
-export interface MessageToolCall {
+  // for now, claude-3-opus will give object, gpt-3/4 will give string
+  // todo: unify this to always be an object
+  input: unknown;
   id: string;
-  function: Function;
-  type: "function";
-}
+};
+
+export type ToolResult = {
+  id: string;
+  result: string;
+  isError: boolean;
+};
+
+export type ToolCallOptions = {
+  toolCall: ToolCall;
+};
+
+export type ToolResultOptions = {
+  toolResult: ToolResult;
+};
+
+export type ToolCallLLMMessageOptions =
+  | ToolResultOptions
+  | ToolCallOptions
+  | {};

packages/core/src/llm/utils.ts

@@ -61,14 +61,24 @@ export function extractText(message: MessageContent): string {
 /**
  * @internal
  */
-export function wrapLLMEvent(
-  originalMethod: LLMChat["chat"],
+export function wrapLLMEvent<
+  AdditionalChatOptions extends object = object,
+  AdditionalMessageOptions extends object = object,
+>(
+  originalMethod: LLMChat<
+    AdditionalChatOptions,
+    AdditionalMessageOptions
+  >["chat"],
   _context: ClassMethodDecoratorContext,
 ) {
   return async function withLLMEvent(
-    this: LLM,
-    ...params: Parameters<LLMChat["chat"]>
-  ): ReturnType<LLMChat["chat"]> {
+    this: LLM<AdditionalChatOptions, AdditionalMessageOptions>,
+    ...params: Parameters<
+      LLMChat<AdditionalChatOptions, AdditionalMessageOptions>["chat"]
+    >
+  ): ReturnType<
+    LLMChat<AdditionalChatOptions, AdditionalMessageOptions>["chat"]
+  > {
     const id = randomUUID();
     getCallbackManager().dispatchEvent("llm-start", {
       payload: {

packages/core/src/memory/ChatMemoryBuffer.ts

@@ -1,3 +1,4 @@
+import type { ChatHistory } from "../ChatHistory.js";
 import type { ChatMessage, LLM } from "../llm/index.js";
 import { SimpleChatStore } from "../storage/chatStore/SimpleChatStore.js";
 import type { BaseChatStore } from "../storage/chatStore/types.js";
@@ -6,25 +7,17 @@ import type { BaseMemory } from "./types.js";
 const DEFAULT_TOKEN_LIMIT_RATIO = 0.75;
 const DEFAULT_TOKEN_LIMIT = 3000;
 
-type ChatMemoryBufferParams<
-  AdditionalMessageOptions extends Record<string, unknown> = Record<
-    string,
-    unknown
-  >,
-> = {
-  tokenLimit?: number;
-  chatStore?: BaseChatStore<AdditionalMessageOptions>;
-  chatStoreKey?: string;
-  chatHistory?: ChatMessage<AdditionalMessageOptions>[];
-  llm?: LLM<Record<string, unknown>, AdditionalMessageOptions>;
-};
+type ChatMemoryBufferParams<AdditionalMessageOptions extends object = object> =
+  {
+    tokenLimit?: number;
+    chatStore?: BaseChatStore<AdditionalMessageOptions>;
+    chatStoreKey?: string;
+    chatHistory?: ChatHistory<AdditionalMessageOptions>;
+    llm?: LLM<object, AdditionalMessageOptions>;
+  };
 
-export class ChatMemoryBuffer<
-  AdditionalMessageOptions extends Record<string, unknown> = Record<
-    string,
-    unknown
-  >,
-> implements BaseMemory<AdditionalMessageOptions>
+export class ChatMemoryBuffer<AdditionalMessageOptions extends object = object>
+  implements BaseMemory<AdditionalMessageOptions>
 {
   tokenLimit: number;
 
@@ -47,7 +40,7 @@ export class ChatMemoryBuffer<
     }
 
     if (init?.chatHistory) {
-      this.chatStore.setMessages(this.chatStoreKey, init.chatHistory);
+      this.chatStore.setMessages(this.chatStoreKey, init.chatHistory.messages);
     }
   }
 

packages/core/src/memory/types.ts

@@ -1,11 +1,6 @@
 import type { ChatMessage } from "../llm/index.js";
 
-export interface BaseMemory<
-  AdditionalMessageOptions extends Record<string, unknown> = Record<
-    string,
-    unknown
-  >,
-> {
+export interface BaseMemory<AdditionalMessageOptions extends object = object> {
   tokenLimit: number;
   get(...args: unknown[]): ChatMessage<AdditionalMessageOptions>[];
   getAll(): ChatMessage<AdditionalMessageOptions>[];

packages/core/src/objects/base.ts

@@ -50,7 +50,7 @@ export abstract class BaseObjectNodeMapping {
 
 type QueryType = string;
 
-export class ObjectRetriever {
+export class ObjectRetriever<T = unknown> {
   _retriever: BaseRetriever;
   _objectNodeMapping: BaseObjectNodeMapping;
 
@@ -68,7 +68,7 @@ export class ObjectRetriever {
   }
 
   // Translating the retrieve method
-  async retrieve(strOrQueryBundle: QueryType): Promise<any> {
+  async retrieve(strOrQueryBundle: QueryType): Promise<T[]> {
     const nodes = await this.retriever.retrieve({ query: strOrQueryBundle });
     const objs = nodes.map((n) => this._objectNodeMapping.fromNode(n.node));
     return objs;
@@ -180,7 +180,7 @@ export class ObjectIndex {
     return this._objectNodeMapping.objNodeMapping();
   }
 
-  async asRetriever(kwargs: any): Promise<ObjectRetriever> {
+  async asRetriever(kwargs: any): Promise<ObjectRetriever<any>> {
     return new ObjectRetriever(
       this._index.asRetriever(kwargs),
       this._objectNodeMapping,

packages/core/src/storage/chatStore/SimpleChatStore.ts

@@ -6,10 +6,7 @@ import type { BaseChatStore } from "./types.js";
  *  This could lead to memory leaks if the messages are not properly cleaned up.
  */
 export class SimpleChatStore<
-  AdditionalMessageOptions extends Record<string, unknown> = Record<
-    string,
-    unknown
-  >,
+  AdditionalMessageOptions extends object = Record<string, unknown>,
 > implements BaseChatStore<AdditionalMessageOptions>
 {
   store: { [key: string]: ChatMessage<AdditionalMessageOptions>[] } = {};

packages/core/src/storage/chatStore/types.ts

@@ -1,10 +1,7 @@
 import type { ChatMessage } from "../../llm/index.js";
 
 export interface BaseChatStore<
-  AdditionalMessageOptions extends Record<string, unknown> = Record<
-    string,
-    unknown
-  >,
+  AdditionalMessageOptions extends object = object,
 > {
   setMessages(
     key: string,

packages/core/src/synthesizers/MultiModalResponseSynthesizer.ts

@@ -91,6 +91,6 @@ export class MultiModalResponseSynthesizer
       prompt,
     });
 
-    return new Response(response.text, nodes);
+    return new Response(response.text, nodesWithScore);
   }
 }

packages/core/src/synthesizers/ResponseSynthesizer.ts

@@ -69,19 +69,21 @@ export class ResponseSynthesizer
     const textChunks: string[] = nodesWithScore.map(({ node }) =>
       node.getContent(this.metadataMode),
     );
-    const nodes = nodesWithScore.map(({ node }) => node);
     if (stream) {
       const response = await this.responseBuilder.getResponse({
         query,
         textChunks,
         stream,
       });
-      return streamConverter(response, (chunk) => new Response(chunk, nodes));
+      return streamConverter(
+        response,
+        (chunk) => new Response(chunk, nodesWithScore),
+      );
     }
     const response = await this.responseBuilder.getResponse({
       query,
       textChunks,
     });
-    return new Response(response, nodes);
+    return new Response(response, nodesWithScore);
   }
 }

packages/core/src/tools/utils.ts

@@ -1,27 +1,33 @@
+import { getCallbackManager } from "../internal/settings/CallbackManager.js";
 import type { BaseTool } from "../types.js";
 import { ToolOutput } from "./types.js";
 
 export async function callToolWithErrorHandling(
   tool: BaseTool,
-  inputDict: { [key: string]: any },
+  input: unknown,
 ): Promise<ToolOutput> {
   if (!tool.call) {
     return new ToolOutput(
       "Error: Tool does not have a call function.",
       tool.metadata.name,
-      { kwargs: inputDict },
+      input,
       null,
     );
   }
   try {
-    const value = await tool.call(inputDict);
-    return new ToolOutput(value, tool.metadata.name, inputDict, value);
-  } catch (e) {
-    return new ToolOutput(
-      `Error: ${e}`,
-      tool.metadata.name,
-      { kwargs: inputDict },
-      e,
+    getCallbackManager().dispatchEvent("llm-tool-call", {
+      payload: {
+        toolCall: {
+          name: tool.metadata.name,
+          input,
+        },
+      },
+    });
+    const value = await tool.call(
+      typeof input === "string" ? JSON.parse(input) : input,
     );
+    return new ToolOutput(value, tool.metadata.name, input, value);
+  } catch (e) {
+    return new ToolOutput(`Error: ${e}`, tool.metadata.name, input, e);
   }
 }

packages/core/src/types.ts

@@ -67,7 +67,7 @@ export interface BaseTool<Input = any> {
   Input extends Known ? ToolMetadata<JSONSchemaType<Input>> : ToolMetadata;
 }
 
-export type ToolWithCall<Input = unknown> = Omit<BaseTool<Input>, "call"> & {
+export type BaseToolWithCall<Input = any> = Omit<BaseTool<Input>, "call"> & {
   call: NonNullable<Pick<BaseTool<Input>, "call">["call"]>;
 };
 

packages/core/tests/type.test.ts

@@ -0,0 +1,56 @@
+import type { ChatMessage, MessageContent, MessageType } from "llamaindex";
+import { expectTypeOf, test } from "vitest";
+import type { ChatResponse } from "../src/index.js";
+
+test("chat message type", () => {
+  // if generic is not provided, `options` is not required
+  expectTypeOf<ChatMessage>().toMatchTypeOf<{
+    content: MessageContent;
+    role: MessageType;
+  }>();
+  expectTypeOf<ChatMessage>().toMatchTypeOf<{
+    content: MessageContent;
+    role: MessageType;
+    options?: object;
+  }>();
+  expectTypeOf<ChatMessage>().not.toMatchTypeOf<{
+    content: MessageContent;
+    role: MessageType;
+    options: Record<string, unknown>;
+  }>();
+  type Options = {
+    a: string;
+    b: number;
+  };
+  expectTypeOf<ChatMessage<Options>>().toMatchTypeOf<{
+    content: MessageContent;
+    role: MessageType;
+    options?: Options;
+  }>();
+});
+
+test("chat response type", () => {
+  // if generic is not provided, `options` is not required
+  expectTypeOf<ChatResponse>().toMatchTypeOf<{
+    message: ChatMessage;
+    raw: object | null;
+  }>();
+  expectTypeOf<ChatResponse>().toMatchTypeOf<{
+    message: ChatMessage;
+    raw: object | null;
+    options?: Record<string, unknown>;
+  }>();
+  expectTypeOf<ChatResponse>().not.toMatchTypeOf<{
+    message: ChatMessage;
+    raw: object | null;
+    options: Record<string, unknown>;
+  }>();
+  type Options = {
+    a: string;
+    b: number;
+  };
+  expectTypeOf<ChatResponse<Options>>().toMatchTypeOf<{
+    message: ChatMessage<Options>;
+    raw: object | null;
+  }>();
+});

packages/edge/.gitignore

@@ -1,3 +1,4 @@
 .turbo
 README.md
 LICENSE
+CHANGELOG.md

packages/edge/CHANGELOG.md

@@ -1,8 +0,0 @@
-# @llamaindex/edge
-
-## 0.2.7
-
-### Patch Changes
-
-- Updated dependencies
-  - @llamaindex/env@0.0.7

packages/edge/e2e/test-edge-runtime/package.json

@@ -5,8 +5,7 @@
   "scripts": {
     "dev": "next dev",
     "build": "next build",
-    "start": "next start",
-    "lint": "next lint"
+    "start": "next start"
   },
   "dependencies": {
     "@llamaindex/edge": "workspace:*",

packages/edge/e2e/test-edge-runtime/src/app/layout.tsx

@@ -1,9 +1,6 @@
 import type { Metadata } from "next";
-import { Inter } from "next/font/google";
 import "./globals.css";
 
-const inter = Inter({ subsets: ["latin"] });
-
 export const metadata: Metadata = {
   title: "Create Next App",
   description: "Generated by create next app",
@@ -18,7 +15,7 @@ export default function RootLayout({
 }>) {
   return (
     <html lang="en">
-      <body className={inter.className}>{children}</body>
+      <body>{children}</body>
     </html>
   );
 }

packages/edge/package.json

@@ -1,10 +1,10 @@
 {
   "name": "@llamaindex/edge",
-  "version": "0.2.8",
+  "version": "0.2.9",
   "license": "MIT",
   "type": "module",
   "dependencies": {
-    "@anthropic-ai/sdk": "^0.18.0",
+    "@anthropic-ai/sdk": "^0.20.4",
     "@aws-crypto/sha256-js": "^5.2.0",
     "@datastax/astra-db-ts": "^0.1.4",
     "@grpc/grpc-js": "^1.10.6",
@@ -78,7 +78,7 @@
     "directory": "packages/edge"
   },
   "scripts": {
-    "copy": "cp -r ../../README.md ../../LICENSE .",
+    "copy": "cp -r ../../README.md ../../LICENSE ../core/CHANGELOG.md .",
     "update:deps": "node scripts/update-deps.js",
     "build:core": "pnpm --filter llamaindex build && cp -r ../core/dist . && rm -rf dist/cjs",
     "build": "pnpm run update:deps && pnpm run build:core && pnpm copy"

packages/eslint-config-custom/package.json

@@ -1,20 +0,0 @@
-{
-  "name": "eslint-config-custom",
-  "private": true,
-  "version": "0.0.0",
-  "main": "index.js",
-  "license": "MIT",
-  "dependencies": {
-    "eslint-config-next": "^13.5.6",
-    "eslint-config-prettier": "^8.10.0",
-    "eslint-config-turbo": "^1.11.3",
-    "eslint-plugin-react": "7.28.0",
-    "@typescript-eslint/eslint-plugin": "^7.5.0"
-  },
-  "publishConfig": {
-    "access": "public"
-  },
-  "devDependencies": {
-    "next": "^13.5.6"
-  }
-}

packages/experimental/CHANGELOG.md

@@ -1,5 +1,13 @@
 # @llamaindex/experimental
 
+## 0.0.12
+
+### Patch Changes
+
+- Updated dependencies [76c3fd6]
+- Updated dependencies [208282d]
+  - llamaindex@0.2.9
+
 ## 0.0.11
 
 ### Patch Changes

packages/experimental/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/experimental",
   "description": "Experimental package for LlamaIndexTS",
-  "version": "0.0.11",
+  "version": "0.0.12",
   "type": "module",
   "types": "dist/type/index.d.ts",
   "main": "dist/cjs/index.js",

packages/tsconfig/base.json

@@ -1,20 +0,0 @@
-{
-  "$schema": "https://json.schemastore.org/tsconfig",
-  "display": "Default",
-  "compilerOptions": {
-    "composite": false,
-    "declaration": true,
-    "declarationMap": true,
-    "esModuleInterop": true,
-    "forceConsistentCasingInFileNames": true,
-    "inlineSources": false,
-    "isolatedModules": true,
-    "moduleResolution": "node",
-    "noUnusedLocals": false,
-    "noUnusedParameters": false,
-    "preserveWatchOutput": true,
-    "skipLibCheck": true,
-    "strict": true
-  },
-  "exclude": ["node_modules"]
-}

packages/tsconfig/nextjs.json

@@ -1,21 +0,0 @@
-{
-  "$schema": "https://json.schemastore.org/tsconfig",
-  "display": "Next.js",
-  "extends": "./base.json",
-  "compilerOptions": {
-    "plugins": [{ "name": "next" }],
-    "allowJs": true,
-    "declaration": false,
-    "declarationMap": false,
-    "incremental": true,
-    "jsx": "preserve",
-    "lib": ["dom", "dom.iterable", "esnext"],
-    "module": "esnext",
-    "noEmit": true,
-    "resolveJsonModule": true,
-    "strict": false,
-    "target": "es5"
-  },
-  "include": ["src", "next-env.d.ts"],
-  "exclude": ["node_modules"]
-}

packages/tsconfig/package.json

@@ -1,9 +0,0 @@
-{
-  "name": "tsconfig",
-  "version": "0.0.0",
-  "private": true,
-  "license": "MIT",
-  "publishConfig": {
-    "access": "public"
-  }
-}

packages/tsconfig/react-library.json

@@ -1,11 +0,0 @@
-{
-  "$schema": "https://json.schemastore.org/tsconfig",
-  "display": "React Library",
-  "extends": "./base.json",
-  "compilerOptions": {
-    "jsx": "react-jsx",
-    "lib": ["ES2015", "DOM"],
-    "module": "ESNext",
-    "target": "es6"
-  }
-}