Merge branch 'main' into ljv/agent-docs

* New introduction and installation tweaks
* New use-case based starter tutorials
* Adding a new Guides section with Agents to start
* Miscellaneous docs bug fixes

.changeset/dry-queens-dream.md

@@ -0,0 +1,5 @@
+---
+"llamaindex": patch
+---
+
+Add vectorStores to storage context to define vector store per modality

.changeset/fuzzy-coins-breathe.md

@@ -0,0 +1,6 @@
+---
+"llamaindex": patch
+"@llamaindex/examples": patch
+---
+
+Added support for accessing Gemini via Vertex AI

.changeset/heavy-files-compare.md

@@ -0,0 +1,5 @@
+---
+"llamaindex": patch
+---
+
+Add system prompt to ContextChatEngine

.eslintrc.cjs

@@ -1,5 +1,3 @@
-const { join } = require("node:path");
-
 module.exports = {
   root: true,
   extends: [
@@ -8,7 +6,7 @@ module.exports = {
     "plugin:@typescript-eslint/recommended-type-checked-only",
   ],
   parserOptions: {
-    project: join(__dirname, "tsconfig.eslint.json"),
+    project: true,
     __tsconfigRootDir: __dirname,
   },
   settings: {
@@ -25,7 +23,6 @@ module.exports = {
         ignoreIIFE: true,
       },
     ],
-    "no-debugger": "error",
     "@typescript-eslint/await-thenable": "off",
     "@typescript-eslint/ban-ts-comment": "off",
     "@typescript-eslint/ban-types": "off",

.github/workflows/test.yml

@@ -85,7 +85,6 @@ jobs:
           if-no-files-found: error
   e2e-core-examples:
     strategy:
-      fail-fast: false
       matrix:
         packages:
           - cloudflare-worker-agent

apps/docs/CHANGELOG.md

@@ -1,60 +1,5 @@
 # docs
 
-## 0.0.25
-
-### Patch Changes
-
-- Updated dependencies [6bc5bdd]
-- Updated dependencies [bf25ff6]
-- Updated dependencies [e6d6576]
-  - llamaindex@0.3.17
-
-## 0.0.24
-
-### Patch Changes
-
-- 631f000: feat: DeepInfra LLM implementation
-- 8832669: Community bedrock support added
-- a29d835: setDocumentHash should be async
-- Updated dependencies [11ae926]
-- Updated dependencies [631f000]
-- Updated dependencies [1378ec4]
-- Updated dependencies [6b1ded4]
-- Updated dependencies [4d4bd85]
-- Updated dependencies [24a9d1e]
-- Updated dependencies [45952de]
-- Updated dependencies [54230f0]
-- Updated dependencies [a29d835]
-- Updated dependencies [73819bf]
-  - llamaindex@0.3.16
-
-## 0.0.23
-
-### Patch Changes
-
-- Updated dependencies [6e156ed]
-- Updated dependencies [265976d]
-- Updated dependencies [8e26f75]
-  - llamaindex@0.3.15
-
-## 0.0.22
-
-### Patch Changes
-
-- Updated dependencies [6ff7576]
-- Updated dependencies [94543de]
-  - llamaindex@0.3.14
-
-## 0.0.21
-
-### Patch Changes
-
-- Updated dependencies [1b1081b]
-- Updated dependencies [37525df]
-- Updated dependencies [660a2b3]
-- Updated dependencies [a1f2475]
-  - llamaindex@0.3.13
-
 ## 0.0.20
 
 ### Patch Changes

apps/docs/docs/getting_started/starter_tutorial/retrieval_augmented_generation.mdx

@@ -10,19 +10,21 @@ import TSConfigSource from "!!raw-loader!../../../../../examples/tsconfig.json";
 
 One of the most common use-cases for LlamaIndex is Retrieval-Augmented Generation or RAG, in which your data is indexed and selectively retrieved to be given to an LLM as source material for responding to a query. You can learn more about the [concepts behind RAG](../concepts).
 
-## Set up the project
+## Before you start
 
-In a new folder, run:
+Make sure you have installed LlamaIndex.TS and have an OpenAI key. If you haven't, check out the [installation](../installation) steps.
+
+You can use [other LLMs](../examples/other_llms) via their APIs; if you would prefer to use local models check out our [local LLM example](../../examples/local_llm).
+
+## Set up
+
+In a new folder:
 
 ```bash npm2yarn
 npm init
 npm install -D typescript @types/node
 ```
 
-Then, check out the [installation](../installation) steps to install LlamaIndex.TS and prepare an OpenAI key.
-
-You can use [other LLMs](../examples/other_llms) via their APIs; if you would prefer to use local models check out our [local LLM example](../../examples/local_llm).
-
 ## Run queries
 
 Create the file `example.ts`. This code will

apps/docs/docs/modules/data_loader.mdx

@@ -6,7 +6,6 @@ import CodeBlock from "@theme/CodeBlock";
 import CodeSource from "!raw-loader!../../../../examples/readers/src/simple-directory-reader";
 import CodeSource2 from "!raw-loader!../../../../examples/readers/src/custom-simple-directory-reader";
 import CodeSource3 from "!raw-loader!../../../../examples/readers/src/llamaparse";
-import CodeSource4 from "!raw-loader!../../../../examples/readers/src/simple-directory-reader-with-llamaparse.ts";
 
 # Loader
 
@@ -22,13 +21,11 @@ It is a simple reader that reads all files from a directory and its subdirectori
 
 <CodeBlock language="ts">{CodeSource}</CodeBlock>
 
-Currently, it supports reading `.txt`, `.pdf`, `.csv`, `.md`, `.docx`, `.htm`, `.html`, `.jpg`, `.jpeg`, `.png` and `.gif` files, but support for other file types is planned.
+Currently, it supports reading `.csv`, `.docx`, `.html`, `.md` and `.pdf` files,
+but support for other file types is planned.
 
-You can override the default reader for all file types, inlcuding unsupported ones, with the `overrideReader` option.
-Additionally, you can override the default reader for specific file types or add support for additional file types with the `fileExtToReader` option.
-Also, you can provide a `defaultReader` as a fallback for files with unsupported extensions. By default it is `TextFileReader`.
-
-SimpleDirectoryReader supports up to 9 concurrent requests. Use the `numWorkers` option to set the number of concurrent requests. By default it runs in sequential mode, i.e. set to 1.
+Also, you can provide a `defaultReader` as a fallback for files with unsupported extensions.
+Or pass new readers for `fileExtToReader` to support more file types.
 
 <CodeBlock language="ts" showLineNumbers metastring="{8-12,17-21}">
   {CodeSource2}
@@ -38,31 +35,14 @@ SimpleDirectoryReader supports up to 9 concurrent requests. Use the `numWorkers`
 
 LlamaParse is an API created by LlamaIndex to efficiently parse files, e.g. it's great at converting PDF tables into markdown.
 
-To use it, first login and get an API key from https://cloud.llamaindex.ai. Make sure to store the key as `apiKey` parameter or in the environment variable `LLAMA_CLOUD_API_KEY`.
+To use it, first login and get an API key from https://cloud.llamaindex.ai. Make sure to store the key in the environment variable `LLAMA_CLOUD_API_KEY`.
 
-Then, you can use the `LlamaParseReader` class to local files and convert them into a parsed document that can be used by LlamaIndex.
-See [LlamaParseReader.ts](https://github.com/run-llama/LlamaIndexTS/blob/main/packages/core/src/readers/LlamaParseReader.ts#L6) for a list of supported file types:
+Then, you can use the `LlamaParseReader` class to read a local PDF file and convert it into a markdown document that can be used by LlamaIndex:
 
 <CodeBlock language="ts">{CodeSource3}</CodeBlock>
 
-Additional options can be set with the `LlamaParseReader` constructor:
-
-- `resultType` can be set to `markdown`, `text` or `.json`. Defaults to `text`
-- `language` primarly helps with OCR recognition. Defaults to `en`. See [../readers/type.ts](https://github.com/run-llama/LlamaIndexTS/blob/main/packages/core/src/readers/type.ts#L20) for a list of supported languages.
-- `parsingInstructions` can help with complicated document structures. See this [LlamaIndex Blog Post](https://www.llamaindex.ai/blog/launching-the-first-genai-native-document-parsing-platform) for an example.
-- `skipDiagonalText` set to true to ignore diagonal text.
-- `invalidateCache` set to true to ignore the LlamaCloud cache. All document are kept in cache for 48hours after the job was completed to avoid processing the same document twice. Can be useful for testing when trying to re-parse the same document with, e.g. different `parsingInstructions`.
-- `gpt4oMode` set to true to use GPT-4o to extract content.
-- `gpt4oApiKey` set the GPT-4o API key. Optional. Lowers the cost of parsing by using your own API key. Your OpenAI account will be charged. Can also be set in the environment variable `LLAMA_CLOUD_GPT4O_API_KEY`.
-- `numWorkers` as in the python version, is set in `SimpleDirectoryReader`. Default is 1.
-
-## LlamaParse with SimpleDirectoryReader
-
-Below a full example of `LlamaParse` integrated in `SimpleDirectoryReader` with additional options.
-
-<CodeBlock language="ts">{CodeSource4}</CodeBlock>
+Alternatively, you can set the [`resultType`](../api/classes/LlamaParseReader.md#resulttype) option to `text` to get the parsed document as a text string.
 
 ## API Reference
 
 - [SimpleDirectoryReader](../api/classes/SimpleDirectoryReader.md)
-- [LlamaParseReader](../api/classes/LlamaParseReader.md)

apps/docs/docs/modules/embeddings/available_embeddings/deepinfra.md

@@ -1,79 +0,0 @@
-# DeepInfra
-
-To use DeepInfra embeddings, you need to import `DeepInfraEmbedding` from llamaindex.
-Check out available embedding models [here](https://deepinfra.com/models/embeddings).
-
-```ts
-import {
-  DeepInfraEmbedding,
-  Settings,
-  Document,
-  VectorStoreIndex,
-} from "llamaindex";
-
-// Update Embed Model
-Settings.embedModel = new DeepInfraEmbedding();
-
-const document = new Document({ text: essay, id_: "essay" });
-
-const index = await VectorStoreIndex.fromDocuments([document]);
-
-const queryEngine = index.asQueryEngine();
-
-const query = "What is the meaning of life?";
-
-const results = await queryEngine.query({
-  query,
-});
-```
-
-By default, DeepInfraEmbedding is using the sentence-transformers/clip-ViT-B-32 model. You can change the model by passing the model parameter to the constructor.
-For example:
-
-```ts
-import { DeepInfraEmbedding } from "llamaindex";
-
-const model = "intfloat/e5-large-v2";
-Settings.embedModel = new DeepInfraEmbedding({
-  model,
-});
-```
-
-You can also set the `maxRetries` and `timeout` parameters when initializing `DeepInfraEmbedding` for better control over the request behavior.
-
-For example:
-
-```ts
-import { DeepInfraEmbedding, Settings } from "llamaindex";
-
-const model = "intfloat/e5-large-v2";
-const maxRetries = 5;
-const timeout = 5000; // 5 seconds
-
-Settings.embedModel = new DeepInfraEmbedding({
-  model,
-  maxRetries,
-  timeout,
-});
-```
-
-Standalone usage:
-
-```ts
-import { DeepInfraEmbedding } from "llamaindex";
-import { config } from "dotenv";
-// For standalone usage, you need to configure DEEPINFRA_API_TOKEN in .env file
-config();
-
-const main = async () => {
-  const model = "intfloat/e5-large-v2";
-  const embeddings = new DeepInfraEmbedding({ model });
-  const text = "What is the meaning of life?";
-  const response = await embeddings.embed([text]);
-  console.log(response);
-};
-
-main();
-```
-
-For questions or feedback, please contact us at [feedback@deepinfra.com](mailto:feedback@deepinfra.com)

apps/docs/docs/modules/evaluation/modules/correctness.md

@@ -21,7 +21,7 @@ export OPENAI_API_KEY=your-api-key
 Import the required modules:
 
 ```ts
-import { CorrectnessEvaluator, OpenAI, Settings, Response } from "llamaindex";
+import { CorrectnessEvaluator, OpenAI, Settings } from "llamaindex";
 ```
 
 Let's setup gpt-4 for better results:
@@ -45,7 +45,7 @@ const evaluator = new CorrectnessEvaluator();
 
 const result = await evaluator.evaluateResponse({
   query,
-  response: new Response(response),
+  response,
 });
 
 console.log(

apps/docs/docs/modules/evaluation/modules/relevancy.md

@@ -21,13 +21,7 @@ export OPENAI_API_KEY=your-api-key
 Import the required modules:
 
 ```ts
-import {
-  RelevancyEvaluator,
-  OpenAI,
-  Settings,
-  Document,
-  VectorStoreIndex,
-} from "llamaindex";
+import { RelevancyEvaluator, OpenAI, Settings } from "llamaindex";
 ```
 
 Let's setup gpt-4 for better results:

apps/docs/docs/modules/llms/available_llms/bedrock.md

@@ -1,61 +0,0 @@
-# Bedrock
-
-## Usage
-
-```ts
-import { BEDROCK_MODELS, Bedrock } from "@llamaindex/community";
-
-Settings.llm = new Bedrock({
-  model: BEDROCK_MODELS.ANTHROPIC_CLAUDE_3_HAIKU,
-  region: "us-east-1", // can be provided via env AWS_REGION
-  credentials: {
-    accessKeyId: "...", // optional and can be provided via env AWS_ACCESS_KEY_ID
-    secretAccessKey: "...", // optional and can be provided via env AWS_SECRET_ACCESS_KEY
-  },
-});
-```
-
-Currently only supports Anthropic models:
-
-```ts
-ANTHROPIC_CLAUDE_INSTANT_1 = "anthropic.claude-instant-v1";
-ANTHROPIC_CLAUDE_2 = "anthropic.claude-v2";
-ANTHROPIC_CLAUDE_2_1 = "anthropic.claude-v2:1";
-ANTHROPIC_CLAUDE_3_SONNET = "anthropic.claude-3-sonnet-20240229-v1:0";
-ANTHROPIC_CLAUDE_3_HAIKU = "anthropic.claude-3-haiku-20240307-v1:0";
-ANTHROPIC_CLAUDE_3_OPUS = "anthropic.claude-3-opus-20240229-v1:0"; // Not currently on Bedrock
-```
-
-Sonnet, Haiku and Opus are multimodal, image_url only supports base64 data url format, e.g. `data:image/jpeg;base64,SGVsbG8sIFdvcmxkIQ==`
-
-## Full Example
-
-```ts
-import { BEDROCK_MODELS, Bedrock } from "llamaindex";
-
-Settings.llm = new Bedrock({
-  model: BEDROCK_MODELS.ANTHROPIC_CLAUDE_3_HAIKU,
-});
-
-async function main() {
-  const document = new Document({ text: essay, id_: "essay" });
-
-  // Load and index documents
-  const index = await VectorStoreIndex.fromDocuments([document]);
-
-  // Create a query engine
-  const queryEngine = index.asQueryEngine({
-    retriever,
-  });
-
-  const query = "What is the meaning of life?";
-
-  // Query
-  const response = await queryEngine.query({
-    query,
-  });
-
-  // Log the response
-  console.log(response.response);
-}
-```

apps/docs/docs/modules/llms/available_llms/deepinfra.md

@@ -1,83 +0,0 @@
-# DeepInfra
-
-Check out available LLMs [here](https://deepinfra.com/models/text-generation).
-
-```ts
-import { DeepInfra, Settings } from "llamaindex";
-
-// Get the API key from `DEEPINFRA_API_TOKEN` environment variable
-import { config } from "dotenv";
-config();
-Settings.llm = new DeepInfra();
-
-// Set the API key
-apiKey = "YOUR_API_KEY";
-Settings.llm = new DeepInfra({ apiKey });
-```
-
-You can setup the apiKey on the environment variables, like:
-
-```bash
-export DEEPINFRA_API_TOKEN="<YOUR_API_KEY>"
-```
-
-## Load and index documents
-
-For this example, we will use a single document. In a real-world scenario, you would have multiple documents to index.
-
-```ts
-const document = new Document({ text: essay, id_: "essay" });
-
-const index = await VectorStoreIndex.fromDocuments([document]);
-```
-
-## Query
-
-```ts
-const queryEngine = index.asQueryEngine();
-
-const query = "What is the meaning of life?";
-
-const results = await queryEngine.query({
-  query,
-});
-```
-
-## Full Example
-
-```ts
-import { DeepInfra, Document, VectorStoreIndex, Settings } from "llamaindex";
-
-// Use custom LLM
-const model = "meta-llama/Meta-Llama-3-8B-Instruct";
-Settings.llm = new DeepInfra({ model, temperature: 0 });
-
-async function main() {
-  const document = new Document({ text: essay, id_: "essay" });
-
-  // Load and index documents
-  const index = await VectorStoreIndex.fromDocuments([document]);
-
-  // get retriever
-  const retriever = index.asRetriever();
-
-  // Create a query engine
-  const queryEngine = index.asQueryEngine({
-    retriever,
-  });
-
-  const query = "What is the meaning of life?";
-
-  // Query
-  const response = await queryEngine.query({
-    query,
-  });
-
-  // Log the response
-  console.log(response.response);
-}
-```
-
-## Feedback
-
-If you have any feedback, please reach out to us at [feedback@deepinfra.com](mailto:feedback@deepinfra.com)

apps/docs/i18n/fr/docusaurus-plugin-content-docs/current/api/classes/BaseDocumentStore.md

@@ -271,7 +271,7 @@ custom_edit_url: null
 
 ### setDocumentHash
 
-▸ `Abstract` **setDocumentHash**(`docId`, `docHash`): `Promise`<`void`\>
+▸ `Abstract` **setDocumentHash**(`docId`, `docHash`): `void`
 
 #### Parameters
 

apps/docs/i18n/hr/docusaurus-plugin-content-docs/current/api/classes/BaseDocumentStore.md

@@ -271,7 +271,7 @@ custom_edit_url: null
 
 ### setDocumentHash
 
-▸ `Abstract` **setDocumentHash**(`docId`, `docHash`): `Promise`<`void`\>
+▸ `Abstract` **setDocumentHash**(`docId`, `docHash`): `void`
 
 #### Parameters
 

apps/docs/i18n/zh-Hans/docusaurus-plugin-content-docs/current/api/classes/BaseDocumentStore.md

@@ -271,7 +271,7 @@ custom_edit_url: null
 
 ### setDocumentHash
 
-▸ `Abstract` **setDocumentHash**(`docId`, `docHash`): `Promise`<`void`\>
+▸ `Abstract` **setDocumentHash**(`docId`, `docHash`): `void`
 
 #### Parameters
 

apps/docs/package.json

@@ -1,6 +1,6 @@
 {
   "name": "docs",
-  "version": "0.0.25",
+  "version": "0.0.20",
   "private": true,
   "scripts": {
     "docusaurus": "docusaurus",

examples/agent/large_toolcall.ts

@@ -1,64 +0,0 @@
-import { FunctionTool, OpenAI, OpenAIAgent } from "llamaindex";
-
-const csvData =
-  "TITLE,RELEASE_YEAR,SCORE,NUMBER_OF_VOTES,DURATION,MAIN_GENRE,MAIN_PRODUCTION\nDavid Attenborough: A Life on Our Planet,2020,9,31180,83,documentary,GB\nInception,2010,8.8,2268288,148,scifi,GB\nForrest Gump,1994,8.8,1994599,142,drama,US\nAnbe Sivam,2003,8.7,20595,160,comedy,IN\nBo Burnham: Inside,2021,8.7,44074,87,comedy,US\nSaving Private Ryan,1998,8.6,1346020,169,drama,US\nDjango Unchained,2012,8.4,1472668,165,western,US\nDangal,2016,8.4,180247,161,action,IN\nBo Burnham: Make Happy,2016,8.4,14356,60,comedy,US\nLouis C.K.: Hilarious,2010,8.4,11973,84,comedy,US\nDave Chappelle: Sticks & Stones,2019,8.4,25687,65,comedy,US\n3 Idiots,2009,8.4,385782,170,comedy,IN\nBlack Friday,2004,8.4,20611,143,crime,IN\nSuper Deluxe,2019,8.4,13680,176,thriller,IN\nWinter on Fire: Ukraine's Fight for Freedom,2015,8.3,17710,98,documentary,UA\nOnce Upon a Time in America,1984,8.3,342335,229,drama,US\nTaxi Driver,1976,8.3,795222,113,crime,US\nLike Stars on Earth,2007,8.3,188234,165,drama,IN\nBo Burnham: What.,2013,8.3,11488,60,comedy,US\nFull Metal Jacket,1987,8.3,723306,116,drama,GB\nWarrior,2011,8.2,463276,140,drama,US\nDrishyam,2015,8.2,79075,163,thriller,IN\nQueen,2014,8.2,64805,146,drama,IN\nPaan Singh Tomar,2012,8.2,35888,135,drama,IN";
-
-const userQuestion = "which are the best comedies after 2010?";
-
-(async () => {
-  // The agent will succeed if we increase `maxTokens` to 1024
-  const llm = new OpenAI({ model: "gpt-4-turbo", maxTokens: 256 });
-
-  type Input = {
-    code: string;
-  };
-  // initiate fake code interpreter
-  const interpreterTool = FunctionTool.from<Input>(
-    ({ code }) => {
-      console.log(
-        `To answer the user's question, call the following code:\n${code}`,
-      );
-      return code;
-    },
-    {
-      name: "interpreter",
-      description:
-        "Execute python code in a Jupyter notebook cell and return any result, stdout, stderr, display_data, and error.",
-      parameters: {
-        type: "object",
-        properties: {
-          code: {
-            type: "string",
-            description: "The python code to execute in a single cell.",
-          },
-        },
-        required: ["code"],
-      },
-    },
-  );
-
-  const systemPrompt =
-    "You are a Python interpreter.\n        - You are given tasks to complete and you run python code to solve them.\n        - The python code runs in a Jupyter notebook. Every time you call $(interpreter) tool, the python code is executed in a separate cell. It's okay to make multiple calls to $(interpreter).\n        - Display visualizations using matplotlib or any other visualization library directly in the notebook. Shouldn't save the visualizations to a file, just return the base64 encoded data.\n        - You can install any pip package (if it exists) if you need to but the usual packages for data analysis are already preinstalled.\n        - You can run any python code you want in a secure environment.";
-
-  const agent = new OpenAIAgent({
-    llm,
-    tools: [interpreterTool],
-    systemPrompt,
-    verbose: true,
-  });
-
-  console.log(`User question: ${userQuestion}\n`);
-
-  await agent.chat({
-    message: [
-      {
-        type: "text",
-        text: userQuestion,
-      },
-      {
-        type: "text",
-        text: `Use data from following CSV raw contents:\n${csvData}`,
-      },
-    ],
-  });
-})();

examples/agent/large_toolcall_with_gpt4o.ts

@@ -1,74 +0,0 @@
-import { FunctionTool, OpenAI, ToolCallOptions } from "llamaindex";
-
-(async () => {
-  // The tool call will generate a partial JSON for `gpt-4-turbo`
-  // See thread: https://community.openai.com/t/gpt-4o-doesnt-consistently-respect-json-schema-on-tool-use/751125/7
-
-  const models = ["gpt-4o", "gpt-4-turbo"];
-  for (const model of models) {
-    const validJSON = await callLLM({ model });
-    console.log(
-      `LLM call resulting in large tool input with '${model}': LLM generates ${validJSON ? "valid" : "invalid"} JSON.`,
-    );
-  }
-})();
-
-async function callLLM(init: Partial<OpenAI>) {
-  const csvData =
-    "Country,Average Height (cm)\nNetherlands,156\nDenmark,158\nNorway,160";
-
-  const userQuestion = "Describe data in this csv";
-
-  // fake code interpreter tool
-  const interpreterTool = FunctionTool.from(
-    ({ code }: { code: string }) => code,
-    {
-      name: "interpreter",
-      description:
-        "Execute python code in a Jupyter notebook cell and return any result, stdout, stderr, display_data, and error.",
-      parameters: {
-        type: "object",
-        properties: {
-          code: {
-            type: "string",
-            description: "The python code to execute in a single cell.",
-          },
-        },
-        required: ["code"],
-      },
-    },
-  );
-
-  const systemPrompt =
-    "You are a Python interpreter.\n- You are given tasks to complete and you run python code to solve them.\n- The python code runs in a Jupyter notebook. Every time you call $(interpreter) tool, the python code is executed in a separate cell. It's okay to make multiple calls to $(interpreter).\n- Display visualizations using matplotlib or any other visualization library directly in the notebook. Shouldn't save the visualizations to a file, just return the base64 encoded data.\n- You can install any pip package (if it exists) if you need to but the usual packages for data analysis are already preinstalled.\n- You can run any python code you want in a secure environment.";
-
-  const llm = new OpenAI(init);
-  const response = await llm.chat({
-    tools: [interpreterTool],
-    messages: [
-      { role: "system", content: systemPrompt },
-      {
-        role: "user",
-        content: [
-          {
-            type: "text",
-            text: userQuestion,
-          },
-          {
-            type: "text",
-            text: `Use data from following CSV raw contents:\n${csvData}`,
-          },
-        ],
-      },
-    ],
-  });
-
-  const options = response.message?.options as ToolCallOptions;
-  const input = options.toolCall[0].input as string;
-  try {
-    JSON.parse(input);
-    return true;
-  } catch {
-    return false;
-  }
-}

examples/deepinfra/chat.ts

@@ -1,19 +0,0 @@
-import { DeepInfra } from "llamaindex";
-
-(async () => {
-  if (!process.env.DEEPINFRA_API_TOKEN) {
-    throw new Error("Please set the DEEPINFRA_API_TOKEN environment variable.");
-  }
-  const deepinfra = new DeepInfra({});
-  const result = await deepinfra.chat({
-    messages: [
-      { content: "You want to talk in rhymes.", role: "system" },
-      {
-        content:
-          "How much wood would a woodchuck chuck if a woodchuck could chuck wood?",
-        role: "user",
-      },
-    ],
-  });
-  console.log(result);
-})();

examples/deepinfra/embedding.ts

@@ -1,17 +0,0 @@
-import { DeepInfraEmbedding } from "llamaindex";
-
-async function main() {
-  // API token can be provided as an environment variable too
-  // using DEEPINFRA_API_TOKEN variable
-  const apiToken = "YOUR_API_TOKEN" ?? process.env.DEEPINFRA_API_TOKEN;
-  const model = "BAAI/bge-large-en-v1.5";
-  const embedModel = new DeepInfraEmbedding({
-    model,
-    apiToken,
-  });
-  const texts = ["hello", "world"];
-  const embeddings = await embedModel.getTextEmbeddingsBatch(texts);
-  console.log(`\nWe have ${embeddings.length} embeddings`);
-}
-
-main().catch(console.error);

examples/multimodal/context.ts

@@ -1,54 +0,0 @@
-// call pnpm tsx multimodal/load.ts first to init the storage
-import {
-  ContextChatEngine,
-  NodeWithScore,
-  ObjectType,
-  OpenAI,
-  RetrievalEndEvent,
-  Settings,
-  VectorStoreIndex,
-} from "llamaindex";
-import { getStorageContext } from "./storage";
-
-// Update chunk size and overlap
-Settings.chunkSize = 512;
-Settings.chunkOverlap = 20;
-
-// Update llm
-Settings.llm = new OpenAI({ model: "gpt-4-turbo", maxTokens: 512 });
-
-// Update callbackManager
-Settings.callbackManager.on("retrieve-end", (event: RetrievalEndEvent) => {
-  const { nodes, query } = event.detail.payload;
-  const imageNodes = nodes.filter(
-    (node: NodeWithScore) => node.node.type === ObjectType.IMAGE_DOCUMENT,
-  );
-  const textNodes = nodes.filter(
-    (node: NodeWithScore) => node.node.type === ObjectType.TEXT,
-  );
-  console.log(
-    `Retrieved ${textNodes.length} text nodes and ${imageNodes.length} image nodes for query: ${query}`,
-  );
-});
-
-async function main() {
-  const storageContext = await getStorageContext();
-  const index = await VectorStoreIndex.init({
-    storageContext,
-  });
-  // topK for text is 0 and for image 1 => we only retrieve one image and no text based on the query
-  const retriever = index.asRetriever({ topK: { TEXT: 0, IMAGE: 1 } });
-  // NOTE: we set the contextRole to "user" (default is "system"). The reason is that GPT-4 does not support
-  // images in a system message
-  const chatEngine = new ContextChatEngine({ retriever, contextRole: "user" });
-
-  // the ContextChatEngine will use the Clip embedding to retrieve the closest image
-  // (the lady in the chair) and use it in the context for the query
-  const response = await chatEngine.chat({
-    message: "What is the name of the painting with the lady in the chair?",
-  });
-
-  console.log(response.response, "\n");
-}
-
-main().catch(console.error);

examples/multimodal/rag.ts

@@ -1,4 +1,5 @@
 import {
+  ImageType,
   MultiModalResponseSynthesizer,
   OpenAI,
   RetrievalEndEvent,
@@ -21,6 +22,8 @@ Settings.callbackManager.on("retrieve-end", (event: RetrievalEndEvent) => {
 });
 
 async function main() {
+  const images: ImageType[] = [];
+
   const storageContext = await getStorageContext();
   const index = await VectorStoreIndex.init({
     nodes: [],
@@ -31,14 +34,13 @@ async function main() {
     responseSynthesizer: new MultiModalResponseSynthesizer(),
     retriever: index.asRetriever({ topK: { TEXT: 3, IMAGE: 1 } }),
   });
-  const stream = await queryEngine.query({
+  const result = await queryEngine.query({
     query: "Tell me more about Vincent van Gogh's famous paintings",
-    stream: true,
   });
-  for await (const chunk of stream) {
-    process.stdout.write(chunk.response);
-  }
-  process.stdout.write("\n");
+  console.log(result.response, "\n");
+  images.forEach((image) =>
+    console.log(`Image retrieved and used in inference: ${image.toString()}`),
+  );
 }
 
 main().catch(console.error);

examples/package.json

@@ -4,22 +4,22 @@
   "version": "0.0.4",
   "dependencies": {
     "@aws-crypto/sha256-js": "^5.2.0",
-    "@datastax/astra-db-ts": "^1.2.1",
+    "@datastax/astra-db-ts": "^1.1.0",
     "@notionhq/client": "^2.2.15",
-    "@pinecone-database/pinecone": "^2.2.2",
+    "@pinecone-database/pinecone": "^2.2.0",
     "@zilliz/milvus2-sdk-node": "^2.4.2",
-    "chromadb": "^1.8.1",
-    "commander": "^12.1.0",
+    "chromadb": "^1.7.3",
+    "commander": "^12.0.0",
     "dotenv": "^16.4.5",
-    "js-tiktoken": "^1.0.12",
-    "llamaindex": "^0.3.14",
-    "mongodb": "^6.7.0",
+    "js-tiktoken": "^1.0.11",
+    "llamaindex": "*",
+    "mongodb": "^6.6.1",
     "pathe": "^1.1.2"
   },
   "devDependencies": {
-    "@types/node": "^20.14.1",
+    "@types/node": "^20.12.11",
     "ts-node": "^10.9.2",
-    "tsx": "^4.11.2",
+    "tsx": "^4.9.3",
     "typescript": "^5.4.5"
   },
   "scripts": {

examples/readers/package.json

@@ -11,7 +11,7 @@
     "start:pdf": "node --import tsx ./src/pdf.ts",
     "start:llamaparse": "node --import tsx ./src/llamaparse.ts",
     "start:notion": "node --import tsx ./src/notion.ts",
-    "start:llamaparse-dir": "node --import tsx ./src/simple-directory-reader-with-llamaparse.ts"
+    "start:llamaparse2": "node --import tsx ./src/llamaparse_2.ts"
   },
   "dependencies": {
     "llamaindex": "*"

examples/readers/src/custom-simple-directory-reader.ts

@@ -1,13 +1,12 @@
-import type { Document, Metadata } from "llamaindex";
-import { FileReader } from "llamaindex";
+import type { BaseReader, Document, Metadata } from "llamaindex";
 import {
   FILE_EXT_TO_READER,
   SimpleDirectoryReader,
 } from "llamaindex/readers/SimpleDirectoryReader";
 import { TextFileReader } from "llamaindex/readers/TextFileReader";
 
-class ZipReader extends FileReader {
-  loadDataAsContent(fileContent: Buffer): Promise<Document<Metadata>[]> {
+class ZipReader implements BaseReader {
+  loadData(...args: any[]): Promise<Document<Metadata>[]> {
     throw new Error("Implement me");
   }
 }

examples/readers/src/llamaparse_2.ts

@@ -0,0 +1,26 @@
+import fs from "fs/promises";
+import { LlamaParseReader } from "llamaindex";
+
+async function main() {
+  // Load PDF using LlamaParse. set apiKey here or in environment variable LLAMA_CLOUD_API_KEY
+  const reader = new LlamaParseReader({
+    resultType: "markdown",
+    language: "en",
+    parsingInstruction:
+      "The provided document is a manga comic book. Most pages do NOT have title. It does not contain tables. Try to reconstruct the dialogue happening in a cohesive way. Output any math equation in LATEX markdown (between $$)",
+  });
+  const documents = await reader.loadData("../data/manga.pdf"); // The manga.pdf in the data folder is just a copy of the TOS, due to copyright laws. You have to place your own. I used "The Manga Guide to Calculus" by Hiroyuki Kojima
+
+  // Assuming documents contain an array of pages or sections
+  const parsedManga = documents.map((page) => page.text).join("\n---\n");
+
+  // Output the parsed manga to .md file. Will be placed in ../example/readers/
+  try {
+    await fs.writeFile("./parsedManga.md", parsedManga);
+    console.log("Output successfully written to parsedManga.md");
+  } catch (err) {
+    console.error("Error writing to file:", err);
+  }
+}
+
+main().catch(console.error);

examples/readers/src/simple-directory-reader-with-llamaparse.ts

@@ -1,35 +0,0 @@
-import {
-  LlamaParseReader,
-  SimpleDirectoryReader,
-  VectorStoreIndex,
-} from "llamaindex";
-
-async function main() {
-  const reader = new SimpleDirectoryReader();
-
-  const docs = await reader.loadData({
-    directoryPath: "../data/parallel", // brk-2022.pdf split into 6 parts
-    numWorkers: 2,
-    // set LlamaParse as the default reader for all file types. Set apiKey here or in environment variable LLAMA_CLOUD_API_KEY
-    overrideReader: new LlamaParseReader({
-      language: "en",
-      resultType: "markdown",
-      parsingInstruction:
-        "The provided files is Berkshire Hathaway's 2022 Annual Report. They contain figures, tables and raw data. Capture the data in a structured format. Mathematical equation should be put out as LATEX markdown (between $$).",
-    }),
-  });
-
-  const index = await VectorStoreIndex.fromDocuments(docs);
-
-  // Query the index
-  const queryEngine = index.asQueryEngine();
-  const response = await queryEngine.query({
-    query:
-      "What is the general strategy for shareholder safety outlined in the report? Use a concrete example with numbers",
-  });
-
-  // Output response
-  console.log(response.toString());
-}
-
-main().catch(console.error);

package.json

@@ -19,15 +19,15 @@
     "new-snapshot": "pnpm run build:release && changeset version --snapshot"
   },
   "devDependencies": {
-    "@changesets/cli": "^2.27.5",
-    "@typescript-eslint/eslint-plugin": "^7.11.0",
+    "@changesets/cli": "^2.27.1",
+    "@typescript-eslint/eslint-plugin": "^7.8.0",
     "eslint": "^8.57.0",
     "eslint-config-next": "^14.2.3",
     "eslint-config-prettier": "^9.1.0",
     "eslint-config-turbo": "^1.13.3",
     "eslint-plugin-react": "7.34.1",
     "husky": "^9.0.11",
-    "lint-staged": "^15.2.5",
+    "lint-staged": "^15.2.2",
     "madge": "^7.0.0",
     "prettier": "^3.2.5",
     "prettier-plugin-organize-imports": "^3.2.4",

packages/autotool/examples/01_node/CHANGELOG.md

@@ -4,36 +4,6 @@
 
 ### Patch Changes
 
-- Updated dependencies [6e156ed]
-- Updated dependencies [265976d]
-- Updated dependencies [8e26f75]
-  - llamaindex@0.3.15
-  - @llamaindex/autotool@0.0.1
-
-## null
-
-### Patch Changes
-
-- Updated dependencies [6ff7576]
-- Updated dependencies [94543de]
-  - llamaindex@0.3.14
-  - @llamaindex/autotool@0.0.1
-
-## null
-
-### Patch Changes
-
-- Updated dependencies [1b1081b]
-- Updated dependencies [37525df]
-- Updated dependencies [660a2b3]
-- Updated dependencies [a1f2475]
-  - llamaindex@0.3.13
-  - @llamaindex/autotool@0.0.1
-
-## null
-
-### Patch Changes
-
 - Updated dependencies [34fb1d8]
   - llamaindex@0.3.12
   - @llamaindex/autotool@0.0.1

packages/autotool/examples/02_nextjs/CHANGELOG.md

@@ -1,62 +1,5 @@
 # @llamaindex/autotool-02-next-example
 
-## 0.1.9
-
-### Patch Changes
-
-- Updated dependencies [6bc5bdd]
-- Updated dependencies [bf25ff6]
-- Updated dependencies [e6d6576]
-  - llamaindex@0.3.17
-  - @llamaindex/autotool@0.0.1
-
-## 0.1.8
-
-### Patch Changes
-
-- Updated dependencies [11ae926]
-- Updated dependencies [631f000]
-- Updated dependencies [1378ec4]
-- Updated dependencies [6b1ded4]
-- Updated dependencies [4d4bd85]
-- Updated dependencies [24a9d1e]
-- Updated dependencies [45952de]
-- Updated dependencies [54230f0]
-- Updated dependencies [a29d835]
-- Updated dependencies [73819bf]
-  - llamaindex@0.3.16
-  - @llamaindex/autotool@0.0.1
-
-## 0.1.7
-
-### Patch Changes
-
-- Updated dependencies [6e156ed]
-- Updated dependencies [265976d]
-- Updated dependencies [8e26f75]
-  - llamaindex@0.3.15
-  - @llamaindex/autotool@0.0.1
-
-## 0.1.6
-
-### Patch Changes
-
-- Updated dependencies [6ff7576]
-- Updated dependencies [94543de]
-  - llamaindex@0.3.14
-  - @llamaindex/autotool@0.0.1
-
-## 0.1.5
-
-### Patch Changes
-
-- Updated dependencies [1b1081b]
-- Updated dependencies [37525df]
-- Updated dependencies [660a2b3]
-- Updated dependencies [a1f2475]
-  - llamaindex@0.3.13
-  - @llamaindex/autotool@0.0.1
-
 ## 0.1.4
 
 ### Patch Changes

packages/autotool/examples/02_nextjs/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/autotool-02-next-example",
   "private": true,
-  "version": "0.1.9",
+  "version": "0.1.4",
   "scripts": {
     "dev": "next dev",
     "build": "next build",

packages/autotool/package.json

@@ -51,7 +51,7 @@
     "unplugin": "^1.10.1"
   },
   "peerDependencies": {
-    "llamaindex": "^0.3.17",
+    "llamaindex": "^0.3.12",
     "openai": "^4",
     "typescript": "^4"
   },

packages/community/CHANGELOG.md

@@ -1,27 +0,0 @@
-# @llamaindex/community
-
-## 0.0.3
-
-### Patch Changes
-
-- Updated dependencies [6bc5bdd]
-- Updated dependencies [bf25ff6]
-- Updated dependencies [e6d6576]
-  - llamaindex@0.3.17
-
-## 0.0.2
-
-### Patch Changes
-
-- 8832669: Community bedrock support added
-- Updated dependencies [11ae926]
-- Updated dependencies [631f000]
-- Updated dependencies [1378ec4]
-- Updated dependencies [6b1ded4]
-- Updated dependencies [4d4bd85]
-- Updated dependencies [24a9d1e]
-- Updated dependencies [45952de]
-- Updated dependencies [54230f0]
-- Updated dependencies [a29d835]
-- Updated dependencies [73819bf]
-  - llamaindex@0.3.16

packages/community/jsr.json

@@ -1,8 +0,0 @@
-{
-  "name": "@llamaindex/community",
-  "version": "0.0.5",
-  "exports": {
-    ".": "./src/index.ts",
-    "./type": "./src/type.ts"
-  }
-}

packages/community/package.json

@@ -1,57 +0,0 @@
-{
-  "name": "@llamaindex/community",
-  "description": "Community package for LlamaIndexTS",
-  "version": "0.0.3",
-  "type": "module",
-  "types": "dist/type/index.d.ts",
-  "main": "dist/cjs/index.js",
-  "exports": {
-    ".": {
-      "import": {
-        "types": "./dist/type/index.d.ts",
-        "default": "./dist/index.js"
-      },
-      "require": {
-        "types": "./dist/type/index.d.ts",
-        "default": "./dist/index.cjs"
-      }
-    },
-    "./llm/bedrock": {
-      "import": {
-        "types": "./dist/type/llm/bedrock.d.ts",
-        "default": "./dist/llm/bedrock/base.js"
-      },
-      "require": {
-        "types": "./dist/type/llm/bedrock.d.ts",
-        "default": "./dist/llm/bedrock/base.cjs"
-      }
-    }
-  },
-  "files": [
-    "dist",
-    "CHANGELOG.md",
-    "!**/*.tsbuildinfo"
-  ],
-  "repository": {
-    "type": "git",
-    "url": "https://github.com/run-llama/LlamaIndexTS.git",
-    "directory": "packages/community"
-  },
-  "scripts": {
-    "build": "rm -rf ./dist && pnpm run build:code && pnpm run build:type",
-    "build:code": "tsup",
-    "build:type": "tsc -p tsconfig.json",
-    "dev": "concurrently \"pnpm run build:esm --watch\" \"pnpm run build:cjs --watch\" \"pnpm run build:type --watch\""
-  },
-  "devDependencies": {
-    "@swc/cli": "^0.3.12",
-    "@swc/core": "^1.5.5",
-    "concurrently": "^8.2.2",
-    "tsup": "^8.1.0"
-  },
-  "dependencies": {
-    "@aws-sdk/client-bedrock-runtime": "^3.582.0",
-    "@types/node": "^20.14.2",
-    "llamaindex": "workspace:*"
-  }
-}

packages/community/src/index.ts

@@ -1 +0,0 @@
-export { BEDROCK_MODELS, Bedrock } from "./llm/bedrock/base.js";

packages/community/src/llm/bedrock/base.ts

@@ -1,352 +0,0 @@
-import {
-  BedrockRuntimeClient,
-  InvokeModelCommand,
-  InvokeModelWithResponseStreamCommand,
-  type BedrockRuntimeClientConfig,
-  type InvokeModelCommandInput,
-  type InvokeModelWithResponseStreamCommandInput,
-} from "@aws-sdk/client-bedrock-runtime";
-
-import type {
-  ChatMessage,
-  ChatResponse,
-  ChatResponseChunk,
-  CompletionResponse,
-  LLMChatParamsNonStreaming,
-  LLMChatParamsStreaming,
-  LLMCompletionParamsNonStreaming,
-  LLMCompletionParamsStreaming,
-  LLMMetadata,
-  ToolCallLLMMessageOptions,
-} from "llamaindex";
-import { ToolCallLLM, streamConverter, wrapLLMEvent } from "llamaindex";
-import type {
-  AnthropicNoneStreamingResponse,
-  AnthropicTextContent,
-  StreamEvent,
-} from "./types.js";
-import {
-  mapChatMessagesToAnthropicMessages,
-  mapMessageContentToMessageContentDetails,
-  toUtf8,
-} from "./utils.js";
-
-export type BedrockAdditionalChatOptions = {};
-
-export type BedrockChatParamsStreaming = LLMChatParamsStreaming<
-  BedrockAdditionalChatOptions,
-  ToolCallLLMMessageOptions
->;
-
-export type BedrockChatStreamResponse = AsyncIterable<
-  ChatResponseChunk<ToolCallLLMMessageOptions>
->;
-
-export type BedrockChatParamsNonStreaming = LLMChatParamsNonStreaming<
-  BedrockAdditionalChatOptions,
-  ToolCallLLMMessageOptions
->;
-
-export type BedrockChatNonStreamResponse =
-  ChatResponse<ToolCallLLMMessageOptions>;
-
-export enum BEDROCK_MODELS {
-  AMAZON_TITAN_TG1_LARGE = "amazon.titan-tg1-large",
-  AMAZON_TITAN_TEXT_EXPRESS_V1 = "amazon.titan-text-express-v1",
-  AI21_J2_GRANDE_INSTRUCT = "ai21.j2-grande-instruct",
-  AI21_J2_JUMBO_INSTRUCT = "ai21.j2-jumbo-instruct",
-  AI21_J2_MID = "ai21.j2-mid",
-  AI21_J2_MID_V1 = "ai21.j2-mid-v1",
-  AI21_J2_ULTRA = "ai21.j2-ultra",
-  AI21_J2_ULTRA_V1 = "ai21.j2-ultra-v1",
-  COHERE_COMMAND_TEXT_V14 = "cohere.command-text-v14",
-  ANTHROPIC_CLAUDE_INSTANT_1 = "anthropic.claude-instant-v1",
-  ANTHROPIC_CLAUDE_1 = "anthropic.claude-v1", // EOF: No longer supported
-  ANTHROPIC_CLAUDE_2 = "anthropic.claude-v2",
-  ANTHROPIC_CLAUDE_2_1 = "anthropic.claude-v2:1",
-  ANTHROPIC_CLAUDE_3_SONNET = "anthropic.claude-3-sonnet-20240229-v1:0",
-  ANTHROPIC_CLAUDE_3_HAIKU = "anthropic.claude-3-haiku-20240307-v1:0",
-  ANTHROPIC_CLAUDE_3_OPUS = "anthropic.claude-3-opus-20240229-v1:0",
-  META_LLAMA2_13B_CHAT = "meta.llama2-13b-chat-v1",
-  META_LLAMA2_70B_CHAT = "meta.llama2-70b-chat-v1",
-  META_LLAMA3_8B_INSTRUCT = "meta.llama3-8b-instruct-v1:0",
-  META_LLAMA3_70B_INSTRUCT = "meta.llama3-70b-instruct-v1:0",
-  MISTRAL_7B_INSTRUCT = "mistral.mistral-7b-instruct-v0:2",
-  MISTRAL_MIXTRAL_7B_INSTRUCT = "mistral.mixtral-8x7b-instruct-v0:1",
-  MISTRAL_MIXTRAL_LARGE_2402 = "mistral.mistral-large-2402-v1:0",
-}
-
-/*
- * Values taken from https://docs.aws.amazon.com/bedrock/latest/userguide/model-parameters.html#model-parameters-claude
- */
-
-const COMPLETION_MODELS = {
-  [BEDROCK_MODELS.AMAZON_TITAN_TG1_LARGE]: 8000,
-  [BEDROCK_MODELS.AMAZON_TITAN_TEXT_EXPRESS_V1]: 8000,
-  [BEDROCK_MODELS.AI21_J2_GRANDE_INSTRUCT]: 8000,
-  [BEDROCK_MODELS.AI21_J2_JUMBO_INSTRUCT]: 8000,
-  [BEDROCK_MODELS.AI21_J2_MID]: 8000,
-  [BEDROCK_MODELS.AI21_J2_MID_V1]: 8000,
-  [BEDROCK_MODELS.AI21_J2_ULTRA]: 8000,
-  [BEDROCK_MODELS.AI21_J2_ULTRA_V1]: 8000,
-  [BEDROCK_MODELS.COHERE_COMMAND_TEXT_V14]: 4096,
-};
-
-const CHAT_ONLY_MODELS = {
-  [BEDROCK_MODELS.ANTHROPIC_CLAUDE_INSTANT_1]: 100000,
-  [BEDROCK_MODELS.ANTHROPIC_CLAUDE_1]: 100000,
-  [BEDROCK_MODELS.ANTHROPIC_CLAUDE_2]: 100000,
-  [BEDROCK_MODELS.ANTHROPIC_CLAUDE_2_1]: 200000,
-  [BEDROCK_MODELS.ANTHROPIC_CLAUDE_3_SONNET]: 200000,
-  [BEDROCK_MODELS.ANTHROPIC_CLAUDE_3_HAIKU]: 200000,
-  [BEDROCK_MODELS.ANTHROPIC_CLAUDE_3_OPUS]: 200000,
-  [BEDROCK_MODELS.META_LLAMA2_13B_CHAT]: 2048,
-  [BEDROCK_MODELS.META_LLAMA2_70B_CHAT]: 4096,
-  [BEDROCK_MODELS.META_LLAMA3_8B_INSTRUCT]: 8192,
-  [BEDROCK_MODELS.META_LLAMA3_70B_INSTRUCT]: 8192,
-  [BEDROCK_MODELS.MISTRAL_7B_INSTRUCT]: 32000,
-  [BEDROCK_MODELS.MISTRAL_MIXTRAL_7B_INSTRUCT]: 32000,
-  [BEDROCK_MODELS.MISTRAL_MIXTRAL_LARGE_2402]: 32000,
-};
-
-const BEDROCK_FOUNDATION_LLMS = { ...COMPLETION_MODELS, ...CHAT_ONLY_MODELS };
-
-/*
- * Only the following models support streaming as
- * per result of Bedrock.Client.list_foundation_models
- * https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/bedrock/client/list_foundation_models.html
- */
-export const STREAMING_MODELS = new Set([
-  BEDROCK_MODELS.AMAZON_TITAN_TG1_LARGE,
-  BEDROCK_MODELS.AMAZON_TITAN_TEXT_EXPRESS_V1,
-  BEDROCK_MODELS.ANTHROPIC_CLAUDE_INSTANT_1,
-  BEDROCK_MODELS.ANTHROPIC_CLAUDE_1,
-  BEDROCK_MODELS.ANTHROPIC_CLAUDE_2,
-  BEDROCK_MODELS.ANTHROPIC_CLAUDE_2_1,
-  BEDROCK_MODELS.ANTHROPIC_CLAUDE_3_SONNET,
-  BEDROCK_MODELS.ANTHROPIC_CLAUDE_3_HAIKU,
-  BEDROCK_MODELS.ANTHROPIC_CLAUDE_3_OPUS,
-  BEDROCK_MODELS.META_LLAMA2_13B_CHAT,
-  BEDROCK_MODELS.META_LLAMA2_70B_CHAT,
-  BEDROCK_MODELS.META_LLAMA3_8B_INSTRUCT,
-  BEDROCK_MODELS.META_LLAMA3_70B_INSTRUCT,
-  BEDROCK_MODELS.MISTRAL_7B_INSTRUCT,
-  BEDROCK_MODELS.MISTRAL_MIXTRAL_7B_INSTRUCT,
-  BEDROCK_MODELS.MISTRAL_MIXTRAL_LARGE_2402,
-]);
-
-abstract class Provider {
-  abstract getTextFromResponse(response: Record<string, any>): string;
-
-  getTextFromStreamResponse(response: Record<string, any>): string {
-    return this.getTextFromResponse(response);
-  }
-
-  abstract getRequestBody<T extends ChatMessage>(
-    metadata: LLMMetadata,
-    messages: T[],
-  ): InvokeModelCommandInput | InvokeModelWithResponseStreamCommandInput;
-}
-
-class AnthropicProvider extends Provider {
-  getResultFromResponse(
-    response: Record<string, any>,
-  ): AnthropicNoneStreamingResponse {
-    return JSON.parse(toUtf8(response.body));
-  }
-
-  getTextFromResponse(response: Record<string, any>): string {
-    const result = this.getResultFromResponse(response);
-    return result.content
-      .filter((item) => item.type === "text")
-      .map((item) => (item as AnthropicTextContent).text)
-      .join(" ");
-  }
-
-  getTextFromStreamResponse(response: Record<string, any>): string {
-    const event: StreamEvent | undefined = response.chunk?.bytes
-      ? JSON.parse(toUtf8(response.chunk?.bytes))
-      : undefined;
-
-    if (event?.type === "content_block_delta") return event.delta.text;
-    return "";
-  }
-
-  getRequestBody<T extends ChatMessage>(
-    metadata: LLMMetadata,
-    messages: T[],
-  ): InvokeModelCommandInput | InvokeModelWithResponseStreamCommandInput {
-    return {
-      modelId: metadata.model,
-      contentType: "application/json",
-      accept: "application/json",
-      body: JSON.stringify({
-        anthropic_version: "bedrock-2023-05-31",
-        messages: mapChatMessagesToAnthropicMessages(messages),
-        max_tokens: metadata.maxTokens,
-        temperature: metadata.temperature,
-        top_p: metadata.topP,
-      }),
-    };
-  }
-}
-
-// Other providers could go here
-const PROVIDERS: { [key: string]: Provider } = {
-  anthropic: new AnthropicProvider(),
-};
-
-const getProvider = (model: string): Provider => {
-  const providerName = model.split(".")[0];
-  if (!(providerName in PROVIDERS)) {
-    throw new Error(
-      `Provider ${providerName} for model ${model} is not supported`,
-    );
-  }
-  return PROVIDERS[providerName];
-};
-
-export type BedrockModelParams = {
-  model: keyof typeof BEDROCK_FOUNDATION_LLMS;
-  temperature?: number;
-  topP?: number;
-  maxTokens?: number;
-};
-
-const DEFAULT_BEDROCK_PARAMS = {
-  temperature: 0.1,
-  topP: 1,
-  maxTokens: 1024, // required by anthropic
-};
-
-export type BedrockParams = BedrockModelParams & BedrockRuntimeClientConfig;
-
-/**
- * ToolCallLLM for Bedrock
- */
-export class Bedrock extends ToolCallLLM<BedrockAdditionalChatOptions> {
-  private client: BedrockRuntimeClient;
-  model: keyof typeof BEDROCK_FOUNDATION_LLMS;
-  temperature: number;
-  topP: number;
-  maxTokens?: number;
-  provider: Provider;
-  topK?: number;
-
-  // there should be no check for env variables. Bedrock can be authenticated in various ways
-  // AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY and AWS_REGION are the env variables used directly by the sdk
-  constructor({
-    temperature,
-    topP,
-    maxTokens,
-    model,
-    ...params
-  }: BedrockParams) {
-    super();
-
-    this.model = model;
-    this.provider = getProvider(this.model);
-    this.maxTokens = maxTokens ?? DEFAULT_BEDROCK_PARAMS.maxTokens;
-    this.temperature = temperature ?? DEFAULT_BEDROCK_PARAMS.temperature;
-    this.topP = topP ?? DEFAULT_BEDROCK_PARAMS.topP;
-    this.client = new BedrockRuntimeClient(params);
-  }
-
-  get supportToolCall(): boolean {
-    return false;
-  }
-
-  get metadata(): LLMMetadata {
-    // NOTE, Anthropic supports top_k but LLMMetadata does not
-    return {
-      model: this.model,
-      temperature: this.temperature,
-      topP: this.topP,
-      maxTokens: this.maxTokens,
-      contextWindow: BEDROCK_FOUNDATION_LLMS[this.model],
-      tokenizer: undefined,
-    };
-  }
-
-  protected async nonStreamChat(
-    params: BedrockChatParamsNonStreaming,
-  ): Promise<BedrockChatNonStreamResponse> {
-    const input = this.provider.getRequestBody(this.metadata, params.messages);
-    const command = new InvokeModelCommand(input);
-    const response = await this.client.send(command);
-    return {
-      raw: response,
-      message: {
-        content: this.provider.getTextFromResponse(response),
-        role: "assistant",
-      },
-    };
-  }
-
-  protected async *streamChat(
-    params: BedrockChatParamsStreaming,
-  ): BedrockChatStreamResponse {
-    if (!STREAMING_MODELS.has(this.model))
-      throw new Error(`The model: ${this.model} does not support streaming`);
-    const input = this.provider.getRequestBody(this.metadata, params.messages);
-    const command = new InvokeModelWithResponseStreamCommand(input);
-    const response = await this.client.send(command);
-
-    if (response.body)
-      yield* streamConverter(response.body, (response) => {
-        return {
-          delta: this.provider.getTextFromStreamResponse(response),
-          raw: response,
-        };
-      });
-  }
-
-  chat(params: BedrockChatParamsStreaming): Promise<BedrockChatStreamResponse>;
-  chat(
-    params: BedrockChatParamsNonStreaming,
-  ): Promise<BedrockChatNonStreamResponse>;
-
-  @wrapLLMEvent
-  async chat(
-    params: BedrockChatParamsStreaming | BedrockChatParamsNonStreaming,
-  ): Promise<BedrockChatStreamResponse | BedrockChatNonStreamResponse> {
-    if (params.stream) return this.streamChat(params);
-    return this.nonStreamChat(params);
-  }
-
-  complete(
-    params: LLMCompletionParamsStreaming,
-  ): Promise<AsyncIterable<CompletionResponse>>;
-  complete(
-    params: LLMCompletionParamsNonStreaming,
-  ): Promise<CompletionResponse>;
-  async complete(
-    params: LLMCompletionParamsStreaming | LLMCompletionParamsNonStreaming,
-  ): Promise<CompletionResponse | AsyncIterable<CompletionResponse>> {
-    const message: ChatMessage = {
-      role: "user",
-      content: mapMessageContentToMessageContentDetails(params.prompt),
-    };
-
-    const input = this.provider.getRequestBody(this.metadata, [message]);
-
-    if (params.stream) {
-      const command = new InvokeModelWithResponseStreamCommand(input);
-      const response = await this.client.send(command);
-      if (response.body)
-        return streamConverter(response.body, (response) => {
-          return {
-            text: this.provider.getTextFromStreamResponse(response),
-            raw: response,
-          };
-        });
-    }
-
-    const command = new InvokeModelCommand(input);
-    const response = await this.client.send(command);
-    return {
-      text: this.provider.getTextFromResponse(response),
-      raw: response,
-    };
-  }
-}

packages/community/src/llm/bedrock/types.ts

@@ -1,109 +0,0 @@
-type Usage = {
-  input_tokens: number;
-  output_tokens: number;
-};
-
-type Message = {
-  id: string;
-  type: string;
-  role: string;
-  content: string[];
-  model: string;
-  stop_reason: string | null;
-  stop_sequence: string | null;
-  usage: Usage;
-};
-
-type ContentBlockStart = {
-  type: "content_block_start";
-  index: number;
-  content_block: {
-    type: string;
-    text: string;
-  };
-};
-
-type Delta = {
-  type: string;
-  text: string;
-};
-
-type ContentBlockDelta = {
-  type: "content_block_delta";
-  index: number;
-  delta: Delta;
-};
-
-type ContentBlockStop = {
-  type: "content_block_stop";
-  index: number;
-};
-
-type MessageDelta = {
-  type: "message_delta";
-  delta: {
-    stop_reason: string;
-    stop_sequence: string | null;
-  };
-  usage: Usage;
-};
-
-type InvocationMetrics = {
-  inputTokenCount: number;
-  outputTokenCount: number;
-  invocationLatency: number;
-  firstByteLatency: number;
-};
-
-type MessageStop = {
-  type: "message_stop";
-  "amazon-bedrock-invocationMetrics": InvocationMetrics;
-};
-
-export type StreamEvent =
-  | { type: "message_start"; message: Message }
-  | ContentBlockStart
-  | ContentBlockDelta
-  | ContentBlockStop
-  | MessageDelta
-  | MessageStop;
-
-export type AnthropicContent = AnthropicTextContent | AnthropicImageContent;
-
-export type AnthropicTextContent = {
-  type: "text";
-  text: string;
-};
-
-export type AnthropicMediaTypes =
-  | "image/jpeg"
-  | "image/png"
-  | "image/webp"
-  | "image/gif";
-
-export type AnthropicImageSource = {
-  type: "base64";
-  media_type: AnthropicMediaTypes;
-  data: string; // base64 encoded image bytes
-};
-
-export type AnthropicImageContent = {
-  type: "image";
-  source: AnthropicImageSource;
-};
-
-export type AnthropicMessage = {
-  role: "user" | "assistant";
-  content: AnthropicContent[];
-};
-
-export type AnthropicNoneStreamingResponse = {
-  id: string;
-  type: "message";
-  role: "assistant";
-  content: AnthropicContent[];
-  model: string;
-  stop_reason: "end_turn" | "max_tokens" | "stop_sequence";
-  stop_sequence?: string;
-  usage: { input_tokens: number; output_tokens: number };
-};

packages/community/src/llm/bedrock/utils.ts

@@ -1,144 +0,0 @@
-import type {
-  ChatMessage,
-  MessageContent,
-  MessageContentDetail,
-} from "llamaindex";
-import type {
-  AnthropicContent,
-  AnthropicImageContent,
-  AnthropicMediaTypes,
-  AnthropicMessage,
-  AnthropicTextContent,
-} from "./types.js";
-
-const ACCEPTED_IMAGE_MIME_TYPES = [
-  "image/jpeg",
-  "image/png",
-  "image/webp",
-  "image/gif",
-];
-
-export const mapMessageContentToMessageContentDetails = (
-  content: MessageContent,
-): MessageContentDetail[] => {
-  return Array.isArray(content) ? content : [{ type: "text", text: content }];
-};
-
-export const mergeNeighboringSameRoleMessages = (
-  messages: AnthropicMessage[],
-): AnthropicMessage[] => {
-  return messages.reduce(
-    (result: AnthropicMessage[], current: AnthropicMessage, index: number) => {
-      if (index > 0 && messages[index - 1].role === current.role) {
-        result[result.length - 1].content = [
-          ...result[result.length - 1].content,
-          ...current.content,
-        ];
-      } else {
-        result.push(current);
-      }
-      return result;
-    },
-    [],
-  );
-};
-
-export const mapMessageContentDetailToAnthropicContent = <
-  T extends MessageContentDetail,
->(
-  detail: T,
-): AnthropicContent => {
-  let content: AnthropicContent;
-
-  if (detail.type === "text") {
-    content = mapTextContent(detail.text);
-  } else if (detail.type === "image_url") {
-    content = mapImageContent(detail.image_url.url);
-  } else {
-    throw new Error("Unsupported content detail type");
-  }
-  return content;
-};
-
-export const mapMessageContentToAnthropicContent = <T extends MessageContent>(
-  content: T,
-): AnthropicContent[] => {
-  return mapMessageContentToMessageContentDetails(content).map(
-    mapMessageContentDetailToAnthropicContent,
-  );
-};
-
-export const mapChatMessagesToAnthropicMessages = <T extends ChatMessage>(
-  messages: T[],
-): AnthropicMessage[] => {
-  const mapped = messages
-    .flatMap((msg: T): AnthropicMessage[] => {
-      return mapMessageContentToMessageContentDetails(msg.content).map(
-        (detail: MessageContentDetail): AnthropicMessage => {
-          const content = mapMessageContentDetailToAnthropicContent(detail);
-
-          return {
-            role: msg.role === "assistant" ? "assistant" : "user",
-            content: [content],
-          };
-        },
-      );
-    })
-    .filter((message: AnthropicMessage) => {
-      const content = message.content[0];
-      if (content.type === "text" && !content.text) return false;
-      if (content.type === "image" && !content.source.data) return false;
-      return true;
-    });
-
-  return mergeNeighboringSameRoleMessages(mapped);
-};
-
-export const mapTextContent = (text: string): AnthropicTextContent => {
-  return { type: "text", text };
-};
-
-export const extractDataUrlComponents = (
-  dataUrl: string,
-): {
-  mimeType: string;
-  base64: string;
-} => {
-  const parts = dataUrl.split(";base64,");
-
-  if (parts.length !== 2 || !parts[0].startsWith("data:")) {
-    throw new Error("Invalid data URL");
-  }
-
-  const mimeType = parts[0].slice(5);
-  const base64 = parts[1];
-
-  return {
-    mimeType,
-    base64,
-  };
-};
-
-export const mapImageContent = (imageUrl: string): AnthropicImageContent => {
-  if (!imageUrl.startsWith("data:"))
-    throw new Error(
-      "For Anthropic please only use base64 data url, e.g.: data:image/jpeg;base64,SGVsbG8sIFdvcmxkIQ==",
-    );
-  const { mimeType, base64: data } = extractDataUrlComponents(imageUrl);
-  if (!ACCEPTED_IMAGE_MIME_TYPES.includes(mimeType))
-    throw new Error(
-      `Anthropic only accepts the following mimeTypes: ${ACCEPTED_IMAGE_MIME_TYPES.join("\n")}`,
-    );
-
-  return {
-    type: "image",
-    source: {
-      type: "base64",
-      media_type: mimeType as AnthropicMediaTypes,
-      data,
-    },
-  };
-};
-
-export const toUtf8 = (input: Uint8Array): string =>
-  new TextDecoder("utf-8").decode(input);

packages/community/tsconfig.json

@@ -1,19 +0,0 @@
-{
-  "extends": "../../tsconfig.json",
-  "compilerOptions": {
-    "rootDir": "./src",
-    "outDir": "./dist/type",
-    "tsBuildInfoFile": "./dist/.tsbuildinfo",
-    "emitDeclarationOnly": true,
-    "module": "ESNext",
-    "moduleResolution": "bundler",
-    "types": ["node"]
-  },
-  "include": ["./src"],
-  "exclude": ["node_modules"],
-  "references": [
-    {
-      "path": "../core/tsconfig.json"
-    }
-  ]
-}

packages/community/tsconfig.script.json

@@ -1,9 +0,0 @@
-{
-  "extends": "../../tsconfig.json",
-  "compilerOptions": {
-    "outDir": "./dist/script/type",
-    "tsBuildInfoFile": "./dist/script/.tsbuildinfo",
-    "emitDeclarationOnly": true
-  },
-  "include": ["./tsup.config.ts"]
-}

packages/community/tsup.config.ts

@@ -1,9 +0,0 @@
-import { defineConfig } from "tsup";
-
-export default defineConfig([
-  {
-    entry: ["src/index.ts", "src/llm/bedrock/base.ts"],
-    format: ["cjs", "esm"],
-    sourcemap: true,
-  },
-]);

packages/core/CHANGELOG.md

@@ -1,52 +1,5 @@
 # llamaindex
 
-## 0.3.17
-
-### Patch Changes
-
-- 6bc5bdd: feat: add cache disabling, fast mode, do not unroll columns mode and custom page seperator to LlamaParseReader
-- bf25ff6: fix: polyfill for cloudflare worker
-- e6d6576: chore: use `unpdf`
-
-## 0.3.16
-
-### Patch Changes
-
-- 11ae926: feat: add numCandidates setting to MongoDBAtlasVectorStore for tuning queries
-- 631f000: feat: DeepInfra LLM implementation
-- 1378ec4: feat: set default model to `gpt-4o`
-- 6b1ded4: add gpt4o-mode, invalidate cache and skip diagonal text to LlamaParseReader
-- 4d4bd85: Show error message if agent tool is called with partial JSON
-- 24a9d1e: add json mode and image retrieval to LlamaParseReader
-- 45952de: add concurrency management for SimpleDirectoryReader
-- 54230f0: feat: Gemini GA release models
-- a29d835: setDocumentHash should be async
-- 73819bf: Unify metadata and ID handling of documents, allow files to be read by `Buffer`
-
-## 0.3.15
-
-### Patch Changes
-
-- 6e156ed: Use images in context chat engine
-- 265976d: fix bug with node decorator
-- 8e26f75: Add retrieval for images using multi-modal messages
-
-## 0.3.14
-
-### Patch Changes
-
-- 6ff7576: Added GPT-4o for Azure
-- 94543de: Added the latest preview gemini models and multi modal images taken into account
-
-## 0.3.13
-
-### Patch Changes
-
-- 1b1081b: Add vectorStores to storage context to define vector store per modality
-- 37525df: Added support for accessing Gemini via Vertex AI
-- 660a2b3: Fix text before heading in markdown reader
-- a1f2475: Add system prompt to ContextChatEngine
-
 ## 0.3.12
 
 ### Patch Changes

packages/core/e2e/CHANGELOG.md

@@ -1,11 +1,5 @@
 # @llamaindex/core-e2e
 
-## 0.0.7
-
-### Patch Changes
-
-- bf25ff6: fix: polyfill for cloudflare worker
-
 ## 0.0.6
 
 ### Patch Changes

packages/core/e2e/examples/cloudflare-worker-agent/CHANGELOG.md

@@ -1,58 +1,5 @@
 # @llamaindex/cloudflare-worker-agent-test
 
-## 0.0.18
-
-### Patch Changes
-
-- bf25ff6: fix: polyfill for cloudflare worker
-- Updated dependencies [6bc5bdd]
-- Updated dependencies [bf25ff6]
-- Updated dependencies [e6d6576]
-  - llamaindex@0.3.17
-
-## 0.0.17
-
-### Patch Changes
-
-- Updated dependencies [11ae926]
-- Updated dependencies [631f000]
-- Updated dependencies [1378ec4]
-- Updated dependencies [6b1ded4]
-- Updated dependencies [4d4bd85]
-- Updated dependencies [24a9d1e]
-- Updated dependencies [45952de]
-- Updated dependencies [54230f0]
-- Updated dependencies [a29d835]
-- Updated dependencies [73819bf]
-  - llamaindex@0.3.16
-
-## 0.0.16
-
-### Patch Changes
-
-- Updated dependencies [6e156ed]
-- Updated dependencies [265976d]
-- Updated dependencies [8e26f75]
-  - llamaindex@0.3.15
-
-## 0.0.15
-
-### Patch Changes
-
-- Updated dependencies [6ff7576]
-- Updated dependencies [94543de]
-  - llamaindex@0.3.14
-
-## 0.0.14
-
-### Patch Changes
-
-- Updated dependencies [1b1081b]
-- Updated dependencies [37525df]
-- Updated dependencies [660a2b3]
-- Updated dependencies [a1f2475]
-  - llamaindex@0.3.13
-
 ## 0.0.13
 
 ### Patch Changes

packages/core/e2e/examples/cloudflare-worker-agent/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/cloudflare-worker-agent-test",
-  "version": "0.0.18",
+  "version": "0.0.13",
   "type": "module",
   "private": true,
   "scripts": {
@@ -12,13 +12,13 @@
     "cf-typegen": "wrangler types"
   },
   "devDependencies": {
-    "@cloudflare/vitest-pool-workers": "^0.4.3",
-    "@cloudflare/workers-types": "^4.20240605.0",
+    "@cloudflare/vitest-pool-workers": "^0.2.6",
+    "@cloudflare/workers-types": "^4.20240502.0",
     "@vitest/runner": "1.3.0",
     "@vitest/snapshot": "1.3.0",
     "typescript": "^5.4.5",
     "vitest": "1.3.0",
-    "wrangler": "^3.60.1"
+    "wrangler": "^3.53.1"
   },
   "dependencies": {
     "llamaindex": "workspace:*"

packages/core/e2e/examples/cloudflare-worker-agent/src/index.ts

@@ -10,12 +10,10 @@ export default {
     const agent = new OpenAIAgent({
       tools: [],
     });
-    console.log(1);
     const responseStream = await agent.chat({
       stream: true,
       message: "Hello? What is the weather today?",
     });
-    console.log(2);
     const textEncoder = new TextEncoder();
     const response = responseStream.pipeThrough<Uint8Array>(
       // @ts-expect-error: see https://github.com/cloudflare/workerd/issues/2067

packages/core/e2e/examples/nextjs-agent/CHANGELOG.md

@@ -1,57 +1,5 @@
 # @llamaindex/next-agent-test
 
-## 0.1.18
-
-### Patch Changes
-
-- Updated dependencies [6bc5bdd]
-- Updated dependencies [bf25ff6]
-- Updated dependencies [e6d6576]
-  - llamaindex@0.3.17
-
-## 0.1.17
-
-### Patch Changes
-
-- Updated dependencies [11ae926]
-- Updated dependencies [631f000]
-- Updated dependencies [1378ec4]
-- Updated dependencies [6b1ded4]
-- Updated dependencies [4d4bd85]
-- Updated dependencies [24a9d1e]
-- Updated dependencies [45952de]
-- Updated dependencies [54230f0]
-- Updated dependencies [a29d835]
-- Updated dependencies [73819bf]
-  - llamaindex@0.3.16
-
-## 0.1.16
-
-### Patch Changes
-
-- Updated dependencies [6e156ed]
-- Updated dependencies [265976d]
-- Updated dependencies [8e26f75]
-  - llamaindex@0.3.15
-
-## 0.1.15
-
-### Patch Changes
-
-- Updated dependencies [6ff7576]
-- Updated dependencies [94543de]
-  - llamaindex@0.3.14
-
-## 0.1.14
-
-### Patch Changes
-
-- Updated dependencies [1b1081b]
-- Updated dependencies [37525df]
-- Updated dependencies [660a2b3]
-- Updated dependencies [a1f2475]
-  - llamaindex@0.3.13
-
 ## 0.1.13
 
 ### Patch Changes

packages/core/e2e/examples/nextjs-agent/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/next-agent-test",
-  "version": "0.1.18",
+  "version": "0.1.13",
   "private": true,
   "scripts": {
     "dev": "next dev",

packages/core/e2e/examples/nextjs-edge-runtime/CHANGELOG.md

@@ -1,57 +1,5 @@
 # test-edge-runtime
 
-## 0.1.17
-
-### Patch Changes
-
-- Updated dependencies [6bc5bdd]
-- Updated dependencies [bf25ff6]
-- Updated dependencies [e6d6576]
-  - llamaindex@0.3.17
-
-## 0.1.16
-
-### Patch Changes
-
-- Updated dependencies [11ae926]
-- Updated dependencies [631f000]
-- Updated dependencies [1378ec4]
-- Updated dependencies [6b1ded4]
-- Updated dependencies [4d4bd85]
-- Updated dependencies [24a9d1e]
-- Updated dependencies [45952de]
-- Updated dependencies [54230f0]
-- Updated dependencies [a29d835]
-- Updated dependencies [73819bf]
-  - llamaindex@0.3.16
-
-## 0.1.15
-
-### Patch Changes
-
-- Updated dependencies [6e156ed]
-- Updated dependencies [265976d]
-- Updated dependencies [8e26f75]
-  - llamaindex@0.3.15
-
-## 0.1.14
-
-### Patch Changes
-
-- Updated dependencies [6ff7576]
-- Updated dependencies [94543de]
-  - llamaindex@0.3.14
-
-## 0.1.13
-
-### Patch Changes
-
-- Updated dependencies [1b1081b]
-- Updated dependencies [37525df]
-- Updated dependencies [660a2b3]
-- Updated dependencies [a1f2475]
-  - llamaindex@0.3.13
-
 ## 0.1.12
 
 ### Patch Changes

packages/core/e2e/examples/nextjs-edge-runtime/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/nextjs-edge-runtime-test",
-  "version": "0.1.17",
+  "version": "0.1.12",
   "private": true,
   "scripts": {
     "dev": "next dev",

packages/core/e2e/examples/waku-query-engine/CHANGELOG.md

@@ -1,57 +1,5 @@
 # @llamaindex/waku-query-engine-test
 
-## 0.0.18
-
-### Patch Changes
-
-- Updated dependencies [6bc5bdd]
-- Updated dependencies [bf25ff6]
-- Updated dependencies [e6d6576]
-  - llamaindex@0.3.17
-
-## 0.0.17
-
-### Patch Changes
-
-- Updated dependencies [11ae926]
-- Updated dependencies [631f000]
-- Updated dependencies [1378ec4]
-- Updated dependencies [6b1ded4]
-- Updated dependencies [4d4bd85]
-- Updated dependencies [24a9d1e]
-- Updated dependencies [45952de]
-- Updated dependencies [54230f0]
-- Updated dependencies [a29d835]
-- Updated dependencies [73819bf]
-  - llamaindex@0.3.16
-
-## 0.0.16
-
-### Patch Changes
-
-- Updated dependencies [6e156ed]
-- Updated dependencies [265976d]
-- Updated dependencies [8e26f75]
-  - llamaindex@0.3.15
-
-## 0.0.15
-
-### Patch Changes
-
-- Updated dependencies [6ff7576]
-- Updated dependencies [94543de]
-  - llamaindex@0.3.14
-
-## 0.0.14
-
-### Patch Changes
-
-- Updated dependencies [1b1081b]
-- Updated dependencies [37525df]
-- Updated dependencies [660a2b3]
-- Updated dependencies [a1f2475]
-  - llamaindex@0.3.13
-
 ## 0.0.13
 
 ### Patch Changes

packages/core/e2e/examples/waku-query-engine/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/waku-query-engine-test",
-  "version": "0.0.18",
+  "version": "0.0.13",
   "type": "module",
   "private": true,
   "scripts": {

packages/core/e2e/fixtures/embeddings/OpenAIEmbedding.ts

@@ -1,14 +1,9 @@
-import {
-  BaseNode,
-  SimilarityType,
-  type BaseEmbedding,
-  type MessageContentDetail,
-} from "llamaindex";
+import { BaseNode, SimilarityType, type BaseEmbedding } from "llamaindex";
 
 export class OpenAIEmbedding implements BaseEmbedding {
   embedBatchSize = 512;
 
-  async getQueryEmbedding(query: MessageContentDetail) {
+  async getQueryEmbedding(query: string) {
     return [0];
   }
 

packages/core/e2e/node/openai.e2e.ts

@@ -191,8 +191,7 @@ For questions about more specific sections, please use the vector_tool.`,
     message:
       "What's the summary of Alex? Does he live in Brazil based on the brief information? Return yes or no.",
   });
-  // not sure if AI calls twice("Alex", "Brazil") or once ("Alex in Brazil") or something else
-  ok(mockCall.mock.callCount() >= 1);
+  strictEqual(mockCall.mock.callCount(), 1);
 
   consola.debug("response:", response.message.content);
   ok(extractText(response.message.content).toLowerCase().includes("no"));

packages/core/e2e/node/snapshot/agent.snap

@@ -20,23 +20,21 @@
           "content": "",
           "role": "assistant",
           "options": {
-            "toolCall": [
-              {
-                "id": "call_sH6QfjsymHW7JFl68j8AY6xg",
-                "name": "Weather",
-                "input": "{\"location\":\"San Francisco\"}"
-              }
-            ]
+            "toolCall": {
+              "id": "call_Enrc7RhNts0qxGaEATOEhV6F",
+              "name": "Weather",
+              "input": "{\"location\":\"San Francisco\"}"
+            }
           }
         },
         {
-          "role": "user",
           "content": "35 degrees and sunny in San Francisco",
+          "role": "user",
           "options": {
             "toolResult": {
               "result": "35 degrees and sunny in San Francisco",
               "isError": false,
-              "id": "call_sH6QfjsymHW7JFl68j8AY6xg"
+              "id": "call_Enrc7RhNts0qxGaEATOEhV6F"
             }
           }
         }
@@ -62,23 +60,21 @@
           "content": "",
           "role": "assistant",
           "options": {
-            "toolCall": [
-              {
-                "id": "call_V7zs8cyDT5FqJhjwBqcCydgA",
-                "name": "unique_id",
-                "input": "{\"firstName\":\"Alex\",\"lastName\":\"Yang\"}"
-              }
-            ]
+            "toolCall": {
+              "id": "call_OYo81prrpK2RLOS9d0Y6kiXB",
+              "name": "unique_id",
+              "input": "{\"firstName\":\"Alex\",\"lastName\":\"Yang\"}"
+            }
           }
         },
         {
-          "role": "user",
           "content": "123456789",
+          "role": "user",
           "options": {
             "toolResult": {
               "result": "123456789",
               "isError": false,
-              "id": "call_V7zs8cyDT5FqJhjwBqcCydgA"
+              "id": "call_OYo81prrpK2RLOS9d0Y6kiXB"
             }
           }
         }
@@ -104,23 +100,21 @@
           "content": "",
           "role": "assistant",
           "options": {
-            "toolCall": [
-              {
-                "id": "call_BrlGGU6GDGWr0hrwXt9qZKyt",
-                "name": "sumNumbers",
-                "input": "{\"a\":1,\"b\":1}"
-              }
-            ]
+            "toolCall": {
+              "id": "call_YUXowHKwOJ6GIWV4EA59x8iF",
+              "name": "sumNumbers",
+              "input": "{\"a\":1,\"b\":1}"
+            }
           }
         },
         {
-          "role": "user",
           "content": "2",
+          "role": "user",
           "options": {
             "toolResult": {
               "result": "2",
               "isError": false,
-              "id": "call_BrlGGU6GDGWr0hrwXt9qZKyt"
+              "id": "call_YUXowHKwOJ6GIWV4EA59x8iF"
             }
           }
         }
@@ -136,13 +130,11 @@
           "content": "",
           "role": "assistant",
           "options": {
-            "toolCall": [
-              {
-                "id": "call_sH6QfjsymHW7JFl68j8AY6xg",
-                "name": "Weather",
-                "input": "{\"location\":\"San Francisco\"}"
-              }
-            ]
+            "toolCall": {
+              "id": "call_Enrc7RhNts0qxGaEATOEhV6F",
+              "name": "Weather",
+              "input": "{\"location\":\"San Francisco\"}"
+            }
           }
         }
       }
@@ -166,13 +158,11 @@
           "content": "",
           "role": "assistant",
           "options": {
-            "toolCall": [
-              {
-                "id": "call_V7zs8cyDT5FqJhjwBqcCydgA",
-                "name": "unique_id",
-                "input": "{\"firstName\":\"Alex\",\"lastName\":\"Yang\"}"
-              }
-            ]
+            "toolCall": {
+              "id": "call_OYo81prrpK2RLOS9d0Y6kiXB",
+              "name": "unique_id",
+              "input": "{\"firstName\":\"Alex\",\"lastName\":\"Yang\"}"
+            }
           }
         }
       }
@@ -196,13 +186,11 @@
           "content": "",
           "role": "assistant",
           "options": {
-            "toolCall": [
-              {
-                "id": "call_BrlGGU6GDGWr0hrwXt9qZKyt",
-                "name": "sumNumbers",
-                "input": "{\"a\":1,\"b\":1}"
-              }
-            ]
+            "toolCall": {
+              "id": "call_YUXowHKwOJ6GIWV4EA59x8iF",
+              "name": "sumNumbers",
+              "input": "{\"a\":1,\"b\":1}"
+            }
           }
         }
       }
@@ -212,7 +200,7 @@
       "response": {
         "raw": null,
         "message": {
-          "content": "1 + 1 is equal to 2.",
+          "content": "The sum of 1 + 1 is 2.",
           "role": "assistant",
           "options": {}
         }

packages/core/e2e/node/snapshot/agent_stream.snap

@@ -20,21 +20,14 @@
           "role": "assistant",
           "content": "",
           "options": {
-            "toolCall": [
-              {
-                "name": "divideNumbers",
-                "id": "call_V4daSdWk9QeYeSMKLBisNbSf",
-                "input": {
-                  "a": 16,
-                  "b": 2
-                }
-              },
-              {
-                "name": "sumNumbers",
-                "id": "call_n2OlBxlaoeMIMVeU9DeDfiPX",
-                "input": "{\"a\": 8, \"b\": 20}"
+            "toolCall": {
+              "name": "divideNumbers",
+              "id": "call_QQh8uKzGHbgBlrhqxZNoWiQT",
+              "input": {
+                "a": 16,
+                "b": 2
               }
-            ]
+            }
           }
         },
         {
@@ -44,7 +37,18 @@
             "toolResult": {
               "result": "8",
               "isError": false,
-              "id": "call_V4daSdWk9QeYeSMKLBisNbSf"
+              "id": "call_QQh8uKzGHbgBlrhqxZNoWiQT"
+            }
+          }
+        },
+        {
+          "role": "assistant",
+          "content": "",
+          "options": {
+            "toolCall": {
+              "name": "sumNumbers",
+              "id": "call_C03ASSDxIFTrYmzSuTxCrItd",
+              "input": "{\"a\": 8, \"b\": 20}"
             }
           }
         },
@@ -55,7 +59,7 @@
             "toolResult": {
               "result": "28",
               "isError": false,
-              "id": "call_n2OlBxlaoeMIMVeU9DeDfiPX"
+              "id": "call_C03ASSDxIFTrYmzSuTxCrItd"
             }
           }
         }
@@ -71,13 +75,11 @@
           "content": "",
           "role": "assistant",
           "options": {
-            "toolCall": [
-              {
-                "name": "sumNumbers",
-                "id": "call_n2OlBxlaoeMIMVeU9DeDfiPX",
-                "input": "{\"a\": 8, \"b\": 20}"
-              }
-            ]
+            "toolCall": {
+              "name": "sumNumbers",
+              "id": "call_C03ASSDxIFTrYmzSuTxCrItd",
+              "input": "{\"a\": 8, \"b\": 20}"
+            }
           }
         }
       }
@@ -87,7 +89,7 @@
       "response": {
         "raw": null,
         "message": {
-          "content": "The result of dividing 16 by 2 is 8. When you add 20 to 8, the total is 28.",
+          "content": "The result of dividing 16 by 2 and then adding 20 is 28.",
           "role": "assistant",
           "options": {}
         }
@@ -100,13 +102,11 @@
       "chunk": {
         "raw": null,
         "options": {
-          "toolCall": [
-            {
-              "name": "divideNumbers",
-              "id": "call_V4daSdWk9QeYeSMKLBisNbSf",
-              "input": "{\"a\": 16, \"b\": 2}"
-            }
-          ]
+          "toolCall": {
+            "name": "divideNumbers",
+            "id": "call_QQh8uKzGHbgBlrhqxZNoWiQT",
+            "input": ""
+          }
         },
         "delta": ""
       }
@@ -116,13 +116,11 @@
       "chunk": {
         "raw": null,
         "options": {
-          "toolCall": [
-            {
-              "name": "divideNumbers",
-              "id": "call_V4daSdWk9QeYeSMKLBisNbSf",
-              "input": "{\"a\": 16, \"b\": 2}"
-            }
-          ]
+          "toolCall": {
+            "name": "divideNumbers",
+            "id": "call_QQh8uKzGHbgBlrhqxZNoWiQT",
+            "input": "{\"a\""
+          }
         },
         "delta": ""
       }
@@ -132,13 +130,11 @@
       "chunk": {
         "raw": null,
         "options": {
-          "toolCall": [
-            {
-              "name": "divideNumbers",
-              "id": "call_V4daSdWk9QeYeSMKLBisNbSf",
-              "input": "{\"a\": 16, \"b\": 2}"
-            }
-          ]
+          "toolCall": {
+            "name": "divideNumbers",
+            "id": "call_QQh8uKzGHbgBlrhqxZNoWiQT",
+            "input": "{\"a\": 16,"
+          }
         },
         "delta": ""
       }
@@ -148,13 +144,11 @@
       "chunk": {
         "raw": null,
         "options": {
-          "toolCall": [
-            {
-              "name": "divideNumbers",
-              "id": "call_V4daSdWk9QeYeSMKLBisNbSf",
-              "input": "{\"a\": 16, \"b\": 2}"
-            }
-          ]
+          "toolCall": {
+            "name": "divideNumbers",
+            "id": "call_QQh8uKzGHbgBlrhqxZNoWiQT",
+            "input": "{\"a\": 16, \"b\": "
+          }
         },
         "delta": ""
       }
@@ -164,13 +158,11 @@
       "chunk": {
         "raw": null,
         "options": {
-          "toolCall": [
-            {
-              "name": "divideNumbers",
-              "id": "call_V4daSdWk9QeYeSMKLBisNbSf",
-              "input": "{\"a\": 16, \"b\": 2}"
-            }
-          ]
+          "toolCall": {
+            "name": "divideNumbers",
+            "id": "call_QQh8uKzGHbgBlrhqxZNoWiQT",
+            "input": "{\"a\": 16, \"b\": 2}"
+          }
         },
         "delta": ""
       }
@@ -180,16 +172,14 @@
       "chunk": {
         "raw": null,
         "options": {
-          "toolCall": [
-            {
-              "name": "divideNumbers",
-              "id": "call_V4daSdWk9QeYeSMKLBisNbSf",
-              "input": {
-                "a": 16,
-                "b": 2
-              }
+          "toolCall": {
+            "name": "divideNumbers",
+            "id": "call_QQh8uKzGHbgBlrhqxZNoWiQT",
+            "input": {
+              "a": 16,
+              "b": 2
             }
-          ]
+          }
         },
         "delta": ""
       }
@@ -199,13 +189,11 @@
       "chunk": {
         "raw": null,
         "options": {
-          "toolCall": [
-            {
-              "name": "sumNumbers",
-              "id": "call_n2OlBxlaoeMIMVeU9DeDfiPX",
-              "input": "{\"a\": 8, \"b\": 20}"
-            }
-          ]
+          "toolCall": {
+            "name": "sumNumbers",
+            "id": "call_C03ASSDxIFTrYmzSuTxCrItd",
+            "input": "{\"a\""
+          }
         },
         "delta": ""
       }
@@ -215,13 +203,11 @@
       "chunk": {
         "raw": null,
         "options": {
-          "toolCall": [
-            {
-              "name": "sumNumbers",
-              "id": "call_n2OlBxlaoeMIMVeU9DeDfiPX",
-              "input": "{\"a\": 8, \"b\": 20}"
-            }
-          ]
+          "toolCall": {
+            "name": "sumNumbers",
+            "id": "call_C03ASSDxIFTrYmzSuTxCrItd",
+            "input": "{\"a\": 8, "
+          }
         },
         "delta": ""
       }
@@ -231,13 +217,11 @@
       "chunk": {
         "raw": null,
         "options": {
-          "toolCall": [
-            {
-              "name": "sumNumbers",
-              "id": "call_n2OlBxlaoeMIMVeU9DeDfiPX",
-              "input": "{\"a\": 8, \"b\": 20}"
-            }
-          ]
+          "toolCall": {
+            "name": "sumNumbers",
+            "id": "call_C03ASSDxIFTrYmzSuTxCrItd",
+            "input": "{\"a\": 8, \"b\": 2"
+          }
         },
         "delta": ""
       }
@@ -247,13 +231,11 @@
       "chunk": {
         "raw": null,
         "options": {
-          "toolCall": [
-            {
-              "name": "sumNumbers",
-              "id": "call_n2OlBxlaoeMIMVeU9DeDfiPX",
-              "input": "{\"a\": 8, \"b\": 20}"
-            }
-          ]
+          "toolCall": {
+            "name": "sumNumbers",
+            "id": "call_C03ASSDxIFTrYmzSuTxCrItd",
+            "input": "{\"a\": 8, \"b\": 20}"
+          }
         },
         "delta": ""
       }
@@ -335,7 +317,7 @@
       "chunk": {
         "raw": null,
         "options": {},
-        "delta": " is"
+        "delta": " and"
       }
     },
     {
@@ -343,7 +325,7 @@
       "chunk": {
         "raw": null,
         "options": {},
-        "delta": " "
+        "delta": " then"
       }
     },
     {
@@ -351,39 +333,7 @@
       "chunk": {
         "raw": null,
         "options": {},
-        "delta": "8"
-      }
-    },
-    {
-      "id": "PRESERVE_1",
-      "chunk": {
-        "raw": null,
-        "options": {},
-        "delta": "."
-      }
-    },
-    {
-      "id": "PRESERVE_1",
-      "chunk": {
-        "raw": null,
-        "options": {},
-        "delta": " When"
-      }
-    },
-    {
-      "id": "PRESERVE_1",
-      "chunk": {
-        "raw": null,
-        "options": {},
-        "delta": " you"
-      }
-    },
-    {
-      "id": "PRESERVE_1",
-      "chunk": {
-        "raw": null,
-        "options": {},
-        "delta": " add"
+        "delta": " adding"
       }
     },
     {
@@ -402,54 +352,6 @@
         "delta": "20"
       }
     },
-    {
-      "id": "PRESERVE_1",
-      "chunk": {
-        "raw": null,
-        "options": {},
-        "delta": " to"
-      }
-    },
-    {
-      "id": "PRESERVE_1",
-      "chunk": {
-        "raw": null,
-        "options": {},
-        "delta": " "
-      }
-    },
-    {
-      "id": "PRESERVE_1",
-      "chunk": {
-        "raw": null,
-        "options": {},
-        "delta": "8"
-      }
-    },
-    {
-      "id": "PRESERVE_1",
-      "chunk": {
-        "raw": null,
-        "options": {},
-        "delta": ","
-      }
-    },
-    {
-      "id": "PRESERVE_1",
-      "chunk": {
-        "raw": null,
-        "options": {},
-        "delta": " the"
-      }
-    },
-    {
-      "id": "PRESERVE_1",
-      "chunk": {
-        "raw": null,
-        "options": {},
-        "delta": " total"
-      }
-    },
     {
       "id": "PRESERVE_1",
       "chunk": {

packages/core/e2e/node/snapshot/agent_with_object_function_call.snap

@@ -20,18 +20,16 @@
           "content": "",
           "role": "assistant",
           "options": {
-            "toolCall": [
-              {
-                "id": "call_uERMumWlJLTO2GW93X6C2W3N",
-                "name": "get_weather",
-                "input": "{\"location\":\"San Francisco\"}"
-              }
-            ]
+            "toolCall": {
+              "id": "call_iY0VO4HvXDT2PtR3CvRSY1uK",
+              "name": "get_weather",
+              "input": "{\"location\":\"San Francisco\"}"
+            }
           }
         },
         {
-          "role": "user",
           "content": "{\n  location: San Francisco,\n  temperature: 72,\n  weather: cloudy,\n  rain_prediction: 0.89\n}",
+          "role": "user",
           "options": {
             "toolResult": {
               "result": {
@@ -41,7 +39,7 @@
                 "rain_prediction": 0.89
               },
               "isError": false,
-              "id": "call_uERMumWlJLTO2GW93X6C2W3N"
+              "id": "call_iY0VO4HvXDT2PtR3CvRSY1uK"
             }
           }
         }
@@ -57,13 +55,11 @@
           "content": "",
           "role": "assistant",
           "options": {
-            "toolCall": [
-              {
-                "id": "call_uERMumWlJLTO2GW93X6C2W3N",
-                "name": "get_weather",
-                "input": "{\"location\":\"San Francisco\"}"
-              }
-            ]
+            "toolCall": {
+              "id": "call_iY0VO4HvXDT2PtR3CvRSY1uK",
+              "name": "get_weather",
+              "input": "{\"location\":\"San Francisco\"}"
+            }
           }
         }
       }

packages/core/e2e/node/snapshot/agent_with_object_retriever.snap

@@ -24,15 +24,6 @@
     },
     {
       "id": "PRESERVE_2",
-      "messages": [
-        {
-          "content": "Context information is below.\n---------------------\nAlex is a male. What's very important, Alex is not in the Brazil.\n---------------------\nGiven the context information and not prior knowledge, answer the query.\nQuery: Brazil\nAnswer:",
-          "role": "user"
-        }
-      ]
-    },
-    {
-      "id": "PRESERVE_3",
       "messages": [
         {
           "content": "Please always use the tools provided to answer a question. Do not rely on prior knowledge.",
@@ -46,39 +37,21 @@
           "content": "",
           "role": "assistant",
           "options": {
-            "toolCall": [
-              {
-                "id": "call_vv1XW3xv4j2us5sZOtzCU2lL",
-                "name": "summary_tool",
-                "input": "{\"query\": \"Alex\"}"
-              },
-              {
-                "id": "call_V36LMHbwUJkEa20A3GoA4wIr",
-                "name": "summary_tool",
-                "input": "{\"query\": \"Brazil\"}"
-              }
-            ]
-          }
-        },
-        {
-          "role": "user",
-          "content": "Alex is not in Brazil.",
-          "options": {
-            "toolResult": {
-              "result": "Alex is not in Brazil.",
-              "isError": false,
-              "id": "call_vv1XW3xv4j2us5sZOtzCU2lL"
+            "toolCall": {
+              "id": "call_VfgZS1NPKpbtzqhM3o1Buzx8",
+              "name": "summary_tool",
+              "input": "{\"query\": \"Alex\"}"
             }
           }
         },
         {
-          "role": "user",
           "content": "Alex is not in Brazil.",
+          "role": "user",
           "options": {
             "toolResult": {
               "result": "Alex is not in Brazil.",
               "isError": false,
-              "id": "call_V36LMHbwUJkEa20A3GoA4wIr"
+              "id": "call_VfgZS1NPKpbtzqhM3o1Buzx8"
             }
           }
         }
@@ -94,18 +67,11 @@
           "content": "",
           "role": "assistant",
           "options": {
-            "toolCall": [
-              {
-                "id": "call_vv1XW3xv4j2us5sZOtzCU2lL",
-                "name": "summary_tool",
-                "input": "{\"query\": \"Alex\"}"
-              },
-              {
-                "id": "call_V36LMHbwUJkEa20A3GoA4wIr",
-                "name": "summary_tool",
-                "input": "{\"query\": \"Brazil\"}"
-              }
-            ]
+            "toolCall": {
+              "id": "call_VfgZS1NPKpbtzqhM3o1Buzx8",
+              "name": "summary_tool",
+              "input": "{\"query\": \"Alex\"}"
+            }
           }
         }
       }
@@ -126,18 +92,7 @@
       "response": {
         "raw": null,
         "message": {
-          "content": "Alex is not in Brazil.",
-          "role": "assistant",
-          "options": {}
-        }
-      }
-    },
-    {
-      "id": "PRESERVE_3",
-      "response": {
-        "raw": null,
-        "message": {
-          "content": "Based on the brief information, Alex is not in Brazil.",
+          "content": "No, Alex does not live in Brazil based on the brief information available.",
           "role": "assistant",
           "options": {}
         }

packages/core/e2e/node/snapshot/anthropic-agent-multiple-chat.snap

@@ -178,15 +178,13 @@
           ],
           "role": "assistant",
           "options": {
-            "toolCall": [
-              {
-                "id": "toolu_017MxwaxaLYkmt4dpP5HKzFe",
-                "name": "getWeather",
-                "input": {
-                  "city": "San Francisco"
-                }
+            "toolCall": {
+              "id": "toolu_01Gy7Gxbx7uGmjVncGH6pubL",
+              "name": "getWeather",
+              "input": {
+                "city": "San Francisco"
               }
-            ]
+            }
           }
         },
         {
@@ -196,7 +194,7 @@
             "toolResult": {
               "result": "The weather in San Francisco is 72 degrees",
               "isError": false,
-              "id": "toolu_017MxwaxaLYkmt4dpP5HKzFe"
+              "id": "toolu_01Gy7Gxbx7uGmjVncGH6pubL"
             }
           }
         }
@@ -260,15 +258,13 @@
           ],
           "role": "assistant",
           "options": {
-            "toolCall": [
-              {
-                "id": "toolu_017MxwaxaLYkmt4dpP5HKzFe",
-                "name": "getWeather",
-                "input": {
-                  "city": "San Francisco"
-                }
+            "toolCall": {
+              "id": "toolu_01Gy7Gxbx7uGmjVncGH6pubL",
+              "name": "getWeather",
+              "input": {
+                "city": "San Francisco"
               }
-            ]
+            }
           }
         },
         {
@@ -278,7 +274,7 @@
             "toolResult": {
               "result": "The weather in San Francisco is 72 degrees",
               "isError": false,
-              "id": "toolu_017MxwaxaLYkmt4dpP5HKzFe"
+              "id": "toolu_01Gy7Gxbx7uGmjVncGH6pubL"
             }
           }
         },
@@ -356,15 +352,13 @@
           ],
           "role": "assistant",
           "options": {
-            "toolCall": [
-              {
-                "id": "toolu_017MxwaxaLYkmt4dpP5HKzFe",
-                "name": "getWeather",
-                "input": {
-                  "city": "San Francisco"
-                }
+            "toolCall": {
+              "id": "toolu_01Gy7Gxbx7uGmjVncGH6pubL",
+              "name": "getWeather",
+              "input": {
+                "city": "San Francisco"
               }
-            ]
+            }
           }
         },
         {
@@ -374,7 +368,7 @@
             "toolResult": {
               "result": "The weather in San Francisco is 72 degrees",
               "isError": false,
-              "id": "toolu_017MxwaxaLYkmt4dpP5HKzFe"
+              "id": "toolu_01Gy7Gxbx7uGmjVncGH6pubL"
             }
           }
         },
@@ -401,15 +395,13 @@
           ],
           "role": "assistant",
           "options": {
-            "toolCall": [
-              {
-                "id": "toolu_013ZnATGKhvPkt2jjxdFheDK",
-                "name": "getWeather",
-                "input": {
-                  "city": "Shanghai"
-                }
+            "toolCall": {
+              "id": "toolu_01NHyahSUqrPjxQk9mvCvvGe",
+              "name": "getWeather",
+              "input": {
+                "city": "Shanghai"
               }
-            ]
+            }
           }
         },
         {
@@ -419,7 +411,7 @@
             "toolResult": {
               "result": "The weather in Shanghai is 72 degrees",
               "isError": false,
-              "id": "toolu_013ZnATGKhvPkt2jjxdFheDK"
+              "id": "toolu_01NHyahSUqrPjxQk9mvCvvGe"
             }
           }
         }
@@ -488,15 +480,13 @@
           ],
           "role": "assistant",
           "options": {
-            "toolCall": [
-              {
-                "id": "toolu_017MxwaxaLYkmt4dpP5HKzFe",
-                "name": "getWeather",
-                "input": {
-                  "city": "San Francisco"
-                }
+            "toolCall": {
+              "id": "toolu_01Gy7Gxbx7uGmjVncGH6pubL",
+              "name": "getWeather",
+              "input": {
+                "city": "San Francisco"
               }
-            ]
+            }
           }
         }
       }
@@ -530,15 +520,13 @@
           ],
           "role": "assistant",
           "options": {
-            "toolCall": [
-              {
-                "id": "toolu_013ZnATGKhvPkt2jjxdFheDK",
-                "name": "getWeather",
-                "input": {
-                  "city": "Shanghai"
-                }
+            "toolCall": {
+              "id": "toolu_01NHyahSUqrPjxQk9mvCvvGe",
+              "name": "getWeather",
+              "input": {
+                "city": "Shanghai"
               }
-            ]
+            }
           }
         }
       }

packages/core/e2e/node/snapshot/anthropic-agent.snap

@@ -5,7 +5,8 @@
       "messages": [
         {
           "role": "user",
-          "content": "What is the weather in San Francisco?"
+          "content": "What is the weather in San Francisco?",
+          "options": {}
         }
       ]
     },
@@ -14,26 +15,25 @@
       "messages": [
         {
           "role": "user",
-          "content": "What is the weather in San Francisco?"
+          "content": "What is the weather in San Francisco?",
+          "options": {}
         },
         {
           "content": [
             {
               "type": "text",
-              "text": "<thinking>\nThe user has asked for the weather in a specific location, San Francisco. The Weather tool is the relevant function to answer this request, as it returns weather information for a given location.\n\nThe Weather tool requires a single parameter:\n- location (string, required): The user has directly provided the location as \"San Francisco\".\n\nSince the required location parameter has been provided, we have enough information to call the Weather tool.\n</thinking>"
+              "text": "<thinking>\nThe user is asking for the weather in a specific location, San Francisco. The Weather function is the relevant tool to answer this request, as it returns weather information for a given location.\n\nThe Weather function has one required parameter:\n- location (string): The user has directly provided the location of \"San Francisco\"\n\nSince the required location parameter has been provided by the user, we have all the necessary information to call the Weather function.\n</thinking>"
             }
           ],
           "role": "assistant",
           "options": {
-            "toolCall": [
-              {
-                "id": "toolu_011YcKkPygw2woJZrVWRRMwH",
-                "name": "Weather",
-                "input": {
-                  "location": "San Francisco"
-                }
+            "toolCall": {
+              "id": "HIDDEN",
+              "name": "Weather",
+              "input": {
+                "location": "San Francisco"
               }
-            ]
+            }
           }
         },
         {
@@ -41,9 +41,8 @@
           "role": "user",
           "options": {
             "toolResult": {
-              "result": "35 degrees and sunny in San Francisco",
               "isError": false,
-              "id": "toolu_011YcKkPygw2woJZrVWRRMwH"
+              "id": "HIDDEN"
             }
           }
         }
@@ -54,7 +53,8 @@
       "messages": [
         {
           "role": "user",
-          "content": "My name is Alex Yang. What is my unique id?"
+          "content": "My name is Alex Yang. What is my unique id?",
+          "options": {}
         }
       ]
     },
@@ -63,27 +63,26 @@
       "messages": [
         {
           "role": "user",
-          "content": "My name is Alex Yang. What is my unique id?"
+          "content": "My name is Alex Yang. What is my unique id?",
+          "options": {}
         },
         {
           "content": [
             {
               "type": "text",
-              "text": "<thinking>\nThe unique_id function takes firstName and lastName as required parameters. The user has provided their first name (Alex) and last name (Yang) in the request, so we have all the necessary information to call the function.\n</thinking>"
+              "text": "<thinking>\nThe unique_id function is the relevant tool to answer the user's request for their unique ID. It requires two parameters:\nfirstName: The user provided their first name, which is \"Alex\"\nlastName: The user also provided their last name, \"Yang\"\nSince the user has provided all the required parameters, we can proceed with calling the unique_id function.\n</thinking>"
             }
           ],
           "role": "assistant",
           "options": {
-            "toolCall": [
-              {
-                "id": "toolu_012ZLUq5SWsghkXUjsEXsB1f",
-                "name": "unique_id",
-                "input": {
-                  "firstName": "Alex",
-                  "lastName": "Yang"
-                }
+            "toolCall": {
+              "id": "HIDDEN",
+              "name": "unique_id",
+              "input": {
+                "firstName": "Alex",
+                "lastName": "Yang"
               }
-            ]
+            }
           }
         },
         {
@@ -91,9 +90,8 @@
           "role": "user",
           "options": {
             "toolResult": {
-              "result": "123456789",
               "isError": false,
-              "id": "toolu_012ZLUq5SWsghkXUjsEXsB1f"
+              "id": "HIDDEN"
             }
           }
         }
@@ -104,7 +102,8 @@
       "messages": [
         {
           "role": "user",
-          "content": "how much is 1 + 1?"
+          "content": "how much is 1 + 1?",
+          "options": {}
         }
       ]
     },
@@ -113,27 +112,26 @@
       "messages": [
         {
           "role": "user",
-          "content": "how much is 1 + 1?"
+          "content": "how much is 1 + 1?",
+          "options": {}
         },
         {
           "content": [
             {
               "type": "text",
-              "text": "<thinking>\nThe user is asking to sum the numbers 1 and 1. The relevant tool is the sumNumbers function, which takes two number parameters a and b.\nThe user has directly provided the values for the required parameters:\na = 1 \nb = 1\nSince all the required parameters are provided, we can proceed with calling the function.\n</thinking>"
+              "text": "<thinking>\nThe user is asking to sum the numbers 1 and 1. The relevant tool to use is the sumNumbers function, which takes two number parameters a and b.\nThe user has directly provided the values for the parameters:\na = 1 \nb = 1\nSince all the required parameters have been provided, we can proceed with calling the function.\n</thinking>"
             }
           ],
           "role": "assistant",
           "options": {
-            "toolCall": [
-              {
-                "id": "toolu_01SQD2XkaXkDNLQ2xaFzmguG",
-                "name": "sumNumbers",
-                "input": {
-                  "a": 1,
-                  "b": 1
-                }
+            "toolCall": {
+              "id": "HIDDEN",
+              "name": "sumNumbers",
+              "input": {
+                "a": 1,
+                "b": 1
               }
-            ]
+            }
           }
         },
         {
@@ -141,9 +139,8 @@
           "role": "user",
           "options": {
             "toolResult": {
-              "result": "2",
               "isError": false,
-              "id": "toolu_01SQD2XkaXkDNLQ2xaFzmguG"
+              "id": "HIDDEN"
             }
           }
         }
@@ -154,25 +151,48 @@
     {
       "id": "PRESERVE_0",
       "response": {
-        "raw": null,
+        "raw": {
+          "id": "HIDDEN",
+          "type": "message",
+          "role": "assistant",
+          "model": "claude-3-opus-20240229",
+          "stop_sequence": null,
+          "usage": {
+            "input_tokens": 462,
+            "output_tokens": 147
+          },
+          "content": [
+            {
+              "type": "text",
+              "text": "<thinking>\nThe user is asking for the weather in a specific location, San Francisco. The Weather function is the relevant tool to answer this request, as it returns weather information for a given location.\n\nThe Weather function has one required parameter:\n- location (string): The user has directly provided the location of \"San Francisco\"\n\nSince the required location parameter has been provided by the user, we have all the necessary information to call the Weather function.\n</thinking>"
+            },
+            {
+              "type": "tool_use",
+              "id": "HIDDEN",
+              "name": "Weather",
+              "input": {
+                "location": "San Francisco"
+              }
+            }
+          ],
+          "stop_reason": "tool_use"
+        },
         "message": {
           "content": [
             {
               "type": "text",
-              "text": "<thinking>\nThe user has asked for the weather in a specific location, San Francisco. The Weather tool is the relevant function to answer this request, as it returns weather information for a given location.\n\nThe Weather tool requires a single parameter:\n- location (string, required): The user has directly provided the location as \"San Francisco\".\n\nSince the required location parameter has been provided, we have enough information to call the Weather tool.\n</thinking>"
+              "text": "<thinking>\nThe user is asking for the weather in a specific location, San Francisco. The Weather function is the relevant tool to answer this request, as it returns weather information for a given location.\n\nThe Weather function has one required parameter:\n- location (string): The user has directly provided the location of \"San Francisco\"\n\nSince the required location parameter has been provided by the user, we have all the necessary information to call the Weather function.\n</thinking>"
             }
           ],
           "role": "assistant",
           "options": {
-            "toolCall": [
-              {
-                "id": "toolu_011YcKkPygw2woJZrVWRRMwH",
-                "name": "Weather",
-                "input": {
-                  "location": "San Francisco"
-                }
+            "toolCall": {
+              "id": "HIDDEN",
+              "name": "Weather",
+              "input": {
+                "location": "San Francisco"
               }
-            ]
+            }
           }
         }
       }
@@ -180,7 +200,24 @@
     {
       "id": "PRESERVE_1",
       "response": {
-        "raw": null,
+        "raw": {
+          "id": "HIDDEN",
+          "type": "message",
+          "role": "assistant",
+          "model": "claude-3-opus-20240229",
+          "stop_sequence": null,
+          "usage": {
+            "input_tokens": 628,
+            "output_tokens": 18
+          },
+          "content": [
+            {
+              "type": "text",
+              "text": "The current weather in San Francisco is 35 degrees and sunny."
+            }
+          ],
+          "stop_reason": "end_turn"
+        },
         "message": {
           "content": [
             {
@@ -196,26 +233,50 @@
     {
       "id": "PRESERVE_2",
       "response": {
-        "raw": null,
+        "raw": {
+          "id": "HIDDEN",
+          "type": "message",
+          "role": "assistant",
+          "model": "claude-3-opus-20240229",
+          "stop_sequence": null,
+          "usage": {
+            "input_tokens": 482,
+            "output_tokens": 152
+          },
+          "content": [
+            {
+              "type": "text",
+              "text": "<thinking>\nThe unique_id function is the relevant tool to answer the user's request for their unique ID. It requires two parameters:\nfirstName: The user provided their first name, which is \"Alex\"\nlastName: The user also provided their last name, \"Yang\"\nSince the user has provided all the required parameters, we can proceed with calling the unique_id function.\n</thinking>"
+            },
+            {
+              "type": "tool_use",
+              "id": "HIDDEN",
+              "name": "unique_id",
+              "input": {
+                "firstName": "Alex",
+                "lastName": "Yang"
+              }
+            }
+          ],
+          "stop_reason": "tool_use"
+        },
         "message": {
           "content": [
             {
               "type": "text",
-              "text": "<thinking>\nThe unique_id function takes firstName and lastName as required parameters. The user has provided their first name (Alex) and last name (Yang) in the request, so we have all the necessary information to call the function.\n</thinking>"
+              "text": "<thinking>\nThe unique_id function is the relevant tool to answer the user's request for their unique ID. It requires two parameters:\nfirstName: The user provided their first name, which is \"Alex\"\nlastName: The user also provided their last name, \"Yang\"\nSince the user has provided all the required parameters, we can proceed with calling the unique_id function.\n</thinking>"
             }
           ],
           "role": "assistant",
           "options": {
-            "toolCall": [
-              {
-                "id": "toolu_012ZLUq5SWsghkXUjsEXsB1f",
-                "name": "unique_id",
-                "input": {
-                  "firstName": "Alex",
-                  "lastName": "Yang"
-                }
+            "toolCall": {
+              "id": "HIDDEN",
+              "name": "unique_id",
+              "input": {
+                "firstName": "Alex",
+                "lastName": "Yang"
               }
-            ]
+            }
           }
         }
       }
@@ -223,7 +284,24 @@
     {
       "id": "PRESERVE_3",
       "response": {
-        "raw": null,
+        "raw": {
+          "id": "HIDDEN",
+          "type": "message",
+          "role": "assistant",
+          "model": "claude-3-opus-20240229",
+          "stop_sequence": null,
+          "usage": {
+            "input_tokens": 648,
+            "output_tokens": 13
+          },
+          "content": [
+            {
+              "type": "text",
+              "text": "Your unique ID is 123456789."
+            }
+          ],
+          "stop_reason": "end_turn"
+        },
         "message": {
           "content": [
             {
@@ -239,26 +317,50 @@
     {
       "id": "PRESERVE_4",
       "response": {
-        "raw": null,
+        "raw": {
+          "id": "HIDDEN",
+          "type": "message",
+          "role": "assistant",
+          "model": "claude-3-opus-20240229",
+          "stop_sequence": null,
+          "usage": {
+            "input_tokens": 498,
+            "output_tokens": 151
+          },
+          "content": [
+            {
+              "type": "text",
+              "text": "<thinking>\nThe user is asking to sum the numbers 1 and 1. The relevant tool to use is the sumNumbers function, which takes two number parameters a and b.\nThe user has directly provided the values for the parameters:\na = 1 \nb = 1\nSince all the required parameters have been provided, we can proceed with calling the function.\n</thinking>"
+            },
+            {
+              "type": "tool_use",
+              "id": "HIDDEN",
+              "name": "sumNumbers",
+              "input": {
+                "a": 1,
+                "b": 1
+              }
+            }
+          ],
+          "stop_reason": "tool_use"
+        },
         "message": {
           "content": [
             {
               "type": "text",
-              "text": "<thinking>\nThe user is asking to sum the numbers 1 and 1. The relevant tool is the sumNumbers function, which takes two number parameters a and b.\nThe user has directly provided the values for the required parameters:\na = 1 \nb = 1\nSince all the required parameters are provided, we can proceed with calling the function.\n</thinking>"
+              "text": "<thinking>\nThe user is asking to sum the numbers 1 and 1. The relevant tool to use is the sumNumbers function, which takes two number parameters a and b.\nThe user has directly provided the values for the parameters:\na = 1 \nb = 1\nSince all the required parameters have been provided, we can proceed with calling the function.\n</thinking>"
             }
           ],
           "role": "assistant",
           "options": {
-            "toolCall": [
-              {
-                "id": "toolu_01SQD2XkaXkDNLQ2xaFzmguG",
-                "name": "sumNumbers",
-                "input": {
-                  "a": 1,
-                  "b": 1
-                }
+            "toolCall": {
+              "id": "HIDDEN",
+              "name": "sumNumbers",
+              "input": {
+                "a": 1,
+                "b": 1
               }
-            ]
+            }
           }
         }
       }
@@ -266,7 +368,24 @@
     {
       "id": "PRESERVE_5",
       "response": {
-        "raw": null,
+        "raw": {
+          "id": "HIDDEN",
+          "type": "message",
+          "role": "assistant",
+          "model": "claude-3-opus-20240229",
+          "stop_sequence": null,
+          "usage": {
+            "input_tokens": 661,
+            "output_tokens": 16
+          },
+          "content": [
+            {
+              "type": "text",
+              "text": "So 1 + 1 = 2."
+            }
+          ],
+          "stop_reason": "end_turn"
+        },
         "message": {
           "content": [
             {

packages/core/e2e/node/snapshot/gpt-4-turbo.snap

@@ -20,23 +20,21 @@
           "content": "",
           "role": "assistant",
           "options": {
-            "toolCall": [
-              {
-                "id": "call_Xa2Kxa2zUE073mnougPWzRlh",
-                "name": "Weather",
-                "input": "{\"location\":\"San Jose\"}"
-              }
-            ]
+            "toolCall": {
+              "id": "call_TRC5kpzyVaV5n2R91sJsBAF1",
+              "name": "Weather",
+              "input": "{\"location\":\"San Jose\"}"
+            }
           }
         },
         {
-          "role": "user",
           "content": "45 degrees and sunny in San Jose",
+          "role": "user",
           "options": {
             "toolResult": {
               "result": "45 degrees and sunny in San Jose",
               "isError": false,
-              "id": "call_Xa2Kxa2zUE073mnougPWzRlh"
+              "id": "call_TRC5kpzyVaV5n2R91sJsBAF1"
             }
           }
         }
@@ -52,13 +50,11 @@
           "content": "",
           "role": "assistant",
           "options": {
-            "toolCall": [
-              {
-                "id": "call_Xa2Kxa2zUE073mnougPWzRlh",
-                "name": "Weather",
-                "input": "{\"location\":\"San Jose\"}"
-              }
-            ]
+            "toolCall": {
+              "id": "call_TRC5kpzyVaV5n2R91sJsBAF1",
+              "name": "Weather",
+              "input": "{\"location\":\"San Jose\"}"
+            }
           }
         }
       }
@@ -68,7 +64,7 @@
       "response": {
         "raw": null,
         "message": {
-          "content": "The current weather in San Jose is 45 degrees and sunny.",
+          "content": "The weather in San Jose is currently 45 degrees and sunny.",
           "role": "assistant",
           "options": {}
         }

packages/core/e2e/node/snapshot/llm-anthropic.snap

@@ -24,7 +24,24 @@
     {
       "id": "PRESERVE_0",
       "response": {
-        "raw": null,
+        "raw": {
+          "id": "HIDDEN",
+          "type": "message",
+          "role": "assistant",
+          "model": "claude-3-opus-20240229",
+          "stop_sequence": null,
+          "usage": {
+            "input_tokens": 8,
+            "output_tokens": 12
+          },
+          "content": [
+            {
+              "type": "text",
+              "text": "Hello! How can I assist you today?"
+            }
+          ],
+          "stop_reason": "end_turn"
+        },
         "message": {
           "content": "Hello! How can I assist you today?",
           "role": "assistant",
@@ -35,7 +52,116 @@
     {
       "id": "PRESERVE_1",
       "response": {
-        "raw": null,
+        "raw": [
+          {
+            "raw": {
+              "type": "content_block_delta",
+              "index": 0,
+              "delta": {
+                "type": "text_delta",
+                "text": "Hello"
+              }
+            },
+            "delta": "Hello",
+            "options": {}
+          },
+          {
+            "raw": {
+              "type": "content_block_delta",
+              "index": 0,
+              "delta": {
+                "type": "text_delta",
+                "text": "!"
+              }
+            },
+            "delta": "!",
+            "options": {}
+          },
+          {
+            "raw": {
+              "type": "content_block_delta",
+              "index": 0,
+              "delta": {
+                "type": "text_delta",
+                "text": " How"
+              }
+            },
+            "delta": " How",
+            "options": {}
+          },
+          {
+            "raw": {
+              "type": "content_block_delta",
+              "index": 0,
+              "delta": {
+                "type": "text_delta",
+                "text": " can"
+              }
+            },
+            "delta": " can",
+            "options": {}
+          },
+          {
+            "raw": {
+              "type": "content_block_delta",
+              "index": 0,
+              "delta": {
+                "type": "text_delta",
+                "text": " I"
+              }
+            },
+            "delta": " I",
+            "options": {}
+          },
+          {
+            "raw": {
+              "type": "content_block_delta",
+              "index": 0,
+              "delta": {
+                "type": "text_delta",
+                "text": " assist"
+              }
+            },
+            "delta": " assist",
+            "options": {}
+          },
+          {
+            "raw": {
+              "type": "content_block_delta",
+              "index": 0,
+              "delta": {
+                "type": "text_delta",
+                "text": " you"
+              }
+            },
+            "delta": " you",
+            "options": {}
+          },
+          {
+            "raw": {
+              "type": "content_block_delta",
+              "index": 0,
+              "delta": {
+                "type": "text_delta",
+                "text": " today"
+              }
+            },
+            "delta": " today",
+            "options": {}
+          },
+          {
+            "raw": {
+              "type": "content_block_delta",
+              "index": 0,
+              "delta": {
+                "type": "text_delta",
+                "text": "?"
+              }
+            },
+            "delta": "?",
+            "options": {}
+          }
+        ],
         "message": {
           "content": "Hello! How can I assist you today?",
           "role": "assistant",
@@ -48,7 +174,14 @@
     {
       "id": "PRESERVE_1",
       "chunk": {
-        "raw": null,
+        "raw": {
+          "type": "content_block_delta",
+          "index": 0,
+          "delta": {
+            "type": "text_delta",
+            "text": "Hello"
+          }
+        },
         "delta": "Hello",
         "options": {}
       }
@@ -56,7 +189,14 @@
     {
       "id": "PRESERVE_1",
       "chunk": {
-        "raw": null,
+        "raw": {
+          "type": "content_block_delta",
+          "index": 0,
+          "delta": {
+            "type": "text_delta",
+            "text": "!"
+          }
+        },
         "delta": "!",
         "options": {}
       }
@@ -64,7 +204,14 @@
     {
       "id": "PRESERVE_1",
       "chunk": {
-        "raw": null,
+        "raw": {
+          "type": "content_block_delta",
+          "index": 0,
+          "delta": {
+            "type": "text_delta",
+            "text": " How"
+          }
+        },
         "delta": " How",
         "options": {}
       }
@@ -72,7 +219,14 @@
     {
       "id": "PRESERVE_1",
       "chunk": {
-        "raw": null,
+        "raw": {
+          "type": "content_block_delta",
+          "index": 0,
+          "delta": {
+            "type": "text_delta",
+            "text": " can"
+          }
+        },
         "delta": " can",
         "options": {}
       }
@@ -80,7 +234,14 @@
     {
       "id": "PRESERVE_1",
       "chunk": {
-        "raw": null,
+        "raw": {
+          "type": "content_block_delta",
+          "index": 0,
+          "delta": {
+            "type": "text_delta",
+            "text": " I"
+          }
+        },
         "delta": " I",
         "options": {}
       }
@@ -88,7 +249,14 @@
     {
       "id": "PRESERVE_1",
       "chunk": {
-        "raw": null,
+        "raw": {
+          "type": "content_block_delta",
+          "index": 0,
+          "delta": {
+            "type": "text_delta",
+            "text": " assist"
+          }
+        },
         "delta": " assist",
         "options": {}
       }
@@ -96,7 +264,14 @@
     {
       "id": "PRESERVE_1",
       "chunk": {
-        "raw": null,
+        "raw": {
+          "type": "content_block_delta",
+          "index": 0,
+          "delta": {
+            "type": "text_delta",
+            "text": " you"
+          }
+        },
         "delta": " you",
         "options": {}
       }
@@ -104,7 +279,14 @@
     {
       "id": "PRESERVE_1",
       "chunk": {
-        "raw": null,
+        "raw": {
+          "type": "content_block_delta",
+          "index": 0,
+          "delta": {
+            "type": "text_delta",
+            "text": " today"
+          }
+        },
         "delta": " today",
         "options": {}
       }
@@ -112,7 +294,14 @@
     {
       "id": "PRESERVE_1",
       "chunk": {
-        "raw": null,
+        "raw": {
+          "type": "content_block_delta",
+          "index": 0,
+          "delta": {
+            "type": "text_delta",
+            "text": "?"
+          }
+        },
         "delta": "?",
         "options": {}
       }

packages/core/e2e/node/snapshot/openai_agent_system_prompt.snap

@@ -28,23 +28,21 @@
           "content": "",
           "role": "assistant",
           "options": {
-            "toolCall": [
-              {
-                "id": "call_1SlugFuJ7rhwsmXd5aRnJhPe",
-                "name": "getWeather",
-                "input": "{\"city\":\"San Francisco\"}"
-              }
-            ]
+            "toolCall": {
+              "id": "call_loaXnCH3WRnbnROZzNbrRkHS",
+              "name": "getWeather",
+              "input": "{\"city\":\"San Francisco\"}"
+            }
           }
         },
         {
-          "role": "user",
           "content": "The weather in San Francisco is 72 degrees",
+          "role": "user",
           "options": {
             "toolResult": {
               "result": "The weather in San Francisco is 72 degrees",
               "isError": false,
-              "id": "call_1SlugFuJ7rhwsmXd5aRnJhPe"
+              "id": "call_loaXnCH3WRnbnROZzNbrRkHS"
             }
           }
         }
@@ -60,13 +58,11 @@
           "content": "",
           "role": "assistant",
           "options": {
-            "toolCall": [
-              {
-                "id": "call_1SlugFuJ7rhwsmXd5aRnJhPe",
-                "name": "getWeather",
-                "input": "{\"city\":\"San Francisco\"}"
-              }
-            ]
+            "toolCall": {
+              "id": "call_loaXnCH3WRnbnROZzNbrRkHS",
+              "name": "getWeather",
+              "input": "{\"city\":\"San Francisco\"}"
+            }
           }
         }
       }
@@ -76,7 +72,7 @@
       "response": {
         "raw": null,
         "message": {
-          "content": "Arhgs the weather in San Francisco be 72 degrees.",
+          "content": "Arhgs! The weather in San Francisco is 72 degrees.",
           "role": "assistant",
           "options": {}
         }

packages/core/e2e/node/snapshot/react-agent-stream.snap

@@ -41,7 +41,7 @@
       "response": {
         "raw": null,
         "message": {
-          "content": "Thought: I need to use a tool to help me answer the question. \nAction: getWeather\nAction Input: {\"city\": \"San Francisco\"}",
+          "content": "Thought: I need to use a tool to help me answer the question.\nAction: getWeather\nAction Input: {\"city\": \"San Francisco\"}",
           "role": "assistant",
           "options": {}
         }
@@ -177,15 +177,7 @@
       "chunk": {
         "raw": null,
         "options": {},
-        "delta": "."
-      }
-    },
-    {
-      "id": "PRESERVE_0",
-      "chunk": {
-        "raw": null,
-        "options": {},
-        "delta": " \n"
+        "delta": ".\n"
       }
     },
     {

packages/core/e2e/node/snapshot/react-agent.snap

@@ -41,7 +41,7 @@
       "response": {
         "raw": null,
         "message": {
-          "content": "Thought: I need to use a tool to help me answer the question. \nAction: getWeather\nAction Input: {\"city\": \"San Francisco\"}",
+          "content": "Thought: I need to use a tool to help me answer the question.\nAction: getWeather\nAction Input: {\"city\": \"San Francisco\"}",
           "role": "assistant",
           "options": {}
         }

packages/core/e2e/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/core-e2e",
   "private": true,
-  "version": "0.0.7",
+  "version": "0.0.6",
   "type": "module",
   "scripts": {
     "e2e": "node --import tsx --import ./mock-register.js --test ./node/*.e2e.ts",

packages/core/jsr.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/core",
-  "version": "0.3.17",
+  "version": "0.3.12",
   "exports": "./src/index.ts",
   "imports": {
     "@llamaindex/env": "jsr:@llamaindex/env@0.1.3"

packages/core/package.json

@@ -1,6 +1,6 @@
 {
   "name": "llamaindex",
-  "version": "0.3.17",
+  "version": "0.3.12",
   "expectedMinorVersion": "3",
   "license": "MIT",
   "type": "module",
@@ -21,60 +21,55 @@
     "llamaindex"
   ],
   "dependencies": {
-    "@anthropic-ai/sdk": "^0.21.1",
+    "@anthropic-ai/sdk": "^0.20.9",
     "@aws-crypto/sha256-js": "^5.2.0",
-    "@datastax/astra-db-ts": "^1.2.1",
-    "@google-cloud/vertexai": "^1.2.0",
-    "@google/generative-ai": "^0.12.0",
-    "@grpc/grpc-js": "^1.10.8",
-    "@huggingface/inference": "^2.7.0",
+    "@datastax/astra-db-ts": "^1.1.0",
+    "@google-cloud/vertexai": "^1.1.0",
+    "@google/generative-ai": "^0.11.0",
+    "@grpc/grpc-js": "^1.10.7",
+    "@huggingface/inference": "^2.6.7",
     "@llamaindex/cloud": "0.0.5",
     "@llamaindex/env": "workspace:*",
-    "@mistralai/mistralai": "^0.4.0",
-    "@pinecone-database/pinecone": "^2.2.2",
+    "@mistralai/mistralai": "^0.2.0",
+    "@pinecone-database/pinecone": "^2.2.0",
     "@qdrant/js-client-rest": "^1.9.0",
-    "@types/lodash": "^4.17.4",
+    "@types/lodash": "^4.17.1",
     "@types/papaparse": "^5.3.14",
     "@types/pg": "^8.11.6",
-    "@xenova/transformers": "^2.17.2",
+    "@xenova/transformers": "^2.17.1",
     "@zilliz/milvus2-sdk-node": "^2.4.2",
-    "ajv": "^8.16.0",
-    "assemblyai": "^4.4.5",
-    "chromadb": "1.8.1",
-    "cohere-ai": "7.9.5",
-    "js-tiktoken": "^1.0.12",
+    "ajv": "^8.13.0",
+    "assemblyai": "^4.4.2",
+    "chromadb": "~1.7.3",
+    "cohere-ai": "^7.9.5",
+    "js-tiktoken": "^1.0.11",
     "lodash": "^4.17.21",
     "magic-bytes.js": "^1.10.0",
     "mammoth": "^1.7.2",
     "md-utils-ts": "^2.0.0",
-    "mongodb": "^6.7.0",
+    "mongodb": "^6.6.1",
     "notion-md-crawler": "^1.0.0",
-    "openai": "^4.48.1",
+    "openai": "^4.46.0",
     "papaparse": "^5.4.1",
     "pathe": "^1.1.2",
-    "pg": "^8.12.0",
+    "pdf2json": "3.0.5",
+    "pg": "^8.11.5",
     "pgvector": "^0.1.8",
     "portkey-ai": "^0.1.16",
     "rake-modified": "^1.0.8",
     "string-strip-html": "^13.4.8",
-    "unpdf": "^0.10.1",
     "wikipedia": "^2.1.2",
-    "wink-nlp": "^2.3.0"
+    "wink-nlp": "^2.2.2"
   },
   "peerDependencies": {
     "@notionhq/client": "^2.2.15"
   },
-  "peerDependenciesMeta": {
-    "@notionhq/client": {
-      "optional": true
-    }
-  },
   "devDependencies": {
     "@notionhq/client": "^2.2.15",
     "@swc/cli": "^0.3.12",
-    "@swc/core": "^1.5.24",
+    "@swc/core": "^1.5.5",
     "concurrently": "^8.2.2",
-    "glob": "^10.4.1",
+    "glob": "^10.3.12",
     "typescript": "^5.4.5"
   },
   "engines": {
@@ -141,8 +136,7 @@
   "files": [
     "dist",
     "CHANGELOG.md",
-    "examples",
-    "!**/*.tsbuildinfo"
+    "examples"
   ],
   "repository": {
     "type": "git",

packages/core/src/Retriever.ts

@@ -1,9 +1,8 @@
 import type { NodeWithScore } from "./Node.js";
 import type { ServiceContext } from "./ServiceContext.js";
-import type { MessageContent } from "./index.edge.js";
 
 export type RetrieveParams = {
-  query: MessageContent;
+  query: string;
   preFilters?: unknown;
 };
 

packages/core/src/agent/anthropic.ts

@@ -39,10 +39,9 @@ export class AnthropicAgent extends AgentRunner<Anthropic> {
   constructor(params: AnthropicAgentParams) {
     super({
       llm:
-        params.llm ??
-        (Settings.llm instanceof Anthropic
+        params.llm ?? Settings.llm instanceof Anthropic
           ? (Settings.llm as Anthropic)
-          : new Anthropic()),
+          : new Anthropic(),
       chatHistory: params.chatHistory ?? [],
       systemPrompt: params.systemPrompt ?? null,
       runner: new AnthropicAgentWorker(),
@@ -93,31 +92,29 @@ export class AnthropicAgent extends AgentRunner<Anthropic> {
     });
     if ("toolCall" in options) {
       const { toolCall } = options;
-      for (const call of toolCall) {
-        const targetTool = tools.find(
-          (tool) => tool.metadata.name === call.name,
-        );
-        const toolOutput = await callTool(
-          targetTool,
-          call,
-          step.context.logger,
-        );
-        step.context.store.toolOutputs.push(toolOutput);
-        step.context.store.messages = [
-          ...step.context.store.messages,
-          {
-            content: stringifyJSONToMessageContent(toolOutput.output),
-            role: "user",
-            options: {
-              toolResult: {
-                result: toolOutput.output,
-                isError: toolOutput.isError,
-                id: call.id,
-              },
+      const targetTool = tools.find(
+        (tool) => tool.metadata.name === toolCall.name,
+      );
+      const toolOutput = await callTool(
+        targetTool,
+        toolCall,
+        step.context.logger,
+      );
+      step.context.store.toolOutputs.push(toolOutput);
+      step.context.store.messages = [
+        ...step.context.store.messages,
+        {
+          content: stringifyJSONToMessageContent(toolOutput.output),
+          role: "user",
+          options: {
+            toolResult: {
+              result: toolOutput.output,
+              isError: toolOutput.isError,
+              id: toolCall.id,
             },
           },
-        ];
-      }
+        },
+      ];
     }
   };
 }

packages/core/src/agent/openai.ts

@@ -36,10 +36,9 @@ export class OpenAIAgent extends AgentRunner<OpenAI> {
   constructor(params: OpenAIAgentParams) {
     super({
       llm:
-        params.llm ??
-        (Settings.llm instanceof OpenAI
+        params.llm ?? Settings.llm instanceof OpenAI
           ? (Settings.llm as OpenAI)
-          : new OpenAI()),
+          : new OpenAI(),
       chatHistory: params.chatHistory ?? [],
       runner: new OpenAIAgentWorker(),
       systemPrompt: params.systemPrompt ?? null,
@@ -76,31 +75,29 @@ export class OpenAIAgent extends AgentRunner<OpenAI> {
       });
       if ("toolCall" in options) {
         const { toolCall } = options;
-        for (const call of toolCall) {
-          const targetTool = tools.find(
-            (tool) => tool.metadata.name === call.name,
-          );
-          const toolOutput = await callTool(
-            targetTool,
-            call,
-            step.context.logger,
-          );
-          step.context.store.toolOutputs.push(toolOutput);
-          step.context.store.messages = [
-            ...step.context.store.messages,
-            {
-              role: "user" as const,
-              content: stringifyJSONToMessageContent(toolOutput.output),
-              options: {
-                toolResult: {
-                  result: toolOutput.output,
-                  isError: toolOutput.isError,
-                  id: call.id,
-                },
+        const targetTool = tools.find(
+          (tool) => tool.metadata.name === toolCall.name,
+        );
+        const toolOutput = await callTool(
+          targetTool,
+          toolCall,
+          step.context.logger,
+        );
+        step.context.store.toolOutputs.push(toolOutput);
+        step.context.store.messages = [
+          ...step.context.store.messages,
+          {
+            role: "user" as const,
+            content: stringifyJSONToMessageContent(toolOutput.output),
+            options: {
+              toolResult: {
+                result: toolOutput.output,
+                isError: toolOutput.isError,
+                id: toolCall.id,
               },
             },
-          ];
-        }
+          },
+        ];
       }
     } else {
       const responseChunkStream = new ReadableStream<
@@ -137,25 +134,23 @@ export class OpenAIAgent extends AgentRunner<OpenAI> {
         for await (const chunk of pipStream) {
           if (chunk.options && "toolCall" in chunk.options) {
             const toolCall = chunk.options.toolCall;
-            toolCall.forEach((toolCall) => {
-              toolCalls.set(toolCall.id, toolCall);
-            });
+            toolCalls.set(toolCall.id, toolCall);
           }
         }
-        step.context.store.messages = [
-          ...step.context.store.messages,
-          {
-            role: "assistant" as const,
-            content: "",
-            options: {
-              toolCall: [...toolCalls.values()],
-            },
-          },
-        ];
         for (const toolCall of toolCalls.values()) {
           const targetTool = tools.find(
             (tool) => tool.metadata.name === toolCall.name,
           );
+          step.context.store.messages = [
+            ...step.context.store.messages,
+            {
+              role: "assistant" as const,
+              content: "",
+              options: {
+                toolCall,
+              },
+            },
+          ];
           const toolOutput = await callTool(
             targetTool,
             toolCall,

packages/core/src/agent/utils.ts

@@ -16,25 +16,10 @@ export async function callTool(
   toolCall: ToolCall | PartialToolCall,
   logger: Logger,
 ): Promise<ToolOutput> {
-  let input: JSONObject;
-  if (typeof toolCall.input === "string") {
-    try {
-      input = JSON.parse(toolCall.input);
-    } catch (e) {
-      const output = `Tool ${toolCall.name} can't be called. Input is not a valid JSON object.`;
-      logger.error(
-        `${output} Try increasing the maxTokens parameter of your LLM. Invalid Input: ${toolCall.input}`,
-      );
-      return {
-        tool,
-        input: {},
-        output,
-        isError: true,
-      };
-    }
-  } else {
-    input = toolCall.input;
-  }
+  const input: JSONObject =
+    typeof toolCall.input === "string"
+      ? JSON.parse(toolCall.input)
+      : toolCall.input;
   if (!tool) {
     logger.error(`Tool ${toolCall.name} does not exist.`);
     const output = `Tool ${toolCall.name} does not exist.`;

packages/core/src/callbacks/CallbackManager.ts

@@ -12,7 +12,6 @@ import type {
   LLMStreamEvent,
   LLMToolCallEvent,
   LLMToolResultEvent,
-  MessageContent,
   RetrievalEndEvent,
   RetrievalStartEvent,
 } from "../llm/types.js";
@@ -100,7 +99,7 @@ export interface StreamCallbackResponse {
 }
 
 export interface RetrievalCallbackResponse {
-  query: MessageContent;
+  query: string;
   nodes: NodeWithScore[];
 }
 

packages/core/src/cloud/LlamaCloudRetriever.ts

@@ -2,9 +2,8 @@ import type { PlatformApi, PlatformApiClient } from "@llamaindex/cloud";
 import type { NodeWithScore } from "../Node.js";
 import { ObjectType, jsonToNode } from "../Node.js";
 import type { BaseRetriever, RetrieveParams } from "../Retriever.js";
+import { Settings } from "../Settings.js";
 import { wrapEventCaller } from "../internal/context/EventCaller.js";
-import { getCallbackManager } from "../internal/settings/CallbackManager.js";
-import { extractText } from "../llm/utils.js";
 import type { ClientParams, CloudConstructorParams } from "./types.js";
 import { DEFAULT_PROJECT_NAME } from "./types.js";
 import { getClient } from "./utils.js";
@@ -71,13 +70,13 @@ export class LlamaCloudRetriever implements BaseRetriever {
       await this.getClient()
     ).pipeline.runSearch(pipelines[0].id, {
       ...this.retrieveParams,
-      query: extractText(query),
+      query,
       searchFilters: preFilters as Record<string, unknown[]>,
     });
 
     const nodes = this.resultNodesToNodeWithScore(results.retrievalNodes);
 
-    getCallbackManager().dispatchEvent("retrieve", {
+    Settings.callbackManager.dispatchEvent("retrieve", {
       query,
       nodes,
     });

packages/core/src/embeddings/ClipEmbedding.ts

@@ -87,4 +87,8 @@ export class ClipEmbedding extends MultiModalEmbedding {
     const { text_embeds } = await (await this.getTextModel())(textInputs);
     return text_embeds.data;
   }
+
+  async getQueryEmbedding(query: string): Promise<number[]> {
+    return this.getTextEmbedding(query);
+  }
 }

packages/core/src/embeddings/CloudflareWorkerEmbedding.ts

@@ -1,14 +0,0 @@
-import type { ImageType } from "../Node.js";
-import { MultiModalEmbedding } from "./MultiModalEmbedding.js";
-
-/**
- * Cloudflare worker doesn't support image embeddings for now
- */
-export class CloudflareWorkerMultiModalEmbedding extends MultiModalEmbedding {
-  getImageEmbedding(images: ImageType): Promise<number[]> {
-    throw new Error("Method not implemented.");
-  }
-  getTextEmbedding(text: string): Promise<number[]> {
-    throw new Error("Method not implemented.");
-  }
-}

packages/core/src/embeddings/DeepInfraEmbedding.ts

@@ -1,152 +0,0 @@
-import { getEnv } from "@llamaindex/env";
-import type { MessageContentDetail } from "../llm/index.js";
-import { extractSingleText } from "../llm/utils.js";
-import { BaseEmbedding } from "./types.js";
-
-const DEFAULT_MODEL = "sentence-transformers/clip-ViT-B-32";
-
-const API_TOKEN_ENV_VARIABLE_NAME = "DEEPINFRA_API_TOKEN";
-
-const API_ROOT = "https://api.deepinfra.com/v1/inference";
-
-const DEFAULT_TIMEOUT = 60 * 1000;
-
-const DEFAULT_MAX_RETRIES = 5;
-
-export interface DeepInfraEmbeddingResponse {
-  embeddings: number[][];
-  request_id: string;
-  inference_status: InferenceStatus;
-}
-
-export interface InferenceStatus {
-  status: string;
-  runtime_ms: number;
-  cost: number;
-  tokens_input: number;
-}
-
-const mapPrefixWithInputs = (prefix: string, inputs: string[]): string[] => {
-  return inputs.map((input) => (prefix ? `${prefix} ${input}` : input));
-};
-
-/**
- * DeepInfraEmbedding is an alias for DeepInfra that implements the BaseEmbedding interface.
- */
-export class DeepInfraEmbedding extends BaseEmbedding {
-  /**
-   * DeepInfra model to use
-   * @default "sentence-transformers/clip-ViT-B-32"
-   * @see https://deepinfra.com/models/embeddings
-   */
-  model: string;
-
-  /**
-   * DeepInfra API token
-   * @see https://deepinfra.com/dash/api_keys
-   * If not provided, it will try to get the token from the environment variable `DEEPINFRA_API_TOKEN`
-   *
-   */
-  apiToken: string;
-
-  /**
-   * Prefix to add to the query
-   * @default ""
-   */
-  queryPrefix: string;
-
-  /**
-   * Prefix to add to the text
-   * @default ""
-   */
-  textPrefix: string;
-
-  /**
-   *
-   * @default 5
-   */
-  maxRetries: number;
-
-  /**
-   *
-   * @default 60 * 1000
-   */
-  timeout: number;
-
-  constructor(init?: Partial<DeepInfraEmbedding>) {
-    super();
-
-    this.model = init?.model ?? DEFAULT_MODEL;
-    this.apiToken = init?.apiToken ?? getEnv(API_TOKEN_ENV_VARIABLE_NAME) ?? "";
-    this.queryPrefix = init?.queryPrefix ?? "";
-    this.textPrefix = init?.textPrefix ?? "";
-    this.maxRetries = init?.maxRetries ?? DEFAULT_MAX_RETRIES;
-    this.timeout = init?.timeout ?? DEFAULT_TIMEOUT;
-  }
-
-  async getTextEmbedding(text: string): Promise<number[]> {
-    const texts = mapPrefixWithInputs(this.textPrefix, [text]);
-    const embeddings = await this.getDeepInfraEmbedding(texts);
-    return embeddings[0];
-  }
-
-  async getQueryEmbedding(
-    query: MessageContentDetail,
-  ): Promise<number[] | null> {
-    const text = extractSingleText(query);
-    if (text) {
-      const queries = mapPrefixWithInputs(this.queryPrefix, [text]);
-      const embeddings = await this.getDeepInfraEmbedding(queries);
-      return embeddings[0];
-    } else {
-      return null;
-    }
-  }
-
-  async getTextEmbeddings(texts: string[]): Promise<number[][]> {
-    const textsWithPrefix = mapPrefixWithInputs(this.textPrefix, texts);
-    return await this.getDeepInfraEmbedding(textsWithPrefix);
-  }
-
-  async getQueryEmbeddings(queries: string[]): Promise<number[][]> {
-    const queriesWithPrefix = mapPrefixWithInputs(this.queryPrefix, queries);
-    return await this.getDeepInfraEmbedding(queriesWithPrefix);
-  }
-
-  private async getDeepInfraEmbedding(inputs: string[]): Promise<number[][]> {
-    const url = this.getUrl(this.model);
-
-    for (let attempt = 0; attempt < this.maxRetries; attempt++) {
-      const controller = new AbortController();
-      const id = setTimeout(() => controller.abort(), this.timeout);
-
-      try {
-        const response = await fetch(url, {
-          method: "POST",
-          headers: {
-            "Content-Type": "application/json",
-            Authorization: `Bearer ${this.apiToken}`,
-          },
-          body: JSON.stringify({ inputs }),
-          signal: controller.signal,
-        });
-        if (!response.ok) {
-          throw new Error(`Request failed with status ${response.status}`);
-        }
-
-        const responseJson: DeepInfraEmbeddingResponse = await response.json();
-        return responseJson.embeddings;
-      } catch (error) {
-        console.error(`Attempt ${attempt + 1} failed: ${error}`);
-      } finally {
-        clearTimeout(id);
-      }
-    }
-
-    throw new Error("Exceeded maximum retries");
-  }
-
-  private getUrl(model: string): string {
-    return `${API_ROOT}/${model}`;
-  }
-}

packages/core/src/embeddings/GeminiEmbedding.ts

@@ -36,4 +36,8 @@ export class GeminiEmbedding extends BaseEmbedding {
   getTextEmbedding(text: string): Promise<number[]> {
     return this.getEmbedding(text);
   }
+
+  getQueryEmbedding(query: string): Promise<number[]> {
+    return this.getTextEmbedding(query);
+  }
 }

packages/core/src/embeddings/HuggingFaceEmbedding.ts

@@ -45,4 +45,8 @@ export class HuggingFaceEmbedding extends BaseEmbedding {
     const output = await extractor(text, { pooling: "mean", normalize: true });
     return Array.from(output.data);
   }
+
+  async getQueryEmbedding(query: string): Promise<number[]> {
+    return this.getTextEmbedding(query);
+  }
 }

packages/core/src/embeddings/MistralAIEmbedding.ts

@@ -30,4 +30,8 @@ export class MistralAIEmbedding extends BaseEmbedding {
   async getTextEmbedding(text: string): Promise<number[]> {
     return this.getMistralAIEmbedding(text);
   }
+
+  async getQueryEmbedding(query: string): Promise<number[]> {
+    return this.getMistralAIEmbedding(query);
+  }
 }

packages/core/src/embeddings/MultiModalEmbedding.ts

@@ -6,8 +6,6 @@ import {
   type BaseNode,
   type ImageType,
 } from "../Node.js";
-import type { MessageContentDetail } from "../llm/types.js";
-import { extractImage, extractSingleText } from "../llm/utils.js";
 import { BaseEmbedding, batchEmbeddings } from "./types.js";
 
 /*
@@ -54,18 +52,4 @@ export abstract class MultiModalEmbedding extends BaseEmbedding {
 
     return nodes;
   }
-
-  async getQueryEmbedding(
-    query: MessageContentDetail,
-  ): Promise<number[] | null> {
-    const image = extractImage(query);
-    if (image) {
-      return await this.getImageEmbedding(image);
-    }
-    const text = extractSingleText(query);
-    if (text) {
-      return await this.getTextEmbedding(text);
-    }
-    return null;
-  }
 }

packages/core/src/embeddings/OpenAIEmbedding.ts

@@ -133,4 +133,13 @@ export class OpenAIEmbedding extends BaseEmbedding {
   async getTextEmbedding(text: string): Promise<number[]> {
     return (await this.getOpenAIEmbedding([text]))[0];
   }
+
+  /**
+   * Get embeddings for a query
+   * @param texts
+   * @param options
+   */
+  async getQueryEmbedding(query: string): Promise<number[]> {
+    return (await this.getOpenAIEmbedding([query]))[0];
+  }
 }

packages/core/src/embeddings/index.ts

@@ -1,4 +1,3 @@
-export { DeepInfraEmbedding } from "./DeepInfraEmbedding.js";
 export * from "./GeminiEmbedding.js";
 export * from "./JinaAIEmbedding.js";
 export * from "./MistralAIEmbedding.js";

packages/core/src/embeddings/types.ts

@@ -1,8 +1,6 @@
 import type { BaseNode } from "../Node.js";
 import { MetadataMode } from "../Node.js";
 import type { TransformComponent } from "../ingestion/types.js";
-import type { MessageContentDetail } from "../llm/types.js";
-import { extractSingleText } from "../llm/utils.js";
 import { SimilarityType, similarity } from "./utils.js";
 
 const DEFAULT_EMBED_BATCH_SIZE = 10;
@@ -21,16 +19,7 @@ export abstract class BaseEmbedding implements TransformComponent {
   }
 
   abstract getTextEmbedding(text: string): Promise<number[]>;
-
-  async getQueryEmbedding(
-    query: MessageContentDetail,
-  ): Promise<number[] | null> {
-    const text = extractSingleText(query);
-    if (text) {
-      return await this.getTextEmbedding(text);
-    }
-    return null;
-  }
+  abstract getQueryEmbedding(query: string): Promise<number[]>;
 
   /**
    * Optionally override this method to retrieve multiple embeddings in a single request

packages/core/src/engines/chat/ContextChatEngine.ts

@@ -3,10 +3,10 @@ import { getHistory } from "../../ChatHistory.js";
 import type { ContextSystemPrompt } from "../../Prompt.js";
 import { Response } from "../../Response.js";
 import type { BaseRetriever } from "../../Retriever.js";
-import { Settings } from "../../Settings.js";
 import { wrapEventCaller } from "../../internal/context/EventCaller.js";
 import type { ChatMessage, ChatResponseChunk, LLM } from "../../llm/index.js";
-import type { MessageContent, MessageType } from "../../llm/types.js";
+import { OpenAI } from "../../llm/index.js";
+import type { MessageContent } from "../../llm/types.js";
 import {
   extractText,
   streamConverter,
@@ -40,16 +40,15 @@ export class ContextChatEngine extends PromptMixin implements ChatEngine {
     contextSystemPrompt?: ContextSystemPrompt;
     nodePostprocessors?: BaseNodePostprocessor[];
     systemPrompt?: string;
-    contextRole?: MessageType;
   }) {
     super();
-    this.chatModel = init.chatModel ?? Settings.llm;
+    this.chatModel =
+      init.chatModel ?? new OpenAI({ model: "gpt-3.5-turbo-16k" });
     this.chatHistory = getHistory(init?.chatHistory);
     this.contextGenerator = new DefaultContextGenerator({
       retriever: init.retriever,
       contextSystemPrompt: init?.contextSystemPrompt,
       nodePostprocessors: init?.nodePostprocessors,
-      contextRole: init?.contextRole,
     });
     this.systemPrompt = init.systemPrompt;
   }

packages/core/src/engines/chat/DefaultContextGenerator.ts

@@ -1,11 +1,9 @@
-import { type NodeWithScore } from "../../Node.js";
+import type { NodeWithScore, TextNode } from "../../Node.js";
 import type { ContextSystemPrompt } from "../../Prompt.js";
 import { defaultContextSystemPrompt } from "../../Prompt.js";
 import type { BaseRetriever } from "../../Retriever.js";
-import type { MessageContent, MessageType } from "../../llm/types.js";
 import type { BaseNodePostprocessor } from "../../postprocessors/index.js";
 import { PromptMixin } from "../../prompts/index.js";
-import { createMessageContent } from "../../synthesizers/utils.js";
 import type { Context, ContextGenerator } from "./types.js";
 
 export class DefaultContextGenerator
@@ -15,13 +13,11 @@ export class DefaultContextGenerator
   retriever: BaseRetriever;
   contextSystemPrompt: ContextSystemPrompt;
   nodePostprocessors: BaseNodePostprocessor[];
-  contextRole: MessageType;
 
   constructor(init: {
     retriever: BaseRetriever;
     contextSystemPrompt?: ContextSystemPrompt;
     nodePostprocessors?: BaseNodePostprocessor[];
-    contextRole?: MessageType;
   }) {
     super();
 
@@ -29,7 +25,6 @@ export class DefaultContextGenerator
     this.contextSystemPrompt =
       init?.contextSystemPrompt ?? defaultContextSystemPrompt;
     this.nodePostprocessors = init.nodePostprocessors || [];
-    this.contextRole = init.contextRole ?? "system";
   }
 
   protected _getPrompts(): { contextSystemPrompt: ContextSystemPrompt } {
@@ -46,10 +41,7 @@ export class DefaultContextGenerator
     }
   }
 
-  private async applyNodePostprocessors(
-    nodes: NodeWithScore[],
-    query: MessageContent,
-  ) {
+  private async applyNodePostprocessors(nodes: NodeWithScore[], query: string) {
     let nodesWithScore = nodes;
 
     for (const postprocessor of this.nodePostprocessors) {
@@ -62,7 +54,7 @@ export class DefaultContextGenerator
     return nodesWithScore;
   }
 
-  async generate(message: MessageContent): Promise<Context> {
+  async generate(message: string): Promise<Context> {
     const sourceNodesWithScore = await this.retriever.retrieve({
       query: message,
     });
@@ -72,15 +64,12 @@ export class DefaultContextGenerator
       message,
     );
 
-    const content = await createMessageContent(
-      this.contextSystemPrompt,
-      nodes.map((r) => r.node),
-    );
-
     return {
       message: {
-        content,
-        role: this.contextRole,
+        content: this.contextSystemPrompt({
+          context: nodes.map((r) => (r.node as TextNode).text).join("\n\n"),
+        }),
+        role: "system",
       },
       nodes,
     };

packages/core/src/engines/chat/SimpleChatEngine.ts

@@ -1,9 +1,9 @@
 import type { ChatHistory } from "../../ChatHistory.js";
 import { getHistory } from "../../ChatHistory.js";
 import { Response } from "../../Response.js";
-import { Settings } from "../../Settings.js";
 import { wrapEventCaller } from "../../internal/context/EventCaller.js";
 import type { ChatResponseChunk, LLM } from "../../llm/index.js";
+import { OpenAI } from "../../llm/index.js";
 import {
   extractText,
   streamConverter,
@@ -25,7 +25,7 @@ export class SimpleChatEngine implements ChatEngine {
 
   constructor(init?: Partial<SimpleChatEngine>) {
     this.chatHistory = getHistory(init?.chatHistory);
-    this.llm = init?.llm ?? Settings.llm;
+    this.llm = init?.llm ?? new OpenAI();
   }
 
   chat(params: ChatEngineParamsStreaming): Promise<AsyncIterable<Response>>;

packages/core/src/indices/BaseIndex.ts

@@ -95,7 +95,7 @@ export abstract class BaseIndex<T> {
       [nodeParserFromSettingsOrContext(this.serviceContext)],
     );
     await this.insertNodes(nodes);
-    await this.docStore.setDocumentHash(document.id_, document.hash);
+    this.docStore.setDocumentHash(document.id_, document.hash);
   }
 
   abstract insertNodes(nodes: BaseNode[]): Promise<void>;

packages/core/src/indices/keyword/index.ts

@@ -29,7 +29,6 @@ import {
 
 import { llmFromSettingsOrContext } from "../../Settings.js";
 import type { LLM } from "../../llm/types.js";
-import { extractText } from "../../llm/utils.js";
 
 export interface KeywordIndexOptions {
   nodes?: BaseNode[];
@@ -86,7 +85,7 @@ abstract class BaseKeywordTableRetriever implements BaseRetriever {
   abstract getKeywords(query: string): Promise<string[]>;
 
   async retrieve({ query }: RetrieveParams): Promise<NodeWithScore[]> {
-    const keywords = await this.getKeywords(extractText(query));
+    const keywords = await this.getKeywords(query);
     const chunkIndicesCount: { [key: string]: number } = {};
     const filteredKeywords = keywords.filter((keyword) =>
       this.indexStruct.table.has(keyword),
@@ -281,7 +280,7 @@ export class KeywordTableIndex extends BaseIndex<KeywordTable> {
 
     await docStore.addDocuments(documents, true);
     for (const doc of documents) {
-      await docStore.setDocumentHash(doc.id_, doc.hash);
+      docStore.setDocumentHash(doc.id_, doc.hash);
     }
 
     const nodes = serviceContext.nodeParser.getNodesFromDocuments(documents);

packages/core/src/indices/summary/index.ts

@@ -11,7 +11,6 @@ import {
 } from "../../Settings.js";
 import { RetrieverQueryEngine } from "../../engines/query/index.js";
 import { wrapEventCaller } from "../../internal/context/EventCaller.js";
-import { extractText } from "../../llm/utils.js";
 import type { BaseNodePostprocessor } from "../../postprocessors/index.js";
 import type { StorageContext } from "../../storage/StorageContext.js";
 import { storageContextFromDefaults } from "../../storage/StorageContext.js";
@@ -138,7 +137,7 @@ export class SummaryIndex extends BaseIndex<IndexList> {
 
     await docStore.addDocuments(documents, true);
     for (const doc of documents) {
-      await docStore.setDocumentHash(doc.id_, doc.hash);
+      docStore.setDocumentHash(doc.id_, doc.hash);
     }
 
     const nodes =
@@ -344,7 +343,7 @@ export class SummaryIndexLLMRetriever implements BaseRetriever {
       const nodesBatch = await this.index.docStore.getNodes(nodeIdsBatch);
 
       const fmtBatchStr = this.formatNodeBatchFn(nodesBatch);
-      const input = { context: fmtBatchStr, query: extractText(query) };
+      const input = { context: fmtBatchStr, query: query };
 
       const llm = llmFromSettingsOrContext(this.serviceContext);
 

packages/core/src/indices/vectorStore/index.ts

@@ -23,7 +23,6 @@ import {
 } from "../../ingestion/strategies/index.js";
 import { wrapEventCaller } from "../../internal/context/EventCaller.js";
 import { getCallbackManager } from "../../internal/settings/CallbackManager.js";
-import type { MessageContent } from "../../llm/types.js";
 import type { BaseNodePostprocessor } from "../../postprocessors/types.js";
 import type { StorageContext } from "../../storage/StorageContext.js";
 import { storageContextFromDefaults } from "../../storage/StorageContext.js";
@@ -31,6 +30,7 @@ import type {
   MetadataFilters,
   VectorStore,
   VectorStoreByType,
+  VectorStoreQuery,
   VectorStoreQueryResult,
 } from "../../storage/index.js";
 import type { BaseIndexStore } from "../../storage/indexStore/types.js";
@@ -422,9 +422,10 @@ export class VectorIndexRetriever implements BaseRetriever {
     let nodesWithScores: NodeWithScore[] = [];
 
     for (const type in vectorStores) {
+      // TODO: add retrieval by using an image as query
       const vectorStore: VectorStore = vectorStores[type as ModalityType]!;
       nodesWithScores = nodesWithScores.concat(
-        await this.retrieveQuery(
+        await this.textRetrieve(
           query,
           type as ModalityType,
           vectorStore,
@@ -446,33 +447,36 @@ export class VectorIndexRetriever implements BaseRetriever {
     return nodesWithScores;
   }
 
-  protected async retrieveQuery(
-    query: MessageContent,
+  protected async textRetrieve(
+    query: string,
     type: ModalityType,
     vectorStore: VectorStore,
     preFilters?: MetadataFilters,
   ): Promise<NodeWithScore[]> {
-    // convert string message to multi-modal format
-    if (typeof query === "string") {
-      query = [{ type: "text", text: query }];
-    }
-    // overwrite embed model if specified, otherwise use the one from the vector store
-    const embedModel = this.index.embedModel ?? vectorStore.embedModel;
-    let nodes: NodeWithScore[] = [];
-    // query each content item (e.g. text or image) separately
-    for (const item of query) {
-      const queryEmbedding = await embedModel.getQueryEmbedding(item);
-      if (queryEmbedding) {
-        const result = await vectorStore.query({
-          queryEmbedding,
-          mode: VectorStoreQueryMode.DEFAULT,
-          similarityTopK: this.topK[type],
-          filters: preFilters ?? undefined,
-        });
-        nodes = nodes.concat(this.buildNodeListFromQueryResult(result));
-      }
-    }
-    return nodes;
+    const q = await this.buildVectorStoreQuery(
+      this.index.embedModel ?? vectorStore.embedModel,
+      query,
+      this.topK[type],
+      preFilters,
+    );
+    const result = await vectorStore.query(q);
+    return this.buildNodeListFromQueryResult(result);
+  }
+
+  protected async buildVectorStoreQuery(
+    embedModel: BaseEmbedding,
+    query: string,
+    similarityTopK: number,
+    preFilters?: MetadataFilters,
+  ): Promise<VectorStoreQuery> {
+    const queryEmbedding = await embedModel.getQueryEmbedding(query);
+
+    return {
+      queryEmbedding,
+      mode: VectorStoreQueryMode.DEFAULT,
+      similarityTopK,
+      filters: preFilters ?? undefined,
+    };
   }
 
   protected buildNodeListFromQueryResult(result: VectorStoreQueryResult) {