Release 0.3.16 (#896)

Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>

.github/workflows/test.yml

@@ -85,6 +85,7 @@ jobs:
           if-no-files-found: error
   e2e-core-examples:
     strategy:
+      fail-fast: false
       matrix:
         packages:
           - cloudflare-worker-agent

apps/docs/CHANGELOG.md

@@ -1,5 +1,41 @@
 # docs
 
+## 0.0.24
+
+### Patch Changes
+
+- 631f000: feat: DeepInfra LLM implementation
+- 8832669: Community bedrock support added
+- a29d835: setDocumentHash should be async
+- Updated dependencies [11ae926]
+- Updated dependencies [631f000]
+- Updated dependencies [1378ec4]
+- Updated dependencies [6b1ded4]
+- Updated dependencies [4d4bd85]
+- Updated dependencies [24a9d1e]
+- Updated dependencies [45952de]
+- Updated dependencies [54230f0]
+- Updated dependencies [a29d835]
+- Updated dependencies [73819bf]
+  - llamaindex@0.3.16
+
+## 0.0.23
+
+### Patch Changes
+
+- Updated dependencies [6e156ed]
+- Updated dependencies [265976d]
+- Updated dependencies [8e26f75]
+  - llamaindex@0.3.15
+
+## 0.0.22
+
+### Patch Changes
+
+- Updated dependencies [6ff7576]
+- Updated dependencies [94543de]
+  - llamaindex@0.3.14
+
 ## 0.0.21
 
 ### Patch Changes

apps/docs/docs/getting_started/starter_tutorial/retrieval_augmented_generation.mdx

@@ -10,21 +10,19 @@ import TSConfigSource from "!!raw-loader!../../../../../examples/tsconfig.json";
 
 One of the most common use-cases for LlamaIndex is Retrieval-Augmented Generation or RAG, in which your data is indexed and selectively retrieved to be given to an LLM as source material for responding to a query. You can learn more about the [concepts behind RAG](../concepts).
 
-## Before you start
+## Set up the project
 
-Make sure you have installed LlamaIndex.TS and have an OpenAI key. If you haven't, check out the [installation](../installation) steps.
-
-You can use [other LLMs](../examples/other_llms) via their APIs; if you would prefer to use local models check out our [local LLM example](../../examples/local_llm).
-
-## Set up
-
-In a new folder:
+In a new folder, run:
 
 ```bash npm2yarn
 npm init
 npm install -D typescript @types/node
 ```
 
+Then, check out the [installation](../installation) steps to install LlamaIndex.TS and prepare an OpenAI key.
+
+You can use [other LLMs](../examples/other_llms) via their APIs; if you would prefer to use local models check out our [local LLM example](../../examples/local_llm).
+
 ## Run queries
 
 Create the file `example.ts`. This code will

apps/docs/docs/modules/data_loader.mdx

@@ -6,6 +6,7 @@ import CodeBlock from "@theme/CodeBlock";
 import CodeSource from "!raw-loader!../../../../examples/readers/src/simple-directory-reader";
 import CodeSource2 from "!raw-loader!../../../../examples/readers/src/custom-simple-directory-reader";
 import CodeSource3 from "!raw-loader!../../../../examples/readers/src/llamaparse";
+import CodeSource4 from "!raw-loader!../../../../examples/readers/src/simple-directory-reader-with-llamaparse.ts";
 
 # Loader
 
@@ -21,11 +22,13 @@ It is a simple reader that reads all files from a directory and its subdirectori
 
 <CodeBlock language="ts">{CodeSource}</CodeBlock>
 
-Currently, it supports reading `.csv`, `.docx`, `.html`, `.md` and `.pdf` files,
-but support for other file types is planned.
+Currently, it supports reading `.txt`, `.pdf`, `.csv`, `.md`, `.docx`, `.htm`, `.html`, `.jpg`, `.jpeg`, `.png` and `.gif` files, but support for other file types is planned.
 
-Also, you can provide a `defaultReader` as a fallback for files with unsupported extensions.
-Or pass new readers for `fileExtToReader` to support more file types.
+You can override the default reader for all file types, inlcuding unsupported ones, with the `overrideReader` option.
+Additionally, you can override the default reader for specific file types or add support for additional file types with the `fileExtToReader` option.
+Also, you can provide a `defaultReader` as a fallback for files with unsupported extensions. By default it is `TextFileReader`.
+
+SimpleDirectoryReader supports up to 9 concurrent requests. Use the `numWorkers` option to set the number of concurrent requests. By default it runs in sequential mode, i.e. set to 1.
 
 <CodeBlock language="ts" showLineNumbers metastring="{8-12,17-21}">
   {CodeSource2}
@@ -35,14 +38,31 @@ Or pass new readers for `fileExtToReader` to support more file types.
 
 LlamaParse is an API created by LlamaIndex to efficiently parse files, e.g. it's great at converting PDF tables into markdown.
 
-To use it, first login and get an API key from https://cloud.llamaindex.ai. Make sure to store the key in the environment variable `LLAMA_CLOUD_API_KEY`.
+To use it, first login and get an API key from https://cloud.llamaindex.ai. Make sure to store the key as `apiKey` parameter or in the environment variable `LLAMA_CLOUD_API_KEY`.
 
-Then, you can use the `LlamaParseReader` class to read a local PDF file and convert it into a markdown document that can be used by LlamaIndex:
+Then, you can use the `LlamaParseReader` class to local files and convert them into a parsed document that can be used by LlamaIndex.
+See [LlamaParseReader.ts](https://github.com/run-llama/LlamaIndexTS/blob/main/packages/core/src/readers/LlamaParseReader.ts#L6) for a list of supported file types:
 
 <CodeBlock language="ts">{CodeSource3}</CodeBlock>
 
-Alternatively, you can set the [`resultType`](../api/classes/LlamaParseReader.md#resulttype) option to `text` to get the parsed document as a text string.
+Additional options can be set with the `LlamaParseReader` constructor:
+
+- `resultType` can be set to `markdown`, `text` or `.json`. Defaults to `text`
+- `language` primarly helps with OCR recognition. Defaults to `en`. See [../readers/type.ts](https://github.com/run-llama/LlamaIndexTS/blob/main/packages/core/src/readers/type.ts#L20) for a list of supported languages.
+- `parsingInstructions` can help with complicated document structures. See this [LlamaIndex Blog Post](https://www.llamaindex.ai/blog/launching-the-first-genai-native-document-parsing-platform) for an example.
+- `skipDiagonalText` set to true to ignore diagonal text.
+- `invalidateCache` set to true to ignore the LlamaCloud cache. All document are kept in cache for 48hours after the job was completed to avoid processing the same document twice. Can be useful for testing when trying to re-parse the same document with, e.g. different `parsingInstructions`.
+- `gpt4oMode` set to true to use GPT-4o to extract content.
+- `gpt4oApiKey` set the GPT-4o API key. Optional. Lowers the cost of parsing by using your own API key. Your OpenAI account will be charged. Can also be set in the environment variable `LLAMA_CLOUD_GPT4O_API_KEY`.
+- `numWorkers` as in the python version, is set in `SimpleDirectoryReader`. Default is 1.
+
+## LlamaParse with SimpleDirectoryReader
+
+Below a full example of `LlamaParse` integrated in `SimpleDirectoryReader` with additional options.
+
+<CodeBlock language="ts">{CodeSource4}</CodeBlock>
 
 ## API Reference
 
 - [SimpleDirectoryReader](../api/classes/SimpleDirectoryReader.md)
+- [LlamaParseReader](../api/classes/LlamaParseReader.md)

apps/docs/docs/modules/embeddings/available_embeddings/deepinfra.md

@@ -0,0 +1,79 @@
+# DeepInfra
+
+To use DeepInfra embeddings, you need to import `DeepInfraEmbedding` from llamaindex.
+Check out available embedding models [here](https://deepinfra.com/models/embeddings).
+
+```ts
+import {
+  DeepInfraEmbedding,
+  Settings,
+  Document,
+  VectorStoreIndex,
+} from "llamaindex";
+
+// Update Embed Model
+Settings.embedModel = new DeepInfraEmbedding();
+
+const document = new Document({ text: essay, id_: "essay" });
+
+const index = await VectorStoreIndex.fromDocuments([document]);
+
+const queryEngine = index.asQueryEngine();
+
+const query = "What is the meaning of life?";
+
+const results = await queryEngine.query({
+  query,
+});
+```
+
+By default, DeepInfraEmbedding is using the sentence-transformers/clip-ViT-B-32 model. You can change the model by passing the model parameter to the constructor.
+For example:
+
+```ts
+import { DeepInfraEmbedding } from "llamaindex";
+
+const model = "intfloat/e5-large-v2";
+Settings.embedModel = new DeepInfraEmbedding({
+  model,
+});
+```
+
+You can also set the `maxRetries` and `timeout` parameters when initializing `DeepInfraEmbedding` for better control over the request behavior.
+
+For example:
+
+```ts
+import { DeepInfraEmbedding, Settings } from "llamaindex";
+
+const model = "intfloat/e5-large-v2";
+const maxRetries = 5;
+const timeout = 5000; // 5 seconds
+
+Settings.embedModel = new DeepInfraEmbedding({
+  model,
+  maxRetries,
+  timeout,
+});
+```
+
+Standalone usage:
+
+```ts
+import { DeepInfraEmbedding } from "llamaindex";
+import { config } from "dotenv";
+// For standalone usage, you need to configure DEEPINFRA_API_TOKEN in .env file
+config();
+
+const main = async () => {
+  const model = "intfloat/e5-large-v2";
+  const embeddings = new DeepInfraEmbedding({ model });
+  const text = "What is the meaning of life?";
+  const response = await embeddings.embed([text]);
+  console.log(response);
+};
+
+main();
+```
+
+For questions or feedback, please contact us at [feedback@deepinfra.com](mailto:feedback@deepinfra.com)

apps/docs/docs/modules/llms/available_llms/bedrock.md

@@ -0,0 +1,61 @@
+# Bedrock
+
+## Usage
+
+```ts
+import { BEDROCK_MODELS, Bedrock } from "@llamaindex/community";
+
+Settings.llm = new Bedrock({
+  model: BEDROCK_MODELS.ANTHROPIC_CLAUDE_3_HAIKU,
+  region: "us-east-1", // can be provided via env AWS_REGION
+  credentials: {
+    accessKeyId: "...", // optional and can be provided via env AWS_ACCESS_KEY_ID
+    secretAccessKey: "...", // optional and can be provided via env AWS_SECRET_ACCESS_KEY
+  },
+});
+```
+
+Currently only supports Anthropic models:
+
+```ts
+ANTHROPIC_CLAUDE_INSTANT_1 = "anthropic.claude-instant-v1";
+ANTHROPIC_CLAUDE_2 = "anthropic.claude-v2";
+ANTHROPIC_CLAUDE_2_1 = "anthropic.claude-v2:1";
+ANTHROPIC_CLAUDE_3_SONNET = "anthropic.claude-3-sonnet-20240229-v1:0";
+ANTHROPIC_CLAUDE_3_HAIKU = "anthropic.claude-3-haiku-20240307-v1:0";
+ANTHROPIC_CLAUDE_3_OPUS = "anthropic.claude-3-opus-20240229-v1:0"; // Not currently on Bedrock
+```
+
+Sonnet, Haiku and Opus are multimodal, image_url only supports base64 data url format, e.g. `data:image/jpeg;base64,SGVsbG8sIFdvcmxkIQ==`
+
+## Full Example
+
+```ts
+import { BEDROCK_MODELS, Bedrock } from "llamaindex";
+
+Settings.llm = new Bedrock({
+  model: BEDROCK_MODELS.ANTHROPIC_CLAUDE_3_HAIKU,
+});
+
+async function main() {
+  const document = new Document({ text: essay, id_: "essay" });
+
+  // Load and index documents
+  const index = await VectorStoreIndex.fromDocuments([document]);
+
+  // Create a query engine
+  const queryEngine = index.asQueryEngine({
+    retriever,
+  });
+
+  const query = "What is the meaning of life?";
+
+  // Query
+  const response = await queryEngine.query({
+    query,
+  });
+
+  // Log the response
+  console.log(response.response);
+}
+```

apps/docs/docs/modules/llms/available_llms/deepinfra.md

@@ -0,0 +1,83 @@
+# DeepInfra
+
+Check out available LLMs [here](https://deepinfra.com/models/text-generation).
+
+```ts
+import { DeepInfra, Settings } from "llamaindex";
+
+// Get the API key from `DEEPINFRA_API_TOKEN` environment variable
+import { config } from "dotenv";
+config();
+Settings.llm = new DeepInfra();
+
+// Set the API key
+apiKey = "YOUR_API_KEY";
+Settings.llm = new DeepInfra({ apiKey });
+```
+
+You can setup the apiKey on the environment variables, like:
+
+```bash
+export DEEPINFRA_API_TOKEN="<YOUR_API_KEY>"
+```
+
+## Load and index documents
+
+For this example, we will use a single document. In a real-world scenario, you would have multiple documents to index.
+
+```ts
+const document = new Document({ text: essay, id_: "essay" });
+
+const index = await VectorStoreIndex.fromDocuments([document]);
+```
+
+## Query
+
+```ts
+const queryEngine = index.asQueryEngine();
+
+const query = "What is the meaning of life?";
+
+const results = await queryEngine.query({
+  query,
+});
+```
+
+## Full Example
+
+```ts
+import { DeepInfra, Document, VectorStoreIndex, Settings } from "llamaindex";
+
+// Use custom LLM
+const model = "meta-llama/Meta-Llama-3-8B-Instruct";
+Settings.llm = new DeepInfra({ model, temperature: 0 });
+
+async function main() {
+  const document = new Document({ text: essay, id_: "essay" });
+
+  // Load and index documents
+  const index = await VectorStoreIndex.fromDocuments([document]);
+
+  // get retriever
+  const retriever = index.asRetriever();
+
+  // Create a query engine
+  const queryEngine = index.asQueryEngine({
+    retriever,
+  });
+
+  const query = "What is the meaning of life?";
+
+  // Query
+  const response = await queryEngine.query({
+    query,
+  });
+
+  // Log the response
+  console.log(response.response);
+}
+```
+
+## Feedback
+
+If you have any feedback, please reach out to us at [feedback@deepinfra.com](mailto:feedback@deepinfra.com)

apps/docs/i18n/fr/docusaurus-plugin-content-docs/current/api/classes/BaseDocumentStore.md

@@ -271,7 +271,7 @@ custom_edit_url: null
 
 ### setDocumentHash
 
-▸ `Abstract` **setDocumentHash**(`docId`, `docHash`): `void`
+▸ `Abstract` **setDocumentHash**(`docId`, `docHash`): `Promise`<`void`\>
 
 #### Parameters
 

apps/docs/i18n/hr/docusaurus-plugin-content-docs/current/api/classes/BaseDocumentStore.md

@@ -271,7 +271,7 @@ custom_edit_url: null
 
 ### setDocumentHash
 
-▸ `Abstract` **setDocumentHash**(`docId`, `docHash`): `void`
+▸ `Abstract` **setDocumentHash**(`docId`, `docHash`): `Promise`<`void`\>
 
 #### Parameters
 

apps/docs/i18n/zh-Hans/docusaurus-plugin-content-docs/current/api/classes/BaseDocumentStore.md

@@ -271,7 +271,7 @@ custom_edit_url: null
 
 ### setDocumentHash
 
-▸ `Abstract` **setDocumentHash**(`docId`, `docHash`): `void`
+▸ `Abstract` **setDocumentHash**(`docId`, `docHash`): `Promise`<`void`\>
 
 #### Parameters
 

apps/docs/package.json

@@ -1,6 +1,6 @@
 {
   "name": "docs",
-  "version": "0.0.21",
+  "version": "0.0.24",
   "private": true,
   "scripts": {
     "docusaurus": "docusaurus",

examples/agent/large_toolcall.ts

@@ -0,0 +1,64 @@
+import { FunctionTool, OpenAI, OpenAIAgent } from "llamaindex";
+
+const csvData =
+  "TITLE,RELEASE_YEAR,SCORE,NUMBER_OF_VOTES,DURATION,MAIN_GENRE,MAIN_PRODUCTION\nDavid Attenborough: A Life on Our Planet,2020,9,31180,83,documentary,GB\nInception,2010,8.8,2268288,148,scifi,GB\nForrest Gump,1994,8.8,1994599,142,drama,US\nAnbe Sivam,2003,8.7,20595,160,comedy,IN\nBo Burnham: Inside,2021,8.7,44074,87,comedy,US\nSaving Private Ryan,1998,8.6,1346020,169,drama,US\nDjango Unchained,2012,8.4,1472668,165,western,US\nDangal,2016,8.4,180247,161,action,IN\nBo Burnham: Make Happy,2016,8.4,14356,60,comedy,US\nLouis C.K.: Hilarious,2010,8.4,11973,84,comedy,US\nDave Chappelle: Sticks & Stones,2019,8.4,25687,65,comedy,US\n3 Idiots,2009,8.4,385782,170,comedy,IN\nBlack Friday,2004,8.4,20611,143,crime,IN\nSuper Deluxe,2019,8.4,13680,176,thriller,IN\nWinter on Fire: Ukraine's Fight for Freedom,2015,8.3,17710,98,documentary,UA\nOnce Upon a Time in America,1984,8.3,342335,229,drama,US\nTaxi Driver,1976,8.3,795222,113,crime,US\nLike Stars on Earth,2007,8.3,188234,165,drama,IN\nBo Burnham: What.,2013,8.3,11488,60,comedy,US\nFull Metal Jacket,1987,8.3,723306,116,drama,GB\nWarrior,2011,8.2,463276,140,drama,US\nDrishyam,2015,8.2,79075,163,thriller,IN\nQueen,2014,8.2,64805,146,drama,IN\nPaan Singh Tomar,2012,8.2,35888,135,drama,IN";
+
+const userQuestion = "which are the best comedies after 2010?";
+
+(async () => {
+  // The agent will succeed if we increase `maxTokens` to 1024
+  const llm = new OpenAI({ model: "gpt-4-turbo", maxTokens: 256 });
+
+  type Input = {
+    code: string;
+  };
+  // initiate fake code interpreter
+  const interpreterTool = FunctionTool.from<Input>(
+    ({ code }) => {
+      console.log(
+        `To answer the user's question, call the following code:\n${code}`,
+      );
+      return code;
+    },
+    {
+      name: "interpreter",
+      description:
+        "Execute python code in a Jupyter notebook cell and return any result, stdout, stderr, display_data, and error.",
+      parameters: {
+        type: "object",
+        properties: {
+          code: {
+            type: "string",
+            description: "The python code to execute in a single cell.",
+          },
+        },
+        required: ["code"],
+      },
+    },
+  );
+
+  const systemPrompt =
+    "You are a Python interpreter.\n        - You are given tasks to complete and you run python code to solve them.\n        - The python code runs in a Jupyter notebook. Every time you call $(interpreter) tool, the python code is executed in a separate cell. It's okay to make multiple calls to $(interpreter).\n        - Display visualizations using matplotlib or any other visualization library directly in the notebook. Shouldn't save the visualizations to a file, just return the base64 encoded data.\n        - You can install any pip package (if it exists) if you need to but the usual packages for data analysis are already preinstalled.\n        - You can run any python code you want in a secure environment.";
+
+  const agent = new OpenAIAgent({
+    llm,
+    tools: [interpreterTool],
+    systemPrompt,
+    verbose: true,
+  });
+
+  console.log(`User question: ${userQuestion}\n`);
+
+  await agent.chat({
+    message: [
+      {
+        type: "text",
+        text: userQuestion,
+      },
+      {
+        type: "text",
+        text: `Use data from following CSV raw contents:\n${csvData}`,
+      },
+    ],
+  });
+})();

examples/deepinfra/chat.ts

@@ -0,0 +1,19 @@
+import { DeepInfra } from "llamaindex";
+
+(async () => {
+  if (!process.env.DEEPINFRA_API_TOKEN) {
+    throw new Error("Please set the DEEPINFRA_API_TOKEN environment variable.");
+  }
+  const deepinfra = new DeepInfra({});
+  const result = await deepinfra.chat({
+    messages: [
+      { content: "You want to talk in rhymes.", role: "system" },
+      {
+        content:
+          "How much wood would a woodchuck chuck if a woodchuck could chuck wood?",
+        role: "user",
+      },
+    ],
+  });
+  console.log(result);
+})();

examples/deepinfra/embedding.ts

@@ -0,0 +1,17 @@
+import { DeepInfraEmbedding } from "llamaindex";
+
+async function main() {
+  // API token can be provided as an environment variable too
+  // using DEEPINFRA_API_TOKEN variable
+  const apiToken = "YOUR_API_TOKEN" ?? process.env.DEEPINFRA_API_TOKEN;
+  const model = "BAAI/bge-large-en-v1.5";
+  const embedModel = new DeepInfraEmbedding({
+    model,
+    apiToken,
+  });
+  const texts = ["hello", "world"];
+  const embeddings = await embedModel.getTextEmbeddingsBatch(texts);
+  console.log(`\nWe have ${embeddings.length} embeddings`);
+}
+
+main().catch(console.error);

examples/multimodal/context.ts

@@ -0,0 +1,54 @@
+// call pnpm tsx multimodal/load.ts first to init the storage
+import {
+  ContextChatEngine,
+  NodeWithScore,
+  ObjectType,
+  OpenAI,
+  RetrievalEndEvent,
+  Settings,
+  VectorStoreIndex,
+} from "llamaindex";
+import { getStorageContext } from "./storage";
+
+// Update chunk size and overlap
+Settings.chunkSize = 512;
+Settings.chunkOverlap = 20;
+
+// Update llm
+Settings.llm = new OpenAI({ model: "gpt-4-turbo", maxTokens: 512 });
+
+// Update callbackManager
+Settings.callbackManager.on("retrieve-end", (event: RetrievalEndEvent) => {
+  const { nodes, query } = event.detail.payload;
+  const imageNodes = nodes.filter(
+    (node: NodeWithScore) => node.node.type === ObjectType.IMAGE_DOCUMENT,
+  );
+  const textNodes = nodes.filter(
+    (node: NodeWithScore) => node.node.type === ObjectType.TEXT,
+  );
+  console.log(
+    `Retrieved ${textNodes.length} text nodes and ${imageNodes.length} image nodes for query: ${query}`,
+  );
+});
+
+async function main() {
+  const storageContext = await getStorageContext();
+  const index = await VectorStoreIndex.init({
+    storageContext,
+  });
+  // topK for text is 0 and for image 1 => we only retrieve one image and no text based on the query
+  const retriever = index.asRetriever({ topK: { TEXT: 0, IMAGE: 1 } });
+  // NOTE: we set the contextRole to "user" (default is "system"). The reason is that GPT-4 does not support
+  // images in a system message
+  const chatEngine = new ContextChatEngine({ retriever, contextRole: "user" });
+
+  // the ContextChatEngine will use the Clip embedding to retrieve the closest image
+  // (the lady in the chair) and use it in the context for the query
+  const response = await chatEngine.chat({
+    message: "What is the name of the painting with the lady in the chair?",
+  });
+
+  console.log(response.response, "\n");
+}
+
+main().catch(console.error);

examples/multimodal/rag.ts

@@ -1,5 +1,4 @@
 import {
-  ImageType,
   MultiModalResponseSynthesizer,
   OpenAI,
   RetrievalEndEvent,
@@ -22,8 +21,6 @@ Settings.callbackManager.on("retrieve-end", (event: RetrievalEndEvent) => {
 });
 
 async function main() {
-  const images: ImageType[] = [];
-
   const storageContext = await getStorageContext();
   const index = await VectorStoreIndex.init({
     nodes: [],
@@ -34,13 +31,14 @@ async function main() {
     responseSynthesizer: new MultiModalResponseSynthesizer(),
     retriever: index.asRetriever({ topK: { TEXT: 3, IMAGE: 1 } }),
   });
-  const result = await queryEngine.query({
+  const stream = await queryEngine.query({
     query: "Tell me more about Vincent van Gogh's famous paintings",
+    stream: true,
   });
-  console.log(result.response, "\n");
-  images.forEach((image) =>
-    console.log(`Image retrieved and used in inference: ${image.toString()}`),
-  );
+  for await (const chunk of stream) {
+    process.stdout.write(chunk.response);
+  }
+  process.stdout.write("\n");
 }
 
 main().catch(console.error);

examples/package.json

@@ -4,22 +4,22 @@
   "version": "0.0.4",
   "dependencies": {
     "@aws-crypto/sha256-js": "^5.2.0",
-    "@datastax/astra-db-ts": "^1.1.0",
+    "@datastax/astra-db-ts": "^1.2.1",
     "@notionhq/client": "^2.2.15",
-    "@pinecone-database/pinecone": "^2.2.0",
+    "@pinecone-database/pinecone": "^2.2.2",
     "@zilliz/milvus2-sdk-node": "^2.4.2",
-    "chromadb": "^1.7.3",
-    "commander": "^12.0.0",
+    "chromadb": "^1.8.1",
+    "commander": "^12.1.0",
     "dotenv": "^16.4.5",
-    "js-tiktoken": "^1.0.11",
-    "llamaindex": "*",
-    "mongodb": "^6.6.1",
+    "js-tiktoken": "^1.0.12",
+    "llamaindex": "^0.3.14",
+    "mongodb": "^6.7.0",
     "pathe": "^1.1.2"
   },
   "devDependencies": {
-    "@types/node": "^20.12.11",
+    "@types/node": "^20.14.1",
     "ts-node": "^10.9.2",
-    "tsx": "^4.9.3",
+    "tsx": "^4.11.2",
     "typescript": "^5.4.5"
   },
   "scripts": {

examples/readers/package.json

@@ -11,7 +11,7 @@
     "start:pdf": "node --import tsx ./src/pdf.ts",
     "start:llamaparse": "node --import tsx ./src/llamaparse.ts",
     "start:notion": "node --import tsx ./src/notion.ts",
-    "start:llamaparse2": "node --import tsx ./src/llamaparse_2.ts"
+    "start:llamaparse-dir": "node --import tsx ./src/simple-directory-reader-with-llamaparse.ts"
   },
   "dependencies": {
     "llamaindex": "*"

examples/readers/src/custom-simple-directory-reader.ts

@@ -1,12 +1,13 @@
-import type { BaseReader, Document, Metadata } from "llamaindex";
+import type { Document, Metadata } from "llamaindex";
+import { FileReader } from "llamaindex";
 import {
   FILE_EXT_TO_READER,
   SimpleDirectoryReader,
 } from "llamaindex/readers/SimpleDirectoryReader";
 import { TextFileReader } from "llamaindex/readers/TextFileReader";
 
-class ZipReader implements BaseReader {
-  loadData(...args: any[]): Promise<Document<Metadata>[]> {
+class ZipReader extends FileReader {
+  loadDataAsContent(fileContent: Buffer): Promise<Document<Metadata>[]> {
     throw new Error("Implement me");
   }
 }

examples/readers/src/llamaparse_2.ts

@@ -1,26 +0,0 @@
-import fs from "fs/promises";
-import { LlamaParseReader } from "llamaindex";
-
-async function main() {
-  // Load PDF using LlamaParse. set apiKey here or in environment variable LLAMA_CLOUD_API_KEY
-  const reader = new LlamaParseReader({
-    resultType: "markdown",
-    language: "en",
-    parsingInstruction:
-      "The provided document is a manga comic book. Most pages do NOT have title. It does not contain tables. Try to reconstruct the dialogue happening in a cohesive way. Output any math equation in LATEX markdown (between $$)",
-  });
-  const documents = await reader.loadData("../data/manga.pdf"); // The manga.pdf in the data folder is just a copy of the TOS, due to copyright laws. You have to place your own. I used "The Manga Guide to Calculus" by Hiroyuki Kojima
-
-  // Assuming documents contain an array of pages or sections
-  const parsedManga = documents.map((page) => page.text).join("\n---\n");
-
-  // Output the parsed manga to .md file. Will be placed in ../example/readers/
-  try {
-    await fs.writeFile("./parsedManga.md", parsedManga);
-    console.log("Output successfully written to parsedManga.md");
-  } catch (err) {
-    console.error("Error writing to file:", err);
-  }
-}
-
-main().catch(console.error);

examples/readers/src/simple-directory-reader-with-llamaparse.ts

@@ -0,0 +1,35 @@
+import {
+  LlamaParseReader,
+  SimpleDirectoryReader,
+  VectorStoreIndex,
+} from "llamaindex";
+
+async function main() {
+  const reader = new SimpleDirectoryReader();
+
+  const docs = await reader.loadData({
+    directoryPath: "../data/parallel", // brk-2022.pdf split into 6 parts
+    numWorkers: 2,
+    // set LlamaParse as the default reader for all file types. Set apiKey here or in environment variable LLAMA_CLOUD_API_KEY
+    overrideReader: new LlamaParseReader({
+      language: "en",
+      resultType: "markdown",
+      parsingInstruction:
+        "The provided files is Berkshire Hathaway's 2022 Annual Report. They contain figures, tables and raw data. Capture the data in a structured format. Mathematical equation should be put out as LATEX markdown (between $$).",
+    }),
+  });
+
+  const index = await VectorStoreIndex.fromDocuments(docs);
+
+  // Query the index
+  const queryEngine = index.asQueryEngine();
+  const response = await queryEngine.query({
+    query:
+      "What is the general strategy for shareholder safety outlined in the report? Use a concrete example with numbers",
+  });
+
+  // Output response
+  console.log(response.toString());
+}
+
+main().catch(console.error);

package.json

@@ -19,15 +19,15 @@
     "new-snapshot": "pnpm run build:release && changeset version --snapshot"
   },
   "devDependencies": {
-    "@changesets/cli": "^2.27.1",
-    "@typescript-eslint/eslint-plugin": "^7.8.0",
+    "@changesets/cli": "^2.27.5",
+    "@typescript-eslint/eslint-plugin": "^7.11.0",
     "eslint": "^8.57.0",
     "eslint-config-next": "^14.2.3",
     "eslint-config-prettier": "^9.1.0",
     "eslint-config-turbo": "^1.13.3",
     "eslint-plugin-react": "7.34.1",
     "husky": "^9.0.11",
-    "lint-staged": "^15.2.2",
+    "lint-staged": "^15.2.5",
     "madge": "^7.0.0",
     "prettier": "^3.2.5",
     "prettier-plugin-organize-imports": "^3.2.4",

packages/autotool/examples/01_node/CHANGELOG.md

@@ -4,6 +4,25 @@
 
 ### Patch Changes
 
+- Updated dependencies [6e156ed]
+- Updated dependencies [265976d]
+- Updated dependencies [8e26f75]
+  - llamaindex@0.3.15
+  - @llamaindex/autotool@0.0.1
+
+## null
+
+### Patch Changes
+
+- Updated dependencies [6ff7576]
+- Updated dependencies [94543de]
+  - llamaindex@0.3.14
+  - @llamaindex/autotool@0.0.1
+
+## null
+
+### Patch Changes
+
 - Updated dependencies [1b1081b]
 - Updated dependencies [37525df]
 - Updated dependencies [660a2b3]

packages/autotool/examples/02_nextjs/CHANGELOG.md

@@ -1,5 +1,41 @@
 # @llamaindex/autotool-02-next-example
 
+## 0.1.8
+
+### Patch Changes
+
+- Updated dependencies [11ae926]
+- Updated dependencies [631f000]
+- Updated dependencies [1378ec4]
+- Updated dependencies [6b1ded4]
+- Updated dependencies [4d4bd85]
+- Updated dependencies [24a9d1e]
+- Updated dependencies [45952de]
+- Updated dependencies [54230f0]
+- Updated dependencies [a29d835]
+- Updated dependencies [73819bf]
+  - llamaindex@0.3.16
+  - @llamaindex/autotool@0.0.1
+
+## 0.1.7
+
+### Patch Changes
+
+- Updated dependencies [6e156ed]
+- Updated dependencies [265976d]
+- Updated dependencies [8e26f75]
+  - llamaindex@0.3.15
+  - @llamaindex/autotool@0.0.1
+
+## 0.1.6
+
+### Patch Changes
+
+- Updated dependencies [6ff7576]
+- Updated dependencies [94543de]
+  - llamaindex@0.3.14
+  - @llamaindex/autotool@0.0.1
+
 ## 0.1.5
 
 ### Patch Changes

packages/autotool/examples/02_nextjs/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/autotool-02-next-example",
   "private": true,
-  "version": "0.1.5",
+  "version": "0.1.8",
   "scripts": {
     "dev": "next dev",
     "build": "next build",

packages/autotool/package.json

@@ -51,7 +51,7 @@
     "unplugin": "^1.10.1"
   },
   "peerDependencies": {
-    "llamaindex": "^0.3.13",
+    "llamaindex": "^0.3.16",
     "openai": "^4",
     "typescript": "^4"
   },

packages/community/.cjs.swcrc

@@ -0,0 +1,12 @@
+{
+  "jsc": {
+    "parser": {
+      "syntax": "typescript"
+    },
+    "target": "esnext"
+  },
+  "module": {
+    "type": "commonjs",
+    "ignoreDynamic": true
+  }
+}

packages/community/.swcrc

@@ -0,0 +1,8 @@
+{
+  "jsc": {
+    "parser": {
+      "syntax": "typescript"
+    },
+    "target": "esnext"
+  }
+}

packages/community/CHANGELOG.md

@@ -0,0 +1,18 @@
+# @llamaindex/community
+
+## 0.0.2
+
+### Patch Changes
+
+- 8832669: Community bedrock support added
+- Updated dependencies [11ae926]
+- Updated dependencies [631f000]
+- Updated dependencies [1378ec4]
+- Updated dependencies [6b1ded4]
+- Updated dependencies [4d4bd85]
+- Updated dependencies [24a9d1e]
+- Updated dependencies [45952de]
+- Updated dependencies [54230f0]
+- Updated dependencies [a29d835]
+- Updated dependencies [73819bf]
+  - llamaindex@0.3.16

packages/community/jsr.json

@@ -0,0 +1,8 @@
+{
+  "name": "@llamaindex/community",
+  "version": "0.0.5",
+  "exports": {
+    ".": "./src/index.ts",
+    "./type": "./src/type.ts"
+  }
+}

packages/community/package.json

@@ -0,0 +1,59 @@
+{
+  "name": "@llamaindex/community",
+  "description": "Community package for LlamaIndexTS",
+  "version": "0.0.2",
+  "type": "module",
+  "types": "dist/type/index.d.ts",
+  "main": "dist/cjs/index.js",
+  "exports": {
+    ".": {
+      "import": {
+        "types": "./dist/type/index.d.ts",
+        "default": "./dist/index.js"
+      },
+      "require": {
+        "types": "./dist/type/index.d.ts",
+        "default": "./dist/cjs/index.js"
+      }
+    },
+    "./*": {
+      "import": {
+        "types": "./dist/type/*.d.ts",
+        "default": "./dist/*.js"
+      },
+      "require": {
+        "types": "./dist/type/*.d.ts",
+        "default": "./dist/cjs/*.js"
+      }
+    }
+  },
+  "files": [
+    "dist",
+    "CHANGELOG.md"
+  ],
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/run-llama/LlamaIndexTS.git",
+    "directory": "packages/community"
+  },
+  "scripts": {
+    "lint": "eslint .",
+    "build": "rm -rf ./dist && pnpm run build:esm && pnpm run build:cjs && pnpm run build:type",
+    "build:esm": "swc src -d dist --strip-leading-paths --config-file ../../.swcrc",
+    "build:cjs": "swc src -d dist/cjs --strip-leading-paths --config-file ../../.cjs.swcrc",
+    "build:type": "tsc -p tsconfig.json",
+    "postbuild": "node -e \"require('fs').writeFileSync('./dist/cjs/package.json', JSON.stringify({ type: 'commonjs' }))\"",
+    "dev": "concurrently \"pnpm run build:esm --watch\" \"pnpm run build:cjs --watch\" \"pnpm run build:type --watch\""
+  },
+  "devDependencies": {
+    "@swc/cli": "^0.3.12",
+    "@swc/core": "^1.5.5",
+    "concurrently": "^8.2.2",
+    "pathe": "^1.1.2"
+  },
+  "dependencies": {
+    "@aws-sdk/client-bedrock-runtime": "^3.582.0",
+    "@types/node": "^20.12.11",
+    "llamaindex": "workspace:*"
+  }
+}

packages/community/src/index.ts

@@ -0,0 +1 @@
+export { BEDROCK_MODELS, Bedrock } from "./llm/bedrock/base.js";

packages/community/src/llm/bedrock/base.ts

@@ -0,0 +1,352 @@
+import {
+  BedrockRuntimeClient,
+  InvokeModelCommand,
+  InvokeModelWithResponseStreamCommand,
+  type BedrockRuntimeClientConfig,
+  type InvokeModelCommandInput,
+  type InvokeModelWithResponseStreamCommandInput,
+} from "@aws-sdk/client-bedrock-runtime";
+
+import type {
+  ChatMessage,
+  ChatResponse,
+  ChatResponseChunk,
+  CompletionResponse,
+  LLMChatParamsNonStreaming,
+  LLMChatParamsStreaming,
+  LLMCompletionParamsNonStreaming,
+  LLMCompletionParamsStreaming,
+  LLMMetadata,
+  ToolCallLLMMessageOptions,
+} from "llamaindex";
+import { ToolCallLLM, streamConverter, wrapLLMEvent } from "llamaindex";
+import type {
+  AnthropicNoneStreamingResponse,
+  AnthropicTextContent,
+  StreamEvent,
+} from "./types.js";
+import {
+  mapChatMessagesToAnthropicMessages,
+  mapMessageContentToMessageContentDetails,
+  toUtf8,
+} from "./utils.js";
+
+export type BedrockAdditionalChatOptions = {};
+
+export type BedrockChatParamsStreaming = LLMChatParamsStreaming<
+  BedrockAdditionalChatOptions,
+  ToolCallLLMMessageOptions
+>;
+
+export type BedrockChatStreamResponse = AsyncIterable<
+  ChatResponseChunk<ToolCallLLMMessageOptions>
+>;
+
+export type BedrockChatParamsNonStreaming = LLMChatParamsNonStreaming<
+  BedrockAdditionalChatOptions,
+  ToolCallLLMMessageOptions
+>;
+
+export type BedrockChatNonStreamResponse =
+  ChatResponse<ToolCallLLMMessageOptions>;
+
+export enum BEDROCK_MODELS {
+  AMAZON_TITAN_TG1_LARGE = "amazon.titan-tg1-large",
+  AMAZON_TITAN_TEXT_EXPRESS_V1 = "amazon.titan-text-express-v1",
+  AI21_J2_GRANDE_INSTRUCT = "ai21.j2-grande-instruct",
+  AI21_J2_JUMBO_INSTRUCT = "ai21.j2-jumbo-instruct",
+  AI21_J2_MID = "ai21.j2-mid",
+  AI21_J2_MID_V1 = "ai21.j2-mid-v1",
+  AI21_J2_ULTRA = "ai21.j2-ultra",
+  AI21_J2_ULTRA_V1 = "ai21.j2-ultra-v1",
+  COHERE_COMMAND_TEXT_V14 = "cohere.command-text-v14",
+  ANTHROPIC_CLAUDE_INSTANT_1 = "anthropic.claude-instant-v1",
+  ANTHROPIC_CLAUDE_1 = "anthropic.claude-v1", // EOF: No longer supported
+  ANTHROPIC_CLAUDE_2 = "anthropic.claude-v2",
+  ANTHROPIC_CLAUDE_2_1 = "anthropic.claude-v2:1",
+  ANTHROPIC_CLAUDE_3_SONNET = "anthropic.claude-3-sonnet-20240229-v1:0",
+  ANTHROPIC_CLAUDE_3_HAIKU = "anthropic.claude-3-haiku-20240307-v1:0",
+  ANTHROPIC_CLAUDE_3_OPUS = "anthropic.claude-3-opus-20240229-v1:0",
+  META_LLAMA2_13B_CHAT = "meta.llama2-13b-chat-v1",
+  META_LLAMA2_70B_CHAT = "meta.llama2-70b-chat-v1",
+  META_LLAMA3_8B_INSTRUCT = "meta.llama3-8b-instruct-v1:0",
+  META_LLAMA3_70B_INSTRUCT = "meta.llama3-70b-instruct-v1:0",
+  MISTRAL_7B_INSTRUCT = "mistral.mistral-7b-instruct-v0:2",
+  MISTRAL_MIXTRAL_7B_INSTRUCT = "mistral.mixtral-8x7b-instruct-v0:1",
+  MISTRAL_MIXTRAL_LARGE_2402 = "mistral.mistral-large-2402-v1:0",
+}
+
+/*
+ * Values taken from https://docs.aws.amazon.com/bedrock/latest/userguide/model-parameters.html#model-parameters-claude
+ */
+
+const COMPLETION_MODELS = {
+  [BEDROCK_MODELS.AMAZON_TITAN_TG1_LARGE]: 8000,
+  [BEDROCK_MODELS.AMAZON_TITAN_TEXT_EXPRESS_V1]: 8000,
+  [BEDROCK_MODELS.AI21_J2_GRANDE_INSTRUCT]: 8000,
+  [BEDROCK_MODELS.AI21_J2_JUMBO_INSTRUCT]: 8000,
+  [BEDROCK_MODELS.AI21_J2_MID]: 8000,
+  [BEDROCK_MODELS.AI21_J2_MID_V1]: 8000,
+  [BEDROCK_MODELS.AI21_J2_ULTRA]: 8000,
+  [BEDROCK_MODELS.AI21_J2_ULTRA_V1]: 8000,
+  [BEDROCK_MODELS.COHERE_COMMAND_TEXT_V14]: 4096,
+};
+
+const CHAT_ONLY_MODELS = {
+  [BEDROCK_MODELS.ANTHROPIC_CLAUDE_INSTANT_1]: 100000,
+  [BEDROCK_MODELS.ANTHROPIC_CLAUDE_1]: 100000,
+  [BEDROCK_MODELS.ANTHROPIC_CLAUDE_2]: 100000,
+  [BEDROCK_MODELS.ANTHROPIC_CLAUDE_2_1]: 200000,
+  [BEDROCK_MODELS.ANTHROPIC_CLAUDE_3_SONNET]: 200000,
+  [BEDROCK_MODELS.ANTHROPIC_CLAUDE_3_HAIKU]: 200000,
+  [BEDROCK_MODELS.ANTHROPIC_CLAUDE_3_OPUS]: 200000,
+  [BEDROCK_MODELS.META_LLAMA2_13B_CHAT]: 2048,
+  [BEDROCK_MODELS.META_LLAMA2_70B_CHAT]: 4096,
+  [BEDROCK_MODELS.META_LLAMA3_8B_INSTRUCT]: 8192,
+  [BEDROCK_MODELS.META_LLAMA3_70B_INSTRUCT]: 8192,
+  [BEDROCK_MODELS.MISTRAL_7B_INSTRUCT]: 32000,
+  [BEDROCK_MODELS.MISTRAL_MIXTRAL_7B_INSTRUCT]: 32000,
+  [BEDROCK_MODELS.MISTRAL_MIXTRAL_LARGE_2402]: 32000,
+};
+
+const BEDROCK_FOUNDATION_LLMS = { ...COMPLETION_MODELS, ...CHAT_ONLY_MODELS };
+
+/*
+ * Only the following models support streaming as
+ * per result of Bedrock.Client.list_foundation_models
+ * https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/bedrock/client/list_foundation_models.html
+ */
+export const STREAMING_MODELS = new Set([
+  BEDROCK_MODELS.AMAZON_TITAN_TG1_LARGE,
+  BEDROCK_MODELS.AMAZON_TITAN_TEXT_EXPRESS_V1,
+  BEDROCK_MODELS.ANTHROPIC_CLAUDE_INSTANT_1,
+  BEDROCK_MODELS.ANTHROPIC_CLAUDE_1,
+  BEDROCK_MODELS.ANTHROPIC_CLAUDE_2,
+  BEDROCK_MODELS.ANTHROPIC_CLAUDE_2_1,
+  BEDROCK_MODELS.ANTHROPIC_CLAUDE_3_SONNET,
+  BEDROCK_MODELS.ANTHROPIC_CLAUDE_3_HAIKU,
+  BEDROCK_MODELS.ANTHROPIC_CLAUDE_3_OPUS,
+  BEDROCK_MODELS.META_LLAMA2_13B_CHAT,
+  BEDROCK_MODELS.META_LLAMA2_70B_CHAT,
+  BEDROCK_MODELS.META_LLAMA3_8B_INSTRUCT,
+  BEDROCK_MODELS.META_LLAMA3_70B_INSTRUCT,
+  BEDROCK_MODELS.MISTRAL_7B_INSTRUCT,
+  BEDROCK_MODELS.MISTRAL_MIXTRAL_7B_INSTRUCT,
+  BEDROCK_MODELS.MISTRAL_MIXTRAL_LARGE_2402,
+]);
+
+abstract class Provider {
+  abstract getTextFromResponse(response: Record<string, any>): string;
+
+  getTextFromStreamResponse(response: Record<string, any>): string {
+    return this.getTextFromResponse(response);
+  }
+
+  abstract getRequestBody<T extends ChatMessage>(
+    metadata: LLMMetadata,
+    messages: T[],
+  ): InvokeModelCommandInput | InvokeModelWithResponseStreamCommandInput;
+}
+
+class AnthropicProvider extends Provider {
+  getResultFromResponse(
+    response: Record<string, any>,
+  ): AnthropicNoneStreamingResponse {
+    return JSON.parse(toUtf8(response.body));
+  }
+
+  getTextFromResponse(response: Record<string, any>): string {
+    const result = this.getResultFromResponse(response);
+    return result.content
+      .filter((item) => item.type === "text")
+      .map((item) => (item as AnthropicTextContent).text)
+      .join(" ");
+  }
+
+  getTextFromStreamResponse(response: Record<string, any>): string {
+    const event: StreamEvent | undefined = response.chunk?.bytes
+      ? JSON.parse(toUtf8(response.chunk?.bytes))
+      : undefined;
+
+    if (event?.type === "content_block_delta") return event.delta.text;
+    return "";
+  }
+
+  getRequestBody<T extends ChatMessage>(
+    metadata: LLMMetadata,
+    messages: T[],
+  ): InvokeModelCommandInput | InvokeModelWithResponseStreamCommandInput {
+    return {
+      modelId: metadata.model,
+      contentType: "application/json",
+      accept: "application/json",
+      body: JSON.stringify({
+        anthropic_version: "bedrock-2023-05-31",
+        messages: mapChatMessagesToAnthropicMessages(messages),
+        max_tokens: metadata.maxTokens,
+        temperature: metadata.temperature,
+        top_p: metadata.topP,
+      }),
+    };
+  }
+}
+
+// Other providers could go here
+const PROVIDERS: { [key: string]: Provider } = {
+  anthropic: new AnthropicProvider(),
+};
+
+const getProvider = (model: string): Provider => {
+  const providerName = model.split(".")[0];
+  if (!(providerName in PROVIDERS)) {
+    throw new Error(
+      `Provider ${providerName} for model ${model} is not supported`,
+    );
+  }
+  return PROVIDERS[providerName];
+};
+
+export type BedrockModelParams = {
+  model: keyof typeof BEDROCK_FOUNDATION_LLMS;
+  temperature?: number;
+  topP?: number;
+  maxTokens?: number;
+};
+
+const DEFAULT_BEDROCK_PARAMS = {
+  temperature: 0.1,
+  topP: 1,
+  maxTokens: 1024, // required by anthropic
+};
+
+export type BedrockParams = BedrockModelParams & BedrockRuntimeClientConfig;
+
+/**
+ * ToolCallLLM for Bedrock
+ */
+export class Bedrock extends ToolCallLLM<BedrockAdditionalChatOptions> {
+  private client: BedrockRuntimeClient;
+  model: keyof typeof BEDROCK_FOUNDATION_LLMS;
+  temperature: number;
+  topP: number;
+  maxTokens?: number;
+  provider: Provider;
+  topK?: number;
+
+  // there should be no check for env variables. Bedrock can be authenticated in various ways
+  // AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY and AWS_REGION are the env variables used directly by the sdk
+  constructor({
+    temperature,
+    topP,
+    maxTokens,
+    model,
+    ...params
+  }: BedrockParams) {
+    super();
+
+    this.model = model;
+    this.provider = getProvider(this.model);
+    this.maxTokens = maxTokens ?? DEFAULT_BEDROCK_PARAMS.maxTokens;
+    this.temperature = temperature ?? DEFAULT_BEDROCK_PARAMS.temperature;
+    this.topP = topP ?? DEFAULT_BEDROCK_PARAMS.topP;
+    this.client = new BedrockRuntimeClient(params);
+  }
+
+  get supportToolCall(): boolean {
+    return false;
+  }
+
+  get metadata(): LLMMetadata {
+    // NOTE, Anthropic supports top_k but LLMMetadata does not
+    return {
+      model: this.model,
+      temperature: this.temperature,
+      topP: this.topP,
+      maxTokens: this.maxTokens,
+      contextWindow: BEDROCK_FOUNDATION_LLMS[this.model],
+      tokenizer: undefined,
+    };
+  }
+
+  protected async nonStreamChat(
+    params: BedrockChatParamsNonStreaming,
+  ): Promise<BedrockChatNonStreamResponse> {
+    const input = this.provider.getRequestBody(this.metadata, params.messages);
+    const command = new InvokeModelCommand(input);
+    const response = await this.client.send(command);
+    return {
+      raw: response,
+      message: {
+        content: this.provider.getTextFromResponse(response),
+        role: "assistant",
+      },
+    };
+  }
+
+  protected async *streamChat(
+    params: BedrockChatParamsStreaming,
+  ): BedrockChatStreamResponse {
+    if (!STREAMING_MODELS.has(this.model))
+      throw new Error(`The model: ${this.model} does not support streaming`);
+    const input = this.provider.getRequestBody(this.metadata, params.messages);
+    const command = new InvokeModelWithResponseStreamCommand(input);
+    const response = await this.client.send(command);
+
+    if (response.body)
+      yield* streamConverter(response.body, (response) => {
+        return {
+          delta: this.provider.getTextFromStreamResponse(response),
+          raw: response,
+        };
+      });
+  }
+
+  chat(params: BedrockChatParamsStreaming): Promise<BedrockChatStreamResponse>;
+  chat(
+    params: BedrockChatParamsNonStreaming,
+  ): Promise<BedrockChatNonStreamResponse>;
+
+  @wrapLLMEvent
+  async chat(
+    params: BedrockChatParamsStreaming | BedrockChatParamsNonStreaming,
+  ): Promise<BedrockChatStreamResponse | BedrockChatNonStreamResponse> {
+    if (params.stream) return this.streamChat(params);
+    return this.nonStreamChat(params);
+  }
+
+  complete(
+    params: LLMCompletionParamsStreaming,
+  ): Promise<AsyncIterable<CompletionResponse>>;
+  complete(
+    params: LLMCompletionParamsNonStreaming,
+  ): Promise<CompletionResponse>;
+  async complete(
+    params: LLMCompletionParamsStreaming | LLMCompletionParamsNonStreaming,
+  ): Promise<CompletionResponse | AsyncIterable<CompletionResponse>> {
+    const message: ChatMessage = {
+      role: "user",
+      content: mapMessageContentToMessageContentDetails(params.prompt),
+    };
+
+    const input = this.provider.getRequestBody(this.metadata, [message]);
+
+    if (params.stream) {
+      const command = new InvokeModelWithResponseStreamCommand(input);
+      const response = await this.client.send(command);
+      if (response.body)
+        return streamConverter(response.body, (response) => {
+          return {
+            text: this.provider.getTextFromStreamResponse(response),
+            raw: response,
+          };
+        });
+    }
+
+    const command = new InvokeModelCommand(input);
+    const response = await this.client.send(command);
+    return {
+      text: this.provider.getTextFromResponse(response),
+      raw: response,
+    };
+  }
+}

packages/community/src/llm/bedrock/types.ts

@@ -0,0 +1,109 @@
+type Usage = {
+  input_tokens: number;
+  output_tokens: number;
+};
+
+type Message = {
+  id: string;
+  type: string;
+  role: string;
+  content: string[];
+  model: string;
+  stop_reason: string | null;
+  stop_sequence: string | null;
+  usage: Usage;
+};
+
+type ContentBlockStart = {
+  type: "content_block_start";
+  index: number;
+  content_block: {
+    type: string;
+    text: string;
+  };
+};
+
+type Delta = {
+  type: string;
+  text: string;
+};
+
+type ContentBlockDelta = {
+  type: "content_block_delta";
+  index: number;
+  delta: Delta;
+};
+
+type ContentBlockStop = {
+  type: "content_block_stop";
+  index: number;
+};
+
+type MessageDelta = {
+  type: "message_delta";
+  delta: {
+    stop_reason: string;
+    stop_sequence: string | null;
+  };
+  usage: Usage;
+};
+
+type InvocationMetrics = {
+  inputTokenCount: number;
+  outputTokenCount: number;
+  invocationLatency: number;
+  firstByteLatency: number;
+};
+
+type MessageStop = {
+  type: "message_stop";
+  "amazon-bedrock-invocationMetrics": InvocationMetrics;
+};
+
+export type StreamEvent =
+  | { type: "message_start"; message: Message }
+  | ContentBlockStart
+  | ContentBlockDelta
+  | ContentBlockStop
+  | MessageDelta
+  | MessageStop;
+
+export type AnthropicContent = AnthropicTextContent | AnthropicImageContent;
+
+export type AnthropicTextContent = {
+  type: "text";
+  text: string;
+};
+
+export type AnthropicMediaTypes =
+  | "image/jpeg"
+  | "image/png"
+  | "image/webp"
+  | "image/gif";
+
+export type AnthropicImageSource = {
+  type: "base64";
+  media_type: AnthropicMediaTypes;
+  data: string; // base64 encoded image bytes
+};
+
+export type AnthropicImageContent = {
+  type: "image";
+  source: AnthropicImageSource;
+};
+
+export type AnthropicMessage = {
+  role: "user" | "assistant";
+  content: AnthropicContent[];
+};
+
+export type AnthropicNoneStreamingResponse = {
+  id: string;
+  type: "message";
+  role: "assistant";
+  content: AnthropicContent[];
+  model: string;
+  stop_reason: "end_turn" | "max_tokens" | "stop_sequence";
+  stop_sequence?: string;
+  usage: { input_tokens: number; output_tokens: number };
+};

packages/community/src/llm/bedrock/utils.ts

@@ -0,0 +1,144 @@
+import type {
+  ChatMessage,
+  MessageContent,
+  MessageContentDetail,
+} from "llamaindex";
+import type {
+  AnthropicContent,
+  AnthropicImageContent,
+  AnthropicMediaTypes,
+  AnthropicMessage,
+  AnthropicTextContent,
+} from "./types.js";
+
+const ACCEPTED_IMAGE_MIME_TYPES = [
+  "image/jpeg",
+  "image/png",
+  "image/webp",
+  "image/gif",
+];
+
+export const mapMessageContentToMessageContentDetails = (
+  content: MessageContent,
+): MessageContentDetail[] => {
+  return Array.isArray(content) ? content : [{ type: "text", text: content }];
+};
+
+export const mergeNeighboringSameRoleMessages = (
+  messages: AnthropicMessage[],
+): AnthropicMessage[] => {
+  return messages.reduce(
+    (result: AnthropicMessage[], current: AnthropicMessage, index: number) => {
+      if (index > 0 && messages[index - 1].role === current.role) {
+        result[result.length - 1].content = [
+          ...result[result.length - 1].content,
+          ...current.content,
+        ];
+      } else {
+        result.push(current);
+      }
+      return result;
+    },
+    [],
+  );
+};
+
+export const mapMessageContentDetailToAnthropicContent = <
+  T extends MessageContentDetail,
+>(
+  detail: T,
+): AnthropicContent => {
+  let content: AnthropicContent;
+
+  if (detail.type === "text") {
+    content = mapTextContent(detail.text);
+  } else if (detail.type === "image_url") {
+    content = mapImageContent(detail.image_url.url);
+  } else {
+    throw new Error("Unsupported content detail type");
+  }
+  return content;
+};
+
+export const mapMessageContentToAnthropicContent = <T extends MessageContent>(
+  content: T,
+): AnthropicContent[] => {
+  return mapMessageContentToMessageContentDetails(content).map(
+    mapMessageContentDetailToAnthropicContent,
+  );
+};
+
+export const mapChatMessagesToAnthropicMessages = <T extends ChatMessage>(
+  messages: T[],
+): AnthropicMessage[] => {
+  const mapped = messages
+    .flatMap((msg: T): AnthropicMessage[] => {
+      return mapMessageContentToMessageContentDetails(msg.content).map(
+        (detail: MessageContentDetail): AnthropicMessage => {
+          const content = mapMessageContentDetailToAnthropicContent(detail);
+
+          return {
+            role: msg.role === "assistant" ? "assistant" : "user",
+            content: [content],
+          };
+        },
+      );
+    })
+    .filter((message: AnthropicMessage) => {
+      const content = message.content[0];
+      if (content.type === "text" && !content.text) return false;
+      if (content.type === "image" && !content.source.data) return false;
+      return true;
+    });
+
+  return mergeNeighboringSameRoleMessages(mapped);
+};
+
+export const mapTextContent = (text: string): AnthropicTextContent => {
+  return { type: "text", text };
+};
+
+export const extractDataUrlComponents = (
+  dataUrl: string,
+): {
+  mimeType: string;
+  base64: string;
+} => {
+  const parts = dataUrl.split(";base64,");
+
+  if (parts.length !== 2 || !parts[0].startsWith("data:")) {
+    throw new Error("Invalid data URL");
+  }
+
+  const mimeType = parts[0].slice(5);
+  const base64 = parts[1];
+
+  return {
+    mimeType,
+    base64,
+  };
+};
+
+export const mapImageContent = (imageUrl: string): AnthropicImageContent => {
+  if (!imageUrl.startsWith("data:"))
+    throw new Error(
+      "For Anthropic please only use base64 data url, e.g.: data:image/jpeg;base64,SGVsbG8sIFdvcmxkIQ==",
+    );
+  const { mimeType, base64: data } = extractDataUrlComponents(imageUrl);
+  if (!ACCEPTED_IMAGE_MIME_TYPES.includes(mimeType))
+    throw new Error(
+      `Anthropic only accepts the following mimeTypes: ${ACCEPTED_IMAGE_MIME_TYPES.join("\n")}`,
+    );
+
+  return {
+    type: "image",
+    source: {
+      type: "base64",
+      media_type: mimeType as AnthropicMediaTypes,
+      data,
+    },
+  };
+};
+
+export const toUtf8 = (input: Uint8Array): string =>
+  new TextDecoder("utf-8").decode(input);

packages/community/tsconfig.json

@@ -0,0 +1,19 @@
+{
+  "extends": "../../tsconfig.json",
+  "compilerOptions": {
+    "rootDir": "./src",
+    "outDir": "./dist/type",
+    "tsBuildInfoFile": "./dist/.tsbuildinfo",
+    "emitDeclarationOnly": true,
+    "module": "node16",
+    "moduleResolution": "node16",
+    "types": ["node"]
+  },
+  "include": ["./src"],
+  "exclude": ["node_modules"],
+  "references": [
+    {
+      "path": "../core/tsconfig.json"
+    }
+  ]
+}

packages/core/CHANGELOG.md

@@ -1,5 +1,35 @@
 # llamaindex
 
+## 0.3.16
+
+### Patch Changes
+
+- 11ae926: feat: add numCandidates setting to MongoDBAtlasVectorStore for tuning queries
+- 631f000: feat: DeepInfra LLM implementation
+- 1378ec4: feat: set default model to `gpt-4o`
+- 6b1ded4: add gpt4o-mode, invalidate cache and skip diagonal text to LlamaParseReader
+- 4d4bd85: Show error message if agent tool is called with partial JSON
+- 24a9d1e: add json mode and image retrieval to LlamaParseReader
+- 45952de: add concurrency management for SimpleDirectoryReader
+- 54230f0: feat: Gemini GA release models
+- a29d835: setDocumentHash should be async
+- 73819bf: Unify metadata and ID handling of documents, allow files to be read by `Buffer`
+
+## 0.3.15
+
+### Patch Changes
+
+- 6e156ed: Use images in context chat engine
+- 265976d: fix bug with node decorator
+- 8e26f75: Add retrieval for images using multi-modal messages
+
+## 0.3.14
+
+### Patch Changes
+
+- 6ff7576: Added GPT-4o for Azure
+- 94543de: Added the latest preview gemini models and multi modal images taken into account
+
 ## 0.3.13
 
 ### Patch Changes

packages/core/e2e/examples/cloudflare-worker-agent/CHANGELOG.md

@@ -1,5 +1,38 @@
 # @llamaindex/cloudflare-worker-agent-test
 
+## 0.0.17
+
+### Patch Changes
+
+- Updated dependencies [11ae926]
+- Updated dependencies [631f000]
+- Updated dependencies [1378ec4]
+- Updated dependencies [6b1ded4]
+- Updated dependencies [4d4bd85]
+- Updated dependencies [24a9d1e]
+- Updated dependencies [45952de]
+- Updated dependencies [54230f0]
+- Updated dependencies [a29d835]
+- Updated dependencies [73819bf]
+  - llamaindex@0.3.16
+
+## 0.0.16
+
+### Patch Changes
+
+- Updated dependencies [6e156ed]
+- Updated dependencies [265976d]
+- Updated dependencies [8e26f75]
+  - llamaindex@0.3.15
+
+## 0.0.15
+
+### Patch Changes
+
+- Updated dependencies [6ff7576]
+- Updated dependencies [94543de]
+  - llamaindex@0.3.14
+
 ## 0.0.14
 
 ### Patch Changes

packages/core/e2e/examples/cloudflare-worker-agent/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/cloudflare-worker-agent-test",
-  "version": "0.0.14",
+  "version": "0.0.17",
   "type": "module",
   "private": true,
   "scripts": {

packages/core/e2e/examples/nextjs-agent/CHANGELOG.md

@@ -1,5 +1,38 @@
 # @llamaindex/next-agent-test
 
+## 0.1.17
+
+### Patch Changes
+
+- Updated dependencies [11ae926]
+- Updated dependencies [631f000]
+- Updated dependencies [1378ec4]
+- Updated dependencies [6b1ded4]
+- Updated dependencies [4d4bd85]
+- Updated dependencies [24a9d1e]
+- Updated dependencies [45952de]
+- Updated dependencies [54230f0]
+- Updated dependencies [a29d835]
+- Updated dependencies [73819bf]
+  - llamaindex@0.3.16
+
+## 0.1.16
+
+### Patch Changes
+
+- Updated dependencies [6e156ed]
+- Updated dependencies [265976d]
+- Updated dependencies [8e26f75]
+  - llamaindex@0.3.15
+
+## 0.1.15
+
+### Patch Changes
+
+- Updated dependencies [6ff7576]
+- Updated dependencies [94543de]
+  - llamaindex@0.3.14
+
 ## 0.1.14
 
 ### Patch Changes

packages/core/e2e/examples/nextjs-agent/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/next-agent-test",
-  "version": "0.1.14",
+  "version": "0.1.17",
   "private": true,
   "scripts": {
     "dev": "next dev",

packages/core/e2e/examples/nextjs-edge-runtime/CHANGELOG.md

@@ -1,5 +1,38 @@
 # test-edge-runtime
 
+## 0.1.16
+
+### Patch Changes
+
+- Updated dependencies [11ae926]
+- Updated dependencies [631f000]
+- Updated dependencies [1378ec4]
+- Updated dependencies [6b1ded4]
+- Updated dependencies [4d4bd85]
+- Updated dependencies [24a9d1e]
+- Updated dependencies [45952de]
+- Updated dependencies [54230f0]
+- Updated dependencies [a29d835]
+- Updated dependencies [73819bf]
+  - llamaindex@0.3.16
+
+## 0.1.15
+
+### Patch Changes
+
+- Updated dependencies [6e156ed]
+- Updated dependencies [265976d]
+- Updated dependencies [8e26f75]
+  - llamaindex@0.3.15
+
+## 0.1.14
+
+### Patch Changes
+
+- Updated dependencies [6ff7576]
+- Updated dependencies [94543de]
+  - llamaindex@0.3.14
+
 ## 0.1.13
 
 ### Patch Changes

packages/core/e2e/examples/nextjs-edge-runtime/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/nextjs-edge-runtime-test",
-  "version": "0.1.13",
+  "version": "0.1.16",
   "private": true,
   "scripts": {
     "dev": "next dev",

packages/core/e2e/examples/waku-query-engine/CHANGELOG.md

@@ -1,5 +1,38 @@
 # @llamaindex/waku-query-engine-test
 
+## 0.0.17
+
+### Patch Changes
+
+- Updated dependencies [11ae926]
+- Updated dependencies [631f000]
+- Updated dependencies [1378ec4]
+- Updated dependencies [6b1ded4]
+- Updated dependencies [4d4bd85]
+- Updated dependencies [24a9d1e]
+- Updated dependencies [45952de]
+- Updated dependencies [54230f0]
+- Updated dependencies [a29d835]
+- Updated dependencies [73819bf]
+  - llamaindex@0.3.16
+
+## 0.0.16
+
+### Patch Changes
+
+- Updated dependencies [6e156ed]
+- Updated dependencies [265976d]
+- Updated dependencies [8e26f75]
+  - llamaindex@0.3.15
+
+## 0.0.15
+
+### Patch Changes
+
+- Updated dependencies [6ff7576]
+- Updated dependencies [94543de]
+  - llamaindex@0.3.14
+
 ## 0.0.14
 
 ### Patch Changes

packages/core/e2e/examples/waku-query-engine/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/waku-query-engine-test",
-  "version": "0.0.14",
+  "version": "0.0.17",
   "type": "module",
   "private": true,
   "scripts": {

packages/core/e2e/fixtures/embeddings/OpenAIEmbedding.ts

@@ -1,9 +1,14 @@
-import { BaseNode, SimilarityType, type BaseEmbedding } from "llamaindex";
+import {
+  BaseNode,
+  SimilarityType,
+  type BaseEmbedding,
+  type MessageContentDetail,
+} from "llamaindex";
 
 export class OpenAIEmbedding implements BaseEmbedding {
   embedBatchSize = 512;
 
-  async getQueryEmbedding(query: string) {
+  async getQueryEmbedding(query: MessageContentDetail) {
     return [0];
   }
 

packages/core/e2e/node/openai.e2e.ts

@@ -191,7 +191,8 @@ For questions about more specific sections, please use the vector_tool.`,
     message:
       "What's the summary of Alex? Does he live in Brazil based on the brief information? Return yes or no.",
   });
-  strictEqual(mockCall.mock.callCount(), 1);
+  // not sure if AI calls twice("Alex", "Brazil") or once ("Alex in Brazil") or something else
+  ok(mockCall.mock.callCount() >= 1);
 
   consola.debug("response:", response.message.content);
   ok(extractText(response.message.content).toLowerCase().includes("no"));

packages/core/e2e/node/snapshot/agent.snap

@@ -20,21 +20,23 @@
           "content": "",
           "role": "assistant",
           "options": {
-            "toolCall": {
-              "id": "call_Enrc7RhNts0qxGaEATOEhV6F",
-              "name": "Weather",
-              "input": "{\"location\":\"San Francisco\"}"
-            }
+            "toolCall": [
+              {
+                "id": "call_sH6QfjsymHW7JFl68j8AY6xg",
+                "name": "Weather",
+                "input": "{\"location\":\"San Francisco\"}"
+              }
+            ]
           }
         },
         {
-          "content": "35 degrees and sunny in San Francisco",
           "role": "user",
+          "content": "35 degrees and sunny in San Francisco",
           "options": {
             "toolResult": {
               "result": "35 degrees and sunny in San Francisco",
               "isError": false,
-              "id": "call_Enrc7RhNts0qxGaEATOEhV6F"
+              "id": "call_sH6QfjsymHW7JFl68j8AY6xg"
             }
           }
         }
@@ -60,21 +62,23 @@
           "content": "",
           "role": "assistant",
           "options": {
-            "toolCall": {
-              "id": "call_OYo81prrpK2RLOS9d0Y6kiXB",
-              "name": "unique_id",
-              "input": "{\"firstName\":\"Alex\",\"lastName\":\"Yang\"}"
-            }
+            "toolCall": [
+              {
+                "id": "call_V7zs8cyDT5FqJhjwBqcCydgA",
+                "name": "unique_id",
+                "input": "{\"firstName\":\"Alex\",\"lastName\":\"Yang\"}"
+              }
+            ]
           }
         },
         {
-          "content": "123456789",
           "role": "user",
+          "content": "123456789",
           "options": {
             "toolResult": {
               "result": "123456789",
               "isError": false,
-              "id": "call_OYo81prrpK2RLOS9d0Y6kiXB"
+              "id": "call_V7zs8cyDT5FqJhjwBqcCydgA"
             }
           }
         }
@@ -100,21 +104,23 @@
           "content": "",
           "role": "assistant",
           "options": {
-            "toolCall": {
-              "id": "call_YUXowHKwOJ6GIWV4EA59x8iF",
-              "name": "sumNumbers",
-              "input": "{\"a\":1,\"b\":1}"
-            }
+            "toolCall": [
+              {
+                "id": "call_BrlGGU6GDGWr0hrwXt9qZKyt",
+                "name": "sumNumbers",
+                "input": "{\"a\":1,\"b\":1}"
+              }
+            ]
           }
         },
         {
-          "content": "2",
           "role": "user",
+          "content": "2",
           "options": {
             "toolResult": {
               "result": "2",
               "isError": false,
-              "id": "call_YUXowHKwOJ6GIWV4EA59x8iF"
+              "id": "call_BrlGGU6GDGWr0hrwXt9qZKyt"
             }
           }
         }
@@ -130,11 +136,13 @@
           "content": "",
           "role": "assistant",
           "options": {
-            "toolCall": {
-              "id": "call_Enrc7RhNts0qxGaEATOEhV6F",
-              "name": "Weather",
-              "input": "{\"location\":\"San Francisco\"}"
-            }
+            "toolCall": [
+              {
+                "id": "call_sH6QfjsymHW7JFl68j8AY6xg",
+                "name": "Weather",
+                "input": "{\"location\":\"San Francisco\"}"
+              }
+            ]
           }
         }
       }
@@ -158,11 +166,13 @@
           "content": "",
           "role": "assistant",
           "options": {
-            "toolCall": {
-              "id": "call_OYo81prrpK2RLOS9d0Y6kiXB",
-              "name": "unique_id",
-              "input": "{\"firstName\":\"Alex\",\"lastName\":\"Yang\"}"
-            }
+            "toolCall": [
+              {
+                "id": "call_V7zs8cyDT5FqJhjwBqcCydgA",
+                "name": "unique_id",
+                "input": "{\"firstName\":\"Alex\",\"lastName\":\"Yang\"}"
+              }
+            ]
           }
         }
       }
@@ -186,11 +196,13 @@
           "content": "",
           "role": "assistant",
           "options": {
-            "toolCall": {
-              "id": "call_YUXowHKwOJ6GIWV4EA59x8iF",
-              "name": "sumNumbers",
-              "input": "{\"a\":1,\"b\":1}"
-            }
+            "toolCall": [
+              {
+                "id": "call_BrlGGU6GDGWr0hrwXt9qZKyt",
+                "name": "sumNumbers",
+                "input": "{\"a\":1,\"b\":1}"
+              }
+            ]
           }
         }
       }
@@ -200,7 +212,7 @@
       "response": {
         "raw": null,
         "message": {
-          "content": "The sum of 1 + 1 is 2.",
+          "content": "1 + 1 is equal to 2.",
           "role": "assistant",
           "options": {}
         }

packages/core/e2e/node/snapshot/agent_stream.snap

@@ -20,14 +20,21 @@
           "role": "assistant",
           "content": "",
           "options": {
-            "toolCall": {
-              "name": "divideNumbers",
-              "id": "call_QQh8uKzGHbgBlrhqxZNoWiQT",
-              "input": {
-                "a": 16,
-                "b": 2
+            "toolCall": [
+              {
+                "name": "divideNumbers",
+                "id": "call_V4daSdWk9QeYeSMKLBisNbSf",
+                "input": {
+                  "a": 16,
+                  "b": 2
+                }
+              },
+              {
+                "name": "sumNumbers",
+                "id": "call_n2OlBxlaoeMIMVeU9DeDfiPX",
+                "input": "{\"a\": 8, \"b\": 20}"
               }
-            }
+            ]
           }
         },
         {
@@ -37,18 +44,7 @@
             "toolResult": {
               "result": "8",
               "isError": false,
-              "id": "call_QQh8uKzGHbgBlrhqxZNoWiQT"
-            }
-          }
-        },
-        {
-          "role": "assistant",
-          "content": "",
-          "options": {
-            "toolCall": {
-              "name": "sumNumbers",
-              "id": "call_C03ASSDxIFTrYmzSuTxCrItd",
-              "input": "{\"a\": 8, \"b\": 20}"
+              "id": "call_V4daSdWk9QeYeSMKLBisNbSf"
             }
           }
         },
@@ -59,7 +55,7 @@
             "toolResult": {
               "result": "28",
               "isError": false,
-              "id": "call_C03ASSDxIFTrYmzSuTxCrItd"
+              "id": "call_n2OlBxlaoeMIMVeU9DeDfiPX"
             }
           }
         }
@@ -75,11 +71,13 @@
           "content": "",
           "role": "assistant",
           "options": {
-            "toolCall": {
-              "name": "sumNumbers",
-              "id": "call_C03ASSDxIFTrYmzSuTxCrItd",
-              "input": "{\"a\": 8, \"b\": 20}"
-            }
+            "toolCall": [
+              {
+                "name": "sumNumbers",
+                "id": "call_n2OlBxlaoeMIMVeU9DeDfiPX",
+                "input": "{\"a\": 8, \"b\": 20}"
+              }
+            ]
           }
         }
       }
@@ -89,7 +87,7 @@
       "response": {
         "raw": null,
         "message": {
-          "content": "The result of dividing 16 by 2 and then adding 20 is 28.",
+          "content": "The result of dividing 16 by 2 is 8. When you add 20 to 8, the total is 28.",
           "role": "assistant",
           "options": {}
         }
@@ -102,84 +100,13 @@
       "chunk": {
         "raw": null,
         "options": {
-          "toolCall": {
-            "name": "divideNumbers",
-            "id": "call_QQh8uKzGHbgBlrhqxZNoWiQT",
-            "input": ""
-          }
-        },
-        "delta": ""
-      }
-    },
-    {
-      "id": "PRESERVE_0",
-      "chunk": {
-        "raw": null,
-        "options": {
-          "toolCall": {
-            "name": "divideNumbers",
-            "id": "call_QQh8uKzGHbgBlrhqxZNoWiQT",
-            "input": "{\"a\""
-          }
-        },
-        "delta": ""
-      }
-    },
-    {
-      "id": "PRESERVE_0",
-      "chunk": {
-        "raw": null,
-        "options": {
-          "toolCall": {
-            "name": "divideNumbers",
-            "id": "call_QQh8uKzGHbgBlrhqxZNoWiQT",
-            "input": "{\"a\": 16,"
-          }
-        },
-        "delta": ""
-      }
-    },
-    {
-      "id": "PRESERVE_0",
-      "chunk": {
-        "raw": null,
-        "options": {
-          "toolCall": {
-            "name": "divideNumbers",
-            "id": "call_QQh8uKzGHbgBlrhqxZNoWiQT",
-            "input": "{\"a\": 16, \"b\": "
-          }
-        },
-        "delta": ""
-      }
-    },
-    {
-      "id": "PRESERVE_0",
-      "chunk": {
-        "raw": null,
-        "options": {
-          "toolCall": {
-            "name": "divideNumbers",
-            "id": "call_QQh8uKzGHbgBlrhqxZNoWiQT",
-            "input": "{\"a\": 16, \"b\": 2}"
-          }
-        },
-        "delta": ""
-      }
-    },
-    {
-      "id": "PRESERVE_0",
-      "chunk": {
-        "raw": null,
-        "options": {
-          "toolCall": {
-            "name": "divideNumbers",
-            "id": "call_QQh8uKzGHbgBlrhqxZNoWiQT",
-            "input": {
-              "a": 16,
-              "b": 2
+          "toolCall": [
+            {
+              "name": "divideNumbers",
+              "id": "call_V4daSdWk9QeYeSMKLBisNbSf",
+              "input": "{\"a\": 16, \"b\": 2}"
             }
-          }
+          ]
         },
         "delta": ""
       }
@@ -189,11 +116,13 @@
       "chunk": {
         "raw": null,
         "options": {
-          "toolCall": {
-            "name": "sumNumbers",
-            "id": "call_C03ASSDxIFTrYmzSuTxCrItd",
-            "input": "{\"a\""
-          }
+          "toolCall": [
+            {
+              "name": "divideNumbers",
+              "id": "call_V4daSdWk9QeYeSMKLBisNbSf",
+              "input": "{\"a\": 16, \"b\": 2}"
+            }
+          ]
         },
         "delta": ""
       }
@@ -203,11 +132,13 @@
       "chunk": {
         "raw": null,
         "options": {
-          "toolCall": {
-            "name": "sumNumbers",
-            "id": "call_C03ASSDxIFTrYmzSuTxCrItd",
-            "input": "{\"a\": 8, "
-          }
+          "toolCall": [
+            {
+              "name": "divideNumbers",
+              "id": "call_V4daSdWk9QeYeSMKLBisNbSf",
+              "input": "{\"a\": 16, \"b\": 2}"
+            }
+          ]
         },
         "delta": ""
       }
@@ -217,11 +148,13 @@
       "chunk": {
         "raw": null,
         "options": {
-          "toolCall": {
-            "name": "sumNumbers",
-            "id": "call_C03ASSDxIFTrYmzSuTxCrItd",
-            "input": "{\"a\": 8, \"b\": 2"
-          }
+          "toolCall": [
+            {
+              "name": "divideNumbers",
+              "id": "call_V4daSdWk9QeYeSMKLBisNbSf",
+              "input": "{\"a\": 16, \"b\": 2}"
+            }
+          ]
         },
         "delta": ""
       }
@@ -231,11 +164,96 @@
       "chunk": {
         "raw": null,
         "options": {
-          "toolCall": {
-            "name": "sumNumbers",
-            "id": "call_C03ASSDxIFTrYmzSuTxCrItd",
-            "input": "{\"a\": 8, \"b\": 20}"
-          }
+          "toolCall": [
+            {
+              "name": "divideNumbers",
+              "id": "call_V4daSdWk9QeYeSMKLBisNbSf",
+              "input": "{\"a\": 16, \"b\": 2}"
+            }
+          ]
+        },
+        "delta": ""
+      }
+    },
+    {
+      "id": "PRESERVE_0",
+      "chunk": {
+        "raw": null,
+        "options": {
+          "toolCall": [
+            {
+              "name": "divideNumbers",
+              "id": "call_V4daSdWk9QeYeSMKLBisNbSf",
+              "input": {
+                "a": 16,
+                "b": 2
+              }
+            }
+          ]
+        },
+        "delta": ""
+      }
+    },
+    {
+      "id": "PRESERVE_0",
+      "chunk": {
+        "raw": null,
+        "options": {
+          "toolCall": [
+            {
+              "name": "sumNumbers",
+              "id": "call_n2OlBxlaoeMIMVeU9DeDfiPX",
+              "input": "{\"a\": 8, \"b\": 20}"
+            }
+          ]
+        },
+        "delta": ""
+      }
+    },
+    {
+      "id": "PRESERVE_0",
+      "chunk": {
+        "raw": null,
+        "options": {
+          "toolCall": [
+            {
+              "name": "sumNumbers",
+              "id": "call_n2OlBxlaoeMIMVeU9DeDfiPX",
+              "input": "{\"a\": 8, \"b\": 20}"
+            }
+          ]
+        },
+        "delta": ""
+      }
+    },
+    {
+      "id": "PRESERVE_0",
+      "chunk": {
+        "raw": null,
+        "options": {
+          "toolCall": [
+            {
+              "name": "sumNumbers",
+              "id": "call_n2OlBxlaoeMIMVeU9DeDfiPX",
+              "input": "{\"a\": 8, \"b\": 20}"
+            }
+          ]
+        },
+        "delta": ""
+      }
+    },
+    {
+      "id": "PRESERVE_0",
+      "chunk": {
+        "raw": null,
+        "options": {
+          "toolCall": [
+            {
+              "name": "sumNumbers",
+              "id": "call_n2OlBxlaoeMIMVeU9DeDfiPX",
+              "input": "{\"a\": 8, \"b\": 20}"
+            }
+          ]
         },
         "delta": ""
       }
@@ -317,7 +335,7 @@
       "chunk": {
         "raw": null,
         "options": {},
-        "delta": " and"
+        "delta": " is"
       }
     },
     {
@@ -325,7 +343,7 @@
       "chunk": {
         "raw": null,
         "options": {},
-        "delta": " then"
+        "delta": " "
       }
     },
     {
@@ -333,7 +351,39 @@
       "chunk": {
         "raw": null,
         "options": {},
-        "delta": " adding"
+        "delta": "8"
+      }
+    },
+    {
+      "id": "PRESERVE_1",
+      "chunk": {
+        "raw": null,
+        "options": {},
+        "delta": "."
+      }
+    },
+    {
+      "id": "PRESERVE_1",
+      "chunk": {
+        "raw": null,
+        "options": {},
+        "delta": " When"
+      }
+    },
+    {
+      "id": "PRESERVE_1",
+      "chunk": {
+        "raw": null,
+        "options": {},
+        "delta": " you"
+      }
+    },
+    {
+      "id": "PRESERVE_1",
+      "chunk": {
+        "raw": null,
+        "options": {},
+        "delta": " add"
       }
     },
     {
@@ -352,6 +402,54 @@
         "delta": "20"
       }
     },
+    {
+      "id": "PRESERVE_1",
+      "chunk": {
+        "raw": null,
+        "options": {},
+        "delta": " to"
+      }
+    },
+    {
+      "id": "PRESERVE_1",
+      "chunk": {
+        "raw": null,
+        "options": {},
+        "delta": " "
+      }
+    },
+    {
+      "id": "PRESERVE_1",
+      "chunk": {
+        "raw": null,
+        "options": {},
+        "delta": "8"
+      }
+    },
+    {
+      "id": "PRESERVE_1",
+      "chunk": {
+        "raw": null,
+        "options": {},
+        "delta": ","
+      }
+    },
+    {
+      "id": "PRESERVE_1",
+      "chunk": {
+        "raw": null,
+        "options": {},
+        "delta": " the"
+      }
+    },
+    {
+      "id": "PRESERVE_1",
+      "chunk": {
+        "raw": null,
+        "options": {},
+        "delta": " total"
+      }
+    },
     {
       "id": "PRESERVE_1",
       "chunk": {

packages/core/e2e/node/snapshot/agent_with_object_function_call.snap

@@ -20,16 +20,18 @@
           "content": "",
           "role": "assistant",
           "options": {
-            "toolCall": {
-              "id": "call_iY0VO4HvXDT2PtR3CvRSY1uK",
-              "name": "get_weather",
-              "input": "{\"location\":\"San Francisco\"}"
-            }
+            "toolCall": [
+              {
+                "id": "call_uERMumWlJLTO2GW93X6C2W3N",
+                "name": "get_weather",
+                "input": "{\"location\":\"San Francisco\"}"
+              }
+            ]
           }
         },
         {
-          "content": "{\n  location: San Francisco,\n  temperature: 72,\n  weather: cloudy,\n  rain_prediction: 0.89\n}",
           "role": "user",
+          "content": "{\n  location: San Francisco,\n  temperature: 72,\n  weather: cloudy,\n  rain_prediction: 0.89\n}",
           "options": {
             "toolResult": {
               "result": {
@@ -39,7 +41,7 @@
                 "rain_prediction": 0.89
               },
               "isError": false,
-              "id": "call_iY0VO4HvXDT2PtR3CvRSY1uK"
+              "id": "call_uERMumWlJLTO2GW93X6C2W3N"
             }
           }
         }
@@ -55,11 +57,13 @@
           "content": "",
           "role": "assistant",
           "options": {
-            "toolCall": {
-              "id": "call_iY0VO4HvXDT2PtR3CvRSY1uK",
-              "name": "get_weather",
-              "input": "{\"location\":\"San Francisco\"}"
-            }
+            "toolCall": [
+              {
+                "id": "call_uERMumWlJLTO2GW93X6C2W3N",
+                "name": "get_weather",
+                "input": "{\"location\":\"San Francisco\"}"
+              }
+            ]
           }
         }
       }

packages/core/e2e/node/snapshot/agent_with_object_retriever.snap

@@ -24,6 +24,15 @@
     },
     {
       "id": "PRESERVE_2",
+      "messages": [
+        {
+          "content": "Context information is below.\n---------------------\nAlex is a male. What's very important, Alex is not in the Brazil.\n---------------------\nGiven the context information and not prior knowledge, answer the query.\nQuery: Brazil\nAnswer:",
+          "role": "user"
+        }
+      ]
+    },
+    {
+      "id": "PRESERVE_3",
       "messages": [
         {
           "content": "Please always use the tools provided to answer a question. Do not rely on prior knowledge.",
@@ -37,21 +46,39 @@
           "content": "",
           "role": "assistant",
           "options": {
-            "toolCall": {
-              "id": "call_VfgZS1NPKpbtzqhM3o1Buzx8",
-              "name": "summary_tool",
-              "input": "{\"query\": \"Alex\"}"
-            }
+            "toolCall": [
+              {
+                "id": "call_vv1XW3xv4j2us5sZOtzCU2lL",
+                "name": "summary_tool",
+                "input": "{\"query\": \"Alex\"}"
+              },
+              {
+                "id": "call_V36LMHbwUJkEa20A3GoA4wIr",
+                "name": "summary_tool",
+                "input": "{\"query\": \"Brazil\"}"
+              }
+            ]
           }
         },
         {
-          "content": "Alex is not in Brazil.",
           "role": "user",
+          "content": "Alex is not in Brazil.",
           "options": {
             "toolResult": {
               "result": "Alex is not in Brazil.",
               "isError": false,
-              "id": "call_VfgZS1NPKpbtzqhM3o1Buzx8"
+              "id": "call_vv1XW3xv4j2us5sZOtzCU2lL"
+            }
+          }
+        },
+        {
+          "role": "user",
+          "content": "Alex is not in Brazil.",
+          "options": {
+            "toolResult": {
+              "result": "Alex is not in Brazil.",
+              "isError": false,
+              "id": "call_V36LMHbwUJkEa20A3GoA4wIr"
             }
           }
         }
@@ -67,11 +94,18 @@
           "content": "",
           "role": "assistant",
           "options": {
-            "toolCall": {
-              "id": "call_VfgZS1NPKpbtzqhM3o1Buzx8",
-              "name": "summary_tool",
-              "input": "{\"query\": \"Alex\"}"
-            }
+            "toolCall": [
+              {
+                "id": "call_vv1XW3xv4j2us5sZOtzCU2lL",
+                "name": "summary_tool",
+                "input": "{\"query\": \"Alex\"}"
+              },
+              {
+                "id": "call_V36LMHbwUJkEa20A3GoA4wIr",
+                "name": "summary_tool",
+                "input": "{\"query\": \"Brazil\"}"
+              }
+            ]
           }
         }
       }
@@ -92,7 +126,18 @@
       "response": {
         "raw": null,
         "message": {
-          "content": "No, Alex does not live in Brazil based on the brief information available.",
+          "content": "Alex is not in Brazil.",
+          "role": "assistant",
+          "options": {}
+        }
+      }
+    },
+    {
+      "id": "PRESERVE_3",
+      "response": {
+        "raw": null,
+        "message": {
+          "content": "Based on the brief information, Alex is not in Brazil.",
           "role": "assistant",
           "options": {}
         }

packages/core/e2e/node/snapshot/anthropic-agent-multiple-chat.snap

@@ -178,13 +178,15 @@
           ],
           "role": "assistant",
           "options": {
-            "toolCall": {
-              "id": "toolu_01Gy7Gxbx7uGmjVncGH6pubL",
-              "name": "getWeather",
-              "input": {
-                "city": "San Francisco"
+            "toolCall": [
+              {
+                "id": "toolu_017MxwaxaLYkmt4dpP5HKzFe",
+                "name": "getWeather",
+                "input": {
+                  "city": "San Francisco"
+                }
               }
-            }
+            ]
           }
         },
         {
@@ -194,7 +196,7 @@
             "toolResult": {
               "result": "The weather in San Francisco is 72 degrees",
               "isError": false,
-              "id": "toolu_01Gy7Gxbx7uGmjVncGH6pubL"
+              "id": "toolu_017MxwaxaLYkmt4dpP5HKzFe"
             }
           }
         }
@@ -258,13 +260,15 @@
           ],
           "role": "assistant",
           "options": {
-            "toolCall": {
-              "id": "toolu_01Gy7Gxbx7uGmjVncGH6pubL",
-              "name": "getWeather",
-              "input": {
-                "city": "San Francisco"
+            "toolCall": [
+              {
+                "id": "toolu_017MxwaxaLYkmt4dpP5HKzFe",
+                "name": "getWeather",
+                "input": {
+                  "city": "San Francisco"
+                }
               }
-            }
+            ]
           }
         },
         {
@@ -274,7 +278,7 @@
             "toolResult": {
               "result": "The weather in San Francisco is 72 degrees",
               "isError": false,
-              "id": "toolu_01Gy7Gxbx7uGmjVncGH6pubL"
+              "id": "toolu_017MxwaxaLYkmt4dpP5HKzFe"
             }
           }
         },
@@ -352,13 +356,15 @@
           ],
           "role": "assistant",
           "options": {
-            "toolCall": {
-              "id": "toolu_01Gy7Gxbx7uGmjVncGH6pubL",
-              "name": "getWeather",
-              "input": {
-                "city": "San Francisco"
+            "toolCall": [
+              {
+                "id": "toolu_017MxwaxaLYkmt4dpP5HKzFe",
+                "name": "getWeather",
+                "input": {
+                  "city": "San Francisco"
+                }
               }
-            }
+            ]
           }
         },
         {
@@ -368,7 +374,7 @@
             "toolResult": {
               "result": "The weather in San Francisco is 72 degrees",
               "isError": false,
-              "id": "toolu_01Gy7Gxbx7uGmjVncGH6pubL"
+              "id": "toolu_017MxwaxaLYkmt4dpP5HKzFe"
             }
           }
         },
@@ -395,13 +401,15 @@
           ],
           "role": "assistant",
           "options": {
-            "toolCall": {
-              "id": "toolu_01NHyahSUqrPjxQk9mvCvvGe",
-              "name": "getWeather",
-              "input": {
-                "city": "Shanghai"
+            "toolCall": [
+              {
+                "id": "toolu_013ZnATGKhvPkt2jjxdFheDK",
+                "name": "getWeather",
+                "input": {
+                  "city": "Shanghai"
+                }
               }
-            }
+            ]
           }
         },
         {
@@ -411,7 +419,7 @@
             "toolResult": {
               "result": "The weather in Shanghai is 72 degrees",
               "isError": false,
-              "id": "toolu_01NHyahSUqrPjxQk9mvCvvGe"
+              "id": "toolu_013ZnATGKhvPkt2jjxdFheDK"
             }
           }
         }
@@ -480,13 +488,15 @@
           ],
           "role": "assistant",
           "options": {
-            "toolCall": {
-              "id": "toolu_01Gy7Gxbx7uGmjVncGH6pubL",
-              "name": "getWeather",
-              "input": {
-                "city": "San Francisco"
+            "toolCall": [
+              {
+                "id": "toolu_017MxwaxaLYkmt4dpP5HKzFe",
+                "name": "getWeather",
+                "input": {
+                  "city": "San Francisco"
+                }
               }
-            }
+            ]
           }
         }
       }
@@ -520,13 +530,15 @@
           ],
           "role": "assistant",
           "options": {
-            "toolCall": {
-              "id": "toolu_01NHyahSUqrPjxQk9mvCvvGe",
-              "name": "getWeather",
-              "input": {
-                "city": "Shanghai"
+            "toolCall": [
+              {
+                "id": "toolu_013ZnATGKhvPkt2jjxdFheDK",
+                "name": "getWeather",
+                "input": {
+                  "city": "Shanghai"
+                }
               }
-            }
+            ]
           }
         }
       }

packages/core/e2e/node/snapshot/anthropic-agent.snap

@@ -5,8 +5,7 @@
       "messages": [
         {
           "role": "user",
-          "content": "What is the weather in San Francisco?",
-          "options": {}
+          "content": "What is the weather in San Francisco?"
         }
       ]
     },
@@ -15,25 +14,26 @@
       "messages": [
         {
           "role": "user",
-          "content": "What is the weather in San Francisco?",
-          "options": {}
+          "content": "What is the weather in San Francisco?"
         },
         {
           "content": [
             {
               "type": "text",
-              "text": "<thinking>\nThe user is asking for the weather in a specific location, San Francisco. The Weather function is the relevant tool to answer this request, as it returns weather information for a given location.\n\nThe Weather function has one required parameter:\n- location (string): The user has directly provided the location of \"San Francisco\"\n\nSince the required location parameter has been provided by the user, we have all the necessary information to call the Weather function.\n</thinking>"
+              "text": "<thinking>\nThe user has asked for the weather in a specific location, San Francisco. The Weather tool is the relevant function to answer this request, as it returns weather information for a given location.\n\nThe Weather tool requires a single parameter:\n- location (string, required): The user has directly provided the location as \"San Francisco\".\n\nSince the required location parameter has been provided, we have enough information to call the Weather tool.\n</thinking>"
             }
           ],
           "role": "assistant",
           "options": {
-            "toolCall": {
-              "id": "HIDDEN",
-              "name": "Weather",
-              "input": {
-                "location": "San Francisco"
+            "toolCall": [
+              {
+                "id": "toolu_011YcKkPygw2woJZrVWRRMwH",
+                "name": "Weather",
+                "input": {
+                  "location": "San Francisco"
+                }
               }
-            }
+            ]
           }
         },
         {
@@ -41,8 +41,9 @@
           "role": "user",
           "options": {
             "toolResult": {
+              "result": "35 degrees and sunny in San Francisco",
               "isError": false,
-              "id": "HIDDEN"
+              "id": "toolu_011YcKkPygw2woJZrVWRRMwH"
             }
           }
         }
@@ -53,8 +54,7 @@
       "messages": [
         {
           "role": "user",
-          "content": "My name is Alex Yang. What is my unique id?",
-          "options": {}
+          "content": "My name is Alex Yang. What is my unique id?"
         }
       ]
     },
@@ -63,26 +63,27 @@
       "messages": [
         {
           "role": "user",
-          "content": "My name is Alex Yang. What is my unique id?",
-          "options": {}
+          "content": "My name is Alex Yang. What is my unique id?"
         },
         {
           "content": [
             {
               "type": "text",
-              "text": "<thinking>\nThe unique_id function is the relevant tool to answer the user's request for their unique ID. It requires two parameters:\nfirstName: The user provided their first name, which is \"Alex\"\nlastName: The user also provided their last name, \"Yang\"\nSince the user has provided all the required parameters, we can proceed with calling the unique_id function.\n</thinking>"
+              "text": "<thinking>\nThe unique_id function takes firstName and lastName as required parameters. The user has provided their first name (Alex) and last name (Yang) in the request, so we have all the necessary information to call the function.\n</thinking>"
             }
           ],
           "role": "assistant",
           "options": {
-            "toolCall": {
-              "id": "HIDDEN",
-              "name": "unique_id",
-              "input": {
-                "firstName": "Alex",
-                "lastName": "Yang"
+            "toolCall": [
+              {
+                "id": "toolu_012ZLUq5SWsghkXUjsEXsB1f",
+                "name": "unique_id",
+                "input": {
+                  "firstName": "Alex",
+                  "lastName": "Yang"
+                }
               }
-            }
+            ]
           }
         },
         {
@@ -90,8 +91,9 @@
           "role": "user",
           "options": {
             "toolResult": {
+              "result": "123456789",
               "isError": false,
-              "id": "HIDDEN"
+              "id": "toolu_012ZLUq5SWsghkXUjsEXsB1f"
             }
           }
         }
@@ -102,8 +104,7 @@
       "messages": [
         {
           "role": "user",
-          "content": "how much is 1 + 1?",
-          "options": {}
+          "content": "how much is 1 + 1?"
         }
       ]
     },
@@ -112,26 +113,27 @@
       "messages": [
         {
           "role": "user",
-          "content": "how much is 1 + 1?",
-          "options": {}
+          "content": "how much is 1 + 1?"
         },
         {
           "content": [
             {
               "type": "text",
-              "text": "<thinking>\nThe user is asking to sum the numbers 1 and 1. The relevant tool to use is the sumNumbers function, which takes two number parameters a and b.\nThe user has directly provided the values for the parameters:\na = 1 \nb = 1\nSince all the required parameters have been provided, we can proceed with calling the function.\n</thinking>"
+              "text": "<thinking>\nThe user is asking to sum the numbers 1 and 1. The relevant tool is the sumNumbers function, which takes two number parameters a and b.\nThe user has directly provided the values for the required parameters:\na = 1 \nb = 1\nSince all the required parameters are provided, we can proceed with calling the function.\n</thinking>"
             }
           ],
           "role": "assistant",
           "options": {
-            "toolCall": {
-              "id": "HIDDEN",
-              "name": "sumNumbers",
-              "input": {
-                "a": 1,
-                "b": 1
+            "toolCall": [
+              {
+                "id": "toolu_01SQD2XkaXkDNLQ2xaFzmguG",
+                "name": "sumNumbers",
+                "input": {
+                  "a": 1,
+                  "b": 1
+                }
               }
-            }
+            ]
           }
         },
         {
@@ -139,8 +141,9 @@
           "role": "user",
           "options": {
             "toolResult": {
+              "result": "2",
               "isError": false,
-              "id": "HIDDEN"
+              "id": "toolu_01SQD2XkaXkDNLQ2xaFzmguG"
             }
           }
         }
@@ -151,48 +154,25 @@
     {
       "id": "PRESERVE_0",
       "response": {
-        "raw": {
-          "id": "HIDDEN",
-          "type": "message",
-          "role": "assistant",
-          "model": "claude-3-opus-20240229",
-          "stop_sequence": null,
-          "usage": {
-            "input_tokens": 462,
-            "output_tokens": 147
-          },
-          "content": [
-            {
-              "type": "text",
-              "text": "<thinking>\nThe user is asking for the weather in a specific location, San Francisco. The Weather function is the relevant tool to answer this request, as it returns weather information for a given location.\n\nThe Weather function has one required parameter:\n- location (string): The user has directly provided the location of \"San Francisco\"\n\nSince the required location parameter has been provided by the user, we have all the necessary information to call the Weather function.\n</thinking>"
-            },
-            {
-              "type": "tool_use",
-              "id": "HIDDEN",
-              "name": "Weather",
-              "input": {
-                "location": "San Francisco"
-              }
-            }
-          ],
-          "stop_reason": "tool_use"
-        },
+        "raw": null,
         "message": {
           "content": [
             {
               "type": "text",
-              "text": "<thinking>\nThe user is asking for the weather in a specific location, San Francisco. The Weather function is the relevant tool to answer this request, as it returns weather information for a given location.\n\nThe Weather function has one required parameter:\n- location (string): The user has directly provided the location of \"San Francisco\"\n\nSince the required location parameter has been provided by the user, we have all the necessary information to call the Weather function.\n</thinking>"
+              "text": "<thinking>\nThe user has asked for the weather in a specific location, San Francisco. The Weather tool is the relevant function to answer this request, as it returns weather information for a given location.\n\nThe Weather tool requires a single parameter:\n- location (string, required): The user has directly provided the location as \"San Francisco\".\n\nSince the required location parameter has been provided, we have enough information to call the Weather tool.\n</thinking>"
             }
           ],
           "role": "assistant",
           "options": {
-            "toolCall": {
-              "id": "HIDDEN",
-              "name": "Weather",
-              "input": {
-                "location": "San Francisco"
+            "toolCall": [
+              {
+                "id": "toolu_011YcKkPygw2woJZrVWRRMwH",
+                "name": "Weather",
+                "input": {
+                  "location": "San Francisco"
+                }
               }
-            }
+            ]
           }
         }
       }
@@ -200,24 +180,7 @@
     {
       "id": "PRESERVE_1",
       "response": {
-        "raw": {
-          "id": "HIDDEN",
-          "type": "message",
-          "role": "assistant",
-          "model": "claude-3-opus-20240229",
-          "stop_sequence": null,
-          "usage": {
-            "input_tokens": 628,
-            "output_tokens": 18
-          },
-          "content": [
-            {
-              "type": "text",
-              "text": "The current weather in San Francisco is 35 degrees and sunny."
-            }
-          ],
-          "stop_reason": "end_turn"
-        },
+        "raw": null,
         "message": {
           "content": [
             {
@@ -233,50 +196,26 @@
     {
       "id": "PRESERVE_2",
       "response": {
-        "raw": {
-          "id": "HIDDEN",
-          "type": "message",
-          "role": "assistant",
-          "model": "claude-3-opus-20240229",
-          "stop_sequence": null,
-          "usage": {
-            "input_tokens": 482,
-            "output_tokens": 152
-          },
-          "content": [
-            {
-              "type": "text",
-              "text": "<thinking>\nThe unique_id function is the relevant tool to answer the user's request for their unique ID. It requires two parameters:\nfirstName: The user provided their first name, which is \"Alex\"\nlastName: The user also provided their last name, \"Yang\"\nSince the user has provided all the required parameters, we can proceed with calling the unique_id function.\n</thinking>"
-            },
-            {
-              "type": "tool_use",
-              "id": "HIDDEN",
-              "name": "unique_id",
-              "input": {
-                "firstName": "Alex",
-                "lastName": "Yang"
-              }
-            }
-          ],
-          "stop_reason": "tool_use"
-        },
+        "raw": null,
         "message": {
           "content": [
             {
               "type": "text",
-              "text": "<thinking>\nThe unique_id function is the relevant tool to answer the user's request for their unique ID. It requires two parameters:\nfirstName: The user provided their first name, which is \"Alex\"\nlastName: The user also provided their last name, \"Yang\"\nSince the user has provided all the required parameters, we can proceed with calling the unique_id function.\n</thinking>"
+              "text": "<thinking>\nThe unique_id function takes firstName and lastName as required parameters. The user has provided their first name (Alex) and last name (Yang) in the request, so we have all the necessary information to call the function.\n</thinking>"
             }
           ],
           "role": "assistant",
           "options": {
-            "toolCall": {
-              "id": "HIDDEN",
-              "name": "unique_id",
-              "input": {
-                "firstName": "Alex",
-                "lastName": "Yang"
+            "toolCall": [
+              {
+                "id": "toolu_012ZLUq5SWsghkXUjsEXsB1f",
+                "name": "unique_id",
+                "input": {
+                  "firstName": "Alex",
+                  "lastName": "Yang"
+                }
               }
-            }
+            ]
           }
         }
       }
@@ -284,24 +223,7 @@
     {
       "id": "PRESERVE_3",
       "response": {
-        "raw": {
-          "id": "HIDDEN",
-          "type": "message",
-          "role": "assistant",
-          "model": "claude-3-opus-20240229",
-          "stop_sequence": null,
-          "usage": {
-            "input_tokens": 648,
-            "output_tokens": 13
-          },
-          "content": [
-            {
-              "type": "text",
-              "text": "Your unique ID is 123456789."
-            }
-          ],
-          "stop_reason": "end_turn"
-        },
+        "raw": null,
         "message": {
           "content": [
             {
@@ -317,50 +239,26 @@
     {
       "id": "PRESERVE_4",
       "response": {
-        "raw": {
-          "id": "HIDDEN",
-          "type": "message",
-          "role": "assistant",
-          "model": "claude-3-opus-20240229",
-          "stop_sequence": null,
-          "usage": {
-            "input_tokens": 498,
-            "output_tokens": 151
-          },
-          "content": [
-            {
-              "type": "text",
-              "text": "<thinking>\nThe user is asking to sum the numbers 1 and 1. The relevant tool to use is the sumNumbers function, which takes two number parameters a and b.\nThe user has directly provided the values for the parameters:\na = 1 \nb = 1\nSince all the required parameters have been provided, we can proceed with calling the function.\n</thinking>"
-            },
-            {
-              "type": "tool_use",
-              "id": "HIDDEN",
-              "name": "sumNumbers",
-              "input": {
-                "a": 1,
-                "b": 1
-              }
-            }
-          ],
-          "stop_reason": "tool_use"
-        },
+        "raw": null,
         "message": {
           "content": [
             {
               "type": "text",
-              "text": "<thinking>\nThe user is asking to sum the numbers 1 and 1. The relevant tool to use is the sumNumbers function, which takes two number parameters a and b.\nThe user has directly provided the values for the parameters:\na = 1 \nb = 1\nSince all the required parameters have been provided, we can proceed with calling the function.\n</thinking>"
+              "text": "<thinking>\nThe user is asking to sum the numbers 1 and 1. The relevant tool is the sumNumbers function, which takes two number parameters a and b.\nThe user has directly provided the values for the required parameters:\na = 1 \nb = 1\nSince all the required parameters are provided, we can proceed with calling the function.\n</thinking>"
             }
           ],
           "role": "assistant",
           "options": {
-            "toolCall": {
-              "id": "HIDDEN",
-              "name": "sumNumbers",
-              "input": {
-                "a": 1,
-                "b": 1
+            "toolCall": [
+              {
+                "id": "toolu_01SQD2XkaXkDNLQ2xaFzmguG",
+                "name": "sumNumbers",
+                "input": {
+                  "a": 1,
+                  "b": 1
+                }
               }
-            }
+            ]
           }
         }
       }
@@ -368,24 +266,7 @@
     {
       "id": "PRESERVE_5",
       "response": {
-        "raw": {
-          "id": "HIDDEN",
-          "type": "message",
-          "role": "assistant",
-          "model": "claude-3-opus-20240229",
-          "stop_sequence": null,
-          "usage": {
-            "input_tokens": 661,
-            "output_tokens": 16
-          },
-          "content": [
-            {
-              "type": "text",
-              "text": "So 1 + 1 = 2."
-            }
-          ],
-          "stop_reason": "end_turn"
-        },
+        "raw": null,
         "message": {
           "content": [
             {

packages/core/e2e/node/snapshot/gpt-4-turbo.snap

@@ -20,21 +20,23 @@
           "content": "",
           "role": "assistant",
           "options": {
-            "toolCall": {
-              "id": "call_TRC5kpzyVaV5n2R91sJsBAF1",
-              "name": "Weather",
-              "input": "{\"location\":\"San Jose\"}"
-            }
+            "toolCall": [
+              {
+                "id": "call_Xa2Kxa2zUE073mnougPWzRlh",
+                "name": "Weather",
+                "input": "{\"location\":\"San Jose\"}"
+              }
+            ]
           }
         },
         {
-          "content": "45 degrees and sunny in San Jose",
           "role": "user",
+          "content": "45 degrees and sunny in San Jose",
           "options": {
             "toolResult": {
               "result": "45 degrees and sunny in San Jose",
               "isError": false,
-              "id": "call_TRC5kpzyVaV5n2R91sJsBAF1"
+              "id": "call_Xa2Kxa2zUE073mnougPWzRlh"
             }
           }
         }
@@ -50,11 +52,13 @@
           "content": "",
           "role": "assistant",
           "options": {
-            "toolCall": {
-              "id": "call_TRC5kpzyVaV5n2R91sJsBAF1",
-              "name": "Weather",
-              "input": "{\"location\":\"San Jose\"}"
-            }
+            "toolCall": [
+              {
+                "id": "call_Xa2Kxa2zUE073mnougPWzRlh",
+                "name": "Weather",
+                "input": "{\"location\":\"San Jose\"}"
+              }
+            ]
           }
         }
       }
@@ -64,7 +68,7 @@
       "response": {
         "raw": null,
         "message": {
-          "content": "The weather in San Jose is currently 45 degrees and sunny.",
+          "content": "The current weather in San Jose is 45 degrees and sunny.",
           "role": "assistant",
           "options": {}
         }

packages/core/e2e/node/snapshot/llm-anthropic.snap

@@ -24,24 +24,7 @@
     {
       "id": "PRESERVE_0",
       "response": {
-        "raw": {
-          "id": "HIDDEN",
-          "type": "message",
-          "role": "assistant",
-          "model": "claude-3-opus-20240229",
-          "stop_sequence": null,
-          "usage": {
-            "input_tokens": 8,
-            "output_tokens": 12
-          },
-          "content": [
-            {
-              "type": "text",
-              "text": "Hello! How can I assist you today?"
-            }
-          ],
-          "stop_reason": "end_turn"
-        },
+        "raw": null,
         "message": {
           "content": "Hello! How can I assist you today?",
           "role": "assistant",
@@ -52,116 +35,7 @@
     {
       "id": "PRESERVE_1",
       "response": {
-        "raw": [
-          {
-            "raw": {
-              "type": "content_block_delta",
-              "index": 0,
-              "delta": {
-                "type": "text_delta",
-                "text": "Hello"
-              }
-            },
-            "delta": "Hello",
-            "options": {}
-          },
-          {
-            "raw": {
-              "type": "content_block_delta",
-              "index": 0,
-              "delta": {
-                "type": "text_delta",
-                "text": "!"
-              }
-            },
-            "delta": "!",
-            "options": {}
-          },
-          {
-            "raw": {
-              "type": "content_block_delta",
-              "index": 0,
-              "delta": {
-                "type": "text_delta",
-                "text": " How"
-              }
-            },
-            "delta": " How",
-            "options": {}
-          },
-          {
-            "raw": {
-              "type": "content_block_delta",
-              "index": 0,
-              "delta": {
-                "type": "text_delta",
-                "text": " can"
-              }
-            },
-            "delta": " can",
-            "options": {}
-          },
-          {
-            "raw": {
-              "type": "content_block_delta",
-              "index": 0,
-              "delta": {
-                "type": "text_delta",
-                "text": " I"
-              }
-            },
-            "delta": " I",
-            "options": {}
-          },
-          {
-            "raw": {
-              "type": "content_block_delta",
-              "index": 0,
-              "delta": {
-                "type": "text_delta",
-                "text": " assist"
-              }
-            },
-            "delta": " assist",
-            "options": {}
-          },
-          {
-            "raw": {
-              "type": "content_block_delta",
-              "index": 0,
-              "delta": {
-                "type": "text_delta",
-                "text": " you"
-              }
-            },
-            "delta": " you",
-            "options": {}
-          },
-          {
-            "raw": {
-              "type": "content_block_delta",
-              "index": 0,
-              "delta": {
-                "type": "text_delta",
-                "text": " today"
-              }
-            },
-            "delta": " today",
-            "options": {}
-          },
-          {
-            "raw": {
-              "type": "content_block_delta",
-              "index": 0,
-              "delta": {
-                "type": "text_delta",
-                "text": "?"
-              }
-            },
-            "delta": "?",
-            "options": {}
-          }
-        ],
+        "raw": null,
         "message": {
           "content": "Hello! How can I assist you today?",
           "role": "assistant",
@@ -174,14 +48,7 @@
     {
       "id": "PRESERVE_1",
       "chunk": {
-        "raw": {
-          "type": "content_block_delta",
-          "index": 0,
-          "delta": {
-            "type": "text_delta",
-            "text": "Hello"
-          }
-        },
+        "raw": null,
         "delta": "Hello",
         "options": {}
       }
@@ -189,14 +56,7 @@
     {
       "id": "PRESERVE_1",
       "chunk": {
-        "raw": {
-          "type": "content_block_delta",
-          "index": 0,
-          "delta": {
-            "type": "text_delta",
-            "text": "!"
-          }
-        },
+        "raw": null,
         "delta": "!",
         "options": {}
       }
@@ -204,14 +64,7 @@
     {
       "id": "PRESERVE_1",
       "chunk": {
-        "raw": {
-          "type": "content_block_delta",
-          "index": 0,
-          "delta": {
-            "type": "text_delta",
-            "text": " How"
-          }
-        },
+        "raw": null,
         "delta": " How",
         "options": {}
       }
@@ -219,14 +72,7 @@
     {
       "id": "PRESERVE_1",
       "chunk": {
-        "raw": {
-          "type": "content_block_delta",
-          "index": 0,
-          "delta": {
-            "type": "text_delta",
-            "text": " can"
-          }
-        },
+        "raw": null,
         "delta": " can",
         "options": {}
       }
@@ -234,14 +80,7 @@
     {
       "id": "PRESERVE_1",
       "chunk": {
-        "raw": {
-          "type": "content_block_delta",
-          "index": 0,
-          "delta": {
-            "type": "text_delta",
-            "text": " I"
-          }
-        },
+        "raw": null,
         "delta": " I",
         "options": {}
       }
@@ -249,14 +88,7 @@
     {
       "id": "PRESERVE_1",
       "chunk": {
-        "raw": {
-          "type": "content_block_delta",
-          "index": 0,
-          "delta": {
-            "type": "text_delta",
-            "text": " assist"
-          }
-        },
+        "raw": null,
         "delta": " assist",
         "options": {}
       }
@@ -264,14 +96,7 @@
     {
       "id": "PRESERVE_1",
       "chunk": {
-        "raw": {
-          "type": "content_block_delta",
-          "index": 0,
-          "delta": {
-            "type": "text_delta",
-            "text": " you"
-          }
-        },
+        "raw": null,
         "delta": " you",
         "options": {}
       }
@@ -279,14 +104,7 @@
     {
       "id": "PRESERVE_1",
       "chunk": {
-        "raw": {
-          "type": "content_block_delta",
-          "index": 0,
-          "delta": {
-            "type": "text_delta",
-            "text": " today"
-          }
-        },
+        "raw": null,
         "delta": " today",
         "options": {}
       }
@@ -294,14 +112,7 @@
     {
       "id": "PRESERVE_1",
       "chunk": {
-        "raw": {
-          "type": "content_block_delta",
-          "index": 0,
-          "delta": {
-            "type": "text_delta",
-            "text": "?"
-          }
-        },
+        "raw": null,
         "delta": "?",
         "options": {}
       }

packages/core/e2e/node/snapshot/openai_agent_system_prompt.snap

@@ -28,21 +28,23 @@
           "content": "",
           "role": "assistant",
           "options": {
-            "toolCall": {
-              "id": "call_loaXnCH3WRnbnROZzNbrRkHS",
-              "name": "getWeather",
-              "input": "{\"city\":\"San Francisco\"}"
-            }
+            "toolCall": [
+              {
+                "id": "call_1SlugFuJ7rhwsmXd5aRnJhPe",
+                "name": "getWeather",
+                "input": "{\"city\":\"San Francisco\"}"
+              }
+            ]
           }
         },
         {
-          "content": "The weather in San Francisco is 72 degrees",
           "role": "user",
+          "content": "The weather in San Francisco is 72 degrees",
           "options": {
             "toolResult": {
               "result": "The weather in San Francisco is 72 degrees",
               "isError": false,
-              "id": "call_loaXnCH3WRnbnROZzNbrRkHS"
+              "id": "call_1SlugFuJ7rhwsmXd5aRnJhPe"
             }
           }
         }
@@ -58,11 +60,13 @@
           "content": "",
           "role": "assistant",
           "options": {
-            "toolCall": {
-              "id": "call_loaXnCH3WRnbnROZzNbrRkHS",
-              "name": "getWeather",
-              "input": "{\"city\":\"San Francisco\"}"
-            }
+            "toolCall": [
+              {
+                "id": "call_1SlugFuJ7rhwsmXd5aRnJhPe",
+                "name": "getWeather",
+                "input": "{\"city\":\"San Francisco\"}"
+              }
+            ]
           }
         }
       }
@@ -72,7 +76,7 @@
       "response": {
         "raw": null,
         "message": {
-          "content": "Arhgs! The weather in San Francisco is 72 degrees.",
+          "content": "Arhgs the weather in San Francisco be 72 degrees.",
           "role": "assistant",
           "options": {}
         }

packages/core/e2e/node/snapshot/react-agent-stream.snap

@@ -41,7 +41,7 @@
       "response": {
         "raw": null,
         "message": {
-          "content": "Thought: I need to use a tool to help me answer the question.\nAction: getWeather\nAction Input: {\"city\": \"San Francisco\"}",
+          "content": "Thought: I need to use a tool to help me answer the question. \nAction: getWeather\nAction Input: {\"city\": \"San Francisco\"}",
           "role": "assistant",
           "options": {}
         }
@@ -177,7 +177,15 @@
       "chunk": {
         "raw": null,
         "options": {},
-        "delta": ".\n"
+        "delta": "."
+      }
+    },
+    {
+      "id": "PRESERVE_0",
+      "chunk": {
+        "raw": null,
+        "options": {},
+        "delta": " \n"
       }
     },
     {

packages/core/e2e/node/snapshot/react-agent.snap

@@ -41,7 +41,7 @@
       "response": {
         "raw": null,
         "message": {
-          "content": "Thought: I need to use a tool to help me answer the question.\nAction: getWeather\nAction Input: {\"city\": \"San Francisco\"}",
+          "content": "Thought: I need to use a tool to help me answer the question. \nAction: getWeather\nAction Input: {\"city\": \"San Francisco\"}",
           "role": "assistant",
           "options": {}
         }

packages/core/jsr.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/core",
-  "version": "0.3.13",
+  "version": "0.3.16",
   "exports": "./src/index.ts",
   "imports": {
     "@llamaindex/env": "jsr:@llamaindex/env@0.1.3"

packages/core/package.json

@@ -1,6 +1,6 @@
 {
   "name": "llamaindex",
-  "version": "0.3.13",
+  "version": "0.3.16",
   "expectedMinorVersion": "3",
   "license": "MIT",
   "type": "module",
@@ -21,45 +21,45 @@
     "llamaindex"
   ],
   "dependencies": {
-    "@anthropic-ai/sdk": "^0.20.9",
+    "@anthropic-ai/sdk": "^0.21.1",
     "@aws-crypto/sha256-js": "^5.2.0",
-    "@datastax/astra-db-ts": "^1.1.0",
-    "@google-cloud/vertexai": "^1.1.0",
-    "@google/generative-ai": "^0.11.0",
-    "@grpc/grpc-js": "^1.10.7",
-    "@huggingface/inference": "^2.6.7",
+    "@datastax/astra-db-ts": "^1.2.1",
+    "@google-cloud/vertexai": "^1.2.0",
+    "@google/generative-ai": "^0.12.0",
+    "@grpc/grpc-js": "^1.10.8",
+    "@huggingface/inference": "^2.7.0",
     "@llamaindex/cloud": "0.0.5",
     "@llamaindex/env": "workspace:*",
-    "@mistralai/mistralai": "^0.2.0",
-    "@pinecone-database/pinecone": "^2.2.0",
+    "@mistralai/mistralai": "^0.4.0",
+    "@pinecone-database/pinecone": "^2.2.2",
     "@qdrant/js-client-rest": "^1.9.0",
-    "@types/lodash": "^4.17.1",
+    "@types/lodash": "^4.17.4",
     "@types/papaparse": "^5.3.14",
     "@types/pg": "^8.11.6",
-    "@xenova/transformers": "^2.17.1",
+    "@xenova/transformers": "^2.17.2",
     "@zilliz/milvus2-sdk-node": "^2.4.2",
-    "ajv": "^8.13.0",
-    "assemblyai": "^4.4.2",
-    "chromadb": "~1.7.3",
-    "cohere-ai": "^7.9.5",
-    "js-tiktoken": "^1.0.11",
+    "ajv": "^8.16.0",
+    "assemblyai": "^4.4.5",
+    "chromadb": "1.8.1",
+    "cohere-ai": "7.9.5",
+    "js-tiktoken": "^1.0.12",
     "lodash": "^4.17.21",
     "magic-bytes.js": "^1.10.0",
     "mammoth": "^1.7.2",
     "md-utils-ts": "^2.0.0",
-    "mongodb": "^6.6.1",
+    "mongodb": "^6.7.0",
     "notion-md-crawler": "^1.0.0",
-    "openai": "^4.46.0",
+    "openai": "^4.48.1",
     "papaparse": "^5.4.1",
     "pathe": "^1.1.2",
-    "pdf2json": "3.0.5",
-    "pg": "^8.11.5",
+    "pdf2json": "3.1.3",
+    "pg": "^8.12.0",
     "pgvector": "^0.1.8",
     "portkey-ai": "^0.1.16",
     "rake-modified": "^1.0.8",
     "string-strip-html": "^13.4.8",
     "wikipedia": "^2.1.2",
-    "wink-nlp": "^2.2.2"
+    "wink-nlp": "^2.3.0"
   },
   "peerDependencies": {
     "@notionhq/client": "^2.2.15"
@@ -67,9 +67,9 @@
   "devDependencies": {
     "@notionhq/client": "^2.2.15",
     "@swc/cli": "^0.3.12",
-    "@swc/core": "^1.5.5",
+    "@swc/core": "^1.5.24",
     "concurrently": "^8.2.2",
-    "glob": "^10.3.12",
+    "glob": "^10.4.1",
     "typescript": "^5.4.5"
   },
   "engines": {

packages/core/src/Retriever.ts

@@ -1,8 +1,9 @@
 import type { NodeWithScore } from "./Node.js";
 import type { ServiceContext } from "./ServiceContext.js";
+import type { MessageContent } from "./index.edge.js";
 
 export type RetrieveParams = {
-  query: string;
+  query: MessageContent;
   preFilters?: unknown;
 };
 

packages/core/src/agent/anthropic.ts

@@ -93,29 +93,31 @@ export class AnthropicAgent extends AgentRunner<Anthropic> {
     });
     if ("toolCall" in options) {
       const { toolCall } = options;
-      const targetTool = tools.find(
-        (tool) => tool.metadata.name === toolCall.name,
-      );
-      const toolOutput = await callTool(
-        targetTool,
-        toolCall,
-        step.context.logger,
-      );
-      step.context.store.toolOutputs.push(toolOutput);
-      step.context.store.messages = [
-        ...step.context.store.messages,
-        {
-          content: stringifyJSONToMessageContent(toolOutput.output),
-          role: "user",
-          options: {
-            toolResult: {
-              result: toolOutput.output,
-              isError: toolOutput.isError,
-              id: toolCall.id,
+      for (const call of toolCall) {
+        const targetTool = tools.find(
+          (tool) => tool.metadata.name === call.name,
+        );
+        const toolOutput = await callTool(
+          targetTool,
+          call,
+          step.context.logger,
+        );
+        step.context.store.toolOutputs.push(toolOutput);
+        step.context.store.messages = [
+          ...step.context.store.messages,
+          {
+            content: stringifyJSONToMessageContent(toolOutput.output),
+            role: "user",
+            options: {
+              toolResult: {
+                result: toolOutput.output,
+                isError: toolOutput.isError,
+                id: call.id,
+              },
             },
           },
-        },
-      ];
+        ];
+      }
     }
   };
 }

packages/core/src/agent/openai.ts

@@ -76,31 +76,34 @@ export class OpenAIAgent extends AgentRunner<OpenAI> {
       });
       if ("toolCall" in options) {
         const { toolCall } = options;
-        const targetTool = tools.find(
-          (tool) => tool.metadata.name === toolCall.name,
-        );
-        const toolOutput = await callTool(
-          targetTool,
-          toolCall,
-          step.context.logger,
-        );
-        step.context.store.toolOutputs.push(toolOutput);
-        step.context.store.messages = [
-          ...step.context.store.messages,
-          {
-            role: "user" as const,
-            content: stringifyJSONToMessageContent(toolOutput.output),
-            options: {
-              toolResult: {
-                result: toolOutput.output,
-                isError: toolOutput.isError,
-                id: toolCall.id,
+        for (const call of toolCall) {
+          const targetTool = tools.find(
+            (tool) => tool.metadata.name === call.name,
+          );
+          const toolOutput = await callTool(
+            targetTool,
+            call,
+            step.context.logger,
+          );
+          step.context.store.toolOutputs.push(toolOutput);
+          step.context.store.messages = [
+            ...step.context.store.messages,
+            {
+              role: "user" as const,
+              content: stringifyJSONToMessageContent(toolOutput.output),
+              options: {
+                toolResult: {
+                  result: toolOutput.output,
+                  isError: toolOutput.isError,
+                  id: call.id,
+                },
               },
             },
-          },
-        ];
+          ];
+        }
       }
     } else {
+      debugger;
       const responseChunkStream = new ReadableStream<
         ChatResponseChunk<ToolCallLLMMessageOptions>
       >({
@@ -135,23 +138,25 @@ export class OpenAIAgent extends AgentRunner<OpenAI> {
         for await (const chunk of pipStream) {
           if (chunk.options && "toolCall" in chunk.options) {
             const toolCall = chunk.options.toolCall;
-            toolCalls.set(toolCall.id, toolCall);
+            toolCall.forEach((toolCall) => {
+              toolCalls.set(toolCall.id, toolCall);
+            });
           }
         }
+        step.context.store.messages = [
+          ...step.context.store.messages,
+          {
+            role: "assistant" as const,
+            content: "",
+            options: {
+              toolCall: [...toolCalls.values()],
+            },
+          },
+        ];
         for (const toolCall of toolCalls.values()) {
           const targetTool = tools.find(
             (tool) => tool.metadata.name === toolCall.name,
           );
-          step.context.store.messages = [
-            ...step.context.store.messages,
-            {
-              role: "assistant" as const,
-              content: "",
-              options: {
-                toolCall,
-              },
-            },
-          ];
           const toolOutput = await callTool(
             targetTool,
             toolCall,

packages/core/src/agent/utils.ts

@@ -16,10 +16,25 @@ export async function callTool(
   toolCall: ToolCall | PartialToolCall,
   logger: Logger,
 ): Promise<ToolOutput> {
-  const input: JSONObject =
-    typeof toolCall.input === "string"
-      ? JSON.parse(toolCall.input)
-      : toolCall.input;
+  let input: JSONObject;
+  if (typeof toolCall.input === "string") {
+    try {
+      input = JSON.parse(toolCall.input);
+    } catch (e) {
+      const output = `Tool ${toolCall.name} can't be called. Input is not a valid JSON object.`;
+      logger.error(
+        `${output} Try increasing the maxTokens parameter of your LLM. Invalid Input: ${toolCall.input}`,
+      );
+      return {
+        tool,
+        input: {},
+        output,
+        isError: true,
+      };
+    }
+  } else {
+    input = toolCall.input;
+  }
   if (!tool) {
     logger.error(`Tool ${toolCall.name} does not exist.`);
     const output = `Tool ${toolCall.name} does not exist.`;

packages/core/src/callbacks/CallbackManager.ts

@@ -12,6 +12,7 @@ import type {
   LLMStreamEvent,
   LLMToolCallEvent,
   LLMToolResultEvent,
+  MessageContent,
   RetrievalEndEvent,
   RetrievalStartEvent,
 } from "../llm/types.js";
@@ -99,7 +100,7 @@ export interface StreamCallbackResponse {
 }
 
 export interface RetrievalCallbackResponse {
-  query: string;
+  query: MessageContent;
   nodes: NodeWithScore[];
 }
 

packages/core/src/cloud/LlamaCloudRetriever.ts

@@ -2,8 +2,9 @@ import type { PlatformApi, PlatformApiClient } from "@llamaindex/cloud";
 import type { NodeWithScore } from "../Node.js";
 import { ObjectType, jsonToNode } from "../Node.js";
 import type { BaseRetriever, RetrieveParams } from "../Retriever.js";
-import { Settings } from "../Settings.js";
 import { wrapEventCaller } from "../internal/context/EventCaller.js";
+import { getCallbackManager } from "../internal/settings/CallbackManager.js";
+import { extractText } from "../llm/utils.js";
 import type { ClientParams, CloudConstructorParams } from "./types.js";
 import { DEFAULT_PROJECT_NAME } from "./types.js";
 import { getClient } from "./utils.js";
@@ -70,13 +71,13 @@ export class LlamaCloudRetriever implements BaseRetriever {
       await this.getClient()
     ).pipeline.runSearch(pipelines[0].id, {
       ...this.retrieveParams,
-      query,
+      query: extractText(query),
       searchFilters: preFilters as Record<string, unknown[]>,
     });
 
     const nodes = this.resultNodesToNodeWithScore(results.retrievalNodes);
 
-    Settings.callbackManager.dispatchEvent("retrieve", {
+    getCallbackManager().dispatchEvent("retrieve", {
       query,
       nodes,
     });

packages/core/src/embeddings/ClipEmbedding.ts

@@ -87,8 +87,4 @@ export class ClipEmbedding extends MultiModalEmbedding {
     const { text_embeds } = await (await this.getTextModel())(textInputs);
     return text_embeds.data;
   }
-
-  async getQueryEmbedding(query: string): Promise<number[]> {
-    return this.getTextEmbedding(query);
-  }
 }

packages/core/src/embeddings/DeepInfraEmbedding.ts

@@ -0,0 +1,152 @@
+import { getEnv } from "@llamaindex/env";
+import type { MessageContentDetail } from "../llm/index.js";
+import { extractSingleText } from "../llm/utils.js";
+import { BaseEmbedding } from "./types.js";
+
+const DEFAULT_MODEL = "sentence-transformers/clip-ViT-B-32";
+
+const API_TOKEN_ENV_VARIABLE_NAME = "DEEPINFRA_API_TOKEN";
+
+const API_ROOT = "https://api.deepinfra.com/v1/inference";
+
+const DEFAULT_TIMEOUT = 60 * 1000;
+
+const DEFAULT_MAX_RETRIES = 5;
+
+export interface DeepInfraEmbeddingResponse {
+  embeddings: number[][];
+  request_id: string;
+  inference_status: InferenceStatus;
+}
+
+export interface InferenceStatus {
+  status: string;
+  runtime_ms: number;
+  cost: number;
+  tokens_input: number;
+}
+
+const mapPrefixWithInputs = (prefix: string, inputs: string[]): string[] => {
+  return inputs.map((input) => (prefix ? `${prefix} ${input}` : input));
+};
+
+/**
+ * DeepInfraEmbedding is an alias for DeepInfra that implements the BaseEmbedding interface.
+ */
+export class DeepInfraEmbedding extends BaseEmbedding {
+  /**
+   * DeepInfra model to use
+   * @default "sentence-transformers/clip-ViT-B-32"
+   * @see https://deepinfra.com/models/embeddings
+   */
+  model: string;
+
+  /**
+   * DeepInfra API token
+   * @see https://deepinfra.com/dash/api_keys
+   * If not provided, it will try to get the token from the environment variable `DEEPINFRA_API_TOKEN`
+   *
+   */
+  apiToken: string;
+
+  /**
+   * Prefix to add to the query
+   * @default ""
+   */
+  queryPrefix: string;
+
+  /**
+   * Prefix to add to the text
+   * @default ""
+   */
+  textPrefix: string;
+
+  /**
+   *
+   * @default 5
+   */
+  maxRetries: number;
+
+  /**
+   *
+   * @default 60 * 1000
+   */
+  timeout: number;
+
+  constructor(init?: Partial<DeepInfraEmbedding>) {
+    super();
+
+    this.model = init?.model ?? DEFAULT_MODEL;
+    this.apiToken = init?.apiToken ?? getEnv(API_TOKEN_ENV_VARIABLE_NAME) ?? "";
+    this.queryPrefix = init?.queryPrefix ?? "";
+    this.textPrefix = init?.textPrefix ?? "";
+    this.maxRetries = init?.maxRetries ?? DEFAULT_MAX_RETRIES;
+    this.timeout = init?.timeout ?? DEFAULT_TIMEOUT;
+  }
+
+  async getTextEmbedding(text: string): Promise<number[]> {
+    const texts = mapPrefixWithInputs(this.textPrefix, [text]);
+    const embeddings = await this.getDeepInfraEmbedding(texts);
+    return embeddings[0];
+  }
+
+  async getQueryEmbedding(
+    query: MessageContentDetail,
+  ): Promise<number[] | null> {
+    const text = extractSingleText(query);
+    if (text) {
+      const queries = mapPrefixWithInputs(this.queryPrefix, [text]);
+      const embeddings = await this.getDeepInfraEmbedding(queries);
+      return embeddings[0];
+    } else {
+      return null;
+    }
+  }
+
+  async getTextEmbeddings(texts: string[]): Promise<number[][]> {
+    const textsWithPrefix = mapPrefixWithInputs(this.textPrefix, texts);
+    return await this.getDeepInfraEmbedding(textsWithPrefix);
+  }
+
+  async getQueryEmbeddings(queries: string[]): Promise<number[][]> {
+    const queriesWithPrefix = mapPrefixWithInputs(this.queryPrefix, queries);
+    return await this.getDeepInfraEmbedding(queriesWithPrefix);
+  }
+
+  private async getDeepInfraEmbedding(inputs: string[]): Promise<number[][]> {
+    const url = this.getUrl(this.model);
+
+    for (let attempt = 0; attempt < this.maxRetries; attempt++) {
+      const controller = new AbortController();
+      const id = setTimeout(() => controller.abort(), this.timeout);
+
+      try {
+        const response = await fetch(url, {
+          method: "POST",
+          headers: {
+            "Content-Type": "application/json",
+            Authorization: `Bearer ${this.apiToken}`,
+          },
+          body: JSON.stringify({ inputs }),
+          signal: controller.signal,
+        });
+        if (!response.ok) {
+          throw new Error(`Request failed with status ${response.status}`);
+        }
+
+        const responseJson: DeepInfraEmbeddingResponse = await response.json();
+        return responseJson.embeddings;
+      } catch (error) {
+        console.error(`Attempt ${attempt + 1} failed: ${error}`);
+      } finally {
+        clearTimeout(id);
+      }
+    }
+
+    throw new Error("Exceeded maximum retries");
+  }
+
+  private getUrl(model: string): string {
+    return `${API_ROOT}/${model}`;
+  }
+}

packages/core/src/embeddings/GeminiEmbedding.ts

@@ -36,8 +36,4 @@ export class GeminiEmbedding extends BaseEmbedding {
   getTextEmbedding(text: string): Promise<number[]> {
     return this.getEmbedding(text);
   }
-
-  getQueryEmbedding(query: string): Promise<number[]> {
-    return this.getTextEmbedding(query);
-  }
 }

packages/core/src/embeddings/HuggingFaceEmbedding.ts

@@ -45,8 +45,4 @@ export class HuggingFaceEmbedding extends BaseEmbedding {
     const output = await extractor(text, { pooling: "mean", normalize: true });
     return Array.from(output.data);
   }
-
-  async getQueryEmbedding(query: string): Promise<number[]> {
-    return this.getTextEmbedding(query);
-  }
 }

packages/core/src/embeddings/MistralAIEmbedding.ts

@@ -30,8 +30,4 @@ export class MistralAIEmbedding extends BaseEmbedding {
   async getTextEmbedding(text: string): Promise<number[]> {
     return this.getMistralAIEmbedding(text);
   }
-
-  async getQueryEmbedding(query: string): Promise<number[]> {
-    return this.getMistralAIEmbedding(query);
-  }
 }

packages/core/src/embeddings/MultiModalEmbedding.ts

@@ -6,6 +6,8 @@ import {
   type BaseNode,
   type ImageType,
 } from "../Node.js";
+import type { MessageContentDetail } from "../llm/types.js";
+import { extractImage, extractSingleText } from "../llm/utils.js";
 import { BaseEmbedding, batchEmbeddings } from "./types.js";
 
 /*
@@ -52,4 +54,18 @@ export abstract class MultiModalEmbedding extends BaseEmbedding {
 
     return nodes;
   }
+
+  async getQueryEmbedding(
+    query: MessageContentDetail,
+  ): Promise<number[] | null> {
+    const image = extractImage(query);
+    if (image) {
+      return await this.getImageEmbedding(image);
+    }
+    const text = extractSingleText(query);
+    if (text) {
+      return await this.getTextEmbedding(text);
+    }
+    return null;
+  }
 }

packages/core/src/embeddings/OpenAIEmbedding.ts

@@ -133,13 +133,4 @@ export class OpenAIEmbedding extends BaseEmbedding {
   async getTextEmbedding(text: string): Promise<number[]> {
     return (await this.getOpenAIEmbedding([text]))[0];
   }
-
-  /**
-   * Get embeddings for a query
-   * @param texts
-   * @param options
-   */
-  async getQueryEmbedding(query: string): Promise<number[]> {
-    return (await this.getOpenAIEmbedding([query]))[0];
-  }
 }

packages/core/src/embeddings/index.ts

@@ -1,3 +1,4 @@
+export { DeepInfraEmbedding } from "./DeepInfraEmbedding.js";
 export * from "./GeminiEmbedding.js";
 export * from "./JinaAIEmbedding.js";
 export * from "./MistralAIEmbedding.js";

packages/core/src/embeddings/types.ts

@@ -1,6 +1,8 @@
 import type { BaseNode } from "../Node.js";
 import { MetadataMode } from "../Node.js";
 import type { TransformComponent } from "../ingestion/types.js";
+import type { MessageContentDetail } from "../llm/types.js";
+import { extractSingleText } from "../llm/utils.js";
 import { SimilarityType, similarity } from "./utils.js";
 
 const DEFAULT_EMBED_BATCH_SIZE = 10;
@@ -19,7 +21,16 @@ export abstract class BaseEmbedding implements TransformComponent {
   }
 
   abstract getTextEmbedding(text: string): Promise<number[]>;
-  abstract getQueryEmbedding(query: string): Promise<number[]>;
+
+  async getQueryEmbedding(
+    query: MessageContentDetail,
+  ): Promise<number[] | null> {
+    const text = extractSingleText(query);
+    if (text) {
+      return await this.getTextEmbedding(text);
+    }
+    return null;
+  }
 
   /**
    * Optionally override this method to retrieve multiple embeddings in a single request

packages/core/src/engines/chat/ContextChatEngine.ts

@@ -3,10 +3,10 @@ import { getHistory } from "../../ChatHistory.js";
 import type { ContextSystemPrompt } from "../../Prompt.js";
 import { Response } from "../../Response.js";
 import type { BaseRetriever } from "../../Retriever.js";
+import { Settings } from "../../Settings.js";
 import { wrapEventCaller } from "../../internal/context/EventCaller.js";
 import type { ChatMessage, ChatResponseChunk, LLM } from "../../llm/index.js";
-import { OpenAI } from "../../llm/index.js";
-import type { MessageContent } from "../../llm/types.js";
+import type { MessageContent, MessageType } from "../../llm/types.js";
 import {
   extractText,
   streamConverter,
@@ -40,15 +40,16 @@ export class ContextChatEngine extends PromptMixin implements ChatEngine {
     contextSystemPrompt?: ContextSystemPrompt;
     nodePostprocessors?: BaseNodePostprocessor[];
     systemPrompt?: string;
+    contextRole?: MessageType;
   }) {
     super();
-    this.chatModel =
-      init.chatModel ?? new OpenAI({ model: "gpt-3.5-turbo-16k" });
+    this.chatModel = init.chatModel ?? Settings.llm;
     this.chatHistory = getHistory(init?.chatHistory);
     this.contextGenerator = new DefaultContextGenerator({
       retriever: init.retriever,
       contextSystemPrompt: init?.contextSystemPrompt,
       nodePostprocessors: init?.nodePostprocessors,
+      contextRole: init?.contextRole,
     });
     this.systemPrompt = init.systemPrompt;
   }

packages/core/src/engines/chat/DefaultContextGenerator.ts

@@ -1,9 +1,11 @@
-import type { NodeWithScore, TextNode } from "../../Node.js";
+import { type NodeWithScore } from "../../Node.js";
 import type { ContextSystemPrompt } from "../../Prompt.js";
 import { defaultContextSystemPrompt } from "../../Prompt.js";
 import type { BaseRetriever } from "../../Retriever.js";
+import type { MessageContent, MessageType } from "../../llm/types.js";
 import type { BaseNodePostprocessor } from "../../postprocessors/index.js";
 import { PromptMixin } from "../../prompts/index.js";
+import { createMessageContent } from "../../synthesizers/utils.js";
 import type { Context, ContextGenerator } from "./types.js";
 
 export class DefaultContextGenerator
@@ -13,11 +15,13 @@ export class DefaultContextGenerator
   retriever: BaseRetriever;
   contextSystemPrompt: ContextSystemPrompt;
   nodePostprocessors: BaseNodePostprocessor[];
+  contextRole: MessageType;
 
   constructor(init: {
     retriever: BaseRetriever;
     contextSystemPrompt?: ContextSystemPrompt;
     nodePostprocessors?: BaseNodePostprocessor[];
+    contextRole?: MessageType;
   }) {
     super();
 
@@ -25,6 +29,7 @@ export class DefaultContextGenerator
     this.contextSystemPrompt =
       init?.contextSystemPrompt ?? defaultContextSystemPrompt;
     this.nodePostprocessors = init.nodePostprocessors || [];
+    this.contextRole = init.contextRole ?? "system";
   }
 
   protected _getPrompts(): { contextSystemPrompt: ContextSystemPrompt } {
@@ -41,7 +46,10 @@ export class DefaultContextGenerator
     }
   }
 
-  private async applyNodePostprocessors(nodes: NodeWithScore[], query: string) {
+  private async applyNodePostprocessors(
+    nodes: NodeWithScore[],
+    query: MessageContent,
+  ) {
     let nodesWithScore = nodes;
 
     for (const postprocessor of this.nodePostprocessors) {
@@ -54,7 +62,7 @@ export class DefaultContextGenerator
     return nodesWithScore;
   }
 
-  async generate(message: string): Promise<Context> {
+  async generate(message: MessageContent): Promise<Context> {
     const sourceNodesWithScore = await this.retriever.retrieve({
       query: message,
     });
@@ -64,12 +72,15 @@ export class DefaultContextGenerator
       message,
     );
 
+    const content = await createMessageContent(
+      this.contextSystemPrompt,
+      nodes.map((r) => r.node),
+    );
+
     return {
       message: {
-        content: this.contextSystemPrompt({
-          context: nodes.map((r) => (r.node as TextNode).text).join("\n\n"),
-        }),
-        role: "system",
+        content,
+        role: this.contextRole,
       },
       nodes,
     };

packages/core/src/engines/chat/SimpleChatEngine.ts

@@ -1,9 +1,9 @@
 import type { ChatHistory } from "../../ChatHistory.js";
 import { getHistory } from "../../ChatHistory.js";
 import { Response } from "../../Response.js";
+import { Settings } from "../../Settings.js";
 import { wrapEventCaller } from "../../internal/context/EventCaller.js";
 import type { ChatResponseChunk, LLM } from "../../llm/index.js";
-import { OpenAI } from "../../llm/index.js";
 import {
   extractText,
   streamConverter,
@@ -25,7 +25,7 @@ export class SimpleChatEngine implements ChatEngine {
 
   constructor(init?: Partial<SimpleChatEngine>) {
     this.chatHistory = getHistory(init?.chatHistory);
-    this.llm = init?.llm ?? new OpenAI();
+    this.llm = init?.llm ?? Settings.llm;
   }
 
   chat(params: ChatEngineParamsStreaming): Promise<AsyncIterable<Response>>;

packages/core/src/indices/BaseIndex.ts

@@ -95,7 +95,7 @@ export abstract class BaseIndex<T> {
       [nodeParserFromSettingsOrContext(this.serviceContext)],
     );
     await this.insertNodes(nodes);
-    this.docStore.setDocumentHash(document.id_, document.hash);
+    await this.docStore.setDocumentHash(document.id_, document.hash);
   }
 
   abstract insertNodes(nodes: BaseNode[]): Promise<void>;

packages/core/src/indices/keyword/index.ts

@@ -29,6 +29,7 @@ import {
 
 import { llmFromSettingsOrContext } from "../../Settings.js";
 import type { LLM } from "../../llm/types.js";
+import { extractText } from "../../llm/utils.js";
 
 export interface KeywordIndexOptions {
   nodes?: BaseNode[];
@@ -85,7 +86,7 @@ abstract class BaseKeywordTableRetriever implements BaseRetriever {
   abstract getKeywords(query: string): Promise<string[]>;
 
   async retrieve({ query }: RetrieveParams): Promise<NodeWithScore[]> {
-    const keywords = await this.getKeywords(query);
+    const keywords = await this.getKeywords(extractText(query));
     const chunkIndicesCount: { [key: string]: number } = {};
     const filteredKeywords = keywords.filter((keyword) =>
       this.indexStruct.table.has(keyword),
@@ -280,7 +281,7 @@ export class KeywordTableIndex extends BaseIndex<KeywordTable> {
 
     await docStore.addDocuments(documents, true);
     for (const doc of documents) {
-      docStore.setDocumentHash(doc.id_, doc.hash);
+      await docStore.setDocumentHash(doc.id_, doc.hash);
     }
 
     const nodes = serviceContext.nodeParser.getNodesFromDocuments(documents);

packages/core/src/indices/summary/index.ts

@@ -11,6 +11,7 @@ import {
 } from "../../Settings.js";
 import { RetrieverQueryEngine } from "../../engines/query/index.js";
 import { wrapEventCaller } from "../../internal/context/EventCaller.js";
+import { extractText } from "../../llm/utils.js";
 import type { BaseNodePostprocessor } from "../../postprocessors/index.js";
 import type { StorageContext } from "../../storage/StorageContext.js";
 import { storageContextFromDefaults } from "../../storage/StorageContext.js";
@@ -137,7 +138,7 @@ export class SummaryIndex extends BaseIndex<IndexList> {
 
     await docStore.addDocuments(documents, true);
     for (const doc of documents) {
-      docStore.setDocumentHash(doc.id_, doc.hash);
+      await docStore.setDocumentHash(doc.id_, doc.hash);
     }
 
     const nodes =
@@ -343,7 +344,7 @@ export class SummaryIndexLLMRetriever implements BaseRetriever {
       const nodesBatch = await this.index.docStore.getNodes(nodeIdsBatch);
 
       const fmtBatchStr = this.formatNodeBatchFn(nodesBatch);
-      const input = { context: fmtBatchStr, query: query };
+      const input = { context: fmtBatchStr, query: extractText(query) };
 
       const llm = llmFromSettingsOrContext(this.serviceContext);
 

packages/core/src/indices/vectorStore/index.ts

@@ -23,6 +23,7 @@ import {
 } from "../../ingestion/strategies/index.js";
 import { wrapEventCaller } from "../../internal/context/EventCaller.js";
 import { getCallbackManager } from "../../internal/settings/CallbackManager.js";
+import type { MessageContent } from "../../llm/types.js";
 import type { BaseNodePostprocessor } from "../../postprocessors/types.js";
 import type { StorageContext } from "../../storage/StorageContext.js";
 import { storageContextFromDefaults } from "../../storage/StorageContext.js";
@@ -30,7 +31,6 @@ import type {
   MetadataFilters,
   VectorStore,
   VectorStoreByType,
-  VectorStoreQuery,
   VectorStoreQueryResult,
 } from "../../storage/index.js";
 import type { BaseIndexStore } from "../../storage/indexStore/types.js";
@@ -422,10 +422,9 @@ export class VectorIndexRetriever implements BaseRetriever {
     let nodesWithScores: NodeWithScore[] = [];
 
     for (const type in vectorStores) {
-      // TODO: add retrieval by using an image as query
       const vectorStore: VectorStore = vectorStores[type as ModalityType]!;
       nodesWithScores = nodesWithScores.concat(
-        await this.textRetrieve(
+        await this.retrieveQuery(
           query,
           type as ModalityType,
           vectorStore,
@@ -447,36 +446,33 @@ export class VectorIndexRetriever implements BaseRetriever {
     return nodesWithScores;
   }
 
-  protected async textRetrieve(
-    query: string,
+  protected async retrieveQuery(
+    query: MessageContent,
     type: ModalityType,
     vectorStore: VectorStore,
     preFilters?: MetadataFilters,
   ): Promise<NodeWithScore[]> {
-    const q = await this.buildVectorStoreQuery(
-      this.index.embedModel ?? vectorStore.embedModel,
-      query,
-      this.topK[type],
-      preFilters,
-    );
-    const result = await vectorStore.query(q);
-    return this.buildNodeListFromQueryResult(result);
-  }
-
-  protected async buildVectorStoreQuery(
-    embedModel: BaseEmbedding,
-    query: string,
-    similarityTopK: number,
-    preFilters?: MetadataFilters,
-  ): Promise<VectorStoreQuery> {
-    const queryEmbedding = await embedModel.getQueryEmbedding(query);
-
-    return {
-      queryEmbedding,
-      mode: VectorStoreQueryMode.DEFAULT,
-      similarityTopK,
-      filters: preFilters ?? undefined,
-    };
+    // convert string message to multi-modal format
+    if (typeof query === "string") {
+      query = [{ type: "text", text: query }];
+    }
+    // overwrite embed model if specified, otherwise use the one from the vector store
+    const embedModel = this.index.embedModel ?? vectorStore.embedModel;
+    let nodes: NodeWithScore[] = [];
+    // query each content item (e.g. text or image) separately
+    for (const item of query) {
+      const queryEmbedding = await embedModel.getQueryEmbedding(item);
+      if (queryEmbedding) {
+        const result = await vectorStore.query({
+          queryEmbedding,
+          mode: VectorStoreQueryMode.DEFAULT,
+          similarityTopK: this.topK[type],
+          filters: preFilters ?? undefined,
+        });
+        nodes = nodes.concat(this.buildNodeListFromQueryResult(result));
+      }
+    }
+    return nodes;
   }
 
   protected buildNodeListFromQueryResult(result: VectorStoreQueryResult) {

packages/core/src/ingestion/strategies/DuplicatesStrategy.ts

@@ -19,7 +19,7 @@ export class DuplicatesStrategy implements TransformComponent {
 
     for (const node of nodes) {
       if (!(node.hash in hashes) && !currentHashes.has(node.hash)) {
-        this.docStore.setDocumentHash(node.id_, node.hash);
+        await this.docStore.setDocumentHash(node.id_, node.hash);
         nodesToRun.push(node);
         currentHashes.add(node.hash);
       }

packages/core/src/internal/decorator/node.ts

@@ -4,12 +4,19 @@ import { getChunkSize } from "../settings/chunk-size.js";
 
 const emitOnce = false;
 
-export function chunkSizeCheck(
-  contentGetter: () => string,
-  _context: ClassMethodDecoratorContext | ClassGetterDecoratorContext,
+export function chunkSizeCheck<
+  This extends BaseNode,
+  Args extends any[],
+  Return,
+>(
+  contentGetter: (this: This, ...args: Args) => string,
+  _context: ClassMethodDecoratorContext<
+    This,
+    (this: This, ...args: Args) => Return
+  >,
 ) {
-  return function <Node extends BaseNode>(this: Node) {
-    const content = contentGetter.call(this);
+  return function (this: This, ...args: Args) {
+    const content = contentGetter.call(this, ...args);
     const chunkSize = getChunkSize();
     const enableChunkSizeCheck = getEnv("ENABLE_CHUNK_SIZE_CHECK") === "true";
     if (

packages/core/src/llm/anthropic.ts

@@ -194,12 +194,15 @@ export class Anthropic extends ToolCallLLM<AnthropicAdditionalChatOptions> {
                     } satisfies TextBlockParam,
                   ]
                 : []),
-              {
-                type: "tool_use",
-                id: options.toolCall.id,
-                name: options.toolCall.name,
-                input: options.toolCall.input,
-              } satisfies ToolUseBlockParam,
+              ...options.toolCall.map(
+                (toolCall) =>
+                  ({
+                    type: "tool_use",
+                    id: toolCall.id,
+                    name: toolCall.name,
+                    input: toolCall.input,
+                  }) satisfies ToolUseBlockParam,
+              ),
             ] satisfies ToolsBetaContentBlock[],
           } satisfies ToolsBetaMessageParam;
         }
@@ -326,7 +329,7 @@ export class Anthropic extends ToolCallLLM<AnthropicAdditionalChatOptions> {
       }
       const response = await anthropic.beta.tools.messages.create(params);
 
-      const toolUseBlock = response.content.find(
+      const toolUseBlock = response.content.filter(
         (content): content is ToolUseBlock => content.type === "tool_use",
       );
 
@@ -340,15 +343,16 @@ export class Anthropic extends ToolCallLLM<AnthropicAdditionalChatOptions> {
               text: content.text,
             })),
           role: "assistant",
-          options: toolUseBlock
-            ? {
-                toolCall: {
-                  id: toolUseBlock.id,
-                  name: toolUseBlock.name,
-                  input: toolUseBlock.input,
-                },
-              }
-            : {},
+          options:
+            toolUseBlock.length > 0
+              ? {
+                  toolCall: toolUseBlock.map((block) => ({
+                    id: block.id,
+                    name: block.name,
+                    input: block.input,
+                  })),
+                }
+              : {},
         },
       };
     } else {

packages/core/src/llm/azure.ts

@@ -17,12 +17,17 @@ const ALL_AZURE_OPENAI_CHAT_MODELS = {
     contextWindow: 16384,
     openAIModel: "gpt-3.5-turbo-16k",
   },
+  "gpt-4o": { contextWindow: 128000, openAIModel: "gpt-4o" },
   "gpt-4": { contextWindow: 8192, openAIModel: "gpt-4" },
   "gpt-4-32k": { contextWindow: 32768, openAIModel: "gpt-4-32k" },
   "gpt-4-turbo": {
     contextWindow: 128000,
     openAIModel: "gpt-4-turbo",
   },
+  "gpt-4-turbo-2024-04-09": {
+    contextWindow: 128000,
+    openAIModel: "gpt-4-turbo",
+  },
   "gpt-4-vision-preview": {
     contextWindow: 128000,
     openAIModel: "gpt-4-vision-preview",
@@ -31,6 +36,10 @@ const ALL_AZURE_OPENAI_CHAT_MODELS = {
     contextWindow: 128000,
     openAIModel: "gpt-4-1106-preview",
   },
+  "gpt-4o-2024-05-13": {
+    contextWindow: 128000,
+    openAIModel: "gpt-4o-2024-05-13",
+  },
 };
 
 const ALL_AZURE_OPENAI_EMBEDDING_MODELS = {
@@ -62,6 +71,8 @@ const ALL_AZURE_API_VERSIONS = [
   "2024-02-01",
   "2024-02-15-preview",
   "2024-03-01-preview",
+  "2024-04-01-preview",
+  "2024-05-01-preview",
 ];
 
 const DEFAULT_API_VERSION = "2023-05-15";

packages/core/src/llm/deepinfra.ts

@@ -0,0 +1,33 @@
+import { getEnv } from "@llamaindex/env";
+import { OpenAI } from "./openai.js";
+
+const ENV_VARIABLE_NAME = "DEEPINFRA_API_TOKEN";
+const DEFAULT_MODEL = "mistralai/Mixtral-8x22B-Instruct-v0.1";
+const BASE_URL = "https://api.deepinfra.com/v1/openai";
+
+export class DeepInfra extends OpenAI {
+  constructor(init?: Partial<OpenAI>) {
+    const {
+      apiKey = getEnv(ENV_VARIABLE_NAME),
+      additionalSessionOptions = {},
+      model = DEFAULT_MODEL,
+      ...rest
+    } = init ?? {};
+
+    if (!apiKey) {
+      throw new Error(
+        `Set DeepInfra API key in ${ENV_VARIABLE_NAME} env variable`,
+      );
+    }
+
+    additionalSessionOptions.baseURL =
+      additionalSessionOptions.baseURL ?? BASE_URL;
+
+    super({
+      apiKey,
+      additionalSessionOptions,
+      model,
+      ...rest,
+    });
+  }
+}

packages/core/src/llm/gemini/base.ts

@@ -35,11 +35,20 @@ export const GEMINI_MODEL_INFO_MAP: Record<GEMINI_MODEL, GeminiModelInfo> = {
   [GEMINI_MODEL.GEMINI_PRO]: { contextWindow: 30720 },
   [GEMINI_MODEL.GEMINI_PRO_VISION]: { contextWindow: 12288 },
   [GEMINI_MODEL.GEMINI_PRO_LATEST]: { contextWindow: 10 ** 6 },
+  // multi-modal/multi turn
+  [GEMINI_MODEL.GEMINI_PRO_1_5_PRO_PREVIEW]: { contextWindow: 10 ** 6 },
+  [GEMINI_MODEL.GEMINI_PRO_1_5_FLASH_PREVIEW]: { contextWindow: 10 ** 6 },
+  [GEMINI_MODEL.GEMINI_PRO_1_5]: { contextWindow: 2 * 10 ** 6 },
+  [GEMINI_MODEL.GEMINI_PRO_1_5_FLASH]: { contextWindow: 10 ** 6 },
 };
 
 const SUPPORT_TOOL_CALL_MODELS: GEMINI_MODEL[] = [
   GEMINI_MODEL.GEMINI_PRO,
   GEMINI_MODEL.GEMINI_PRO_VISION,
+  GEMINI_MODEL.GEMINI_PRO_1_5_PRO_PREVIEW,
+  GEMINI_MODEL.GEMINI_PRO_1_5_FLASH_PREVIEW,
+  GEMINI_MODEL.GEMINI_PRO_1_5,
+  GEMINI_MODEL.GEMINI_PRO_1_5_FLASH,
 ];
 
 const DEFAULT_GEMINI_PARAMS = {

packages/core/src/llm/gemini/types.ts

@@ -51,6 +51,10 @@ export enum GEMINI_MODEL {
   GEMINI_PRO = "gemini-pro",
   GEMINI_PRO_VISION = "gemini-pro-vision",
   GEMINI_PRO_LATEST = "gemini-1.5-pro-latest",
+  GEMINI_PRO_1_5_PRO_PREVIEW = "gemini-1.5-pro-preview-0514",
+  GEMINI_PRO_1_5_FLASH_PREVIEW = "gemini-1.5-flash-preview-0514",
+  GEMINI_PRO_1_5 = "gemini-1.5-pro-001",
+  GEMINI_PRO_1_5_FLASH = "gemini-1.5-flash-001",
 }
 
 export interface GeminiModelInfo {

packages/core/src/llm/gemini/utils.ts

@@ -99,7 +99,13 @@ export const cleanParts = (
 ): GeminiMessageContent => {
   return {
     ...message,
-    parts: message.parts.filter((part) => part.text?.trim()),
+    parts: message.parts.filter(
+      (part) =>
+        part.text?.trim() ||
+        part.inlineData ||
+        part.fileData ||
+        part.functionCall,
+    ),
   };
 };
 
@@ -147,24 +153,28 @@ export class GeminiHelper {
   public static mergeNeighboringSameRoleMessages(
     messages: GeminiMessageContent[],
   ): GeminiMessageContent[] {
-    return messages.reduce(
-      (
-        result: GeminiMessageContent[],
-        current: GeminiMessageContent,
-        index: number,
-      ) => {
-        if (index > 0 && messages[index - 1].role === current.role) {
-          result[result.length - 1].parts = [
-            ...result[result.length - 1].parts,
-            ...current.parts,
-          ];
-        } else {
-          result.push(current);
-        }
-        return result;
-      },
-      [],
-    );
+    return messages
+      .map(cleanParts)
+      .filter((message) => message.parts.length)
+      .reduce(
+        (
+          result: GeminiMessageContent[],
+          current: GeminiMessageContent,
+          index: number,
+          original: GeminiMessageContent[],
+        ) => {
+          if (index > 0 && original[index - 1].role === current.role) {
+            result[result.length - 1].parts = [
+              ...result[result.length - 1].parts,
+              ...current.parts,
+            ];
+          } else {
+            result.push(current);
+          }
+          return result;
+        },
+        [],
+      );
   }
 
   public static messageContentToGeminiParts(content: MessageContent): Part[] {

packages/core/src/llm/huggingface.ts

@@ -1,7 +1,4 @@
-import {
-  HfInference,
-  type Options as HfInferenceOptions,
-} from "@huggingface/inference";
+import { HfInference } from "@huggingface/inference";
 import type {
   PreTrainedModel,
   PreTrainedTokenizer,
@@ -20,12 +17,49 @@ import type {
 } from "./types.js";
 import { streamConverter, wrapLLMEvent } from "./utils.js";
 
+// TODO workaround issue with @huggingface/inference@2.7.0
+interface HfInferenceOptions {
+  /**
+   * (Default: true) Boolean. If a request 503s and wait_for_model is set to false, the request will be retried with the same parameters but with wait_for_model set to true.
+   */
+  retry_on_error?: boolean;
+  /**
+   * (Default: true). Boolean. There is a cache layer on Inference API (serverless) to speedup requests we have already seen. Most models can use those results as is as models are deterministic (meaning the results will be the same anyway). However if you use a non deterministic model, you can set this parameter to prevent the caching mechanism from being used resulting in a real new query.
+   */
+  use_cache?: boolean;
+  /**
+   * (Default: false). Boolean. Do not load the model if it's not already available.
+   */
+  dont_load_model?: boolean;
+  /**
+   * (Default: false). Boolean to use GPU instead of CPU for inference (requires Startup plan at least).
+   */
+  use_gpu?: boolean;
+  /**
+   * (Default: false) Boolean. If the model is not ready, wait for it instead of receiving 503. It limits the number of requests required to get your inference done. It is advised to only set this flag to true after receiving a 503 error as it will limit hanging in your application to known places.
+   */
+  wait_for_model?: boolean;
+  /**
+   * Custom fetch function to use instead of the default one, for example to use a proxy or edit headers.
+   */
+  fetch?: typeof fetch;
+  /**
+   * Abort Controller signal to use for request interruption.
+   */
+  signal?: AbortSignal;
+  /**
+   * (Default: "same-origin"). String | Boolean. Credentials to use for the request. If this is a string, it will be passed straight on. If it's a boolean, true will be "include" and false will not send credentials at all.
+   */
+  includeCredentials?: string | boolean;
+}
+
 const DEFAULT_PARAMS = {
   temperature: 0.1,
   topP: 1,
   maxTokens: undefined,
   contextWindow: 3900,
 };
+
 export type HFConfig = Partial<typeof DEFAULT_PARAMS> &
   HfInferenceOptions & {
     model: string;
@@ -139,7 +173,7 @@ export class HuggingFaceInferenceAPI extends BaseLLM {
       inputs: this.messagesToPrompt(params.messages),
       parameters: this.metadata,
     });
-    yield* streamConverter(stream, (chunk) => ({
+    yield* streamConverter(stream, (chunk: any) => ({
       delta: chunk.token.text,
       raw: chunk,
     }));

packages/core/src/llm/index.ts

@@ -4,8 +4,10 @@ export {
   ALL_AVAILABLE_V3_MODELS,
   Anthropic,
 } from "./anthropic.js";
+export { ToolCallLLM } from "./base.js";
 export { FireworksLLM } from "./fireworks.js";
 export { Gemini, GeminiSession } from "./gemini/base.js";
+export { streamConverter, wrapLLMEvent } from "./utils.js";
 
 export {
   GEMINI_MODEL,
@@ -23,6 +25,7 @@ export * from "./openai.js";
 export { Portkey } from "./portkey.js";
 export * from "./replicate_ai.js";
 // Note: The type aliases for replicate are to simplify usage for Llama 2 (we're using replicate for Llama 2 support)
+export { DeepInfra } from "./deepinfra.js";
 export { Ollama, type OllamaParams } from "./ollama.js";
 export {
   ALL_AVAILABLE_REPLICATE_MODELS,
@@ -32,5 +35,6 @@ export {
   ReplicateLLM,
   ReplicateSession,
 } from "./replicate_ai.js";
+
 export { TogetherLLM } from "./together.js";
 export * from "./types.js";

packages/core/src/llm/ollama.ts

@@ -191,10 +191,6 @@ export class Ollama
     return this.getEmbedding(text);
   }
 
-  async getQueryEmbedding(query: string): Promise<number[]> {
-    return this.getEmbedding(query);
-  }
-
   // Inherited from OllamaBase
 
   push(

packages/core/src/llm/openai.ts

@@ -110,7 +110,6 @@ export const GPT4_MODELS = {
   "gpt-4-1106-preview": { contextWindow: 128000 },
   "gpt-4-0125-preview": { contextWindow: 128000 },
   "gpt-4-vision-preview": { contextWindow: 128000 },
-  // fixme: wait for openai documentation
   "gpt-4o": { contextWindow: 128000 },
   "gpt-4o-2024-05-13": { contextWindow: 128000 },
 };
@@ -185,7 +184,7 @@ export class OpenAI extends ToolCallLLM<OpenAIAdditionalChatOptions> {
     },
   ) {
     super();
-    this.model = init?.model ?? "gpt-3.5-turbo";
+    this.model = init?.model ?? "gpt-4o";
     this.temperature = init?.temperature ?? 0.1;
     this.topP = init?.topP ?? 1;
     this.maxTokens = init?.maxTokens ?? undefined;
@@ -279,19 +278,19 @@ export class OpenAI extends ToolCallLLM<OpenAIAdditionalChatOptions> {
         return {
           role: "assistant",
           content: extractText(message.content),
-          tool_calls: [
-            {
-              id: options.toolCall.id,
+          tool_calls: options.toolCall.map((toolCall) => {
+            return {
+              id: toolCall.id,
               type: "function",
               function: {
-                name: options.toolCall.name,
+                name: toolCall.name,
                 arguments:
-                  typeof options.toolCall.input === "string"
-                    ? options.toolCall.input
-                    : JSON.stringify(options.toolCall.input),
+                  typeof toolCall.input === "string"
+                    ? toolCall.input
+                    : JSON.stringify(toolCall.input),
               },
-            },
-          ],
+            };
+          }),
         } satisfies ChatCompletionAssistantMessageParam;
       } else if (message.role === "user") {
         return {
@@ -380,12 +379,13 @@ export class OpenAI extends ToolCallLLM<OpenAIAdditionalChatOptions> {
         role: response.choices[0].message.role,
         options: response.choices[0].message?.tool_calls
           ? {
-              toolCall: {
-                id: response.choices[0].message.tool_calls[0].id,
-                name: response.choices[0].message.tool_calls[0].function.name,
-                input:
-                  response.choices[0].message.tool_calls[0].function.arguments,
-              },
+              toolCall: response.choices[0].message.tool_calls.map(
+                (toolCall) => ({
+                  id: toolCall.id,
+                  name: toolCall.function.name,
+                  input: toolCall.function.arguments,
+                }),
+              ),
             }
           : {},
       },
@@ -459,10 +459,10 @@ export class OpenAI extends ToolCallLLM<OpenAIAdditionalChatOptions> {
       yield {
         raw: part,
         options: shouldEmitToolCall
-          ? { toolCall: shouldEmitToolCall }
+          ? { toolCall: [shouldEmitToolCall] }
           : currentToolCall
             ? {
-                toolCall: currentToolCall,
+                toolCall: [currentToolCall],
               }
             : {},
         delta: choice.delta.content ?? "",