Release 0.3.16 (#896)

Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>

.github/workflows/test.yml

@@ -85,6 +85,7 @@ jobs:
           if-no-files-found: error
   e2e-core-examples:
     strategy:
+      fail-fast: false
       matrix:
         packages:
           - cloudflare-worker-agent

apps/docs/CHANGELOG.md

@@ -1,5 +1,24 @@
 # docs
 
+## 0.0.24
+
+### Patch Changes
+
+- 631f000: feat: DeepInfra LLM implementation
+- 8832669: Community bedrock support added
+- a29d835: setDocumentHash should be async
+- Updated dependencies [11ae926]
+- Updated dependencies [631f000]
+- Updated dependencies [1378ec4]
+- Updated dependencies [6b1ded4]
+- Updated dependencies [4d4bd85]
+- Updated dependencies [24a9d1e]
+- Updated dependencies [45952de]
+- Updated dependencies [54230f0]
+- Updated dependencies [a29d835]
+- Updated dependencies [73819bf]
+  - llamaindex@0.3.16
+
 ## 0.0.23
 
 ### Patch Changes

apps/docs/docs/modules/data_loader.mdx

@@ -6,6 +6,7 @@ import CodeBlock from "@theme/CodeBlock";
 import CodeSource from "!raw-loader!../../../../examples/readers/src/simple-directory-reader";
 import CodeSource2 from "!raw-loader!../../../../examples/readers/src/custom-simple-directory-reader";
 import CodeSource3 from "!raw-loader!../../../../examples/readers/src/llamaparse";
+import CodeSource4 from "!raw-loader!../../../../examples/readers/src/simple-directory-reader-with-llamaparse.ts";
 
 # Loader
 
@@ -21,11 +22,13 @@ It is a simple reader that reads all files from a directory and its subdirectori
 
 <CodeBlock language="ts">{CodeSource}</CodeBlock>
 
-Currently, it supports reading `.csv`, `.docx`, `.html`, `.md` and `.pdf` files,
-but support for other file types is planned.
+Currently, it supports reading `.txt`, `.pdf`, `.csv`, `.md`, `.docx`, `.htm`, `.html`, `.jpg`, `.jpeg`, `.png` and `.gif` files, but support for other file types is planned.
 
-Also, you can provide a `defaultReader` as a fallback for files with unsupported extensions.
-Or pass new readers for `fileExtToReader` to support more file types.
+You can override the default reader for all file types, inlcuding unsupported ones, with the `overrideReader` option.
+Additionally, you can override the default reader for specific file types or add support for additional file types with the `fileExtToReader` option.
+Also, you can provide a `defaultReader` as a fallback for files with unsupported extensions. By default it is `TextFileReader`.
+
+SimpleDirectoryReader supports up to 9 concurrent requests. Use the `numWorkers` option to set the number of concurrent requests. By default it runs in sequential mode, i.e. set to 1.
 
 <CodeBlock language="ts" showLineNumbers metastring="{8-12,17-21}">
   {CodeSource2}
@@ -35,14 +38,31 @@ Or pass new readers for `fileExtToReader` to support more file types.
 
 LlamaParse is an API created by LlamaIndex to efficiently parse files, e.g. it's great at converting PDF tables into markdown.
 
-To use it, first login and get an API key from https://cloud.llamaindex.ai. Make sure to store the key in the environment variable `LLAMA_CLOUD_API_KEY`.
+To use it, first login and get an API key from https://cloud.llamaindex.ai. Make sure to store the key as `apiKey` parameter or in the environment variable `LLAMA_CLOUD_API_KEY`.
 
-Then, you can use the `LlamaParseReader` class to read a local PDF file and convert it into a markdown document that can be used by LlamaIndex:
+Then, you can use the `LlamaParseReader` class to local files and convert them into a parsed document that can be used by LlamaIndex.
+See [LlamaParseReader.ts](https://github.com/run-llama/LlamaIndexTS/blob/main/packages/core/src/readers/LlamaParseReader.ts#L6) for a list of supported file types:
 
 <CodeBlock language="ts">{CodeSource3}</CodeBlock>
 
-Alternatively, you can set the [`resultType`](../api/classes/LlamaParseReader.md#resulttype) option to `text` to get the parsed document as a text string.
+Additional options can be set with the `LlamaParseReader` constructor:
+
+- `resultType` can be set to `markdown`, `text` or `.json`. Defaults to `text`
+- `language` primarly helps with OCR recognition. Defaults to `en`. See [../readers/type.ts](https://github.com/run-llama/LlamaIndexTS/blob/main/packages/core/src/readers/type.ts#L20) for a list of supported languages.
+- `parsingInstructions` can help with complicated document structures. See this [LlamaIndex Blog Post](https://www.llamaindex.ai/blog/launching-the-first-genai-native-document-parsing-platform) for an example.
+- `skipDiagonalText` set to true to ignore diagonal text.
+- `invalidateCache` set to true to ignore the LlamaCloud cache. All document are kept in cache for 48hours after the job was completed to avoid processing the same document twice. Can be useful for testing when trying to re-parse the same document with, e.g. different `parsingInstructions`.
+- `gpt4oMode` set to true to use GPT-4o to extract content.
+- `gpt4oApiKey` set the GPT-4o API key. Optional. Lowers the cost of parsing by using your own API key. Your OpenAI account will be charged. Can also be set in the environment variable `LLAMA_CLOUD_GPT4O_API_KEY`.
+- `numWorkers` as in the python version, is set in `SimpleDirectoryReader`. Default is 1.
+
+## LlamaParse with SimpleDirectoryReader
+
+Below a full example of `LlamaParse` integrated in `SimpleDirectoryReader` with additional options.
+
+<CodeBlock language="ts">{CodeSource4}</CodeBlock>
 
 ## API Reference
 
 - [SimpleDirectoryReader](../api/classes/SimpleDirectoryReader.md)
+- [LlamaParseReader](../api/classes/LlamaParseReader.md)

apps/docs/docs/modules/embeddings/available_embeddings/deepinfra.md

@@ -0,0 +1,79 @@
+# DeepInfra
+
+To use DeepInfra embeddings, you need to import `DeepInfraEmbedding` from llamaindex.
+Check out available embedding models [here](https://deepinfra.com/models/embeddings).
+
+```ts
+import {
+  DeepInfraEmbedding,
+  Settings,
+  Document,
+  VectorStoreIndex,
+} from "llamaindex";
+
+// Update Embed Model
+Settings.embedModel = new DeepInfraEmbedding();
+
+const document = new Document({ text: essay, id_: "essay" });
+
+const index = await VectorStoreIndex.fromDocuments([document]);
+
+const queryEngine = index.asQueryEngine();
+
+const query = "What is the meaning of life?";
+
+const results = await queryEngine.query({
+  query,
+});
+```
+
+By default, DeepInfraEmbedding is using the sentence-transformers/clip-ViT-B-32 model. You can change the model by passing the model parameter to the constructor.
+For example:
+
+```ts
+import { DeepInfraEmbedding } from "llamaindex";
+
+const model = "intfloat/e5-large-v2";
+Settings.embedModel = new DeepInfraEmbedding({
+  model,
+});
+```
+
+You can also set the `maxRetries` and `timeout` parameters when initializing `DeepInfraEmbedding` for better control over the request behavior.
+
+For example:
+
+```ts
+import { DeepInfraEmbedding, Settings } from "llamaindex";
+
+const model = "intfloat/e5-large-v2";
+const maxRetries = 5;
+const timeout = 5000; // 5 seconds
+
+Settings.embedModel = new DeepInfraEmbedding({
+  model,
+  maxRetries,
+  timeout,
+});
+```
+
+Standalone usage:
+
+```ts
+import { DeepInfraEmbedding } from "llamaindex";
+import { config } from "dotenv";
+// For standalone usage, you need to configure DEEPINFRA_API_TOKEN in .env file
+config();
+
+const main = async () => {
+  const model = "intfloat/e5-large-v2";
+  const embeddings = new DeepInfraEmbedding({ model });
+  const text = "What is the meaning of life?";
+  const response = await embeddings.embed([text]);
+  console.log(response);
+};
+
+main();
+```
+
+For questions or feedback, please contact us at [feedback@deepinfra.com](mailto:feedback@deepinfra.com)

apps/docs/docs/modules/llms/available_llms/bedrock.md

@@ -0,0 +1,61 @@
+# Bedrock
+
+## Usage
+
+```ts
+import { BEDROCK_MODELS, Bedrock } from "@llamaindex/community";
+
+Settings.llm = new Bedrock({
+  model: BEDROCK_MODELS.ANTHROPIC_CLAUDE_3_HAIKU,
+  region: "us-east-1", // can be provided via env AWS_REGION
+  credentials: {
+    accessKeyId: "...", // optional and can be provided via env AWS_ACCESS_KEY_ID
+    secretAccessKey: "...", // optional and can be provided via env AWS_SECRET_ACCESS_KEY
+  },
+});
+```
+
+Currently only supports Anthropic models:
+
+```ts
+ANTHROPIC_CLAUDE_INSTANT_1 = "anthropic.claude-instant-v1";
+ANTHROPIC_CLAUDE_2 = "anthropic.claude-v2";
+ANTHROPIC_CLAUDE_2_1 = "anthropic.claude-v2:1";
+ANTHROPIC_CLAUDE_3_SONNET = "anthropic.claude-3-sonnet-20240229-v1:0";
+ANTHROPIC_CLAUDE_3_HAIKU = "anthropic.claude-3-haiku-20240307-v1:0";
+ANTHROPIC_CLAUDE_3_OPUS = "anthropic.claude-3-opus-20240229-v1:0"; // Not currently on Bedrock
+```
+
+Sonnet, Haiku and Opus are multimodal, image_url only supports base64 data url format, e.g. `data:image/jpeg;base64,SGVsbG8sIFdvcmxkIQ==`
+
+## Full Example
+
+```ts
+import { BEDROCK_MODELS, Bedrock } from "llamaindex";
+
+Settings.llm = new Bedrock({
+  model: BEDROCK_MODELS.ANTHROPIC_CLAUDE_3_HAIKU,
+});
+
+async function main() {
+  const document = new Document({ text: essay, id_: "essay" });
+
+  // Load and index documents
+  const index = await VectorStoreIndex.fromDocuments([document]);
+
+  // Create a query engine
+  const queryEngine = index.asQueryEngine({
+    retriever,
+  });
+
+  const query = "What is the meaning of life?";
+
+  // Query
+  const response = await queryEngine.query({
+    query,
+  });
+
+  // Log the response
+  console.log(response.response);
+}
+```

apps/docs/docs/modules/llms/available_llms/deepinfra.md

@@ -0,0 +1,83 @@
+# DeepInfra
+
+Check out available LLMs [here](https://deepinfra.com/models/text-generation).
+
+```ts
+import { DeepInfra, Settings } from "llamaindex";
+
+// Get the API key from `DEEPINFRA_API_TOKEN` environment variable
+import { config } from "dotenv";
+config();
+Settings.llm = new DeepInfra();
+
+// Set the API key
+apiKey = "YOUR_API_KEY";
+Settings.llm = new DeepInfra({ apiKey });
+```
+
+You can setup the apiKey on the environment variables, like:
+
+```bash
+export DEEPINFRA_API_TOKEN="<YOUR_API_KEY>"
+```
+
+## Load and index documents
+
+For this example, we will use a single document. In a real-world scenario, you would have multiple documents to index.
+
+```ts
+const document = new Document({ text: essay, id_: "essay" });
+
+const index = await VectorStoreIndex.fromDocuments([document]);
+```
+
+## Query
+
+```ts
+const queryEngine = index.asQueryEngine();
+
+const query = "What is the meaning of life?";
+
+const results = await queryEngine.query({
+  query,
+});
+```
+
+## Full Example
+
+```ts
+import { DeepInfra, Document, VectorStoreIndex, Settings } from "llamaindex";
+
+// Use custom LLM
+const model = "meta-llama/Meta-Llama-3-8B-Instruct";
+Settings.llm = new DeepInfra({ model, temperature: 0 });
+
+async function main() {
+  const document = new Document({ text: essay, id_: "essay" });
+
+  // Load and index documents
+  const index = await VectorStoreIndex.fromDocuments([document]);
+
+  // get retriever
+  const retriever = index.asRetriever();
+
+  // Create a query engine
+  const queryEngine = index.asQueryEngine({
+    retriever,
+  });
+
+  const query = "What is the meaning of life?";
+
+  // Query
+  const response = await queryEngine.query({
+    query,
+  });
+
+  // Log the response
+  console.log(response.response);
+}
+```
+
+## Feedback
+
+If you have any feedback, please reach out to us at [feedback@deepinfra.com](mailto:feedback@deepinfra.com)

apps/docs/i18n/fr/docusaurus-plugin-content-docs/current/api/classes/BaseDocumentStore.md

@@ -271,7 +271,7 @@ custom_edit_url: null
 
 ### setDocumentHash
 
-▸ `Abstract` **setDocumentHash**(`docId`, `docHash`): `void`
+▸ `Abstract` **setDocumentHash**(`docId`, `docHash`): `Promise`<`void`\>
 
 #### Parameters
 

apps/docs/i18n/hr/docusaurus-plugin-content-docs/current/api/classes/BaseDocumentStore.md

@@ -271,7 +271,7 @@ custom_edit_url: null
 
 ### setDocumentHash
 
-▸ `Abstract` **setDocumentHash**(`docId`, `docHash`): `void`
+▸ `Abstract` **setDocumentHash**(`docId`, `docHash`): `Promise`<`void`\>
 
 #### Parameters
 

apps/docs/i18n/zh-Hans/docusaurus-plugin-content-docs/current/api/classes/BaseDocumentStore.md

@@ -271,7 +271,7 @@ custom_edit_url: null
 
 ### setDocumentHash
 
-▸ `Abstract` **setDocumentHash**(`docId`, `docHash`): `void`
+▸ `Abstract` **setDocumentHash**(`docId`, `docHash`): `Promise`<`void`\>
 
 #### Parameters
 

apps/docs/package.json

@@ -1,6 +1,6 @@
 {
   "name": "docs",
-  "version": "0.0.23",
+  "version": "0.0.24",
   "private": true,
   "scripts": {
     "docusaurus": "docusaurus",

examples/agent/large_toolcall.ts

@@ -0,0 +1,64 @@
+import { FunctionTool, OpenAI, OpenAIAgent } from "llamaindex";
+
+const csvData =
+  "TITLE,RELEASE_YEAR,SCORE,NUMBER_OF_VOTES,DURATION,MAIN_GENRE,MAIN_PRODUCTION\nDavid Attenborough: A Life on Our Planet,2020,9,31180,83,documentary,GB\nInception,2010,8.8,2268288,148,scifi,GB\nForrest Gump,1994,8.8,1994599,142,drama,US\nAnbe Sivam,2003,8.7,20595,160,comedy,IN\nBo Burnham: Inside,2021,8.7,44074,87,comedy,US\nSaving Private Ryan,1998,8.6,1346020,169,drama,US\nDjango Unchained,2012,8.4,1472668,165,western,US\nDangal,2016,8.4,180247,161,action,IN\nBo Burnham: Make Happy,2016,8.4,14356,60,comedy,US\nLouis C.K.: Hilarious,2010,8.4,11973,84,comedy,US\nDave Chappelle: Sticks & Stones,2019,8.4,25687,65,comedy,US\n3 Idiots,2009,8.4,385782,170,comedy,IN\nBlack Friday,2004,8.4,20611,143,crime,IN\nSuper Deluxe,2019,8.4,13680,176,thriller,IN\nWinter on Fire: Ukraine's Fight for Freedom,2015,8.3,17710,98,documentary,UA\nOnce Upon a Time in America,1984,8.3,342335,229,drama,US\nTaxi Driver,1976,8.3,795222,113,crime,US\nLike Stars on Earth,2007,8.3,188234,165,drama,IN\nBo Burnham: What.,2013,8.3,11488,60,comedy,US\nFull Metal Jacket,1987,8.3,723306,116,drama,GB\nWarrior,2011,8.2,463276,140,drama,US\nDrishyam,2015,8.2,79075,163,thriller,IN\nQueen,2014,8.2,64805,146,drama,IN\nPaan Singh Tomar,2012,8.2,35888,135,drama,IN";
+
+const userQuestion = "which are the best comedies after 2010?";
+
+(async () => {
+  // The agent will succeed if we increase `maxTokens` to 1024
+  const llm = new OpenAI({ model: "gpt-4-turbo", maxTokens: 256 });
+
+  type Input = {
+    code: string;
+  };
+  // initiate fake code interpreter
+  const interpreterTool = FunctionTool.from<Input>(
+    ({ code }) => {
+      console.log(
+        `To answer the user's question, call the following code:\n${code}`,
+      );
+      return code;
+    },
+    {
+      name: "interpreter",
+      description:
+        "Execute python code in a Jupyter notebook cell and return any result, stdout, stderr, display_data, and error.",
+      parameters: {
+        type: "object",
+        properties: {
+          code: {
+            type: "string",
+            description: "The python code to execute in a single cell.",
+          },
+        },
+        required: ["code"],
+      },
+    },
+  );
+
+  const systemPrompt =
+    "You are a Python interpreter.\n        - You are given tasks to complete and you run python code to solve them.\n        - The python code runs in a Jupyter notebook. Every time you call $(interpreter) tool, the python code is executed in a separate cell. It's okay to make multiple calls to $(interpreter).\n        - Display visualizations using matplotlib or any other visualization library directly in the notebook. Shouldn't save the visualizations to a file, just return the base64 encoded data.\n        - You can install any pip package (if it exists) if you need to but the usual packages for data analysis are already preinstalled.\n        - You can run any python code you want in a secure environment.";
+
+  const agent = new OpenAIAgent({
+    llm,
+    tools: [interpreterTool],
+    systemPrompt,
+    verbose: true,
+  });
+
+  console.log(`User question: ${userQuestion}\n`);
+
+  await agent.chat({
+    message: [
+      {
+        type: "text",
+        text: userQuestion,
+      },
+      {
+        type: "text",
+        text: `Use data from following CSV raw contents:\n${csvData}`,
+      },
+    ],
+  });
+})();

examples/deepinfra/chat.ts

@@ -0,0 +1,19 @@
+import { DeepInfra } from "llamaindex";
+
+(async () => {
+  if (!process.env.DEEPINFRA_API_TOKEN) {
+    throw new Error("Please set the DEEPINFRA_API_TOKEN environment variable.");
+  }
+  const deepinfra = new DeepInfra({});
+  const result = await deepinfra.chat({
+    messages: [
+      { content: "You want to talk in rhymes.", role: "system" },
+      {
+        content:
+          "How much wood would a woodchuck chuck if a woodchuck could chuck wood?",
+        role: "user",
+      },
+    ],
+  });
+  console.log(result);
+})();

examples/deepinfra/embedding.ts

@@ -0,0 +1,17 @@
+import { DeepInfraEmbedding } from "llamaindex";
+
+async function main() {
+  // API token can be provided as an environment variable too
+  // using DEEPINFRA_API_TOKEN variable
+  const apiToken = "YOUR_API_TOKEN" ?? process.env.DEEPINFRA_API_TOKEN;
+  const model = "BAAI/bge-large-en-v1.5";
+  const embedModel = new DeepInfraEmbedding({
+    model,
+    apiToken,
+  });
+  const texts = ["hello", "world"];
+  const embeddings = await embedModel.getTextEmbeddingsBatch(texts);
+  console.log(`\nWe have ${embeddings.length} embeddings`);
+}
+
+main().catch(console.error);

examples/package.json

@@ -4,22 +4,22 @@
   "version": "0.0.4",
   "dependencies": {
     "@aws-crypto/sha256-js": "^5.2.0",
-    "@datastax/astra-db-ts": "^1.1.0",
+    "@datastax/astra-db-ts": "^1.2.1",
     "@notionhq/client": "^2.2.15",
-    "@pinecone-database/pinecone": "^2.2.0",
+    "@pinecone-database/pinecone": "^2.2.2",
     "@zilliz/milvus2-sdk-node": "^2.4.2",
-    "chromadb": "^1.7.3",
-    "commander": "^12.0.0",
+    "chromadb": "^1.8.1",
+    "commander": "^12.1.0",
     "dotenv": "^16.4.5",
-    "js-tiktoken": "^1.0.11",
-    "llamaindex": "*",
-    "mongodb": "^6.6.1",
+    "js-tiktoken": "^1.0.12",
+    "llamaindex": "^0.3.14",
+    "mongodb": "^6.7.0",
     "pathe": "^1.1.2"
   },
   "devDependencies": {
-    "@types/node": "^20.12.11",
+    "@types/node": "^20.14.1",
     "ts-node": "^10.9.2",
-    "tsx": "^4.9.3",
+    "tsx": "^4.11.2",
     "typescript": "^5.4.5"
   },
   "scripts": {

examples/readers/package.json

@@ -11,7 +11,7 @@
     "start:pdf": "node --import tsx ./src/pdf.ts",
     "start:llamaparse": "node --import tsx ./src/llamaparse.ts",
     "start:notion": "node --import tsx ./src/notion.ts",
-    "start:llamaparse2": "node --import tsx ./src/llamaparse_2.ts"
+    "start:llamaparse-dir": "node --import tsx ./src/simple-directory-reader-with-llamaparse.ts"
   },
   "dependencies": {
     "llamaindex": "*"

examples/readers/src/custom-simple-directory-reader.ts

@@ -1,12 +1,13 @@
-import type { BaseReader, Document, Metadata } from "llamaindex";
+import type { Document, Metadata } from "llamaindex";
+import { FileReader } from "llamaindex";
 import {
   FILE_EXT_TO_READER,
   SimpleDirectoryReader,
 } from "llamaindex/readers/SimpleDirectoryReader";
 import { TextFileReader } from "llamaindex/readers/TextFileReader";
 
-class ZipReader implements BaseReader {
-  loadData(...args: any[]): Promise<Document<Metadata>[]> {
+class ZipReader extends FileReader {
+  loadDataAsContent(fileContent: Buffer): Promise<Document<Metadata>[]> {
     throw new Error("Implement me");
   }
 }

examples/readers/src/llamaparse_2.ts

@@ -1,26 +0,0 @@
-import fs from "fs/promises";
-import { LlamaParseReader } from "llamaindex";
-
-async function main() {
-  // Load PDF using LlamaParse. set apiKey here or in environment variable LLAMA_CLOUD_API_KEY
-  const reader = new LlamaParseReader({
-    resultType: "markdown",
-    language: "en",
-    parsingInstruction:
-      "The provided document is a manga comic book. Most pages do NOT have title. It does not contain tables. Try to reconstruct the dialogue happening in a cohesive way. Output any math equation in LATEX markdown (between $$)",
-  });
-  const documents = await reader.loadData("../data/manga.pdf"); // The manga.pdf in the data folder is just a copy of the TOS, due to copyright laws. You have to place your own. I used "The Manga Guide to Calculus" by Hiroyuki Kojima
-
-  // Assuming documents contain an array of pages or sections
-  const parsedManga = documents.map((page) => page.text).join("\n---\n");
-
-  // Output the parsed manga to .md file. Will be placed in ../example/readers/
-  try {
-    await fs.writeFile("./parsedManga.md", parsedManga);
-    console.log("Output successfully written to parsedManga.md");
-  } catch (err) {
-    console.error("Error writing to file:", err);
-  }
-}
-
-main().catch(console.error);

examples/readers/src/simple-directory-reader-with-llamaparse.ts

@@ -0,0 +1,35 @@
+import {
+  LlamaParseReader,
+  SimpleDirectoryReader,
+  VectorStoreIndex,
+} from "llamaindex";
+
+async function main() {
+  const reader = new SimpleDirectoryReader();
+
+  const docs = await reader.loadData({
+    directoryPath: "../data/parallel", // brk-2022.pdf split into 6 parts
+    numWorkers: 2,
+    // set LlamaParse as the default reader for all file types. Set apiKey here or in environment variable LLAMA_CLOUD_API_KEY
+    overrideReader: new LlamaParseReader({
+      language: "en",
+      resultType: "markdown",
+      parsingInstruction:
+        "The provided files is Berkshire Hathaway's 2022 Annual Report. They contain figures, tables and raw data. Capture the data in a structured format. Mathematical equation should be put out as LATEX markdown (between $$).",
+    }),
+  });
+
+  const index = await VectorStoreIndex.fromDocuments(docs);
+
+  // Query the index
+  const queryEngine = index.asQueryEngine();
+  const response = await queryEngine.query({
+    query:
+      "What is the general strategy for shareholder safety outlined in the report? Use a concrete example with numbers",
+  });
+
+  // Output response
+  console.log(response.toString());
+}
+
+main().catch(console.error);

package.json

@@ -19,15 +19,15 @@
     "new-snapshot": "pnpm run build:release && changeset version --snapshot"
   },
   "devDependencies": {
-    "@changesets/cli": "^2.27.1",
-    "@typescript-eslint/eslint-plugin": "^7.8.0",
+    "@changesets/cli": "^2.27.5",
+    "@typescript-eslint/eslint-plugin": "^7.11.0",
     "eslint": "^8.57.0",
     "eslint-config-next": "^14.2.3",
     "eslint-config-prettier": "^9.1.0",
     "eslint-config-turbo": "^1.13.3",
     "eslint-plugin-react": "7.34.1",
     "husky": "^9.0.11",
-    "lint-staged": "^15.2.2",
+    "lint-staged": "^15.2.5",
     "madge": "^7.0.0",
     "prettier": "^3.2.5",
     "prettier-plugin-organize-imports": "^3.2.4",

packages/autotool/examples/02_nextjs/CHANGELOG.md

@@ -1,5 +1,22 @@
 # @llamaindex/autotool-02-next-example
 
+## 0.1.8
+
+### Patch Changes
+
+- Updated dependencies [11ae926]
+- Updated dependencies [631f000]
+- Updated dependencies [1378ec4]
+- Updated dependencies [6b1ded4]
+- Updated dependencies [4d4bd85]
+- Updated dependencies [24a9d1e]
+- Updated dependencies [45952de]
+- Updated dependencies [54230f0]
+- Updated dependencies [a29d835]
+- Updated dependencies [73819bf]
+  - llamaindex@0.3.16
+  - @llamaindex/autotool@0.0.1
+
 ## 0.1.7
 
 ### Patch Changes

packages/autotool/examples/02_nextjs/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/autotool-02-next-example",
   "private": true,
-  "version": "0.1.7",
+  "version": "0.1.8",
   "scripts": {
     "dev": "next dev",
     "build": "next build",

packages/autotool/package.json

@@ -51,7 +51,7 @@
     "unplugin": "^1.10.1"
   },
   "peerDependencies": {
-    "llamaindex": "^0.3.15",
+    "llamaindex": "^0.3.16",
     "openai": "^4",
     "typescript": "^4"
   },

packages/community/.cjs.swcrc

@@ -0,0 +1,12 @@
+{
+  "jsc": {
+    "parser": {
+      "syntax": "typescript"
+    },
+    "target": "esnext"
+  },
+  "module": {
+    "type": "commonjs",
+    "ignoreDynamic": true
+  }
+}

packages/community/.swcrc

@@ -0,0 +1,8 @@
+{
+  "jsc": {
+    "parser": {
+      "syntax": "typescript"
+    },
+    "target": "esnext"
+  }
+}

packages/community/CHANGELOG.md

@@ -0,0 +1,18 @@
+# @llamaindex/community
+
+## 0.0.2
+
+### Patch Changes
+
+- 8832669: Community bedrock support added
+- Updated dependencies [11ae926]
+- Updated dependencies [631f000]
+- Updated dependencies [1378ec4]
+- Updated dependencies [6b1ded4]
+- Updated dependencies [4d4bd85]
+- Updated dependencies [24a9d1e]
+- Updated dependencies [45952de]
+- Updated dependencies [54230f0]
+- Updated dependencies [a29d835]
+- Updated dependencies [73819bf]
+  - llamaindex@0.3.16

packages/community/jsr.json

@@ -0,0 +1,8 @@
+{
+  "name": "@llamaindex/community",
+  "version": "0.0.5",
+  "exports": {
+    ".": "./src/index.ts",
+    "./type": "./src/type.ts"
+  }
+}

packages/community/package.json

@@ -0,0 +1,59 @@
+{
+  "name": "@llamaindex/community",
+  "description": "Community package for LlamaIndexTS",
+  "version": "0.0.2",
+  "type": "module",
+  "types": "dist/type/index.d.ts",
+  "main": "dist/cjs/index.js",
+  "exports": {
+    ".": {
+      "import": {
+        "types": "./dist/type/index.d.ts",
+        "default": "./dist/index.js"
+      },
+      "require": {
+        "types": "./dist/type/index.d.ts",
+        "default": "./dist/cjs/index.js"
+      }
+    },
+    "./*": {
+      "import": {
+        "types": "./dist/type/*.d.ts",
+        "default": "./dist/*.js"
+      },
+      "require": {
+        "types": "./dist/type/*.d.ts",
+        "default": "./dist/cjs/*.js"
+      }
+    }
+  },
+  "files": [
+    "dist",
+    "CHANGELOG.md"
+  ],
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/run-llama/LlamaIndexTS.git",
+    "directory": "packages/community"
+  },
+  "scripts": {
+    "lint": "eslint .",
+    "build": "rm -rf ./dist && pnpm run build:esm && pnpm run build:cjs && pnpm run build:type",
+    "build:esm": "swc src -d dist --strip-leading-paths --config-file ../../.swcrc",
+    "build:cjs": "swc src -d dist/cjs --strip-leading-paths --config-file ../../.cjs.swcrc",
+    "build:type": "tsc -p tsconfig.json",
+    "postbuild": "node -e \"require('fs').writeFileSync('./dist/cjs/package.json', JSON.stringify({ type: 'commonjs' }))\"",
+    "dev": "concurrently \"pnpm run build:esm --watch\" \"pnpm run build:cjs --watch\" \"pnpm run build:type --watch\""
+  },
+  "devDependencies": {
+    "@swc/cli": "^0.3.12",
+    "@swc/core": "^1.5.5",
+    "concurrently": "^8.2.2",
+    "pathe": "^1.1.2"
+  },
+  "dependencies": {
+    "@aws-sdk/client-bedrock-runtime": "^3.582.0",
+    "@types/node": "^20.12.11",
+    "llamaindex": "workspace:*"
+  }
+}

packages/community/src/index.ts

@@ -0,0 +1 @@
+export { BEDROCK_MODELS, Bedrock } from "./llm/bedrock/base.js";

packages/community/src/llm/bedrock/base.ts

@@ -0,0 +1,352 @@
+import {
+  BedrockRuntimeClient,
+  InvokeModelCommand,
+  InvokeModelWithResponseStreamCommand,
+  type BedrockRuntimeClientConfig,
+  type InvokeModelCommandInput,
+  type InvokeModelWithResponseStreamCommandInput,
+} from "@aws-sdk/client-bedrock-runtime";
+
+import type {
+  ChatMessage,
+  ChatResponse,
+  ChatResponseChunk,
+  CompletionResponse,
+  LLMChatParamsNonStreaming,
+  LLMChatParamsStreaming,
+  LLMCompletionParamsNonStreaming,
+  LLMCompletionParamsStreaming,
+  LLMMetadata,
+  ToolCallLLMMessageOptions,
+} from "llamaindex";
+import { ToolCallLLM, streamConverter, wrapLLMEvent } from "llamaindex";
+import type {
+  AnthropicNoneStreamingResponse,
+  AnthropicTextContent,
+  StreamEvent,
+} from "./types.js";
+import {
+  mapChatMessagesToAnthropicMessages,
+  mapMessageContentToMessageContentDetails,
+  toUtf8,
+} from "./utils.js";
+
+export type BedrockAdditionalChatOptions = {};
+
+export type BedrockChatParamsStreaming = LLMChatParamsStreaming<
+  BedrockAdditionalChatOptions,
+  ToolCallLLMMessageOptions
+>;
+
+export type BedrockChatStreamResponse = AsyncIterable<
+  ChatResponseChunk<ToolCallLLMMessageOptions>
+>;
+
+export type BedrockChatParamsNonStreaming = LLMChatParamsNonStreaming<
+  BedrockAdditionalChatOptions,
+  ToolCallLLMMessageOptions
+>;
+
+export type BedrockChatNonStreamResponse =
+  ChatResponse<ToolCallLLMMessageOptions>;
+
+export enum BEDROCK_MODELS {
+  AMAZON_TITAN_TG1_LARGE = "amazon.titan-tg1-large",
+  AMAZON_TITAN_TEXT_EXPRESS_V1 = "amazon.titan-text-express-v1",
+  AI21_J2_GRANDE_INSTRUCT = "ai21.j2-grande-instruct",
+  AI21_J2_JUMBO_INSTRUCT = "ai21.j2-jumbo-instruct",
+  AI21_J2_MID = "ai21.j2-mid",
+  AI21_J2_MID_V1 = "ai21.j2-mid-v1",
+  AI21_J2_ULTRA = "ai21.j2-ultra",
+  AI21_J2_ULTRA_V1 = "ai21.j2-ultra-v1",
+  COHERE_COMMAND_TEXT_V14 = "cohere.command-text-v14",
+  ANTHROPIC_CLAUDE_INSTANT_1 = "anthropic.claude-instant-v1",
+  ANTHROPIC_CLAUDE_1 = "anthropic.claude-v1", // EOF: No longer supported
+  ANTHROPIC_CLAUDE_2 = "anthropic.claude-v2",
+  ANTHROPIC_CLAUDE_2_1 = "anthropic.claude-v2:1",
+  ANTHROPIC_CLAUDE_3_SONNET = "anthropic.claude-3-sonnet-20240229-v1:0",
+  ANTHROPIC_CLAUDE_3_HAIKU = "anthropic.claude-3-haiku-20240307-v1:0",
+  ANTHROPIC_CLAUDE_3_OPUS = "anthropic.claude-3-opus-20240229-v1:0",
+  META_LLAMA2_13B_CHAT = "meta.llama2-13b-chat-v1",
+  META_LLAMA2_70B_CHAT = "meta.llama2-70b-chat-v1",
+  META_LLAMA3_8B_INSTRUCT = "meta.llama3-8b-instruct-v1:0",
+  META_LLAMA3_70B_INSTRUCT = "meta.llama3-70b-instruct-v1:0",
+  MISTRAL_7B_INSTRUCT = "mistral.mistral-7b-instruct-v0:2",
+  MISTRAL_MIXTRAL_7B_INSTRUCT = "mistral.mixtral-8x7b-instruct-v0:1",
+  MISTRAL_MIXTRAL_LARGE_2402 = "mistral.mistral-large-2402-v1:0",
+}
+
+/*
+ * Values taken from https://docs.aws.amazon.com/bedrock/latest/userguide/model-parameters.html#model-parameters-claude
+ */
+
+const COMPLETION_MODELS = {
+  [BEDROCK_MODELS.AMAZON_TITAN_TG1_LARGE]: 8000,
+  [BEDROCK_MODELS.AMAZON_TITAN_TEXT_EXPRESS_V1]: 8000,
+  [BEDROCK_MODELS.AI21_J2_GRANDE_INSTRUCT]: 8000,
+  [BEDROCK_MODELS.AI21_J2_JUMBO_INSTRUCT]: 8000,
+  [BEDROCK_MODELS.AI21_J2_MID]: 8000,
+  [BEDROCK_MODELS.AI21_J2_MID_V1]: 8000,
+  [BEDROCK_MODELS.AI21_J2_ULTRA]: 8000,
+  [BEDROCK_MODELS.AI21_J2_ULTRA_V1]: 8000,
+  [BEDROCK_MODELS.COHERE_COMMAND_TEXT_V14]: 4096,
+};
+
+const CHAT_ONLY_MODELS = {
+  [BEDROCK_MODELS.ANTHROPIC_CLAUDE_INSTANT_1]: 100000,
+  [BEDROCK_MODELS.ANTHROPIC_CLAUDE_1]: 100000,
+  [BEDROCK_MODELS.ANTHROPIC_CLAUDE_2]: 100000,
+  [BEDROCK_MODELS.ANTHROPIC_CLAUDE_2_1]: 200000,
+  [BEDROCK_MODELS.ANTHROPIC_CLAUDE_3_SONNET]: 200000,
+  [BEDROCK_MODELS.ANTHROPIC_CLAUDE_3_HAIKU]: 200000,
+  [BEDROCK_MODELS.ANTHROPIC_CLAUDE_3_OPUS]: 200000,
+  [BEDROCK_MODELS.META_LLAMA2_13B_CHAT]: 2048,
+  [BEDROCK_MODELS.META_LLAMA2_70B_CHAT]: 4096,
+  [BEDROCK_MODELS.META_LLAMA3_8B_INSTRUCT]: 8192,
+  [BEDROCK_MODELS.META_LLAMA3_70B_INSTRUCT]: 8192,
+  [BEDROCK_MODELS.MISTRAL_7B_INSTRUCT]: 32000,
+  [BEDROCK_MODELS.MISTRAL_MIXTRAL_7B_INSTRUCT]: 32000,
+  [BEDROCK_MODELS.MISTRAL_MIXTRAL_LARGE_2402]: 32000,
+};
+
+const BEDROCK_FOUNDATION_LLMS = { ...COMPLETION_MODELS, ...CHAT_ONLY_MODELS };
+
+/*
+ * Only the following models support streaming as
+ * per result of Bedrock.Client.list_foundation_models
+ * https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/bedrock/client/list_foundation_models.html
+ */
+export const STREAMING_MODELS = new Set([
+  BEDROCK_MODELS.AMAZON_TITAN_TG1_LARGE,
+  BEDROCK_MODELS.AMAZON_TITAN_TEXT_EXPRESS_V1,
+  BEDROCK_MODELS.ANTHROPIC_CLAUDE_INSTANT_1,
+  BEDROCK_MODELS.ANTHROPIC_CLAUDE_1,
+  BEDROCK_MODELS.ANTHROPIC_CLAUDE_2,
+  BEDROCK_MODELS.ANTHROPIC_CLAUDE_2_1,
+  BEDROCK_MODELS.ANTHROPIC_CLAUDE_3_SONNET,
+  BEDROCK_MODELS.ANTHROPIC_CLAUDE_3_HAIKU,
+  BEDROCK_MODELS.ANTHROPIC_CLAUDE_3_OPUS,
+  BEDROCK_MODELS.META_LLAMA2_13B_CHAT,
+  BEDROCK_MODELS.META_LLAMA2_70B_CHAT,
+  BEDROCK_MODELS.META_LLAMA3_8B_INSTRUCT,
+  BEDROCK_MODELS.META_LLAMA3_70B_INSTRUCT,
+  BEDROCK_MODELS.MISTRAL_7B_INSTRUCT,
+  BEDROCK_MODELS.MISTRAL_MIXTRAL_7B_INSTRUCT,
+  BEDROCK_MODELS.MISTRAL_MIXTRAL_LARGE_2402,
+]);
+
+abstract class Provider {
+  abstract getTextFromResponse(response: Record<string, any>): string;
+
+  getTextFromStreamResponse(response: Record<string, any>): string {
+    return this.getTextFromResponse(response);
+  }
+
+  abstract getRequestBody<T extends ChatMessage>(
+    metadata: LLMMetadata,
+    messages: T[],
+  ): InvokeModelCommandInput | InvokeModelWithResponseStreamCommandInput;
+}
+
+class AnthropicProvider extends Provider {
+  getResultFromResponse(
+    response: Record<string, any>,
+  ): AnthropicNoneStreamingResponse {
+    return JSON.parse(toUtf8(response.body));
+  }
+
+  getTextFromResponse(response: Record<string, any>): string {
+    const result = this.getResultFromResponse(response);
+    return result.content
+      .filter((item) => item.type === "text")
+      .map((item) => (item as AnthropicTextContent).text)
+      .join(" ");
+  }
+
+  getTextFromStreamResponse(response: Record<string, any>): string {
+    const event: StreamEvent | undefined = response.chunk?.bytes
+      ? JSON.parse(toUtf8(response.chunk?.bytes))
+      : undefined;
+
+    if (event?.type === "content_block_delta") return event.delta.text;
+    return "";
+  }
+
+  getRequestBody<T extends ChatMessage>(
+    metadata: LLMMetadata,
+    messages: T[],
+  ): InvokeModelCommandInput | InvokeModelWithResponseStreamCommandInput {
+    return {
+      modelId: metadata.model,
+      contentType: "application/json",
+      accept: "application/json",
+      body: JSON.stringify({
+        anthropic_version: "bedrock-2023-05-31",
+        messages: mapChatMessagesToAnthropicMessages(messages),
+        max_tokens: metadata.maxTokens,
+        temperature: metadata.temperature,
+        top_p: metadata.topP,
+      }),
+    };
+  }
+}
+
+// Other providers could go here
+const PROVIDERS: { [key: string]: Provider } = {
+  anthropic: new AnthropicProvider(),
+};
+
+const getProvider = (model: string): Provider => {
+  const providerName = model.split(".")[0];
+  if (!(providerName in PROVIDERS)) {
+    throw new Error(
+      `Provider ${providerName} for model ${model} is not supported`,
+    );
+  }
+  return PROVIDERS[providerName];
+};
+
+export type BedrockModelParams = {
+  model: keyof typeof BEDROCK_FOUNDATION_LLMS;
+  temperature?: number;
+  topP?: number;
+  maxTokens?: number;
+};
+
+const DEFAULT_BEDROCK_PARAMS = {
+  temperature: 0.1,
+  topP: 1,
+  maxTokens: 1024, // required by anthropic
+};
+
+export type BedrockParams = BedrockModelParams & BedrockRuntimeClientConfig;
+
+/**
+ * ToolCallLLM for Bedrock
+ */
+export class Bedrock extends ToolCallLLM<BedrockAdditionalChatOptions> {
+  private client: BedrockRuntimeClient;
+  model: keyof typeof BEDROCK_FOUNDATION_LLMS;
+  temperature: number;
+  topP: number;
+  maxTokens?: number;
+  provider: Provider;
+  topK?: number;
+
+  // there should be no check for env variables. Bedrock can be authenticated in various ways
+  // AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY and AWS_REGION are the env variables used directly by the sdk
+  constructor({
+    temperature,
+    topP,
+    maxTokens,
+    model,
+    ...params
+  }: BedrockParams) {
+    super();
+
+    this.model = model;
+    this.provider = getProvider(this.model);
+    this.maxTokens = maxTokens ?? DEFAULT_BEDROCK_PARAMS.maxTokens;
+    this.temperature = temperature ?? DEFAULT_BEDROCK_PARAMS.temperature;
+    this.topP = topP ?? DEFAULT_BEDROCK_PARAMS.topP;
+    this.client = new BedrockRuntimeClient(params);
+  }
+
+  get supportToolCall(): boolean {
+    return false;
+  }
+
+  get metadata(): LLMMetadata {
+    // NOTE, Anthropic supports top_k but LLMMetadata does not
+    return {
+      model: this.model,
+      temperature: this.temperature,
+      topP: this.topP,
+      maxTokens: this.maxTokens,
+      contextWindow: BEDROCK_FOUNDATION_LLMS[this.model],
+      tokenizer: undefined,
+    };
+  }
+
+  protected async nonStreamChat(
+    params: BedrockChatParamsNonStreaming,
+  ): Promise<BedrockChatNonStreamResponse> {
+    const input = this.provider.getRequestBody(this.metadata, params.messages);
+    const command = new InvokeModelCommand(input);
+    const response = await this.client.send(command);
+    return {
+      raw: response,
+      message: {
+        content: this.provider.getTextFromResponse(response),
+        role: "assistant",
+      },
+    };
+  }
+
+  protected async *streamChat(
+    params: BedrockChatParamsStreaming,
+  ): BedrockChatStreamResponse {
+    if (!STREAMING_MODELS.has(this.model))
+      throw new Error(`The model: ${this.model} does not support streaming`);
+    const input = this.provider.getRequestBody(this.metadata, params.messages);
+    const command = new InvokeModelWithResponseStreamCommand(input);
+    const response = await this.client.send(command);
+
+    if (response.body)
+      yield* streamConverter(response.body, (response) => {
+        return {
+          delta: this.provider.getTextFromStreamResponse(response),
+          raw: response,
+        };
+      });
+  }
+
+  chat(params: BedrockChatParamsStreaming): Promise<BedrockChatStreamResponse>;
+  chat(
+    params: BedrockChatParamsNonStreaming,
+  ): Promise<BedrockChatNonStreamResponse>;
+
+  @wrapLLMEvent
+  async chat(
+    params: BedrockChatParamsStreaming | BedrockChatParamsNonStreaming,
+  ): Promise<BedrockChatStreamResponse | BedrockChatNonStreamResponse> {
+    if (params.stream) return this.streamChat(params);
+    return this.nonStreamChat(params);
+  }
+
+  complete(
+    params: LLMCompletionParamsStreaming,
+  ): Promise<AsyncIterable<CompletionResponse>>;
+  complete(
+    params: LLMCompletionParamsNonStreaming,
+  ): Promise<CompletionResponse>;
+  async complete(
+    params: LLMCompletionParamsStreaming | LLMCompletionParamsNonStreaming,
+  ): Promise<CompletionResponse | AsyncIterable<CompletionResponse>> {
+    const message: ChatMessage = {
+      role: "user",
+      content: mapMessageContentToMessageContentDetails(params.prompt),
+    };
+
+    const input = this.provider.getRequestBody(this.metadata, [message]);
+
+    if (params.stream) {
+      const command = new InvokeModelWithResponseStreamCommand(input);
+      const response = await this.client.send(command);
+      if (response.body)
+        return streamConverter(response.body, (response) => {
+          return {
+            text: this.provider.getTextFromStreamResponse(response),
+            raw: response,
+          };
+        });
+    }
+
+    const command = new InvokeModelCommand(input);
+    const response = await this.client.send(command);
+    return {
+      text: this.provider.getTextFromResponse(response),
+      raw: response,
+    };
+  }
+}

packages/community/src/llm/bedrock/types.ts

@@ -0,0 +1,109 @@
+type Usage = {
+  input_tokens: number;
+  output_tokens: number;
+};
+
+type Message = {
+  id: string;
+  type: string;
+  role: string;
+  content: string[];
+  model: string;
+  stop_reason: string | null;
+  stop_sequence: string | null;
+  usage: Usage;
+};
+
+type ContentBlockStart = {
+  type: "content_block_start";
+  index: number;
+  content_block: {
+    type: string;
+    text: string;
+  };
+};
+
+type Delta = {
+  type: string;
+  text: string;
+};
+
+type ContentBlockDelta = {
+  type: "content_block_delta";
+  index: number;
+  delta: Delta;
+};
+
+type ContentBlockStop = {
+  type: "content_block_stop";
+  index: number;
+};
+
+type MessageDelta = {
+  type: "message_delta";
+  delta: {
+    stop_reason: string;
+    stop_sequence: string | null;
+  };
+  usage: Usage;
+};
+
+type InvocationMetrics = {
+  inputTokenCount: number;
+  outputTokenCount: number;
+  invocationLatency: number;
+  firstByteLatency: number;
+};
+
+type MessageStop = {
+  type: "message_stop";
+  "amazon-bedrock-invocationMetrics": InvocationMetrics;
+};
+
+export type StreamEvent =
+  | { type: "message_start"; message: Message }
+  | ContentBlockStart
+  | ContentBlockDelta
+  | ContentBlockStop
+  | MessageDelta
+  | MessageStop;
+
+export type AnthropicContent = AnthropicTextContent | AnthropicImageContent;
+
+export type AnthropicTextContent = {
+  type: "text";
+  text: string;
+};
+
+export type AnthropicMediaTypes =
+  | "image/jpeg"
+  | "image/png"
+  | "image/webp"
+  | "image/gif";
+
+export type AnthropicImageSource = {
+  type: "base64";
+  media_type: AnthropicMediaTypes;
+  data: string; // base64 encoded image bytes
+};
+
+export type AnthropicImageContent = {
+  type: "image";
+  source: AnthropicImageSource;
+};
+
+export type AnthropicMessage = {
+  role: "user" | "assistant";
+  content: AnthropicContent[];
+};
+
+export type AnthropicNoneStreamingResponse = {
+  id: string;
+  type: "message";
+  role: "assistant";
+  content: AnthropicContent[];
+  model: string;
+  stop_reason: "end_turn" | "max_tokens" | "stop_sequence";
+  stop_sequence?: string;
+  usage: { input_tokens: number; output_tokens: number };
+};

packages/community/src/llm/bedrock/utils.ts

@@ -0,0 +1,144 @@
+import type {
+  ChatMessage,
+  MessageContent,
+  MessageContentDetail,
+} from "llamaindex";
+import type {
+  AnthropicContent,
+  AnthropicImageContent,
+  AnthropicMediaTypes,
+  AnthropicMessage,
+  AnthropicTextContent,
+} from "./types.js";
+
+const ACCEPTED_IMAGE_MIME_TYPES = [
+  "image/jpeg",
+  "image/png",
+  "image/webp",
+  "image/gif",
+];
+
+export const mapMessageContentToMessageContentDetails = (
+  content: MessageContent,
+): MessageContentDetail[] => {
+  return Array.isArray(content) ? content : [{ type: "text", text: content }];
+};
+
+export const mergeNeighboringSameRoleMessages = (
+  messages: AnthropicMessage[],
+): AnthropicMessage[] => {
+  return messages.reduce(
+    (result: AnthropicMessage[], current: AnthropicMessage, index: number) => {
+      if (index > 0 && messages[index - 1].role === current.role) {
+        result[result.length - 1].content = [
+          ...result[result.length - 1].content,
+          ...current.content,
+        ];
+      } else {
+        result.push(current);
+      }
+      return result;
+    },
+    [],
+  );
+};
+
+export const mapMessageContentDetailToAnthropicContent = <
+  T extends MessageContentDetail,
+>(
+  detail: T,
+): AnthropicContent => {
+  let content: AnthropicContent;
+
+  if (detail.type === "text") {
+    content = mapTextContent(detail.text);
+  } else if (detail.type === "image_url") {
+    content = mapImageContent(detail.image_url.url);
+  } else {
+    throw new Error("Unsupported content detail type");
+  }
+  return content;
+};
+
+export const mapMessageContentToAnthropicContent = <T extends MessageContent>(
+  content: T,
+): AnthropicContent[] => {
+  return mapMessageContentToMessageContentDetails(content).map(
+    mapMessageContentDetailToAnthropicContent,
+  );
+};
+
+export const mapChatMessagesToAnthropicMessages = <T extends ChatMessage>(
+  messages: T[],
+): AnthropicMessage[] => {
+  const mapped = messages
+    .flatMap((msg: T): AnthropicMessage[] => {
+      return mapMessageContentToMessageContentDetails(msg.content).map(
+        (detail: MessageContentDetail): AnthropicMessage => {
+          const content = mapMessageContentDetailToAnthropicContent(detail);
+
+          return {
+            role: msg.role === "assistant" ? "assistant" : "user",
+            content: [content],
+          };
+        },
+      );
+    })
+    .filter((message: AnthropicMessage) => {
+      const content = message.content[0];
+      if (content.type === "text" && !content.text) return false;
+      if (content.type === "image" && !content.source.data) return false;
+      return true;
+    });
+
+  return mergeNeighboringSameRoleMessages(mapped);
+};
+
+export const mapTextContent = (text: string): AnthropicTextContent => {
+  return { type: "text", text };
+};
+
+export const extractDataUrlComponents = (
+  dataUrl: string,
+): {
+  mimeType: string;
+  base64: string;
+} => {
+  const parts = dataUrl.split(";base64,");
+
+  if (parts.length !== 2 || !parts[0].startsWith("data:")) {
+    throw new Error("Invalid data URL");
+  }
+
+  const mimeType = parts[0].slice(5);
+  const base64 = parts[1];
+
+  return {
+    mimeType,
+    base64,
+  };
+};
+
+export const mapImageContent = (imageUrl: string): AnthropicImageContent => {
+  if (!imageUrl.startsWith("data:"))
+    throw new Error(
+      "For Anthropic please only use base64 data url, e.g.: data:image/jpeg;base64,SGVsbG8sIFdvcmxkIQ==",
+    );
+  const { mimeType, base64: data } = extractDataUrlComponents(imageUrl);
+  if (!ACCEPTED_IMAGE_MIME_TYPES.includes(mimeType))
+    throw new Error(
+      `Anthropic only accepts the following mimeTypes: ${ACCEPTED_IMAGE_MIME_TYPES.join("\n")}`,
+    );
+
+  return {
+    type: "image",
+    source: {
+      type: "base64",
+      media_type: mimeType as AnthropicMediaTypes,
+      data,
+    },
+  };
+};
+
+export const toUtf8 = (input: Uint8Array): string =>
+  new TextDecoder("utf-8").decode(input);

packages/community/tsconfig.json

@@ -0,0 +1,19 @@
+{
+  "extends": "../../tsconfig.json",
+  "compilerOptions": {
+    "rootDir": "./src",
+    "outDir": "./dist/type",
+    "tsBuildInfoFile": "./dist/.tsbuildinfo",
+    "emitDeclarationOnly": true,
+    "module": "node16",
+    "moduleResolution": "node16",
+    "types": ["node"]
+  },
+  "include": ["./src"],
+  "exclude": ["node_modules"],
+  "references": [
+    {
+      "path": "../core/tsconfig.json"
+    }
+  ]
+}

packages/core/CHANGELOG.md

@@ -1,5 +1,20 @@
 # llamaindex
 
+## 0.3.16
+
+### Patch Changes
+
+- 11ae926: feat: add numCandidates setting to MongoDBAtlasVectorStore for tuning queries
+- 631f000: feat: DeepInfra LLM implementation
+- 1378ec4: feat: set default model to `gpt-4o`
+- 6b1ded4: add gpt4o-mode, invalidate cache and skip diagonal text to LlamaParseReader
+- 4d4bd85: Show error message if agent tool is called with partial JSON
+- 24a9d1e: add json mode and image retrieval to LlamaParseReader
+- 45952de: add concurrency management for SimpleDirectoryReader
+- 54230f0: feat: Gemini GA release models
+- a29d835: setDocumentHash should be async
+- 73819bf: Unify metadata and ID handling of documents, allow files to be read by `Buffer`
+
 ## 0.3.15
 
 ### Patch Changes

packages/core/e2e/examples/cloudflare-worker-agent/CHANGELOG.md

@@ -1,5 +1,21 @@
 # @llamaindex/cloudflare-worker-agent-test
 
+## 0.0.17
+
+### Patch Changes
+
+- Updated dependencies [11ae926]
+- Updated dependencies [631f000]
+- Updated dependencies [1378ec4]
+- Updated dependencies [6b1ded4]
+- Updated dependencies [4d4bd85]
+- Updated dependencies [24a9d1e]
+- Updated dependencies [45952de]
+- Updated dependencies [54230f0]
+- Updated dependencies [a29d835]
+- Updated dependencies [73819bf]
+  - llamaindex@0.3.16
+
 ## 0.0.16
 
 ### Patch Changes

packages/core/e2e/examples/cloudflare-worker-agent/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/cloudflare-worker-agent-test",
-  "version": "0.0.16",
+  "version": "0.0.17",
   "type": "module",
   "private": true,
   "scripts": {

packages/core/e2e/examples/nextjs-agent/CHANGELOG.md

@@ -1,5 +1,21 @@
 # @llamaindex/next-agent-test
 
+## 0.1.17
+
+### Patch Changes
+
+- Updated dependencies [11ae926]
+- Updated dependencies [631f000]
+- Updated dependencies [1378ec4]
+- Updated dependencies [6b1ded4]
+- Updated dependencies [4d4bd85]
+- Updated dependencies [24a9d1e]
+- Updated dependencies [45952de]
+- Updated dependencies [54230f0]
+- Updated dependencies [a29d835]
+- Updated dependencies [73819bf]
+  - llamaindex@0.3.16
+
 ## 0.1.16
 
 ### Patch Changes

packages/core/e2e/examples/nextjs-agent/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/next-agent-test",
-  "version": "0.1.16",
+  "version": "0.1.17",
   "private": true,
   "scripts": {
     "dev": "next dev",

packages/core/e2e/examples/nextjs-edge-runtime/CHANGELOG.md

@@ -1,5 +1,21 @@
 # test-edge-runtime
 
+## 0.1.16
+
+### Patch Changes
+
+- Updated dependencies [11ae926]
+- Updated dependencies [631f000]
+- Updated dependencies [1378ec4]
+- Updated dependencies [6b1ded4]
+- Updated dependencies [4d4bd85]
+- Updated dependencies [24a9d1e]
+- Updated dependencies [45952de]
+- Updated dependencies [54230f0]
+- Updated dependencies [a29d835]
+- Updated dependencies [73819bf]
+  - llamaindex@0.3.16
+
 ## 0.1.15
 
 ### Patch Changes

packages/core/e2e/examples/nextjs-edge-runtime/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/nextjs-edge-runtime-test",
-  "version": "0.1.15",
+  "version": "0.1.16",
   "private": true,
   "scripts": {
     "dev": "next dev",

packages/core/e2e/examples/waku-query-engine/CHANGELOG.md

@@ -1,5 +1,21 @@
 # @llamaindex/waku-query-engine-test
 
+## 0.0.17
+
+### Patch Changes
+
+- Updated dependencies [11ae926]
+- Updated dependencies [631f000]
+- Updated dependencies [1378ec4]
+- Updated dependencies [6b1ded4]
+- Updated dependencies [4d4bd85]
+- Updated dependencies [24a9d1e]
+- Updated dependencies [45952de]
+- Updated dependencies [54230f0]
+- Updated dependencies [a29d835]
+- Updated dependencies [73819bf]
+  - llamaindex@0.3.16
+
 ## 0.0.16
 
 ### Patch Changes

packages/core/e2e/examples/waku-query-engine/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/waku-query-engine-test",
-  "version": "0.0.16",
+  "version": "0.0.17",
   "type": "module",
   "private": true,
   "scripts": {

packages/core/e2e/node/openai.e2e.ts

@@ -191,7 +191,8 @@ For questions about more specific sections, please use the vector_tool.`,
     message:
       "What's the summary of Alex? Does he live in Brazil based on the brief information? Return yes or no.",
   });
-  strictEqual(mockCall.mock.callCount(), 1);
+  // not sure if AI calls twice("Alex", "Brazil") or once ("Alex in Brazil") or something else
+  ok(mockCall.mock.callCount() >= 1);
 
   consola.debug("response:", response.message.content);
   ok(extractText(response.message.content).toLowerCase().includes("no"));

packages/core/e2e/node/snapshot/agent.snap

@@ -20,21 +20,23 @@
           "content": "",
           "role": "assistant",
           "options": {
-            "toolCall": {
-              "id": "call_Enrc7RhNts0qxGaEATOEhV6F",
-              "name": "Weather",
-              "input": "{\"location\":\"San Francisco\"}"
-            }
+            "toolCall": [
+              {
+                "id": "call_sH6QfjsymHW7JFl68j8AY6xg",
+                "name": "Weather",
+                "input": "{\"location\":\"San Francisco\"}"
+              }
+            ]
           }
         },
         {
-          "content": "35 degrees and sunny in San Francisco",
           "role": "user",
+          "content": "35 degrees and sunny in San Francisco",
           "options": {
             "toolResult": {
               "result": "35 degrees and sunny in San Francisco",
               "isError": false,
-              "id": "call_Enrc7RhNts0qxGaEATOEhV6F"
+              "id": "call_sH6QfjsymHW7JFl68j8AY6xg"
             }
           }
         }
@@ -60,21 +62,23 @@
           "content": "",
           "role": "assistant",
           "options": {
-            "toolCall": {
-              "id": "call_OYo81prrpK2RLOS9d0Y6kiXB",
-              "name": "unique_id",
-              "input": "{\"firstName\":\"Alex\",\"lastName\":\"Yang\"}"
-            }
+            "toolCall": [
+              {
+                "id": "call_V7zs8cyDT5FqJhjwBqcCydgA",
+                "name": "unique_id",
+                "input": "{\"firstName\":\"Alex\",\"lastName\":\"Yang\"}"
+              }
+            ]
           }
         },
         {
-          "content": "123456789",
           "role": "user",
+          "content": "123456789",
           "options": {
             "toolResult": {
               "result": "123456789",
               "isError": false,
-              "id": "call_OYo81prrpK2RLOS9d0Y6kiXB"
+              "id": "call_V7zs8cyDT5FqJhjwBqcCydgA"
             }
           }
         }
@@ -100,21 +104,23 @@
           "content": "",
           "role": "assistant",
           "options": {
-            "toolCall": {
-              "id": "call_YUXowHKwOJ6GIWV4EA59x8iF",
-              "name": "sumNumbers",
-              "input": "{\"a\":1,\"b\":1}"
-            }
+            "toolCall": [
+              {
+                "id": "call_BrlGGU6GDGWr0hrwXt9qZKyt",
+                "name": "sumNumbers",
+                "input": "{\"a\":1,\"b\":1}"
+              }
+            ]
           }
         },
         {
-          "content": "2",
           "role": "user",
+          "content": "2",
           "options": {
             "toolResult": {
               "result": "2",
               "isError": false,
-              "id": "call_YUXowHKwOJ6GIWV4EA59x8iF"
+              "id": "call_BrlGGU6GDGWr0hrwXt9qZKyt"
             }
           }
         }
@@ -130,11 +136,13 @@
           "content": "",
           "role": "assistant",
           "options": {
-            "toolCall": {
-              "id": "call_Enrc7RhNts0qxGaEATOEhV6F",
-              "name": "Weather",
-              "input": "{\"location\":\"San Francisco\"}"
-            }
+            "toolCall": [
+              {
+                "id": "call_sH6QfjsymHW7JFl68j8AY6xg",
+                "name": "Weather",
+                "input": "{\"location\":\"San Francisco\"}"
+              }
+            ]
           }
         }
       }
@@ -158,11 +166,13 @@
           "content": "",
           "role": "assistant",
           "options": {
-            "toolCall": {
-              "id": "call_OYo81prrpK2RLOS9d0Y6kiXB",
-              "name": "unique_id",
-              "input": "{\"firstName\":\"Alex\",\"lastName\":\"Yang\"}"
-            }
+            "toolCall": [
+              {
+                "id": "call_V7zs8cyDT5FqJhjwBqcCydgA",
+                "name": "unique_id",
+                "input": "{\"firstName\":\"Alex\",\"lastName\":\"Yang\"}"
+              }
+            ]
           }
         }
       }
@@ -186,11 +196,13 @@
           "content": "",
           "role": "assistant",
           "options": {
-            "toolCall": {
-              "id": "call_YUXowHKwOJ6GIWV4EA59x8iF",
-              "name": "sumNumbers",
-              "input": "{\"a\":1,\"b\":1}"
-            }
+            "toolCall": [
+              {
+                "id": "call_BrlGGU6GDGWr0hrwXt9qZKyt",
+                "name": "sumNumbers",
+                "input": "{\"a\":1,\"b\":1}"
+              }
+            ]
           }
         }
       }
@@ -200,7 +212,7 @@
       "response": {
         "raw": null,
         "message": {
-          "content": "The sum of 1 + 1 is 2.",
+          "content": "1 + 1 is equal to 2.",
           "role": "assistant",
           "options": {}
         }

packages/core/e2e/node/snapshot/agent_stream.snap

@@ -20,14 +20,21 @@
           "role": "assistant",
           "content": "",
           "options": {
-            "toolCall": {
-              "name": "divideNumbers",
-              "id": "call_QQh8uKzGHbgBlrhqxZNoWiQT",
-              "input": {
-                "a": 16,
-                "b": 2
+            "toolCall": [
+              {
+                "name": "divideNumbers",
+                "id": "call_V4daSdWk9QeYeSMKLBisNbSf",
+                "input": {
+                  "a": 16,
+                  "b": 2
+                }
+              },
+              {
+                "name": "sumNumbers",
+                "id": "call_n2OlBxlaoeMIMVeU9DeDfiPX",
+                "input": "{\"a\": 8, \"b\": 20}"
               }
-            }
+            ]
           }
         },
         {
@@ -37,18 +44,7 @@
             "toolResult": {
               "result": "8",
               "isError": false,
-              "id": "call_QQh8uKzGHbgBlrhqxZNoWiQT"
-            }
-          }
-        },
-        {
-          "role": "assistant",
-          "content": "",
-          "options": {
-            "toolCall": {
-              "name": "sumNumbers",
-              "id": "call_C03ASSDxIFTrYmzSuTxCrItd",
-              "input": "{\"a\": 8, \"b\": 20}"
+              "id": "call_V4daSdWk9QeYeSMKLBisNbSf"
             }
           }
         },
@@ -59,7 +55,7 @@
             "toolResult": {
               "result": "28",
               "isError": false,
-              "id": "call_C03ASSDxIFTrYmzSuTxCrItd"
+              "id": "call_n2OlBxlaoeMIMVeU9DeDfiPX"
             }
           }
         }
@@ -75,11 +71,13 @@
           "content": "",
           "role": "assistant",
           "options": {
-            "toolCall": {
-              "name": "sumNumbers",
-              "id": "call_C03ASSDxIFTrYmzSuTxCrItd",
-              "input": "{\"a\": 8, \"b\": 20}"
-            }
+            "toolCall": [
+              {
+                "name": "sumNumbers",
+                "id": "call_n2OlBxlaoeMIMVeU9DeDfiPX",
+                "input": "{\"a\": 8, \"b\": 20}"
+              }
+            ]
           }
         }
       }
@@ -89,7 +87,7 @@
       "response": {
         "raw": null,
         "message": {
-          "content": "The result of dividing 16 by 2 and then adding 20 is 28.",
+          "content": "The result of dividing 16 by 2 is 8. When you add 20 to 8, the total is 28.",
           "role": "assistant",
           "options": {}
         }
@@ -102,84 +100,13 @@
       "chunk": {
         "raw": null,
         "options": {
-          "toolCall": {
-            "name": "divideNumbers",
-            "id": "call_QQh8uKzGHbgBlrhqxZNoWiQT",
-            "input": ""
-          }
-        },
-        "delta": ""
-      }
-    },
-    {
-      "id": "PRESERVE_0",
-      "chunk": {
-        "raw": null,
-        "options": {
-          "toolCall": {
-            "name": "divideNumbers",
-            "id": "call_QQh8uKzGHbgBlrhqxZNoWiQT",
-            "input": "{\"a\""
-          }
-        },
-        "delta": ""
-      }
-    },
-    {
-      "id": "PRESERVE_0",
-      "chunk": {
-        "raw": null,
-        "options": {
-          "toolCall": {
-            "name": "divideNumbers",
-            "id": "call_QQh8uKzGHbgBlrhqxZNoWiQT",
-            "input": "{\"a\": 16,"
-          }
-        },
-        "delta": ""
-      }
-    },
-    {
-      "id": "PRESERVE_0",
-      "chunk": {
-        "raw": null,
-        "options": {
-          "toolCall": {
-            "name": "divideNumbers",
-            "id": "call_QQh8uKzGHbgBlrhqxZNoWiQT",
-            "input": "{\"a\": 16, \"b\": "
-          }
-        },
-        "delta": ""
-      }
-    },
-    {
-      "id": "PRESERVE_0",
-      "chunk": {
-        "raw": null,
-        "options": {
-          "toolCall": {
-            "name": "divideNumbers",
-            "id": "call_QQh8uKzGHbgBlrhqxZNoWiQT",
-            "input": "{\"a\": 16, \"b\": 2}"
-          }
-        },
-        "delta": ""
-      }
-    },
-    {
-      "id": "PRESERVE_0",
-      "chunk": {
-        "raw": null,
-        "options": {
-          "toolCall": {
-            "name": "divideNumbers",
-            "id": "call_QQh8uKzGHbgBlrhqxZNoWiQT",
-            "input": {
-              "a": 16,
-              "b": 2
+          "toolCall": [
+            {
+              "name": "divideNumbers",
+              "id": "call_V4daSdWk9QeYeSMKLBisNbSf",
+              "input": "{\"a\": 16, \"b\": 2}"
             }
-          }
+          ]
         },
         "delta": ""
       }
@@ -189,11 +116,13 @@
       "chunk": {
         "raw": null,
         "options": {
-          "toolCall": {
-            "name": "sumNumbers",
-            "id": "call_C03ASSDxIFTrYmzSuTxCrItd",
-            "input": "{\"a\""
-          }
+          "toolCall": [
+            {
+              "name": "divideNumbers",
+              "id": "call_V4daSdWk9QeYeSMKLBisNbSf",
+              "input": "{\"a\": 16, \"b\": 2}"
+            }
+          ]
         },
         "delta": ""
       }
@@ -203,11 +132,13 @@
       "chunk": {
         "raw": null,
         "options": {
-          "toolCall": {
-            "name": "sumNumbers",
-            "id": "call_C03ASSDxIFTrYmzSuTxCrItd",
-            "input": "{\"a\": 8, "
-          }
+          "toolCall": [
+            {
+              "name": "divideNumbers",
+              "id": "call_V4daSdWk9QeYeSMKLBisNbSf",
+              "input": "{\"a\": 16, \"b\": 2}"
+            }
+          ]
         },
         "delta": ""
       }
@@ -217,11 +148,13 @@
       "chunk": {
         "raw": null,
         "options": {
-          "toolCall": {
-            "name": "sumNumbers",
-            "id": "call_C03ASSDxIFTrYmzSuTxCrItd",
-            "input": "{\"a\": 8, \"b\": 2"
-          }
+          "toolCall": [
+            {
+              "name": "divideNumbers",
+              "id": "call_V4daSdWk9QeYeSMKLBisNbSf",
+              "input": "{\"a\": 16, \"b\": 2}"
+            }
+          ]
         },
         "delta": ""
       }
@@ -231,11 +164,96 @@
       "chunk": {
         "raw": null,
         "options": {
-          "toolCall": {
-            "name": "sumNumbers",
-            "id": "call_C03ASSDxIFTrYmzSuTxCrItd",
-            "input": "{\"a\": 8, \"b\": 20}"
-          }
+          "toolCall": [
+            {
+              "name": "divideNumbers",
+              "id": "call_V4daSdWk9QeYeSMKLBisNbSf",
+              "input": "{\"a\": 16, \"b\": 2}"
+            }
+          ]
+        },
+        "delta": ""
+      }
+    },
+    {
+      "id": "PRESERVE_0",
+      "chunk": {
+        "raw": null,
+        "options": {
+          "toolCall": [
+            {
+              "name": "divideNumbers",
+              "id": "call_V4daSdWk9QeYeSMKLBisNbSf",
+              "input": {
+                "a": 16,
+                "b": 2
+              }
+            }
+          ]
+        },
+        "delta": ""
+      }
+    },
+    {
+      "id": "PRESERVE_0",
+      "chunk": {
+        "raw": null,
+        "options": {
+          "toolCall": [
+            {
+              "name": "sumNumbers",
+              "id": "call_n2OlBxlaoeMIMVeU9DeDfiPX",
+              "input": "{\"a\": 8, \"b\": 20}"
+            }
+          ]
+        },
+        "delta": ""
+      }
+    },
+    {
+      "id": "PRESERVE_0",
+      "chunk": {
+        "raw": null,
+        "options": {
+          "toolCall": [
+            {
+              "name": "sumNumbers",
+              "id": "call_n2OlBxlaoeMIMVeU9DeDfiPX",
+              "input": "{\"a\": 8, \"b\": 20}"
+            }
+          ]
+        },
+        "delta": ""
+      }
+    },
+    {
+      "id": "PRESERVE_0",
+      "chunk": {
+        "raw": null,
+        "options": {
+          "toolCall": [
+            {
+              "name": "sumNumbers",
+              "id": "call_n2OlBxlaoeMIMVeU9DeDfiPX",
+              "input": "{\"a\": 8, \"b\": 20}"
+            }
+          ]
+        },
+        "delta": ""
+      }
+    },
+    {
+      "id": "PRESERVE_0",
+      "chunk": {
+        "raw": null,
+        "options": {
+          "toolCall": [
+            {
+              "name": "sumNumbers",
+              "id": "call_n2OlBxlaoeMIMVeU9DeDfiPX",
+              "input": "{\"a\": 8, \"b\": 20}"
+            }
+          ]
         },
         "delta": ""
       }
@@ -317,7 +335,7 @@
       "chunk": {
         "raw": null,
         "options": {},
-        "delta": " and"
+        "delta": " is"
       }
     },
     {
@@ -325,7 +343,7 @@
       "chunk": {
         "raw": null,
         "options": {},
-        "delta": " then"
+        "delta": " "
       }
     },
     {
@@ -333,7 +351,39 @@
       "chunk": {
         "raw": null,
         "options": {},
-        "delta": " adding"
+        "delta": "8"
+      }
+    },
+    {
+      "id": "PRESERVE_1",
+      "chunk": {
+        "raw": null,
+        "options": {},
+        "delta": "."
+      }
+    },
+    {
+      "id": "PRESERVE_1",
+      "chunk": {
+        "raw": null,
+        "options": {},
+        "delta": " When"
+      }
+    },
+    {
+      "id": "PRESERVE_1",
+      "chunk": {
+        "raw": null,
+        "options": {},
+        "delta": " you"
+      }
+    },
+    {
+      "id": "PRESERVE_1",
+      "chunk": {
+        "raw": null,
+        "options": {},
+        "delta": " add"
       }
     },
     {
@@ -352,6 +402,54 @@
         "delta": "20"
       }
     },
+    {
+      "id": "PRESERVE_1",
+      "chunk": {
+        "raw": null,
+        "options": {},
+        "delta": " to"
+      }
+    },
+    {
+      "id": "PRESERVE_1",
+      "chunk": {
+        "raw": null,
+        "options": {},
+        "delta": " "
+      }
+    },
+    {
+      "id": "PRESERVE_1",
+      "chunk": {
+        "raw": null,
+        "options": {},
+        "delta": "8"
+      }
+    },
+    {
+      "id": "PRESERVE_1",
+      "chunk": {
+        "raw": null,
+        "options": {},
+        "delta": ","
+      }
+    },
+    {
+      "id": "PRESERVE_1",
+      "chunk": {
+        "raw": null,
+        "options": {},
+        "delta": " the"
+      }
+    },
+    {
+      "id": "PRESERVE_1",
+      "chunk": {
+        "raw": null,
+        "options": {},
+        "delta": " total"
+      }
+    },
     {
       "id": "PRESERVE_1",
       "chunk": {

packages/core/e2e/node/snapshot/agent_with_object_function_call.snap

@@ -20,16 +20,18 @@
           "content": "",
           "role": "assistant",
           "options": {
-            "toolCall": {
-              "id": "call_iY0VO4HvXDT2PtR3CvRSY1uK",
-              "name": "get_weather",
-              "input": "{\"location\":\"San Francisco\"}"
-            }
+            "toolCall": [
+              {
+                "id": "call_uERMumWlJLTO2GW93X6C2W3N",
+                "name": "get_weather",
+                "input": "{\"location\":\"San Francisco\"}"
+              }
+            ]
           }
         },
         {
-          "content": "{\n  location: San Francisco,\n  temperature: 72,\n  weather: cloudy,\n  rain_prediction: 0.89\n}",
           "role": "user",
+          "content": "{\n  location: San Francisco,\n  temperature: 72,\n  weather: cloudy,\n  rain_prediction: 0.89\n}",
           "options": {
             "toolResult": {
               "result": {
@@ -39,7 +41,7 @@
                 "rain_prediction": 0.89
               },
               "isError": false,
-              "id": "call_iY0VO4HvXDT2PtR3CvRSY1uK"
+              "id": "call_uERMumWlJLTO2GW93X6C2W3N"
             }
           }
         }
@@ -55,11 +57,13 @@
           "content": "",
           "role": "assistant",
           "options": {
-            "toolCall": {
-              "id": "call_iY0VO4HvXDT2PtR3CvRSY1uK",
-              "name": "get_weather",
-              "input": "{\"location\":\"San Francisco\"}"
-            }
+            "toolCall": [
+              {
+                "id": "call_uERMumWlJLTO2GW93X6C2W3N",
+                "name": "get_weather",
+                "input": "{\"location\":\"San Francisco\"}"
+              }
+            ]
           }
         }
       }

packages/core/e2e/node/snapshot/agent_with_object_retriever.snap

@@ -24,6 +24,15 @@
     },
     {
       "id": "PRESERVE_2",
+      "messages": [
+        {
+          "content": "Context information is below.\n---------------------\nAlex is a male. What's very important, Alex is not in the Brazil.\n---------------------\nGiven the context information and not prior knowledge, answer the query.\nQuery: Brazil\nAnswer:",
+          "role": "user"
+        }
+      ]
+    },
+    {
+      "id": "PRESERVE_3",
       "messages": [
         {
           "content": "Please always use the tools provided to answer a question. Do not rely on prior knowledge.",
@@ -37,21 +46,39 @@
           "content": "",
           "role": "assistant",
           "options": {
-            "toolCall": {
-              "id": "call_VfgZS1NPKpbtzqhM3o1Buzx8",
-              "name": "summary_tool",
-              "input": "{\"query\": \"Alex\"}"
-            }
+            "toolCall": [
+              {
+                "id": "call_vv1XW3xv4j2us5sZOtzCU2lL",
+                "name": "summary_tool",
+                "input": "{\"query\": \"Alex\"}"
+              },
+              {
+                "id": "call_V36LMHbwUJkEa20A3GoA4wIr",
+                "name": "summary_tool",
+                "input": "{\"query\": \"Brazil\"}"
+              }
+            ]
           }
         },
         {
-          "content": "Alex is not in Brazil.",
           "role": "user",
+          "content": "Alex is not in Brazil.",
           "options": {
             "toolResult": {
               "result": "Alex is not in Brazil.",
               "isError": false,
-              "id": "call_VfgZS1NPKpbtzqhM3o1Buzx8"
+              "id": "call_vv1XW3xv4j2us5sZOtzCU2lL"
+            }
+          }
+        },
+        {
+          "role": "user",
+          "content": "Alex is not in Brazil.",
+          "options": {
+            "toolResult": {
+              "result": "Alex is not in Brazil.",
+              "isError": false,
+              "id": "call_V36LMHbwUJkEa20A3GoA4wIr"
             }
           }
         }
@@ -67,11 +94,18 @@
           "content": "",
           "role": "assistant",
           "options": {
-            "toolCall": {
-              "id": "call_VfgZS1NPKpbtzqhM3o1Buzx8",
-              "name": "summary_tool",
-              "input": "{\"query\": \"Alex\"}"
-            }
+            "toolCall": [
+              {
+                "id": "call_vv1XW3xv4j2us5sZOtzCU2lL",
+                "name": "summary_tool",
+                "input": "{\"query\": \"Alex\"}"
+              },
+              {
+                "id": "call_V36LMHbwUJkEa20A3GoA4wIr",
+                "name": "summary_tool",
+                "input": "{\"query\": \"Brazil\"}"
+              }
+            ]
           }
         }
       }
@@ -92,7 +126,18 @@
       "response": {
         "raw": null,
         "message": {
-          "content": "No, Alex does not live in Brazil based on the brief information available.",
+          "content": "Alex is not in Brazil.",
+          "role": "assistant",
+          "options": {}
+        }
+      }
+    },
+    {
+      "id": "PRESERVE_3",
+      "response": {
+        "raw": null,
+        "message": {
+          "content": "Based on the brief information, Alex is not in Brazil.",
           "role": "assistant",
           "options": {}
         }

packages/core/e2e/node/snapshot/anthropic-agent-multiple-chat.snap

@@ -178,13 +178,15 @@
           ],
           "role": "assistant",
           "options": {
-            "toolCall": {
-              "id": "toolu_01Gy7Gxbx7uGmjVncGH6pubL",
-              "name": "getWeather",
-              "input": {
-                "city": "San Francisco"
+            "toolCall": [
+              {
+                "id": "toolu_017MxwaxaLYkmt4dpP5HKzFe",
+                "name": "getWeather",
+                "input": {
+                  "city": "San Francisco"
+                }
               }
-            }
+            ]
           }
         },
         {
@@ -194,7 +196,7 @@
             "toolResult": {
               "result": "The weather in San Francisco is 72 degrees",
               "isError": false,
-              "id": "toolu_01Gy7Gxbx7uGmjVncGH6pubL"
+              "id": "toolu_017MxwaxaLYkmt4dpP5HKzFe"
             }
           }
         }
@@ -258,13 +260,15 @@
           ],
           "role": "assistant",
           "options": {
-            "toolCall": {
-              "id": "toolu_01Gy7Gxbx7uGmjVncGH6pubL",
-              "name": "getWeather",
-              "input": {
-                "city": "San Francisco"
+            "toolCall": [
+              {
+                "id": "toolu_017MxwaxaLYkmt4dpP5HKzFe",
+                "name": "getWeather",
+                "input": {
+                  "city": "San Francisco"
+                }
               }
-            }
+            ]
           }
         },
         {
@@ -274,7 +278,7 @@
             "toolResult": {
               "result": "The weather in San Francisco is 72 degrees",
               "isError": false,
-              "id": "toolu_01Gy7Gxbx7uGmjVncGH6pubL"
+              "id": "toolu_017MxwaxaLYkmt4dpP5HKzFe"
             }
           }
         },
@@ -352,13 +356,15 @@
           ],
           "role": "assistant",
           "options": {
-            "toolCall": {
-              "id": "toolu_01Gy7Gxbx7uGmjVncGH6pubL",
-              "name": "getWeather",
-              "input": {
-                "city": "San Francisco"
+            "toolCall": [
+              {
+                "id": "toolu_017MxwaxaLYkmt4dpP5HKzFe",
+                "name": "getWeather",
+                "input": {
+                  "city": "San Francisco"
+                }
               }
-            }
+            ]
           }
         },
         {
@@ -368,7 +374,7 @@
             "toolResult": {
               "result": "The weather in San Francisco is 72 degrees",
               "isError": false,
-              "id": "toolu_01Gy7Gxbx7uGmjVncGH6pubL"
+              "id": "toolu_017MxwaxaLYkmt4dpP5HKzFe"
             }
           }
         },
@@ -395,13 +401,15 @@
           ],
           "role": "assistant",
           "options": {
-            "toolCall": {
-              "id": "toolu_01NHyahSUqrPjxQk9mvCvvGe",
-              "name": "getWeather",
-              "input": {
-                "city": "Shanghai"
+            "toolCall": [
+              {
+                "id": "toolu_013ZnATGKhvPkt2jjxdFheDK",
+                "name": "getWeather",
+                "input": {
+                  "city": "Shanghai"
+                }
               }
-            }
+            ]
           }
         },
         {
@@ -411,7 +419,7 @@
             "toolResult": {
               "result": "The weather in Shanghai is 72 degrees",
               "isError": false,
-              "id": "toolu_01NHyahSUqrPjxQk9mvCvvGe"
+              "id": "toolu_013ZnATGKhvPkt2jjxdFheDK"
             }
           }
         }
@@ -480,13 +488,15 @@
           ],
           "role": "assistant",
           "options": {
-            "toolCall": {
-              "id": "toolu_01Gy7Gxbx7uGmjVncGH6pubL",
-              "name": "getWeather",
-              "input": {
-                "city": "San Francisco"
+            "toolCall": [
+              {
+                "id": "toolu_017MxwaxaLYkmt4dpP5HKzFe",
+                "name": "getWeather",
+                "input": {
+                  "city": "San Francisco"
+                }
               }
-            }
+            ]
           }
         }
       }
@@ -520,13 +530,15 @@
           ],
           "role": "assistant",
           "options": {
-            "toolCall": {
-              "id": "toolu_01NHyahSUqrPjxQk9mvCvvGe",
-              "name": "getWeather",
-              "input": {
-                "city": "Shanghai"
+            "toolCall": [
+              {
+                "id": "toolu_013ZnATGKhvPkt2jjxdFheDK",
+                "name": "getWeather",
+                "input": {
+                  "city": "Shanghai"
+                }
               }
-            }
+            ]
           }
         }
       }

packages/core/e2e/node/snapshot/anthropic-agent.snap

@@ -5,8 +5,7 @@
       "messages": [
         {
           "role": "user",
-          "content": "What is the weather in San Francisco?",
-          "options": {}
+          "content": "What is the weather in San Francisco?"
         }
       ]
     },
@@ -15,25 +14,26 @@
       "messages": [
         {
           "role": "user",
-          "content": "What is the weather in San Francisco?",
-          "options": {}
+          "content": "What is the weather in San Francisco?"
         },
         {
           "content": [
             {
               "type": "text",
-              "text": "<thinking>\nThe user is asking for the weather in a specific location, San Francisco. The Weather function is the relevant tool to answer this request, as it returns weather information for a given location.\n\nThe Weather function has one required parameter:\n- location (string): The user has directly provided the location of \"San Francisco\"\n\nSince the required location parameter has been provided by the user, we have all the necessary information to call the Weather function.\n</thinking>"
+              "text": "<thinking>\nThe user has asked for the weather in a specific location, San Francisco. The Weather tool is the relevant function to answer this request, as it returns weather information for a given location.\n\nThe Weather tool requires a single parameter:\n- location (string, required): The user has directly provided the location as \"San Francisco\".\n\nSince the required location parameter has been provided, we have enough information to call the Weather tool.\n</thinking>"
             }
           ],
           "role": "assistant",
           "options": {
-            "toolCall": {
-              "id": "HIDDEN",
-              "name": "Weather",
-              "input": {
-                "location": "San Francisco"
+            "toolCall": [
+              {
+                "id": "toolu_011YcKkPygw2woJZrVWRRMwH",
+                "name": "Weather",
+                "input": {
+                  "location": "San Francisco"
+                }
               }
-            }
+            ]
           }
         },
         {
@@ -41,8 +41,9 @@
           "role": "user",
           "options": {
             "toolResult": {
+              "result": "35 degrees and sunny in San Francisco",
               "isError": false,
-              "id": "HIDDEN"
+              "id": "toolu_011YcKkPygw2woJZrVWRRMwH"
             }
           }
         }
@@ -53,8 +54,7 @@
       "messages": [
         {
           "role": "user",
-          "content": "My name is Alex Yang. What is my unique id?",
-          "options": {}
+          "content": "My name is Alex Yang. What is my unique id?"
         }
       ]
     },
@@ -63,26 +63,27 @@
       "messages": [
         {
           "role": "user",
-          "content": "My name is Alex Yang. What is my unique id?",
-          "options": {}
+          "content": "My name is Alex Yang. What is my unique id?"
         },
         {
           "content": [
             {
               "type": "text",
-              "text": "<thinking>\nThe unique_id function is the relevant tool to answer the user's request for their unique ID. It requires two parameters:\nfirstName: The user provided their first name, which is \"Alex\"\nlastName: The user also provided their last name, \"Yang\"\nSince the user has provided all the required parameters, we can proceed with calling the unique_id function.\n</thinking>"
+              "text": "<thinking>\nThe unique_id function takes firstName and lastName as required parameters. The user has provided their first name (Alex) and last name (Yang) in the request, so we have all the necessary information to call the function.\n</thinking>"
             }
           ],
           "role": "assistant",
           "options": {
-            "toolCall": {
-              "id": "HIDDEN",
-              "name": "unique_id",
-              "input": {
-                "firstName": "Alex",
-                "lastName": "Yang"
+            "toolCall": [
+              {
+                "id": "toolu_012ZLUq5SWsghkXUjsEXsB1f",
+                "name": "unique_id",
+                "input": {
+                  "firstName": "Alex",
+                  "lastName": "Yang"
+                }
               }
-            }
+            ]
           }
         },
         {
@@ -90,8 +91,9 @@
           "role": "user",
           "options": {
             "toolResult": {
+              "result": "123456789",
               "isError": false,
-              "id": "HIDDEN"
+              "id": "toolu_012ZLUq5SWsghkXUjsEXsB1f"
             }
           }
         }
@@ -102,8 +104,7 @@
       "messages": [
         {
           "role": "user",
-          "content": "how much is 1 + 1?",
-          "options": {}
+          "content": "how much is 1 + 1?"
         }
       ]
     },
@@ -112,26 +113,27 @@
       "messages": [
         {
           "role": "user",
-          "content": "how much is 1 + 1?",
-          "options": {}
+          "content": "how much is 1 + 1?"
         },
         {
           "content": [
             {
               "type": "text",
-              "text": "<thinking>\nThe user is asking to sum the numbers 1 and 1. The relevant tool to use is the sumNumbers function, which takes two number parameters a and b.\nThe user has directly provided the values for the parameters:\na = 1 \nb = 1\nSince all the required parameters have been provided, we can proceed with calling the function.\n</thinking>"
+              "text": "<thinking>\nThe user is asking to sum the numbers 1 and 1. The relevant tool is the sumNumbers function, which takes two number parameters a and b.\nThe user has directly provided the values for the required parameters:\na = 1 \nb = 1\nSince all the required parameters are provided, we can proceed with calling the function.\n</thinking>"
             }
           ],
           "role": "assistant",
           "options": {
-            "toolCall": {
-              "id": "HIDDEN",
-              "name": "sumNumbers",
-              "input": {
-                "a": 1,
-                "b": 1
+            "toolCall": [
+              {
+                "id": "toolu_01SQD2XkaXkDNLQ2xaFzmguG",
+                "name": "sumNumbers",
+                "input": {
+                  "a": 1,
+                  "b": 1
+                }
               }
-            }
+            ]
           }
         },
         {
@@ -139,8 +141,9 @@
           "role": "user",
           "options": {
             "toolResult": {
+              "result": "2",
               "isError": false,
-              "id": "HIDDEN"
+              "id": "toolu_01SQD2XkaXkDNLQ2xaFzmguG"
             }
           }
         }
@@ -151,48 +154,25 @@
     {
       "id": "PRESERVE_0",
       "response": {
-        "raw": {
-          "id": "HIDDEN",
-          "type": "message",
-          "role": "assistant",
-          "model": "claude-3-opus-20240229",
-          "stop_sequence": null,
-          "usage": {
-            "input_tokens": 462,
-            "output_tokens": 147
-          },
-          "content": [
-            {
-              "type": "text",
-              "text": "<thinking>\nThe user is asking for the weather in a specific location, San Francisco. The Weather function is the relevant tool to answer this request, as it returns weather information for a given location.\n\nThe Weather function has one required parameter:\n- location (string): The user has directly provided the location of \"San Francisco\"\n\nSince the required location parameter has been provided by the user, we have all the necessary information to call the Weather function.\n</thinking>"
-            },
-            {
-              "type": "tool_use",
-              "id": "HIDDEN",
-              "name": "Weather",
-              "input": {
-                "location": "San Francisco"
-              }
-            }
-          ],
-          "stop_reason": "tool_use"
-        },
+        "raw": null,
         "message": {
           "content": [
             {
               "type": "text",
-              "text": "<thinking>\nThe user is asking for the weather in a specific location, San Francisco. The Weather function is the relevant tool to answer this request, as it returns weather information for a given location.\n\nThe Weather function has one required parameter:\n- location (string): The user has directly provided the location of \"San Francisco\"\n\nSince the required location parameter has been provided by the user, we have all the necessary information to call the Weather function.\n</thinking>"
+              "text": "<thinking>\nThe user has asked for the weather in a specific location, San Francisco. The Weather tool is the relevant function to answer this request, as it returns weather information for a given location.\n\nThe Weather tool requires a single parameter:\n- location (string, required): The user has directly provided the location as \"San Francisco\".\n\nSince the required location parameter has been provided, we have enough information to call the Weather tool.\n</thinking>"
             }
           ],
           "role": "assistant",
           "options": {
-            "toolCall": {
-              "id": "HIDDEN",
-              "name": "Weather",
-              "input": {
-                "location": "San Francisco"
+            "toolCall": [
+              {
+                "id": "toolu_011YcKkPygw2woJZrVWRRMwH",
+                "name": "Weather",
+                "input": {
+                  "location": "San Francisco"
+                }
               }
-            }
+            ]
           }
         }
       }
@@ -200,24 +180,7 @@
     {
       "id": "PRESERVE_1",
       "response": {
-        "raw": {
-          "id": "HIDDEN",
-          "type": "message",
-          "role": "assistant",
-          "model": "claude-3-opus-20240229",
-          "stop_sequence": null,
-          "usage": {
-            "input_tokens": 628,
-            "output_tokens": 18
-          },
-          "content": [
-            {
-              "type": "text",
-              "text": "The current weather in San Francisco is 35 degrees and sunny."
-            }
-          ],
-          "stop_reason": "end_turn"
-        },
+        "raw": null,
         "message": {
           "content": [
             {
@@ -233,50 +196,26 @@
     {
       "id": "PRESERVE_2",
       "response": {
-        "raw": {
-          "id": "HIDDEN",
-          "type": "message",
-          "role": "assistant",
-          "model": "claude-3-opus-20240229",
-          "stop_sequence": null,
-          "usage": {
-            "input_tokens": 482,
-            "output_tokens": 152
-          },
-          "content": [
-            {
-              "type": "text",
-              "text": "<thinking>\nThe unique_id function is the relevant tool to answer the user's request for their unique ID. It requires two parameters:\nfirstName: The user provided their first name, which is \"Alex\"\nlastName: The user also provided their last name, \"Yang\"\nSince the user has provided all the required parameters, we can proceed with calling the unique_id function.\n</thinking>"
-            },
-            {
-              "type": "tool_use",
-              "id": "HIDDEN",
-              "name": "unique_id",
-              "input": {
-                "firstName": "Alex",
-                "lastName": "Yang"
-              }
-            }
-          ],
-          "stop_reason": "tool_use"
-        },
+        "raw": null,
         "message": {
           "content": [
             {
               "type": "text",
-              "text": "<thinking>\nThe unique_id function is the relevant tool to answer the user's request for their unique ID. It requires two parameters:\nfirstName: The user provided their first name, which is \"Alex\"\nlastName: The user also provided their last name, \"Yang\"\nSince the user has provided all the required parameters, we can proceed with calling the unique_id function.\n</thinking>"
+              "text": "<thinking>\nThe unique_id function takes firstName and lastName as required parameters. The user has provided their first name (Alex) and last name (Yang) in the request, so we have all the necessary information to call the function.\n</thinking>"
             }
           ],
           "role": "assistant",
           "options": {
-            "toolCall": {
-              "id": "HIDDEN",
-              "name": "unique_id",
-              "input": {
-                "firstName": "Alex",
-                "lastName": "Yang"
+            "toolCall": [
+              {
+                "id": "toolu_012ZLUq5SWsghkXUjsEXsB1f",
+                "name": "unique_id",
+                "input": {
+                  "firstName": "Alex",
+                  "lastName": "Yang"
+                }
               }
-            }
+            ]
           }
         }
       }
@@ -284,24 +223,7 @@
     {
       "id": "PRESERVE_3",
       "response": {
-        "raw": {
-          "id": "HIDDEN",
-          "type": "message",
-          "role": "assistant",
-          "model": "claude-3-opus-20240229",
-          "stop_sequence": null,
-          "usage": {
-            "input_tokens": 648,
-            "output_tokens": 13
-          },
-          "content": [
-            {
-              "type": "text",
-              "text": "Your unique ID is 123456789."
-            }
-          ],
-          "stop_reason": "end_turn"
-        },
+        "raw": null,
         "message": {
           "content": [
             {
@@ -317,50 +239,26 @@
     {
       "id": "PRESERVE_4",
       "response": {
-        "raw": {
-          "id": "HIDDEN",
-          "type": "message",
-          "role": "assistant",
-          "model": "claude-3-opus-20240229",
-          "stop_sequence": null,
-          "usage": {
-            "input_tokens": 498,
-            "output_tokens": 151
-          },
-          "content": [
-            {
-              "type": "text",
-              "text": "<thinking>\nThe user is asking to sum the numbers 1 and 1. The relevant tool to use is the sumNumbers function, which takes two number parameters a and b.\nThe user has directly provided the values for the parameters:\na = 1 \nb = 1\nSince all the required parameters have been provided, we can proceed with calling the function.\n</thinking>"
-            },
-            {
-              "type": "tool_use",
-              "id": "HIDDEN",
-              "name": "sumNumbers",
-              "input": {
-                "a": 1,
-                "b": 1
-              }
-            }
-          ],
-          "stop_reason": "tool_use"
-        },
+        "raw": null,
         "message": {
           "content": [
             {
               "type": "text",
-              "text": "<thinking>\nThe user is asking to sum the numbers 1 and 1. The relevant tool to use is the sumNumbers function, which takes two number parameters a and b.\nThe user has directly provided the values for the parameters:\na = 1 \nb = 1\nSince all the required parameters have been provided, we can proceed with calling the function.\n</thinking>"
+              "text": "<thinking>\nThe user is asking to sum the numbers 1 and 1. The relevant tool is the sumNumbers function, which takes two number parameters a and b.\nThe user has directly provided the values for the required parameters:\na = 1 \nb = 1\nSince all the required parameters are provided, we can proceed with calling the function.\n</thinking>"
             }
           ],
           "role": "assistant",
           "options": {
-            "toolCall": {
-              "id": "HIDDEN",
-              "name": "sumNumbers",
-              "input": {
-                "a": 1,
-                "b": 1
+            "toolCall": [
+              {
+                "id": "toolu_01SQD2XkaXkDNLQ2xaFzmguG",
+                "name": "sumNumbers",
+                "input": {
+                  "a": 1,
+                  "b": 1
+                }
               }
-            }
+            ]
           }
         }
       }
@@ -368,24 +266,7 @@
     {
       "id": "PRESERVE_5",
       "response": {
-        "raw": {
-          "id": "HIDDEN",
-          "type": "message",
-          "role": "assistant",
-          "model": "claude-3-opus-20240229",
-          "stop_sequence": null,
-          "usage": {
-            "input_tokens": 661,
-            "output_tokens": 16
-          },
-          "content": [
-            {
-              "type": "text",
-              "text": "So 1 + 1 = 2."
-            }
-          ],
-          "stop_reason": "end_turn"
-        },
+        "raw": null,
         "message": {
           "content": [
             {

packages/core/e2e/node/snapshot/gpt-4-turbo.snap

@@ -20,21 +20,23 @@
           "content": "",
           "role": "assistant",
           "options": {
-            "toolCall": {
-              "id": "call_TRC5kpzyVaV5n2R91sJsBAF1",
-              "name": "Weather",
-              "input": "{\"location\":\"San Jose\"}"
-            }
+            "toolCall": [
+              {
+                "id": "call_Xa2Kxa2zUE073mnougPWzRlh",
+                "name": "Weather",
+                "input": "{\"location\":\"San Jose\"}"
+              }
+            ]
           }
         },
         {
-          "content": "45 degrees and sunny in San Jose",
           "role": "user",
+          "content": "45 degrees and sunny in San Jose",
           "options": {
             "toolResult": {
               "result": "45 degrees and sunny in San Jose",
               "isError": false,
-              "id": "call_TRC5kpzyVaV5n2R91sJsBAF1"
+              "id": "call_Xa2Kxa2zUE073mnougPWzRlh"
             }
           }
         }
@@ -50,11 +52,13 @@
           "content": "",
           "role": "assistant",
           "options": {
-            "toolCall": {
-              "id": "call_TRC5kpzyVaV5n2R91sJsBAF1",
-              "name": "Weather",
-              "input": "{\"location\":\"San Jose\"}"
-            }
+            "toolCall": [
+              {
+                "id": "call_Xa2Kxa2zUE073mnougPWzRlh",
+                "name": "Weather",
+                "input": "{\"location\":\"San Jose\"}"
+              }
+            ]
           }
         }
       }
@@ -64,7 +68,7 @@
       "response": {
         "raw": null,
         "message": {
-          "content": "The weather in San Jose is currently 45 degrees and sunny.",
+          "content": "The current weather in San Jose is 45 degrees and sunny.",
           "role": "assistant",
           "options": {}
         }

packages/core/e2e/node/snapshot/llm-anthropic.snap

@@ -24,24 +24,7 @@
     {
       "id": "PRESERVE_0",
       "response": {
-        "raw": {
-          "id": "HIDDEN",
-          "type": "message",
-          "role": "assistant",
-          "model": "claude-3-opus-20240229",
-          "stop_sequence": null,
-          "usage": {
-            "input_tokens": 8,
-            "output_tokens": 12
-          },
-          "content": [
-            {
-              "type": "text",
-              "text": "Hello! How can I assist you today?"
-            }
-          ],
-          "stop_reason": "end_turn"
-        },
+        "raw": null,
         "message": {
           "content": "Hello! How can I assist you today?",
           "role": "assistant",
@@ -52,116 +35,7 @@
     {
       "id": "PRESERVE_1",
       "response": {
-        "raw": [
-          {
-            "raw": {
-              "type": "content_block_delta",
-              "index": 0,
-              "delta": {
-                "type": "text_delta",
-                "text": "Hello"
-              }
-            },
-            "delta": "Hello",
-            "options": {}
-          },
-          {
-            "raw": {
-              "type": "content_block_delta",
-              "index": 0,
-              "delta": {
-                "type": "text_delta",
-                "text": "!"
-              }
-            },
-            "delta": "!",
-            "options": {}
-          },
-          {
-            "raw": {
-              "type": "content_block_delta",
-              "index": 0,
-              "delta": {
-                "type": "text_delta",
-                "text": " How"
-              }
-            },
-            "delta": " How",
-            "options": {}
-          },
-          {
-            "raw": {
-              "type": "content_block_delta",
-              "index": 0,
-              "delta": {
-                "type": "text_delta",
-                "text": " can"
-              }
-            },
-            "delta": " can",
-            "options": {}
-          },
-          {
-            "raw": {
-              "type": "content_block_delta",
-              "index": 0,
-              "delta": {
-                "type": "text_delta",
-                "text": " I"
-              }
-            },
-            "delta": " I",
-            "options": {}
-          },
-          {
-            "raw": {
-              "type": "content_block_delta",
-              "index": 0,
-              "delta": {
-                "type": "text_delta",
-                "text": " assist"
-              }
-            },
-            "delta": " assist",
-            "options": {}
-          },
-          {
-            "raw": {
-              "type": "content_block_delta",
-              "index": 0,
-              "delta": {
-                "type": "text_delta",
-                "text": " you"
-              }
-            },
-            "delta": " you",
-            "options": {}
-          },
-          {
-            "raw": {
-              "type": "content_block_delta",
-              "index": 0,
-              "delta": {
-                "type": "text_delta",
-                "text": " today"
-              }
-            },
-            "delta": " today",
-            "options": {}
-          },
-          {
-            "raw": {
-              "type": "content_block_delta",
-              "index": 0,
-              "delta": {
-                "type": "text_delta",
-                "text": "?"
-              }
-            },
-            "delta": "?",
-            "options": {}
-          }
-        ],
+        "raw": null,
         "message": {
           "content": "Hello! How can I assist you today?",
           "role": "assistant",
@@ -174,14 +48,7 @@
     {
       "id": "PRESERVE_1",
       "chunk": {
-        "raw": {
-          "type": "content_block_delta",
-          "index": 0,
-          "delta": {
-            "type": "text_delta",
-            "text": "Hello"
-          }
-        },
+        "raw": null,
         "delta": "Hello",
         "options": {}
       }
@@ -189,14 +56,7 @@
     {
       "id": "PRESERVE_1",
       "chunk": {
-        "raw": {
-          "type": "content_block_delta",
-          "index": 0,
-          "delta": {
-            "type": "text_delta",
-            "text": "!"
-          }
-        },
+        "raw": null,
         "delta": "!",
         "options": {}
       }
@@ -204,14 +64,7 @@
     {
       "id": "PRESERVE_1",
       "chunk": {
-        "raw": {
-          "type": "content_block_delta",
-          "index": 0,
-          "delta": {
-            "type": "text_delta",
-            "text": " How"
-          }
-        },
+        "raw": null,
         "delta": " How",
         "options": {}
       }
@@ -219,14 +72,7 @@
     {
       "id": "PRESERVE_1",
       "chunk": {
-        "raw": {
-          "type": "content_block_delta",
-          "index": 0,
-          "delta": {
-            "type": "text_delta",
-            "text": " can"
-          }
-        },
+        "raw": null,
         "delta": " can",
         "options": {}
       }
@@ -234,14 +80,7 @@
     {
       "id": "PRESERVE_1",
       "chunk": {
-        "raw": {
-          "type": "content_block_delta",
-          "index": 0,
-          "delta": {
-            "type": "text_delta",
-            "text": " I"
-          }
-        },
+        "raw": null,
         "delta": " I",
         "options": {}
       }
@@ -249,14 +88,7 @@
     {
       "id": "PRESERVE_1",
       "chunk": {
-        "raw": {
-          "type": "content_block_delta",
-          "index": 0,
-          "delta": {
-            "type": "text_delta",
-            "text": " assist"
-          }
-        },
+        "raw": null,
         "delta": " assist",
         "options": {}
       }
@@ -264,14 +96,7 @@
     {
       "id": "PRESERVE_1",
       "chunk": {
-        "raw": {
-          "type": "content_block_delta",
-          "index": 0,
-          "delta": {
-            "type": "text_delta",
-            "text": " you"
-          }
-        },
+        "raw": null,
         "delta": " you",
         "options": {}
       }
@@ -279,14 +104,7 @@
     {
       "id": "PRESERVE_1",
       "chunk": {
-        "raw": {
-          "type": "content_block_delta",
-          "index": 0,
-          "delta": {
-            "type": "text_delta",
-            "text": " today"
-          }
-        },
+        "raw": null,
         "delta": " today",
         "options": {}
       }
@@ -294,14 +112,7 @@
     {
       "id": "PRESERVE_1",
       "chunk": {
-        "raw": {
-          "type": "content_block_delta",
-          "index": 0,
-          "delta": {
-            "type": "text_delta",
-            "text": "?"
-          }
-        },
+        "raw": null,
         "delta": "?",
         "options": {}
       }

packages/core/e2e/node/snapshot/openai_agent_system_prompt.snap

@@ -28,21 +28,23 @@
           "content": "",
           "role": "assistant",
           "options": {
-            "toolCall": {
-              "id": "call_loaXnCH3WRnbnROZzNbrRkHS",
-              "name": "getWeather",
-              "input": "{\"city\":\"San Francisco\"}"
-            }
+            "toolCall": [
+              {
+                "id": "call_1SlugFuJ7rhwsmXd5aRnJhPe",
+                "name": "getWeather",
+                "input": "{\"city\":\"San Francisco\"}"
+              }
+            ]
           }
         },
         {
-          "content": "The weather in San Francisco is 72 degrees",
           "role": "user",
+          "content": "The weather in San Francisco is 72 degrees",
           "options": {
             "toolResult": {
               "result": "The weather in San Francisco is 72 degrees",
               "isError": false,
-              "id": "call_loaXnCH3WRnbnROZzNbrRkHS"
+              "id": "call_1SlugFuJ7rhwsmXd5aRnJhPe"
             }
           }
         }
@@ -58,11 +60,13 @@
           "content": "",
           "role": "assistant",
           "options": {
-            "toolCall": {
-              "id": "call_loaXnCH3WRnbnROZzNbrRkHS",
-              "name": "getWeather",
-              "input": "{\"city\":\"San Francisco\"}"
-            }
+            "toolCall": [
+              {
+                "id": "call_1SlugFuJ7rhwsmXd5aRnJhPe",
+                "name": "getWeather",
+                "input": "{\"city\":\"San Francisco\"}"
+              }
+            ]
           }
         }
       }
@@ -72,7 +76,7 @@
       "response": {
         "raw": null,
         "message": {
-          "content": "Arhgs! The weather in San Francisco is 72 degrees.",
+          "content": "Arhgs the weather in San Francisco be 72 degrees.",
           "role": "assistant",
           "options": {}
         }

packages/core/e2e/node/snapshot/react-agent-stream.snap

@@ -41,7 +41,7 @@
       "response": {
         "raw": null,
         "message": {
-          "content": "Thought: I need to use a tool to help me answer the question.\nAction: getWeather\nAction Input: {\"city\": \"San Francisco\"}",
+          "content": "Thought: I need to use a tool to help me answer the question. \nAction: getWeather\nAction Input: {\"city\": \"San Francisco\"}",
           "role": "assistant",
           "options": {}
         }
@@ -177,7 +177,15 @@
       "chunk": {
         "raw": null,
         "options": {},
-        "delta": ".\n"
+        "delta": "."
+      }
+    },
+    {
+      "id": "PRESERVE_0",
+      "chunk": {
+        "raw": null,
+        "options": {},
+        "delta": " \n"
       }
     },
     {

packages/core/e2e/node/snapshot/react-agent.snap

@@ -41,7 +41,7 @@
       "response": {
         "raw": null,
         "message": {
-          "content": "Thought: I need to use a tool to help me answer the question.\nAction: getWeather\nAction Input: {\"city\": \"San Francisco\"}",
+          "content": "Thought: I need to use a tool to help me answer the question. \nAction: getWeather\nAction Input: {\"city\": \"San Francisco\"}",
           "role": "assistant",
           "options": {}
         }

packages/core/jsr.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/core",
-  "version": "0.3.15",
+  "version": "0.3.16",
   "exports": "./src/index.ts",
   "imports": {
     "@llamaindex/env": "jsr:@llamaindex/env@0.1.3"

packages/core/package.json

@@ -1,6 +1,6 @@
 {
   "name": "llamaindex",
-  "version": "0.3.15",
+  "version": "0.3.16",
   "expectedMinorVersion": "3",
   "license": "MIT",
   "type": "module",
@@ -21,45 +21,45 @@
     "llamaindex"
   ],
   "dependencies": {
-    "@anthropic-ai/sdk": "^0.20.9",
+    "@anthropic-ai/sdk": "^0.21.1",
     "@aws-crypto/sha256-js": "^5.2.0",
-    "@datastax/astra-db-ts": "^1.1.0",
-    "@google-cloud/vertexai": "^1.1.0",
-    "@google/generative-ai": "^0.11.0",
-    "@grpc/grpc-js": "^1.10.7",
-    "@huggingface/inference": "^2.6.7",
+    "@datastax/astra-db-ts": "^1.2.1",
+    "@google-cloud/vertexai": "^1.2.0",
+    "@google/generative-ai": "^0.12.0",
+    "@grpc/grpc-js": "^1.10.8",
+    "@huggingface/inference": "^2.7.0",
     "@llamaindex/cloud": "0.0.5",
     "@llamaindex/env": "workspace:*",
-    "@mistralai/mistralai": "^0.2.0",
-    "@pinecone-database/pinecone": "^2.2.0",
+    "@mistralai/mistralai": "^0.4.0",
+    "@pinecone-database/pinecone": "^2.2.2",
     "@qdrant/js-client-rest": "^1.9.0",
-    "@types/lodash": "^4.17.1",
+    "@types/lodash": "^4.17.4",
     "@types/papaparse": "^5.3.14",
     "@types/pg": "^8.11.6",
-    "@xenova/transformers": "^2.17.1",
+    "@xenova/transformers": "^2.17.2",
     "@zilliz/milvus2-sdk-node": "^2.4.2",
-    "ajv": "^8.13.0",
-    "assemblyai": "^4.4.2",
-    "chromadb": "~1.7.3",
-    "cohere-ai": "^7.9.5",
-    "js-tiktoken": "^1.0.11",
+    "ajv": "^8.16.0",
+    "assemblyai": "^4.4.5",
+    "chromadb": "1.8.1",
+    "cohere-ai": "7.9.5",
+    "js-tiktoken": "^1.0.12",
     "lodash": "^4.17.21",
     "magic-bytes.js": "^1.10.0",
     "mammoth": "^1.7.2",
     "md-utils-ts": "^2.0.0",
-    "mongodb": "^6.6.1",
+    "mongodb": "^6.7.0",
     "notion-md-crawler": "^1.0.0",
-    "openai": "^4.46.0",
+    "openai": "^4.48.1",
     "papaparse": "^5.4.1",
     "pathe": "^1.1.2",
-    "pdf2json": "3.0.5",
-    "pg": "^8.11.5",
+    "pdf2json": "3.1.3",
+    "pg": "^8.12.0",
     "pgvector": "^0.1.8",
     "portkey-ai": "^0.1.16",
     "rake-modified": "^1.0.8",
     "string-strip-html": "^13.4.8",
     "wikipedia": "^2.1.2",
-    "wink-nlp": "^2.2.2"
+    "wink-nlp": "^2.3.0"
   },
   "peerDependencies": {
     "@notionhq/client": "^2.2.15"
@@ -67,9 +67,9 @@
   "devDependencies": {
     "@notionhq/client": "^2.2.15",
     "@swc/cli": "^0.3.12",
-    "@swc/core": "^1.5.5",
+    "@swc/core": "^1.5.24",
     "concurrently": "^8.2.2",
-    "glob": "^10.3.12",
+    "glob": "^10.4.1",
     "typescript": "^5.4.5"
   },
   "engines": {

packages/core/src/agent/anthropic.ts

@@ -93,29 +93,31 @@ export class AnthropicAgent extends AgentRunner<Anthropic> {
     });
     if ("toolCall" in options) {
       const { toolCall } = options;
-      const targetTool = tools.find(
-        (tool) => tool.metadata.name === toolCall.name,
-      );
-      const toolOutput = await callTool(
-        targetTool,
-        toolCall,
-        step.context.logger,
-      );
-      step.context.store.toolOutputs.push(toolOutput);
-      step.context.store.messages = [
-        ...step.context.store.messages,
-        {
-          content: stringifyJSONToMessageContent(toolOutput.output),
-          role: "user",
-          options: {
-            toolResult: {
-              result: toolOutput.output,
-              isError: toolOutput.isError,
-              id: toolCall.id,
+      for (const call of toolCall) {
+        const targetTool = tools.find(
+          (tool) => tool.metadata.name === call.name,
+        );
+        const toolOutput = await callTool(
+          targetTool,
+          call,
+          step.context.logger,
+        );
+        step.context.store.toolOutputs.push(toolOutput);
+        step.context.store.messages = [
+          ...step.context.store.messages,
+          {
+            content: stringifyJSONToMessageContent(toolOutput.output),
+            role: "user",
+            options: {
+              toolResult: {
+                result: toolOutput.output,
+                isError: toolOutput.isError,
+                id: call.id,
+              },
             },
           },
-        },
-      ];
+        ];
+      }
     }
   };
 }

packages/core/src/agent/openai.ts

@@ -76,31 +76,34 @@ export class OpenAIAgent extends AgentRunner<OpenAI> {
       });
       if ("toolCall" in options) {
         const { toolCall } = options;
-        const targetTool = tools.find(
-          (tool) => tool.metadata.name === toolCall.name,
-        );
-        const toolOutput = await callTool(
-          targetTool,
-          toolCall,
-          step.context.logger,
-        );
-        step.context.store.toolOutputs.push(toolOutput);
-        step.context.store.messages = [
-          ...step.context.store.messages,
-          {
-            role: "user" as const,
-            content: stringifyJSONToMessageContent(toolOutput.output),
-            options: {
-              toolResult: {
-                result: toolOutput.output,
-                isError: toolOutput.isError,
-                id: toolCall.id,
+        for (const call of toolCall) {
+          const targetTool = tools.find(
+            (tool) => tool.metadata.name === call.name,
+          );
+          const toolOutput = await callTool(
+            targetTool,
+            call,
+            step.context.logger,
+          );
+          step.context.store.toolOutputs.push(toolOutput);
+          step.context.store.messages = [
+            ...step.context.store.messages,
+            {
+              role: "user" as const,
+              content: stringifyJSONToMessageContent(toolOutput.output),
+              options: {
+                toolResult: {
+                  result: toolOutput.output,
+                  isError: toolOutput.isError,
+                  id: call.id,
+                },
               },
             },
-          },
-        ];
+          ];
+        }
       }
     } else {
+      debugger;
       const responseChunkStream = new ReadableStream<
         ChatResponseChunk<ToolCallLLMMessageOptions>
       >({
@@ -135,23 +138,25 @@ export class OpenAIAgent extends AgentRunner<OpenAI> {
         for await (const chunk of pipStream) {
           if (chunk.options && "toolCall" in chunk.options) {
             const toolCall = chunk.options.toolCall;
-            toolCalls.set(toolCall.id, toolCall);
+            toolCall.forEach((toolCall) => {
+              toolCalls.set(toolCall.id, toolCall);
+            });
           }
         }
+        step.context.store.messages = [
+          ...step.context.store.messages,
+          {
+            role: "assistant" as const,
+            content: "",
+            options: {
+              toolCall: [...toolCalls.values()],
+            },
+          },
+        ];
         for (const toolCall of toolCalls.values()) {
           const targetTool = tools.find(
             (tool) => tool.metadata.name === toolCall.name,
           );
-          step.context.store.messages = [
-            ...step.context.store.messages,
-            {
-              role: "assistant" as const,
-              content: "",
-              options: {
-                toolCall,
-              },
-            },
-          ];
           const toolOutput = await callTool(
             targetTool,
             toolCall,

packages/core/src/agent/utils.ts

@@ -16,10 +16,25 @@ export async function callTool(
   toolCall: ToolCall | PartialToolCall,
   logger: Logger,
 ): Promise<ToolOutput> {
-  const input: JSONObject =
-    typeof toolCall.input === "string"
-      ? JSON.parse(toolCall.input)
-      : toolCall.input;
+  let input: JSONObject;
+  if (typeof toolCall.input === "string") {
+    try {
+      input = JSON.parse(toolCall.input);
+    } catch (e) {
+      const output = `Tool ${toolCall.name} can't be called. Input is not a valid JSON object.`;
+      logger.error(
+        `${output} Try increasing the maxTokens parameter of your LLM. Invalid Input: ${toolCall.input}`,
+      );
+      return {
+        tool,
+        input: {},
+        output,
+        isError: true,
+      };
+    }
+  } else {
+    input = toolCall.input;
+  }
   if (!tool) {
     logger.error(`Tool ${toolCall.name} does not exist.`);
     const output = `Tool ${toolCall.name} does not exist.`;

packages/core/src/embeddings/DeepInfraEmbedding.ts

@@ -0,0 +1,152 @@
+import { getEnv } from "@llamaindex/env";
+import type { MessageContentDetail } from "../llm/index.js";
+import { extractSingleText } from "../llm/utils.js";
+import { BaseEmbedding } from "./types.js";
+
+const DEFAULT_MODEL = "sentence-transformers/clip-ViT-B-32";
+
+const API_TOKEN_ENV_VARIABLE_NAME = "DEEPINFRA_API_TOKEN";
+
+const API_ROOT = "https://api.deepinfra.com/v1/inference";
+
+const DEFAULT_TIMEOUT = 60 * 1000;
+
+const DEFAULT_MAX_RETRIES = 5;
+
+export interface DeepInfraEmbeddingResponse {
+  embeddings: number[][];
+  request_id: string;
+  inference_status: InferenceStatus;
+}
+
+export interface InferenceStatus {
+  status: string;
+  runtime_ms: number;
+  cost: number;
+  tokens_input: number;
+}
+
+const mapPrefixWithInputs = (prefix: string, inputs: string[]): string[] => {
+  return inputs.map((input) => (prefix ? `${prefix} ${input}` : input));
+};
+
+/**
+ * DeepInfraEmbedding is an alias for DeepInfra that implements the BaseEmbedding interface.
+ */
+export class DeepInfraEmbedding extends BaseEmbedding {
+  /**
+   * DeepInfra model to use
+   * @default "sentence-transformers/clip-ViT-B-32"
+   * @see https://deepinfra.com/models/embeddings
+   */
+  model: string;
+
+  /**
+   * DeepInfra API token
+   * @see https://deepinfra.com/dash/api_keys
+   * If not provided, it will try to get the token from the environment variable `DEEPINFRA_API_TOKEN`
+   *
+   */
+  apiToken: string;
+
+  /**
+   * Prefix to add to the query
+   * @default ""
+   */
+  queryPrefix: string;
+
+  /**
+   * Prefix to add to the text
+   * @default ""
+   */
+  textPrefix: string;
+
+  /**
+   *
+   * @default 5
+   */
+  maxRetries: number;
+
+  /**
+   *
+   * @default 60 * 1000
+   */
+  timeout: number;
+
+  constructor(init?: Partial<DeepInfraEmbedding>) {
+    super();
+
+    this.model = init?.model ?? DEFAULT_MODEL;
+    this.apiToken = init?.apiToken ?? getEnv(API_TOKEN_ENV_VARIABLE_NAME) ?? "";
+    this.queryPrefix = init?.queryPrefix ?? "";
+    this.textPrefix = init?.textPrefix ?? "";
+    this.maxRetries = init?.maxRetries ?? DEFAULT_MAX_RETRIES;
+    this.timeout = init?.timeout ?? DEFAULT_TIMEOUT;
+  }
+
+  async getTextEmbedding(text: string): Promise<number[]> {
+    const texts = mapPrefixWithInputs(this.textPrefix, [text]);
+    const embeddings = await this.getDeepInfraEmbedding(texts);
+    return embeddings[0];
+  }
+
+  async getQueryEmbedding(
+    query: MessageContentDetail,
+  ): Promise<number[] | null> {
+    const text = extractSingleText(query);
+    if (text) {
+      const queries = mapPrefixWithInputs(this.queryPrefix, [text]);
+      const embeddings = await this.getDeepInfraEmbedding(queries);
+      return embeddings[0];
+    } else {
+      return null;
+    }
+  }
+
+  async getTextEmbeddings(texts: string[]): Promise<number[][]> {
+    const textsWithPrefix = mapPrefixWithInputs(this.textPrefix, texts);
+    return await this.getDeepInfraEmbedding(textsWithPrefix);
+  }
+
+  async getQueryEmbeddings(queries: string[]): Promise<number[][]> {
+    const queriesWithPrefix = mapPrefixWithInputs(this.queryPrefix, queries);
+    return await this.getDeepInfraEmbedding(queriesWithPrefix);
+  }
+
+  private async getDeepInfraEmbedding(inputs: string[]): Promise<number[][]> {
+    const url = this.getUrl(this.model);
+
+    for (let attempt = 0; attempt < this.maxRetries; attempt++) {
+      const controller = new AbortController();
+      const id = setTimeout(() => controller.abort(), this.timeout);
+
+      try {
+        const response = await fetch(url, {
+          method: "POST",
+          headers: {
+            "Content-Type": "application/json",
+            Authorization: `Bearer ${this.apiToken}`,
+          },
+          body: JSON.stringify({ inputs }),
+          signal: controller.signal,
+        });
+        if (!response.ok) {
+          throw new Error(`Request failed with status ${response.status}`);
+        }
+
+        const responseJson: DeepInfraEmbeddingResponse = await response.json();
+        return responseJson.embeddings;
+      } catch (error) {
+        console.error(`Attempt ${attempt + 1} failed: ${error}`);
+      } finally {
+        clearTimeout(id);
+      }
+    }
+
+    throw new Error("Exceeded maximum retries");
+  }
+
+  private getUrl(model: string): string {
+    return `${API_ROOT}/${model}`;
+  }
+}

packages/core/src/embeddings/index.ts

@@ -1,3 +1,4 @@
+export { DeepInfraEmbedding } from "./DeepInfraEmbedding.js";
 export * from "./GeminiEmbedding.js";
 export * from "./JinaAIEmbedding.js";
 export * from "./MistralAIEmbedding.js";

packages/core/src/indices/BaseIndex.ts

@@ -95,7 +95,7 @@ export abstract class BaseIndex<T> {
       [nodeParserFromSettingsOrContext(this.serviceContext)],
     );
     await this.insertNodes(nodes);
-    this.docStore.setDocumentHash(document.id_, document.hash);
+    await this.docStore.setDocumentHash(document.id_, document.hash);
   }
 
   abstract insertNodes(nodes: BaseNode[]): Promise<void>;

packages/core/src/indices/keyword/index.ts

@@ -281,7 +281,7 @@ export class KeywordTableIndex extends BaseIndex<KeywordTable> {
 
     await docStore.addDocuments(documents, true);
     for (const doc of documents) {
-      docStore.setDocumentHash(doc.id_, doc.hash);
+      await docStore.setDocumentHash(doc.id_, doc.hash);
     }
 
     const nodes = serviceContext.nodeParser.getNodesFromDocuments(documents);

packages/core/src/indices/summary/index.ts

@@ -138,7 +138,7 @@ export class SummaryIndex extends BaseIndex<IndexList> {
 
     await docStore.addDocuments(documents, true);
     for (const doc of documents) {
-      docStore.setDocumentHash(doc.id_, doc.hash);
+      await docStore.setDocumentHash(doc.id_, doc.hash);
     }
 
     const nodes =

packages/core/src/ingestion/strategies/DuplicatesStrategy.ts

@@ -19,7 +19,7 @@ export class DuplicatesStrategy implements TransformComponent {
 
     for (const node of nodes) {
       if (!(node.hash in hashes) && !currentHashes.has(node.hash)) {
-        this.docStore.setDocumentHash(node.id_, node.hash);
+        await this.docStore.setDocumentHash(node.id_, node.hash);
         nodesToRun.push(node);
         currentHashes.add(node.hash);
       }

packages/core/src/llm/anthropic.ts

@@ -194,12 +194,15 @@ export class Anthropic extends ToolCallLLM<AnthropicAdditionalChatOptions> {
                     } satisfies TextBlockParam,
                   ]
                 : []),
-              {
-                type: "tool_use",
-                id: options.toolCall.id,
-                name: options.toolCall.name,
-                input: options.toolCall.input,
-              } satisfies ToolUseBlockParam,
+              ...options.toolCall.map(
+                (toolCall) =>
+                  ({
+                    type: "tool_use",
+                    id: toolCall.id,
+                    name: toolCall.name,
+                    input: toolCall.input,
+                  }) satisfies ToolUseBlockParam,
+              ),
             ] satisfies ToolsBetaContentBlock[],
           } satisfies ToolsBetaMessageParam;
         }
@@ -326,7 +329,7 @@ export class Anthropic extends ToolCallLLM<AnthropicAdditionalChatOptions> {
       }
       const response = await anthropic.beta.tools.messages.create(params);
 
-      const toolUseBlock = response.content.find(
+      const toolUseBlock = response.content.filter(
         (content): content is ToolUseBlock => content.type === "tool_use",
       );
 
@@ -340,15 +343,16 @@ export class Anthropic extends ToolCallLLM<AnthropicAdditionalChatOptions> {
               text: content.text,
             })),
           role: "assistant",
-          options: toolUseBlock
-            ? {
-                toolCall: {
-                  id: toolUseBlock.id,
-                  name: toolUseBlock.name,
-                  input: toolUseBlock.input,
-                },
-              }
-            : {},
+          options:
+            toolUseBlock.length > 0
+              ? {
+                  toolCall: toolUseBlock.map((block) => ({
+                    id: block.id,
+                    name: block.name,
+                    input: block.input,
+                  })),
+                }
+              : {},
         },
       };
     } else {

packages/core/src/llm/deepinfra.ts

@@ -0,0 +1,33 @@
+import { getEnv } from "@llamaindex/env";
+import { OpenAI } from "./openai.js";
+
+const ENV_VARIABLE_NAME = "DEEPINFRA_API_TOKEN";
+const DEFAULT_MODEL = "mistralai/Mixtral-8x22B-Instruct-v0.1";
+const BASE_URL = "https://api.deepinfra.com/v1/openai";
+
+export class DeepInfra extends OpenAI {
+  constructor(init?: Partial<OpenAI>) {
+    const {
+      apiKey = getEnv(ENV_VARIABLE_NAME),
+      additionalSessionOptions = {},
+      model = DEFAULT_MODEL,
+      ...rest
+    } = init ?? {};
+
+    if (!apiKey) {
+      throw new Error(
+        `Set DeepInfra API key in ${ENV_VARIABLE_NAME} env variable`,
+      );
+    }
+
+    additionalSessionOptions.baseURL =
+      additionalSessionOptions.baseURL ?? BASE_URL;
+
+    super({
+      apiKey,
+      additionalSessionOptions,
+      model,
+      ...rest,
+    });
+  }
+}

packages/core/src/llm/gemini/base.ts

@@ -38,6 +38,8 @@ export const GEMINI_MODEL_INFO_MAP: Record<GEMINI_MODEL, GeminiModelInfo> = {
   // multi-modal/multi turn
   [GEMINI_MODEL.GEMINI_PRO_1_5_PRO_PREVIEW]: { contextWindow: 10 ** 6 },
   [GEMINI_MODEL.GEMINI_PRO_1_5_FLASH_PREVIEW]: { contextWindow: 10 ** 6 },
+  [GEMINI_MODEL.GEMINI_PRO_1_5]: { contextWindow: 2 * 10 ** 6 },
+  [GEMINI_MODEL.GEMINI_PRO_1_5_FLASH]: { contextWindow: 10 ** 6 },
 };
 
 const SUPPORT_TOOL_CALL_MODELS: GEMINI_MODEL[] = [
@@ -45,6 +47,8 @@ const SUPPORT_TOOL_CALL_MODELS: GEMINI_MODEL[] = [
   GEMINI_MODEL.GEMINI_PRO_VISION,
   GEMINI_MODEL.GEMINI_PRO_1_5_PRO_PREVIEW,
   GEMINI_MODEL.GEMINI_PRO_1_5_FLASH_PREVIEW,
+  GEMINI_MODEL.GEMINI_PRO_1_5,
+  GEMINI_MODEL.GEMINI_PRO_1_5_FLASH,
 ];
 
 const DEFAULT_GEMINI_PARAMS = {

packages/core/src/llm/gemini/types.ts

@@ -53,6 +53,8 @@ export enum GEMINI_MODEL {
   GEMINI_PRO_LATEST = "gemini-1.5-pro-latest",
   GEMINI_PRO_1_5_PRO_PREVIEW = "gemini-1.5-pro-preview-0514",
   GEMINI_PRO_1_5_FLASH_PREVIEW = "gemini-1.5-flash-preview-0514",
+  GEMINI_PRO_1_5 = "gemini-1.5-pro-001",
+  GEMINI_PRO_1_5_FLASH = "gemini-1.5-flash-001",
 }
 
 export interface GeminiModelInfo {

packages/core/src/llm/huggingface.ts

@@ -1,7 +1,4 @@
-import {
-  HfInference,
-  type Options as HfInferenceOptions,
-} from "@huggingface/inference";
+import { HfInference } from "@huggingface/inference";
 import type {
   PreTrainedModel,
   PreTrainedTokenizer,
@@ -20,12 +17,49 @@ import type {
 } from "./types.js";
 import { streamConverter, wrapLLMEvent } from "./utils.js";
 
+// TODO workaround issue with @huggingface/inference@2.7.0
+interface HfInferenceOptions {
+  /**
+   * (Default: true) Boolean. If a request 503s and wait_for_model is set to false, the request will be retried with the same parameters but with wait_for_model set to true.
+   */
+  retry_on_error?: boolean;
+  /**
+   * (Default: true). Boolean. There is a cache layer on Inference API (serverless) to speedup requests we have already seen. Most models can use those results as is as models are deterministic (meaning the results will be the same anyway). However if you use a non deterministic model, you can set this parameter to prevent the caching mechanism from being used resulting in a real new query.
+   */
+  use_cache?: boolean;
+  /**
+   * (Default: false). Boolean. Do not load the model if it's not already available.
+   */
+  dont_load_model?: boolean;
+  /**
+   * (Default: false). Boolean to use GPU instead of CPU for inference (requires Startup plan at least).
+   */
+  use_gpu?: boolean;
+  /**
+   * (Default: false) Boolean. If the model is not ready, wait for it instead of receiving 503. It limits the number of requests required to get your inference done. It is advised to only set this flag to true after receiving a 503 error as it will limit hanging in your application to known places.
+   */
+  wait_for_model?: boolean;
+  /**
+   * Custom fetch function to use instead of the default one, for example to use a proxy or edit headers.
+   */
+  fetch?: typeof fetch;
+  /**
+   * Abort Controller signal to use for request interruption.
+   */
+  signal?: AbortSignal;
+  /**
+   * (Default: "same-origin"). String | Boolean. Credentials to use for the request. If this is a string, it will be passed straight on. If it's a boolean, true will be "include" and false will not send credentials at all.
+   */
+  includeCredentials?: string | boolean;
+}
+
 const DEFAULT_PARAMS = {
   temperature: 0.1,
   topP: 1,
   maxTokens: undefined,
   contextWindow: 3900,
 };
+
 export type HFConfig = Partial<typeof DEFAULT_PARAMS> &
   HfInferenceOptions & {
     model: string;
@@ -139,7 +173,7 @@ export class HuggingFaceInferenceAPI extends BaseLLM {
       inputs: this.messagesToPrompt(params.messages),
       parameters: this.metadata,
     });
-    yield* streamConverter(stream, (chunk) => ({
+    yield* streamConverter(stream, (chunk: any) => ({
       delta: chunk.token.text,
       raw: chunk,
     }));

packages/core/src/llm/index.ts

@@ -4,8 +4,10 @@ export {
   ALL_AVAILABLE_V3_MODELS,
   Anthropic,
 } from "./anthropic.js";
+export { ToolCallLLM } from "./base.js";
 export { FireworksLLM } from "./fireworks.js";
 export { Gemini, GeminiSession } from "./gemini/base.js";
+export { streamConverter, wrapLLMEvent } from "./utils.js";
 
 export {
   GEMINI_MODEL,
@@ -23,6 +25,7 @@ export * from "./openai.js";
 export { Portkey } from "./portkey.js";
 export * from "./replicate_ai.js";
 // Note: The type aliases for replicate are to simplify usage for Llama 2 (we're using replicate for Llama 2 support)
+export { DeepInfra } from "./deepinfra.js";
 export { Ollama, type OllamaParams } from "./ollama.js";
 export {
   ALL_AVAILABLE_REPLICATE_MODELS,
@@ -32,5 +35,6 @@ export {
   ReplicateLLM,
   ReplicateSession,
 } from "./replicate_ai.js";
+
 export { TogetherLLM } from "./together.js";
 export * from "./types.js";

packages/core/src/llm/openai.ts

@@ -110,7 +110,6 @@ export const GPT4_MODELS = {
   "gpt-4-1106-preview": { contextWindow: 128000 },
   "gpt-4-0125-preview": { contextWindow: 128000 },
   "gpt-4-vision-preview": { contextWindow: 128000 },
-  // fixme: wait for openai documentation
   "gpt-4o": { contextWindow: 128000 },
   "gpt-4o-2024-05-13": { contextWindow: 128000 },
 };
@@ -185,7 +184,7 @@ export class OpenAI extends ToolCallLLM<OpenAIAdditionalChatOptions> {
     },
   ) {
     super();
-    this.model = init?.model ?? "gpt-3.5-turbo";
+    this.model = init?.model ?? "gpt-4o";
     this.temperature = init?.temperature ?? 0.1;
     this.topP = init?.topP ?? 1;
     this.maxTokens = init?.maxTokens ?? undefined;
@@ -279,19 +278,19 @@ export class OpenAI extends ToolCallLLM<OpenAIAdditionalChatOptions> {
         return {
           role: "assistant",
           content: extractText(message.content),
-          tool_calls: [
-            {
-              id: options.toolCall.id,
+          tool_calls: options.toolCall.map((toolCall) => {
+            return {
+              id: toolCall.id,
               type: "function",
               function: {
-                name: options.toolCall.name,
+                name: toolCall.name,
                 arguments:
-                  typeof options.toolCall.input === "string"
-                    ? options.toolCall.input
-                    : JSON.stringify(options.toolCall.input),
+                  typeof toolCall.input === "string"
+                    ? toolCall.input
+                    : JSON.stringify(toolCall.input),
               },
-            },
-          ],
+            };
+          }),
         } satisfies ChatCompletionAssistantMessageParam;
       } else if (message.role === "user") {
         return {
@@ -380,12 +379,13 @@ export class OpenAI extends ToolCallLLM<OpenAIAdditionalChatOptions> {
         role: response.choices[0].message.role,
         options: response.choices[0].message?.tool_calls
           ? {
-              toolCall: {
-                id: response.choices[0].message.tool_calls[0].id,
-                name: response.choices[0].message.tool_calls[0].function.name,
-                input:
-                  response.choices[0].message.tool_calls[0].function.arguments,
-              },
+              toolCall: response.choices[0].message.tool_calls.map(
+                (toolCall) => ({
+                  id: toolCall.id,
+                  name: toolCall.function.name,
+                  input: toolCall.function.arguments,
+                }),
+              ),
             }
           : {},
       },
@@ -459,10 +459,10 @@ export class OpenAI extends ToolCallLLM<OpenAIAdditionalChatOptions> {
       yield {
         raw: part,
         options: shouldEmitToolCall
-          ? { toolCall: shouldEmitToolCall }
+          ? { toolCall: [shouldEmitToolCall] }
           : currentToolCall
             ? {
-                toolCall: currentToolCall,
+                toolCall: [currentToolCall],
               }
             : {},
         delta: choice.delta.content ?? "",

packages/core/src/llm/types.ts

@@ -212,7 +212,7 @@ export type ToolResult = {
 };
 
 export type ToolCallOptions = {
-  toolCall: ToolCall | PartialToolCall;
+  toolCall: (ToolCall | PartialToolCall)[];
 };
 
 export type ToolResultOptions = {

packages/core/src/readers/CSVReader.ts

@@ -1,15 +1,14 @@
-import { fs } from "@llamaindex/env";
 import type { ParseConfig } from "papaparse";
 import Papa from "papaparse";
 import { Document } from "../Node.js";
-import type { FileReader } from "./type.js";
+import { FileReader } from "./type.js";
 
 /**
  * papaparse-based csv parser
  * @class CSVReader
  * @implements BaseReader
  */
-export class PapaCSVReader implements FileReader {
+export class PapaCSVReader extends FileReader {
   private concatRows: boolean;
   private colJoiner: string;
   private rowJoiner: string;
@@ -27,6 +26,7 @@ export class PapaCSVReader implements FileReader {
     rowJoiner: string = "\n",
     papaConfig?: ParseConfig,
   ) {
+    super();
     this.concatRows = concatRows;
     this.colJoiner = colJoiner;
     this.rowJoiner = rowJoiner;
@@ -39,9 +39,8 @@ export class PapaCSVReader implements FileReader {
    * @param {GenericFileSystem} [fs=DEFAULT_FS] - The file system to use for reading the file.
    * @returns {Promise<Document[]>}
    */
-  async loadData(file: string): Promise<Document[]> {
-    const fileContent = await fs.readFile(file, "utf-8");
-    const result = Papa.parse(fileContent, this.papaConfig);
+  async loadDataAsContent(fileContent: Buffer): Promise<Document[]> {
+    const result = Papa.parse(fileContent.toString("utf-8"), this.papaConfig);
     const textList = result.data.map((row: any) => {
       // Compatible with header row mode
       const rowValues = Object.values(row).map((value) => String(value));

packages/core/src/readers/DocxReader.ts

@@ -1,13 +1,11 @@
-import { fs } from "@llamaindex/env";
 import mammoth from "mammoth";
 import { Document } from "../Node.js";
-import type { FileReader } from "./type.js";
+import { FileReader } from "./type.js";
 
-export class DocxReader implements FileReader {
+export class DocxReader extends FileReader {
   /** DocxParser */
-  async loadData(file: string): Promise<Document[]> {
-    const dataBuffer = await fs.readFile(file);
-    const { value } = await mammoth.extractRawText({ buffer: dataBuffer });
-    return [new Document({ text: value, id_: file })];
+  async loadDataAsContent(fileContent: Buffer): Promise<Document[]> {
+    const { value } = await mammoth.extractRawText({ buffer: fileContent });
+    return [new Document({ text: value })];
   }
 }

packages/core/src/readers/HTMLReader.ts

@@ -1,6 +1,5 @@
-import { fs } from "@llamaindex/env";
 import { Document } from "../Node.js";
-import type { FileReader } from "./type.js";
+import { FileReader } from "./type.js";
 
 /**
  * Extract the significant text from an arbitrary HTML document.
@@ -9,18 +8,18 @@ import type { FileReader } from "./type.js";
  * All other tags are removed, and the inner text is kept intact.
  * Html entities (e.g., &) are not decoded.
  */
-export class HTMLReader implements FileReader {
+export class HTMLReader extends FileReader {
   /**
    * Public method for this reader.
    * Required by BaseReader interface.
    * @param file Path/name of the file to be loaded.
    * @returns Promise<Document[]> A Promise object, eventually yielding zero or one Document parsed from the HTML content of the specified file.
    */
-  async loadData(file: string): Promise<Document[]> {
-    const dataBuffer = await fs.readFile(file, "utf-8");
+  async loadDataAsContent(fileContent: Buffer): Promise<Document[]> {
+    const dataBuffer = fileContent.toString("utf-8");
     const htmlOptions = this.getOptions();
     const content = await this.parseContent(dataBuffer, htmlOptions);
-    return [new Document({ text: content, id_: file })];
+    return [new Document({ text: content })];
   }
 
   /**

packages/core/src/readers/ImageReader.ts

@@ -1,12 +1,11 @@
-import { fs } from "@llamaindex/env";
 import type { Document } from "../Node.js";
 import { ImageDocument } from "../Node.js";
-import type { FileReader } from "./type.js";
+import { FileReader } from "./type.js";
 
 /**
  * Reads the content of an image file into a Document object (which stores the image file as a Blob).
  */
-export class ImageReader implements FileReader {
+export class ImageReader extends FileReader {
   /**
    * Public method for this reader.
    * Required by BaseReader interface.
@@ -14,9 +13,8 @@ export class ImageReader implements FileReader {
    * @param fs fs wrapper interface for getting the file content.
    * @returns Promise<Document[]> A Promise object, eventually yielding zero or one ImageDocument of the specified file.
    */
-  async loadData(file: string): Promise<Document[]> {
-    const dataBuffer = await fs.readFile(file);
-    const blob = new Blob([dataBuffer]);
-    return [new ImageDocument({ image: blob, id_: file })];
+  async loadDataAsContent(fileContent: Buffer): Promise<Document[]> {
+    const blob = new Blob([fileContent]);
+    return [new ImageDocument({ image: blob })];
   }
 }

packages/core/src/readers/LlamaParseReader.ts

@@ -1,57 +1,112 @@
 import { fs, getEnv } from "@llamaindex/env";
 import { filetypemime } from "magic-bytes.js";
 import { Document } from "../Node.js";
-import type { FileReader, Language, ResultType } from "./type.js";
+import { FileReader, type Language, type ResultType } from "./type.js";
 
 const SupportedFiles: { [key: string]: string } = {
   ".pdf": "application/pdf",
+  // Documents and Presentations
+  ".602": "application/x-t602",
+  ".abw": "application/x-abiword",
+  ".cgm": "image/cgm",
+  ".cwk": "application/x-cwk",
   ".doc": "application/msword",
   ".docx":
     "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
   ".docm": "application/vnd.ms-word.document.macroEnabled.12",
   ".dot": "application/msword",
+  ".dotm": "application/vnd.ms-word.template.macroEnabled.12",
   ".dotx":
     "application/vnd.openxmlformats-officedocument.wordprocessingml.template",
-  ".dotm": "application/vnd.ms-word.template.macroEnabled.12",
-  ".rtf": "application/rtf",
-  ".wps": "application/vnd.ms-works",
-  ".wpd": "application/wordperfect",
-  ".sxw": "application/vnd.sun.xml.writer",
-  ".stw": "application/vnd.sun.xml.writer.template",
-  ".sxg": "application/vnd.sun.xml.writer.global",
-  ".pages": "application/x-iwork-pages-sffpages",
+  ".hwp": "application/x-hwp",
+  ".key": "application/x-iwork-keynote-sffkey",
+  ".lwp": "application/vnd.lotus-wordpro",
   ".mw": "application/macwriteii",
   ".mcw": "application/macwriteii",
-  ".uot": "application/x-uo",
-  ".uof": "application/vnd.uoml+xml",
-  ".uos": "application/vnd.sun.xml.calc",
-  ".uop": "application/vnd.openofficeorg.presentation",
+  ".pages": "application/x-iwork-pages-sffpages",
+  ".pbd": "application/x-pagemaker",
   ".ppt": "application/vnd.ms-powerpoint",
+  ".pptm": "application/vnd.ms-powerpoint.presentation.macroEnabled.12",
   ".pptx":
     "application/vnd.openxmlformats-officedocument.presentationml.presentation",
   ".pot": "application/vnd.ms-powerpoint",
-  ".pptm": "application/vnd.ms-powerpoint.presentation.macroEnabled.12",
+  ".potm": "application/vnd.ms-powerpoint.template.macroEnabled.12",
   ".potx":
     "application/vnd.openxmlformats-officedocument.presentationml.template",
-  ".potm": "application/vnd.ms-powerpoint.template.macroEnabled.12",
-  ".key": "application/x-iwork-keynote-sffkey",
-  ".odp": "application/vnd.oasis.opendocument.presentation",
-  ".odg": "application/vnd.oasis.opendocument.graphics",
-  ".otp": "application/vnd.oasis.opendocument.presentation-template",
-  ".fopd": "application/vnd.oasis.opendocument.presentation",
-  ".sxi": "application/vnd.sun.xml.impress",
+  ".rtf": "application/rtf",
+  ".sda": "application/vnd.stardivision.draw",
+  ".sdd": "application/vnd.stardivision.impress",
+  ".sdp": "application/sdp",
+  ".sdw": "application/vnd.stardivision.writer",
+  ".sgl": "application/vnd.stardivision.writer",
   ".sti": "application/vnd.sun.xml.impress.template",
+  ".sxi": "application/vnd.sun.xml.impress",
+  ".sxw": "application/vnd.sun.xml.writer",
+  ".stw": "application/vnd.sun.xml.writer.template",
+  ".sxg": "application/vnd.sun.xml.writer.global",
+  ".txt": "text/plain",
+  ".uof": "application/vnd.uoml+xml",
+  ".uop": "application/vnd.openofficeorg.presentation",
+  ".uot": "application/x-uo",
+  ".vor": "application/vnd.stardivision.writer",
+  ".wpd": "application/wordperfect",
+  ".wps": "application/vnd.ms-works",
+  ".xml": "application/xml",
+  ".zabw": "application/x-abiword",
+  // Images
   ".epub": "application/epub+zip",
-  ".html": "text/html",
+  ".jpg": "image/jpeg",
+  ".jpeg": "image/jpeg",
+  ".png": "image/png",
+  ".gif": "image/gif",
+  ".bmp": "image/bmp",
+  ".svg": "image/svg+xml",
+  ".tiff": "image/tiff",
+  ".webp": "image/webp",
+  // Web
   ".htm": "text/html",
+  ".html": "text/html",
+  // Spreadsheets
+  ".xlsx": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
+  ".xls": "application/vnd.ms-excel",
+  ".xlsm": "application/vnd.ms-excel.sheet.macroEnabled.12",
+  ".xlsb": "application/vnd.ms-excel.sheet.binary.macroEnabled.12",
+  ".xlw": "application/vnd.ms-excel",
+  ".csv": "text/csv",
+  ".dif": "application/x-dif",
+  ".sylk": "text/vnd.sylk",
+  ".slk": "text/vnd.sylk",
+  ".prn": "application/x-prn",
+  ".numbers": "application/x-iwork-numbers-sffnumbers",
+  ".et": "application/vnd.ms-excel",
+  ".ods": "application/vnd.oasis.opendocument.spreadsheet",
+  ".fods": "application/vnd.oasis.opendocument.spreadsheet",
+  ".uos1": "application/vnd.uoml+xml",
+  ".uos2": "application/vnd.uoml+xml",
+  ".dbf": "application/vnd.dbf",
+  ".wk1": "application/vnd.lotus-1-2-3",
+  ".wk2": "application/vnd.lotus-1-2-3",
+  ".wk3": "application/vnd.lotus-1-2-3",
+  ".wk4": "application/vnd.lotus-1-2-3",
+  ".wks": "application/vnd.lotus-1-2-3",
+  ".123": "application/vnd.lotus-1-2-3",
+  ".wq1": "application/x-lotus",
+  ".wq2": "application/x-lotus",
+  ".wb1": "application/x-quattro-pro",
+  ".wb2": "application/x-quattro-pro",
+  ".wb3": "application/x-quattro-pro",
+  ".qpw": "application/x-quattro-pro",
+  ".xlr": "application/vnd.ms-works",
+  ".eth": "application/ethos",
+  ".tsv": "text/tab-separated-values",
 };
 
 /**
  * Represents a reader for parsing files using the LlamaParse API.
  * See https://github.com/run-llama/llama_parse
  */
-export class LlamaParseReader implements FileReader {
-  // The API key for the LlamaParse API.
+export class LlamaParseReader extends FileReader {
+  // The API key for the LlamaParse API. Can be set as an environment variable: LLAMA_CLOUD_API_KEY
   apiKey: string;
   // The base URL of the Llama Parsing API.
   baseUrl: string = "https://api.cloud.llamaindex.ai/api/parsing";
@@ -67,8 +122,18 @@ export class LlamaParseReader implements FileReader {
   language: Language = "en";
   // The parsing instruction for the parser.
   parsingInstruction: string = "";
+  // If set to true, the parser will ignore diagonal text (when the text rotation in degrees modulo 90 is not 0).
+  skipDiagonalText: boolean = false;
+  // If set to true, the cache will be ignored and the document re-processes. All document are kept in cache for 48hours after the job was completed to avoid processing the same document twice.
+  invalidateCache: boolean = false;
+  // Whether to use gpt-4o to extract text from documents.
+  gpt4oMode: boolean = false;
+  // The API key for the GPT-4o API. Optional, lowers the cost of parsing. Can be set as an env variable: LLAMA_CLOUD_GPT4O_API_KEY.
+  gpt4oApiKey?: string;
+  // numWorkers is implemented in SimpleDirectoryReader
 
   constructor(params: Partial<LlamaParseReader> = {}) {
+    super();
     Object.assign(this, params);
     params.apiKey = params.apiKey ?? getEnv("LLAMA_CLOUD_API_KEY");
     if (!params.apiKey) {
@@ -77,19 +142,34 @@ export class LlamaParseReader implements FileReader {
       );
     }
     this.apiKey = params.apiKey;
+
+    if (params.gpt4oMode) {
+      params.gpt4oApiKey =
+        params.gpt4oApiKey ?? getEnv("LLAMA_CLOUD_GPT4O_API_KEY");
+
+      this.gpt4oApiKey = params.gpt4oApiKey;
+    }
   }
 
-  async loadData(file: string): Promise<Document[]> {
-    const metadata = { file_path: file };
-
+  // Create a job for the LlamaParse API
+  private async createJob(data: Buffer): Promise<string> {
     // Load data, set the mime type
-    const data = await fs.readFile(file);
     const mimeType = await this.getMimeType(data);
 
+    if (this.verbose) {
+      console.log(`Starting load for file with mimeType: ${mimeType}`);
+    }
+
     const body = new FormData();
-    body.set("file", new Blob([data], { type: mimeType }), file);
+    body.set("file", new Blob([data], { type: mimeType }));
     body.append("language", this.language);
-    body.append("parsingInstruction", this.parsingInstruction);
+    body.append("parsing_instruction", this.parsingInstruction);
+    body.append("skip_diagonal_text", this.skipDiagonalText.toString());
+    body.append("invalidate_cache", this.invalidateCache.toString());
+    body.append("gpt4o_mode", this.gpt4oMode.toString());
+    if (this.gpt4oMode && this.gpt4oApiKey) {
+      body.append("gpt4o_api_key", this.gpt4oApiKey);
+    }
 
     const headers = {
       Authorization: `Bearer ${this.apiKey}`,
@@ -97,7 +177,7 @@ export class LlamaParseReader implements FileReader {
 
     // Send the request, start job
     const url = `${this.baseUrl}/upload`;
-    let response = await fetch(url, {
+    const response = await fetch(url, {
       signal: AbortSignal.timeout(this.maxTimeout * 1000),
       method: "POST",
       body,
@@ -107,33 +187,29 @@ export class LlamaParseReader implements FileReader {
       throw new Error(`Failed to parse the file: ${await response.text()}`);
     }
     const jsonResponse = await response.json();
+    return jsonResponse.id;
+  }
 
-    // Check the status of the job, return when done
-    const jobId = jsonResponse.id;
-    if (this.verbose) {
-      console.log(`Started parsing the file under job id ${jobId}`);
-    }
+  // Get the result of the job
+  private async getJobResult(jobId: string, resultType: string): Promise<any> {
+    const resultUrl = `${this.baseUrl}/job/${jobId}/result/${resultType}`;
+    const statusUrl = `${this.baseUrl}/job/${jobId}`;
+    const headers = { Authorization: `Bearer ${this.apiKey}` };
 
-    const resultUrl = `${this.baseUrl}/job/${jobId}/result/${this.resultType}`;
-
-    const start = Date.now();
+    const signal = AbortSignal.timeout(this.maxTimeout * 1000);
     let tries = 0;
     while (true) {
       await new Promise((resolve) =>
         setTimeout(resolve, this.checkInterval * 1000),
       );
-      response = await fetch(resultUrl, {
-        headers,
-        signal: AbortSignal.timeout(this.maxTimeout * 1000),
-      });
 
-      if (!response.ok) {
-        const end = Date.now();
-        if (end - start > this.maxTimeout * 1000) {
-          throw new Error(
-            `Timeout while parsing the file: ${await response.text()}`,
-          );
-        }
+      // Check the job status. If unsuccessful response, checks if maximum timeout has been reached. If reached, throws an error
+      const statusResponse = await fetch(statusUrl, {
+        headers,
+        signal,
+      });
+      if (!statusResponse.ok) {
+        signal.throwIfAborted();
         if (this.verbose && tries % 10 === 0) {
           process.stdout.write(".");
         }
@@ -141,16 +217,139 @@ export class LlamaParseReader implements FileReader {
         continue;
       }
 
-      const resultJson = await response.json();
-      return [
-        new Document({
-          text: resultJson[this.resultType],
-          metadata: metadata,
-        }),
-      ];
+      // If response is succesful, check status of job. Allowed values "PENDING", "SUCCESS", "ERROR", "CANCELED"
+      const statusJson = await statusResponse.json();
+      const status = statusJson.status;
+      // If job has completed, return the result
+      if (status === "SUCCESS") {
+        const resultResponse = await fetch(resultUrl, {
+          headers,
+          signal,
+        });
+        if (!resultResponse.ok) {
+          throw new Error(
+            `Failed to fetch result: ${await resultResponse.text()}`,
+          );
+        }
+        return resultResponse.json();
+        // If job is still pending, check if maximum timeout has been reached. If reached, throws an error
+      } else if (status === "PENDING") {
+        signal.throwIfAborted();
+        if (this.verbose && tries % 10 === 0) {
+          process.stdout.write(".");
+        }
+        tries++;
+      } else {
+        throw new Error(
+          `Failed to parse the file: ${jobId}, status: ${status}`,
+        );
+      }
     }
   }
 
+  /**
+   * Loads data from a file and returns an array of Document objects.
+   * To be used with resultType = "text" and "markdown"
+   *
+   * @param {Buffer} fileContent - The content of the file to be loaded.
+   * @return {Promise<Document[]>} A Promise object that resolves to an array of Document objects.
+   */
+  async loadDataAsContent(fileContent: Buffer): Promise<Document[]> {
+    // Creates a job for the file
+    const jobId = await this.createJob(fileContent);
+    if (this.verbose) {
+      console.log(`Started parsing the file under job id ${jobId}`);
+    }
+
+    // Return results as Document objects
+    const resultJson = await this.getJobResult(jobId, this.resultType);
+    return [
+      new Document({
+        text: resultJson[this.resultType],
+      }),
+    ];
+  }
+  /**
+   * Loads data from a file and returns its contents as a JSON object.
+   * To be used with resultType = "json"
+   *
+   * @param {string} file - The path to the file to be loaded.
+   * @return {Promise<Record<string, any>>} A Promise that resolves to the JSON object.
+   */
+  async loadJson(file: string): Promise<Record<string, any>> {
+    const data = await fs.readFile(file);
+    // Creates a job for the file
+    const jobId = await this.createJob(data);
+    if (this.verbose) {
+      console.log(`Started parsing the file under job id ${jobId}`);
+    }
+
+    // Return results as JSON object
+    const resultJson = await this.getJobResult(jobId, "json");
+    resultJson.job_id = jobId;
+    resultJson.file_path = file;
+    return resultJson;
+  }
+
+  /**
+   * Downloads and saves images from a given JSON result to a specified download path.
+   * Currently only supports resultType = "json"
+   *
+   * @param {Record<string, any>[]} jsonResult - The JSON result containing image information.
+   * @param {string} downloadPath - The path to save the downloaded images.
+   * @return {Promise<Record<string, any>[]>} A Promise that resolves to an array of image objects.
+   */
+  async getImages(
+    jsonResult: Record<string, any>[],
+    downloadPath: string,
+  ): Promise<Record<string, any>[]> {
+    const headers = { Authorization: `Bearer ${this.apiKey}` };
+
+    // Create download directory if it doesn't exist (Actually check for write access, not existence, since fsPromises does not have a `existsSync` method)
+    if (!fs.access(downloadPath)) {
+      await fs.mkdir(downloadPath, { recursive: true });
+    }
+
+    const images: Record<string, any>[] = [];
+    for (const result of jsonResult) {
+      const jobId = result.job_id;
+      for (const page of result.pages) {
+        if (this.verbose) {
+          console.log(`> Image for page ${page.page}: ${page.images}`);
+        }
+        for (const image of page.images) {
+          const imageName = image.name;
+          // Get the full path
+          let imagePath = `${downloadPath}/${jobId}-${imageName}`;
+
+          if (!imagePath.endsWith(".png") && !imagePath.endsWith(".jpg")) {
+            imagePath += ".png";
+          }
+
+          // Get a valid image path
+          image.path = imagePath;
+          image.job_id = jobId;
+          image.original_pdf_path = result.file_path;
+          image.page_number = page.page;
+
+          const imageUrl = `${this.baseUrl}/job/${jobId}/result/image/${imageName}`;
+          const response = await fetch(imageUrl, { headers });
+          if (!response.ok) {
+            throw new Error(
+              `Failed to download image: ${await response.text()}`,
+            );
+          }
+          const arrayBuffer = await response.arrayBuffer();
+          const buffer = Buffer.from(arrayBuffer);
+          await fs.writeFile(imagePath, buffer);
+
+          images.push(image);
+        }
+      }
+    }
+    return images;
+  }
+
   private async getMimeType(data: Buffer): Promise<string> {
     const mimes = filetypemime(data);
     const validMime = mimes.find((mime) =>

packages/core/src/readers/MarkdownReader.ts

@@ -1,6 +1,5 @@
-import { fs } from "@llamaindex/env";
 import { Document } from "../Node.js";
-import type { FileReader } from "./type.js";
+import { FileReader } from "./type.js";
 
 type MarkdownTuple = [string | null, string];
 
@@ -8,7 +7,7 @@ type MarkdownTuple = [string | null, string];
  * Extract text from markdown files.
  * Returns dictionary with keys as headers and values as the text between headers.
  */
-export class MarkdownReader implements FileReader {
+export class MarkdownReader extends FileReader {
   private _removeHyperlinks: boolean;
   private _removeImages: boolean;
 
@@ -17,6 +16,7 @@ export class MarkdownReader implements FileReader {
    * @param {boolean} [removeImages=true] - Indicates whether images should be removed.
    */
   constructor(removeHyperlinks: boolean = true, removeImages: boolean = true) {
+    super();
     this._removeHyperlinks = removeHyperlinks;
     this._removeImages = removeImages;
   }
@@ -89,18 +89,17 @@ export class MarkdownReader implements FileReader {
     return this.markdownToTups(modifiedContent);
   }
 
-  async loadData(file: string): Promise<Document[]> {
-    const content = await fs.readFile(file, "utf-8");
+  async loadDataAsContent(fileContent: Buffer): Promise<Document[]> {
+    const content = fileContent.toString("utf-8");
     const tups = this.parseTups(content);
     const results: Document[] = [];
     let counter = 0;
     for (const [header, value] of tups) {
-      const id_ = `${file}_${counter}`;
       if (header) {
         const text = `\n\n${header}\n${value}`;
-        results.push(new Document({ text, id_ }));
+        results.push(new Document({ text }));
       } else {
-        results.push(new Document({ text: value, id_ }));
+        results.push(new Document({ text: value }));
       }
       counter += 1;
     }

packages/core/src/readers/PDFReader.ts

@@ -1,27 +1,24 @@
-import { fs } from "@llamaindex/env";
 import { Document } from "../Node.js";
-import type { BaseReader } from "./type.js";
+import { FileReader } from "./type.js";
 
 /**
  * Read the text of a PDF
  */
-export class PDFReader implements BaseReader {
-  async loadData(file: string): Promise<Document[]> {
-    const content = await fs.readFile(file);
-    const pages = await readPDF(content);
+export class PDFReader extends FileReader {
+  async loadDataAsContent(fileContent: Buffer): Promise<Document[]> {
+    const pages = await readPDF(fileContent);
     return pages.map((text, page) => {
-      const id_ = `${file}_${page + 1}`;
       const metadata = {
         page_number: page + 1,
       };
-      return new Document({ text, id_, metadata });
+      return new Document({ text, metadata });
     });
   }
 }
 
 async function readPDF(data: Buffer): Promise<string[]> {
   const parser = await import("pdf2json").then(
-    ({ default: Pdfparser }) => new Pdfparser(null, 1),
+    ({ default: Pdfparser }) => new Pdfparser(null, true),
   );
   const text = await new Promise<string>((resolve, reject) => {
     parser.on("pdfParser_dataError", (error) => {

packages/core/src/readers/SimpleDirectoryReader.edge.ts

@@ -1,8 +1,9 @@
-import { fs, path } from "@llamaindex/env";
-import { Document, type Metadata } from "../Node.js";
+import { path } from "@llamaindex/env";
+import { Document } from "../Node.js";
 import { walk } from "../storage/FileSystem.js";
 import { TextFileReader } from "./TextFileReader.js";
-import type { BaseReader } from "./type.js";
+import type { BaseReader, FileReader } from "./type.js";
+import pLimit from "./utils.js";
 
 type ReaderCallback = (
   category: "file" | "directory",
@@ -18,10 +19,21 @@ enum ReaderStatus {
 
 export type SimpleDirectoryReaderLoadDataParams = {
   directoryPath: string;
-  defaultReader?: BaseReader | null;
-  fileExtToReader?: Record<string, BaseReader>;
+  // Fallback Reader, defaults to TextFileReader
+  defaultReader?: FileReader | null;
+  // Map file extensions individually to readers
+  fileExtToReader?: Record<string, FileReader>;
+  // Number of workers, defaults to 1. Must be between 1 and 9.
+  numWorkers?: number;
+  // Overrides reader for all file extensions
+  overrideReader?: FileReader;
 };
 
+type ProcessFileParams = Omit<
+  SimpleDirectoryReaderLoadDataParams,
+  "directoryPath"
+>;
+
 /**
  * Read all the documents in a directory.
  * By default, supports the list of file types
@@ -45,8 +57,14 @@ export class SimpleDirectoryReader implements BaseReader {
       directoryPath,
       defaultReader = new TextFileReader(),
       fileExtToReader,
+      numWorkers = 1,
+      overrideReader,
     } = params;
 
+    if (numWorkers < 1 || numWorkers > 9) {
+      throw new Error("The number of workers must be between 1 - 9.");
+    }
+
     // Observer can decide to skip the directory
     if (
       !this.doObserverCheck("directory", directoryPath, ReaderStatus.STARTED)
@@ -54,59 +72,85 @@ export class SimpleDirectoryReader implements BaseReader {
       return [];
     }
 
-    const docs: Document[] = [];
+    // Crates a queue of file paths each worker accesses individually
+    const filePathQueue: string[] = [];
+
     for await (const filePath of walk(directoryPath)) {
-      try {
-        const fileExt = path.extname(filePath).slice(1).toLowerCase();
+      filePathQueue.push(filePath);
+    }
 
-        // Observer can decide to skip each file
-        if (!this.doObserverCheck("file", filePath, ReaderStatus.STARTED)) {
-          // Skip this file
-          continue;
-        }
+    const processFileParams: ProcessFileParams = {
+      defaultReader,
+      fileExtToReader,
+      overrideReader,
+    };
 
-        let reader: BaseReader;
+    // Uses pLimit to control number of parallel requests
+    const limit = pLimit(numWorkers);
+    const workerPromises = filePathQueue.map((filePath) =>
+      limit(() => this.processFile(filePath, processFileParams)),
+    );
 
-        if (fileExtToReader && fileExt in fileExtToReader) {
-          reader = fileExtToReader[fileExt];
-        } else if (defaultReader != null) {
-          reader = defaultReader;
-        } else {
-          const msg = `No reader for file extension of ${filePath}`;
-          console.warn(msg);
+    const results: Document[][] = await Promise.all(workerPromises);
 
-          // In an error condition, observer's false cancels the whole process.
-          if (
-            !this.doObserverCheck("file", filePath, ReaderStatus.ERROR, msg)
-          ) {
-            return [];
-          }
+    // After successful import of all files, directory completion
+    // is only a notification for observer, cannot be cancelled.
+    this.doObserverCheck("directory", directoryPath, ReaderStatus.COMPLETE);
 
-          continue;
-        }
+    return results.flat();
+  }
 
-        const fileDocs = await reader.loadData(filePath, fs);
-        fileDocs.forEach(addMetaData(filePath));
+  private async processFile(
+    filePath: string,
+    params: ProcessFileParams,
+  ): Promise<Document[]> {
+    const docs: Document[] = [];
 
-        // Observer can still cancel addition of the resulting docs from this file
-        if (this.doObserverCheck("file", filePath, ReaderStatus.COMPLETE)) {
-          docs.push(...fileDocs);
-        }
-      } catch (e) {
-        const msg = `Error reading file ${filePath}: ${e}`;
-        console.error(msg);
+    try {
+      const fileExt = path.extname(filePath).slice(1).toLowerCase();
+
+      // Observer can decide to skip each file
+      if (!this.doObserverCheck("file", filePath, ReaderStatus.STARTED)) {
+        // Skip this file
+        return [];
+      }
+
+      let reader: FileReader;
+
+      if (params.overrideReader) {
+        reader = params.overrideReader;
+      } else if (params.fileExtToReader && fileExt in params.fileExtToReader) {
+        reader = params.fileExtToReader[fileExt];
+      } else if (params.defaultReader != null) {
+        reader = params.defaultReader;
+      } else {
+        const msg = `No reader for file extension of ${filePath}`;
+        console.warn(msg);
 
         // In an error condition, observer's false cancels the whole process.
         if (!this.doObserverCheck("file", filePath, ReaderStatus.ERROR, msg)) {
           return [];
         }
+
+        return [];
+      }
+
+      const fileDocs = await reader.loadData(filePath);
+
+      // Observer can still cancel addition of the resulting docs from this file
+      if (this.doObserverCheck("file", filePath, ReaderStatus.COMPLETE)) {
+        docs.push(...fileDocs);
+      }
+    } catch (e) {
+      const msg = `Error reading file ${filePath}: ${e}`;
+      console.error(msg);
+
+      // In an error condition, observer's false cancels the whole process.
+      if (!this.doObserverCheck("file", filePath, ReaderStatus.ERROR, msg)) {
+        return [];
       }
     }
 
-    // After successful import of all files, directory completion
-    // is only a notification for observer, cannot be cancelled.
-    this.doObserverCheck("directory", directoryPath, ReaderStatus.COMPLETE);
-
     return docs;
   }
 
@@ -122,10 +166,3 @@ export class SimpleDirectoryReader implements BaseReader {
     return true;
   }
 }
-
-function addMetaData(filePath: string): (doc: Document<Metadata>) => void {
-  return (doc: Document<Metadata>) => {
-    doc.metadata["file_path"] = path.resolve(filePath);
-    doc.metadata["file_name"] = path.basename(filePath);
-  };
-}

packages/core/src/readers/SimpleDirectoryReader.ts

@@ -10,9 +10,9 @@ import {
   type SimpleDirectoryReaderLoadDataParams,
 } from "./SimpleDirectoryReader.edge.js";
 import { TextFileReader } from "./TextFileReader.js";
-import type { BaseReader } from "./type.js";
+import type { FileReader } from "./type.js";
 
-export const FILE_EXT_TO_READER: Record<string, BaseReader> = {
+export const FILE_EXT_TO_READER: Record<string, FileReader> = {
   txt: new TextFileReader(),
   pdf: new PDFReader(),
   csv: new PapaCSVReader(),

packages/core/src/readers/TextFileReader.ts

@@ -1,14 +1,13 @@
-import { fs } from "@llamaindex/env";
 import { Document } from "../Node.js";
-import type { BaseReader } from "./type.js";
+import { FileReader } from "./type.js";
 
 /**
  * Read a .txt file
  */
 
-export class TextFileReader implements BaseReader {
-  async loadData(file: string): Promise<Document[]> {
-    const dataBuffer = await fs.readFile(file, "utf-8");
-    return [new Document({ text: dataBuffer, id_: file })];
+export class TextFileReader extends FileReader {
+  async loadDataAsContent(fileContent: Buffer): Promise<Document[]> {
+    const dataBuffer = fileContent.toString("utf-8");
+    return [new Document({ text: dataBuffer })];
   }
 }

packages/core/src/readers/type.ts

@@ -1,3 +1,4 @@
+import { fs, path } from "@llamaindex/env";
 import type { Document } from "../Node.js";
 
 /**
@@ -8,10 +9,26 @@ export interface BaseReader {
 }
 
 /**
- * A reader takes file paths and imports data into Document objects.
+ * A FileReader takes file paths and imports data into Document objects.
  */
-export interface FileReader extends BaseReader {
-  loadData(filePath: string): Promise<Document[]>;
+export abstract class FileReader implements BaseReader {
+  abstract loadDataAsContent(fileContent: Buffer): Promise<Document[]>;
+
+  async loadData(filePath: string): Promise<Document[]> {
+    const fileContent = await fs.readFile(filePath);
+    const docs = await this.loadDataAsContent(fileContent);
+    docs.forEach(FileReader.addMetaData(filePath));
+    return docs;
+  }
+
+  static addMetaData(filePath: string) {
+    return (doc: Document, index: number) => {
+      // generate id as loadDataAsContent is only responsible for the content
+      doc.id_ = `${filePath}_${index + 1}`;
+      doc.metadata["file_path"] = path.resolve(filePath);
+      doc.metadata["file_name"] = path.basename(filePath);
+    };
+  }
 }
 
 // For LlamaParseReader.ts

packages/core/src/readers/utils.ts

@@ -0,0 +1,106 @@
+// Note: this code is taken from p-limit 5.0.0 and modified to work with non NodeJS envs by removing AsyncResource which seems not be needed in our case and also it's not recommended to used anymore. If we need to preserve some state between async calls better use `AsyncLocalStorage`.
+// Also removed dependency to yocto-queue by using normal Array
+
+export type LimitFunction = {
+  /**
+	The number of promises that are currently running.
+	*/
+  readonly activeCount: number;
+
+  /**
+	The number of promises that are waiting to run (i.e. their internal `fn` was not called yet).
+	*/
+  readonly pendingCount: number;
+
+  /**
+	Discard pending promises that are waiting to run.
+
+	This might be useful if you want to teardown the queue at the end of your program's lifecycle or discard any function calls referencing an intermediary state of your app.
+
+	Note: This does not cancel promises that are already running.
+	*/
+  clearQueue: () => void;
+
+  /**
+	@param fn - Promise-returning/async function.
+	@param arguments - Any arguments to pass through to `fn`. Support for passing arguments on to the `fn` is provided in order to be able to avoid creating unnecessary closures. You probably don't need this optimization unless you're pushing a lot of functions.
+	@returns The promise returned by calling `fn(...arguments)`.
+	*/
+  <Arguments extends unknown[], ReturnType>(
+    fn: (...arguments_: Arguments) => PromiseLike<ReturnType> | ReturnType,
+    ...arguments_: Arguments
+  ): Promise<ReturnType>;
+};
+
+export default function pLimit(concurrency: number): LimitFunction {
+  if (
+    !(
+      (Number.isInteger(concurrency) ||
+        concurrency === Number.POSITIVE_INFINITY) &&
+      concurrency > 0
+    )
+  ) {
+    throw new TypeError("Expected `concurrency` to be a number from 1 and up");
+  }
+
+  const queue = new Array();
+  let activeCount = 0;
+
+  const next = () => {
+    activeCount--;
+
+    if (queue.length > 0) {
+      queue.shift()();
+    }
+  };
+
+  const run = async (function_: any, resolve: any, arguments_: any) => {
+    activeCount++;
+
+    const result = (async () => function_(...arguments_))();
+    resolve(result);
+
+    try {
+      await result;
+    } catch {}
+
+    next();
+  };
+
+  const enqueue = (function_: any, resolve: any, arguments_: any) => {
+    queue.push(run.bind(undefined, function_, resolve, arguments_));
+
+    (async () => {
+      // This function needs to wait until the next microtask before comparing
+      // `activeCount` to `concurrency`, because `activeCount` is updated asynchronously
+      // when the run function is dequeued and called. The comparison in the if-statement
+      // needs to happen asynchronously as well to get an up-to-date value for `activeCount`.
+      await Promise.resolve();
+
+      if (activeCount < concurrency && queue.length > 0) {
+        queue.shift()();
+      }
+    })();
+  };
+
+  const generator = (function_: any, ...arguments_: any) =>
+    new Promise((resolve) => {
+      enqueue(function_, resolve, arguments_);
+    });
+
+  Object.defineProperties(generator, {
+    activeCount: {
+      get: () => activeCount,
+    },
+    pendingCount: {
+      get: () => queue.length,
+    },
+    clearQueue: {
+      value() {
+        queue.length = 0;
+      },
+    },
+  });
+
+  return generator as LimitFunction;
+}

packages/core/src/storage/docStore/types.ts

@@ -32,7 +32,7 @@ export abstract class BaseDocumentStore {
   abstract documentExists(docId: string): Promise<boolean>;
 
   // Hash
-  abstract setDocumentHash(docId: string, docHash: string): void;
+  abstract setDocumentHash(docId: string, docHash: string): Promise<void>;
 
   abstract getDocumentHash(docId: string): Promise<string | undefined>;
 

packages/core/src/storage/vectorStore/MongoDBAtlasVectorStore.ts

@@ -3,6 +3,7 @@ import type { BulkWriteOptions, Collection } from "mongodb";
 import { MongoClient } from "mongodb";
 import type { BaseNode } from "../../Node.js";
 import { MetadataMode } from "../../Node.js";
+import { BaseEmbedding } from "../../embeddings/types.js";
 import {
   VectorStoreBase,
   type MetadataFilters,
@@ -23,7 +24,10 @@ function toMongoDBFilter(
   return filters;
 }
 
-// MongoDB Atlas Vector Store class implementing VectorStore
+/**
+ * Vector store that uses MongoDB Atlas for storage and vector search.
+ * This store uses the $vectorSearch aggregation stage to perform vector similarity search.
+ */
 export class MongoDBAtlasVectorSearch
   extends VectorStoreBase
   implements VectorStoreNoEmbedModel
@@ -31,22 +35,73 @@ export class MongoDBAtlasVectorSearch
   storesText: boolean = true;
   flatMetadata: boolean = true;
 
+  /**
+   * The used MongoClient. If not given, a new MongoClient is created based on the MONGODB_URI env variable.
+   */
   mongodbClient: MongoClient;
+
+  /**
+   * Name of the vector index. If invalid, Mongo will silently ignore this issue and return 0 results.
+   *
+   * Default: "default"
+   */
   indexName: string;
+
+  /**
+   * Name of the key containing the embedding vector.
+   *
+   * Default: "embedding"
+   */
   embeddingKey: string;
+
+  /**
+   * Name of the key containing the node id.
+   *
+   * Default: "id"
+   */
   idKey: string;
+
+  /**
+   * Name of the key containing the node text.
+   *
+   * Default: "text"
+   */
   textKey: string;
+
+  /**
+   * Name of the key containing the node metadata.
+   *
+   * Default: "metadata"
+   */
   metadataKey: string;
+
+  /**
+   * Options to pass to the insertMany function when adding nodes.
+   */
   insertOptions?: BulkWriteOptions;
+
+  /**
+   * Function to determine the number of candidates to retrieve for a given query.
+   * In case your results are not good, you might tune this value.
+   *
+   * {@link https://www.mongodb.com/docs/atlas/atlas-vector-search/vector-search-stage/|Run Vector Search Queries}
+   *
+   * {@link https://arxiv.org/abs/1603.09320|Efficient and robust approximate nearest neighbor search using Hierarchical Navigable Small World graphs}
+   *
+   *
+   * Default: query.similarityTopK * 10
+   */
+  numCandidates: (query: VectorStoreQuery) => number;
   private collection: Collection;
 
   constructor(
     init: Partial<MongoDBAtlasVectorSearch> & {
       dbName: string;
       collectionName: string;
+      embedModel?: BaseEmbedding;
     },
   ) {
-    super();
+    super(init.embedModel);
     if (init.mongodbClient) {
       this.mongodbClient = init.mongodbClient;
     } else {
@@ -67,9 +122,17 @@ export class MongoDBAtlasVectorSearch
     this.idKey = init.idKey ?? "id";
     this.textKey = init.textKey ?? "text";
     this.metadataKey = init.metadataKey ?? "metadata";
+    this.numCandidates =
+      init.numCandidates ?? ((query) => query.similarityTopK * 10);
     this.insertOptions = init.insertOptions;
   }
 
+  /**
+   * Add nodes to the vector store.
+   *
+   * @param nodes Nodes to add to the vector store
+   * @returns List of node ids that were added
+   */
   async add(nodes: BaseNode[]): Promise<string[]> {
     if (!nodes || nodes.length === 0) {
       return [];
@@ -99,8 +162,14 @@ export class MongoDBAtlasVectorSearch
     return nodes.map((node) => node.id_);
   }
 
+  /**
+   * Delete nodes from the vector store with the given redDocId.
+   *
+   * @param refDocId The refDocId of the nodes to delete
+   * @param deleteOptions Options to pass to the deleteOne function
+   */
   async delete(refDocId: string, deleteOptions?: any): Promise<void> {
-    await this.collection.deleteOne(
+    await this.collection.deleteMany(
       {
         [`${this.metadataKey}.ref_doc_id`]: refDocId,
       },
@@ -112,6 +181,12 @@ export class MongoDBAtlasVectorSearch
     return this.mongodbClient;
   }
 
+  /**
+   * Perform a vector similarity search query.
+   *
+   * @param query The query to run
+   * @returns List of nodes and their similarities
+   */
   async query(
     query: VectorStoreQuery,
     options?: any,
@@ -119,7 +194,7 @@ export class MongoDBAtlasVectorSearch
     const params: any = {
       queryVector: query.queryEmbedding,
       path: this.embeddingKey,
-      numCandidates: query.similarityTopK * 10,
+      numCandidates: this.numCandidates(query),
       limit: query.similarityTopK,
       index: this.indexName,
     };

packages/core/src/storage/vectorStore/SimpleVectorStore.ts

@@ -151,13 +151,20 @@ export class SimpleVectorStore
 
   async persist(
     persistPath: string = path.join(DEFAULT_PERSIST_DIR, "vector_store.json"),
+  ): Promise<void> {
+    await SimpleVectorStore.persistData(persistPath, this.data);
+  }
+
+  protected static async persistData(
+    persistPath: string,
+    data: SimpleVectorStoreData,
   ): Promise<void> {
     const dirPath = path.dirname(persistPath);
     if (!(await exists(dirPath))) {
       await fs.mkdir(dirPath);
     }
 
-    await fs.writeFile(persistPath, JSON.stringify(this.data));
+    await fs.writeFile(persistPath, JSON.stringify(data));
   }
 
   static async fromPersistPath(
@@ -177,6 +184,11 @@ export class SimpleVectorStore
       console.error(
         `No valid data found at path: ${persistPath} starting new store.`,
       );
+      // persist empty data, to ignore this error in the future
+      await SimpleVectorStore.persistData(
+        persistPath,
+        new SimpleVectorStoreData(),
+      );
     }
 
     const data = new SimpleVectorStoreData();

packages/experimental/CHANGELOG.md

@@ -1,5 +1,21 @@
 # @llamaindex/experimental
 
+## 0.0.33
+
+### Patch Changes
+
+- Updated dependencies [11ae926]
+- Updated dependencies [631f000]
+- Updated dependencies [1378ec4]
+- Updated dependencies [6b1ded4]
+- Updated dependencies [4d4bd85]
+- Updated dependencies [24a9d1e]
+- Updated dependencies [45952de]
+- Updated dependencies [54230f0]
+- Updated dependencies [a29d835]
+- Updated dependencies [73819bf]
+  - llamaindex@0.3.16
+
 ## 0.0.32
 
 ### Patch Changes

packages/experimental/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/experimental",
   "description": "Experimental package for LlamaIndexTS",
-  "version": "0.0.32",
+  "version": "0.0.33",
   "type": "module",
   "types": "dist/type/index.d.ts",
   "main": "dist/cjs/index.js",