Release 0.5.6 (#1048)

Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>

.github/workflows/release.yml

@@ -67,3 +67,4 @@ jobs:
         with:
           commit_message: "chore: update lock file"
           branch: changeset-release/main
+          file_pattern: "pnpm-lock.yaml"

apps/docs/CHANGELOG.md

@@ -1,5 +1,54 @@
 # docs
 
+## 0.0.47
+
+### Patch Changes
+
+- Updated dependencies [2562244]
+- Updated dependencies [325aa51]
+- Updated dependencies [ab700ea]
+- Updated dependencies [92f0782]
+- Updated dependencies [6cf6ae6]
+- Updated dependencies [b7cfe5b]
+  - llamaindex@0.5.6
+
+## 0.0.46
+
+### Patch Changes
+
+- Updated dependencies [b974eea]
+  - llamaindex@0.5.5
+
+## 0.0.45
+
+### Patch Changes
+
+- Updated dependencies [1a65ead]
+  - llamaindex@0.5.4
+
+## 0.0.44
+
+### Patch Changes
+
+- Updated dependencies [9bbbc67]
+- Updated dependencies [b3681bf]
+  - llamaindex@0.5.3
+
+## 0.0.43
+
+### Patch Changes
+
+- llamaindex@0.5.2
+
+## 0.0.42
+
+### Patch Changes
+
+- 2774681: Add mixedbread's embeddings and reranking API
+- Updated dependencies [2774681]
+- Updated dependencies [a0f424e]
+  - llamaindex@0.5.1
+
 ## 0.0.41
 
 ### Patch Changes

apps/docs/docs/modules/data_loaders/discord.mdx

@@ -0,0 +1,34 @@
+import CodeBlock from "@theme/CodeBlock";
+import CodeSource from "!raw-loader!../../../../../examples/readers/src/discord";
+
+# DiscordReader
+
+DiscordReader is a simple data loader that reads all messages in a given Discord channel and returns them as Document objects.
+It uses the [@discordjs/rest](https://github.com/discordjs/discord.js/tree/main/packages/rest) library to fetch the messages.
+
+## Usage
+
+First step is to create a Discord Application and generating a bot token [here](https://discord.com/developers/applications).
+In your Discord Application, go to the `OAuth2` tab and generate an invite URL by selecting `bot` and click `Read Messages/View Channels` as wells as `Read Message History`.
+This will invite the bot with the necessary permissions to read messages.
+Copy the URL in your browser and select the server you want your bot to join.
+
+<CodeBlock language="ts">{CodeSource}</CodeBlock>
+
+### Params
+
+#### DiscordReader()
+
+- `discordToken?`: The Discord bot token.
+- `makeRequest?`: Optionally provide a custom request function for edge environments, e.g. `fetch`. See discord.js for more info.
+
+#### DiscordReader.loadData
+
+- `channelIDs`: The ID(s) of discord channels as an array of strings.
+- `limit?`: Optionally limit the number of messages to read
+- `additionalInfo?`: An optional flag to include embedded messages and attachment urls in the document.
+- `oldestFirst?`: An optional flag to return the oldest messages first.
+
+## API Reference
+
+- [DiscordReader](../../api/classes/DiscordReader.md)

apps/docs/docs/modules/data_loaders/llama_parse/index.mdx

@@ -42,10 +42,13 @@ They can be divided into two groups.
 - `fastMode?` Optional. Set to true to use the fast mode. This mode will skip OCR of images, and table/heading reconstruction. Note: Non-compatible with `gpt4oMode`.
 - `doNotUnrollColumns?` Optional. Set to true to keep the text according to document layout. Reduce reconstruction accuracy, and LLMs/embeddings performances in most cases.
 - `pageSeparator?` Optional. The page separator to use. Defaults is `\\n---\\n`.
-- `gpt4oMode` set to true to use GPT-4o to extract content. Default is `false`.
-- `gpt4oApiKey?` Optional. Set the GPT-4o API key. Lowers the cost of parsing by using your own API key. Your OpenAI account will be charged. Can also be set in the environment variable `LLAMA_CLOUD_GPT4O_API_KEY`.
+- `gpt4oMode` Deprecated. Use vendorMultimodal params. Set to true to use GPT-4o to extract content. Default is `false`.
+- `gpt4oApiKey?` Deprecated. Use vendorMultimodal params. Optional. Set the GPT-4o API key. Lowers the cost of parsing by using your own API key. Your OpenAI account will be charged. Can also be set in the environment variable `LLAMA_CLOUD_GPT4O_API_KEY`.
 - `boundingBox?` Optional. Specify an area of the document to parse. Expects the bounding box margins as a string in clockwise order, e.g. `boundingBox = "0.1,0,0,0"` to not parse the top 10% of the document.
 - `targetPages?` Optional. Specify which pages to parse by specifying them as a comma-separated list. First page is `0`.
+- `useVendorMultimodalModel` set to true to use a multimodal model. Default is `false`.
+- `vendorMultimodalModel?` Optional. Specify which multimodal model to use. Default is GPT4o. See [here](https://docs.cloud.llamaindex.ai/llamaparse/features/multimodal) for a list of available models and cost.
+- `vendorMultimodalApiKey?` Optional. Set the multimodal model API key. Can also be set in the environment variable `LLAMA_CLOUD_VENDOR_MULTIMODAL_API_KEY`.
 - `numWorkers` as in the python version, is set in `SimpleDirectoryReader`. Default is 1.
 
 ### LlamaParse with SimpleDirectoryReader

apps/docs/docs/modules/embeddings/available_embeddings/mixedbreadai.md

@@ -0,0 +1,100 @@
+# MixedbreadAI
+
+Welcome to the mixedbread embeddings guide! This guide will help you use the mixedbread ai's API to generate embeddings for your text documents, ensuring you get the most relevant information, just like picking the freshest bread from the bakery.
+
+To find out more about the latest features, updates, and available models, visit [mixedbread.ai](https://mixedbread-ai.com/).
+
+## Table of Contents
+
+1. [Setup](#setup)
+2. [Usage with LlamaIndex](#integration-with-llamaindex)
+3. [Embeddings with Custom Parameters](#embeddings-with-custom-parameters)
+
+## Setup
+
+First, you will need to install the `llamaindex` package.
+
+```bash
+pnpm install llamaindex
+```
+
+Next, sign up for an API key at [mixedbread.ai](https://mixedbread.ai/). Once you have your API key, you can import the necessary modules and create a new instance of the `MixedbreadAIEmbeddings` class.
+
+```ts
+import { MixedbreadAIEmbeddings, Document, Settings } from "llamaindex";
+```
+
+## Usage with LlamaIndex
+
+This section will guide you through integrating mixedbread embeddings with LlamaIndex for more advanced usage.
+
+### Step 1: Load and Index Documents
+
+For this example, we will use a single document. In a real-world scenario, you would have multiple documents to index, like a variety of breads in a bakery.
+
+```ts
+Settings.embedModel = new MixedbreadAIEmbeddings({
+  apiKey: "<MIXEDBREAD_API_KEY>",
+  model: "mixedbread-ai/mxbai-embed-large-v1",
+});
+
+const document = new Document({
+  text: "The true source of happiness.",
+  id_: "bread",
+});
+
+const index = await VectorStoreIndex.fromDocuments([document]);
+```
+
+### Step 2: Create a Query Engine
+
+Combine the retriever and the embed model to create a query engine. This setup ensures that your queries are processed to provide the best results, like arranging the bread in the order of freshness and quality.
+
+Models can require prompts to generate embeddings for queries, in the 'mixedbread-ai/mxbai-embed-large-v1' model's case, the prompt is `Represent this sentence for searching relevant passages:`.
+
+```ts
+const queryEngine = index.asQueryEngine();
+
+const query =
+  "Represent this sentence for searching relevant passages: What is bread?";
+
+// Log the response
+const results = await queryEngine.query(query);
+console.log(results); // Serving up the freshest, most relevant results.
+```
+
+## Embeddings with Custom Parameters
+
+This section will guide you through generating embeddings with custom parameters and usage with f.e. matryoshka and binary embeddings.
+
+### Step 1: Create an Instance of MixedbreadAIEmbeddings
+
+Create a new instance of the `MixedbreadAIEmbeddings` class with custom parameters. For example, to use the `mixedbread-ai/mxbai-embed-large-v1` model with a batch size of 64, normalized embeddings, and binary encoding format:
+
+```ts
+const embeddings = new MixedbreadAIEmbeddings({
+  apiKey: "<MIXEDBREAD_API_KEY>",
+  model: "mixedbread-ai/mxbai-embed-large-v1",
+  batchSize: 64,
+  normalized: true,
+  dimensions: 512,
+  encodingFormat: MixedbreadAI.EncodingFormat.Binary,
+});
+```
+
+### Step 2: Define Texts
+
+Define the texts you want to generate embeddings for.
+
+```ts
+const texts = ["Bread is life", "Bread is love"];
+```
+
+### Step 3: Generate Embeddings
+
+Use the `embedDocuments` method to generate embeddings for the texts.
+
+```ts
+const result = await embeddings.embedDocuments(texts);
+console.log(result); // Perfectly customized embeddings, ready to serve.
+```

apps/docs/docs/modules/node_postprocessors/cohere_reranker.md

@@ -39,8 +39,9 @@ const index = await VectorStoreIndex.fromDocuments([document]);
 The default value for `similarityTopK` is 2. This means that only the most similar document will be returned. To retrieve more results, you can increase the value of `similarityTopK`.
 
 ```ts
-const retriever = index.asRetriever();
-retriever.similarityTopK = 5;
+const retriever = index.asRetriever({
+  similarityTopK: 5,
+});
 ```
 
 ## Create a new instance of the CohereRerank class

apps/docs/docs/modules/node_postprocessors/jinaai_reranker.md

@@ -39,8 +39,9 @@ const index = await VectorStoreIndex.fromDocuments([document]);
 The default value for `similarityTopK` is 2. This means that only the most similar document will be returned. To retrieve more results, you can increase the value of `similarityTopK`.
 
 ```ts
-const retriever = index.asRetriever();
-retriever.similarityTopK = 5;
+const retriever = index.asRetriever({
+  similarityTopK: 5,
+});
 ```
 
 ## Create a new instance of the JinaAIReranker class

apps/docs/docs/modules/node_postprocessors/mixedbreadiai_reranker.md

@@ -0,0 +1,165 @@
+# MixedbreadAI
+
+Welcome to the mixedbread ai reranker guide! This guide will help you use mixedbread ai's API to rerank search query results, ensuring you get the most relevant information, just like picking the freshest bread from the bakery.
+
+To find out more about the latest features and updates, visit the [mixedbread.ai](https://mixedbread.ai/).
+
+## Table of Contents
+
+1. [Setup](#setup)
+2. [Usage with LlamaIndex](#integration-with-llamaindex)
+3. [Simple Reranking Guide](#simple-reranking-guide)
+4. [Reranking with Objects](#reranking-with-objects)
+
+## Setup
+
+First, you will need to install the `llamaindex` package.
+
+```bash
+pnpm install llamaindex
+```
+
+Next, sign up for an API key at [mixedbread.ai](https://mixedbread.ai/). Once you have your API key, you can import the necessary modules and create a new instance of the `MixedbreadAIReranker` class.
+
+```ts
+import {
+  MixedbreadAIReranker,
+  Document,
+  OpenAI,
+  VectorStoreIndex,
+  Settings,
+} from "llamaindex";
+```
+
+## Usage with LlamaIndex
+
+This section will guide you through integrating mixedbread's reranker with LlamaIndex.
+
+### Step 1: Load and Index Documents
+
+For this example, we will use a single document. In a real-world scenario, you would have multiple documents to index, like a variety of breads in a bakery.
+
+```ts
+const document = new Document({
+  text: "This is a sample document.",
+  id_: "sampleDoc",
+});
+
+Settings.llm = new OpenAI({ model: "gpt-3.5-turbo", temperature: 0.1 });
+
+const index = await VectorStoreIndex.fromDocuments([document]);
+```
+
+### Step 2: Increase Similarity TopK
+
+The default value for `similarityTopK` is 2, which means only the most similar document will be returned. To get more results, like picking a variety of fresh breads, you can increase the value of `similarityTopK`.
+
+```ts
+const retriever = index.asRetriever({
+  similarityTopK: 5,
+});
+```
+
+### Step 3: Create a MixedbreadAIReranker Instance
+
+Create a new instance of the `MixedbreadAIReranker` class.
+
+```ts
+const nodePostprocessor = new MixedbreadAIReranker({
+  apiKey: "<MIXEDBREAD_API_KEY>",
+  topN: 4,
+});
+```
+
+### Step 4: Create a Query Engine
+
+Combine the retriever and node postprocessor to create a query engine. This setup ensures that your queries are processed and reranked to provide the best results, like arranging the bread in the order of freshness and quality.
+
+```ts
+const queryEngine = index.asQueryEngine({
+  retriever,
+  nodePostprocessors: [nodePostprocessor],
+});
+
+// Log the response
+const response = await queryEngine.query("Where did the author grow up?");
+console.log(response);
+```
+
+With mixedbread's Reranker, you're all set to serve up the most relevant and well-ordered results, just like a skilled baker arranging their best breads for eager customers. Enjoy the perfect blend of technology and culinary delight!
+
+## Simple Reranking Guide
+
+This section will guide you through a simple reranking process using mixedbread ai.
+
+### Step 1: Create an Instance of MixedbreadAIReranker
+
+Create a new instance of the `MixedbreadAIReranker` class, passing in your API key and the number of results you want to return. It's like setting up your bakery to offer a specific number of freshly baked items.
+
+```ts
+const reranker = new MixedbreadAIReranker({
+  apiKey: "<MIXEDBREAD_API_KEY>",
+  topN: 4,
+});
+```
+
+### Step 2: Define Nodes and Query
+
+Define the nodes (documents) you want to rerank and the query.
+
+```ts
+const nodes = [
+  { node: new BaseNode("To bake bread you need flour") },
+  { node: new BaseNode("To bake bread you need yeast") },
+];
+const query = "What do you need to bake bread?";
+```
+
+### Step 3: Perform Reranking
+
+Use the `postprocessNodes` method to rerank the nodes based on the query.
+
+```ts
+const result = await reranker.postprocessNodes(nodes, query);
+console.log(result); // Like pulling freshly baked nodes out of the oven.
+```
+
+## Reranking with Objects
+
+This section will guide you through reranking when working with objects.
+
+### Step 1: Create an Instance of MixedbreadAIReranker
+
+Create a new instance of the `MixedbreadAIReranker` class, just like before.
+
+```ts
+const reranker = new MixedbreadAIReranker({
+  apiKey: "<MIXEDBREAD_API_KEY>",
+  model: "mixedbread-ai/mxbai-rerank-large-v1",
+  topK: 5,
+  rankFields: ["title", "content"],
+  returnInput: true,
+  maxRetries: 5,
+});
+```
+
+### Step 2: Define Documents and Query
+
+Define the documents (objects) you want to rerank and the query.
+
+```ts
+const documents = [
+  { title: "Bread Recipe", content: "To bake bread you need flour" },
+  { title: "Bread Recipe", content: "To bake bread you need yeast" },
+];
+const query = "What do you need to bake bread?";
+```
+
+### Step 3: Perform Reranking
+
+Use the `rerank` method to reorder the documents based on the query.
+
+```ts
+const result = await reranker.rerank(documents, query);
+console.log(result); // Perfectly customized results, ready to serve.
+```

apps/docs/docs/modules/query_engines/metadata_filtering.md

@@ -75,7 +75,7 @@ const queryEngine = index.asQueryEngine({
       {
         key: "dogId",
         value: "2",
-        filterType: "ExactMatch",
+        operator: "==",
       },
     ],
   },
@@ -88,6 +88,8 @@ const response = await queryEngine.query({
 console.log(response.toString());
 ```
 
+Besides using the equal operator (`==`), you can also use a whole set of different [operators](../../api/interfaces/MetadataFilter.md#operator) to filter your documents.
+
 ## Full Code
 
 ```ts
@@ -135,7 +137,7 @@ async function main() {
           {
             key: "dogId",
             value: "2",
-            filterType: "ExactMatch",
+            operator: "==",
           },
         ],
       },
@@ -156,3 +158,4 @@ main();
 
 - [VectorStoreIndex](../../api/classes/VectorStoreIndex.md)
 - [ChromaVectorStore](../../api/classes/ChromaVectorStore.md)
+- [MetadataFilter](../../api/interfaces/MetadataFilter.md)

apps/docs/docs/modules/retriever.md

@@ -7,8 +7,9 @@ sidebar_position: 5
 A retriever in LlamaIndex is what is used to fetch `Node`s from an index using a query string. Aa `VectorIndexRetriever` will fetch the top-k most similar nodes. Meanwhile, a `SummaryIndexRetriever` will fetch all nodes no matter the query.
 
 ```typescript
-const retriever = vector_index.asRetriever();
-retriever.similarityTopK = 3;
+const retriever = vectorIndex.asRetriever({
+  similarityTopK: 3,
+});
 
 // Fetch nodes!
 const nodesWithScore = await retriever.retrieve({ query: "query string" });

apps/docs/package.json

@@ -1,6 +1,6 @@
 {
   "name": "docs",
-  "version": "0.0.41",
+  "version": "0.0.47",
   "private": true,
   "scripts": {
     "docusaurus": "docusaurus",
@@ -15,29 +15,29 @@
     "typecheck": "tsc"
   },
   "dependencies": {
-    "@docusaurus/core": "^3.3.2",
-    "@docusaurus/remark-plugin-npm2yarn": "^3.3.2",
+    "@docusaurus/core": "3.4.0",
+    "@docusaurus/remark-plugin-npm2yarn": "3.4.0",
     "@llamaindex/examples": "workspace:*",
-    "@mdx-js/react": "^3.0.1",
-    "clsx": "^2.1.1",
+    "@mdx-js/react": "3.0.1",
+    "clsx": "2.1.1",
     "llamaindex": "workspace:*",
-    "postcss": "^8.4.38",
-    "prism-react-renderer": "^2.3.1",
-    "raw-loader": "^4.0.2",
-    "react": "^18.3.1",
-    "react-dom": "^18.3.1"
+    "postcss": "8.4.39",
+    "prism-react-renderer": "2.3.1",
+    "raw-loader": "4.0.2",
+    "react": "18.3.1",
+    "react-dom": "18.3.1"
   },
   "devDependencies": {
-    "@docusaurus/module-type-aliases": "3.3.2",
-    "@docusaurus/preset-classic": "^3.3.2",
-    "@docusaurus/theme-classic": "^3.3.2",
-    "@docusaurus/types": "^3.3.2",
-    "@tsconfig/docusaurus": "^2.0.3",
+    "@docusaurus/module-type-aliases": "3.4.0",
+    "@docusaurus/preset-classic": "3.4.0",
+    "@docusaurus/theme-classic": "3.4.0",
+    "@docusaurus/types": "3.4.0",
+    "@tsconfig/docusaurus": "2.0.3",
     "@types/node": "^20.12.11",
-    "docusaurus-plugin-typedoc": "^1.0.1",
-    "typedoc": "^0.25.13",
-    "typedoc-plugin-markdown": "^4.0.1",
-    "typescript": "^5.5.2"
+    "docusaurus-plugin-typedoc": "1.0.3",
+    "typedoc": "0.26.4",
+    "typedoc-plugin-markdown": "4.1.2",
+    "typescript": "^5.5.3"
   },
   "browserslist": {
     "production": [

examples/chatEngine.ts

@@ -16,8 +16,9 @@ Settings.chunkSize = 512;
 async function main() {
   const document = new Document({ text: essay });
   const index = await VectorStoreIndex.fromDocuments([document]);
-  const retriever = index.asRetriever();
-  retriever.similarityTopK = 5;
+  const retriever = index.asRetriever({
+    similarityTopK: 5,
+  });
   const chatEngine = new ContextChatEngine({ retriever });
   const rl = readline.createInterface({ input, output });
 

examples/chromadb/preFilters.ts

@@ -40,7 +40,7 @@ async function main() {
           {
             key: "dogId",
             value: "2",
-            filterType: "ExactMatch",
+            operator: "==",
           },
         ],
       },

examples/lowlevel.ts

@@ -27,11 +27,13 @@ import {
     },
   ];
 
-  const stream = await responseSynthesizer.synthesize({
-    query: "What age am I?",
-    nodesWithScore,
-    stream: true,
-  });
+  const stream = await responseSynthesizer.synthesize(
+    {
+      query: "What age am I?",
+      nodesWithScore,
+    },
+    true,
+  );
   for await (const chunk of stream) {
     process.stdout.write(chunk.response);
   }

examples/metadata-filter/milvus.ts

@@ -0,0 +1,40 @@
+import { MilvusVectorStore, VectorStoreIndex } from "llamaindex";
+
+const collectionName = "movie_reviews";
+
+async function main() {
+  try {
+    const milvus = new MilvusVectorStore({ collection: collectionName });
+    const index = await VectorStoreIndex.fromVectorStore(milvus);
+    const retriever = index.asRetriever({ similarityTopK: 20 });
+
+    console.log("\n=====\nQuerying the index with filters");
+    const queryEngineWithFilters = index.asQueryEngine({
+      retriever,
+      preFilters: {
+        filters: [
+          {
+            key: "document_id",
+            value: "./data/movie_reviews.csv_37",
+            operator: "==",
+          },
+          {
+            key: "document_id",
+            value: "./data/movie_reviews.csv_37",
+            operator: "!=",
+          },
+        ],
+        condition: "or",
+      },
+    });
+    const resultAfterFilter = await queryEngineWithFilters.query({
+      query: "Get all movie titles.",
+    });
+    console.log(`Query from ${resultAfterFilter.sourceNodes?.length} nodes`);
+    console.log(resultAfterFilter.response);
+  } catch (e) {
+    console.error(e);
+  }
+}
+
+void main();

examples/metadata-filter/simple.ts

@@ -0,0 +1,143 @@
+import {
+  Document,
+  Settings,
+  SimpleDocumentStore,
+  VectorStoreIndex,
+  storageContextFromDefaults,
+} from "llamaindex";
+
+Settings.callbackManager.on("retrieve-end", (event) => {
+  const { nodes } = event.detail;
+  console.log("Number of retrieved nodes:", nodes.length);
+});
+
+async function getDataSource() {
+  const docs = [
+    new Document({
+      text: "The dog is brown",
+      metadata: {
+        dogId: "1",
+        private: true,
+      },
+    }),
+    new Document({
+      text: "The dog is yellow",
+      metadata: {
+        dogId: "2",
+        private: false,
+      },
+    }),
+    new Document({
+      text: "The dog is red",
+      metadata: {
+        dogId: "3",
+        private: false,
+      },
+    }),
+  ];
+  const storageContext = await storageContextFromDefaults({
+    persistDir: "./cache",
+  });
+  const numberOfDocs = Object.keys(
+    (storageContext.docStore as SimpleDocumentStore).toDict(),
+  ).length;
+  if (numberOfDocs === 0) {
+    // Generate the data source if it's empty
+    return await VectorStoreIndex.fromDocuments(docs, {
+      storageContext,
+    });
+  }
+  return await VectorStoreIndex.init({
+    storageContext,
+  });
+}
+
+async function main() {
+  const index = await getDataSource();
+  console.log(
+    "=============\nQuerying index with no filters. The output should be any color.",
+  );
+  const queryEngineNoFilters = index.asQueryEngine({
+    similarityTopK: 3,
+  });
+  const noFilterResponse = await queryEngineNoFilters.query({
+    query: "What is the color of the dog?",
+  });
+  console.log("No filter response:", noFilterResponse.toString());
+
+  console.log(
+    "\n=============\nQuerying index with dogId 2 and private false. The output always should be red.",
+  );
+  const queryEngineEQ = index.asQueryEngine({
+    preFilters: {
+      filters: [
+        {
+          key: "private",
+          value: "false",
+          operator: "==",
+        },
+        {
+          key: "dogId",
+          value: "3",
+          operator: "==",
+        },
+      ],
+    },
+    similarityTopK: 3,
+  });
+  const responseEQ = await queryEngineEQ.query({
+    query: "What is the color of the dog?",
+  });
+  console.log("Filter with dogId 2 response:", responseEQ.toString());
+
+  console.log(
+    "\n=============\nQuerying index with dogId IN (1, 3). The output should be brown and red.",
+  );
+  const queryEngineIN = index.asQueryEngine({
+    preFilters: {
+      filters: [
+        {
+          key: "dogId",
+          value: ["1", "3"],
+          operator: "in",
+        },
+      ],
+    },
+    similarityTopK: 3,
+  });
+  const responseIN = await queryEngineIN.query({
+    query: "What is the color of the dog?",
+  });
+  console.log("Filter with dogId IN (1, 3) response:", responseIN.toString());
+
+  console.log(
+    "\n=============\nQuerying index with dogId IN (1, 3). The output should be any.",
+  );
+  const queryEngineOR = index.asQueryEngine({
+    preFilters: {
+      filters: [
+        {
+          key: "private",
+          value: "false",
+          operator: "==",
+        },
+        {
+          key: "dogId",
+          value: ["1", "3"],
+          operator: "in",
+        },
+      ],
+      condition: "or",
+    },
+    similarityTopK: 3,
+  });
+  const responseOR = await queryEngineOR.query({
+    query: "What is the color of the dog?",
+  });
+  console.log(
+    "Filter with dogId with OR operator response:",
+    responseOR.toString(),
+  );
+}
+
+void main();

examples/multimodal/jina.ts

@@ -0,0 +1,83 @@
+import {
+  ImageDocument,
+  JinaAIEmbedding,
+  similarity,
+  SimilarityType,
+  SimpleDirectoryReader,
+} from "llamaindex";
+import path from "path";
+
+async function main() {
+  const jina = new JinaAIEmbedding({
+    model: "jina-clip-v1",
+  });
+
+  // Get text embeddings
+  const text1 = "a car";
+  const textEmbedding1 = await jina.getTextEmbedding(text1);
+  const text2 = "a football match";
+  const textEmbedding2 = await jina.getTextEmbedding(text2);
+
+  // Get image embedding
+  const image =
+    "https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/football-match.jpg";
+  const imageEmbedding = await jina.getImageEmbedding(image);
+
+  // Calc similarity between text and image
+  const sim1 = similarity(
+    textEmbedding1,
+    imageEmbedding,
+    SimilarityType.DEFAULT,
+  );
+  const sim2 = similarity(
+    textEmbedding2,
+    imageEmbedding,
+    SimilarityType.DEFAULT,
+  );
+
+  console.log(`Similarity between "${text1}" and the image is ${sim1}`);
+  console.log(`Similarity between "${text2}" and the image is ${sim2}`);
+
+  // Get multiple text embeddings
+  const textEmbeddings = await jina.getTextEmbeddings([text1, text2]);
+  const sim3 = similarity(
+    textEmbeddings[0],
+    textEmbeddings[1],
+    SimilarityType.DEFAULT,
+  );
+  console.log(
+    `Similarity between the two texts "${text1}" and "${text2}" is ${sim3}`,
+  );
+
+  // Get multiple image embeddings
+  const catImg1 =
+    "https://i.pinimg.com/600x315/21/48/7e/21487e8e0970dd366dafaed6ab25d8d8.jpg";
+  const catImg2 =
+    "https://i.pinimg.com/736x/c9/f2/3e/c9f23e212529f13f19bad5602d84b78b.jpg";
+  const imageEmbeddings = await jina.getImageEmbeddings([catImg1, catImg2]);
+  const sim4 = similarity(
+    imageEmbeddings[0],
+    imageEmbeddings[1],
+    SimilarityType.DEFAULT,
+  );
+  console.log(`Similarity between the two online cat images is ${sim4}`);
+
+  // Get image embeddings from multiple local files
+  const documents = await new SimpleDirectoryReader().loadData({
+    directoryPath: path.join("multimodal", "data"),
+  });
+  const localImages = documents
+    .filter((doc) => doc instanceof ImageDocument)
+    .slice(0, 2); // Get only the first two images
+  const localImageEmbeddings = await jina.getImageEmbeddings(
+    localImages.map((doc) => (doc as ImageDocument).image),
+  );
+  const sim5 = similarity(
+    localImageEmbeddings[0],
+    localImageEmbeddings[1],
+    SimilarityType.DEFAULT,
+  );
+  console.log(`Similarity between the two local images is ${sim5}`);
+}
+
+void main();

examples/package.json

@@ -9,7 +9,7 @@
     "@llamaindex/core": "^0.1.0",
     "@notionhq/client": "^2.2.15",
     "@pinecone-database/pinecone": "^2.2.2",
-    "@zilliz/milvus2-sdk-node": "^2.4.2",
+    "@zilliz/milvus2-sdk-node": "^2.4.4",
     "chromadb": "^1.8.1",
     "commander": "^12.1.0",
     "dotenv": "^16.4.5",
@@ -21,7 +21,7 @@
   "devDependencies": {
     "@types/node": "^20.14.1",
     "tsx": "^4.15.6",
-    "typescript": "^5.5.2"
+    "typescript": "^5.5.3"
   },
   "scripts": {
     "lint": "eslint ."

examples/qdrantdb/preFilters.ts

@@ -64,7 +64,7 @@ async function main() {
           {
             key: "dogId",
             value: "2",
-            filterType: "ExactMatch",
+            operator: "==",
           },
         ],
       },

examples/readers/package.json

@@ -12,7 +12,8 @@
     "start:llamaparse": "node --import tsx ./src/llamaparse.ts",
     "start:notion": "node --import tsx ./src/notion.ts",
     "start:llamaparse-dir": "node --import tsx ./src/simple-directory-reader-with-llamaparse.ts",
-    "start:llamaparse-json": "node --import tsx ./src/llamaparse-json.ts"
+    "start:llamaparse-json": "node --import tsx ./src/llamaparse-json.ts",
+    "start:discord": "node --import tsx ./src/discord.ts"
   },
   "dependencies": {
     "llamaindex": "*"
@@ -20,6 +21,6 @@
   "devDependencies": {
     "@types/node": "^20.12.11",
     "tsx": "^4.15.6",
-    "typescript": "^5.5.2"
+    "typescript": "^5.5.3"
   }
 }

examples/readers/src/discord.ts

@@ -0,0 +1,20 @@
+import { DiscordReader } from "llamaindex";
+
+async function main() {
+  // Create an instance of the DiscordReader. Set token here or DISCORD_TOKEN environment variable
+  const discordReader = new DiscordReader();
+
+  // Specify the channel IDs you want to read messages from as an arry of strings
+  const channelIds = ["721374320794009630", "719596376261918720"];
+
+  // Specify the number of messages to fetch per channel
+  const limit = 10;
+
+  // Load messages from the specified channel
+  const messages = await discordReader.loadData(channelIds, limit, true);
+
+  // Print out the messages
+  console.log(messages);
+}
+
+main().catch(console.error);

examples/rerankers/CohereReranker.ts

@@ -15,9 +15,9 @@ async function main() {
 
   const index = await VectorStoreIndex.fromDocuments([document]);
 
-  const retriever = index.asRetriever();
-
-  retriever.similarityTopK = 5;
+  const retriever = index.asRetriever({
+    similarityTopK: 5,
+  });
 
   const nodePostprocessor = new CohereRerank({
     apiKey: "<COHERE_API_KEY>",

examples/vectorIndexCustomize.ts

@@ -18,8 +18,9 @@ async function main() {
 
   const index = await VectorStoreIndex.fromDocuments([document]);
 
-  const retriever = index.asRetriever();
-  retriever.similarityTopK = 5;
+  const retriever = index.asRetriever({
+    similarityTopK: 5,
+  });
   const nodePostprocessor = new SimilarityPostprocessor({
     similarityCutoff: 0.7,
   });

package.json

@@ -21,19 +21,19 @@
     "@changesets/cli": "^2.27.5",
     "@typescript-eslint/eslint-plugin": "^7.13.1",
     "eslint": "^8.57.0",
-    "eslint-config-next": "^14.2.4",
+    "eslint-config-next": "^14.2.5",
     "eslint-config-prettier": "^9.1.0",
     "eslint-config-turbo": "^2.0.5",
-    "eslint-plugin-react": "7.34.1",
+    "eslint-plugin-react": "7.34.3",
     "husky": "^9.0.11",
     "lint-staged": "^15.2.7",
     "madge": "^7.0.0",
     "prettier": "^3.3.2",
-    "prettier-plugin-organize-imports": "^3.2.4",
+    "prettier-plugin-organize-imports": "^4.0.0",
     "turbo": "^2.0.5",
-    "typescript": "^5.5.2"
+    "typescript": "^5.5.3"
   },
-  "packageManager": "pnpm@9.4.0",
+  "packageManager": "pnpm@9.5.0",
   "pnpm": {
     "overrides": {
       "trim": "1.0.1",

packages/autotool/examples/01_node/package.json

@@ -5,7 +5,7 @@
   "dependencies": {
     "@llamaindex/autotool": "workspace:*",
     "llamaindex": "workspace:*",
-    "openai": "^4.52.0"
+    "openai": "^4.52.5"
   },
   "devDependencies": {
     "tsx": "^4.15.6"

packages/autotool/examples/02_nextjs/CHANGELOG.md

@@ -1,5 +1,59 @@
 # @llamaindex/autotool-02-next-example
 
+## 0.1.31
+
+### Patch Changes
+
+- Updated dependencies [2562244]
+- Updated dependencies [325aa51]
+- Updated dependencies [ab700ea]
+- Updated dependencies [92f0782]
+- Updated dependencies [6cf6ae6]
+- Updated dependencies [b7cfe5b]
+  - llamaindex@0.5.6
+  - @llamaindex/autotool@2.0.0
+
+## 0.1.30
+
+### Patch Changes
+
+- Updated dependencies [b974eea]
+  - llamaindex@0.5.5
+  - @llamaindex/autotool@2.0.0
+
+## 0.1.29
+
+### Patch Changes
+
+- Updated dependencies [1a65ead]
+  - llamaindex@0.5.4
+  - @llamaindex/autotool@2.0.0
+
+## 0.1.28
+
+### Patch Changes
+
+- Updated dependencies [9bbbc67]
+- Updated dependencies [b3681bf]
+  - llamaindex@0.5.3
+  - @llamaindex/autotool@2.0.0
+
+## 0.1.27
+
+### Patch Changes
+
+- llamaindex@0.5.2
+- @llamaindex/autotool@2.0.0
+
+## 0.1.26
+
+### Patch Changes
+
+- Updated dependencies [2774681]
+- Updated dependencies [a0f424e]
+  - llamaindex@0.5.1
+  - @llamaindex/autotool@2.0.0
+
 ## 0.1.25
 
 ### Patch Changes

packages/autotool/examples/02_nextjs/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/autotool-02-next-example",
   "private": true,
-  "version": "0.1.25",
+  "version": "0.1.31",
   "scripts": {
     "dev": "next dev",
     "build": "next build",
@@ -14,7 +14,7 @@
     "class-variance-authority": "^0.7.0",
     "dotenv": "^16.3.1",
     "llamaindex": "workspace:*",
-    "lucide-react": "^0.378.0",
+    "lucide-react": "^0.407.0",
     "next": "14.3.0-canary.51",
     "react": "^18.3.1",
     "react-dom": "^18.3.1",
@@ -32,6 +32,6 @@
     "cross-env": "^7.0.3",
     "postcss": "^8.4.32",
     "tailwindcss": "^3.4.4",
-    "typescript": "^5.5.2"
+    "typescript": "^5.5.3"
   }
 }

packages/autotool/package.json

@@ -47,11 +47,11 @@
   "dependencies": {
     "@swc/core": "^1.6.3",
     "jotai": "^2.8.3",
-    "typedoc": "^0.25.13",
+    "typedoc": "^0.26.4",
     "unplugin": "^1.10.1"
   },
   "peerDependencies": {
-    "llamaindex": "^0.5.0",
+    "llamaindex": "^0.5.6",
     "openai": "^4",
     "typescript": "^4"
   },
@@ -72,11 +72,11 @@
     "@types/node": "^20.12.11",
     "bunchee": "5.3.0-beta.0",
     "llamaindex": "workspace:*",
-    "next": "14.2.3",
+    "next": "14.2.5",
     "rollup": "^4.18.0",
     "tsx": "^4.15.6",
-    "typescript": "^5.5.2",
-    "vitest": "^1.6.0",
+    "typescript": "^5.5.3",
+    "vitest": "^2.0.2",
     "webpack": "^5.92.1"
   }
 }

packages/cloud/CHANGELOG.md

@@ -1,5 +1,11 @@
 # @llamaindex/cloud
 
+## 0.2.0
+
+### Minor Changes
+
+- 3ed6acc: feat: cloud api change
+
 ## 0.1.4
 
 ### Patch Changes

packages/cloud/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/cloud",
-  "version": "0.1.4",
+  "version": "0.2.0",
   "type": "module",
   "license": "MIT",
   "scripts": {

packages/community/CHANGELOG.md

@@ -1,5 +1,32 @@
 # @llamaindex/community
 
+## 0.0.23
+
+### Patch Changes
+
+- Updated dependencies [6cf6ae6]
+  - @llamaindex/core@0.1.3
+
+## 0.0.22
+
+### Patch Changes
+
+- Updated dependencies [b974eea]
+  - @llamaindex/core@0.1.2
+
+## 0.0.21
+
+### Patch Changes
+
+- Updated dependencies [b3681bf]
+  - @llamaindex/core@0.1.1
+
+## 0.0.20
+
+### Patch Changes
+
+- 56746c2: fix: llama3 patched to handle empty content (can happen with system) and added max tokens export
+
 ## 0.0.19
 
 ### Patch Changes

packages/community/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/community",
   "description": "Community package for LlamaIndexTS",
-  "version": "0.0.19",
+  "version": "0.0.23",
   "type": "module",
   "types": "dist/type/index.d.ts",
   "main": "dist/cjs/index.js",
@@ -46,7 +46,7 @@
     "bunchee": "5.3.0-beta.0"
   },
   "dependencies": {
-    "@aws-sdk/client-bedrock-runtime": "^3.600.0",
+    "@aws-sdk/client-bedrock-runtime": "^3.613.0",
     "@llamaindex/core": "workspace:*"
   }
 }

packages/community/src/index.ts

@@ -1 +1,5 @@
-export { BEDROCK_MODELS, Bedrock } from "./llm/bedrock/base.js";
+export {
+  BEDROCK_MODELS,
+  BEDROCK_MODEL_MAX_TOKENS,
+  Bedrock,
+} from "./llm/bedrock/base.js";

packages/community/src/llm/bedrock/base.ts

@@ -150,6 +150,18 @@ export type BedrockModelParams = {
   maxTokens?: number;
 };
 
+export const BEDROCK_MODEL_MAX_TOKENS: Partial<Record<BEDROCK_MODELS, number>> =
+  {
+    [BEDROCK_MODELS.ANTHROPIC_CLAUDE_3_SONNET]: 4096,
+    [BEDROCK_MODELS.ANTHROPIC_CLAUDE_3_HAIKU]: 4096,
+    [BEDROCK_MODELS.ANTHROPIC_CLAUDE_3_OPUS]: 4096,
+    [BEDROCK_MODELS.ANTHROPIC_CLAUDE_3_5_SONNET]: 4096,
+    [BEDROCK_MODELS.META_LLAMA2_13B_CHAT]: 2048,
+    [BEDROCK_MODELS.META_LLAMA2_70B_CHAT]: 2048,
+    [BEDROCK_MODELS.META_LLAMA3_8B_INSTRUCT]: 2048,
+    [BEDROCK_MODELS.META_LLAMA3_70B_INSTRUCT]: 2048,
+  };
+
 const DEFAULT_BEDROCK_PARAMS = {
   temperature: 0.1,
   topP: 1,

packages/community/src/llm/bedrock/utils.ts

@@ -154,10 +154,10 @@ export const mapChatMessagesToMetaMessages = <T extends ChatMessage>(
   messages: T[],
 ): MetaMessage[] => {
   return messages.map((msg) => {
-    let content: string;
+    let content: string = "";
     if (typeof msg.content === "string") {
       content = msg.content;
-    } else {
+    } else if (msg.content.length) {
       content = (msg.content[0] as MessageContentTextDetail).text;
     }
     return {

packages/core/CHANGELOG.md

@@ -1,5 +1,23 @@
 # @llamaindex/core
 
+## 0.1.3
+
+### Patch Changes
+
+- 6cf6ae6: feat: abstract query type
+
+## 0.1.2
+
+### Patch Changes
+
+- b974eea: Add support for Metadata filters
+
+## 0.1.1
+
+### Patch Changes
+
+- b3681bf: fix: DataCloneError when using FunctionTool
+
 ## 0.1.0
 
 ### Minor Changes

packages/core/package.json

@@ -1,9 +1,23 @@
 {
   "name": "@llamaindex/core",
   "type": "module",
-  "version": "0.1.0",
+  "version": "0.1.3",
   "description": "LlamaIndex Core Module",
   "exports": {
+    "./query-engine": {
+      "require": {
+        "types": "./dist/query-engine/index.d.cts",
+        "default": "./dist/query-engine/index.cjs"
+      },
+      "import": {
+        "types": "./dist/query-engine/index.d.ts",
+        "default": "./dist/query-engine/index.js"
+      },
+      "default": {
+        "types": "./dist/query-engine/index.d.ts",
+        "default": "./dist/query-engine/index.js"
+      }
+    },
     "./llms": {
       "require": {
         "types": "./dist/llms/index.d.cts",
@@ -32,6 +46,20 @@
         "default": "./dist/decorator/index.js"
       }
     },
+    "./embeddings": {
+      "require": {
+        "types": "./dist/embeddings/index.d.cts",
+        "default": "./dist/embeddings/index.cjs"
+      },
+      "import": {
+        "types": "./dist/embeddings/index.d.ts",
+        "default": "./dist/embeddings/index.js"
+      },
+      "default": {
+        "types": "./dist/embeddings/index.d.ts",
+        "default": "./dist/embeddings/index.js"
+      }
+    },
     "./global": {
       "require": {
         "types": "./dist/global/index.d.cts",

packages/core/src/embeddings/base.ts

@@ -1,9 +1,8 @@
-import type { MessageContentDetail } from "@llamaindex/core/llms";
-import type { BaseNode } from "@llamaindex/core/schema";
-import { MetadataMode } from "@llamaindex/core/schema";
-import { extractSingleText } from "@llamaindex/core/utils";
 import { type Tokenizers } from "@llamaindex/env";
-import type { TransformComponent } from "../ingestion/types.js";
+import type { MessageContentDetail } from "../llms";
+import type { TransformComponent } from "../schema";
+import { BaseNode, MetadataMode } from "../schema";
+import { extractSingleText } from "../utils";
 import { truncateMaxTokens } from "./tokenizer.js";
 import { SimilarityType, similarity } from "./utils.js";
 
@@ -17,7 +16,13 @@ export type EmbeddingInfo = {
   tokenizer?: Tokenizers;
 };
 
-export abstract class BaseEmbedding implements TransformComponent {
+export type BaseEmbeddingOptions = {
+  logProgress?: boolean;
+};
+
+export abstract class BaseEmbedding
+  implements TransformComponent<BaseEmbeddingOptions>
+{
   embedBatchSize = DEFAULT_EMBED_BATCH_SIZE;
   embedInfo?: EmbeddingInfo;
 
@@ -45,7 +50,7 @@ export abstract class BaseEmbedding implements TransformComponent {
    * Optionally override this method to retrieve multiple embeddings in a single request
    * @param texts
    */
-  async getTextEmbeddings(texts: string[]): Promise<Array<number[]>> {
+  getTextEmbeddings = async (texts: string[]): Promise<Array<number[]>> => {
     const embeddings: number[][] = [];
 
     for (const text of texts) {
@@ -54,7 +59,7 @@ export abstract class BaseEmbedding implements TransformComponent {
     }
 
     return embeddings;
-  }
+  };
 
   /**
    * Get embeddings for a batch of texts
@@ -63,22 +68,23 @@ export abstract class BaseEmbedding implements TransformComponent {
    */
   async getTextEmbeddingsBatch(
     texts: string[],
-    options?: {
-      logProgress?: boolean;
-    },
+    options?: BaseEmbeddingOptions,
   ): Promise<Array<number[]>> {
     return await batchEmbeddings(
       texts,
-      this.getTextEmbeddings.bind(this),
+      this.getTextEmbeddings,
       this.embedBatchSize,
       options,
     );
   }
 
-  async transform(nodes: BaseNode[], _options?: any): Promise<BaseNode[]> {
+  async transform(
+    nodes: BaseNode[],
+    options?: BaseEmbeddingOptions,
+  ): Promise<BaseNode[]> {
     const texts = nodes.map((node) => node.getContent(MetadataMode.EMBED));
 
-    const embeddings = await this.getTextEmbeddingsBatch(texts, _options);
+    const embeddings = await this.getTextEmbeddingsBatch(texts, options);
 
     for (let i = 0; i < nodes.length; i++) {
       nodes[i].embedding = embeddings[i];
@@ -104,9 +110,7 @@ export async function batchEmbeddings<T>(
   values: T[],
   embedFunc: EmbedFunc<T>,
   chunkSize: number,
-  options?: {
-    logProgress?: boolean;
-  },
+  options?: BaseEmbeddingOptions,
 ): Promise<Array<number[]>> {
   const resultEmbeddings: Array<number[]> = [];
 

packages/core/src/embeddings/index.ts

@@ -0,0 +1,4 @@
+export { BaseEmbedding, batchEmbeddings } from "./base";
+export type { BaseEmbeddingOptions, EmbeddingInfo } from "./base";
+export { truncateMaxTokens } from "./tokenizer";
+export { DEFAULT_SIMILARITY_TOP_K, SimilarityType, similarity } from "./utils";

packages/core/src/embeddings/utils.ts

@@ -0,0 +1,64 @@
+export const DEFAULT_SIMILARITY_TOP_K = 2;
+
+/**
+ * Similarity type
+ * Default is cosine similarity. Dot product and negative Euclidean distance are also supported.
+ */
+export enum SimilarityType {
+  DEFAULT = "cosine",
+  DOT_PRODUCT = "dot_product",
+  EUCLIDEAN = "euclidean",
+}
+
+/**
+ * The similarity between two embeddings.
+ * @param embedding1
+ * @param embedding2
+ * @param mode
+ * @returns similarity score with higher numbers meaning the two embeddings are more similar
+ */
+
+export function similarity(
+  embedding1: number[],
+  embedding2: number[],
+  mode: SimilarityType = SimilarityType.DEFAULT,
+): number {
+  if (embedding1.length !== embedding2.length) {
+    throw new Error("Embedding length mismatch");
+  }
+
+  // NOTE I've taken enough Kahan to know that we should probably leave the
+  // numeric programming to numeric programmers. The naive approach here
+  // will probably cause some avoidable loss of floating point precision
+  // ml-distance is worth watching although they currently also use the naive
+  // formulas
+  function norm(x: number[]): number {
+    let result = 0;
+    for (let i = 0; i < x.length; i++) {
+      result += x[i] * x[i];
+    }
+    return Math.sqrt(result);
+  }
+
+  switch (mode) {
+    case SimilarityType.EUCLIDEAN: {
+      const difference = embedding1.map((x, i) => x - embedding2[i]);
+      return -norm(difference);
+    }
+    case SimilarityType.DOT_PRODUCT: {
+      let result = 0;
+      for (let i = 0; i < embedding1.length; i++) {
+        result += embedding1[i] * embedding2[i];
+      }
+      return result;
+    }
+    case SimilarityType.DEFAULT: {
+      return (
+        similarity(embedding1, embedding2, SimilarityType.DOT_PRODUCT) /
+        (norm(embedding1) * norm(embedding2))
+      );
+    }
+    default:
+      throw new Error("Not implemented yet");
+  }
+}

packages/core/src/global/settings/callback-manager.ts

@@ -105,9 +105,7 @@ export class CallbackManager {
     }
     queueMicrotask(() => {
       cbs.forEach((handler) =>
-        handler(
-          LlamaIndexCustomEvent.fromEvent(event, structuredClone(detail)),
-        ),
+        handler(LlamaIndexCustomEvent.fromEvent(event, { ...detail })),
       );
     });
   }

packages/core/src/query-engine/base.ts

@@ -0,0 +1,29 @@
+import type { MessageContent } from "../llms";
+import { EngineResponse, type NodeWithScore } from "../schema";
+
+/**
+ * @link https://docs.llamaindex.ai/en/stable/api_reference/schema/?h=querybundle#llama_index.core.schema.QueryBundle
+ *
+ *  We don't have `image_path` here, because it is included in the `query` field.
+ */
+export type QueryBundle = {
+  query: MessageContent;
+  customEmbeddings?: string[];
+  embeddings?: number[];
+};
+
+export type QueryType = string | QueryBundle;
+
+export interface BaseQueryEngine {
+  query(
+    strOrQueryBundle: QueryType,
+    stream: true,
+  ): Promise<AsyncIterable<EngineResponse>>;
+  query(strOrQueryBundle: QueryType, stream?: false): Promise<EngineResponse>;
+
+  synthesize?(
+    strOrQueryBundle: QueryType,
+    nodes: NodeWithScore[],
+    additionalSources?: Iterator<NodeWithScore>,
+  ): Promise<EngineResponse>;
+}

packages/core/src/query-engine/index.ts

@@ -0,0 +1 @@
+export type { BaseQueryEngine, QueryBundle, QueryType } from "./base";

packages/core/src/schema/index.ts

@@ -1,2 +1,4 @@
 export * from "./node";
+export type { TransformComponent } from "./type";
+export { EngineResponse } from "./type/engine–response";
 export * from "./zod";

packages/core/src/schema/type.ts

@@ -0,0 +1,5 @@
+import type { BaseNode } from "./node";
+
+export interface TransformComponent<Options extends Record<string, unknown>> {
+  transform(nodes: BaseNode[], options?: Options): Promise<BaseNode[]>;
+}

packages/core/src/schema/type/engine–response.ts

@@ -1,20 +1,16 @@
-import type {
-  ChatMessage,
-  ChatResponse,
-  ChatResponseChunk,
-} from "@llamaindex/core/llms";
-import type { NodeWithScore } from "@llamaindex/core/schema";
-import { extractText } from "@llamaindex/core/utils";
+import type { ChatMessage, ChatResponse, ChatResponseChunk } from "../../llms";
+import { extractText } from "../../utils";
+import type { Metadata, NodeWithScore } from "../node";
 
 export class EngineResponse implements ChatResponse, ChatResponseChunk {
   sourceNodes?: NodeWithScore[];
 
-  metadata: Record<string, unknown> = {};
+  metadata: Metadata = {};
 
   message: ChatMessage;
   raw: object | null;
 
-  #stream: boolean;
+  readonly stream: boolean;
 
   private constructor(
     chatResponse: ChatResponse,
@@ -24,7 +20,7 @@ export class EngineResponse implements ChatResponse, ChatResponseChunk {
     this.message = chatResponse.message;
     this.raw = chatResponse.raw;
     this.sourceNodes = sourceNodes;
-    this.#stream = stream;
+    this.stream = stream;
   }
 
   static fromResponse(
@@ -70,13 +66,15 @@ export class EngineResponse implements ChatResponse, ChatResponseChunk {
     );
   }
 
-  // @deprecated use 'message' instead
+  /**
+   * @deprecated Use `message` instead.
+   */
   get response(): string {
     return extractText(this.message.content);
   }
 
   get delta(): string {
-    if (!this.#stream) {
+    if (!this.stream) {
       console.warn(
         "delta is only available for streaming responses. Consider using 'message' instead.",
       );
@@ -84,7 +82,7 @@ export class EngineResponse implements ChatResponse, ChatResponseChunk {
     return extractText(this.message.content);
   }
 
-  toString() {
+  toString(): string {
     return this.response ?? "";
   }
 }

packages/core/src/utils/llms.ts

@@ -3,15 +3,22 @@ import type {
   MessageContentDetail,
   MessageContentTextDetail,
 } from "../llms";
+import type { QueryType } from "../query-engine";
 import type { ImageType } from "../schema";
 
 /**
- * Extracts just the text from a multi-modal message or the message itself if it's just text.
+ * Extracts just the text whether from
+ *  a multi-modal message
+ *  a single text message
+ *  or a query
  *
  * @param message The message to extract text from.
  * @returns The extracted text
  */
-export function extractText(message: MessageContent): string {
+export function extractText(message: MessageContent | QueryType): string {
+  if (typeof message === "object" && "query" in message) {
+    return extractText(message.query);
+  }
   if (typeof message !== "string" && !Array.isArray(message)) {
     console.warn(
       "extractText called with non-MessageContent message, this is likely a bug.",

packages/core/tests/embeddings.test.ts

@@ -1,6 +1,6 @@
+import { truncateMaxTokens } from "@llamaindex/core/embeddings";
 import { Tokenizers, tokenizers } from "@llamaindex/env";
 import { describe, expect, test } from "vitest";
-import { truncateMaxTokens } from "../../src/embeddings/tokenizer.js";
 
 describe("truncateMaxTokens", () => {
   const tokenizer = tokenizers.tokenizer(Tokenizers.CL100K_BASE);

packages/core/tests/event-system.test.ts

@@ -6,6 +6,9 @@ declare module "@llamaindex/core/global" {
     test: {
       value: number;
     };
+    functionTest: {
+      fn: ({ x }: { x: number }) => string;
+    };
   }
 }
 
@@ -42,6 +45,25 @@ describe("event system", () => {
     expect(callback).toHaveBeenCalledTimes(1);
   });
 
+  test("dispatch function tool event", async () => {
+    const testFunction = ({ x }: { x: number }) => `${x * 2}`;
+    let callback;
+    Settings.callbackManager.on(
+      "functionTest",
+      (callback = vi.fn((event) => {
+        const data = event.detail;
+        expect(data.fn).toBe(testFunction);
+      })),
+    );
+
+    Settings.callbackManager.dispatchEvent("functionTest", {
+      fn: testFunction,
+    });
+    expect(callback).toHaveBeenCalledTimes(0);
+    await new Promise((resolve) => process.nextTick(resolve));
+    expect(callback).toHaveBeenCalledTimes(1);
+  });
+
   // rollup doesn't support decorators for now
   // test('wrap event caller', async () => {
   //   class A {

packages/core/tests/package.json

@@ -7,6 +7,6 @@
   },
   "devDependencies": {
     "@llamaindex/core": "workspace:*",
-    "vitest": "^1.6.0"
+    "vitest": "^2.0.2"
   }
 }

packages/env/package.json

@@ -68,11 +68,11 @@
   },
   "devDependencies": {
     "@aws-crypto/sha256-js": "^5.2.0",
-    "@swc/cli": "^0.3.12",
+    "@swc/cli": "^0.4.0",
     "@swc/core": "^1.6.3",
     "concurrently": "^8.2.2",
     "pathe": "^1.1.2",
-    "vitest": "^1.6.0"
+    "vitest": "^2.0.2"
   },
   "dependencies": {
     "@types/lodash": "^4.17.5",

packages/experimental/CHANGELOG.md

@@ -1,5 +1,53 @@
 # @llamaindex/experimental
 
+## 0.0.56
+
+### Patch Changes
+
+- Updated dependencies [2562244]
+- Updated dependencies [325aa51]
+- Updated dependencies [ab700ea]
+- Updated dependencies [92f0782]
+- Updated dependencies [6cf6ae6]
+- Updated dependencies [b7cfe5b]
+  - llamaindex@0.5.6
+
+## 0.0.55
+
+### Patch Changes
+
+- Updated dependencies [b974eea]
+  - llamaindex@0.5.5
+
+## 0.0.54
+
+### Patch Changes
+
+- Updated dependencies [1a65ead]
+  - llamaindex@0.5.4
+
+## 0.0.53
+
+### Patch Changes
+
+- Updated dependencies [9bbbc67]
+- Updated dependencies [b3681bf]
+  - llamaindex@0.5.3
+
+## 0.0.52
+
+### Patch Changes
+
+- llamaindex@0.5.2
+
+## 0.0.51
+
+### Patch Changes
+
+- Updated dependencies [2774681]
+- Updated dependencies [a0f424e]
+  - llamaindex@0.5.1
+
 ## 0.0.50
 
 ### Patch Changes

packages/experimental/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/experimental",
   "description": "Experimental package for LlamaIndexTS",
-  "version": "0.0.50",
+  "version": "0.0.56",
   "type": "module",
   "types": "dist/type/index.d.ts",
   "main": "dist/cjs/index.js",
@@ -56,7 +56,7 @@
   },
   "devDependencies": {
     "@aws-crypto/sha256-js": "^5.2.0",
-    "@swc/cli": "^0.3.12",
+    "@swc/cli": "^0.4.0",
     "@swc/core": "^1.6.3",
     "@types/jsonpath": "^0.2.4",
     "concurrently": "^8.2.2",

packages/llamaindex/CHANGELOG.md

@@ -1,5 +1,55 @@
 # llamaindex
 
+## 0.5.6
+
+### Patch Changes
+
+- 2562244: feat: add gpt4o-mini
+- 325aa51: Implement Jina embedding through Jina api
+- ab700ea: Add missing authentication to LlamaCloudIndex.fromDocuments
+- 92f0782: feat: use query bundle
+- 6cf6ae6: feat: abstract query type
+- b7cfe5b: fix: passing max_token option to replicate's api call
+- Updated dependencies [6cf6ae6]
+  - @llamaindex/core@0.1.3
+
+## 0.5.5
+
+### Patch Changes
+
+- b974eea: Add support for Metadata filters
+- Updated dependencies [b974eea]
+  - @llamaindex/core@0.1.2
+
+## 0.5.4
+
+### Patch Changes
+
+- 1a65ead: feat: add vendorMultimodal params to LlamaParseReader
+
+## 0.5.3
+
+### Patch Changes
+
+- 9bbbc67: feat: add a reader for Discord messages
+- b3681bf: fix: DataCloneError when using FunctionTool
+- Updated dependencies [b3681bf]
+  - @llamaindex/core@0.1.1
+
+## 0.5.2
+
+### Patch Changes
+
+- Updated dependencies [3ed6acc]
+  - @llamaindex/cloud@0.2.0
+
+## 0.5.1
+
+### Patch Changes
+
+- 2774681: Add mixedbread's embeddings and reranking API
+- a0f424e: corrected the regex in the react.ts file in extractToolUse & extractJsonStr functions, as mentioned in https://github.com/run-llama/LlamaIndexTS/issues/1019
+
 ## 0.5.0
 
 ### Minor Changes

packages/llamaindex/e2e/examples/cloudflare-worker-agent/CHANGELOG.md

@@ -1,5 +1,53 @@
 # @llamaindex/cloudflare-worker-agent-test
 
+## 0.0.40
+
+### Patch Changes
+
+- Updated dependencies [2562244]
+- Updated dependencies [325aa51]
+- Updated dependencies [ab700ea]
+- Updated dependencies [92f0782]
+- Updated dependencies [6cf6ae6]
+- Updated dependencies [b7cfe5b]
+  - llamaindex@0.5.6
+
+## 0.0.39
+
+### Patch Changes
+
+- Updated dependencies [b974eea]
+  - llamaindex@0.5.5
+
+## 0.0.38
+
+### Patch Changes
+
+- Updated dependencies [1a65ead]
+  - llamaindex@0.5.4
+
+## 0.0.37
+
+### Patch Changes
+
+- Updated dependencies [9bbbc67]
+- Updated dependencies [b3681bf]
+  - llamaindex@0.5.3
+
+## 0.0.36
+
+### Patch Changes
+
+- llamaindex@0.5.2
+
+## 0.0.35
+
+### Patch Changes
+
+- Updated dependencies [2774681]
+- Updated dependencies [a0f424e]
+  - llamaindex@0.5.1
+
 ## 0.0.34
 
 ### Patch Changes

packages/llamaindex/e2e/examples/cloudflare-worker-agent/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/cloudflare-worker-agent-test",
-  "version": "0.0.34",
+  "version": "0.0.40",
   "type": "module",
   "private": true,
   "scripts": {
@@ -12,13 +12,13 @@
     "cf-typegen": "wrangler types"
   },
   "devDependencies": {
-    "@cloudflare/vitest-pool-workers": "^0.4.3",
-    "@cloudflare/workers-types": "^4.20240605.0",
-    "@vitest/runner": "1.3.0",
-    "@vitest/snapshot": "1.3.0",
-    "typescript": "^5.5.2",
-    "vitest": "1.3.0",
-    "wrangler": "^3.60.1"
+    "@cloudflare/vitest-pool-workers": "^0.4.10",
+    "@cloudflare/workers-types": "^4.20240620.0",
+    "@vitest/runner": "1.5.3",
+    "@vitest/snapshot": "1.5.3",
+    "typescript": "^5.5.3",
+    "vitest": "1.5.3",
+    "wrangler": "^3.63.2"
   },
   "dependencies": {
     "llamaindex": "workspace:*"

packages/llamaindex/e2e/examples/nextjs-agent/CHANGELOG.md

@@ -1,5 +1,53 @@
 # @llamaindex/next-agent-test
 
+## 0.1.40
+
+### Patch Changes
+
+- Updated dependencies [2562244]
+- Updated dependencies [325aa51]
+- Updated dependencies [ab700ea]
+- Updated dependencies [92f0782]
+- Updated dependencies [6cf6ae6]
+- Updated dependencies [b7cfe5b]
+  - llamaindex@0.5.6
+
+## 0.1.39
+
+### Patch Changes
+
+- Updated dependencies [b974eea]
+  - llamaindex@0.5.5
+
+## 0.1.38
+
+### Patch Changes
+
+- Updated dependencies [1a65ead]
+  - llamaindex@0.5.4
+
+## 0.1.37
+
+### Patch Changes
+
+- Updated dependencies [9bbbc67]
+- Updated dependencies [b3681bf]
+  - llamaindex@0.5.3
+
+## 0.1.36
+
+### Patch Changes
+
+- llamaindex@0.5.2
+
+## 0.1.35
+
+### Patch Changes
+
+- Updated dependencies [2774681]
+- Updated dependencies [a0f424e]
+  - llamaindex@0.5.1
+
 ## 0.1.34
 
 ### Patch Changes

packages/llamaindex/e2e/examples/nextjs-agent/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/next-agent-test",
-  "version": "0.1.34",
+  "version": "0.1.40",
   "private": true,
   "scripts": {
     "dev": "next dev",
@@ -11,7 +11,7 @@
   "dependencies": {
     "ai": "^3.2.1",
     "llamaindex": "workspace:*",
-    "next": "14.2.4",
+    "next": "14.2.5",
     "react": "18.3.1",
     "react-dom": "18.3.1"
   },
@@ -20,9 +20,9 @@
     "@types/react": "^18.3.3",
     "@types/react-dom": "^18.3.0",
     "eslint": "^8.57.0",
-    "eslint-config-next": "14.2.3",
+    "eslint-config-next": "14.2.5",
     "postcss": "^8",
     "tailwindcss": "^3.4.4",
-    "typescript": "^5.5.2"
+    "typescript": "^5.5.3"
   }
 }

packages/llamaindex/e2e/examples/nextjs-edge-runtime/CHANGELOG.md

@@ -1,5 +1,53 @@
 # test-edge-runtime
 
+## 0.1.39
+
+### Patch Changes
+
+- Updated dependencies [2562244]
+- Updated dependencies [325aa51]
+- Updated dependencies [ab700ea]
+- Updated dependencies [92f0782]
+- Updated dependencies [6cf6ae6]
+- Updated dependencies [b7cfe5b]
+  - llamaindex@0.5.6
+
+## 0.1.38
+
+### Patch Changes
+
+- Updated dependencies [b974eea]
+  - llamaindex@0.5.5
+
+## 0.1.37
+
+### Patch Changes
+
+- Updated dependencies [1a65ead]
+  - llamaindex@0.5.4
+
+## 0.1.36
+
+### Patch Changes
+
+- Updated dependencies [9bbbc67]
+- Updated dependencies [b3681bf]
+  - llamaindex@0.5.3
+
+## 0.1.35
+
+### Patch Changes
+
+- llamaindex@0.5.2
+
+## 0.1.34
+
+### Patch Changes
+
+- Updated dependencies [2774681]
+- Updated dependencies [a0f424e]
+  - llamaindex@0.5.1
+
 ## 0.1.33
 
 ### Patch Changes

packages/llamaindex/e2e/examples/nextjs-edge-runtime/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/nextjs-edge-runtime-test",
-  "version": "0.1.33",
+  "version": "0.1.39",
   "private": true,
   "scripts": {
     "dev": "next dev",
@@ -9,7 +9,7 @@
   },
   "dependencies": {
     "llamaindex": "workspace:*",
-    "next": "14.2.4",
+    "next": "14.2.5",
     "react": "^18.3.1",
     "react-dom": "^18.3.1"
   },
@@ -17,6 +17,6 @@
     "@types/node": "^20.12.11",
     "@types/react": "^18.3.3",
     "@types/react-dom": "^18.3.0",
-    "typescript": "^5.5.2"
+    "typescript": "^5.5.3"
   }
 }

packages/llamaindex/e2e/examples/nextjs-node-runtime/CHANGELOG.md

@@ -1,5 +1,53 @@
 # @llamaindex/next-node-runtime
 
+## 0.0.21
+
+### Patch Changes
+
+- Updated dependencies [2562244]
+- Updated dependencies [325aa51]
+- Updated dependencies [ab700ea]
+- Updated dependencies [92f0782]
+- Updated dependencies [6cf6ae6]
+- Updated dependencies [b7cfe5b]
+  - llamaindex@0.5.6
+
+## 0.0.20
+
+### Patch Changes
+
+- Updated dependencies [b974eea]
+  - llamaindex@0.5.5
+
+## 0.0.19
+
+### Patch Changes
+
+- Updated dependencies [1a65ead]
+  - llamaindex@0.5.4
+
+## 0.0.18
+
+### Patch Changes
+
+- Updated dependencies [9bbbc67]
+- Updated dependencies [b3681bf]
+  - llamaindex@0.5.3
+
+## 0.0.17
+
+### Patch Changes
+
+- llamaindex@0.5.2
+
+## 0.0.16
+
+### Patch Changes
+
+- Updated dependencies [2774681]
+- Updated dependencies [a0f424e]
+  - llamaindex@0.5.1
+
 ## 0.0.15
 
 ### Patch Changes

packages/llamaindex/e2e/examples/nextjs-node-runtime/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/next-node-runtime-test",
-  "version": "0.0.15",
+  "version": "0.0.21",
   "private": true,
   "scripts": {
     "dev": "next dev",
@@ -10,7 +10,7 @@
   },
   "dependencies": {
     "llamaindex": "workspace:*",
-    "next": "14.2.4",
+    "next": "14.2.5",
     "react": "18.3.1",
     "react-dom": "18.3.1"
   },
@@ -19,9 +19,9 @@
     "@types/react": "^18.3.3",
     "@types/react-dom": "^18.3.0",
     "eslint": "^8.57.0",
-    "eslint-config-next": "14.2.3",
+    "eslint-config-next": "14.2.5",
     "postcss": "^8",
     "tailwindcss": "^3.4.4",
-    "typescript": "^5.5.2"
+    "typescript": "^5.5.3"
   }
 }

packages/llamaindex/e2e/examples/waku-query-engine/CHANGELOG.md

@@ -1,5 +1,53 @@
 # @llamaindex/waku-query-engine-test
 
+## 0.0.40
+
+### Patch Changes
+
+- Updated dependencies [2562244]
+- Updated dependencies [325aa51]
+- Updated dependencies [ab700ea]
+- Updated dependencies [92f0782]
+- Updated dependencies [6cf6ae6]
+- Updated dependencies [b7cfe5b]
+  - llamaindex@0.5.6
+
+## 0.0.39
+
+### Patch Changes
+
+- Updated dependencies [b974eea]
+  - llamaindex@0.5.5
+
+## 0.0.38
+
+### Patch Changes
+
+- Updated dependencies [1a65ead]
+  - llamaindex@0.5.4
+
+## 0.0.37
+
+### Patch Changes
+
+- Updated dependencies [9bbbc67]
+- Updated dependencies [b3681bf]
+  - llamaindex@0.5.3
+
+## 0.0.36
+
+### Patch Changes
+
+- llamaindex@0.5.2
+
+## 0.0.35
+
+### Patch Changes
+
+- Updated dependencies [2774681]
+- Updated dependencies [a0f424e]
+  - llamaindex@0.5.1
+
 ## 0.0.34
 
 ### Patch Changes

packages/llamaindex/e2e/examples/waku-query-engine/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/waku-query-engine-test",
-  "version": "0.0.34",
+  "version": "0.0.40",
   "type": "module",
   "private": true,
   "scripts": {
@@ -10,16 +10,16 @@
   },
   "dependencies": {
     "llamaindex": "workspace:*",
-    "react": "19.0.0-canary-e3ebcd54b-20240405",
-    "react-dom": "19.0.0-canary-e3ebcd54b-20240405",
-    "react-server-dom-webpack": "19.0.0-canary-e3ebcd54b-20240405",
-    "waku": "0.20.1"
+    "react": "19.0.0-beta-e7d213dfb0-20240507",
+    "react-dom": "19.0.0-beta-e7d213dfb0-20240507",
+    "react-server-dom-webpack": "19.0.0-beta-e7d213dfb0-20240507",
+    "waku": "0.20.2"
   },
   "devDependencies": {
-    "@types/react": "18.3.1",
+    "@types/react": "18.3.3",
     "@types/react-dom": "18.3.0",
     "autoprefixer": "10.4.19",
-    "tailwindcss": "3.4.3",
-    "typescript": "5.4.5"
+    "tailwindcss": "3.4.4",
+    "typescript": "5.5.3"
   }
 }

packages/llamaindex/package.json

@@ -1,6 +1,6 @@
 {
   "name": "llamaindex",
-  "version": "0.5.0",
+  "version": "0.5.6",
   "license": "MIT",
   "type": "module",
   "keywords": [
@@ -20,18 +20,20 @@
     "llamaindex"
   ],
   "dependencies": {
-    "@anthropic-ai/sdk": "^0.21.1",
+    "@anthropic-ai/sdk": "0.21.1",
     "@aws-crypto/sha256-js": "^5.2.0",
     "@azure/identity": "^4.2.1",
     "@datastax/astra-db-ts": "^1.2.1",
+    "@discordjs/rest": "^2.3.0",
     "@google-cloud/vertexai": "^1.2.0",
-    "@google/generative-ai": "^0.12.0",
-    "@grpc/grpc-js": "^1.10.8",
+    "@google/generative-ai": "0.12.0",
+    "@grpc/grpc-js": "^1.10.11",
     "@huggingface/inference": "^2.7.0",
     "@llamaindex/cloud": "workspace:*",
     "@llamaindex/core": "workspace:*",
     "@llamaindex/env": "workspace:*",
-    "@mistralai/mistralai": "^0.4.0",
+    "@mistralai/mistralai": "^0.5.0",
+    "@mixedbread-ai/sdk": "^2.2.11",
     "@pinecone-database/pinecone": "^2.2.2",
     "@qdrant/js-client-rest": "^1.9.0",
     "@types/lodash": "^4.17.4",
@@ -39,11 +41,12 @@
     "@types/papaparse": "^5.3.14",
     "@types/pg": "^8.11.6",
     "@xenova/transformers": "^2.17.2",
-    "@zilliz/milvus2-sdk-node": "^2.4.2",
+    "@zilliz/milvus2-sdk-node": "^2.4.4",
     "ajv": "^8.16.0",
-    "assemblyai": "^4.4.5",
+    "assemblyai": "^4.6.0",
     "chromadb": "1.8.1",
-    "cohere-ai": "7.9.5",
+    "cohere-ai": "7.10.6",
+    "discord-api-types": "^0.37.92",
     "groq-sdk": "^0.5.0",
     "js-tiktoken": "^1.0.12",
     "lodash": "^4.17.21",
@@ -52,16 +55,16 @@
     "md-utils-ts": "^2.0.0",
     "mongodb": "^6.7.0",
     "notion-md-crawler": "^1.0.0",
-    "openai": "^4.52.0",
+    "openai": "^4.52.5",
     "papaparse": "^5.4.1",
     "pathe": "^1.1.2",
     "pg": "^8.12.0",
-    "pgvector": "^0.1.8",
-    "portkey-ai": "^0.1.16",
+    "pgvector": "^0.2.0",
+    "portkey-ai": "0.1.16",
     "rake-modified": "^1.0.8",
     "string-strip-html": "^13.4.8",
     "tiktoken": "^1.0.15",
-    "unpdf": "^0.10.1",
+    "unpdf": "^0.11.0",
     "wikipedia": "^2.1.2",
     "wink-nlp": "^2.3.0",
     "zod": "^3.23.8"
@@ -76,11 +79,11 @@
   },
   "devDependencies": {
     "@notionhq/client": "^2.2.15",
-    "@swc/cli": "^0.3.12",
+    "@swc/cli": "^0.4.0",
     "@swc/core": "^1.6.3",
     "concurrently": "^8.2.2",
-    "glob": "^10.4.2",
-    "typescript": "^5.5.2"
+    "glob": "^11.0.0",
+    "typescript": "^5.5.3"
   },
   "engines": {
     "node": ">=18.0.0"

packages/llamaindex/src/QuestionGenerator.ts

@@ -1,4 +1,6 @@
 import type { LLM, ToolMetadata } from "@llamaindex/core/llms";
+import type { QueryType } from "@llamaindex/core/query-engine";
+import { extractText } from "@llamaindex/core/utils";
 import { SubQuestionOutputParser } from "./OutputParser.js";
 import type { SubQuestionPrompt } from "./Prompt.js";
 import { buildToolsText, defaultSubQuestionPrompt } from "./Prompt.js";
@@ -43,9 +45,12 @@ export class LLMQuestionGenerator
     }
   }
 
-  async generate(tools: ToolMetadata[], query: string): Promise<SubQuestion[]> {
+  async generate(
+    tools: ToolMetadata[],
+    query: QueryType,
+  ): Promise<SubQuestion[]> {
     const toolsStr = buildToolsText(tools);
-    const queryStr = query;
+    const queryStr = extractText(query);
     const prediction = (
       await this.llm.complete({
         prompt: this.prompt({

packages/llamaindex/src/ServiceContext.ts

@@ -1,7 +1,7 @@
+import type { BaseEmbedding } from "@llamaindex/core/embeddings";
 import type { LLM } from "@llamaindex/core/llms";
 import { PromptHelper } from "./PromptHelper.js";
 import { OpenAIEmbedding } from "./embeddings/OpenAIEmbedding.js";
-import type { BaseEmbedding } from "./embeddings/types.js";
 import { OpenAI } from "./llm/openai.js";
 import { SimpleNodeParser } from "./nodeParsers/SimpleNodeParser.js";
 import type { NodeParser } from "./nodeParsers/types.js";

packages/llamaindex/src/Settings.ts

@@ -7,10 +7,10 @@ import { OpenAI } from "./llm/openai.js";
 import { PromptHelper } from "./PromptHelper.js";
 import { SimpleNodeParser } from "./nodeParsers/SimpleNodeParser.js";
 
+import type { BaseEmbedding } from "@llamaindex/core/embeddings";
 import type { LLM } from "@llamaindex/core/llms";
 import { AsyncLocalStorage, getEnv } from "@llamaindex/env";
 import type { ServiceContext } from "./ServiceContext.js";
-import type { BaseEmbedding } from "./embeddings/types.js";
 import {
   getEmbeddedModel,
   setEmbeddedModel,

packages/llamaindex/src/agent/base.ts

@@ -5,10 +5,10 @@ import type {
   MessageContent,
   ToolOutput,
 } from "@llamaindex/core/llms";
+import { EngineResponse } from "@llamaindex/core/schema";
 import { wrapEventCaller } from "@llamaindex/core/utils";
 import { ReadableStream, TransformStream, randomUUID } from "@llamaindex/env";
 import { ChatHistory } from "../ChatHistory.js";
-import { EngineResponse } from "../EngineResponse.js";
 import { Settings } from "../Settings.js";
 import {
   type ChatEngine,

packages/llamaindex/src/agent/react.ts

@@ -66,7 +66,7 @@ function reasonFormatter(reason: Reason): string | Promise<string> {
 }
 
 function extractJsonStr(text: string): string {
-  const pattern = /\{.*}/s;
+  const pattern = /\{.*\}/s;
   const match = text.match(pattern);
 
   if (!match) {
@@ -98,7 +98,7 @@ function extractToolUse(
   inputText: string,
 ): [thought: string, action: string, input: string] {
   const pattern =
-    /\s*Thought: (.*?)\nAction: ([a-zA-Z0-9_]+).*?\.*Input: .*?(\{.*?})/s;
+    /\s*Thought: (.*?)\nAction: ([a-zA-Z0-9_]+).*?\.*Input: .*?(\{.*?\})/s;
 
   const match = inputText.match(pattern);
 

packages/llamaindex/src/cloud/LlamaCloudIndex.ts

@@ -1,7 +1,6 @@
-import type { Document } from "@llamaindex/core/schema";
+import type { Document, TransformComponent } from "@llamaindex/core/schema";
 import type { BaseRetriever } from "../Retriever.js";
 import { RetrieverQueryEngine } from "../engines/query/RetrieverQueryEngine.js";
-import type { TransformComponent } from "../ingestion/types.js";
 import type { BaseNodePostprocessor } from "../postprocessors/types.js";
 import type { BaseSynthesizer } from "../synthesizers/types.js";
 import type { QueryEngine } from "../types.js";
@@ -148,11 +147,12 @@ export class LlamaCloudIndex {
   static async fromDocuments(
     params: {
       documents: Document[];
-      transformations?: TransformComponent[];
+      transformations?: TransformComponent<any>[];
       verbose?: boolean;
     } & CloudConstructorParams,
   ): Promise<LlamaCloudIndex> {
-    const defaultTransformations: TransformComponent[] = [
+    initService(params);
+    const defaultTransformations: TransformComponent<any>[] = [
       new SimpleNodeParser(),
       new OpenAIEmbedding({
         apiKey: getEnv("OPENAI_API_KEY"),

packages/llamaindex/src/cloud/config.ts

@@ -3,20 +3,19 @@ import type {
   PipelineCreate,
   PipelineType,
 } from "@llamaindex/cloud/api";
-import { BaseNode } from "@llamaindex/core/schema";
+import { BaseNode, type TransformComponent } from "@llamaindex/core/schema";
 import { OpenAIEmbedding } from "../embeddings/OpenAIEmbedding.js";
-import type { TransformComponent } from "../ingestion/types.js";
 import { SimpleNodeParser } from "../nodeParsers/SimpleNodeParser.js";
 
 export type GetPipelineCreateParams = {
   pipelineName: string;
   pipelineType: PipelineType;
-  transformations?: TransformComponent[];
+  transformations?: TransformComponent<any>[];
   inputNodes?: BaseNode[];
 };
 
 function getTransformationConfig(
-  transformation: TransformComponent,
+  transformation: TransformComponent<any>,
 ): ConfiguredTransformationItem {
   if (transformation instanceof SimpleNodeParser) {
     return {

packages/llamaindex/src/constants.ts

@@ -4,6 +4,5 @@ export const DEFAULT_NUM_OUTPUTS = 256;
 export const DEFAULT_CHUNK_SIZE = 1024;
 export const DEFAULT_CHUNK_OVERLAP = 20;
 export const DEFAULT_CHUNK_OVERLAP_RATIO = 0.1;
-export const DEFAULT_SIMILARITY_TOP_K = 2;
 
 export const DEFAULT_PADDING = 5;

packages/llamaindex/src/embeddings/DeepInfraEmbedding.ts

@@ -1,7 +1,7 @@
+import { BaseEmbedding } from "@llamaindex/core/embeddings";
 import type { MessageContentDetail } from "@llamaindex/core/llms";
 import { extractSingleText } from "@llamaindex/core/utils";
 import { getEnv } from "@llamaindex/env";
-import { BaseEmbedding } from "./types.js";
 
 const DEFAULT_MODEL = "sentence-transformers/clip-ViT-B-32";
 
@@ -103,10 +103,10 @@ export class DeepInfraEmbedding extends BaseEmbedding {
     }
   }
 
-  async getTextEmbeddings(texts: string[]): Promise<number[][]> {
+  getTextEmbeddings = async (texts: string[]): Promise<number[][]> => {
     const textsWithPrefix = mapPrefixWithInputs(this.textPrefix, texts);
-    return await this.getDeepInfraEmbedding(textsWithPrefix);
-  }
+    return this.getDeepInfraEmbedding(textsWithPrefix);
+  };
 
   async getQueryEmbeddings(queries: string[]): Promise<number[][]> {
     const queriesWithPrefix = mapPrefixWithInputs(this.queryPrefix, queries);

packages/llamaindex/src/embeddings/GeminiEmbedding.ts

@@ -1,6 +1,6 @@
+import { BaseEmbedding } from "@llamaindex/core/embeddings";
 import { GeminiSession, GeminiSessionStore } from "../llm/gemini/base.js";
 import { GEMINI_BACKENDS } from "../llm/gemini/types.js";
-import { BaseEmbedding } from "./types.js";
 
 export enum GEMINI_EMBEDDING_MODEL {
   EMBEDDING_001 = "embedding-001",

packages/llamaindex/src/embeddings/HuggingFaceEmbedding.ts

@@ -1,6 +1,6 @@
 import { HfInference } from "@huggingface/inference";
+import { BaseEmbedding } from "@llamaindex/core/embeddings";
 import { lazyLoadTransformers } from "../internal/deps/transformers.js";
-import { BaseEmbedding } from "./types.js";
 
 export enum HuggingFaceEmbeddingModelType {
   XENOVA_ALL_MINILM_L6_V2 = "Xenova/all-MiniLM-L6-v2",
@@ -91,11 +91,11 @@ export class HuggingFaceInferenceAPIEmbedding extends BaseEmbedding {
     return res as number[];
   }
 
-  async getTextEmbeddings(texts: string[]): Promise<Array<number[]>> {
+  getTextEmbeddings = async (texts: string[]): Promise<Array<number[]>> => {
     const res = await this.hf.featureExtraction({
       model: this.model,
       inputs: texts,
     });
     return res as number[][];
-  }
+  };
 }

packages/llamaindex/src/embeddings/JinaAIEmbedding.ts

@@ -1,29 +1,127 @@
 import { getEnv } from "@llamaindex/env";
-import { OpenAIEmbedding } from "./OpenAIEmbedding.js";
+import { imageToDataUrl } from "../internal/utils.js";
+import type { ImageType } from "../Node.js";
+import { MultiModalEmbedding } from "./MultiModalEmbedding.js";
 
-export class JinaAIEmbedding extends OpenAIEmbedding {
-  constructor(init?: Partial<OpenAIEmbedding>) {
-    const {
-      apiKey = getEnv("JINAAI_API_KEY"),
-      additionalSessionOptions = {},
-      model = "jina-embeddings-v2-base-en",
-      ...rest
-    } = init ?? {};
+function isLocal(url: ImageType): boolean {
+  if (url instanceof Blob) return true;
+  return new URL(url).protocol === "file:";
+}
 
+export type JinaEmbeddingRequest = {
+  input: Array<{ text: string } | { url: string } | { bytes: string }>;
+  model?: string;
+  encoding_type?: "float" | "binary" | "ubinary";
+};
+
+export type JinaEmbeddingResponse = {
+  model: string;
+  object: string;
+  usage: {
+    total_tokens: number;
+    prompt_tokens: number;
+  };
+  data: Array<{
+    object: string;
+    index: number;
+    embedding: number[];
+  }>;
+};
+
+const JINA_MULTIMODAL_MODELS = ["jina-clip-v1"];
+
+export class JinaAIEmbedding extends MultiModalEmbedding {
+  apiKey: string;
+  model: string;
+  baseURL: string;
+
+  async getTextEmbedding(text: string): Promise<number[]> {
+    const result = await this.getJinaEmbedding({ input: [{ text }] });
+    return result.data[0].embedding;
+  }
+
+  async getImageEmbedding(image: ImageType): Promise<number[]> {
+    const img = await this.getImageInput(image);
+    const result = await this.getJinaEmbedding({ input: [img] });
+    return result.data[0].embedding;
+  }
+
+  // Retrieve multiple text embeddings in a single request
+  getTextEmbeddings = async (texts: string[]): Promise<Array<number[]>> => {
+    const input = texts.map((text) => ({ text }));
+    const result = await this.getJinaEmbedding({ input });
+    return result.data.map((d) => d.embedding);
+  };
+
+  // Retrieve multiple image embeddings in a single request
+  async getImageEmbeddings(images: ImageType[]): Promise<number[][]> {
+    const input = await Promise.all(
+      images.map((img) => this.getImageInput(img)),
+    );
+    const result = await this.getJinaEmbedding({ input });
+    return result.data.map((d) => d.embedding);
+  }
+
+  constructor(init?: Partial<JinaAIEmbedding>) {
+    super();
+    const apiKey = init?.apiKey ?? getEnv("JINAAI_API_KEY");
     if (!apiKey) {
       throw new Error(
         "Set Jina AI API Key in JINAAI_API_KEY env variable. Get one for free or top up your key at https://jina.ai/embeddings",
       );
     }
+    this.apiKey = apiKey;
+    this.model = init?.model ?? "jina-embeddings-v2-base-en";
+    this.baseURL = init?.baseURL ?? "https://api.jina.ai/v1/embeddings";
+    init?.embedBatchSize && (this.embedBatchSize = init?.embedBatchSize);
+  }
 
-    additionalSessionOptions.baseURL =
-      additionalSessionOptions.baseURL ?? "https://api.jina.ai/v1";
+  private async getImageInput(
+    image: ImageType,
+  ): Promise<{ bytes: string } | { url: string }> {
+    if (isLocal(image)) {
+      const base64 = await imageToDataUrl(image);
+      const bytes = base64.split(",")[1];
+      return { bytes };
+    } else {
+      return { url: image.toString() };
+    }
+  }
 
-    super({
-      apiKey,
-      additionalSessionOptions,
-      model,
-      ...rest,
+  private async getJinaEmbedding(
+    input: JinaEmbeddingRequest,
+  ): Promise<JinaEmbeddingResponse> {
+    // if input includes image, check if model supports multimodal embeddings
+    if (
+      input.input.some((i) => "url" in i || "bytes" in i) &&
+      !JINA_MULTIMODAL_MODELS.includes(this.model)
+    ) {
+      throw new Error(
+        `Model ${this.model} does not support image embeddings. Use ${JINA_MULTIMODAL_MODELS.join(", ")}`,
+      );
+    }
+
+    const response = await fetch(this.baseURL, {
+      method: "POST",
+      headers: {
+        "Content-Type": "application/json",
+        Authorization: `Bearer ${this.apiKey}`,
+      },
+      body: JSON.stringify({
+        model: this.model,
+        encoding_type: "float",
+        ...input,
+      }),
     });
+    if (!response.ok) {
+      throw new Error(
+        `Request ${this.baseURL} failed with status ${response.status}`,
+      );
+    }
+    const result: JinaEmbeddingResponse = await response.json();
+    return {
+      ...result,
+      data: result.data.sort((a, b) => a.index - b.index), // Sort resulting embeddings by index
+    };
   }
 }

packages/llamaindex/src/embeddings/MistralAIEmbedding.ts

@@ -1,5 +1,5 @@
+import { BaseEmbedding } from "@llamaindex/core/embeddings";
 import { MistralAISession } from "../llm/mistral.js";
-import { BaseEmbedding } from "./types.js";
 
 export enum MistralAIEmbeddingModelType {
   MISTRAL_EMBED = "mistral-embed",

packages/llamaindex/src/embeddings/MixedbreadAIEmbeddings.ts

@@ -0,0 +1,170 @@
+import { BaseEmbedding, type EmbeddingInfo } from "@llamaindex/core/embeddings";
+import { getEnv } from "@llamaindex/env";
+import { MixedbreadAI, MixedbreadAIClient } from "@mixedbread-ai/sdk";
+
+type EmbeddingsRequestWithoutInput = Omit<
+  MixedbreadAI.EmbeddingsRequest,
+  "input"
+>;
+
+/**
+ * Interface extending EmbeddingsParams with additional
+ * parameters specific to the MixedbreadAIEmbeddings class.
+ */
+export interface MixedbreadAIEmbeddingsParams
+  extends Omit<EmbeddingsRequestWithoutInput, "model"> {
+  /**
+   * The model to use for generating embeddings.
+   * @default {"mixedbread-ai/mxbai-embed-large-v1"}
+   */
+  model?: string;
+
+  /**
+   * The API key to use.
+   * @default {process.env.MXBAI_API_KEY}
+   */
+  apiKey?: string;
+
+  /**
+   * The base URL for the API.
+   */
+  baseUrl?: string;
+
+  /**
+   * The maximum number of documents to embed in a single request.
+   * @default {128}
+   */
+  embedBatchSize?: number;
+
+  /**
+   * The embed info for the model.
+   */
+  embedInfo?: EmbeddingInfo;
+
+  /**
+   * The maximum number of retries to attempt.
+   * @default {3}
+   */
+  maxRetries?: number;
+
+  /**
+   * Timeouts for the request.
+   */
+  timeoutInSeconds?: number;
+}
+
+/**
+ * Class for generating embeddings using the mixedbread ai API.
+ *
+ * This class leverages the model "mixedbread-ai/mxbai-embed-large-v1" to generate
+ * embeddings for text documents. The embeddings can be used for various NLP tasks
+ * such as similarity comparison, clustering, or as features in machine learning models.
+ *
+ * @example
+ * const mxbai = new MixedbreadAIEmbeddings({ apiKey: 'your-api-key' });
+ * const texts = ["Baking bread is fun", "I love baking"];
+ * const result = await mxbai.getTextEmbeddings(texts);
+ * console.log(result);
+ *
+ * @example
+ * const mxbai = new MixedbreadAIEmbeddings({
+ *  apiKey: 'your-api-key',
+ *  model: 'mixedbread-ai/mxbai-embed-large-v1',
+ *  encodingFormat: MixedbreadAI.EncodingFormat.Binary,
+ *  dimensions: 512,
+ *  normalized: true,
+ * });
+ * const query = "Represent this sentence for searching relevant passages: Is baking bread fun?";
+ * const result = await mxbai.getTextEmbedding(query);
+ * console.log(result);
+ */
+export class MixedbreadAIEmbeddings extends BaseEmbedding {
+  requestParams: EmbeddingsRequestWithoutInput;
+  requestOptions: MixedbreadAIClient.RequestOptions;
+  private client: MixedbreadAIClient;
+
+  /**
+   * Constructor for MixedbreadAIEmbeddings.
+   * @param {Partial<MixedbreadAIEmbeddingsParams>} params - An optional object with properties to configure the instance.
+   * @throws {Error} If the API key is not provided or found in the environment variables.
+   * @throws {Error} If the batch size exceeds 256.
+   */
+  constructor(params?: Partial<MixedbreadAIEmbeddingsParams>) {
+    super();
+
+    const apiKey = params?.apiKey ?? getEnv("MXBAI_API_KEY");
+    if (!apiKey) {
+      throw new Error(
+        "mixedbread ai API key not found. Either provide it in the constructor or set the 'MXBAI_API_KEY' environment variable.",
+      );
+    }
+    if (params?.embedBatchSize && params?.embedBatchSize > 256) {
+      throw new Error(
+        "The maximum batch size for mixedbread ai embeddings API is 256.",
+      );
+    }
+
+    this.embedBatchSize = params?.embedBatchSize ?? 128;
+    this.embedInfo = params?.embedInfo;
+    this.requestParams = {
+      model: params?.model ?? "mixedbread-ai/mxbai-embed-large-v1",
+      normalized: params?.normalized,
+      dimensions: params?.dimensions,
+      encodingFormat: params?.encodingFormat,
+      truncationStrategy: params?.truncationStrategy,
+      prompt: params?.prompt,
+    };
+    this.requestOptions = {
+      timeoutInSeconds: params?.timeoutInSeconds,
+      maxRetries: params?.maxRetries ?? 3,
+      // Support for this already exists in the python sdk and will be added to the js sdk soon
+      // @ts-ignore
+      additionalHeaders: {
+        "user-agent": "@mixedbread-ai/llamaindex-ts-sdk",
+      },
+    };
+    this.client = new MixedbreadAIClient({
+      apiKey,
+      environment: params?.baseUrl,
+    });
+  }
+
+  /**
+   * Generates an embedding for a single text.
+   * @param {string} text - A string to generate an embedding for.
+   * @returns {Promise<number[]>} A Promise that resolves to an array of numbers representing the embedding.
+   *
+   * @example
+   * const query = "Represent this sentence for searching relevant passages: Is baking bread fun?";
+   * const result = await mxbai.getTextEmbedding(text);
+   * console.log(result);
+   */
+  async getTextEmbedding(text: string): Promise<number[]> {
+    return (await this.getTextEmbeddings([text]))[0];
+  }
+
+  /**
+   * Generates embeddings for an array of texts.
+   * @param {string[]} texts - An array of strings to generate embeddings for.
+   * @returns {Promise<Array<number[]>>} A Promise that resolves to an array of embeddings.
+   *
+   * @example
+   * const texts = ["Baking bread is fun", "I love baking"];
+   * const result = await mxbai.getTextEmbeddings(texts);
+   * console.log(result);
+   */
+  getTextEmbeddings = async (texts: string[]): Promise<Array<number[]>> => {
+    if (texts.length === 0) {
+      return [];
+    }
+
+    const response = await this.client.embeddings(
+      {
+        ...this.requestParams,
+        input: texts,
+      },
+      this.requestOptions,
+    );
+    return response.data.map((d) => d.embedding as number[]);
+  };
+}

packages/llamaindex/src/embeddings/MultiModalEmbedding.ts

@@ -1,3 +1,4 @@
+import { BaseEmbedding, batchEmbeddings } from "@llamaindex/core/embeddings";
 import type { MessageContentDetail } from "@llamaindex/core/llms";
 import {
   ImageNode,
@@ -8,7 +9,6 @@ import {
   type ImageType,
 } from "@llamaindex/core/schema";
 import { extractImage, extractSingleText } from "@llamaindex/core/utils";
-import { BaseEmbedding, batchEmbeddings } from "./types.js";
 
 /*
  * Base class for Multi Modal embeddings.

packages/llamaindex/src/embeddings/OllamaEmbedding.ts

@@ -1,5 +1,5 @@
+import type { BaseEmbedding } from "@llamaindex/core/embeddings";
 import { Ollama } from "../llm/ollama.js";
-import type { BaseEmbedding } from "./types.js";
 
 /**
  * OllamaEmbedding is an alias for Ollama that implements the BaseEmbedding interface.

packages/llamaindex/src/embeddings/OpenAIEmbedding.ts

@@ -1,3 +1,4 @@
+import { BaseEmbedding } from "@llamaindex/core/embeddings";
 import { Tokenizers } from "@llamaindex/env";
 import type { ClientOptions as OpenAIClientOptions } from "openai";
 import type { AzureOpenAIConfig } from "../llm/azure.js";
@@ -8,7 +9,6 @@ import {
 } from "../llm/azure.js";
 import type { OpenAISession } from "../llm/openai.js";
 import { getOpenAISession } from "../llm/openai.js";
-import { BaseEmbedding } from "./types.js";
 
 export const ALL_OPENAI_EMBEDDING_MODELS = {
   "text-embedding-ada-002": {
@@ -132,9 +132,9 @@ export class OpenAIEmbedding extends BaseEmbedding {
    * Get embeddings for a batch of texts
    * @param texts
    */
-  async getTextEmbeddings(texts: string[]): Promise<number[][]> {
-    return await this.getOpenAIEmbedding(texts);
-  }
+  getTextEmbeddings = async (texts: string[]): Promise<number[][]> => {
+    return this.getOpenAIEmbedding(texts);
+  };
 
   /**
    * Get embeddings for a single text

packages/llamaindex/src/embeddings/index.ts

@@ -1,12 +1,12 @@
+export * from "@llamaindex/core/embeddings";
 export { DeepInfraEmbedding } from "./DeepInfraEmbedding.js";
 export { FireworksEmbedding } from "./fireworks.js";
 export * from "./GeminiEmbedding.js";
 export { HuggingFaceInferenceAPIEmbedding } from "./HuggingFaceEmbedding.js";
 export * from "./JinaAIEmbedding.js";
 export * from "./MistralAIEmbedding.js";
+export * from "./MixedbreadAIEmbeddings.js";
 export * from "./MultiModalEmbedding.js";
 export { OllamaEmbedding } from "./OllamaEmbedding.js";
 export * from "./OpenAIEmbedding.js";
 export { TogetherEmbedding } from "./together.js";
-export * from "./types.js";
-export * from "./utils.js";

packages/llamaindex/src/embeddings/utils.ts

@@ -1,256 +0,0 @@
-import type { ImageType } from "@llamaindex/core/schema";
-import { fs } from "@llamaindex/env";
-import _ from "lodash";
-import { filetypemime } from "magic-bytes.js";
-import { DEFAULT_SIMILARITY_TOP_K } from "../constants.js";
-import type { VectorStoreQueryMode } from "../storage/vectorStore/types.js";
-
-/**
- * Similarity type
- * Default is cosine similarity. Dot product and negative Euclidean distance are also supported.
- */
-export enum SimilarityType {
-  DEFAULT = "cosine",
-  DOT_PRODUCT = "dot_product",
-  EUCLIDEAN = "euclidean",
-}
-
-/**
- * The similarity between two embeddings.
- * @param embedding1
- * @param embedding2
- * @param mode
- * @returns similarity score with higher numbers meaning the two embeddings are more similar
- */
-
-export function similarity(
-  embedding1: number[],
-  embedding2: number[],
-  mode: SimilarityType = SimilarityType.DEFAULT,
-): number {
-  if (embedding1.length !== embedding2.length) {
-    throw new Error("Embedding length mismatch");
-  }
-
-  // NOTE I've taken enough Kahan to know that we should probably leave the
-  // numeric programming to numeric programmers. The naive approach here
-  // will probably cause some avoidable loss of floating point precision
-  // ml-distance is worth watching although they currently also use the naive
-  // formulas
-  function norm(x: number[]): number {
-    let result = 0;
-    for (let i = 0; i < x.length; i++) {
-      result += x[i] * x[i];
-    }
-    return Math.sqrt(result);
-  }
-
-  switch (mode) {
-    case SimilarityType.EUCLIDEAN: {
-      const difference = embedding1.map((x, i) => x - embedding2[i]);
-      return -norm(difference);
-    }
-    case SimilarityType.DOT_PRODUCT: {
-      let result = 0;
-      for (let i = 0; i < embedding1.length; i++) {
-        result += embedding1[i] * embedding2[i];
-      }
-      return result;
-    }
-    case SimilarityType.DEFAULT: {
-      return (
-        similarity(embedding1, embedding2, SimilarityType.DOT_PRODUCT) /
-        (norm(embedding1) * norm(embedding2))
-      );
-    }
-    default:
-      throw new Error("Not implemented yet");
-  }
-}
-
-/**
- * Get the top K embeddings from a list of embeddings ordered by similarity to the query.
- * @param queryEmbedding
- * @param embeddings list of embeddings to consider
- * @param similarityTopK max number of embeddings to return, default 2
- * @param embeddingIds ids of embeddings in the embeddings list
- * @param similarityCutoff minimum similarity score
- * @returns
- */
-// eslint-disable-next-line max-params
-export function getTopKEmbeddings(
-  queryEmbedding: number[],
-  embeddings: number[][],
-  similarityTopK: number = DEFAULT_SIMILARITY_TOP_K,
-  embeddingIds: any[] | null = null,
-  similarityCutoff: number | null = null,
-): [number[], any[]] {
-  if (embeddingIds == null) {
-    embeddingIds = Array(embeddings.length).map((_, i) => i);
-  }
-
-  if (embeddingIds.length !== embeddings.length) {
-    throw new Error(
-      "getTopKEmbeddings: embeddings and embeddingIds length mismatch",
-    );
-  }
-
-  const similarities: { similarity: number; id: number }[] = [];
-
-  for (let i = 0; i < embeddings.length; i++) {
-    const sim = similarity(queryEmbedding, embeddings[i]);
-    if (similarityCutoff == null || sim > similarityCutoff) {
-      similarities.push({ similarity: sim, id: embeddingIds[i] });
-    }
-  }
-
-  similarities.sort((a, b) => b.similarity - a.similarity); // Reverse sort
-
-  const resultSimilarities: number[] = [];
-  const resultIds: any[] = [];
-
-  for (let i = 0; i < similarityTopK; i++) {
-    if (i >= similarities.length) {
-      break;
-    }
-    resultSimilarities.push(similarities[i].similarity);
-    resultIds.push(similarities[i].id);
-  }
-
-  return [resultSimilarities, resultIds];
-}
-
-// eslint-disable-next-line max-params
-export function getTopKEmbeddingsLearner(
-  queryEmbedding: number[],
-  embeddings: number[][],
-  similarityTopK?: number,
-  embeddingsIds?: any[],
-  queryMode?: VectorStoreQueryMode,
-): [number[], any[]] {
-  throw new Error("Not implemented yet");
-}
-
-// eslint-disable-next-line max-params
-export function getTopKMMREmbeddings(
-  queryEmbedding: number[],
-  embeddings: number[][],
-  similarityFn: ((...args: any[]) => number) | null = null,
-  similarityTopK: number | null = null,
-  embeddingIds: any[] | null = null,
-  _similarityCutoff: number | null = null,
-  mmrThreshold: number | null = null,
-): [number[], any[]] {
-  const threshold = mmrThreshold || 0.5;
-  similarityFn = similarityFn || similarity;
-
-  if (embeddingIds === null || embeddingIds.length === 0) {
-    embeddingIds = Array.from({ length: embeddings.length }, (_, i) => i);
-  }
-  const fullEmbedMap = new Map(embeddingIds.map((value, i) => [value, i]));
-  const embedMap = new Map(fullEmbedMap);
-  const embedSimilarity: Map<any, number> = new Map();
-  let score: number = Number.NEGATIVE_INFINITY;
-  let highScoreId: any | null = null;
-
-  for (let i = 0; i < embeddings.length; i++) {
-    const emb = embeddings[i];
-    const similarity = similarityFn(queryEmbedding, emb);
-    embedSimilarity.set(embeddingIds[i], similarity);
-    if (similarity * threshold > score) {
-      highScoreId = embeddingIds[i];
-      score = similarity * threshold;
-    }
-  }
-
-  const results: [number, any][] = [];
-
-  const embeddingLength = embeddings.length;
-  const similarityTopKCount = similarityTopK || embeddingLength;
-
-  while (results.length < Math.min(similarityTopKCount, embeddingLength)) {
-    results.push([score, highScoreId]);
-    embedMap.delete(highScoreId);
-    const recentEmbeddingId = highScoreId;
-    score = Number.NEGATIVE_INFINITY;
-    for (const embedId of Array.from(embedMap.keys())) {
-      const overlapWithRecent = similarityFn(
-        embeddings[embedMap.get(embedId)!],
-        embeddings[fullEmbedMap.get(recentEmbeddingId)!],
-      );
-      if (
-        threshold * embedSimilarity.get(embedId)! -
-          (1 - threshold) * overlapWithRecent >
-        score
-      ) {
-        score =
-          threshold * embedSimilarity.get(embedId)! -
-          (1 - threshold) * overlapWithRecent;
-        highScoreId = embedId;
-      }
-    }
-  }
-
-  const resultSimilarities = results.map(([s, _]) => s);
-  const resultIds = results.map(([_, n]) => n);
-
-  return [resultSimilarities, resultIds];
-}
-
-async function blobToDataUrl(input: Blob) {
-  const buffer = Buffer.from(await input.arrayBuffer());
-  const mimes = filetypemime(buffer);
-  if (mimes.length < 1) {
-    throw new Error("Unsupported image type");
-  }
-  return "data:" + mimes[0] + ";base64," + buffer.toString("base64");
-}
-
-export async function imageToString(input: ImageType): Promise<string> {
-  if (input instanceof Blob) {
-    // if the image is a Blob, convert it to a base64 data URL
-    return await blobToDataUrl(input);
-  } else if (_.isString(input)) {
-    return input;
-  } else if (input instanceof URL) {
-    return input.toString();
-  } else {
-    throw new Error(`Unsupported input type: ${typeof input}`);
-  }
-}
-
-export function stringToImage(input: string): ImageType {
-  if (input.startsWith("data:")) {
-    // if the input is a base64 data URL, convert it back to a Blob
-    const base64Data = input.split(",")[1];
-    const byteArray = Buffer.from(base64Data, "base64");
-    return new Blob([byteArray]);
-  } else if (input.startsWith("http://") || input.startsWith("https://")) {
-    return new URL(input);
-  } else if (_.isString(input)) {
-    return input;
-  } else {
-    throw new Error(`Unsupported input type: ${typeof input}`);
-  }
-}
-
-export async function imageToDataUrl(input: ImageType): Promise<string> {
-  // first ensure, that the input is a Blob
-  if (
-    (input instanceof URL && input.protocol === "file:") ||
-    _.isString(input)
-  ) {
-    // string or file URL
-    const dataBuffer = await fs.readFile(
-      input instanceof URL ? input.pathname : input,
-    );
-    input = new Blob([dataBuffer]);
-  } else if (!(input instanceof Blob)) {
-    if (input instanceof URL) {
-      throw new Error(`Unsupported URL with protocol: ${input.protocol}`);
-    } else {
-      throw new Error(`Unsupported input type: ${typeof input}`);
-    }
-  }
-  return await blobToDataUrl(input);
-}

packages/llamaindex/src/engines/chat/CondenseQuestionChatEngine.ts

@@ -1,4 +1,5 @@
 import type { ChatMessage, LLM } from "@llamaindex/core/llms";
+import type { EngineResponse } from "@llamaindex/core/schema";
 import {
   extractText,
   streamReducer,
@@ -6,7 +7,6 @@ import {
 } from "@llamaindex/core/utils";
 import type { ChatHistory } from "../../ChatHistory.js";
 import { getHistory } from "../../ChatHistory.js";
-import type { EngineResponse } from "../../EngineResponse.js";
 import type { CondenseQuestionPrompt } from "../../Prompt.js";
 import {
   defaultCondenseQuestionPrompt,
@@ -109,7 +109,8 @@ export class CondenseQuestionChatEngine
       return streamReducer({
         stream,
         initialValue: "",
-        reducer: (accumulator, part) => (accumulator += part.response),
+        reducer: (accumulator, part) =>
+          (accumulator += extractText(part.message.content)),
         finished: (accumulator) => {
           chatHistory.addMessage({ content: accumulator, role: "assistant" });
         },
@@ -118,7 +119,10 @@ export class CondenseQuestionChatEngine
     const response = await this.queryEngine.query({
       query: condensedQuestion,
     });
-    chatHistory.addMessage({ content: response.response, role: "assistant" });
+    chatHistory.addMessage({
+      content: response.message.content,
+      role: "assistant",
+    });
 
     return response;
   }

packages/llamaindex/src/engines/chat/ContextChatEngine.ts

@@ -4,6 +4,7 @@ import type {
   MessageContent,
   MessageType,
 } from "@llamaindex/core/llms";
+import { EngineResponse } from "@llamaindex/core/schema";
 import {
   extractText,
   streamConverter,
@@ -12,7 +13,6 @@ import {
 } from "@llamaindex/core/utils";
 import type { ChatHistory } from "../../ChatHistory.js";
 import { getHistory } from "../../ChatHistory.js";
-import { EngineResponse } from "../../EngineResponse.js";
 import type { ContextSystemPrompt } from "../../Prompt.js";
 import type { BaseRetriever } from "../../Retriever.js";
 import { Settings } from "../../Settings.js";

packages/llamaindex/src/engines/chat/SimpleChatEngine.ts

@@ -1,4 +1,5 @@
 import type { LLM } from "@llamaindex/core/llms";
+import { EngineResponse } from "@llamaindex/core/schema";
 import {
   streamConverter,
   streamReducer,
@@ -6,7 +7,6 @@ import {
 } from "@llamaindex/core/utils";
 import type { ChatHistory } from "../../ChatHistory.js";
 import { getHistory } from "../../ChatHistory.js";
-import { EngineResponse } from "../../EngineResponse.js";
 import { Settings } from "../../Settings.js";
 import type {
   ChatEngine,

packages/llamaindex/src/engines/chat/types.ts

@@ -1,7 +1,6 @@
 import type { ChatMessage, MessageContent } from "@llamaindex/core/llms";
-import type { NodeWithScore } from "@llamaindex/core/schema";
+import { EngineResponse, type NodeWithScore } from "@llamaindex/core/schema";
 import type { ChatHistory } from "../../ChatHistory.js";
-import type { EngineResponse } from "../../EngineResponse.js";
 
 /**
  * Represents the base parameters for ChatEngine.

packages/llamaindex/src/engines/query/RetrieverQueryEngine.ts

@@ -1,6 +1,5 @@
-import type { NodeWithScore } from "@llamaindex/core/schema";
+import { EngineResponse, type NodeWithScore } from "@llamaindex/core/schema";
 import { wrapEventCaller } from "@llamaindex/core/utils";
-import type { EngineResponse } from "../../EngineResponse.js";
 import type { BaseNodePostprocessor } from "../../postprocessors/index.js";
 import { PromptMixin } from "../../prompts/Mixin.js";
 import type { BaseRetriever } from "../../Retriever.js";
@@ -78,11 +77,13 @@ export class RetrieverQueryEngine extends PromptMixin implements QueryEngine {
     const { query, stream } = params;
     const nodesWithScore = await this.retrieve(query);
     if (stream) {
-      return this.responseSynthesizer.synthesize({
-        query,
-        nodesWithScore,
-        stream: true,
-      });
+      return this.responseSynthesizer.synthesize(
+        {
+          query,
+          nodesWithScore,
+        },
+        true,
+      );
     }
     return this.responseSynthesizer.synthesize({
       query,

packages/llamaindex/src/engines/query/RouterQueryEngine.ts

@@ -1,5 +1,6 @@
-import type { NodeWithScore } from "@llamaindex/core/schema";
-import { EngineResponse } from "../../EngineResponse.js";
+import type { QueryType } from "@llamaindex/core/query-engine";
+import { EngineResponse, type NodeWithScore } from "@llamaindex/core/schema";
+import { extractText } from "@llamaindex/core/utils";
 import type { ServiceContext } from "../../ServiceContext.js";
 import { llmFromSettingsOrContext } from "../../Settings.js";
 import { PromptMixin } from "../../prompts/index.js";
@@ -7,7 +8,6 @@ import type { BaseSelector } from "../../selectors/index.js";
 import { LLMSingleSelector } from "../../selectors/index.js";
 import { TreeSummarize } from "../../synthesizers/index.js";
 import type {
-  QueryBundle,
   QueryEngine,
   QueryEngineParamsNonStreaming,
   QueryEngineParamsStreaming,
@@ -25,7 +25,7 @@ type RouterQueryEngineMetadata = {
 async function combineResponses(
   summarizer: TreeSummarize,
   responses: EngineResponse[],
-  queryBundle: QueryBundle,
+  queryType: QueryType,
   verbose: boolean = false,
 ): Promise<EngineResponse> {
   if (verbose) {
@@ -40,11 +40,11 @@ async function combineResponses(
       sourceNodes.push(...response.sourceNodes);
     }
 
-    responseStrs.push(response.response);
+    responseStrs.push(extractText(response.message.content));
   }
 
   const summary = await summarizer.getResponse({
-    query: queryBundle.queryStr,
+    query: extractText(queryType),
     textChunks: responseStrs,
   });
 
@@ -117,7 +117,7 @@ export class RouterQueryEngine extends PromptMixin implements QueryEngine {
   ): Promise<EngineResponse | AsyncIterable<EngineResponse>> {
     const { query, stream } = params;
 
-    const response = await this.queryRoute({ queryStr: query });
+    const response = await this.queryRoute(query);
 
     if (stream) {
       throw new Error("Streaming is not supported yet.");
@@ -126,8 +126,8 @@ export class RouterQueryEngine extends PromptMixin implements QueryEngine {
     return response;
   }
 
-  private async queryRoute(queryBundle: QueryBundle): Promise<EngineResponse> {
-    const result = await this.selector.select(this.metadatas, queryBundle);
+  private async queryRoute(query: QueryType): Promise<EngineResponse> {
+    const result = await this.selector.select(this.metadatas, query);
 
     if (result.selections.length > 1) {
       const responses: EngineResponse[] = [];
@@ -142,7 +142,7 @@ export class RouterQueryEngine extends PromptMixin implements QueryEngine {
         const selectedQueryEngine = this.queryEngines[engineInd.index];
         responses.push(
           await selectedQueryEngine.query({
-            query: queryBundle.queryStr,
+            query: extractText(query),
           }),
         );
       }
@@ -151,7 +151,7 @@ export class RouterQueryEngine extends PromptMixin implements QueryEngine {
         const finalResponse = await combineResponses(
           this.summarizer,
           responses,
-          queryBundle,
+          query,
           this.verbose,
         );
 
@@ -179,7 +179,7 @@ export class RouterQueryEngine extends PromptMixin implements QueryEngine {
       }
 
       const finalResponse = await selectedQueryEngine.query({
-        query: queryBundle.queryStr,
+        query: extractText(query),
       });
 
       // add selected result

packages/llamaindex/src/engines/query/SubQuestionQueryEngine.ts

@@ -1,6 +1,8 @@
-import type { NodeWithScore } from "@llamaindex/core/schema";
-import { TextNode } from "@llamaindex/core/schema";
-import type { EngineResponse } from "../../EngineResponse.js";
+import {
+  EngineResponse,
+  TextNode,
+  type NodeWithScore,
+} from "@llamaindex/core/schema";
 import { LLMQuestionGenerator } from "../../QuestionGenerator.js";
 import type { ServiceContext } from "../../ServiceContext.js";
 import { PromptMixin } from "../../prompts/Mixin.js";
@@ -10,20 +12,18 @@ import {
   ResponseSynthesizer,
 } from "../../synthesizers/index.js";
 
-import type {
-  QueryEngine,
-  QueryEngineParamsNonStreaming,
-  QueryEngineParamsStreaming,
-} from "../../types.js";
-
 import type { BaseTool, ToolMetadata } from "@llamaindex/core/llms";
+import type { BaseQueryEngine, QueryType } from "@llamaindex/core/query-engine";
 import { wrapEventCaller } from "@llamaindex/core/utils";
 import type { BaseQuestionGenerator, SubQuestion } from "./types.js";
 
 /**
  * SubQuestionQueryEngine decomposes a question into subquestions and then
  */
-export class SubQuestionQueryEngine extends PromptMixin implements QueryEngine {
+export class SubQuestionQueryEngine
+  extends PromptMixin
+  implements BaseQueryEngine
+{
   responseSynthesizer: BaseSynthesizer;
   questionGen: BaseQuestionGenerator;
   queryEngines: BaseTool[];
@@ -73,15 +73,13 @@ export class SubQuestionQueryEngine extends PromptMixin implements QueryEngine {
     });
   }
 
-  query(
-    params: QueryEngineParamsStreaming,
-  ): Promise<AsyncIterable<EngineResponse>>;
-  query(params: QueryEngineParamsNonStreaming): Promise<EngineResponse>;
+  query(query: QueryType, stream: true): Promise<AsyncIterable<EngineResponse>>;
+  query(query: QueryType, stream?: false): Promise<EngineResponse>;
   @wrapEventCaller
   async query(
-    params: QueryEngineParamsStreaming | QueryEngineParamsNonStreaming,
+    query: QueryType,
+    stream?: boolean,
   ): Promise<EngineResponse | AsyncIterable<EngineResponse>> {
-    const { query, stream } = params;
     const subQuestions = await this.questionGen.generate(this.metadatas, query);
 
     const subQNodes = await Promise.all(
@@ -92,16 +90,21 @@ export class SubQuestionQueryEngine extends PromptMixin implements QueryEngine {
       .filter((node) => node !== null)
       .map((node) => node as NodeWithScore);
     if (stream) {
-      return this.responseSynthesizer.synthesize({
+      return this.responseSynthesizer.synthesize(
+        {
+          query,
+          nodesWithScore,
+        },
+        true,
+      );
+    }
+    return this.responseSynthesizer.synthesize(
+      {
         query,
         nodesWithScore,
-        stream: true,
-      });
-    }
-    return this.responseSynthesizer.synthesize({
-      query,
-      nodesWithScore,
-    });
+      },
+      false,
+    );
   }
 
   private async querySubQ(subQ: SubQuestion): Promise<NodeWithScore | null> {

packages/llamaindex/src/engines/query/types.ts

@@ -1,10 +1,11 @@
 import type { ToolMetadata } from "@llamaindex/core/llms";
+import type { QueryType } from "@llamaindex/core/query-engine";
 
 /**
  * QuestionGenerators generate new questions for the LLM using tools and a user query.
  */
 export interface BaseQuestionGenerator {
-  generate(tools: ToolMetadata[], query: string): Promise<SubQuestion[]>;
+  generate(tools: ToolMetadata[], query: QueryType): Promise<SubQuestion[]>;
 }
 
 export interface SubQuestion {

packages/llamaindex/src/evaluation/Correctness.ts

@@ -74,7 +74,7 @@ export class CorrectnessEvaluator extends PromptMixin implements BaseEvaluator {
       {
         role: "user",
         content: defaultUserPrompt({
-          query,
+          query: extractText(query),
           generatedAnswer: response,
           referenceAnswer: reference || "(NO REFERENCE ANSWER SUPPLIED)",
         }),
@@ -106,7 +106,7 @@ export class CorrectnessEvaluator extends PromptMixin implements BaseEvaluator {
     query,
     response,
   }: EvaluatorResponseParams): Promise<EvaluationResult> {
-    const responseStr = response?.response;
+    const responseStr = extractText(response?.message.content);
     const contexts = [];
 
     if (response) {

packages/llamaindex/src/evaluation/Faithfulness.ts

@@ -1,4 +1,5 @@
 import { Document, MetadataMode } from "@llamaindex/core/schema";
+import { extractText } from "@llamaindex/core/utils";
 import type { ServiceContext } from "../ServiceContext.js";
 import { SummaryIndex } from "../indices/summary/index.js";
 import { PromptMixin } from "../prompts/Mixin.js";
@@ -132,7 +133,7 @@ export class FaithfulnessEvaluator
     query,
     response,
   }: EvaluatorResponseParams): Promise<EvaluationResult> {
-    const responseStr = response?.response;
+    const responseStr = extractText(response?.message.content);
     const contexts = [];
 
     if (response) {

packages/llamaindex/src/evaluation/Relevancy.ts

@@ -1,4 +1,5 @@
 import { Document, MetadataMode } from "@llamaindex/core/schema";
+import { extractText } from "@llamaindex/core/utils";
 import type { ServiceContext } from "../ServiceContext.js";
 import { SummaryIndex } from "../indices/summary/index.js";
 import { PromptMixin } from "../prompts/Mixin.js";
@@ -121,7 +122,7 @@ export class RelevancyEvaluator extends PromptMixin implements BaseEvaluator {
     query,
     response,
   }: EvaluatorResponseParams): Promise<EvaluationResult> {
-    const responseStr = response?.response;
+    const responseStr = extractText(response?.message.content);
     const contexts = [];
 
     if (response) {

packages/llamaindex/src/evaluation/types.ts

@@ -1,7 +1,8 @@
-import { EngineResponse } from "../EngineResponse.js";
+import type { QueryType } from "@llamaindex/core/query-engine";
+import type { EngineResponse } from "@llamaindex/core/schema";
 
 export type EvaluationResult = {
-  query?: string;
+  query?: QueryType;
   contexts?: string[];
   response: string | null;
   score: number;
@@ -13,7 +14,7 @@ export type EvaluationResult = {
 };
 
 export type EvaluatorParams = {
-  query: string | null;
+  query: QueryType;
   response: string;
   contexts?: string[];
   reference?: string;
@@ -21,7 +22,7 @@ export type EvaluatorParams = {
 };
 
 export type EvaluatorResponseParams = {
-  query: string | null;
+  query: QueryType;
   response: EngineResponse;
 };
 export interface BaseEvaluator {

packages/llamaindex/src/extractors/types.ts

@@ -1,12 +1,11 @@
-import type { BaseNode } from "@llamaindex/core/schema";
+import type { BaseNode, TransformComponent } from "@llamaindex/core/schema";
 import { MetadataMode, TextNode } from "@llamaindex/core/schema";
-import type { TransformComponent } from "../ingestion/types.js";
 import { defaultNodeTextTemplate } from "./prompts.js";
 
 /*
  * Abstract class for all extractors.
  */
-export abstract class BaseExtractor implements TransformComponent {
+export abstract class BaseExtractor implements TransformComponent<any> {
   isTextNodeOnly: boolean = true;
   showProgress: boolean = true;
   metadataMode: MetadataMode = MetadataMode.ALL;

packages/llamaindex/src/index.edge.ts

@@ -29,7 +29,6 @@ export * from "./ChatHistory.js";
 export * from "./cloud/index.js";
 export * from "./constants.js";
 export * from "./embeddings/index.js";
-export * from "./EngineResponse.js";
 export * from "./engines/chat/index.js";
 export * from "./engines/query/index.js";
 export * from "./evaluation/index.js";