Release 0.5.8 (#1067)

Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>

apps/docs/CHANGELOG.md

@@ -1,5 +1,32 @@
 # docs
 
+## 0.0.49
+
+### Patch Changes
+
+- Updated dependencies [3d5ba08]
+- Updated dependencies [d917cdc]
+  - llamaindex@0.5.8
+
+## 0.0.48
+
+### Patch Changes
+
+- Updated dependencies [ec59acd]
+  - llamaindex@0.5.7
+
+## 0.0.47
+
+### Patch Changes
+
+- Updated dependencies [2562244]
+- Updated dependencies [325aa51]
+- Updated dependencies [ab700ea]
+- Updated dependencies [92f0782]
+- Updated dependencies [6cf6ae6]
+- Updated dependencies [b7cfe5b]
+  - llamaindex@0.5.6
+
 ## 0.0.46
 
 ### Patch Changes

apps/docs/docs/modules/node_postprocessors/cohere_reranker.md

@@ -39,8 +39,9 @@ const index = await VectorStoreIndex.fromDocuments([document]);
 The default value for `similarityTopK` is 2. This means that only the most similar document will be returned. To retrieve more results, you can increase the value of `similarityTopK`.
 
 ```ts
-const retriever = index.asRetriever();
-retriever.similarityTopK = 5;
+const retriever = index.asRetriever({
+  similarityTopK: 5,
+});
 ```
 
 ## Create a new instance of the CohereRerank class

apps/docs/docs/modules/node_postprocessors/jinaai_reranker.md

@@ -39,8 +39,9 @@ const index = await VectorStoreIndex.fromDocuments([document]);
 The default value for `similarityTopK` is 2. This means that only the most similar document will be returned. To retrieve more results, you can increase the value of `similarityTopK`.
 
 ```ts
-const retriever = index.asRetriever();
-retriever.similarityTopK = 5;
+const retriever = index.asRetriever({
+  similarityTopK: 5,
+});
 ```
 
 ## Create a new instance of the JinaAIReranker class

apps/docs/docs/modules/node_postprocessors/mixedbreadiai_reranker.md

@@ -55,8 +55,9 @@ const index = await VectorStoreIndex.fromDocuments([document]);
 The default value for `similarityTopK` is 2, which means only the most similar document will be returned. To get more results, like picking a variety of fresh breads, you can increase the value of `similarityTopK`.
 
 ```ts
-const retriever = index.asRetriever();
-retriever.similarityTopK = 5;
+const retriever = index.asRetriever({
+  similarityTopK: 5,
+});
 ```
 
 ### Step 3: Create a MixedbreadAIReranker Instance

apps/docs/docs/modules/query_engines/metadata_filtering.md

@@ -88,6 +88,8 @@ const response = await queryEngine.query({
 console.log(response.toString());
 ```
 
+Besides using the equal operator (`==`), you can also use a whole set of different [operators](../../api/interfaces/MetadataFilter.md#operator) to filter your documents.
+
 ## Full Code
 
 ```ts
@@ -156,3 +158,4 @@ main();
 
 - [VectorStoreIndex](../../api/classes/VectorStoreIndex.md)
 - [ChromaVectorStore](../../api/classes/ChromaVectorStore.md)
+- [MetadataFilter](../../api/interfaces/MetadataFilter.md)

apps/docs/docs/modules/retriever.md

@@ -7,8 +7,9 @@ sidebar_position: 5
 A retriever in LlamaIndex is what is used to fetch `Node`s from an index using a query string. Aa `VectorIndexRetriever` will fetch the top-k most similar nodes. Meanwhile, a `SummaryIndexRetriever` will fetch all nodes no matter the query.
 
 ```typescript
-const retriever = vector_index.asRetriever();
-retriever.similarityTopK = 3;
+const retriever = vectorIndex.asRetriever({
+  similarityTopK: 3,
+});
 
 // Fetch nodes!
 const nodesWithScore = await retriever.retrieve({ query: "query string" });

apps/docs/package.json

@@ -1,6 +1,6 @@
 {
   "name": "docs",
-  "version": "0.0.46",
+  "version": "0.0.49",
   "private": true,
   "scripts": {
     "docusaurus": "docusaurus",

examples/chatEngine.ts

@@ -16,8 +16,9 @@ Settings.chunkSize = 512;
 async function main() {
   const document = new Document({ text: essay });
   const index = await VectorStoreIndex.fromDocuments([document]);
-  const retriever = index.asRetriever();
-  retriever.similarityTopK = 5;
+  const retriever = index.asRetriever({
+    similarityTopK: 5,
+  });
   const chatEngine = new ContextChatEngine({ retriever });
   const rl = readline.createInterface({ input, output });
 

examples/lowlevel.ts

@@ -27,11 +27,13 @@ import {
     },
   ];
 
-  const stream = await responseSynthesizer.synthesize({
-    query: "What age am I?",
-    nodesWithScore,
-    stream: true,
-  });
+  const stream = await responseSynthesizer.synthesize(
+    {
+      query: "What age am I?",
+      nodesWithScore,
+    },
+    true,
+  );
   for await (const chunk of stream) {
     process.stdout.write(chunk.response);
   }

examples/multimodal/jina.ts

@@ -0,0 +1,83 @@
+import {
+  ImageDocument,
+  JinaAIEmbedding,
+  similarity,
+  SimilarityType,
+  SimpleDirectoryReader,
+} from "llamaindex";
+import path from "path";
+
+async function main() {
+  const jina = new JinaAIEmbedding({
+    model: "jina-clip-v1",
+  });
+
+  // Get text embeddings
+  const text1 = "a car";
+  const textEmbedding1 = await jina.getTextEmbedding(text1);
+  const text2 = "a football match";
+  const textEmbedding2 = await jina.getTextEmbedding(text2);
+
+  // Get image embedding
+  const image =
+    "https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/football-match.jpg";
+  const imageEmbedding = await jina.getImageEmbedding(image);
+
+  // Calc similarity between text and image
+  const sim1 = similarity(
+    textEmbedding1,
+    imageEmbedding,
+    SimilarityType.DEFAULT,
+  );
+  const sim2 = similarity(
+    textEmbedding2,
+    imageEmbedding,
+    SimilarityType.DEFAULT,
+  );
+
+  console.log(`Similarity between "${text1}" and the image is ${sim1}`);
+  console.log(`Similarity between "${text2}" and the image is ${sim2}`);
+
+  // Get multiple text embeddings
+  const textEmbeddings = await jina.getTextEmbeddings([text1, text2]);
+  const sim3 = similarity(
+    textEmbeddings[0],
+    textEmbeddings[1],
+    SimilarityType.DEFAULT,
+  );
+  console.log(
+    `Similarity between the two texts "${text1}" and "${text2}" is ${sim3}`,
+  );
+
+  // Get multiple image embeddings
+  const catImg1 =
+    "https://i.pinimg.com/600x315/21/48/7e/21487e8e0970dd366dafaed6ab25d8d8.jpg";
+  const catImg2 =
+    "https://i.pinimg.com/736x/c9/f2/3e/c9f23e212529f13f19bad5602d84b78b.jpg";
+  const imageEmbeddings = await jina.getImageEmbeddings([catImg1, catImg2]);
+  const sim4 = similarity(
+    imageEmbeddings[0],
+    imageEmbeddings[1],
+    SimilarityType.DEFAULT,
+  );
+  console.log(`Similarity between the two online cat images is ${sim4}`);
+
+  // Get image embeddings from multiple local files
+  const documents = await new SimpleDirectoryReader().loadData({
+    directoryPath: path.join("multimodal", "data"),
+  });
+  const localImages = documents
+    .filter((doc) => doc instanceof ImageDocument)
+    .slice(0, 2); // Get only the first two images
+  const localImageEmbeddings = await jina.getImageEmbeddings(
+    localImages.map((doc) => (doc as ImageDocument).image),
+  );
+  const sim5 = similarity(
+    localImageEmbeddings[0],
+    localImageEmbeddings[1],
+    SimilarityType.DEFAULT,
+  );
+  console.log(`Similarity between the two local images is ${sim5}`);
+}
+
+void main();

examples/readers/package.json

@@ -11,6 +11,7 @@
     "start:pdf": "node --import tsx ./src/pdf.ts",
     "start:llamaparse": "node --import tsx ./src/llamaparse.ts",
     "start:notion": "node --import tsx ./src/notion.ts",
+    "start:assemblyai": "node --import tsx ./src/assemblyai.ts",
     "start:llamaparse-dir": "node --import tsx ./src/simple-directory-reader-with-llamaparse.ts",
     "start:llamaparse-json": "node --import tsx ./src/llamaparse-json.ts",
     "start:discord": "node --import tsx ./src/discord.ts"

examples/rerankers/CohereReranker.ts

@@ -15,9 +15,9 @@ async function main() {
 
   const index = await VectorStoreIndex.fromDocuments([document]);
 
-  const retriever = index.asRetriever();
-
-  retriever.similarityTopK = 5;
+  const retriever = index.asRetriever({
+    similarityTopK: 5,
+  });
 
   const nodePostprocessor = new CohereRerank({
     apiKey: "<COHERE_API_KEY>",

examples/vectorIndexCustomize.ts

@@ -18,8 +18,9 @@ async function main() {
 
   const index = await VectorStoreIndex.fromDocuments([document]);
 
-  const retriever = index.asRetriever();
-  retriever.similarityTopK = 5;
+  const retriever = index.asRetriever({
+    similarityTopK: 5,
+  });
   const nodePostprocessor = new SimilarityPostprocessor({
     similarityCutoff: 0.7,
   });

packages/autotool/examples/02_nextjs/CHANGELOG.md

@@ -1,5 +1,35 @@
 # @llamaindex/autotool-02-next-example
 
+## 0.1.33
+
+### Patch Changes
+
+- Updated dependencies [3d5ba08]
+- Updated dependencies [d917cdc]
+  - llamaindex@0.5.8
+  - @llamaindex/autotool@2.0.0
+
+## 0.1.32
+
+### Patch Changes
+
+- Updated dependencies [ec59acd]
+  - llamaindex@0.5.7
+  - @llamaindex/autotool@2.0.0
+
+## 0.1.31
+
+### Patch Changes
+
+- Updated dependencies [2562244]
+- Updated dependencies [325aa51]
+- Updated dependencies [ab700ea]
+- Updated dependencies [92f0782]
+- Updated dependencies [6cf6ae6]
+- Updated dependencies [b7cfe5b]
+  - llamaindex@0.5.6
+  - @llamaindex/autotool@2.0.0
+
 ## 0.1.30
 
 ### Patch Changes

packages/autotool/examples/02_nextjs/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/autotool-02-next-example",
   "private": true,
-  "version": "0.1.30",
+  "version": "0.1.33",
   "scripts": {
     "dev": "next dev",
     "build": "next build",

packages/autotool/package.json

@@ -51,7 +51,7 @@
     "unplugin": "^1.10.1"
   },
   "peerDependencies": {
-    "llamaindex": "^0.5.5",
+    "llamaindex": "^0.5.8",
     "openai": "^4",
     "typescript": "^4"
   },

packages/community/CHANGELOG.md

@@ -1,5 +1,12 @@
 # @llamaindex/community
 
+## 0.0.23
+
+### Patch Changes
+
+- Updated dependencies [6cf6ae6]
+  - @llamaindex/core@0.1.3
+
 ## 0.0.22
 
 ### Patch Changes

packages/community/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/community",
   "description": "Community package for LlamaIndexTS",
-  "version": "0.0.22",
+  "version": "0.0.23",
   "type": "module",
   "types": "dist/type/index.d.ts",
   "main": "dist/cjs/index.js",

packages/core/CHANGELOG.md

@@ -1,5 +1,11 @@
 # @llamaindex/core
 
+## 0.1.3
+
+### Patch Changes
+
+- 6cf6ae6: feat: abstract query type
+
 ## 0.1.2
 
 ### Patch Changes

packages/core/package.json

@@ -1,9 +1,23 @@
 {
   "name": "@llamaindex/core",
   "type": "module",
-  "version": "0.1.2",
+  "version": "0.1.3",
   "description": "LlamaIndex Core Module",
   "exports": {
+    "./query-engine": {
+      "require": {
+        "types": "./dist/query-engine/index.d.cts",
+        "default": "./dist/query-engine/index.cjs"
+      },
+      "import": {
+        "types": "./dist/query-engine/index.d.ts",
+        "default": "./dist/query-engine/index.js"
+      },
+      "default": {
+        "types": "./dist/query-engine/index.d.ts",
+        "default": "./dist/query-engine/index.js"
+      }
+    },
     "./llms": {
       "require": {
         "types": "./dist/llms/index.d.cts",

packages/core/src/query-engine/base.ts

@@ -0,0 +1,29 @@
+import type { MessageContent } from "../llms";
+import { EngineResponse, type NodeWithScore } from "../schema";
+
+/**
+ * @link https://docs.llamaindex.ai/en/stable/api_reference/schema/?h=querybundle#llama_index.core.schema.QueryBundle
+ *
+ *  We don't have `image_path` here, because it is included in the `query` field.
+ */
+export type QueryBundle = {
+  query: MessageContent;
+  customEmbeddings?: string[];
+  embeddings?: number[];
+};
+
+export type QueryType = string | QueryBundle;
+
+export interface BaseQueryEngine {
+  query(
+    strOrQueryBundle: QueryType,
+    stream: true,
+  ): Promise<AsyncIterable<EngineResponse>>;
+  query(strOrQueryBundle: QueryType, stream?: false): Promise<EngineResponse>;
+
+  synthesize?(
+    strOrQueryBundle: QueryType,
+    nodes: NodeWithScore[],
+    additionalSources?: Iterator<NodeWithScore>,
+  ): Promise<EngineResponse>;
+}

packages/core/src/query-engine/index.ts

@@ -0,0 +1 @@
+export type { BaseQueryEngine, QueryBundle, QueryType } from "./base";

packages/core/src/schema/index.ts

@@ -1,3 +1,4 @@
 export * from "./node";
 export type { TransformComponent } from "./type";
+export { EngineResponse } from "./type/engine–response";
 export * from "./zod";

packages/core/src/schema/type/engine–response.ts

@@ -1,20 +1,16 @@
-import type {
-  ChatMessage,
-  ChatResponse,
-  ChatResponseChunk,
-} from "@llamaindex/core/llms";
-import type { NodeWithScore } from "@llamaindex/core/schema";
-import { extractText } from "@llamaindex/core/utils";
+import type { ChatMessage, ChatResponse, ChatResponseChunk } from "../../llms";
+import { extractText } from "../../utils";
+import type { Metadata, NodeWithScore } from "../node";
 
 export class EngineResponse implements ChatResponse, ChatResponseChunk {
   sourceNodes?: NodeWithScore[];
 
-  metadata: Record<string, unknown> = {};
+  metadata: Metadata = {};
 
   message: ChatMessage;
   raw: object | null;
 
-  #stream: boolean;
+  readonly stream: boolean;
 
   private constructor(
     chatResponse: ChatResponse,
@@ -24,7 +20,7 @@ export class EngineResponse implements ChatResponse, ChatResponseChunk {
     this.message = chatResponse.message;
     this.raw = chatResponse.raw;
     this.sourceNodes = sourceNodes;
-    this.#stream = stream;
+    this.stream = stream;
   }
 
   static fromResponse(
@@ -70,13 +66,15 @@ export class EngineResponse implements ChatResponse, ChatResponseChunk {
     );
   }
 
-  // @deprecated use 'message' instead
+  /**
+   * @deprecated Use `message` instead.
+   */
   get response(): string {
     return extractText(this.message.content);
   }
 
   get delta(): string {
-    if (!this.#stream) {
+    if (!this.stream) {
       console.warn(
         "delta is only available for streaming responses. Consider using 'message' instead.",
       );
@@ -84,7 +82,7 @@ export class EngineResponse implements ChatResponse, ChatResponseChunk {
     return extractText(this.message.content);
   }
 
-  toString() {
+  toString(): string {
     return this.response ?? "";
   }
 }

packages/core/src/utils/llms.ts

@@ -3,15 +3,22 @@ import type {
   MessageContentDetail,
   MessageContentTextDetail,
 } from "../llms";
+import type { QueryType } from "../query-engine";
 import type { ImageType } from "../schema";
 
 /**
- * Extracts just the text from a multi-modal message or the message itself if it's just text.
+ * Extracts just the text whether from
+ *  a multi-modal message
+ *  a single text message
+ *  or a query
  *
  * @param message The message to extract text from.
  * @returns The extracted text
  */
-export function extractText(message: MessageContent): string {
+export function extractText(message: MessageContent | QueryType): string {
+  if (typeof message === "object" && "query" in message) {
+    return extractText(message.query);
+  }
   if (typeof message !== "string" && !Array.isArray(message)) {
     console.warn(
       "extractText called with non-MessageContent message, this is likely a bug.",

packages/experimental/CHANGELOG.md

@@ -1,5 +1,32 @@
 # @llamaindex/experimental
 
+## 0.0.58
+
+### Patch Changes
+
+- Updated dependencies [3d5ba08]
+- Updated dependencies [d917cdc]
+  - llamaindex@0.5.8
+
+## 0.0.57
+
+### Patch Changes
+
+- Updated dependencies [ec59acd]
+  - llamaindex@0.5.7
+
+## 0.0.56
+
+### Patch Changes
+
+- Updated dependencies [2562244]
+- Updated dependencies [325aa51]
+- Updated dependencies [ab700ea]
+- Updated dependencies [92f0782]
+- Updated dependencies [6cf6ae6]
+- Updated dependencies [b7cfe5b]
+  - llamaindex@0.5.6
+
 ## 0.0.55
 
 ### Patch Changes

packages/experimental/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/experimental",
   "description": "Experimental package for LlamaIndexTS",
-  "version": "0.0.55",
+  "version": "0.0.58",
   "type": "module",
   "types": "dist/type/index.d.ts",
   "main": "dist/cjs/index.js",

packages/llamaindex/CHANGELOG.md

@@ -1,5 +1,31 @@
 # llamaindex
 
+## 0.5.8
+
+### Patch Changes
+
+- 3d5ba08: fix: update user agent in AssemblyAI
+- d917cdc: Add azure interpreter tool to tool factory
+
+## 0.5.7
+
+### Patch Changes
+
+- ec59acd: fix: bundling issue with pnpm
+
+## 0.5.6
+
+### Patch Changes
+
+- 2562244: feat: add gpt4o-mini
+- 325aa51: Implement Jina embedding through Jina api
+- ab700ea: Add missing authentication to LlamaCloudIndex.fromDocuments
+- 92f0782: feat: use query bundle
+- 6cf6ae6: feat: abstract query type
+- b7cfe5b: fix: passing max_token option to replicate's api call
+- Updated dependencies [6cf6ae6]
+  - @llamaindex/core@0.1.3
+
 ## 0.5.5
 
 ### Patch Changes

packages/llamaindex/e2e/examples/cloudflare-worker-agent/CHANGELOG.md

@@ -1,5 +1,32 @@
 # @llamaindex/cloudflare-worker-agent-test
 
+## 0.0.42
+
+### Patch Changes
+
+- Updated dependencies [3d5ba08]
+- Updated dependencies [d917cdc]
+  - llamaindex@0.5.8
+
+## 0.0.41
+
+### Patch Changes
+
+- Updated dependencies [ec59acd]
+  - llamaindex@0.5.7
+
+## 0.0.40
+
+### Patch Changes
+
+- Updated dependencies [2562244]
+- Updated dependencies [325aa51]
+- Updated dependencies [ab700ea]
+- Updated dependencies [92f0782]
+- Updated dependencies [6cf6ae6]
+- Updated dependencies [b7cfe5b]
+  - llamaindex@0.5.6
+
 ## 0.0.39
 
 ### Patch Changes

packages/llamaindex/e2e/examples/cloudflare-worker-agent/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/cloudflare-worker-agent-test",
-  "version": "0.0.39",
+  "version": "0.0.42",
   "type": "module",
   "private": true,
   "scripts": {

packages/llamaindex/e2e/examples/nextjs-agent/CHANGELOG.md

@@ -1,5 +1,32 @@
 # @llamaindex/next-agent-test
 
+## 0.1.42
+
+### Patch Changes
+
+- Updated dependencies [3d5ba08]
+- Updated dependencies [d917cdc]
+  - llamaindex@0.5.8
+
+## 0.1.41
+
+### Patch Changes
+
+- Updated dependencies [ec59acd]
+  - llamaindex@0.5.7
+
+## 0.1.40
+
+### Patch Changes
+
+- Updated dependencies [2562244]
+- Updated dependencies [325aa51]
+- Updated dependencies [ab700ea]
+- Updated dependencies [92f0782]
+- Updated dependencies [6cf6ae6]
+- Updated dependencies [b7cfe5b]
+  - llamaindex@0.5.6
+
 ## 0.1.39
 
 ### Patch Changes

packages/llamaindex/e2e/examples/nextjs-agent/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/next-agent-test",
-  "version": "0.1.39",
+  "version": "0.1.42",
   "private": true,
   "scripts": {
     "dev": "next dev",

packages/llamaindex/e2e/examples/nextjs-edge-runtime/CHANGELOG.md

@@ -1,5 +1,32 @@
 # test-edge-runtime
 
+## 0.1.41
+
+### Patch Changes
+
+- Updated dependencies [3d5ba08]
+- Updated dependencies [d917cdc]
+  - llamaindex@0.5.8
+
+## 0.1.40
+
+### Patch Changes
+
+- Updated dependencies [ec59acd]
+  - llamaindex@0.5.7
+
+## 0.1.39
+
+### Patch Changes
+
+- Updated dependencies [2562244]
+- Updated dependencies [325aa51]
+- Updated dependencies [ab700ea]
+- Updated dependencies [92f0782]
+- Updated dependencies [6cf6ae6]
+- Updated dependencies [b7cfe5b]
+  - llamaindex@0.5.6
+
 ## 0.1.38
 
 ### Patch Changes

packages/llamaindex/e2e/examples/nextjs-edge-runtime/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/nextjs-edge-runtime-test",
-  "version": "0.1.38",
+  "version": "0.1.41",
   "private": true,
   "scripts": {
     "dev": "next dev",

packages/llamaindex/e2e/examples/nextjs-node-runtime/CHANGELOG.md

@@ -1,5 +1,32 @@
 # @llamaindex/next-node-runtime
 
+## 0.0.23
+
+### Patch Changes
+
+- Updated dependencies [3d5ba08]
+- Updated dependencies [d917cdc]
+  - llamaindex@0.5.8
+
+## 0.0.22
+
+### Patch Changes
+
+- Updated dependencies [ec59acd]
+  - llamaindex@0.5.7
+
+## 0.0.21
+
+### Patch Changes
+
+- Updated dependencies [2562244]
+- Updated dependencies [325aa51]
+- Updated dependencies [ab700ea]
+- Updated dependencies [92f0782]
+- Updated dependencies [6cf6ae6]
+- Updated dependencies [b7cfe5b]
+  - llamaindex@0.5.6
+
 ## 0.0.20
 
 ### Patch Changes

packages/llamaindex/e2e/examples/nextjs-node-runtime/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/next-node-runtime-test",
-  "version": "0.0.20",
+  "version": "0.0.23",
   "private": true,
   "scripts": {
     "dev": "next dev",

packages/llamaindex/e2e/examples/waku-query-engine/CHANGELOG.md

@@ -1,5 +1,32 @@
 # @llamaindex/waku-query-engine-test
 
+## 0.0.42
+
+### Patch Changes
+
+- Updated dependencies [3d5ba08]
+- Updated dependencies [d917cdc]
+  - llamaindex@0.5.8
+
+## 0.0.41
+
+### Patch Changes
+
+- Updated dependencies [ec59acd]
+  - llamaindex@0.5.7
+
+## 0.0.40
+
+### Patch Changes
+
+- Updated dependencies [2562244]
+- Updated dependencies [325aa51]
+- Updated dependencies [ab700ea]
+- Updated dependencies [92f0782]
+- Updated dependencies [6cf6ae6]
+- Updated dependencies [b7cfe5b]
+  - llamaindex@0.5.6
+
 ## 0.0.39
 
 ### Patch Changes

packages/llamaindex/e2e/examples/waku-query-engine/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/waku-query-engine-test",
-  "version": "0.0.39",
+  "version": "0.0.42",
   "type": "module",
   "private": true,
   "scripts": {

packages/llamaindex/package.json

@@ -1,6 +1,6 @@
 {
   "name": "llamaindex",
-  "version": "0.5.5",
+  "version": "0.5.8",
   "license": "MIT",
   "type": "module",
   "keywords": [

packages/llamaindex/src/QuestionGenerator.ts

@@ -1,4 +1,6 @@
 import type { LLM, ToolMetadata } from "@llamaindex/core/llms";
+import type { QueryType } from "@llamaindex/core/query-engine";
+import { extractText } from "@llamaindex/core/utils";
 import { SubQuestionOutputParser } from "./OutputParser.js";
 import type { SubQuestionPrompt } from "./Prompt.js";
 import { buildToolsText, defaultSubQuestionPrompt } from "./Prompt.js";
@@ -43,9 +45,12 @@ export class LLMQuestionGenerator
     }
   }
 
-  async generate(tools: ToolMetadata[], query: string): Promise<SubQuestion[]> {
+  async generate(
+    tools: ToolMetadata[],
+    query: QueryType,
+  ): Promise<SubQuestion[]> {
     const toolsStr = buildToolsText(tools);
-    const queryStr = query;
+    const queryStr = extractText(query);
     const prediction = (
       await this.llm.complete({
         prompt: this.prompt({

packages/llamaindex/src/agent/base.ts

@@ -5,10 +5,10 @@ import type {
   MessageContent,
   ToolOutput,
 } from "@llamaindex/core/llms";
+import { EngineResponse } from "@llamaindex/core/schema";
 import { wrapEventCaller } from "@llamaindex/core/utils";
 import { ReadableStream, TransformStream, randomUUID } from "@llamaindex/env";
 import { ChatHistory } from "../ChatHistory.js";
-import { EngineResponse } from "../EngineResponse.js";
 import { Settings } from "../Settings.js";
 import {
   type ChatEngine,

packages/llamaindex/src/cloud/LlamaCloudIndex.ts

@@ -151,6 +151,7 @@ export class LlamaCloudIndex {
       verbose?: boolean;
     } & CloudConstructorParams,
   ): Promise<LlamaCloudIndex> {
+    initService(params);
     const defaultTransformations: TransformComponent<any>[] = [
       new SimpleNodeParser(),
       new OpenAIEmbedding({

packages/llamaindex/src/embeddings/JinaAIEmbedding.ts

@@ -1,29 +1,127 @@
 import { getEnv } from "@llamaindex/env";
-import { OpenAIEmbedding } from "./OpenAIEmbedding.js";
+import { imageToDataUrl } from "../internal/utils.js";
+import type { ImageType } from "../Node.js";
+import { MultiModalEmbedding } from "./MultiModalEmbedding.js";
 
-export class JinaAIEmbedding extends OpenAIEmbedding {
-  constructor(init?: Partial<OpenAIEmbedding>) {
-    const {
-      apiKey = getEnv("JINAAI_API_KEY"),
-      additionalSessionOptions = {},
-      model = "jina-embeddings-v2-base-en",
-      ...rest
-    } = init ?? {};
+function isLocal(url: ImageType): boolean {
+  if (url instanceof Blob) return true;
+  return new URL(url).protocol === "file:";
+}
 
+export type JinaEmbeddingRequest = {
+  input: Array<{ text: string } | { url: string } | { bytes: string }>;
+  model?: string;
+  encoding_type?: "float" | "binary" | "ubinary";
+};
+
+export type JinaEmbeddingResponse = {
+  model: string;
+  object: string;
+  usage: {
+    total_tokens: number;
+    prompt_tokens: number;
+  };
+  data: Array<{
+    object: string;
+    index: number;
+    embedding: number[];
+  }>;
+};
+
+const JINA_MULTIMODAL_MODELS = ["jina-clip-v1"];
+
+export class JinaAIEmbedding extends MultiModalEmbedding {
+  apiKey: string;
+  model: string;
+  baseURL: string;
+
+  async getTextEmbedding(text: string): Promise<number[]> {
+    const result = await this.getJinaEmbedding({ input: [{ text }] });
+    return result.data[0].embedding;
+  }
+
+  async getImageEmbedding(image: ImageType): Promise<number[]> {
+    const img = await this.getImageInput(image);
+    const result = await this.getJinaEmbedding({ input: [img] });
+    return result.data[0].embedding;
+  }
+
+  // Retrieve multiple text embeddings in a single request
+  getTextEmbeddings = async (texts: string[]): Promise<Array<number[]>> => {
+    const input = texts.map((text) => ({ text }));
+    const result = await this.getJinaEmbedding({ input });
+    return result.data.map((d) => d.embedding);
+  };
+
+  // Retrieve multiple image embeddings in a single request
+  async getImageEmbeddings(images: ImageType[]): Promise<number[][]> {
+    const input = await Promise.all(
+      images.map((img) => this.getImageInput(img)),
+    );
+    const result = await this.getJinaEmbedding({ input });
+    return result.data.map((d) => d.embedding);
+  }
+
+  constructor(init?: Partial<JinaAIEmbedding>) {
+    super();
+    const apiKey = init?.apiKey ?? getEnv("JINAAI_API_KEY");
     if (!apiKey) {
       throw new Error(
         "Set Jina AI API Key in JINAAI_API_KEY env variable. Get one for free or top up your key at https://jina.ai/embeddings",
       );
     }
+    this.apiKey = apiKey;
+    this.model = init?.model ?? "jina-embeddings-v2-base-en";
+    this.baseURL = init?.baseURL ?? "https://api.jina.ai/v1/embeddings";
+    init?.embedBatchSize && (this.embedBatchSize = init?.embedBatchSize);
+  }
 
-    additionalSessionOptions.baseURL =
-      additionalSessionOptions.baseURL ?? "https://api.jina.ai/v1";
+  private async getImageInput(
+    image: ImageType,
+  ): Promise<{ bytes: string } | { url: string }> {
+    if (isLocal(image)) {
+      const base64 = await imageToDataUrl(image);
+      const bytes = base64.split(",")[1];
+      return { bytes };
+    } else {
+      return { url: image.toString() };
+    }
+  }
 
-    super({
-      apiKey,
-      additionalSessionOptions,
-      model,
-      ...rest,
+  private async getJinaEmbedding(
+    input: JinaEmbeddingRequest,
+  ): Promise<JinaEmbeddingResponse> {
+    // if input includes image, check if model supports multimodal embeddings
+    if (
+      input.input.some((i) => "url" in i || "bytes" in i) &&
+      !JINA_MULTIMODAL_MODELS.includes(this.model)
+    ) {
+      throw new Error(
+        `Model ${this.model} does not support image embeddings. Use ${JINA_MULTIMODAL_MODELS.join(", ")}`,
+      );
+    }
+
+    const response = await fetch(this.baseURL, {
+      method: "POST",
+      headers: {
+        "Content-Type": "application/json",
+        Authorization: `Bearer ${this.apiKey}`,
+      },
+      body: JSON.stringify({
+        model: this.model,
+        encoding_type: "float",
+        ...input,
+      }),
     });
+    if (!response.ok) {
+      throw new Error(
+        `Request ${this.baseURL} failed with status ${response.status}`,
+      );
+    }
+    const result: JinaEmbeddingResponse = await response.json();
+    return {
+      ...result,
+      data: result.data.sort((a, b) => a.index - b.index), // Sort resulting embeddings by index
+    };
   }
 }

packages/llamaindex/src/engines/chat/CondenseQuestionChatEngine.ts

@@ -1,4 +1,5 @@
 import type { ChatMessage, LLM } from "@llamaindex/core/llms";
+import type { EngineResponse } from "@llamaindex/core/schema";
 import {
   extractText,
   streamReducer,
@@ -6,7 +7,6 @@ import {
 } from "@llamaindex/core/utils";
 import type { ChatHistory } from "../../ChatHistory.js";
 import { getHistory } from "../../ChatHistory.js";
-import type { EngineResponse } from "../../EngineResponse.js";
 import type { CondenseQuestionPrompt } from "../../Prompt.js";
 import {
   defaultCondenseQuestionPrompt,
@@ -109,7 +109,8 @@ export class CondenseQuestionChatEngine
       return streamReducer({
         stream,
         initialValue: "",
-        reducer: (accumulator, part) => (accumulator += part.response),
+        reducer: (accumulator, part) =>
+          (accumulator += extractText(part.message.content)),
         finished: (accumulator) => {
           chatHistory.addMessage({ content: accumulator, role: "assistant" });
         },
@@ -118,7 +119,10 @@ export class CondenseQuestionChatEngine
     const response = await this.queryEngine.query({
       query: condensedQuestion,
     });
-    chatHistory.addMessage({ content: response.response, role: "assistant" });
+    chatHistory.addMessage({
+      content: response.message.content,
+      role: "assistant",
+    });
 
     return response;
   }

packages/llamaindex/src/engines/chat/ContextChatEngine.ts

@@ -4,6 +4,7 @@ import type {
   MessageContent,
   MessageType,
 } from "@llamaindex/core/llms";
+import { EngineResponse } from "@llamaindex/core/schema";
 import {
   extractText,
   streamConverter,
@@ -12,7 +13,6 @@ import {
 } from "@llamaindex/core/utils";
 import type { ChatHistory } from "../../ChatHistory.js";
 import { getHistory } from "../../ChatHistory.js";
-import { EngineResponse } from "../../EngineResponse.js";
 import type { ContextSystemPrompt } from "../../Prompt.js";
 import type { BaseRetriever } from "../../Retriever.js";
 import { Settings } from "../../Settings.js";

packages/llamaindex/src/engines/chat/SimpleChatEngine.ts

@@ -1,4 +1,5 @@
 import type { LLM } from "@llamaindex/core/llms";
+import { EngineResponse } from "@llamaindex/core/schema";
 import {
   streamConverter,
   streamReducer,
@@ -6,7 +7,6 @@ import {
 } from "@llamaindex/core/utils";
 import type { ChatHistory } from "../../ChatHistory.js";
 import { getHistory } from "../../ChatHistory.js";
-import { EngineResponse } from "../../EngineResponse.js";
 import { Settings } from "../../Settings.js";
 import type {
   ChatEngine,

packages/llamaindex/src/engines/chat/types.ts

@@ -1,7 +1,6 @@
 import type { ChatMessage, MessageContent } from "@llamaindex/core/llms";
-import type { NodeWithScore } from "@llamaindex/core/schema";
+import { EngineResponse, type NodeWithScore } from "@llamaindex/core/schema";
 import type { ChatHistory } from "../../ChatHistory.js";
-import type { EngineResponse } from "../../EngineResponse.js";
 
 /**
  * Represents the base parameters for ChatEngine.

packages/llamaindex/src/engines/query/RetrieverQueryEngine.ts

@@ -1,6 +1,5 @@
-import type { NodeWithScore } from "@llamaindex/core/schema";
+import { EngineResponse, type NodeWithScore } from "@llamaindex/core/schema";
 import { wrapEventCaller } from "@llamaindex/core/utils";
-import type { EngineResponse } from "../../EngineResponse.js";
 import type { BaseNodePostprocessor } from "../../postprocessors/index.js";
 import { PromptMixin } from "../../prompts/Mixin.js";
 import type { BaseRetriever } from "../../Retriever.js";
@@ -78,11 +77,13 @@ export class RetrieverQueryEngine extends PromptMixin implements QueryEngine {
     const { query, stream } = params;
     const nodesWithScore = await this.retrieve(query);
     if (stream) {
-      return this.responseSynthesizer.synthesize({
-        query,
-        nodesWithScore,
-        stream: true,
-      });
+      return this.responseSynthesizer.synthesize(
+        {
+          query,
+          nodesWithScore,
+        },
+        true,
+      );
     }
     return this.responseSynthesizer.synthesize({
       query,

packages/llamaindex/src/engines/query/RouterQueryEngine.ts

@@ -1,5 +1,6 @@
-import type { NodeWithScore } from "@llamaindex/core/schema";
-import { EngineResponse } from "../../EngineResponse.js";
+import type { QueryType } from "@llamaindex/core/query-engine";
+import { EngineResponse, type NodeWithScore } from "@llamaindex/core/schema";
+import { extractText } from "@llamaindex/core/utils";
 import type { ServiceContext } from "../../ServiceContext.js";
 import { llmFromSettingsOrContext } from "../../Settings.js";
 import { PromptMixin } from "../../prompts/index.js";
@@ -7,7 +8,6 @@ import type { BaseSelector } from "../../selectors/index.js";
 import { LLMSingleSelector } from "../../selectors/index.js";
 import { TreeSummarize } from "../../synthesizers/index.js";
 import type {
-  QueryBundle,
   QueryEngine,
   QueryEngineParamsNonStreaming,
   QueryEngineParamsStreaming,
@@ -25,7 +25,7 @@ type RouterQueryEngineMetadata = {
 async function combineResponses(
   summarizer: TreeSummarize,
   responses: EngineResponse[],
-  queryBundle: QueryBundle,
+  queryType: QueryType,
   verbose: boolean = false,
 ): Promise<EngineResponse> {
   if (verbose) {
@@ -40,11 +40,11 @@ async function combineResponses(
       sourceNodes.push(...response.sourceNodes);
     }
 
-    responseStrs.push(response.response);
+    responseStrs.push(extractText(response.message.content));
   }
 
   const summary = await summarizer.getResponse({
-    query: queryBundle.queryStr,
+    query: extractText(queryType),
     textChunks: responseStrs,
   });
 
@@ -117,7 +117,7 @@ export class RouterQueryEngine extends PromptMixin implements QueryEngine {
   ): Promise<EngineResponse | AsyncIterable<EngineResponse>> {
     const { query, stream } = params;
 
-    const response = await this.queryRoute({ queryStr: query });
+    const response = await this.queryRoute(query);
 
     if (stream) {
       throw new Error("Streaming is not supported yet.");
@@ -126,8 +126,8 @@ export class RouterQueryEngine extends PromptMixin implements QueryEngine {
     return response;
   }
 
-  private async queryRoute(queryBundle: QueryBundle): Promise<EngineResponse> {
-    const result = await this.selector.select(this.metadatas, queryBundle);
+  private async queryRoute(query: QueryType): Promise<EngineResponse> {
+    const result = await this.selector.select(this.metadatas, query);
 
     if (result.selections.length > 1) {
       const responses: EngineResponse[] = [];
@@ -142,7 +142,7 @@ export class RouterQueryEngine extends PromptMixin implements QueryEngine {
         const selectedQueryEngine = this.queryEngines[engineInd.index];
         responses.push(
           await selectedQueryEngine.query({
-            query: queryBundle.queryStr,
+            query: extractText(query),
           }),
         );
       }
@@ -151,7 +151,7 @@ export class RouterQueryEngine extends PromptMixin implements QueryEngine {
         const finalResponse = await combineResponses(
           this.summarizer,
           responses,
-          queryBundle,
+          query,
           this.verbose,
         );
 
@@ -179,7 +179,7 @@ export class RouterQueryEngine extends PromptMixin implements QueryEngine {
       }
 
       const finalResponse = await selectedQueryEngine.query({
-        query: queryBundle.queryStr,
+        query: extractText(query),
       });
 
       // add selected result

packages/llamaindex/src/engines/query/SubQuestionQueryEngine.ts

@@ -1,6 +1,8 @@
-import type { NodeWithScore } from "@llamaindex/core/schema";
-import { TextNode } from "@llamaindex/core/schema";
-import type { EngineResponse } from "../../EngineResponse.js";
+import {
+  EngineResponse,
+  TextNode,
+  type NodeWithScore,
+} from "@llamaindex/core/schema";
 import { LLMQuestionGenerator } from "../../QuestionGenerator.js";
 import type { ServiceContext } from "../../ServiceContext.js";
 import { PromptMixin } from "../../prompts/Mixin.js";
@@ -10,20 +12,18 @@ import {
   ResponseSynthesizer,
 } from "../../synthesizers/index.js";
 
-import type {
-  QueryEngine,
-  QueryEngineParamsNonStreaming,
-  QueryEngineParamsStreaming,
-} from "../../types.js";
-
 import type { BaseTool, ToolMetadata } from "@llamaindex/core/llms";
+import type { BaseQueryEngine, QueryType } from "@llamaindex/core/query-engine";
 import { wrapEventCaller } from "@llamaindex/core/utils";
 import type { BaseQuestionGenerator, SubQuestion } from "./types.js";
 
 /**
  * SubQuestionQueryEngine decomposes a question into subquestions and then
  */
-export class SubQuestionQueryEngine extends PromptMixin implements QueryEngine {
+export class SubQuestionQueryEngine
+  extends PromptMixin
+  implements BaseQueryEngine
+{
   responseSynthesizer: BaseSynthesizer;
   questionGen: BaseQuestionGenerator;
   queryEngines: BaseTool[];
@@ -73,15 +73,13 @@ export class SubQuestionQueryEngine extends PromptMixin implements QueryEngine {
     });
   }
 
-  query(
-    params: QueryEngineParamsStreaming,
-  ): Promise<AsyncIterable<EngineResponse>>;
-  query(params: QueryEngineParamsNonStreaming): Promise<EngineResponse>;
+  query(query: QueryType, stream: true): Promise<AsyncIterable<EngineResponse>>;
+  query(query: QueryType, stream?: false): Promise<EngineResponse>;
   @wrapEventCaller
   async query(
-    params: QueryEngineParamsStreaming | QueryEngineParamsNonStreaming,
+    query: QueryType,
+    stream?: boolean,
   ): Promise<EngineResponse | AsyncIterable<EngineResponse>> {
-    const { query, stream } = params;
     const subQuestions = await this.questionGen.generate(this.metadatas, query);
 
     const subQNodes = await Promise.all(
@@ -92,16 +90,21 @@ export class SubQuestionQueryEngine extends PromptMixin implements QueryEngine {
       .filter((node) => node !== null)
       .map((node) => node as NodeWithScore);
     if (stream) {
-      return this.responseSynthesizer.synthesize({
+      return this.responseSynthesizer.synthesize(
+        {
+          query,
+          nodesWithScore,
+        },
+        true,
+      );
+    }
+    return this.responseSynthesizer.synthesize(
+      {
         query,
         nodesWithScore,
-        stream: true,
-      });
-    }
-    return this.responseSynthesizer.synthesize({
-      query,
-      nodesWithScore,
-    });
+      },
+      false,
+    );
   }
 
   private async querySubQ(subQ: SubQuestion): Promise<NodeWithScore | null> {

packages/llamaindex/src/engines/query/types.ts

@@ -1,10 +1,11 @@
 import type { ToolMetadata } from "@llamaindex/core/llms";
+import type { QueryType } from "@llamaindex/core/query-engine";
 
 /**
  * QuestionGenerators generate new questions for the LLM using tools and a user query.
  */
 export interface BaseQuestionGenerator {
-  generate(tools: ToolMetadata[], query: string): Promise<SubQuestion[]>;
+  generate(tools: ToolMetadata[], query: QueryType): Promise<SubQuestion[]>;
 }
 
 export interface SubQuestion {

packages/llamaindex/src/evaluation/Correctness.ts

@@ -74,7 +74,7 @@ export class CorrectnessEvaluator extends PromptMixin implements BaseEvaluator {
       {
         role: "user",
         content: defaultUserPrompt({
-          query,
+          query: extractText(query),
           generatedAnswer: response,
           referenceAnswer: reference || "(NO REFERENCE ANSWER SUPPLIED)",
         }),
@@ -106,7 +106,7 @@ export class CorrectnessEvaluator extends PromptMixin implements BaseEvaluator {
     query,
     response,
   }: EvaluatorResponseParams): Promise<EvaluationResult> {
-    const responseStr = response?.response;
+    const responseStr = extractText(response?.message.content);
     const contexts = [];
 
     if (response) {

packages/llamaindex/src/evaluation/Faithfulness.ts

@@ -1,4 +1,5 @@
 import { Document, MetadataMode } from "@llamaindex/core/schema";
+import { extractText } from "@llamaindex/core/utils";
 import type { ServiceContext } from "../ServiceContext.js";
 import { SummaryIndex } from "../indices/summary/index.js";
 import { PromptMixin } from "../prompts/Mixin.js";
@@ -132,7 +133,7 @@ export class FaithfulnessEvaluator
     query,
     response,
   }: EvaluatorResponseParams): Promise<EvaluationResult> {
-    const responseStr = response?.response;
+    const responseStr = extractText(response?.message.content);
     const contexts = [];
 
     if (response) {

packages/llamaindex/src/evaluation/Relevancy.ts

@@ -1,4 +1,5 @@
 import { Document, MetadataMode } from "@llamaindex/core/schema";
+import { extractText } from "@llamaindex/core/utils";
 import type { ServiceContext } from "../ServiceContext.js";
 import { SummaryIndex } from "../indices/summary/index.js";
 import { PromptMixin } from "../prompts/Mixin.js";
@@ -121,7 +122,7 @@ export class RelevancyEvaluator extends PromptMixin implements BaseEvaluator {
     query,
     response,
   }: EvaluatorResponseParams): Promise<EvaluationResult> {
-    const responseStr = response?.response;
+    const responseStr = extractText(response?.message.content);
     const contexts = [];
 
     if (response) {

packages/llamaindex/src/evaluation/types.ts

@@ -1,7 +1,8 @@
-import { EngineResponse } from "../EngineResponse.js";
+import type { QueryType } from "@llamaindex/core/query-engine";
+import type { EngineResponse } from "@llamaindex/core/schema";
 
 export type EvaluationResult = {
-  query?: string;
+  query?: QueryType;
   contexts?: string[];
   response: string | null;
   score: number;
@@ -13,7 +14,7 @@ export type EvaluationResult = {
 };
 
 export type EvaluatorParams = {
-  query: string | null;
+  query: QueryType;
   response: string;
   contexts?: string[];
   reference?: string;
@@ -21,7 +22,7 @@ export type EvaluatorParams = {
 };
 
 export type EvaluatorResponseParams = {
-  query: string | null;
+  query: QueryType;
   response: EngineResponse;
 };
 export interface BaseEvaluator {

packages/llamaindex/src/index.edge.ts

@@ -29,7 +29,6 @@ export * from "./ChatHistory.js";
 export * from "./cloud/index.js";
 export * from "./constants.js";
 export * from "./embeddings/index.js";
-export * from "./EngineResponse.js";
 export * from "./engines/chat/index.js";
 export * from "./engines/query/index.js";
 export * from "./evaluation/index.js";

packages/llamaindex/src/index.ts

@@ -15,4 +15,5 @@ export { type VertexGeminiSessionOptions } from "./llm/gemini/types.js";
 export { GeminiVertexSession } from "./llm/gemini/vertex.js";
 
 // Expose AzureDynamicSessionTool for node.js runtime only
+export { JinaAIEmbedding } from "./embeddings/JinaAIEmbedding.js";
 export { AzureDynamicSessionTool } from "./tools/AzureDynamicSessionTool.node.js";

packages/llamaindex/src/indices/vectorStore/index.ts

@@ -404,7 +404,7 @@ export class VectorIndexRetriever implements BaseRetriever {
   }
 
   /**
-   * @deprecated, pass topK in constructor instead
+   * @deprecated, pass similarityTopK or topK in constructor instead or directly modify topK
    */
   set similarityTopK(similarityTopK: number) {
     this.topK[ModalityType.TEXT] = similarityTopK;

packages/llamaindex/src/llm/openai.ts

@@ -106,6 +106,8 @@ export const GPT4_MODELS = {
   "gpt-4-vision-preview": { contextWindow: 128000 },
   "gpt-4o": { contextWindow: 128000 },
   "gpt-4o-2024-05-13": { contextWindow: 128000 },
+  "gpt-4o-mini": { contextWindow: 128000 },
+  "gpt-4o-mini-2024-07-18": { contextWindow: 128000 },
 };
 
 // NOTE we don't currently support gpt-3.5-turbo-instruct and don't plan to in the near future

packages/llamaindex/src/llm/replicate_ai.ts

@@ -323,6 +323,7 @@ If a question does not make any sense, or is not factually coherent, explain why
         prompt,
         system_prompt: systemPrompt,
         temperature: this.temperature,
+        max_tokens: this.maxTokens,
         top_p: this.topP,
       },
     };

packages/llamaindex/src/next.ts

@@ -32,6 +32,12 @@ export default function withLlamaIndex(config: any) {
       "@google-cloud/vertexai": false,
       "groq-sdk": false,
     };
+    // Following lines will fix issues with onnxruntime-node when using pnpm
+    // See: https://github.com/vercel/next.js/issues/43433
+    webpackConfig.externals.push({
+      "onnxruntime-node": "commonjs onnxruntime-node",
+      sharp: "commonjs sharp",
+    });
     return webpackConfig;
   };
   return config;

packages/llamaindex/src/prompts/Mixin.ts

@@ -75,6 +75,7 @@ export class PromptMixin {
   }
 
   // Must be implemented by subclasses
+  // fixme: says must but never implemented
   protected _getPrompts(): PromptsDict {
     return {};
   }

packages/llamaindex/src/readers/AssemblyAIReader.ts

@@ -11,7 +11,14 @@ import { AssemblyAI } from "assemblyai";
 import type { BaseReader } from "./type.js";
 
 type AssemblyAIOptions = Partial<BaseServiceParams>;
-
+const defaultOptions = {
+  userAgent: {
+    integration: {
+      name: "LlamaIndexTS",
+      version: "1.0.1",
+    },
+  },
+};
 /**
  * Base class for AssemblyAI Readers.
  */
@@ -37,7 +44,10 @@ abstract class AssemblyAIReader implements BaseReader {
       );
     }
 
-    this.client = new AssemblyAI(options as BaseServiceParams);
+    this.client = new AssemblyAI({
+      ...defaultOptions,
+      ...options,
+    } as BaseServiceParams);
   }
 
   abstract loadData(params: TranscribeParams | string): Promise<Document[]>;

packages/llamaindex/src/selectors/base.ts

@@ -1,5 +1,6 @@
+import type { QueryType } from "@llamaindex/core/query-engine";
 import { PromptMixin } from "../prompts/Mixin.js";
-import type { QueryBundle, ToolMetadataOnlyDescription } from "../types.js";
+import type { ToolMetadataOnlyDescription } from "../types.js";
 
 export interface SingleSelection {
   index: number;
@@ -10,8 +11,6 @@ export type SelectorResult = {
   selections: SingleSelection[];
 };
 
-type QueryType = string | QueryBundle;
-
 function wrapChoice(
   choice: string | ToolMetadataOnlyDescription,
 ): ToolMetadataOnlyDescription {
@@ -22,25 +21,16 @@ function wrapChoice(
   }
 }
 
-function wrapQuery(query: QueryType): QueryBundle {
-  if (typeof query === "string") {
-    return { queryStr: query };
-  }
-
-  return query;
-}
-
 type MetadataType = string | ToolMetadataOnlyDescription;
 
 export abstract class BaseSelector extends PromptMixin {
   async select(choices: MetadataType[], query: QueryType) {
-    const metadatas = choices.map((choice) => wrapChoice(choice));
-    const queryBundle = wrapQuery(query);
-    return await this._select(metadatas, queryBundle);
+    const metadata = choices.map((choice) => wrapChoice(choice));
+    return await this._select(metadata, query);
   }
 
   abstract _select(
     choices: ToolMetadataOnlyDescription[],
-    query: QueryBundle,
+    query: QueryType,
   ): Promise<SelectorResult>;
 }

packages/llamaindex/src/selectors/llmSelectors.ts

@@ -1,9 +1,10 @@
 import type { LLM } from "@llamaindex/core/llms";
+import type { QueryBundle } from "@llamaindex/core/query-engine";
+import { extractText } from "@llamaindex/core/utils";
 import type { Answer } from "../outputParsers/selectors.js";
 import { SelectionOutputParser } from "../outputParsers/selectors.js";
 import type {
   BaseOutputParser,
-  QueryBundle,
   StructuredOutput,
   ToolMetadataOnlyDescription,
 } from "../types.js";
@@ -39,19 +40,17 @@ function structuredOutputToSelectorResult(
   return { selections };
 }
 
-type LLMPredictorType = LLM;
-
 /**
  * A selector that uses the LLM to select a single or multiple choices from a list of choices.
  */
 export class LLMMultiSelector extends BaseSelector {
-  llm: LLMPredictorType;
+  llm: LLM;
   prompt: MultiSelectPrompt;
   maxOutputs: number;
   outputParser: BaseOutputParser<StructuredOutput<Answer[]>>;
 
   constructor(init: {
-    llm: LLMPredictorType;
+    llm: LLM;
     prompt?: MultiSelectPrompt;
     maxOutputs?: number;
     outputParser?: BaseOutputParser<StructuredOutput<Answer[]>>;
@@ -88,7 +87,7 @@ export class LLMMultiSelector extends BaseSelector {
     const prompt = this.prompt(
       choicesText.length,
       choicesText,
-      query.queryStr,
+      extractText(query.query),
       this.maxOutputs,
     );
 
@@ -116,12 +115,12 @@ export class LLMMultiSelector extends BaseSelector {
  * A selector that uses the LLM to select a single choice from a list of choices.
  */
 export class LLMSingleSelector extends BaseSelector {
-  llm: LLMPredictorType;
+  llm: LLM;
   prompt: SingleSelectPrompt;
   outputParser: BaseOutputParser<StructuredOutput<Answer[]>>;
 
   constructor(init: {
-    llm: LLMPredictorType;
+    llm: LLM;
     prompt?: SingleSelectPrompt;
     outputParser?: BaseOutputParser<StructuredOutput<Answer[]>>;
   }) {
@@ -152,7 +151,11 @@ export class LLMSingleSelector extends BaseSelector {
   ): Promise<SelectorResult> {
     const choicesText = buildChoicesText(choices);
 
-    const prompt = this.prompt(choicesText.length, choicesText, query.queryStr);
+    const prompt = this.prompt(
+      choicesText.length,
+      choicesText,
+      extractText(query.query),
+    );
 
     const formattedPrompt = this.outputParser.format(prompt);
 

packages/llamaindex/src/storage/vectorStore/MilvusVectorStore.ts

@@ -20,7 +20,6 @@ import {
   metadataDictToNode,
   nodeToMetadata,
   parseArrayValue,
-  parseNumberValue,
   parsePrimitiveValue,
 } from "./utils.js";
 
@@ -60,7 +59,7 @@ function parseScalarFilters(scalarFilters: MetadataFilters): string {
       case ">":
       case ">=": {
         filters.push(
-          `metadata["${filter.key}"] ${filter.operator} ${parseNumberValue(filter.value)}`,
+          `metadata["${filter.key}"] ${filter.operator} ${parsePrimitiveValue(filter.value)}`,
         );
         break;
       }

packages/llamaindex/src/storage/vectorStore/SimpleVectorStore.ts

@@ -21,7 +21,6 @@ import {
 import {
   nodeToMetadata,
   parseArrayValue,
-  parseNumberValue,
   parsePrimitiveValue,
 } from "./utils.js";
 
@@ -43,46 +42,43 @@ const OPERATOR_TO_FILTER: {
   ) => boolean;
 } = {
   [FilterOperator.EQ]: ({ key, value }, metadata) => {
-    return parsePrimitiveValue(metadata[key]) === parsePrimitiveValue(value);
+    return metadata[key] === parsePrimitiveValue(value);
   },
   [FilterOperator.NE]: ({ key, value }, metadata) => {
-    return parsePrimitiveValue(metadata[key]) !== parsePrimitiveValue(value);
+    return metadata[key] !== parsePrimitiveValue(value);
   },
   [FilterOperator.IN]: ({ key, value }, metadata) => {
-    return parseArrayValue(value).includes(parsePrimitiveValue(metadata[key]));
+    return !!parseArrayValue(value).find((v) => metadata[key] === v);
   },
   [FilterOperator.NIN]: ({ key, value }, metadata) => {
-    return !parseArrayValue(value).includes(parsePrimitiveValue(metadata[key]));
+    return !parseArrayValue(value).find((v) => metadata[key] === v);
   },
   [FilterOperator.ANY]: ({ key, value }, metadata) => {
-    return parseArrayValue(value).some((v) =>
-      parseArrayValue(metadata[key]).includes(v),
-    );
+    if (!Array.isArray(metadata[key])) return false;
+    return parseArrayValue(value).some((v) => metadata[key].includes(v));
   },
   [FilterOperator.ALL]: ({ key, value }, metadata) => {
-    return parseArrayValue(value).every((v) =>
-      parseArrayValue(metadata[key]).includes(v),
-    );
+    if (!Array.isArray(metadata[key])) return false;
+    return parseArrayValue(value).every((v) => metadata[key].includes(v));
   },
   [FilterOperator.TEXT_MATCH]: ({ key, value }, metadata) => {
-    return parsePrimitiveValue(metadata[key]).includes(
-      parsePrimitiveValue(value),
-    );
+    return metadata[key].includes(parsePrimitiveValue(value));
   },
   [FilterOperator.CONTAINS]: ({ key, value }, metadata) => {
-    return parseArrayValue(metadata[key]).includes(parsePrimitiveValue(value));
+    if (!Array.isArray(metadata[key])) return false;
+    return !!parseArrayValue(metadata[key]).find((v) => v === value);
   },
   [FilterOperator.GT]: ({ key, value }, metadata) => {
-    return parseNumberValue(metadata[key]) > parseNumberValue(value);
+    return metadata[key] > parsePrimitiveValue(value);
   },
   [FilterOperator.LT]: ({ key, value }, metadata) => {
-    return parseNumberValue(metadata[key]) < parseNumberValue(value);
+    return metadata[key] < parsePrimitiveValue(value);
   },
   [FilterOperator.GTE]: ({ key, value }, metadata) => {
-    return parseNumberValue(metadata[key]) >= parseNumberValue(value);
+    return metadata[key] >= parsePrimitiveValue(value);
   },
   [FilterOperator.LTE]: ({ key, value }, metadata) => {
-    return parseNumberValue(metadata[key]) <= parseNumberValue(value);
+    return metadata[key] <= parsePrimitiveValue(value);
   },
 };
 
@@ -97,7 +93,8 @@ const buildFilterFn = (
   const { filters, condition } = preFilters;
   const queryCondition = condition || "and"; // default to and
 
-  const itemFilterFn = (filter: MetadataFilter) => {
+  const itemFilterFn = (filter: MetadataFilter): boolean => {
+    if (metadata[filter.key] === undefined) return false; // always return false if the metadata key is not present
     const metadataLookupFn = OPERATOR_TO_FILTER[filter.operator];
     if (!metadataLookupFn)
       throw new Error(`Unsupported operator: ${filter.operator}`);

packages/llamaindex/src/storage/vectorStore/utils.ts

@@ -79,24 +79,23 @@ export function metadataDictToNode(
   }
 }
 
-export const parseNumberValue = (value: MetadataFilterValue): number => {
-  if (typeof value !== "number") throw new Error("Value must be a number");
-  return value;
-};
-
-export const parsePrimitiveValue = (value: MetadataFilterValue): string => {
+export const parsePrimitiveValue = (
+  value: MetadataFilterValue,
+): string | number => {
   if (typeof value !== "number" && typeof value !== "string") {
     throw new Error("Value must be a string or number");
   }
-  return value.toString();
+  return value;
 };
 
-export const parseArrayValue = (value: MetadataFilterValue): string[] => {
+export const parseArrayValue = (
+  value: MetadataFilterValue,
+): string[] | number[] => {
   const isPrimitiveArray =
     Array.isArray(value) &&
     value.every((v) => typeof v === "string" || typeof v === "number");
   if (!isPrimitiveArray) {
     throw new Error("Value must be an array of strings or numbers");
   }
-  return value.map(String);
+  return value;
 };

packages/llamaindex/src/synthesizers/MultiModalResponseSynthesizer.ts

@@ -1,16 +1,11 @@
-import { MetadataMode } from "@llamaindex/core/schema";
+import { EngineResponse, MetadataMode } from "@llamaindex/core/schema";
 import { streamConverter } from "@llamaindex/core/utils";
-import { EngineResponse } from "../EngineResponse.js";
 import type { ServiceContext } from "../ServiceContext.js";
 import { llmFromSettingsOrContext } from "../Settings.js";
 import { PromptMixin } from "../prompts/Mixin.js";
 import type { TextQaPrompt } from "./../Prompt.js";
 import { defaultTextQaPrompt } from "./../Prompt.js";
-import type {
-  BaseSynthesizer,
-  SynthesizeParamsNonStreaming,
-  SynthesizeParamsStreaming,
-} from "./types.js";
+import type { BaseSynthesizer, SynthesizeQuery } from "./types.js";
 import { createMessageContent } from "./utils.js";
 
 export class MultiModalResponseSynthesizer
@@ -48,21 +43,22 @@ export class MultiModalResponseSynthesizer
   }
 
   synthesize(
-    params: SynthesizeParamsStreaming,
+    query: SynthesizeQuery,
+    stream: true,
   ): Promise<AsyncIterable<EngineResponse>>;
-  synthesize(params: SynthesizeParamsNonStreaming): Promise<EngineResponse>;
-  async synthesize({
-    query,
-    nodesWithScore,
-    stream,
-  }: SynthesizeParamsStreaming | SynthesizeParamsNonStreaming): Promise<
-    AsyncIterable<EngineResponse> | EngineResponse
-  > {
+  synthesize(query: SynthesizeQuery, stream?: false): Promise<EngineResponse>;
+  async synthesize(
+    query: SynthesizeQuery,
+    stream?: boolean,
+  ): Promise<AsyncIterable<EngineResponse> | EngineResponse> {
+    const { nodesWithScore } = query;
     const nodes = nodesWithScore.map(({ node }) => node);
     const prompt = await createMessageContent(
       this.textQATemplate,
       nodes,
-      { query },
+      // fixme: wtf type is this?
+      // { query },
+      {},
       this.metadataMode,
     );
 

packages/llamaindex/src/synthesizers/ResponseSynthesizer.ts

@@ -1,6 +1,5 @@
-import { MetadataMode } from "@llamaindex/core/schema";
+import { EngineResponse, MetadataMode } from "@llamaindex/core/schema";
 import { streamConverter } from "@llamaindex/core/utils";
-import { EngineResponse } from "../EngineResponse.js";
 import type { ServiceContext } from "../ServiceContext.js";
 import { PromptMixin } from "../prompts/Mixin.js";
 import type { ResponseBuilderPrompts } from "./builders.js";
@@ -8,8 +7,7 @@ import { getResponseBuilder } from "./builders.js";
 import type {
   BaseSynthesizer,
   ResponseBuilder,
-  SynthesizeParamsNonStreaming,
-  SynthesizeParamsStreaming,
+  SynthesizeQuery,
 } from "./types.js";
 
 /**
@@ -56,33 +54,37 @@ export class ResponseSynthesizer
   }
 
   synthesize(
-    params: SynthesizeParamsStreaming,
+    query: SynthesizeQuery,
+    stream: true,
   ): Promise<AsyncIterable<EngineResponse>>;
-  synthesize(params: SynthesizeParamsNonStreaming): Promise<EngineResponse>;
-  async synthesize({
-    query,
-    nodesWithScore,
-    stream,
-  }: SynthesizeParamsStreaming | SynthesizeParamsNonStreaming): Promise<
-    AsyncIterable<EngineResponse> | EngineResponse
-  > {
+  synthesize(query: SynthesizeQuery, stream?: false): Promise<EngineResponse>;
+  async synthesize(
+    query: SynthesizeQuery,
+    stream?: boolean,
+  ): Promise<AsyncIterable<EngineResponse> | EngineResponse> {
+    const { nodesWithScore } = query;
     const textChunks: string[] = nodesWithScore.map(({ node }) =>
       node.getContent(this.metadataMode),
     );
     if (stream) {
-      const response = await this.responseBuilder.getResponse({
-        query,
-        textChunks,
-        stream,
-      });
+      const response = await this.responseBuilder.getResponse(
+        {
+          ...query,
+          textChunks,
+        },
+        true,
+      );
       return streamConverter(response, (chunk) =>
         EngineResponse.fromResponse(chunk, true, nodesWithScore),
       );
     }
-    const response = await this.responseBuilder.getResponse({
-      query,
-      textChunks,
-    });
+    const response = await this.responseBuilder.getResponse(
+      {
+        ...query,
+        textChunks,
+      },
+      false,
+    );
     return EngineResponse.fromResponse(response, false, nodesWithScore);
   }
 }

packages/llamaindex/src/synthesizers/builders.ts

@@ -1,5 +1,6 @@
 import type { LLM } from "@llamaindex/core/llms";
-import { streamConverter } from "@llamaindex/core/utils";
+import type { QueryType } from "@llamaindex/core/query-engine";
+import { extractText, streamConverter } from "@llamaindex/core/utils";
 import type {
   RefinePrompt,
   SimplePrompt,
@@ -19,11 +20,7 @@ import {
   llmFromSettingsOrContext,
   promptHelperFromSettingsOrContext,
 } from "../Settings.js";
-import type {
-  ResponseBuilder,
-  ResponseBuilderParamsNonStreaming,
-  ResponseBuilderParamsStreaming,
-} from "./types.js";
+import type { ResponseBuilder, ResponseBuilderQuery } from "./types.js";
 
 /**
  * Response modes of the response synthesizer
@@ -48,20 +45,16 @@ export class SimpleResponseBuilder implements ResponseBuilder {
   }
 
   getResponse(
-    params: ResponseBuilderParamsStreaming,
+    query: ResponseBuilderQuery,
+    stream: true,
   ): Promise<AsyncIterable<string>>;
-  getResponse(params: ResponseBuilderParamsNonStreaming): Promise<string>;
-  async getResponse({
-    query,
-    textChunks,
-    stream,
-  }:
-    | ResponseBuilderParamsStreaming
-    | ResponseBuilderParamsNonStreaming): Promise<
-    AsyncIterable<string> | string
-  > {
+  getResponse(query: ResponseBuilderQuery, stream?: false): Promise<string>;
+  async getResponse(
+    { query, textChunks }: ResponseBuilderQuery,
+    stream?: boolean,
+  ): Promise<AsyncIterable<string> | string> {
     const input = {
-      query,
+      query: extractText(query),
       context: textChunks.join("\n\n"),
     };
 
@@ -122,19 +115,14 @@ export class Refine extends PromptMixin implements ResponseBuilder {
   }
 
   getResponse(
-    params: ResponseBuilderParamsStreaming,
+    query: ResponseBuilderQuery,
+    stream: true,
   ): Promise<AsyncIterable<string>>;
-  getResponse(params: ResponseBuilderParamsNonStreaming): Promise<string>;
-  async getResponse({
-    query,
-    textChunks,
-    prevResponse,
-    stream,
-  }:
-    | ResponseBuilderParamsStreaming
-    | ResponseBuilderParamsNonStreaming): Promise<
-    AsyncIterable<string> | string
-  > {
+  getResponse(query: ResponseBuilderQuery, stream?: false): Promise<string>;
+  async getResponse(
+    { query, textChunks, prevResponse }: ResponseBuilderQuery,
+    stream?: boolean,
+  ): Promise<AsyncIterable<string> | string> {
     let response: AsyncIterable<string> | string | undefined = prevResponse;
 
     for (let i = 0; i < textChunks.length; i++) {
@@ -160,12 +148,12 @@ export class Refine extends PromptMixin implements ResponseBuilder {
   }
 
   private async giveResponseSingle(
-    queryStr: string,
+    query: QueryType,
     textChunk: string,
     stream: boolean,
   ) {
     const textQATemplate: SimplePrompt = (input) =>
-      this.textQATemplate({ ...input, query: queryStr });
+      this.textQATemplate({ ...input, query: extractText(query) });
     const textChunks = this.promptHelper.repack(textQATemplate, [textChunk]);
 
     let response: AsyncIterable<string> | string | undefined = undefined;
@@ -183,7 +171,7 @@ export class Refine extends PromptMixin implements ResponseBuilder {
       } else {
         response = await this.refineResponseSingle(
           response as string,
-          queryStr,
+          query,
           chunk,
           stream && lastChunk,
         );
@@ -196,12 +184,12 @@ export class Refine extends PromptMixin implements ResponseBuilder {
   // eslint-disable-next-line max-params
   private async refineResponseSingle(
     initialReponse: string,
-    queryStr: string,
+    query: QueryType,
     textChunk: string,
     stream: boolean,
   ) {
     const refineTemplate: SimplePrompt = (input) =>
-      this.refineTemplate({ ...input, query: queryStr });
+      this.refineTemplate({ ...input, query: extractText(query) });
 
     const textChunks = this.promptHelper.repack(refineTemplate, [textChunk]);
 
@@ -240,23 +228,24 @@ export class Refine extends PromptMixin implements ResponseBuilder {
  */
 export class CompactAndRefine extends Refine {
   getResponse(
-    params: ResponseBuilderParamsStreaming,
+    query: ResponseBuilderQuery,
+    stream: true,
   ): Promise<AsyncIterable<string>>;
-  getResponse(params: ResponseBuilderParamsNonStreaming): Promise<string>;
-  async getResponse({
-    query,
-    textChunks,
-    prevResponse,
-    stream,
-  }:
-    | ResponseBuilderParamsStreaming
-    | ResponseBuilderParamsNonStreaming): Promise<
-    AsyncIterable<string> | string
-  > {
+  getResponse(query: ResponseBuilderQuery, stream?: false): Promise<string>;
+  async getResponse(
+    { query, textChunks, prevResponse }: ResponseBuilderQuery,
+    stream?: boolean,
+  ): Promise<AsyncIterable<string> | string> {
     const textQATemplate: SimplePrompt = (input) =>
-      this.textQATemplate({ ...input, query: query });
+      this.textQATemplate({
+        ...input,
+        query: extractText(query),
+      });
     const refineTemplate: SimplePrompt = (input) =>
-      this.refineTemplate({ ...input, query: query });
+      this.refineTemplate({
+        ...input,
+        query: extractText(query),
+      });
 
     const maxPrompt = getBiggestPrompt([textQATemplate, refineTemplate]);
     const newTexts = this.promptHelper.repack(maxPrompt, textChunks);
@@ -266,10 +255,12 @@ export class CompactAndRefine extends Refine {
       prevResponse,
     };
     if (stream) {
-      return super.getResponse({
-        ...params,
-        stream,
-      });
+      return super.getResponse(
+        {
+          ...params,
+        },
+        true,
+      );
     }
     return super.getResponse(params);
   }
@@ -309,18 +300,14 @@ export class TreeSummarize extends PromptMixin implements ResponseBuilder {
   }
 
   getResponse(
-    params: ResponseBuilderParamsStreaming,
+    query: ResponseBuilderQuery,
+    stream: true,
   ): Promise<AsyncIterable<string>>;
-  getResponse(params: ResponseBuilderParamsNonStreaming): Promise<string>;
-  async getResponse({
-    query,
-    textChunks,
-    stream,
-  }:
-    | ResponseBuilderParamsStreaming
-    | ResponseBuilderParamsNonStreaming): Promise<
-    AsyncIterable<string> | string
-  > {
+  getResponse(query: ResponseBuilderQuery, stream?: false): Promise<string>;
+  async getResponse(
+    { query, textChunks }: ResponseBuilderQuery,
+    stream?: boolean,
+  ): Promise<AsyncIterable<string> | string> {
     if (!textChunks || textChunks.length === 0) {
       throw new Error("Must have at least one text chunk");
     }
@@ -335,7 +322,7 @@ export class TreeSummarize extends PromptMixin implements ResponseBuilder {
       const params = {
         prompt: this.summaryTemplate({
           context: packedTextChunks[0],
-          query,
+          query: extractText(query),
         }),
       };
       if (stream) {
@@ -349,7 +336,7 @@ export class TreeSummarize extends PromptMixin implements ResponseBuilder {
           this.llm.complete({
             prompt: this.summaryTemplate({
               context: chunk,
-              query,
+              query: extractText(query),
             }),
           }),
         ),
@@ -360,10 +347,12 @@ export class TreeSummarize extends PromptMixin implements ResponseBuilder {
         textChunks: summaries.map((s) => s.text),
       };
       if (stream) {
-        return this.getResponse({
-          ...params,
-          stream,
-        });
+        return this.getResponse(
+          {
+            ...params,
+          },
+          true,
+        );
       }
       return this.getResponse(params);
     }

packages/llamaindex/src/synthesizers/types.ts

@@ -1,56 +1,40 @@
-import type { NodeWithScore } from "@llamaindex/core/schema";
-import type { EngineResponse } from "../EngineResponse.js";
+import type { QueryType } from "@llamaindex/core/query-engine";
+import { EngineResponse, type NodeWithScore } from "@llamaindex/core/schema";
 import type { PromptMixin } from "../prompts/Mixin.js";
 
-export interface SynthesizeParamsBase {
-  query: string;
+export interface SynthesizeQuery {
+  query: QueryType;
   nodesWithScore: NodeWithScore[];
 }
 
-export interface SynthesizeParamsStreaming extends SynthesizeParamsBase {
-  stream: true;
-}
-
-export interface SynthesizeParamsNonStreaming extends SynthesizeParamsBase {
-  stream?: false | null;
-}
-
+// todo(himself65): Move this to @llamaindex/core/schema
 /**
  * A BaseSynthesizer is used to generate a response from a query and a list of nodes.
  */
 export interface BaseSynthesizer {
   synthesize(
-    params: SynthesizeParamsStreaming,
+    query: SynthesizeQuery,
+    stream: true,
   ): Promise<AsyncIterable<EngineResponse>>;
-  synthesize(params: SynthesizeParamsNonStreaming): Promise<EngineResponse>;
+  synthesize(query: SynthesizeQuery, stream?: false): Promise<EngineResponse>;
 }
 
-export interface ResponseBuilderParamsBase {
-  query: string;
+export interface ResponseBuilderQuery {
+  query: QueryType;
   textChunks: string[];
   prevResponse?: string;
 }
 
-export interface ResponseBuilderParamsStreaming
-  extends ResponseBuilderParamsBase {
-  stream: true;
-}
-
-export interface ResponseBuilderParamsNonStreaming
-  extends ResponseBuilderParamsBase {
-  stream?: false | null;
-}
-
 /**
  * A ResponseBuilder is used in a response synthesizer to generate a response from multiple response chunks.
  */
 export interface ResponseBuilder extends Partial<PromptMixin> {
   /**
    * Get the response from a query and a list of text chunks.
-   * @param params
    */
   getResponse(
-    params: ResponseBuilderParamsStreaming,
+    query: ResponseBuilderQuery,
+    stream: true,
   ): Promise<AsyncIterable<string>>;
-  getResponse(params: ResponseBuilderParamsNonStreaming): Promise<string>;
+  getResponse(query: ResponseBuilderQuery, stream?: false): Promise<string>;
 }

packages/llamaindex/src/tools/AzureDynamicSessionTool.node.ts

@@ -177,7 +177,7 @@ export class AzureDynamicSessionTool
   /**
    * The endpoint of the Azure pool management service.
    * This is where the tool will send requests to interact with the session pool.
-   * If not provided, the tool will use the value of the `AZURE_CONTAINER_APP_SESSION_POOL_MANAGEMENT_ENDPOINT` environment variable.
+   * If not provided, the tool will use the value of the `AZURE_POOL_MANAGEMENT_ENDPOINT` environment variable.
    */
   private poolManagementEndpoint: string;
 
@@ -191,14 +191,12 @@ export class AzureDynamicSessionTool
     this.sessionId = params?.sessionId || randomUUID();
     this.poolManagementEndpoint =
       params?.poolManagementEndpoint ||
-      (getEnv("AZURE_CONTAINER_APP_SESSION_POOL_MANAGEMENT_ENDPOINT") ?? "");
+      (getEnv("AZURE_POOL_MANAGEMENT_ENDPOINT") ?? "");
     this.azureADTokenProvider =
       params?.azureADTokenProvider ?? getAzureADTokenProvider();
 
     if (!this.poolManagementEndpoint) {
-      throw new Error(
-        "AZURE_CONTAINER_APP_SESSION_POOL_MANAGEMENT_ENDPOINT must be defined.",
-      );
+      throw new Error("AZURE_POOL_MANAGEMENT_ENDPOINT must be defined.");
     }
   }
 

packages/llamaindex/src/tools/QueryEngineTool.ts

@@ -1,6 +1,6 @@
 import type { BaseTool, ToolMetadata } from "@llamaindex/core/llms";
+import type { BaseQueryEngine } from "@llamaindex/core/query-engine";
 import type { JSONSchemaType } from "ajv";
-import type { QueryEngine } from "../types.js";
 
 const DEFAULT_NAME = "query_engine_tool";
 const DEFAULT_DESCRIPTION =
@@ -18,7 +18,7 @@ const DEFAULT_PARAMETERS: JSONSchemaType<QueryEngineParam> = {
 };
 
 export type QueryEngineToolParams = {
-  queryEngine: QueryEngine;
+  queryEngine: BaseQueryEngine;
   metadata: ToolMetadata<JSONSchemaType<QueryEngineParam>>;
 };
 
@@ -27,7 +27,7 @@ export type QueryEngineParam = {
 };
 
 export class QueryEngineTool implements BaseTool<QueryEngineParam> {
-  private queryEngine: QueryEngine;
+  private queryEngine: BaseQueryEngine;
   metadata: ToolMetadata<JSONSchemaType<QueryEngineParam>>;
 
   constructor({ queryEngine, metadata }: QueryEngineToolParams) {
@@ -42,6 +42,6 @@ export class QueryEngineTool implements BaseTool<QueryEngineParam> {
   async call({ query }: QueryEngineParam) {
     const response = await this.queryEngine.query({ query });
 
-    return response.response;
+    return response.message.content;
   }
 }

packages/llamaindex/src/tools/ToolsFactory.ts

@@ -1,12 +1,18 @@
-import { WikipediaTool } from "./WikipediaTool.js";
+import {
+  AzureDynamicSessionTool,
+  type AzureDynamicSessionToolParams,
+} from "./AzureDynamicSessionTool.node.js";
+import { WikipediaTool, type WikipediaToolParams } from "./WikipediaTool.js";
 
 export namespace ToolsFactory {
   type ToolsMap = {
     [Tools.Wikipedia]: typeof WikipediaTool;
+    [Tools.AzureCodeInterpreter]: typeof AzureDynamicSessionTool;
   };
 
   export enum Tools {
     Wikipedia = "wikipedia.WikipediaToolSpec",
+    AzureCodeInterpreter = "azure_code_interpreter.AzureCodeInterpreterToolSpec",
   }
 
   export async function createTool<Tool extends Tools>(
@@ -14,7 +20,15 @@ export namespace ToolsFactory {
     ...params: ConstructorParameters<ToolsMap[Tool]>
   ): Promise<InstanceType<ToolsMap[Tool]>> {
     if (key === Tools.Wikipedia) {
-      return new WikipediaTool(...params) as InstanceType<ToolsMap[Tool]>;
+      return new WikipediaTool(
+        ...(params as WikipediaToolParams[]),
+      ) as InstanceType<ToolsMap[Tool]>;
+    }
+
+    if (key === Tools.AzureCodeInterpreter) {
+      return new AzureDynamicSessionTool(
+        ...(params as AzureDynamicSessionToolParams[]),
+      ) as InstanceType<ToolsMap[Tool]>;
     }
 
     throw new Error(

packages/llamaindex/src/types.ts

@@ -2,7 +2,7 @@
  * Top level types to avoid circular dependencies
  */
 import type { ToolMetadata } from "@llamaindex/core/llms";
-import type { EngineResponse } from "./EngineResponse.js";
+import type { EngineResponse } from "@llamaindex/core/schema";
 
 /**
  * Parameters for sending a query.
@@ -52,16 +52,4 @@ export interface StructuredOutput<T> {
 
 export type ToolMetadataOnlyDescription = Pick<ToolMetadata, "description">;
 
-export class QueryBundle {
-  queryStr: string;
-
-  constructor(queryStr: string) {
-    this.queryStr = queryStr;
-  }
-
-  toString(): string {
-    return this.queryStr;
-  }
-}
-
 export type UUID = `${string}-${string}-${string}-${string}-${string}`;

packages/llamaindex/tests/vectorStores/SimpleVectorStore.test.ts

@@ -87,6 +87,19 @@ describe("SimpleVectorStore", () => {
         title: "No filter",
         expected: 3,
       },
+      {
+        title: "Filter with non-exist key",
+        filters: {
+          filters: [
+            {
+              key: "non-exist-key",
+              value: "cat",
+              operator: "==",
+            },
+          ],
+        },
+        expected: 0,
+      },
       {
         title: "Filter EQ",
         filters: {