Release 0.6.9 (#1252)

Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>

.github/workflows/test.yml

@@ -13,8 +13,10 @@ concurrency:
   cancel-in-progress: true
 
 env:
-  POSTGRES_USER: runneradmin
   POSTGRES_HOST_AUTH_METHOD: trust
+  TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }}
+  TURBO_TEAM: ${{ vars.TURBO_TEAM }}
+  TURBO_REMOTE_ONLY: true
 
 jobs:
   e2e:

README.md

@@ -167,11 +167,13 @@ export async function chatWithAgent(
       // ... adding your tools here
     ],
   });
-  const responseStream = await agent.chat({
-    stream: true,
-    message: question,
-    chatHistory: prevMessages,
-  });
+  const responseStream = await agent.chat(
+    {
+      message: question,
+      chatHistory: prevMessages,
+    },
+    true,
+  );
   const uiStream = createStreamableUI(<div>loading...</div>);
   responseStream
     .pipeTo(

apps/docs/CHANGELOG.md

@@ -1,5 +1,56 @@
 # docs
 
+## 0.0.78
+
+### Patch Changes
+
+- llamaindex@0.6.9
+
+## 0.0.77
+
+### Patch Changes
+
+- Updated dependencies [8b7fdba]
+  - llamaindex@0.6.8
+
+## 0.0.76
+
+### Patch Changes
+
+- Updated dependencies [23bcc37]
+  - llamaindex@0.6.7
+
+## 0.0.75
+
+### Patch Changes
+
+- Updated dependencies [d902cc3]
+- Updated dependencies [025ffe6]
+- Updated dependencies [a659574]
+  - llamaindex@0.6.6
+
+## 0.0.74
+
+### Patch Changes
+
+- Updated dependencies [e9714db]
+  - llamaindex@0.6.5
+
+## 0.0.73
+
+### Patch Changes
+
+- Updated dependencies [b48bcc3]
+  - llamaindex@0.6.4
+
+## 0.0.72
+
+### Patch Changes
+
+- Updated dependencies [2cd1383]
+- Updated dependencies [5c4badb]
+  - llamaindex@0.6.3
+
 ## 0.0.71
 
 ### Patch Changes

apps/docs/package.json

@@ -1,6 +1,6 @@
 {
   "name": "docs",
-  "version": "0.0.71",
+  "version": "0.0.78",
   "private": true,
   "scripts": {
     "docusaurus": "docusaurus",

examples/anthropic/chat_interactive.ts

@@ -18,7 +18,7 @@ import readline from "node:readline/promises";
   });
   const chatEngine = new SimpleChatEngine({
     llm,
-    chatHistory,
+    memory: chatHistory,
   });
   const rl = readline.createInterface({ input, output });
 

examples/lowlevel.ts

@@ -1,7 +1,7 @@
 import {
   Document,
+  getResponseSynthesizer,
   NodeWithScore,
-  ResponseSynthesizer,
   SentenceSplitter,
   TextNode,
 } from "llamaindex";
@@ -14,7 +14,7 @@ import {
 
   console.log(nodes);
 
-  const responseSynthesizer = new ResponseSynthesizer();
+  const responseSynthesizer = getResponseSynthesizer("compact");
 
   const nodesWithScore: NodeWithScore[] = [
     {
@@ -30,7 +30,7 @@ import {
   const stream = await responseSynthesizer.synthesize(
     {
       query: "What age am I?",
-      nodesWithScore,
+      nodes: nodesWithScore,
     },
     true,
   );

examples/multimodal/context.ts

@@ -1,4 +1,5 @@
 // call pnpm tsx multimodal/load.ts first to init the storage
+import { extractText } from "@llamaindex/core/utils";
 import {
   ContextChatEngine,
   NodeWithScore,
@@ -25,8 +26,9 @@ Settings.callbackManager.on("retrieve-end", (event) => {
   const textNodes = nodes.filter(
     (node: NodeWithScore) => node.node.type === ObjectType.TEXT,
   );
+  const text = extractText(query);
   console.log(
-    `Retrieved ${textNodes.length} text nodes and ${imageNodes.length} image nodes for query: ${query}`,
+    `Retrieved ${textNodes.length} text nodes and ${imageNodes.length} image nodes for query: ${text}`,
   );
 });
 

examples/multimodal/rag.ts

@@ -1,5 +1,6 @@
+import { extractText } from "@llamaindex/core/utils";
 import {
-  MultiModalResponseSynthesizer,
+  getResponseSynthesizer,
   OpenAI,
   Settings,
   VectorStoreIndex,
@@ -16,7 +17,8 @@ Settings.llm = new OpenAI({ model: "gpt-4-turbo", maxTokens: 512 });
 // Update callbackManager
 Settings.callbackManager.on("retrieve-end", (event) => {
   const { nodes, query } = event.detail;
-  console.log(`Retrieved ${nodes.length} nodes for query: ${query}`);
+  const text = extractText(query);
+  console.log(`Retrieved ${nodes.length} nodes for query: ${text}`);
 });
 
 async function main() {
@@ -27,7 +29,7 @@ async function main() {
   });
 
   const queryEngine = index.asQueryEngine({
-    responseSynthesizer: new MultiModalResponseSynthesizer(),
+    responseSynthesizer: getResponseSynthesizer("multi_modal"),
     retriever: index.asRetriever({ topK: { TEXT: 3, IMAGE: 1 } }),
   });
   const stream = await queryEngine.query({

examples/pg-vector-store/load-docs.ts

@@ -40,7 +40,11 @@ async function main(args: any) {
     const rdr = new SimpleDirectoryReader(callback);
     const docs = await rdr.loadData({ directoryPath: sourceDir });
 
-    const pgvs = new PGVectorStore();
+    const pgvs = new PGVectorStore({
+      clientConfig: {
+        connectionString: process.env.PG_CONNECTION_STRING,
+      },
+    });
     pgvs.setCollection(sourceDir);
     await pgvs.clearCollection();
 

examples/pg-vector-store/query.ts

@@ -7,7 +7,11 @@ async function main() {
   });
 
   try {
-    const pgvs = new PGVectorStore();
+    const pgvs = new PGVectorStore({
+      clientConfig: {
+        connectionString: process.env.PG_CONNECTION_STRING,
+      },
+    });
     // Optional - set your collection name, default is no filter on this field.
     // pgvs.setCollection();
 

examples/prompts/promptMixin.ts

@@ -1,8 +1,7 @@
 import {
   Document,
+  getResponseSynthesizer,
   PromptTemplate,
-  ResponseSynthesizer,
-  TreeSummarize,
   TreeSummarizePrompt,
   VectorStoreIndex,
 } from "llamaindex";
@@ -27,9 +26,7 @@ async function main() {
 
   const query = "The quick brown fox jumps over the lazy dog";
 
-  const responseSynthesizer = new ResponseSynthesizer({
-    responseBuilder: new TreeSummarize(),
-  });
+  const responseSynthesizer = getResponseSynthesizer("tree_summarize");
 
   const queryEngine = index.asQueryEngine({
     responseSynthesizer,

examples/readers/src/csv.ts

@@ -1,8 +1,7 @@
 import {
-  CompactAndRefine,
+  getResponseSynthesizer,
   OpenAI,
   PromptTemplate,
-  ResponseSynthesizer,
   Settings,
   VectorStoreIndex,
 } from "llamaindex";
@@ -29,8 +28,8 @@ Given the CSV file, generate me Typescript code to answer the question: {query}.
 `,
   });
 
-  const responseSynthesizer = new ResponseSynthesizer({
-    responseBuilder: new CompactAndRefine(undefined, csvPrompt),
+  const responseSynthesizer = getResponseSynthesizer("compact", {
+    textQATemplate: csvPrompt,
   });
 
   const queryEngine = index.asQueryEngine({ responseSynthesizer });

examples/readers/src/llamaparse-json.ts

@@ -1,3 +1,4 @@
+import { createMessageContent } from "@llamaindex/core/utils";
 import {
   Document,
   ImageNode,
@@ -6,7 +7,6 @@ import {
   PromptTemplate,
   VectorStoreIndex,
 } from "llamaindex";
-import { createMessageContent } from "llamaindex/synthesizers/utils";
 
 const reader = new LlamaParseReader();
 async function main() {

examples/vectorIndexAnthropic.ts

@@ -2,12 +2,10 @@ import fs from "node:fs/promises";
 
 import {
   Anthropic,
-  CompactAndRefine,
   Document,
-  ResponseSynthesizer,
   Settings,
   VectorStoreIndex,
-  anthropicTextQaPrompt,
+  getResponseSynthesizer,
 } from "llamaindex";
 
 // Update llm to use Anthropic
@@ -23,9 +21,7 @@ async function main() {
   const document = new Document({ text: essay, id_: path });
 
   // Split text and create embeddings. Store them in a VectorStoreIndex
-  const responseSynthesizer = new ResponseSynthesizer({
-    responseBuilder: new CompactAndRefine(undefined, anthropicTextQaPrompt),
-  });
+  const responseSynthesizer = getResponseSynthesizer("compact");
 
   const index = await VectorStoreIndex.fromDocuments([document]);
 

examples/vectorIndexFromVectorStore.ts

@@ -1,11 +1,10 @@
 import {
+  getResponseSynthesizer,
   OpenAI,
   OpenAIEmbedding,
-  ResponseSynthesizer,
   RetrieverQueryEngine,
   Settings,
   TextNode,
-  TreeSummarize,
   VectorIndexRetriever,
   VectorStore,
   VectorStoreIndex,
@@ -165,10 +164,7 @@ async function main() {
       similarityTopK: 500,
     });
 
-    const responseSynthesizer = new ResponseSynthesizer({
-      responseBuilder: new TreeSummarize(),
-    });
-
+    const responseSynthesizer = getResponseSynthesizer("tree_summarize");
     return new RetrieverQueryEngine(retriever, responseSynthesizer, {
       filter,
     });

packages/autotool/CHANGELOG.md

@@ -1,5 +1,56 @@
 # @llamaindex/autotool
 
+## 3.0.9
+
+### Patch Changes
+
+- llamaindex@0.6.9
+
+## 3.0.8
+
+### Patch Changes
+
+- Updated dependencies [8b7fdba]
+  - llamaindex@0.6.8
+
+## 3.0.7
+
+### Patch Changes
+
+- Updated dependencies [23bcc37]
+  - llamaindex@0.6.7
+
+## 3.0.6
+
+### Patch Changes
+
+- Updated dependencies [d902cc3]
+- Updated dependencies [025ffe6]
+- Updated dependencies [a659574]
+  - llamaindex@0.6.6
+
+## 3.0.5
+
+### Patch Changes
+
+- Updated dependencies [e9714db]
+  - llamaindex@0.6.5
+
+## 3.0.4
+
+### Patch Changes
+
+- Updated dependencies [b48bcc3]
+  - llamaindex@0.6.4
+
+## 3.0.3
+
+### Patch Changes
+
+- Updated dependencies [2cd1383]
+- Updated dependencies [5c4badb]
+  - llamaindex@0.6.3
+
 ## 3.0.2
 
 ### Patch Changes

packages/autotool/examples/01_node/CHANGELOG.md

@@ -1,5 +1,63 @@
 # @llamaindex/autotool-01-node-example
 
+## 0.0.18
+
+### Patch Changes
+
+- llamaindex@0.6.9
+- @llamaindex/autotool@3.0.9
+
+## 0.0.17
+
+### Patch Changes
+
+- Updated dependencies [8b7fdba]
+  - llamaindex@0.6.8
+  - @llamaindex/autotool@3.0.8
+
+## 0.0.16
+
+### Patch Changes
+
+- Updated dependencies [23bcc37]
+  - llamaindex@0.6.7
+  - @llamaindex/autotool@3.0.7
+
+## 0.0.15
+
+### Patch Changes
+
+- Updated dependencies [d902cc3]
+- Updated dependencies [025ffe6]
+- Updated dependencies [a659574]
+  - llamaindex@0.6.6
+  - @llamaindex/autotool@3.0.6
+
+## 0.0.14
+
+### Patch Changes
+
+- Updated dependencies [e9714db]
+  - llamaindex@0.6.5
+  - @llamaindex/autotool@3.0.5
+
+## 0.0.13
+
+### Patch Changes
+
+- Updated dependencies [b48bcc3]
+  - llamaindex@0.6.4
+  - @llamaindex/autotool@3.0.4
+
+## 0.0.12
+
+### Patch Changes
+
+- Updated dependencies [2cd1383]
+- Updated dependencies [5c4badb]
+  - llamaindex@0.6.3
+  - @llamaindex/autotool@3.0.3
+
 ## 0.0.11
 
 ### Patch Changes

packages/autotool/examples/01_node/package.json

@@ -13,5 +13,5 @@
   "scripts": {
     "start": "node --import tsx --import @llamaindex/autotool/node ./src/index.ts"
   },
-  "version": "0.0.11"
+  "version": "0.0.18"
 }

packages/autotool/examples/02_nextjs/CHANGELOG.md

@@ -1,5 +1,63 @@
 # @llamaindex/autotool-02-next-example
 
+## 0.1.62
+
+### Patch Changes
+
+- llamaindex@0.6.9
+- @llamaindex/autotool@3.0.9
+
+## 0.1.61
+
+### Patch Changes
+
+- Updated dependencies [8b7fdba]
+  - llamaindex@0.6.8
+  - @llamaindex/autotool@3.0.8
+
+## 0.1.60
+
+### Patch Changes
+
+- Updated dependencies [23bcc37]
+  - llamaindex@0.6.7
+  - @llamaindex/autotool@3.0.7
+
+## 0.1.59
+
+### Patch Changes
+
+- Updated dependencies [d902cc3]
+- Updated dependencies [025ffe6]
+- Updated dependencies [a659574]
+  - llamaindex@0.6.6
+  - @llamaindex/autotool@3.0.6
+
+## 0.1.58
+
+### Patch Changes
+
+- Updated dependencies [e9714db]
+  - llamaindex@0.6.5
+  - @llamaindex/autotool@3.0.5
+
+## 0.1.57
+
+### Patch Changes
+
+- Updated dependencies [b48bcc3]
+  - llamaindex@0.6.4
+  - @llamaindex/autotool@3.0.4
+
+## 0.1.56
+
+### Patch Changes
+
+- Updated dependencies [2cd1383]
+- Updated dependencies [5c4badb]
+  - llamaindex@0.6.3
+  - @llamaindex/autotool@3.0.3
+
 ## 0.1.55
 
 ### Patch Changes

packages/autotool/examples/02_nextjs/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/autotool-02-next-example",
   "private": true,
-  "version": "0.1.55",
+  "version": "0.1.62",
   "scripts": {
     "dev": "next dev",
     "build": "next build",

packages/autotool/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/autotool",
   "type": "module",
-  "version": "3.0.2",
+  "version": "3.0.9",
   "description": "auto transpile your JS function to LLM Agent compatible",
   "files": [
     "dist",

packages/cloud/CHANGELOG.md

@@ -1,5 +1,23 @@
 # @llamaindex/cloud
 
+## 0.2.8
+
+### Patch Changes
+
+- ac41ed3: feat: bump cloud sdk version
+
+## 0.2.7
+
+### Patch Changes
+
+- fb36eff: fix: backport for node.js 18
+
+  There could have one missing API in the node.js 18, so we need to backport it to make it work.
+
+- d24d3d1: fix: print warning when llama parse reader has error
+- Updated dependencies [2cd1383]
+  - @llamaindex/core@0.2.3
+
 ## 0.2.6
 
 ### Patch Changes

packages/cloud/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/cloud",
-  "version": "0.2.6",
+  "version": "0.2.8",
   "type": "module",
   "license": "MIT",
   "scripts": {
@@ -50,12 +50,12 @@
   "devDependencies": {
     "@hey-api/client-fetch": "^0.2.4",
     "@hey-api/openapi-ts": "^0.53.0",
-    "@llamaindex/core": "workspace:^0.2.2",
+    "@llamaindex/core": "workspace:^0.2.3",
     "@llamaindex/env": "workspace:^0.1.11",
     "bunchee": "5.3.2"
   },
   "peerDependencies": {
-    "@llamaindex/core": "workspace:^0.2.2",
+    "@llamaindex/core": "workspace:^0.2.3",
     "@llamaindex/env": "workspace:^0.1.11"
   },
   "dependencies": {

packages/cloud/src/reader.ts

@@ -170,6 +170,15 @@ export class LlamaParseReader extends FileReader {
   vendorMultimodalModelName?: string | undefined;
   // The API key for the multimodal API. Can also be set as an env variable: LLAMA_CLOUD_VENDOR_MULTIMODAL_API_KEY
   vendorMultimodalApiKey?: string | undefined;
+
+  webhookUrl?: string | undefined;
+  premiumMode?: boolean | undefined;
+  takeScreenshot?: boolean | undefined;
+  disableOcr?: boolean | undefined;
+  disableReconstruction?: boolean | undefined;
+  inputS3Path?: string | undefined;
+  outputS3PathPrefix?: string | undefined;
+
   // numWorkers is implemented in SimpleDirectoryReader
   stdout?: WriteStream | undefined;
 
@@ -229,20 +238,18 @@ export class LlamaParseReader extends FileReader {
   }
 
   // Create a job for the LlamaParse API
-  private async createJob(
-    data: Uint8Array,
-    fileName: string = "unknown",
-  ): Promise<string> {
+  private async createJob(data: Uint8Array): Promise<string> {
     // Load data, set the mime type
-    const { mime, extension } = await LlamaParseReader.getMimeType(data);
+    const { mime } = await LlamaParseReader.getMimeType(data);
 
     if (this.verbose) {
-      const name = fileName ? fileName : extension;
-      console.log(`Starting load for ${name} file`);
+      console.log("Started uploading the file");
     }
 
     const body = {
-      file: new File([data], fileName, { type: mime }),
+      file: new Blob([data], {
+        type: mime,
+      }),
       language: this.language,
       parsing_instruction: this.parsingInstruction,
       skip_diagonal_text: this.skipDiagonalText,
@@ -260,13 +267,13 @@ export class LlamaParseReader extends FileReader {
       use_vendor_multimodal_model: this.useVendorMultimodalModel,
       vendor_multimodal_model_name: this.vendorMultimodalModelName,
       vendor_multimodal_api_key: this.vendorMultimodalApiKey,
-      // fixme: does these fields need to be set?
-      webhook_url: undefined,
-      take_screenshot: undefined,
-      disable_ocr: undefined,
-      disable_reconstruction: undefined,
-      input_s3_path: undefined,
-      output_s3_path_prefix: undefined,
+      premium_mode: this.premiumMode,
+      webhook_url: this.webhookUrl,
+      take_screenshot: this.takeScreenshot,
+      disable_ocr: this.disableOcr,
+      disable_reconstruction: this.disableReconstruction,
+      input_s3_path: this.inputS3Path,
+      output_s3_path_prefix: this.outputS3PathPrefix,
     } satisfies {
       [Key in keyof Body_upload_file_api_v1_parsing_upload_post]-?:
         | Body_upload_file_api_v1_parsing_upload_post[Key]
@@ -373,14 +380,10 @@ export class LlamaParseReader extends FileReader {
    * To be used with resultType = "text" and "markdown"
    *
    * @param {Uint8Array} fileContent - The content of the file to be loaded.
-   * @param {string} [fileName] - The optional name of the file to be loaded.
    * @return {Promise<Document[]>} A Promise object that resolves to an array of Document objects.
    */
-  async loadDataAsContent(
-    fileContent: Uint8Array,
-    fileName?: string,
-  ): Promise<Document[]> {
-    return this.createJob(fileContent, fileName)
+  async loadDataAsContent(fileContent: Uint8Array): Promise<Document[]> {
+    return this.createJob(fileContent)
       .then(async (jobId) => {
         if (this.verbose) {
           console.log(`Started parsing the file under job id ${jobId}`);
@@ -403,6 +406,7 @@ export class LlamaParseReader extends FileReader {
       })
       .catch((error) => {
         if (this.ignoreErrors) {
+          console.warn(`Error while parsing the file: ${error.message}`);
           return [];
         } else {
           throw error;
@@ -437,8 +441,8 @@ export class LlamaParseReader extends FileReader {
       resultJson.file_path = isFilePath ? filePathOrContent : undefined;
       return [resultJson];
     } catch (e) {
-      console.error(`Error while parsing the file under job id ${jobId}`, e);
       if (this.ignoreErrors) {
+        console.error(`Error while parsing the file under job id ${jobId}`, e);
         return [];
       } else {
         throw e;

packages/cloud/turbo.json

@@ -0,0 +1,8 @@
+{
+  "extends": ["//"],
+  "tasks": {
+    "build": {
+      "outputs": ["dist/**", "src/client/**"]
+    }
+  }
+}

packages/community/CHANGELOG.md

@@ -1,5 +1,35 @@
 # @llamaindex/community
 
+## 0.0.40
+
+### Patch Changes
+
+- 50e6b57: feat: add Amazon Bedrock Retriever
+- Updated dependencies [8b7fdba]
+  - @llamaindex/core@0.2.6
+
+## 0.0.39
+
+### Patch Changes
+
+- Updated dependencies [d902cc3]
+  - @llamaindex/core@0.2.5
+
+## 0.0.38
+
+### Patch Changes
+
+- Updated dependencies [b48bcc3]
+  - @llamaindex/core@0.2.4
+  - @llamaindex/env@0.1.12
+
+## 0.0.37
+
+### Patch Changes
+
+- Updated dependencies [2cd1383]
+  - @llamaindex/core@0.2.3
+
 ## 0.0.36
 
 ### Patch Changes

packages/community/README.md

@@ -7,6 +7,7 @@
 - Bedrock support for the Anthropic Claude Models [usage](https://ts.llamaindex.ai/modules/llms/available_llms/bedrock)
 - Bedrock support for the Meta LLama 2, 3 and 3.1 Models [usage](https://ts.llamaindex.ai/modules/llms/available_llms/bedrock)
 - Meta LLama3.1 405b tool call support
+- Bedrock support for querying Knowledge Base
 
 ## LICENSE
 

packages/community/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/community",
   "description": "Community package for LlamaIndexTS",
-  "version": "0.0.36",
+  "version": "0.0.40",
   "type": "module",
   "types": "dist/type/index.d.ts",
   "main": "dist/cjs/index.js",
@@ -47,6 +47,7 @@
   },
   "dependencies": {
     "@aws-sdk/client-bedrock-runtime": "^3.642.0",
+    "@aws-sdk/client-bedrock-agent-runtime": "^3.642.0",
     "@llamaindex/core": "workspace:*",
     "@llamaindex/env": "workspace:*"
   }

packages/community/src/index.ts

@@ -3,3 +3,4 @@ export {
   BEDROCK_MODEL_MAX_TOKENS,
   Bedrock,
 } from "./llm/bedrock/index.js";
+export { AmazonKnowledgeBaseRetriever } from "./retrievers/bedrock.js";

packages/community/src/retrievers/bedrock.ts

@@ -0,0 +1,165 @@
+import type { KnowledgeBaseVectorSearchConfiguration } from "@aws-sdk/client-bedrock-agent-runtime";
+import {
+  BedrockAgentRuntimeClient,
+  type BedrockAgentRuntimeClientConfig,
+  type RetrievalFilter,
+  RetrieveCommand,
+  type SearchType,
+} from "@aws-sdk/client-bedrock-agent-runtime";
+import type { QueryBundle } from "@llamaindex/core/query-engine";
+import { BaseRetriever } from "@llamaindex/core/retriever";
+import { Document, type NodeWithScore } from "@llamaindex/core/schema";
+import { extractText } from "@llamaindex/core/utils";
+
+/**
+ * Interface for the arguments required to initialize an
+ * AmazonKnowledgeBaseRetriever instance.
+ */
+export interface AmazonKnowledgeBaseRetrieverArgs {
+  knowledgeBaseId: string;
+  topK: number;
+  region: string;
+  clientOptions?: BedrockAgentRuntimeClientConfig;
+  filter?: RetrievalFilter;
+  overrideSearchType?: SearchType;
+}
+
+/**
+ * Class for interacting with Amazon Bedrock Knowledge Bases, a RAG workflow oriented service
+ * Extends the BaseRetriever class.
+ * @example
+ * ```typescript
+ * const retriever = new AmazonKnowledgeBaseRetriever({
+ *   topK: 10,
+ *   knowledgeBaseId: "YOUR_KNOWLEDGE_BASE_ID",
+ *   region: "us-east-2",
+ *   clientOptions: {
+ *     credentials: {
+ *       accessKeyId: "YOUR_ACCESS_KEY_ID",
+ *       secretAccessKey: "YOUR_SECRET_ACCESS_KEY",
+ *     },
+ *   },
+ * });
+ *
+ * const docs = await retriever.retrieve({query: "How are clouds formed?"});
+ * ```
+ */
+export class AmazonKnowledgeBaseRetriever extends BaseRetriever {
+  static lc_name() {
+    return "AmazonKnowledgeBaseRetriever";
+  }
+
+  lc_namespace = ["llamaindex", "retrievers", "amazon_bedrock_knowledge_base"];
+
+  knowledgeBaseId: string;
+
+  topK: number;
+
+  bedrockAgentRuntimeClient: BedrockAgentRuntimeClient;
+
+  filter: RetrievalFilter | undefined;
+
+  overrideSearchType: SearchType | undefined;
+
+  constructor({
+    knowledgeBaseId,
+    topK = 10,
+    clientOptions,
+    region,
+    filter,
+    overrideSearchType,
+  }: AmazonKnowledgeBaseRetrieverArgs) {
+    super();
+
+    this.topK = topK;
+    this.filter = filter;
+    this.overrideSearchType = overrideSearchType;
+    this.bedrockAgentRuntimeClient = new BedrockAgentRuntimeClient({
+      region,
+      ...clientOptions,
+    });
+    this.knowledgeBaseId = knowledgeBaseId;
+  }
+
+  /**
+   * Cleans the result text by replacing sequences of whitespace with a
+   * single space and removing ellipses.
+   * @param resText The result text to clean.
+   * @returns The cleaned result text.
+   */
+  cleanResult(resText: string) {
+    const res = resText.replace(/\s+/g, " ").replace(/\.\.\./g, "");
+    return res;
+  }
+
+  async queryKnowledgeBase(
+    query: QueryBundle,
+    topK: number,
+    filter?: RetrievalFilter,
+    overrideSearchType?: SearchType,
+  ): Promise<NodeWithScore[]> {
+    const retrieveCommand = new RetrieveCommand({
+      knowledgeBaseId: this.knowledgeBaseId,
+      retrievalQuery: {
+        text: extractText(query),
+      },
+      retrievalConfiguration: {
+        vectorSearchConfiguration: {
+          numberOfResults: topK,
+          overrideSearchType,
+          filter,
+        } as KnowledgeBaseVectorSearchConfiguration,
+      },
+    });
+
+    const retrieveResponse =
+      await this.bedrockAgentRuntimeClient.send(retrieveCommand);
+
+    return (
+      retrieveResponse.retrievalResults?.map((result) => {
+        let source;
+        switch (result.location?.type) {
+          case "CONFLUENCE":
+            source = result.location?.confluenceLocation?.url;
+            break;
+          case "S3":
+            source = result.location?.s3Location?.uri;
+            break;
+          case "SALESFORCE":
+            source = result.location?.salesforceLocation?.url;
+            break;
+          case "SHAREPOINT":
+            source = result.location?.sharePointLocation?.url;
+            break;
+          case "WEB":
+            source = result.location?.webLocation?.url;
+            break;
+          default:
+            source = result.location?.s3Location?.uri;
+            break;
+        }
+
+        return {
+          node: new Document({
+            text: this.cleanResult(result.content?.text || ""),
+            metadata: {
+              source,
+              score: result.score,
+              ...result.metadata,
+            },
+          }),
+          score: result.score ?? 1.0,
+        };
+      }) ?? []
+    );
+  }
+
+  async _retrieve(query: QueryBundle): Promise<NodeWithScore[]> {
+    return await this.queryKnowledgeBase(
+      query,
+      this.topK,
+      this.filter,
+      this.overrideSearchType,
+    );
+  }
+}

packages/core/CHANGELOG.md

@@ -1,5 +1,43 @@
 # @llamaindex/core
 
+## 0.2.6
+
+### Patch Changes
+
+- 8b7fdba: refactor: move chat engine & retriever into core.
+
+  - `chatHistory` in BaseChatEngine now returns `ChatMessage[] | Promise<ChatMessage[]>`, instead of `BaseMemory`
+  - update `retrieve-end` type
+
+## 0.2.5
+
+### Patch Changes
+
+- d902cc3: Fix context not being sent using ContextChatEngine
+
+## 0.2.4
+
+### Patch Changes
+
+- b48bcc3: feat: add `load-transformers` event type when loading `@xenova/transformers` module
+
+  This would benefit user who want to customize the transformer env.
+
+- Updated dependencies [b48bcc3]
+  - @llamaindex/env@0.1.12
+
+## 0.2.3
+
+### Patch Changes
+
+- 2cd1383: refactor: align `response-synthesizers` & `chat-engine` module
+
+  - builtin event system
+  - correct class extends
+  - aligin APIs, naming with llama-index python
+  - move stream out of first parameter to second parameter for the better tyep checking
+  - remove JSONQueryEngine in `@llamaindex/experimental`, as the code quality is not satisify and we will bring it back later
+
 ## 0.2.2
 
 ### Patch Changes

packages/core/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/core",
   "type": "module",
-  "version": "0.2.2",
+  "version": "0.2.6",
   "description": "LlamaIndex Core Module",
   "exports": {
     "./node-parser": {
@@ -185,6 +185,48 @@
         "types": "./dist/storage/chat-store/index.d.ts",
         "default": "./dist/storage/chat-store/index.js"
       }
+    },
+    "./response-synthesizers": {
+      "require": {
+        "types": "./dist/response-synthesizers/index.d.cts",
+        "default": "./dist/response-synthesizers/index.cjs"
+      },
+      "import": {
+        "types": "./dist/response-synthesizers/index.d.ts",
+        "default": "./dist/response-synthesizers/index.js"
+      },
+      "default": {
+        "types": "./dist/response-synthesizers/index.d.ts",
+        "default": "./dist/response-synthesizers/index.js"
+      }
+    },
+    "./chat-engine": {
+      "require": {
+        "types": "./dist/chat-engine/index.d.cts",
+        "default": "./dist/chat-engine/index.cjs"
+      },
+      "import": {
+        "types": "./dist/chat-engine/index.d.ts",
+        "default": "./dist/chat-engine/index.js"
+      },
+      "default": {
+        "types": "./dist/chat-engine/index.d.ts",
+        "default": "./dist/chat-engine/index.js"
+      }
+    },
+    "./retriever": {
+      "require": {
+        "types": "./dist/retriever/index.d.cts",
+        "default": "./dist/retriever/index.cjs"
+      },
+      "import": {
+        "types": "./dist/retriever/index.d.ts",
+        "default": "./dist/retriever/index.js"
+      },
+      "default": {
+        "types": "./dist/retriever/index.d.ts",
+        "default": "./dist/retriever/index.js"
+      }
     }
   },
   "files": [
@@ -210,6 +252,7 @@
   "dependencies": {
     "@llamaindex/env": "workspace:*",
     "@types/node": "^22.5.1",
+    "magic-bytes.js": "^1.10.0",
     "zod": "^3.23.8"
   }
 }

packages/core/src/chat-engine/index.ts

@@ -0,0 +1,36 @@
+import type { ChatMessage, MessageContent } from "../llms";
+import type { BaseMemory } from "../memory";
+import { EngineResponse } from "../schema";
+
+export interface BaseChatEngineParams<
+  AdditionalMessageOptions extends object = object,
+> {
+  message: MessageContent;
+  /**
+   * Optional chat history if you want to customize the chat history.
+   */
+  chatHistory?:
+    | ChatMessage<AdditionalMessageOptions>[]
+    | BaseMemory<AdditionalMessageOptions>;
+}
+
+export interface StreamingChatEngineParams<
+  AdditionalMessageOptions extends object = object,
+> extends BaseChatEngineParams<AdditionalMessageOptions> {
+  stream: true;
+}
+
+export interface NonStreamingChatEngineParams<
+  AdditionalMessageOptions extends object = object,
+> extends BaseChatEngineParams<AdditionalMessageOptions> {
+  stream?: false;
+}
+
+export abstract class BaseChatEngine {
+  abstract chat(params: NonStreamingChatEngineParams): Promise<EngineResponse>;
+  abstract chat(
+    params: StreamingChatEngineParams,
+  ): Promise<AsyncIterable<EngineResponse>>;
+
+  abstract chatHistory: ChatMessage[] | Promise<ChatMessage[]>;
+}

packages/core/src/global/settings/callback-manager.ts

@@ -6,8 +6,14 @@ import type {
   ToolCall,
   ToolOutput,
 } from "../../llms";
+import type { QueryEndEvent, QueryStartEvent } from "../../query-engine";
+import type {
+  SynthesizeEndEvent,
+  SynthesizeStartEvent,
+} from "../../response-synthesizers";
+import type { RetrieveEndEvent, RetrieveStartEvent } from "../../retriever";
 import { TextNode } from "../../schema";
-import { EventCaller, getEventCaller } from "../../utils/event-caller";
+import { EventCaller, getEventCaller } from "../../utils";
 import type { UUID } from "../type";
 
 export type LLMStartEvent = {
@@ -60,6 +66,12 @@ export interface LlamaIndexEventMaps {
   "chunking-end": ChunkingEndEvent;
   "node-parsing-start": NodeParsingStartEvent;
   "node-parsing-end": NodeParsingEndEvent;
+  "query-start": QueryStartEvent;
+  "query-end": QueryEndEvent;
+  "synthesize-start": SynthesizeStartEvent;
+  "synthesize-end": SynthesizeEndEvent;
+  "retrieve-start": RetrieveStartEvent;
+  "retrieve-end": RetrieveEndEvent;
 }
 
 export class LlamaIndexCustomEvent<T = any> extends CustomEvent<T> {
@@ -119,16 +131,29 @@ export class CallbackManager {
   dispatchEvent<K extends keyof LlamaIndexEventMaps>(
     event: K,
     detail: LlamaIndexEventMaps[K],
+    sync = false,
   ) {
     const cbs = this.#handlers.get(event);
     if (!cbs) {
       return;
     }
-    queueMicrotask(() => {
+    if (typeof queueMicrotask === "undefined") {
+      console.warn(
+        "queueMicrotask is not available, dispatching synchronously",
+      );
+      sync = true;
+    }
+    if (sync) {
       cbs.forEach((handler) =>
         handler(LlamaIndexCustomEvent.fromEvent(event, { ...detail })),
       );
-    });
+    } else {
+      queueMicrotask(() => {
+        cbs.forEach((handler) =>
+          handler(LlamaIndexCustomEvent.fromEvent(event, { ...detail })),
+        );
+      });
+    }
   }
 }
 

packages/core/src/indices/prompt-helper.ts

@@ -1,10 +1,13 @@
 import { type Tokenizer, tokenizers } from "@llamaindex/env";
 import {
   DEFAULT_CHUNK_OVERLAP_RATIO,
+  DEFAULT_CHUNK_SIZE,
   DEFAULT_CONTEXT_WINDOW,
   DEFAULT_NUM_OUTPUTS,
   DEFAULT_PADDING,
+  Settings,
 } from "../global";
+import type { LLMMetadata } from "../llms";
 import { SentenceSplitter } from "../node-parser";
 import type { PromptTemplate } from "../prompts";
 
@@ -133,4 +136,29 @@ export class PromptHelper {
     const combinedStr = textChunks.join("\n\n");
     return textSplitter.splitText(combinedStr);
   }
+
+  static fromLLMMetadata(
+    metadata: LLMMetadata,
+    options?: {
+      chunkOverlapRatio?: number;
+      chunkSizeLimit?: number;
+      tokenizer?: Tokenizer;
+      separator?: string;
+    },
+  ) {
+    const {
+      chunkOverlapRatio = DEFAULT_CHUNK_OVERLAP_RATIO,
+      chunkSizeLimit = DEFAULT_CHUNK_SIZE,
+      tokenizer = Settings.tokenizer,
+      separator = " ",
+    } = options ?? {};
+    return new PromptHelper({
+      contextWindow: metadata.contextWindow,
+      numOutput: metadata.maxTokens ?? DEFAULT_NUM_OUTPUTS,
+      chunkOverlapRatio,
+      chunkSizeLimit,
+      tokenizer,
+      separator,
+    });
+  }
 }

packages/core/src/memory/base.ts

@@ -1,5 +1,5 @@
 import { Settings } from "../global";
-import type { ChatMessage, MessageContent } from "../llms";
+import type { ChatMessage } from "../llms";
 import { type BaseChatStore, SimpleChatStore } from "../storage/chat-store";
 import { extractText } from "../utils";
 
@@ -12,15 +12,36 @@ export const DEFAULT_CHAT_STORE_KEY = "chat_history";
 export abstract class BaseMemory<
   AdditionalMessageOptions extends object = object,
 > {
+  /**
+   * Retrieves messages from the memory, optionally including transient messages.
+   * Compared to getAllMessages, this method a) allows for transient messages to be included in the retrieval and b) may return a subset of the total messages by applying a token limit.
+   * @param transientMessages Optional array of temporary messages to be included in the retrieval.
+   * These messages are not stored in the memory but are considered for the current interaction.
+   * @returns An array of chat messages, either synchronously or as a Promise.
+   */
   abstract getMessages(
-    input?: MessageContent | undefined,
+    transientMessages?: ChatMessage<AdditionalMessageOptions>[] | undefined,
   ):
     | ChatMessage<AdditionalMessageOptions>[]
     | Promise<ChatMessage<AdditionalMessageOptions>[]>;
+
+  /**
+   * Retrieves all messages stored in the memory.
+   * @returns An array of all chat messages, either synchronously or as a Promise.
+   */
   abstract getAllMessages():
     | ChatMessage<AdditionalMessageOptions>[]
     | Promise<ChatMessage<AdditionalMessageOptions>[]>;
+
+  /**
+   * Adds a new message to the memory.
+   * @param messages The chat message to be added to the memory.
+   */
   abstract put(messages: ChatMessage<AdditionalMessageOptions>): void;
+
+  /**
+   * Clears all messages from the memory.
+   */
   abstract reset(): void;
 
   protected _tokenCountForMessages(messages: ChatMessage[]): number {

packages/core/src/memory/chat-memory-buffer.ts

@@ -1,5 +1,5 @@
 import { Settings } from "../global";
-import type { ChatMessage, LLM, MessageContent } from "../llms";
+import type { ChatMessage, LLM } from "../llms";
 import { type BaseChatStore } from "../storage/chat-store";
 import { BaseChatStoreMemory, DEFAULT_TOKEN_LIMIT_RATIO } from "./base";
 
@@ -34,7 +34,7 @@ export class ChatMemoryBuffer<
   }
 
   getMessages(
-    input?: MessageContent | undefined,
+    transientMessages?: ChatMessage<AdditionalMessageOptions>[] | undefined,
     initialTokenCount: number = 0,
   ) {
     const messages = this.getAllMessages();
@@ -43,16 +43,22 @@ export class ChatMemoryBuffer<
       throw new Error("Initial token count exceeds token limit");
     }
 
-    let messageCount = messages.length;
-    let currentMessages = messages.slice(-messageCount);
-    let tokenCount = this._tokenCountForMessages(messages) + initialTokenCount;
+    // Add input messages as transient messages
+    const messagesWithInput = transientMessages
+      ? [...transientMessages, ...messages]
+      : messages;
+
+    let messageCount = messagesWithInput.length;
+    let currentMessages = messagesWithInput.slice(-messageCount);
+    let tokenCount =
+      this._tokenCountForMessages(messagesWithInput) + initialTokenCount;
 
     while (tokenCount > this.tokenLimit && messageCount > 1) {
       messageCount -= 1;
-      if (messages.at(-messageCount)!.role === "assistant") {
+      if (messagesWithInput.at(-messageCount)!.role === "assistant") {
         messageCount -= 1;
       }
-      currentMessages = messages.slice(-messageCount);
+      currentMessages = messagesWithInput.slice(-messageCount);
       tokenCount =
         this._tokenCountForMessages(currentMessages) + initialTokenCount;
     }
@@ -60,6 +66,6 @@ export class ChatMemoryBuffer<
     if (tokenCount > this.tokenLimit && messageCount <= 0) {
       return [];
     }
-    return messages.slice(-messageCount);
+    return messagesWithInput.slice(-messageCount);
   }
 }

packages/core/src/memory/summary-memory.ts

@@ -114,18 +114,22 @@ export class ChatSummaryMemoryBuffer extends BaseMemory {
     }
   }
 
-  private calcCurrentRequestMessages() {
-    // TODO: check order: currently, we're sending:
+  private calcCurrentRequestMessages(transientMessages?: ChatMessage[]) {
+    // currently, we're sending:
     // system messages first, then transient messages and then the messages that describe the conversation so far
-    return [...this.systemMessages, ...this.calcConversationMessages(true)];
+    return [
+      ...this.systemMessages,
+      ...(transientMessages ? transientMessages : []),
+      ...this.calcConversationMessages(true),
+    ];
   }
 
   reset() {
     this.messages = [];
   }
 
-  async getMessages(): Promise<ChatMessage[]> {
-    const requestMessages = this.calcCurrentRequestMessages();
+  async getMessages(transientMessages?: ChatMessage[]): Promise<ChatMessage[]> {
+    const requestMessages = this.calcCurrentRequestMessages(transientMessages);
 
     // get tokens of current request messages and the transient messages
     const tokens = requestMessages.reduce(
@@ -149,7 +153,7 @@ export class ChatSummaryMemoryBuffer extends BaseMemory {
       // TODO: we still might have too many tokens
       // e.g. too large system messages or transient messages
       // how should we deal with that?
-      return this.calcCurrentRequestMessages();
+      return this.calcCurrentRequestMessages(transientMessages);
     }
     return requestMessages;
   }

packages/core/src/query-engine/base.ts

@@ -1,5 +1,9 @@
+import { randomUUID } from "@llamaindex/env";
+import { Settings } from "../global";
 import type { MessageContent } from "../llms";
+import { PromptMixin } from "../prompts";
 import { EngineResponse, type NodeWithScore } from "../schema";
+import { wrapEventCaller } from "../utils";
 
 /**
  * @link https://docs.llamaindex.ai/en/stable/api_reference/schema/?h=querybundle#llama_index.core.schema.QueryBundle
@@ -14,16 +18,52 @@ export type QueryBundle = {
 
 export type QueryType = string | QueryBundle;
 
-export interface BaseQueryEngine {
-  query(
-    strOrQueryBundle: QueryType,
-    stream: true,
-  ): Promise<AsyncIterable<EngineResponse>>;
-  query(strOrQueryBundle: QueryType, stream?: false): Promise<EngineResponse>;
+export type BaseQueryParams = {
+  query: QueryType;
+};
 
-  synthesize?(
-    strOrQueryBundle: QueryType,
-    nodes: NodeWithScore[],
-    additionalSources?: Iterator<NodeWithScore>,
-  ): Promise<EngineResponse>;
+export interface StreamingQueryParams extends BaseQueryParams {
+  stream: true;
+}
+
+export interface NonStreamingQueryParams extends BaseQueryParams {
+  stream?: false;
+}
+
+export type QueryFn = (
+  strOrQueryBundle: QueryType,
+  stream?: boolean,
+) => Promise<AsyncIterable<EngineResponse> | EngineResponse>;
+
+export abstract class BaseQueryEngine extends PromptMixin {
+  protected constructor(protected readonly _query: QueryFn) {
+    super();
+  }
+
+  async retrieve(params: QueryType): Promise<NodeWithScore[]> {
+    throw new Error(
+      "This query engine does not support retrieve, use query directly",
+    );
+  }
+
+  query(params: StreamingQueryParams): Promise<AsyncIterable<EngineResponse>>;
+  query(params: NonStreamingQueryParams): Promise<EngineResponse>;
+  @wrapEventCaller
+  async query(
+    params: StreamingQueryParams | NonStreamingQueryParams,
+  ): Promise<EngineResponse | AsyncIterable<EngineResponse>> {
+    const { stream, query } = params;
+    const id = randomUUID();
+    const callbackManager = Settings.callbackManager;
+    callbackManager.dispatchEvent("query-start", {
+      id,
+      query,
+    });
+    const response = await this._query(query, stream);
+    callbackManager.dispatchEvent("query-end", {
+      id,
+      response,
+    });
+    return response;
+  }
 }

packages/core/src/query-engine/index.ts

@@ -1 +1,2 @@
-export type { BaseQueryEngine, QueryBundle, QueryType } from "./base";
+export { BaseQueryEngine, type QueryBundle, type QueryType } from "./base";
+export type { QueryEndEvent, QueryStartEvent } from "./type";

packages/core/src/query-engine/type.ts

@@ -0,0 +1,12 @@
+import { EngineResponse } from "../schema";
+import type { QueryType } from "./base";
+
+export type QueryStartEvent = {
+  id: string;
+  query: QueryType;
+};
+
+export type QueryEndEvent = {
+  id: string;
+  response: EngineResponse | AsyncIterable<EngineResponse>;
+};

packages/core/src/response-synthesizers/base-synthesizer.ts

@@ -0,0 +1,58 @@
+import { randomUUID } from "@llamaindex/env";
+import { Settings } from "../global";
+import { PromptHelper } from "../indices";
+import type { LLM, MessageContent } from "../llms";
+import { PromptMixin } from "../prompts";
+import { EngineResponse, type NodeWithScore } from "../schema";
+import type { SynthesizeQuery } from "./type";
+
+export type BaseSynthesizerOptions = {
+  llm?: LLM;
+  promptHelper?: PromptHelper;
+};
+
+export abstract class BaseSynthesizer extends PromptMixin {
+  llm: LLM;
+  promptHelper: PromptHelper;
+
+  protected constructor(options: Partial<BaseSynthesizerOptions>) {
+    super();
+    this.llm = options.llm ?? Settings.llm;
+    this.promptHelper =
+      options.promptHelper ?? PromptHelper.fromLLMMetadata(this.llm.metadata);
+  }
+
+  protected abstract getResponse(
+    query: MessageContent,
+    textChunks: NodeWithScore[],
+    stream: boolean,
+  ): Promise<EngineResponse | AsyncIterable<EngineResponse>>;
+
+  synthesize(
+    query: SynthesizeQuery,
+    stream: true,
+  ): Promise<AsyncIterable<EngineResponse>>;
+  synthesize(query: SynthesizeQuery, stream?: false): Promise<EngineResponse>;
+  async synthesize(
+    query: SynthesizeQuery,
+    stream = false,
+  ): Promise<EngineResponse | AsyncIterable<EngineResponse>> {
+    const callbackManager = Settings.callbackManager;
+    const id = randomUUID();
+    callbackManager.dispatchEvent("synthesize-start", { id, query });
+    let response: EngineResponse | AsyncIterable<EngineResponse>;
+    if (query.nodes.length === 0) {
+      if (stream) {
+        response = EngineResponse.fromResponse("Empty Response", true);
+      } else {
+        response = EngineResponse.fromResponse("Empty Response", false);
+      }
+    } else {
+      const queryMessage: MessageContent =
+        typeof query.query === "string" ? query.query : query.query.query;
+      response = await this.getResponse(queryMessage, query.nodes, stream);
+    }
+    callbackManager.dispatchEvent("synthesize-end", { id, query, response });
+    return response;
+  }
+}

packages/core/src/response-synthesizers/factory.ts

@@ -1,108 +1,52 @@
-import { getBiggestPrompt, type PromptHelper } from "@llamaindex/core/indices";
-import type { LLM } from "@llamaindex/core/llms";
+import { z } from "zod";
+import { getBiggestPrompt } from "../indices";
+import type { MessageContent } from "../llms";
 import {
-  PromptMixin,
   defaultRefinePrompt,
   defaultTextQAPrompt,
   defaultTreeSummarizePrompt,
   type ModuleRecord,
-  type PromptsRecord,
   type RefinePrompt,
   type TextQAPrompt,
   type TreeSummarizePrompt,
-} from "@llamaindex/core/prompts";
-import type { QueryType } from "@llamaindex/core/query-engine";
-import { extractText, streamConverter } from "@llamaindex/core/utils";
-import type { ServiceContext } from "../ServiceContext.js";
+} from "../prompts";
 import {
-  llmFromSettingsOrContext,
-  promptHelperFromSettingsOrContext,
-} from "../Settings.js";
-import type { ResponseBuilder, ResponseBuilderQuery } from "./types.js";
+  EngineResponse,
+  MetadataMode,
+  type NodeWithScore,
+  TextNode,
+} from "../schema";
+import { createMessageContent, extractText, streamConverter } from "../utils";
+import {
+  BaseSynthesizer,
+  type BaseSynthesizerOptions,
+} from "./base-synthesizer";
 
-/**
- * Response modes of the response synthesizer
- */
-enum ResponseMode {
-  REFINE = "refine",
-  COMPACT = "compact",
-  TREE_SUMMARIZE = "tree_summarize",
-  SIMPLE = "simple",
-}
+const responseModeSchema = z.enum([
+  "refine",
+  "compact",
+  "tree_summarize",
+  "multi_modal",
+]);
 
-/**
- * A response builder that just concatenates responses.
- */
-export class SimpleResponseBuilder
-  extends PromptMixin
-  implements ResponseBuilder
-{
-  llm: LLM;
-  textQATemplate: TextQAPrompt;
-
-  constructor(serviceContext?: ServiceContext, textQATemplate?: TextQAPrompt) {
-    super();
-    this.llm = llmFromSettingsOrContext(serviceContext);
-    this.textQATemplate = textQATemplate ?? defaultTextQAPrompt;
-  }
-
-  protected _getPrompts(): PromptsRecord {
-    return {
-      textQATemplate: this.textQATemplate,
-    };
-  }
-  protected _updatePrompts(prompts: { textQATemplate: TextQAPrompt }): void {
-    if (prompts.textQATemplate) {
-      this.textQATemplate = prompts.textQATemplate;
-    }
-  }
-  protected _getPromptModules(): ModuleRecord {
-    return {};
-  }
-
-  getResponse(
-    query: ResponseBuilderQuery,
-    stream: true,
-  ): Promise<AsyncIterable<string>>;
-  getResponse(query: ResponseBuilderQuery, stream?: false): Promise<string>;
-  async getResponse(
-    { query, textChunks }: ResponseBuilderQuery,
-    stream?: boolean,
-  ): Promise<AsyncIterable<string> | string> {
-    const prompt = this.textQATemplate.format({
-      query: extractText(query),
-      context: textChunks.join("\n\n"),
-    });
-    if (stream) {
-      const response = await this.llm.complete({ prompt, stream: true });
-      return streamConverter(response, (chunk) => chunk.text);
-    } else {
-      const response = await this.llm.complete({ prompt, stream: false });
-      return response.text;
-    }
-  }
-}
+export type ResponseMode = z.infer<typeof responseModeSchema>;
 
 /**
  * A response builder that uses the query to ask the LLM generate a better response using multiple text chunks.
  */
-export class Refine extends PromptMixin implements ResponseBuilder {
-  llm: LLM;
-  promptHelper: PromptHelper;
+class Refine extends BaseSynthesizer {
   textQATemplate: TextQAPrompt;
   refineTemplate: RefinePrompt;
 
   constructor(
-    serviceContext?: ServiceContext,
-    textQATemplate?: TextQAPrompt,
-    refineTemplate?: RefinePrompt,
+    options: BaseSynthesizerOptions & {
+      textQATemplate?: TextQAPrompt | undefined;
+      refineTemplate?: RefinePrompt | undefined;
+    },
   ) {
-    super();
-
-    this.llm = llmFromSettingsOrContext(serviceContext);
-    this.promptHelper = promptHelperFromSettingsOrContext(serviceContext);
-    this.textQATemplate = textQATemplate ?? defaultTextQAPrompt;
-    this.refineTemplate = refineTemplate ?? defaultRefinePrompt;
+    super(options);
+    this.textQATemplate = options.textQATemplate ?? defaultTextQAPrompt;
+    this.refineTemplate = options.refineTemplate ?? defaultRefinePrompt;
   }
 
   protected _getPromptModules(): ModuleRecord {
@@ -132,41 +76,47 @@ export class Refine extends PromptMixin implements ResponseBuilder {
     }
   }
 
-  getResponse(
-    query: ResponseBuilderQuery,
-    stream: true,
-  ): Promise<AsyncIterable<string>>;
-  getResponse(query: ResponseBuilderQuery, stream?: false): Promise<string>;
   async getResponse(
-    { query, textChunks, prevResponse }: ResponseBuilderQuery,
-    stream?: boolean,
-  ): Promise<AsyncIterable<string> | string> {
-    let response: AsyncIterable<string> | string | undefined = prevResponse;
+    query: MessageContent,
+    nodes: NodeWithScore[],
+    stream: boolean,
+  ): Promise<EngineResponse | AsyncIterable<EngineResponse>> {
+    let response: AsyncIterable<string> | string | undefined = undefined;
+    const textChunks = nodes.map(({ node }) =>
+      node.getContent(MetadataMode.LLM),
+    );
 
     for (let i = 0; i < textChunks.length; i++) {
-      const chunk = textChunks[i]!;
+      const text = textChunks[i]!;
       const lastChunk = i === textChunks.length - 1;
       if (!response) {
         response = await this.giveResponseSingle(
           query,
-          chunk,
+          text,
           !!stream && lastChunk,
         );
       } else {
         response = await this.refineResponseSingle(
           response as string,
           query,
-          chunk,
+          text,
           !!stream && lastChunk,
         );
       }
     }
 
-    return response ?? "Empty Response";
+    // fixme: no source nodes provided, cannot fix right now due to lack of context
+    if (typeof response === "string") {
+      return EngineResponse.fromResponse(response, false);
+    } else {
+      return streamConverter(response!, (text) =>
+        EngineResponse.fromResponse(text, true),
+      );
+    }
   }
 
   private async giveResponseSingle(
-    query: QueryType,
+    query: MessageContent,
     textChunk: string,
     stream: boolean,
   ): Promise<AsyncIterable<string> | string> {
@@ -203,10 +153,10 @@ export class Refine extends PromptMixin implements ResponseBuilder {
   // eslint-disable-next-line max-params
   private async refineResponseSingle(
     initialReponse: string,
-    query: QueryType,
+    query: MessageContent,
     textChunk: string,
     stream: boolean,
-  ) {
+  ): Promise<AsyncIterable<string> | string> {
     const refineTemplate: RefinePrompt = this.refineTemplate.partialFormat({
       query: extractText(query),
     });
@@ -246,59 +196,54 @@ export class Refine extends PromptMixin implements ResponseBuilder {
 /**
  * CompactAndRefine is a slight variation of Refine that first compacts the text chunks into the smallest possible number of chunks.
  */
-export class CompactAndRefine extends Refine {
-  getResponse(
-    query: ResponseBuilderQuery,
-    stream: true,
-  ): Promise<AsyncIterable<string>>;
-  getResponse(query: ResponseBuilderQuery, stream?: false): Promise<string>;
+class CompactAndRefine extends Refine {
   async getResponse(
-    { query, textChunks, prevResponse }: ResponseBuilderQuery,
-    stream?: boolean,
-  ): Promise<AsyncIterable<string> | string> {
+    query: MessageContent,
+    nodes: NodeWithScore[],
+    stream: boolean,
+  ): Promise<EngineResponse | AsyncIterable<EngineResponse>> {
     const textQATemplate: TextQAPrompt = this.textQATemplate.partialFormat({
       query: extractText(query),
     });
     const refineTemplate: RefinePrompt = this.refineTemplate.partialFormat({
       query: extractText(query),
     });
+    const textChunks = nodes.map(({ node }) =>
+      node.getContent(MetadataMode.LLM),
+    );
 
     const maxPrompt = getBiggestPrompt([textQATemplate, refineTemplate]);
     const newTexts = this.promptHelper.repack(maxPrompt, textChunks);
-    const params = {
-      query,
-      textChunks: newTexts,
-      prevResponse,
-    };
+    const newNodes = newTexts.map((text) => new TextNode({ text }));
     if (stream) {
       return super.getResponse(
-        {
-          ...params,
-        },
+        query,
+        newNodes.map((node) => ({ node })),
         true,
       );
     }
-    return super.getResponse(params);
+    return super.getResponse(
+      query,
+      newNodes.map((node) => ({ node })),
+      false,
+    );
   }
 }
 
 /**
  * TreeSummarize repacks the text chunks into the smallest possible number of chunks and then summarizes them, then recursively does so until there's one chunk left.
  */
-export class TreeSummarize extends PromptMixin implements ResponseBuilder {
-  llm: LLM;
-  promptHelper: PromptHelper;
+class TreeSummarize extends BaseSynthesizer {
   summaryTemplate: TreeSummarizePrompt;
 
   constructor(
-    serviceContext?: ServiceContext,
-    summaryTemplate?: TreeSummarizePrompt,
+    options: BaseSynthesizerOptions & {
+      summaryTemplate?: TreeSummarizePrompt;
+    },
   ) {
-    super();
-
-    this.llm = llmFromSettingsOrContext(serviceContext);
-    this.promptHelper = promptHelperFromSettingsOrContext(serviceContext);
-    this.summaryTemplate = summaryTemplate ?? defaultTreeSummarizePrompt;
+    super(options);
+    this.summaryTemplate =
+      options.summaryTemplate ?? defaultTreeSummarizePrompt;
   }
 
   protected _getPromptModules(): ModuleRecord {
@@ -319,15 +264,14 @@ export class TreeSummarize extends PromptMixin implements ResponseBuilder {
     }
   }
 
-  getResponse(
-    query: ResponseBuilderQuery,
-    stream: true,
-  ): Promise<AsyncIterable<string>>;
-  getResponse(query: ResponseBuilderQuery, stream?: false): Promise<string>;
   async getResponse(
-    { query, textChunks }: ResponseBuilderQuery,
-    stream?: boolean,
-  ): Promise<AsyncIterable<string> | string> {
+    query: MessageContent,
+    nodes: NodeWithScore[],
+    stream: boolean,
+  ): Promise<EngineResponse | AsyncIterable<EngineResponse>> {
+    const textChunks = nodes.map(({ node }) =>
+      node.getContent(MetadataMode.LLM),
+    );
     if (!textChunks || textChunks.length === 0) {
       throw new Error("Must have at least one text chunk");
     }
@@ -347,9 +291,14 @@ export class TreeSummarize extends PromptMixin implements ResponseBuilder {
       };
       if (stream) {
         const response = await this.llm.complete({ ...params, stream });
-        return streamConverter(response, (chunk) => chunk.text);
+        return streamConverter(response, (chunk) =>
+          EngineResponse.fromResponse(chunk.text, true),
+        );
       }
-      return (await this.llm.complete(params)).text;
+      return EngineResponse.fromResponse(
+        (await this.llm.complete(params)).text,
+        false,
+      );
     } else {
       const summaries = await Promise.all(
         packedTextChunks.map((chunk) =>
@@ -362,40 +311,118 @@ export class TreeSummarize extends PromptMixin implements ResponseBuilder {
         ),
       );
 
-      const params = {
-        query,
-        textChunks: summaries.map((s) => s.text),
-      };
       if (stream) {
         return this.getResponse(
-          {
-            ...params,
-          },
+          query,
+          summaries.map((s) => ({
+            node: new TextNode({
+              text: s.text,
+            }),
+          })),
           true,
         );
       }
-      return this.getResponse(params);
+      return this.getResponse(
+        query,
+        summaries.map((s) => ({
+          node: new TextNode({
+            text: s.text,
+          }),
+        })),
+        false,
+      );
     }
   }
 }
 
-export function getResponseBuilder(
-  serviceContext?: ServiceContext,
-  responseMode?: ResponseMode,
-): ResponseBuilder {
-  switch (responseMode) {
-    case ResponseMode.SIMPLE:
-      return new SimpleResponseBuilder(serviceContext);
-    case ResponseMode.REFINE:
-      return new Refine(serviceContext);
-    case ResponseMode.TREE_SUMMARIZE:
-      return new TreeSummarize(serviceContext);
-    default:
-      return new CompactAndRefine(serviceContext);
+class MultiModal extends BaseSynthesizer {
+  metadataMode: MetadataMode;
+  textQATemplate: TextQAPrompt;
+
+  constructor({
+    textQATemplate,
+    metadataMode,
+    ...options
+  }: BaseSynthesizerOptions & {
+    textQATemplate?: TextQAPrompt;
+    metadataMode?: MetadataMode;
+  } = {}) {
+    super(options);
+
+    this.metadataMode = metadataMode ?? MetadataMode.NONE;
+    this.textQATemplate = textQATemplate ?? defaultTextQAPrompt;
+  }
+
+  protected _getPromptModules(): ModuleRecord {
+    return {};
+  }
+
+  protected _getPrompts(): { textQATemplate: TextQAPrompt } {
+    return {
+      textQATemplate: this.textQATemplate,
+    };
+  }
+
+  protected _updatePrompts(promptsDict: {
+    textQATemplate: TextQAPrompt;
+  }): void {
+    if (promptsDict.textQATemplate) {
+      this.textQATemplate = promptsDict.textQATemplate;
+    }
+  }
+
+  protected async getResponse(
+    query: MessageContent,
+    nodes: NodeWithScore[],
+    stream: boolean,
+  ): Promise<EngineResponse | AsyncIterable<EngineResponse>> {
+    const prompt = await createMessageContent(
+      this.textQATemplate,
+      nodes.map(({ node }) => node),
+      // this might not be good as this remove the image information
+      { query: extractText(query) },
+      this.metadataMode,
+    );
+
+    const llm = this.llm;
+
+    if (stream) {
+      const response = await llm.complete({
+        prompt,
+        stream,
+      });
+      return streamConverter(response, ({ text }) =>
+        EngineResponse.fromResponse(text, true),
+      );
+    }
+    const response = await llm.complete({
+      prompt,
+    });
+    return EngineResponse.fromResponse(response.text, false);
   }
 }
 
-export type ResponseBuilderPrompts =
-  | TextQAPrompt
-  | TreeSummarizePrompt
-  | RefinePrompt;
+export function getResponseSynthesizer(
+  mode: ResponseMode,
+  options: BaseSynthesizerOptions & {
+    textQATemplate?: TextQAPrompt;
+    refineTemplate?: RefinePrompt;
+    summaryTemplate?: TreeSummarizePrompt;
+    metadataMode?: MetadataMode;
+  } = {},
+) {
+  switch (mode) {
+    case "compact": {
+      return new CompactAndRefine(options);
+    }
+    case "refine": {
+      return new Refine(options);
+    }
+    case "tree_summarize": {
+      return new TreeSummarize(options);
+    }
+    case "multi_modal": {
+      return new MultiModal(options);
+    }
+  }
+}

packages/core/src/response-synthesizers/index.ts

@@ -0,0 +1,10 @@
+export {
+  BaseSynthesizer,
+  type BaseSynthesizerOptions,
+} from "./base-synthesizer";
+export { getResponseSynthesizer, type ResponseMode } from "./factory";
+export type {
+  SynthesizeEndEvent,
+  SynthesizeQuery,
+  SynthesizeStartEvent,
+} from "./type";

packages/core/src/response-synthesizers/type.ts

@@ -0,0 +1,19 @@
+import type { QueryType } from "../query-engine";
+import { EngineResponse, type NodeWithScore } from "../schema";
+
+export type SynthesizeQuery = {
+  query: QueryType;
+  nodes: NodeWithScore[];
+  additionalSourceNodes?: NodeWithScore[];
+};
+
+export type SynthesizeStartEvent = {
+  id: string;
+  query: SynthesizeQuery;
+};
+
+export type SynthesizeEndEvent = {
+  id: string;
+  query: SynthesizeQuery;
+  response: EngineResponse | AsyncIterable<EngineResponse>;
+};

packages/core/src/retriever/index.ts

@@ -0,0 +1,112 @@
+import { randomUUID } from "@llamaindex/env";
+import { Settings } from "../global";
+import type { MessageContent } from "../llms";
+import { PromptMixin } from "../prompts";
+import type { QueryBundle, QueryType } from "../query-engine";
+import { BaseNode, IndexNode, type NodeWithScore, ObjectType } from "../schema";
+
+export type RetrieveParams = {
+  query: MessageContent;
+  preFilters?: unknown;
+};
+
+export type RetrieveStartEvent = {
+  id: string;
+  query: QueryBundle;
+};
+
+export type RetrieveEndEvent = {
+  id: string;
+  query: QueryBundle;
+  nodes: NodeWithScore[];
+};
+
+export abstract class BaseRetriever extends PromptMixin {
+  objectMap: Map<string, unknown> = new Map();
+
+  protected _updatePrompts() {}
+  protected _getPrompts() {
+    return {};
+  }
+
+  protected _getPromptModules() {
+    return {};
+  }
+
+  protected constructor() {
+    super();
+  }
+
+  public async retrieve(params: QueryType): Promise<NodeWithScore[]> {
+    const cb = Settings.callbackManager;
+    const queryBundle = typeof params === "string" ? { query: params } : params;
+    const id = randomUUID();
+    cb.dispatchEvent("retrieve-start", { id, query: queryBundle });
+    let response = await this._retrieve(queryBundle);
+    response = await this._handleRecursiveRetrieval(queryBundle, response);
+    cb.dispatchEvent("retrieve-end", {
+      id,
+      query: queryBundle,
+      nodes: response,
+    });
+    return response;
+  }
+
+  abstract _retrieve(params: QueryBundle): Promise<NodeWithScore[]>;
+
+  async _handleRecursiveRetrieval(
+    params: QueryBundle,
+    nodes: NodeWithScore[],
+  ): Promise<NodeWithScore[]> {
+    const retrievedNodes = [];
+    for (const { node, score = 1.0 } of nodes) {
+      if (node.type === ObjectType.INDEX) {
+        const indexNode = node as IndexNode;
+        const object = this.objectMap.get(indexNode.indexId);
+        if (object !== undefined) {
+          retrievedNodes.push(
+            ...this._retrieveFromObject(object, params, score),
+          );
+        } else {
+          retrievedNodes.push({ node, score });
+        }
+      } else {
+        retrievedNodes.push({ node, score });
+      }
+    }
+    return nodes;
+  }
+
+  _retrieveFromObject(
+    object: unknown,
+    queryBundle: QueryBundle,
+    score: number,
+  ): NodeWithScore[] {
+    if (object == null) {
+      throw new TypeError("Object is not retrievable");
+    }
+    if (typeof object !== "object") {
+      throw new TypeError("Object is not retrievable");
+    }
+    if ("node" in object && object.node instanceof BaseNode) {
+      return [
+        {
+          node: object.node,
+          score:
+            "score" in object && typeof object.score === "number"
+              ? object.score
+              : score,
+        },
+      ];
+    }
+    if (object instanceof BaseNode) {
+      return [{ node: object, score }];
+    } else {
+      // todo: support other types
+      // BaseQueryEngine
+      // BaseRetriever
+      // QueryComponent
+      throw new TypeError("Object is not retrievable");
+    }
+  }
+}

packages/core/src/utils/index.ts

@@ -1,4 +1,4 @@
-export { wrapEventCaller } from "./event-caller";
+export { EventCaller, getEventCaller, wrapEventCaller } from "./event-caller";
 
 export async function* streamConverter<S, D>(
   stream: AsyncIterable<S>,
@@ -47,10 +47,12 @@ export async function* streamReducer<S, D>(params: {
 export { wrapLLMEvent } from "./wrap-llm-event";
 
 export {
+  createMessageContent,
   extractDataUrlComponents,
   extractImage,
   extractSingleText,
   extractText,
+  imageToDataUrl,
   messagesToHistory,
   toToolDescriptions,
 } from "./llms";

packages/core/src/utils/llms.ts

@@ -1,3 +1,5 @@
+import { fs } from "@llamaindex/env";
+import { filetypemime } from "magic-bytes.js";
 import type {
   ChatMessage,
   MessageContent,
@@ -5,8 +7,16 @@ import type {
   MessageContentTextDetail,
   ToolMetadata,
 } from "../llms";
+import type { BasePromptTemplate } from "../prompts";
 import type { QueryType } from "../query-engine";
 import type { ImageType } from "../schema";
+import {
+  type BaseNode,
+  ImageNode,
+  MetadataMode,
+  ModalityType,
+  splitNodesByType,
+} from "../schema";
 
 /**
  * Extracts just the text whether from
@@ -107,3 +117,99 @@ export function toToolDescriptions(tools: ToolMetadata[]): string {
 
   return JSON.stringify(toolsObj, null, 4);
 }
+
+async function blobToDataUrl(input: Blob) {
+  const buffer = Buffer.from(await input.arrayBuffer());
+  const mimes = filetypemime(buffer);
+  if (mimes.length < 1) {
+    throw new Error("Unsupported image type");
+  }
+  return "data:" + mimes[0] + ";base64," + buffer.toString("base64");
+}
+
+export async function imageToDataUrl(
+  input: ImageType | Uint8Array,
+): Promise<string> {
+  // first ensure, that the input is a Blob
+  if (
+    (input instanceof URL && input.protocol === "file:") ||
+    typeof input === "string"
+  ) {
+    // string or file URL
+    const dataBuffer = await fs.readFile(
+      input instanceof URL ? input.pathname : input,
+    );
+    input = new Blob([dataBuffer]);
+  } else if (!(input instanceof Blob)) {
+    if (input instanceof URL) {
+      throw new Error(`Unsupported URL with protocol: ${input.protocol}`);
+    } else if (input instanceof Uint8Array) {
+      input = new Blob([input]); // convert Uint8Array to Blob
+    } else {
+      throw new Error(`Unsupported input type: ${typeof input}`);
+    }
+  }
+  return await blobToDataUrl(input);
+}
+
+// eslint-disable-next-line max-params
+async function createContentPerModality(
+  prompt: BasePromptTemplate,
+  type: ModalityType,
+  nodes: BaseNode[],
+  extraParams: Record<string, string>,
+  metadataMode: MetadataMode,
+): Promise<MessageContentDetail[]> {
+  switch (type) {
+    case ModalityType.TEXT:
+      return [
+        {
+          type: "text",
+          text: prompt.format({
+            ...extraParams,
+            context: nodes.map((r) => r.getContent(metadataMode)).join("\n\n"),
+          }),
+        },
+      ];
+    case ModalityType.IMAGE:
+      const images: MessageContentDetail[] = await Promise.all(
+        (nodes as ImageNode[]).map(async (node) => {
+          return {
+            type: "image_url",
+            image_url: {
+              url: await imageToDataUrl(node.image),
+            },
+          } satisfies MessageContentDetail;
+        }),
+      );
+      return images;
+    default:
+      return [];
+  }
+}
+
+export async function createMessageContent(
+  prompt: BasePromptTemplate,
+  nodes: BaseNode[],
+  extraParams: Record<string, string> = {},
+  metadataMode: MetadataMode = MetadataMode.NONE,
+): Promise<MessageContentDetail[]> {
+  const content: MessageContentDetail[] = [];
+  const nodeMap = splitNodesByType(nodes);
+  for (const type in nodeMap) {
+    // for each retrieved modality type, create message content
+    const nodes = nodeMap[type as ModalityType];
+    if (nodes) {
+      content.push(
+        ...(await createContentPerModality(
+          prompt,
+          type as ModalityType,
+          nodes,
+          extraParams,
+          metadataMode,
+        )),
+      );
+    }
+  }
+  return content;
+}

packages/core/tests/memory/chat-memory-buffer.test.ts

@@ -0,0 +1,74 @@
+import { Settings } from "@llamaindex/core/global";
+import type { ChatMessage } from "@llamaindex/core/llms";
+import { ChatMemoryBuffer } from "@llamaindex/core/memory";
+import { beforeEach, describe, expect, test } from "vitest";
+
+describe("ChatMemoryBuffer", () => {
+  beforeEach(() => {
+    // Mock the Settings.llm
+    (Settings.llm as any) = {
+      metadata: {
+        contextWindow: 1000,
+      },
+    };
+  });
+
+  test("constructor initializes with custom token limit", () => {
+    const buffer = new ChatMemoryBuffer({ tokenLimit: 500 });
+    expect(buffer.tokenLimit).toBe(500);
+  });
+
+  test("getMessages returns all messages when under token limit", () => {
+    const messages: ChatMessage[] = [
+      { role: "user", content: "Hello" },
+      { role: "assistant", content: "Hi there!" },
+      { role: "user", content: "How are you?" },
+    ];
+    const buffer = new ChatMemoryBuffer({
+      tokenLimit: 1000,
+      chatHistory: messages,
+    });
+
+    const result = buffer.getMessages();
+    expect(result).toEqual(messages);
+  });
+
+  test("getMessages truncates messages when over token limit", () => {
+    const messages: ChatMessage[] = [
+      { role: "user", content: "This is a long message" },
+      { role: "assistant", content: "This is also a long reply" },
+      { role: "user", content: "Short" },
+    ];
+    const buffer = new ChatMemoryBuffer({
+      tokenLimit: 5, // limit to only allow the last message
+      chatHistory: messages,
+    });
+
+    const result = buffer.getMessages();
+    expect(result).toEqual([{ role: "user", content: "Short" }]);
+  });
+
+  test("getMessages handles input messages", () => {
+    const storedMessages: ChatMessage[] = [
+      { role: "user", content: "Hello" },
+      { role: "assistant", content: "Hi there!" },
+    ];
+    const buffer = new ChatMemoryBuffer({
+      tokenLimit: 50,
+      chatHistory: storedMessages,
+    });
+
+    const inputMessages: ChatMessage[] = [
+      { role: "user", content: "New message" },
+    ];
+    const result = buffer.getMessages(inputMessages);
+    expect(result).toEqual([...inputMessages, ...storedMessages]);
+  });
+
+  test("getMessages throws error when initial token count exceeds limit", () => {
+    const buffer = new ChatMemoryBuffer({ tokenLimit: 10 });
+    expect(() => buffer.getMessages(undefined, 20)).toThrow(
+      "Initial token count exceeds token limit",
+    );
+  });
+});

packages/env/CHANGELOG.md

@@ -1,5 +1,13 @@
 # @llamaindex/env
 
+## 0.1.12
+
+### Patch Changes
+
+- b48bcc3: feat: add `load-transformers` event type when loading `@xenova/transformers` module
+
+  This would benefit user who want to customize the transformer env.
+
 ## 0.1.11
 
 ### Patch Changes

packages/env/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/env",
   "description": "environment wrapper, supports all JS environment including node, deno, bun, edge runtime, and cloudflare worker",
-  "version": "0.1.11",
+  "version": "0.1.12",
   "type": "module",
   "types": "dist/type/index.d.ts",
   "main": "dist/cjs/index.js",
@@ -74,16 +74,18 @@
     "@aws-crypto/sha256-js": "^5.2.0",
     "@swc/cli": "^0.4.0",
     "@swc/core": "^1.7.22",
+    "@xenova/transformers": "^2.17.2",
     "concurrently": "^8.2.2",
     "pathe": "^1.1.2",
+    "tiktoken": "^1.0.16",
     "vitest": "^2.0.5"
   },
   "dependencies": {
-    "@types/lodash": "^4.17.7",
     "@types/node": "^22.5.1"
   },
   "peerDependencies": {
     "@aws-crypto/sha256-js": "^5.2.0",
+    "@xenova/transformers": "^2.17.2",
     "js-tiktoken": "^1.0.12",
     "pathe": "^1.1.2",
     "tiktoken": "^1.0.15"
@@ -92,8 +94,17 @@
     "@aws-crypto/sha256-js": {
       "optional": true
     },
+    "@xenova/transformers": {
+      "optional": true
+    },
     "pathe": {
       "optional": true
+    },
+    "tiktoken": {
+      "optional": true
+    },
+    "js-tiktoken": {
+      "optional": true
     }
   }
 }

packages/env/src/index.browser.ts

@@ -6,6 +6,12 @@
 import "./global-check.js";
 export * from "./web-polyfill.js";
 
+export {
+  loadTransformers,
+  setTransformers,
+  type LoadTransformerEvent,
+  type OnLoad,
+} from "./multi-model/index.browser.js";
 export { Tokenizers, tokenizers, type Tokenizer } from "./tokenizers/js.js";
 
 // @ts-expect-error

packages/env/src/index.edge-light.ts

@@ -6,4 +6,10 @@
 import "./global-check.js";
 export * from "./node-polyfill.js";
 
+export {
+  loadTransformers,
+  setTransformers,
+  type LoadTransformerEvent,
+  type OnLoad,
+} from "./multi-model/index.non-nodejs.js";
 export { Tokenizers, tokenizers, type Tokenizer } from "./tokenizers/js.js";

packages/env/src/index.ts

@@ -33,6 +33,12 @@ export function createSHA256(): SHA256 {
   };
 }
 
+export {
+  loadTransformers,
+  setTransformers,
+  type LoadTransformerEvent,
+  type OnLoad,
+} from "./multi-model/index.js";
 export { Tokenizers, tokenizers, type Tokenizer } from "./tokenizers/node.js";
 export {
   AsyncLocalStorage,

packages/env/src/index.workerd.ts

@@ -13,4 +13,10 @@ export function getEnv(name: string): string | undefined {
   return INTERNAL_ENV[name];
 }
 
+export {
+  loadTransformers,
+  setTransformers,
+  type LoadTransformerEvent,
+  type OnLoad,
+} from "./multi-model/index.non-nodejs.js";
 export { Tokenizers, tokenizers, type Tokenizer } from "./tokenizers/js.js";

packages/env/src/multi-model/index.browser.ts

@@ -0,0 +1,20 @@
+import { getTransformers, setTransformers, type OnLoad } from "./shared.js";
+
+export {
+  setTransformers,
+  type LoadTransformerEvent,
+  type OnLoad,
+} from "./shared.js";
+export async function loadTransformers(onLoad: OnLoad) {
+  if (getTransformers() === null) {
+    setTransformers(
+      // @ts-expect-error
+      await import("https://cdn.jsdelivr.net/npm/@xenova/transformers@2.17.2"),
+    );
+  } else {
+    return getTransformers()!;
+  }
+  const transformer = getTransformers()!;
+  onLoad(transformer);
+  return transformer;
+}

packages/env/src/multi-model/index.non-nodejs.ts

@@ -0,0 +1,35 @@
+import { getTransformers, setTransformers, type OnLoad } from "./shared.js";
+export {
+  setTransformers,
+  type LoadTransformerEvent,
+  type OnLoad,
+} from "./shared.js";
+
+export async function loadTransformers(onLoad: OnLoad) {
+  if (getTransformers() === null) {
+    /**
+     * If you see this warning, it means that the current environment does not support the transformer.
+     *  because "@xeonva/transformers" highly depends on Node.js APIs.
+     *
+     * One possible solution is to fix their implementation to make it work in the non-Node.js environment,
+     *  but it's not worth the effort because Edge Runtime and Cloudflare Workers are not the for heavy Machine Learning task.
+     *
+     * Or you can provide an RPC server that runs the transformer in a Node.js environment.
+     * Or you just run the code in a Node.js environment.
+     *
+     * Refs: https://github.com/xenova/transformers.js/issues/309
+     */
+    console.warn(
+      '"@xenova/transformers" is not officially supported in this environment, some features may not work as expected.',
+    );
+    setTransformers(
+      // @ts-expect-error
+      await import("@xenova/transformers/dist/transformers"),
+    );
+  } else {
+    return getTransformers()!;
+  }
+  const transformer = getTransformers()!;
+  onLoad(transformer);
+  return transformer;
+}

packages/env/src/multi-model/index.ts

@@ -0,0 +1,20 @@
+import { getTransformers, setTransformers, type OnLoad } from "./shared.js";
+
+export {
+  setTransformers,
+  type LoadTransformerEvent,
+  type OnLoad,
+} from "./shared.js";
+
+export async function loadTransformers(onLoad: OnLoad) {
+  if (getTransformers() === null) {
+    setTransformers(await import("@xenova/transformers"));
+  } else {
+    return getTransformers()!;
+  }
+  const transformer = getTransformers()!;
+
+  onLoad(transformer);
+
+  return transformer;
+}

packages/env/src/multi-model/shared.ts

@@ -0,0 +1,17 @@
+let transformer: typeof import("@xenova/transformers") | null = null;
+
+export function getTransformers() {
+  return transformer;
+}
+
+export function setTransformers(t: typeof import("@xenova/transformers")) {
+  transformer = t;
+}
+
+export type OnLoad = (
+  transformer: typeof import("@xenova/transformers"),
+) => void;
+
+export type LoadTransformerEvent = {
+  transformer: typeof import("@xenova/transformers");
+};

packages/experimental/CHANGELOG.md

@@ -1,5 +1,64 @@
 # @llamaindex/experimental
 
+## 0.0.87
+
+### Patch Changes
+
+- llamaindex@0.6.9
+
+## 0.0.86
+
+### Patch Changes
+
+- Updated dependencies [8b7fdba]
+  - llamaindex@0.6.8
+
+## 0.0.85
+
+### Patch Changes
+
+- Updated dependencies [23bcc37]
+  - llamaindex@0.6.7
+
+## 0.0.84
+
+### Patch Changes
+
+- Updated dependencies [d902cc3]
+- Updated dependencies [025ffe6]
+- Updated dependencies [a659574]
+  - llamaindex@0.6.6
+
+## 0.0.83
+
+### Patch Changes
+
+- Updated dependencies [e9714db]
+  - llamaindex@0.6.5
+
+## 0.0.82
+
+### Patch Changes
+
+- Updated dependencies [b48bcc3]
+  - llamaindex@0.6.4
+
+## 0.0.81
+
+### Patch Changes
+
+- 2cd1383: refactor: align `response-synthesizers` & `chat-engine` module
+
+  - builtin event system
+  - correct class extends
+  - aligin APIs, naming with llama-index python
+  - move stream out of first parameter to second parameter for the better tyep checking
+  - remove JSONQueryEngine in `@llamaindex/experimental`, as the code quality is not satisify and we will bring it back later
+
+- Updated dependencies [2cd1383]
+- Updated dependencies [5c4badb]
+  - llamaindex@0.6.3
+
 ## 0.0.80
 
 ### Patch Changes

packages/experimental/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/experimental",
   "description": "Experimental package for LlamaIndexTS",
-  "version": "0.0.80",
+  "version": "0.0.87",
   "type": "module",
   "types": "dist/type/index.d.ts",
   "main": "dist/cjs/index.js",

packages/experimental/src/engines/query/JSONQueryEngine.ts

@@ -1,211 +0,0 @@
-import jsonpath from "jsonpath";
-
-import { EngineResponse } from "llamaindex";
-
-import { serviceContextFromDefaults, type ServiceContext } from "llamaindex";
-
-import type {
-  QueryEngine,
-  QueryEngineParamsNonStreaming,
-  QueryEngineParamsStreaming,
-} from "llamaindex";
-
-import {
-  defaultJsonPathPrompt,
-  defaultResponseSynthesizePrompt,
-  type JSONPathPrompt,
-  type ResponseSynthesisPrompt,
-} from "./prompt.js";
-
-export type JSONSchemaType = Record<string, unknown>;
-
-function removeExtraQuotes(expr: string) {
-  let startIndex = 0;
-  let endIndex = expr.length;
-
-  // Trim the leading backticks and single quotes
-  while (
-    startIndex < endIndex &&
-    (expr[startIndex] === "`" || expr[startIndex] === "'")
-  ) {
-    startIndex++;
-  }
-
-  // Trim the trailing backticks and single quotes
-  while (
-    endIndex > startIndex &&
-    (expr[endIndex - 1] === "`" || expr[endIndex - 1] === "'")
-  ) {
-    endIndex--;
-  }
-
-  // Return the trimmed substring
-  return expr.substring(startIndex, endIndex);
-}
-
-export const defaultOutputProcessor = async ({
-  llmOutput,
-  jsonValue,
-}: {
-  llmOutput: string;
-  jsonValue: JSONSchemaType;
-}): Promise<Record<string, unknown>[]> => {
-  const expressions = llmOutput
-    .split(",")
-    .map((expr) => removeExtraQuotes(expr.trim()));
-
-  const results: Record<string, unknown>[] = [];
-
-  for (const expression of expressions) {
-    // get the key for example content from $.content
-    const key = expression.split(".").pop();
-
-    try {
-      const datums = jsonpath.query(jsonValue, expression);
-
-      if (!key) throw new Error(`Invalid JSON Path: ${expression}`);
-
-      for (const datum of datums) {
-        // in case there is a filter like [?(@.username=='simon')] without a key ie: $..comments[?(@.username=='simon').content]
-        if (key.includes("==")) {
-          results.push(datum);
-          continue;
-        }
-
-        results.push({
-          [key]: datum,
-        });
-      }
-    } catch (err) {
-      throw new Error(`Invalid JSON Path: ${expression}`);
-    }
-  }
-
-  return results;
-};
-
-type OutputProcessor = typeof defaultOutputProcessor;
-
-/**
- * A JSON query engine that uses JSONPath to query a JSON object.
- */
-export class JSONQueryEngine implements QueryEngine {
-  jsonValue: JSONSchemaType;
-  jsonSchema: JSONSchemaType;
-  serviceContext: ServiceContext;
-  outputProcessor: OutputProcessor;
-  verbose: boolean;
-  jsonPathPrompt: JSONPathPrompt;
-  synthesizeResponse: boolean;
-  responseSynthesisPrompt: ResponseSynthesisPrompt;
-
-  constructor(init: {
-    jsonValue: JSONSchemaType;
-    jsonSchema: JSONSchemaType;
-    serviceContext?: ServiceContext;
-    jsonPathPrompt?: JSONPathPrompt;
-    outputProcessor?: OutputProcessor;
-    synthesizeResponse?: boolean;
-    responseSynthesisPrompt?: ResponseSynthesisPrompt;
-    verbose?: boolean;
-  }) {
-    this.jsonValue = init.jsonValue;
-    this.jsonSchema = init.jsonSchema;
-    this.serviceContext = init.serviceContext ?? serviceContextFromDefaults({});
-    this.jsonPathPrompt = init.jsonPathPrompt ?? defaultJsonPathPrompt;
-    this.outputProcessor = init.outputProcessor ?? defaultOutputProcessor;
-    this.verbose = init.verbose ?? false;
-    this.synthesizeResponse = init.synthesizeResponse ?? true;
-    this.responseSynthesisPrompt =
-      init.responseSynthesisPrompt ?? defaultResponseSynthesizePrompt;
-  }
-
-  getPrompts(): Record<string, unknown> {
-    return {
-      jsonPathPrompt: this.jsonPathPrompt,
-      responseSynthesisPrompt: this.responseSynthesisPrompt,
-    };
-  }
-
-  updatePrompts(prompts: {
-    jsonPathPrompt?: JSONPathPrompt;
-    responseSynthesisPrompt?: ResponseSynthesisPrompt;
-  }): void {
-    if (prompts.jsonPathPrompt) {
-      this.jsonPathPrompt = prompts.jsonPathPrompt;
-    }
-    if (prompts.responseSynthesisPrompt) {
-      this.responseSynthesisPrompt = prompts.responseSynthesisPrompt;
-    }
-  }
-
-  getPromptModules(): Record<string, unknown> {
-    return {};
-  }
-
-  getSchemaContext(): string {
-    return JSON.stringify(this.jsonSchema);
-  }
-
-  query(
-    params: QueryEngineParamsStreaming,
-  ): Promise<AsyncIterable<EngineResponse>>;
-  query(params: QueryEngineParamsNonStreaming): Promise<EngineResponse>;
-  async query(
-    params: QueryEngineParamsStreaming | QueryEngineParamsNonStreaming,
-  ): Promise<EngineResponse | AsyncIterable<EngineResponse>> {
-    const { query, stream } = params;
-
-    if (stream) {
-      throw new Error("Streaming is not supported");
-    }
-
-    const schema = this.getSchemaContext();
-
-    const { text: jsonPathResponse } = await this.serviceContext.llm.complete({
-      prompt: this.jsonPathPrompt({ query, schema }),
-    });
-
-    if (this.verbose) {
-      console.log(
-        `> JSONPath Instructions:\n\`\`\`\n${jsonPathResponse}\n\`\`\`\n`,
-      );
-    }
-
-    const jsonPathOutput = await this.outputProcessor({
-      llmOutput: jsonPathResponse,
-      jsonValue: this.jsonValue,
-    });
-
-    if (this.verbose) {
-      console.log(`> JSONPath Output: ${jsonPathOutput}\n`);
-    }
-
-    let responseStr;
-
-    if (this.synthesizeResponse) {
-      responseStr = await this.serviceContext.llm.complete({
-        prompt: this.responseSynthesisPrompt({
-          query,
-          jsonSchema: schema,
-          jsonPath: jsonPathResponse,
-          jsonPathValue: JSON.stringify(jsonPathOutput),
-        }),
-      });
-
-      responseStr = responseStr.text;
-    } else {
-      responseStr = JSON.stringify(jsonPathOutput);
-    }
-
-    const responseMetadata = {
-      jsonPathResponse,
-    };
-
-    const response = EngineResponse.fromResponse(responseStr, false);
-
-    response.metadata = responseMetadata;
-
-    return response;
-  }
-}

packages/experimental/src/engines/query/index.ts

@@ -1 +0,0 @@
-export * from "./JSONQueryEngine.js";

packages/experimental/src/engines/query/prompt.ts

@@ -1,36 +0,0 @@
-export const defaultJsonPathPrompt = ({
-  query,
-  schema,
-}: {
-  query: string;
-  schema: string;
-}) => `
-We have provided a JSON schema below:
-${schema}
-Given a task, respond with a JSON Path query that can retrieve data from a JSON value that matches the schema.
-Task: ${query}
-JSONPath: 
-`;
-
-export type JSONPathPrompt = typeof defaultJsonPathPrompt;
-
-export const defaultResponseSynthesizePrompt = ({
-  query,
-  jsonSchema,
-  jsonPath,
-  jsonPathValue,
-}: {
-  query: string;
-  jsonSchema: string;
-  jsonPath: string;
-  jsonPathValue: string;
-}) => `
-Given a query, synthesize a response to satisfy the query using the JSON results. Only include details that are relevant to the query. If you don't know the answer, then say that.
-JSON Schema: ${jsonSchema}
-JSON Path: ${jsonPath}
-Value at path: ${jsonPathValue}
-Query: ${query}
-Response: 
-`;
-
-export type ResponseSynthesisPrompt = typeof defaultResponseSynthesizePrompt;

packages/experimental/src/index.ts

@@ -1 +0,0 @@
-export * from "./engines/query/index.js";

packages/llamaindex/CHANGELOG.md

@@ -1,5 +1,91 @@
 # llamaindex
 
+## 0.6.9
+
+### Patch Changes
+
+- Updated dependencies [ac41ed3]
+  - @llamaindex/cloud@0.2.8
+
+## 0.6.8
+
+### Patch Changes
+
+- 8b7fdba: refactor: move chat engine & retriever into core.
+
+  - `chatHistory` in BaseChatEngine now returns `ChatMessage[] | Promise<ChatMessage[]>`, instead of `BaseMemory`
+  - update `retrieve-end` type
+
+- Updated dependencies [8b7fdba]
+  - @llamaindex/core@0.2.6
+  - @llamaindex/openai@0.1.8
+  - @llamaindex/groq@0.0.7
+
+## 0.6.7
+
+### Patch Changes
+
+- 23bcc37: fix: add `serializer` in doc store
+
+  `PostgresDocumentStore` now will not use JSON.stringify for better performance
+
+## 0.6.6
+
+### Patch Changes
+
+- d902cc3: Fix context not being sent using ContextChatEngine
+- 025ffe6: fix: update `PostgresKVStore` constructor params
+- a659574: Adds upstash vector store as a storage
+- Updated dependencies [d902cc3]
+  - @llamaindex/core@0.2.5
+  - @llamaindex/openai@0.1.7
+  - @llamaindex/groq@0.0.6
+
+## 0.6.5
+
+### Patch Changes
+
+- e9714db: feat: update `PGVectorStore`
+
+  - move constructor parameter `config.user` | `config.database` | `config.password` | `config.connectionString` into `config.clientConfig`
+  - if you pass `pg.Client` or `pg.Pool` instance to `PGVectorStore`, move it to `config.client`, setting `config.shouldConnect` to false if it's already connected
+  - default value of `PGVectorStore.collection` is now `"data"` instead of `""` (empty string)
+
+## 0.6.4
+
+### Patch Changes
+
+- b48bcc3: feat: add `load-transformers` event type when loading `@xenova/transformers` module
+
+  This would benefit user who want to customize the transformer env.
+
+- Updated dependencies [b48bcc3]
+  - @llamaindex/core@0.2.4
+  - @llamaindex/env@0.1.12
+  - @llamaindex/openai@0.1.6
+  - @llamaindex/groq@0.0.5
+
+## 0.6.3
+
+### Patch Changes
+
+- 2cd1383: refactor: align `response-synthesizers` & `chat-engine` module
+
+  - builtin event system
+  - correct class extends
+  - aligin APIs, naming with llama-index python
+  - move stream out of first parameter to second parameter for the better tyep checking
+  - remove JSONQueryEngine in `@llamaindex/experimental`, as the code quality is not satisify and we will bring it back later
+
+- 5c4badb: Extend JinaAPIEmbedding parameters
+- Updated dependencies [fb36eff]
+- Updated dependencies [d24d3d1]
+- Updated dependencies [2cd1383]
+  - @llamaindex/cloud@0.2.7
+  - @llamaindex/core@0.2.3
+  - @llamaindex/openai@0.1.5
+  - @llamaindex/groq@0.0.4
+
 ## 0.6.2
 
 ### Patch Changes

packages/llamaindex/e2e/.env.ci

@@ -0,0 +1 @@
+POSTGRES_USER=runner

packages/llamaindex/e2e/examples/cloudflare-worker-agent/CHANGELOG.md

@@ -1,5 +1,56 @@
 # @llamaindex/cloudflare-worker-agent-test
 
+## 0.0.71
+
+### Patch Changes
+
+- llamaindex@0.6.9
+
+## 0.0.70
+
+### Patch Changes
+
+- Updated dependencies [8b7fdba]
+  - llamaindex@0.6.8
+
+## 0.0.69
+
+### Patch Changes
+
+- Updated dependencies [23bcc37]
+  - llamaindex@0.6.7
+
+## 0.0.68
+
+### Patch Changes
+
+- Updated dependencies [d902cc3]
+- Updated dependencies [025ffe6]
+- Updated dependencies [a659574]
+  - llamaindex@0.6.6
+
+## 0.0.67
+
+### Patch Changes
+
+- Updated dependencies [e9714db]
+  - llamaindex@0.6.5
+
+## 0.0.66
+
+### Patch Changes
+
+- Updated dependencies [b48bcc3]
+  - llamaindex@0.6.4
+
+## 0.0.65
+
+### Patch Changes
+
+- Updated dependencies [2cd1383]
+- Updated dependencies [5c4badb]
+  - llamaindex@0.6.3
+
 ## 0.0.64
 
 ### Patch Changes

packages/llamaindex/e2e/examples/cloudflare-worker-agent/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/cloudflare-worker-agent-test",
-  "version": "0.0.64",
+  "version": "0.0.71",
   "type": "module",
   "private": true,
   "scripts": {

packages/llamaindex/e2e/examples/cloudflare-worker-agent/tsconfig.json

@@ -100,7 +100,8 @@
 
     /* Completeness */
     // "skipDefaultLibCheck": true,                      /* Skip type checking .d.ts files that are included with TypeScript. */
-    "skipLibCheck": true /* Skip type checking all .d.ts files. */
+    "skipLibCheck": true /* Skip type checking all .d.ts files. */,
+    "tsBuildInfoFile": "./dist/.tsbuildinfo"
   },
   "exclude": ["test"]
 }

packages/llamaindex/e2e/examples/llama-parse-browser/CHANGELOG.md

@@ -1,5 +1,20 @@
 # @llamaindex/llama-parse-browser-test
 
+## 0.0.4
+
+### Patch Changes
+
+- Updated dependencies [ac41ed3]
+  - @llamaindex/cloud@0.2.8
+
+## 0.0.3
+
+### Patch Changes
+
+- Updated dependencies [fb36eff]
+- Updated dependencies [d24d3d1]
+  - @llamaindex/cloud@0.2.7
+
 ## 0.0.2
 
 ### Patch Changes

packages/llamaindex/e2e/examples/llama-parse-browser/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/llama-parse-browser-test",
   "private": true,
-  "version": "0.0.2",
+  "version": "0.0.4",
   "type": "module",
   "scripts": {
     "dev": "vite",

packages/llamaindex/e2e/examples/nextjs-agent/CHANGELOG.md

@@ -1,5 +1,56 @@
 # @llamaindex/next-agent-test
 
+## 0.1.71
+
+### Patch Changes
+
+- llamaindex@0.6.9
+
+## 0.1.70
+
+### Patch Changes
+
+- Updated dependencies [8b7fdba]
+  - llamaindex@0.6.8
+
+## 0.1.69
+
+### Patch Changes
+
+- Updated dependencies [23bcc37]
+  - llamaindex@0.6.7
+
+## 0.1.68
+
+### Patch Changes
+
+- Updated dependencies [d902cc3]
+- Updated dependencies [025ffe6]
+- Updated dependencies [a659574]
+  - llamaindex@0.6.6
+
+## 0.1.67
+
+### Patch Changes
+
+- Updated dependencies [e9714db]
+  - llamaindex@0.6.5
+
+## 0.1.66
+
+### Patch Changes
+
+- Updated dependencies [b48bcc3]
+  - llamaindex@0.6.4
+
+## 0.1.65
+
+### Patch Changes
+
+- Updated dependencies [2cd1383]
+- Updated dependencies [5c4badb]
+  - llamaindex@0.6.3
+
 ## 0.1.64
 
 ### Patch Changes

packages/llamaindex/e2e/examples/nextjs-agent/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/next-agent-test",
-  "version": "0.1.64",
+  "version": "0.1.71",
   "private": true,
   "scripts": {
     "dev": "next dev",

packages/llamaindex/e2e/examples/nextjs-edge-runtime/CHANGELOG.md

@@ -1,5 +1,56 @@
 # test-edge-runtime
 
+## 0.1.70
+
+### Patch Changes
+
+- llamaindex@0.6.9
+
+## 0.1.69
+
+### Patch Changes
+
+- Updated dependencies [8b7fdba]
+  - llamaindex@0.6.8
+
+## 0.1.68
+
+### Patch Changes
+
+- Updated dependencies [23bcc37]
+  - llamaindex@0.6.7
+
+## 0.1.67
+
+### Patch Changes
+
+- Updated dependencies [d902cc3]
+- Updated dependencies [025ffe6]
+- Updated dependencies [a659574]
+  - llamaindex@0.6.6
+
+## 0.1.66
+
+### Patch Changes
+
+- Updated dependencies [e9714db]
+  - llamaindex@0.6.5
+
+## 0.1.65
+
+### Patch Changes
+
+- Updated dependencies [b48bcc3]
+  - llamaindex@0.6.4
+
+## 0.1.64
+
+### Patch Changes
+
+- Updated dependencies [2cd1383]
+- Updated dependencies [5c4badb]
+  - llamaindex@0.6.3
+
 ## 0.1.63
 
 ### Patch Changes

packages/llamaindex/e2e/examples/nextjs-edge-runtime/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/nextjs-edge-runtime-test",
-  "version": "0.1.63",
+  "version": "0.1.70",
   "private": true,
   "scripts": {
     "dev": "next dev",

packages/llamaindex/e2e/examples/nextjs-edge-runtime/src/app/globals.css

@@ -1,107 +0,0 @@
-:root {
-  --max-width: 1100px;
-  --border-radius: 12px;
-  --font-mono: ui-monospace, Menlo, Monaco, "Cascadia Mono", "Segoe UI Mono",
-    "Roboto Mono", "Oxygen Mono", "Ubuntu Monospace", "Source Code Pro",
-    "Fira Mono", "Droid Sans Mono", "Courier New", monospace;
-
-  --foreground-rgb: 0, 0, 0;
-  --background-start-rgb: 214, 219, 220;
-  --background-end-rgb: 255, 255, 255;
-
-  --primary-glow: conic-gradient(
-    from 180deg at 50% 50%,
-    #16abff33 0deg,
-    #0885ff33 55deg,
-    #54d6ff33 120deg,
-    #0071ff33 160deg,
-    transparent 360deg
-  );
-  --secondary-glow: radial-gradient(
-    rgba(255, 255, 255, 1),
-    rgba(255, 255, 255, 0)
-  );
-
-  --tile-start-rgb: 239, 245, 249;
-  --tile-end-rgb: 228, 232, 233;
-  --tile-border: conic-gradient(
-    #00000080,
-    #00000040,
-    #00000030,
-    #00000020,
-    #00000010,
-    #00000010,
-    #00000080
-  );
-
-  --callout-rgb: 238, 240, 241;
-  --callout-border-rgb: 172, 175, 176;
-  --card-rgb: 180, 185, 188;
-  --card-border-rgb: 131, 134, 135;
-}
-
-@media (prefers-color-scheme: dark) {
-  :root {
-    --foreground-rgb: 255, 255, 255;
-    --background-start-rgb: 0, 0, 0;
-    --background-end-rgb: 0, 0, 0;
-
-    --primary-glow: radial-gradient(rgba(1, 65, 255, 0.4), rgba(1, 65, 255, 0));
-    --secondary-glow: linear-gradient(
-      to bottom right,
-      rgba(1, 65, 255, 0),
-      rgba(1, 65, 255, 0),
-      rgba(1, 65, 255, 0.3)
-    );
-
-    --tile-start-rgb: 2, 13, 46;
-    --tile-end-rgb: 2, 5, 19;
-    --tile-border: conic-gradient(
-      #ffffff80,
-      #ffffff40,
-      #ffffff30,
-      #ffffff20,
-      #ffffff10,
-      #ffffff10,
-      #ffffff80
-    );
-
-    --callout-rgb: 20, 20, 20;
-    --callout-border-rgb: 108, 108, 108;
-    --card-rgb: 100, 100, 100;
-    --card-border-rgb: 200, 200, 200;
-  }
-}
-
-* {
-  box-sizing: border-box;
-  padding: 0;
-  margin: 0;
-}
-
-html,
-body {
-  max-width: 100vw;
-  overflow-x: hidden;
-}
-
-body {
-  color: rgb(var(--foreground-rgb));
-  background: linear-gradient(
-      to bottom,
-      transparent,
-      rgb(var(--background-end-rgb))
-    )
-    rgb(var(--background-start-rgb));
-}
-
-a {
-  color: inherit;
-  text-decoration: none;
-}
-
-@media (prefers-color-scheme: dark) {
-  html {
-    color-scheme: dark;
-  }
-}

packages/llamaindex/e2e/examples/nextjs-edge-runtime/src/utils/llm.ts

@@ -1,6 +1,6 @@
 // test runtime
 import "llamaindex";
-import { ClipEmbedding } from "llamaindex/embeddings/ClipEmbedding";
+import { ClipEmbedding } from "llamaindex";
 import "llamaindex/readers/SimpleDirectoryReader";
 
 // @ts-expect-error

packages/llamaindex/e2e/examples/nextjs-node-runtime/CHANGELOG.md

@@ -1,5 +1,56 @@
 # @llamaindex/next-node-runtime
 
+## 0.0.52
+
+### Patch Changes
+
+- llamaindex@0.6.9
+
+## 0.0.51
+
+### Patch Changes
+
+- Updated dependencies [8b7fdba]
+  - llamaindex@0.6.8
+
+## 0.0.50
+
+### Patch Changes
+
+- Updated dependencies [23bcc37]
+  - llamaindex@0.6.7
+
+## 0.0.49
+
+### Patch Changes
+
+- Updated dependencies [d902cc3]
+- Updated dependencies [025ffe6]
+- Updated dependencies [a659574]
+  - llamaindex@0.6.6
+
+## 0.0.48
+
+### Patch Changes
+
+- Updated dependencies [e9714db]
+  - llamaindex@0.6.5
+
+## 0.0.47
+
+### Patch Changes
+
+- Updated dependencies [b48bcc3]
+  - llamaindex@0.6.4
+
+## 0.0.46
+
+### Patch Changes
+
+- Updated dependencies [2cd1383]
+- Updated dependencies [5c4badb]
+  - llamaindex@0.6.3
+
 ## 0.0.45
 
 ### Patch Changes

packages/llamaindex/e2e/examples/nextjs-node-runtime/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/next-node-runtime-test",
-  "version": "0.0.45",
+  "version": "0.0.52",
   "private": true,
   "scripts": {
     "dev": "next dev",

packages/llamaindex/e2e/examples/waku-query-engine/CHANGELOG.md

@@ -1,5 +1,56 @@
 # @llamaindex/waku-query-engine-test
 
+## 0.0.71
+
+### Patch Changes
+
+- llamaindex@0.6.9
+
+## 0.0.70
+
+### Patch Changes
+
+- Updated dependencies [8b7fdba]
+  - llamaindex@0.6.8
+
+## 0.0.69
+
+### Patch Changes
+
+- Updated dependencies [23bcc37]
+  - llamaindex@0.6.7
+
+## 0.0.68
+
+### Patch Changes
+
+- Updated dependencies [d902cc3]
+- Updated dependencies [025ffe6]
+- Updated dependencies [a659574]
+  - llamaindex@0.6.6
+
+## 0.0.67
+
+### Patch Changes
+
+- Updated dependencies [e9714db]
+  - llamaindex@0.6.5
+
+## 0.0.66
+
+### Patch Changes
+
+- Updated dependencies [b48bcc3]
+  - llamaindex@0.6.4
+
+## 0.0.65
+
+### Patch Changes
+
+- Updated dependencies [2cd1383]
+- Updated dependencies [5c4badb]
+  - llamaindex@0.6.3
+
 ## 0.0.64
 
 ### Patch Changes

packages/llamaindex/e2e/examples/waku-query-engine/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/waku-query-engine-test",
-  "version": "0.0.64",
+  "version": "0.0.71",
   "type": "module",
   "private": true,
   "scripts": {

packages/llamaindex/e2e/examples/waku-query-engine/src/actions.ts

@@ -1,7 +1,7 @@
 "use server";
-import { Document, VectorStoreIndex, type QueryEngine } from "llamaindex";
+import { BaseQueryEngine, Document, VectorStoreIndex } from "llamaindex";
 import { readFile } from "node:fs/promises";
-let _queryEngine: QueryEngine;
+let _queryEngine: BaseQueryEngine;
 
 async function lazyLoadQueryEngine() {
   if (!_queryEngine) {

packages/llamaindex/e2e/node/embedding/clip.e2e.ts

@@ -1,8 +1,41 @@
-import { ClipEmbedding, ImageNode } from "llamaindex";
+import type { LoadTransformerEvent } from "@llamaindex/env";
+import { setTransformers } from "@llamaindex/env";
+import { ClipEmbedding, ImageNode, Settings } from "llamaindex";
 import assert from "node:assert";
-import { test } from "node:test";
+import { type Mock, test } from "node:test";
+
+let callback: Mock<(event: any) => void>;
+test.before(() => {
+  callback = test.mock.fn((event: any) => {
+    const { transformer } = event.detail as LoadTransformerEvent;
+    assert.ok(transformer);
+    assert.ok(transformer.env);
+  });
+  Settings.callbackManager.on("load-transformers", callback);
+});
+
+test.beforeEach(() => {
+  callback.mock.resetCalls();
+});
 
 await test("clip embedding", async (t) => {
+  await t.test("should trigger load transformer event", async () => {
+    const nodes = [
+      new ImageNode({
+        image: new URL(
+          "../../fixtures/img/llamaindex-white.png",
+          import.meta.url,
+        ),
+      }),
+    ];
+    assert.equal(callback.mock.callCount(), 0);
+    const clipEmbedding = new ClipEmbedding();
+    assert.equal(callback.mock.callCount(), 0);
+    const result = await clipEmbedding(nodes);
+    assert.strictEqual(result.length, 1);
+    assert.equal(callback.mock.callCount(), 1);
+  });
+
   await t.test("init & get image embedding", async () => {
     const clipEmbedding = new ClipEmbedding();
     const imgUrl = new URL(
@@ -27,4 +60,25 @@ await test("clip embedding", async (t) => {
     assert.strictEqual(result.length, 1);
     assert.ok(result[0]!.embedding);
   });
+
+  await t.test("custom transformer", async () => {
+    const transformers = await import("@xenova/transformers");
+    const getter = test.mock.fn((t, k, r) => {
+      return Reflect.get(t, k, r);
+    });
+    setTransformers(
+      new Proxy(transformers, {
+        get: getter,
+      }),
+    );
+    const clipEmbedding = new ClipEmbedding();
+    const imgUrl = new URL(
+      "../../fixtures/img/llamaindex-white.png",
+      import.meta.url,
+    );
+    assert.equal(getter.mock.callCount(), 0);
+    const vec = await clipEmbedding.getImageEmbedding(imgUrl);
+    assert.ok(vec);
+    assert.ok(getter.mock.callCount() > 0);
+  });
 });

packages/llamaindex/e2e/node/vector-store/pg-vector-store.e2e.ts

@@ -1,3 +1,5 @@
+/* eslint-disable turbo/no-undeclared-env-vars */
+import { config } from "dotenv";
 import { Document, VectorStoreQueryMode } from "llamaindex";
 import { PGVectorStore } from "llamaindex/vector-store/PGVectorStore";
 import assert from "node:assert";
@@ -5,43 +7,56 @@ import { test } from "node:test";
 import pg from "pg";
 import { registerTypes } from "pgvector/pg";
 
-let pgClient: pg.Client | pg.Pool;
-test.afterEach(async () => {
-  await pgClient.end();
-});
+config({ path: [".env.local", ".env", ".env.ci"] });
 
-await test("init with client", async () => {
-  pgClient = new pg.Client({
-    database: "llamaindex_node_test",
-  });
+const pgConfig = {
+  user: process.env.POSTGRES_USER ?? "user",
+  password: process.env.POSTGRES_PASSWORD ?? "password",
+  database: "llamaindex_node_test",
+};
+
+await test("init with client", async (t) => {
+  const pgClient = new pg.Client(pgConfig);
   await pgClient.connect();
   await pgClient.query("CREATE EXTENSION IF NOT EXISTS vector");
   await registerTypes(pgClient);
-  const vectorStore = new PGVectorStore(pgClient);
+  t.after(async () => {
+    await pgClient.end();
+  });
+  const vectorStore = new PGVectorStore({
+    client: pgClient,
+    shouldConnect: false,
+  });
   assert.deepStrictEqual(await vectorStore.client(), pgClient);
 });
 
-await test("init with pool", async () => {
-  pgClient = new pg.Pool({
-    database: "llamaindex_node_test",
-  });
+await test("init with pool", async (t) => {
+  const pgClient = new pg.Pool(pgConfig);
   await pgClient.query("CREATE EXTENSION IF NOT EXISTS vector");
   const client = await pgClient.connect();
+  await client.query("CREATE EXTENSION IF NOT EXISTS vector");
   await registerTypes(client);
-  const vectorStore = new PGVectorStore(client);
+  t.after(async () => {
+    client.release();
+    await pgClient.end();
+  });
+  const vectorStore = new PGVectorStore({
+    shouldConnect: false,
+    client,
+  });
   assert.deepStrictEqual(await vectorStore.client(), client);
-  client.release();
 });
 
-await test("init without client", async () => {
-  const vectorStore = new PGVectorStore({
-    database: "llamaindex_node_test",
+await test("init without client", async (t) => {
+  const vectorStore = new PGVectorStore({ clientConfig: pgConfig });
+  const pgClient = (await vectorStore.client()) as pg.Client;
+  t.after(async () => {
+    await pgClient.end();
   });
-  pgClient = (await vectorStore.client()) as pg.Client;
   assert.notDeepStrictEqual(pgClient, undefined);
 });
 
-await test("simple node", async () => {
+await test("simple node", async (t) => {
   const dimensions = 3;
   const schemaName =
     "llamaindex_vector_store_test_" + Math.random().toString(36).substring(7);
@@ -52,10 +67,14 @@ await test("simple node", async () => {
     embedding: [0.1, 0.2, 0.3],
   });
   const vectorStore = new PGVectorStore({
-    database: "llamaindex_node_test",
+    clientConfig: pgConfig,
     dimensions,
     schemaName,
   });
+  const pgClient = (await vectorStore.client()) as pg.Client;
+  t.after(async () => {
+    await pgClient.end();
+  });
 
   await vectorStore.add([node]);
 
@@ -85,6 +104,4 @@ await test("simple node", async () => {
     });
     assert.deepStrictEqual(result.nodes, []);
   }
-
-  pgClient = (await vectorStore.client()) as pg.Client;
 });

packages/llamaindex/e2e/package.json

@@ -4,14 +4,15 @@
   "version": "0.0.7",
   "type": "module",
   "scripts": {
-    "e2e": "node --import tsx --import ./mock-register.js --test ./node/*.e2e.ts",
-    "e2e:nomock": "node --import tsx --test ./node/*.e2e.ts",
-    "e2e:updatesnap": "UPDATE_SNAPSHOT=1 node --import tsx --test ./node/*.e2e.ts"
+    "e2e": "node --import tsx --import ./mock-register.js --test ./node/**/*.e2e.ts",
+    "e2e:nomock": "node --import tsx --test ./node/**/*.e2e.ts",
+    "e2e:updatesnap": "UPDATE_SNAPSHOT=1 node --import tsx --test ./node/**/*.e2e.ts"
   },
   "devDependencies": {
-    "@faker-js/faker": "^8.4.1",
+    "@faker-js/faker": "^9.0.1",
     "@types/node": "^22.5.1",
     "consola": "^3.2.3",
+    "dotenv": "^16.4.5",
     "llamaindex": "workspace:*",
     "tsx": "^4.19.0"
   }

packages/llamaindex/package.json

@@ -1,6 +1,6 @@
 {
   "name": "llamaindex",
-  "version": "0.6.2",
+  "version": "0.6.9",
   "license": "MIT",
   "type": "module",
   "keywords": [
@@ -33,8 +33,8 @@
     "@llamaindex/cloud": "workspace:*",
     "@llamaindex/core": "workspace:*",
     "@llamaindex/env": "workspace:*",
-    "@llamaindex/openai": "workspace:*",
     "@llamaindex/groq": "workspace:*",
+    "@llamaindex/openai": "workspace:*",
     "@mistralai/mistralai": "^1.0.4",
     "@mixedbread-ai/sdk": "^2.2.11",
     "@pinecone-database/pinecone": "^3.0.2",
@@ -43,7 +43,7 @@
     "@types/node": "^22.5.1",
     "@types/papaparse": "^5.3.14",
     "@types/pg": "^8.11.8",
-    "@xenova/transformers": "^2.17.2",
+    "@upstash/vector": "^1.1.5",
     "@zilliz/milvus2-sdk-node": "^2.4.6",
     "ajv": "^8.17.1",
     "assemblyai": "^4.7.0",
@@ -91,6 +91,7 @@
     "@notionhq/client": "^2.2.15",
     "@swc/cli": "^0.4.0",
     "@swc/core": "^1.7.22",
+    "@xenova/transformers": "^2.17.2",
     "concurrently": "^8.2.2",
     "glob": "^11.0.0",
     "pg": "^8.12.0",

packages/llamaindex/src/Retriever.ts

@@ -1,20 +0,0 @@
-import type { NodeWithScore } from "@llamaindex/core/schema";
-import type { ServiceContext } from "./ServiceContext.js";
-import type { MessageContent } from "./index.edge.js";
-
-export type RetrieveParams = {
-  query: MessageContent;
-  preFilters?: unknown;
-};
-
-/**
- * Retrievers retrieve the nodes that most closely match our query in similarity.
- */
-export interface BaseRetriever {
-  retrieve(params: RetrieveParams): Promise<NodeWithScore[]>;
-
-  /**
-   * @deprecated to be deprecated soon
-   */
-  serviceContext?: ServiceContext | undefined;
-}

packages/llamaindex/src/ServiceContext.ts

@@ -9,6 +9,8 @@ import { OpenAI, OpenAIEmbedding } from "@llamaindex/openai";
 
 /**
  * The ServiceContext is a collection of components that are used in different parts of the application.
+ *
+ * @deprecated This will no longer supported, please use `Settings` instead.
  */
 export interface ServiceContext {
   llm: LLM;

packages/llamaindex/src/Settings.ts

@@ -12,6 +12,7 @@ import {
   type NodeParser,
   SentenceSplitter,
 } from "@llamaindex/core/node-parser";
+import type { LoadTransformerEvent } from "@llamaindex/env";
 import { AsyncLocalStorage, getEnv } from "@llamaindex/env";
 import type { ServiceContext } from "./ServiceContext.js";
 import {
@@ -20,6 +21,12 @@ import {
   withEmbeddedModel,
 } from "./internal/settings/EmbedModel.js";
 
+declare module "@llamaindex/core/global" {
+  interface LlamaIndexEventMaps {
+    "load-transformers": LoadTransformerEvent;
+  }
+}
+
 export type PromptConfig = {
   llm?: string;
   lang?: string;

packages/llamaindex/src/agent/anthropic.ts

@@ -1,9 +1,9 @@
-import { Settings } from "../Settings.js";
 import type {
-  ChatEngineParamsNonStreaming,
-  ChatEngineParamsStreaming,
-  EngineResponse,
-} from "../index.edge.js";
+  NonStreamingChatEngineParams,
+  StreamingChatEngineParams,
+} from "@llamaindex/core/chat-engine";
+import type { EngineResponse } from "@llamaindex/core/schema";
+import { Settings } from "../Settings.js";
 import { Anthropic } from "../llm/anthropic.js";
 import { LLMAgent, LLMAgentWorker, type LLMAgentParams } from "./llm.js";
 
@@ -24,12 +24,13 @@ export class AnthropicAgent extends LLMAgent {
     });
   }
 
-  async chat(params: ChatEngineParamsNonStreaming): Promise<EngineResponse>;
-  async chat(params: ChatEngineParamsStreaming): Promise<never>;
+  async chat(params: NonStreamingChatEngineParams): Promise<EngineResponse>;
+  async chat(params: StreamingChatEngineParams): Promise<never>;
   override async chat(
-    params: ChatEngineParamsNonStreaming | ChatEngineParamsStreaming,
+    params: NonStreamingChatEngineParams | StreamingChatEngineParams,
   ) {
-    if (params.stream) {
+    const { stream } = params;
+    if (stream) {
       // Anthropic does support this, but looks like it's not supported in the LITS LLM
       throw new Error("Anthropic does not support streaming");
     }

packages/llamaindex/src/agent/base.ts

@@ -1,3 +1,8 @@
+import {
+  BaseChatEngine,
+  type NonStreamingChatEngineParams,
+  type StreamingChatEngineParams,
+} from "@llamaindex/core/chat-engine";
 import type {
   BaseToolWithCall,
   ChatMessage,
@@ -10,11 +15,6 @@ import { EngineResponse } from "@llamaindex/core/schema";
 import { wrapEventCaller } from "@llamaindex/core/utils";
 import { randomUUID } from "@llamaindex/env";
 import { Settings } from "../Settings.js";
-import {
-  type ChatEngine,
-  type ChatEngineParamsNonStreaming,
-  type ChatEngineParamsStreaming,
-} from "../engines/chat/index.js";
 import { consoleLogger, emptyLogger } from "../internal/logger.js";
 import { isReadableStream } from "../internal/utils.js";
 import { ObjectRetriever } from "../objects/index.js";
@@ -207,8 +207,7 @@ export abstract class AgentRunner<
   >
     ? AdditionalMessageOptions
     : never,
-> implements ChatEngine
-{
+> extends BaseChatEngine {
   readonly #llm: AI;
   readonly #tools:
     | BaseToolWithCall[]
@@ -259,6 +258,7 @@ export abstract class AgentRunner<
   protected constructor(
     params: AgentRunnerParams<AI, Store, AdditionalMessageOptions>,
   ) {
+    super();
     const { llm, chatHistory, systemPrompt, runner, tools, verbose } = params;
     this.#llm = llm;
     this.#chatHistory = chatHistory;
@@ -345,20 +345,19 @@ export abstract class AgentRunner<
     });
   }
 
-  async chat(params: ChatEngineParamsNonStreaming): Promise<EngineResponse>;
+  async chat(params: NonStreamingChatEngineParams): Promise<EngineResponse>;
   async chat(
-    params: ChatEngineParamsStreaming,
+    params: StreamingChatEngineParams,
   ): Promise<ReadableStream<EngineResponse>>;
   @wrapEventCaller
   async chat(
-    params: ChatEngineParamsNonStreaming | ChatEngineParamsStreaming,
+    params: NonStreamingChatEngineParams | StreamingChatEngineParams,
   ): Promise<EngineResponse | ReadableStream<EngineResponse>> {
     let chatHistory: ChatMessage<AdditionalMessageOptions>[] = [];
 
     if (params.chatHistory instanceof BaseMemory) {
-      chatHistory = (await params.chatHistory.getMessages(
-        params.message,
-      )) as ChatMessage<AdditionalMessageOptions>[];
+      chatHistory =
+        (await params.chatHistory.getMessages()) as ChatMessage<AdditionalMessageOptions>[];
     } else {
       chatHistory =
         params.chatHistory as ChatMessage<AdditionalMessageOptions>[];

packages/llamaindex/src/cloud/LlamaCloudIndex.ts

@@ -1,9 +1,8 @@
+import type { BaseQueryEngine } from "@llamaindex/core/query-engine";
+import type { BaseSynthesizer } from "@llamaindex/core/response-synthesizers";
 import type { Document, TransformComponent } from "@llamaindex/core/schema";
-import type { BaseRetriever } from "../Retriever.js";
 import { RetrieverQueryEngine } from "../engines/query/RetrieverQueryEngine.js";
 import type { BaseNodePostprocessor } from "../postprocessors/types.js";
-import type { BaseSynthesizer } from "../synthesizers/types.js";
-import type { QueryEngine } from "../types.js";
 import type { CloudRetrieveParams } from "./LlamaCloudRetriever.js";
 import { LlamaCloudRetriever } from "./LlamaCloudRetriever.js";
 import { getPipelineCreate } from "./config.js";
@@ -12,6 +11,7 @@ import { getAppBaseUrl, getProjectId, initService } from "./utils.js";
 
 import { PipelinesService, ProjectsService } from "@llamaindex/cloud/api";
 import { SentenceSplitter } from "@llamaindex/core/node-parser";
+import type { BaseRetriever } from "@llamaindex/core/retriever";
 import { getEnv } from "@llamaindex/env";
 import { OpenAIEmbedding } from "@llamaindex/openai";
 import { Settings } from "../Settings.js";
@@ -300,7 +300,7 @@ export class LlamaCloudIndex {
       preFilters?: unknown;
       nodePostprocessors?: BaseNodePostprocessor[];
     } & CloudRetrieveParams,
-  ): QueryEngine {
+  ): BaseQueryEngine {
     const retriever = new LlamaCloudRetriever({
       ...this.params,
       ...params,

packages/llamaindex/src/cloud/LlamaCloudRetriever.ts

@@ -4,11 +4,12 @@ import {
   type RetrievalParams,
   type TextNodeWithScore,
 } from "@llamaindex/cloud/api";
-import { DEFAULT_PROJECT_NAME, Settings } from "@llamaindex/core/global";
+import { DEFAULT_PROJECT_NAME } from "@llamaindex/core/global";
+import type { QueryBundle } from "@llamaindex/core/query-engine";
+import { BaseRetriever } from "@llamaindex/core/retriever";
 import type { NodeWithScore } from "@llamaindex/core/schema";
 import { jsonToNode, ObjectType } from "@llamaindex/core/schema";
-import { extractText, wrapEventCaller } from "@llamaindex/core/utils";
-import type { BaseRetriever, RetrieveParams } from "../Retriever.js";
+import { extractText } from "@llamaindex/core/utils";
 import type { ClientParams, CloudConstructorParams } from "./type.js";
 import { getProjectId, initService } from "./utils.js";
 
@@ -17,7 +18,7 @@ export type CloudRetrieveParams = Omit<
   "query" | "search_filters" | "dense_similarity_top_k"
 > & { similarityTopK?: number; filters?: MetadataFilters };
 
-export class LlamaCloudRetriever implements BaseRetriever {
+export class LlamaCloudRetriever extends BaseRetriever {
   clientParams: ClientParams;
   retrieveParams: CloudRetrieveParams;
   organizationId?: string;
@@ -36,12 +37,13 @@ export class LlamaCloudRetriever implements BaseRetriever {
       return {
         // Currently LlamaCloud only supports text nodes
         node: textNode,
-        score: node.score,
+        score: node.score ?? undefined,
       };
     });
   }
 
   constructor(params: CloudConstructorParams & CloudRetrieveParams) {
+    super();
     this.clientParams = { apiKey: params.apiKey, baseUrl: params.baseUrl };
     initService(this.clientParams);
     this.retrieveParams = params;
@@ -54,11 +56,7 @@ export class LlamaCloudRetriever implements BaseRetriever {
     }
   }
 
-  @wrapEventCaller
-  async retrieve({
-    query,
-    preFilters,
-  }: RetrieveParams): Promise<NodeWithScore[]> {
+  async _retrieve(query: QueryBundle): Promise<NodeWithScore[]> {
     const { data: pipelines } =
       await PipelinesService.searchPipelinesApiV1PipelinesGet({
         query: {
@@ -97,19 +95,11 @@ export class LlamaCloudRetriever implements BaseRetriever {
         body: {
           ...this.retrieveParams,
           query: extractText(query),
-          search_filters:
-            this.retrieveParams.filters ?? (preFilters as MetadataFilters),
+          search_filters: this.retrieveParams.filters as MetadataFilters,
           dense_similarity_top_k: this.retrieveParams.similarityTopK!,
         },
       });
 
-    const nodesWithScores = this.resultNodesToNodeWithScore(
-      results.retrieval_nodes,
-    );
-    Settings.callbackManager.dispatchEvent("retrieve-end", {
-      query,
-      nodes: nodesWithScores,
-    });
-    return nodesWithScores;
+    return this.resultNodesToNodeWithScore(results.retrieval_nodes);
   }
 }

packages/llamaindex/src/embeddings/ClipEmbedding.ts

@@ -1,17 +1,26 @@
 import { MultiModalEmbedding } from "@llamaindex/core/embeddings";
 import type { ImageType } from "@llamaindex/core/schema";
 import _ from "lodash";
-import { lazyLoadTransformers } from "../internal/deps/transformers.js";
 // only import type, to avoid bundling error
+import { loadTransformers } from "@llamaindex/env";
 import type {
   CLIPTextModelWithProjection,
   CLIPVisionModelWithProjection,
   PreTrainedTokenizer,
   Processor,
 } from "@xenova/transformers";
+import { Settings } from "../Settings.js";
 
 async function readImage(input: ImageType) {
-  const { RawImage } = await lazyLoadTransformers();
+  const { RawImage } = await loadTransformers((transformer) => {
+    Settings.callbackManager.dispatchEvent(
+      "load-transformers",
+      {
+        transformer,
+      },
+      true,
+    );
+  });
   if (input instanceof Blob) {
     return await RawImage.fromBlob(input);
   } else if (_.isString(input) || input instanceof URL) {
@@ -40,7 +49,15 @@ export class ClipEmbedding extends MultiModalEmbedding {
   }
 
   async getTokenizer() {
-    const { AutoTokenizer } = await lazyLoadTransformers();
+    const { AutoTokenizer } = await loadTransformers((transformer) => {
+      Settings.callbackManager.dispatchEvent(
+        "load-transformers",
+        {
+          transformer,
+        },
+        true,
+      );
+    });
     if (!this.tokenizer) {
       this.tokenizer = await AutoTokenizer.from_pretrained(this.modelType);
     }
@@ -48,7 +65,15 @@ export class ClipEmbedding extends MultiModalEmbedding {
   }
 
   async getProcessor() {
-    const { AutoProcessor } = await lazyLoadTransformers();
+    const { AutoProcessor } = await loadTransformers((transformer) => {
+      Settings.callbackManager.dispatchEvent(
+        "load-transformers",
+        {
+          transformer,
+        },
+        true,
+      );
+    });
     if (!this.processor) {
       this.processor = await AutoProcessor.from_pretrained(this.modelType);
     }
@@ -56,7 +81,17 @@ export class ClipEmbedding extends MultiModalEmbedding {
   }
 
   async getVisionModel() {
-    const { CLIPVisionModelWithProjection } = await lazyLoadTransformers();
+    const { CLIPVisionModelWithProjection } = await loadTransformers(
+      (transformer) => {
+        Settings.callbackManager.dispatchEvent(
+          "load-transformers",
+          {
+            transformer,
+          },
+          true,
+        );
+      },
+    );
     if (!this.visionModel) {
       this.visionModel = await CLIPVisionModelWithProjection.from_pretrained(
         this.modelType,
@@ -67,7 +102,17 @@ export class ClipEmbedding extends MultiModalEmbedding {
   }
 
   async getTextModel() {
-    const { CLIPTextModelWithProjection } = await lazyLoadTransformers();
+    const { CLIPTextModelWithProjection } = await loadTransformers(
+      (transformer) => {
+        Settings.callbackManager.dispatchEvent(
+          "load-transformers",
+          {
+            transformer,
+          },
+          true,
+        );
+      },
+    );
     if (!this.textModel) {
       this.textModel = await CLIPTextModelWithProjection.from_pretrained(
         this.modelType,

packages/llamaindex/src/embeddings/HuggingFaceEmbedding.ts

@@ -1,6 +1,7 @@
 import { HfInference } from "@huggingface/inference";
 import { BaseEmbedding } from "@llamaindex/core/embeddings";
-import { lazyLoadTransformers } from "../internal/deps/transformers.js";
+import { loadTransformers } from "@llamaindex/env";
+import { Settings } from "../Settings.js";
 
 export enum HuggingFaceEmbeddingModelType {
   XENOVA_ALL_MINILM_L6_V2 = "Xenova/all-MiniLM-L6-v2",
@@ -33,7 +34,15 @@ export class HuggingFaceEmbedding extends BaseEmbedding {
 
   async getExtractor() {
     if (!this.extractor) {
-      const { pipeline } = await lazyLoadTransformers();
+      const { pipeline } = await loadTransformers((transformer) => {
+        Settings.callbackManager.dispatchEvent(
+          "load-transformers",
+          {
+            transformer,
+          },
+          true,
+        );
+      });
       this.extractor = await pipeline("feature-extraction", this.modelType, {
         quantized: this.quantized,
       });

packages/llamaindex/src/embeddings/JinaAIEmbedding.ts

@@ -20,8 +20,9 @@ export type JinaEmbeddingRequest = {
   input: Array<{ text: string } | { url: string } | { bytes: string }>;
   model?: string;
   encoding_type?: EncodingType;
-  task_type?: TaskType;
+  task?: TaskType;
   dimensions?: number;
+  late_chunking?: boolean;
 };
 
 export type JinaEmbeddingResponse = {
@@ -44,9 +45,10 @@ export class JinaAIEmbedding extends MultiModalEmbedding {
   apiKey: string;
   model: string;
   baseURL: string;
-  taskType: TaskType | undefined;
+  task?: TaskType | undefined;
   encodingType?: EncodingType | undefined;
   dimensions?: number | undefined;
+  late_chunking?: boolean | undefined;
 
   async getTextEmbedding(text: string): Promise<number[]> {
     const result = await this.getJinaEmbedding({ input: [{ text }] });
@@ -87,8 +89,10 @@ export class JinaAIEmbedding extends MultiModalEmbedding {
     this.model = init?.model ?? "jina-embeddings-v3";
     this.baseURL = init?.baseURL ?? "https://api.jina.ai/v1/embeddings";
     init?.embedBatchSize && (this.embedBatchSize = init?.embedBatchSize);
-    this.taskType = init?.taskType;
+    this.task = init?.task;
     this.encodingType = init?.encodingType;
+    this.dimensions = init?.dimensions;
+    this.late_chunking = init?.late_chunking;
   }
 
   private async getImageInput(
@@ -125,8 +129,11 @@ export class JinaAIEmbedding extends MultiModalEmbedding {
       body: JSON.stringify({
         model: this.model,
         encoding_type: this.encodingType ?? "float",
-        ...(this.taskType && { task_type: this.taskType }),
+        ...(this.task && { task: this.task }),
         ...(this.dimensions !== undefined && { dimensions: this.dimensions }),
+        ...(this.late_chunking !== undefined && {
+          late_chunking: this.late_chunking,
+        }),
         ...params,
       }),
     });

packages/llamaindex/src/embeddings/index.ts

@@ -9,3 +9,5 @@ export * from "./MixedbreadAIEmbeddings.js";
 export { OllamaEmbedding } from "./OllamaEmbedding.js";
 export * from "./OpenAIEmbedding.js";
 export { TogetherEmbedding } from "./together.js";
+// ClipEmbedding might not work in non-node.js runtime, but it doesn't have side effects
+export { ClipEmbedding, ClipEmbeddingModelType } from "./ClipEmbedding.js";

packages/llamaindex/src/engines/chat/CondenseQuestionChatEngine.ts

@@ -1,11 +1,16 @@
+import {
+  BaseChatEngine,
+  type NonStreamingChatEngineParams,
+  type StreamingChatEngineParams,
+} from "@llamaindex/core/chat-engine";
 import type { ChatMessage, LLM } from "@llamaindex/core/llms";
 import { BaseMemory, ChatMemoryBuffer } from "@llamaindex/core/memory";
 import {
   type CondenseQuestionPrompt,
   defaultCondenseQuestionPrompt,
   type ModuleRecord,
-  PromptMixin,
 } from "@llamaindex/core/prompts";
+import type { BaseQueryEngine } from "@llamaindex/core/query-engine";
 import type { EngineResponse } from "@llamaindex/core/schema";
 import {
   extractText,
@@ -15,12 +20,6 @@ import {
 } from "@llamaindex/core/utils";
 import type { ServiceContext } from "../../ServiceContext.js";
 import { llmFromSettingsOrContext } from "../../Settings.js";
-import type { QueryEngine } from "../../types.js";
-import type {
-  ChatEngine,
-  ChatEngineParamsNonStreaming,
-  ChatEngineParamsStreaming,
-} from "./types.js";
 
 /**
  * CondenseQuestionChatEngine is used in conjunction with a Index (for example VectorStoreIndex).
@@ -32,18 +31,18 @@ import type {
  * underlying data. It performs less well when the chat messages are not questions about the
  * data, or are very referential to previous context.
  */
-
-export class CondenseQuestionChatEngine
-  extends PromptMixin
-  implements ChatEngine
-{
-  queryEngine: QueryEngine;
-  chatHistory: BaseMemory;
+export class CondenseQuestionChatEngine extends BaseChatEngine {
+  queryEngine: BaseQueryEngine;
+  memory: BaseMemory;
   llm: LLM;
   condenseMessagePrompt: CondenseQuestionPrompt;
 
+  get chatHistory() {
+    return this.memory.getMessages();
+  }
+
   constructor(init: {
-    queryEngine: QueryEngine;
+    queryEngine: BaseQueryEngine;
     chatHistory: ChatMessage[];
     serviceContext?: ServiceContext;
     condenseMessagePrompt?: CondenseQuestionPrompt;
@@ -51,7 +50,7 @@ export class CondenseQuestionChatEngine
     super();
 
     this.queryEngine = init.queryEngine;
-    this.chatHistory = new ChatMemoryBuffer({
+    this.memory = new ChatMemoryBuffer({
       chatHistory: init?.chatHistory,
     });
     this.llm = llmFromSettingsOrContext(init?.serviceContext);
@@ -78,9 +77,7 @@ export class CondenseQuestionChatEngine
   }
 
   private async condenseQuestion(chatHistory: BaseMemory, question: string) {
-    const chatHistoryStr = messagesToHistory(
-      await chatHistory.getMessages(question),
-    );
+    const chatHistoryStr = messagesToHistory(await chatHistory.getMessages());
 
     return this.llm.complete({
       prompt: this.condenseMessagePrompt.format({
@@ -90,23 +87,23 @@ export class CondenseQuestionChatEngine
     });
   }
 
+  chat(params: NonStreamingChatEngineParams): Promise<EngineResponse>;
   chat(
-    params: ChatEngineParamsStreaming,
+    params: StreamingChatEngineParams,
   ): Promise<AsyncIterable<EngineResponse>>;
-  chat(params: ChatEngineParamsNonStreaming): Promise<EngineResponse>;
   @wrapEventCaller
   async chat(
-    params: ChatEngineParamsStreaming | ChatEngineParamsNonStreaming,
+    params: NonStreamingChatEngineParams | StreamingChatEngineParams,
   ): Promise<EngineResponse | AsyncIterable<EngineResponse>> {
     const { message, stream } = params;
     const chatHistory = params.chatHistory
       ? new ChatMemoryBuffer({
           chatHistory:
             params.chatHistory instanceof BaseMemory
-              ? await params.chatHistory.getMessages(message)
+              ? await params.chatHistory.getMessages()
               : params.chatHistory,
         })
-      : this.chatHistory;
+      : this.memory;
 
     const condensedQuestion = (
       await this.condenseQuestion(chatHistory, extractText(message))
@@ -140,6 +137,6 @@ export class CondenseQuestionChatEngine
   }
 
   reset() {
-    this.chatHistory.reset();
+    this.memory.reset();
   }
 }