Release 0.6.9 (#1252)

Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>

.github/workflows/test.yml

@@ -13,8 +13,10 @@ concurrency:
   cancel-in-progress: true
 
 env:
-  POSTGRES_USER: runneradmin
   POSTGRES_HOST_AUTH_METHOD: trust
+  TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }}
+  TURBO_TEAM: ${{ vars.TURBO_TEAM }}
+  TURBO_REMOTE_ONLY: true
 
 jobs:
   e2e:
@@ -104,6 +106,7 @@ jobs:
           - nextjs-edge-runtime
           - nextjs-node-runtime
           - waku-query-engine
+          - llama-parse-browser
     runs-on: ubuntu-latest
     name: Build LlamaIndex Example (${{ matrix.packages }})
     steps:
@@ -145,6 +148,9 @@ jobs:
       - name: Pack @llamaindex/openai
         run: pnpm pack --pack-destination ${{ runner.temp }}
         working-directory: packages/llm/openai
+      - name: Pack @llamaindex/groq
+        run: pnpm pack --pack-destination ${{ runner.temp }}
+        working-directory: packages/llm/groq
       - name: Pack @llamaindex/core
         run: pnpm pack --pack-destination ${{ runner.temp }}
         working-directory: packages/core

README.md

@@ -167,11 +167,13 @@ export async function chatWithAgent(
       // ... adding your tools here
     ],
   });
-  const responseStream = await agent.chat({
-    stream: true,
-    message: question,
-    chatHistory: prevMessages,
-  });
+  const responseStream = await agent.chat(
+    {
+      message: question,
+      chatHistory: prevMessages,
+    },
+    true,
+  );
   const uiStream = createStreamableUI(<div>loading...</div>);
   responseStream
     .pipeTo(

apps/docs/CHANGELOG.md

@@ -1,5 +1,73 @@
 # docs
 
+## 0.0.78
+
+### Patch Changes
+
+- llamaindex@0.6.9
+
+## 0.0.77
+
+### Patch Changes
+
+- Updated dependencies [8b7fdba]
+  - llamaindex@0.6.8
+
+## 0.0.76
+
+### Patch Changes
+
+- Updated dependencies [23bcc37]
+  - llamaindex@0.6.7
+
+## 0.0.75
+
+### Patch Changes
+
+- Updated dependencies [d902cc3]
+- Updated dependencies [025ffe6]
+- Updated dependencies [a659574]
+  - llamaindex@0.6.6
+
+## 0.0.74
+
+### Patch Changes
+
+- Updated dependencies [e9714db]
+  - llamaindex@0.6.5
+
+## 0.0.73
+
+### Patch Changes
+
+- Updated dependencies [b48bcc3]
+  - llamaindex@0.6.4
+
+## 0.0.72
+
+### Patch Changes
+
+- Updated dependencies [2cd1383]
+- Updated dependencies [5c4badb]
+  - llamaindex@0.6.3
+
+## 0.0.71
+
+### Patch Changes
+
+- Updated dependencies [749b43a]
+  - llamaindex@0.6.2
+
+## 0.0.70
+
+### Patch Changes
+
+- Updated dependencies [fbd5e01]
+- Updated dependencies [6b70c54]
+- Updated dependencies [1a6137b]
+- Updated dependencies [85c2e19]
+  - llamaindex@0.6.1
+
 ## 0.0.69
 
 ### Patch Changes

apps/docs/docs/modules/agent/_category_.yml

@@ -1,2 +1,2 @@
 label: "Agents"
-position: 3
+position: 10

apps/docs/docs/modules/chat_engine.md

@@ -1,5 +1,5 @@
 ---
-sidebar_position: 4
+sidebar_position: 13
 ---
 
 # ChatEngine

apps/docs/docs/modules/data_index.md

@@ -1,5 +1,5 @@
 ---
-sidebar_position: 4
+sidebar_position: 12
 ---
 
 # Index
@@ -8,6 +8,7 @@ An index is the basic container and organization for your data. LlamaIndex.TS su
 
 - `VectorStoreIndex` - will send the top-k `Node`s to the LLM when generating a response. The default top-k is 2.
 - `SummaryIndex` - will send every `Node` in the index to the LLM in order to generate a response
+- `KeywordTableIndex` extracts and provides keywords from `Node`s to the LLM
 
 ```typescript
 import { Document, VectorStoreIndex } from "llamaindex";

apps/docs/docs/modules/data_loaders/index.mdx

@@ -6,6 +6,19 @@ import CodeSource2 from "!raw-loader!../../../../../examples/readers/src/custom-
 
 Before you can start indexing your documents, you need to load them into memory.
 
+All "basic" data loaders can be seen below, mapped to their respective filetypes in `SimpleDirectoryReader`. More loaders are shown in the sidebar on the left.
+Additionally the following loaders exist without separate documentation:
+
+- `AssemblyAIReader` transcribes audio using [AssemblyAI](https://www.assemblyai.com/).
+  - [AudioTranscriptReader](../../api/classes/AudioTranscriptReader.md): loads entire transcript as a single document.
+  - [AudioTranscriptParagraphsReader](../../api/classes/AudioTranscriptParagraphsReader.md): creates a document per paragraph.
+  - [AudioTranscriptSentencesReader](../../api/classes/AudioTranscriptSentencesReader.md): creates a document per sentence.
+  - [AudioSubtitlesReader](../../api/classes/AudioTranscriptParagraphsReader.md): creates a document containing the subtitles of a transcript.
+- [NotionReader](../../api/classes/NotionReader.md) loads [Notion](https://www.notion.so/) pages.
+- [SimpleMongoReader](../../api/classes/SimpleMongoReader) loads data from a [MongoDB](https://www.mongodb.com/).
+
+Check the [LlamaIndexTS Github](https://github.com/run-llama/LlamaIndexTS) for the most up to date overview of integrations.
+
 ## SimpleDirectoryReader
 
 [![Open in StackBlitz](https://developer.stackblitz.com/img/open_in_stackblitz.svg)](https://stackblitz.com/github/run-llama/LlamaIndexTS/tree/main/examples/readers?file=src/simple-directory-reader.ts&title=Simple%20Directory%20Reader)

apps/docs/docs/modules/data_stores/_category_.yml

@@ -0,0 +1,2 @@
+label: "Data Stores"
+position: 2

apps/docs/docs/modules/data_stores/chat_stores/_category_.yml

@@ -0,0 +1 @@
+label: "Chat Stores"

apps/docs/docs/modules/data_stores/chat_stores/index.md

@@ -0,0 +1,13 @@
+# Chat Stores
+
+Chat stores manage chat history by storing sequences of messages in a structured way, ensuring the order of messages is maintained for accurate conversation flow.
+
+## Available Chat Stores
+
+- [SimpleChatStore](../../../api/classes/SimpleChatStore.md): A simple in-memory chat store with support for [persisting](../index.md#local-storage) data to disk.
+
+Check the [LlamaIndexTS Github](https://github.com/run-llama/LlamaIndexTS) for the most up to date overview of integrations.
+
+## API Reference
+
+- [BaseChatStore](../../../api/interfaces/BaseChatStore.md)

apps/docs/docs/modules/data_stores/doc_stores/_category_.yml

@@ -0,0 +1,2 @@
+label: "Document Stores"
+position: 2

apps/docs/docs/modules/data_stores/doc_stores/index.md

@@ -0,0 +1,14 @@
+# Document Stores
+
+Document stores contain ingested document chunks, i.e. [Node](../../documents_and_nodes/index.md)s.
+
+## Available Document Stores
+
+- [SimpleDocumentStore](../../../api/classes/SimpleDocumentStore.md): A simple in-memory document store with support for [persisting](../index.md#local-storage) data to disk.
+- [PostgresDocumentStore](../../../api/classes/PostgresDocumentStore.md): A PostgreSQL document store, see [PostgreSQL Storage](../index.md#postgresql-storage).
+
+Check the [LlamaIndexTS Github](https://github.com/run-llama/LlamaIndexTS) for the most up to date overview of integrations.
+
+## API Reference
+
+- [BaseDocumentStore](../../../api/classes/BaseDocumentStore.md)

apps/docs/docs/modules/data_stores/index.md

@@ -1,7 +1,3 @@
----
-sidebar_position: 7
----
-
 # Storage
 
 Storage in LlamaIndex.TS works automatically once you've configured a
@@ -57,4 +53,4 @@ const index = await VectorStoreIndex.fromDocuments([document], {
 
 ## API Reference
 
-- [StorageContext](../api/interfaces/StorageContext.md)
+- [StorageContext](../../api/interfaces/StorageContext.md)

apps/docs/docs/modules/data_stores/index_stores/_category_.yml

@@ -0,0 +1,2 @@
+label: "Index Stores"
+position: 3

apps/docs/docs/modules/data_stores/index_stores/index.md

@@ -0,0 +1,14 @@
+# Index Stores
+
+Index stores are underlying storage components that contain metadata(i.e. information created when indexing) about the [index](../../data_index.md) itself.
+
+## Available Index Stores
+
+- [SimpleIndexStore](../../../api/classes/SimpleIndexStore.md): A simple in-memory index store with support for [persisting](../index.md#local-storage) data to disk.
+- [PostgresIndexStore](../../../api/classes/PostgresIndexStore.md): A PostgreSQL index store, , see [PostgreSQL Storage](../index.md#postgresql-storage).
+
+Check the [LlamaIndexTS Github](https://github.com/run-llama/LlamaIndexTS) for the most up to date overview of integrations.
+
+## API Reference
+
+- [BaseIndexStore](../../../api/classes/BaseIndexStore.md)

apps/docs/docs/modules/data_stores/kv_stores/_category_.yml

@@ -0,0 +1,2 @@
+label: "Key-Value Stores"
+position: 4

apps/docs/docs/modules/data_stores/kv_stores/index.md

@@ -0,0 +1,14 @@
+# Key-Value Stores
+
+Key-Value Stores represent underlying storage components used in [Document Stores](../doc_stores/index.md) and [Index Stores](../index_stores/index.md)
+
+## Available Key-Value Stores
+
+- [SimpleKVStore](../../../api/classes/SimpleKVStore.md): A simple Key-Value store with support of [persisting](../index.md#local-storage) data to disk.
+- [PostgresKVStore](../../../api/classes/PostgresKVStore.md): A PostgreSQL Key-Value store, see [PostgreSQL Storage](../index.md#postgresql-storage).
+
+Check the [LlamaIndexTS Github](https://github.com/run-llama/LlamaIndexTS) for the most up to date overview of integrations.
+
+## API Reference
+
+- [BaseKVStore](../../../api/classes/BaseKVStore.md)

apps/docs/docs/modules/data_stores/vector_stores/index.md

@@ -0,0 +1,22 @@
+# Vector Stores
+
+Vector stores save embedding vectors of your ingested document chunks.
+
+## Available Vector Stores
+
+Available Vector Stores are shown on the sidebar to the left. Additionally the following integrations exist without separate documentation:
+
+- [SimpleVectorStore](../../../api/classes/SimpleVectorStore.md): A simple in-memory vector store with optional [persistance](../index.md#local-storage) to disk.
+- [AstraDBVectorStore](../../../api/classes/AstraDBVectorStore.md): A cloud-native, scalable Database-as-a-Service built on Apache Cassandra, see [datastax.com](https://www.datastax.com/products/datastax-astra)
+- [ChromaVectorStore](../../../api/classes/ChromaVectorStore.md): An open-source vector database, focused on ease of use and performance, see [trychroma.com](https://www.trychroma.com/)
+- [MilvusVectorStore](../../../api/classes/MilvusVectorStore.md): An open-source, high-performance, highly scalable vector database, see [milvus.io](https://milvus.io/)
+- [MongoDBAtlasVectorSearch](../../../api/classes/MongoDBAtlasVectorSearch.md): A cloud-based vector search solution for MongoDB, see [mongodb.com](https://www.mongodb.com/products/platform/atlas-vector-search)
+- [PGVectorStore](../../../api/classes/PGVectorStore.md): An open-source vector store built on PostgreSQL, see [pgvector Github](https://github.com/pgvector/pgvector)
+- [PineconeVectorStore](../../../api/classes/PineconeVectorStore.md): A managed, cloud-native vector database, see [pinecone.io](https://www.pinecone.io/)
+- [WeaviateVectorStore](../../../api/classes/WeaviateVectorStore.md): An open-source, ai-native vector database, see [weaviate.io](https://weaviate.io/)
+
+Check the [LlamaIndexTS Github](https://github.com/run-llama/LlamaIndexTS) for the most up to date overview of integrations.
+
+## API Reference
+
+- [VectorStoreBase](../../../api/classes/VectorStoreBase.md)

apps/docs/docs/modules/data_stores/vector_stores/qdrant.md

@@ -1,5 +1,7 @@
 # Qdrant Vector Store
 
+[qdrant.tech](https://qdrant.tech/)
+
 To run this example, you need to have a Qdrant instance running. You can run it with Docker:
 
 ```bash
@@ -87,4 +89,4 @@ main().catch(console.error);
 
 ## API Reference
 
-- [QdrantVectorStore](../../api/classes/QdrantVectorStore.md)
+- [QdrantVectorStore](../../../api/classes/QdrantVectorStore.md)

apps/docs/docs/modules/documents_and_nodes/index.md

@@ -1,7 +1,3 @@
----
-sidebar_position: 1
----
-
 # Documents and Nodes
 
 `Document`s and `Node`s are the basic building blocks of any index. While the API for these objects is similar, `Document` objects represent entire files, while `Node`s are smaller pieces of that original document, that are suitable for an LLM and Q&A.

apps/docs/docs/modules/embeddings/_category_.yml

@@ -1,2 +1,2 @@
 label: "Embeddings"
-position: 3
+position: 6

apps/docs/docs/modules/embeddings/available_embeddings/mixedbreadai.md

@@ -7,7 +7,7 @@ To find out more about the latest features, updates, and available models, visit
 ## Table of Contents
 
 1. [Setup](#setup)
-2. [Usage with LlamaIndex](#integration-with-llamaindex)
+2. [Usage with LlamaIndex](#usage-with-llamaindex)
 3. [Embeddings with Custom Parameters](#embeddings-with-custom-parameters)
 
 ## Setup

apps/docs/docs/modules/embeddings/index.md

@@ -16,6 +16,16 @@ Settings.embedModel = new OpenAIEmbedding({
 
 For local embeddings, you can use the [HuggingFace](./available_embeddings/huggingface.md) embedding model.
 
+## Available Embeddings
+
+Most available embeddings are listed in the sidebar on the left.
+Additionally the following integrations exist without separate documentation:
+
+- [ClipEmbedding](../../api/classes/ClipEmbedding.md) using `@xenova/transformers`
+- [FireworksEmbedding](../../api/classes/FireworksEmbedding.md) see [fireworks.ai](https://fireworks.ai/)
+
+Check the [LlamaIndexTS Github](https://github.com/run-llama/LlamaIndexTS) for the most up to date overview of integrations.
+
 ## API Reference
 
 - [OpenAIEmbedding](../../api/classes/OpenAIEmbedding.md)

apps/docs/docs/modules/evaluation/_category_.yml

@@ -1,2 +1,2 @@
 label: "Evaluating"
-position: 3
+position: 9

apps/docs/docs/modules/ingestion_pipeline/_category_.yml

@@ -1,2 +1,2 @@
 label: "Ingestion Pipeline"
-position: 2
+position: 4

apps/docs/docs/modules/llms/_category_.yml

@@ -1,2 +1,2 @@
 label: "LLMs"
-position: 3
+position: 5

apps/docs/docs/modules/llms/available_llms/fireworks.md

@@ -1,6 +1,6 @@
 # Fireworks LLM
 
-Fireworks.ai focus on production use cases for open source LLMs, offering speed and quality.
+[Fireworks.ai](https://fireworks.ai/) focus on production use cases for open source LLMs, offering speed and quality.
 
 ## Usage
 

apps/docs/docs/modules/llms/index.md

@@ -1,7 +1,3 @@
----
-sidebar_position: 3
----
-
 # Large Language Models (LLMs)
 
 The LLM is responsible for reading text and generating natural language responses to queries. By default, LlamaIndex.TS uses `gpt-3.5-turbo`.
@@ -30,6 +26,15 @@ export AZURE_OPENAI_DEPLOYMENT="gpt-4" # or some other deployment name
 
 For local LLMs, currently we recommend the use of [Ollama](./available_llms/ollama.md) LLM.
 
+## Available LLMs
+
+Most available LLMs are listed in the sidebar on the left. Additionally the following integrations exist without separate documentation:
+
+- [HuggingFaceLLM](../../api/classes/HuggingFaceLLM.md) and [HuggingFaceInferenceAPI](../../api/classes/HuggingFaceInferenceAPI.md).
+- [ReplicateLLM](../../api/classes/ReplicateLLM.md) see [replicate.com](https://replicate.com/)
+
+Check the [LlamaIndexTS Github](https://github.com/run-llama/LlamaIndexTS) for the most up to date overview of integrations.
+
 ## API Reference
 
 - [OpenAI](../../api/classes/OpenAI.md)

apps/docs/docs/modules/node_parser.md

@@ -1,5 +1,5 @@
 ---
-sidebar_position: 4
+sidebar_position: 11
 ---
 
 # NodeParser

apps/docs/docs/modules/node_postprocessors/index.md

@@ -107,3 +107,4 @@ const filteredNodes = processor.postprocessNodes(nodes);
 ## API Reference
 
 - [SimilarityPostprocessor](../../api/classes/SimilarityPostprocessor.md)
+- [MetadataReplacementPostProcessor](../../api/classes/MetadataReplacementPostProcessor.md)

apps/docs/docs/modules/node_postprocessors/mixedbreadiai_reranker.md

@@ -7,7 +7,7 @@ To find out more about the latest features and updates, visit the [mixedbread.ai
 ## Table of Contents
 
 1. [Setup](#setup)
-2. [Usage with LlamaIndex](#integration-with-llamaindex)
+2. [Usage with LlamaIndex](#usage-with-llamaindex)
 3. [Simple Reranking Guide](#simple-reranking-guide)
 4. [Reranking with Objects](#reranking-with-objects)
 

apps/docs/docs/modules/prompt/_category_.yml

@@ -1,2 +1,2 @@
 label: "Prompts"
-position: 0
+position: 7

apps/docs/docs/modules/prompt/index.md

@@ -73,6 +73,5 @@ const response = await queryEngine.query({
 
 ## API Reference
 
-- [TextQaPrompt](../../api/type-aliases/TextQaPrompt.md)
 - [ResponseSynthesizer](../../api/classes/ResponseSynthesizer.md)
 - [CompactAndRefine](../../api/classes/CompactAndRefine.md)

apps/docs/docs/modules/query_engines/_category_.yml

@@ -1,2 +1,2 @@
 label: "Query Engines"
-position: 2
+position: 8

apps/docs/docs/modules/response_synthesizer.md

@@ -1,5 +1,5 @@
 ---
-sidebar_position: 6
+sidebar_position: 15
 ---
 
 # ResponseSynthesizer

apps/docs/docs/modules/retriever.md

@@ -1,5 +1,5 @@
 ---
-sidebar_position: 5
+sidebar_position: 14
 ---
 
 # Retriever

apps/docs/package.json

@@ -1,6 +1,6 @@
 {
   "name": "docs",
-  "version": "0.0.69",
+  "version": "0.0.78",
   "private": true,
   "scripts": {
     "docusaurus": "docusaurus",
@@ -37,7 +37,7 @@
     "docusaurus-plugin-typedoc": "1.0.5",
     "typedoc": "0.26.6",
     "typedoc-plugin-markdown": "4.2.6",
-    "typescript": "^5.5.4"
+    "typescript": "^5.6.2"
   },
   "browserslist": {
     "production": [

examples/anthropic/chat_interactive.ts

@@ -1,4 +1,4 @@
-import { Anthropic, SimpleChatEngine, SimpleChatHistory } from "llamaindex";
+import { Anthropic, ChatMemoryBuffer, SimpleChatEngine } from "llamaindex";
 import { stdin as input, stdout as output } from "node:process";
 import readline from "node:readline/promises";
 
@@ -8,8 +8,8 @@ import readline from "node:readline/promises";
     model: "claude-3-opus",
   });
   // chatHistory will store all the messages in the conversation
-  const chatHistory = new SimpleChatHistory({
-    messages: [
+  const chatHistory = new ChatMemoryBuffer({
+    chatHistory: [
       {
         content: "You want to talk in rhymes.",
         role: "system",
@@ -18,7 +18,7 @@ import readline from "node:readline/promises";
   });
   const chatEngine = new SimpleChatEngine({
     llm,
-    chatHistory,
+    memory: chatHistory,
   });
   const rl = readline.createInterface({ input, output });
 

examples/chatHistory.ts

@@ -2,10 +2,10 @@ import { stdin as input, stdout as output } from "node:process";
 import readline from "node:readline/promises";
 
 import {
+  ChatSummaryMemoryBuffer,
   OpenAI,
   Settings,
   SimpleChatEngine,
-  SummaryChatHistory,
 } from "llamaindex";
 
 if (process.env.NODE_ENV === "development") {
@@ -18,7 +18,7 @@ async function main() {
   // Set maxTokens to 75% of the context window size of 4096
   // This will trigger the summarizer once the chat history reaches 25% of the context window size (1024 tokens)
   const llm = new OpenAI({ model: "gpt-3.5-turbo", maxTokens: 4096 * 0.75 });
-  const chatHistory = new SummaryChatHistory({ llm });
+  const chatHistory = new ChatSummaryMemoryBuffer({ llm });
   const chatEngine = new SimpleChatEngine({ llm });
   const rl = readline.createInterface({ input, output });
 

examples/groq.ts

@@ -1,12 +1,23 @@
 import fs from "node:fs/promises";
 
-import { Document, Groq, Settings, VectorStoreIndex } from "llamaindex";
+import {
+  Document,
+  Groq,
+  HuggingFaceEmbedding,
+  Settings,
+  VectorStoreIndex,
+} from "llamaindex";
 
 // Update llm to use Groq
 Settings.llm = new Groq({
   apiKey: process.env.GROQ_API_KEY,
 });
 
+// Use HuggingFace for embeddings
+Settings.embedModel = new HuggingFaceEmbedding({
+  modelType: "Xenova/all-mpnet-base-v2",
+});
+
 async function main() {
   // Load essay from abramov.txt in Node
   const path = "node_modules/llamaindex/examples/abramov.txt";

examples/lowlevel.ts

@@ -1,7 +1,7 @@
 import {
   Document,
+  getResponseSynthesizer,
   NodeWithScore,
-  ResponseSynthesizer,
   SentenceSplitter,
   TextNode,
 } from "llamaindex";
@@ -14,7 +14,7 @@ import {
 
   console.log(nodes);
 
-  const responseSynthesizer = new ResponseSynthesizer();
+  const responseSynthesizer = getResponseSynthesizer("compact");
 
   const nodesWithScore: NodeWithScore[] = [
     {
@@ -30,7 +30,7 @@ import {
   const stream = await responseSynthesizer.synthesize(
     {
       query: "What age am I?",
-      nodesWithScore,
+      nodes: nodesWithScore,
     },
     true,
   );

examples/multimodal/context.ts

@@ -1,4 +1,5 @@
 // call pnpm tsx multimodal/load.ts first to init the storage
+import { extractText } from "@llamaindex/core/utils";
 import {
   ContextChatEngine,
   NodeWithScore,
@@ -25,8 +26,9 @@ Settings.callbackManager.on("retrieve-end", (event) => {
   const textNodes = nodes.filter(
     (node: NodeWithScore) => node.node.type === ObjectType.TEXT,
   );
+  const text = extractText(query);
   console.log(
-    `Retrieved ${textNodes.length} text nodes and ${imageNodes.length} image nodes for query: ${query}`,
+    `Retrieved ${textNodes.length} text nodes and ${imageNodes.length} image nodes for query: ${text}`,
   );
 });
 

examples/multimodal/rag.ts

@@ -1,5 +1,6 @@
+import { extractText } from "@llamaindex/core/utils";
 import {
-  MultiModalResponseSynthesizer,
+  getResponseSynthesizer,
   OpenAI,
   Settings,
   VectorStoreIndex,
@@ -16,7 +17,8 @@ Settings.llm = new OpenAI({ model: "gpt-4-turbo", maxTokens: 512 });
 // Update callbackManager
 Settings.callbackManager.on("retrieve-end", (event) => {
   const { nodes, query } = event.detail;
-  console.log(`Retrieved ${nodes.length} nodes for query: ${query}`);
+  const text = extractText(query);
+  console.log(`Retrieved ${nodes.length} nodes for query: ${text}`);
 });
 
 async function main() {
@@ -27,7 +29,7 @@ async function main() {
   });
 
   const queryEngine = index.asQueryEngine({
-    responseSynthesizer: new MultiModalResponseSynthesizer(),
+    responseSynthesizer: getResponseSynthesizer("multi_modal"),
     retriever: index.asRetriever({ topK: { TEXT: 3, IMAGE: 1 } }),
   });
   const stream = await queryEngine.query({

examples/package.json

@@ -21,7 +21,7 @@
   "devDependencies": {
     "@types/node": "^22.5.1",
     "tsx": "^4.19.0",
-    "typescript": "^5.5.4"
+    "typescript": "^5.6.2"
   },
   "scripts": {
     "lint": "eslint ."

examples/pg-vector-store/load-docs.ts

@@ -40,7 +40,11 @@ async function main(args: any) {
     const rdr = new SimpleDirectoryReader(callback);
     const docs = await rdr.loadData({ directoryPath: sourceDir });
 
-    const pgvs = new PGVectorStore();
+    const pgvs = new PGVectorStore({
+      clientConfig: {
+        connectionString: process.env.PG_CONNECTION_STRING,
+      },
+    });
     pgvs.setCollection(sourceDir);
     await pgvs.clearCollection();
 

examples/pg-vector-store/query.ts

@@ -7,7 +7,11 @@ async function main() {
   });
 
   try {
-    const pgvs = new PGVectorStore();
+    const pgvs = new PGVectorStore({
+      clientConfig: {
+        connectionString: process.env.PG_CONNECTION_STRING,
+      },
+    });
     // Optional - set your collection name, default is no filter on this field.
     // pgvs.setCollection();
 

examples/prompts/promptMixin.ts

@@ -1,8 +1,7 @@
 import {
   Document,
+  getResponseSynthesizer,
   PromptTemplate,
-  ResponseSynthesizer,
-  TreeSummarize,
   TreeSummarizePrompt,
   VectorStoreIndex,
 } from "llamaindex";
@@ -27,9 +26,7 @@ async function main() {
 
   const query = "The quick brown fox jumps over the lazy dog";
 
-  const responseSynthesizer = new ResponseSynthesizer({
-    responseBuilder: new TreeSummarize(),
-  });
+  const responseSynthesizer = getResponseSynthesizer("tree_summarize");
 
   const queryEngine = index.asQueryEngine({
     responseSynthesizer,

examples/readers/package.json

@@ -23,6 +23,6 @@
   "devDependencies": {
     "@types/node": "^22.5.1",
     "tsx": "^4.19.0",
-    "typescript": "^5.5.4"
+    "typescript": "^5.6.2"
   }
 }

examples/readers/src/csv.ts

@@ -1,8 +1,7 @@
 import {
-  CompactAndRefine,
+  getResponseSynthesizer,
   OpenAI,
   PromptTemplate,
-  ResponseSynthesizer,
   Settings,
   VectorStoreIndex,
 } from "llamaindex";
@@ -29,8 +28,8 @@ Given the CSV file, generate me Typescript code to answer the question: {query}.
 `,
   });
 
-  const responseSynthesizer = new ResponseSynthesizer({
-    responseBuilder: new CompactAndRefine(undefined, csvPrompt),
+  const responseSynthesizer = getResponseSynthesizer("compact", {
+    textQATemplate: csvPrompt,
   });
 
   const queryEngine = index.asQueryEngine({ responseSynthesizer });

examples/readers/src/llamaparse-json.ts

@@ -1,3 +1,4 @@
+import { createMessageContent } from "@llamaindex/core/utils";
 import {
   Document,
   ImageNode,
@@ -6,7 +7,6 @@ import {
   PromptTemplate,
   VectorStoreIndex,
 } from "llamaindex";
-import { createMessageContent } from "llamaindex/synthesizers/utils";
 
 const reader = new LlamaParseReader();
 async function main() {

examples/vectorIndexAnthropic.ts

@@ -2,12 +2,10 @@ import fs from "node:fs/promises";
 
 import {
   Anthropic,
-  CompactAndRefine,
   Document,
-  ResponseSynthesizer,
   Settings,
   VectorStoreIndex,
-  anthropicTextQaPrompt,
+  getResponseSynthesizer,
 } from "llamaindex";
 
 // Update llm to use Anthropic
@@ -23,9 +21,7 @@ async function main() {
   const document = new Document({ text: essay, id_: path });
 
   // Split text and create embeddings. Store them in a VectorStoreIndex
-  const responseSynthesizer = new ResponseSynthesizer({
-    responseBuilder: new CompactAndRefine(undefined, anthropicTextQaPrompt),
-  });
+  const responseSynthesizer = getResponseSynthesizer("compact");
 
   const index = await VectorStoreIndex.fromDocuments([document]);
 

examples/vectorIndexFromVectorStore.ts

@@ -1,11 +1,10 @@
 import {
+  getResponseSynthesizer,
   OpenAI,
   OpenAIEmbedding,
-  ResponseSynthesizer,
   RetrieverQueryEngine,
   Settings,
   TextNode,
-  TreeSummarize,
   VectorIndexRetriever,
   VectorStore,
   VectorStoreIndex,
@@ -165,10 +164,7 @@ async function main() {
       similarityTopK: 500,
     });
 
-    const responseSynthesizer = new ResponseSynthesizer({
-      responseBuilder: new TreeSummarize(),
-    });
-
+    const responseSynthesizer = getResponseSynthesizer("tree_summarize");
     return new RetrieverQueryEngine(retriever, responseSynthesizer, {
       filter,
     });

package.json

@@ -2,9 +2,9 @@
   "name": "@llamaindex/monorepo",
   "private": true,
   "scripts": {
-    "build": "turbo run build --filter=\"!docs\" --filter=\"!*-test\" --filter=\"!*-example\"",
-    "build:release": "turbo run build lint test --filter=\"!docs\" --filter=\"!*-test\" --filter=\"!*-example\"",
-    "dev": "turbo run dev",
+    "build": "turbo run build",
+    "build:release": "turbo run build --filter=\"./packages/*\"",
+    "dev": "turbo run dev --filter=\"./packages/*\"",
     "format": "prettier --ignore-unknown --cache --check .",
     "format:write": "prettier --ignore-unknown --write .",
     "lint": "turbo run lint",
@@ -31,7 +31,7 @@
     "prettier": "^3.3.3",
     "prettier-plugin-organize-imports": "^4.0.0",
     "turbo": "^2.1.0",
-    "typescript": "^5.5.4"
+    "typescript": "^5.6.2"
   },
   "packageManager": "pnpm@9.5.0",
   "pnpm": {

packages/autotool/CHANGELOG.md

@@ -1,5 +1,77 @@
 # @llamaindex/autotool
 
+## 3.0.9
+
+### Patch Changes
+
+- llamaindex@0.6.9
+
+## 3.0.8
+
+### Patch Changes
+
+- Updated dependencies [8b7fdba]
+  - llamaindex@0.6.8
+
+## 3.0.7
+
+### Patch Changes
+
+- Updated dependencies [23bcc37]
+  - llamaindex@0.6.7
+
+## 3.0.6
+
+### Patch Changes
+
+- Updated dependencies [d902cc3]
+- Updated dependencies [025ffe6]
+- Updated dependencies [a659574]
+  - llamaindex@0.6.6
+
+## 3.0.5
+
+### Patch Changes
+
+- Updated dependencies [e9714db]
+  - llamaindex@0.6.5
+
+## 3.0.4
+
+### Patch Changes
+
+- Updated dependencies [b48bcc3]
+  - llamaindex@0.6.4
+
+## 3.0.3
+
+### Patch Changes
+
+- Updated dependencies [2cd1383]
+- Updated dependencies [5c4badb]
+  - llamaindex@0.6.3
+
+## 3.0.2
+
+### Patch Changes
+
+- Updated dependencies [749b43a]
+  - llamaindex@0.6.2
+
+## 3.0.1
+
+### Patch Changes
+
+- 1a6137b: feat: experimental support for browser
+
+  If you see bundler issue in next.js edge runtime, please bump to `next@14` latest version.
+
+- Updated dependencies [fbd5e01]
+- Updated dependencies [6b70c54]
+- Updated dependencies [1a6137b]
+- Updated dependencies [85c2e19]
+  - llamaindex@0.6.1
+
 ## 3.0.0
 
 ### Patch Changes

packages/autotool/examples/01_node/CHANGELOG.md

@@ -1,5 +1,82 @@
 # @llamaindex/autotool-01-node-example
 
+## 0.0.18
+
+### Patch Changes
+
+- llamaindex@0.6.9
+- @llamaindex/autotool@3.0.9
+
+## 0.0.17
+
+### Patch Changes
+
+- Updated dependencies [8b7fdba]
+  - llamaindex@0.6.8
+  - @llamaindex/autotool@3.0.8
+
+## 0.0.16
+
+### Patch Changes
+
+- Updated dependencies [23bcc37]
+  - llamaindex@0.6.7
+  - @llamaindex/autotool@3.0.7
+
+## 0.0.15
+
+### Patch Changes
+
+- Updated dependencies [d902cc3]
+- Updated dependencies [025ffe6]
+- Updated dependencies [a659574]
+  - llamaindex@0.6.6
+  - @llamaindex/autotool@3.0.6
+
+## 0.0.14
+
+### Patch Changes
+
+- Updated dependencies [e9714db]
+  - llamaindex@0.6.5
+  - @llamaindex/autotool@3.0.5
+
+## 0.0.13
+
+### Patch Changes
+
+- Updated dependencies [b48bcc3]
+  - llamaindex@0.6.4
+  - @llamaindex/autotool@3.0.4
+
+## 0.0.12
+
+### Patch Changes
+
+- Updated dependencies [2cd1383]
+- Updated dependencies [5c4badb]
+  - llamaindex@0.6.3
+  - @llamaindex/autotool@3.0.3
+
+## 0.0.11
+
+### Patch Changes
+
+- Updated dependencies [749b43a]
+  - llamaindex@0.6.2
+  - @llamaindex/autotool@3.0.2
+
+## 0.0.10
+
+### Patch Changes
+
+- Updated dependencies [fbd5e01]
+- Updated dependencies [6b70c54]
+- Updated dependencies [1a6137b]
+- Updated dependencies [85c2e19]
+  - llamaindex@0.6.1
+  - @llamaindex/autotool@3.0.1
+
 ## 0.0.9
 
 ### Patch Changes

packages/autotool/examples/01_node/package.json

@@ -13,5 +13,5 @@
   "scripts": {
     "start": "node --import tsx --import @llamaindex/autotool/node ./src/index.ts"
   },
-  "version": "0.0.9"
+  "version": "0.0.18"
 }

packages/autotool/examples/02_nextjs/CHANGELOG.md

@@ -1,5 +1,82 @@
 # @llamaindex/autotool-02-next-example
 
+## 0.1.62
+
+### Patch Changes
+
+- llamaindex@0.6.9
+- @llamaindex/autotool@3.0.9
+
+## 0.1.61
+
+### Patch Changes
+
+- Updated dependencies [8b7fdba]
+  - llamaindex@0.6.8
+  - @llamaindex/autotool@3.0.8
+
+## 0.1.60
+
+### Patch Changes
+
+- Updated dependencies [23bcc37]
+  - llamaindex@0.6.7
+  - @llamaindex/autotool@3.0.7
+
+## 0.1.59
+
+### Patch Changes
+
+- Updated dependencies [d902cc3]
+- Updated dependencies [025ffe6]
+- Updated dependencies [a659574]
+  - llamaindex@0.6.6
+  - @llamaindex/autotool@3.0.6
+
+## 0.1.58
+
+### Patch Changes
+
+- Updated dependencies [e9714db]
+  - llamaindex@0.6.5
+  - @llamaindex/autotool@3.0.5
+
+## 0.1.57
+
+### Patch Changes
+
+- Updated dependencies [b48bcc3]
+  - llamaindex@0.6.4
+  - @llamaindex/autotool@3.0.4
+
+## 0.1.56
+
+### Patch Changes
+
+- Updated dependencies [2cd1383]
+- Updated dependencies [5c4badb]
+  - llamaindex@0.6.3
+  - @llamaindex/autotool@3.0.3
+
+## 0.1.55
+
+### Patch Changes
+
+- Updated dependencies [749b43a]
+  - llamaindex@0.6.2
+  - @llamaindex/autotool@3.0.2
+
+## 0.1.54
+
+### Patch Changes
+
+- Updated dependencies [fbd5e01]
+- Updated dependencies [6b70c54]
+- Updated dependencies [1a6137b]
+- Updated dependencies [85c2e19]
+  - llamaindex@0.6.1
+  - @llamaindex/autotool@3.0.1
+
 ## 0.1.53
 
 ### Patch Changes

packages/autotool/examples/02_nextjs/actions.ts

@@ -5,9 +5,9 @@ import { runWithStreamableUI } from "@/context";
 import "@/tool";
 import { convertTools } from "@llamaindex/autotool";
 import { createStreamableUI } from "ai/rsc";
-import type { JSX } from "react";
+import type { ReactNode } from "react";
 
-export async function chatWithAI(message: string): Promise<JSX.Element> {
+export async function chatWithAI(message: string): Promise<ReactNode> {
   const agent = new OpenAIAgent({
     tools: convertTools("llamaindex"),
   });
@@ -25,7 +25,7 @@ export async function chatWithAI(message: string): Promise<JSX.Element> {
               uiStream.append("\n");
             },
             write: async (message) => {
-              uiStream.append(message.response.delta);
+              uiStream.append(message.response);
             },
             close: () => {
               uiStream.done();

packages/autotool/examples/02_nextjs/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/autotool-02-next-example",
   "private": true,
-  "version": "0.1.53",
+  "version": "0.1.62",
   "scripts": {
     "dev": "next dev",
     "build": "next build",
@@ -32,6 +32,6 @@
     "cross-env": "^7.0.3",
     "postcss": "^8.4.41",
     "tailwindcss": "^3.4.10",
-    "typescript": "^5.5.4"
+    "typescript": "^5.6.2"
   }
 }

packages/autotool/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/autotool",
   "type": "module",
-  "version": "3.0.0",
+  "version": "3.0.9",
   "description": "auto transpile your JS function to LLM Agent compatible",
   "files": [
     "dist",
@@ -51,7 +51,7 @@
     "unplugin": "^1.12.2"
   },
   "peerDependencies": {
-    "llamaindex": "^0.6.0",
+    "llamaindex": "workspace:*",
     "openai": "^4",
     "typescript": "^4"
   },
@@ -72,10 +72,10 @@
     "@types/node": "^22.5.1",
     "bunchee": "5.3.2",
     "llamaindex": "workspace:*",
-    "next": "14.2.7",
+    "next": "14.2.11",
     "rollup": "^4.21.2",
     "tsx": "^4.19.0",
-    "typescript": "^5.5.4",
+    "typescript": "^5.6.2",
     "vitest": "^2.0.5",
     "webpack": "^5.94.0"
   }

packages/autotool/src/compiler.ts

@@ -9,7 +9,7 @@ import td from "typedoc";
 import type { SourceMapCompact } from "unplugin";
 import type { InfoString } from "./internal";
 
-export const isToolFile = (url: string) => /tool\.[jt]sx?$/.test(url);
+export const isToolFile = (url: string) => /\.tool\.[jt]sx?$/.test(url);
 export const isJSorTS = (url: string) => /\.m?[jt]sx?$/.test(url);
 
 async function parseRoot(entryPoint: string) {
@@ -28,7 +28,7 @@ async function parseRoot(entryPoint: string) {
   if (project) {
     return app.serializer.projectToObject(project, process.cwd());
   }
-  throw new Error("Failed to parse root");
+  throw new Error(`Failed to parse root ${entryPoint}`);
 }
 
 export async function transformAutoTool(

packages/cloud/CHANGELOG.md

@@ -1,5 +1,47 @@
 # @llamaindex/cloud
 
+## 0.2.8
+
+### Patch Changes
+
+- ac41ed3: feat: bump cloud sdk version
+
+## 0.2.7
+
+### Patch Changes
+
+- fb36eff: fix: backport for node.js 18
+
+  There could have one missing API in the node.js 18, so we need to backport it to make it work.
+
+- d24d3d1: fix: print warning when llama parse reader has error
+- Updated dependencies [2cd1383]
+  - @llamaindex/core@0.2.3
+
+## 0.2.6
+
+### Patch Changes
+
+- b42adeb: fix: get job result in llama parse reader
+- Updated dependencies [749b43a]
+  - @llamaindex/core@0.2.2
+
+## 0.2.5
+
+### Patch Changes
+
+- 85c2e19: feat: `@llamaindex/cloud` package update
+
+  - Bump to latest openapi schema
+  - Move LlamaParse class from llamaindex, this will allow you use llamaparse in more non-node.js environment
+
+- Updated dependencies [ac07e3c]
+- Updated dependencies [70ccb4a]
+- Updated dependencies [1a6137b]
+- Updated dependencies [ac07e3c]
+  - @llamaindex/core@0.2.1
+  - @llamaindex/env@0.1.11
+
 ## 0.2.4
 
 ### Patch Changes

packages/cloud/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/cloud",
-  "version": "0.2.4",
+  "version": "0.2.8",
   "type": "module",
   "license": "MIT",
   "scripts": {
@@ -26,6 +26,20 @@
         "types": "./dist/api.d.ts",
         "default": "./dist/api.js"
       }
+    },
+    "./reader": {
+      "require": {
+        "types": "./dist/reader.d.cts",
+        "default": "./dist/reader.cjs"
+      },
+      "import": {
+        "types": "./dist/reader.d.ts",
+        "default": "./dist/reader.js"
+      },
+      "default": {
+        "types": "./dist/reader.d.ts",
+        "default": "./dist/reader.js"
+      }
     }
   },
   "repository": {
@@ -36,6 +50,15 @@
   "devDependencies": {
     "@hey-api/client-fetch": "^0.2.4",
     "@hey-api/openapi-ts": "^0.53.0",
+    "@llamaindex/core": "workspace:^0.2.3",
+    "@llamaindex/env": "workspace:^0.1.11",
     "bunchee": "5.3.2"
+  },
+  "peerDependencies": {
+    "@llamaindex/core": "workspace:^0.2.3",
+    "@llamaindex/env": "workspace:^0.1.11"
+  },
+  "dependencies": {
+    "magic-bytes.js": "^1.10.0"
   }
 }

packages/cloud/src/reader.ts

@@ -1,92 +1,17 @@
+import { createClient, createConfig, type Client } from "@hey-api/client-fetch";
 import { Document, FileReader } from "@llamaindex/core/schema";
 import { fs, getEnv } from "@llamaindex/env";
 import { filetypeinfo } from "magic-bytes.js";
+import {
+  ParsingService,
+  type Body_upload_file_api_v1_parsing_upload_post,
+  type ParserLanguages,
+} from "./api";
+import { sleep } from "./utils";
+
+export type Language = ParserLanguages;
 
 export type ResultType = "text" | "markdown" | "json";
-export type Language =
-  | "abq"
-  | "ady"
-  | "af"
-  | "ang"
-  | "ar"
-  | "as"
-  | "ava"
-  | "az"
-  | "be"
-  | "bg"
-  | "bh"
-  | "bho"
-  | "bn"
-  | "bs"
-  | "ch_sim"
-  | "ch_tra"
-  | "che"
-  | "cs"
-  | "cy"
-  | "da"
-  | "dar"
-  | "de"
-  | "en"
-  | "es"
-  | "et"
-  | "fa"
-  | "fr"
-  | "ga"
-  | "gom"
-  | "hi"
-  | "hr"
-  | "hu"
-  | "id"
-  | "inh"
-  | "is"
-  | "it"
-  | "ja"
-  | "kbd"
-  | "kn"
-  | "ko"
-  | "ku"
-  | "la"
-  | "lbe"
-  | "lez"
-  | "lt"
-  | "lv"
-  | "mah"
-  | "mai"
-  | "mi"
-  | "mn"
-  | "mr"
-  | "ms"
-  | "mt"
-  | "ne"
-  | "new"
-  | "nl"
-  | "no"
-  | "oc"
-  | "pi"
-  | "pl"
-  | "pt"
-  | "ro"
-  | "ru"
-  | "rs_cyrillic"
-  | "rs_latin"
-  | "sck"
-  | "sk"
-  | "sl"
-  | "sq"
-  | "sv"
-  | "sw"
-  | "ta"
-  | "tab"
-  | "te"
-  | "th"
-  | "tjk"
-  | "tl"
-  | "tr"
-  | "ug"
-  | "uk"
-  | "ur"
-  | "uz"
-  | "vi";
 
 const SUPPORT_FILE_EXT: string[] = [
   ".pdf",
@@ -181,6 +106,15 @@ const SUPPORT_FILE_EXT: string[] = [
   ".tsv",
 ];
 
+//todo: should move into @llamaindex/env
+type WriteStream = {
+  write: (text: string) => void;
+};
+
+// Do not modify this variable or cause type errors
+// eslint-disable-next-line no-var
+var process: any;
+
 /**
  * Represents a reader for parsing files using the LlamaParse API.
  * See https://github.com/run-llama/llama_parse
@@ -188,8 +122,8 @@ const SUPPORT_FILE_EXT: string[] = [
 export class LlamaParseReader extends FileReader {
   // The API key for the LlamaParse API. Can be set as an environment variable: LLAMA_CLOUD_API_KEY
   apiKey: string;
-  // The base URL of the Llama Parsing API.
-  baseUrl: string = "https://api.cloud.llamaindex.ai/api/parsing";
+  // The base URL of the Llama Cloud Platform.
+  baseUrl: string = "https://api.cloud.llamaindex.ai";
   // The result type for the parser.
   resultType: ResultType = "text";
   // The interval in seconds to check if the parsing is done.
@@ -199,7 +133,7 @@ export class LlamaParseReader extends FileReader {
   // Whether to print the progress of the parsing.
   verbose = true;
   // The language of the text to parse.
-  language: Language = "en";
+  language: ParserLanguages[] = ["en"];
   // The parsing instruction for the parser. Backend default is an empty string.
   parsingInstruction?: string | undefined;
   // Wether to ignore diagonal text (when the text rotation in degrees is not 0, 90, 180 or 270, so not a horizontal or vertical text). Backend default is false.
@@ -236,22 +170,48 @@ export class LlamaParseReader extends FileReader {
   vendorMultimodalModelName?: string | undefined;
   // The API key for the multimodal API. Can also be set as an env variable: LLAMA_CLOUD_VENDOR_MULTIMODAL_API_KEY
   vendorMultimodalApiKey?: string | undefined;
+
+  webhookUrl?: string | undefined;
+  premiumMode?: boolean | undefined;
+  takeScreenshot?: boolean | undefined;
+  disableOcr?: boolean | undefined;
+  disableReconstruction?: boolean | undefined;
+  inputS3Path?: string | undefined;
+  outputS3PathPrefix?: string | undefined;
+
   // numWorkers is implemented in SimpleDirectoryReader
+  stdout?: WriteStream | undefined;
+
+  readonly #client: Client;
 
   constructor(
-    params: Partial<LlamaParseReader> & {
+    params: Partial<Omit<LlamaParseReader, "language" | "apiKey">> & {
+      language?: ParserLanguages | ParserLanguages[] | undefined;
       apiKey?: string | undefined;
     } = {},
   ) {
     super();
     Object.assign(this, params);
-    params.apiKey = params.apiKey ?? getEnv("LLAMA_CLOUD_API_KEY");
-    if (!params.apiKey) {
+    this.language = Array.isArray(this.language)
+      ? this.language
+      : [this.language];
+    this.stdout =
+      (params.stdout ?? typeof process !== "undefined")
+        ? process!.stdout
+        : undefined;
+    const apiKey = params.apiKey ?? getEnv("LLAMA_CLOUD_API_KEY");
+    if (!apiKey) {
       throw new Error(
         "API Key is required for LlamaParseReader. Please pass the apiKey parameter or set the LLAMA_CLOUD_API_KEY environment variable.",
       );
     }
-    this.apiKey = params.apiKey;
+    this.apiKey = apiKey;
+    if (this.baseUrl.endsWith("/")) {
+      this.baseUrl = this.baseUrl.slice(0, -"/".length);
+    }
+    if (this.baseUrl.endsWith("/api/parsing")) {
+      this.baseUrl = this.baseUrl.slice(0, -"/api/parsing".length);
+    }
 
     if (params.gpt4oMode) {
       params.gpt4oApiKey =
@@ -266,126 +226,145 @@ export class LlamaParseReader extends FileReader {
 
       this.vendorMultimodalApiKey = params.vendorMultimodalApiKey;
     }
+
+    this.#client = createClient(
+      createConfig({
+        headers: {
+          Authorization: `Bearer ${this.apiKey}`,
+        },
+        baseUrl: this.baseUrl,
+      }),
+    );
   }
 
   // Create a job for the LlamaParse API
-  private async createJob(
-    data: Uint8Array,
-    fileName?: string,
-  ): Promise<string> {
+  private async createJob(data: Uint8Array): Promise<string> {
     // Load data, set the mime type
-    const { mime, extension } = await LlamaParseReader.getMimeType(data);
+    const { mime } = await LlamaParseReader.getMimeType(data);
 
     if (this.verbose) {
-      const name = fileName ? fileName : extension;
-      console.log(`Starting load for ${name} file`);
+      console.log("Started uploading the file");
     }
 
-    const body = new FormData();
-    body.set("file", new Blob([data], { type: mime }), fileName);
-
-    const LlamaParseBodyParams = {
+    const body = {
+      file: new Blob([data], {
+        type: mime,
+      }),
       language: this.language,
       parsing_instruction: this.parsingInstruction,
-      skip_diagonal_text: this.skipDiagonalText?.toString(),
-      invalidate_cache: this.invalidateCache?.toString(),
-      do_not_cache: this.doNotCache?.toString(),
-      fast_mode: this.fastMode?.toString(),
-      do_not_unroll_columns: this.doNotUnrollColumns?.toString(),
+      skip_diagonal_text: this.skipDiagonalText,
+      invalidate_cache: this.invalidateCache,
+      do_not_cache: this.doNotCache,
+      fast_mode: this.fastMode,
+      do_not_unroll_columns: this.doNotUnrollColumns,
       page_separator: this.pageSeparator,
       page_prefix: this.pagePrefix,
       page_suffix: this.pageSuffix,
-      gpt4o_mode: this.gpt4oMode?.toString(),
+      gpt4o_mode: this.gpt4oMode,
       gpt4o_api_key: this.gpt4oApiKey,
       bounding_box: this.boundingBox,
       target_pages: this.targetPages,
-      use_vendor_multimodal_model: this.useVendorMultimodalModel?.toString(),
+      use_vendor_multimodal_model: this.useVendorMultimodalModel,
       vendor_multimodal_model_name: this.vendorMultimodalModelName,
       vendor_multimodal_api_key: this.vendorMultimodalApiKey,
-    };
+      premium_mode: this.premiumMode,
+      webhook_url: this.webhookUrl,
+      take_screenshot: this.takeScreenshot,
+      disable_ocr: this.disableOcr,
+      disable_reconstruction: this.disableReconstruction,
+      input_s3_path: this.inputS3Path,
+      output_s3_path_prefix: this.outputS3PathPrefix,
+    } satisfies {
+      [Key in keyof Body_upload_file_api_v1_parsing_upload_post]-?:
+        | Body_upload_file_api_v1_parsing_upload_post[Key]
+        | undefined;
+    } as unknown as Body_upload_file_api_v1_parsing_upload_post;
 
-    // Filter out params with invalid values that would cause issues on the backend.
-    const filteredParams = this.filterSpecificParams(LlamaParseBodyParams, [
-      "page_separator",
-      "page_prefix",
-      "page_suffix",
-      "bounding_box",
-      "target_pages",
-    ]);
-
-    // Appends body with any defined LlamaParseBodyParams
-    Object.entries(filteredParams).forEach(([key, value]) => {
-      if (value !== undefined) {
-        body.append(key, value);
-      }
-    });
-
-    const headers = {
-      Authorization: `Bearer ${this.apiKey}`,
-    };
-
-    // Send the request, start job
-    const url = `${this.baseUrl}/upload`;
-    const response = await fetch(url, {
+    const response = await ParsingService.uploadFileApiV1ParsingUploadPost({
+      client: this.#client,
+      throwOnError: true,
       signal: AbortSignal.timeout(this.maxTimeout * 1000),
-      method: "POST",
       body,
-      headers,
     });
-    if (!response.ok) {
-      throw new Error(`Failed to parse the file: ${await response.text()}`);
-    }
-    const jsonResponse = await response.json();
-    return jsonResponse.id;
+
+    return response.data.id;
   }
 
   // Get the result of the job
-  private async getJobResult(jobId: string, resultType: string): Promise<any> {
-    const resultUrl = `${this.baseUrl}/job/${jobId}/result/${resultType}`;
-    const statusUrl = `${this.baseUrl}/job/${jobId}`;
-    const headers = { Authorization: `Bearer ${this.apiKey}` };
-
+  private async getJobResult(
+    jobId: string,
+    resultType: "text" | "json" | "markdown",
+  ): Promise<any> {
     const signal = AbortSignal.timeout(this.maxTimeout * 1000);
     let tries = 0;
     while (true) {
-      await new Promise((resolve) =>
-        setTimeout(resolve, this.checkInterval * 1000),
-      );
+      await sleep(this.checkInterval * 1000);
 
       // Check the job status. If unsuccessful response, checks if maximum timeout has been reached. If reached, throws an error
-      const statusResponse = await fetch(statusUrl, {
-        headers,
+      const result = await ParsingService.getJobApiV1ParsingJobJobIdGet({
+        client: this.#client,
+        throwOnError: true,
+        path: {
+          job_id: jobId,
+        },
         signal,
       });
-      if (!statusResponse.ok) {
-        signal.throwIfAborted();
-        if (this.verbose && tries % 10 === 0) {
-          process.stdout.write(".");
-        }
-        tries++;
-        continue;
-      }
+      const { data } = result;
 
-      // If response is succesful, check status of job. Allowed values "PENDING", "SUCCESS", "ERROR", "CANCELED"
-      const statusJson = await statusResponse.json();
-      const status = statusJson.status;
+      const status = (data as Record<string, unknown>)["status"];
       // If job has completed, return the result
       if (status === "SUCCESS") {
-        const resultResponse = await fetch(resultUrl, {
-          headers,
-          signal,
-        });
-        if (!resultResponse.ok) {
-          throw new Error(
-            `Failed to fetch result: ${await resultResponse.text()}`,
-          );
+        let result;
+        switch (resultType) {
+          case "json": {
+            result =
+              await ParsingService.getJobJsonResultApiV1ParsingJobJobIdResultJsonGet(
+                {
+                  client: this.#client,
+                  throwOnError: true,
+                  path: {
+                    job_id: jobId,
+                  },
+                  signal,
+                },
+              );
+            break;
+          }
+          case "markdown": {
+            result =
+              await ParsingService.getJobResultApiV1ParsingJobJobIdResultMarkdownGet(
+                {
+                  client: this.#client,
+                  throwOnError: true,
+                  path: {
+                    job_id: jobId,
+                  },
+                  signal,
+                },
+              );
+            break;
+          }
+          case "text": {
+            result =
+              await ParsingService.getJobTextResultApiV1ParsingJobJobIdResultTextGet(
+                {
+                  client: this.#client,
+                  throwOnError: true,
+                  path: {
+                    job_id: jobId,
+                  },
+                  signal,
+                },
+              );
+            break;
+          }
         }
-        return resultResponse.json();
+        return result.data;
         // If job is still pending, check if maximum timeout has been reached. If reached, throws an error
       } else if (status === "PENDING") {
         signal.throwIfAborted();
         if (this.verbose && tries % 10 === 0) {
-          process.stdout.write(".");
+          this.stdout?.write(".");
         }
         tries++;
       } else {
@@ -401,43 +380,38 @@ export class LlamaParseReader extends FileReader {
    * To be used with resultType = "text" and "markdown"
    *
    * @param {Uint8Array} fileContent - The content of the file to be loaded.
-   * @param {string} [fileName] - The optional name of the file to be loaded.
    * @return {Promise<Document[]>} A Promise object that resolves to an array of Document objects.
    */
-  async loadDataAsContent(
-    fileContent: Uint8Array,
-    fileName?: string,
-  ): Promise<Document[]> {
-    let jobId;
-    try {
-      // Creates a job for the file
-      jobId = await this.createJob(fileContent, fileName);
-      if (this.verbose) {
-        console.log(`Started parsing the file under job id ${jobId}`);
-      }
+  async loadDataAsContent(fileContent: Uint8Array): Promise<Document[]> {
+    return this.createJob(fileContent)
+      .then(async (jobId) => {
+        if (this.verbose) {
+          console.log(`Started parsing the file under job id ${jobId}`);
+        }
 
-      // Return results as Document objects
-      const jobResults = await this.getJobResult(jobId, this.resultType);
-      const resultText = jobResults[this.resultType];
+        // Return results as Document objects
+        const jobResults = await this.getJobResult(jobId, this.resultType);
+        const resultText = jobResults[this.resultType];
 
-      // Split the text by separator if splitByPage is true
-      if (this.splitByPage) {
-        return this.splitTextBySeparator(resultText);
-      }
+        // Split the text by separator if splitByPage is true
+        if (this.splitByPage) {
+          return this.splitTextBySeparator(resultText);
+        }
 
-      return [
-        new Document({
-          text: resultText,
-        }),
-      ];
-    } catch (e) {
-      console.error(`Error while parsing file under job id ${jobId}`, e);
-      if (this.ignoreErrors) {
-        return [];
-      } else {
-        throw e;
-      }
-    }
+        return [
+          new Document({
+            text: resultText,
+          }),
+        ];
+      })
+      .catch((error) => {
+        if (this.ignoreErrors) {
+          console.warn(`Error while parsing the file: ${error.message}`);
+          return [];
+        } else {
+          throw error;
+        }
+      });
   }
   /**
    * Loads data from a file and returns an array of JSON objects.
@@ -467,8 +441,8 @@ export class LlamaParseReader extends FileReader {
       resultJson.file_path = isFilePath ? filePathOrContent : undefined;
       return [resultJson];
     } catch (e) {
-      console.error(`Error while parsing the file under job id ${jobId}`, e);
       if (this.ignoreErrors) {
+        console.error(`Error while parsing the file under job id ${jobId}`, e);
         return [];
       } else {
         throw e;
@@ -551,15 +525,20 @@ export class LlamaParseReader extends FileReader {
     imagePath: string,
     jobId: string,
   ): Promise<void> {
-    const headers = { Authorization: `Bearer ${this.apiKey}` };
-    // Construct the image URL
-    const imageUrl = `${this.baseUrl}/job/${jobId}/result/image/${imageName}`;
-    const response = await fetch(imageUrl, { headers });
-    if (!response.ok) {
-      throw new Error(`Failed to download image: ${await response.text()}`);
+    const response =
+      await ParsingService.getJobImageResultApiV1ParsingJobJobIdResultImageNameGet(
+        {
+          client: this.#client,
+          path: {
+            job_id: jobId,
+            name: imageName,
+          },
+        },
+      );
+    if (response.error) {
+      throw new Error(`Failed to download image: ${response.error.detail}`);
     }
-    // Convert the response to an ArrayBuffer and then to a Buffer
-    const arrayBuffer = await response.arrayBuffer();
+    const arrayBuffer = (await response.data) as ArrayBuffer;
     const buffer = new Uint8Array(arrayBuffer);
     // Write the image buffer to the specified imagePath
     await fs.writeFile(imagePath, buffer);

packages/cloud/src/utils.ts

@@ -0,0 +1,3 @@
+export async function sleep(ms: number): Promise<void> {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}

packages/cloud/tsconfig.json

@@ -8,8 +8,17 @@
     "moduleResolution": "Bundler",
     "skipLibCheck": true,
     "strict": true,
-    "lib": ["DOM", "ESNext"]
+    "lib": ["DOM", "ESNext"],
+    "types": []
   },
   "include": ["./src"],
-  "exclude": ["node_modules"]
+  "exclude": ["node_modules"],
+  "references": [
+    {
+      "path": "../core/tsconfig.json"
+    },
+    {
+      "path": "../env/tsconfig.json"
+    }
+  ]
 }

packages/cloud/turbo.json

@@ -0,0 +1,8 @@
+{
+  "extends": ["//"],
+  "tasks": {
+    "build": {
+      "outputs": ["dist/**", "src/client/**"]
+    }
+  }
+}

packages/community/CHANGELOG.md

@@ -1,5 +1,53 @@
 # @llamaindex/community
 
+## 0.0.40
+
+### Patch Changes
+
+- 50e6b57: feat: add Amazon Bedrock Retriever
+- Updated dependencies [8b7fdba]
+  - @llamaindex/core@0.2.6
+
+## 0.0.39
+
+### Patch Changes
+
+- Updated dependencies [d902cc3]
+  - @llamaindex/core@0.2.5
+
+## 0.0.38
+
+### Patch Changes
+
+- Updated dependencies [b48bcc3]
+  - @llamaindex/core@0.2.4
+  - @llamaindex/env@0.1.12
+
+## 0.0.37
+
+### Patch Changes
+
+- Updated dependencies [2cd1383]
+  - @llamaindex/core@0.2.3
+
+## 0.0.36
+
+### Patch Changes
+
+- Updated dependencies [749b43a]
+  - @llamaindex/core@0.2.2
+
+## 0.0.35
+
+### Patch Changes
+
+- Updated dependencies [ac07e3c]
+- Updated dependencies [70ccb4a]
+- Updated dependencies [1a6137b]
+- Updated dependencies [ac07e3c]
+  - @llamaindex/core@0.2.1
+  - @llamaindex/env@0.1.11
+
 ## 0.0.34
 
 ### Patch Changes

packages/community/README.md

@@ -7,6 +7,7 @@
 - Bedrock support for the Anthropic Claude Models [usage](https://ts.llamaindex.ai/modules/llms/available_llms/bedrock)
 - Bedrock support for the Meta LLama 2, 3 and 3.1 Models [usage](https://ts.llamaindex.ai/modules/llms/available_llms/bedrock)
 - Meta LLama3.1 405b tool call support
+- Bedrock support for querying Knowledge Base
 
 ## LICENSE
 

packages/community/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/community",
   "description": "Community package for LlamaIndexTS",
-  "version": "0.0.34",
+  "version": "0.0.40",
   "type": "module",
   "types": "dist/type/index.d.ts",
   "main": "dist/cjs/index.js",
@@ -47,6 +47,7 @@
   },
   "dependencies": {
     "@aws-sdk/client-bedrock-runtime": "^3.642.0",
+    "@aws-sdk/client-bedrock-agent-runtime": "^3.642.0",
     "@llamaindex/core": "workspace:*",
     "@llamaindex/env": "workspace:*"
   }

packages/community/src/index.ts

@@ -3,3 +3,4 @@ export {
   BEDROCK_MODEL_MAX_TOKENS,
   Bedrock,
 } from "./llm/bedrock/index.js";
+export { AmazonKnowledgeBaseRetriever } from "./retrievers/bedrock.js";

packages/community/src/retrievers/bedrock.ts

@@ -0,0 +1,165 @@
+import type { KnowledgeBaseVectorSearchConfiguration } from "@aws-sdk/client-bedrock-agent-runtime";
+import {
+  BedrockAgentRuntimeClient,
+  type BedrockAgentRuntimeClientConfig,
+  type RetrievalFilter,
+  RetrieveCommand,
+  type SearchType,
+} from "@aws-sdk/client-bedrock-agent-runtime";
+import type { QueryBundle } from "@llamaindex/core/query-engine";
+import { BaseRetriever } from "@llamaindex/core/retriever";
+import { Document, type NodeWithScore } from "@llamaindex/core/schema";
+import { extractText } from "@llamaindex/core/utils";
+
+/**
+ * Interface for the arguments required to initialize an
+ * AmazonKnowledgeBaseRetriever instance.
+ */
+export interface AmazonKnowledgeBaseRetrieverArgs {
+  knowledgeBaseId: string;
+  topK: number;
+  region: string;
+  clientOptions?: BedrockAgentRuntimeClientConfig;
+  filter?: RetrievalFilter;
+  overrideSearchType?: SearchType;
+}
+
+/**
+ * Class for interacting with Amazon Bedrock Knowledge Bases, a RAG workflow oriented service
+ * Extends the BaseRetriever class.
+ * @example
+ * ```typescript
+ * const retriever = new AmazonKnowledgeBaseRetriever({
+ *   topK: 10,
+ *   knowledgeBaseId: "YOUR_KNOWLEDGE_BASE_ID",
+ *   region: "us-east-2",
+ *   clientOptions: {
+ *     credentials: {
+ *       accessKeyId: "YOUR_ACCESS_KEY_ID",
+ *       secretAccessKey: "YOUR_SECRET_ACCESS_KEY",
+ *     },
+ *   },
+ * });
+ *
+ * const docs = await retriever.retrieve({query: "How are clouds formed?"});
+ * ```
+ */
+export class AmazonKnowledgeBaseRetriever extends BaseRetriever {
+  static lc_name() {
+    return "AmazonKnowledgeBaseRetriever";
+  }
+
+  lc_namespace = ["llamaindex", "retrievers", "amazon_bedrock_knowledge_base"];
+
+  knowledgeBaseId: string;
+
+  topK: number;
+
+  bedrockAgentRuntimeClient: BedrockAgentRuntimeClient;
+
+  filter: RetrievalFilter | undefined;
+
+  overrideSearchType: SearchType | undefined;
+
+  constructor({
+    knowledgeBaseId,
+    topK = 10,
+    clientOptions,
+    region,
+    filter,
+    overrideSearchType,
+  }: AmazonKnowledgeBaseRetrieverArgs) {
+    super();
+
+    this.topK = topK;
+    this.filter = filter;
+    this.overrideSearchType = overrideSearchType;
+    this.bedrockAgentRuntimeClient = new BedrockAgentRuntimeClient({
+      region,
+      ...clientOptions,
+    });
+    this.knowledgeBaseId = knowledgeBaseId;
+  }
+
+  /**
+   * Cleans the result text by replacing sequences of whitespace with a
+   * single space and removing ellipses.
+   * @param resText The result text to clean.
+   * @returns The cleaned result text.
+   */
+  cleanResult(resText: string) {
+    const res = resText.replace(/\s+/g, " ").replace(/\.\.\./g, "");
+    return res;
+  }
+
+  async queryKnowledgeBase(
+    query: QueryBundle,
+    topK: number,
+    filter?: RetrievalFilter,
+    overrideSearchType?: SearchType,
+  ): Promise<NodeWithScore[]> {
+    const retrieveCommand = new RetrieveCommand({
+      knowledgeBaseId: this.knowledgeBaseId,
+      retrievalQuery: {
+        text: extractText(query),
+      },
+      retrievalConfiguration: {
+        vectorSearchConfiguration: {
+          numberOfResults: topK,
+          overrideSearchType,
+          filter,
+        } as KnowledgeBaseVectorSearchConfiguration,
+      },
+    });
+
+    const retrieveResponse =
+      await this.bedrockAgentRuntimeClient.send(retrieveCommand);
+
+    return (
+      retrieveResponse.retrievalResults?.map((result) => {
+        let source;
+        switch (result.location?.type) {
+          case "CONFLUENCE":
+            source = result.location?.confluenceLocation?.url;
+            break;
+          case "S3":
+            source = result.location?.s3Location?.uri;
+            break;
+          case "SALESFORCE":
+            source = result.location?.salesforceLocation?.url;
+            break;
+          case "SHAREPOINT":
+            source = result.location?.sharePointLocation?.url;
+            break;
+          case "WEB":
+            source = result.location?.webLocation?.url;
+            break;
+          default:
+            source = result.location?.s3Location?.uri;
+            break;
+        }
+
+        return {
+          node: new Document({
+            text: this.cleanResult(result.content?.text || ""),
+            metadata: {
+              source,
+              score: result.score,
+              ...result.metadata,
+            },
+          }),
+          score: result.score ?? 1.0,
+        };
+      }) ?? []
+    );
+  }
+
+  async _retrieve(query: QueryBundle): Promise<NodeWithScore[]> {
+    return await this.queryKnowledgeBase(
+      query,
+      this.topK,
+      this.filter,
+      this.overrideSearchType,
+    );
+  }
+}

packages/core/CHANGELOG.md

@@ -1,5 +1,61 @@
 # @llamaindex/core
 
+## 0.2.6
+
+### Patch Changes
+
+- 8b7fdba: refactor: move chat engine & retriever into core.
+
+  - `chatHistory` in BaseChatEngine now returns `ChatMessage[] | Promise<ChatMessage[]>`, instead of `BaseMemory`
+  - update `retrieve-end` type
+
+## 0.2.5
+
+### Patch Changes
+
+- d902cc3: Fix context not being sent using ContextChatEngine
+
+## 0.2.4
+
+### Patch Changes
+
+- b48bcc3: feat: add `load-transformers` event type when loading `@xenova/transformers` module
+
+  This would benefit user who want to customize the transformer env.
+
+- Updated dependencies [b48bcc3]
+  - @llamaindex/env@0.1.12
+
+## 0.2.3
+
+### Patch Changes
+
+- 2cd1383: refactor: align `response-synthesizers` & `chat-engine` module
+
+  - builtin event system
+  - correct class extends
+  - aligin APIs, naming with llama-index python
+  - move stream out of first parameter to second parameter for the better tyep checking
+  - remove JSONQueryEngine in `@llamaindex/experimental`, as the code quality is not satisify and we will bring it back later
+
+## 0.2.2
+
+### Patch Changes
+
+- 749b43a: fix: clip embedding transform function
+
+## 0.2.1
+
+### Patch Changes
+
+- ac07e3c: fix: replace instanceof check with `.type` check
+- 70ccb4a: Allow arbitrary types in workflow's StartEvent and StopEvent
+- ac07e3c: fix: add `console.warn` when import dual module
+- Updated dependencies [ac07e3c]
+- Updated dependencies [1a6137b]
+- Updated dependencies [ac07e3c]
+  - @llamaindex/env@0.1.11
+
 ## 0.2.0
 
 ### Minor Changes

packages/core/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/core",
   "type": "module",
-  "version": "0.2.0",
+  "version": "0.2.6",
   "description": "LlamaIndex Core Module",
   "exports": {
     "./node-parser": {
@@ -157,6 +157,76 @@
         "types": "./dist/workflow/index.d.ts",
         "default": "./dist/workflow/index.js"
       }
+    },
+    "./memory": {
+      "require": {
+        "types": "./dist/memory/index.d.cts",
+        "default": "./dist/memory/index.cjs"
+      },
+      "import": {
+        "types": "./dist/memory/index.d.ts",
+        "default": "./dist/memory/index.js"
+      },
+      "default": {
+        "types": "./dist/memory/index.d.ts",
+        "default": "./dist/memory/index.js"
+      }
+    },
+    "./storage/chat-store": {
+      "require": {
+        "types": "./dist/storage/chat-store/index.d.cts",
+        "default": "./dist/storage/chat-store/index.cjs"
+      },
+      "import": {
+        "types": "./dist/storage/chat-store/index.d.ts",
+        "default": "./dist/storage/chat-store/index.js"
+      },
+      "default": {
+        "types": "./dist/storage/chat-store/index.d.ts",
+        "default": "./dist/storage/chat-store/index.js"
+      }
+    },
+    "./response-synthesizers": {
+      "require": {
+        "types": "./dist/response-synthesizers/index.d.cts",
+        "default": "./dist/response-synthesizers/index.cjs"
+      },
+      "import": {
+        "types": "./dist/response-synthesizers/index.d.ts",
+        "default": "./dist/response-synthesizers/index.js"
+      },
+      "default": {
+        "types": "./dist/response-synthesizers/index.d.ts",
+        "default": "./dist/response-synthesizers/index.js"
+      }
+    },
+    "./chat-engine": {
+      "require": {
+        "types": "./dist/chat-engine/index.d.cts",
+        "default": "./dist/chat-engine/index.cjs"
+      },
+      "import": {
+        "types": "./dist/chat-engine/index.d.ts",
+        "default": "./dist/chat-engine/index.js"
+      },
+      "default": {
+        "types": "./dist/chat-engine/index.d.ts",
+        "default": "./dist/chat-engine/index.js"
+      }
+    },
+    "./retriever": {
+      "require": {
+        "types": "./dist/retriever/index.d.cts",
+        "default": "./dist/retriever/index.cjs"
+      },
+      "import": {
+        "types": "./dist/retriever/index.d.ts",
+        "default": "./dist/retriever/index.js"
+      },
+      "default": {
+        "types": "./dist/retriever/index.d.ts",
+        "default": "./dist/retriever/index.js"
+      }
     }
   },
   "files": [
@@ -172,14 +242,17 @@
     "url": "https://github.com/himself65/LlamaIndexTS.git"
   },
   "devDependencies": {
+    "@edge-runtime/vm": "^4.0.3",
     "ajv": "^8.17.1",
     "bunchee": "5.3.2",
+    "happy-dom": "^15.7.4",
     "natural": "^8.0.1",
     "python-format-js": "^1.4.3"
   },
   "dependencies": {
     "@llamaindex/env": "workspace:*",
     "@types/node": "^22.5.1",
+    "magic-bytes.js": "^1.10.0",
     "zod": "^3.23.8"
   }
 }

packages/core/src/chat-engine/index.ts

@@ -0,0 +1,36 @@
+import type { ChatMessage, MessageContent } from "../llms";
+import type { BaseMemory } from "../memory";
+import { EngineResponse } from "../schema";
+
+export interface BaseChatEngineParams<
+  AdditionalMessageOptions extends object = object,
+> {
+  message: MessageContent;
+  /**
+   * Optional chat history if you want to customize the chat history.
+   */
+  chatHistory?:
+    | ChatMessage<AdditionalMessageOptions>[]
+    | BaseMemory<AdditionalMessageOptions>;
+}
+
+export interface StreamingChatEngineParams<
+  AdditionalMessageOptions extends object = object,
+> extends BaseChatEngineParams<AdditionalMessageOptions> {
+  stream: true;
+}
+
+export interface NonStreamingChatEngineParams<
+  AdditionalMessageOptions extends object = object,
+> extends BaseChatEngineParams<AdditionalMessageOptions> {
+  stream?: false;
+}
+
+export abstract class BaseChatEngine {
+  abstract chat(params: NonStreamingChatEngineParams): Promise<EngineResponse>;
+  abstract chat(
+    params: StreamingChatEngineParams,
+  ): Promise<AsyncIterable<EngineResponse>>;
+
+  abstract chatHistory: ChatMessage[] | Promise<ChatMessage[]>;
+}

packages/core/src/embeddings/base.ts

@@ -23,23 +23,34 @@ export abstract class BaseEmbedding extends TransformComponent {
   embedBatchSize = DEFAULT_EMBED_BATCH_SIZE;
   embedInfo?: EmbeddingInfo;
 
-  constructor() {
-    super(
-      async (
-        nodes: BaseNode[],
-        options?: BaseEmbeddingOptions,
-      ): Promise<BaseNode[]> => {
-        const texts = nodes.map((node) => node.getContent(MetadataMode.EMBED));
+  protected constructor(
+    transformFn?: (
+      nodes: BaseNode[],
+      options?: BaseEmbeddingOptions,
+    ) => Promise<BaseNode[]>,
+  ) {
+    if (transformFn) {
+      super(transformFn);
+    } else {
+      super(
+        async (
+          nodes: BaseNode[],
+          options?: BaseEmbeddingOptions,
+        ): Promise<BaseNode[]> => {
+          const texts = nodes.map((node) =>
+            node.getContent(MetadataMode.EMBED),
+          );
 
-        const embeddings = await this.getTextEmbeddingsBatch(texts, options);
+          const embeddings = await this.getTextEmbeddingsBatch(texts, options);
 
-        for (let i = 0; i < nodes.length; i++) {
-          nodes[i]!.embedding = embeddings[i];
-        }
+          for (let i = 0; i < nodes.length; i++) {
+            nodes[i]!.embedding = embeddings[i];
+          }
 
-        return nodes;
-      },
-    );
+          return nodes;
+        },
+      );
+    }
   }
 
   similarity(

packages/core/src/embeddings/index.ts

@@ -1,4 +1,5 @@
 export { BaseEmbedding, batchEmbeddings } from "./base";
 export type { BaseEmbeddingOptions, EmbeddingInfo } from "./base";
+export { MultiModalEmbedding } from "./muti-model";
 export { truncateMaxTokens } from "./tokenizer";
 export { DEFAULT_SIMILARITY_TOP_K, SimilarityType, similarity } from "./utils";

packages/core/src/embeddings/muti-model.ts

@@ -0,0 +1,81 @@
+import type { MessageContentDetail } from "../llms";
+import {
+  ImageNode,
+  MetadataMode,
+  ModalityType,
+  splitNodesByType,
+  type BaseNode,
+  type ImageType,
+} from "../schema";
+import { extractImage, extractSingleText } from "../utils";
+import {
+  BaseEmbedding,
+  batchEmbeddings,
+  type BaseEmbeddingOptions,
+} from "./base";
+
+/*
+ * Base class for Multi Modal embeddings.
+ */
+export abstract class MultiModalEmbedding extends BaseEmbedding {
+  abstract getImageEmbedding(images: ImageType): Promise<number[]>;
+
+  protected constructor() {
+    super(
+      async (
+        nodes: BaseNode[],
+        options?: BaseEmbeddingOptions,
+      ): Promise<BaseNode[]> => {
+        const nodeMap = splitNodesByType(nodes);
+        const imageNodes = nodeMap[ModalityType.IMAGE] ?? [];
+        const textNodes = nodeMap[ModalityType.TEXT] ?? [];
+
+        const embeddings = await batchEmbeddings(
+          textNodes.map((node) => node.getContent(MetadataMode.EMBED)),
+          this.getTextEmbeddings.bind(this),
+          this.embedBatchSize,
+          options,
+        );
+        for (let i = 0; i < textNodes.length; i++) {
+          textNodes[i]!.embedding = embeddings[i];
+        }
+
+        const imageEmbeddings = await batchEmbeddings(
+          imageNodes.map((n) => (n as ImageNode).image),
+          this.getImageEmbeddings.bind(this),
+          this.embedBatchSize,
+          options,
+        );
+        for (let i = 0; i < imageNodes.length; i++) {
+          imageNodes[i]!.embedding = imageEmbeddings[i];
+        }
+
+        return nodes;
+      },
+    );
+  }
+
+  /**
+   * Optionally override this method to retrieve multiple image embeddings in a single request
+   * @param images
+   */
+  async getImageEmbeddings(images: ImageType[]): Promise<number[][]> {
+    return Promise.all(
+      images.map((imgFilePath) => this.getImageEmbedding(imgFilePath)),
+    );
+  }
+
+  async getQueryEmbedding(
+    query: MessageContentDetail,
+  ): Promise<number[] | null> {
+    const image = extractImage(query);
+    if (image) {
+      return await this.getImageEmbedding(image);
+    }
+    const text = extractSingleText(query);
+    if (text) {
+      return await this.getTextEmbedding(text);
+    }
+    return null;
+  }
+}

packages/core/src/global/settings/callback-manager.ts

@@ -6,8 +6,14 @@ import type {
   ToolCall,
   ToolOutput,
 } from "../../llms";
+import type { QueryEndEvent, QueryStartEvent } from "../../query-engine";
+import type {
+  SynthesizeEndEvent,
+  SynthesizeStartEvent,
+} from "../../response-synthesizers";
+import type { RetrieveEndEvent, RetrieveStartEvent } from "../../retriever";
 import { TextNode } from "../../schema";
-import { EventCaller, getEventCaller } from "../../utils/event-caller";
+import { EventCaller, getEventCaller } from "../../utils";
 import type { UUID } from "../type";
 
 export type LLMStartEvent = {
@@ -60,6 +66,12 @@ export interface LlamaIndexEventMaps {
   "chunking-end": ChunkingEndEvent;
   "node-parsing-start": NodeParsingStartEvent;
   "node-parsing-end": NodeParsingEndEvent;
+  "query-start": QueryStartEvent;
+  "query-end": QueryEndEvent;
+  "synthesize-start": SynthesizeStartEvent;
+  "synthesize-end": SynthesizeEndEvent;
+  "retrieve-start": RetrieveStartEvent;
+  "retrieve-end": RetrieveEndEvent;
 }
 
 export class LlamaIndexCustomEvent<T = any> extends CustomEvent<T> {
@@ -119,16 +131,29 @@ export class CallbackManager {
   dispatchEvent<K extends keyof LlamaIndexEventMaps>(
     event: K,
     detail: LlamaIndexEventMaps[K],
+    sync = false,
   ) {
     const cbs = this.#handlers.get(event);
     if (!cbs) {
       return;
     }
-    queueMicrotask(() => {
+    if (typeof queueMicrotask === "undefined") {
+      console.warn(
+        "queueMicrotask is not available, dispatching synchronously",
+      );
+      sync = true;
+    }
+    if (sync) {
       cbs.forEach((handler) =>
         handler(LlamaIndexCustomEvent.fromEvent(event, { ...detail })),
       );
-    });
+    } else {
+      queueMicrotask(() => {
+        cbs.forEach((handler) =>
+          handler(LlamaIndexCustomEvent.fromEvent(event, { ...detail })),
+        );
+      });
+    }
   }
 }
 

packages/core/src/indices/prompt-helper.ts

@@ -1,10 +1,13 @@
 import { type Tokenizer, tokenizers } from "@llamaindex/env";
 import {
   DEFAULT_CHUNK_OVERLAP_RATIO,
+  DEFAULT_CHUNK_SIZE,
   DEFAULT_CONTEXT_WINDOW,
   DEFAULT_NUM_OUTPUTS,
   DEFAULT_PADDING,
+  Settings,
 } from "../global";
+import type { LLMMetadata } from "../llms";
 import { SentenceSplitter } from "../node-parser";
 import type { PromptTemplate } from "../prompts";
 
@@ -133,4 +136,29 @@ export class PromptHelper {
     const combinedStr = textChunks.join("\n\n");
     return textSplitter.splitText(combinedStr);
   }
+
+  static fromLLMMetadata(
+    metadata: LLMMetadata,
+    options?: {
+      chunkOverlapRatio?: number;
+      chunkSizeLimit?: number;
+      tokenizer?: Tokenizer;
+      separator?: string;
+    },
+  ) {
+    const {
+      chunkOverlapRatio = DEFAULT_CHUNK_OVERLAP_RATIO,
+      chunkSizeLimit = DEFAULT_CHUNK_SIZE,
+      tokenizer = Settings.tokenizer,
+      separator = " ",
+    } = options ?? {};
+    return new PromptHelper({
+      contextWindow: metadata.contextWindow,
+      numOutput: metadata.maxTokens ?? DEFAULT_NUM_OUTPUTS,
+      chunkOverlapRatio,
+      chunkSizeLimit,
+      tokenizer,
+      separator,
+    });
+  }
 }

packages/core/src/memory/base.ts

@@ -0,0 +1,83 @@
+import { Settings } from "../global";
+import type { ChatMessage } from "../llms";
+import { type BaseChatStore, SimpleChatStore } from "../storage/chat-store";
+import { extractText } from "../utils";
+
+export const DEFAULT_TOKEN_LIMIT_RATIO = 0.75;
+export const DEFAULT_CHAT_STORE_KEY = "chat_history";
+
+/**
+ * A ChatMemory is used to keep the state of back and forth chat messages
+ */
+export abstract class BaseMemory<
+  AdditionalMessageOptions extends object = object,
+> {
+  /**
+   * Retrieves messages from the memory, optionally including transient messages.
+   * Compared to getAllMessages, this method a) allows for transient messages to be included in the retrieval and b) may return a subset of the total messages by applying a token limit.
+   * @param transientMessages Optional array of temporary messages to be included in the retrieval.
+   * These messages are not stored in the memory but are considered for the current interaction.
+   * @returns An array of chat messages, either synchronously or as a Promise.
+   */
+  abstract getMessages(
+    transientMessages?: ChatMessage<AdditionalMessageOptions>[] | undefined,
+  ):
+    | ChatMessage<AdditionalMessageOptions>[]
+    | Promise<ChatMessage<AdditionalMessageOptions>[]>;
+
+  /**
+   * Retrieves all messages stored in the memory.
+   * @returns An array of all chat messages, either synchronously or as a Promise.
+   */
+  abstract getAllMessages():
+    | ChatMessage<AdditionalMessageOptions>[]
+    | Promise<ChatMessage<AdditionalMessageOptions>[]>;
+
+  /**
+   * Adds a new message to the memory.
+   * @param messages The chat message to be added to the memory.
+   */
+  abstract put(messages: ChatMessage<AdditionalMessageOptions>): void;
+
+  /**
+   * Clears all messages from the memory.
+   */
+  abstract reset(): void;
+
+  protected _tokenCountForMessages(messages: ChatMessage[]): number {
+    if (messages.length === 0) {
+      return 0;
+    }
+
+    const tokenizer = Settings.tokenizer;
+    const str = messages.map((m) => extractText(m.content)).join(" ");
+    return tokenizer.encode(str).length;
+  }
+}
+
+export abstract class BaseChatStoreMemory<
+  AdditionalMessageOptions extends object = object,
+> extends BaseMemory<AdditionalMessageOptions> {
+  protected constructor(
+    public chatStore: BaseChatStore<AdditionalMessageOptions> = new SimpleChatStore<AdditionalMessageOptions>(),
+    public chatStoreKey: string = DEFAULT_CHAT_STORE_KEY,
+  ) {
+    super();
+  }
+
+  getAllMessages(): ChatMessage<AdditionalMessageOptions>[] {
+    return this.chatStore.getMessages(this.chatStoreKey);
+  }
+
+  put(messages: ChatMessage<AdditionalMessageOptions>) {
+    this.chatStore.addMessage(this.chatStoreKey, messages);
+  }
+
+  set(messages: ChatMessage<AdditionalMessageOptions>[]) {
+    this.chatStore.setMessages(this.chatStoreKey, messages);
+  }
+
+  reset() {
+    this.chatStore.deleteMessages(this.chatStoreKey);
+  }
+}

packages/core/src/memory/chat-memory-buffer.ts

@@ -0,0 +1,71 @@
+import { Settings } from "../global";
+import type { ChatMessage, LLM } from "../llms";
+import { type BaseChatStore } from "../storage/chat-store";
+import { BaseChatStoreMemory, DEFAULT_TOKEN_LIMIT_RATIO } from "./base";
+
+type ChatMemoryBufferOptions<AdditionalMessageOptions extends object = object> =
+  {
+    tokenLimit?: number | undefined;
+    chatStore?: BaseChatStore<AdditionalMessageOptions> | undefined;
+    chatStoreKey?: string | undefined;
+    chatHistory?: ChatMessage<AdditionalMessageOptions>[] | undefined;
+    llm?: LLM<object, AdditionalMessageOptions> | undefined;
+  };
+
+export class ChatMemoryBuffer<
+  AdditionalMessageOptions extends object = object,
+> extends BaseChatStoreMemory<AdditionalMessageOptions> {
+  tokenLimit: number;
+
+  constructor(
+    options?: Partial<ChatMemoryBufferOptions<AdditionalMessageOptions>>,
+  ) {
+    super(options?.chatStore, options?.chatStoreKey);
+
+    const llm = options?.llm ?? Settings.llm;
+    const contextWindow = llm.metadata.contextWindow;
+    this.tokenLimit =
+      options?.tokenLimit ??
+      Math.ceil(contextWindow * DEFAULT_TOKEN_LIMIT_RATIO);
+
+    if (options?.chatHistory) {
+      this.chatStore.setMessages(this.chatStoreKey, options.chatHistory);
+    }
+  }
+
+  getMessages(
+    transientMessages?: ChatMessage<AdditionalMessageOptions>[] | undefined,
+    initialTokenCount: number = 0,
+  ) {
+    const messages = this.getAllMessages();
+
+    if (initialTokenCount > this.tokenLimit) {
+      throw new Error("Initial token count exceeds token limit");
+    }
+
+    // Add input messages as transient messages
+    const messagesWithInput = transientMessages
+      ? [...transientMessages, ...messages]
+      : messages;
+
+    let messageCount = messagesWithInput.length;
+    let currentMessages = messagesWithInput.slice(-messageCount);
+    let tokenCount =
+      this._tokenCountForMessages(messagesWithInput) + initialTokenCount;
+
+    while (tokenCount > this.tokenLimit && messageCount > 1) {
+      messageCount -= 1;
+      if (messagesWithInput.at(-messageCount)!.role === "assistant") {
+        messageCount -= 1;
+      }
+      currentMessages = messagesWithInput.slice(-messageCount);
+      tokenCount =
+        this._tokenCountForMessages(currentMessages) + initialTokenCount;
+    }
+
+    if (tokenCount > this.tokenLimit && messageCount <= 0) {
+      return [];
+    }
+    return messagesWithInput.slice(-messageCount);
+  }
+}

packages/core/src/memory/index.ts

@@ -0,0 +1,3 @@
+export { BaseMemory } from "./base";
+export { ChatMemoryBuffer } from "./chat-memory-buffer";
+export { ChatSummaryMemoryBuffer } from "./summary-memory";

packages/core/src/memory/summary-memory.ts

@@ -1,73 +1,11 @@
-import type { ChatMessage, LLM, MessageType } from "@llamaindex/core/llms";
-import {
-  defaultSummaryPrompt,
-  type SummaryPrompt,
-} from "@llamaindex/core/prompts";
-import { extractText, messagesToHistory } from "@llamaindex/core/utils";
-import { tokenizers, type Tokenizer } from "@llamaindex/env";
-import { OpenAI } from "@llamaindex/openai";
+import { type Tokenizer, tokenizers } from "@llamaindex/env";
+import { Settings } from "../global";
+import type { ChatMessage, LLM, MessageType } from "../llms";
+import { defaultSummaryPrompt, type SummaryPrompt } from "../prompts";
+import { extractText, messagesToHistory } from "../utils";
+import { BaseMemory } from "./base";
 
-/**
- * A ChatHistory is used to keep the state of back and forth chat messages
- */
-export abstract class ChatHistory<
-  AdditionalMessageOptions extends object = object,
-> {
-  abstract get messages(): ChatMessage<AdditionalMessageOptions>[];
-  /**
-   * Adds a message to the chat history.
-   * @param message
-   */
-  abstract addMessage(message: ChatMessage<AdditionalMessageOptions>): void;
-
-  /**
-   * Returns the messages that should be used as input to the LLM.
-   */
-  abstract requestMessages(
-    transientMessages?: ChatMessage<AdditionalMessageOptions>[],
-  ): Promise<ChatMessage<AdditionalMessageOptions>[]>;
-
-  /**
-   * Resets the chat history so that it's empty.
-   */
-  abstract reset(): void;
-
-  /**
-   * Returns the new messages since the last call to this function (or since calling the constructor)
-   */
-  abstract newMessages(): ChatMessage<AdditionalMessageOptions>[];
-}
-
-export class SimpleChatHistory extends ChatHistory {
-  messages: ChatMessage[];
-  private messagesBefore: number;
-
-  constructor(init?: { messages?: ChatMessage[] | undefined }) {
-    super();
-    this.messages = init?.messages ?? [];
-    this.messagesBefore = this.messages.length;
-  }
-
-  addMessage(message: ChatMessage) {
-    this.messages.push(message);
-  }
-
-  async requestMessages(transientMessages?: ChatMessage[]) {
-    return [...(transientMessages ?? []), ...this.messages];
-  }
-
-  reset() {
-    this.messages = [];
-  }
-
-  newMessages() {
-    const newMessages = this.messages.slice(this.messagesBefore);
-    this.messagesBefore = this.messages.length;
-    return newMessages;
-  }
-}
-
-export class SummaryChatHistory extends ChatHistory {
+export class ChatSummaryMemoryBuffer extends BaseMemory {
   /**
    * Tokenizer function that converts text to tokens,
    *  this is used to calculate the number of tokens in a message.
@@ -77,20 +15,18 @@ export class SummaryChatHistory extends ChatHistory {
   messages: ChatMessage[];
   summaryPrompt: SummaryPrompt;
   llm: LLM;
-  private messagesBefore: number;
 
-  constructor(init?: Partial<SummaryChatHistory>) {
+  constructor(options?: Partial<ChatSummaryMemoryBuffer>) {
     super();
-    this.messages = init?.messages ?? [];
-    this.messagesBefore = this.messages.length;
-    this.summaryPrompt = init?.summaryPrompt ?? defaultSummaryPrompt;
-    this.llm = init?.llm ?? new OpenAI();
+    this.messages = options?.messages ?? [];
+    this.summaryPrompt = options?.summaryPrompt ?? defaultSummaryPrompt;
+    this.llm = options?.llm ?? Settings.llm;
     if (!this.llm.metadata.maxTokens) {
       throw new Error(
         "LLM maxTokens is not set. Needed so the summarizer ensures the context window size of the LLM.",
       );
     }
-    this.tokenizer = init?.tokenizer ?? tokenizers.tokenizer();
+    this.tokenizer = options?.tokenizer ?? tokenizers.tokenizer();
     this.tokensToSummarize =
       this.llm.metadata.contextWindow - this.llm.metadata.maxTokens;
     if (this.tokensToSummarize < this.llm.metadata.contextWindow * 0.25) {
@@ -128,12 +64,8 @@ export class SummaryChatHistory extends ChatHistory {
     return { content: response.message.content, role: "memory" };
   }
 
-  addMessage(message: ChatMessage) {
-    this.messages.push(message);
-  }
-
   // Find last summary message
-  private getLastSummaryIndex(): number | null {
+  private get lastSummaryIndex(): number | null {
     const reversedMessages = this.messages.slice().reverse();
     const index = reversedMessages.findIndex(
       (message) => message.role === "memory",
@@ -145,7 +77,7 @@ export class SummaryChatHistory extends ChatHistory {
   }
 
   public getLastSummary(): ChatMessage | null {
-    const lastSummaryIndex = this.getLastSummaryIndex();
+    const lastSummaryIndex = this.lastSummaryIndex;
     return lastSummaryIndex ? this.messages[lastSummaryIndex]! : null;
   }
 
@@ -165,7 +97,7 @@ export class SummaryChatHistory extends ChatHistory {
    * If there's a memory, uses all messages after the last summary message.
    */
   private calcConversationMessages(transformSummary?: boolean): ChatMessage[] {
-    const lastSummaryIndex = this.getLastSummaryIndex();
+    const lastSummaryIndex = this.lastSummaryIndex;
     if (!lastSummaryIndex) {
       // there's no memory, so just use all non-system messages
       return this.nonSystemMessages;
@@ -183,7 +115,7 @@ export class SummaryChatHistory extends ChatHistory {
   }
 
   private calcCurrentRequestMessages(transientMessages?: ChatMessage[]) {
-    // TODO: check order: currently, we're sending:
+    // currently, we're sending:
     // system messages first, then transient messages and then the messages that describe the conversation so far
     return [
       ...this.systemMessages,
@@ -192,7 +124,11 @@ export class SummaryChatHistory extends ChatHistory {
     ];
   }
 
-  async requestMessages(transientMessages?: ChatMessage[]) {
+  reset() {
+    this.messages = [];
+  }
+
+  async getMessages(transientMessages?: ChatMessage[]): Promise<ChatMessage[]> {
     const requestMessages = this.calcCurrentRequestMessages(transientMessages);
 
     // get tokens of current request messages and the transient messages
@@ -222,22 +158,11 @@ export class SummaryChatHistory extends ChatHistory {
     return requestMessages;
   }
 
-  reset() {
-    this.messages = [];
+  async getAllMessages(): Promise<ChatMessage[]> {
+    return this.getMessages();
   }
 
-  newMessages() {
-    const newMessages = this.messages.slice(this.messagesBefore);
-    this.messagesBefore = this.messages.length;
-    return newMessages;
+  put(message: ChatMessage) {
+    this.messages.push(message);
   }
 }
-
-export function getHistory(
-  chatHistory?: ChatMessage[] | ChatHistory,
-): ChatHistory {
-  if (chatHistory instanceof ChatHistory) {
-    return chatHistory;
-  }
-  return new SimpleChatHistory({ messages: chatHistory });
-}

packages/core/src/query-engine/base.ts

@@ -1,5 +1,9 @@
+import { randomUUID } from "@llamaindex/env";
+import { Settings } from "../global";
 import type { MessageContent } from "../llms";
+import { PromptMixin } from "../prompts";
 import { EngineResponse, type NodeWithScore } from "../schema";
+import { wrapEventCaller } from "../utils";
 
 /**
  * @link https://docs.llamaindex.ai/en/stable/api_reference/schema/?h=querybundle#llama_index.core.schema.QueryBundle
@@ -14,16 +18,52 @@ export type QueryBundle = {
 
 export type QueryType = string | QueryBundle;
 
-export interface BaseQueryEngine {
-  query(
-    strOrQueryBundle: QueryType,
-    stream: true,
-  ): Promise<AsyncIterable<EngineResponse>>;
-  query(strOrQueryBundle: QueryType, stream?: false): Promise<EngineResponse>;
+export type BaseQueryParams = {
+  query: QueryType;
+};
 
-  synthesize?(
-    strOrQueryBundle: QueryType,
-    nodes: NodeWithScore[],
-    additionalSources?: Iterator<NodeWithScore>,
-  ): Promise<EngineResponse>;
+export interface StreamingQueryParams extends BaseQueryParams {
+  stream: true;
+}
+
+export interface NonStreamingQueryParams extends BaseQueryParams {
+  stream?: false;
+}
+
+export type QueryFn = (
+  strOrQueryBundle: QueryType,
+  stream?: boolean,
+) => Promise<AsyncIterable<EngineResponse> | EngineResponse>;
+
+export abstract class BaseQueryEngine extends PromptMixin {
+  protected constructor(protected readonly _query: QueryFn) {
+    super();
+  }
+
+  async retrieve(params: QueryType): Promise<NodeWithScore[]> {
+    throw new Error(
+      "This query engine does not support retrieve, use query directly",
+    );
+  }
+
+  query(params: StreamingQueryParams): Promise<AsyncIterable<EngineResponse>>;
+  query(params: NonStreamingQueryParams): Promise<EngineResponse>;
+  @wrapEventCaller
+  async query(
+    params: StreamingQueryParams | NonStreamingQueryParams,
+  ): Promise<EngineResponse | AsyncIterable<EngineResponse>> {
+    const { stream, query } = params;
+    const id = randomUUID();
+    const callbackManager = Settings.callbackManager;
+    callbackManager.dispatchEvent("query-start", {
+      id,
+      query,
+    });
+    const response = await this._query(query, stream);
+    callbackManager.dispatchEvent("query-end", {
+      id,
+      response,
+    });
+    return response;
+  }
 }

packages/core/src/query-engine/index.ts

@@ -1 +1,2 @@
-export type { BaseQueryEngine, QueryBundle, QueryType } from "./base";
+export { BaseQueryEngine, type QueryBundle, type QueryType } from "./base";
+export type { QueryEndEvent, QueryStartEvent } from "./type";

packages/core/src/query-engine/type.ts

@@ -0,0 +1,12 @@
+import { EngineResponse } from "../schema";
+import type { QueryType } from "./base";
+
+export type QueryStartEvent = {
+  id: string;
+  query: QueryType;
+};
+
+export type QueryEndEvent = {
+  id: string;
+  response: EngineResponse | AsyncIterable<EngineResponse>;
+};

packages/core/src/response-synthesizers/base-synthesizer.ts

@@ -0,0 +1,58 @@
+import { randomUUID } from "@llamaindex/env";
+import { Settings } from "../global";
+import { PromptHelper } from "../indices";
+import type { LLM, MessageContent } from "../llms";
+import { PromptMixin } from "../prompts";
+import { EngineResponse, type NodeWithScore } from "../schema";
+import type { SynthesizeQuery } from "./type";
+
+export type BaseSynthesizerOptions = {
+  llm?: LLM;
+  promptHelper?: PromptHelper;
+};
+
+export abstract class BaseSynthesizer extends PromptMixin {
+  llm: LLM;
+  promptHelper: PromptHelper;
+
+  protected constructor(options: Partial<BaseSynthesizerOptions>) {
+    super();
+    this.llm = options.llm ?? Settings.llm;
+    this.promptHelper =
+      options.promptHelper ?? PromptHelper.fromLLMMetadata(this.llm.metadata);
+  }
+
+  protected abstract getResponse(
+    query: MessageContent,
+    textChunks: NodeWithScore[],
+    stream: boolean,
+  ): Promise<EngineResponse | AsyncIterable<EngineResponse>>;
+
+  synthesize(
+    query: SynthesizeQuery,
+    stream: true,
+  ): Promise<AsyncIterable<EngineResponse>>;
+  synthesize(query: SynthesizeQuery, stream?: false): Promise<EngineResponse>;
+  async synthesize(
+    query: SynthesizeQuery,
+    stream = false,
+  ): Promise<EngineResponse | AsyncIterable<EngineResponse>> {
+    const callbackManager = Settings.callbackManager;
+    const id = randomUUID();
+    callbackManager.dispatchEvent("synthesize-start", { id, query });
+    let response: EngineResponse | AsyncIterable<EngineResponse>;
+    if (query.nodes.length === 0) {
+      if (stream) {
+        response = EngineResponse.fromResponse("Empty Response", true);
+      } else {
+        response = EngineResponse.fromResponse("Empty Response", false);
+      }
+    } else {
+      const queryMessage: MessageContent =
+        typeof query.query === "string" ? query.query : query.query.query;
+      response = await this.getResponse(queryMessage, query.nodes, stream);
+    }
+    callbackManager.dispatchEvent("synthesize-end", { id, query, response });
+    return response;
+  }
+}

packages/core/src/response-synthesizers/factory.ts

@@ -1,108 +1,52 @@
-import { getBiggestPrompt, type PromptHelper } from "@llamaindex/core/indices";
-import type { LLM } from "@llamaindex/core/llms";
+import { z } from "zod";
+import { getBiggestPrompt } from "../indices";
+import type { MessageContent } from "../llms";
 import {
-  PromptMixin,
   defaultRefinePrompt,
   defaultTextQAPrompt,
   defaultTreeSummarizePrompt,
   type ModuleRecord,
-  type PromptsRecord,
   type RefinePrompt,
   type TextQAPrompt,
   type TreeSummarizePrompt,
-} from "@llamaindex/core/prompts";
-import type { QueryType } from "@llamaindex/core/query-engine";
-import { extractText, streamConverter } from "@llamaindex/core/utils";
-import type { ServiceContext } from "../ServiceContext.js";
+} from "../prompts";
 import {
-  llmFromSettingsOrContext,
-  promptHelperFromSettingsOrContext,
-} from "../Settings.js";
-import type { ResponseBuilder, ResponseBuilderQuery } from "./types.js";
+  EngineResponse,
+  MetadataMode,
+  type NodeWithScore,
+  TextNode,
+} from "../schema";
+import { createMessageContent, extractText, streamConverter } from "../utils";
+import {
+  BaseSynthesizer,
+  type BaseSynthesizerOptions,
+} from "./base-synthesizer";
 
-/**
- * Response modes of the response synthesizer
- */
-enum ResponseMode {
-  REFINE = "refine",
-  COMPACT = "compact",
-  TREE_SUMMARIZE = "tree_summarize",
-  SIMPLE = "simple",
-}
+const responseModeSchema = z.enum([
+  "refine",
+  "compact",
+  "tree_summarize",
+  "multi_modal",
+]);
 
-/**
- * A response builder that just concatenates responses.
- */
-export class SimpleResponseBuilder
-  extends PromptMixin
-  implements ResponseBuilder
-{
-  llm: LLM;
-  textQATemplate: TextQAPrompt;
-
-  constructor(serviceContext?: ServiceContext, textQATemplate?: TextQAPrompt) {
-    super();
-    this.llm = llmFromSettingsOrContext(serviceContext);
-    this.textQATemplate = textQATemplate ?? defaultTextQAPrompt;
-  }
-
-  protected _getPrompts(): PromptsRecord {
-    return {
-      textQATemplate: this.textQATemplate,
-    };
-  }
-  protected _updatePrompts(prompts: { textQATemplate: TextQAPrompt }): void {
-    if (prompts.textQATemplate) {
-      this.textQATemplate = prompts.textQATemplate;
-    }
-  }
-  protected _getPromptModules(): ModuleRecord {
-    return {};
-  }
-
-  getResponse(
-    query: ResponseBuilderQuery,
-    stream: true,
-  ): Promise<AsyncIterable<string>>;
-  getResponse(query: ResponseBuilderQuery, stream?: false): Promise<string>;
-  async getResponse(
-    { query, textChunks }: ResponseBuilderQuery,
-    stream?: boolean,
-  ): Promise<AsyncIterable<string> | string> {
-    const prompt = this.textQATemplate.format({
-      query: extractText(query),
-      context: textChunks.join("\n\n"),
-    });
-    if (stream) {
-      const response = await this.llm.complete({ prompt, stream: true });
-      return streamConverter(response, (chunk) => chunk.text);
-    } else {
-      const response = await this.llm.complete({ prompt, stream: false });
-      return response.text;
-    }
-  }
-}
+export type ResponseMode = z.infer<typeof responseModeSchema>;
 
 /**
  * A response builder that uses the query to ask the LLM generate a better response using multiple text chunks.
  */
-export class Refine extends PromptMixin implements ResponseBuilder {
-  llm: LLM;
-  promptHelper: PromptHelper;
+class Refine extends BaseSynthesizer {
   textQATemplate: TextQAPrompt;
   refineTemplate: RefinePrompt;
 
   constructor(
-    serviceContext?: ServiceContext,
-    textQATemplate?: TextQAPrompt,
-    refineTemplate?: RefinePrompt,
+    options: BaseSynthesizerOptions & {
+      textQATemplate?: TextQAPrompt | undefined;
+      refineTemplate?: RefinePrompt | undefined;
+    },
   ) {
-    super();
-
-    this.llm = llmFromSettingsOrContext(serviceContext);
-    this.promptHelper = promptHelperFromSettingsOrContext(serviceContext);
-    this.textQATemplate = textQATemplate ?? defaultTextQAPrompt;
-    this.refineTemplate = refineTemplate ?? defaultRefinePrompt;
+    super(options);
+    this.textQATemplate = options.textQATemplate ?? defaultTextQAPrompt;
+    this.refineTemplate = options.refineTemplate ?? defaultRefinePrompt;
   }
 
   protected _getPromptModules(): ModuleRecord {
@@ -132,41 +76,47 @@ export class Refine extends PromptMixin implements ResponseBuilder {
     }
   }
 
-  getResponse(
-    query: ResponseBuilderQuery,
-    stream: true,
-  ): Promise<AsyncIterable<string>>;
-  getResponse(query: ResponseBuilderQuery, stream?: false): Promise<string>;
   async getResponse(
-    { query, textChunks, prevResponse }: ResponseBuilderQuery,
-    stream?: boolean,
-  ): Promise<AsyncIterable<string> | string> {
-    let response: AsyncIterable<string> | string | undefined = prevResponse;
+    query: MessageContent,
+    nodes: NodeWithScore[],
+    stream: boolean,
+  ): Promise<EngineResponse | AsyncIterable<EngineResponse>> {
+    let response: AsyncIterable<string> | string | undefined = undefined;
+    const textChunks = nodes.map(({ node }) =>
+      node.getContent(MetadataMode.LLM),
+    );
 
     for (let i = 0; i < textChunks.length; i++) {
-      const chunk = textChunks[i]!;
+      const text = textChunks[i]!;
       const lastChunk = i === textChunks.length - 1;
       if (!response) {
         response = await this.giveResponseSingle(
           query,
-          chunk,
+          text,
           !!stream && lastChunk,
         );
       } else {
         response = await this.refineResponseSingle(
           response as string,
           query,
-          chunk,
+          text,
           !!stream && lastChunk,
         );
       }
     }
 
-    return response ?? "Empty Response";
+    // fixme: no source nodes provided, cannot fix right now due to lack of context
+    if (typeof response === "string") {
+      return EngineResponse.fromResponse(response, false);
+    } else {
+      return streamConverter(response!, (text) =>
+        EngineResponse.fromResponse(text, true),
+      );
+    }
   }
 
   private async giveResponseSingle(
-    query: QueryType,
+    query: MessageContent,
     textChunk: string,
     stream: boolean,
   ): Promise<AsyncIterable<string> | string> {
@@ -203,10 +153,10 @@ export class Refine extends PromptMixin implements ResponseBuilder {
   // eslint-disable-next-line max-params
   private async refineResponseSingle(
     initialReponse: string,
-    query: QueryType,
+    query: MessageContent,
     textChunk: string,
     stream: boolean,
-  ) {
+  ): Promise<AsyncIterable<string> | string> {
     const refineTemplate: RefinePrompt = this.refineTemplate.partialFormat({
       query: extractText(query),
     });
@@ -246,59 +196,54 @@ export class Refine extends PromptMixin implements ResponseBuilder {
 /**
  * CompactAndRefine is a slight variation of Refine that first compacts the text chunks into the smallest possible number of chunks.
  */
-export class CompactAndRefine extends Refine {
-  getResponse(
-    query: ResponseBuilderQuery,
-    stream: true,
-  ): Promise<AsyncIterable<string>>;
-  getResponse(query: ResponseBuilderQuery, stream?: false): Promise<string>;
+class CompactAndRefine extends Refine {
   async getResponse(
-    { query, textChunks, prevResponse }: ResponseBuilderQuery,
-    stream?: boolean,
-  ): Promise<AsyncIterable<string> | string> {
+    query: MessageContent,
+    nodes: NodeWithScore[],
+    stream: boolean,
+  ): Promise<EngineResponse | AsyncIterable<EngineResponse>> {
     const textQATemplate: TextQAPrompt = this.textQATemplate.partialFormat({
       query: extractText(query),
     });
     const refineTemplate: RefinePrompt = this.refineTemplate.partialFormat({
       query: extractText(query),
     });
+    const textChunks = nodes.map(({ node }) =>
+      node.getContent(MetadataMode.LLM),
+    );
 
     const maxPrompt = getBiggestPrompt([textQATemplate, refineTemplate]);
     const newTexts = this.promptHelper.repack(maxPrompt, textChunks);
-    const params = {
-      query,
-      textChunks: newTexts,
-      prevResponse,
-    };
+    const newNodes = newTexts.map((text) => new TextNode({ text }));
     if (stream) {
       return super.getResponse(
-        {
-          ...params,
-        },
+        query,
+        newNodes.map((node) => ({ node })),
         true,
       );
     }
-    return super.getResponse(params);
+    return super.getResponse(
+      query,
+      newNodes.map((node) => ({ node })),
+      false,
+    );
   }
 }
 
 /**
  * TreeSummarize repacks the text chunks into the smallest possible number of chunks and then summarizes them, then recursively does so until there's one chunk left.
  */
-export class TreeSummarize extends PromptMixin implements ResponseBuilder {
-  llm: LLM;
-  promptHelper: PromptHelper;
+class TreeSummarize extends BaseSynthesizer {
   summaryTemplate: TreeSummarizePrompt;
 
   constructor(
-    serviceContext?: ServiceContext,
-    summaryTemplate?: TreeSummarizePrompt,
+    options: BaseSynthesizerOptions & {
+      summaryTemplate?: TreeSummarizePrompt;
+    },
   ) {
-    super();
-
-    this.llm = llmFromSettingsOrContext(serviceContext);
-    this.promptHelper = promptHelperFromSettingsOrContext(serviceContext);
-    this.summaryTemplate = summaryTemplate ?? defaultTreeSummarizePrompt;
+    super(options);
+    this.summaryTemplate =
+      options.summaryTemplate ?? defaultTreeSummarizePrompt;
   }
 
   protected _getPromptModules(): ModuleRecord {
@@ -319,15 +264,14 @@ export class TreeSummarize extends PromptMixin implements ResponseBuilder {
     }
   }
 
-  getResponse(
-    query: ResponseBuilderQuery,
-    stream: true,
-  ): Promise<AsyncIterable<string>>;
-  getResponse(query: ResponseBuilderQuery, stream?: false): Promise<string>;
   async getResponse(
-    { query, textChunks }: ResponseBuilderQuery,
-    stream?: boolean,
-  ): Promise<AsyncIterable<string> | string> {
+    query: MessageContent,
+    nodes: NodeWithScore[],
+    stream: boolean,
+  ): Promise<EngineResponse | AsyncIterable<EngineResponse>> {
+    const textChunks = nodes.map(({ node }) =>
+      node.getContent(MetadataMode.LLM),
+    );
     if (!textChunks || textChunks.length === 0) {
       throw new Error("Must have at least one text chunk");
     }
@@ -347,9 +291,14 @@ export class TreeSummarize extends PromptMixin implements ResponseBuilder {
       };
       if (stream) {
         const response = await this.llm.complete({ ...params, stream });
-        return streamConverter(response, (chunk) => chunk.text);
+        return streamConverter(response, (chunk) =>
+          EngineResponse.fromResponse(chunk.text, true),
+        );
       }
-      return (await this.llm.complete(params)).text;
+      return EngineResponse.fromResponse(
+        (await this.llm.complete(params)).text,
+        false,
+      );
     } else {
       const summaries = await Promise.all(
         packedTextChunks.map((chunk) =>
@@ -362,40 +311,118 @@ export class TreeSummarize extends PromptMixin implements ResponseBuilder {
         ),
       );
 
-      const params = {
-        query,
-        textChunks: summaries.map((s) => s.text),
-      };
       if (stream) {
         return this.getResponse(
-          {
-            ...params,
-          },
+          query,
+          summaries.map((s) => ({
+            node: new TextNode({
+              text: s.text,
+            }),
+          })),
           true,
         );
       }
-      return this.getResponse(params);
+      return this.getResponse(
+        query,
+        summaries.map((s) => ({
+          node: new TextNode({
+            text: s.text,
+          }),
+        })),
+        false,
+      );
     }
   }
 }
 
-export function getResponseBuilder(
-  serviceContext?: ServiceContext,
-  responseMode?: ResponseMode,
-): ResponseBuilder {
-  switch (responseMode) {
-    case ResponseMode.SIMPLE:
-      return new SimpleResponseBuilder(serviceContext);
-    case ResponseMode.REFINE:
-      return new Refine(serviceContext);
-    case ResponseMode.TREE_SUMMARIZE:
-      return new TreeSummarize(serviceContext);
-    default:
-      return new CompactAndRefine(serviceContext);
+class MultiModal extends BaseSynthesizer {
+  metadataMode: MetadataMode;
+  textQATemplate: TextQAPrompt;
+
+  constructor({
+    textQATemplate,
+    metadataMode,
+    ...options
+  }: BaseSynthesizerOptions & {
+    textQATemplate?: TextQAPrompt;
+    metadataMode?: MetadataMode;
+  } = {}) {
+    super(options);
+
+    this.metadataMode = metadataMode ?? MetadataMode.NONE;
+    this.textQATemplate = textQATemplate ?? defaultTextQAPrompt;
+  }
+
+  protected _getPromptModules(): ModuleRecord {
+    return {};
+  }
+
+  protected _getPrompts(): { textQATemplate: TextQAPrompt } {
+    return {
+      textQATemplate: this.textQATemplate,
+    };
+  }
+
+  protected _updatePrompts(promptsDict: {
+    textQATemplate: TextQAPrompt;
+  }): void {
+    if (promptsDict.textQATemplate) {
+      this.textQATemplate = promptsDict.textQATemplate;
+    }
+  }
+
+  protected async getResponse(
+    query: MessageContent,
+    nodes: NodeWithScore[],
+    stream: boolean,
+  ): Promise<EngineResponse | AsyncIterable<EngineResponse>> {
+    const prompt = await createMessageContent(
+      this.textQATemplate,
+      nodes.map(({ node }) => node),
+      // this might not be good as this remove the image information
+      { query: extractText(query) },
+      this.metadataMode,
+    );
+
+    const llm = this.llm;
+
+    if (stream) {
+      const response = await llm.complete({
+        prompt,
+        stream,
+      });
+      return streamConverter(response, ({ text }) =>
+        EngineResponse.fromResponse(text, true),
+      );
+    }
+    const response = await llm.complete({
+      prompt,
+    });
+    return EngineResponse.fromResponse(response.text, false);
   }
 }
 
-export type ResponseBuilderPrompts =
-  | TextQAPrompt
-  | TreeSummarizePrompt
-  | RefinePrompt;
+export function getResponseSynthesizer(
+  mode: ResponseMode,
+  options: BaseSynthesizerOptions & {
+    textQATemplate?: TextQAPrompt;
+    refineTemplate?: RefinePrompt;
+    summaryTemplate?: TreeSummarizePrompt;
+    metadataMode?: MetadataMode;
+  } = {},
+) {
+  switch (mode) {
+    case "compact": {
+      return new CompactAndRefine(options);
+    }
+    case "refine": {
+      return new Refine(options);
+    }
+    case "tree_summarize": {
+      return new TreeSummarize(options);
+    }
+    case "multi_modal": {
+      return new MultiModal(options);
+    }
+  }
+}

packages/core/src/response-synthesizers/index.ts

@@ -0,0 +1,10 @@
+export {
+  BaseSynthesizer,
+  type BaseSynthesizerOptions,
+} from "./base-synthesizer";
+export { getResponseSynthesizer, type ResponseMode } from "./factory";
+export type {
+  SynthesizeEndEvent,
+  SynthesizeQuery,
+  SynthesizeStartEvent,
+} from "./type";

packages/core/src/response-synthesizers/type.ts

@@ -0,0 +1,19 @@
+import type { QueryType } from "../query-engine";
+import { EngineResponse, type NodeWithScore } from "../schema";
+
+export type SynthesizeQuery = {
+  query: QueryType;
+  nodes: NodeWithScore[];
+  additionalSourceNodes?: NodeWithScore[];
+};
+
+export type SynthesizeStartEvent = {
+  id: string;
+  query: SynthesizeQuery;
+};
+
+export type SynthesizeEndEvent = {
+  id: string;
+  query: SynthesizeQuery;
+  response: EngineResponse | AsyncIterable<EngineResponse>;
+};

packages/core/src/retriever/index.ts

@@ -0,0 +1,112 @@
+import { randomUUID } from "@llamaindex/env";
+import { Settings } from "../global";
+import type { MessageContent } from "../llms";
+import { PromptMixin } from "../prompts";
+import type { QueryBundle, QueryType } from "../query-engine";
+import { BaseNode, IndexNode, type NodeWithScore, ObjectType } from "../schema";
+
+export type RetrieveParams = {
+  query: MessageContent;
+  preFilters?: unknown;
+};
+
+export type RetrieveStartEvent = {
+  id: string;
+  query: QueryBundle;
+};
+
+export type RetrieveEndEvent = {
+  id: string;
+  query: QueryBundle;
+  nodes: NodeWithScore[];
+};
+
+export abstract class BaseRetriever extends PromptMixin {
+  objectMap: Map<string, unknown> = new Map();
+
+  protected _updatePrompts() {}
+  protected _getPrompts() {
+    return {};
+  }
+
+  protected _getPromptModules() {
+    return {};
+  }
+
+  protected constructor() {
+    super();
+  }
+
+  public async retrieve(params: QueryType): Promise<NodeWithScore[]> {
+    const cb = Settings.callbackManager;
+    const queryBundle = typeof params === "string" ? { query: params } : params;
+    const id = randomUUID();
+    cb.dispatchEvent("retrieve-start", { id, query: queryBundle });
+    let response = await this._retrieve(queryBundle);
+    response = await this._handleRecursiveRetrieval(queryBundle, response);
+    cb.dispatchEvent("retrieve-end", {
+      id,
+      query: queryBundle,
+      nodes: response,
+    });
+    return response;
+  }
+
+  abstract _retrieve(params: QueryBundle): Promise<NodeWithScore[]>;
+
+  async _handleRecursiveRetrieval(
+    params: QueryBundle,
+    nodes: NodeWithScore[],
+  ): Promise<NodeWithScore[]> {
+    const retrievedNodes = [];
+    for (const { node, score = 1.0 } of nodes) {
+      if (node.type === ObjectType.INDEX) {
+        const indexNode = node as IndexNode;
+        const object = this.objectMap.get(indexNode.indexId);
+        if (object !== undefined) {
+          retrievedNodes.push(
+            ...this._retrieveFromObject(object, params, score),
+          );
+        } else {
+          retrievedNodes.push({ node, score });
+        }
+      } else {
+        retrievedNodes.push({ node, score });
+      }
+    }
+    return nodes;
+  }
+
+  _retrieveFromObject(
+    object: unknown,
+    queryBundle: QueryBundle,
+    score: number,
+  ): NodeWithScore[] {
+    if (object == null) {
+      throw new TypeError("Object is not retrievable");
+    }
+    if (typeof object !== "object") {
+      throw new TypeError("Object is not retrievable");
+    }
+    if ("node" in object && object.node instanceof BaseNode) {
+      return [
+        {
+          node: object.node,
+          score:
+            "score" in object && typeof object.score === "number"
+              ? object.score
+              : score,
+        },
+      ];
+    }
+    if (object instanceof BaseNode) {
+      return [{ node: object, score }];
+    } else {
+      // todo: support other types
+      // BaseQueryEngine
+      // BaseRetriever
+      // QueryComponent
+      throw new TypeError("Object is not retrievable");
+    }
+  }
+}

packages/core/src/schema/node.ts

@@ -437,9 +437,16 @@ export function splitNodesByType(nodes: BaseNode[]): NodesByType {
 
   for (const node of nodes) {
     let type: ModalityType;
-    if (node instanceof ImageNode) {
+    if (
+      node.type === ObjectType.IMAGE ||
+      node.type === ObjectType.IMAGE_DOCUMENT
+    ) {
       type = ModalityType.IMAGE;
-    } else if (node instanceof TextNode) {
+    } else if (
+      node.type === ObjectType.TEXT ||
+      node.type === ObjectType.DOCUMENT ||
+      node.type === ObjectType.INDEX
+    ) {
       type = ModalityType.TEXT;
     } else {
       throw new Error(`Unknown node type: ${node.type}`);
@@ -465,28 +472,36 @@ export function buildNodeFromSplits(
   };
 
   textSplits.forEach((textChunk, i) => {
-    if (doc instanceof ImageDocument) {
+    if (
+      doc.type === ObjectType.IMAGE ||
+      doc.type === ObjectType.IMAGE_DOCUMENT
+    ) {
+      const imageDoc = doc as ImageNode;
       const imageNode = new ImageNode({
-        id_: idGenerator(i, doc),
+        id_: idGenerator(i, imageDoc),
         text: textChunk,
-        image: doc.image,
-        embedding: doc.embedding,
-        excludedEmbedMetadataKeys: [...doc.excludedEmbedMetadataKeys],
-        excludedLlmMetadataKeys: [...doc.excludedLlmMetadataKeys],
-        metadataSeparator: doc.metadataSeparator,
-        textTemplate: doc.textTemplate,
+        image: imageDoc.image,
+        embedding: imageDoc.embedding,
+        excludedEmbedMetadataKeys: [...imageDoc.excludedEmbedMetadataKeys],
+        excludedLlmMetadataKeys: [...imageDoc.excludedLlmMetadataKeys],
+        metadataSeparator: imageDoc.metadataSeparator,
+        textTemplate: imageDoc.textTemplate,
         relationships: { ...relationships },
       });
       nodes.push(imageNode);
-    } else if (doc instanceof Document || doc instanceof TextNode) {
+    } else if (
+      doc.type === ObjectType.DOCUMENT ||
+      doc.type === ObjectType.TEXT
+    ) {
+      const textDoc = doc as TextNode;
       const node = new TextNode({
-        id_: idGenerator(i, doc),
+        id_: idGenerator(i, textDoc),
         text: textChunk,
-        embedding: doc.embedding,
-        excludedEmbedMetadataKeys: [...doc.excludedEmbedMetadataKeys],
-        excludedLlmMetadataKeys: [...doc.excludedLlmMetadataKeys],
-        metadataSeparator: doc.metadataSeparator,
-        textTemplate: doc.textTemplate,
+        embedding: textDoc.embedding,
+        excludedEmbedMetadataKeys: [...textDoc.excludedEmbedMetadataKeys],
+        excludedLlmMetadataKeys: [...textDoc.excludedLlmMetadataKeys],
+        metadataSeparator: textDoc.metadataSeparator,
+        textTemplate: textDoc.textTemplate,
         relationships: { ...relationships },
       });
       nodes.push(node);

packages/core/src/storage/chat-store/base-chat-store.ts

@@ -0,0 +1,19 @@
+import type { ChatMessage } from "../../llms";
+
+export abstract class BaseChatStore<
+  AdditionalMessageOptions extends object = object,
+> {
+  abstract setMessages(
+    key: string,
+    messages: ChatMessage<AdditionalMessageOptions>[],
+  ): void;
+  abstract getMessages(key: string): ChatMessage<AdditionalMessageOptions>[];
+  abstract addMessage(
+    key: string,
+    message: ChatMessage<AdditionalMessageOptions>,
+    idx?: number,
+  ): void;
+  abstract deleteMessages(key: string): void;
+  abstract deleteMessage(key: string, idx: number): void;
+  abstract getKeys(): IterableIterator<string>;
+}

packages/core/src/storage/chat-store/index.ts

@@ -0,0 +1,2 @@
+export { BaseChatStore } from "./base-chat-store";
+export { SimpleChatStore } from "./simple-chat-store";

packages/core/src/storage/chat-store/simple-chat-store.ts

@@ -0,0 +1,43 @@
+import type { ChatMessage } from "../../llms";
+import { BaseChatStore } from "./base-chat-store";
+
+export class SimpleChatStore<
+  AdditionalMessageOptions extends object = object,
+> extends BaseChatStore<AdditionalMessageOptions> {
+  #store = new Map<string, ChatMessage<AdditionalMessageOptions>[]>();
+  setMessages(key: string, messages: ChatMessage<AdditionalMessageOptions>[]) {
+    this.#store.set(key, messages);
+  }
+
+  getMessages(key: string) {
+    return this.#store.get(key) ?? [];
+  }
+
+  addMessage(
+    key: string,
+    message: ChatMessage<AdditionalMessageOptions>,
+    idx?: number,
+  ) {
+    const messages = this.#store.get(key) ?? [];
+    if (idx === undefined) {
+      messages.push(message);
+    } else {
+      messages.splice(idx, 0, message);
+    }
+    this.#store.set(key, messages);
+  }
+
+  deleteMessages(key: string) {
+    this.#store.delete(key);
+  }
+
+  deleteMessage(key: string, idx: number) {
+    const messages = this.#store.get(key) ?? [];
+    messages.splice(idx, 1);
+    this.#store.set(key, messages);
+  }
+
+  getKeys() {
+    return this.#store.keys();
+  }
+}

packages/core/src/utils/index.ts

@@ -1,4 +1,4 @@
-export { wrapEventCaller } from "./event-caller";
+export { EventCaller, getEventCaller, wrapEventCaller } from "./event-caller";
 
 export async function* streamConverter<S, D>(
   stream: AsyncIterable<S>,
@@ -47,10 +47,12 @@ export async function* streamReducer<S, D>(params: {
 export { wrapLLMEvent } from "./wrap-llm-event";
 
 export {
+  createMessageContent,
   extractDataUrlComponents,
   extractImage,
   extractSingleText,
   extractText,
+  imageToDataUrl,
   messagesToHistory,
   toToolDescriptions,
 } from "./llms";

packages/core/src/utils/llms.ts

@@ -1,3 +1,5 @@
+import { fs } from "@llamaindex/env";
+import { filetypemime } from "magic-bytes.js";
 import type {
   ChatMessage,
   MessageContent,
@@ -5,8 +7,16 @@ import type {
   MessageContentTextDetail,
   ToolMetadata,
 } from "../llms";
+import type { BasePromptTemplate } from "../prompts";
 import type { QueryType } from "../query-engine";
 import type { ImageType } from "../schema";
+import {
+  type BaseNode,
+  ImageNode,
+  MetadataMode,
+  ModalityType,
+  splitNodesByType,
+} from "../schema";
 
 /**
  * Extracts just the text whether from
@@ -107,3 +117,99 @@ export function toToolDescriptions(tools: ToolMetadata[]): string {
 
   return JSON.stringify(toolsObj, null, 4);
 }
+
+async function blobToDataUrl(input: Blob) {
+  const buffer = Buffer.from(await input.arrayBuffer());
+  const mimes = filetypemime(buffer);
+  if (mimes.length < 1) {
+    throw new Error("Unsupported image type");
+  }
+  return "data:" + mimes[0] + ";base64," + buffer.toString("base64");
+}
+
+export async function imageToDataUrl(
+  input: ImageType | Uint8Array,
+): Promise<string> {
+  // first ensure, that the input is a Blob
+  if (
+    (input instanceof URL && input.protocol === "file:") ||
+    typeof input === "string"
+  ) {
+    // string or file URL
+    const dataBuffer = await fs.readFile(
+      input instanceof URL ? input.pathname : input,
+    );
+    input = new Blob([dataBuffer]);
+  } else if (!(input instanceof Blob)) {
+    if (input instanceof URL) {
+      throw new Error(`Unsupported URL with protocol: ${input.protocol}`);
+    } else if (input instanceof Uint8Array) {
+      input = new Blob([input]); // convert Uint8Array to Blob
+    } else {
+      throw new Error(`Unsupported input type: ${typeof input}`);
+    }
+  }
+  return await blobToDataUrl(input);
+}
+
+// eslint-disable-next-line max-params
+async function createContentPerModality(
+  prompt: BasePromptTemplate,
+  type: ModalityType,
+  nodes: BaseNode[],
+  extraParams: Record<string, string>,
+  metadataMode: MetadataMode,
+): Promise<MessageContentDetail[]> {
+  switch (type) {
+    case ModalityType.TEXT:
+      return [
+        {
+          type: "text",
+          text: prompt.format({
+            ...extraParams,
+            context: nodes.map((r) => r.getContent(metadataMode)).join("\n\n"),
+          }),
+        },
+      ];
+    case ModalityType.IMAGE:
+      const images: MessageContentDetail[] = await Promise.all(
+        (nodes as ImageNode[]).map(async (node) => {
+          return {
+            type: "image_url",
+            image_url: {
+              url: await imageToDataUrl(node.image),
+            },
+          } satisfies MessageContentDetail;
+        }),
+      );
+      return images;
+    default:
+      return [];
+  }
+}
+
+export async function createMessageContent(
+  prompt: BasePromptTemplate,
+  nodes: BaseNode[],
+  extraParams: Record<string, string> = {},
+  metadataMode: MetadataMode = MetadataMode.NONE,
+): Promise<MessageContentDetail[]> {
+  const content: MessageContentDetail[] = [];
+  const nodeMap = splitNodesByType(nodes);
+  for (const type in nodeMap) {
+    // for each retrieved modality type, create message content
+    const nodes = nodeMap[type as ModalityType];
+    if (nodes) {
+      content.push(
+        ...(await createContentPerModality(
+          prompt,
+          type as ModalityType,
+          nodes,
+          extraParams,
+          metadataMode,
+        )),
+      );
+    }
+  }
+  return content;
+}