Release 0.3.15 (#884)

Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>

.changeset/dry-queens-dream.md

@@ -1,5 +0,0 @@
----
-"llamaindex": patch
----
-
-Add vectorStores to storage context to define vector store per modality

.changeset/fuzzy-coins-breathe.md

@@ -1,6 +0,0 @@
----
-"llamaindex": patch
-"@llamaindex/examples": patch
----
-
-Added support for accessing Gemini via Vertex AI

.changeset/heavy-files-compare.md

@@ -1,5 +0,0 @@
----
-"llamaindex": patch
----
-
-Add system prompt to ContextChatEngine

apps/docs/CHANGELOG.md

@@ -1,5 +1,32 @@
 # docs
 
+## 0.0.23
+
+### Patch Changes
+
+- Updated dependencies [6e156ed]
+- Updated dependencies [265976d]
+- Updated dependencies [8e26f75]
+  - llamaindex@0.3.15
+
+## 0.0.22
+
+### Patch Changes
+
+- Updated dependencies [6ff7576]
+- Updated dependencies [94543de]
+  - llamaindex@0.3.14
+
+## 0.0.21
+
+### Patch Changes
+
+- Updated dependencies [1b1081b]
+- Updated dependencies [37525df]
+- Updated dependencies [660a2b3]
+- Updated dependencies [a1f2475]
+  - llamaindex@0.3.13
+
 ## 0.0.20
 
 ### Patch Changes

apps/docs/docs/getting_started/starter_tutorial/retrieval_augmented_generation.mdx

@@ -10,21 +10,19 @@ import TSConfigSource from "!!raw-loader!../../../../../examples/tsconfig.json";
 
 One of the most common use-cases for LlamaIndex is Retrieval-Augmented Generation or RAG, in which your data is indexed and selectively retrieved to be given to an LLM as source material for responding to a query. You can learn more about the [concepts behind RAG](../concepts).
 
-## Before you start
+## Set up the project
 
-Make sure you have installed LlamaIndex.TS and have an OpenAI key. If you haven't, check out the [installation](../installation) steps.
-
-You can use [other LLMs](../examples/other_llms) via their APIs; if you would prefer to use local models check out our [local LLM example](../../examples/local_llm).
-
-## Set up
-
-In a new folder:
+In a new folder, run:
 
 ```bash npm2yarn
 npm init
 npm install -D typescript @types/node
 ```
 
+Then, check out the [installation](../installation) steps to install LlamaIndex.TS and prepare an OpenAI key.
+
+You can use [other LLMs](../examples/other_llms) via their APIs; if you would prefer to use local models check out our [local LLM example](../../examples/local_llm).
+
 ## Run queries
 
 Create the file `example.ts`. This code will

apps/docs/package.json

@@ -1,6 +1,6 @@
 {
   "name": "docs",
-  "version": "0.0.20",
+  "version": "0.0.23",
   "private": true,
   "scripts": {
     "docusaurus": "docusaurus",

examples/multimodal/context.ts

@@ -0,0 +1,54 @@
+// call pnpm tsx multimodal/load.ts first to init the storage
+import {
+  ContextChatEngine,
+  NodeWithScore,
+  ObjectType,
+  OpenAI,
+  RetrievalEndEvent,
+  Settings,
+  VectorStoreIndex,
+} from "llamaindex";
+import { getStorageContext } from "./storage";
+
+// Update chunk size and overlap
+Settings.chunkSize = 512;
+Settings.chunkOverlap = 20;
+
+// Update llm
+Settings.llm = new OpenAI({ model: "gpt-4-turbo", maxTokens: 512 });
+
+// Update callbackManager
+Settings.callbackManager.on("retrieve-end", (event: RetrievalEndEvent) => {
+  const { nodes, query } = event.detail.payload;
+  const imageNodes = nodes.filter(
+    (node: NodeWithScore) => node.node.type === ObjectType.IMAGE_DOCUMENT,
+  );
+  const textNodes = nodes.filter(
+    (node: NodeWithScore) => node.node.type === ObjectType.TEXT,
+  );
+  console.log(
+    `Retrieved ${textNodes.length} text nodes and ${imageNodes.length} image nodes for query: ${query}`,
+  );
+});
+
+async function main() {
+  const storageContext = await getStorageContext();
+  const index = await VectorStoreIndex.init({
+    storageContext,
+  });
+  // topK for text is 0 and for image 1 => we only retrieve one image and no text based on the query
+  const retriever = index.asRetriever({ topK: { TEXT: 0, IMAGE: 1 } });
+  // NOTE: we set the contextRole to "user" (default is "system"). The reason is that GPT-4 does not support
+  // images in a system message
+  const chatEngine = new ContextChatEngine({ retriever, contextRole: "user" });
+
+  // the ContextChatEngine will use the Clip embedding to retrieve the closest image
+  // (the lady in the chair) and use it in the context for the query
+  const response = await chatEngine.chat({
+    message: "What is the name of the painting with the lady in the chair?",
+  });
+
+  console.log(response.response, "\n");
+}
+
+main().catch(console.error);

examples/multimodal/rag.ts

@@ -1,5 +1,4 @@
 import {
-  ImageType,
   MultiModalResponseSynthesizer,
   OpenAI,
   RetrievalEndEvent,
@@ -22,8 +21,6 @@ Settings.callbackManager.on("retrieve-end", (event: RetrievalEndEvent) => {
 });
 
 async function main() {
-  const images: ImageType[] = [];
-
   const storageContext = await getStorageContext();
   const index = await VectorStoreIndex.init({
     nodes: [],
@@ -34,13 +31,14 @@ async function main() {
     responseSynthesizer: new MultiModalResponseSynthesizer(),
     retriever: index.asRetriever({ topK: { TEXT: 3, IMAGE: 1 } }),
   });
-  const result = await queryEngine.query({
+  const stream = await queryEngine.query({
     query: "Tell me more about Vincent van Gogh's famous paintings",
+    stream: true,
   });
-  console.log(result.response, "\n");
-  images.forEach((image) =>
-    console.log(`Image retrieved and used in inference: ${image.toString()}`),
-  );
+  for await (const chunk of stream) {
+    process.stdout.write(chunk.response);
+  }
+  process.stdout.write("\n");
 }
 
 main().catch(console.error);

packages/autotool/examples/01_node/CHANGELOG.md

@@ -4,6 +4,36 @@
 
 ### Patch Changes
 
+- Updated dependencies [6e156ed]
+- Updated dependencies [265976d]
+- Updated dependencies [8e26f75]
+  - llamaindex@0.3.15
+  - @llamaindex/autotool@0.0.1
+
+## null
+
+### Patch Changes
+
+- Updated dependencies [6ff7576]
+- Updated dependencies [94543de]
+  - llamaindex@0.3.14
+  - @llamaindex/autotool@0.0.1
+
+## null
+
+### Patch Changes
+
+- Updated dependencies [1b1081b]
+- Updated dependencies [37525df]
+- Updated dependencies [660a2b3]
+- Updated dependencies [a1f2475]
+  - llamaindex@0.3.13
+  - @llamaindex/autotool@0.0.1
+
+## null
+
+### Patch Changes
+
 - Updated dependencies [34fb1d8]
   - llamaindex@0.3.12
   - @llamaindex/autotool@0.0.1

packages/autotool/examples/02_nextjs/CHANGELOG.md

@@ -1,5 +1,35 @@
 # @llamaindex/autotool-02-next-example
 
+## 0.1.7
+
+### Patch Changes
+
+- Updated dependencies [6e156ed]
+- Updated dependencies [265976d]
+- Updated dependencies [8e26f75]
+  - llamaindex@0.3.15
+  - @llamaindex/autotool@0.0.1
+
+## 0.1.6
+
+### Patch Changes
+
+- Updated dependencies [6ff7576]
+- Updated dependencies [94543de]
+  - llamaindex@0.3.14
+  - @llamaindex/autotool@0.0.1
+
+## 0.1.5
+
+### Patch Changes
+
+- Updated dependencies [1b1081b]
+- Updated dependencies [37525df]
+- Updated dependencies [660a2b3]
+- Updated dependencies [a1f2475]
+  - llamaindex@0.3.13
+  - @llamaindex/autotool@0.0.1
+
 ## 0.1.4
 
 ### Patch Changes

packages/autotool/examples/02_nextjs/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/autotool-02-next-example",
   "private": true,
-  "version": "0.1.4",
+  "version": "0.1.7",
   "scripts": {
     "dev": "next dev",
     "build": "next build",

packages/autotool/package.json

@@ -51,7 +51,7 @@
     "unplugin": "^1.10.1"
   },
   "peerDependencies": {
-    "llamaindex": "^0.3.12",
+    "llamaindex": "^0.3.15",
     "openai": "^4",
     "typescript": "^4"
   },

packages/core/CHANGELOG.md

@@ -1,5 +1,29 @@
 # llamaindex
 
+## 0.3.15
+
+### Patch Changes
+
+- 6e156ed: Use images in context chat engine
+- 265976d: fix bug with node decorator
+- 8e26f75: Add retrieval for images using multi-modal messages
+
+## 0.3.14
+
+### Patch Changes
+
+- 6ff7576: Added GPT-4o for Azure
+- 94543de: Added the latest preview gemini models and multi modal images taken into account
+
+## 0.3.13
+
+### Patch Changes
+
+- 1b1081b: Add vectorStores to storage context to define vector store per modality
+- 37525df: Added support for accessing Gemini via Vertex AI
+- 660a2b3: Fix text before heading in markdown reader
+- a1f2475: Add system prompt to ContextChatEngine
+
 ## 0.3.12
 
 ### Patch Changes

packages/core/e2e/examples/cloudflare-worker-agent/CHANGELOG.md

@@ -1,5 +1,32 @@
 # @llamaindex/cloudflare-worker-agent-test
 
+## 0.0.16
+
+### Patch Changes
+
+- Updated dependencies [6e156ed]
+- Updated dependencies [265976d]
+- Updated dependencies [8e26f75]
+  - llamaindex@0.3.15
+
+## 0.0.15
+
+### Patch Changes
+
+- Updated dependencies [6ff7576]
+- Updated dependencies [94543de]
+  - llamaindex@0.3.14
+
+## 0.0.14
+
+### Patch Changes
+
+- Updated dependencies [1b1081b]
+- Updated dependencies [37525df]
+- Updated dependencies [660a2b3]
+- Updated dependencies [a1f2475]
+  - llamaindex@0.3.13
+
 ## 0.0.13
 
 ### Patch Changes

packages/core/e2e/examples/cloudflare-worker-agent/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/cloudflare-worker-agent-test",
-  "version": "0.0.13",
+  "version": "0.0.16",
   "type": "module",
   "private": true,
   "scripts": {

packages/core/e2e/examples/nextjs-agent/CHANGELOG.md

@@ -1,5 +1,32 @@
 # @llamaindex/next-agent-test
 
+## 0.1.16
+
+### Patch Changes
+
+- Updated dependencies [6e156ed]
+- Updated dependencies [265976d]
+- Updated dependencies [8e26f75]
+  - llamaindex@0.3.15
+
+## 0.1.15
+
+### Patch Changes
+
+- Updated dependencies [6ff7576]
+- Updated dependencies [94543de]
+  - llamaindex@0.3.14
+
+## 0.1.14
+
+### Patch Changes
+
+- Updated dependencies [1b1081b]
+- Updated dependencies [37525df]
+- Updated dependencies [660a2b3]
+- Updated dependencies [a1f2475]
+  - llamaindex@0.3.13
+
 ## 0.1.13
 
 ### Patch Changes

packages/core/e2e/examples/nextjs-agent/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/next-agent-test",
-  "version": "0.1.13",
+  "version": "0.1.16",
   "private": true,
   "scripts": {
     "dev": "next dev",

packages/core/e2e/examples/nextjs-edge-runtime/CHANGELOG.md

@@ -1,5 +1,32 @@
 # test-edge-runtime
 
+## 0.1.15
+
+### Patch Changes
+
+- Updated dependencies [6e156ed]
+- Updated dependencies [265976d]
+- Updated dependencies [8e26f75]
+  - llamaindex@0.3.15
+
+## 0.1.14
+
+### Patch Changes
+
+- Updated dependencies [6ff7576]
+- Updated dependencies [94543de]
+  - llamaindex@0.3.14
+
+## 0.1.13
+
+### Patch Changes
+
+- Updated dependencies [1b1081b]
+- Updated dependencies [37525df]
+- Updated dependencies [660a2b3]
+- Updated dependencies [a1f2475]
+  - llamaindex@0.3.13
+
 ## 0.1.12
 
 ### Patch Changes

packages/core/e2e/examples/nextjs-edge-runtime/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/nextjs-edge-runtime-test",
-  "version": "0.1.12",
+  "version": "0.1.15",
   "private": true,
   "scripts": {
     "dev": "next dev",

packages/core/e2e/examples/waku-query-engine/CHANGELOG.md

@@ -1,5 +1,32 @@
 # @llamaindex/waku-query-engine-test
 
+## 0.0.16
+
+### Patch Changes
+
+- Updated dependencies [6e156ed]
+- Updated dependencies [265976d]
+- Updated dependencies [8e26f75]
+  - llamaindex@0.3.15
+
+## 0.0.15
+
+### Patch Changes
+
+- Updated dependencies [6ff7576]
+- Updated dependencies [94543de]
+  - llamaindex@0.3.14
+
+## 0.0.14
+
+### Patch Changes
+
+- Updated dependencies [1b1081b]
+- Updated dependencies [37525df]
+- Updated dependencies [660a2b3]
+- Updated dependencies [a1f2475]
+  - llamaindex@0.3.13
+
 ## 0.0.13
 
 ### Patch Changes

packages/core/e2e/examples/waku-query-engine/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/waku-query-engine-test",
-  "version": "0.0.13",
+  "version": "0.0.16",
   "type": "module",
   "private": true,
   "scripts": {

packages/core/e2e/fixtures/embeddings/OpenAIEmbedding.ts

@@ -1,9 +1,14 @@
-import { BaseNode, SimilarityType, type BaseEmbedding } from "llamaindex";
+import {
+  BaseNode,
+  SimilarityType,
+  type BaseEmbedding,
+  type MessageContentDetail,
+} from "llamaindex";
 
 export class OpenAIEmbedding implements BaseEmbedding {
   embedBatchSize = 512;
 
-  async getQueryEmbedding(query: string) {
+  async getQueryEmbedding(query: MessageContentDetail) {
     return [0];
   }
 

packages/core/jsr.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/core",
-  "version": "0.3.12",
+  "version": "0.3.15",
   "exports": "./src/index.ts",
   "imports": {
     "@llamaindex/env": "jsr:@llamaindex/env@0.1.3"

packages/core/package.json

@@ -1,6 +1,6 @@
 {
   "name": "llamaindex",
-  "version": "0.3.12",
+  "version": "0.3.15",
   "expectedMinorVersion": "3",
   "license": "MIT",
   "type": "module",

packages/core/src/Retriever.ts

@@ -1,8 +1,9 @@
 import type { NodeWithScore } from "./Node.js";
 import type { ServiceContext } from "./ServiceContext.js";
+import type { MessageContent } from "./index.edge.js";
 
 export type RetrieveParams = {
-  query: string;
+  query: MessageContent;
   preFilters?: unknown;
 };
 

packages/core/src/agent/anthropic.ts

@@ -39,9 +39,10 @@ export class AnthropicAgent extends AgentRunner<Anthropic> {
   constructor(params: AnthropicAgentParams) {
     super({
       llm:
-        params.llm ?? Settings.llm instanceof Anthropic
+        params.llm ??
+        (Settings.llm instanceof Anthropic
           ? (Settings.llm as Anthropic)
-          : new Anthropic(),
+          : new Anthropic()),
       chatHistory: params.chatHistory ?? [],
       systemPrompt: params.systemPrompt ?? null,
       runner: new AnthropicAgentWorker(),

packages/core/src/agent/openai.ts

@@ -36,9 +36,10 @@ export class OpenAIAgent extends AgentRunner<OpenAI> {
   constructor(params: OpenAIAgentParams) {
     super({
       llm:
-        params.llm ?? Settings.llm instanceof OpenAI
+        params.llm ??
+        (Settings.llm instanceof OpenAI
           ? (Settings.llm as OpenAI)
-          : new OpenAI(),
+          : new OpenAI()),
       chatHistory: params.chatHistory ?? [],
       runner: new OpenAIAgentWorker(),
       systemPrompt: params.systemPrompt ?? null,

packages/core/src/callbacks/CallbackManager.ts

@@ -12,6 +12,7 @@ import type {
   LLMStreamEvent,
   LLMToolCallEvent,
   LLMToolResultEvent,
+  MessageContent,
   RetrievalEndEvent,
   RetrievalStartEvent,
 } from "../llm/types.js";
@@ -99,7 +100,7 @@ export interface StreamCallbackResponse {
 }
 
 export interface RetrievalCallbackResponse {
-  query: string;
+  query: MessageContent;
   nodes: NodeWithScore[];
 }
 

packages/core/src/cloud/LlamaCloudRetriever.ts

@@ -2,8 +2,9 @@ import type { PlatformApi, PlatformApiClient } from "@llamaindex/cloud";
 import type { NodeWithScore } from "../Node.js";
 import { ObjectType, jsonToNode } from "../Node.js";
 import type { BaseRetriever, RetrieveParams } from "../Retriever.js";
-import { Settings } from "../Settings.js";
 import { wrapEventCaller } from "../internal/context/EventCaller.js";
+import { getCallbackManager } from "../internal/settings/CallbackManager.js";
+import { extractText } from "../llm/utils.js";
 import type { ClientParams, CloudConstructorParams } from "./types.js";
 import { DEFAULT_PROJECT_NAME } from "./types.js";
 import { getClient } from "./utils.js";
@@ -70,13 +71,13 @@ export class LlamaCloudRetriever implements BaseRetriever {
       await this.getClient()
     ).pipeline.runSearch(pipelines[0].id, {
       ...this.retrieveParams,
-      query,
+      query: extractText(query),
       searchFilters: preFilters as Record<string, unknown[]>,
     });
 
     const nodes = this.resultNodesToNodeWithScore(results.retrievalNodes);
 
-    Settings.callbackManager.dispatchEvent("retrieve", {
+    getCallbackManager().dispatchEvent("retrieve", {
       query,
       nodes,
     });

packages/core/src/embeddings/ClipEmbedding.ts

@@ -87,8 +87,4 @@ export class ClipEmbedding extends MultiModalEmbedding {
     const { text_embeds } = await (await this.getTextModel())(textInputs);
     return text_embeds.data;
   }
-
-  async getQueryEmbedding(query: string): Promise<number[]> {
-    return this.getTextEmbedding(query);
-  }
 }

packages/core/src/embeddings/GeminiEmbedding.ts

@@ -36,8 +36,4 @@ export class GeminiEmbedding extends BaseEmbedding {
   getTextEmbedding(text: string): Promise<number[]> {
     return this.getEmbedding(text);
   }
-
-  getQueryEmbedding(query: string): Promise<number[]> {
-    return this.getTextEmbedding(query);
-  }
 }

packages/core/src/embeddings/HuggingFaceEmbedding.ts

@@ -45,8 +45,4 @@ export class HuggingFaceEmbedding extends BaseEmbedding {
     const output = await extractor(text, { pooling: "mean", normalize: true });
     return Array.from(output.data);
   }
-
-  async getQueryEmbedding(query: string): Promise<number[]> {
-    return this.getTextEmbedding(query);
-  }
 }

packages/core/src/embeddings/MistralAIEmbedding.ts

@@ -30,8 +30,4 @@ export class MistralAIEmbedding extends BaseEmbedding {
   async getTextEmbedding(text: string): Promise<number[]> {
     return this.getMistralAIEmbedding(text);
   }
-
-  async getQueryEmbedding(query: string): Promise<number[]> {
-    return this.getMistralAIEmbedding(query);
-  }
 }

packages/core/src/embeddings/MultiModalEmbedding.ts

@@ -6,6 +6,8 @@ import {
   type BaseNode,
   type ImageType,
 } from "../Node.js";
+import type { MessageContentDetail } from "../llm/types.js";
+import { extractImage, extractSingleText } from "../llm/utils.js";
 import { BaseEmbedding, batchEmbeddings } from "./types.js";
 
 /*
@@ -52,4 +54,18 @@ export abstract class MultiModalEmbedding extends BaseEmbedding {
 
     return nodes;
   }
+
+  async getQueryEmbedding(
+    query: MessageContentDetail,
+  ): Promise<number[] | null> {
+    const image = extractImage(query);
+    if (image) {
+      return await this.getImageEmbedding(image);
+    }
+    const text = extractSingleText(query);
+    if (text) {
+      return await this.getTextEmbedding(text);
+    }
+    return null;
+  }
 }

packages/core/src/embeddings/OpenAIEmbedding.ts

@@ -133,13 +133,4 @@ export class OpenAIEmbedding extends BaseEmbedding {
   async getTextEmbedding(text: string): Promise<number[]> {
     return (await this.getOpenAIEmbedding([text]))[0];
   }
-
-  /**
-   * Get embeddings for a query
-   * @param texts
-   * @param options
-   */
-  async getQueryEmbedding(query: string): Promise<number[]> {
-    return (await this.getOpenAIEmbedding([query]))[0];
-  }
 }

packages/core/src/embeddings/types.ts

@@ -1,6 +1,8 @@
 import type { BaseNode } from "../Node.js";
 import { MetadataMode } from "../Node.js";
 import type { TransformComponent } from "../ingestion/types.js";
+import type { MessageContentDetail } from "../llm/types.js";
+import { extractSingleText } from "../llm/utils.js";
 import { SimilarityType, similarity } from "./utils.js";
 
 const DEFAULT_EMBED_BATCH_SIZE = 10;
@@ -19,7 +21,16 @@ export abstract class BaseEmbedding implements TransformComponent {
   }
 
   abstract getTextEmbedding(text: string): Promise<number[]>;
-  abstract getQueryEmbedding(query: string): Promise<number[]>;
+
+  async getQueryEmbedding(
+    query: MessageContentDetail,
+  ): Promise<number[] | null> {
+    const text = extractSingleText(query);
+    if (text) {
+      return await this.getTextEmbedding(text);
+    }
+    return null;
+  }
 
   /**
    * Optionally override this method to retrieve multiple embeddings in a single request

packages/core/src/engines/chat/ContextChatEngine.ts

@@ -3,10 +3,10 @@ import { getHistory } from "../../ChatHistory.js";
 import type { ContextSystemPrompt } from "../../Prompt.js";
 import { Response } from "../../Response.js";
 import type { BaseRetriever } from "../../Retriever.js";
+import { Settings } from "../../Settings.js";
 import { wrapEventCaller } from "../../internal/context/EventCaller.js";
 import type { ChatMessage, ChatResponseChunk, LLM } from "../../llm/index.js";
-import { OpenAI } from "../../llm/index.js";
-import type { MessageContent } from "../../llm/types.js";
+import type { MessageContent, MessageType } from "../../llm/types.js";
 import {
   extractText,
   streamConverter,
@@ -40,15 +40,16 @@ export class ContextChatEngine extends PromptMixin implements ChatEngine {
     contextSystemPrompt?: ContextSystemPrompt;
     nodePostprocessors?: BaseNodePostprocessor[];
     systemPrompt?: string;
+    contextRole?: MessageType;
   }) {
     super();
-    this.chatModel =
-      init.chatModel ?? new OpenAI({ model: "gpt-3.5-turbo-16k" });
+    this.chatModel = init.chatModel ?? Settings.llm;
     this.chatHistory = getHistory(init?.chatHistory);
     this.contextGenerator = new DefaultContextGenerator({
       retriever: init.retriever,
       contextSystemPrompt: init?.contextSystemPrompt,
       nodePostprocessors: init?.nodePostprocessors,
+      contextRole: init?.contextRole,
     });
     this.systemPrompt = init.systemPrompt;
   }

packages/core/src/engines/chat/DefaultContextGenerator.ts

@@ -1,9 +1,11 @@
-import type { NodeWithScore, TextNode } from "../../Node.js";
+import { type NodeWithScore } from "../../Node.js";
 import type { ContextSystemPrompt } from "../../Prompt.js";
 import { defaultContextSystemPrompt } from "../../Prompt.js";
 import type { BaseRetriever } from "../../Retriever.js";
+import type { MessageContent, MessageType } from "../../llm/types.js";
 import type { BaseNodePostprocessor } from "../../postprocessors/index.js";
 import { PromptMixin } from "../../prompts/index.js";
+import { createMessageContent } from "../../synthesizers/utils.js";
 import type { Context, ContextGenerator } from "./types.js";
 
 export class DefaultContextGenerator
@@ -13,11 +15,13 @@ export class DefaultContextGenerator
   retriever: BaseRetriever;
   contextSystemPrompt: ContextSystemPrompt;
   nodePostprocessors: BaseNodePostprocessor[];
+  contextRole: MessageType;
 
   constructor(init: {
     retriever: BaseRetriever;
     contextSystemPrompt?: ContextSystemPrompt;
     nodePostprocessors?: BaseNodePostprocessor[];
+    contextRole?: MessageType;
   }) {
     super();
 
@@ -25,6 +29,7 @@ export class DefaultContextGenerator
     this.contextSystemPrompt =
       init?.contextSystemPrompt ?? defaultContextSystemPrompt;
     this.nodePostprocessors = init.nodePostprocessors || [];
+    this.contextRole = init.contextRole ?? "system";
   }
 
   protected _getPrompts(): { contextSystemPrompt: ContextSystemPrompt } {
@@ -41,7 +46,10 @@ export class DefaultContextGenerator
     }
   }
 
-  private async applyNodePostprocessors(nodes: NodeWithScore[], query: string) {
+  private async applyNodePostprocessors(
+    nodes: NodeWithScore[],
+    query: MessageContent,
+  ) {
     let nodesWithScore = nodes;
 
     for (const postprocessor of this.nodePostprocessors) {
@@ -54,7 +62,7 @@ export class DefaultContextGenerator
     return nodesWithScore;
   }
 
-  async generate(message: string): Promise<Context> {
+  async generate(message: MessageContent): Promise<Context> {
     const sourceNodesWithScore = await this.retriever.retrieve({
       query: message,
     });
@@ -64,12 +72,15 @@ export class DefaultContextGenerator
       message,
     );
 
+    const content = await createMessageContent(
+      this.contextSystemPrompt,
+      nodes.map((r) => r.node),
+    );
+
     return {
       message: {
-        content: this.contextSystemPrompt({
-          context: nodes.map((r) => (r.node as TextNode).text).join("\n\n"),
-        }),
-        role: "system",
+        content,
+        role: this.contextRole,
       },
       nodes,
     };

packages/core/src/engines/chat/SimpleChatEngine.ts

@@ -1,9 +1,9 @@
 import type { ChatHistory } from "../../ChatHistory.js";
 import { getHistory } from "../../ChatHistory.js";
 import { Response } from "../../Response.js";
+import { Settings } from "../../Settings.js";
 import { wrapEventCaller } from "../../internal/context/EventCaller.js";
 import type { ChatResponseChunk, LLM } from "../../llm/index.js";
-import { OpenAI } from "../../llm/index.js";
 import {
   extractText,
   streamConverter,
@@ -25,7 +25,7 @@ export class SimpleChatEngine implements ChatEngine {
 
   constructor(init?: Partial<SimpleChatEngine>) {
     this.chatHistory = getHistory(init?.chatHistory);
-    this.llm = init?.llm ?? new OpenAI();
+    this.llm = init?.llm ?? Settings.llm;
   }
 
   chat(params: ChatEngineParamsStreaming): Promise<AsyncIterable<Response>>;

packages/core/src/indices/keyword/index.ts

@@ -29,6 +29,7 @@ import {
 
 import { llmFromSettingsOrContext } from "../../Settings.js";
 import type { LLM } from "../../llm/types.js";
+import { extractText } from "../../llm/utils.js";
 
 export interface KeywordIndexOptions {
   nodes?: BaseNode[];
@@ -85,7 +86,7 @@ abstract class BaseKeywordTableRetriever implements BaseRetriever {
   abstract getKeywords(query: string): Promise<string[]>;
 
   async retrieve({ query }: RetrieveParams): Promise<NodeWithScore[]> {
-    const keywords = await this.getKeywords(query);
+    const keywords = await this.getKeywords(extractText(query));
     const chunkIndicesCount: { [key: string]: number } = {};
     const filteredKeywords = keywords.filter((keyword) =>
       this.indexStruct.table.has(keyword),

packages/core/src/indices/summary/index.ts

@@ -11,6 +11,7 @@ import {
 } from "../../Settings.js";
 import { RetrieverQueryEngine } from "../../engines/query/index.js";
 import { wrapEventCaller } from "../../internal/context/EventCaller.js";
+import { extractText } from "../../llm/utils.js";
 import type { BaseNodePostprocessor } from "../../postprocessors/index.js";
 import type { StorageContext } from "../../storage/StorageContext.js";
 import { storageContextFromDefaults } from "../../storage/StorageContext.js";
@@ -343,7 +344,7 @@ export class SummaryIndexLLMRetriever implements BaseRetriever {
       const nodesBatch = await this.index.docStore.getNodes(nodeIdsBatch);
 
       const fmtBatchStr = this.formatNodeBatchFn(nodesBatch);
-      const input = { context: fmtBatchStr, query: query };
+      const input = { context: fmtBatchStr, query: extractText(query) };
 
       const llm = llmFromSettingsOrContext(this.serviceContext);
 

packages/core/src/indices/vectorStore/index.ts

@@ -23,6 +23,7 @@ import {
 } from "../../ingestion/strategies/index.js";
 import { wrapEventCaller } from "../../internal/context/EventCaller.js";
 import { getCallbackManager } from "../../internal/settings/CallbackManager.js";
+import type { MessageContent } from "../../llm/types.js";
 import type { BaseNodePostprocessor } from "../../postprocessors/types.js";
 import type { StorageContext } from "../../storage/StorageContext.js";
 import { storageContextFromDefaults } from "../../storage/StorageContext.js";
@@ -30,7 +31,6 @@ import type {
   MetadataFilters,
   VectorStore,
   VectorStoreByType,
-  VectorStoreQuery,
   VectorStoreQueryResult,
 } from "../../storage/index.js";
 import type { BaseIndexStore } from "../../storage/indexStore/types.js";
@@ -422,10 +422,9 @@ export class VectorIndexRetriever implements BaseRetriever {
     let nodesWithScores: NodeWithScore[] = [];
 
     for (const type in vectorStores) {
-      // TODO: add retrieval by using an image as query
       const vectorStore: VectorStore = vectorStores[type as ModalityType]!;
       nodesWithScores = nodesWithScores.concat(
-        await this.textRetrieve(
+        await this.retrieveQuery(
           query,
           type as ModalityType,
           vectorStore,
@@ -447,36 +446,33 @@ export class VectorIndexRetriever implements BaseRetriever {
     return nodesWithScores;
   }
 
-  protected async textRetrieve(
-    query: string,
+  protected async retrieveQuery(
+    query: MessageContent,
     type: ModalityType,
     vectorStore: VectorStore,
     preFilters?: MetadataFilters,
   ): Promise<NodeWithScore[]> {
-    const q = await this.buildVectorStoreQuery(
-      this.index.embedModel ?? vectorStore.embedModel,
-      query,
-      this.topK[type],
-      preFilters,
-    );
-    const result = await vectorStore.query(q);
-    return this.buildNodeListFromQueryResult(result);
-  }
-
-  protected async buildVectorStoreQuery(
-    embedModel: BaseEmbedding,
-    query: string,
-    similarityTopK: number,
-    preFilters?: MetadataFilters,
-  ): Promise<VectorStoreQuery> {
-    const queryEmbedding = await embedModel.getQueryEmbedding(query);
-
-    return {
-      queryEmbedding,
-      mode: VectorStoreQueryMode.DEFAULT,
-      similarityTopK,
-      filters: preFilters ?? undefined,
-    };
+    // convert string message to multi-modal format
+    if (typeof query === "string") {
+      query = [{ type: "text", text: query }];
+    }
+    // overwrite embed model if specified, otherwise use the one from the vector store
+    const embedModel = this.index.embedModel ?? vectorStore.embedModel;
+    let nodes: NodeWithScore[] = [];
+    // query each content item (e.g. text or image) separately
+    for (const item of query) {
+      const queryEmbedding = await embedModel.getQueryEmbedding(item);
+      if (queryEmbedding) {
+        const result = await vectorStore.query({
+          queryEmbedding,
+          mode: VectorStoreQueryMode.DEFAULT,
+          similarityTopK: this.topK[type],
+          filters: preFilters ?? undefined,
+        });
+        nodes = nodes.concat(this.buildNodeListFromQueryResult(result));
+      }
+    }
+    return nodes;
   }
 
   protected buildNodeListFromQueryResult(result: VectorStoreQueryResult) {

packages/core/src/internal/decorator/node.ts

@@ -4,12 +4,19 @@ import { getChunkSize } from "../settings/chunk-size.js";
 
 const emitOnce = false;
 
-export function chunkSizeCheck(
-  contentGetter: () => string,
-  _context: ClassMethodDecoratorContext | ClassGetterDecoratorContext,
+export function chunkSizeCheck<
+  This extends BaseNode,
+  Args extends any[],
+  Return,
+>(
+  contentGetter: (this: This, ...args: Args) => string,
+  _context: ClassMethodDecoratorContext<
+    This,
+    (this: This, ...args: Args) => Return
+  >,
 ) {
-  return function <Node extends BaseNode>(this: Node) {
-    const content = contentGetter.call(this);
+  return function (this: This, ...args: Args) {
+    const content = contentGetter.call(this, ...args);
     const chunkSize = getChunkSize();
     const enableChunkSizeCheck = getEnv("ENABLE_CHUNK_SIZE_CHECK") === "true";
     if (

packages/core/src/llm/azure.ts

@@ -17,12 +17,17 @@ const ALL_AZURE_OPENAI_CHAT_MODELS = {
     contextWindow: 16384,
     openAIModel: "gpt-3.5-turbo-16k",
   },
+  "gpt-4o": { contextWindow: 128000, openAIModel: "gpt-4o" },
   "gpt-4": { contextWindow: 8192, openAIModel: "gpt-4" },
   "gpt-4-32k": { contextWindow: 32768, openAIModel: "gpt-4-32k" },
   "gpt-4-turbo": {
     contextWindow: 128000,
     openAIModel: "gpt-4-turbo",
   },
+  "gpt-4-turbo-2024-04-09": {
+    contextWindow: 128000,
+    openAIModel: "gpt-4-turbo",
+  },
   "gpt-4-vision-preview": {
     contextWindow: 128000,
     openAIModel: "gpt-4-vision-preview",
@@ -31,6 +36,10 @@ const ALL_AZURE_OPENAI_CHAT_MODELS = {
     contextWindow: 128000,
     openAIModel: "gpt-4-1106-preview",
   },
+  "gpt-4o-2024-05-13": {
+    contextWindow: 128000,
+    openAIModel: "gpt-4o-2024-05-13",
+  },
 };
 
 const ALL_AZURE_OPENAI_EMBEDDING_MODELS = {
@@ -62,6 +71,8 @@ const ALL_AZURE_API_VERSIONS = [
   "2024-02-01",
   "2024-02-15-preview",
   "2024-03-01-preview",
+  "2024-04-01-preview",
+  "2024-05-01-preview",
 ];
 
 const DEFAULT_API_VERSION = "2023-05-15";

packages/core/src/llm/gemini/base.ts

@@ -35,11 +35,16 @@ export const GEMINI_MODEL_INFO_MAP: Record<GEMINI_MODEL, GeminiModelInfo> = {
   [GEMINI_MODEL.GEMINI_PRO]: { contextWindow: 30720 },
   [GEMINI_MODEL.GEMINI_PRO_VISION]: { contextWindow: 12288 },
   [GEMINI_MODEL.GEMINI_PRO_LATEST]: { contextWindow: 10 ** 6 },
+  // multi-modal/multi turn
+  [GEMINI_MODEL.GEMINI_PRO_1_5_PRO_PREVIEW]: { contextWindow: 10 ** 6 },
+  [GEMINI_MODEL.GEMINI_PRO_1_5_FLASH_PREVIEW]: { contextWindow: 10 ** 6 },
 };
 
 const SUPPORT_TOOL_CALL_MODELS: GEMINI_MODEL[] = [
   GEMINI_MODEL.GEMINI_PRO,
   GEMINI_MODEL.GEMINI_PRO_VISION,
+  GEMINI_MODEL.GEMINI_PRO_1_5_PRO_PREVIEW,
+  GEMINI_MODEL.GEMINI_PRO_1_5_FLASH_PREVIEW,
 ];
 
 const DEFAULT_GEMINI_PARAMS = {

packages/core/src/llm/gemini/types.ts

@@ -51,6 +51,8 @@ export enum GEMINI_MODEL {
   GEMINI_PRO = "gemini-pro",
   GEMINI_PRO_VISION = "gemini-pro-vision",
   GEMINI_PRO_LATEST = "gemini-1.5-pro-latest",
+  GEMINI_PRO_1_5_PRO_PREVIEW = "gemini-1.5-pro-preview-0514",
+  GEMINI_PRO_1_5_FLASH_PREVIEW = "gemini-1.5-flash-preview-0514",
 }
 
 export interface GeminiModelInfo {

packages/core/src/llm/gemini/utils.ts

@@ -99,7 +99,13 @@ export const cleanParts = (
 ): GeminiMessageContent => {
   return {
     ...message,
-    parts: message.parts.filter((part) => part.text?.trim()),
+    parts: message.parts.filter(
+      (part) =>
+        part.text?.trim() ||
+        part.inlineData ||
+        part.fileData ||
+        part.functionCall,
+    ),
   };
 };
 
@@ -147,24 +153,28 @@ export class GeminiHelper {
   public static mergeNeighboringSameRoleMessages(
     messages: GeminiMessageContent[],
   ): GeminiMessageContent[] {
-    return messages.reduce(
-      (
-        result: GeminiMessageContent[],
-        current: GeminiMessageContent,
-        index: number,
-      ) => {
-        if (index > 0 && messages[index - 1].role === current.role) {
-          result[result.length - 1].parts = [
-            ...result[result.length - 1].parts,
-            ...current.parts,
-          ];
-        } else {
-          result.push(current);
-        }
-        return result;
-      },
-      [],
-    );
+    return messages
+      .map(cleanParts)
+      .filter((message) => message.parts.length)
+      .reduce(
+        (
+          result: GeminiMessageContent[],
+          current: GeminiMessageContent,
+          index: number,
+          original: GeminiMessageContent[],
+        ) => {
+          if (index > 0 && original[index - 1].role === current.role) {
+            result[result.length - 1].parts = [
+              ...result[result.length - 1].parts,
+              ...current.parts,
+            ];
+          } else {
+            result.push(current);
+          }
+          return result;
+        },
+        [],
+      );
   }
 
   public static messageContentToGeminiParts(content: MessageContent): Part[] {

packages/core/src/llm/ollama.ts

@@ -191,10 +191,6 @@ export class Ollama
     return this.getEmbedding(text);
   }
 
-  async getQueryEmbedding(query: string): Promise<number[]> {
-    return this.getEmbedding(query);
-  }
-
   // Inherited from OllamaBase
 
   push(

packages/core/src/llm/types.ts

@@ -4,10 +4,10 @@ import type { BaseEvent } from "../internal/type.js";
 import type { BaseTool, JSONObject, ToolOutput, UUID } from "../types.js";
 
 export type RetrievalStartEvent = BaseEvent<{
-  query: string;
+  query: MessageContent;
 }>;
 export type RetrievalEndEvent = BaseEvent<{
-  query: string;
+  query: MessageContent;
   nodes: NodeWithScore[];
 }>;
 export type LLMStartEvent = BaseEvent<{

packages/core/src/llm/utils.ts

@@ -1,4 +1,5 @@
 import { AsyncLocalStorage, randomUUID } from "@llamaindex/env";
+import type { ImageType } from "../Node.js";
 import { getCallbackManager } from "../internal/settings/CallbackManager.js";
 import type {
   ChatResponse,
@@ -6,6 +7,7 @@ import type {
   LLM,
   LLMChat,
   MessageContent,
+  MessageContentDetail,
   MessageContentTextDetail,
 } from "./types.js";
 
@@ -62,7 +64,7 @@ export async function* streamReducer<S, D>(params: {
 export function extractText(message: MessageContent): string {
   if (typeof message !== "string" && !Array.isArray(message)) {
     console.warn(
-      "extractText called with non-string message, this is likely a bug.",
+      "extractText called with non-MessageContent message, this is likely a bug.",
     );
     return `${message}`;
   } else if (typeof message !== "string" && Array.isArray(message)) {
@@ -77,6 +79,34 @@ export function extractText(message: MessageContent): string {
   }
 }
 
+/**
+ * Extracts a single text from a multi-modal message content
+ *
+ * @param message The message to extract images from.
+ * @returns The extracted images
+ */
+export function extractSingleText(
+  message: MessageContentDetail,
+): string | null {
+  if (message.type === "text") {
+    return message.text;
+  }
+  return null;
+}
+
+/**
+ * Extracts an image from a multi-modal message content
+ *
+ * @param message The message to extract images from.
+ * @returns The extracted images
+ */
+export function extractImage(message: MessageContentDetail): ImageType | null {
+  if (message.type === "image_url") {
+    return new URL(message.image_url.url);
+  }
+  return null;
+}
+
 export const extractDataUrlComponents = (
   dataUrl: string,
 ): {

packages/core/src/postprocessors/rerankers/CohereRerank.ts

@@ -2,6 +2,8 @@ import { CohereClient } from "cohere-ai";
 
 import type { NodeWithScore } from "../../Node.js";
 import { MetadataMode } from "../../Node.js";
+import type { MessageContent } from "../../llm/types.js";
+import { extractText } from "../../llm/utils.js";
 import type { BaseNodePostprocessor } from "../types.js";
 
 type CohereRerankOptions = {
@@ -46,7 +48,7 @@ export class CohereRerank implements BaseNodePostprocessor {
    */
   async postprocessNodes(
     nodes: NodeWithScore[],
-    query?: string,
+    query?: MessageContent,
   ): Promise<NodeWithScore[]> {
     if (this.client === null) {
       throw new Error("CohereRerank client is null");
@@ -61,7 +63,7 @@ export class CohereRerank implements BaseNodePostprocessor {
     }
 
     const results = await this.client.rerank({
-      query,
+      query: extractText(query),
       model: this.model,
       topN: this.topN,
       documents: nodes.map((n) => n.node.getContent(MetadataMode.ALL)),

packages/core/src/postprocessors/rerankers/JinaAIReranker.ts

@@ -1,6 +1,8 @@
 import { getEnv } from "@llamaindex/env";
 import type { NodeWithScore } from "../../Node.js";
 import { MetadataMode } from "../../Node.js";
+import type { MessageContent } from "../../llm/types.js";
+import { extractText } from "../../llm/utils.js";
 import type { BaseNodePostprocessor } from "../types.js";
 
 interface JinaAIRerankerResult {
@@ -62,7 +64,7 @@ export class JinaAIReranker implements BaseNodePostprocessor {
 
   async postprocessNodes(
     nodes: NodeWithScore[],
-    query?: string,
+    query?: MessageContent,
   ): Promise<NodeWithScore[]> {
     if (nodes.length === 0) {
       return [];
@@ -73,7 +75,7 @@ export class JinaAIReranker implements BaseNodePostprocessor {
     }
 
     const documents = nodes.map((n) => n.node.getContent(MetadataMode.ALL));
-    const results = await this.rerank(query, documents, this.topN);
+    const results = await this.rerank(extractText(query), documents, this.topN);
     const newNodes: NodeWithScore[] = [];
 
     for (const result of results) {

packages/core/src/postprocessors/types.ts

@@ -1,4 +1,5 @@
 import type { NodeWithScore } from "../Node.js";
+import type { MessageContent } from "../llm/types.js";
 
 export interface BaseNodePostprocessor {
   /**
@@ -9,6 +10,6 @@ export interface BaseNodePostprocessor {
    */
   postprocessNodes(
     nodes: NodeWithScore[],
-    query?: string,
+    query?: MessageContent,
   ): Promise<NodeWithScore[]>;
 }

packages/core/src/readers/MarkdownReader.ts

@@ -43,6 +43,8 @@ export class MarkdownReader implements FileReader {
             continue;
           }
           markdownTups.push([currentHeader, currentText]);
+        } else if (currentText) {
+          markdownTups.push([null, currentText]);
         }
 
         currentHeader = line;

packages/core/src/storage/vectorStore/SimpleVectorStore.ts

@@ -151,20 +151,13 @@ export class SimpleVectorStore
 
   async persist(
     persistPath: string = path.join(DEFAULT_PERSIST_DIR, "vector_store.json"),
-  ): Promise<void> {
-    await SimpleVectorStore.persistData(persistPath, this.data);
-  }
-
-  protected static async persistData(
-    persistPath: string,
-    data: SimpleVectorStoreData,
   ): Promise<void> {
     const dirPath = path.dirname(persistPath);
     if (!(await exists(dirPath))) {
       await fs.mkdir(dirPath);
     }
 
-    await fs.writeFile(persistPath, JSON.stringify(data));
+    await fs.writeFile(persistPath, JSON.stringify(this.data));
   }
 
   static async fromPersistPath(
@@ -184,11 +177,6 @@ export class SimpleVectorStore
       console.error(
         `No valid data found at path: ${persistPath} starting new store.`,
       );
-      // persist empty data, to ignore this error in the future
-      await SimpleVectorStore.persistData(
-        persistPath,
-        new SimpleVectorStoreData(),
-      );
     }
 
     const data = new SimpleVectorStoreData();

packages/core/src/synthesizers/MultiModalResponseSynthesizer.ts

@@ -1,10 +1,8 @@
-import type { ImageNode } from "../Node.js";
-import { MetadataMode, ModalityType, splitNodesByType } from "../Node.js";
+import { MetadataMode } from "../Node.js";
 import { Response } from "../Response.js";
 import type { ServiceContext } from "../ServiceContext.js";
 import { llmFromSettingsOrContext } from "../Settings.js";
-import { imageToDataUrl } from "../embeddings/index.js";
-import type { MessageContentDetail } from "../llm/types.js";
+import { streamConverter } from "../llm/utils.js";
 import { PromptMixin } from "../prompts/Mixin.js";
 import type { TextQaPrompt } from "./../Prompt.js";
 import { defaultTextQaPrompt } from "./../Prompt.js";
@@ -13,6 +11,7 @@ import type {
   SynthesizeParamsNonStreaming,
   SynthesizeParamsStreaming,
 } from "./types.js";
+import { createMessageContent } from "./utils.js";
 
 export class MultiModalResponseSynthesizer
   extends PromptMixin
@@ -59,41 +58,29 @@ export class MultiModalResponseSynthesizer
   }: SynthesizeParamsStreaming | SynthesizeParamsNonStreaming): Promise<
     AsyncIterable<Response> | Response
   > {
-    if (stream) {
-      throw new Error("streaming not implemented");
-    }
     const nodes = nodesWithScore.map(({ node }) => node);
-    const nodeMap = splitNodesByType(nodes);
-    const imageNodes: ImageNode[] =
-      (nodeMap[ModalityType.IMAGE] as ImageNode[]) ?? [];
-    const textNodes = nodeMap[ModalityType.TEXT] ?? [];
-    const textChunks = textNodes.map((node) =>
-      node.getContent(this.metadataMode),
+    const prompt = await createMessageContent(
+      this.textQATemplate,
+      nodes,
+      { query },
+      this.metadataMode,
     );
-    // TODO: use builders to generate context
-    const context = textChunks.join("\n\n");
-    const textPrompt = this.textQATemplate({ context, query });
-    const images = await Promise.all(
-      imageNodes.map(async (node: ImageNode) => {
-        return {
-          type: "image_url",
-          image_url: {
-            url: await imageToDataUrl(node.image),
-          },
-        } as MessageContentDetail;
-      }),
-    );
-    const prompt: MessageContentDetail[] = [
-      { type: "text", text: textPrompt },
-      ...images,
-    ];
 
     const llm = llmFromSettingsOrContext(this.serviceContext);
 
+    if (stream) {
+      const response = await llm.complete({
+        prompt,
+        stream,
+      });
+      return streamConverter(
+        response,
+        ({ text }) => new Response(text, nodesWithScore),
+      );
+    }
     const response = await llm.complete({
       prompt,
     });
-
     return new Response(response.text, nodesWithScore);
   }
 }

packages/core/src/synthesizers/utils.ts

@@ -0,0 +1,72 @@
+import {
+  ImageNode,
+  MetadataMode,
+  ModalityType,
+  splitNodesByType,
+  type BaseNode,
+} from "../Node.js";
+import type { SimplePrompt } from "../Prompt.js";
+import { imageToDataUrl } from "../embeddings/utils.js";
+import type { MessageContentDetail } from "../llm/types.js";
+
+export async function createMessageContent(
+  prompt: SimplePrompt,
+  nodes: BaseNode[],
+  extraParams: Record<string, string | undefined> = {},
+  metadataMode: MetadataMode = MetadataMode.NONE,
+): Promise<MessageContentDetail[]> {
+  const content: MessageContentDetail[] = [];
+  const nodeMap = splitNodesByType(nodes);
+  for (const type in nodeMap) {
+    // for each retrieved modality type, create message content
+    const nodes = nodeMap[type as ModalityType];
+    if (nodes) {
+      content.push(
+        ...(await createContentPerModality(
+          prompt,
+          type as ModalityType,
+          nodes,
+          extraParams,
+          metadataMode,
+        )),
+      );
+    }
+  }
+  return content;
+}
+
+// eslint-disable-next-line max-params
+async function createContentPerModality(
+  prompt: SimplePrompt,
+  type: ModalityType,
+  nodes: BaseNode[],
+  extraParams: Record<string, string | undefined>,
+  metadataMode: MetadataMode,
+): Promise<MessageContentDetail[]> {
+  switch (type) {
+    case ModalityType.TEXT:
+      return [
+        {
+          type: "text",
+          text: prompt({
+            ...extraParams,
+            context: nodes.map((r) => r.getContent(metadataMode)).join("\n\n"),
+          }),
+        },
+      ];
+    case ModalityType.IMAGE:
+      const images: MessageContentDetail[] = await Promise.all(
+        (nodes as ImageNode[]).map(async (node) => {
+          return {
+            type: "image_url",
+            image_url: {
+              url: await imageToDataUrl(node.image),
+            },
+          } satisfies MessageContentDetail;
+        }),
+      );
+      return images;
+    default:
+      return [];
+  }
+}

packages/core/tests/Node.test.ts

@@ -53,7 +53,7 @@ describe("TextNode", () => {
         "endCharIdx": undefined,
         "excludedEmbedMetadataKeys": [],
         "excludedLlmMetadataKeys": [],
-        "hash": "nTSKdUTYqR52MPv/brvb4RTGeqedTEqG9QN8KSAj2Do=",
+        "hash": "Z6SWgFPlalaeblMGQGw0KS3qKgmZdEWXKfzEp/K+QN0=",
         "id_": Any<String>,
         "metadata": {
           "something": 1,

packages/experimental/CHANGELOG.md

@@ -1,5 +1,32 @@
 # @llamaindex/experimental
 
+## 0.0.32
+
+### Patch Changes
+
+- Updated dependencies [6e156ed]
+- Updated dependencies [265976d]
+- Updated dependencies [8e26f75]
+  - llamaindex@0.3.15
+
+## 0.0.31
+
+### Patch Changes
+
+- Updated dependencies [6ff7576]
+- Updated dependencies [94543de]
+  - llamaindex@0.3.14
+
+## 0.0.30
+
+### Patch Changes
+
+- Updated dependencies [1b1081b]
+- Updated dependencies [37525df]
+- Updated dependencies [660a2b3]
+- Updated dependencies [a1f2475]
+  - llamaindex@0.3.13
+
 ## 0.0.29
 
 ### Patch Changes

packages/experimental/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/experimental",
   "description": "Experimental package for LlamaIndexTS",
-  "version": "0.0.29",
+  "version": "0.0.32",
   "type": "module",
   "types": "dist/type/index.d.ts",
   "main": "dist/cjs/index.js",