Release 0.3.20 (#457)

Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>

CHANGELOG.md

@@ -1,5 +1,26 @@
 # create-llama
 
+## 0.3.20
+
+### Patch Changes
+
+- 27d2499: Bump the LlamaCloud library and fix breaking changes (Python).
+
+## 0.3.19
+
+### Patch Changes
+
+- f9a057d: Add support multimodal indexes (e.g. from LlamaCloud)
+- aedd73d: bump: chat-ui
+
+## 0.3.18
+
+### Patch Changes
+
+- fe90a7e: chore: bump ai v4
+- 02b2473: Show streaming errors in Python, optimize system prompts for tool usage and set the weather tool as default for the Agentic RAG use case
+- 63e961e: Use auto_routed retriever mode for LlamaCloudIndex
+
 ## 0.3.17
 
 ### Patch Changes

helpers/env-variables.ts

@@ -13,6 +13,12 @@ import {
 
 import { TSYSTEMS_LLMHUB_API_URL } from "./providers/llmhub";
 
+const DEFAULT_SYSTEM_PROMPT =
+  "You are a helpful assistant who helps users with their questions.";
+
+const DATA_SOURCES_PROMPT =
+  "You have access to a knowledge base including the facts that you should start with to find the answer for the user question. Use the query engine tool to retrieve the facts from the knowledge base.";
+
 export type EnvVar = {
   name?: string;
   description?: string;
@@ -449,9 +455,6 @@ const getSystemPromptEnv = (
   dataSources?: TemplateDataSource[],
   template?: TemplateType,
 ): EnvVar[] => {
-  const defaultSystemPrompt =
-    "You are a helpful assistant who helps users with their questions.";
-
   const systemPromptEnv: EnvVar[] = [];
   // build tool system prompt by merging all tool system prompts
   // multiagent template doesn't need system prompt
@@ -466,9 +469,12 @@ const getSystemPromptEnv = (
       }
     });
 
-    const systemPrompt = toolSystemPrompt
-      ? `\"${toolSystemPrompt}\"`
-      : defaultSystemPrompt;
+    const systemPrompt =
+      "'" +
+      DEFAULT_SYSTEM_PROMPT +
+      (dataSources?.length ? `\n${DATA_SOURCES_PROMPT}` : "") +
+      (toolSystemPrompt ? `\n${toolSystemPrompt}` : "") +
+      "'";
 
     systemPromptEnv.push({
       name: "SYSTEM_PROMPT",

helpers/python.ts

@@ -37,21 +37,21 @@ const getAdditionalDependencies = (
     case "mongo": {
       dependencies.push({
         name: "llama-index-vector-stores-mongodb",
-        version: "^0.3.1",
+        version: "^0.6.0",
       });
       break;
     }
     case "pg": {
       dependencies.push({
         name: "llama-index-vector-stores-postgres",
-        version: "^0.2.5",
+        version: "^0.3.2",
       });
       break;
     }
     case "pinecone": {
       dependencies.push({
         name: "llama-index-vector-stores-pinecone",
-        version: "^0.2.1",
+        version: "^0.4.1",
         constraints: {
           python: ">=3.11,<3.13",
         },
@@ -61,7 +61,7 @@ const getAdditionalDependencies = (
     case "milvus": {
       dependencies.push({
         name: "llama-index-vector-stores-milvus",
-        version: "^0.2.0",
+        version: "^0.3.0",
       });
       dependencies.push({
         name: "pymilvus",
@@ -72,14 +72,14 @@ const getAdditionalDependencies = (
     case "astra": {
       dependencies.push({
         name: "llama-index-vector-stores-astra-db",
-        version: "^0.2.0",
+        version: "^0.4.0",
       });
       break;
     }
     case "qdrant": {
       dependencies.push({
         name: "llama-index-vector-stores-qdrant",
-        version: "^0.3.0",
+        version: "^0.4.0",
         constraints: {
           python: ">=3.11,<3.13",
         },
@@ -89,21 +89,21 @@ const getAdditionalDependencies = (
     case "chroma": {
       dependencies.push({
         name: "llama-index-vector-stores-chroma",
-        version: "^0.2.0",
+        version: "^0.4.0",
       });
       break;
     }
     case "weaviate": {
       dependencies.push({
         name: "llama-index-vector-stores-weaviate",
-        version: "^1.1.1",
+        version: "^1.2.3",
       });
       break;
     }
     case "llamacloud":
       dependencies.push({
         name: "llama-index-indices-managed-llama-cloud",
-        version: "^0.3.1",
+        version: "^0.6.3",
       });
       break;
   }
@@ -122,13 +122,13 @@ const getAdditionalDependencies = (
         case "web":
           dependencies.push({
             name: "llama-index-readers-web",
-            version: "^0.2.2",
+            version: "^0.3.0",
           });
           break;
         case "db":
           dependencies.push({
             name: "llama-index-readers-database",
-            version: "^0.2.0",
+            version: "^0.3.0",
           });
           dependencies.push({
             name: "pymysql",
@@ -167,15 +167,15 @@ const getAdditionalDependencies = (
       if (templateType !== "multiagent") {
         dependencies.push({
           name: "llama-index-llms-openai",
-          version: "^0.2.0",
+          version: "^0.3.2",
         });
         dependencies.push({
           name: "llama-index-embeddings-openai",
-          version: "^0.2.3",
+          version: "^0.3.1",
         });
         dependencies.push({
           name: "llama-index-agent-openai",
-          version: "^0.3.0",
+          version: "^0.4.0",
         });
       }
       break;
@@ -524,7 +524,7 @@ export const installPythonTemplate = async ({
     if (observability === "llamatrace") {
       addOnDependencies.push({
         name: "llama-index-callbacks-arize-phoenix",
-        version: "^0.2.1",
+        version: "^0.3.0",
         constraints: {
           python: ">=3.11,<3.13",
         },

helpers/tools.ts

@@ -41,7 +41,7 @@ export const supportedTools: Tool[] = [
     dependencies: [
       {
         name: "llama-index-tools-google",
-        version: "^0.2.0",
+        version: "^0.3.0",
       },
     ],
     supportedFrameworks: ["fastapi"],
@@ -71,8 +71,7 @@ export const supportedTools: Tool[] = [
       {
         name: TOOL_SYSTEM_PROMPT_ENV_VAR,
         description: "System prompt for DuckDuckGo search tool.",
-        value: `You are a DuckDuckGo search agent. 
-You can use the duckduckgo search tool to get information from the web to answer user questions.
+        value: `You have access to the duckduckgo search tool. Use it to get information from the web to answer user questions.
 For better results, you can specify the region parameter to get results from a specific region but it's optional.`,
       },
     ],
@@ -83,18 +82,11 @@ For better results, you can specify the region parameter to get results from a s
     dependencies: [
       {
         name: "llama-index-tools-wikipedia",
-        version: "^0.2.0",
+        version: "^0.3.0",
       },
     ],
     supportedFrameworks: ["fastapi", "express", "nextjs"],
     type: ToolType.LLAMAHUB,
-    envVars: [
-      {
-        name: TOOL_SYSTEM_PROMPT_ENV_VAR,
-        description: "System prompt for wiki tool.",
-        value: `You are a Wikipedia agent. You help users to get information from Wikipedia.`,
-      },
-    ],
   },
   {
     display: "Weather",
@@ -102,13 +94,6 @@ For better results, you can specify the region parameter to get results from a s
     dependencies: [],
     supportedFrameworks: ["fastapi", "express", "nextjs"],
     type: ToolType.LOCAL,
-    envVars: [
-      {
-        name: TOOL_SYSTEM_PROMPT_ENV_VAR,
-        description: "System prompt for weather tool.",
-        value: `You are a weather forecast agent. You help users to get the weather forecast for a given location.`,
-      },
-    ],
   },
   {
     display: "Document generator",
@@ -211,14 +196,6 @@ For better results, you can specify the region parameter to get results from a s
     },
     supportedFrameworks: ["fastapi", "express", "nextjs"],
     type: ToolType.LOCAL,
-    envVars: [
-      {
-        name: TOOL_SYSTEM_PROMPT_ENV_VAR,
-        description: "System prompt for openapi action tool.",
-        value:
-          "You are an OpenAPI action agent. You help users to make requests to the provided OpenAPI schema.",
-      },
-    ],
   },
   {
     display: "Image Generator",
@@ -231,11 +208,6 @@ For better results, you can specify the region parameter to get results from a s
         description:
           "STABILITY_API_KEY key is required to run image generator. Get it here: https://platform.stability.ai/account/keys",
       },
-      {
-        name: TOOL_SYSTEM_PROMPT_ENV_VAR,
-        description: "System prompt for image generator tool.",
-        value: `You are an image generator agent. You help users to generate images using the Stability API.`,
-      },
     ],
   },
   {

package.json

@@ -1,6 +1,6 @@
 {
   "name": "create-llama",
-  "version": "0.3.17",
+  "version": "0.3.20",
   "description": "Create LlamaIndex-powered apps with one command",
   "keywords": [
     "rag",

questions/simple.ts

@@ -131,7 +131,7 @@ const convertAnswers = async (
   > = {
     rag: {
       template: "streaming",
-      tools: getTools(["wikipedia.WikipediaToolSpec"]),
+      tools: getTools(["weather"]),
       frontend: true,
       dataSources: [EXAMPLE_FILE],
     },

templates/components/agents/python/blog/app/agents/researcher.py

@@ -1,4 +1,3 @@
-import os
 from textwrap import dedent
 from typing import List
 
@@ -6,42 +5,24 @@ from app.engine.index import IndexConfig, get_index
 from app.engine.tools import ToolFactory
 from app.workflows.single import FunctionCallingAgent
 from llama_index.core.chat_engine.types import ChatMessage
-from llama_index.core.tools import QueryEngineTool, ToolMetadata
+from app.engine.tools.query_engine import get_query_engine_tool
 
 
-def _create_query_engine_tool(params=None) -> QueryEngineTool:
-    """
-    Provide an agent worker that can be used to query the index.
-    """
-    # Add query tool if index exists
-    index_config = IndexConfig(**(params or {}))
-    index = get_index(index_config)
-    if index is None:
-        return None
-    top_k = int(os.getenv("TOP_K", 0))
-    query_engine = index.as_query_engine(
-        **({"similarity_top_k": top_k} if top_k != 0 else {})
-    )
-    return QueryEngineTool(
-        query_engine=query_engine,
-        metadata=ToolMetadata(
-            name="query_index",
-            description="""
-                Use this tool to retrieve information about the text corpus from the index.
-            """,
-        ),
-    )
-
-
-def _get_research_tools(**kwargs) -> QueryEngineTool:
+def _get_research_tools(**kwargs):
     """
     Researcher take responsibility for retrieving information.
     Try init wikipedia or duckduckgo tool if available.
     """
     tools = []
-    query_engine_tool = _create_query_engine_tool(**kwargs)
-    if query_engine_tool is not None:
-        tools.append(query_engine_tool)
+    # Create query engine tool
+    index_config = IndexConfig(**kwargs)
+    index = get_index(index_config)
+    if index is not None:
+        query_engine_tool = get_query_engine_tool(index=index)
+        if query_engine_tool is not None:
+            tools.append(query_engine_tool)
+
+    # Create duckduckgo tool
     researcher_tool_names = [
         "duckduckgo_search",
         "duckduckgo_image_search",

templates/components/agents/python/financial_report/app/workflows/financial_report.py

@@ -1,8 +1,8 @@
-import os
 from typing import Any, Dict, List, Optional
 
 from app.engine.index import IndexConfig, get_index
 from app.engine.tools import ToolFactory
+from app.engine.tools.query_engine import get_query_engine_tool
 from app.workflows.events import AgentRunEvent
 from app.workflows.tools import (
     call_tools,
@@ -10,7 +10,6 @@ from app.workflows.tools import (
 )
 from llama_index.core import Settings
 from llama_index.core.base.llms.types import ChatMessage, MessageRole
-from llama_index.core.indices.vector_store import VectorStoreIndex
 from llama_index.core.llms.function_calling import FunctionCallingLLM
 from llama_index.core.memory import ChatMemoryBuffer
 from llama_index.core.tools import FunctionTool, QueryEngineTool, ToolSelection
@@ -27,16 +26,16 @@ from llama_index.core.workflow import (
 def create_workflow(
     chat_history: Optional[List[ChatMessage]] = None,
     params: Optional[Dict[str, Any]] = None,
-    filters: Optional[List[Any]] = None,
+    **kwargs,
 ) -> Workflow:
+    # Create query engine tool
     index_config = IndexConfig(**params)
-    index: VectorStoreIndex = get_index(config=index_config)
+    index = get_index(index_config)
     if index is None:
-        query_engine_tool = None
-    else:
-        top_k = int(os.getenv("TOP_K", 10))
-        query_engine = index.as_query_engine(similarity_top_k=top_k, filters=filters)
-        query_engine_tool = QueryEngineTool.from_defaults(query_engine=query_engine)
+        raise ValueError(
+            "Index is not found. Try run generation script to create the index first."
+        )
+    query_engine_tool = get_query_engine_tool(index=index)
 
     configured_tools: Dict[str, FunctionTool] = ToolFactory.from_env(map_result=True)  # type: ignore
     code_interpreter_tool = configured_tools.get("interpret")

templates/components/agents/python/form_filling/app/workflows/form_filling.py

@@ -1,16 +1,7 @@
-import os
 from typing import Any, Dict, List, Optional
 
-from app.engine.index import IndexConfig, get_index
-from app.engine.tools import ToolFactory
-from app.workflows.events import AgentRunEvent
-from app.workflows.tools import (
-    call_tools,
-    chat_with_tools,
-)
 from llama_index.core import Settings
 from llama_index.core.base.llms.types import ChatMessage, MessageRole
-from llama_index.core.indices.vector_store import VectorStoreIndex
 from llama_index.core.llms.function_calling import FunctionCallingLLM
 from llama_index.core.memory import ChatMemoryBuffer
 from llama_index.core.tools import FunctionTool, QueryEngineTool, ToolSelection
@@ -23,24 +14,28 @@ from llama_index.core.workflow import (
     step,
 )
 
+from app.engine.index import IndexConfig, get_index
+from app.engine.tools import ToolFactory
+from app.engine.tools.query_engine import get_query_engine_tool
+from app.workflows.events import AgentRunEvent
+from app.workflows.tools import (
+    call_tools,
+    chat_with_tools,
+)
+
 
 def create_workflow(
     chat_history: Optional[List[ChatMessage]] = None,
     params: Optional[Dict[str, Any]] = None,
-    filters: Optional[List[Any]] = None,
+    **kwargs,
 ) -> Workflow:
-    if params is None:
-        params = {}
-    if filters is None:
-        filters = []
+    # Create query engine tool
     index_config = IndexConfig(**params)
-    index: VectorStoreIndex = get_index(config=index_config)
+    index = get_index(index_config)
     if index is None:
         query_engine_tool = None
     else:
-        top_k = int(os.getenv("TOP_K", 10))
-        query_engine = index.as_query_engine(similarity_top_k=top_k, filters=filters)
-        query_engine_tool = QueryEngineTool.from_defaults(query_engine=query_engine)
+        query_engine_tool = get_query_engine_tool(index=index)
 
     configured_tools = ToolFactory.from_env(map_result=True)
     extractor_tool = configured_tools.get("extract_questions")  # type: ignore

templates/components/agents/typescript/blog/workflow/agents.ts

@@ -1,15 +1,15 @@
 import { ChatMessage } from "llamaindex";
 import { getTool } from "../engine/tools";
 import { FunctionCallingAgent } from "./single-agent";
-import { getQueryEngineTools } from "./tools";
+import { getQueryEngineTool } from "./tools";
 
 export const createResearcher = async (chatHistory: ChatMessage[]) => {
-  const queryEngineTools = await getQueryEngineTools();
+  const queryEngineTool = await getQueryEngineTool();
   const tools = [
     await getTool("wikipedia_tool"),
     await getTool("duckduckgo_search"),
     await getTool("image_generator"),
-    ...(queryEngineTools ? queryEngineTools : []),
+    queryEngineTool,
   ].filter((tool) => tool !== undefined);
 
   return new FunctionCallingAgent({

templates/components/agents/typescript/financial_report/workflow/factory.ts

@@ -1,7 +1,7 @@
 import { ChatMessage, ToolCallLLM } from "llamaindex";
 import { getTool } from "../engine/tools";
 import { FinancialReportWorkflow } from "./fin-report";
-import { getQueryEngineTools } from "./tools";
+import { getQueryEngineTool } from "./tools";
 
 const TIMEOUT = 360 * 1000;
 
@@ -9,11 +9,19 @@ export async function createWorkflow(options: {
   chatHistory: ChatMessage[];
   llm?: ToolCallLLM;
 }) {
+  const queryEngineTool = await getQueryEngineTool();
+  const codeInterpreterTool = await getTool("interpreter");
+  const documentGeneratorTool = await getTool("document_generator");
+
+  if (!queryEngineTool || !codeInterpreterTool || !documentGeneratorTool) {
+    throw new Error("One or more required tools are not defined");
+  }
+
   return new FinancialReportWorkflow({
     chatHistory: options.chatHistory,
-    queryEngineTools: (await getQueryEngineTools()) || [],
-    codeInterpreterTool: (await getTool("interpreter"))!,
-    documentGeneratorTool: (await getTool("document_generator"))!,
+    queryEngineTool,
+    codeInterpreterTool,
+    documentGeneratorTool,
     llm: options.llm,
     timeout: TIMEOUT,
   });

templates/components/agents/typescript/financial_report/workflow/fin-report.ts

@@ -45,7 +45,7 @@ export class FinancialReportWorkflow extends Workflow<
 > {
   llm: ToolCallLLM;
   memory: ChatMemoryBuffer;
-  queryEngineTools: BaseToolWithCall[];
+  queryEngineTool: BaseToolWithCall;
   codeInterpreterTool: BaseToolWithCall;
   documentGeneratorTool: BaseToolWithCall;
   systemPrompt?: string;
@@ -53,7 +53,7 @@ export class FinancialReportWorkflow extends Workflow<
   constructor(options: {
     llm?: ToolCallLLM;
     chatHistory: ChatMessage[];
-    queryEngineTools: BaseToolWithCall[];
+    queryEngineTool: BaseToolWithCall;
     codeInterpreterTool: BaseToolWithCall;
     documentGeneratorTool: BaseToolWithCall;
     systemPrompt?: string;
@@ -70,7 +70,7 @@ export class FinancialReportWorkflow extends Workflow<
       throw new Error("LLM is not a ToolCallLLM");
     }
     this.systemPrompt = options.systemPrompt ?? DEFAULT_SYSTEM_PROMPT;
-    this.queryEngineTools = options.queryEngineTools;
+    this.queryEngineTool = options.queryEngineTool;
     this.codeInterpreterTool = options.codeInterpreterTool;
 
     this.documentGeneratorTool = options.documentGeneratorTool;
@@ -153,10 +153,11 @@ export class FinancialReportWorkflow extends Workflow<
   > => {
     const chatHistory = ev.data.input;
 
-    const tools = [this.codeInterpreterTool, this.documentGeneratorTool];
-    if (this.queryEngineTools) {
-      tools.push(...this.queryEngineTools);
-    }
+    const tools = [
+      this.codeInterpreterTool,
+      this.documentGeneratorTool,
+      this.queryEngineTool,
+    ];
 
     const toolCallResponse = await chatWithTools(this.llm, tools, chatHistory);
 
@@ -189,10 +190,7 @@ export class FinancialReportWorkflow extends Workflow<
           toolCalls: toolCallResponse.toolCalls,
         });
       default:
-        if (
-          this.queryEngineTools &&
-          this.queryEngineTools.some((tool) => tool.metadata.name === toolName)
-        ) {
+        if (this.queryEngineTool.metadata.name === toolName) {
           return new ResearchEvent({
             toolCalls: toolCallResponse.toolCalls,
           });
@@ -216,7 +214,7 @@ export class FinancialReportWorkflow extends Workflow<
     const { toolCalls } = ev.data;
 
     const toolMsgs = await callTools({
-      tools: this.queryEngineTools,
+      tools: [this.queryEngineTool],
       toolCalls,
       ctx,
       agentName: "Researcher",

templates/components/agents/typescript/form_filling/workflow/factory.ts

@@ -1,7 +1,7 @@
 import { ChatMessage, ToolCallLLM } from "llamaindex";
 import { getTool } from "../engine/tools";
 import { FormFillingWorkflow } from "./form-filling";
-import { getQueryEngineTools } from "./tools";
+import { getQueryEngineTool } from "./tools";
 
 const TIMEOUT = 360 * 1000;
 
@@ -9,11 +9,18 @@ export async function createWorkflow(options: {
   chatHistory: ChatMessage[];
   llm?: ToolCallLLM;
 }) {
+  const extractorTool = await getTool("extract_missing_cells");
+  const fillMissingCellsTool = await getTool("fill_missing_cells");
+
+  if (!extractorTool || !fillMissingCellsTool) {
+    throw new Error("One or more required tools are not defined");
+  }
+
   return new FormFillingWorkflow({
     chatHistory: options.chatHistory,
-    queryEngineTools: (await getQueryEngineTools()) || [],
-    extractorTool: (await getTool("extract_missing_cells"))!,
-    fillMissingCellsTool: (await getTool("fill_missing_cells"))!,
+    queryEngineTool: (await getQueryEngineTool()) || undefined,
+    extractorTool,
+    fillMissingCellsTool,
     llm: options.llm,
     timeout: TIMEOUT,
   });

templates/components/agents/typescript/form_filling/workflow/form-filling.ts

@@ -48,7 +48,7 @@ export class FormFillingWorkflow extends Workflow<
   llm: ToolCallLLM;
   memory: ChatMemoryBuffer;
   extractorTool: BaseToolWithCall;
-  queryEngineTools?: BaseToolWithCall[];
+  queryEngineTool?: BaseToolWithCall;
   fillMissingCellsTool: BaseToolWithCall;
   systemPrompt?: string;
 
@@ -56,7 +56,7 @@ export class FormFillingWorkflow extends Workflow<
     llm?: ToolCallLLM;
     chatHistory: ChatMessage[];
     extractorTool: BaseToolWithCall;
-    queryEngineTools?: BaseToolWithCall[];
+    queryEngineTool?: BaseToolWithCall;
     fillMissingCellsTool: BaseToolWithCall;
     systemPrompt?: string;
     verbose?: boolean;
@@ -73,7 +73,7 @@ export class FormFillingWorkflow extends Workflow<
     }
     this.systemPrompt = options.systemPrompt ?? DEFAULT_SYSTEM_PROMPT;
     this.extractorTool = options.extractorTool;
-    this.queryEngineTools = options.queryEngineTools;
+    this.queryEngineTool = options.queryEngineTool;
     this.fillMissingCellsTool = options.fillMissingCellsTool;
 
     this.memory = new ChatMemoryBuffer({
@@ -156,8 +156,8 @@ export class FormFillingWorkflow extends Workflow<
     const chatHistory = ev.data.input;
 
     const tools = [this.extractorTool, this.fillMissingCellsTool];
-    if (this.queryEngineTools) {
-      tools.push(...this.queryEngineTools);
+    if (this.queryEngineTool) {
+      tools.push(this.queryEngineTool);
     }
 
     const toolCallResponse = await chatWithTools(this.llm, tools, chatHistory);
@@ -192,8 +192,8 @@ export class FormFillingWorkflow extends Workflow<
         });
       default:
         if (
-          this.queryEngineTools &&
-          this.queryEngineTools.some((tool) => tool.metadata.name === toolName)
+          this.queryEngineTool &&
+          this.queryEngineTool.metadata.name === toolName
         ) {
           return new FindAnswersEvent({
             toolCalls: toolCallResponse.toolCalls,
@@ -232,7 +232,7 @@ export class FormFillingWorkflow extends Workflow<
     ev: FindAnswersEvent,
   ): Promise<InputEvent> => {
     const { toolCalls } = ev.data;
-    if (!this.queryEngineTools) {
+    if (!this.queryEngineTool) {
       throw new Error("Query engine tool is not available");
     }
     ctx.sendEvent(
@@ -243,7 +243,7 @@ export class FormFillingWorkflow extends Workflow<
       }),
     );
     const toolMsgs = await callTools({
-      tools: this.queryEngineTools,
+      tools: [this.queryEngineTool],
       toolCalls,
       ctx,
       agentName: "Researcher",

templates/components/engines/python/agent/engine.py

@@ -1,18 +1,18 @@
 import os
 from typing import List
 
-from app.engine.index import IndexConfig, get_index
-from app.engine.tools import ToolFactory
 from llama_index.core.agent import AgentRunner
 from llama_index.core.callbacks import CallbackManager
 from llama_index.core.settings import Settings
 from llama_index.core.tools import BaseTool
-from llama_index.core.tools.query_engine import QueryEngineTool
+
+from app.engine.index import IndexConfig, get_index
+from app.engine.tools import ToolFactory
+from app.engine.tools.query_engine import get_query_engine_tool
 
 
-def get_chat_engine(filters=None, params=None, event_handlers=None, **kwargs):
+def get_chat_engine(params=None, event_handlers=None, **kwargs):
     system_prompt = os.getenv("SYSTEM_PROMPT")
-    top_k = int(os.getenv("TOP_K", 0))
     tools: List[BaseTool] = []
     callback_manager = CallbackManager(handlers=event_handlers or [])
 
@@ -20,10 +20,7 @@ def get_chat_engine(filters=None, params=None, event_handlers=None, **kwargs):
     index_config = IndexConfig(callback_manager=callback_manager, **(params or {}))
     index = get_index(index_config)
     if index is not None:
-        query_engine = index.as_query_engine(
-            filters=filters, **({"similarity_top_k": top_k} if top_k != 0 else {})
-        )
-        query_engine_tool = QueryEngineTool.from_defaults(query_engine=query_engine)
+        query_engine_tool = get_query_engine_tool(index, **kwargs)
         tools.append(query_engine_tool)
 
     # Add additional tools

templates/components/engines/python/agent/tools/query_engine.py

@@ -0,0 +1,187 @@
+import os
+from typing import Any, Dict, List, Optional, Sequence
+
+from llama_index.core import get_response_synthesizer
+from llama_index.core.base.base_query_engine import BaseQueryEngine
+from llama_index.core.base.response.schema import RESPONSE_TYPE, Response
+from llama_index.core.multi_modal_llms import MultiModalLLM
+from llama_index.core.prompts.base import BasePromptTemplate
+from llama_index.core.prompts.default_prompt_selectors import (
+    DEFAULT_TEXT_QA_PROMPT_SEL,
+)
+from llama_index.core.query_engine.multi_modal import _get_image_and_text_nodes
+from llama_index.core.response_synthesizers.base import BaseSynthesizer, QueryTextType
+from llama_index.core.schema import (
+    ImageNode,
+    NodeWithScore,
+)
+from llama_index.core.tools.query_engine import QueryEngineTool
+from llama_index.core.types import RESPONSE_TEXT_TYPE
+
+from app.settings import get_multi_modal_llm
+
+
+def create_query_engine(index, **kwargs) -> BaseQueryEngine:
+    """
+    Create a query engine for the given index.
+
+    Args:
+        index: The index to create a query engine for.
+        params (optional): Additional parameters for the query engine, e.g: similarity_top_k
+    """
+
+    top_k = int(os.getenv("TOP_K", 0))
+    if top_k != 0 and kwargs.get("filters") is None:
+        kwargs["similarity_top_k"] = top_k
+    multimodal_llm = get_multi_modal_llm()
+    if multimodal_llm:
+        kwargs["response_synthesizer"] = MultiModalSynthesizer(
+            multimodal_model=multimodal_llm,
+        )
+
+    # If index is index is LlamaCloudIndex
+    # use auto_routed mode for better query results
+    if index.__class__.__name__ == "LlamaCloudIndex":
+        if kwargs.get("retrieval_mode") is None:
+            kwargs["retrieval_mode"] = "auto_routed"
+        if multimodal_llm:
+            kwargs["retrieve_image_nodes"] = True
+    return index.as_query_engine(**kwargs)
+
+
+def get_query_engine_tool(
+    index,
+    name: Optional[str] = None,
+    description: Optional[str] = None,
+    **kwargs,
+) -> QueryEngineTool:
+    """
+    Get a query engine tool for the given index.
+
+    Args:
+        index: The index to create a query engine for.
+        name (optional): The name of the tool.
+        description (optional): The description of the tool.
+    """
+    if name is None:
+        name = "query_index"
+    if description is None:
+        description = (
+            "Use this tool to retrieve information about the text corpus from an index."
+        )
+    query_engine = create_query_engine(index, **kwargs)
+    return QueryEngineTool.from_defaults(
+        query_engine=query_engine,
+        name=name,
+        description=description,
+    )
+
+
+class MultiModalSynthesizer(BaseSynthesizer):
+    """
+    A synthesizer that summarizes text nodes and uses a multi-modal LLM to generate a response.
+    """
+
+    def __init__(
+        self,
+        multimodal_model: MultiModalLLM,
+        response_synthesizer: Optional[BaseSynthesizer] = None,
+        text_qa_template: Optional[BasePromptTemplate] = None,
+        *args,
+        **kwargs,
+    ):
+        super().__init__(*args, **kwargs)
+        self._multi_modal_llm = multimodal_model
+        self._response_synthesizer = response_synthesizer or get_response_synthesizer()
+        self._text_qa_template = text_qa_template or DEFAULT_TEXT_QA_PROMPT_SEL
+
+    def _get_prompts(self, **kwargs) -> Dict[str, Any]:
+        return {
+            "text_qa_template": self._text_qa_template,
+        }
+
+    def _update_prompts(self, prompts: Dict[str, Any]) -> None:
+        if "text_qa_template" in prompts:
+            self._text_qa_template = prompts["text_qa_template"]
+
+    async def aget_response(
+        self,
+        *args,
+        **response_kwargs: Any,
+    ) -> RESPONSE_TEXT_TYPE:
+        return await self._response_synthesizer.aget_response(*args, **response_kwargs)
+
+    def get_response(self, *args, **kwargs) -> RESPONSE_TEXT_TYPE:
+        return self._response_synthesizer.get_response(*args, **kwargs)
+
+    async def asynthesize(
+        self,
+        query: QueryTextType,
+        nodes: List[NodeWithScore],
+        additional_source_nodes: Optional[Sequence[NodeWithScore]] = None,
+        **response_kwargs: Any,
+    ) -> RESPONSE_TYPE:
+        image_nodes, text_nodes = _get_image_and_text_nodes(nodes)
+
+        if len(image_nodes) == 0:
+            return await self._response_synthesizer.asynthesize(query, text_nodes)
+
+        # Summarize the text nodes to avoid exceeding the token limit
+        text_response = str(
+            await self._response_synthesizer.asynthesize(query, text_nodes)
+        )
+
+        fmt_prompt = self._text_qa_template.format(
+            context_str=text_response,
+            query_str=query.query_str,  # type: ignore
+        )
+
+        llm_response = await self._multi_modal_llm.acomplete(
+            prompt=fmt_prompt,
+            image_documents=[
+                image_node.node
+                for image_node in image_nodes
+                if isinstance(image_node.node, ImageNode)
+            ],
+        )
+
+        return Response(
+            response=str(llm_response),
+            source_nodes=nodes,
+            metadata={"text_nodes": text_nodes, "image_nodes": image_nodes},
+        )
+
+    def synthesize(
+        self,
+        query: QueryTextType,
+        nodes: List[NodeWithScore],
+        additional_source_nodes: Optional[Sequence[NodeWithScore]] = None,
+        **response_kwargs: Any,
+    ) -> RESPONSE_TYPE:
+        image_nodes, text_nodes = _get_image_and_text_nodes(nodes)
+
+        if len(image_nodes) == 0:
+            return self._response_synthesizer.synthesize(query, text_nodes)
+
+        # Summarize the text nodes to avoid exceeding the token limit
+        text_response = str(self._response_synthesizer.synthesize(query, text_nodes))
+
+        fmt_prompt = self._text_qa_template.format(
+            context_str=text_response,
+            query_str=query.query_str,  # type: ignore
+        )
+
+        llm_response = self._multi_modal_llm.complete(
+            prompt=fmt_prompt,
+            image_documents=[
+                image_node.node
+                for image_node in image_nodes
+                if isinstance(image_node.node, ImageNode)
+            ],
+        )
+
+        return Response(
+            response=str(llm_response),
+            source_nodes=nodes,
+            metadata={"text_nodes": text_nodes, "image_nodes": image_nodes},
+        )

templates/components/engines/python/chat/engine.py

@@ -9,7 +9,7 @@ from llama_index.core.memory import ChatMemoryBuffer
 from llama_index.core.settings import Settings
 
 
-def get_chat_engine(filters=None, params=None, event_handlers=None, **kwargs):
+def get_chat_engine(params=None, event_handlers=None, **kwargs):
     system_prompt = os.getenv("SYSTEM_PROMPT")
     citation_prompt = os.getenv("SYSTEM_CITATION_PROMPT", None)
     top_k = int(os.getenv("TOP_K", 0))
@@ -33,10 +33,9 @@ def get_chat_engine(filters=None, params=None, event_handlers=None, **kwargs):
                 "StorageContext is empty - call 'poetry run generate' to generate the storage first"
             ),
         )
-
-    retriever = index.as_retriever(
-        filters=filters, **({"similarity_top_k": top_k} if top_k != 0 else {})
-    )
+    if top_k != 0 and kwargs.get("similarity_top_k") is None:
+        kwargs["similarity_top_k"] = top_k
+    retriever = index.as_retriever(**kwargs)
 
     return CondensePlusContextChatEngine(
         llm=llm,

templates/components/engines/typescript/agent/chat.ts

@@ -1,14 +1,9 @@
-import {
-  BaseChatEngine,
-  BaseToolWithCall,
-  LLMAgent,
-  QueryEngineTool,
-} from "llamaindex";
+import { BaseChatEngine, BaseToolWithCall, LLMAgent } from "llamaindex";
 import fs from "node:fs/promises";
 import path from "node:path";
 import { getDataSource } from "./index";
-import { generateFilters } from "./queryFilter";
 import { createTools } from "./tools";
+import { createQueryEngineTool } from "./tools/query-engine";
 
 export async function createChatEngine(documentIds?: string[], params?: any) {
   const tools: BaseToolWithCall[] = [];
@@ -17,17 +12,7 @@ export async function createChatEngine(documentIds?: string[], params?: any) {
   // Delete this code if you don't have a data source
   const index = await getDataSource(params);
   if (index) {
-    tools.push(
-      new QueryEngineTool({
-        queryEngine: index.asQueryEngine({
-          preFilters: generateFilters(documentIds || []),
-        }),
-        metadata: {
-          name: "data_query_engine",
-          description: `A query engine for documents from your data source.`,
-        },
-      }),
-    );
+    tools.push(createQueryEngineTool(index, { documentIds }));
   }
 
   const configFile = path.join("config", "tools.json");

templates/components/engines/typescript/agent/tools/interpreter.ts

@@ -116,7 +116,9 @@ export class InterpreterTool implements BaseTool<InterpreterParameter> {
           const fileName = path.basename(filePath);
           const localFilePath = path.join(this.uploadedFilesDir, fileName);
           const content = fs.readFileSync(localFilePath);
-          await this.codeInterpreter?.files.write(filePath, content);
+
+          const arrayBuffer = new Uint8Array(content).buffer;
+          await this.codeInterpreter?.files.write(filePath, arrayBuffer);
         }
       } catch (error) {
         console.error("Got error when uploading files to sandbox", error);

templates/components/engines/typescript/agent/tools/query-engine.ts

@@ -0,0 +1,57 @@
+import {
+  BaseQueryEngine,
+  CloudRetrieveParams,
+  LlamaCloudIndex,
+  MetadataFilters,
+  QueryEngineTool,
+  VectorStoreIndex,
+} from "llamaindex";
+import { generateFilters } from "../queryFilter";
+
+interface QueryEngineParams {
+  documentIds?: string[];
+  topK?: number;
+}
+
+export function createQueryEngineTool(
+  index: VectorStoreIndex | LlamaCloudIndex,
+  params?: QueryEngineParams,
+  name?: string,
+  description?: string,
+): QueryEngineTool {
+  return new QueryEngineTool({
+    queryEngine: createQueryEngine(index, params),
+    metadata: {
+      name: name || "query_engine",
+      description:
+        description ||
+        `Use this tool to retrieve information about the text corpus from an index.`,
+    },
+  });
+}
+
+function createQueryEngine(
+  index: VectorStoreIndex | LlamaCloudIndex,
+  params?: QueryEngineParams,
+): BaseQueryEngine {
+  const baseQueryParams = {
+    similarityTopK:
+      params?.topK ??
+      (process.env.TOP_K ? parseInt(process.env.TOP_K) : undefined),
+  };
+
+  if (index instanceof LlamaCloudIndex) {
+    return index.asQueryEngine({
+      ...baseQueryParams,
+      retrieval_mode: "auto_routed",
+      preFilters: generateFilters(
+        params?.documentIds || [],
+      ) as CloudRetrieveParams["filters"],
+    });
+  }
+
+  return index.asQueryEngine({
+    ...baseQueryParams,
+    preFilters: generateFilters(params?.documentIds || []) as MetadataFilters,
+  });
+}

templates/components/multiagent/python/app/api/routers/chat.py

@@ -1,12 +1,13 @@
 import logging
 
+from fastapi import APIRouter, BackgroundTasks, HTTPException, Request, status
+
 from app.api.routers.models import (
     ChatData,
 )
 from app.api.routers.vercel_response import VercelStreamResponse
 from app.engine.query_filter import generate_filters
 from app.workflows import create_workflow
-from fastapi import APIRouter, BackgroundTasks, HTTPException, Request, status
 
 chat_router = r = APIRouter()
 
@@ -28,7 +29,9 @@ async def chat(
         params = data.data or {}
 
         workflow = create_workflow(
-            chat_history=messages, params=params, filters=filters
+            chat_history=messages,
+            params=params,
+            filters=filters,
         )
 
         event_handler = workflow.run(input=last_message_content, streaming=True)

templates/components/multiagent/python/app/api/routers/vercel_response.py

@@ -19,6 +19,7 @@ class VercelStreamResponse(StreamingResponse):
 
     TEXT_PREFIX = "0:"
     DATA_PREFIX = "8:"
+    ERROR_PREFIX = "3:"
 
     def __init__(self, request: Request, chat_data: ChatData, *args, **kwargs):
         self.request = request
@@ -41,13 +42,16 @@ class VercelStreamResponse(StreamingResponse):
 
                     yield output
         except asyncio.CancelledError:
-            logger.info("Stopping workflow")
-            await event_handler.cancel_run()
+            logger.warning("Workflow has been cancelled!")
         except Exception as e:
             logger.error(
                 f"Unexpected error in content_generator: {str(e)}", exc_info=True
             )
+            yield self.convert_error(
+                "An unexpected error occurred while processing your request, preventing the creation of a final answer. Please try again."
+            )
         finally:
+            await event_handler.cancel_run()
             logger.info("The stream has been stopped!")
 
     def _create_stream(
@@ -107,6 +111,11 @@ class VercelStreamResponse(StreamingResponse):
         data_str = json.dumps(data)
         return f"{cls.DATA_PREFIX}[{data_str}]\n"
 
+    @classmethod
+    def convert_error(cls, error: str):
+        error_str = json.dumps(error)
+        return f"{cls.ERROR_PREFIX}{error_str}\n"
+
     @staticmethod
     async def _generate_next_questions(chat_history: List[Message], response: str):
         questions = await NextQuestionSuggestion.suggest_next_questions(

templates/components/multiagent/typescript/express/chat.controller.ts

@@ -1,4 +1,4 @@
-import { Message, streamToResponse } from "ai";
+import { LlamaIndexAdapter, Message } from "ai";
 import { Request, Response } from "express";
 import {
   convertToChatHistory,
@@ -28,7 +28,20 @@ export const chat = async (req: Request, res: Response) => {
     const { stream, dataStream } =
       await createStreamFromWorkflowContext(context);
 
-    return streamToResponse(stream, res, {}, dataStream);
+    const streamResponse = LlamaIndexAdapter.toDataStreamResponse(stream, {
+      data: dataStream,
+    });
+    if (streamResponse.body) {
+      const reader = streamResponse.body.getReader();
+      while (true) {
+        const { done, value } = await reader.read();
+        if (done) {
+          res.end();
+          return;
+        }
+        res.write(value);
+      }
+    }
   } catch (error) {
     console.error("[LlamaIndex]", error);
     return res.status(500).json({

templates/components/multiagent/typescript/nextjs/route.ts

@@ -1,5 +1,5 @@
 import { initObservability } from "@/app/observability";
-import { StreamingTextResponse, type Message } from "ai";
+import { LlamaIndexAdapter, type Message } from "ai";
 import { NextRequest, NextResponse } from "next/server";
 import { initSettings } from "./engine/settings";
 import {
@@ -41,9 +41,9 @@ export async function POST(request: NextRequest) {
     });
     const { stream, dataStream } =
       await createStreamFromWorkflowContext(context);
-
-    // Return the two streams in one response
-    return new StreamingTextResponse(stream, {}, dataStream);
+    return LlamaIndexAdapter.toDataStreamResponse(stream, {
+      data: dataStream,
+    });
   } catch (error) {
     console.error("[LlamaIndex]", error);
     return NextResponse.json(

templates/components/multiagent/typescript/workflow/stream.ts

@@ -3,20 +3,15 @@ import {
   WorkflowContext,
   WorkflowEvent,
 } from "@llamaindex/workflow";
-import {
-  StreamData,
-  createStreamDataTransformer,
-  trimStartOfStreamHelper,
-} from "ai";
-import { ChatResponseChunk } from "llamaindex";
+import { StreamData } from "ai";
+import { ChatResponseChunk, EngineResponse } from "llamaindex";
+import { ReadableStream } from "stream/web";
 import { AgentRunEvent } from "./type";
 
 export async function createStreamFromWorkflowContext<Input, Output, Context>(
   context: WorkflowContext<Input, Output, Context>,
-): Promise<{ stream: ReadableStream<string>; dataStream: StreamData }> {
-  const trimStartOfStream = trimStartOfStreamHelper();
+): Promise<{ stream: ReadableStream<EngineResponse>; dataStream: StreamData }> {
   const dataStream = new StreamData();
-  const encoder = new TextEncoder();
   let generator: AsyncGenerator<ChatResponseChunk> | undefined;
 
   const closeStreams = (controller: ReadableStreamDefaultController) => {
@@ -24,10 +19,10 @@ export async function createStreamFromWorkflowContext<Input, Output, Context>(
     dataStream.close();
   };
 
-  const mainStream = new ReadableStream({
+  const stream = new ReadableStream<EngineResponse>({
     async start(controller) {
       // Kickstart the stream by sending an empty string
-      controller.enqueue(encoder.encode(""));
+      controller.enqueue({ delta: "" } as EngineResponse);
     },
     async pull(controller) {
       while (!generator) {
@@ -46,17 +41,14 @@ export async function createStreamFromWorkflowContext<Input, Output, Context>(
         closeStreams(controller);
         return;
       }
-      const text = trimStartOfStream(chunk.delta ?? "");
-      if (text) {
-        controller.enqueue(encoder.encode(text));
+      const delta = chunk.delta ?? "";
+      if (delta) {
+        controller.enqueue({ delta } as EngineResponse);
       }
     },
   });
 
-  return {
-    stream: mainStream.pipeThrough(createStreamDataTransformer()),
-    dataStream,
-  };
+  return { stream, dataStream };
 }
 
 function handleEvent(

templates/components/multiagent/typescript/workflow/tools.ts

@@ -5,7 +5,6 @@ import {
   ChatMessage,
   ChatResponse,
   ChatResponseChunk,
-  LlamaCloudIndex,
   PartialToolCall,
   QueryEngineTool,
   ToolCall,
@@ -14,58 +13,17 @@ import {
 } from "llamaindex";
 import crypto from "node:crypto";
 import { getDataSource } from "../engine";
+import { createQueryEngineTool } from "../engine/tools/query-engine";
 import { AgentRunEvent } from "./type";
 
-export const getQueryEngineTools = async (): Promise<
-  QueryEngineTool[] | null
-> => {
-  const topK = process.env.TOP_K ? parseInt(process.env.TOP_K) : undefined;
-
+export const getQueryEngineTool = async (): Promise<QueryEngineTool | null> => {
   const index = await getDataSource();
+
   if (!index) {
     return null;
   }
-  // index is LlamaCloudIndex use two query engine tools
-  if (index instanceof LlamaCloudIndex) {
-    return [
-      new QueryEngineTool({
-        queryEngine: index.asQueryEngine({
-          similarityTopK: topK,
-          retrieval_mode: "files_via_content",
-        }),
-        metadata: {
-          name: "document_retriever",
-          description: `Document retriever that retrieves entire documents from the corpus.
-  ONLY use for research questions that may require searching over entire research reports.
-  Will be slower and more expensive than chunk-level retrieval but may be necessary.`,
-        },
-      }),
-      new QueryEngineTool({
-        queryEngine: index.asQueryEngine({
-          similarityTopK: topK,
-          retrieval_mode: "chunks",
-        }),
-        metadata: {
-          name: "chunk_retriever",
-          description: `Retrieves a small set of relevant document chunks from the corpus.
-      Use for research questions that want to look up specific facts from the knowledge corpus,
-      and need entire documents.`,
-        },
-      }),
-    ];
-  } else {
-    return [
-      new QueryEngineTool({
-        queryEngine: index.asQueryEngine({
-          similarityTopK: topK,
-        }),
-        metadata: {
-          name: "retriever",
-          description: `Use this tool to retrieve information about the text corpus from the index.`,
-        },
-      }),
-    ];
-  }
+
+  return createQueryEngineTool(index);
 };
 
 /**

templates/components/settings/python/settings.py

@@ -1,8 +1,17 @@
 import os
-from typing import Dict
+from typing import Dict, Optional
 
+from llama_index.core.multi_modal_llms import MultiModalLLM
 from llama_index.core.settings import Settings
 
+# `Settings` does not support setting `MultiModalLLM`
+# so we use a global variable to store it
+_multi_modal_llm: Optional[MultiModalLLM] = None
+
+
+def get_multi_modal_llm():
+    return _multi_modal_llm
+
 
 def init_settings():
     model_provider = os.getenv("MODEL_PROVIDER")
@@ -60,14 +69,21 @@ def init_openai():
     from llama_index.core.constants import DEFAULT_TEMPERATURE
     from llama_index.embeddings.openai import OpenAIEmbedding
     from llama_index.llms.openai import OpenAI
+    from llama_index.multi_modal_llms.openai import OpenAIMultiModal
+    from llama_index.multi_modal_llms.openai.utils import GPT4V_MODELS
 
     max_tokens = os.getenv("LLM_MAX_TOKENS")
+    model_name = os.getenv("MODEL", "gpt-4o-mini")
     Settings.llm = OpenAI(
-        model=os.getenv("MODEL", "gpt-4o-mini"),
+        model=model_name,
         temperature=float(os.getenv("LLM_TEMPERATURE", DEFAULT_TEMPERATURE)),
         max_tokens=int(max_tokens) if max_tokens is not None else None,
     )
 
+    if model_name in GPT4V_MODELS:
+        global _multi_modal_llm
+        _multi_modal_llm = OpenAIMultiModal(model=model_name)
+
     dimensions = os.getenv("EMBEDDING_DIM")
     Settings.embed_model = OpenAIEmbedding(
         model=os.getenv("EMBEDDING_MODEL", "text-embedding-3-small"),

templates/components/vectordbs/python/llamacloud/generate.py

@@ -1,5 +1,4 @@
 # flake8: noqa: E402
-import os
 
 from dotenv import load_dotenv
 
@@ -7,62 +6,24 @@ load_dotenv()
 
 import logging
 
-from app.engine.index import get_client, get_index
+from llama_index.core.readers import SimpleDirectoryReader
+from tqdm import tqdm
+
+from app.engine.index import get_index
 from app.engine.service import LLamaCloudFileService  # type: ignore
 from app.settings import init_settings
-from llama_cloud import PipelineType
-from llama_index.core.readers import SimpleDirectoryReader
-from llama_index.core.settings import Settings
 
 logging.basicConfig(level=logging.INFO)
 logger = logging.getLogger()
 
 
-def ensure_index(index):
-    project_id = index._get_project_id()
-    client = get_client()
-    pipelines = client.pipelines.search_pipelines(
-        project_id=project_id,
-        pipeline_name=index.name,
-        pipeline_type=PipelineType.MANAGED.value,
-    )
-    if len(pipelines) == 0:
-        from llama_index.embeddings.openai import OpenAIEmbedding
-
-        if not isinstance(Settings.embed_model, OpenAIEmbedding):
-            raise ValueError(
-                "Creating a new pipeline with a non-OpenAI embedding model is not supported."
-            )
-        client.pipelines.upsert_pipeline(
-            project_id=project_id,
-            request={
-                "name": index.name,
-                "embedding_config": {
-                    "type": "OPENAI_EMBEDDING",
-                    "component": {
-                        "api_key": os.getenv("OPENAI_API_KEY"),  # editable
-                        "model_name": os.getenv("EMBEDDING_MODEL"),
-                    },
-                },
-                "transform_config": {
-                    "mode": "auto",
-                    "config": {
-                        "chunk_size": Settings.chunk_size,  # editable
-                        "chunk_overlap": Settings.chunk_overlap,  # editable
-                    },
-                },
-            },
-        )
-
-
 def generate_datasource():
     init_settings()
     logger.info("Generate index for the provided data")
 
-    index = get_index()
-    ensure_index(index)
-    project_id = index._get_project_id()
-    pipeline_id = index._get_pipeline_id()
+    index = get_index(create_if_missing=True)
+    if index is None:
+        raise ValueError("Index not found and could not be created")
 
     # use SimpleDirectoryReader to retrieve the files to process
     reader = SimpleDirectoryReader(
@@ -72,14 +33,30 @@ def generate_datasource():
     files_to_process = reader.input_files
 
     # add each file to the LlamaCloud pipeline
-    for input_file in files_to_process:
+    error_files = []
+    for input_file in tqdm(
+        files_to_process,
+        desc="Processing files",
+        unit="file",
+    ):
         with open(input_file, "rb") as f:
-            logger.info(
+            logger.debug(
                 f"Adding file {input_file} to pipeline {index.name} in project {index.project_name}"
             )
-            LLamaCloudFileService.add_file_to_pipeline(
-                project_id, pipeline_id, f, custom_metadata={}
-            )
+            try:
+                LLamaCloudFileService.add_file_to_pipeline(
+                    index.project.id,
+                    index.pipeline.id,
+                    f,
+                    custom_metadata={},
+                    wait_for_processing=False,
+                )
+            except Exception as e:
+                error_files.append(input_file)
+                logger.error(f"Error adding file {input_file}: {e}")
+
+    if error_files:
+        logger.error(f"Failed to add the following files: {error_files}")
 
     logger.info("Finished generating the index")
 

templates/components/vectordbs/python/llamacloud/index.py

@@ -2,10 +2,12 @@ import logging
 import os
 from typing import Optional
 
+from llama_cloud import PipelineType
 from llama_index.core.callbacks import CallbackManager
 from llama_index.core.ingestion.api_utils import (
     get_client as llama_cloud_get_client,
 )
+from llama_index.core.settings import Settings
 from llama_index.indices.managed.llama_cloud import LlamaCloudIndex
 from pydantic import BaseModel, Field, field_validator
 
@@ -82,14 +84,63 @@ class IndexConfig(BaseModel):
         }
 
 
-def get_index(config: IndexConfig = None):
+def get_index(
+    config: IndexConfig = None,
+    create_if_missing: bool = False,
+):
     if config is None:
         config = IndexConfig()
-    index = LlamaCloudIndex(**config.to_index_kwargs())
-
-    return index
+    # Check whether the index exists
+    try:
+        index = LlamaCloudIndex(**config.to_index_kwargs())
+        return index
+    except ValueError:
+        logger.warning("Index not found")
+        if create_if_missing:
+            logger.info("Creating index")
+            _create_index(config)
+            return LlamaCloudIndex(**config.to_index_kwargs())
+        return None
 
 
 def get_client():
     config = LlamaCloudConfig()
     return llama_cloud_get_client(**config.to_client_kwargs())
+
+
+def _create_index(
+    config: IndexConfig,
+):
+    client = get_client()
+    pipeline_name = config.llama_cloud_pipeline_config.pipeline
+
+    pipelines = client.pipelines.search_pipelines(
+        pipeline_name=pipeline_name,
+        pipeline_type=PipelineType.MANAGED.value,
+    )
+    if len(pipelines) == 0:
+        from llama_index.embeddings.openai import OpenAIEmbedding
+
+        if not isinstance(Settings.embed_model, OpenAIEmbedding):
+            raise ValueError(
+                "Creating a new pipeline with a non-OpenAI embedding model is not supported."
+            )
+        client.pipelines.upsert_pipeline(
+            request={
+                "name": pipeline_name,
+                "embedding_config": {
+                    "type": "OPENAI_EMBEDDING",
+                    "component": {
+                        "api_key": os.getenv("OPENAI_API_KEY"),  # editable
+                        "model_name": os.getenv("EMBEDDING_MODEL"),
+                    },
+                },
+                "transform_config": {
+                    "mode": "auto",
+                    "config": {
+                        "chunk_size": Settings.chunk_size,  # editable
+                        "chunk_overlap": Settings.chunk_overlap,  # editable
+                    },
+                },
+            },
+        )

templates/components/vectordbs/python/llamacloud/service.py

@@ -1,18 +1,18 @@
-from io import BytesIO
 import logging
 import os
 import time
-from typing import Any, Dict, List, Optional, Set, Tuple, Union
 import typing
+from io import BytesIO
+from typing import Any, Dict, List, Optional, Set, Tuple, Union
 
+import requests
 from fastapi import BackgroundTasks
 from llama_cloud import ManagedIngestionStatus, PipelineFileCreateCustomMetadataValue
+from llama_index.core.schema import NodeWithScore
 from pydantic import BaseModel
-import requests
+
 from app.api.routers.models import SourceNodes
 from app.engine.index import get_client
-from llama_index.core.schema import NodeWithScore
-
 
 logger = logging.getLogger("uvicorn")
 
@@ -64,27 +64,34 @@ class LLamaCloudFileService:
         pipeline_id: str,
         upload_file: Union[typing.IO, Tuple[str, BytesIO]],
         custom_metadata: Optional[Dict[str, PipelineFileCreateCustomMetadataValue]],
+        wait_for_processing: bool = True,
     ) -> str:
         client = get_client()
         file = client.files.upload_file(project_id=project_id, upload_file=upload_file)
+        file_id = file.id
         files = [
             {
-                "file_id": file.id,
-                "custom_metadata": {"file_id": file.id, **(custom_metadata or {})},
+                "file_id": file_id,
+                "custom_metadata": {"file_id": file_id, **(custom_metadata or {})},
             }
         ]
         files = client.pipelines.add_files_to_pipeline(pipeline_id, request=files)
 
+        if not wait_for_processing:
+            return file_id
+
         # Wait 2s for the file to be processed
         max_attempts = 20
         attempt = 0
         while attempt < max_attempts:
-            result = client.pipelines.get_pipeline_file_status(pipeline_id, file.id)
+            result = client.pipelines.get_pipeline_file_status(
+                file_id=file_id, pipeline_id=pipeline_id
+            )
             if result.status == ManagedIngestionStatus.ERROR:
                 raise Exception(f"File processing failed: {str(result)}")
             if result.status == ManagedIngestionStatus.SUCCESS:
                 # File is ingested - return the file id
-                return file.id
+                return file_id
             attempt += 1
             time.sleep(0.1)  # Sleep for 100ms
         raise Exception(

templates/types/extractor/fastapi/app/ui/components/upload.py

@@ -2,6 +2,7 @@ import os
 from typing import List
 
 import reflex as rx
+
 from app.engine.generate import generate_datasource
 
 
@@ -78,10 +79,10 @@ def upload_component() -> rx.Component:
             UploadedFilesState.uploaded_files,
             lambda file: rx.card(
                 rx.stack(
-                    rx.text(file.file_name, size="sm"),
+                    rx.text(file.file_name, size="2"),
                     rx.button(
                         "x",
-                        size="sm",
+                        size="2",
                         on_click=UploadedFilesState.remove_file(file.file_name),
                     ),
                     justify="between",

templates/types/extractor/fastapi/pyproject.toml

@@ -14,7 +14,7 @@ fastapi = "^0.109.1"
 uvicorn = { extras = ["standard"], version = "^0.23.2" }
 python-dotenv = "^1.0.0"
 pydantic = "<2.10"
-llama-index = "^0.11.1"
+llama-index = "^0.12.1"
 cachetools = "^5.3.3"
 reflex = "^0.6.2.post1"
 

templates/types/streaming/express/README-template.md

@@ -8,7 +8,7 @@ First, install the dependencies:
 npm install
 ```
 
-Second, generate the embeddings of the documents in the `./data` directory (if this folder exists - otherwise, skip this step):
+Second, generate the embeddings of the documents in the `./data` directory:
 
 ```
 npm run generate

templates/types/streaming/express/package.json

@@ -16,7 +16,7 @@
     "lint": "eslint ."
   },
   "dependencies": {
-    "ai": "3.3.42",
+    "ai": "4.0.3",
     "cors": "^2.8.5",
     "dotenv": "^16.3.1",
     "duck-duck-scrape": "^2.2.5",

templates/types/streaming/express/src/controllers/chat.controller.ts

@@ -1,4 +1,4 @@
-import { LlamaIndexAdapter, Message, StreamData, streamToResponse } from "ai";
+import { LlamaIndexAdapter, Message, StreamData } from "ai";
 import { Request, Response } from "express";
 import { ChatMessage, Settings } from "llamaindex";
 import { createChatEngine } from "./engine/chat";
@@ -43,7 +43,7 @@ export const chat = async (req: Request, res: Response) => {
       });
     });
 
-    const onFinal = (content: string) => {
+    const onCompletion = (content: string) => {
       chatHistory.push({ role: "assistant", content: content });
       generateNextQuestions(chatHistory)
         .then((questions: string[]) => {
@@ -59,8 +59,21 @@ export const chat = async (req: Request, res: Response) => {
         });
     };
 
-    const stream = LlamaIndexAdapter.toDataStream(response, { onFinal });
-    return streamToResponse(stream, res, {}, vercelStreamData);
+    const streamResponse = LlamaIndexAdapter.toDataStreamResponse(response, {
+      data: vercelStreamData,
+      callbacks: { onCompletion },
+    });
+    if (streamResponse.body) {
+      const reader = streamResponse.body.getReader();
+      while (true) {
+        const { done, value } = await reader.read();
+        if (done) {
+          res.end();
+          return;
+        }
+        res.write(value);
+      }
+    }
   } catch (error) {
     console.error("[LlamaIndex]", error);
     return res.status(500).json({

templates/types/streaming/fastapi/README-template.md

@@ -15,7 +15,7 @@ Then check the parameters that have been pre-configured in the `.env` file in th
 
 If you are using any tools or data sources, you can update their config files in the `config` folder.
 
-Second, generate the embeddings of the documents in the `./data` directory (if this folder exists - otherwise, skip this step):
+Second, generate the embeddings of the documents in the `./data` directory:
 
 ```
 poetry run generate

templates/types/streaming/fastapi/app/api/routers/vercel_response.py

@@ -22,6 +22,7 @@ class VercelStreamResponse(StreamingResponse):
 
     TEXT_PREFIX = "0:"
     DATA_PREFIX = "8:"
+    ERROR_PREFIX = "3:"
 
     def __init__(
         self,
@@ -53,17 +54,26 @@ class VercelStreamResponse(StreamingResponse):
         # Merge the chat response generator and the event generator
         combine = stream.merge(chat_response_generator, event_generator)
         is_stream_started = False
-        async with combine.stream() as streamer:
-            async for output in streamer:
-                if not is_stream_started:
-                    is_stream_started = True
-                    # Stream a blank message to start displaying the response in the UI
-                    yield cls.convert_text("")
+        try:
+            async with combine.stream() as streamer:
+                async for output in streamer:
+                    if await request.is_disconnected():
+                        break
 
-                yield output
+                    if not is_stream_started:
+                        is_stream_started = True
+                        # Stream a blank message to start displaying the response in the UI
+                        yield cls.convert_text("")
 
-                if await request.is_disconnected():
-                    break
+                    yield output
+        except Exception:
+            logger.exception("Error in stream response")
+            yield cls.convert_error(
+                "An unexpected error occurred while processing your request, preventing the creation of a final answer. Please try again."
+            )
+        finally:
+            # Ensure event handler is marked as done even if connection breaks
+            event_handler.is_done = True
 
     @classmethod
     async def _event_generator(cls, event_handler: EventCallbackHandler):
@@ -131,6 +141,11 @@ class VercelStreamResponse(StreamingResponse):
         data_str = json.dumps(data)
         return f"{cls.DATA_PREFIX}[{data_str}]\n"
 
+    @classmethod
+    def convert_error(cls, error: str):
+        error_str = json.dumps(error)
+        return f"{cls.ERROR_PREFIX}{error_str}\n"
+
     @staticmethod
     def _process_response_nodes(
         source_nodes: List[NodeWithScore],

templates/types/streaming/fastapi/app/services/file.py

@@ -249,6 +249,7 @@ class FileService:
             index.pipeline.id,
             upload_file,
             custom_metadata={},
+            wait_for_processing=True,
         )
         return doc_id
 

templates/types/streaming/fastapi/pyproject.toml

@@ -19,7 +19,7 @@ python-dotenv = "^1.0.0"
 pydantic = "<2.10"
 aiostream = "^0.5.2"
 cachetools = "^5.3.3"
-llama-index = "^0.11.17"
+llama-index = "^0.12.1"
 rich = "^13.9.4"
 
 [tool.poetry.group.dev.dependencies]

templates/types/streaming/nextjs/README-template.md

@@ -8,7 +8,7 @@ First, install the dependencies:
 npm install
 ```
 
-Second, generate the embeddings of the documents in the `./data` directory (if this folder exists - otherwise, skip this step):
+Second, generate the embeddings of the documents in the `./data` directory:
 
 ```
 npm run generate

templates/types/streaming/nextjs/app/api/chat/route.ts

@@ -56,7 +56,7 @@ export async function POST(request: NextRequest) {
       });
     });
 
-    const onFinal = (content: string) => {
+    const onCompletion = (content: string) => {
       chatHistory.push({ role: "assistant", content: content });
       generateNextQuestions(chatHistory)
         .then((questions: string[]) => {
@@ -74,7 +74,7 @@ export async function POST(request: NextRequest) {
 
     return LlamaIndexAdapter.toDataStreamResponse(response, {
       data: vercelStreamData,
-      callbacks: { onFinal },
+      callbacks: { onCompletion },
     });
   } catch (error) {
     console.error("[LlamaIndex]", error);

templates/types/streaming/nextjs/app/api/sandbox/route.ts

@@ -92,7 +92,8 @@ export async function POST(req: Request) {
       const localFilePath = path.join("output", "uploaded", fileName);
       const fileContent = await fs.readFile(localFilePath);
 
-      await sbx.files.write(sandboxFilePath, fileContent);
+      const arrayBuffer = new Uint8Array(fileContent).buffer;
+      await sbx.files.write(sandboxFilePath, arrayBuffer);
       console.log(`Copied file to ${sandboxFilePath} in ${sbx.sandboxID}`);
     });
   }

templates/types/streaming/nextjs/app/components/chat-section.tsx

@@ -1,8 +1,7 @@
 "use client";
 
 import { ChatSection as ChatSectionUI } from "@llamaindex/chat-ui";
-import "@llamaindex/chat-ui/styles/code.css";
-import "@llamaindex/chat-ui/styles/katex.css";
+import "@llamaindex/chat-ui/styles/markdown.css";
 import "@llamaindex/chat-ui/styles/pdf.css";
 import { useChat } from "ai/react";
 import CustomChatInput from "./ui/chat/chat-input";
@@ -15,7 +14,13 @@ export default function ChatSection() {
     api: `${backend}/api/chat`,
     onError: (error: unknown) => {
       if (!(error instanceof Error)) throw error;
-      alert(JSON.parse(error.message).detail);
+      let errorMessage: string;
+      try {
+        errorMessage = JSON.parse(error.message).detail;
+      } catch (e) {
+        errorMessage = error.message;
+      }
+      alert(errorMessage);
     },
   });
   return (

templates/types/streaming/nextjs/package.json

@@ -16,8 +16,8 @@
     "@radix-ui/react-select": "^2.1.1",
     "@radix-ui/react-slot": "^1.0.2",
     "@radix-ui/react-tabs": "^1.1.0",
-    "@llamaindex/chat-ui": "0.0.9",
-    "ai": "3.4.33",
+    "@llamaindex/chat-ui": "0.0.12",
+    "ai": "4.0.3",
     "ajv": "^8.12.0",
     "class-variance-authority": "^0.7.0",
     "clsx": "^2.1.1",