Release 0.3.21 (#459)

Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>

CHANGELOG.md

@@ -1,5 +1,54 @@
 # create-llama
 
+## 0.3.21
+
+### Patch Changes
+
+- 95227a7: Add query endpoint
+
+## 0.3.20
+
+### Patch Changes
+
+- 27d2499: Bump the LlamaCloud library and fix breaking changes (Python).
+
+## 0.3.19
+
+### Patch Changes
+
+- f9a057d: Add support multimodal indexes (e.g. from LlamaCloud)
+- aedd73d: bump: chat-ui
+
+## 0.3.18
+
+### Patch Changes
+
+- fe90a7e: chore: bump ai v4
+- 02b2473: Show streaming errors in Python, optimize system prompts for tool usage and set the weather tool as default for the Agentic RAG use case
+- 63e961e: Use auto_routed retriever mode for LlamaCloudIndex
+
+## 0.3.17
+
+### Patch Changes
+
+- 28c8808: Add fly.io deployment
+- 0a7dfcf: Generate NEXT_PUBLIC_CHAT_API for NextJS backend to specify alternative backend
+
+## 0.3.16
+
+### Patch Changes
+
+- 8b371d8: Set pydantic version to <2.10 to avoid incompatibility with llama-index.
+- 30fe269: Deactive duckduckgo tool for TS
+- 30fe269: Replace DuckDuckGo by Wikipedia tool for agentic template
+
+## 0.3.15
+
+### Patch Changes
+
+- fc5b266: Improve DX for Python template (use one deployment instead of two)
+- f8f97d2: Add support for python 3.13
+
 ## 0.3.14
 
 ### Patch Changes

create-app.ts

@@ -7,17 +7,16 @@ import { getOnline } from "./helpers/is-online";
 import { isWriteable } from "./helpers/is-writeable";
 import { makeDir } from "./helpers/make-dir";
 
-import fs from "fs";
 import terminalLink from "terminal-link";
 import type { InstallTemplateArgs, TemplateObservability } from "./helpers";
 import { installTemplate } from "./helpers";
-import { writeDevcontainer } from "./helpers/devcontainer";
 import { templatesDir } from "./helpers/dir";
 import { toolsRequireConfig } from "./helpers/tools";
+import { configVSCode } from "./helpers/vscode";
 
 export type InstallAppArgs = Omit<
   InstallTemplateArgs,
-  "appName" | "root" | "isOnline" | "customApiPath"
+  "appName" | "root" | "isOnline" | "port"
 > & {
   appPath: string;
   frontend: boolean;
@@ -35,7 +34,6 @@ export async function createApp({
   communityProjectConfig,
   llamapack,
   vectorDb,
-  externalPort,
   postInstallAction,
   dataSources,
   tools,
@@ -81,7 +79,6 @@ export async function createApp({
     communityProjectConfig,
     llamapack,
     vectorDb,
-    externalPort,
     postInstallAction,
     dataSources,
     tools,
@@ -90,31 +87,22 @@ export async function createApp({
     agents,
   };
 
-  if (frontend) {
-    // install backend
-    const backendRoot = path.join(root, "backend");
-    await makeDir(backendRoot);
-    await installTemplate({ ...args, root: backendRoot, backend: true });
+  // Install backend
+  await installTemplate({ ...args, backend: true });
+
+  if (frontend && framework === "fastapi") {
     // install frontend
-    const frontendRoot = path.join(root, "frontend");
+    const frontendRoot = path.join(root, ".frontend");
     await makeDir(frontendRoot);
     await installTemplate({
       ...args,
       root: frontendRoot,
       framework: "nextjs",
-      customApiPath: `http://localhost:${externalPort ?? 8000}/api/chat`,
       backend: false,
     });
-    // copy readme for fullstack
-    await fs.promises.copyFile(
-      path.join(templatesDir, "README-fullstack.md"),
-      path.join(root, "README.md"),
-    );
-  } else {
-    await installTemplate({ ...args, backend: true });
   }
 
-  await writeDevcontainer(root, templatesDir, framework, frontend);
+  await configVSCode(root, templatesDir, framework);
 
   process.chdir(root);
   if (tryGitInit(root)) {

e2e/python/resolve_dependencies.spec.ts

@@ -63,7 +63,6 @@ if (
             vectorDb,
             tools: "none",
             port: 3000,
-            externalPort: 8000,
             postInstallAction: "none",
             templateUI: undefined,
             appType: "--no-frontend",
@@ -101,7 +100,6 @@ if (
             vectorDb: "none",
             tools: tool,
             port: 3000,
-            externalPort: 8000,
             postInstallAction: "none",
             templateUI: undefined,
             appType: "--no-frontend",
@@ -135,7 +133,6 @@ if (
             vectorDb: "none",
             tools: "none",
             port: 3000,
-            externalPort: 8000,
             postInstallAction: "none",
             templateUI: undefined,
             appType: "--no-frontend",
@@ -169,7 +166,6 @@ if (
             vectorDb: "none",
             tools: "none",
             port: 3000,
-            externalPort: 8000,
             postInstallAction: "none",
             templateUI: undefined,
             appType: "--no-frontend",

e2e/shared/extractor_template.spec.ts

@@ -20,8 +20,7 @@ if (
   dataSource === "--example-file"
 ) {
   test.describe("Test extractor template", async () => {
-    let frontendPort: number;
-    let backendPort: number;
+    let appPort: number;
     let name: string;
     let appProcess: ChildProcess;
     let cwd: string;
@@ -29,16 +28,14 @@ if (
     // Create extractor app
     test.beforeAll(async () => {
       cwd = await createTestDir();
-      frontendPort = Math.floor(Math.random() * 10000) + 10000;
-      backendPort = frontendPort + 1;
+      appPort = Math.floor(Math.random() * 10000) + 10000;
       const result = await runCreateLlama({
         cwd,
         templateType: "extractor",
         templateFramework: "fastapi",
         dataSource: "--example-file",
         vectorDb: "none",
-        port: frontendPort,
-        externalPort: backendPort,
+        port: appPort,
         postInstallAction: "runApp",
       });
       name = result.projectName;
@@ -54,7 +51,7 @@ if (
       expect(dirExists).toBeTruthy();
     });
     test("Frontend should have a title", async ({ page }) => {
-      await page.goto(`http://localhost:${frontendPort}`);
+      await page.goto(`http://localhost:${appPort}`);
       await expect(page.getByText("Built by LlamaIndex")).toBeVisible({
         timeout: 2000 * 60,
       });

e2e/shared/multiagent_template.spec.ts

@@ -16,7 +16,7 @@ const templateFramework: TemplateFramework = process.env.FRAMEWORK
 const dataSource: string = "--example-file";
 const templateUI: TemplateUI = "shadcn";
 const templatePostInstallAction: TemplatePostInstallAction = "runApp";
-const appType: AppType = templateFramework === "nextjs" ? "" : "--frontend";
+const appType: AppType = templateFramework === "fastapi" ? "--frontend" : "";
 const userMessage = "Write a blog post about physical standards for letters";
 const templateAgents = ["financial_report", "blog", "form_filling"];
 
@@ -27,7 +27,6 @@ for (const agents of templateAgents) {
       "The multiagent template currently only works with files. We also only run on Linux to speed up tests.",
     );
     let port: number;
-    let externalPort: number;
     let cwd: string;
     let name: string;
     let appProcess: ChildProcess;
@@ -36,7 +35,6 @@ for (const agents of templateAgents) {
 
     test.beforeAll(async () => {
       port = Math.floor(Math.random() * 10000) + 10000;
-      externalPort = port + 1;
       cwd = await createTestDir();
       const result = await runCreateLlama({
         cwd,
@@ -45,7 +43,6 @@ for (const agents of templateAgents) {
         dataSource,
         vectorDb,
         port,
-        externalPort,
         postInstallAction: templatePostInstallAction,
         templateUI,
         appType,
@@ -61,6 +58,10 @@ for (const agents of templateAgents) {
     });
 
     test("Frontend should have a title", async ({ page }) => {
+      test.skip(
+        templatePostInstallAction !== "runApp" ||
+          templateFramework === "express",
+      );
       await page.goto(`http://localhost:${port}`);
       await expect(page.getByText("Built by LlamaIndex")).toBeVisible();
     });
@@ -69,7 +70,10 @@ for (const agents of templateAgents) {
       page,
     }) => {
       test.skip(
-        agents === "financial_report" || agents === "form_filling",
+        templatePostInstallAction !== "runApp" ||
+          agents === "financial_report" ||
+          agents === "form_filling" ||
+          templateFramework === "express",
         "Skip chat tests for financial report and form filling.",
       );
       await page.goto(`http://localhost:${port}`);

e2e/shared/streaming_template.spec.ts

@@ -22,7 +22,7 @@ const templatePostInstallAction: TemplatePostInstallAction = "runApp";
 const llamaCloudProjectName = "create-llama";
 const llamaCloudIndexName = "e2e-test";
 
-const appType: AppType = templateFramework === "nextjs" ? "" : "--frontend";
+const appType: AppType = templateFramework === "fastapi" ? "--frontend" : "";
 const userMessage =
   dataSource !== "--no-files" ? "Physical standard for letters" : "Hello";
 
@@ -35,7 +35,6 @@ test.describe(`Test streaming template ${templateFramework} ${dataSource} ${temp
   }
 
   let port: number;
-  let externalPort: number;
   let cwd: string;
   let name: string;
   let appProcess: ChildProcess;
@@ -44,7 +43,6 @@ test.describe(`Test streaming template ${templateFramework} ${dataSource} ${temp
 
   test.beforeAll(async () => {
     port = Math.floor(Math.random() * 10000) + 10000;
-    externalPort = port + 1;
     cwd = await createTestDir();
     const result = await runCreateLlama({
       cwd,
@@ -53,7 +51,6 @@ test.describe(`Test streaming template ${templateFramework} ${dataSource} ${temp
       dataSource,
       vectorDb,
       port,
-      externalPort,
       postInstallAction: templatePostInstallAction,
       templateUI,
       appType,
@@ -68,8 +65,11 @@ test.describe(`Test streaming template ${templateFramework} ${dataSource} ${temp
     const dirExists = fs.existsSync(path.join(cwd, name));
     expect(dirExists).toBeTruthy();
   });
+
   test("Frontend should have a title", async ({ page }) => {
-    test.skip(templatePostInstallAction !== "runApp");
+    test.skip(
+      templatePostInstallAction !== "runApp" || templateFramework === "express",
+    );
     await page.goto(`http://localhost:${port}`);
     await expect(page.getByText("Built by LlamaIndex")).toBeVisible();
   });
@@ -77,7 +77,9 @@ test.describe(`Test streaming template ${templateFramework} ${dataSource} ${temp
   test("Frontend should be able to submit a message and receive a response", async ({
     page,
   }) => {
-    test.skip(templatePostInstallAction !== "runApp");
+    test.skip(
+      templatePostInstallAction !== "runApp" || templateFramework === "express",
+    );
     await page.goto(`http://localhost:${port}`);
     await page.fill("form textarea", userMessage);
     const [response] = await Promise.all([
@@ -102,7 +104,7 @@ test.describe(`Test streaming template ${templateFramework} ${dataSource} ${temp
     test.skip(templatePostInstallAction !== "runApp");
     test.skip(templateFramework === "nextjs");
     const response = await request.post(
-      `http://localhost:${externalPort}/api/chat/request`,
+      `http://localhost:${port}/api/chat/request`,
       {
         data: {
           messages: [

e2e/typescript/resolve_dependencies.spec.ts

@@ -56,7 +56,6 @@ test.describe("Test resolve TS dependencies", () => {
       dataSource: dataSource,
       vectorDb: vectorDb,
       port: 3000,
-      externalPort: 8000,
       postInstallAction: "none",
       templateUI: undefined,
       appType: templateFramework === "nextjs" ? "" : "--no-frontend",

e2e/utils.ts

@@ -25,7 +25,6 @@ export type RunCreateLlamaOptions = {
   dataSource: string;
   vectorDb: TemplateVectorDB;
   port: number;
-  externalPort: number;
   postInstallAction: TemplatePostInstallAction;
   templateUI?: TemplateUI;
   appType?: AppType;
@@ -44,7 +43,6 @@ export async function runCreateLlama({
   dataSource,
   vectorDb,
   port,
-  externalPort,
   postInstallAction,
   templateUI,
   appType,
@@ -93,8 +91,6 @@ export async function runCreateLlama({
     "--use-pnpm",
     "--port",
     port,
-    "--external-port",
-    externalPort,
     "--post-install-action",
     postInstallAction,
     "--tools",
@@ -142,12 +138,7 @@ export async function runCreateLlama({
 
   // Wait for app to start
   if (postInstallAction === "runApp") {
-    await checkAppHasStarted(
-      appType === "--frontend",
-      templateFramework,
-      port,
-      externalPort,
-    );
+    await waitPorts([port]);
   } else if (postInstallAction === "dependencies") {
     await waitForProcess(appProcess, 1000 * 60); // wait 1 min for dependencies to be resolved
   } else {
@@ -167,19 +158,6 @@ export async function createTestDir() {
   return cwd;
 }
 
-// eslint-disable-next-line max-params
-async function checkAppHasStarted(
-  frontend: boolean,
-  framework: TemplateFramework,
-  port: number,
-  externalPort: number,
-) {
-  const portsToWait = frontend
-    ? [port, externalPort]
-    : [framework === "nextjs" ? port : externalPort];
-  await waitPorts(portsToWait);
-}
-
 async function waitPorts(ports: number[]): Promise<void> {
   const waitForPort = async (port: number): Promise<void> => {
     await waitPort({

helpers/copy.ts

@@ -61,6 +61,9 @@ export const assetRelocator = (name: string) => {
     case "README-template.md": {
       return "README.md";
     }
+    case "vscode_settings.json": {
+      return "settings.json";
+    }
     default: {
       return name;
     }

helpers/env-variables.ts

@@ -13,6 +13,12 @@ import {
 
 import { TSYSTEMS_LLMHUB_API_URL } from "./providers/llmhub";
 
+const DEFAULT_SYSTEM_PROMPT =
+  "You are a helpful assistant who helps users with their questions.";
+
+const DATA_SOURCES_PROMPT =
+  "You have access to a knowledge base including the facts that you should start with to find the answer for the user question. Use the query engine tool to retrieve the facts from the knowledge base.";
+
 export type EnvVar = {
   name?: string;
   description?: string;
@@ -407,6 +413,13 @@ const getFrameworkEnvs = (
       ],
     );
   }
+  if (framework === "nextjs") {
+    result.push({
+      name: "NEXT_PUBLIC_CHAT_API",
+      description:
+        "The API for the chat endpoint. Set when using a custom backend (e.g. Express). Use full URL like http://localhost:8000/api/chat",
+    });
+  }
   return result;
 };
 
@@ -442,9 +455,6 @@ const getSystemPromptEnv = (
   dataSources?: TemplateDataSource[],
   template?: TemplateType,
 ): EnvVar[] => {
-  const defaultSystemPrompt =
-    "You are a helpful assistant who helps users with their questions.";
-
   const systemPromptEnv: EnvVar[] = [];
   // build tool system prompt by merging all tool system prompts
   // multiagent template doesn't need system prompt
@@ -459,9 +469,12 @@ const getSystemPromptEnv = (
       }
     });
 
-    const systemPrompt = toolSystemPrompt
-      ? `\"${toolSystemPrompt}\"`
-      : defaultSystemPrompt;
+    const systemPrompt =
+      "'" +
+      DEFAULT_SYSTEM_PROMPT +
+      (dataSources?.length ? `\n${DATA_SOURCES_PROMPT}` : "") +
+      (toolSystemPrompt ? `\n${toolSystemPrompt}` : "") +
+      "'";
 
     systemPromptEnv.push({
       name: "SYSTEM_PROMPT",
@@ -553,7 +566,7 @@ export const createBackendEnvFile = async (
     | "framework"
     | "dataSources"
     | "template"
-    | "externalPort"
+    | "port"
     | "tools"
     | "observability"
   >,
@@ -570,7 +583,7 @@ export const createBackendEnvFile = async (
     ...getModelEnvs(opts.modelConfig),
     ...getEngineEnvs(),
     ...getVectorDBEnvs(opts.vectorDb, opts.framework),
-    ...getFrameworkEnvs(opts.framework, opts.externalPort),
+    ...getFrameworkEnvs(opts.framework, opts.port),
     ...getToolEnvs(opts.tools),
     ...getTemplateEnvs(opts.template),
     ...getObservabilityEnvs(opts.observability),
@@ -585,18 +598,10 @@ export const createBackendEnvFile = async (
 export const createFrontendEnvFile = async (
   root: string,
   opts: {
-    customApiPath?: string;
     vectorDb?: TemplateVectorDB;
   },
 ) => {
   const defaultFrontendEnvs = [
-    {
-      name: "NEXT_PUBLIC_CHAT_API",
-      description: "The backend API for chat endpoint.",
-      value: opts.customApiPath
-        ? opts.customApiPath
-        : "http://localhost:8000/api/chat",
-    },
     {
       name: "NEXT_PUBLIC_USE_LLAMACLOUD",
       description: "Let's the user change indexes in LlamaCloud projects",

helpers/index.ts

@@ -225,7 +225,6 @@ export const installTemplate = async (
   } else {
     // this is a frontend for a full-stack app, create .env file with model information
     await createFrontendEnvFile(props.root, {
-      customApiPath: props.customApiPath,
       vectorDb: props.vectorDb,
     });
   }

helpers/python.ts

@@ -20,6 +20,7 @@ interface Dependency {
   name: string;
   version?: string;
   extras?: string[];
+  constraints?: Record<string, string>;
 }
 
 const getAdditionalDependencies = (
@@ -36,28 +37,31 @@ const getAdditionalDependencies = (
     case "mongo": {
       dependencies.push({
         name: "llama-index-vector-stores-mongodb",
-        version: "^0.3.1",
+        version: "^0.6.0",
       });
       break;
     }
     case "pg": {
       dependencies.push({
         name: "llama-index-vector-stores-postgres",
-        version: "^0.2.5",
+        version: "^0.3.2",
       });
       break;
     }
     case "pinecone": {
       dependencies.push({
         name: "llama-index-vector-stores-pinecone",
-        version: "^0.2.1",
+        version: "^0.4.1",
+        constraints: {
+          python: ">=3.11,<3.13",
+        },
       });
       break;
     }
     case "milvus": {
       dependencies.push({
         name: "llama-index-vector-stores-milvus",
-        version: "^0.2.0",
+        version: "^0.3.0",
       });
       dependencies.push({
         name: "pymilvus",
@@ -68,35 +72,38 @@ const getAdditionalDependencies = (
     case "astra": {
       dependencies.push({
         name: "llama-index-vector-stores-astra-db",
-        version: "^0.2.0",
+        version: "^0.4.0",
       });
       break;
     }
     case "qdrant": {
       dependencies.push({
         name: "llama-index-vector-stores-qdrant",
-        version: "^0.3.0",
+        version: "^0.4.0",
+        constraints: {
+          python: ">=3.11,<3.13",
+        },
       });
       break;
     }
     case "chroma": {
       dependencies.push({
         name: "llama-index-vector-stores-chroma",
-        version: "^0.2.0",
+        version: "^0.4.0",
       });
       break;
     }
     case "weaviate": {
       dependencies.push({
         name: "llama-index-vector-stores-weaviate",
-        version: "^1.1.1",
+        version: "^1.2.3",
       });
       break;
     }
     case "llamacloud":
       dependencies.push({
         name: "llama-index-indices-managed-llama-cloud",
-        version: "^0.3.1",
+        version: "^0.6.3",
       });
       break;
   }
@@ -115,13 +122,13 @@ const getAdditionalDependencies = (
         case "web":
           dependencies.push({
             name: "llama-index-readers-web",
-            version: "^0.2.2",
+            version: "^0.3.0",
           });
           break;
         case "db":
           dependencies.push({
             name: "llama-index-readers-database",
-            version: "^0.2.0",
+            version: "^0.3.0",
           });
           dependencies.push({
             name: "pymysql",
@@ -160,15 +167,15 @@ const getAdditionalDependencies = (
       if (templateType !== "multiagent") {
         dependencies.push({
           name: "llama-index-llms-openai",
-          version: "^0.2.0",
+          version: "^0.3.2",
         });
         dependencies.push({
           name: "llama-index-embeddings-openai",
-          version: "^0.2.3",
+          version: "^0.3.1",
         });
         dependencies.push({
           name: "llama-index-agent-openai",
-          version: "^0.3.0",
+          version: "^0.4.0",
         });
       }
       break;
@@ -279,14 +286,19 @@ const mergePoetryDependencies = (
     value.version = dependency.version ?? value.version;
     value.extras = dependency.extras ?? value.extras;
 
+    // Merge constraints if they exist
+    if (dependency.constraints) {
+      value = { ...value, ...dependency.constraints };
+    }
+
     if (value.version === undefined) {
       throw new Error(
         `Dependency "${dependency.name}" is missing attribute "version"!`,
       );
     }
 
-    // Serialize separately only if extras are provided
-    if (value.extras && value.extras.length > 0) {
+    // Serialize as object if there are any additional properties
+    if (Object.keys(value).length > 1) {
       existingDependencies[dependency.name] = value;
     } else {
       // Otherwise, serialize just the version string
@@ -512,7 +524,10 @@ export const installPythonTemplate = async ({
     if (observability === "llamatrace") {
       addOnDependencies.push({
         name: "llama-index-callbacks-arize-phoenix",
-        version: "^0.2.1",
+        version: "^0.3.0",
+        constraints: {
+          python: ">=3.11,<3.13",
+        },
       });
     }
 
@@ -533,9 +548,4 @@ export const installPythonTemplate = async ({
   if (postInstallAction === "runApp" || postInstallAction === "dependencies") {
     installPythonDependencies();
   }
-
-  // Copy deployment files for python
-  await copy("**", root, {
-    cwd: path.join(compPath, "deployments", "python"),
-  });
 };

helpers/run-app.ts

@@ -1,40 +1,39 @@
-import { ChildProcess, SpawnOptions, spawn } from "child_process";
-import path from "path";
+import { SpawnOptions, spawn } from "child_process";
 import { TemplateFramework } from "./types";
 
 const createProcess = (
   command: string,
   args: string[],
   options: SpawnOptions,
-) => {
-  return spawn(command, args, {
-    ...options,
-    shell: true,
-  })
-    .on("exit", function (code) {
-      if (code !== 0) {
-        console.log(`Child process exited with code=${code}`);
-        process.exit(1);
-      }
+): Promise<void> => {
+  return new Promise((resolve, reject) => {
+    spawn(command, args, {
+      ...options,
+      shell: true,
     })
-    .on("error", function (err) {
-      console.log("Error when running chill process: ", err);
-      process.exit(1);
-    });
+      .on("exit", function (code) {
+        if (code !== 0) {
+          console.log(`Child process exited with code=${code}`);
+          reject(code);
+        } else {
+          resolve();
+        }
+      })
+      .on("error", function (err) {
+        console.log("Error when running child process: ", err);
+        reject(err);
+      });
+  });
 };
 
-export function runReflexApp(
-  appPath: string,
-  frontendPort?: number,
-  backendPort?: number,
-) {
-  const commandArgs = ["run", "reflex", "run"];
-  if (frontendPort) {
-    commandArgs.push("--frontend-port", frontendPort.toString());
-  }
-  if (backendPort) {
-    commandArgs.push("--backend-port", backendPort.toString());
-  }
+export function runReflexApp(appPath: string, port: number) {
+  const commandArgs = [
+    "run",
+    "reflex",
+    "run",
+    "--frontend-port",
+    port.toString(),
+  ];
   return createProcess("poetry", commandArgs, {
     stdio: "inherit",
     cwd: appPath,
@@ -42,11 +41,10 @@ export function runReflexApp(
 }
 
 export function runFastAPIApp(appPath: string, port: number) {
-  const commandArgs = ["run", "uvicorn", "main:app", "--port=" + port];
-
-  return createProcess("poetry", commandArgs, {
+  return createProcess("poetry", ["run", "dev"], {
     stdio: "inherit",
     cwd: appPath,
+    env: { ...process.env, APP_PORT: `${port}` },
   });
 }
 
@@ -61,39 +59,23 @@ export function runTSApp(appPath: string, port: number) {
 export async function runApp(
   appPath: string,
   template: string,
-  frontend: boolean,
   framework: TemplateFramework,
   port?: number,
-  externalPort?: number,
-): Promise<any> {
-  const processes: ChildProcess[] = [];
+): Promise<void> {
+  try {
+    // Start the app
+    const defaultPort =
+      framework === "nextjs" || template === "extractor" ? 3000 : 8000;
 
-  // Callback to kill all sub processes if the main process is killed
-  process.on("exit", () => {
-    console.log("Killing app processes...");
-    processes.forEach((p) => p.kill());
-  });
-
-  // Default sub app paths
-  const backendPath = path.join(appPath, "backend");
-  const frontendPath = path.join(appPath, "frontend");
-
-  if (template === "extractor") {
-    processes.push(runReflexApp(appPath, port, externalPort));
+    const appRunner =
+      template === "extractor"
+        ? runReflexApp
+        : framework === "fastapi"
+          ? runFastAPIApp
+          : runTSApp;
+    await appRunner(appPath, port || defaultPort);
+  } catch (error) {
+    console.error("Failed to run app:", error);
+    throw error;
   }
-  if (template === "streaming" || template === "multiagent") {
-    if (framework === "fastapi" || framework === "express") {
-      const backendRunner = framework === "fastapi" ? runFastAPIApp : runTSApp;
-      if (frontend) {
-        processes.push(backendRunner(backendPath, externalPort || 8000));
-        processes.push(runTSApp(frontendPath, port || 3000));
-      } else {
-        processes.push(backendRunner(appPath, externalPort || 8000));
-      }
-    } else if (framework === "nextjs") {
-      processes.push(runTSApp(appPath, port || 3000));
-    }
-  }
-
-  return Promise.all(processes);
 }

helpers/tools.ts

@@ -41,7 +41,7 @@ export const supportedTools: Tool[] = [
     dependencies: [
       {
         name: "llama-index-tools-google",
-        version: "^0.2.0",
+        version: "^0.3.0",
       },
     ],
     supportedFrameworks: ["fastapi"],
@@ -62,17 +62,16 @@ export const supportedTools: Tool[] = [
     dependencies: [
       {
         name: "duckduckgo-search",
-        version: "6.1.7",
+        version: "^6.3.5",
       },
     ],
-    supportedFrameworks: ["fastapi", "nextjs", "express"],
+    supportedFrameworks: ["fastapi"], // TODO: Re-enable this tool once the duck-duck-scrape TypeScript library works again
     type: ToolType.LOCAL,
     envVars: [
       {
         name: TOOL_SYSTEM_PROMPT_ENV_VAR,
         description: "System prompt for DuckDuckGo search tool.",
-        value: `You are a DuckDuckGo search agent. 
-You can use the duckduckgo search tool to get information from the web to answer user questions.
+        value: `You have access to the duckduckgo search tool. Use it to get information from the web to answer user questions.
 For better results, you can specify the region parameter to get results from a specific region but it's optional.`,
       },
     ],
@@ -83,18 +82,11 @@ For better results, you can specify the region parameter to get results from a s
     dependencies: [
       {
         name: "llama-index-tools-wikipedia",
-        version: "^0.2.0",
+        version: "^0.3.0",
       },
     ],
     supportedFrameworks: ["fastapi", "express", "nextjs"],
     type: ToolType.LLAMAHUB,
-    envVars: [
-      {
-        name: TOOL_SYSTEM_PROMPT_ENV_VAR,
-        description: "System prompt for wiki tool.",
-        value: `You are a Wikipedia agent. You help users to get information from Wikipedia.`,
-      },
-    ],
   },
   {
     display: "Weather",
@@ -102,13 +94,6 @@ For better results, you can specify the region parameter to get results from a s
     dependencies: [],
     supportedFrameworks: ["fastapi", "express", "nextjs"],
     type: ToolType.LOCAL,
-    envVars: [
-      {
-        name: TOOL_SYSTEM_PROMPT_ENV_VAR,
-        description: "System prompt for weather tool.",
-        value: `You are a weather forecast agent. You help users to get the weather forecast for a given location.`,
-      },
-    ],
   },
   {
     display: "Document generator",
@@ -211,14 +196,6 @@ For better results, you can specify the region parameter to get results from a s
     },
     supportedFrameworks: ["fastapi", "express", "nextjs"],
     type: ToolType.LOCAL,
-    envVars: [
-      {
-        name: TOOL_SYSTEM_PROMPT_ENV_VAR,
-        description: "System prompt for openapi action tool.",
-        value:
-          "You are an OpenAPI action agent. You help users to make requests to the provided OpenAPI schema.",
-      },
-    ],
   },
   {
     display: "Image Generator",
@@ -231,11 +208,6 @@ For better results, you can specify the region parameter to get results from a s
         description:
           "STABILITY_API_KEY key is required to run image generator. Get it here: https://platform.stability.ai/account/keys",
       },
-      {
-        name: TOOL_SYSTEM_PROMPT_ENV_VAR,
-        description: "System prompt for image generator tool.",
-        value: `You are an image generator agent. You help users to generate images using the Stability API.`,
-      },
     ],
   },
   {

helpers/types.ts

@@ -89,14 +89,13 @@ export interface InstallTemplateArgs {
   framework: TemplateFramework;
   ui: TemplateUI;
   dataSources: TemplateDataSource[];
-  customApiPath?: string;
   modelConfig: ModelConfig;
   llamaCloudKey?: string;
   useLlamaParse?: boolean;
   communityProjectConfig?: CommunityProjectConfig;
   llamapack?: string;
   vectorDb?: TemplateVectorDB;
-  externalPort?: number;
+  port?: number;
   postInstallAction?: TemplatePostInstallAction;
   tools?: Tool[];
   observability?: TemplateObservability;

helpers/typescript.ts

@@ -241,14 +241,12 @@ export const installTSTemplate = async ({
     vectorDb,
   });
 
-  if (postInstallAction === "runApp" || postInstallAction === "dependencies") {
+  if (
+    backend &&
+    (postInstallAction === "runApp" || postInstallAction === "dependencies")
+  ) {
     await installTSDependencies(packageJson, packageManager, isOnline);
   }
-
-  // Copy deployment files for typescript
-  await copy("**", root, {
-    cwd: path.join(compPath, "deployments", "typescript"),
-  });
 };
 
 async function updatePackageJson({

helpers/vscode.ts

@@ -1,40 +1,26 @@
 import fs from "fs";
 import path from "path";
+import { assetRelocator, copy } from "./copy";
 import { TemplateFramework } from "./types";
 
 function renderDevcontainerContent(
   templatesDir: string,
   framework: TemplateFramework,
-  frontend: boolean,
 ) {
   const devcontainerJson: any = JSON.parse(
     fs.readFileSync(path.join(templatesDir, "devcontainer.json"), "utf8"),
   );
 
   // Modify postCreateCommand
-  if (frontend) {
-    devcontainerJson.postCreateCommand =
-      framework === "fastapi"
-        ? "cd backend && poetry install && cd ../frontend && npm install"
-        : "cd backend && npm install && cd ../frontend && npm install";
-  } else {
-    devcontainerJson.postCreateCommand =
-      framework === "fastapi" ? "poetry install" : "npm install";
-  }
+  devcontainerJson.postCreateCommand =
+    framework === "fastapi" ? "poetry install" : "npm install";
 
   // Modify containerEnv
   if (framework === "fastapi") {
-    if (frontend) {
-      devcontainerJson.containerEnv = {
-        ...devcontainerJson.containerEnv,
-        PYTHONPATH: "${PYTHONPATH}:${workspaceFolder}/backend",
-      };
-    } else {
-      devcontainerJson.containerEnv = {
-        ...devcontainerJson.containerEnv,
-        PYTHONPATH: "${PYTHONPATH}:${workspaceFolder}",
-      };
-    }
+    devcontainerJson.containerEnv = {
+      ...devcontainerJson.containerEnv,
+      PYTHONPATH: "${PYTHONPATH}:${workspaceFolder}",
+    };
   }
 
   return JSON.stringify(devcontainerJson, null, 2);
@@ -44,7 +30,6 @@ export const writeDevcontainer = async (
   root: string,
   templatesDir: string,
   framework: TemplateFramework,
-  frontend: boolean,
 ) => {
   const devcontainerDir = path.join(root, ".devcontainer");
   if (fs.existsSync(devcontainerDir)) {
@@ -54,7 +39,6 @@ export const writeDevcontainer = async (
   const devcontainerContent = renderDevcontainerContent(
     templatesDir,
     framework,
-    frontend,
   );
   fs.mkdirSync(devcontainerDir);
   await fs.promises.writeFile(
@@ -62,3 +46,25 @@ export const writeDevcontainer = async (
     devcontainerContent,
   );
 };
+
+export const copyVSCodeSettings = async (
+  root: string,
+  templatesDir: string,
+) => {
+  const vscodeDir = path.join(root, ".vscode");
+  await copy("vscode_settings.json", vscodeDir, {
+    cwd: templatesDir,
+    rename: assetRelocator,
+  });
+};
+
+export const configVSCode = async (
+  root: string,
+  templatesDir: string,
+  framework: TemplateFramework,
+) => {
+  await writeDevcontainer(root, templatesDir, framework);
+  if (framework === "fastapi") {
+    await copyVSCodeSettings(root, templatesDir);
+  }
+};

index.ts

@@ -134,13 +134,6 @@ const program = new Command(packageJson.name)
     `
 
   Select UI port.
-`,
-  )
-  .option(
-    "--external-port <external>",
-    `
-
-  Select external port.
 `,
   )
   .option(
@@ -333,7 +326,6 @@ async function run(): Promise<void> {
     ...answers,
     appPath: resolvedProjectPath,
     packageManager,
-    externalPort: options.externalPort,
   });
 
   if (answers.postInstallAction === "VSCode") {
@@ -362,14 +354,7 @@ Please check ${cyan(
     }
   } else if (answers.postInstallAction === "runApp") {
     console.log(`Running app in ${root}...`);
-    await runApp(
-      root,
-      answers.template,
-      answers.frontend,
-      answers.framework,
-      options.port,
-      options.externalPort,
-    );
+    await runApp(root, answers.template, answers.framework, options.port);
   }
 }
 

package.json

@@ -1,6 +1,6 @@
 {
   "name": "create-llama",
-  "version": "0.3.14",
+  "version": "0.3.21",
   "description": "Create LlamaIndex-powered apps with one command",
   "keywords": [
     "rag",

questions/questions.ts

@@ -1,4 +1,4 @@
-import { blue, green } from "picocolors";
+import { blue } from "picocolors";
 import prompts from "prompts";
 import { isCI } from ".";
 import { COMMUNITY_OWNER, COMMUNITY_REPO } from "../helpers/constant";
@@ -123,24 +123,17 @@ export const askProQuestions = async (program: QuestionArgs) => {
   }
 
   if (
-    (program.framework === "express" || program.framework === "fastapi") &&
+    program.framework === "fastapi" &&
     (program.template === "streaming" || program.template === "multiagent")
   ) {
     // if a backend-only framework is selected, ask whether we should create a frontend
     if (program.frontend === undefined) {
       const styledNextJS = blue("NextJS");
-      const styledBackend = green(
-        program.framework === "express"
-          ? "Express "
-          : program.framework === "fastapi"
-            ? "FastAPI (Python) "
-            : "",
-      );
       const { frontend } = await prompts({
         onState: onPromptState,
         type: "toggle",
         name: "frontend",
-        message: `Would you like to generate a ${styledNextJS} frontend for your ${styledBackend}backend?`,
+        message: `Would you like to generate a ${styledNextJS} frontend for your FastAPI backend?`,
         initial: false,
         active: "Yes",
         inactive: "No",

questions/simple.ts

@@ -131,7 +131,7 @@ const convertAnswers = async (
   > = {
     rag: {
       template: "streaming",
-      tools: getTools(["duckduckgo"]),
+      tools: getTools(["weather"]),
       frontend: true,
       dataSources: [EXAMPLE_FILE],
     },

questions/types.ts

@@ -2,7 +2,7 @@ import { InstallAppArgs } from "../create-app";
 
 export type QuestionResults = Omit<
   InstallAppArgs,
-  "appPath" | "packageManager" | "externalPort"
+  "appPath" | "packageManager"
 >;
 
 export type PureQuestionArgs = {

templates/.gitignore

@@ -1,3 +0,0 @@
-__pycache__
-poetry.lock
-storage

templates/README-fullstack.md

@@ -1,18 +0,0 @@
-This is a [LlamaIndex](https://www.llamaindex.ai/) project bootstrapped with [`create-llama`](https://github.com/run-llama/LlamaIndexTS/tree/main/packages/create-llama).
-
-## Getting Started
-
-First, startup the backend as described in the [backend README](./backend/README.md).
-
-Second, run the development server of the frontend as described in the [frontend README](./frontend/README.md).
-
-Open [http://localhost:3000](http://localhost:3000) with your browser to see the result.
-
-## Learn More
-
-To learn more about LlamaIndex, take a look at the following resources:
-
-- [LlamaIndex Documentation](https://docs.llamaindex.ai) - learn about LlamaIndex (Python features).
-- [LlamaIndexTS Documentation](https://ts.llamaindex.ai) - learn about LlamaIndex (Typescript features).
-
-You can check out [the LlamaIndexTS GitHub repository](https://github.com/run-llama/LlamaIndexTS) - your feedback and contributions are welcome!

templates/components/agents/python/blog/README-template.md

@@ -32,7 +32,7 @@ poetry run generate
 Third, run the development server:
 
 ```shell
-poetry run python main.py
+poetry run dev
 ```
 
 Per default, the example is using the explicit workflow. You can change the example by setting the `EXAMPLE_TYPE` environment variable to `choreography` or `orchestrator`.
@@ -47,14 +47,18 @@ curl --location 'localhost:8000/api/chat' \
 
 You can start editing the API by modifying `app/api/routers/chat.py` or `app/examples/workflow.py`. The API auto-updates as you save the files.
 
-Open [http://localhost:8000/docs](http://localhost:8000/docs) with your browser to see the Swagger UI of the API.
+Open [http://localhost:8000](http://localhost:8000) with your browser to start the app.
 
-The API allows CORS for all origins to simplify development. You can change this behavior by setting the `ENVIRONMENT` environment variable to `prod`:
+To start the app optimized for **production**, run:
 
 ```
-ENVIRONMENT=prod poetry run python main.py
+poetry run prod
 ```
 
+## Deployments
+
+For production deployments, check the [DEPLOY.md](DEPLOY.md) file.
+
 ## Learn More
 
 To learn more about LlamaIndex, take a look at the following resources:

templates/components/agents/python/blog/app/agents/researcher.py

@@ -1,4 +1,3 @@
-import os
 from textwrap import dedent
 from typing import List
 
@@ -6,42 +5,24 @@ from app.engine.index import IndexConfig, get_index
 from app.engine.tools import ToolFactory
 from app.workflows.single import FunctionCallingAgent
 from llama_index.core.chat_engine.types import ChatMessage
-from llama_index.core.tools import QueryEngineTool, ToolMetadata
+from app.engine.tools.query_engine import get_query_engine_tool
 
 
-def _create_query_engine_tool(params=None) -> QueryEngineTool:
-    """
-    Provide an agent worker that can be used to query the index.
-    """
-    # Add query tool if index exists
-    index_config = IndexConfig(**(params or {}))
-    index = get_index(index_config)
-    if index is None:
-        return None
-    top_k = int(os.getenv("TOP_K", 0))
-    query_engine = index.as_query_engine(
-        **({"similarity_top_k": top_k} if top_k != 0 else {})
-    )
-    return QueryEngineTool(
-        query_engine=query_engine,
-        metadata=ToolMetadata(
-            name="query_index",
-            description="""
-                Use this tool to retrieve information about the text corpus from the index.
-            """,
-        ),
-    )
-
-
-def _get_research_tools(**kwargs) -> QueryEngineTool:
+def _get_research_tools(**kwargs):
     """
     Researcher take responsibility for retrieving information.
     Try init wikipedia or duckduckgo tool if available.
     """
     tools = []
-    query_engine_tool = _create_query_engine_tool(**kwargs)
-    if query_engine_tool is not None:
-        tools.append(query_engine_tool)
+    # Create query engine tool
+    index_config = IndexConfig(**kwargs)
+    index = get_index(index_config)
+    if index is not None:
+        query_engine_tool = get_query_engine_tool(index=index)
+        if query_engine_tool is not None:
+            tools.append(query_engine_tool)
+
+    # Create duckduckgo tool
     researcher_tool_names = [
         "duckduckgo_search",
         "duckduckgo_image_search",

templates/components/agents/python/financial_report/README-template.md

@@ -21,7 +21,7 @@ poetry run generate
 Third, run the development server:
 
 ```shell
-poetry run python main.py
+poetry run dev
 ```
 
 The example provides one streaming API endpoint `/api/chat`.
@@ -35,14 +35,18 @@ curl --location 'localhost:8000/api/chat' \
 
 You can start editing the API by modifying `app/api/routers/chat.py` or `app/workflows/financial_report.py`. The API auto-updates as you save the files.
 
-Open [http://localhost:8000/docs](http://localhost:8000/docs) with your browser to see the Swagger UI of the API.
+Open [http://localhost:8000](http://localhost:8000) with your browser to start the app.
 
-The API allows CORS for all origins to simplify development. You can change this behavior by setting the `ENVIRONMENT` environment variable to `prod`:
+To start the app optimized for **production**, run:
 
 ```
-ENVIRONMENT=prod poetry run python main.py
+poetry run prod
 ```
 
+## Deployments
+
+For production deployments, check the [DEPLOY.md](DEPLOY.md) file.
+
 ## Learn More
 
 To learn more about LlamaIndex, take a look at the following resources:

templates/components/agents/python/financial_report/app/workflows/financial_report.py

@@ -1,8 +1,8 @@
-import os
 from typing import Any, Dict, List, Optional
 
 from app.engine.index import IndexConfig, get_index
 from app.engine.tools import ToolFactory
+from app.engine.tools.query_engine import get_query_engine_tool
 from app.workflows.events import AgentRunEvent
 from app.workflows.tools import (
     call_tools,
@@ -10,7 +10,6 @@ from app.workflows.tools import (
 )
 from llama_index.core import Settings
 from llama_index.core.base.llms.types import ChatMessage, MessageRole
-from llama_index.core.indices.vector_store import VectorStoreIndex
 from llama_index.core.llms.function_calling import FunctionCallingLLM
 from llama_index.core.memory import ChatMemoryBuffer
 from llama_index.core.tools import FunctionTool, QueryEngineTool, ToolSelection
@@ -27,16 +26,16 @@ from llama_index.core.workflow import (
 def create_workflow(
     chat_history: Optional[List[ChatMessage]] = None,
     params: Optional[Dict[str, Any]] = None,
-    filters: Optional[List[Any]] = None,
+    **kwargs,
 ) -> Workflow:
+    # Create query engine tool
     index_config = IndexConfig(**params)
-    index: VectorStoreIndex = get_index(config=index_config)
+    index = get_index(index_config)
     if index is None:
-        query_engine_tool = None
-    else:
-        top_k = int(os.getenv("TOP_K", 10))
-        query_engine = index.as_query_engine(similarity_top_k=top_k, filters=filters)
-        query_engine_tool = QueryEngineTool.from_defaults(query_engine=query_engine)
+        raise ValueError(
+            "Index is not found. Try run generation script to create the index first."
+        )
+    query_engine_tool = get_query_engine_tool(index=index)
 
     configured_tools: Dict[str, FunctionTool] = ToolFactory.from_env(map_result=True)  # type: ignore
     code_interpreter_tool = configured_tools.get("interpret")

templates/components/agents/python/form_filling/README-template.md

@@ -16,7 +16,7 @@ Make sure you have the `OPENAI_API_KEY` set.
 Second, run the development server:
 
 ```shell
-poetry run python main.py
+poetry run dev
 ```
 
 ## Use Case: Filling Financial CSV Template
@@ -41,14 +41,18 @@ curl --location 'localhost:8000/api/chat' \
 
 You can start editing the API by modifying `app/api/routers/chat.py` or `app/workflows/form_filling.py`. The API auto-updates as you save the files.
 
-Open [http://localhost:8000/docs](http://localhost:8000/docs) with your browser to see the Swagger UI of the API.
+Open [http://localhost:8000](http://localhost:8000) with your browser to start the app.
 
-The API allows CORS for all origins to simplify development. You can change this behavior by setting the `ENVIRONMENT` environment variable to `prod`:
+To start the app optimized for **production**, run:
 
 ```
-ENVIRONMENT=prod poetry run python main.py
+poetry run prod
 ```
 
+## Deployments
+
+For production deployments, check the [DEPLOY.md](DEPLOY.md) file.
+
 ## Learn More
 
 To learn more about LlamaIndex, take a look at the following resources:

templates/components/agents/python/form_filling/app/workflows/form_filling.py

@@ -1,16 +1,7 @@
-import os
 from typing import Any, Dict, List, Optional
 
-from app.engine.index import IndexConfig, get_index
-from app.engine.tools import ToolFactory
-from app.workflows.events import AgentRunEvent
-from app.workflows.tools import (
-    call_tools,
-    chat_with_tools,
-)
 from llama_index.core import Settings
 from llama_index.core.base.llms.types import ChatMessage, MessageRole
-from llama_index.core.indices.vector_store import VectorStoreIndex
 from llama_index.core.llms.function_calling import FunctionCallingLLM
 from llama_index.core.memory import ChatMemoryBuffer
 from llama_index.core.tools import FunctionTool, QueryEngineTool, ToolSelection
@@ -23,24 +14,28 @@ from llama_index.core.workflow import (
     step,
 )
 
+from app.engine.index import IndexConfig, get_index
+from app.engine.tools import ToolFactory
+from app.engine.tools.query_engine import get_query_engine_tool
+from app.workflows.events import AgentRunEvent
+from app.workflows.tools import (
+    call_tools,
+    chat_with_tools,
+)
+
 
 def create_workflow(
     chat_history: Optional[List[ChatMessage]] = None,
     params: Optional[Dict[str, Any]] = None,
-    filters: Optional[List[Any]] = None,
+    **kwargs,
 ) -> Workflow:
-    if params is None:
-        params = {}
-    if filters is None:
-        filters = []
+    # Create query engine tool
     index_config = IndexConfig(**params)
-    index: VectorStoreIndex = get_index(config=index_config)
+    index = get_index(index_config)
     if index is None:
         query_engine_tool = None
     else:
-        top_k = int(os.getenv("TOP_K", 10))
-        query_engine = index.as_query_engine(similarity_top_k=top_k, filters=filters)
-        query_engine_tool = QueryEngineTool.from_defaults(query_engine=query_engine)
+        query_engine_tool = get_query_engine_tool(index=index)
 
     configured_tools = ToolFactory.from_env(map_result=True)
     extractor_tool = configured_tools.get("extract_questions")  # type: ignore

templates/components/agents/typescript/blog/workflow/agents.ts

@@ -1,15 +1,15 @@
 import { ChatMessage } from "llamaindex";
 import { getTool } from "../engine/tools";
 import { FunctionCallingAgent } from "./single-agent";
-import { getQueryEngineTools } from "./tools";
+import { getQueryEngineTool } from "./tools";
 
 export const createResearcher = async (chatHistory: ChatMessage[]) => {
-  const queryEngineTools = await getQueryEngineTools();
+  const queryEngineTool = await getQueryEngineTool();
   const tools = [
     await getTool("wikipedia_tool"),
     await getTool("duckduckgo_search"),
     await getTool("image_generator"),
-    ...(queryEngineTools ? queryEngineTools : []),
+    queryEngineTool,
   ].filter((tool) => tool !== undefined);
 
   return new FunctionCallingAgent({

templates/components/agents/typescript/financial_report/workflow/factory.ts

@@ -1,7 +1,7 @@
 import { ChatMessage, ToolCallLLM } from "llamaindex";
 import { getTool } from "../engine/tools";
 import { FinancialReportWorkflow } from "./fin-report";
-import { getQueryEngineTools } from "./tools";
+import { getQueryEngineTool } from "./tools";
 
 const TIMEOUT = 360 * 1000;
 
@@ -9,11 +9,19 @@ export async function createWorkflow(options: {
   chatHistory: ChatMessage[];
   llm?: ToolCallLLM;
 }) {
+  const queryEngineTool = await getQueryEngineTool();
+  const codeInterpreterTool = await getTool("interpreter");
+  const documentGeneratorTool = await getTool("document_generator");
+
+  if (!queryEngineTool || !codeInterpreterTool || !documentGeneratorTool) {
+    throw new Error("One or more required tools are not defined");
+  }
+
   return new FinancialReportWorkflow({
     chatHistory: options.chatHistory,
-    queryEngineTools: (await getQueryEngineTools()) || [],
-    codeInterpreterTool: (await getTool("interpreter"))!,
-    documentGeneratorTool: (await getTool("document_generator"))!,
+    queryEngineTool,
+    codeInterpreterTool,
+    documentGeneratorTool,
     llm: options.llm,
     timeout: TIMEOUT,
   });

templates/components/agents/typescript/financial_report/workflow/fin-report.ts

@@ -45,7 +45,7 @@ export class FinancialReportWorkflow extends Workflow<
 > {
   llm: ToolCallLLM;
   memory: ChatMemoryBuffer;
-  queryEngineTools: BaseToolWithCall[];
+  queryEngineTool: BaseToolWithCall;
   codeInterpreterTool: BaseToolWithCall;
   documentGeneratorTool: BaseToolWithCall;
   systemPrompt?: string;
@@ -53,7 +53,7 @@ export class FinancialReportWorkflow extends Workflow<
   constructor(options: {
     llm?: ToolCallLLM;
     chatHistory: ChatMessage[];
-    queryEngineTools: BaseToolWithCall[];
+    queryEngineTool: BaseToolWithCall;
     codeInterpreterTool: BaseToolWithCall;
     documentGeneratorTool: BaseToolWithCall;
     systemPrompt?: string;
@@ -70,7 +70,7 @@ export class FinancialReportWorkflow extends Workflow<
       throw new Error("LLM is not a ToolCallLLM");
     }
     this.systemPrompt = options.systemPrompt ?? DEFAULT_SYSTEM_PROMPT;
-    this.queryEngineTools = options.queryEngineTools;
+    this.queryEngineTool = options.queryEngineTool;
     this.codeInterpreterTool = options.codeInterpreterTool;
 
     this.documentGeneratorTool = options.documentGeneratorTool;
@@ -153,10 +153,11 @@ export class FinancialReportWorkflow extends Workflow<
   > => {
     const chatHistory = ev.data.input;
 
-    const tools = [this.codeInterpreterTool, this.documentGeneratorTool];
-    if (this.queryEngineTools) {
-      tools.push(...this.queryEngineTools);
-    }
+    const tools = [
+      this.codeInterpreterTool,
+      this.documentGeneratorTool,
+      this.queryEngineTool,
+    ];
 
     const toolCallResponse = await chatWithTools(this.llm, tools, chatHistory);
 
@@ -189,10 +190,7 @@ export class FinancialReportWorkflow extends Workflow<
           toolCalls: toolCallResponse.toolCalls,
         });
       default:
-        if (
-          this.queryEngineTools &&
-          this.queryEngineTools.some((tool) => tool.metadata.name === toolName)
-        ) {
+        if (this.queryEngineTool.metadata.name === toolName) {
           return new ResearchEvent({
             toolCalls: toolCallResponse.toolCalls,
           });
@@ -216,7 +214,7 @@ export class FinancialReportWorkflow extends Workflow<
     const { toolCalls } = ev.data;
 
     const toolMsgs = await callTools({
-      tools: this.queryEngineTools,
+      tools: [this.queryEngineTool],
       toolCalls,
       ctx,
       agentName: "Researcher",

templates/components/agents/typescript/form_filling/workflow/factory.ts

@@ -1,7 +1,7 @@
 import { ChatMessage, ToolCallLLM } from "llamaindex";
 import { getTool } from "../engine/tools";
 import { FormFillingWorkflow } from "./form-filling";
-import { getQueryEngineTools } from "./tools";
+import { getQueryEngineTool } from "./tools";
 
 const TIMEOUT = 360 * 1000;
 
@@ -9,11 +9,18 @@ export async function createWorkflow(options: {
   chatHistory: ChatMessage[];
   llm?: ToolCallLLM;
 }) {
+  const extractorTool = await getTool("extract_missing_cells");
+  const fillMissingCellsTool = await getTool("fill_missing_cells");
+
+  if (!extractorTool || !fillMissingCellsTool) {
+    throw new Error("One or more required tools are not defined");
+  }
+
   return new FormFillingWorkflow({
     chatHistory: options.chatHistory,
-    queryEngineTools: (await getQueryEngineTools()) || [],
-    extractorTool: (await getTool("extract_missing_cells"))!,
-    fillMissingCellsTool: (await getTool("fill_missing_cells"))!,
+    queryEngineTool: (await getQueryEngineTool()) || undefined,
+    extractorTool,
+    fillMissingCellsTool,
     llm: options.llm,
     timeout: TIMEOUT,
   });

templates/components/agents/typescript/form_filling/workflow/form-filling.ts

@@ -48,7 +48,7 @@ export class FormFillingWorkflow extends Workflow<
   llm: ToolCallLLM;
   memory: ChatMemoryBuffer;
   extractorTool: BaseToolWithCall;
-  queryEngineTools?: BaseToolWithCall[];
+  queryEngineTool?: BaseToolWithCall;
   fillMissingCellsTool: BaseToolWithCall;
   systemPrompt?: string;
 
@@ -56,7 +56,7 @@ export class FormFillingWorkflow extends Workflow<
     llm?: ToolCallLLM;
     chatHistory: ChatMessage[];
     extractorTool: BaseToolWithCall;
-    queryEngineTools?: BaseToolWithCall[];
+    queryEngineTool?: BaseToolWithCall;
     fillMissingCellsTool: BaseToolWithCall;
     systemPrompt?: string;
     verbose?: boolean;
@@ -73,7 +73,7 @@ export class FormFillingWorkflow extends Workflow<
     }
     this.systemPrompt = options.systemPrompt ?? DEFAULT_SYSTEM_PROMPT;
     this.extractorTool = options.extractorTool;
-    this.queryEngineTools = options.queryEngineTools;
+    this.queryEngineTool = options.queryEngineTool;
     this.fillMissingCellsTool = options.fillMissingCellsTool;
 
     this.memory = new ChatMemoryBuffer({
@@ -156,8 +156,8 @@ export class FormFillingWorkflow extends Workflow<
     const chatHistory = ev.data.input;
 
     const tools = [this.extractorTool, this.fillMissingCellsTool];
-    if (this.queryEngineTools) {
-      tools.push(...this.queryEngineTools);
+    if (this.queryEngineTool) {
+      tools.push(this.queryEngineTool);
     }
 
     const toolCallResponse = await chatWithTools(this.llm, tools, chatHistory);
@@ -192,8 +192,8 @@ export class FormFillingWorkflow extends Workflow<
         });
       default:
         if (
-          this.queryEngineTools &&
-          this.queryEngineTools.some((tool) => tool.metadata.name === toolName)
+          this.queryEngineTool &&
+          this.queryEngineTool.metadata.name === toolName
         ) {
           return new FindAnswersEvent({
             toolCalls: toolCallResponse.toolCalls,
@@ -232,7 +232,7 @@ export class FormFillingWorkflow extends Workflow<
     ev: FindAnswersEvent,
   ): Promise<InputEvent> => {
     const { toolCalls } = ev.data;
-    if (!this.queryEngineTools) {
+    if (!this.queryEngineTool) {
       throw new Error("Query engine tool is not available");
     }
     ctx.sendEvent(
@@ -243,7 +243,7 @@ export class FormFillingWorkflow extends Workflow<
       }),
     );
     const toolMsgs = await callTools({
-      tools: this.queryEngineTools,
+      tools: [this.queryEngineTool],
       toolCalls,
       ctx,
       agentName: "Researcher",

templates/components/engines/python/agent/engine.py

@@ -1,18 +1,18 @@
 import os
 from typing import List
 
-from app.engine.index import IndexConfig, get_index
-from app.engine.tools import ToolFactory
 from llama_index.core.agent import AgentRunner
 from llama_index.core.callbacks import CallbackManager
 from llama_index.core.settings import Settings
 from llama_index.core.tools import BaseTool
-from llama_index.core.tools.query_engine import QueryEngineTool
+
+from app.engine.index import IndexConfig, get_index
+from app.engine.tools import ToolFactory
+from app.engine.tools.query_engine import get_query_engine_tool
 
 
-def get_chat_engine(filters=None, params=None, event_handlers=None, **kwargs):
+def get_chat_engine(params=None, event_handlers=None, **kwargs):
     system_prompt = os.getenv("SYSTEM_PROMPT")
-    top_k = int(os.getenv("TOP_K", 0))
     tools: List[BaseTool] = []
     callback_manager = CallbackManager(handlers=event_handlers or [])
 
@@ -20,10 +20,7 @@ def get_chat_engine(filters=None, params=None, event_handlers=None, **kwargs):
     index_config = IndexConfig(callback_manager=callback_manager, **(params or {}))
     index = get_index(index_config)
     if index is not None:
-        query_engine = index.as_query_engine(
-            filters=filters, **({"similarity_top_k": top_k} if top_k != 0 else {})
-        )
-        query_engine_tool = QueryEngineTool.from_defaults(query_engine=query_engine)
+        query_engine_tool = get_query_engine_tool(index, **kwargs)
         tools.append(query_engine_tool)
 
     # Add additional tools

templates/components/engines/python/agent/tools/query_engine.py

@@ -0,0 +1,187 @@
+import os
+from typing import Any, Dict, List, Optional, Sequence
+
+from llama_index.core import get_response_synthesizer
+from llama_index.core.base.base_query_engine import BaseQueryEngine
+from llama_index.core.base.response.schema import RESPONSE_TYPE, Response
+from llama_index.core.multi_modal_llms import MultiModalLLM
+from llama_index.core.prompts.base import BasePromptTemplate
+from llama_index.core.prompts.default_prompt_selectors import (
+    DEFAULT_TEXT_QA_PROMPT_SEL,
+)
+from llama_index.core.query_engine.multi_modal import _get_image_and_text_nodes
+from llama_index.core.response_synthesizers.base import BaseSynthesizer, QueryTextType
+from llama_index.core.schema import (
+    ImageNode,
+    NodeWithScore,
+)
+from llama_index.core.tools.query_engine import QueryEngineTool
+from llama_index.core.types import RESPONSE_TEXT_TYPE
+
+from app.settings import get_multi_modal_llm
+
+
+def create_query_engine(index, **kwargs) -> BaseQueryEngine:
+    """
+    Create a query engine for the given index.
+
+    Args:
+        index: The index to create a query engine for.
+        params (optional): Additional parameters for the query engine, e.g: similarity_top_k
+    """
+
+    top_k = int(os.getenv("TOP_K", 0))
+    if top_k != 0 and kwargs.get("filters") is None:
+        kwargs["similarity_top_k"] = top_k
+    multimodal_llm = get_multi_modal_llm()
+    if multimodal_llm:
+        kwargs["response_synthesizer"] = MultiModalSynthesizer(
+            multimodal_model=multimodal_llm,
+        )
+
+    # If index is index is LlamaCloudIndex
+    # use auto_routed mode for better query results
+    if index.__class__.__name__ == "LlamaCloudIndex":
+        if kwargs.get("retrieval_mode") is None:
+            kwargs["retrieval_mode"] = "auto_routed"
+        if multimodal_llm:
+            kwargs["retrieve_image_nodes"] = True
+    return index.as_query_engine(**kwargs)
+
+
+def get_query_engine_tool(
+    index,
+    name: Optional[str] = None,
+    description: Optional[str] = None,
+    **kwargs,
+) -> QueryEngineTool:
+    """
+    Get a query engine tool for the given index.
+
+    Args:
+        index: The index to create a query engine for.
+        name (optional): The name of the tool.
+        description (optional): The description of the tool.
+    """
+    if name is None:
+        name = "query_index"
+    if description is None:
+        description = (
+            "Use this tool to retrieve information about the text corpus from an index."
+        )
+    query_engine = create_query_engine(index, **kwargs)
+    return QueryEngineTool.from_defaults(
+        query_engine=query_engine,
+        name=name,
+        description=description,
+    )
+
+
+class MultiModalSynthesizer(BaseSynthesizer):
+    """
+    A synthesizer that summarizes text nodes and uses a multi-modal LLM to generate a response.
+    """
+
+    def __init__(
+        self,
+        multimodal_model: MultiModalLLM,
+        response_synthesizer: Optional[BaseSynthesizer] = None,
+        text_qa_template: Optional[BasePromptTemplate] = None,
+        *args,
+        **kwargs,
+    ):
+        super().__init__(*args, **kwargs)
+        self._multi_modal_llm = multimodal_model
+        self._response_synthesizer = response_synthesizer or get_response_synthesizer()
+        self._text_qa_template = text_qa_template or DEFAULT_TEXT_QA_PROMPT_SEL
+
+    def _get_prompts(self, **kwargs) -> Dict[str, Any]:
+        return {
+            "text_qa_template": self._text_qa_template,
+        }
+
+    def _update_prompts(self, prompts: Dict[str, Any]) -> None:
+        if "text_qa_template" in prompts:
+            self._text_qa_template = prompts["text_qa_template"]
+
+    async def aget_response(
+        self,
+        *args,
+        **response_kwargs: Any,
+    ) -> RESPONSE_TEXT_TYPE:
+        return await self._response_synthesizer.aget_response(*args, **response_kwargs)
+
+    def get_response(self, *args, **kwargs) -> RESPONSE_TEXT_TYPE:
+        return self._response_synthesizer.get_response(*args, **kwargs)
+
+    async def asynthesize(
+        self,
+        query: QueryTextType,
+        nodes: List[NodeWithScore],
+        additional_source_nodes: Optional[Sequence[NodeWithScore]] = None,
+        **response_kwargs: Any,
+    ) -> RESPONSE_TYPE:
+        image_nodes, text_nodes = _get_image_and_text_nodes(nodes)
+
+        if len(image_nodes) == 0:
+            return await self._response_synthesizer.asynthesize(query, text_nodes)
+
+        # Summarize the text nodes to avoid exceeding the token limit
+        text_response = str(
+            await self._response_synthesizer.asynthesize(query, text_nodes)
+        )
+
+        fmt_prompt = self._text_qa_template.format(
+            context_str=text_response,
+            query_str=query.query_str,  # type: ignore
+        )
+
+        llm_response = await self._multi_modal_llm.acomplete(
+            prompt=fmt_prompt,
+            image_documents=[
+                image_node.node
+                for image_node in image_nodes
+                if isinstance(image_node.node, ImageNode)
+            ],
+        )
+
+        return Response(
+            response=str(llm_response),
+            source_nodes=nodes,
+            metadata={"text_nodes": text_nodes, "image_nodes": image_nodes},
+        )
+
+    def synthesize(
+        self,
+        query: QueryTextType,
+        nodes: List[NodeWithScore],
+        additional_source_nodes: Optional[Sequence[NodeWithScore]] = None,
+        **response_kwargs: Any,
+    ) -> RESPONSE_TYPE:
+        image_nodes, text_nodes = _get_image_and_text_nodes(nodes)
+
+        if len(image_nodes) == 0:
+            return self._response_synthesizer.synthesize(query, text_nodes)
+
+        # Summarize the text nodes to avoid exceeding the token limit
+        text_response = str(self._response_synthesizer.synthesize(query, text_nodes))
+
+        fmt_prompt = self._text_qa_template.format(
+            context_str=text_response,
+            query_str=query.query_str,  # type: ignore
+        )
+
+        llm_response = self._multi_modal_llm.complete(
+            prompt=fmt_prompt,
+            image_documents=[
+                image_node.node
+                for image_node in image_nodes
+                if isinstance(image_node.node, ImageNode)
+            ],
+        )
+
+        return Response(
+            response=str(llm_response),
+            source_nodes=nodes,
+            metadata={"text_nodes": text_nodes, "image_nodes": image_nodes},
+        )

templates/components/engines/python/chat/engine.py

@@ -9,7 +9,7 @@ from llama_index.core.memory import ChatMemoryBuffer
 from llama_index.core.settings import Settings
 
 
-def get_chat_engine(filters=None, params=None, event_handlers=None, **kwargs):
+def get_chat_engine(params=None, event_handlers=None, **kwargs):
     system_prompt = os.getenv("SYSTEM_PROMPT")
     citation_prompt = os.getenv("SYSTEM_CITATION_PROMPT", None)
     top_k = int(os.getenv("TOP_K", 0))
@@ -33,10 +33,9 @@ def get_chat_engine(filters=None, params=None, event_handlers=None, **kwargs):
                 "StorageContext is empty - call 'poetry run generate' to generate the storage first"
             ),
         )
-
-    retriever = index.as_retriever(
-        filters=filters, **({"similarity_top_k": top_k} if top_k != 0 else {})
-    )
+    if top_k != 0 and kwargs.get("similarity_top_k") is None:
+        kwargs["similarity_top_k"] = top_k
+    retriever = index.as_retriever(**kwargs)
 
     return CondensePlusContextChatEngine(
         llm=llm,

templates/components/engines/typescript/agent/chat.ts

@@ -1,14 +1,9 @@
-import {
-  BaseChatEngine,
-  BaseToolWithCall,
-  LLMAgent,
-  QueryEngineTool,
-} from "llamaindex";
+import { BaseChatEngine, BaseToolWithCall, LLMAgent } from "llamaindex";
 import fs from "node:fs/promises";
 import path from "node:path";
 import { getDataSource } from "./index";
-import { generateFilters } from "./queryFilter";
 import { createTools } from "./tools";
+import { createQueryEngineTool } from "./tools/query-engine";
 
 export async function createChatEngine(documentIds?: string[], params?: any) {
   const tools: BaseToolWithCall[] = [];
@@ -17,17 +12,7 @@ export async function createChatEngine(documentIds?: string[], params?: any) {
   // Delete this code if you don't have a data source
   const index = await getDataSource(params);
   if (index) {
-    tools.push(
-      new QueryEngineTool({
-        queryEngine: index.asQueryEngine({
-          preFilters: generateFilters(documentIds || []),
-        }),
-        metadata: {
-          name: "data_query_engine",
-          description: `A query engine for documents from your data source.`,
-        },
-      }),
-    );
+    tools.push(createQueryEngineTool(index, { documentIds }));
   }
 
   const configFile = path.join("config", "tools.json");

templates/components/engines/typescript/agent/tools/interpreter.ts

@@ -116,7 +116,9 @@ export class InterpreterTool implements BaseTool<InterpreterParameter> {
           const fileName = path.basename(filePath);
           const localFilePath = path.join(this.uploadedFilesDir, fileName);
           const content = fs.readFileSync(localFilePath);
-          await this.codeInterpreter?.files.write(filePath, content);
+
+          const arrayBuffer = new Uint8Array(content).buffer;
+          await this.codeInterpreter?.files.write(filePath, arrayBuffer);
         }
       } catch (error) {
         console.error("Got error when uploading files to sandbox", error);

templates/components/engines/typescript/agent/tools/query-engine.ts

@@ -0,0 +1,57 @@
+import {
+  BaseQueryEngine,
+  CloudRetrieveParams,
+  LlamaCloudIndex,
+  MetadataFilters,
+  QueryEngineTool,
+  VectorStoreIndex,
+} from "llamaindex";
+import { generateFilters } from "../queryFilter";
+
+interface QueryEngineParams {
+  documentIds?: string[];
+  topK?: number;
+}
+
+export function createQueryEngineTool(
+  index: VectorStoreIndex | LlamaCloudIndex,
+  params?: QueryEngineParams,
+  name?: string,
+  description?: string,
+): QueryEngineTool {
+  return new QueryEngineTool({
+    queryEngine: createQueryEngine(index, params),
+    metadata: {
+      name: name || "query_engine",
+      description:
+        description ||
+        `Use this tool to retrieve information about the text corpus from an index.`,
+    },
+  });
+}
+
+function createQueryEngine(
+  index: VectorStoreIndex | LlamaCloudIndex,
+  params?: QueryEngineParams,
+): BaseQueryEngine {
+  const baseQueryParams = {
+    similarityTopK:
+      params?.topK ??
+      (process.env.TOP_K ? parseInt(process.env.TOP_K) : undefined),
+  };
+
+  if (index instanceof LlamaCloudIndex) {
+    return index.asQueryEngine({
+      ...baseQueryParams,
+      retrieval_mode: "auto_routed",
+      preFilters: generateFilters(
+        params?.documentIds || [],
+      ) as CloudRetrieveParams["filters"],
+    });
+  }
+
+  return index.asQueryEngine({
+    ...baseQueryParams,
+    preFilters: generateFilters(params?.documentIds || []) as MetadataFilters,
+  });
+}

templates/components/multiagent/python/app/api/routers/chat.py

@@ -1,12 +1,13 @@
 import logging
 
+from fastapi import APIRouter, BackgroundTasks, HTTPException, Request, status
+
 from app.api.routers.models import (
     ChatData,
 )
 from app.api.routers.vercel_response import VercelStreamResponse
 from app.engine.query_filter import generate_filters
 from app.workflows import create_workflow
-from fastapi import APIRouter, BackgroundTasks, HTTPException, Request, status
 
 chat_router = r = APIRouter()
 
@@ -28,7 +29,9 @@ async def chat(
         params = data.data or {}
 
         workflow = create_workflow(
-            chat_history=messages, params=params, filters=filters
+            chat_history=messages,
+            params=params,
+            filters=filters,
         )
 
         event_handler = workflow.run(input=last_message_content, streaming=True)

templates/components/multiagent/python/app/api/routers/vercel_response.py

@@ -19,6 +19,7 @@ class VercelStreamResponse(StreamingResponse):
 
     TEXT_PREFIX = "0:"
     DATA_PREFIX = "8:"
+    ERROR_PREFIX = "3:"
 
     def __init__(self, request: Request, chat_data: ChatData, *args, **kwargs):
         self.request = request
@@ -41,13 +42,16 @@ class VercelStreamResponse(StreamingResponse):
 
                     yield output
         except asyncio.CancelledError:
-            logger.info("Stopping workflow")
-            await event_handler.cancel_run()
+            logger.warning("Workflow has been cancelled!")
         except Exception as e:
             logger.error(
                 f"Unexpected error in content_generator: {str(e)}", exc_info=True
             )
+            yield self.convert_error(
+                "An unexpected error occurred while processing your request, preventing the creation of a final answer. Please try again."
+            )
         finally:
+            await event_handler.cancel_run()
             logger.info("The stream has been stopped!")
 
     def _create_stream(
@@ -107,6 +111,11 @@ class VercelStreamResponse(StreamingResponse):
         data_str = json.dumps(data)
         return f"{cls.DATA_PREFIX}[{data_str}]\n"
 
+    @classmethod
+    def convert_error(cls, error: str):
+        error_str = json.dumps(error)
+        return f"{cls.ERROR_PREFIX}{error_str}\n"
+
     @staticmethod
     async def _generate_next_questions(chat_history: List[Message], response: str):
         questions = await NextQuestionSuggestion.suggest_next_questions(

templates/components/multiagent/typescript/express/chat.controller.ts

@@ -1,4 +1,4 @@
-import { Message, streamToResponse } from "ai";
+import { LlamaIndexAdapter, Message } from "ai";
 import { Request, Response } from "express";
 import {
   convertToChatHistory,
@@ -28,7 +28,20 @@ export const chat = async (req: Request, res: Response) => {
     const { stream, dataStream } =
       await createStreamFromWorkflowContext(context);
 
-    return streamToResponse(stream, res, {}, dataStream);
+    const streamResponse = LlamaIndexAdapter.toDataStreamResponse(stream, {
+      data: dataStream,
+    });
+    if (streamResponse.body) {
+      const reader = streamResponse.body.getReader();
+      while (true) {
+        const { done, value } = await reader.read();
+        if (done) {
+          res.end();
+          return;
+        }
+        res.write(value);
+      }
+    }
   } catch (error) {
     console.error("[LlamaIndex]", error);
     return res.status(500).json({

templates/components/multiagent/typescript/nextjs/route.ts

@@ -1,5 +1,5 @@
 import { initObservability } from "@/app/observability";
-import { StreamingTextResponse, type Message } from "ai";
+import { LlamaIndexAdapter, type Message } from "ai";
 import { NextRequest, NextResponse } from "next/server";
 import { initSettings } from "./engine/settings";
 import {
@@ -41,9 +41,9 @@ export async function POST(request: NextRequest) {
     });
     const { stream, dataStream } =
       await createStreamFromWorkflowContext(context);
-
-    // Return the two streams in one response
-    return new StreamingTextResponse(stream, {}, dataStream);
+    return LlamaIndexAdapter.toDataStreamResponse(stream, {
+      data: dataStream,
+    });
   } catch (error) {
     console.error("[LlamaIndex]", error);
     return NextResponse.json(

templates/components/multiagent/typescript/workflow/stream.ts

@@ -3,20 +3,15 @@ import {
   WorkflowContext,
   WorkflowEvent,
 } from "@llamaindex/workflow";
-import {
-  StreamData,
-  createStreamDataTransformer,
-  trimStartOfStreamHelper,
-} from "ai";
-import { ChatResponseChunk } from "llamaindex";
+import { StreamData } from "ai";
+import { ChatResponseChunk, EngineResponse } from "llamaindex";
+import { ReadableStream } from "stream/web";
 import { AgentRunEvent } from "./type";
 
 export async function createStreamFromWorkflowContext<Input, Output, Context>(
   context: WorkflowContext<Input, Output, Context>,
-): Promise<{ stream: ReadableStream<string>; dataStream: StreamData }> {
-  const trimStartOfStream = trimStartOfStreamHelper();
+): Promise<{ stream: ReadableStream<EngineResponse>; dataStream: StreamData }> {
   const dataStream = new StreamData();
-  const encoder = new TextEncoder();
   let generator: AsyncGenerator<ChatResponseChunk> | undefined;
 
   const closeStreams = (controller: ReadableStreamDefaultController) => {
@@ -24,10 +19,10 @@ export async function createStreamFromWorkflowContext<Input, Output, Context>(
     dataStream.close();
   };
 
-  const mainStream = new ReadableStream({
+  const stream = new ReadableStream<EngineResponse>({
     async start(controller) {
       // Kickstart the stream by sending an empty string
-      controller.enqueue(encoder.encode(""));
+      controller.enqueue({ delta: "" } as EngineResponse);
     },
     async pull(controller) {
       while (!generator) {
@@ -46,17 +41,14 @@ export async function createStreamFromWorkflowContext<Input, Output, Context>(
         closeStreams(controller);
         return;
       }
-      const text = trimStartOfStream(chunk.delta ?? "");
-      if (text) {
-        controller.enqueue(encoder.encode(text));
+      const delta = chunk.delta ?? "";
+      if (delta) {
+        controller.enqueue({ delta } as EngineResponse);
       }
     },
   });
 
-  return {
-    stream: mainStream.pipeThrough(createStreamDataTransformer()),
-    dataStream,
-  };
+  return { stream, dataStream };
 }
 
 function handleEvent(

templates/components/multiagent/typescript/workflow/tools.ts

@@ -5,7 +5,6 @@ import {
   ChatMessage,
   ChatResponse,
   ChatResponseChunk,
-  LlamaCloudIndex,
   PartialToolCall,
   QueryEngineTool,
   ToolCall,
@@ -14,58 +13,17 @@ import {
 } from "llamaindex";
 import crypto from "node:crypto";
 import { getDataSource } from "../engine";
+import { createQueryEngineTool } from "../engine/tools/query-engine";
 import { AgentRunEvent } from "./type";
 
-export const getQueryEngineTools = async (): Promise<
-  QueryEngineTool[] | null
-> => {
-  const topK = process.env.TOP_K ? parseInt(process.env.TOP_K) : undefined;
-
+export const getQueryEngineTool = async (): Promise<QueryEngineTool | null> => {
   const index = await getDataSource();
+
   if (!index) {
     return null;
   }
-  // index is LlamaCloudIndex use two query engine tools
-  if (index instanceof LlamaCloudIndex) {
-    return [
-      new QueryEngineTool({
-        queryEngine: index.asQueryEngine({
-          similarityTopK: topK,
-          retrieval_mode: "files_via_content",
-        }),
-        metadata: {
-          name: "document_retriever",
-          description: `Document retriever that retrieves entire documents from the corpus.
-  ONLY use for research questions that may require searching over entire research reports.
-  Will be slower and more expensive than chunk-level retrieval but may be necessary.`,
-        },
-      }),
-      new QueryEngineTool({
-        queryEngine: index.asQueryEngine({
-          similarityTopK: topK,
-          retrieval_mode: "chunks",
-        }),
-        metadata: {
-          name: "chunk_retriever",
-          description: `Retrieves a small set of relevant document chunks from the corpus.
-      Use for research questions that want to look up specific facts from the knowledge corpus,
-      and need entire documents.`,
-        },
-      }),
-    ];
-  } else {
-    return [
-      new QueryEngineTool({
-        queryEngine: index.asQueryEngine({
-          similarityTopK: topK,
-        }),
-        metadata: {
-          name: "retriever",
-          description: `Use this tool to retrieve information about the text corpus from the index.`,
-        },
-      }),
-    ];
-  }
+
+  return createQueryEngineTool(index);
 };
 
 /**

templates/components/services/python/suggestion.py

@@ -60,10 +60,12 @@ class NextQuestionSuggestion:
             return None
 
     @classmethod
-    def _extract_questions(cls, text: str) -> List[str]:
+    def _extract_questions(cls, text: str) -> List[str] | None:
         content_match = re.search(r"```(.*?)```", text, re.DOTALL)
-        content = content_match.group(1) if content_match else ""
-        return content.strip().split("\n")
+        content = content_match.group(1) if content_match else None
+        if not content:
+            return None
+        return [q.strip() for q in content.split("\n") if q.strip()]
 
     @classmethod
     async def suggest_next_questions(

templates/components/settings/python/settings.py

@@ -1,8 +1,17 @@
 import os
-from typing import Dict
+from typing import Dict, Optional
 
+from llama_index.core.multi_modal_llms import MultiModalLLM
 from llama_index.core.settings import Settings
 
+# `Settings` does not support setting `MultiModalLLM`
+# so we use a global variable to store it
+_multi_modal_llm: Optional[MultiModalLLM] = None
+
+
+def get_multi_modal_llm():
+    return _multi_modal_llm
+
 
 def init_settings():
     model_provider = os.getenv("MODEL_PROVIDER")
@@ -60,14 +69,21 @@ def init_openai():
     from llama_index.core.constants import DEFAULT_TEMPERATURE
     from llama_index.embeddings.openai import OpenAIEmbedding
     from llama_index.llms.openai import OpenAI
+    from llama_index.multi_modal_llms.openai import OpenAIMultiModal
+    from llama_index.multi_modal_llms.openai.utils import GPT4V_MODELS
 
     max_tokens = os.getenv("LLM_MAX_TOKENS")
+    model_name = os.getenv("MODEL", "gpt-4o-mini")
     Settings.llm = OpenAI(
-        model=os.getenv("MODEL", "gpt-4o-mini"),
+        model=model_name,
         temperature=float(os.getenv("LLM_TEMPERATURE", DEFAULT_TEMPERATURE)),
         max_tokens=int(max_tokens) if max_tokens is not None else None,
     )
 
+    if model_name in GPT4V_MODELS:
+        global _multi_modal_llm
+        _multi_modal_llm = OpenAIMultiModal(model=model_name)
+
     dimensions = os.getenv("EMBEDDING_DIM")
     Settings.embed_model = OpenAIEmbedding(
         model=os.getenv("EMBEDDING_MODEL", "text-embedding-3-small"),

templates/components/vectordbs/python/llamacloud/generate.py

@@ -1,5 +1,4 @@
 # flake8: noqa: E402
-import os
 
 from dotenv import load_dotenv
 
@@ -7,62 +6,24 @@ load_dotenv()
 
 import logging
 
-from app.engine.index import get_client, get_index
+from llama_index.core.readers import SimpleDirectoryReader
+from tqdm import tqdm
+
+from app.engine.index import get_index
 from app.engine.service import LLamaCloudFileService  # type: ignore
 from app.settings import init_settings
-from llama_cloud import PipelineType
-from llama_index.core.readers import SimpleDirectoryReader
-from llama_index.core.settings import Settings
 
 logging.basicConfig(level=logging.INFO)
 logger = logging.getLogger()
 
 
-def ensure_index(index):
-    project_id = index._get_project_id()
-    client = get_client()
-    pipelines = client.pipelines.search_pipelines(
-        project_id=project_id,
-        pipeline_name=index.name,
-        pipeline_type=PipelineType.MANAGED.value,
-    )
-    if len(pipelines) == 0:
-        from llama_index.embeddings.openai import OpenAIEmbedding
-
-        if not isinstance(Settings.embed_model, OpenAIEmbedding):
-            raise ValueError(
-                "Creating a new pipeline with a non-OpenAI embedding model is not supported."
-            )
-        client.pipelines.upsert_pipeline(
-            project_id=project_id,
-            request={
-                "name": index.name,
-                "embedding_config": {
-                    "type": "OPENAI_EMBEDDING",
-                    "component": {
-                        "api_key": os.getenv("OPENAI_API_KEY"),  # editable
-                        "model_name": os.getenv("EMBEDDING_MODEL"),
-                    },
-                },
-                "transform_config": {
-                    "mode": "auto",
-                    "config": {
-                        "chunk_size": Settings.chunk_size,  # editable
-                        "chunk_overlap": Settings.chunk_overlap,  # editable
-                    },
-                },
-            },
-        )
-
-
 def generate_datasource():
     init_settings()
     logger.info("Generate index for the provided data")
 
-    index = get_index()
-    ensure_index(index)
-    project_id = index._get_project_id()
-    pipeline_id = index._get_pipeline_id()
+    index = get_index(create_if_missing=True)
+    if index is None:
+        raise ValueError("Index not found and could not be created")
 
     # use SimpleDirectoryReader to retrieve the files to process
     reader = SimpleDirectoryReader(
@@ -72,14 +33,30 @@ def generate_datasource():
     files_to_process = reader.input_files
 
     # add each file to the LlamaCloud pipeline
-    for input_file in files_to_process:
+    error_files = []
+    for input_file in tqdm(
+        files_to_process,
+        desc="Processing files",
+        unit="file",
+    ):
         with open(input_file, "rb") as f:
-            logger.info(
+            logger.debug(
                 f"Adding file {input_file} to pipeline {index.name} in project {index.project_name}"
             )
-            LLamaCloudFileService.add_file_to_pipeline(
-                project_id, pipeline_id, f, custom_metadata={}
-            )
+            try:
+                LLamaCloudFileService.add_file_to_pipeline(
+                    index.project.id,
+                    index.pipeline.id,
+                    f,
+                    custom_metadata={},
+                    wait_for_processing=False,
+                )
+            except Exception as e:
+                error_files.append(input_file)
+                logger.error(f"Error adding file {input_file}: {e}")
+
+    if error_files:
+        logger.error(f"Failed to add the following files: {error_files}")
 
     logger.info("Finished generating the index")
 

templates/components/vectordbs/python/llamacloud/index.py

@@ -2,10 +2,12 @@ import logging
 import os
 from typing import Optional
 
+from llama_cloud import PipelineType
 from llama_index.core.callbacks import CallbackManager
 from llama_index.core.ingestion.api_utils import (
     get_client as llama_cloud_get_client,
 )
+from llama_index.core.settings import Settings
 from llama_index.indices.managed.llama_cloud import LlamaCloudIndex
 from pydantic import BaseModel, Field, field_validator
 
@@ -82,14 +84,63 @@ class IndexConfig(BaseModel):
         }
 
 
-def get_index(config: IndexConfig = None):
+def get_index(
+    config: IndexConfig = None,
+    create_if_missing: bool = False,
+):
     if config is None:
         config = IndexConfig()
-    index = LlamaCloudIndex(**config.to_index_kwargs())
-
-    return index
+    # Check whether the index exists
+    try:
+        index = LlamaCloudIndex(**config.to_index_kwargs())
+        return index
+    except ValueError:
+        logger.warning("Index not found")
+        if create_if_missing:
+            logger.info("Creating index")
+            _create_index(config)
+            return LlamaCloudIndex(**config.to_index_kwargs())
+        return None
 
 
 def get_client():
     config = LlamaCloudConfig()
     return llama_cloud_get_client(**config.to_client_kwargs())
+
+
+def _create_index(
+    config: IndexConfig,
+):
+    client = get_client()
+    pipeline_name = config.llama_cloud_pipeline_config.pipeline
+
+    pipelines = client.pipelines.search_pipelines(
+        pipeline_name=pipeline_name,
+        pipeline_type=PipelineType.MANAGED.value,
+    )
+    if len(pipelines) == 0:
+        from llama_index.embeddings.openai import OpenAIEmbedding
+
+        if not isinstance(Settings.embed_model, OpenAIEmbedding):
+            raise ValueError(
+                "Creating a new pipeline with a non-OpenAI embedding model is not supported."
+            )
+        client.pipelines.upsert_pipeline(
+            request={
+                "name": pipeline_name,
+                "embedding_config": {
+                    "type": "OPENAI_EMBEDDING",
+                    "component": {
+                        "api_key": os.getenv("OPENAI_API_KEY"),  # editable
+                        "model_name": os.getenv("EMBEDDING_MODEL"),
+                    },
+                },
+                "transform_config": {
+                    "mode": "auto",
+                    "config": {
+                        "chunk_size": Settings.chunk_size,  # editable
+                        "chunk_overlap": Settings.chunk_overlap,  # editable
+                    },
+                },
+            },
+        )

templates/components/vectordbs/python/llamacloud/service.py

@@ -1,18 +1,18 @@
-from io import BytesIO
 import logging
 import os
 import time
-from typing import Any, Dict, List, Optional, Set, Tuple, Union
 import typing
+from io import BytesIO
+from typing import Any, Dict, List, Optional, Set, Tuple, Union
 
+import requests
 from fastapi import BackgroundTasks
 from llama_cloud import ManagedIngestionStatus, PipelineFileCreateCustomMetadataValue
+from llama_index.core.schema import NodeWithScore
 from pydantic import BaseModel
-import requests
+
 from app.api.routers.models import SourceNodes
 from app.engine.index import get_client
-from llama_index.core.schema import NodeWithScore
-
 
 logger = logging.getLogger("uvicorn")
 
@@ -64,27 +64,34 @@ class LLamaCloudFileService:
         pipeline_id: str,
         upload_file: Union[typing.IO, Tuple[str, BytesIO]],
         custom_metadata: Optional[Dict[str, PipelineFileCreateCustomMetadataValue]],
+        wait_for_processing: bool = True,
     ) -> str:
         client = get_client()
         file = client.files.upload_file(project_id=project_id, upload_file=upload_file)
+        file_id = file.id
         files = [
             {
-                "file_id": file.id,
-                "custom_metadata": {"file_id": file.id, **(custom_metadata or {})},
+                "file_id": file_id,
+                "custom_metadata": {"file_id": file_id, **(custom_metadata or {})},
             }
         ]
         files = client.pipelines.add_files_to_pipeline(pipeline_id, request=files)
 
+        if not wait_for_processing:
+            return file_id
+
         # Wait 2s for the file to be processed
         max_attempts = 20
         attempt = 0
         while attempt < max_attempts:
-            result = client.pipelines.get_pipeline_file_status(pipeline_id, file.id)
+            result = client.pipelines.get_pipeline_file_status(
+                file_id=file_id, pipeline_id=pipeline_id
+            )
             if result.status == ManagedIngestionStatus.ERROR:
                 raise Exception(f"File processing failed: {str(result)}")
             if result.status == ManagedIngestionStatus.SUCCESS:
                 # File is ingested - return the file id
-                return file.id
+                return file_id
             attempt += 1
             time.sleep(0.1)  # Sleep for 100ms
         raise Exception(

templates/types/extractor/fastapi/app/ui/components/upload.py

@@ -2,6 +2,7 @@ import os
 from typing import List
 
 import reflex as rx
+
 from app.engine.generate import generate_datasource
 
 
@@ -78,10 +79,10 @@ def upload_component() -> rx.Component:
             UploadedFilesState.uploaded_files,
             lambda file: rx.card(
                 rx.stack(
-                    rx.text(file.file_name, size="sm"),
+                    rx.text(file.file_name, size="2"),
                     rx.button(
                         "x",
-                        size="sm",
+                        size="2",
                         on_click=UploadedFilesState.remove_file(file.file_name),
                     ),
                     justify="between",

templates/types/extractor/fastapi/pyproject.toml

@@ -13,7 +13,8 @@ python = "^3.11,<4.0"
 fastapi = "^0.109.1"
 uvicorn = { extras = ["standard"], version = "^0.23.2" }
 python-dotenv = "^1.0.0"
-llama-index = "^0.11.1"
+pydantic = "<2.10"
+llama-index = "^0.12.1"
 cachetools = "^5.3.3"
 reflex = "^0.6.2.post1"
 

templates/types/streaming/express/README-template.md

@@ -8,7 +8,7 @@ First, install the dependencies:
 npm install
 ```
 
-Second, generate the embeddings of the documents in the `./data` directory (if this folder exists - otherwise, skip this step):
+Second, generate the embeddings of the documents in the `./data` directory:
 
 ```
 npm run generate

templates/types/streaming/express/package.json

@@ -16,7 +16,7 @@
     "lint": "eslint ."
   },
   "dependencies": {
-    "ai": "3.3.42",
+    "ai": "4.0.3",
     "cors": "^2.8.5",
     "dotenv": "^16.3.1",
     "duck-duck-scrape": "^2.2.5",

templates/types/streaming/express/src/controllers/chat.controller.ts

@@ -1,4 +1,4 @@
-import { LlamaIndexAdapter, Message, StreamData, streamToResponse } from "ai";
+import { LlamaIndexAdapter, Message, StreamData } from "ai";
 import { Request, Response } from "express";
 import { ChatMessage, Settings } from "llamaindex";
 import { createChatEngine } from "./engine/chat";
@@ -43,7 +43,7 @@ export const chat = async (req: Request, res: Response) => {
       });
     });
 
-    const onFinal = (content: string) => {
+    const onCompletion = (content: string) => {
       chatHistory.push({ role: "assistant", content: content });
       generateNextQuestions(chatHistory)
         .then((questions: string[]) => {
@@ -59,8 +59,21 @@ export const chat = async (req: Request, res: Response) => {
         });
     };
 
-    const stream = LlamaIndexAdapter.toDataStream(response, { onFinal });
-    return streamToResponse(stream, res, {}, vercelStreamData);
+    const streamResponse = LlamaIndexAdapter.toDataStreamResponse(response, {
+      data: vercelStreamData,
+      callbacks: { onCompletion },
+    });
+    if (streamResponse.body) {
+      const reader = streamResponse.body.getReader();
+      while (true) {
+        const { done, value } = await reader.read();
+        if (done) {
+          res.end();
+          return;
+        }
+        res.write(value);
+      }
+    }
   } catch (error) {
     console.error("[LlamaIndex]", error);
     return res.status(500).json({

templates/types/streaming/fastapi/DEPLOY.md

@@ -0,0 +1,73 @@
+## Deployments
+
+### Using [Fly.io](https://fly.io/):
+
+First, install [flyctl](https://fly.io/docs/flyctl/install/) and then authenticate with your Fly.io account:
+
+```shell
+fly auth login
+```
+
+Then, run this command and follow the prompts to deploy the app.:
+
+```shell
+fly launch
+```
+
+And to open the app in your browser:
+
+```shell
+fly apps open
+```
+
+> **Note**: The app will use the values from the `.env` file by default to simplify the deployment. Make sure all the needed environment variables in the [.env](.env) file are set. For production environments, you should not use the `.env` file, but [set the variables in Fly.io](https://fly.io/docs/rails/the-basics/configuration/) instead.
+
+#### Documents
+
+If you're having documents in the `./data` folder, run the following command to generate vector embeddings of the documents:
+
+```
+fly console --machine <machine_id> --command "poetry run generate"
+```
+
+Where `machine_id` is the ID of the machine where the app is running. You can show the running machines with the `fly machines` command.
+
+> **Note**: Using documents will make the app stateful. As Fly.io is a stateless app, you should use [LlamaCloud](https://docs.cloud.llamaindex.ai/llamacloud/getting_started) or a vector database to store the embeddings of the documents. This applies also for document uploads by the user.
+
+### Using Docker
+
+First, build an image for the app:
+
+```
+docker build -t <your_image_name> .
+```
+
+Then, start the app by running the image:
+
+```
+docker run \
+  -v $(pwd)/.env:/app/.env \ # Use ENV variables and configuration from your file-system
+  -v $(pwd)/config:/app/config \
+  -v $(pwd)/storage:/app/storage \ # Use your file system to store vector embeddings
+  -p 8000:8000 \
+  <your_image_name>
+```
+
+Open [http://localhost:8000](http://localhost:8000) with your browser to start the app.
+
+#### Documents
+
+If you're having documents in the `./data` folder, run the following command to generate vector embeddings of the documents:
+
+```
+docker run \
+  --rm \
+  -v $(pwd)/.env:/app/.env \ # Use ENV variables and configuration from your file-system
+  -v $(pwd)/config:/app/config \
+  -v $(pwd)/data:/app/data \ # Use your local folder to read the data
+  -v $(pwd)/storage:/app/storage \ # Use your file system to store the vector database
+  <your_image_name> \
+  poetry run generate
+```
+
+The app will then be able to answer questions about the documents in the `./data` folder.

templates/types/streaming/fastapi/Dockerfile

@@ -1,4 +1,19 @@
-FROM python:3.11 as build
+# ====================================
+# Build the frontend
+# ====================================
+FROM node:20 AS frontend
+
+WORKDIR /app/frontend
+
+COPY .frontend /app/frontend
+
+RUN npm install && npm run build
+
+
+# ====================================
+# Backend
+# ====================================
+FROM python:3.11 AS build
 
 WORKDIR /app
 
@@ -19,8 +34,17 @@ COPY ./pyproject.toml ./poetry.lock* /app/
 RUN poetry install --no-root --no-cache --only main
 
 # ====================================
-FROM build as release
+# Release
+# ====================================
+FROM build AS release
+
+COPY --from=frontend /app/frontend/out /app/static
 
 COPY . .
 
-CMD ["python", "main.py"]
+# Remove frontend code
+RUN rm -rf .frontend
+
+EXPOSE 8000
+
+CMD ["poetry", "run", "prod"]

templates/types/streaming/fastapi/README-template.md

@@ -15,18 +15,20 @@ Then check the parameters that have been pre-configured in the `.env` file in th
 
 If you are using any tools or data sources, you can update their config files in the `config` folder.
 
-Second, generate the embeddings of the documents in the `./data` directory (if this folder exists - otherwise, skip this step):
+Second, generate the embeddings of the documents in the `./data` directory:
 
 ```
 poetry run generate
 ```
 
-Third, run the development server:
+Third, run the app:
 
 ```
-python main.py
+poetry run dev
 ```
 
+Open [http://localhost:8000](http://localhost:8000) with your browser to start the app.
+
 The example provides two different API endpoints:
 
 1. `/api/chat` - a streaming chat endpoint
@@ -50,47 +52,15 @@ curl --location 'localhost:8000/api/chat/request' \
 
 You can start editing the API endpoints by modifying `app/api/routers/chat.py`. The endpoints auto-update as you save the file. You can delete the endpoint you're not using.
 
-Open [http://localhost:8000/docs](http://localhost:8000/docs) with your browser to see the Swagger UI of the API.
-
-The API allows CORS for all origins to simplify development. You can change this behavior by setting the `ENVIRONMENT` environment variable to `prod`:
+To start the app optimized for **production**, run:
 
 ```
-ENVIRONMENT=prod python main.py
+poetry run prod
 ```
 
-## Using Docker
+## Deployments
 
-1. Build an image for the FastAPI app:
-
-```
-docker build -t <your_backend_image_name> .
-```
-
-2. Generate embeddings:
-
-Parse the data and generate the vector embeddings if the `./data` folder exists - otherwise, skip this step:
-
-```
-docker run \
-  --rm \
-  -v $(pwd)/.env:/app/.env \ # Use ENV variables and configuration from your file-system
-  -v $(pwd)/config:/app/config \
-  -v $(pwd)/data:/app/data \ # Use your local folder to read the data
-  -v $(pwd)/storage:/app/storage \ # Use your file system to store the vector database
-  <your_backend_image_name> \
-  poetry run generate
-```
-
-3. Start the API:
-
-```
-docker run \
-  -v $(pwd)/.env:/app/.env \ # Use ENV variables and configuration from your file-system
-  -v $(pwd)/config:/app/config \
-  -v $(pwd)/storage:/app/storage \ # Use your file system to store gea vector database
-  -p 8000:8000 \
-  <your_backend_image_name>
-```
+For production deployments, check the [DEPLOY.md](DEPLOY.md) file.
 
 ## Learn More
 

templates/types/streaming/fastapi/app/api/routers/__init__.py

@@ -3,11 +3,13 @@ from fastapi import APIRouter
 from .chat import chat_router  # noqa: F401
 from .chat_config import config_router  # noqa: F401
 from .upload import file_upload_router  # noqa: F401
+from .query import query_router  # noqa: F401
 
 api_router = APIRouter()
 api_router.include_router(chat_router, prefix="/chat")
 api_router.include_router(config_router, prefix="/chat/config")
 api_router.include_router(file_upload_router, prefix="/chat/upload")
+api_router.include_router(query_router, prefix="/query")
 
 # Dynamically adding additional routers if they exist
 try:

templates/types/streaming/fastapi/app/api/routers/query.py

@@ -0,0 +1,25 @@
+import logging
+
+from fastapi import APIRouter
+from app.engine.index import IndexConfig, get_index
+from llama_index.core.base.base_query_engine import BaseQueryEngine
+
+
+query_router = r = APIRouter()
+
+logger = logging.getLogger("uvicorn")
+
+
+def get_query_engine() -> BaseQueryEngine:
+    index_config = IndexConfig(**{})
+    index = get_index(index_config)
+    return index.as_query_engine()
+
+
+@r.get("/")
+async def query_request(
+    query: str,
+) -> str:
+    query_engine = get_query_engine()
+    response = await query_engine.aquery(query)
+    return response.response

templates/types/streaming/fastapi/app/api/routers/vercel_response.py

@@ -22,6 +22,7 @@ class VercelStreamResponse(StreamingResponse):
 
     TEXT_PREFIX = "0:"
     DATA_PREFIX = "8:"
+    ERROR_PREFIX = "3:"
 
     def __init__(
         self,
@@ -53,17 +54,26 @@ class VercelStreamResponse(StreamingResponse):
         # Merge the chat response generator and the event generator
         combine = stream.merge(chat_response_generator, event_generator)
         is_stream_started = False
-        async with combine.stream() as streamer:
-            async for output in streamer:
-                if not is_stream_started:
-                    is_stream_started = True
-                    # Stream a blank message to start displaying the response in the UI
-                    yield cls.convert_text("")
+        try:
+            async with combine.stream() as streamer:
+                async for output in streamer:
+                    if await request.is_disconnected():
+                        break
 
-                yield output
+                    if not is_stream_started:
+                        is_stream_started = True
+                        # Stream a blank message to start displaying the response in the UI
+                        yield cls.convert_text("")
 
-                if await request.is_disconnected():
-                    break
+                    yield output
+        except Exception:
+            logger.exception("Error in stream response")
+            yield cls.convert_error(
+                "An unexpected error occurred while processing your request, preventing the creation of a final answer. Please try again."
+            )
+        finally:
+            # Ensure event handler is marked as done even if connection breaks
+            event_handler.is_done = True
 
     @classmethod
     async def _event_generator(cls, event_handler: EventCallbackHandler):
@@ -131,6 +141,11 @@ class VercelStreamResponse(StreamingResponse):
         data_str = json.dumps(data)
         return f"{cls.DATA_PREFIX}[{data_str}]\n"
 
+    @classmethod
+    def convert_error(cls, error: str):
+        error_str = json.dumps(error)
+        return f"{cls.ERROR_PREFIX}{error_str}\n"
+
     @staticmethod
     def _process_response_nodes(
         source_nodes: List[NodeWithScore],

templates/types/streaming/fastapi/app/config.py

@@ -1 +1,4 @@
+import os
+
 DATA_DIR = "data"
+STATIC_DIR = os.getenv("STATIC_DIR", "static")

templates/types/streaming/fastapi/app/middlewares/frontend.py

@@ -0,0 +1,78 @@
+import logging
+from typing import Set
+
+import httpx
+from fastapi import Request
+from fastapi.responses import StreamingResponse
+
+logger = logging.getLogger("uvicorn")
+
+
+class FrontendProxyMiddleware:
+    """
+    Proxy requests to the frontend development server
+    """
+
+    def __init__(
+        self,
+        app,
+        frontend_endpoint: str,
+        excluded_paths: Set[str],
+    ):
+        self.app = app
+        self.excluded_paths = excluded_paths
+        self.frontend_endpoint = frontend_endpoint
+
+    async def _request_frontend(
+        self,
+        request: Request,
+        path: str,
+        timeout: float = 60.0,
+    ):
+        async with httpx.AsyncClient(timeout=timeout) as client:
+            url = f"{self.frontend_endpoint}/{path}"
+            if request.query_params:
+                url = f"{url}?{request.query_params}"
+
+            headers = dict(request.headers)
+            try:
+                body = await request.body() if request.method != "GET" else None
+
+                response = await client.request(
+                    method=request.method,
+                    url=url,
+                    headers=headers,
+                    content=body,
+                    follow_redirects=True,
+                )
+
+                response_headers = dict(response.headers)
+                response_headers.pop("content-encoding", None)
+                response_headers.pop("content-length", None)
+
+                return StreamingResponse(
+                    response.iter_bytes(),
+                    status_code=response.status_code,
+                    headers=response_headers,
+                )
+            except Exception as e:
+                logger.error(f"Proxy error: {str(e)}")
+                raise
+
+    def _is_excluded_path(self, path: str) -> bool:
+        return any(
+            path.startswith(excluded_path) for excluded_path in self.excluded_paths
+        )
+
+    async def __call__(self, scope, receive, send):
+        if scope["type"] != "http":
+            return await self.app(scope, receive, send)
+
+        request = Request(scope, receive)
+        path = request.url.path
+
+        if self._is_excluded_path(path):
+            return await self.app(scope, receive, send)
+
+        response = await self._request_frontend(request, path.lstrip("/"))
+        return await response(scope, receive, send)

templates/types/streaming/fastapi/app/services/file.py

@@ -249,6 +249,7 @@ class FileService:
             index.pipeline.id,
             upload_file,
             custom_metadata={},
+            wait_for_processing=True,
         )
         return doc_id
 

templates/types/streaming/fastapi/gitignore

@@ -2,3 +2,4 @@ __pycache__
 storage
 .env
 output
+static/

templates/types/streaming/fastapi/main.py

@@ -1,5 +1,5 @@
 # flake8: noqa: E402
-from app.config import DATA_DIR
+from app.config import DATA_DIR, STATIC_DIR
 from dotenv import load_dotenv
 
 load_dotenv()
@@ -9,10 +9,10 @@ import os
 
 import uvicorn
 from app.api.routers import api_router
+from app.middlewares.frontend import FrontendProxyMiddleware
 from app.observability import init_observability
 from app.settings import init_settings
 from fastapi import FastAPI
-from fastapi.middleware.cors import CORSMiddleware
 from fastapi.responses import RedirectResponse
 from fastapi.staticfiles import StaticFiles
 
@@ -24,38 +24,43 @@ init_observability()
 environment = os.getenv("ENVIRONMENT", "dev")  # Default to 'development' if not set
 logger = logging.getLogger("uvicorn")
 
-if environment == "dev":
-    logger.warning("Running in development mode - allowing CORS for all origins")
-    app.add_middleware(
-        CORSMiddleware,
-        allow_origins=["*"],
-        allow_credentials=True,
-        allow_methods=["*"],
-        allow_headers=["*"],
-    )
 
-    # Redirect to documentation page when accessing base URL
-    @app.get("/")
-    async def redirect_to_docs():
-        return RedirectResponse(url="/docs")
-
-
-def mount_static_files(directory, path):
+def mount_static_files(directory, path, html=False):
     if os.path.exists(directory):
         logger.info(f"Mounting static files '{directory}' at '{path}'")
         app.mount(
             path,
-            StaticFiles(directory=directory, check_dir=False),
+            StaticFiles(directory=directory, check_dir=False, html=html),
             name=f"{directory}-static",
         )
 
 
+app.include_router(api_router, prefix="/api")
+
 # Mount the data files to serve the file viewer
 mount_static_files(DATA_DIR, "/api/files/data")
 # Mount the output files from tools
 mount_static_files("output", "/api/files/output")
 
-app.include_router(api_router, prefix="/api")
+if environment == "dev":
+    frontend_endpoint = os.getenv("FRONTEND_ENDPOINT")
+    if frontend_endpoint:
+        app.add_middleware(
+            FrontendProxyMiddleware,
+            frontend_endpoint=frontend_endpoint,
+            excluded_paths=set(
+                route.path for route in app.routes if hasattr(route, "path")
+            ),
+        )
+    else:
+        logger.warning("No frontend endpoint - starting API server only")
+
+        @app.get("/")
+        async def redirect_to_docs():
+            return RedirectResponse(url="/docs")
+else:
+    # Mount the frontend static files (production)
+    mount_static_files(STATIC_DIR, "/", html=True)
 
 if __name__ == "__main__":
     app_host = os.getenv("APP_HOST", "0.0.0.0")

templates/types/streaming/fastapi/pyproject.toml

@@ -7,15 +7,20 @@ readme = "README.md"
 
 [tool.poetry.scripts]
 generate = "app.engine.generate:generate_datasource"
+dev = "run:dev" # Starts the app in dev mode
+prod = "run:prod" # Starts the app in prod mode
+build = "run:build" # Builds the frontend assets and copies them to the static directory
 
 [tool.poetry.dependencies]
-python = ">=3.11,<3.13"
+python = ">=3.11,<3.14"
 fastapi = "^0.109.1"
 uvicorn = { extras = ["standard"], version = "^0.23.2" }
 python-dotenv = "^1.0.0"
+pydantic = "<2.10"
 aiostream = "^0.5.2"
 cachetools = "^5.3.3"
-llama-index = "^0.11.17"
+llama-index = "^0.12.1"
+rich = "^13.9.4"
 
 [tool.poetry.group.dev.dependencies]
 mypy = "^1.8.0"

templates/types/streaming/fastapi/run.py

@@ -0,0 +1,275 @@
+import asyncio
+import os
+import shutil
+import socket
+from asyncio.subprocess import Process
+from pathlib import Path
+from shutil import which
+from subprocess import CalledProcessError, run
+
+import dotenv
+import rich
+
+dotenv.load_dotenv()
+
+
+FRONTEND_DIR = Path(os.getenv("FRONTEND_DIR", ".frontend"))
+DEFAULT_FRONTEND_PORT = 3000
+STATIC_DIR = Path(os.getenv("STATIC_DIR", "static"))
+
+
+def build():
+    """
+    Build the frontend and copy the static files to the backend.
+
+    Raises:
+        SystemError: If any build step fails
+    """
+    static_dir = Path("static")
+
+    try:
+        package_manager = _get_node_package_manager()
+        _install_frontend_dependencies()
+
+        rich.print("\n[bold]Building the frontend[/bold]")
+        run([package_manager, "run", "build"], cwd=FRONTEND_DIR, check=True)
+
+        if static_dir.exists():
+            shutil.rmtree(static_dir)
+        static_dir.mkdir(exist_ok=True)
+
+        shutil.copytree(FRONTEND_DIR / "out", static_dir, dirs_exist_ok=True)
+
+        rich.print(
+            "\n[bold]Built frontend successfully![/bold]"
+            "\n[bold]Run: 'poetry run prod' to start the app[/bold]"
+            "\n[bold]Don't forget to update the .env file![/bold]"
+        )
+    except CalledProcessError as e:
+        raise SystemError(f"Build failed during {e.cmd}") from e
+    except Exception as e:
+        raise SystemError(f"Build failed: {str(e)}") from e
+
+
+def dev():
+    asyncio.run(start_development_servers())
+
+
+def prod():
+    asyncio.run(start_production_server())
+
+
+async def start_development_servers():
+    """
+    Start both frontend and backend development servers.
+    Frontend runs with hot reloading, backend runs FastAPI server.
+
+    Raises:
+        SystemError: If either server fails to start
+    """
+    rich.print("\n[bold]Starting development servers[/bold]")
+
+    try:
+        processes = []
+        if _is_frontend_included():
+            frontend_process, frontend_port = await _run_frontend()
+            processes.append(frontend_process)
+            backend_process = await _run_backend(
+                envs={
+                    "ENVIRONMENT": "dev",
+                    "FRONTEND_ENDPOINT": f"http://localhost:{frontend_port}",
+                },
+            )
+            processes.append(backend_process)
+        else:
+            backend_process = await _run_backend(
+                envs={"ENVIRONMENT": "dev"},
+            )
+            processes.append(backend_process)
+
+        try:
+            # Wait for processes to complete
+            await asyncio.gather(*[process.wait() for process in processes])
+        except (asyncio.CancelledError, KeyboardInterrupt):
+            rich.print("\n[bold yellow]Shutting down...[/bold yellow]")
+        finally:
+            # Terminate both processes
+            for process in processes:
+                process.terminate()
+                try:
+                    await asyncio.wait_for(process.wait(), timeout=5)
+                except asyncio.TimeoutError:
+                    process.kill()
+
+    except Exception as e:
+        raise SystemError(f"Failed to start development servers: {str(e)}") from e
+
+
+async def start_production_server():
+    if _is_frontend_included():
+        is_frontend_built = (FRONTEND_DIR / "out" / "index.html").exists()
+        is_frontend_static_dir_exists = STATIC_DIR.exists()
+        if not is_frontend_built or not is_frontend_static_dir_exists:
+            build()
+
+    try:
+        process = await _run_backend(
+            envs={"ENVIRONMENT": "prod"},
+        )
+        await process.wait()
+    except Exception as e:
+        raise SystemError(f"Failed to start production server: {str(e)}") from e
+    finally:
+        process.terminate()
+        try:
+            await asyncio.wait_for(process.wait(), timeout=5)
+        except asyncio.TimeoutError:
+            process.kill()
+
+
+async def _run_frontend(
+    port: int = DEFAULT_FRONTEND_PORT,
+    timeout: int = 5,
+) -> tuple[Process, int]:
+    """
+    Start the frontend development server and return its process and port.
+
+    Returns:
+        tuple[Process, int]: The frontend process and the port it's running on
+    """
+    # Install dependencies
+    _install_frontend_dependencies()
+
+    port = _find_free_port(start_port=DEFAULT_FRONTEND_PORT)
+    package_manager = _get_node_package_manager()
+    frontend_process = await asyncio.create_subprocess_exec(
+        package_manager,
+        "run",
+        "dev",
+        "-p",
+        str(port),
+        cwd=FRONTEND_DIR,
+    )
+    rich.print(
+        f"\n[bold]Waiting for frontend to start, port: {port}, process id: {frontend_process.pid}[/bold]"
+    )
+    # Block until the frontend is accessible
+    for _ in range(timeout):
+        await asyncio.sleep(1)
+        # Check if the frontend is accessible (port is open) or frontend_process is running
+        if frontend_process.returncode is not None:
+            raise RuntimeError("Could not start frontend dev server")
+        if not _is_bindable_port(port):
+            rich.print(
+                f"\n[bold green]Frontend dev server is running on port {port}[/bold green]"
+            )
+            return frontend_process, port
+    raise TimeoutError(f"Frontend dev server failed to start within {timeout} seconds")
+
+
+async def _run_backend(
+    envs: dict[str, str | None] = {},
+) -> Process:
+    """
+    Start the backend development server.
+
+    Args:
+        frontend_port: The port number the frontend is running on
+    Returns:
+        Process: The backend process
+    """
+    # Merge environment variables
+    envs = {**os.environ, **(envs or {})}
+    rich.print("\n[bold]Starting backend FastAPI server...[/bold]")
+    poetry_executable = _get_poetry_executable()
+    return await asyncio.create_subprocess_exec(
+        poetry_executable,
+        "run",
+        "python",
+        "main.py",
+        env=envs,
+    )
+
+
+def _install_frontend_dependencies():
+    package_manager = _get_node_package_manager()
+    rich.print(
+        f"\n[bold]Installing frontend dependencies using {Path(package_manager).name}. It might take a while...[/bold]"
+    )
+    run([package_manager, "install"], cwd=".frontend", check=True)
+
+
+def _get_node_package_manager() -> str:
+    """
+    Check for available package managers and return the preferred one.
+    Returns 'pnpm' if installed, falls back to 'npm'.
+    Raises SystemError if neither is installed.
+
+    Returns:
+        str: The full path to the available package manager executable
+    """
+    # On Windows, we need to check for .cmd extensions
+    pnpm_cmds = ["pnpm", "pnpm.cmd"]
+    npm_cmds = ["npm", "npm.cmd"]
+
+    for cmd in pnpm_cmds:
+        cmd_path = which(cmd)
+        if cmd_path is not None:
+            return cmd_path
+
+    for cmd in npm_cmds:
+        cmd_path = which(cmd)
+        if cmd_path is not None:
+            return cmd_path
+
+    raise SystemError(
+        "Neither pnpm nor npm is installed. Please install Node.js and a package manager first."
+    )
+
+
+def _get_poetry_executable() -> str:
+    """
+    Check for available Poetry executables and return the preferred one.
+    Returns 'poetry' if installed, falls back to 'poetry.cmd'.
+    Raises SystemError if neither is installed.
+
+    Returns:
+        str: The full path to the available Poetry executable
+    """
+    poetry_cmds = ["poetry", "poetry.cmd"]
+    for cmd in poetry_cmds:
+        cmd_path = which(cmd)
+        if cmd_path is not None:
+            return cmd_path
+    raise SystemError("Poetry is not installed. Please install Poetry first.")
+
+
+def _is_bindable_port(port: int) -> bool:
+    """Check if a port is available by attempting to connect to it."""
+    with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
+        try:
+            # Try to connect to the port
+            s.connect(("localhost", port))
+            # If we can connect, port is in use
+            return False
+        except ConnectionRefusedError:
+            # Connection refused means port is available
+            return True
+        except socket.error:
+            # Other socket errors also likely mean port is available
+            return True
+
+
+def _find_free_port(start_port: int) -> int:
+    """
+    Find a free port starting from the given port number.
+    """
+    for port in range(start_port, 65535):
+        if _is_bindable_port(port):
+            return port
+    raise SystemError("No free port found")
+
+
+def _is_frontend_included() -> bool:
+    """Check if the app has frontend"""
+    return FRONTEND_DIR.exists()

templates/types/streaming/nextjs/Dockerfile

@@ -0,0 +1,16 @@
+FROM node:20-alpine as build
+
+WORKDIR /app
+
+# Install dependencies
+COPY package.json package-lock.* ./
+RUN npm install
+
+# Build the application
+COPY . .
+RUN npm run build
+
+# ====================================
+FROM build as release
+
+CMD ["npm", "run", "start"]

templates/types/streaming/nextjs/README-template.md

@@ -8,7 +8,7 @@ First, install the dependencies:
 npm install
 ```
 
-Second, generate the embeddings of the documents in the `./data` directory (if this folder exists - otherwise, skip this step):
+Second, generate the embeddings of the documents in the `./data` directory:
 
 ```
 npm run generate

templates/types/streaming/nextjs/app/api/chat/route.ts

@@ -56,7 +56,7 @@ export async function POST(request: NextRequest) {
       });
     });
 
-    const onFinal = (content: string) => {
+    const onCompletion = (content: string) => {
       chatHistory.push({ role: "assistant", content: content });
       generateNextQuestions(chatHistory)
         .then((questions: string[]) => {
@@ -74,7 +74,7 @@ export async function POST(request: NextRequest) {
 
     return LlamaIndexAdapter.toDataStreamResponse(response, {
       data: vercelStreamData,
-      callbacks: { onFinal },
+      callbacks: { onCompletion },
     });
   } catch (error) {
     console.error("[LlamaIndex]", error);

templates/types/streaming/nextjs/app/api/sandbox/route.ts

@@ -92,7 +92,8 @@ export async function POST(req: Request) {
       const localFilePath = path.join("output", "uploaded", fileName);
       const fileContent = await fs.readFile(localFilePath);
 
-      await sbx.files.write(sandboxFilePath, fileContent);
+      const arrayBuffer = new Uint8Array(fileContent).buffer;
+      await sbx.files.write(sandboxFilePath, arrayBuffer);
       console.log(`Copied file to ${sandboxFilePath} in ${sbx.sandboxID}`);
     });
   }

templates/types/streaming/nextjs/app/components/chat-section.tsx

@@ -1,8 +1,7 @@
 "use client";
 
 import { ChatSection as ChatSectionUI } from "@llamaindex/chat-ui";
-import "@llamaindex/chat-ui/styles/code.css";
-import "@llamaindex/chat-ui/styles/katex.css";
+import "@llamaindex/chat-ui/styles/markdown.css";
 import "@llamaindex/chat-ui/styles/pdf.css";
 import { useChat } from "ai/react";
 import CustomChatInput from "./ui/chat/chat-input";
@@ -15,7 +14,13 @@ export default function ChatSection() {
     api: `${backend}/api/chat`,
     onError: (error: unknown) => {
       if (!(error instanceof Error)) throw error;
-      alert(JSON.parse(error.message).detail);
+      let errorMessage: string;
+      try {
+        errorMessage = JSON.parse(error.message).detail;
+      } catch (e) {
+        errorMessage = error.message;
+      }
+      alert(errorMessage);
     },
   });
   return (

templates/types/streaming/nextjs/package.json

@@ -16,8 +16,8 @@
     "@radix-ui/react-select": "^2.1.1",
     "@radix-ui/react-slot": "^1.0.2",
     "@radix-ui/react-tabs": "^1.1.0",
-    "@llamaindex/chat-ui": "0.0.9",
-    "ai": "3.4.33",
+    "@llamaindex/chat-ui": "0.0.12",
+    "ai": "4.0.3",
     "ajv": "^8.12.0",
     "class-variance-authority": "^0.7.0",
     "clsx": "^2.1.1",

templates/vscode_settings.json

@@ -0,0 +1,3 @@
+{
+  "python.envFile": ""
+}