Release 0.3.15 (#438)

Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>

CHANGELOG.md

@@ -1,5 +1,81 @@
 # create-llama
 
+## 0.3.15
+
+### Patch Changes
+
+- fc5b266: Improve DX for Python template (use one deployment instead of two)
+- f8f97d2: Add support for python 3.13
+
+## 0.3.14
+
+### Patch Changes
+
+- 00f0b3a: fix: dont include user message in chat history
+- 4663dec: chore: bump react19 rc
+- 44b34fb: chore: update eslint 9, nextjs 15, react 19
+- 6925676: feat: use latest chat UI
+
+## 0.3.13
+
+### Patch Changes
+
+- 282eaa0: Ensure that the index and document store are created when uploading a file with no available index.
+
+## 0.3.12
+
+### Patch Changes
+
+- 6edea6a: Optimize generated workflow code for Python
+- 8431b78: Optimize Typescript multi-agent code
+- 8431b78: Add form filling use case (Typescript)
+
+## 0.3.11
+
+### Patch Changes
+
+- 2b8aaa8: Add support for local models via Hugging Face
+- b9570b2: Fix: use generic LLMAgent instead of OpenAIAgent (adds support for Gemini and Anthropic for Agentic RAG)
+- 1fe21f8: Fix the highlight.js issue with the Next.js static build
+- 00009ae: feat: import pdf css
+
+## 0.3.10
+
+### Patch Changes
+
+- 9172fed: feat: bump LITS 0.8.2
+- 78ccde7: feat: use llamaindex chat-ui for nextjs frontend
+
+## 0.3.9
+
+### Patch Changes
+
+- ed59927: Add form filling use case (Python)
+
+## 0.3.8
+
+### Patch Changes
+
+- 4a83469: Add multi-agent financial report for Typescript (and update LITS to 0.7.10)
+
+## 0.3.7
+
+### Patch Changes
+
+- fa80378: DocumentInfo working with relative URLs
+
+## 0.3.6
+
+### Patch Changes
+
+- 0182368: Fix the streaming issue to prevent the UI from hanging.
+
+## 0.3.5
+
+### Patch Changes
+
+- 2209409: Add financial report as the default use case in the multi-agent template (Python).
+
 ## 0.3.4
 
 ### Patch Changes

create-app.ts

@@ -7,7 +7,6 @@ import { getOnline } from "./helpers/is-online";
 import { isWriteable } from "./helpers/is-writeable";
 import { makeDir } from "./helpers/make-dir";
 
-import fs from "fs";
 import terminalLink from "terminal-link";
 import type { InstallTemplateArgs, TemplateObservability } from "./helpers";
 import { installTemplate } from "./helpers";
@@ -35,12 +34,13 @@ export async function createApp({
   communityProjectConfig,
   llamapack,
   vectorDb,
-  externalPort,
+  port,
   postInstallAction,
   dataSources,
   tools,
   useLlamaParse,
   observability,
+  agents,
 }: InstallAppArgs): Promise<void> {
   const root = path.resolve(appPath);
 
@@ -80,36 +80,29 @@ export async function createApp({
     communityProjectConfig,
     llamapack,
     vectorDb,
-    externalPort,
+    port,
     postInstallAction,
     dataSources,
     tools,
     useLlamaParse,
     observability,
+    agents,
   };
 
-  if (frontend) {
-    // install backend
-    const backendRoot = path.join(root, "backend");
-    await makeDir(backendRoot);
-    await installTemplate({ ...args, root: backendRoot, backend: true });
+  // Install backend
+  await installTemplate({ ...args, backend: true });
+
+  if (frontend && framework === "fastapi") {
     // install frontend
-    const frontendRoot = path.join(root, "frontend");
+    const frontendRoot = path.join(root, ".frontend");
     await makeDir(frontendRoot);
     await installTemplate({
       ...args,
       root: frontendRoot,
       framework: "nextjs",
-      customApiPath: `http://localhost:${externalPort ?? 8000}/api/chat`,
+      customApiPath: `http://localhost:${port ?? 8000}/api/chat`,
       backend: false,
     });
-    // copy readme for fullstack
-    await fs.promises.copyFile(
-      path.join(templatesDir, "README-fullstack.md"),
-      path.join(root, "README.md"),
-    );
-  } else {
-    await installTemplate({ ...args, backend: true });
   }
 
   await writeDevcontainer(root, templatesDir, framework, frontend);

e2e/python/resolve_dependencies.spec.ts

@@ -63,7 +63,6 @@ if (
             vectorDb,
             tools: "none",
             port: 3000,
-            externalPort: 8000,
             postInstallAction: "none",
             templateUI: undefined,
             appType: "--no-frontend",
@@ -101,7 +100,6 @@ if (
             vectorDb: "none",
             tools: tool,
             port: 3000,
-            externalPort: 8000,
             postInstallAction: "none",
             templateUI: undefined,
             appType: "--no-frontend",
@@ -135,7 +133,6 @@ if (
             vectorDb: "none",
             tools: "none",
             port: 3000,
-            externalPort: 8000,
             postInstallAction: "none",
             templateUI: undefined,
             appType: "--no-frontend",
@@ -169,7 +166,6 @@ if (
             vectorDb: "none",
             tools: "none",
             port: 3000,
-            externalPort: 8000,
             postInstallAction: "none",
             templateUI: undefined,
             appType: "--no-frontend",

e2e/shared/extractor_template.spec.ts

@@ -20,8 +20,7 @@ if (
   dataSource === "--example-file"
 ) {
   test.describe("Test extractor template", async () => {
-    let frontendPort: number;
-    let backendPort: number;
+    let appPort: number;
     let name: string;
     let appProcess: ChildProcess;
     let cwd: string;
@@ -29,16 +28,14 @@ if (
     // Create extractor app
     test.beforeAll(async () => {
       cwd = await createTestDir();
-      frontendPort = Math.floor(Math.random() * 10000) + 10000;
-      backendPort = frontendPort + 1;
+      appPort = Math.floor(Math.random() * 10000) + 10000;
       const result = await runCreateLlama({
         cwd,
         templateType: "extractor",
         templateFramework: "fastapi",
         dataSource: "--example-file",
         vectorDb: "none",
-        port: frontendPort,
-        externalPort: backendPort,
+        port: appPort,
         postInstallAction: "runApp",
       });
       name = result.projectName;
@@ -54,7 +51,7 @@ if (
       expect(dirExists).toBeTruthy();
     });
     test("Frontend should have a title", async ({ page }) => {
-      await page.goto(`http://localhost:${frontendPort}`);
+      await page.goto(`http://localhost:${appPort}`);
       await expect(page.getByText("Built by LlamaIndex")).toBeVisible({
         timeout: 2000 * 60,
       });

e2e/shared/multiagent_template.spec.ts

@@ -16,70 +16,82 @@ const templateFramework: TemplateFramework = process.env.FRAMEWORK
 const dataSource: string = "--example-file";
 const templateUI: TemplateUI = "shadcn";
 const templatePostInstallAction: TemplatePostInstallAction = "runApp";
-const appType: AppType = templateFramework === "nextjs" ? "" : "--frontend";
+const appType: AppType = templateFramework === "fastapi" ? "--frontend" : "";
 const userMessage = "Write a blog post about physical standards for letters";
+const templateAgents = ["financial_report", "blog", "form_filling"];
 
-test.describe(`Test multiagent template ${templateFramework} ${dataSource} ${templateUI} ${appType} ${templatePostInstallAction}`, async () => {
-  test.skip(
-    process.platform !== "linux" || process.env.DATASOURCE === "--no-files",
-    "The multiagent template currently only works with files. We also only run on Linux to speed up tests.",
-  );
-  let port: number;
-  let externalPort: number;
-  let cwd: string;
-  let name: string;
-  let appProcess: ChildProcess;
-  // Only test without using vector db for now
-  const vectorDb = "none";
-
-  test.beforeAll(async () => {
-    port = Math.floor(Math.random() * 10000) + 10000;
-    externalPort = port + 1;
-    cwd = await createTestDir();
-    const result = await runCreateLlama({
-      cwd,
-      templateType: "multiagent",
-      templateFramework,
-      dataSource,
-      vectorDb,
-      port,
-      externalPort,
-      postInstallAction: templatePostInstallAction,
-      templateUI,
-      appType,
-    });
-    name = result.projectName;
-    appProcess = result.appProcess;
-  });
-
-  test("App folder should exist", async () => {
-    const dirExists = fs.existsSync(path.join(cwd, name));
-    expect(dirExists).toBeTruthy();
-  });
-
-  test("Frontend should have a title", async ({ page }) => {
-    await page.goto(`http://localhost:${port}`);
-    await expect(page.getByText("Built by LlamaIndex")).toBeVisible();
-  });
-
-  test("Frontend should be able to submit a message and receive the start of a streamed response", async ({
-    page,
-  }) => {
-    await page.goto(`http://localhost:${port}`);
-    await page.fill("form textarea", userMessage);
-
-    const responsePromise = page.waitForResponse((res) =>
-      res.url().includes("/api/chat"),
+for (const agents of templateAgents) {
+  test.describe(`Test multiagent template ${agents} ${templateFramework} ${dataSource} ${templateUI} ${appType} ${templatePostInstallAction}`, async () => {
+    test.skip(
+      process.platform !== "linux" || process.env.DATASOURCE === "--no-files",
+      "The multiagent template currently only works with files. We also only run on Linux to speed up tests.",
     );
+    let port: number;
+    let cwd: string;
+    let name: string;
+    let appProcess: ChildProcess;
+    // Only test without using vector db for now
+    const vectorDb = "none";
 
-    await page.click("form button[type=submit]");
+    test.beforeAll(async () => {
+      port = Math.floor(Math.random() * 10000) + 10000;
+      cwd = await createTestDir();
+      const result = await runCreateLlama({
+        cwd,
+        templateType: "multiagent",
+        templateFramework,
+        dataSource,
+        vectorDb,
+        port,
+        postInstallAction: templatePostInstallAction,
+        templateUI,
+        appType,
+        agents,
+      });
+      name = result.projectName;
+      appProcess = result.appProcess;
+    });
 
-    const response = await responsePromise;
-    expect(response.ok()).toBeTruthy();
+    test("App folder should exist", async () => {
+      const dirExists = fs.existsSync(path.join(cwd, name));
+      expect(dirExists).toBeTruthy();
+    });
+
+    test("Frontend should have a title", async ({ page }) => {
+      test.skip(
+        templatePostInstallAction !== "runApp" ||
+          templateFramework === "express",
+      );
+      await page.goto(`http://localhost:${port}`);
+      await expect(page.getByText("Built by LlamaIndex")).toBeVisible();
+    });
+
+    test("Frontend should be able to submit a message and receive the start of a streamed response", async ({
+      page,
+    }) => {
+      test.skip(
+        templatePostInstallAction !== "runApp" ||
+          agents === "financial_report" ||
+          agents === "form_filling" ||
+          templateFramework === "express",
+        "Skip chat tests for financial report and form filling.",
+      );
+      await page.goto(`http://localhost:${port}`);
+      await page.fill("form textarea", userMessage);
+
+      const responsePromise = page.waitForResponse((res) =>
+        res.url().includes("/api/chat"),
+      );
+
+      await page.click("form button[type=submit]");
+
+      const response = await responsePromise;
+      expect(response.ok()).toBeTruthy();
+    });
+
+    // clean processes
+    test.afterAll(async () => {
+      appProcess?.kill();
+    });
   });
-
-  // clean processes
-  test.afterAll(async () => {
-    appProcess?.kill();
-  });
-});
+}

e2e/shared/streaming_template.spec.ts

@@ -22,7 +22,7 @@ const templatePostInstallAction: TemplatePostInstallAction = "runApp";
 const llamaCloudProjectName = "create-llama";
 const llamaCloudIndexName = "e2e-test";
 
-const appType: AppType = templateFramework === "nextjs" ? "" : "--frontend";
+const appType: AppType = templateFramework === "fastapi" ? "--frontend" : "";
 const userMessage =
   dataSource !== "--no-files" ? "Physical standard for letters" : "Hello";
 
@@ -35,7 +35,6 @@ test.describe(`Test streaming template ${templateFramework} ${dataSource} ${temp
   }
 
   let port: number;
-  let externalPort: number;
   let cwd: string;
   let name: string;
   let appProcess: ChildProcess;
@@ -44,7 +43,6 @@ test.describe(`Test streaming template ${templateFramework} ${dataSource} ${temp
 
   test.beforeAll(async () => {
     port = Math.floor(Math.random() * 10000) + 10000;
-    externalPort = port + 1;
     cwd = await createTestDir();
     const result = await runCreateLlama({
       cwd,
@@ -53,7 +51,6 @@ test.describe(`Test streaming template ${templateFramework} ${dataSource} ${temp
       dataSource,
       vectorDb,
       port,
-      externalPort,
       postInstallAction: templatePostInstallAction,
       templateUI,
       appType,
@@ -68,8 +65,11 @@ test.describe(`Test streaming template ${templateFramework} ${dataSource} ${temp
     const dirExists = fs.existsSync(path.join(cwd, name));
     expect(dirExists).toBeTruthy();
   });
+
   test("Frontend should have a title", async ({ page }) => {
-    test.skip(templatePostInstallAction !== "runApp");
+    test.skip(
+      templatePostInstallAction !== "runApp" || templateFramework === "express",
+    );
     await page.goto(`http://localhost:${port}`);
     await expect(page.getByText("Built by LlamaIndex")).toBeVisible();
   });
@@ -77,7 +77,9 @@ test.describe(`Test streaming template ${templateFramework} ${dataSource} ${temp
   test("Frontend should be able to submit a message and receive a response", async ({
     page,
   }) => {
-    test.skip(templatePostInstallAction !== "runApp");
+    test.skip(
+      templatePostInstallAction !== "runApp" || templateFramework === "express",
+    );
     await page.goto(`http://localhost:${port}`);
     await page.fill("form textarea", userMessage);
     const [response] = await Promise.all([
@@ -102,7 +104,7 @@ test.describe(`Test streaming template ${templateFramework} ${dataSource} ${temp
     test.skip(templatePostInstallAction !== "runApp");
     test.skip(templateFramework === "nextjs");
     const response = await request.post(
-      `http://localhost:${externalPort}/api/chat/request`,
+      `http://localhost:${port}/api/chat/request`,
       {
         data: {
           messages: [

e2e/typescript/resolve_dependencies.spec.ts

@@ -56,7 +56,6 @@ test.describe("Test resolve TS dependencies", () => {
       dataSource: dataSource,
       vectorDb: vectorDb,
       port: 3000,
-      externalPort: 8000,
       postInstallAction: "none",
       templateUI: undefined,
       appType: templateFramework === "nextjs" ? "" : "--no-frontend",

e2e/utils.ts

@@ -25,7 +25,6 @@ export type RunCreateLlamaOptions = {
   dataSource: string;
   vectorDb: TemplateVectorDB;
   port: number;
-  externalPort: number;
   postInstallAction: TemplatePostInstallAction;
   templateUI?: TemplateUI;
   appType?: AppType;
@@ -34,6 +33,7 @@ export type RunCreateLlamaOptions = {
   tools?: string;
   useLlamaParse?: boolean;
   observability?: string;
+  agents?: string;
 };
 
 export async function runCreateLlama({
@@ -43,7 +43,6 @@ export async function runCreateLlama({
   dataSource,
   vectorDb,
   port,
-  externalPort,
   postInstallAction,
   templateUI,
   appType,
@@ -52,6 +51,7 @@ export async function runCreateLlama({
   tools,
   useLlamaParse,
   observability,
+  agents,
 }: RunCreateLlamaOptions): Promise<CreateLlamaResult> {
   if (!process.env.OPENAI_API_KEY || !process.env.LLAMA_CLOUD_API_KEY) {
     throw new Error(
@@ -88,21 +88,15 @@ export async function runCreateLlama({
     ...dataSourceArgs,
     "--vector-db",
     vectorDb,
-    "--open-ai-key",
-    process.env.OPENAI_API_KEY,
     "--use-pnpm",
     "--port",
     port,
-    "--external-port",
-    externalPort,
     "--post-install-action",
     postInstallAction,
     "--tools",
     tools ?? "none",
     "--observability",
     "none",
-    "--llama-cloud-key",
-    process.env.LLAMA_CLOUD_API_KEY,
   ];
 
   if (templateUI) {
@@ -119,6 +113,9 @@ export async function runCreateLlama({
   if (observability) {
     commandArgs.push("--observability", observability);
   }
+  if (templateType === "multiagent" && agents) {
+    commandArgs.push("--agents", agents);
+  }
 
   const command = commandArgs.join(" ");
   console.log(`running command '${command}' in ${cwd}`);
@@ -141,12 +138,7 @@ export async function runCreateLlama({
 
   // Wait for app to start
   if (postInstallAction === "runApp") {
-    await checkAppHasStarted(
-      appType === "--frontend",
-      templateFramework,
-      port,
-      externalPort,
-    );
+    await waitPorts([port]);
   } else if (postInstallAction === "dependencies") {
     await waitForProcess(appProcess, 1000 * 60); // wait 1 min for dependencies to be resolved
   } else {
@@ -166,19 +158,6 @@ export async function createTestDir() {
   return cwd;
 }
 
-// eslint-disable-next-line max-params
-async function checkAppHasStarted(
-  frontend: boolean,
-  framework: TemplateFramework,
-  port: number,
-  externalPort: number,
-) {
-  const portsToWait = frontend
-    ? [port, externalPort]
-    : [framework === "nextjs" ? port : externalPort];
-  await waitPorts(portsToWait);
-}
-
 async function waitPorts(ports: number[]): Promise<void> {
   const waitForPort = async (port: number): Promise<void> => {
     await waitPort({

helpers/datasources.ts

@@ -11,6 +11,25 @@ export const EXAMPLE_FILE: TemplateDataSource = {
   },
 };
 
+export const EXAMPLE_10K_SEC_FILES: TemplateDataSource[] = [
+  {
+    type: "file",
+    config: {
+      url: new URL(
+        "https://s2.q4cdn.com/470004039/files/doc_earnings/2023/q4/filing/_10-K-Q4-2023-As-Filed.pdf",
+      ),
+    },
+  },
+  {
+    type: "file",
+    config: {
+      url: new URL(
+        "https://ir.tesla.com/_flysystem/s3/sec/000162828024002390/tsla-20231231-gen.pdf",
+      ),
+    },
+  },
+];
+
 export function getDataSources(
   files?: string,
   exampleFile?: boolean,

helpers/devcontainer.ts

@@ -5,36 +5,21 @@ import { TemplateFramework } from "./types";
 function renderDevcontainerContent(
   templatesDir: string,
   framework: TemplateFramework,
-  frontend: boolean,
 ) {
   const devcontainerJson: any = JSON.parse(
     fs.readFileSync(path.join(templatesDir, "devcontainer.json"), "utf8"),
   );
 
   // Modify postCreateCommand
-  if (frontend) {
-    devcontainerJson.postCreateCommand =
-      framework === "fastapi"
-        ? "cd backend && poetry install && cd ../frontend && npm install"
-        : "cd backend && npm install && cd ../frontend && npm install";
-  } else {
-    devcontainerJson.postCreateCommand =
-      framework === "fastapi" ? "poetry install" : "npm install";
-  }
+  devcontainerJson.postCreateCommand =
+    framework === "fastapi" ? "poetry install" : "npm install";
 
   // Modify containerEnv
   if (framework === "fastapi") {
-    if (frontend) {
-      devcontainerJson.containerEnv = {
-        ...devcontainerJson.containerEnv,
-        PYTHONPATH: "${PYTHONPATH}:${workspaceFolder}/backend",
-      };
-    } else {
-      devcontainerJson.containerEnv = {
-        ...devcontainerJson.containerEnv,
-        PYTHONPATH: "${PYTHONPATH}:${workspaceFolder}",
-      };
-    }
+    devcontainerJson.containerEnv = {
+      ...devcontainerJson.containerEnv,
+      PYTHONPATH: "${PYTHONPATH}:${workspaceFolder}",
+    };
   }
 
   return JSON.stringify(devcontainerJson, null, 2);
@@ -54,7 +39,6 @@ export const writeDevcontainer = async (
   const devcontainerContent = renderDevcontainerContent(
     templatesDir,
     framework,
-    frontend,
   );
   fs.mkdirSync(devcontainerDir);
   await fs.promises.writeFile(

helpers/env-variables.ts

@@ -217,7 +217,13 @@ Otherwise, use CHROMA_HOST and CHROMA_PORT config above`,
         },
       ];
     default:
-      return [];
+      return [
+        {
+          name: "STORAGE_CACHE_DIR",
+          description: "The directory to store the local storage cache.",
+          value: ".cache",
+        },
+      ];
   }
 };
 
@@ -336,6 +342,20 @@ const getModelEnvs = (modelConfig: ModelConfig): EnvVar[] => {
           },
         ]
       : []),
+    ...(modelConfig.provider === "huggingface"
+      ? [
+          {
+            name: "EMBEDDING_BACKEND",
+            description:
+              "The backend to use for the Sentence Transformers embedding model, either 'torch', 'onnx', or 'openvino'. Defaults to 'onnx'.",
+          },
+          {
+            name: "EMBEDDING_TRUST_REMOTE_CODE",
+            description:
+              "Whether to trust remote code for the embedding model, required for some models with custom code.",
+          },
+        ]
+      : []),
     ...(modelConfig.provider === "t-systems"
       ? [
           {
@@ -533,7 +553,7 @@ export const createBackendEnvFile = async (
     | "framework"
     | "dataSources"
     | "template"
-    | "externalPort"
+    | "port"
     | "tools"
     | "observability"
   >,
@@ -550,7 +570,7 @@ export const createBackendEnvFile = async (
     ...getModelEnvs(opts.modelConfig),
     ...getEngineEnvs(),
     ...getVectorDBEnvs(opts.vectorDb, opts.framework),
-    ...getFrameworkEnvs(opts.framework, opts.externalPort),
+    ...getFrameworkEnvs(opts.framework, opts.port),
     ...getToolEnvs(opts.tools),
     ...getTemplateEnvs(opts.template),
     ...getObservabilityEnvs(opts.observability),

helpers/index.ts

@@ -96,6 +96,12 @@ async function generateContextData(
   }
 }
 
+const downloadFile = async (url: string, destPath: string) => {
+  const response = await fetch(url);
+  const fileBuffer = await response.arrayBuffer();
+  await fsExtra.writeFile(destPath, Buffer.from(fileBuffer));
+};
+
 const prepareContextData = async (
   root: string,
   dataSources: TemplateDataSource[],
@@ -103,12 +109,28 @@ const prepareContextData = async (
   await makeDir(path.join(root, "data"));
   for (const dataSource of dataSources) {
     const dataSourceConfig = dataSource?.config as FileSourceConfig;
-    // Copy local data
-    const dataPath = dataSourceConfig.path;
-
-    const destPath = path.join(root, "data", path.basename(dataPath));
-    console.log("Copying data from path:", dataPath);
-    await fsExtra.copy(dataPath, destPath);
+    // If the path is URLs, download the data and save it to the data directory
+    if ("url" in dataSourceConfig) {
+      console.log(
+        "Downloading file from URL:",
+        dataSourceConfig.url.toString(),
+      );
+      const destPath = path.join(
+        root,
+        "data",
+        path.basename(dataSourceConfig.url.toString()),
+      );
+      await downloadFile(dataSourceConfig.url.toString(), destPath);
+    } else {
+      // Copy local data
+      console.log("Copying data from path:", dataSourceConfig.path);
+      const destPath = path.join(
+        root,
+        "data",
+        path.basename(dataSourceConfig.path),
+      );
+      await fsExtra.copy(dataSourceConfig.path, destPath);
+    }
   }
 };
 

helpers/providers/huggingface.ts

@@ -0,0 +1,68 @@
+import prompts from "prompts";
+import { ModelConfigParams } from ".";
+import { questionHandlers, toChoice } from "../../questions/utils";
+
+const MODELS = ["HuggingFaceH4/zephyr-7b-alpha"];
+type ModelData = {
+  dimensions: number;
+};
+const EMBEDDING_MODELS: Record<string, ModelData> = {
+  "BAAI/bge-small-en-v1.5": { dimensions: 384 },
+  "BAAI/bge-base-en-v1.5": { dimensions: 768 },
+  "BAAI/bge-large-en-v1.5": { dimensions: 1024 },
+  "sentence-transformers/all-MiniLM-L6-v2": { dimensions: 384 },
+  "sentence-transformers/all-mpnet-base-v2": { dimensions: 768 },
+  "intfloat/multilingual-e5-large": { dimensions: 1024 },
+  "mixedbread-ai/mxbai-embed-large-v1": { dimensions: 1024 },
+  "nomic-ai/nomic-embed-text-v1.5": { dimensions: 768 },
+};
+
+const DEFAULT_MODEL = MODELS[0];
+const DEFAULT_EMBEDDING_MODEL = Object.keys(EMBEDDING_MODELS)[0];
+const DEFAULT_DIMENSIONS = Object.values(EMBEDDING_MODELS)[0].dimensions;
+
+type HuggingfaceQuestionsParams = {
+  askModels: boolean;
+};
+
+export async function askHuggingfaceQuestions({
+  askModels,
+}: HuggingfaceQuestionsParams): Promise<ModelConfigParams> {
+  const config: ModelConfigParams = {
+    model: DEFAULT_MODEL,
+    embeddingModel: DEFAULT_EMBEDDING_MODEL,
+    dimensions: DEFAULT_DIMENSIONS,
+    isConfigured(): boolean {
+      return true;
+    },
+  };
+
+  if (askModels) {
+    const { model } = await prompts(
+      {
+        type: "select",
+        name: "model",
+        message: "Which Hugging Face model would you like to use?",
+        choices: MODELS.map(toChoice),
+        initial: 0,
+      },
+      questionHandlers,
+    );
+    config.model = model;
+
+    const { embeddingModel } = await prompts(
+      {
+        type: "select",
+        name: "embeddingModel",
+        message: "Which embedding model would you like to use?",
+        choices: Object.keys(EMBEDDING_MODELS).map(toChoice),
+        initial: 0,
+      },
+      questionHandlers,
+    );
+    config.embeddingModel = embeddingModel;
+    config.dimensions = EMBEDDING_MODELS[embeddingModel].dimensions;
+  }
+
+  return config;
+}

helpers/providers/index.ts

@@ -5,6 +5,7 @@ import { askAnthropicQuestions } from "./anthropic";
 import { askAzureQuestions } from "./azure";
 import { askGeminiQuestions } from "./gemini";
 import { askGroqQuestions } from "./groq";
+import { askHuggingfaceQuestions } from "./huggingface";
 import { askLLMHubQuestions } from "./llmhub";
 import { askMistralQuestions } from "./mistral";
 import { askOllamaQuestions } from "./ollama";
@@ -39,6 +40,7 @@ export async function askModelConfig({
 
     if (framework === "fastapi") {
       choices.push({ title: "T-Systems", value: "t-systems" });
+      choices.push({ title: "Huggingface", value: "huggingface" });
     }
     const { provider } = await prompts(
       {
@@ -76,6 +78,9 @@ export async function askModelConfig({
     case "t-systems":
       modelConfig = await askLLMHubQuestions({ askModels });
       break;
+    case "huggingface":
+      modelConfig = await askHuggingfaceQuestions({ askModels });
+      break;
     default:
       modelConfig = await askOpenAIQuestions({
         openAiKey,

helpers/providers/openai.ts

@@ -3,6 +3,7 @@ import ora from "ora";
 import { red } from "picocolors";
 import prompts from "prompts";
 import { ModelConfigParams, ModelConfigQuestionsParams } from ".";
+import { isCI } from "../../questions";
 import { questionHandlers } from "../../questions/utils";
 
 const OPENAI_API_URL = "https://api.openai.com/v1";
@@ -30,7 +31,7 @@ export async function askOpenAIQuestions({
     },
   };
 
-  if (!config.apiKey) {
+  if (!config.apiKey && !isCI) {
     const { key } = await prompts(
       {
         type: "text",

helpers/python.ts

@@ -20,6 +20,7 @@ interface Dependency {
   name: string;
   version?: string;
   extras?: string[];
+  constraints?: Record<string, string>;
 }
 
 const getAdditionalDependencies = (
@@ -51,6 +52,9 @@ const getAdditionalDependencies = (
       dependencies.push({
         name: "llama-index-vector-stores-pinecone",
         version: "^0.2.1",
+        constraints: {
+          python: ">=3.11,<3.13",
+        },
       });
       break;
     }
@@ -76,6 +80,9 @@ const getAdditionalDependencies = (
       dependencies.push({
         name: "llama-index-vector-stores-qdrant",
         version: "^0.3.0",
+        constraints: {
+          python: ">=3.11,<3.13",
+        },
       });
       break;
     }
@@ -234,6 +241,21 @@ const getAdditionalDependencies = (
         version: "0.2.4",
       });
       break;
+    case "huggingface":
+      dependencies.push({
+        name: "llama-index-llms-huggingface",
+        version: "^0.3.5",
+      });
+      dependencies.push({
+        name: "llama-index-embeddings-huggingface",
+        version: "^0.3.1",
+      });
+      dependencies.push({
+        name: "optimum",
+        version: "^1.23.3",
+        extras: ["onnxruntime"],
+      });
+      break;
     case "t-systems":
       dependencies.push({
         name: "llama-index-agent-openai",
@@ -264,14 +286,19 @@ const mergePoetryDependencies = (
     value.version = dependency.version ?? value.version;
     value.extras = dependency.extras ?? value.extras;
 
+    // Merge constraints if they exist
+    if (dependency.constraints) {
+      value = { ...value, ...dependency.constraints };
+    }
+
     if (value.version === undefined) {
       throw new Error(
         `Dependency "${dependency.name}" is missing attribute "version"!`,
       );
     }
 
-    // Serialize separately only if extras are provided
-    if (value.extras && value.extras.length > 0) {
+    // Serialize as object if there are any additional properties
+    if (Object.keys(value).length > 1) {
       existingDependencies[dependency.name] = value;
     } else {
       // Otherwise, serialize just the version string
@@ -362,6 +389,7 @@ export const installPythonTemplate = async ({
   postInstallAction,
   observability,
   modelConfig,
+  agents,
 }: Pick<
   InstallTemplateArgs,
   | "root"
@@ -373,6 +401,7 @@ export const installPythonTemplate = async ({
   | "postInstallAction"
   | "observability"
   | "modelConfig"
+  | "agents"
 >) => {
   console.log("\nInitializing Python project with template:", template, "\n");
   let templatePath;
@@ -443,6 +472,24 @@ export const installPythonTemplate = async ({
       cwd: path.join(compPath, "engines", "python", engine),
     });
 
+    // Copy agent code
+    if (template === "multiagent") {
+      if (agents) {
+        await copy("**", path.join(root), {
+          parents: true,
+          cwd: path.join(compPath, "agents", "python", agents),
+          rename: assetRelocator,
+        });
+      } else {
+        console.log(
+          red(
+            "There is no agent selected for multi-agent template. Please pick an agent to use via --agents flag.",
+          ),
+        );
+        process.exit(1);
+      }
+    }
+
     // Copy router code
     await copyRouterCode(root, tools ?? []);
   }
@@ -478,6 +525,9 @@ export const installPythonTemplate = async ({
       addOnDependencies.push({
         name: "llama-index-callbacks-arize-phoenix",
         version: "^0.2.1",
+        constraints: {
+          python: ">=3.11,<3.13",
+        },
       });
     }
 

helpers/run-app.ts

@@ -1,40 +1,39 @@
-import { ChildProcess, SpawnOptions, spawn } from "child_process";
-import path from "path";
+import { SpawnOptions, spawn } from "child_process";
 import { TemplateFramework } from "./types";
 
 const createProcess = (
   command: string,
   args: string[],
   options: SpawnOptions,
-) => {
-  return spawn(command, args, {
-    ...options,
-    shell: true,
-  })
-    .on("exit", function (code) {
-      if (code !== 0) {
-        console.log(`Child process exited with code=${code}`);
-        process.exit(1);
-      }
+): Promise<void> => {
+  return new Promise((resolve, reject) => {
+    spawn(command, args, {
+      ...options,
+      shell: true,
     })
-    .on("error", function (err) {
-      console.log("Error when running chill process: ", err);
-      process.exit(1);
-    });
+      .on("exit", function (code) {
+        if (code !== 0) {
+          console.log(`Child process exited with code=${code}`);
+          reject(code);
+        } else {
+          resolve();
+        }
+      })
+      .on("error", function (err) {
+        console.log("Error when running child process: ", err);
+        reject(err);
+      });
+  });
 };
 
-export function runReflexApp(
-  appPath: string,
-  frontendPort?: number,
-  backendPort?: number,
-) {
-  const commandArgs = ["run", "reflex", "run"];
-  if (frontendPort) {
-    commandArgs.push("--frontend-port", frontendPort.toString());
-  }
-  if (backendPort) {
-    commandArgs.push("--backend-port", backendPort.toString());
-  }
+export function runReflexApp(appPath: string, port: number) {
+  const commandArgs = [
+    "run",
+    "reflex",
+    "run",
+    "--frontend-port",
+    port.toString(),
+  ];
   return createProcess("poetry", commandArgs, {
     stdio: "inherit",
     cwd: appPath,
@@ -42,11 +41,10 @@ export function runReflexApp(
 }
 
 export function runFastAPIApp(appPath: string, port: number) {
-  const commandArgs = ["run", "uvicorn", "main:app", "--port=" + port];
-
-  return createProcess("poetry", commandArgs, {
+  return createProcess("poetry", ["run", "dev"], {
     stdio: "inherit",
     cwd: appPath,
+    env: { ...process.env, APP_PORT: `${port}` },
   });
 }
 
@@ -61,39 +59,23 @@ export function runTSApp(appPath: string, port: number) {
 export async function runApp(
   appPath: string,
   template: string,
-  frontend: boolean,
   framework: TemplateFramework,
   port?: number,
-  externalPort?: number,
-): Promise<any> {
-  const processes: ChildProcess[] = [];
+): Promise<void> {
+  try {
+    // Start the app
+    const defaultPort =
+      framework === "nextjs" || template === "extractor" ? 3000 : 8000;
 
-  // Callback to kill all sub processes if the main process is killed
-  process.on("exit", () => {
-    console.log("Killing app processes...");
-    processes.forEach((p) => p.kill());
-  });
-
-  // Default sub app paths
-  const backendPath = path.join(appPath, "backend");
-  const frontendPath = path.join(appPath, "frontend");
-
-  if (template === "extractor") {
-    processes.push(runReflexApp(appPath, port, externalPort));
+    const appRunner =
+      template === "extractor"
+        ? runReflexApp
+        : framework === "fastapi"
+          ? runFastAPIApp
+          : runTSApp;
+    await appRunner(appPath, port || defaultPort);
+  } catch (error) {
+    console.error("Failed to run app:", error);
+    throw error;
   }
-  if (template === "streaming" || template === "multiagent") {
-    if (framework === "fastapi" || framework === "express") {
-      const backendRunner = framework === "fastapi" ? runFastAPIApp : runTSApp;
-      if (frontend) {
-        processes.push(backendRunner(backendPath, externalPort || 8000));
-        processes.push(runTSApp(frontendPath, port || 3000));
-      } else {
-        processes.push(backendRunner(appPath, externalPort || 8000));
-      }
-    } else if (framework === "nextjs") {
-      processes.push(runTSApp(appPath, port || 3000));
-    }
-  }
-
-  return Promise.all(processes);
 }

helpers/tools.ts

@@ -62,7 +62,7 @@ export const supportedTools: Tool[] = [
     dependencies: [
       {
         name: "duckduckgo-search",
-        version: "6.1.7",
+        version: "^6.3.5",
       },
     ],
     supportedFrameworks: ["fastapi", "nextjs", "express"],
@@ -267,6 +267,22 @@ For better results, you can specify the region parameter to get results from a s
       },
     ],
   },
+  {
+    display: "Form Filling",
+    name: "form_filling",
+    supportedFrameworks: ["fastapi"],
+    type: ToolType.LOCAL,
+    dependencies: [
+      {
+        name: "pandas",
+        version: "^2.2.3",
+      },
+      {
+        name: "tabulate",
+        version: "^0.9.0",
+      },
+    ],
+  },
 ];
 
 export const getTool = (toolName: string): Tool | undefined => {

helpers/types.ts

@@ -9,6 +9,7 @@ export type ModelProvider =
   | "gemini"
   | "mistral"
   | "azure-openai"
+  | "huggingface"
   | "t-systems";
 export type ModelConfig = {
   provider: ModelProvider;
@@ -48,10 +49,15 @@ export type TemplateDataSource = {
 };
 export type TemplateDataSourceType = "file" | "web" | "db";
 export type TemplateObservability = "none" | "traceloop" | "llamatrace";
+export type TemplateAgents = "financial_report" | "blog" | "form_filling";
 // Config for both file and folder
-export type FileSourceConfig = {
-  path: string;
-};
+export type FileSourceConfig =
+  | {
+      path: string;
+    }
+  | {
+      url: URL;
+    };
 export type WebSourceConfig = {
   baseUrl?: string;
   prefix?: string;
@@ -90,8 +96,9 @@ export interface InstallTemplateArgs {
   communityProjectConfig?: CommunityProjectConfig;
   llamapack?: string;
   vectorDb?: TemplateVectorDB;
-  externalPort?: number;
+  port?: number;
   postInstallAction?: TemplatePostInstallAction;
   tools?: Tool[];
   observability?: TemplateObservability;
+  agents?: TemplateAgents;
 }

helpers/typescript.ts

@@ -1,7 +1,7 @@
 import fs from "fs/promises";
 import os from "os";
 import path from "path";
-import { bold, cyan, yellow } from "picocolors";
+import { bold, cyan, red, yellow } from "picocolors";
 import { assetRelocator, copy } from "../helpers/copy";
 import { callPackageManager } from "../helpers/install";
 import { templatesDir } from "./dir";
@@ -26,6 +26,7 @@ export const installTSTemplate = async ({
   tools,
   dataSources,
   useLlamaParse,
+  agents,
 }: InstallTemplateArgs & { backend: boolean }) => {
   console.log(bold(`Using ${packageManager}.`));
 
@@ -57,11 +58,9 @@ export const installTSTemplate = async ({
       console.log("\nUsing static site generation\n");
     } else {
       if (vectorDb === "milvus") {
-        nextConfigJson.experimental.serverComponentsExternalPackages =
-          nextConfigJson.experimental.serverComponentsExternalPackages ?? [];
-        nextConfigJson.experimental.serverComponentsExternalPackages.push(
-          "@zilliz/milvus2-sdk-node",
-        );
+        nextConfigJson.serverExternalPackages =
+          nextConfigJson.serverExternalPackages ?? [];
+        nextConfigJson.serverExternalPackages.push("@zilliz/milvus2-sdk-node");
       }
     }
     await fs.writeFile(
@@ -132,6 +131,34 @@ export const installTSTemplate = async ({
       cwd: path.join(multiagentPath, "workflow"),
     });
 
+    // Copy agents use case code for multiagent template
+    if (agents) {
+      console.log("\nCopying agent:", agents, "\n");
+      const useCasePath = path.join(compPath, "agents", "typescript", agents);
+      const agentsCodePath = path.join(useCasePath, "workflow");
+
+      // Copy agent codes
+      await copy("**", path.join(root, relativeEngineDestPath, "workflow"), {
+        parents: true,
+        cwd: agentsCodePath,
+        rename: assetRelocator,
+      });
+
+      // Copy use case files to project root
+      await copy("*.*", path.join(root), {
+        parents: true,
+        cwd: useCasePath,
+        rename: assetRelocator,
+      });
+    } else {
+      console.log(
+        red(
+          "There is no agent selected for multi-agent template. Please pick an agent to use via --agents flag.",
+        ),
+      );
+      process.exit(1);
+    }
+
     if (framework === "nextjs") {
       // patch route.ts file
       await copy("**", path.join(root, relativeEngineDestPath), {
@@ -214,7 +241,10 @@ export const installTSTemplate = async ({
     vectorDb,
   });
 
-  if (postInstallAction === "runApp" || postInstallAction === "dependencies") {
+  if (
+    backend &&
+    (postInstallAction === "runApp" || postInstallAction === "dependencies")
+  ) {
     await installTSDependencies(packageJson, packageManager, isOnline);
   }
 

index.ts

@@ -134,13 +134,6 @@ const program = new Command(packageJson.name)
     `
 
   Select UI port.
-`,
-  )
-  .option(
-    "--external-port <external>",
-    `
-
-  Select external port.
 `,
   )
   .option(
@@ -208,6 +201,13 @@ const program = new Command(packageJson.name)
 `,
     false,
   )
+  .option(
+    "--agents <agents>",
+    `
+
+  Select which agents to use for the multi-agent template (e.g: financial_report, blog).
+`,
+  )
   .allowUnknownOption()
   .parse(process.argv);
 
@@ -326,7 +326,7 @@ async function run(): Promise<void> {
     ...answers,
     appPath: resolvedProjectPath,
     packageManager,
-    externalPort: options.externalPort,
+    port: options.port,
   });
 
   if (answers.postInstallAction === "VSCode") {
@@ -355,14 +355,7 @@ Please check ${cyan(
     }
   } else if (answers.postInstallAction === "runApp") {
     console.log(`Running app in ${root}...`);
-    await runApp(
-      root,
-      answers.template,
-      answers.frontend,
-      answers.framework,
-      options.port,
-      options.externalPort,
-    );
+    await runApp(root, answers.template, answers.framework, options.port);
   }
 }
 

package.json

@@ -1,6 +1,6 @@
 {
   "name": "create-llama",
-  "version": "0.3.4",
+  "version": "0.3.15",
   "description": "Create LlamaIndex-powered apps with one command",
   "keywords": [
     "rag",

questions/index.ts

@@ -4,10 +4,12 @@ import { askProQuestions } from "./questions";
 import { askSimpleQuestions } from "./simple";
 import { QuestionArgs, QuestionResults } from "./types";
 
+export const isCI = ciInfo.isCI || process.env.PLAYWRIGHT_TEST === "1";
+
 export const askQuestions = async (
   args: QuestionArgs,
 ): Promise<QuestionResults> => {
-  if (ciInfo.isCI || process.env.PLAYWRIGHT_TEST === "1") {
+  if (isCI) {
     return await getCIQuestionResults(args);
   } else if (args.pro) {
     // TODO: refactor pro questions to return a result object

questions/questions.ts

@@ -1,5 +1,6 @@
-import { blue, green } from "picocolors";
+import { blue } from "picocolors";
 import prompts from "prompts";
+import { isCI } from ".";
 import { COMMUNITY_OWNER, COMMUNITY_REPO } from "../helpers/constant";
 import { EXAMPLE_FILE } from "../helpers/datasources";
 import { getAvailableLlamapackOptions } from "../helpers/llama-pack";
@@ -122,24 +123,17 @@ export const askProQuestions = async (program: QuestionArgs) => {
   }
 
   if (
-    (program.framework === "express" || program.framework === "fastapi") &&
+    program.framework === "fastapi" &&
     (program.template === "streaming" || program.template === "multiagent")
   ) {
     // if a backend-only framework is selected, ask whether we should create a frontend
     if (program.frontend === undefined) {
       const styledNextJS = blue("NextJS");
-      const styledBackend = green(
-        program.framework === "express"
-          ? "Express "
-          : program.framework === "fastapi"
-            ? "FastAPI (Python) "
-            : "",
-      );
       const { frontend } = await prompts({
         onState: onPromptState,
         type: "toggle",
         name: "frontend",
-        message: `Would you like to generate a ${styledNextJS} frontend for your ${styledBackend}backend?`,
+        message: `Would you like to generate a ${styledNextJS} frontend for your FastAPI backend?`,
         initial: false,
         active: "Yes",
         inactive: "No",
@@ -177,6 +171,34 @@ export const askProQuestions = async (program: QuestionArgs) => {
     program.observability = observability;
   }
 
+  // Ask agents
+  if (program.template === "multiagent" && !program.agents) {
+    const { agents } = await prompts(
+      {
+        type: "select",
+        name: "agents",
+        message: "Which agents would you like to use?",
+        choices: [
+          {
+            title: "Financial report (generate a financial report)",
+            value: "financial_report",
+          },
+          {
+            title: "Form filling (fill missing value in a CSV file)",
+            value: "form_filling",
+          },
+          {
+            title: "Blog writer (Write a blog post)",
+            value: "blog_writer",
+          },
+        ],
+        initial: 0,
+      },
+      questionHandlers,
+    );
+    program.agents = agents;
+  }
+
   if (!program.modelConfig) {
     const modelConfig = await askModelConfig({
       openAiKey: program.openAiKey,
@@ -358,7 +380,7 @@ export const askProQuestions = async (program: QuestionArgs) => {
 
   // Ask for LlamaCloud API key when using a LlamaCloud index or LlamaParse
   if (isUsingLlamaCloud || program.useLlamaParse) {
-    if (!program.llamaCloudKey) {
+    if (!program.llamaCloudKey && !isCI) {
       // if already set, don't ask again
       // Ask for LlamaCloud API key
       const { llamaCloudKey } = await prompts(

questions/simple.ts

@@ -1,5 +1,5 @@
 import prompts from "prompts";
-import { EXAMPLE_FILE } from "../helpers/datasources";
+import { EXAMPLE_10K_SEC_FILES, EXAMPLE_FILE } from "../helpers/datasources";
 import { askModelConfig } from "../helpers/providers";
 import { getTools } from "../helpers/tools";
 import { ModelConfig, TemplateFramework } from "../helpers/types";
@@ -9,7 +9,8 @@ import { askPostInstallAction, questionHandlers } from "./utils";
 type AppType =
   | "rag"
   | "code_artifact"
-  | "multiagent"
+  | "financial_report_agent"
+  | "form_filling"
   | "extractor"
   | "data_scientist";
 
@@ -31,9 +32,16 @@ export const askSimpleQuestions = async (
       choices: [
         { title: "Agentic RAG", value: "rag" },
         { title: "Data Scientist", value: "data_scientist" },
+        {
+          title: "Financial Report Generator (using Workflows)",
+          value: "financial_report_agent",
+        },
+        {
+          title: "Form Filler (using Workflows)",
+          value: "form_filling",
+        },
         { title: "Code Artifact Agent", value: "code_artifact" },
-        { title: "Multi-Agent Report Gen", value: "multiagent" },
-        { title: "Structured extraction", value: "extractor" },
+        { title: "Information Extractor", value: "extractor" },
       ],
     },
     questionHandlers,
@@ -42,6 +50,7 @@ export const askSimpleQuestions = async (
   let language: TemplateFramework = "fastapi";
   let llamaCloudKey = args.llamaCloudKey;
   let useLlamaCloud = false;
+
   if (appType !== "extractor") {
     const { language: newLanguage } = await prompts(
       {
@@ -113,7 +122,10 @@ const convertAnswers = async (
   };
   const lookup: Record<
     AppType,
-    Pick<QuestionResults, "template" | "tools" | "frontend" | "dataSources"> & {
+    Pick<
+      QuestionResults,
+      "template" | "tools" | "frontend" | "dataSources" | "agents"
+    > & {
       modelConfig?: ModelConfig;
     }
   > = {
@@ -137,16 +149,21 @@ const convertAnswers = async (
       dataSources: [],
       modelConfig: MODEL_GPT4o,
     },
-    multiagent: {
+    financial_report_agent: {
       template: "multiagent",
-      tools: getTools([
-        "document_generator",
-        "wikipedia.WikipediaToolSpec",
-        "duckduckgo",
-        "img_gen",
-      ]),
+      agents: "financial_report",
+      tools: getTools(["document_generator", "interpreter"]),
+      dataSources: EXAMPLE_10K_SEC_FILES,
       frontend: true,
-      dataSources: [EXAMPLE_FILE],
+      modelConfig: MODEL_GPT4o,
+    },
+    form_filling: {
+      template: "multiagent",
+      agents: "form_filling",
+      tools: getTools(["form_filling"]),
+      dataSources: EXAMPLE_10K_SEC_FILES,
+      frontend: true,
+      modelConfig: MODEL_GPT4o,
     },
     extractor: {
       template: "extractor",

questions/types.ts

@@ -2,7 +2,7 @@ import { InstallAppArgs } from "../create-app";
 
 export type QuestionResults = Omit<
   InstallAppArgs,
-  "appPath" | "packageManager" | "externalPort"
+  "appPath" | "packageManager"
 >;
 
 export type PureQuestionArgs = {

templates/.gitignore

@@ -1,3 +0,0 @@
-__pycache__
-poetry.lock
-storage

templates/README-fullstack.md

@@ -1,18 +0,0 @@
-This is a [LlamaIndex](https://www.llamaindex.ai/) project bootstrapped with [`create-llama`](https://github.com/run-llama/LlamaIndexTS/tree/main/packages/create-llama).
-
-## Getting Started
-
-First, startup the backend as described in the [backend README](./backend/README.md).
-
-Second, run the development server of the frontend as described in the [frontend README](./frontend/README.md).
-
-Open [http://localhost:3000](http://localhost:3000) with your browser to see the result.
-
-## Learn More
-
-To learn more about LlamaIndex, take a look at the following resources:
-
-- [LlamaIndex Documentation](https://docs.llamaindex.ai) - learn about LlamaIndex (Python features).
-- [LlamaIndexTS Documentation](https://ts.llamaindex.ai) - learn about LlamaIndex (Typescript features).
-
-You can check out [the LlamaIndexTS GitHub repository](https://github.com/run-llama/LlamaIndexTS) - your feedback and contributions are welcome!

templates/components/agents/python/blog/README-template.md

@@ -1,5 +1,3 @@
-This is a [LlamaIndex](https://www.llamaindex.ai/) multi-agents project using [Workflows](https://docs.llamaindex.ai/en/stable/understanding/workflows/).
-
 ## Overview
 
 This example is using three agents to generate a blog post:
@@ -10,9 +8,9 @@ This example is using three agents to generate a blog post:
 
 There are three different methods how the agents can interact to reach their goal:
 
-1. [Choreography](./app/examples/choreography.py) - the agents decide themselves to delegate a task to another agent
-1. [Orchestrator](./app/examples/orchestrator.py) - a central orchestrator decides which agent should execute a task
-1. [Explicit Workflow](./app/examples/workflow.py) - a pre-defined workflow specific for the task is used to execute the tasks
+1. [Choreography](./app/agents/choreography.py) - the agents decide themselves to delegate a task to another agent
+1. [Orchestrator](./app/agents/orchestrator.py) - a central orchestrator decides which agent should execute a task
+1. [Explicit Workflow](./app/agents/workflow.py) - a pre-defined workflow specific for the task is used to execute the tasks
 
 ## Getting Started
 
@@ -25,7 +23,6 @@ poetry install
 ```
 
 Then check the parameters that have been pre-configured in the `.env` file in this directory. (E.g. you might need to configure an `OPENAI_API_KEY` if you're using OpenAI as model provider).
-
 Second, generate the embeddings of the documents in the `./data` directory:
 
 ```shell
@@ -35,11 +32,10 @@ poetry run generate
 Third, run the development server:
 
 ```shell
-poetry run python main.py
+poetry run dev
 ```
 
 Per default, the example is using the explicit workflow. You can change the example by setting the `EXAMPLE_TYPE` environment variable to `choreography` or `orchestrator`.
-
 The example provides one streaming API endpoint `/api/chat`.
 You can test the endpoint with the following curl request:
 
@@ -51,12 +47,12 @@ curl --location 'localhost:8000/api/chat' \
 
 You can start editing the API by modifying `app/api/routers/chat.py` or `app/examples/workflow.py`. The API auto-updates as you save the files.
 
-Open [http://localhost:8000/docs](http://localhost:8000/docs) with your browser to see the Swagger UI of the API.
+Open [http://localhost:8000](http://localhost:8000) with your browser to start the app.
 
-The API allows CORS for all origins to simplify development. You can change this behavior by setting the `ENVIRONMENT` environment variable to `prod`:
+To start the app in **production**, run:
 
 ```
-ENVIRONMENT=prod poetry run python main.py
+poetry run prod
 ```
 
 ## Learn More
@@ -65,5 +61,4 @@ To learn more about LlamaIndex, take a look at the following resources:
 
 - [LlamaIndex Documentation](https://docs.llamaindex.ai) - learn about LlamaIndex.
 - [Workflows Introduction](https://docs.llamaindex.ai/en/stable/understanding/workflows/) - learn about LlamaIndex workflows.
-
-You can check out [the LlamaIndex GitHub repository](https://github.com/run-llama/llama_index) - your feedback and contributions are welcome!
+  You can check out [the LlamaIndex GitHub repository](https://github.com/run-llama/llama_index) - your feedback and contributions are welcome!

templates/components/agents/python/blog/app/agents/choreography.py

@@ -1,10 +1,10 @@
 from textwrap import dedent
 from typing import List, Optional
 
-from app.agents.multi import AgentCallingAgent
-from app.agents.single import FunctionCallingAgent
-from app.examples.publisher import create_publisher
-from app.examples.researcher import create_researcher
+from app.agents.publisher import create_publisher
+from app.agents.researcher import create_researcher
+from app.workflows.multi import AgentCallingAgent
+from app.workflows.single import FunctionCallingAgent
 from llama_index.core.chat_engine.types import ChatMessage
 
 

templates/components/agents/python/blog/app/agents/orchestrator.py

@@ -1,10 +1,10 @@
 from textwrap import dedent
 from typing import List, Optional
 
-from app.agents.multi import AgentOrchestrator
-from app.agents.single import FunctionCallingAgent
-from app.examples.publisher import create_publisher
-from app.examples.researcher import create_researcher
+from app.agents.publisher import create_publisher
+from app.agents.researcher import create_researcher
+from app.workflows.multi import AgentOrchestrator
+from app.workflows.single import FunctionCallingAgent
 from llama_index.core.chat_engine.types import ChatMessage
 
 

templates/components/agents/python/blog/app/agents/publisher.py

@@ -1,8 +1,8 @@
 from textwrap import dedent
 from typing import List, Tuple
 
-from app.agents.single import FunctionCallingAgent
 from app.engine.tools import ToolFactory
+from app.workflows.single import FunctionCallingAgent
 from llama_index.core.chat_engine.types import ChatMessage
 from llama_index.core.tools import FunctionTool
 
@@ -11,11 +11,11 @@ def get_publisher_tools() -> Tuple[List[FunctionTool], str, str]:
     tools = []
     # Get configured tools from the tools.yaml file
     configured_tools = ToolFactory.from_env(map_result=True)
-    if "document_generator" in configured_tools.keys():
-        tools.extend(configured_tools["document_generator"])
+    if "generate_document" in configured_tools.keys():
+        tools.append(configured_tools["generate_document"])
         prompt_instructions = dedent("""
             Normally, reply the blog post content to the user directly. 
-            But if user requested to generate a file, use the document_generator tool to generate the file and reply the link to the file.
+            But if user requested to generate a file, use the generate_document tool to generate the file and reply the link to the file.
         """)
         description = "Expert in publishing the blog post, able to publish the blog post in PDF or HTML format."
     else:

templates/components/agents/python/blog/app/agents/researcher.py

@@ -2,9 +2,9 @@ import os
 from textwrap import dedent
 from typing import List
 
-from app.agents.single import FunctionCallingAgent
 from app.engine.index import IndexConfig, get_index
 from app.engine.tools import ToolFactory
+from app.workflows.single import FunctionCallingAgent
 from llama_index.core.chat_engine.types import ChatMessage
 from llama_index.core.tools import QueryEngineTool, ToolMetadata
 
@@ -42,11 +42,15 @@ def _get_research_tools(**kwargs) -> QueryEngineTool:
     query_engine_tool = _create_query_engine_tool(**kwargs)
     if query_engine_tool is not None:
         tools.append(query_engine_tool)
-    researcher_tool_names = ["duckduckgo", "wikipedia.WikipediaToolSpec"]
+    researcher_tool_names = [
+        "duckduckgo_search",
+        "duckduckgo_image_search",
+        "wikipedia.WikipediaToolSpec",
+    ]
     configured_tools = ToolFactory.from_env(map_result=True)
     for tool_name, tool in configured_tools.items():
         if tool_name in researcher_tool_names:
-            tools.extend(tool)
+            tools.append(tool)
     return tools
 
 

templates/components/agents/python/blog/app/agents/workflow.py

@@ -1,9 +1,9 @@
 from textwrap import dedent
 from typing import AsyncGenerator, List, Optional
 
-from app.agents.single import AgentRunEvent, AgentRunResult, FunctionCallingAgent
-from app.examples.publisher import create_publisher
-from app.examples.researcher import create_researcher
+from app.agents.publisher import create_publisher
+from app.agents.researcher import create_researcher
+from app.workflows.single import AgentRunEvent, AgentRunResult, FunctionCallingAgent
 from llama_index.core.chat_engine.types import ChatMessage
 from llama_index.core.prompts import PromptTemplate
 from llama_index.core.settings import Settings
@@ -238,7 +238,9 @@ class BlogPostWorkflow(Workflow):
         publisher: FunctionCallingAgent,
     ) -> StopEvent:
         try:
-            result: AgentRunResult = await self.run_agent(ctx, publisher, ev.input)
+            result: AgentRunResult = await self.run_agent(
+                ctx, publisher, ev.input, streaming=ctx.data["streaming"]
+            )
             return StopEvent(result=result)
         except Exception as e:
             ctx.write_event_to_stream(

templates/components/agents/python/blog/app/workflows/__init__.py

@@ -0,0 +1,3 @@
+from .blog import create_workflow
+
+__all__ = ["create_workflow"]

templates/components/agents/python/blog/app/workflows/blog.py

@@ -2,19 +2,20 @@ import logging
 import os
 from typing import List, Optional
 
-from app.examples.choreography import create_choreography
-from app.examples.orchestrator import create_orchestrator
-from app.examples.workflow import create_workflow
+from app.agents.choreography import create_choreography
+from app.agents.orchestrator import create_orchestrator
+from app.agents.workflow import create_workflow as create_blog_workflow
 from llama_index.core.chat_engine.types import ChatMessage
 from llama_index.core.workflow import Workflow
 
 logger = logging.getLogger("uvicorn")
 
 
-def get_chat_engine(
+def create_workflow(
     chat_history: Optional[List[ChatMessage]] = None, **kwargs
 ) -> Workflow:
-    # TODO: the EXAMPLE_TYPE could be passed as a chat config parameter?
+    # Chat filters are not supported yet
+    kwargs.pop("filters", None)
     agent_type = os.getenv("EXAMPLE_TYPE", "").lower()
     match agent_type:
         case "choreography":
@@ -22,7 +23,7 @@ def get_chat_engine(
         case "orchestrator":
             agent = create_orchestrator(chat_history, **kwargs)
         case _:
-            agent = create_workflow(chat_history, **kwargs)
+            agent = create_blog_workflow(chat_history, **kwargs)
 
     logger.info(f"Using agent pattern: {agent_type}")
 

templates/components/agents/python/blog/app/workflows/multi.py

@@ -1,7 +1,7 @@
 from typing import Any, List
 
-from app.agents.planner import StructuredPlannerAgent
-from app.agents.single import (
+from app.workflows.planner import StructuredPlannerAgent
+from app.workflows.single import (
     AgentRunResult,
     ContextAwareTool,
     FunctionCallingAgent,

templates/components/agents/python/blog/app/workflows/planner.py

@@ -2,7 +2,7 @@ import uuid
 from enum import Enum
 from typing import Any, AsyncGenerator, Dict, List, Optional, Tuple, Union
 
-from app.agents.single import AgentRunEvent, AgentRunResult, FunctionCallingAgent
+from app.workflows.single import AgentRunEvent, AgentRunResult, FunctionCallingAgent
 from llama_index.core.agent.runner.planner import (
     DEFAULT_INITIAL_PLAN_PROMPT,
     DEFAULT_PLAN_REFINE_PROMPT,

templates/components/agents/python/blog/app/workflows/single.py

@@ -1,4 +1,5 @@
 from abc import abstractmethod
+from enum import Enum
 from typing import Any, AsyncGenerator, List, Optional
 
 from llama_index.core.llms import ChatMessage, ChatResponse
@@ -15,7 +16,7 @@ from llama_index.core.workflow import (
     Workflow,
     step,
 )
-from pydantic import BaseModel
+from pydantic import BaseModel, Field
 
 
 class InputEvent(Event):
@@ -26,17 +27,27 @@ class ToolCallEvent(Event):
     tool_calls: list[ToolSelection]
 
 
+class AgentRunEventType(Enum):
+    TEXT = "text"
+    PROGRESS = "progress"
+
+
 class AgentRunEvent(Event):
     name: str
-    _msg: str
+    msg: str
+    event_type: AgentRunEventType = Field(default=AgentRunEventType.TEXT)
+    data: Optional[dict] = None
 
-    @property
-    def msg(self):
-        return self._msg
-
-    @msg.setter
-    def msg(self, value):
-        self._msg = value
+    def to_response(self) -> dict:
+        return {
+            "type": "agent",
+            "data": {
+                "agent": self.name,
+                "type": self.event_type.value,
+                "text": self.msg,
+                "data": self.data,
+            },
+        }
 
 
 class AgentRunResult(BaseModel):

templates/components/agents/python/financial_report/README-template.md

@@ -0,0 +1,53 @@
+This is a [LlamaIndex](https://www.llamaindex.ai/) multi-agents project using [Workflows](https://docs.llamaindex.ai/en/stable/understanding/workflows/).
+
+## Getting Started
+
+First, setup the environment with poetry:
+
+> **_Note:_** This step is not needed if you are using the dev-container.
+
+```shell
+poetry install
+```
+
+Then check the parameters that have been pre-configured in the `.env` file in this directory. (E.g. you might need to configure an `OPENAI_API_KEY` if you're using OpenAI as model provider and `E2B_API_KEY` for the [E2B's code interpreter tool](https://e2b.dev/docs)).
+
+Second, generate the embeddings of the documents in the `./data` directory:
+
+```shell
+poetry run generate
+```
+
+Third, run the development server:
+
+```shell
+poetry run dev
+```
+
+The example provides one streaming API endpoint `/api/chat`.
+You can test the endpoint with the following curl request:
+
+```
+curl --location 'localhost:8000/api/chat' \
+--header 'Content-Type: application/json' \
+--data '{ "messages": [{ "role": "user", "content": "Create a report comparing the finances of Apple and Tesla" }] }'
+```
+
+You can start editing the API by modifying `app/api/routers/chat.py` or `app/workflows/financial_report.py`. The API auto-updates as you save the files.
+
+Open [http://localhost:8000](http://localhost:8000) with your browser to start the app.
+
+To start the app in **production**, run:
+
+```
+poetry run prod
+```
+
+## Learn More
+
+To learn more about LlamaIndex, take a look at the following resources:
+
+- [LlamaIndex Documentation](https://docs.llamaindex.ai) - learn about LlamaIndex.
+- [Workflows Introduction](https://docs.llamaindex.ai/en/stable/understanding/workflows/) - learn about LlamaIndex workflows.
+
+You can check out [the LlamaIndex GitHub repository](https://github.com/run-llama/llama_index) - your feedback and contributions are welcome!

templates/components/agents/python/financial_report/app/workflows/__init__.py

@@ -0,0 +1,3 @@
+from .financial_report import create_workflow
+
+__all__ = ["create_workflow"]

templates/components/agents/python/financial_report/app/workflows/financial_report.py

@@ -0,0 +1,298 @@
+import os
+from typing import Any, Dict, List, Optional
+
+from app.engine.index import IndexConfig, get_index
+from app.engine.tools import ToolFactory
+from app.workflows.events import AgentRunEvent
+from app.workflows.tools import (
+    call_tools,
+    chat_with_tools,
+)
+from llama_index.core import Settings
+from llama_index.core.base.llms.types import ChatMessage, MessageRole
+from llama_index.core.indices.vector_store import VectorStoreIndex
+from llama_index.core.llms.function_calling import FunctionCallingLLM
+from llama_index.core.memory import ChatMemoryBuffer
+from llama_index.core.tools import FunctionTool, QueryEngineTool, ToolSelection
+from llama_index.core.workflow import (
+    Context,
+    Event,
+    StartEvent,
+    StopEvent,
+    Workflow,
+    step,
+)
+
+
+def create_workflow(
+    chat_history: Optional[List[ChatMessage]] = None,
+    params: Optional[Dict[str, Any]] = None,
+    filters: Optional[List[Any]] = None,
+) -> Workflow:
+    index_config = IndexConfig(**params)
+    index: VectorStoreIndex = get_index(config=index_config)
+    if index is None:
+        query_engine_tool = None
+    else:
+        top_k = int(os.getenv("TOP_K", 10))
+        query_engine = index.as_query_engine(similarity_top_k=top_k, filters=filters)
+        query_engine_tool = QueryEngineTool.from_defaults(query_engine=query_engine)
+
+    configured_tools: Dict[str, FunctionTool] = ToolFactory.from_env(map_result=True)  # type: ignore
+    code_interpreter_tool = configured_tools.get("interpret")
+    document_generator_tool = configured_tools.get("generate_document")
+
+    return FinancialReportWorkflow(
+        query_engine_tool=query_engine_tool,
+        code_interpreter_tool=code_interpreter_tool,
+        document_generator_tool=document_generator_tool,
+        chat_history=chat_history,
+    )
+
+
+class InputEvent(Event):
+    input: List[ChatMessage]
+    response: bool = False
+
+
+class ResearchEvent(Event):
+    input: list[ToolSelection]
+
+
+class AnalyzeEvent(Event):
+    input: list[ToolSelection] | ChatMessage
+
+
+class ReportEvent(Event):
+    input: list[ToolSelection]
+
+
+class FinancialReportWorkflow(Workflow):
+    """
+    A workflow to generate a financial report using indexed documents.
+
+    Requirements:
+    - Indexed documents containing financial data and a query engine tool to search them
+    - A code interpreter tool to analyze data and generate reports
+    - A document generator tool to create report files
+
+    Steps:
+    1. LLM Input: The LLM determines the next step based on function calling.
+       For example, if the model requests the query engine tool, it returns a ResearchEvent;
+       if it requests document generation, it returns a ReportEvent.
+    2. Research: Uses the query engine to find relevant chunks from indexed documents.
+       After gathering information, it requests analysis (step 3).
+    3. Analyze: Uses a custom prompt to analyze research results and can call the code
+       interpreter tool for visualization or calculation. Returns results to the LLM.
+    4. Report: Uses the document generator tool to create a report. Returns results to the LLM.
+    """
+
+    _default_system_prompt = """
+    You are a financial analyst who are given a set of tools to help you.
+    It's good to using appropriate tools for the user request and always use the information from the tools, don't make up anything yourself.
+    For the query engine tool, you should break down the user request into a list of queries and call the tool with the queries.
+    """
+
+    def __init__(
+        self,
+        query_engine_tool: QueryEngineTool,
+        code_interpreter_tool: FunctionTool,
+        document_generator_tool: FunctionTool,
+        llm: Optional[FunctionCallingLLM] = None,
+        timeout: int = 360,
+        chat_history: Optional[List[ChatMessage]] = None,
+        system_prompt: Optional[str] = None,
+    ):
+        super().__init__(timeout=timeout)
+        self.system_prompt = system_prompt or self._default_system_prompt
+        self.chat_history = chat_history or []
+        self.query_engine_tool = query_engine_tool
+        self.code_interpreter_tool = code_interpreter_tool
+        self.document_generator_tool = document_generator_tool
+        assert (
+            query_engine_tool is not None
+        ), "Query engine tool is not found. Try run generation script or upload a document file first."
+        assert code_interpreter_tool is not None, "Code interpreter tool is required"
+        assert (
+            document_generator_tool is not None
+        ), "Document generator tool is required"
+        self.tools = [
+            self.query_engine_tool,
+            self.code_interpreter_tool,
+            self.document_generator_tool,
+        ]
+        self.llm: FunctionCallingLLM = llm or Settings.llm
+        assert isinstance(self.llm, FunctionCallingLLM)
+        self.memory = ChatMemoryBuffer.from_defaults(
+            llm=self.llm, chat_history=self.chat_history
+        )
+
+    @step()
+    async def prepare_chat_history(self, ctx: Context, ev: StartEvent) -> InputEvent:
+        ctx.data["input"] = ev.input
+
+        if self.system_prompt:
+            system_msg = ChatMessage(
+                role=MessageRole.SYSTEM, content=self.system_prompt
+            )
+            self.memory.put(system_msg)
+
+        # Add user input to memory
+        self.memory.put(ChatMessage(role=MessageRole.USER, content=ev.input))
+
+        return InputEvent(input=self.memory.get())
+
+    @step()
+    async def handle_llm_input(  # type: ignore
+        self,
+        ctx: Context,
+        ev: InputEvent,
+    ) -> ResearchEvent | AnalyzeEvent | ReportEvent | StopEvent:
+        """
+        Handle an LLM input and decide the next step.
+        """
+        # Always use the latest chat history from the input
+        chat_history: list[ChatMessage] = ev.input
+
+        # Get tool calls
+        response = await chat_with_tools(
+            self.llm,
+            self.tools,  # type: ignore
+            chat_history,
+        )
+        if not response.has_tool_calls():
+            # If no tool call, return the response generator
+            return StopEvent(result=response.generator)
+        # calling different tools at the same time is not supported at the moment
+        # add an error message to tell the AI to process step by step
+        if response.is_calling_different_tools():
+            self.memory.put(
+                ChatMessage(
+                    role=MessageRole.ASSISTANT,
+                    content="Cannot call different tools at the same time. Try calling one tool at a time.",
+                )
+            )
+            return InputEvent(input=self.memory.get())
+        self.memory.put(response.tool_call_message)
+        match response.tool_name():
+            case self.code_interpreter_tool.metadata.name:
+                return AnalyzeEvent(input=response.tool_calls)
+            case self.document_generator_tool.metadata.name:
+                return ReportEvent(input=response.tool_calls)
+            case self.query_engine_tool.metadata.name:
+                return ResearchEvent(input=response.tool_calls)
+            case _:
+                raise ValueError(f"Unknown tool: {response.tool_name()}")
+
+    @step()
+    async def research(self, ctx: Context, ev: ResearchEvent) -> AnalyzeEvent:
+        """
+        Do a research to gather information for the user's request.
+        A researcher should have these tools: query engine, search engine, etc.
+        """
+        ctx.write_event_to_stream(
+            AgentRunEvent(
+                name="Researcher",
+                msg="Starting research",
+            )
+        )
+        tool_calls = ev.input
+
+        tool_messages = await call_tools(
+            ctx=ctx,
+            agent_name="Researcher",
+            tools=[self.query_engine_tool],
+            tool_calls=tool_calls,
+        )
+        self.memory.put_messages(tool_messages)
+        return AnalyzeEvent(
+            input=ChatMessage(
+                role=MessageRole.ASSISTANT,
+                content="I've finished the research. Please analyze the result.",
+            ),
+        )
+
+    @step()
+    async def analyze(self, ctx: Context, ev: AnalyzeEvent) -> InputEvent:
+        """
+        Analyze the research result.
+        """
+        ctx.write_event_to_stream(
+            AgentRunEvent(
+                name="Analyst",
+                msg="Starting analysis",
+            )
+        )
+        event_requested_by_workflow_llm = isinstance(ev.input, list)
+        # Requested by the workflow LLM Input step, it's a tool call
+        if event_requested_by_workflow_llm:
+            # Set the tool calls
+            tool_calls = ev.input
+        else:
+            # Otherwise, it's triggered by the research step
+            # Use a custom prompt and independent memory for the analyst agent
+            analysis_prompt = """
+            You are a financial analyst, you are given a research result and a set of tools to help you.
+            Always use the given information, don't make up anything yourself. If there is not enough information, you can asking for more information.
+            If you have enough numerical information, it's good to include some charts/visualizations to the report so you can use the code interpreter tool to generate a report.
+            """
+            # This is handled by analyst agent
+            # Clone the shared memory to avoid conflicting with the workflow.
+            chat_history = self.memory.get()
+            chat_history.append(
+                ChatMessage(role=MessageRole.SYSTEM, content=analysis_prompt)
+            )
+            chat_history.append(ev.input)  # type: ignore
+            # Check if the analyst agent needs to call tools
+            response = await chat_with_tools(
+                self.llm,
+                [self.code_interpreter_tool],
+                chat_history,
+            )
+            if not response.has_tool_calls():
+                # If no tool call, fallback analyst message to the workflow
+                analyst_msg = ChatMessage(
+                    role=MessageRole.ASSISTANT,
+                    content=await response.full_response(),
+                )
+                self.memory.put(analyst_msg)
+                return InputEvent(input=self.memory.get())
+            else:
+                # Set the tool calls and the tool call message to the memory
+                tool_calls = response.tool_calls
+                self.memory.put(response.tool_call_message)
+
+        # Call tools
+        tool_messages = await call_tools(
+            ctx=ctx,
+            agent_name="Analyst",
+            tools=[self.code_interpreter_tool],
+            tool_calls=tool_calls,  # type: ignore
+        )
+        self.memory.put_messages(tool_messages)
+
+        # Fallback to the input with the latest chat history
+        return InputEvent(input=self.memory.get())
+
+    @step()
+    async def report(self, ctx: Context, ev: ReportEvent) -> InputEvent:
+        """
+        Generate a report based on the analysis result.
+        """
+        ctx.write_event_to_stream(
+            AgentRunEvent(
+                name="Reporter",
+                msg="Starting report generation",
+            )
+        )
+        tool_calls = ev.input
+        tool_messages = await call_tools(
+            ctx=ctx,
+            agent_name="Reporter",
+            tools=[self.document_generator_tool],
+            tool_calls=tool_calls,
+        )
+        self.memory.put_messages(tool_messages)
+
+        # After the tool calls, fallback to the input with the latest chat history
+        return InputEvent(input=self.memory.get())

templates/components/agents/python/form_filling/README-template.md

@@ -0,0 +1,59 @@
+This is a [LlamaIndex](https://www.llamaindex.ai/) multi-agents project using [Workflows](https://docs.llamaindex.ai/en/stable/understanding/workflows/).
+
+## Getting Started
+
+First, setup the environment with poetry:
+
+> **_Note:_** This step is not needed if you are using the dev-container.
+
+```shell
+poetry install
+```
+
+Then check the parameters that have been pre-configured in the `.env` file in this directory.
+Make sure you have the `OPENAI_API_KEY` set.
+
+Second, run the development server:
+
+```shell
+poetry run dev
+```
+
+## Use Case: Filling Financial CSV Template
+
+To reproduce the use case, start the [frontend](../frontend/README.md) and follow these steps in the frontend:
+
+1. Upload the Apple and Tesla financial reports from the [data](./data) directory. Just send an empty message.
+2. Upload the CSV file [sec_10k_template.csv](./sec_10k_template.csv) and send the message "Fill the missing cells in the CSV file".
+
+The agent will fill the missing cells by retrieving the information from the uploaded financial reports and return a new CSV file with the filled cells.
+
+### API endpoints
+
+The example provides one streaming API endpoint `/api/chat`.
+You can test the endpoint with the following curl request:
+
+```
+curl --location 'localhost:8000/api/chat' \
+--header 'Content-Type: application/json' \
+--data '{ "messages": [{ "role": "user", "content": "What can you do?" }] }'
+```
+
+You can start editing the API by modifying `app/api/routers/chat.py` or `app/workflows/form_filling.py`. The API auto-updates as you save the files.
+
+Open [http://localhost:8000](http://localhost:8000) with your browser to start the app.
+
+To start the app in **production**, run:
+
+```
+poetry run prod
+```
+
+## Learn More
+
+To learn more about LlamaIndex, take a look at the following resources:
+
+- [LlamaIndex Documentation](https://docs.llamaindex.ai) - learn about LlamaIndex.
+- [Workflows Introduction](https://docs.llamaindex.ai/en/stable/understanding/workflows/) - learn about LlamaIndex workflows.
+
+You can check out [the LlamaIndex GitHub repository](https://github.com/run-llama/llama_index) - your feedback and contributions are welcome!

templates/components/agents/python/form_filling/app/workflows/__init__.py

@@ -0,0 +1,3 @@
+from .form_filling import create_workflow
+
+__all__ = ["create_workflow"]

templates/components/agents/python/form_filling/app/workflows/form_filling.py

@@ -0,0 +1,241 @@
+import os
+from typing import Any, Dict, List, Optional
+
+from app.engine.index import IndexConfig, get_index
+from app.engine.tools import ToolFactory
+from app.workflows.events import AgentRunEvent
+from app.workflows.tools import (
+    call_tools,
+    chat_with_tools,
+)
+from llama_index.core import Settings
+from llama_index.core.base.llms.types import ChatMessage, MessageRole
+from llama_index.core.indices.vector_store import VectorStoreIndex
+from llama_index.core.llms.function_calling import FunctionCallingLLM
+from llama_index.core.memory import ChatMemoryBuffer
+from llama_index.core.tools import FunctionTool, QueryEngineTool, ToolSelection
+from llama_index.core.workflow import (
+    Context,
+    Event,
+    StartEvent,
+    StopEvent,
+    Workflow,
+    step,
+)
+
+
+def create_workflow(
+    chat_history: Optional[List[ChatMessage]] = None,
+    params: Optional[Dict[str, Any]] = None,
+    filters: Optional[List[Any]] = None,
+) -> Workflow:
+    if params is None:
+        params = {}
+    if filters is None:
+        filters = []
+    index_config = IndexConfig(**params)
+    index: VectorStoreIndex = get_index(config=index_config)
+    if index is None:
+        query_engine_tool = None
+    else:
+        top_k = int(os.getenv("TOP_K", 10))
+        query_engine = index.as_query_engine(similarity_top_k=top_k, filters=filters)
+        query_engine_tool = QueryEngineTool.from_defaults(query_engine=query_engine)
+
+    configured_tools = ToolFactory.from_env(map_result=True)
+    extractor_tool = configured_tools.get("extract_questions")  # type: ignore
+    filling_tool = configured_tools.get("fill_form")  # type: ignore
+
+    workflow = FormFillingWorkflow(
+        query_engine_tool=query_engine_tool,
+        extractor_tool=extractor_tool,  # type: ignore
+        filling_tool=filling_tool,  # type: ignore
+        chat_history=chat_history,
+    )
+
+    return workflow
+
+
+class InputEvent(Event):
+    input: List[ChatMessage]
+    response: bool = False
+
+
+class ExtractMissingCellsEvent(Event):
+    tool_calls: list[ToolSelection]
+
+
+class FindAnswersEvent(Event):
+    tool_calls: list[ToolSelection]
+
+
+class FillEvent(Event):
+    tool_calls: list[ToolSelection]
+
+
+class FormFillingWorkflow(Workflow):
+    """
+    A predefined workflow for filling missing cells in a CSV file.
+    Required tools:
+    - query_engine: A query engine to query for the answers to the questions.
+    - extract_question: Extract missing cells in a CSV file and generate questions to fill them.
+    - answer_question: Query for the answers to the questions.
+
+    Flow:
+    1. Extract missing cells in a CSV file and generate questions to fill them.
+    2. Query for the answers to the questions.
+    3. Fill the missing cells with the answers.
+    """
+
+    _default_system_prompt = """
+    You are a helpful assistant who helps fill missing cells in a CSV file.
+    Only extract missing cells from CSV files.
+    Only use provided data - never make up any information yourself. Fill N/A if an answer is not found.
+    If there is no query engine tool or the gathered information has many N/A values indicating the questions don't match the data, respond with a warning and ask the user to upload a different file or connect to a knowledge base.
+    """
+
+    def __init__(
+        self,
+        query_engine_tool: Optional[QueryEngineTool],
+        extractor_tool: FunctionTool,
+        filling_tool: FunctionTool,
+        llm: Optional[FunctionCallingLLM] = None,
+        timeout: int = 360,
+        chat_history: Optional[List[ChatMessage]] = None,
+        system_prompt: Optional[str] = None,
+    ):
+        super().__init__(timeout=timeout)
+        self.system_prompt = system_prompt or self._default_system_prompt
+        self.chat_history = chat_history or []
+        self.query_engine_tool = query_engine_tool
+        self.extractor_tool = extractor_tool
+        self.filling_tool = filling_tool
+        if self.extractor_tool is None or self.filling_tool is None:
+            raise ValueError("Extractor and filling tools are required.")
+        self.tools = [self.extractor_tool, self.filling_tool]
+        if self.query_engine_tool is not None:
+            self.tools.append(self.query_engine_tool)  # type: ignore
+        self.llm: FunctionCallingLLM = llm or Settings.llm
+        if not isinstance(self.llm, FunctionCallingLLM):
+            raise ValueError("FormFillingWorkflow only supports FunctionCallingLLM.")
+        self.memory = ChatMemoryBuffer.from_defaults(
+            llm=self.llm, chat_history=self.chat_history
+        )
+
+    @step()
+    async def start(self, ctx: Context, ev: StartEvent) -> InputEvent:
+        ctx.data["input"] = ev.input
+
+        if self.system_prompt:
+            system_msg = ChatMessage(
+                role=MessageRole.SYSTEM, content=self.system_prompt
+            )
+            self.memory.put(system_msg)
+
+        user_input = ev.input
+        user_msg = ChatMessage(role=MessageRole.USER, content=user_input)
+        self.memory.put(user_msg)
+
+        chat_history = self.memory.get()
+        return InputEvent(input=chat_history)
+
+    @step()
+    async def handle_llm_input(  # type: ignore
+        self,
+        ctx: Context,
+        ev: InputEvent,
+    ) -> ExtractMissingCellsEvent | FillEvent | StopEvent:
+        """
+        Handle an LLM input and decide the next step.
+        """
+        chat_history: list[ChatMessage] = ev.input
+        response = await chat_with_tools(
+            self.llm,
+            self.tools,
+            chat_history,
+        )
+        if not response.has_tool_calls():
+            return StopEvent(result=response.generator)
+        # calling different tools at the same time is not supported at the moment
+        # add an error message to tell the AI to process step by step
+        if response.is_calling_different_tools():
+            self.memory.put(
+                ChatMessage(
+                    role=MessageRole.ASSISTANT,
+                    content="Cannot call different tools at the same time. Try calling one tool at a time.",
+                )
+            )
+            return InputEvent(input=self.memory.get())
+        self.memory.put(response.tool_call_message)
+        match response.tool_name():
+            case self.extractor_tool.metadata.name:
+                return ExtractMissingCellsEvent(tool_calls=response.tool_calls)
+            case self.query_engine_tool.metadata.name:
+                return FindAnswersEvent(tool_calls=response.tool_calls)
+            case self.filling_tool.metadata.name:
+                return FillEvent(tool_calls=response.tool_calls)
+            case _:
+                raise ValueError(f"Unknown tool: {response.tool_name()}")
+
+    @step()
+    async def extract_missing_cells(
+        self, ctx: Context, ev: ExtractMissingCellsEvent
+    ) -> InputEvent | FindAnswersEvent:
+        """
+        Extract missing cells in a CSV file and generate questions to fill them.
+        """
+        ctx.write_event_to_stream(
+            AgentRunEvent(
+                name="Extractor",
+                msg="Extracting missing cells",
+            )
+        )
+        # Call the extract questions tool
+        tool_messages = await call_tools(
+            agent_name="Extractor",
+            tools=[self.extractor_tool],
+            ctx=ctx,
+            tool_calls=ev.tool_calls,
+        )
+        self.memory.put_messages(tool_messages)
+        return InputEvent(input=self.memory.get())
+
+    @step()
+    async def find_answers(self, ctx: Context, ev: FindAnswersEvent) -> InputEvent:
+        """
+        Call answer questions tool to query for the answers to the questions.
+        """
+        ctx.write_event_to_stream(
+            AgentRunEvent(
+                name="Researcher",
+                msg="Finding answers for missing cells",
+            )
+        )
+        tool_messages = await call_tools(
+            ctx=ctx,
+            agent_name="Researcher",
+            tools=[self.query_engine_tool],
+            tool_calls=ev.tool_calls,
+        )
+        self.memory.put_messages(tool_messages)
+        return InputEvent(input=self.memory.get())
+
+    @step()
+    async def fill_cells(self, ctx: Context, ev: FillEvent) -> InputEvent:
+        """
+        Call fill cells tool to fill the missing cells with the answers.
+        """
+        ctx.write_event_to_stream(
+            AgentRunEvent(
+                name="Processor",
+                msg="Filling missing cells",
+            )
+        )
+        tool_messages = await call_tools(
+            agent_name="Processor",
+            tools=[self.filling_tool],
+            ctx=ctx,
+            tool_calls=ev.tool_calls,
+        )
+        self.memory.put_messages(tool_messages)
+        return InputEvent(input=self.memory.get())

templates/components/agents/python/form_filling/sec_10k_template.csv

@@ -0,0 +1,17 @@
+Parameter,2023 Apple (AAPL),2023 Tesla (TSLA)
+Revenue,,
+Net Income,,
+Earnings Per Share (EPS),,
+Debt-to-Equity Ratio,,
+Current Ratio,,
+Gross Margin,,
+Operating Margin,,
+Net Profit Margin,,
+Inventory Turnover,,
+Accounts Receivable Turnover,,
+Capital Expenditure,,
+Research and Development Expense,,
+Market Cap,,
+Price to Earnings Ratio,,
+Dividend Yield,,
+Year-over-Year Growth Rate,,

templates/components/agents/typescript/blog/workflow/agents.ts

@@ -1,19 +1,16 @@
 import { ChatMessage } from "llamaindex";
+import { getTool } from "../engine/tools";
 import { FunctionCallingAgent } from "./single-agent";
-import { getQueryEngineTool, lookupTools } from "./tools";
+import { getQueryEngineTools } from "./tools";
 
-export const createResearcher = async (
-  chatHistory: ChatMessage[],
-  params?: any,
-) => {
-  const queryEngineTool = await getQueryEngineTool(params);
-  const tools = (
-    await lookupTools([
-      "wikipedia_tool",
-      "duckduckgo_search",
-      "image_generator",
-    ])
-  ).concat(queryEngineTool ? [queryEngineTool] : []);
+export const createResearcher = async (chatHistory: ChatMessage[]) => {
+  const queryEngineTools = await getQueryEngineTools();
+  const tools = [
+    await getTool("wikipedia_tool"),
+    await getTool("duckduckgo_search"),
+    await getTool("image_generator"),
+    ...(queryEngineTools ? queryEngineTools : []),
+  ].filter((tool) => tool !== undefined);
 
   return new FunctionCallingAgent({
     name: "researcher",
@@ -81,17 +78,17 @@ Example:
 };
 
 export const createPublisher = async (chatHistory: ChatMessage[]) => {
-  const tools = await lookupTools(["document_generator"]);
+  const tool = await getTool("document_generator");
   let systemPrompt = `You are an expert in publishing blog posts. You are given a task to publish a blog post. 
 If the writer says that there was an error, you should reply with the error and not publish the post.`;
-  if (tools.length > 0) {
+  if (tool) {
     systemPrompt = `${systemPrompt}. 
 If the user requests to generate a file, use the document_generator tool to generate the file and reply with the link to the file.
 Otherwise, simply return the content of the post.`;
   }
   return new FunctionCallingAgent({
     name: "publisher",
-    tools: tools,
+    tools: tool ? [tool] : [],
     systemPrompt: systemPrompt,
     chatHistory,
   });

templates/components/agents/typescript/blog/workflow/factory.ts

@@ -0,0 +1,291 @@
+import {
+  HandlerContext,
+  StartEvent,
+  StopEvent,
+  Workflow,
+  WorkflowContext,
+  WorkflowEvent,
+} from "@llamaindex/workflow";
+import { ChatMessage, ChatResponseChunk, Settings } from "llamaindex";
+import {
+  createPublisher,
+  createResearcher,
+  createReviewer,
+  createWriter,
+} from "./agents";
+import {
+  FunctionCallingAgent,
+  FunctionCallingAgentInput,
+} from "./single-agent";
+import { AgentInput, AgentRunEvent } from "./type";
+
+const TIMEOUT = 360 * 1000;
+const MAX_ATTEMPTS = 2;
+
+class ResearchEvent extends WorkflowEvent<{ input: string }> {}
+class WriteEvent extends WorkflowEvent<{
+  input: string;
+  isGood: boolean;
+}> {}
+class ReviewEvent extends WorkflowEvent<{ input: string }> {}
+class PublishEvent extends WorkflowEvent<{ input: string }> {}
+
+type BlogContext = {
+  task: string;
+  attempts: number;
+  result: string;
+};
+
+export const createWorkflow = ({
+  chatHistory,
+  params,
+}: {
+  chatHistory: ChatMessage[];
+  params?: any;
+}) => {
+  const runAgent = async (
+    context: HandlerContext<BlogContext>,
+    agent: FunctionCallingAgent,
+    input: FunctionCallingAgentInput,
+  ) => {
+    const agentContext = agent.run(input, {
+      streaming: input.streaming ?? false,
+    });
+    for await (const event of agentContext) {
+      if (event instanceof AgentRunEvent) {
+        context.sendEvent(event);
+      }
+      if (event instanceof StopEvent) {
+        return event;
+      }
+    }
+    return null;
+  };
+
+  const start = async (
+    context: HandlerContext<BlogContext>,
+    ev: StartEvent<AgentInput>,
+  ) => {
+    const chatHistoryStr = chatHistory
+      .map((msg) => `${msg.role}: ${msg.content}`)
+      .join("\n");
+
+    // Decision-making process
+    const decision = await decideWorkflow(
+      ev.data.message.toString(),
+      chatHistoryStr,
+    );
+
+    if (decision !== "publish") {
+      return new ResearchEvent({
+        input: `Research for this task: ${JSON.stringify(context.data.task)}`,
+      });
+    } else {
+      return new PublishEvent({
+        input: `Publish content based on the chat history\n${chatHistoryStr}\n\n and task: ${context.data.task}`,
+      });
+    }
+  };
+
+  const decideWorkflow = async (task: string, chatHistoryStr: string) => {
+    const llm = Settings.llm;
+
+    const prompt = `You are an expert in decision-making, helping people write and publish blog posts.
+If the user is asking for a file or to publish content, respond with 'publish'.
+If the user requests to write or update a blog post, respond with 'not_publish'.
+
+Here is the chat history:
+${chatHistoryStr}
+
+The current user request is:
+${task}
+
+Given the chat history and the new user request, decide whether to publish based on existing information.
+Decision (respond with either 'not_publish' or 'publish'):`;
+
+    const output = await llm.complete({ prompt: prompt });
+    const decision = output.text.trim().toLowerCase();
+    return decision === "publish" ? "publish" : "research";
+  };
+
+  const research = async (
+    context: HandlerContext<BlogContext>,
+    ev: ResearchEvent,
+  ) => {
+    const researcher = await createResearcher(chatHistory);
+    const researchRes = await runAgent(context, researcher, {
+      displayName: "Researcher",
+      message: ev.data.input,
+    });
+    const researchResult = researchRes?.data;
+
+    return new WriteEvent({
+      input: `Write a blog post given this task: ${JSON.stringify(
+        context.data.task,
+      )} using this research content: ${researchResult}`,
+      isGood: false,
+    });
+  };
+
+  const write = async (
+    context: HandlerContext<BlogContext>,
+    ev: WriteEvent,
+  ) => {
+    const writer = createWriter(chatHistory);
+    context.data.attempts = context.data.attempts + 1;
+    const tooManyAttempts = context.data.attempts > MAX_ATTEMPTS;
+    if (tooManyAttempts) {
+      context.sendEvent(
+        new AgentRunEvent({
+          agent: "writer",
+          text: `Too many attempts (${MAX_ATTEMPTS}) to write the blog post. Proceeding with the current version.`,
+          type: "text",
+        }),
+      );
+    }
+
+    if (ev.data.isGood || tooManyAttempts) {
+      // the blog post is good or too many attempts
+      // stream the final content
+      const result = await runAgent(context, writer, {
+        message: `Based on the reviewer's feedback, refine the post and return only the final version of the post. Here's the current version: ${ev.data.input}`,
+        displayName: "Writer",
+        streaming: true,
+      });
+      return result as unknown as StopEvent<AsyncGenerator<ChatResponseChunk>>;
+    }
+
+    const writeRes = await runAgent(context, writer, {
+      message: ev.data.input,
+      displayName: "Writer",
+      streaming: false,
+    });
+    const writeResult = writeRes?.data;
+    context.data.result = writeResult; // store the last result
+
+    return new ReviewEvent({ input: writeResult });
+  };
+
+  const review = async (
+    context: HandlerContext<BlogContext>,
+    ev: ReviewEvent,
+  ) => {
+    const reviewer = createReviewer(chatHistory);
+    const reviewResult = (await runAgent(context, reviewer, {
+      message: ev.data.input,
+      displayName: "Reviewer",
+      streaming: false,
+    })) as unknown as StopEvent<string>;
+    const reviewResultStr = reviewResult.data;
+    const oldContent = context.data.result;
+    const postIsGood = reviewResultStr.toLowerCase().includes("post is good");
+    context.sendEvent(
+      new AgentRunEvent({
+        agent: "reviewer",
+        text: `The post is ${postIsGood ? "" : "not "}good enough for publishing. Sending back to the writer${
+          postIsGood ? " for publication." : "."
+        }`,
+        type: "text",
+      }),
+    );
+    if (postIsGood) {
+      return new WriteEvent({
+        input: "",
+        isGood: true,
+      });
+    }
+
+    return new WriteEvent({
+      input: `Improve the writing of a given blog post by using a given review.
+            Blog post:
+            \`\`\`
+            ${oldContent}
+            \`\`\`
+
+            Review:
+            \`\`\`
+            ${reviewResult}
+            \`\`\``,
+      isGood: false,
+    });
+  };
+
+  const publish = async (
+    context: HandlerContext<BlogContext>,
+    ev: PublishEvent,
+  ) => {
+    const publisher = await createPublisher(chatHistory);
+
+    const publishResult = await runAgent(context, publisher, {
+      message: `${ev.data.input}`,
+      displayName: "Publisher",
+      streaming: true,
+    });
+    return publishResult as unknown as StopEvent<
+      AsyncGenerator<ChatResponseChunk>
+    >;
+  };
+
+  const workflow: Workflow<
+    BlogContext,
+    AgentInput,
+    string | AsyncGenerator<boolean | ChatResponseChunk>
+  > = new Workflow();
+
+  workflow.addStep(
+    {
+      inputs: [StartEvent<AgentInput>],
+      outputs: [ResearchEvent, PublishEvent],
+    },
+    start,
+  );
+
+  workflow.addStep(
+    {
+      inputs: [ResearchEvent],
+      outputs: [WriteEvent],
+    },
+    research,
+  );
+
+  workflow.addStep(
+    {
+      inputs: [WriteEvent],
+      outputs: [ReviewEvent, StopEvent<AsyncGenerator<ChatResponseChunk>>],
+    },
+    write,
+  );
+
+  workflow.addStep(
+    {
+      inputs: [ReviewEvent],
+      outputs: [WriteEvent],
+    },
+    review,
+  );
+
+  workflow.addStep(
+    {
+      inputs: [PublishEvent],
+      outputs: [StopEvent],
+    },
+    publish,
+  );
+
+  // Overload run method to initialize the context
+  workflow.run = function (
+    input: AgentInput,
+  ): WorkflowContext<
+    AgentInput,
+    string | AsyncGenerator<boolean | ChatResponseChunk>,
+    BlogContext
+  > {
+    return Workflow.prototype.run.call(workflow, new StartEvent(input), {
+      task: input.message.toString(),
+      attempts: 0,
+      result: "",
+    });
+  };
+
+  return workflow;
+};

templates/components/agents/typescript/financial_report/README-template.md

@@ -0,0 +1,47 @@
+This is a [LlamaIndex](https://www.llamaindex.ai/) project using [Next.js](https://nextjs.org/) bootstrapped with [`create-llama`](https://github.com/run-llama/LlamaIndexTS/tree/main/packages/create-llama).
+
+## Getting Started
+
+First, install the dependencies:
+
+```
+npm install
+```
+
+Then check the parameters that have been pre-configured in the `.env` file in this directory.
+Make sure you have the `OPENAI_API_KEY` set.
+
+Second, generate the embeddings of the documents in the `./data` directory:
+
+```
+npm run generate
+```
+
+Third, run the development server:
+
+```
+npm run dev
+```
+
+Open [http://localhost:3000](http://localhost:3000) with your browser to see the chat UI.
+
+## Use Case: Filling Financial CSV Template
+
+You can start by sending an request on the chat UI to create a report comparing the finances of Apple and Tesla.
+Or you can test the `/api/chat` endpoint with the following curl request:
+
+```
+curl --location 'localhost:3000/api/chat' \
+--header 'Content-Type: application/json' \
+--data '{ "messages": [{ "role": "user", "content": "Create a report comparing the finances of Apple and Tesla" }] }'
+```
+
+## Learn More
+
+To learn more about LlamaIndex, take a look at the following resources:
+
+- [LlamaIndex Documentation](https://docs.llamaindex.ai) - learn about LlamaIndex (Python features).
+- [LlamaIndexTS Documentation](https://ts.llamaindex.ai/docs/llamaindex) - learn about LlamaIndex (Typescript features).
+- [Workflows Introduction](https://ts.llamaindex.ai/docs/llamaindex/guide/workflow) - learn about LlamaIndexTS workflows.
+
+You can check out [the LlamaIndexTS GitHub repository](https://github.com/run-llama/LlamaIndexTS) - your feedback and contributions are welcome!

templates/components/agents/typescript/financial_report/workflow/factory.ts

@@ -0,0 +1,20 @@
+import { ChatMessage, ToolCallLLM } from "llamaindex";
+import { getTool } from "../engine/tools";
+import { FinancialReportWorkflow } from "./fin-report";
+import { getQueryEngineTools } from "./tools";
+
+const TIMEOUT = 360 * 1000;
+
+export async function createWorkflow(options: {
+  chatHistory: ChatMessage[];
+  llm?: ToolCallLLM;
+}) {
+  return new FinancialReportWorkflow({
+    chatHistory: options.chatHistory,
+    queryEngineTools: (await getQueryEngineTools()) || [],
+    codeInterpreterTool: (await getTool("interpreter"))!,
+    documentGeneratorTool: (await getTool("document_generator"))!,
+    llm: options.llm,
+    timeout: TIMEOUT,
+  });
+}

templates/components/agents/typescript/financial_report/workflow/fin-report.ts

@@ -0,0 +1,322 @@
+import {
+  HandlerContext,
+  StartEvent,
+  StopEvent,
+  Workflow,
+  WorkflowEvent,
+} from "@llamaindex/workflow";
+import {
+  BaseToolWithCall,
+  ChatMemoryBuffer,
+  ChatMessage,
+  ChatResponseChunk,
+  Settings,
+  ToolCall,
+  ToolCallLLM,
+} from "llamaindex";
+import { callTools, chatWithTools } from "./tools";
+import { AgentInput, AgentRunEvent } from "./type";
+
+// Create a custom event type
+class InputEvent extends WorkflowEvent<{ input: ChatMessage[] }> {}
+
+class ResearchEvent extends WorkflowEvent<{
+  toolCalls: ToolCall[];
+}> {}
+
+class AnalyzeEvent extends WorkflowEvent<{
+  input: ChatMessage | ToolCall[];
+}> {}
+
+class ReportGenerationEvent extends WorkflowEvent<{
+  toolCalls: ToolCall[];
+}> {}
+
+const DEFAULT_SYSTEM_PROMPT = `
+You are a financial analyst who are given a set of tools to help you.
+It's good to using appropriate tools for the user request and always use the information from the tools, don't make up anything yourself.
+For the query engine tool, you should break down the user request into a list of queries and call the tool with the queries.
+`;
+
+export class FinancialReportWorkflow extends Workflow<
+  null,
+  AgentInput,
+  ChatResponseChunk
+> {
+  llm: ToolCallLLM;
+  memory: ChatMemoryBuffer;
+  queryEngineTools: BaseToolWithCall[];
+  codeInterpreterTool: BaseToolWithCall;
+  documentGeneratorTool: BaseToolWithCall;
+  systemPrompt?: string;
+
+  constructor(options: {
+    llm?: ToolCallLLM;
+    chatHistory: ChatMessage[];
+    queryEngineTools: BaseToolWithCall[];
+    codeInterpreterTool: BaseToolWithCall;
+    documentGeneratorTool: BaseToolWithCall;
+    systemPrompt?: string;
+    verbose?: boolean;
+    timeout?: number;
+  }) {
+    super({
+      verbose: options?.verbose ?? false,
+      timeout: options?.timeout ?? 360,
+    });
+
+    this.llm = options.llm ?? (Settings.llm as ToolCallLLM);
+    if (!(this.llm instanceof ToolCallLLM)) {
+      throw new Error("LLM is not a ToolCallLLM");
+    }
+    this.systemPrompt = options.systemPrompt ?? DEFAULT_SYSTEM_PROMPT;
+    this.queryEngineTools = options.queryEngineTools;
+    this.codeInterpreterTool = options.codeInterpreterTool;
+
+    this.documentGeneratorTool = options.documentGeneratorTool;
+    this.memory = new ChatMemoryBuffer({
+      llm: this.llm,
+      chatHistory: options.chatHistory,
+    });
+
+    // Add steps
+    this.addStep(
+      {
+        inputs: [StartEvent<AgentInput>],
+        outputs: [InputEvent],
+      },
+      this.prepareChatHistory,
+    );
+
+    this.addStep(
+      {
+        inputs: [InputEvent],
+        outputs: [
+          InputEvent,
+          ResearchEvent,
+          AnalyzeEvent,
+          ReportGenerationEvent,
+          StopEvent,
+        ],
+      },
+      this.handleLLMInput,
+    );
+
+    this.addStep(
+      {
+        inputs: [ResearchEvent],
+        outputs: [AnalyzeEvent],
+      },
+      this.handleResearch,
+    );
+
+    this.addStep(
+      {
+        inputs: [AnalyzeEvent],
+        outputs: [InputEvent],
+      },
+      this.handleAnalyze,
+    );
+
+    this.addStep(
+      {
+        inputs: [ReportGenerationEvent],
+        outputs: [InputEvent],
+      },
+      this.handleReportGeneration,
+    );
+  }
+
+  prepareChatHistory = async (
+    ctx: HandlerContext<null>,
+    ev: StartEvent<AgentInput>,
+  ): Promise<InputEvent> => {
+    const { message } = ev.data;
+
+    if (this.systemPrompt) {
+      this.memory.put({ role: "system", content: this.systemPrompt });
+    }
+    this.memory.put({ role: "user", content: message });
+
+    return new InputEvent({ input: this.memory.getMessages() });
+  };
+
+  handleLLMInput = async (
+    ctx: HandlerContext<null>,
+    ev: InputEvent,
+  ): Promise<
+    | InputEvent
+    | ResearchEvent
+    | AnalyzeEvent
+    | ReportGenerationEvent
+    | StopEvent
+  > => {
+    const chatHistory = ev.data.input;
+
+    const tools = [this.codeInterpreterTool, this.documentGeneratorTool];
+    if (this.queryEngineTools) {
+      tools.push(...this.queryEngineTools);
+    }
+
+    const toolCallResponse = await chatWithTools(this.llm, tools, chatHistory);
+
+    if (!toolCallResponse.hasToolCall()) {
+      return new StopEvent(toolCallResponse.responseGenerator);
+    }
+
+    if (toolCallResponse.hasMultipleTools()) {
+      this.memory.put({
+        role: "assistant",
+        content:
+          "Calling different tools is not allowed. Please only use multiple calls of the same tool.",
+      });
+      return new InputEvent({ input: this.memory.getMessages() });
+    }
+
+    // Put the LLM tool call message into the memory
+    // And trigger the next step according to the tool call
+    if (toolCallResponse.toolCallMessage) {
+      this.memory.put(toolCallResponse.toolCallMessage);
+    }
+    const toolName = toolCallResponse.getToolNames()[0];
+    switch (toolName) {
+      case this.codeInterpreterTool.metadata.name:
+        return new AnalyzeEvent({
+          input: toolCallResponse.toolCalls,
+        });
+      case this.documentGeneratorTool.metadata.name:
+        return new ReportGenerationEvent({
+          toolCalls: toolCallResponse.toolCalls,
+        });
+      default:
+        if (
+          this.queryEngineTools &&
+          this.queryEngineTools.some((tool) => tool.metadata.name === toolName)
+        ) {
+          return new ResearchEvent({
+            toolCalls: toolCallResponse.toolCalls,
+          });
+        }
+        throw new Error(`Unknown tool: ${toolName}`);
+    }
+  };
+
+  handleResearch = async (
+    ctx: HandlerContext<null>,
+    ev: ResearchEvent,
+  ): Promise<AnalyzeEvent> => {
+    ctx.sendEvent(
+      new AgentRunEvent({
+        agent: "Researcher",
+        text: "Researching data",
+        type: "text",
+      }),
+    );
+
+    const { toolCalls } = ev.data;
+
+    const toolMsgs = await callTools({
+      tools: this.queryEngineTools,
+      toolCalls,
+      ctx,
+      agentName: "Researcher",
+    });
+    for (const toolMsg of toolMsgs) {
+      this.memory.put(toolMsg);
+    }
+    return new AnalyzeEvent({
+      input: {
+        role: "assistant",
+        content:
+          "I have finished researching the data, please analyze the data.",
+      },
+    });
+  };
+
+  /**
+   * Analyze a research result or a tool call for code interpreter from the LLM
+   */
+  handleAnalyze = async (
+    ctx: HandlerContext<null>,
+    ev: AnalyzeEvent,
+  ): Promise<InputEvent> => {
+    ctx.sendEvent(
+      new AgentRunEvent({
+        agent: "Analyst",
+        text: `Starting analysis`,
+        type: "text",
+      }),
+    );
+    // Request by workflow LLM, input is a list of tool calls
+    let toolCalls: ToolCall[] = [];
+    if (Array.isArray(ev.data.input)) {
+      toolCalls = ev.data.input;
+    } else {
+      // Requested by Researcher, input is a ChatMessage
+      // We start new LLM chat specifically for analyzing the data
+      const analysisPrompt = `
+      You are an expert in analyzing financial data.
+      You are given a set of financial data to analyze. Your task is to analyze the financial data and return a report.
+      Your response should include a detailed analysis of the financial data, including any trends, patterns, or insights that you find.
+      Construct the analysis in textual format; including tables would be great!
+      Don't need to synthesize the data, just analyze and provide your findings.
+      `;
+
+      // Clone the current chat history
+      // Add the analysis system prompt and the message from the researcher
+      const newChatHistory = [
+        ...this.memory.getMessages(),
+        { role: "system", content: analysisPrompt },
+        ev.data.input,
+      ];
+      const toolCallResponse = await chatWithTools(
+        this.llm,
+        [this.codeInterpreterTool],
+        newChatHistory as ChatMessage[],
+      );
+
+      if (!toolCallResponse.hasToolCall()) {
+        this.memory.put(await toolCallResponse.asFullResponse());
+        return new InputEvent({
+          input: this.memory.getMessages(),
+        });
+      } else {
+        this.memory.put(toolCallResponse.toolCallMessage);
+        toolCalls = toolCallResponse.toolCalls;
+      }
+    }
+
+    // Call the tools
+    const toolMsgs = await callTools({
+      tools: [this.codeInterpreterTool],
+      toolCalls,
+      ctx,
+      agentName: "Analyst",
+    });
+    for (const toolMsg of toolMsgs) {
+      this.memory.put(toolMsg);
+    }
+
+    return new InputEvent({
+      input: this.memory.getMessages(),
+    });
+  };
+
+  handleReportGeneration = async (
+    ctx: HandlerContext<null>,
+    ev: ReportGenerationEvent,
+  ): Promise<InputEvent> => {
+    const { toolCalls } = ev.data;
+
+    const toolMsgs = await callTools({
+      tools: [this.documentGeneratorTool],
+      toolCalls,
+      ctx,
+      agentName: "Reporter",
+    });
+    for (const toolMsg of toolMsgs) {
+      this.memory.put(toolMsg);
+    }
+    return new InputEvent({ input: this.memory.getMessages() });
+  };
+}

templates/components/agents/typescript/form_filling/README-template.md

@@ -0,0 +1,37 @@
+This is a [LlamaIndex](https://www.llamaindex.ai/) project using [Next.js](https://nextjs.org/) bootstrapped with [`create-llama`](https://github.com/run-llama/LlamaIndexTS/tree/main/packages/create-llama).
+
+## Getting Started
+
+First, install the dependencies:
+
+```
+npm install
+```
+
+Then check the parameters that have been pre-configured in the `.env` file in this directory.
+Make sure you have the `OPENAI_API_KEY` set.
+
+Second, run the development server:
+
+```
+npm run dev
+```
+
+Open [http://localhost:3000](http://localhost:3000) with your browser to see the chat UI.
+
+## Use Case: Filling Financial CSV Template
+
+1. Upload the Apple and Tesla financial reports from the [data](./data) directory. Just send an empty message.
+2. Upload the CSV file [sec_10k_template.csv](./sec_10k_template.csv) and send the message "Fill the missing cells in the CSV file".
+
+The agent will fill the missing cells by retrieving the information from the uploaded financial reports and return a new CSV file with the filled cells.
+
+## Learn More
+
+To learn more about LlamaIndex, take a look at the following resources:
+
+- [LlamaIndex Documentation](https://docs.llamaindex.ai) - learn about LlamaIndex (Python features).
+- [LlamaIndexTS Documentation](https://ts.llamaindex.ai/docs/llamaindex) - learn about LlamaIndex (Typescript features).
+- [Workflows Introduction](https://ts.llamaindex.ai/docs/llamaindex/guide/workflow) - learn about LlamaIndexTS workflows.
+
+You can check out [the LlamaIndexTS GitHub repository](https://github.com/run-llama/LlamaIndexTS) - your feedback and contributions are welcome!

templates/components/agents/typescript/form_filling/sec_10k_template.csv

@@ -0,0 +1,17 @@
+Parameter,2023 Apple (AAPL),2023 Tesla (TSLA)
+Revenue,,
+Net Income,,
+Earnings Per Share (EPS),,
+Debt-to-Equity Ratio,,
+Current Ratio,,
+Gross Margin,,
+Operating Margin,,
+Net Profit Margin,,
+Inventory Turnover,,
+Accounts Receivable Turnover,,
+Capital Expenditure,,
+Research and Development Expense,,
+Market Cap,,
+Price to Earnings Ratio,,
+Dividend Yield,,
+Year-over-Year Growth Rate,,

templates/components/agents/typescript/form_filling/workflow/factory.ts

@@ -0,0 +1,20 @@
+import { ChatMessage, ToolCallLLM } from "llamaindex";
+import { getTool } from "../engine/tools";
+import { FormFillingWorkflow } from "./form-filling";
+import { getQueryEngineTools } from "./tools";
+
+const TIMEOUT = 360 * 1000;
+
+export async function createWorkflow(options: {
+  chatHistory: ChatMessage[];
+  llm?: ToolCallLLM;
+}) {
+  return new FormFillingWorkflow({
+    chatHistory: options.chatHistory,
+    queryEngineTools: (await getQueryEngineTools()) || [],
+    extractorTool: (await getTool("extract_missing_cells"))!,
+    fillMissingCellsTool: (await getTool("fill_missing_cells"))!,
+    llm: options.llm,
+    timeout: TIMEOUT,
+  });
+}

templates/components/agents/typescript/form_filling/workflow/form-filling.ts

@@ -0,0 +1,275 @@
+import {
+  HandlerContext,
+  StartEvent,
+  StopEvent,
+  Workflow,
+  WorkflowEvent,
+} from "@llamaindex/workflow";
+import {
+  BaseToolWithCall,
+  ChatMemoryBuffer,
+  ChatMessage,
+  ChatResponseChunk,
+  Settings,
+  ToolCall,
+  ToolCallLLM,
+} from "llamaindex";
+import { callTools, chatWithTools } from "./tools";
+import { AgentInput, AgentRunEvent } from "./type";
+
+// Create a custom event type
+class InputEvent extends WorkflowEvent<{ input: ChatMessage[] }> {}
+
+class ExtractMissingCellsEvent extends WorkflowEvent<{
+  toolCalls: ToolCall[];
+}> {}
+
+class FindAnswersEvent extends WorkflowEvent<{
+  toolCalls: ToolCall[];
+}> {}
+
+class FillMissingCellsEvent extends WorkflowEvent<{
+  toolCalls: ToolCall[];
+}> {}
+
+const DEFAULT_SYSTEM_PROMPT = `
+You are a helpful assistant who helps fill missing cells in a CSV file.
+Only use the information from the retriever tool - don't make up any information yourself. Fill N/A if an answer is not found.
+If there is no retriever tool or the gathered information has many N/A values indicating the questions don't match the data, respond with a warning and ask the user to upload a different file or connect to a knowledge base.
+You can make multiple tool calls at once but only call with the same tool.
+Only use the local file path for the tools.
+`;
+
+export class FormFillingWorkflow extends Workflow<
+  null,
+  AgentInput,
+  ChatResponseChunk
+> {
+  llm: ToolCallLLM;
+  memory: ChatMemoryBuffer;
+  extractorTool: BaseToolWithCall;
+  queryEngineTools?: BaseToolWithCall[];
+  fillMissingCellsTool: BaseToolWithCall;
+  systemPrompt?: string;
+
+  constructor(options: {
+    llm?: ToolCallLLM;
+    chatHistory: ChatMessage[];
+    extractorTool: BaseToolWithCall;
+    queryEngineTools?: BaseToolWithCall[];
+    fillMissingCellsTool: BaseToolWithCall;
+    systemPrompt?: string;
+    verbose?: boolean;
+    timeout?: number;
+  }) {
+    super({
+      verbose: options?.verbose ?? false,
+      timeout: options?.timeout ?? 360,
+    });
+
+    this.llm = options.llm ?? (Settings.llm as ToolCallLLM);
+    if (!(this.llm instanceof ToolCallLLM)) {
+      throw new Error("LLM is not a ToolCallLLM");
+    }
+    this.systemPrompt = options.systemPrompt ?? DEFAULT_SYSTEM_PROMPT;
+    this.extractorTool = options.extractorTool;
+    this.queryEngineTools = options.queryEngineTools;
+    this.fillMissingCellsTool = options.fillMissingCellsTool;
+
+    this.memory = new ChatMemoryBuffer({
+      llm: this.llm,
+      chatHistory: options.chatHistory,
+    });
+
+    // Add steps
+    this.addStep(
+      {
+        inputs: [StartEvent<AgentInput>],
+        outputs: [InputEvent],
+      },
+      this.prepareChatHistory,
+    );
+
+    this.addStep(
+      {
+        inputs: [InputEvent],
+        outputs: [
+          InputEvent,
+          ExtractMissingCellsEvent,
+          FindAnswersEvent,
+          FillMissingCellsEvent,
+          StopEvent,
+        ],
+      },
+      this.handleLLMInput,
+    );
+
+    this.addStep(
+      {
+        inputs: [ExtractMissingCellsEvent],
+        outputs: [InputEvent],
+      },
+      this.handleExtractMissingCells,
+    );
+
+    this.addStep(
+      {
+        inputs: [FindAnswersEvent],
+        outputs: [InputEvent],
+      },
+      this.handleFindAnswers,
+    );
+
+    this.addStep(
+      {
+        inputs: [FillMissingCellsEvent],
+        outputs: [InputEvent],
+      },
+      this.handleFillMissingCells,
+    );
+  }
+
+  prepareChatHistory = async (
+    ctx: HandlerContext<null>,
+    ev: StartEvent<AgentInput>,
+  ): Promise<InputEvent> => {
+    const { message } = ev.data;
+
+    if (this.systemPrompt) {
+      this.memory.put({ role: "system", content: this.systemPrompt });
+    }
+    this.memory.put({ role: "user", content: message });
+
+    return new InputEvent({ input: this.memory.getMessages() });
+  };
+
+  handleLLMInput = async (
+    ctx: HandlerContext<null>,
+    ev: InputEvent,
+  ): Promise<
+    | InputEvent
+    | ExtractMissingCellsEvent
+    | FindAnswersEvent
+    | FillMissingCellsEvent
+    | StopEvent
+  > => {
+    const chatHistory = ev.data.input;
+
+    const tools = [this.extractorTool, this.fillMissingCellsTool];
+    if (this.queryEngineTools) {
+      tools.push(...this.queryEngineTools);
+    }
+
+    const toolCallResponse = await chatWithTools(this.llm, tools, chatHistory);
+
+    if (!toolCallResponse.hasToolCall()) {
+      return new StopEvent(toolCallResponse.responseGenerator);
+    }
+
+    if (toolCallResponse.hasMultipleTools()) {
+      this.memory.put({
+        role: "assistant",
+        content:
+          "Calling different tools is not allowed. Please only use multiple calls of the same tool.",
+      });
+      return new InputEvent({ input: this.memory.getMessages() });
+    }
+
+    // Put the LLM tool call message into the memory
+    // And trigger the next step according to the tool call
+    if (toolCallResponse.toolCallMessage) {
+      this.memory.put(toolCallResponse.toolCallMessage);
+    }
+    const toolName = toolCallResponse.getToolNames()[0];
+    switch (toolName) {
+      case this.extractorTool.metadata.name:
+        return new ExtractMissingCellsEvent({
+          toolCalls: toolCallResponse.toolCalls,
+        });
+      case this.fillMissingCellsTool.metadata.name:
+        return new FillMissingCellsEvent({
+          toolCalls: toolCallResponse.toolCalls,
+        });
+      default:
+        if (
+          this.queryEngineTools &&
+          this.queryEngineTools.some((tool) => tool.metadata.name === toolName)
+        ) {
+          return new FindAnswersEvent({
+            toolCalls: toolCallResponse.toolCalls,
+          });
+        }
+        throw new Error(`Unknown tool: ${toolName}`);
+    }
+  };
+
+  handleExtractMissingCells = async (
+    ctx: HandlerContext<null>,
+    ev: ExtractMissingCellsEvent,
+  ): Promise<InputEvent> => {
+    ctx.sendEvent(
+      new AgentRunEvent({
+        agent: "CSVExtractor",
+        text: "Extracting missing cells",
+        type: "text",
+      }),
+    );
+    const { toolCalls } = ev.data;
+    const toolMsgs = await callTools({
+      tools: [this.extractorTool],
+      toolCalls,
+      ctx,
+      agentName: "CSVExtractor",
+    });
+    for (const toolMsg of toolMsgs) {
+      this.memory.put(toolMsg);
+    }
+    return new InputEvent({ input: this.memory.getMessages() });
+  };
+
+  handleFindAnswers = async (
+    ctx: HandlerContext<null>,
+    ev: FindAnswersEvent,
+  ): Promise<InputEvent> => {
+    const { toolCalls } = ev.data;
+    if (!this.queryEngineTools) {
+      throw new Error("Query engine tool is not available");
+    }
+    ctx.sendEvent(
+      new AgentRunEvent({
+        agent: "Researcher",
+        text: "Finding answers",
+        type: "text",
+      }),
+    );
+    const toolMsgs = await callTools({
+      tools: this.queryEngineTools,
+      toolCalls,
+      ctx,
+      agentName: "Researcher",
+    });
+
+    for (const toolMsg of toolMsgs) {
+      this.memory.put(toolMsg);
+    }
+    return new InputEvent({ input: this.memory.getMessages() });
+  };
+
+  handleFillMissingCells = async (
+    ctx: HandlerContext<null>,
+    ev: FillMissingCellsEvent,
+  ): Promise<InputEvent> => {
+    const { toolCalls } = ev.data;
+
+    const toolMsgs = await callTools({
+      tools: [this.fillMissingCellsTool],
+      toolCalls,
+      ctx,
+      agentName: "Processor",
+    });
+    for (const toolMsg of toolMsgs) {
+      this.memory.put(toolMsg);
+    }
+    return new InputEvent({ input: this.memory.getMessages() });
+  };
+}

templates/components/engines/python/agent/tools/__init__.py

@@ -56,7 +56,7 @@ class ToolFactory:
             A dictionary of tool names to lists of FunctionTools if map_result is True,
             otherwise a list of FunctionTools.
         """
-        tools: Union[Dict[str, List[FunctionTool]], List[FunctionTool]] = (
+        tools: Union[Dict[str, FunctionTool], List[FunctionTool]] = (
             {} if map_result else []
         )
 
@@ -69,7 +69,9 @@ class ToolFactory:
                             tool_type, tool_name, config
                         )
                         if map_result:
-                            tools[tool_name] = loaded_tools  # type: ignore
+                            tools.update(  # type: ignore
+                                {tool.metadata.name: tool for tool in loaded_tools}
+                            )
                         else:
                             tools.extend(loaded_tools)  # type: ignore
 

templates/components/engines/python/agent/tools/form_filling.py

@@ -0,0 +1,224 @@
+import logging
+import os
+import uuid
+from textwrap import dedent
+from typing import Optional
+
+import pandas as pd
+from app.services.file import FileService
+from llama_index.core import Settings
+from llama_index.core.prompts import PromptTemplate
+from llama_index.core.tools import FunctionTool
+from pydantic import BaseModel, Field
+
+logger = logging.getLogger(__name__)
+
+
+class MissingCell(BaseModel):
+    """
+    A missing cell in a table.
+    """
+
+    row_index: int = Field(description="The index of the row of the missing cell")
+    column_index: int = Field(description="The index of the column of the missing cell")
+    question_to_answer: str = Field(
+        description="The question to answer to fill the missing cell"
+    )
+
+
+class MissingCells(BaseModel):
+    """
+    A list of missing cells.
+    """
+
+    missing_cells: list[MissingCell] = Field(description="The missing cells")
+
+
+class CellValue(BaseModel):
+    row_index: int = Field(description="The row index of the cell")
+    column_index: int = Field(description="The column index of the cell")
+    value: str = Field(
+        description="The value of the cell. Should be a concise value (numerical value or specific value)"
+    )
+
+
+class FormFillingTool:
+    """
+    Fill out missing cells in a CSV file using information from the knowledge base.
+    """
+
+    save_dir: str = os.path.join("output", "tools")
+
+    # Default prompt for extracting questions
+    # Replace the default prompt with a custom prompt by setting the EXTRACT_QUESTIONS_PROMPT environment variable.
+    _default_extract_questions_prompt = dedent(
+        """
+        You are a data analyst. You are given a table with missing cells.
+        Your task is to identify the missing cells and the questions needed to fill them.
+        IMPORTANT: Column indices should be 0-based, where the first data column is index 1 
+        (index 0 is typically the row names/index column).
+
+        # Instructions:
+        - Understand the entire content of the table and the topics of the table.
+        - Identify the missing cells and the meaning of the data in the cells.
+        - For each missing cell, provide the row index and the correct column index (remember: first data column is 1).
+        - For each missing cell, provide the question needed to fill the cell (it's important to provide the question that is relevant to the topic of the table).
+        - Since the cell's value should be concise, the question should request a numerical answer or a specific value.
+
+        # Example:
+        # |    | Name | Age | City |
+        # |----|------|-----|------|
+        # | 0  | John |     | Paris|
+        # | 1  | Mary |     |      |
+        # | 2  |      | 30  |      |
+        #
+        # Your thoughts:
+        # - The table is about people's names, ages, and cities.
+        # - Row: 1, Column: 1 (Age column), Question: "How old is Mary? Please provide only the numerical answer."
+        # - Row: 1, Column: 2 (City column), Question: "In which city does Mary live? Please provide only the city name."
+        
+
+        Please provide your answer in the requested format.
+        # Here is your task:
+
+        - Table content:
+        {table_content}
+
+        - Your answer:
+        """
+    )
+
+    def extract_questions(
+        self,
+        file_path: Optional[str] = None,
+        file_content: Optional[str] = None,
+    ) -> dict:
+        """
+        Use this tool to extract missing cells in a CSV file and generate questions to fill them.
+        Pass either the path to the CSV file or the content of the CSV file.
+
+        Args:
+            file_path (Optional[str]): The local file path to the CSV file to extract missing cells from (Don't pass a sandbox path).
+            file_content (Optional[str]): The content of the CSV file to extract missing cells from.
+
+        Returns:
+            dict: A dictionary containing the missing cells and their corresponding questions.
+        """
+        extract_questions_prompt = os.getenv(
+            "EXTRACT_QUESTIONS_PROMPT", self._default_extract_questions_prompt
+        )
+        if file_path is None and file_content is None:
+            raise ValueError("Either `file_path` or `file_content` must be provided")
+
+        table_content = None
+
+        if file_path:
+            file_name, file_extension = self._get_file_name_and_extension(
+                file_path, file_content
+            )
+
+            try:
+                df = pd.read_csv(file_path)
+            except FileNotFoundError as e:
+                return {
+                    "error": str(e),
+                    "message": "Please check and update the file path and ensure it's a local path - not a sandbox path.",
+                }
+
+            table_content = df.to_markdown()
+            if table_content is None:
+                raise ValueError("Could not convert the table to markdown")
+        if file_content:
+            table_content = file_content
+
+        if table_content is None:
+            raise ValueError("Table content not found")
+
+        response: MissingCells = Settings.llm.structured_predict(
+            output_cls=MissingCells,
+            prompt=PromptTemplate(extract_questions_prompt),
+            table_content=table_content,
+        )
+        return response.model_dump()
+
+    def fill_form(
+        self,
+        cell_values: list[CellValue],
+        file_path: Optional[str] = None,
+        file_content: Optional[str] = None,
+    ) -> dict:
+        """
+        Use this tool to fill cell values into a CSV file.
+        Requires cell values to be used for filling out, as well as either the path to the CSV file or the content of the CSV file.
+
+        Args:
+            cell_values (list[CellValue]): The cell values used to fill out the CSV file (call `extract_questions` and query engine to construct the cell values).
+            file_path (Optional[str]): The local file path to the CSV file that should be filled out (not as sandbox path).
+            file_content (Optional[str]): The content of the CSV file that should be filled out.
+
+        Returns:
+            dict: A dictionary containing the content and metadata of the filled-out file.
+        """
+        file_name, file_extension = self._get_file_name_and_extension(
+            file_path, file_content
+        )
+        df = pd.read_csv(file_path)
+
+        # Fill the dataframe with the cell values
+        filled_df = df.copy()
+        for cell_value in cell_values:
+            if not isinstance(cell_value, CellValue):
+                cell_value = CellValue(**cell_value)
+            filled_df.iloc[cell_value.row_index, cell_value.column_index] = (
+                cell_value.value
+            )
+
+        # Save the filled table to a new CSV file
+        csv_content: str = filled_df.to_csv(index=False)
+        file_metadata = FileService.save_file(
+            content=csv_content,
+            file_name=f"{file_name}_filled.csv",
+            save_dir=self.save_dir,
+        )
+
+        new_content: str = filled_df.to_markdown()
+        result = {
+            "filled_content": new_content,
+            "filled_file": file_metadata,
+        }
+        return result
+
+    def _get_file_name_and_extension(
+        self, file_path: Optional[str], file_content: Optional[str]
+    ) -> tuple[str, str]:
+        if file_path is None and file_content is None:
+            raise ValueError("Either `file_path` or `file_content` must be provided")
+
+        if file_path is None:
+            file_name = str(uuid.uuid4())
+            file_extension = ".csv"
+        else:
+            file_name, file_extension = os.path.splitext(file_path)
+            if file_extension != ".csv":
+                raise ValueError("Form filling is only supported for CSV files")
+
+        return file_name, file_extension
+
+    def _save_output(self, file_name: str, output: str) -> dict:
+        """
+        Save the output to a file.
+        """
+        file_metadata = FileService.save_file(
+            content=output,
+            file_name=file_name,
+            save_dir=self.save_dir,
+        )
+        return file_metadata.model_dump()
+
+
+def get_tools(**kwargs):
+    tool = FormFillingTool()
+    return [
+        FunctionTool.from_defaults(tool.extract_questions),
+        FunctionTool.from_defaults(tool.fill_form),
+    ]

templates/components/engines/typescript/agent/chat.ts

@@ -1,7 +1,7 @@
 import {
   BaseChatEngine,
   BaseToolWithCall,
-  OpenAIAgent,
+  LLMAgent,
   QueryEngineTool,
 } from "llamaindex";
 import fs from "node:fs/promises";
@@ -42,7 +42,7 @@ export async function createChatEngine(documentIds?: string[], params?: any) {
     tools.push(...(await createTools(toolConfig)));
   }
 
-  const agent = new OpenAIAgent({
+  const agent = new LLMAgent({
     tools,
     systemPrompt: process.env.SYSTEM_PROMPT,
   }) as unknown as BaseChatEngine;

templates/components/engines/typescript/agent/tools/form-filling.ts

@@ -0,0 +1,296 @@
+import { JSONSchemaType } from "ajv";
+import fs from "fs";
+import { BaseTool, Settings, ToolMetadata } from "llamaindex";
+import Papa from "papaparse";
+import path from "path";
+import { saveDocument } from "../../llamaindex/documents/helper";
+
+type ExtractMissingCellsParameter = {
+  filePath: string;
+};
+
+export type MissingCell = {
+  rowIndex: number;
+  columnIndex: number;
+  question: string;
+};
+
+const CSV_EXTRACTION_PROMPT = `You are a data analyst. You are given a table with missing cells.
+Your task is to identify the missing cells and the questions needed to fill them.
+IMPORTANT: Column indices should be 0-based
+
+# Instructions:
+- Understand the entire content of the table and the topics of the table.
+- Identify the missing cells and the meaning of the data in the cells.
+- For each missing cell, provide the row index and the correct column index (remember: first data column is 1).
+- For each missing cell, provide the question needed to fill the cell (it's important to provide the question that is relevant to the topic of the table).
+- Since the cell's value should be concise, the question should request a numerical answer or a specific value.
+- Finally, only return the answer in JSON format with the following schema:
+{
+  "missing_cells": [
+    {
+      "rowIndex": number,
+      "columnIndex": number,
+      "question": string
+    }
+  ]
+}
+- If there are no missing cells, return an empty array.
+- The answer is only the JSON object, nothing else and don't wrap it inside markdown code block.
+
+# Example:
+# |    | Name | Age | City |
+# |----|------|-----|------|
+# | 0  | John |     | Paris|
+# | 1  | Mary |     |      |
+# | 2  |      | 30  |      |
+#
+# Your thoughts:
+# - The table is about people's names, ages, and cities.
+# - Row: 1, Column: 2 (Age column), Question: "How old is Mary? Please provide only the numerical answer."
+# - Row: 1, Column: 3 (City column), Question: "In which city does Mary live? Please provide only the city name."
+# Your answer:
+# {
+#   "missing_cells": [
+#     {
+#       "rowIndex": 1,
+#       "columnIndex": 2,
+#       "question": "How old is Mary? Please provide only the numerical answer."
+#     },
+#     {
+#       "rowIndex": 1,
+#       "columnIndex": 3,
+#       "question": "In which city does Mary live? Please provide only the city name."
+#     }
+#   ]
+# }
+
+
+# Here is your task:
+
+- Table content:
+{table_content}
+
+- Your answer:
+`;
+
+const DEFAULT_METADATA: ToolMetadata<
+  JSONSchemaType<ExtractMissingCellsParameter>
+> = {
+  name: "extract_missing_cells",
+  description: `Use this tool to extract missing cells in a CSV file and generate questions to fill them. This tool only works with local file path.`,
+  parameters: {
+    type: "object",
+    properties: {
+      filePath: {
+        type: "string",
+        description: "The local file path to the CSV file.",
+      },
+    },
+    required: ["filePath"],
+  },
+};
+
+export interface ExtractMissingCellsParams {
+  metadata?: ToolMetadata<JSONSchemaType<ExtractMissingCellsParameter>>;
+}
+
+export class ExtractMissingCellsTool
+  implements BaseTool<ExtractMissingCellsParameter>
+{
+  metadata: ToolMetadata<JSONSchemaType<ExtractMissingCellsParameter>>;
+  defaultExtractionPrompt: string;
+
+  constructor(params: ExtractMissingCellsParams) {
+    this.metadata = params.metadata ?? DEFAULT_METADATA;
+    this.defaultExtractionPrompt = CSV_EXTRACTION_PROMPT;
+  }
+
+  private readCsvFile(filePath: string): Promise<string[][]> {
+    return new Promise((resolve, reject) => {
+      fs.readFile(filePath, "utf8", (err, data) => {
+        if (err) {
+          reject(err);
+          return;
+        }
+
+        const parsedData = Papa.parse<string[]>(data, {
+          skipEmptyLines: false,
+        });
+
+        if (parsedData.errors.length) {
+          reject(parsedData.errors);
+          return;
+        }
+
+        // Ensure all rows have the same number of columns as the header
+        const maxColumns = parsedData.data[0].length;
+        const paddedRows = parsedData.data.map((row) => {
+          return [...row, ...Array(maxColumns - row.length).fill("")];
+        });
+
+        resolve(paddedRows);
+      });
+    });
+  }
+
+  private formatToMarkdownTable(data: string[][]): string {
+    if (data.length === 0) return "";
+
+    const maxColumns = data[0].length;
+
+    const headerRow = `| ${data[0].join(" | ")} |`;
+    const separatorRow = `| ${Array(maxColumns).fill("---").join(" | ")} |`;
+
+    const dataRows = data.slice(1).map((row) => {
+      return `| ${row.join(" | ")} |`;
+    });
+
+    return [headerRow, separatorRow, ...dataRows].join("\n");
+  }
+
+  async call(input: ExtractMissingCellsParameter): Promise<MissingCell[]> {
+    const { filePath } = input;
+    let tableContent: string[][];
+    try {
+      tableContent = await this.readCsvFile(filePath);
+    } catch (error) {
+      throw new Error(
+        `Failed to read CSV file. Make sure that you are reading a local file path (not a sandbox path).`,
+      );
+    }
+
+    const prompt = this.defaultExtractionPrompt.replace(
+      "{table_content}",
+      this.formatToMarkdownTable(tableContent),
+    );
+
+    const llm = Settings.llm;
+    const response = await llm.complete({
+      prompt,
+    });
+    const rawAnswer = response.text;
+    const parsedResponse = JSON.parse(rawAnswer) as {
+      missing_cells: MissingCell[];
+    };
+    if (!parsedResponse.missing_cells) {
+      throw new Error(
+        "The answer is not in the correct format. There should be a missing_cells array.",
+      );
+    }
+    const answer = parsedResponse.missing_cells;
+
+    return answer;
+  }
+}
+
+type FillMissingCellsParameter = {
+  filePath: string;
+  cells: {
+    rowIndex: number;
+    columnIndex: number;
+    answer: string;
+  }[];
+};
+
+const FILL_CELLS_METADATA: ToolMetadata<
+  JSONSchemaType<FillMissingCellsParameter>
+> = {
+  name: "fill_missing_cells",
+  description: `Use this tool to fill missing cells in a CSV file with provided answers. This tool only works with local file path.`,
+  parameters: {
+    type: "object",
+    properties: {
+      filePath: {
+        type: "string",
+        description: "The local file path to the CSV file.",
+      },
+      cells: {
+        type: "array",
+        items: {
+          type: "object",
+          properties: {
+            rowIndex: { type: "number" },
+            columnIndex: { type: "number" },
+            answer: { type: "string" },
+          },
+          required: ["rowIndex", "columnIndex", "answer"],
+        },
+        description: "Array of cells to fill with their answers",
+      },
+    },
+    required: ["filePath", "cells"],
+  },
+};
+
+export interface FillMissingCellsParams {
+  metadata?: ToolMetadata<JSONSchemaType<FillMissingCellsParameter>>;
+}
+
+export class FillMissingCellsTool
+  implements BaseTool<FillMissingCellsParameter>
+{
+  metadata: ToolMetadata<JSONSchemaType<FillMissingCellsParameter>>;
+
+  constructor(params: FillMissingCellsParams = {}) {
+    this.metadata = params.metadata ?? FILL_CELLS_METADATA;
+  }
+
+  async call(input: FillMissingCellsParameter): Promise<string> {
+    const { filePath, cells } = input;
+
+    // Read the CSV file
+    const fileContent = await new Promise<string>((resolve, reject) => {
+      fs.readFile(filePath, "utf8", (err, data) => {
+        if (err) {
+          reject(err);
+        } else {
+          resolve(data);
+        }
+      });
+    });
+
+    // Parse CSV with PapaParse
+    const parseResult = Papa.parse<string[]>(fileContent, {
+      header: false, // Ensure the header is not treated as a separate object
+      skipEmptyLines: false, // Ensure empty lines are not skipped
+    });
+
+    if (parseResult.errors.length) {
+      throw new Error(
+        "Failed to parse CSV file: " + parseResult.errors[0].message,
+      );
+    }
+
+    const rows = parseResult.data;
+
+    // Fill the cells with answers
+    for (const cell of cells) {
+      // Adjust rowIndex to start from 1 for data rows
+      const adjustedRowIndex = cell.rowIndex + 1;
+      if (
+        adjustedRowIndex < rows.length &&
+        cell.columnIndex < rows[adjustedRowIndex].length
+      ) {
+        rows[adjustedRowIndex][cell.columnIndex] = cell.answer;
+      }
+    }
+
+    // Convert back to CSV format
+    const updatedContent = Papa.unparse(rows, {
+      delimiter: parseResult.meta.delimiter,
+    });
+
+    // Use the helper function to write the file
+    const parsedPath = path.parse(filePath);
+    const newFileName = `${parsedPath.name}-filled${parsedPath.ext}`;
+    const newFilePath = path.join("output/tools", newFileName);
+
+    const newFileUrl = await saveDocument(newFilePath, updatedContent);
+
+    return (
+      "Successfully filled missing cells in the CSV file. File URL to show to the user: " +
+      newFileUrl
+    );
+  }
+}

templates/components/engines/typescript/agent/tools/index.ts

@@ -1,11 +1,19 @@
 import { BaseToolWithCall } from "llamaindex";
 import { ToolsFactory } from "llamaindex/tools/ToolsFactory";
+import fs from "node:fs/promises";
+import path from "node:path";
 import { CodeGeneratorTool, CodeGeneratorToolParams } from "./code-generator";
 import {
   DocumentGenerator,
   DocumentGeneratorParams,
 } from "./document-generator";
 import { DuckDuckGoSearchTool, DuckDuckGoToolParams } from "./duckduckgo";
+import {
+  ExtractMissingCellsParams,
+  ExtractMissingCellsTool,
+  FillMissingCellsParams,
+  FillMissingCellsTool,
+} from "./form-filling";
 import { ImgGeneratorTool, ImgGeneratorToolParams } from "./img-gen";
 import { InterpreterTool, InterpreterToolParams } from "./interpreter";
 import { OpenAPIActionTool } from "./openapi-action";
@@ -54,6 +62,12 @@ const toolFactory: Record<string, ToolCreator> = {
   document_generator: async (config: unknown) => {
     return [new DocumentGenerator(config as DocumentGeneratorParams)];
   },
+  form_filling: async (config: unknown) => {
+    return [
+      new ExtractMissingCellsTool(config as ExtractMissingCellsParams),
+      new FillMissingCellsTool(config as FillMissingCellsParams),
+    ];
+  },
 };
 
 async function createLocalTools(
@@ -70,3 +84,19 @@ async function createLocalTools(
 
   return tools;
 }
+
+export async function getConfiguredTools(
+  configPath?: string,
+): Promise<BaseToolWithCall[]> {
+  const configFile = path.join(configPath ?? "config", "tools.json");
+  const toolConfig = JSON.parse(await fs.readFile(configFile, "utf8"));
+  const tools = await createTools(toolConfig);
+  return tools;
+}
+
+export async function getTool(
+  toolName: string,
+): Promise<BaseToolWithCall | undefined> {
+  const tools = await getConfiguredTools();
+  return tools.find((tool) => tool.metadata.name === toolName);
+}

templates/components/llamaindex/typescript/documents/helper.ts

@@ -13,7 +13,7 @@ const MIME_TYPE_TO_EXT: Record<string, string> = {
     "docx",
 };
 
-const UPLOADED_FOLDER = "output/uploaded";
+export const UPLOADED_FOLDER = "output/uploaded";
 
 export async function storeAndParseFile(
   name: string,

templates/components/llamaindex/typescript/documents/pipeline.ts

@@ -3,6 +3,7 @@ import {
   IngestionPipeline,
   Settings,
   SimpleNodeParser,
+  storageContextFromDefaults,
   VectorStoreIndex,
 } from "llamaindex";
 
@@ -28,11 +29,20 @@ export async function runPipeline(
     return documents.map((document) => document.id_);
   } else {
     // Initialize a new index with the documents
-    const newIndex = await VectorStoreIndex.fromDocuments(documents);
-    newIndex.storageContext.docStore.persist();
     console.log(
       "Got empty index, created new index with the uploaded documents",
     );
+    const persistDir = process.env.STORAGE_CACHE_DIR;
+    if (!persistDir) {
+      throw new Error("STORAGE_CACHE_DIR environment variable is required!");
+    }
+    const storageContext = await storageContextFromDefaults({
+      persistDir,
+    });
+    const newIndex = await VectorStoreIndex.fromDocuments(documents, {
+      storageContext,
+    });
+    await newIndex.storageContext.docStore.persist();
     return documents.map((document) => document.id_);
   }
 }

templates/components/llamaindex/typescript/documents/upload.ts

@@ -1,7 +1,5 @@
 import { Document, LLamaCloudFileService, VectorStoreIndex } from "llamaindex";
 import { LlamaCloudIndex } from "llamaindex/cloud/LlamaCloudIndex";
-import fs from "node:fs/promises";
-import path from "node:path";
 import { DocumentFile } from "../streaming/annotations";
 import { parseFile, storeFile } from "./helper";
 import { runPipeline } from "./pipeline";
@@ -18,8 +16,8 @@ export async function uploadDocument(
   // Store file
   const fileMetadata = await storeFile(name, fileBuffer, mimeType);
 
-  // If the file is csv and has codeExecutorTool, we don't need to index the file.
-  if (mimeType === "text/csv" && (await hasCodeExecutorTool())) {
+  // Do not index csv files
+  if (mimeType === "text/csv") {
     return fileMetadata;
   }
   let documentIds: string[] = [];
@@ -49,7 +47,11 @@ export async function uploadDocument(
     }
   } else {
     // run the pipeline for other vector store indexes
-    const documents: Document[] = await parseFile(fileBuffer, name, mimeType);
+    const documents: Document[] = await parseFile(
+      fileBuffer,
+      fileMetadata.name,
+      mimeType,
+    );
     documentIds = await runPipeline(index, documents);
   }
 
@@ -57,14 +59,3 @@ export async function uploadDocument(
   fileMetadata.refs = documentIds;
   return fileMetadata;
 }
-
-const hasCodeExecutorTool = async () => {
-  const codeExecutorTools = ["interpreter", "artifact"];
-
-  const configFile = path.join("config", "tools.json");
-  const toolConfig = JSON.parse(await fs.readFile(configFile, "utf8"));
-
-  const localTools = toolConfig.local || {};
-  // Check if local tools contains codeExecutorTools
-  return codeExecutorTools.some((tool) => localTools[tool] !== undefined);
-};

templates/components/llamaindex/typescript/streaming/annotations.ts

@@ -1,5 +1,11 @@
 import { JSONValue, Message } from "ai";
-import { MessageContent, MessageContentDetail } from "llamaindex";
+import {
+  ChatMessage,
+  MessageContent,
+  MessageContentDetail,
+  MessageType,
+} from "llamaindex";
+import { UPLOADED_FOLDER } from "../documents/helper";
 
 export type DocumentFileType = "csv" | "pdf" | "txt" | "docx";
 
@@ -58,6 +64,45 @@ export function retrieveMessageContent(messages: Message[]): MessageContent {
   ];
 }
 
+export function convertToChatHistory(messages: Message[]): ChatMessage[] {
+  if (!messages || !Array.isArray(messages)) {
+    return [];
+  }
+  const agentHistory = retrieveAgentHistoryMessage(messages);
+  if (agentHistory) {
+    const previousMessages = messages.slice(0, -1);
+    return [...previousMessages, agentHistory].map((msg) => ({
+      role: msg.role as MessageType,
+      content: msg.content,
+    }));
+  }
+  return messages.map((msg) => ({
+    role: msg.role as MessageType,
+    content: msg.content,
+  }));
+}
+
+function retrieveAgentHistoryMessage(
+  messages: Message[],
+  maxAgentMessages = 10,
+): ChatMessage | null {
+  const agentAnnotations = getAnnotations<{ agent: string; text: string }>(
+    messages,
+    { role: "assistant", type: "agent" },
+  ).slice(-maxAgentMessages);
+
+  if (agentAnnotations.length > 0) {
+    const messageContent =
+      "Here is the previous conversation of agents:\n" +
+      agentAnnotations.map((annotation) => annotation.data.text).join("\n");
+    return {
+      role: "assistant",
+      content: messageContent,
+    };
+  }
+  return null;
+}
+
 function getFileContent(file: DocumentFile): string {
   let defaultContent = `=====File: ${file.name}=====\n`;
   // Include file URL if it's available
@@ -84,6 +129,10 @@ function getFileContent(file: DocumentFile): string {
   const sandboxFilePath = `/tmp/${file.name}`;
   defaultContent += `Sandbox file path (instruction: only use sandbox path for artifact or code interpreter tool): ${sandboxFilePath}\n`;
 
+  // Include local file path
+  const localFilePath = `${UPLOADED_FOLDER}/${file.name}`;
+  defaultContent += `Local file path (instruction: use for local tool that requires a local path): ${localFilePath}\n`;
+
   return defaultContent;
 }
 
@@ -127,13 +176,10 @@ function retrieveLatestArtifact(messages: Message[]): MessageContentDetail[] {
 }
 
 function convertAnnotations(messages: Message[]): MessageContentDetail[] {
-  // annotations from the last user message that has annotations
-  const annotations: Annotation[] =
-    messages
-      .slice()
-      .reverse()
-      .find((message) => message.role === "user" && message.annotations)
-      ?.annotations?.map(getValidAnnotation) || [];
+  // get all annotations from user messages
+  const annotations: Annotation[] = messages
+    .filter((message) => message.role === "user" && message.annotations)
+    .flatMap((message) => message.annotations?.map(getValidAnnotation) || []);
   if (annotations.length === 0) return [];
 
   const content: MessageContentDetail[] = [];

templates/components/loaders/typescript/file/loader.ts

@@ -1,7 +1,7 @@
 import {
   FILE_EXT_TO_READER,
   SimpleDirectoryReader,
-} from "llamaindex/readers/SimpleDirectoryReader";
+} from "llamaindex/readers/index";
 
 export const DATA_DIR = "./data";
 

templates/components/loaders/typescript/llama_parse/loader.ts

@@ -2,7 +2,7 @@ import { LlamaParseReader } from "llamaindex";
 import {
   FILE_EXT_TO_READER,
   SimpleDirectoryReader,
-} from "llamaindex/readers/SimpleDirectoryReader";
+} from "llamaindex/readers/index";
 
 export const DATA_DIR = "./data";
 

templates/components/multiagent/python/app/api/routers/chat.py

@@ -4,7 +4,8 @@ from app.api.routers.models import (
     ChatData,
 )
 from app.api.routers.vercel_response import VercelStreamResponse
-from app.engine.engine import get_chat_engine
+from app.engine.query_filter import generate_filters
+from app.workflows import create_workflow
 from fastapi import APIRouter, BackgroundTasks, HTTPException, Request, status
 
 chat_router = r = APIRouter()
@@ -22,19 +23,20 @@ async def chat(
         last_message_content = data.get_last_message_content()
         messages = data.get_history_messages(include_agent_messages=True)
 
-        # The chat API supports passing private document filters and chat params
-        # but agent workflow does not support them yet
-        # ignore chat params and use all documents for now
-        # TODO: generate filters based on doc_ids
+        doc_ids = data.get_chat_document_ids()
+        filters = generate_filters(doc_ids)
         params = data.data or {}
-        engine = get_chat_engine(chat_history=messages, params=params)
 
-        event_handler = engine.run(input=last_message_content, streaming=True)
+        workflow = create_workflow(
+            chat_history=messages, params=params, filters=filters
+        )
+
+        event_handler = workflow.run(input=last_message_content, streaming=True)
         return VercelStreamResponse(
             request=request,
             chat_data=data,
             event_handler=event_handler,
-            events=engine.stream_events(),
+            events=workflow.stream_events(),
         )
     except Exception as e:
         logger.exception("Error in chat engine", exc_info=True)

templates/components/multiagent/python/app/api/routers/vercel_response.py

@@ -1,10 +1,9 @@
 import asyncio
 import json
 import logging
-from typing import AsyncGenerator, List
+from typing import AsyncGenerator, Awaitable, List
 
 from aiostream import stream
-from app.agents.single import AgentRunEvent, AgentRunResult
 from app.api.routers.models import ChatData, Message
 from app.api.services.suggestion import NextQuestionSuggestion
 from fastapi import Request
@@ -28,7 +27,6 @@ class VercelStreamResponse(StreamingResponse):
         super().__init__(content=content)
 
     async def content_generator(self, event_handler, events):
-        logger.info("Starting content_generator")
         stream = self._create_stream(
             self.request, self.chat_data, event_handler, events
         )
@@ -56,8 +54,8 @@ class VercelStreamResponse(StreamingResponse):
         self,
         request: Request,
         chat_data: ChatData,
-        event_handler: AgentRunResult | AsyncGenerator,
-        events: AsyncGenerator[AgentRunEvent, None],
+        event_handler: Awaitable,
+        events: AsyncGenerator,
         verbose: bool = True,
     ):
         # Yield the text response
@@ -65,15 +63,17 @@ class VercelStreamResponse(StreamingResponse):
             result = await event_handler
             final_response = ""
 
-            if isinstance(result, AgentRunResult):
-                for token in result.response.message.content:
-                    final_response += token
-                    yield self.convert_text(token)
-
             if isinstance(result, AsyncGenerator):
                 async for token in result:
-                    final_response += token.delta
+                    final_response += str(token.delta)
                     yield self.convert_text(token.delta)
+            else:
+                if hasattr(result, "response"):
+                    content = result.response.message.content
+                    if content:
+                        for token in content:
+                            final_response += str(token)
+                            yield self.convert_text(token)
 
             # Generate next questions if next question prompt is configured
             question_data = await self._generate_next_questions(
@@ -87,7 +87,7 @@ class VercelStreamResponse(StreamingResponse):
         # Yield the events from the event handler
         async def _event_generator():
             async for event in events:
-                event_response = self._event_to_response(event)
+                event_response = event.to_response()
                 if verbose:
                     logger.debug(event_response)
                 if event_response is not None:
@@ -96,13 +96,6 @@ class VercelStreamResponse(StreamingResponse):
         combine = stream.merge(_chat_response_generator(), _event_generator())
         return combine
 
-    @staticmethod
-    def _event_to_response(event: AgentRunEvent) -> dict:
-        return {
-            "type": "agent",
-            "data": {"agent": event.name, "text": event.msg},
-        }
-
     @classmethod
     def convert_text(cls, token: str):
         # Escape newlines and double quotes to avoid breaking the stream

templates/components/multiagent/python/app/workflows/events.py

@@ -0,0 +1,27 @@
+from enum import Enum
+from typing import Optional
+
+from llama_index.core.workflow import Event
+
+
+class AgentRunEventType(Enum):
+    TEXT = "text"
+    PROGRESS = "progress"
+
+
+class AgentRunEvent(Event):
+    name: str
+    msg: str
+    event_type: AgentRunEventType = AgentRunEventType.TEXT
+    data: Optional[dict] = None
+
+    def to_response(self) -> dict:
+        return {
+            "type": "agent",
+            "data": {
+                "agent": self.name,
+                "type": self.event_type.value,
+                "text": self.msg,
+                "data": self.data,
+            },
+        }

templates/components/multiagent/python/app/workflows/function_calling_agent.py

@@ -0,0 +1,121 @@
+from typing import Any, List, Optional
+
+from app.workflows.events import AgentRunEvent
+from app.workflows.tools import ToolCallResponse, call_tools, chat_with_tools
+from llama_index.core.base.llms.types import ChatMessage
+from llama_index.core.llms.function_calling import FunctionCallingLLM
+from llama_index.core.memory import ChatMemoryBuffer
+from llama_index.core.settings import Settings
+from llama_index.core.tools.types import BaseTool
+from llama_index.core.workflow import (
+    Context,
+    Event,
+    StartEvent,
+    StopEvent,
+    Workflow,
+    step,
+)
+
+
+class InputEvent(Event):
+    input: list[ChatMessage]
+
+
+class ToolCallEvent(Event):
+    input: ToolCallResponse
+
+
+class FunctionCallingAgent(Workflow):
+    """
+    A simple workflow to request LLM with tools independently.
+    You can share the previous chat history to provide the context for the LLM.
+    """
+
+    def __init__(
+        self,
+        *args: Any,
+        llm: FunctionCallingLLM | None = None,
+        chat_history: Optional[List[ChatMessage]] = None,
+        tools: List[BaseTool] | None = None,
+        system_prompt: str | None = None,
+        verbose: bool = False,
+        timeout: float = 360.0,
+        name: str,
+        write_events: bool = True,
+        **kwargs: Any,
+    ) -> None:
+        super().__init__(*args, verbose=verbose, timeout=timeout, **kwargs)  # type: ignore
+        self.tools = tools or []
+        self.name = name
+        self.write_events = write_events
+
+        if llm is None:
+            llm = Settings.llm
+        self.llm = llm
+        if not self.llm.metadata.is_function_calling_model:
+            raise ValueError("The provided LLM must support function calling.")
+
+        self.system_prompt = system_prompt
+
+        self.memory = ChatMemoryBuffer.from_defaults(
+            llm=self.llm, chat_history=chat_history
+        )
+        self.sources = []  # type: ignore
+
+    @step()
+    async def prepare_chat_history(self, ctx: Context, ev: StartEvent) -> InputEvent:
+        # clear sources
+        self.sources = []
+
+        # set streaming
+        ctx.data["streaming"] = getattr(ev, "streaming", False)
+
+        # set system prompt
+        if self.system_prompt is not None:
+            system_msg = ChatMessage(role="system", content=self.system_prompt)
+            self.memory.put(system_msg)
+
+        # get user input
+        user_input = ev.input
+        user_msg = ChatMessage(role="user", content=user_input)
+        self.memory.put(user_msg)
+
+        if self.write_events:
+            ctx.write_event_to_stream(
+                AgentRunEvent(name=self.name, msg=f"Start to work on: {user_input}")
+            )
+
+        return InputEvent(input=self.memory.get())
+
+    @step()
+    async def handle_llm_input(
+        self,
+        ctx: Context,
+        ev: InputEvent,
+    ) -> ToolCallEvent | StopEvent:
+        chat_history = ev.input
+
+        response = await chat_with_tools(
+            self.llm,
+            self.tools,
+            chat_history,
+        )
+        is_tool_call = isinstance(response, ToolCallResponse)
+        if not is_tool_call:
+            if ctx.data["streaming"]:
+                return StopEvent(result=response)
+            else:
+                full_response = ""
+                async for chunk in response.generator:
+                    full_response += chunk.message.content
+                return StopEvent(result=full_response)
+        return ToolCallEvent(input=response)
+
+    @step()
+    async def handle_tool_calls(self, ctx: Context, ev: ToolCallEvent) -> InputEvent:
+        tool_calls = ev.input.tool_calls
+        tool_call_message = ev.input.tool_call_message
+        self.memory.put(tool_call_message)
+        tool_messages = await call_tools(self.name, self.tools, ctx, tool_calls)
+        self.memory.put_messages(tool_messages)
+        return InputEvent(input=self.memory.get())

templates/components/multiagent/python/app/workflows/tools.py

@@ -0,0 +1,227 @@
+import logging
+import uuid
+from abc import ABC, abstractmethod
+from typing import Any, AsyncGenerator, Callable, Optional
+
+from app.workflows.events import AgentRunEvent, AgentRunEventType
+from llama_index.core.base.llms.types import ChatMessage, ChatResponse, MessageRole
+from llama_index.core.llms.function_calling import FunctionCallingLLM
+from llama_index.core.tools import (
+    BaseTool,
+    FunctionTool,
+    ToolOutput,
+    ToolSelection,
+)
+from llama_index.core.workflow import Context
+from pydantic import BaseModel, ConfigDict
+
+logger = logging.getLogger("uvicorn")
+
+
+class ContextAwareTool(FunctionTool, ABC):
+    @abstractmethod
+    async def acall(self, ctx: Context, input: Any) -> ToolOutput:  # type: ignore
+        pass
+
+
+class ChatWithToolsResponse(BaseModel):
+    """
+    A tool call response from chat_with_tools.
+    """
+
+    tool_calls: Optional[list[ToolSelection]]
+    tool_call_message: Optional[ChatMessage]
+    generator: Optional[AsyncGenerator[ChatResponse | None, None]]
+
+    model_config = ConfigDict(arbitrary_types_allowed=True)
+
+    def is_calling_different_tools(self) -> bool:
+        tool_names = {tool_call.tool_name for tool_call in self.tool_calls}
+        return len(tool_names) > 1
+
+    def has_tool_calls(self) -> bool:
+        return self.tool_calls is not None and len(self.tool_calls) > 0
+
+    def tool_name(self) -> str:
+        assert self.has_tool_calls()
+        assert not self.is_calling_different_tools()
+        return self.tool_calls[0].tool_name
+
+    async def full_response(self) -> str:
+        assert self.generator is not None
+        full_response = ""
+        async for chunk in self.generator:
+            full_response += chunk.message.content
+        return full_response
+
+
+async def chat_with_tools(  # type: ignore
+    llm: FunctionCallingLLM,
+    tools: list[BaseTool],
+    chat_history: list[ChatMessage],
+) -> ChatWithToolsResponse:
+    """
+    Request LLM to call tools or not.
+    This function doesn't change the memory.
+    """
+    generator = _tool_call_generator(llm, tools, chat_history)
+    is_tool_call = await generator.__anext__()
+    if is_tool_call:
+        # Last chunk is the full response
+        # Wait for the last chunk
+        full_response = None
+        async for chunk in generator:
+            full_response = chunk
+        assert isinstance(full_response, ChatResponse)
+        return ChatWithToolsResponse(
+            tool_calls=llm.get_tool_calls_from_response(full_response),
+            tool_call_message=full_response.message,
+            generator=None,
+        )
+    else:
+        return ChatWithToolsResponse(
+            tool_calls=None,
+            tool_call_message=None,
+            generator=generator,
+        )
+
+
+async def call_tools(
+    ctx: Context,
+    agent_name: str,
+    tools: list[BaseTool],
+    tool_calls: list[ToolSelection],
+    emit_agent_events: bool = True,
+) -> list[ChatMessage]:
+    if len(tool_calls) == 0:
+        return []
+
+    tools_by_name = {tool.metadata.get_name(): tool for tool in tools}
+    if len(tool_calls) == 1:
+        return [
+            await call_tool(
+                ctx,
+                tools_by_name[tool_calls[0].tool_name],
+                tool_calls[0],
+                lambda msg: ctx.write_event_to_stream(
+                    AgentRunEvent(
+                        name=agent_name,
+                        msg=msg,
+                    )
+                ),
+            )
+        ]
+    # Multiple tool calls, show progress
+    tool_msgs: list[ChatMessage] = []
+
+    progress_id = str(uuid.uuid4())
+    total_steps = len(tool_calls)
+    if emit_agent_events:
+        ctx.write_event_to_stream(
+            AgentRunEvent(
+                name=agent_name,
+                msg=f"Making {total_steps} tool calls",
+            )
+        )
+    for i, tool_call in enumerate(tool_calls):
+        tool = tools_by_name.get(tool_call.tool_name)
+        if not tool:
+            tool_msgs.append(
+                ChatMessage(
+                    role=MessageRole.ASSISTANT,
+                    content=f"Tool {tool_call.tool_name} does not exist",
+                )
+            )
+            continue
+        tool_msg = await call_tool(
+            ctx,
+            tool,
+            tool_call,
+            event_emitter=lambda msg: ctx.write_event_to_stream(
+                AgentRunEvent(
+                    name=agent_name,
+                    msg=msg,
+                    event_type=AgentRunEventType.PROGRESS,
+                    data={
+                        "id": progress_id,
+                        "total": total_steps,
+                        "current": i,
+                    },
+                )
+            ),
+        )
+        tool_msgs.append(tool_msg)
+    return tool_msgs
+
+
+async def call_tool(
+    ctx: Context,
+    tool: BaseTool,
+    tool_call: ToolSelection,
+    event_emitter: Optional[Callable[[str], None]],
+) -> ChatMessage:
+    if event_emitter:
+        event_emitter(
+            f"Calling tool {tool_call.tool_name}, {str(tool_call.tool_kwargs)}"
+        )
+    try:
+        if isinstance(tool, ContextAwareTool):
+            if ctx is None:
+                raise ValueError("Context is required for context aware tool")
+            # inject context for calling an context aware tool
+            response = await tool.acall(ctx=ctx, **tool_call.tool_kwargs)
+        else:
+            response = await tool.acall(**tool_call.tool_kwargs)  # type: ignore
+        return ChatMessage(
+            role=MessageRole.TOOL,
+            content=str(response.raw_output),
+            additional_kwargs={
+                "tool_call_id": tool_call.tool_id,
+                "name": tool.metadata.get_name(),
+            },
+        )
+    except Exception as e:
+        logger.error(f"Got error in tool {tool_call.tool_name}: {str(e)}")
+        if event_emitter:
+            event_emitter(f"Got error in tool {tool_call.tool_name}: {str(e)}")
+        return ChatMessage(
+            role=MessageRole.TOOL,
+            content=f"Error: {str(e)}",
+            additional_kwargs={
+                "tool_call_id": tool_call.tool_id,
+                "name": tool.metadata.get_name(),
+            },
+        )
+
+
+async def _tool_call_generator(
+    llm: FunctionCallingLLM,
+    tools: list[BaseTool],
+    chat_history: list[ChatMessage],
+) -> AsyncGenerator[ChatResponse | bool, None]:
+    response_stream = await llm.astream_chat_with_tools(
+        tools,
+        chat_history=chat_history,
+        allow_parallel_tool_calls=False,
+    )
+
+    full_response = None
+    yielded_indicator = False
+    async for chunk in response_stream:
+        if "tool_calls" not in chunk.message.additional_kwargs:
+            # Yield a boolean to indicate whether the response is a tool call
+            if not yielded_indicator:
+                yield False
+                yielded_indicator = True
+
+            # if not a tool call, yield the chunks!
+            yield chunk  # type: ignore
+        elif not yielded_indicator:
+            # Yield the indicator for a tool call
+            yield True
+            yielded_indicator = True
+
+        full_response = chunk
+
+    if full_response:
+        yield full_response  # type: ignore

templates/components/multiagent/typescript/express/chat.controller.ts

@@ -1,36 +1,34 @@
-import { StopEvent } from "@llamaindex/core/workflow";
 import { Message, streamToResponse } from "ai";
 import { Request, Response } from "express";
-import { ChatResponseChunk } from "llamaindex";
+import {
+  convertToChatHistory,
+  retrieveMessageContent,
+} from "./llamaindex/streaming/annotations";
 import { createWorkflow } from "./workflow/factory";
-import { toDataStream, workflowEventsToStreamData } from "./workflow/stream";
+import { createStreamFromWorkflowContext } from "./workflow/stream";
 
 export const chat = async (req: Request, res: Response) => {
   try {
-    const { messages, data }: { messages: Message[]; data?: any } = req.body;
-    const userMessage = messages.pop();
-    if (!messages || !userMessage || userMessage.role !== "user") {
+    const { messages }: { messages: Message[] } = req.body;
+    if (!messages || messages.length === 0) {
       return res.status(400).json({
-        error:
-          "messages are required in the request body and the last message must be from the user",
+        error: "messages are required in the request body",
       });
     }
+    const chatHistory = convertToChatHistory(messages);
+    const userMessageContent = retrieveMessageContent(messages);
 
-    const agent = createWorkflow(messages, data);
-    const result = agent.run<AsyncGenerator<ChatResponseChunk>>(
-      userMessage.content,
-    ) as unknown as Promise<StopEvent<AsyncGenerator<ChatResponseChunk>>>;
+    const workflow = await createWorkflow({ chatHistory });
 
-    // convert the workflow events to a vercel AI stream data object
-    const agentStreamData = await workflowEventsToStreamData(
-      agent.streamEvents(),
-    );
-    // convert the workflow result to a vercel AI content stream
-    const stream = toDataStream(result, {
-      onFinal: () => agentStreamData.close(),
+    const context = workflow.run({
+      message: userMessageContent,
+      streaming: true,
     });
 
-    return streamToResponse(stream, res, {}, agentStreamData);
+    const { stream, dataStream } =
+      await createStreamFromWorkflowContext(context);
+
+    return streamToResponse(stream, res, {}, dataStream);
   } catch (error) {
     console.error("[LlamaIndex]", error);
     return res.status(500).json({

templates/components/multiagent/typescript/nextjs/route.ts

@@ -1,11 +1,14 @@
 import { initObservability } from "@/app/observability";
-import { StopEvent } from "@llamaindex/core/workflow";
-import { Message, StreamingTextResponse } from "ai";
-import { ChatResponseChunk } from "llamaindex";
+import { StreamingTextResponse, type Message } from "ai";
 import { NextRequest, NextResponse } from "next/server";
 import { initSettings } from "./engine/settings";
+import {
+  convertToChatHistory,
+  isValidMessages,
+  retrieveMessageContent,
+} from "./llamaindex/streaming/annotations";
 import { createWorkflow } from "./workflow/factory";
-import { toDataStream, workflowEventsToStreamData } from "./workflow/stream";
+import { createStreamFromWorkflowContext } from "./workflow/stream";
 
 initObservability();
 initSettings();
@@ -16,9 +19,8 @@ export const dynamic = "force-dynamic";
 export async function POST(request: NextRequest) {
   try {
     const body = await request.json();
-    const { messages, data }: { messages: Message[]; data?: any } = body;
-    const userMessage = messages.pop();
-    if (!messages || !userMessage || userMessage.role !== "user") {
+    const { messages }: { messages: Message[]; data?: any } = body;
+    if (!isValidMessages(messages)) {
       return NextResponse.json(
         {
           error:
@@ -28,20 +30,20 @@ export async function POST(request: NextRequest) {
       );
     }
 
-    const agent = createWorkflow(messages, data);
-    // TODO: fix type in agent.run in LITS
-    const result = agent.run<AsyncGenerator<ChatResponseChunk>>(
-      userMessage.content,
-    ) as unknown as Promise<StopEvent<AsyncGenerator<ChatResponseChunk>>>;
-    // convert the workflow events to a vercel AI stream data object
-    const agentStreamData = await workflowEventsToStreamData(
-      agent.streamEvents(),
-    );
-    // convert the workflow result to a vercel AI content stream
-    const stream = toDataStream(result, {
-      onFinal: () => agentStreamData.close(),
+    const chatHistory = convertToChatHistory(messages);
+    const userMessageContent = retrieveMessageContent(messages);
+
+    const workflow = await createWorkflow({ chatHistory });
+
+    const context = workflow.run({
+      message: userMessageContent,
+      streaming: true,
     });
-    return new StreamingTextResponse(stream, {}, agentStreamData);
+    const { stream, dataStream } =
+      await createStreamFromWorkflowContext(context);
+
+    // Return the two streams in one response
+    return new StreamingTextResponse(stream, {}, dataStream);
   } catch (error) {
     console.error("[LlamaIndex]", error);
     return NextResponse.json(

templates/components/multiagent/typescript/workflow/factory.ts

@@ -1,230 +0,0 @@
-import {
-  Context,
-  StartEvent,
-  StopEvent,
-  Workflow,
-  WorkflowEvent,
-} from "@llamaindex/core/workflow";
-import { Message } from "ai";
-import { ChatMessage, ChatResponseChunk, Settings } from "llamaindex";
-import { getAnnotations } from "../llamaindex/streaming/annotations";
-import {
-  createPublisher,
-  createResearcher,
-  createReviewer,
-  createWriter,
-} from "./agents";
-import { AgentInput, AgentRunEvent } from "./type";
-
-const TIMEOUT = 360 * 1000;
-const MAX_ATTEMPTS = 2;
-
-class ResearchEvent extends WorkflowEvent<{ input: string }> {}
-class WriteEvent extends WorkflowEvent<{
-  input: string;
-  isGood: boolean;
-}> {}
-class ReviewEvent extends WorkflowEvent<{ input: string }> {}
-class PublishEvent extends WorkflowEvent<{ input: string }> {}
-
-const prepareChatHistory = (chatHistory: Message[]): ChatMessage[] => {
-  // By default, the chat history only contains the assistant and user messages
-  // all the agents messages are stored in annotation data which is not visible to the LLM
-
-  const MAX_AGENT_MESSAGES = 10;
-  const agentAnnotations = getAnnotations<{ agent: string; text: string }>(
-    chatHistory,
-    { role: "assistant", type: "agent" },
-  ).slice(-MAX_AGENT_MESSAGES);
-
-  const agentMessages = agentAnnotations
-    .map(
-      (annotation) =>
-        `\n<${annotation.data.agent}>\n${annotation.data.text}\n</${annotation.data.agent}>`,
-    )
-    .join("\n");
-
-  const agentContent = agentMessages
-    ? "Here is the previous conversation of agents:\n" + agentMessages
-    : "";
-
-  if (agentContent) {
-    const agentMessage: ChatMessage = {
-      role: "assistant",
-      content: agentContent,
-    };
-    return [
-      ...chatHistory.slice(0, -1),
-      agentMessage,
-      chatHistory.slice(-1)[0],
-    ] as ChatMessage[];
-  }
-  return chatHistory as ChatMessage[];
-};
-
-export const createWorkflow = (messages: Message[], params?: any) => {
-  const chatHistoryWithAgentMessages = prepareChatHistory(messages);
-  const runAgent = async (
-    context: Context,
-    agent: Workflow,
-    input: AgentInput,
-  ) => {
-    const run = agent.run(new StartEvent({ input }));
-    for await (const event of agent.streamEvents()) {
-      if (event.data instanceof AgentRunEvent) {
-        context.writeEventToStream(event.data);
-      }
-    }
-    return await run;
-  };
-
-  const start = async (context: Context, ev: StartEvent) => {
-    context.set("task", ev.data.input);
-
-    const chatHistoryStr = chatHistoryWithAgentMessages
-      .map((msg) => `${msg.role}: ${msg.content}`)
-      .join("\n");
-
-    // Decision-making process
-    const decision = await decideWorkflow(ev.data.input, chatHistoryStr);
-
-    if (decision !== "publish") {
-      return new ResearchEvent({
-        input: `Research for this task: ${ev.data.input}`,
-      });
-    } else {
-      return new PublishEvent({
-        input: `Publish content based on the chat history\n${chatHistoryStr}\n\n and task: ${ev.data.input}`,
-      });
-    }
-  };
-
-  const decideWorkflow = async (task: string, chatHistoryStr: string) => {
-    const llm = Settings.llm;
-
-    const prompt = `You are an expert in decision-making, helping people write and publish blog posts.
-If the user is asking for a file or to publish content, respond with 'publish'.
-If the user requests to write or update a blog post, respond with 'not_publish'.
-
-Here is the chat history:
-${chatHistoryStr}
-
-The current user request is:
-${task}
-
-Given the chat history and the new user request, decide whether to publish based on existing information.
-Decision (respond with either 'not_publish' or 'publish'):`;
-
-    const output = await llm.complete({ prompt: prompt });
-    const decision = output.text.trim().toLowerCase();
-    return decision === "publish" ? "publish" : "research";
-  };
-
-  const research = async (context: Context, ev: ResearchEvent) => {
-    const researcher = await createResearcher(
-      chatHistoryWithAgentMessages,
-      params,
-    );
-    const researchRes = await runAgent(context, researcher, {
-      message: ev.data.input,
-    });
-    const researchResult = researchRes.data.result;
-    return new WriteEvent({
-      input: `Write a blog post given this task: ${context.get("task")} using this research content: ${researchResult}`,
-      isGood: false,
-    });
-  };
-
-  const write = async (context: Context, ev: WriteEvent) => {
-    const writer = createWriter(chatHistoryWithAgentMessages);
-
-    context.set("attempts", context.get("attempts", 0) + 1);
-    const tooManyAttempts = context.get("attempts") > MAX_ATTEMPTS;
-    if (tooManyAttempts) {
-      context.writeEventToStream(
-        new AgentRunEvent({
-          name: "writer",
-          msg: `Too many attempts (${MAX_ATTEMPTS}) to write the blog post. Proceeding with the current version.`,
-        }),
-      );
-    }
-
-    if (ev.data.isGood || tooManyAttempts) {
-      // the blog post is good or too many attempts
-      // stream the final content
-      const result = await runAgent(context, writer, {
-        message: `Based on the reviewer's feedback, refine the post and return only the final version of the post. Here's the current version: ${ev.data.input}`,
-        streaming: true,
-      });
-      return result as unknown as StopEvent<AsyncGenerator<ChatResponseChunk>>;
-    }
-
-    const writeRes = await runAgent(context, writer, {
-      message: ev.data.input,
-    });
-    const writeResult = writeRes.data.result;
-    context.set("result", writeResult); // store the last result
-    return new ReviewEvent({ input: writeResult });
-  };
-
-  const review = async (context: Context, ev: ReviewEvent) => {
-    const reviewer = createReviewer(chatHistoryWithAgentMessages);
-    const reviewRes = await reviewer.run(
-      new StartEvent<AgentInput>({ input: { message: ev.data.input } }),
-    );
-    const reviewResult = reviewRes.data.result;
-    const oldContent = context.get("result");
-    const postIsGood = reviewResult.toLowerCase().includes("post is good");
-    context.writeEventToStream(
-      new AgentRunEvent({
-        name: "reviewer",
-        msg: `The post is ${postIsGood ? "" : "not "}good enough for publishing. Sending back to the writer${
-          postIsGood ? " for publication." : "."
-        }`,
-      }),
-    );
-    if (postIsGood) {
-      return new WriteEvent({
-        input: "",
-        isGood: true,
-      });
-    }
-
-    return new WriteEvent({
-      input: `Improve the writing of a given blog post by using a given review.
-            Blog post:
-            \`\`\`
-            ${oldContent}
-            \`\`\`
-
-            Review:
-            \`\`\`
-            ${reviewResult}
-            \`\`\``,
-      isGood: false,
-    });
-  };
-
-  const publish = async (context: Context, ev: PublishEvent) => {
-    const publisher = await createPublisher(chatHistoryWithAgentMessages);
-
-    const publishResult = await runAgent(context, publisher, {
-      message: `${ev.data.input}`,
-      streaming: true,
-    });
-    return publishResult as unknown as StopEvent<
-      AsyncGenerator<ChatResponseChunk>
-    >;
-  };
-
-  const workflow = new Workflow({ timeout: TIMEOUT, validate: true });
-  workflow.addStep(StartEvent, start, {
-    outputs: [ResearchEvent, PublishEvent],
-  });
-  workflow.addStep(ResearchEvent, research, { outputs: WriteEvent });
-  workflow.addStep(WriteEvent, write, { outputs: [ReviewEvent, StopEvent] });
-  workflow.addStep(ReviewEvent, review, { outputs: WriteEvent });
-  workflow.addStep(PublishEvent, publish, { outputs: StopEvent });
-
-  return workflow;
-};

templates/components/multiagent/typescript/workflow/single-agent.ts

@@ -1,22 +1,21 @@
 import {
-  Context,
+  HandlerContext,
   StartEvent,
   StopEvent,
   Workflow,
   WorkflowEvent,
-} from "@llamaindex/core/workflow";
+} from "@llamaindex/workflow";
 import {
   BaseToolWithCall,
   ChatMemoryBuffer,
   ChatMessage,
-  ChatResponse,
   ChatResponseChunk,
+  QueryEngineTool,
   Settings,
   ToolCall,
   ToolCallLLM,
-  ToolCallLLMMessageOptions,
-  callTool,
 } from "llamaindex";
+import { callTools, chatWithTools } from "./tools";
 import { AgentInput, AgentRunEvent } from "./type";
 
 class InputEvent extends WorkflowEvent<{
@@ -27,11 +26,23 @@ class ToolCallEvent extends WorkflowEvent<{
   toolCalls: ToolCall[];
 }> {}
 
-export class FunctionCallingAgent extends Workflow {
+type FunctionCallingAgentContextData = {
+  streaming: boolean;
+};
+
+export type FunctionCallingAgentInput = AgentInput & {
+  displayName: string;
+};
+
+export class FunctionCallingAgent extends Workflow<
+  FunctionCallingAgentContextData,
+  FunctionCallingAgentInput,
+  string | AsyncGenerator<boolean | ChatResponseChunk<object>>
+> {
   name: string;
   llm: ToolCallLLM;
   memory: ChatMemoryBuffer;
-  tools: BaseToolWithCall[];
+  tools: BaseToolWithCall[] | QueryEngineTool[];
   systemPrompt?: string;
   writeEvents: boolean;
   role?: string;
@@ -53,7 +64,9 @@ export class FunctionCallingAgent extends Workflow {
     });
     this.name = options?.name;
     this.llm = options.llm ?? (Settings.llm as ToolCallLLM);
-    this.checkToolCallSupport();
+    if (!(this.llm instanceof ToolCallLLM)) {
+      throw new Error("LLM is not a ToolCallLLM");
+    }
     this.memory = new ChatMemoryBuffer({
       llm: this.llm,
       chatHistory: options.chatHistory,
@@ -64,173 +77,103 @@ export class FunctionCallingAgent extends Workflow {
     this.role = options?.role;
 
     // add steps
-    this.addStep(StartEvent<AgentInput>, this.prepareChatHistory, {
-      outputs: InputEvent,
-    });
-    this.addStep(InputEvent, this.handleLLMInput, {
-      outputs: [ToolCallEvent, StopEvent],
-    });
-    this.addStep(ToolCallEvent, this.handleToolCalls, {
-      outputs: InputEvent,
-    });
+    this.addStep(
+      {
+        inputs: [StartEvent<AgentInput>],
+        outputs: [InputEvent],
+      },
+      this.prepareChatHistory,
+    );
+    this.addStep(
+      {
+        inputs: [InputEvent],
+        outputs: [ToolCallEvent, StopEvent],
+      },
+      this.handleLLMInput,
+    );
+    this.addStep(
+      {
+        inputs: [ToolCallEvent],
+        outputs: [InputEvent],
+      },
+      this.handleToolCalls,
+    );
   }
 
   private get chatHistory() {
     return this.memory.getMessages();
   }
 
-  private async prepareChatHistory(
-    ctx: Context,
+  prepareChatHistory = async (
+    ctx: HandlerContext<FunctionCallingAgentContextData>,
     ev: StartEvent<AgentInput>,
-  ): Promise<InputEvent> {
-    const { message, streaming } = ev.data.input;
-    ctx.set("streaming", streaming);
+  ): Promise<InputEvent> => {
+    const { message, streaming } = ev.data;
+    ctx.data.streaming = streaming ?? false;
     this.writeEvent(`Start to work on: ${message}`, ctx);
     if (this.systemPrompt) {
       this.memory.put({ role: "system", content: this.systemPrompt });
     }
     this.memory.put({ role: "user", content: message });
     return new InputEvent({ input: this.chatHistory });
-  }
+  };
 
-  private async handleLLMInput(
-    ctx: Context,
+  handleLLMInput = async (
+    ctx: HandlerContext<FunctionCallingAgentContextData>,
     ev: InputEvent,
-  ): Promise<StopEvent<string | AsyncGenerator> | ToolCallEvent> {
-    if (ctx.get("streaming")) {
-      return await this.handleLLMInputStream(ctx, ev);
+  ): Promise<StopEvent<string | AsyncGenerator> | ToolCallEvent> => {
+    const toolCallResponse = await chatWithTools(
+      this.llm,
+      this.tools,
+      this.chatHistory,
+    );
+    if (toolCallResponse.toolCallMessage) {
+      this.memory.put(toolCallResponse.toolCallMessage);
     }
 
-    const result = await this.llm.chat({
-      messages: this.chatHistory,
-      tools: this.tools,
-    });
-    this.memory.put(result.message);
-
-    const toolCalls = this.getToolCallsFromResponse(result);
-    if (toolCalls.length) {
-      return new ToolCallEvent({ toolCalls });
+    if (toolCallResponse.hasToolCall()) {
+      return new ToolCallEvent({ toolCalls: toolCallResponse.toolCalls });
     }
-    this.writeEvent("Finished task", ctx);
-    return new StopEvent({ result: result.message.content.toString() });
-  }
 
-  private async handleLLMInputStream(
-    context: Context,
-    ev: InputEvent,
-  ): Promise<StopEvent<AsyncGenerator> | ToolCallEvent> {
-    const { llm, tools, memory } = this;
-    const llmArgs = { messages: this.chatHistory, tools };
-
-    const responseGenerator = async function* () {
-      const responseStream = await llm.chat({ ...llmArgs, stream: true });
-
-      let fullResponse = null;
-      let yieldedIndicator = false;
-      for await (const chunk of responseStream) {
-        const hasToolCalls = chunk.options && "toolCall" in chunk.options;
-        if (!hasToolCalls) {
-          if (!yieldedIndicator) {
-            yield false;
-            yieldedIndicator = true;
-          }
-          yield chunk;
-        } else if (!yieldedIndicator) {
-          yield true;
-          yieldedIndicator = true;
-        }
-
-        fullResponse = chunk;
+    if (ctx.data.streaming) {
+      if (!toolCallResponse.responseGenerator) {
+        throw new Error("No streaming response");
       }
-
-      if (fullResponse?.options && Object.keys(fullResponse.options).length) {
-        memory.put({
-          role: "assistant",
-          content: "",
-          options: fullResponse.options,
-        });
-        yield fullResponse;
-      }
-    };
-
-    const generator = responseGenerator();
-    const isToolCall = await generator.next();
-    if (isToolCall.value) {
-      const fullResponse = await generator.next();
-      const toolCalls = this.getToolCallsFromResponse(
-        fullResponse.value as ChatResponseChunk<ToolCallLLMMessageOptions>,
-      );
-      return new ToolCallEvent({ toolCalls });
+      return new StopEvent(toolCallResponse.responseGenerator);
     }
 
-    this.writeEvent("Finished task", context);
-    return new StopEvent({ result: generator });
-  }
+    const fullResponse = await toolCallResponse.asFullResponse();
+    this.memory.put(fullResponse);
+    return new StopEvent(fullResponse.content.toString());
+  };
 
-  private async handleToolCalls(
-    ctx: Context,
+  handleToolCalls = async (
+    ctx: HandlerContext<FunctionCallingAgentContextData>,
     ev: ToolCallEvent,
-  ): Promise<InputEvent> {
+  ): Promise<InputEvent> => {
     const { toolCalls } = ev.data;
 
-    const toolMsgs: ChatMessage[] = [];
-
-    for (const call of toolCalls) {
-      const targetTool = this.tools.find(
-        (tool) => tool.metadata.name === call.name,
-      );
-      // TODO: make logger optional in callTool in framework
-      const toolOutput = await callTool(targetTool, call, {
-        log: () => {},
-        error: console.error.bind(console),
-        warn: () => {},
-      });
-      toolMsgs.push({
-        content: JSON.stringify(toolOutput.output),
-        role: "user",
-        options: {
-          toolResult: {
-            result: toolOutput.output,
-            isError: toolOutput.isError,
-            id: call.id,
-          },
-        },
-      });
-    }
+    const toolMsgs = await callTools({
+      tools: this.tools,
+      toolCalls,
+      ctx,
+      agentName: this.name,
+    });
 
     for (const msg of toolMsgs) {
       this.memory.put(msg);
     }
 
     return new InputEvent({ input: this.memory.getMessages() });
-  }
+  };
 
-  private writeEvent(msg: string, context: Context) {
+  writeEvent = (
+    msg: string,
+    ctx: HandlerContext<FunctionCallingAgentContextData>,
+  ) => {
     if (!this.writeEvents) return;
-    context.writeEventToStream({
-      data: new AgentRunEvent({ name: this.name, msg }),
-    });
-  }
-
-  private checkToolCallSupport() {
-    const { supportToolCall } = this.llm as ToolCallLLM;
-    if (!supportToolCall) throw new Error("LLM does not support tool calls");
-  }
-
-  private getToolCallsFromResponse(
-    response:
-      | ChatResponse<ToolCallLLMMessageOptions>
-      | ChatResponseChunk<ToolCallLLMMessageOptions>,
-  ): ToolCall[] {
-    let options;
-    if ("message" in response) {
-      options = response.message.options;
-    } else {
-      options = response.options;
-    }
-    if (options && "toolCall" in options) {
-      return options.toolCall as ToolCall[];
-    }
-    return [];
-  }
+    ctx.sendEvent(
+      new AgentRunEvent({ agent: this.name, text: msg, type: "text" }),
+    );
+  };
 }

templates/components/multiagent/typescript/workflow/stream.ts

@@ -1,65 +1,77 @@
-import { StopEvent } from "@llamaindex/core/workflow";
 import {
-  createCallbacksTransformer,
-  createStreamDataTransformer,
+  StopEvent,
+  WorkflowContext,
+  WorkflowEvent,
+} from "@llamaindex/workflow";
+import {
   StreamData,
+  createStreamDataTransformer,
   trimStartOfStreamHelper,
-  type AIStreamCallbacksAndOptions,
 } from "ai";
 import { ChatResponseChunk } from "llamaindex";
 import { AgentRunEvent } from "./type";
 
-export function toDataStream(
-  result: Promise<StopEvent<AsyncGenerator<ChatResponseChunk>>>,
-  callbacks?: AIStreamCallbacksAndOptions,
-) {
-  return toReadableStream(result)
-    .pipeThrough(createCallbacksTransformer(callbacks))
-    .pipeThrough(createStreamDataTransformer());
-}
-
-function toReadableStream(
-  result: Promise<StopEvent<AsyncGenerator<ChatResponseChunk>>>,
-) {
+export async function createStreamFromWorkflowContext<Input, Output, Context>(
+  context: WorkflowContext<Input, Output, Context>,
+): Promise<{ stream: ReadableStream<string>; dataStream: StreamData }> {
   const trimStartOfStream = trimStartOfStreamHelper();
-  return new ReadableStream<string>({
-    start(controller) {
-      controller.enqueue(""); // Kickstart the stream
+  const dataStream = new StreamData();
+  const encoder = new TextEncoder();
+  let generator: AsyncGenerator<ChatResponseChunk> | undefined;
+
+  const closeStreams = (controller: ReadableStreamDefaultController) => {
+    controller.close();
+    dataStream.close();
+  };
+
+  const mainStream = new ReadableStream({
+    async start(controller) {
+      // Kickstart the stream by sending an empty string
+      controller.enqueue(encoder.encode(""));
     },
-    async pull(controller): Promise<void> {
-      const stopEvent = await result;
-      const generator = stopEvent.data.result;
-      const { value, done } = await generator.next();
+    async pull(controller) {
+      while (!generator) {
+        // get next event from workflow context
+        const { value: event, done } =
+          await context[Symbol.asyncIterator]().next();
+        if (done) {
+          closeStreams(controller);
+          return;
+        }
+        generator = handleEvent(event, dataStream);
+      }
+
+      const { value: chunk, done } = await generator.next();
       if (done) {
-        controller.close();
+        closeStreams(controller);
         return;
       }
-
-      const text = trimStartOfStream(value.delta ?? "");
-      if (text) controller.enqueue(text);
+      const text = trimStartOfStream(chunk.delta ?? "");
+      if (text) {
+        controller.enqueue(encoder.encode(text));
+      }
     },
   });
+
+  return {
+    stream: mainStream.pipeThrough(createStreamDataTransformer()),
+    dataStream,
+  };
 }
 
-export async function workflowEventsToStreamData(
-  events: AsyncIterable<AgentRunEvent>,
-): Promise<StreamData> {
-  const streamData = new StreamData();
-
-  (async () => {
-    for await (const event of events) {
-      if (event instanceof AgentRunEvent) {
-        const { name, msg } = event.data;
-        if ((streamData as any).isClosed) {
-          break;
-        }
-        streamData.appendMessageAnnotation({
-          type: "agent",
-          data: { agent: name, text: msg },
-        });
-      }
-    }
-  })();
-
-  return streamData;
+function handleEvent(
+  event: WorkflowEvent<any>,
+  dataStream: StreamData,
+): AsyncGenerator<ChatResponseChunk> | undefined {
+  // Handle for StopEvent
+  if (event instanceof StopEvent) {
+    return event.data as AsyncGenerator<ChatResponseChunk>;
+  }
+  // Handle for AgentRunEvent
+  if (event instanceof AgentRunEvent) {
+    dataStream.appendMessageAnnotation({
+      type: "agent",
+      data: event.data,
+    });
+  }
 }

templates/components/multiagent/typescript/workflow/tools.ts

@@ -1,54 +1,342 @@
-import fs from "fs/promises";
-import { BaseToolWithCall, QueryEngineTool } from "llamaindex";
-import path from "path";
+import { HandlerContext } from "@llamaindex/workflow";
+import {
+  BaseToolWithCall,
+  callTool,
+  ChatMessage,
+  ChatResponse,
+  ChatResponseChunk,
+  LlamaCloudIndex,
+  PartialToolCall,
+  QueryEngineTool,
+  ToolCall,
+  ToolCallLLM,
+  ToolCallLLMMessageOptions,
+} from "llamaindex";
+import crypto from "node:crypto";
 import { getDataSource } from "../engine";
-import { createTools } from "../engine/tools/index";
+import { AgentRunEvent } from "./type";
 
-export const getQueryEngineTool = async (
-  params?: any,
-): Promise<QueryEngineTool | null> => {
-  const index = await getDataSource(params);
+export const getQueryEngineTools = async (): Promise<
+  QueryEngineTool[] | null
+> => {
+  const topK = process.env.TOP_K ? parseInt(process.env.TOP_K) : undefined;
+
+  const index = await getDataSource();
   if (!index) {
     return null;
   }
+  // index is LlamaCloudIndex use two query engine tools
+  if (index instanceof LlamaCloudIndex) {
+    return [
+      new QueryEngineTool({
+        queryEngine: index.asQueryEngine({
+          similarityTopK: topK,
+          retrieval_mode: "files_via_content",
+        }),
+        metadata: {
+          name: "document_retriever",
+          description: `Document retriever that retrieves entire documents from the corpus.
+  ONLY use for research questions that may require searching over entire research reports.
+  Will be slower and more expensive than chunk-level retrieval but may be necessary.`,
+        },
+      }),
+      new QueryEngineTool({
+        queryEngine: index.asQueryEngine({
+          similarityTopK: topK,
+          retrieval_mode: "chunks",
+        }),
+        metadata: {
+          name: "chunk_retriever",
+          description: `Retrieves a small set of relevant document chunks from the corpus.
+      Use for research questions that want to look up specific facts from the knowledge corpus,
+      and need entire documents.`,
+        },
+      }),
+    ];
+  } else {
+    return [
+      new QueryEngineTool({
+        queryEngine: index.asQueryEngine({
+          similarityTopK: topK,
+        }),
+        metadata: {
+          name: "retriever",
+          description: `Use this tool to retrieve information about the text corpus from the index.`,
+        },
+      }),
+    ];
+  }
+};
 
-  const topK = process.env.TOP_K ? parseInt(process.env.TOP_K) : undefined;
-  return new QueryEngineTool({
-    queryEngine: index.asQueryEngine({
-      similarityTopK: topK,
-    }),
-    metadata: {
-      name: "query_index",
-      description: `Use this tool to retrieve information about the text corpus from the index.`,
+/**
+ * Call multiple tools and return the tool messages
+ */
+export const callTools = async <T>({
+  tools,
+  toolCalls,
+  ctx,
+  agentName,
+  writeEvent = true,
+}: {
+  toolCalls: ToolCall[];
+  tools: BaseToolWithCall[];
+  ctx: HandlerContext<T>;
+  agentName: string;
+  writeEvent?: boolean;
+}): Promise<ChatMessage[]> => {
+  const toolMsgs: ChatMessage[] = [];
+  if (toolCalls.length === 0) {
+    return toolMsgs;
+  }
+  if (toolCalls.length === 1) {
+    const tool = tools.find((tool) => tool.metadata.name === toolCalls[0].name);
+    if (!tool) {
+      throw new Error(`Tool ${toolCalls[0].name} not found`);
+    }
+    return [
+      await callSingleTool(
+        tool,
+        toolCalls[0],
+        writeEvent
+          ? (msg: string) => {
+              ctx.sendEvent(
+                new AgentRunEvent({
+                  agent: agentName,
+                  text: msg,
+                  type: "text",
+                }),
+              );
+            }
+          : undefined,
+      ),
+    ];
+  }
+  // Multiple tool calls, show events in progress
+  const progressId = crypto.randomUUID();
+  const totalSteps = toolCalls.length;
+  let currentStep = 0;
+  for (const toolCall of toolCalls) {
+    const tool = tools.find((tool) => tool.metadata.name === toolCall.name);
+    if (!tool) {
+      throw new Error(`Tool ${toolCall.name} not found`);
+    }
+    const toolMsg = await callSingleTool(tool, toolCall, (msg: string) => {
+      ctx.sendEvent(
+        new AgentRunEvent({
+          agent: agentName,
+          text: msg,
+          type: "progress",
+          data: {
+            id: progressId,
+            total: totalSteps,
+            current: currentStep,
+          },
+        }),
+      );
+      currentStep++;
+    });
+    toolMsgs.push(toolMsg);
+  }
+  return toolMsgs;
+};
+
+export const callSingleTool = async (
+  tool: BaseToolWithCall,
+  toolCall: ToolCall,
+  eventEmitter?: (msg: string) => void,
+): Promise<ChatMessage> => {
+  if (eventEmitter) {
+    eventEmitter(
+      `Calling tool ${toolCall.name} with input: ${JSON.stringify(toolCall.input)}`,
+    );
+  }
+
+  const toolOutput = await callTool(tool, toolCall, {
+    log: () => {},
+    error: (...args: unknown[]) => {
+      console.error(`Tool ${toolCall.name} got error:`, ...args);
+      if (eventEmitter) {
+        eventEmitter(`Tool ${toolCall.name} got error: ${args.join(" ")}`);
+      }
+      return {
+        content: JSON.stringify({
+          error: args.join(" "),
+        }),
+        role: "user",
+        options: {
+          toolResult: {
+            id: toolCall.id,
+            result: JSON.stringify({
+              error: args.join(" "),
+            }),
+            isError: true,
+          },
+        },
+      };
     },
+    warn: () => {},
+  });
+
+  return {
+    content: JSON.stringify(toolOutput.output),
+    role: "user",
+    options: {
+      toolResult: {
+        result: toolOutput.output,
+        isError: toolOutput.isError,
+        id: toolCall.id,
+      },
+    },
+  };
+};
+
+class ChatWithToolsResponse {
+  toolCalls: ToolCall[];
+  toolCallMessage?: ChatMessage;
+  responseGenerator?: AsyncGenerator<ChatResponseChunk>;
+
+  constructor(options: {
+    toolCalls: ToolCall[];
+    toolCallMessage?: ChatMessage;
+    responseGenerator?: AsyncGenerator<ChatResponseChunk>;
+  }) {
+    this.toolCalls = options.toolCalls;
+    this.toolCallMessage = options.toolCallMessage;
+    this.responseGenerator = options.responseGenerator;
+  }
+
+  hasMultipleTools() {
+    const uniqueToolNames = new Set(this.getToolNames());
+    return uniqueToolNames.size > 1;
+  }
+
+  hasToolCall() {
+    return this.toolCalls.length > 0;
+  }
+
+  getToolNames() {
+    return this.toolCalls.map((toolCall) => toolCall.name);
+  }
+
+  async asFullResponse(): Promise<ChatMessage> {
+    if (!this.responseGenerator) {
+      throw new Error("No response generator");
+    }
+    let fullResponse = "";
+    for await (const chunk of this.responseGenerator) {
+      fullResponse += chunk.delta;
+    }
+    return {
+      role: "assistant",
+      content: fullResponse,
+    };
+  }
+}
+
+export const chatWithTools = async (
+  llm: ToolCallLLM,
+  tools: BaseToolWithCall[],
+  messages: ChatMessage[],
+): Promise<ChatWithToolsResponse> => {
+  const responseGenerator = async function* (): AsyncGenerator<
+    boolean | ChatResponseChunk,
+    void,
+    unknown
+  > {
+    const responseStream = await llm.chat({ messages, tools, stream: true });
+
+    let fullResponse = null;
+    let yieldedIndicator = false;
+    const toolCallMap = new Map();
+    for await (const chunk of responseStream) {
+      const hasToolCalls = chunk.options && "toolCall" in chunk.options;
+      if (!hasToolCalls) {
+        if (!yieldedIndicator) {
+          yield false;
+          yieldedIndicator = true;
+        }
+        yield chunk;
+      } else if (!yieldedIndicator) {
+        yield true;
+        yieldedIndicator = true;
+      }
+
+      if (chunk.options && "toolCall" in chunk.options) {
+        for (const toolCall of chunk.options.toolCall as PartialToolCall[]) {
+          if (toolCall.id) {
+            toolCallMap.set(toolCall.id, toolCall);
+          }
+        }
+      }
+
+      if (
+        hasToolCalls &&
+        (chunk.raw as any)?.choices?.[0]?.finish_reason !== null
+      ) {
+        // Update the fullResponse with the tool calls
+        const toolCalls = Array.from(toolCallMap.values());
+        fullResponse = {
+          ...chunk,
+          options: {
+            ...chunk.options,
+            toolCall: toolCalls,
+          },
+        };
+      }
+    }
+
+    if (fullResponse) {
+      yield fullResponse;
+    }
+  };
+
+  const generator = responseGenerator();
+  const isToolCall = await generator.next();
+
+  if (isToolCall.value) {
+    // If it's a tool call, we need to wait for the full response
+    let fullResponse = null;
+    for await (const chunk of generator) {
+      fullResponse = chunk;
+    }
+
+    if (fullResponse) {
+      const responseChunk = fullResponse as ChatResponseChunk;
+      const toolCalls = getToolCallsFromResponse(responseChunk);
+      return new ChatWithToolsResponse({
+        toolCalls,
+        toolCallMessage: {
+          options: responseChunk.options,
+          role: "assistant",
+          content: "",
+        },
+      });
+    } else {
+      throw new Error("Cannot get tool calls from response");
+    }
+  }
+
+  return new ChatWithToolsResponse({
+    toolCalls: [],
+    responseGenerator: generator as AsyncGenerator<ChatResponseChunk>,
   });
 };
 
-export const getAvailableTools = async () => {
-  const configFile = path.join("config", "tools.json");
-  let toolConfig: any;
-  const tools: BaseToolWithCall[] = [];
-  try {
-    toolConfig = JSON.parse(await fs.readFile(configFile, "utf8"));
-  } catch (e) {
-    console.info(`Could not read ${configFile} file. Using no tools.`);
-  }
-  if (toolConfig) {
-    tools.push(...(await createTools(toolConfig)));
-  }
-  const queryEngineTool = await getQueryEngineTool();
-  if (queryEngineTool) {
-    tools.push(queryEngineTool);
+export const getToolCallsFromResponse = (
+  response:
+    | ChatResponse<ToolCallLLMMessageOptions>
+    | ChatResponseChunk<ToolCallLLMMessageOptions>,
+): ToolCall[] => {
+  let options;
+
+  if ("message" in response) {
+    options = response.message.options;
+  } else {
+    options = response.options;
   }
 
-  return tools;
-};
-
-export const lookupTools = async (
-  toolNames: string[],
-): Promise<BaseToolWithCall[]> => {
-  const availableTools = await getAvailableTools();
-  return availableTools.filter((tool) =>
-    toolNames.includes(tool.metadata.name),
-  );
+  if (options && "toolCall" in options) {
+    return options.toolCall as ToolCall[];
+  }
+  return [];
 };

templates/components/multiagent/typescript/workflow/type.ts

@@ -1,11 +1,24 @@
-import { WorkflowEvent } from "@llamaindex/core/workflow";
+import { WorkflowEvent } from "@llamaindex/workflow";
+import { MessageContent } from "llamaindex";
 
 export type AgentInput = {
-  message: string;
+  message: MessageContent;
   streaming?: boolean;
 };
 
+export type AgentRunEventType = "text" | "progress";
+
+export type ProgressEventData = {
+  id: string;
+  total: number;
+  current: number;
+};
+
+export type AgentRunEventData = ProgressEventData;
+
 export class AgentRunEvent extends WorkflowEvent<{
-  name: string;
-  msg: string;
+  agent: string;
+  text: string;
+  type: AgentRunEventType;
+  data?: AgentRunEventData;
 }> {}

templates/components/settings/python/settings.py

@@ -21,6 +21,8 @@ def init_settings():
             init_mistral()
         case "azure-openai":
             init_azure_openai()
+        case "huggingface":
+            init_huggingface()
         case "t-systems":
             from .llmhub import init_llmhub
 
@@ -138,6 +140,42 @@ def init_fastembed():
     )
 
 
+def init_huggingface_embedding():
+    try:
+        from llama_index.embeddings.huggingface import HuggingFaceEmbedding
+    except ImportError:
+        raise ImportError(
+            "Hugging Face support is not installed. Please install it with `poetry add llama-index-embeddings-huggingface`"
+        )
+
+    embedding_model = os.getenv("EMBEDDING_MODEL", "all-MiniLM-L6-v2")
+    backend = os.getenv("EMBEDDING_BACKEND", "onnx")  # "torch", "onnx", or "openvino"
+    trust_remote_code = (
+        os.getenv("EMBEDDING_TRUST_REMOTE_CODE", "false").lower() == "true"
+    )
+
+    Settings.embed_model = HuggingFaceEmbedding(
+        model_name=embedding_model,
+        trust_remote_code=trust_remote_code,
+        backend=backend,
+    )
+
+
+def init_huggingface():
+    try:
+        from llama_index.llms.huggingface import HuggingFaceLLM
+    except ImportError:
+        raise ImportError(
+            "Hugging Face support is not installed. Please install it with `poetry add llama-index-llms-huggingface` and `poetry add llama-index-embeddings-huggingface`"
+        )
+
+    Settings.llm = HuggingFaceLLM(
+        model_name=os.getenv("MODEL"),
+        tokenizer_name=os.getenv("MODEL"),
+    )
+    init_huggingface_embedding()
+
+
 def init_groq():
     try:
         from llama_index.llms.groq import Groq

templates/components/vectordbs/typescript/none/generate.ts

@@ -5,7 +5,6 @@ import * as dotenv from "dotenv";
 
 import { getDocuments } from "./loader";
 import { initSettings } from "./settings";
-import { STORAGE_CACHE_DIR } from "./shared";
 
 // Load environment variables from local .env file
 dotenv.config();
@@ -20,9 +19,13 @@ async function getRuntime(func: any) {
 async function generateDatasource() {
   console.log(`Generating storage context...`);
   // Split documents, create embeddings and store them in the storage context
+  const persistDir = process.env.STORAGE_CACHE_DIR;
+  if (!persistDir) {
+    throw new Error("STORAGE_CACHE_DIR environment variable is required!");
+  }
   const ms = await getRuntime(async () => {
     const storageContext = await storageContextFromDefaults({
-      persistDir: STORAGE_CACHE_DIR,
+      persistDir,
     });
     const documents = await getDocuments();
 

templates/components/vectordbs/typescript/none/index.ts

@@ -1,10 +1,13 @@
 import { SimpleDocumentStore, VectorStoreIndex } from "llamaindex";
 import { storageContextFromDefaults } from "llamaindex/storage/StorageContext";
-import { STORAGE_CACHE_DIR } from "./shared";
 
 export async function getDataSource(params?: any) {
+  const persistDir = process.env.STORAGE_CACHE_DIR;
+  if (!persistDir) {
+    throw new Error("STORAGE_CACHE_DIR environment variable is required!");
+  }
   const storageContext = await storageContextFromDefaults({
-    persistDir: `${STORAGE_CACHE_DIR}`,
+    persistDir,
   });
 
   const numberOfDocs = Object.keys(

templates/components/vectordbs/typescript/none/shared.ts

@@ -1 +0,0 @@
-export const STORAGE_CACHE_DIR = "./cache";

templates/types/streaming/express/eslint.config.mjs

@@ -0,0 +1,31 @@
+import { FlatCompat } from "@eslint/eslintrc";
+import js from "@eslint/js";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+import tseslint from "typescript-eslint";
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+const compat = new FlatCompat({
+  baseDirectory: __dirname,
+  recommendedConfig: js.configs.recommended,
+  allConfig: js.configs.all,
+});
+
+export default [
+  ...compat.extends("eslint:recommended", "prettier"),
+  ...tseslint.configs.recommended,
+  {
+    ignores: ["prettier.config.cjs"],
+  },
+  { files: ["**/*.{ts}"] },
+  {
+    rules: {
+      "max-params": ["error", 4],
+      "prefer-const": "error",
+      "@typescript-eslint/no-explicit-any": "off",
+      "@typescript-eslint/ban-ts-comment": "off",
+      "@typescript-eslint/no-unused-vars": "off",
+    },
+  },
+];

templates/types/streaming/express/eslintrc.json

@@ -1,10 +0,0 @@
-{
-  "extends": ["eslint:recommended", "prettier"],
-  "rules": {
-    "max-params": ["error", 4],
-    "prefer-const": "error"
-  },
-  "parserOptions": {
-    "sourceType": "module"
-  }
-}

templates/types/streaming/express/index.ts

@@ -1,4 +1,3 @@
-/* eslint-disable turbo/no-undeclared-env-vars */
 import cors from "cors";
 import "dotenv/config";
 import express, { Express, Request, Response } from "express";

templates/types/streaming/express/package.json

@@ -12,30 +12,34 @@
     "format:write": "prettier --ignore-unknown --write .",
     "build": "tsup index.ts --format esm --dts",
     "start": "node dist/index.js",
-    "dev": "concurrently \"tsup index.ts --format esm --dts --watch\" \"nodemon --watch dist/index.js\""
+    "dev": "concurrently \"tsup index.ts --format esm --dts --watch\" \"nodemon --watch dist/index.js\"",
+    "lint": "eslint ."
   },
   "dependencies": {
-    "@llamaindex/core": "^0.2.6",
     "ai": "3.3.42",
     "cors": "^2.8.5",
     "dotenv": "^16.3.1",
     "duck-duck-scrape": "^2.2.5",
     "express": "^4.18.2",
-    "llamaindex": "0.6.22",
+    "llamaindex": "0.8.2",
     "pdf2json": "3.0.5",
     "ajv": "^8.12.0",
     "@e2b/code-interpreter": "0.0.9-beta.3",
     "got": "^14.4.1",
     "@apidevtools/swagger-parser": "^10.1.0",
     "formdata-node": "^6.0.3",
-    "marked": "^14.1.2"
+    "marked": "^14.1.2",
+    "papaparse": "^5.4.1"
   },
   "devDependencies": {
     "@types/cors": "^2.8.16",
     "@types/express": "^4.17.21",
     "@types/node": "^20.9.5",
+    "typescript-eslint": "^8.14.0",
+    "@llamaindex/workflow": "^0.0.3",
+    "@types/papaparse": "^5.3.15",
     "concurrently": "^8.2.2",
-    "eslint": "^8.54.0",
+    "eslint": "^9.14.0",
     "eslint-config-prettier": "^8.10.0",
     "nodemon": "^3.0.1",
     "prettier": "^3.2.5",

templates/types/streaming/express/src/controllers/chat.controller.ts

@@ -32,7 +32,7 @@ export const chat = async (req: Request, res: Response) => {
 
     // Setup callbacks
     const callbackManager = createCallbackManager(vercelStreamData);
-    const chatHistory: ChatMessage[] = messages as ChatMessage[];
+    const chatHistory: ChatMessage[] = messages.slice(0, -1) as ChatMessage[];
 
     // Calling LlamaIndex's ChatEngine to get a streamed response
     const response = await Settings.withCallbackManager(callbackManager, () => {

templates/types/streaming/fastapi/README-template.md

@@ -21,12 +21,14 @@ Second, generate the embeddings of the documents in the `./data` directory (if t
 poetry run generate
 ```
 
-Third, run the development server:
+Third, run the app:
 
 ```
-python main.py
+poetry run dev
 ```
 
+Open [http://localhost:8000](http://localhost:8000) with your browser to start the app.
+
 The example provides two different API endpoints:
 
 1. `/api/chat` - a streaming chat endpoint
@@ -50,12 +52,10 @@ curl --location 'localhost:8000/api/chat/request' \
 
 You can start editing the API endpoints by modifying `app/api/routers/chat.py`. The endpoints auto-update as you save the file. You can delete the endpoint you're not using.
 
-Open [http://localhost:8000/docs](http://localhost:8000/docs) with your browser to see the Swagger UI of the API.
-
-The API allows CORS for all origins to simplify development. You can change this behavior by setting the `ENVIRONMENT` environment variable to `prod`:
+To start the app in **production**, run:
 
 ```
-ENVIRONMENT=prod python main.py
+poetry run prod
 ```
 
 ## Using Docker

templates/types/streaming/fastapi/app/api/routers/models.py

@@ -60,9 +60,11 @@ class AnnotationFileData(BaseModel):
         # Include document IDs if it's available
         if file.refs is not None:
             default_content += f"Document IDs: {file.refs}\n"
-        # Include sandbox file path
+        # file path
         sandbox_file_path = f"/tmp/{file.name}"
+        local_file_path = f"output/uploaded/{file.name}"
         default_content += f"Sandbox file path (instruction: only use sandbox path for artifact or code interpreter tool): {sandbox_file_path}\n"
+        default_content += f"Local file path (instruction: Use for local tools: form filling, extractor): {local_file_path}\n"
         return default_content
 
     def to_llm_content(self) -> Optional[str]:
@@ -128,24 +130,29 @@ class ChatData(BaseModel):
 
     def get_last_message_content(self) -> str:
         """
-        Get the content of the last message along with the data content if available.
-        Fallback to use data content from previous messages
+        Get the content of the last message along with the data content from all user messages
         """
         if len(self.messages) == 0:
             raise ValueError("There is not any message in the chat")
+
         last_message = self.messages[-1]
         message_content = last_message.content
-        for message in reversed(self.messages):
+
+        # Collect annotation contents from all user messages
+        all_annotation_contents: List[str] = []
+        for message in self.messages:
             if message.role == MessageRole.USER and message.annotations is not None:
                 annotation_contents = filter(
                     None,
                     [annotation.to_content() for annotation in message.annotations],
                 )
-                if not annotation_contents:
-                    continue
-                annotation_text = "\n".join(annotation_contents)
-                message_content = f"{message_content}\n{annotation_text}"
-                break
+                all_annotation_contents.extend(annotation_contents)
+
+        # Add all annotation contents if any exist
+        if len(all_annotation_contents) > 0:
+            annotation_text = "\n".join(all_annotation_contents)
+            message_content = f"{message_content}\n{annotation_text}"
+
         return message_content
 
     def _get_agent_messages(self, max_messages: int = 10) -> List[str]:

templates/types/streaming/fastapi/app/config.py

@@ -1 +1,4 @@
+import os
+
 DATA_DIR = "data"
+STATIC_DIR = os.getenv("STATIC_DIR", "static")

templates/types/streaming/fastapi/app/middlewares/frontend.py

@@ -0,0 +1,78 @@
+import logging
+from typing import Set
+
+import httpx
+from fastapi import Request
+from fastapi.responses import StreamingResponse
+
+logger = logging.getLogger("uvicorn")
+
+
+class FrontendProxyMiddleware:
+    """
+    Proxy requests to the frontend development server
+    """
+
+    def __init__(
+        self,
+        app,
+        frontend_endpoint: str,
+        excluded_paths: Set[str],
+    ):
+        self.app = app
+        self.excluded_paths = excluded_paths
+        self.frontend_endpoint = frontend_endpoint
+
+    async def _request_frontend(
+        self,
+        request: Request,
+        path: str,
+        timeout: float = 60.0,
+    ):
+        async with httpx.AsyncClient(timeout=timeout) as client:
+            url = f"{self.frontend_endpoint}/{path}"
+            if request.query_params:
+                url = f"{url}?{request.query_params}"
+
+            headers = dict(request.headers)
+            try:
+                body = await request.body() if request.method != "GET" else None
+
+                response = await client.request(
+                    method=request.method,
+                    url=url,
+                    headers=headers,
+                    content=body,
+                    follow_redirects=True,
+                )
+
+                response_headers = dict(response.headers)
+                response_headers.pop("content-encoding", None)
+                response_headers.pop("content-length", None)
+
+                return StreamingResponse(
+                    response.iter_bytes(),
+                    status_code=response.status_code,
+                    headers=response_headers,
+                )
+            except Exception as e:
+                logger.error(f"Proxy error: {str(e)}")
+                raise
+
+    def _is_excluded_path(self, path: str) -> bool:
+        return any(
+            path.startswith(excluded_path) for excluded_path in self.excluded_paths
+        )
+
+    async def __call__(self, scope, receive, send):
+        if scope["type"] != "http":
+            return await self.app(scope, receive, send)
+
+        request = Request(scope, receive)
+        path = request.url.path
+
+        if self._is_excluded_path(path):
+            return await self.app(scope, receive, send)
+
+        response = await self._request_frontend(request, path.lstrip("/"))
+        return await response(scope, receive, send)

templates/types/streaming/fastapi/app/observability.py

@@ -1,2 +1,3 @@
+# TODO: You can add observability here. For templates re-start `create-llama` with `--pro` flag to generate a new project with observability.
 def init_observability():
     pass

templates/types/streaming/fastapi/app/services/file.py

@@ -6,7 +6,7 @@ import re
 import uuid
 from io import BytesIO
 from pathlib import Path
-from typing import Dict, List, Optional, Tuple
+from typing import List, Optional, Tuple
 
 from llama_index.core import VectorStoreIndex
 from llama_index.core.ingestion import IngestionPipeline
@@ -14,7 +14,6 @@ from llama_index.core.readers.file.base import (
     _try_loading_included_file_formats as get_file_loaders_map,
 )
 from llama_index.core.schema import Document
-from llama_index.core.tools.function_tool import FunctionTool
 from llama_index.indices.managed.llama_cloud.base import LlamaCloudIndex
 from llama_index.readers.file import FlatReader
 from pydantic import BaseModel, Field
@@ -78,10 +77,8 @@ class FileService:
             save_dir=PRIVATE_STORE_PATH,
         )
 
-        tools = _get_available_tools()
-        code_executor_tools = ["interpreter", "artifact"]
-        # If the file is CSV and there is a code executor tool, we don't need to index.
-        if extension == "csv" and any(tool in tools for tool in code_executor_tools):
+        # Don't index csv files (they are handled by tools)
+        if extension == "csv":
             return document_file
         else:
             # Insert the file into the index and update document ids to the file metadata
@@ -245,13 +242,11 @@ class FileService:
         except ImportError as e:
             raise ValueError("LlamaCloudFileService is not found") from e
 
-        project_id = index._get_project_id()
-        pipeline_id = index._get_pipeline_id()
         # LlamaCloudIndex is a managed index so we can directly use the files
         upload_file = (file_name, BytesIO(file_data))
         doc_id = LLamaCloudFileService.add_file_to_pipeline(
-            project_id,
-            pipeline_id,
+            index.project.id,
+            index.pipeline.id,
             upload_file,
             custom_metadata={},
         )
@@ -283,18 +278,3 @@ def _default_file_loaders_map():
     default_loaders[".txt"] = FlatReader
     default_loaders[".csv"] = FlatReader
     return default_loaders
-
-
-def _get_available_tools() -> Dict[str, List[FunctionTool]]:
-    try:
-        from app.engine.tools import ToolFactory  # type: ignore
-    except ImportError:
-        logger.warning("ToolFactory not found, no tools will be available")
-        return {}
-
-    try:
-        tools = ToolFactory.from_env(map_result=True)
-        return tools  # type: ignore
-    except Exception as e:
-        logger.error(f"Error loading tools from environment: {str(e)}")
-        raise ValueError(f"Failed to get available tools: {str(e)}") from e

templates/types/streaming/fastapi/gitignore

@@ -2,3 +2,4 @@ __pycache__
 storage
 .env
 output
+static/

templates/types/streaming/fastapi/main.py

@@ -1,5 +1,5 @@
 # flake8: noqa: E402
-from app.config import DATA_DIR
+from app.config import DATA_DIR, STATIC_DIR
 from dotenv import load_dotenv
 
 load_dotenv()
@@ -9,10 +9,10 @@ import os
 
 import uvicorn
 from app.api.routers import api_router
+from app.middlewares.frontend import FrontendProxyMiddleware
 from app.observability import init_observability
 from app.settings import init_settings
 from fastapi import FastAPI
-from fastapi.middleware.cors import CORSMiddleware
 from fastapi.responses import RedirectResponse
 from fastapi.staticfiles import StaticFiles
 
@@ -24,38 +24,43 @@ init_observability()
 environment = os.getenv("ENVIRONMENT", "dev")  # Default to 'development' if not set
 logger = logging.getLogger("uvicorn")
 
-if environment == "dev":
-    logger.warning("Running in development mode - allowing CORS for all origins")
-    app.add_middleware(
-        CORSMiddleware,
-        allow_origins=["*"],
-        allow_credentials=True,
-        allow_methods=["*"],
-        allow_headers=["*"],
-    )
 
-    # Redirect to documentation page when accessing base URL
-    @app.get("/")
-    async def redirect_to_docs():
-        return RedirectResponse(url="/docs")
-
-
-def mount_static_files(directory, path):
+def mount_static_files(directory, path, html=False):
     if os.path.exists(directory):
         logger.info(f"Mounting static files '{directory}' at '{path}'")
         app.mount(
             path,
-            StaticFiles(directory=directory, check_dir=False),
+            StaticFiles(directory=directory, check_dir=False, html=html),
             name=f"{directory}-static",
         )
 
 
+app.include_router(api_router, prefix="/api")
+
 # Mount the data files to serve the file viewer
 mount_static_files(DATA_DIR, "/api/files/data")
 # Mount the output files from tools
 mount_static_files("output", "/api/files/output")
 
-app.include_router(api_router, prefix="/api")
+if environment == "dev":
+    frontend_endpoint = os.getenv("FRONTEND_ENDPOINT")
+    if frontend_endpoint:
+        app.add_middleware(
+            FrontendProxyMiddleware,
+            frontend_endpoint=frontend_endpoint,
+            excluded_paths=set(
+                route.path for route in app.routes if hasattr(route, "path")
+            ),
+        )
+    else:
+        logger.warning("No frontend endpoint - starting API server only")
+
+        @app.get("/")
+        async def redirect_to_docs():
+            return RedirectResponse(url="/docs")
+else:
+    # Mount the frontend static files (production)
+    mount_static_files(STATIC_DIR, "/", html=True)
 
 if __name__ == "__main__":
     app_host = os.getenv("APP_HOST", "0.0.0.0")

templates/types/streaming/fastapi/pyproject.toml

@@ -7,15 +7,19 @@ readme = "README.md"
 
 [tool.poetry.scripts]
 generate = "app.engine.generate:generate_datasource"
+dev = "run:dev" # Starts the app in dev mode
+prod = "run:prod" # Starts the app in prod mode
+build = "run:build" # Builds the frontend assets and copies them to the static directory
 
 [tool.poetry.dependencies]
-python = ">=3.11,<3.13"
+python = ">=3.11,<3.14"
 fastapi = "^0.109.1"
 uvicorn = { extras = ["standard"], version = "^0.23.2" }
 python-dotenv = "^1.0.0"
 aiostream = "^0.5.2"
 cachetools = "^5.3.3"
 llama-index = "^0.11.17"
+rich = "^13.9.4"
 
 [tool.poetry.group.dev.dependencies]
 mypy = "^1.8.0"

templates/types/streaming/fastapi/run.py

@@ -0,0 +1,275 @@
+import asyncio
+import os
+import shutil
+import socket
+from asyncio.subprocess import Process
+from pathlib import Path
+from shutil import which
+from subprocess import CalledProcessError, run
+
+import dotenv
+import rich
+
+dotenv.load_dotenv()
+
+
+FRONTEND_DIR = Path(os.getenv("FRONTEND_DIR", ".frontend"))
+DEFAULT_FRONTEND_PORT = 3000
+STATIC_DIR = Path(os.getenv("STATIC_DIR", "static"))
+
+
+def build():
+    """
+    Build the frontend and copy the static files to the backend.
+
+    Raises:
+        SystemError: If any build step fails
+    """
+    static_dir = Path("static")
+
+    try:
+        package_manager = _get_node_package_manager()
+        _install_frontend_dependencies()
+
+        rich.print("\n[bold]Building the frontend[/bold]")
+        run([package_manager, "run", "build"], cwd=FRONTEND_DIR, check=True)
+
+        if static_dir.exists():
+            shutil.rmtree(static_dir)
+        static_dir.mkdir(exist_ok=True)
+
+        shutil.copytree(FRONTEND_DIR / "out", static_dir, dirs_exist_ok=True)
+
+        rich.print(
+            "\n[bold]Built frontend successfully![/bold]"
+            "\n[bold]Run: 'poetry run prod' to start the app[/bold]"
+            "\n[bold]Don't forget to update the .env file![/bold]"
+        )
+    except CalledProcessError as e:
+        raise SystemError(f"Build failed during {e.cmd}") from e
+    except Exception as e:
+        raise SystemError(f"Build failed: {str(e)}") from e
+
+
+def dev():
+    asyncio.run(start_development_servers())
+
+
+def prod():
+    asyncio.run(start_production_server())
+
+
+async def start_development_servers():
+    """
+    Start both frontend and backend development servers.
+    Frontend runs with hot reloading, backend runs FastAPI server.
+
+    Raises:
+        SystemError: If either server fails to start
+    """
+    rich.print("\n[bold]Starting development servers[/bold]")
+
+    try:
+        processes = []
+        if _is_frontend_included():
+            frontend_process, frontend_port = await _run_frontend()
+            processes.append(frontend_process)
+            backend_process = await _run_backend(
+                envs={
+                    "ENVIRONMENT": "dev",
+                    "FRONTEND_ENDPOINT": f"http://localhost:{frontend_port}",
+                },
+            )
+            processes.append(backend_process)
+        else:
+            backend_process = await _run_backend(
+                envs={"ENVIRONMENT": "dev"},
+            )
+            processes.append(backend_process)
+
+        try:
+            # Wait for processes to complete
+            await asyncio.gather(*[process.wait() for process in processes])
+        except (asyncio.CancelledError, KeyboardInterrupt):
+            rich.print("\n[bold yellow]Shutting down...[/bold yellow]")
+        finally:
+            # Terminate both processes
+            for process in processes:
+                process.terminate()
+                try:
+                    await asyncio.wait_for(process.wait(), timeout=5)
+                except asyncio.TimeoutError:
+                    process.kill()
+
+    except Exception as e:
+        raise SystemError(f"Failed to start development servers: {str(e)}") from e
+
+
+async def start_production_server():
+    if _is_frontend_included():
+        is_frontend_built = (FRONTEND_DIR / "out" / "index.html").exists()
+        is_frontend_static_dir_exists = STATIC_DIR.exists()
+        if not is_frontend_built or not is_frontend_static_dir_exists:
+            build()
+
+    try:
+        process = await _run_backend(
+            envs={"ENVIRONMENT": "prod"},
+        )
+        await process.wait()
+    except Exception as e:
+        raise SystemError(f"Failed to start production server: {str(e)}") from e
+    finally:
+        process.terminate()
+        try:
+            await asyncio.wait_for(process.wait(), timeout=5)
+        except asyncio.TimeoutError:
+            process.kill()
+
+
+async def _run_frontend(
+    port: int = DEFAULT_FRONTEND_PORT,
+    timeout: int = 5,
+) -> tuple[Process, int]:
+    """
+    Start the frontend development server and return its process and port.
+
+    Returns:
+        tuple[Process, int]: The frontend process and the port it's running on
+    """
+    # Install dependencies
+    _install_frontend_dependencies()
+
+    port = _find_free_port(start_port=DEFAULT_FRONTEND_PORT)
+    package_manager = _get_node_package_manager()
+    frontend_process = await asyncio.create_subprocess_exec(
+        package_manager,
+        "run",
+        "dev",
+        "-p",
+        str(port),
+        cwd=FRONTEND_DIR,
+    )
+    rich.print(
+        f"\n[bold]Waiting for frontend to start, port: {port}, process id: {frontend_process.pid}[/bold]"
+    )
+    # Block until the frontend is accessible
+    for _ in range(timeout):
+        await asyncio.sleep(1)
+        # Check if the frontend is accessible (port is open) or frontend_process is running
+        if frontend_process.returncode is not None:
+            raise RuntimeError("Could not start frontend dev server")
+        if not _is_bindable_port(port):
+            rich.print(
+                f"\n[bold green]Frontend dev server is running on port {port}[/bold green]"
+            )
+            return frontend_process, port
+    raise TimeoutError(f"Frontend dev server failed to start within {timeout} seconds")
+
+
+async def _run_backend(
+    envs: dict[str, str | None] = {},
+) -> Process:
+    """
+    Start the backend development server.
+
+    Args:
+        frontend_port: The port number the frontend is running on
+    Returns:
+        Process: The backend process
+    """
+    # Merge environment variables
+    envs = {**os.environ, **(envs or {})}
+    rich.print("\n[bold]Starting backend FastAPI server...[/bold]")
+    poetry_executable = _get_poetry_executable()
+    return await asyncio.create_subprocess_exec(
+        poetry_executable,
+        "run",
+        "python",
+        "main.py",
+        env=envs,
+    )
+
+
+def _install_frontend_dependencies():
+    package_manager = _get_node_package_manager()
+    rich.print(
+        f"\n[bold]Installing frontend dependencies using {Path(package_manager).name}. It might take a while...[/bold]"
+    )
+    run([package_manager, "install"], cwd=".frontend", check=True)
+
+
+def _get_node_package_manager() -> str:
+    """
+    Check for available package managers and return the preferred one.
+    Returns 'pnpm' if installed, falls back to 'npm'.
+    Raises SystemError if neither is installed.
+
+    Returns:
+        str: The full path to the available package manager executable
+    """
+    # On Windows, we need to check for .cmd extensions
+    pnpm_cmds = ["pnpm", "pnpm.cmd"]
+    npm_cmds = ["npm", "npm.cmd"]
+
+    for cmd in pnpm_cmds:
+        cmd_path = which(cmd)
+        if cmd_path is not None:
+            return cmd_path
+
+    for cmd in npm_cmds:
+        cmd_path = which(cmd)
+        if cmd_path is not None:
+            return cmd_path
+
+    raise SystemError(
+        "Neither pnpm nor npm is installed. Please install Node.js and a package manager first."
+    )
+
+
+def _get_poetry_executable() -> str:
+    """
+    Check for available Poetry executables and return the preferred one.
+    Returns 'poetry' if installed, falls back to 'poetry.cmd'.
+    Raises SystemError if neither is installed.
+
+    Returns:
+        str: The full path to the available Poetry executable
+    """
+    poetry_cmds = ["poetry", "poetry.cmd"]
+    for cmd in poetry_cmds:
+        cmd_path = which(cmd)
+        if cmd_path is not None:
+            return cmd_path
+    raise SystemError("Poetry is not installed. Please install Poetry first.")
+
+
+def _is_bindable_port(port: int) -> bool:
+    """Check if a port is available by attempting to connect to it."""
+    with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
+        try:
+            # Try to connect to the port
+            s.connect(("localhost", port))
+            # If we can connect, port is in use
+            return False
+        except ConnectionRefusedError:
+            # Connection refused means port is available
+            return True
+        except socket.error:
+            # Other socket errors also likely mean port is available
+            return True
+
+
+def _find_free_port(start_port: int) -> int:
+    """
+    Find a free port starting from the given port number.
+    """
+    for port in range(start_port, 65535):
+        if _is_bindable_port(port):
+            return port
+    raise SystemError("No free port found")
+
+
+def _is_frontend_included() -> bool:
+    """Check if the app has frontend"""
+    return FRONTEND_DIR.exists()

templates/types/streaming/nextjs/app/api/chat/route.ts

@@ -45,7 +45,7 @@ export async function POST(request: NextRequest) {
 
     // Setup callbacks
     const callbackManager = createCallbackManager(vercelStreamData);
-    const chatHistory: ChatMessage[] = messages as ChatMessage[];
+    const chatHistory: ChatMessage[] = messages.slice(0, -1) as ChatMessage[];
 
     // Calling LlamaIndex's ChatEngine to get a streamed response
     const response = await Settings.withCallbackManager(callbackManager, () => {

templates/types/streaming/nextjs/app/api/files/[...slug]/route.ts

@@ -9,9 +9,9 @@ import { DATA_DIR } from "../../chat/engine/loader";
  */
 export async function GET(
   _request: NextRequest,
-  { params }: { params: { slug: string[] } },
+  { params }: { params: Promise<{ slug: string[] }> },
 ) {
-  const slug = params.slug;
+  const slug = (await params).slug;
 
   if (!slug) {
     return NextResponse.json({ detail: "Missing file slug" }, { status: 400 });
@@ -21,7 +21,7 @@ export async function GET(
     return NextResponse.json({ detail: "Invalid file path" }, { status: 400 });
   }
 
-  const [folder, ...pathTofile] = params.slug; // data, file.pdf
+  const [folder, ...pathTofile] = slug; // data, file.pdf
   const allowedFolders = ["data", "output"];
 
   if (!allowedFolders.includes(folder)) {

templates/types/streaming/nextjs/app/components/chat-section.tsx

@@ -1,57 +1,27 @@
 "use client";
 
+import { ChatSection as ChatSectionUI } from "@llamaindex/chat-ui";
+import "@llamaindex/chat-ui/styles/code.css";
+import "@llamaindex/chat-ui/styles/katex.css";
+import "@llamaindex/chat-ui/styles/pdf.css";
 import { useChat } from "ai/react";
-import { useState } from "react";
-import { ChatInput, ChatMessages } from "./ui/chat";
+import CustomChatInput from "./ui/chat/chat-input";
+import CustomChatMessages from "./ui/chat/chat-messages";
 import { useClientConfig } from "./ui/chat/hooks/use-config";
 
 export default function ChatSection() {
   const { backend } = useClientConfig();
-  const [requestData, setRequestData] = useState<any>();
-  const {
-    messages,
-    input,
-    isLoading,
-    handleSubmit,
-    handleInputChange,
-    reload,
-    stop,
-    append,
-    setInput,
-  } = useChat({
-    body: { data: requestData },
+  const handler = useChat({
     api: `${backend}/api/chat`,
-    headers: {
-      "Content-Type": "application/json", // using JSON because of vercel/ai 2.2.26
-    },
     onError: (error: unknown) => {
       if (!(error instanceof Error)) throw error;
-      const message = JSON.parse(error.message);
-      alert(message.detail);
+      alert(JSON.parse(error.message).detail);
     },
-    sendExtraMessageFields: true,
   });
-
   return (
-    <div className="space-y-4 w-full h-full flex flex-col">
-      <ChatMessages
-        messages={messages}
-        isLoading={isLoading}
-        reload={reload}
-        stop={stop}
-        append={append}
-      />
-      <ChatInput
-        input={input}
-        handleSubmit={handleSubmit}
-        handleInputChange={handleInputChange}
-        isLoading={isLoading}
-        messages={messages}
-        append={append}
-        setInput={setInput}
-        requestParams={{ params: requestData }}
-        setRequestData={setRequestData}
-      />
-    </div>
+    <ChatSectionUI handler={handler} className="w-full h-full">
+      <CustomChatMessages />
+      <CustomChatInput />
+    </ChatSectionUI>
   );
 }