Release 0.5.5 (#564)

Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>

.github/workflows/e2e.yml

@@ -2,8 +2,12 @@ name: E2E Tests
 on:
   push:
     branches: [main]
+    paths-ignore:
+      - "llama-index-server/**"
   pull_request:
     branches: [main]
+    paths-ignore:
+      - "llama-index-server/**"
 
 env:
   POETRY_VERSION: "1.6.1"
@@ -67,6 +71,8 @@ jobs:
           LLAMA_CLOUD_API_KEY: ${{ secrets.LLAMA_CLOUD_API_KEY }}
           FRAMEWORK: ${{ matrix.frameworks }}
           DATASOURCE: ${{ matrix.datasources }}
+          PYTHONIOENCODING: utf-8
+          PYTHONLEGACYWINDOWSSTDIO: utf-8
         working-directory: .
 
       - uses: actions/upload-artifact@v4
@@ -86,7 +92,7 @@ jobs:
         node-version: [18, 20]
         python-version: ["3.11"]
         os: [macos-latest, windows-latest, ubuntu-22.04]
-        frameworks: ["nextjs", "express"]
+        frameworks: ["nextjs"]
         datasources: ["--no-files", "--example-file", "--llamacloud"]
     defaults:
       run:

.github/workflows/release_llama_index_server.yml

@@ -0,0 +1,130 @@
+name: Release llama-index-server
+
+on:
+  push:
+    branches:
+      - main
+    paths:
+      - "llama-index-server/**"
+      - ".github/workflows/release_llama_index_server.yml"
+  pull_request:
+    types:
+      - closed
+
+concurrency: ${{ github.workflow }}-${{ github.ref }}
+
+jobs:
+  release:
+    name: Create Release PR
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: ./llama-index-server
+    if: |
+      github.event_name == 'push' && 
+      !startsWith(github.ref, 'refs/heads/release/llama-index-server-v')
+
+    steps:
+      - name: Checkout Repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+
+      - name: Install Poetry
+        run: |
+          curl -sSL https://install.python-poetry.org | python3 -
+
+      - name: Install dependencies
+        run: poetry install
+
+      - name: Setup Git
+        run: |
+          git config --global user.email "github-actions[bot]@users.noreply.github.com"
+          git config --global user.name "github-actions[bot]"
+
+      - name: Bump patch version
+        run: |
+          poetry version patch
+          git add pyproject.toml
+          git commit -m "chore(release): bump version to $(poetry version -s)"
+
+      - name: Get current version
+        id: get_version
+        run: |
+          version=$(poetry version -s)
+          echo "current_version=${version}" >> "$GITHUB_OUTPUT"
+
+      - name: Create Release PR
+        uses: peter-evans/create-pull-request@v6
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          commit-message: "Release: llama-index-server v${{ steps.get_version.outputs.current_version }}"
+          title: "Release: llama-index-server v${{ steps.get_version.outputs.current_version }}"
+          body: |
+            This PR was automatically created to release a new version of the llama-index-server package.
+
+            Version: ${{ steps.get_version.outputs.current_version }}
+
+            Please review the changes and merge to trigger the release.
+          branch: release/llama-index-server-v${{ steps.get_version.outputs.current_version }}
+          base: main
+          labels: release, llama-index-server
+
+  publish:
+    name: Publish to PyPI
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: ./llama-index-server
+    if: |
+      github.event_name == 'pull_request' && 
+      github.event.pull_request.merged == true && 
+      startsWith(github.event.pull_request.title, 'Release: llama-index-server') &&
+      startsWith(github.event.pull_request.head.ref, 'release/llama-index-server-v')
+
+    steps:
+      - name: Checkout Repository
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+
+      - name: Install Poetry
+        run: |
+          curl -sSL https://install.python-poetry.org | python3 -
+
+      - name: Install dependencies
+        run: poetry install
+
+      - name: Get current version
+        id: get_version
+        run: |
+          version=$(poetry version -s)
+          echo "current_version=${version}" >> "$GITHUB_OUTPUT"
+
+      - name: Build and publish to PyPI
+        uses: JRubics/poetry-publish@v2.1
+        with:
+          python_version: "3.11"
+          pypi_token: ${{ secrets.PYPI_TOKEN }}
+          package_directory: "llama-index-server"
+          poetry_install_options: "--without dev"
+
+      - name: Create GitHub Release
+        uses: softprops/action-gh-release@v2
+        with:
+          tag_name: llama-index-server-v${{ steps.get_version.outputs.current_version }}
+          name: "llama-index-server v${{ steps.get_version.outputs.current_version }}"
+          body: |
+            Release of llama-index-server v${{ steps.get_version.outputs.current_version }}
+          draft: false
+          prerelease: false
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/test_llama_index_server.yml

@@ -0,0 +1,111 @@
+name: Build Package
+
+on:
+  pull_request:
+
+env:
+  POETRY_VERSION: "1.8.3"
+  PYTHON_VERSION: "3.9"
+
+jobs:
+  unit-test:
+    name: Unit Tests
+    runs-on: ${{ matrix.os }}
+    defaults:
+      run:
+        working-directory: llama-index-server
+    strategy:
+      matrix:
+        os: [ubuntu-latest, windows-latest]
+        python-version: ["3.9"]
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install Poetry
+        run: pipx install poetry==${{ env.POETRY_VERSION }}
+
+      - name: Set up python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+          cache: "poetry"
+
+      - name: Configure Poetry
+        run: |
+          poetry config virtualenvs.create true
+          poetry config virtualenvs.in-project true
+          poetry env use python
+
+      - name: Install dependencies
+        shell: bash
+        run: poetry install --with dev
+
+      - name: Run unit tests
+        shell: bash
+        run: |
+          poetry run pytest tests
+
+  type-check:
+    name: Type Check
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: llama-index-server
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install Poetry
+        run: pipx install poetry==${{ env.POETRY_VERSION }}
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
+          cache: "poetry"
+
+      - name: Configure Poetry
+        run: |
+          poetry config virtualenvs.create true
+          poetry config virtualenvs.in-project true
+          poetry env use python
+
+      - name: Install dependencies
+        shell: bash
+        run: poetry install --with dev
+
+      - name: Run mypy
+        shell: bash
+        run: poetry run mypy llama_index
+
+  build:
+    needs: [unit-test, type-check]
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: llama-index-server
+    steps:
+      - uses: actions/checkout@v4
+      - name: Install Poetry
+        run: pipx install poetry==${{ env.POETRY_VERSION }}
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
+      - name: Clear python cache
+        shell: bash
+        run: poetry cache clear --all pypi
+      - name: Build package
+        shell: bash
+        run: poetry build
+      - name: Test installing built package
+        shell: bash
+        run: python -m pip install .
+      - name: Test import
+        shell: bash
+        working-directory: ${{ vars.RUNNER_TEMP }}
+        run: python -c "from llama_index.server import LlamaIndexServer"
+      - name: Upload artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: llama-index-server
+          path: llama-index-server/dist/

.gitignore

@@ -8,6 +8,7 @@ node_modules
 
 # testing
 coverage
+.coverage
 
 # next.js
 .next/
@@ -48,6 +49,13 @@ e2e/cache
 
 # Python
 .mypy_cache/
+venv/
+.venv/
+dist/
+.__pycache__
+__pycache__
+.python-version
+.ui
 
 # build artifacts
 create-llama-*.tgz

CHANGELOG.md

@@ -1,5 +1,46 @@
 # create-llama
 
+## 0.5.5
+
+### Patch Changes
+
+- ee85320: The default custom deep research component does not work.
+
+## 0.5.4
+
+### Patch Changes
+
+- 7c3b279: Support code generation of event components using an LLM (Python)
+
+## 0.5.3
+
+### Patch Changes
+
+- 76ec360: Update templates to use new chat ui config
+
+## 0.5.2
+
+### Patch Changes
+
+- c9f8f8d: Use custom component for deep research use case
+
+## 0.5.1
+
+### Patch Changes
+
+- 08b3e07: Simplify the local index code.
+
+## 0.5.0
+
+### Minor Changes
+
+- 54c9e2f: Simplified generated code using LlamaIndexServer
+
+### Patch Changes
+
+- 0e4ecfa: fix: add trycatch for generating error
+- ee69ce7: bump: chat-ui and tailwind v4
+
 ## 0.4.0
 
 ### Minor Changes

create-app.ts

@@ -90,7 +90,7 @@ export async function createApp({
   // Install backend
   await installTemplate({ ...args, backend: true });
 
-  if (frontend && framework === "fastapi") {
+  if (frontend && framework === "fastapi" && template !== "llamaindexserver") {
     // install frontend
     const frontendRoot = path.join(root, ".frontend");
     await makeDir(frontendRoot);
@@ -110,7 +110,7 @@ export async function createApp({
     console.log();
   }
 
-  if (toolsRequireConfig(tools)) {
+  if (toolsRequireConfig(tools) && template !== "llamaindexserver") {
     const configFile =
       framework === "fastapi" ? "config/tools.yaml" : "config/tools.json";
     console.log(

e2e/shared/llamaindexserver_template.spec.ts

@@ -16,15 +16,17 @@ const templateFramework: TemplateFramework = process.env.FRAMEWORK
 const dataSource: string = "--example-file";
 const templateUI: TemplateUI = "shadcn";
 const templatePostInstallAction: TemplatePostInstallAction = "runApp";
-const appType: AppType = templateFramework === "fastapi" ? "--frontend" : "";
+const appType: AppType = "--frontend";
 const userMessage = "Write a blog post about physical standards for letters";
-const templateUseCases = ["financial_report", "blog", "form_filling"];
+const templateUseCases = ["financial_report", "agentic_rag", "deep_research"];
 
 for (const useCase of templateUseCases) {
-  test.describe(`Test multiagent template ${useCase} ${templateFramework} ${dataSource} ${templateUI} ${appType} ${templatePostInstallAction}`, async () => {
+  test.describe(`Test use case ${useCase} ${templateFramework} ${dataSource} ${templateUI} ${appType} ${templatePostInstallAction}`, async () => {
     test.skip(
-      process.platform !== "linux" || process.env.DATASOURCE === "--no-files",
-      "The multiagent template currently only works with files. We also only run on Linux to speed up tests.",
+      process.platform !== "linux" ||
+        process.env.DATASOURCE === "--no-files" ||
+        templateFramework === "express",
+      "The llamaindexserver template currently only works with nextjs, fastapi. We also only run on Linux to speed up tests.",
     );
     let port: number;
     let cwd: string;
@@ -38,7 +40,7 @@ for (const useCase of templateUseCases) {
       cwd = await createTestDir();
       const result = await runCreateLlama({
         cwd,
-        templateType: "multiagent",
+        templateType: "llamaindexserver",
         templateFramework,
         dataSource,
         vectorDb,
@@ -72,9 +74,9 @@ for (const useCase of templateUseCases) {
       test.skip(
         templatePostInstallAction !== "runApp" ||
           useCase === "financial_report" ||
-          useCase === "form_filling" ||
+          useCase === "deep_research" ||
           templateFramework === "express",
-        "Skip chat tests for financial report and form filling.",
+        "Skip chat tests for financial report and deep research.",
       );
       await page.goto(`http://localhost:${port}`);
       await page.fill("form textarea", userMessage);
@@ -86,6 +88,12 @@ for (const useCase of templateUseCases) {
       await page.click("form button[type=submit]");
 
       const response = await responsePromise;
+      console.log(`Response status: ${response.status()}`);
+      const responseBody = await response
+        .text()
+        .catch((e) => `Error reading body: ${e}`);
+      console.log(`Response body: ${responseBody}`);
+
       expect(response.ok()).toBeTruthy();
     });
 

e2e/utils.ts

@@ -113,7 +113,12 @@ export async function runCreateLlama({
   if (observability) {
     commandArgs.push("--observability", observability);
   }
-  if ((templateType === "multiagent" || templateType === "reflex") && useCase) {
+  if (
+    (templateType === "multiagent" ||
+      templateType === "reflex" ||
+      templateType === "llamaindexserver") &&
+    useCase
+  ) {
     commandArgs.push("--use-case", useCase);
   }
 

helpers/env-variables.ts

@@ -44,6 +44,7 @@ const renderEnvVar = (envVars: EnvVar[]): string => {
 const getVectorDBEnvs = (
   vectorDb?: TemplateVectorDB,
   framework?: TemplateFramework,
+  template?: TemplateType,
 ): EnvVar[] => {
   if (!vectorDb || !framework) {
     return [];
@@ -168,7 +169,7 @@ const getVectorDBEnvs = (
           description:
             "The organization ID for the LlamaCloud project (uses default organization if not specified)",
         },
-        ...(framework === "nextjs"
+        ...(framework === "nextjs" && template !== "llamaindexserver"
           ? // activate index selector per default (not needed for non-NextJS backends as it's handled by createFrontendEnvFile)
             [
               {
@@ -223,13 +224,15 @@ Otherwise, use CHROMA_HOST and CHROMA_PORT config above`,
         },
       ];
     default:
-      return [
-        {
-          name: "STORAGE_CACHE_DIR",
-          description: "The directory to store the local storage cache.",
-          value: ".cache",
-        },
-      ];
+      return template !== "llamaindexserver"
+        ? [
+            {
+              name: "STORAGE_CACHE_DIR",
+              description: "The directory to store the local storage cache.",
+              value: ".cache",
+            },
+          ]
+        : [];
   }
 };
 
@@ -382,38 +385,42 @@ const getModelEnvs = (modelConfig: ModelConfig): EnvVar[] => {
 
 const getFrameworkEnvs = (
   framework: TemplateFramework,
+  template: TemplateType,
   port?: number,
 ): EnvVar[] => {
   const sPort = port?.toString() || "8000";
-  const result: EnvVar[] = [
-    {
-      name: "FILESERVER_URL_PREFIX",
-      description:
-        "FILESERVER_URL_PREFIX is the URL prefix of the server storing the images generated by the interpreter.",
-      value:
-        framework === "nextjs"
-          ? // FIXME: if we are using nextjs, port should be 3000
-            "http://localhost:3000/api/files"
-          : `http://localhost:${sPort}/api/files`,
-    },
-  ];
+  const result: EnvVar[] =
+    template !== "llamaindexserver"
+      ? [
+          {
+            name: "FILESERVER_URL_PREFIX",
+            description:
+              "FILESERVER_URL_PREFIX is the URL prefix of the server storing the images generated by the interpreter.",
+            value:
+              framework === "nextjs"
+                ? // FIXME: if we are using nextjs, port should be 3000
+                  "http://localhost:3000/api/files"
+                : `http://localhost:${sPort}/api/files`,
+          },
+        ]
+      : [];
   if (framework === "fastapi") {
     result.push(
       ...[
         {
           name: "APP_HOST",
-          description: "The address to start the backend app.",
+          description: "The address to start the FastAPI app.",
           value: "0.0.0.0",
         },
         {
           name: "APP_PORT",
-          description: "The port to start the backend app.",
+          description: "The port to start the FastAPI app.",
           value: sPort,
         },
       ],
     );
   }
-  if (framework === "nextjs") {
+  if (framework === "nextjs" && template !== "llamaindexserver") {
     result.push({
       name: "NEXT_PUBLIC_CHAT_API",
       description:
@@ -569,25 +576,41 @@ export const createBackendEnvFile = async (
     | "port"
     | "tools"
     | "observability"
+    | "useLlamaParse"
   >,
 ) => {
   // Init env values
   const envFileName = ".env";
   const envVars: EnvVar[] = [
-    {
-      name: "LLAMA_CLOUD_API_KEY",
-      description: `The Llama Cloud API key.`,
-      value: opts.llamaCloudKey,
-    },
-    // Add environment variables of each component
-    ...getModelEnvs(opts.modelConfig),
-    ...getEngineEnvs(),
-    ...getVectorDBEnvs(opts.vectorDb, opts.framework),
-    ...getFrameworkEnvs(opts.framework, opts.port),
+    ...(opts.useLlamaParse
+      ? [
+          {
+            name: "LLAMA_CLOUD_API_KEY",
+            description: `The Llama Cloud API key.`,
+            value: opts.llamaCloudKey,
+          },
+        ]
+      : []),
+    ...getVectorDBEnvs(opts.vectorDb, opts.framework, opts.template),
     ...getToolEnvs(opts.tools),
-    ...getTemplateEnvs(opts.template),
-    ...getObservabilityEnvs(opts.observability),
-    ...getSystemPromptEnv(opts.tools, opts.dataSources, opts.template),
+    ...getFrameworkEnvs(opts.framework, opts.template, opts.port),
+    // Add environment variables of each component
+    ...(opts.template === "llamaindexserver"
+      ? [
+          {
+            name: "OPENAI_API_KEY",
+            description: "The OpenAI API key to use.",
+            value: opts.modelConfig.apiKey,
+          },
+        ]
+      : [
+          // don't use this stuff for llama-indexserver
+          ...getModelEnvs(opts.modelConfig),
+          ...getEngineEnvs(),
+          ...getTemplateEnvs(opts.template),
+          ...getObservabilityEnvs(opts.observability),
+          ...getSystemPromptEnv(opts.tools, opts.dataSources, opts.template),
+        ]),
   ];
   // Render and write env file
   const content = renderEnvVar(envVars);

helpers/index.ts

@@ -1,7 +1,7 @@
 import { callPackageManager } from "./install";
 
 import path from "path";
-import { cyan } from "picocolors";
+import picocolors, { cyan } from "picocolors";
 
 import fsExtra from "fs-extra";
 import { writeLoadersConfig } from "./datasources";
@@ -41,7 +41,11 @@ const checkForGenerateScript = (
     missingSettings.push("your LLAMA_CLOUD_API_KEY");
   }
 
-  if (vectorDb !== "none" && vectorDb !== "llamacloud") {
+  if (
+    vectorDb !== undefined &&
+    vectorDb !== "none" &&
+    vectorDb !== "llamacloud"
+  ) {
     missingSettings.push("your Vector DB environment variables");
   }
 
@@ -92,7 +96,7 @@ async function generateContextData(
     }
 
     const settingsMessage = `After setting ${missingSettings.join(" and ")}, run ${runGenerate} to generate the context data.`;
-    console.log(`\n${settingsMessage}\n\n`);
+    console.log(picocolors.yellow(`\n${settingsMessage}\n\n`));
   }
 }
 
@@ -166,6 +170,17 @@ export const installTemplate = async (
 
   if (props.framework === "fastapi") {
     await installPythonTemplate(props);
+  } else {
+    await installTSTemplate(props);
+  }
+
+  // write configurations
+  if (props.template !== "llamaindexserver") {
+    await writeToolsConfig(
+      props.root,
+      props.tools,
+      props.framework === "fastapi" ? ConfigFileType.YAML : ConfigFileType.JSON,
+    );
     if (props.vectorDb !== "llamacloud") {
       // write loaders configuration (currently Python only)
       // not needed for LlamaCloud as it has its own loaders
@@ -175,26 +190,13 @@ export const installTemplate = async (
         props.useLlamaParse,
       );
     }
-  } else {
-    await installTSTemplate(props);
   }
 
-  // write tools configuration
-  await writeToolsConfig(
-    props.root,
-    props.tools,
-    props.framework === "fastapi" ? ConfigFileType.YAML : ConfigFileType.JSON,
-  );
-
   if (props.backend) {
     // This is a backend, so we need to copy the test data and create the env file.
 
     // Copy the environment file to the target directory.
-    if (
-      props.template === "streaming" ||
-      props.template === "multiagent" ||
-      props.template === "reflex"
-    ) {
+    if (props.template !== "community" && props.template !== "llamapack") {
       await createBackendEnvFile(props.root, props);
     }
 

helpers/python.ts

@@ -12,6 +12,7 @@ import {
   InstallTemplateArgs,
   ModelConfig,
   TemplateDataSource,
+  TemplateObservability,
   TemplateType,
   TemplateVectorDB,
 } from "./types";
@@ -29,6 +30,7 @@ const getAdditionalDependencies = (
   dataSources?: TemplateDataSource[],
   tools?: Tool[],
   templateType?: TemplateType,
+  observability?: TemplateObservability,
 ) => {
   const dependencies: Dependency[] = [];
 
@@ -103,7 +105,7 @@ const getAdditionalDependencies = (
     case "llamacloud":
       dependencies.push({
         name: "llama-index-indices-managed-llama-cloud",
-        version: "^0.6.3",
+        version: "0.6.3",
       });
       break;
   }
@@ -268,6 +270,21 @@ const getAdditionalDependencies = (
       break;
   }
 
+  if (observability && observability !== "none") {
+    if (observability === "traceloop") {
+      dependencies.push({
+        name: "traceloop-sdk",
+        version: "^0.15.11",
+      });
+    }
+    if (observability === "llamatrace") {
+      dependencies.push({
+        name: "llama-index-callbacks-arize-phoenix",
+        version: "^0.3.0",
+      });
+    }
+  }
+
   return dependencies;
 };
 
@@ -379,47 +396,24 @@ export const installPythonDependencies = (
   }
 };
 
-export const installPythonTemplate = async ({
-  appName,
+const installLegacyPythonTemplate = async ({
   root,
   template,
-  framework,
   vectorDb,
-  postInstallAction,
-  modelConfig,
   dataSources,
   tools,
-  useLlamaParse,
   useCase,
   observability,
 }: Pick<
   InstallTemplateArgs,
-  | "appName"
   | "root"
   | "template"
-  | "framework"
   | "vectorDb"
-  | "postInstallAction"
-  | "modelConfig"
   | "dataSources"
   | "tools"
-  | "useLlamaParse"
   | "useCase"
   | "observability"
 >) => {
-  console.log("\nInitializing Python project with template:", template, "\n");
-  let templatePath;
-  if (template === "reflex") {
-    templatePath = path.join(templatesDir, "types", "reflex");
-  } else {
-    templatePath = path.join(templatesDir, "types", "streaming", framework);
-  }
-  await copy("**", root, {
-    parents: true,
-    cwd: templatePath,
-    rename: assetRelocator,
-  });
-
   const compPath = path.join(templatesDir, "components");
   const enginePath = path.join(root, "app", "engine");
 
@@ -509,34 +503,7 @@ export const installPythonTemplate = async ({
     }
   }
 
-  console.log("Adding additional dependencies");
-
-  const addOnDependencies = getAdditionalDependencies(
-    modelConfig,
-    vectorDb,
-    dataSources,
-    tools,
-    template,
-  );
-
   if (observability && observability !== "none") {
-    if (observability === "traceloop") {
-      addOnDependencies.push({
-        name: "traceloop-sdk",
-        version: "^0.15.11",
-      });
-    }
-
-    if (observability === "llamatrace") {
-      addOnDependencies.push({
-        name: "llama-index-callbacks-arize-phoenix",
-        version: "^0.3.0",
-        constraints: {
-          python: ">=3.11,<3.13",
-        },
-      });
-    }
-
     const templateObservabilityPath = path.join(
       templatesDir,
       "components",
@@ -548,6 +515,133 @@ export const installPythonTemplate = async ({
       cwd: templateObservabilityPath,
     });
   }
+};
+
+const installLlamaIndexServerTemplate = async ({
+  root,
+  useCase,
+  useLlamaParse,
+}: Pick<InstallTemplateArgs, "root" | "useCase" | "useLlamaParse">) => {
+  if (!useCase) {
+    console.log(
+      red(
+        `There is no use case selected. Please pick a use case to use via --use-case flag.`,
+      ),
+    );
+    process.exit(1);
+  }
+
+  await copy("workflow.py", path.join(root, "app"), {
+    parents: true,
+    cwd: path.join(templatesDir, "components", "workflows", "python", useCase),
+  });
+
+  // Copy custom UI component code
+  await copy(`*`, path.join(root, "components"), {
+    parents: true,
+    cwd: path.join(templatesDir, "components", "ui", "workflows", useCase),
+  });
+
+  if (useLlamaParse) {
+    await copy("index.py", path.join(root, "app"), {
+      parents: true,
+      cwd: path.join(
+        templatesDir,
+        "components",
+        "vectordbs",
+        "llamaindexserver",
+        "llamacloud",
+        "python",
+      ),
+    });
+    // TODO: Consider moving generate.py to app folder.
+    await copy("generate.py", path.join(root), {
+      parents: true,
+      cwd: path.join(
+        templatesDir,
+        "components",
+        "vectordbs",
+        "llamaindexserver",
+        "llamacloud",
+        "python",
+      ),
+    });
+  }
+  // Copy README.md
+  await copy("README-template.md", path.join(root), {
+    parents: true,
+    cwd: path.join(templatesDir, "components", "workflows", "python", useCase),
+    rename: assetRelocator,
+  });
+};
+
+export const installPythonTemplate = async ({
+  appName,
+  root,
+  template,
+  framework,
+  vectorDb,
+  postInstallAction,
+  modelConfig,
+  dataSources,
+  tools,
+  useLlamaParse,
+  useCase,
+  observability,
+}: Pick<
+  InstallTemplateArgs,
+  | "appName"
+  | "root"
+  | "template"
+  | "framework"
+  | "vectorDb"
+  | "postInstallAction"
+  | "modelConfig"
+  | "dataSources"
+  | "tools"
+  | "useLlamaParse"
+  | "useCase"
+  | "observability"
+>) => {
+  console.log("\nInitializing Python project with template:", template, "\n");
+  let templatePath;
+  if (template === "reflex") {
+    templatePath = path.join(templatesDir, "types", "reflex");
+  } else {
+    templatePath = path.join(templatesDir, "types", template, framework);
+  }
+  await copy("**", root, {
+    parents: true,
+    cwd: templatePath,
+    rename: assetRelocator,
+  });
+
+  if (template === "llamaindexserver") {
+    await installLlamaIndexServerTemplate({
+      root,
+      useCase,
+      useLlamaParse,
+    });
+  } else {
+    await installLegacyPythonTemplate({
+      root,
+      template,
+      vectorDb,
+      dataSources,
+      tools,
+      useCase,
+      observability,
+    });
+  }
+
+  console.log("Adding additional dependencies");
+  const addOnDependencies = getAdditionalDependencies(
+    modelConfig,
+    vectorDb,
+    dataSources,
+    tools,
+    template,
+  );
 
   await addDependencies(root, addOnDependencies);
 

helpers/tools.ts

@@ -124,7 +124,7 @@ For better results, you can specify the region parameter to get results from a s
     dependencies: [
       {
         name: "e2b_code_interpreter",
-        version: "1.0.3",
+        version: "1.1.1",
       },
     ],
     supportedFrameworks: ["fastapi", "express", "nextjs"],
@@ -155,7 +155,7 @@ For better results, you can specify the region parameter to get results from a s
     dependencies: [
       {
         name: "e2b_code_interpreter",
-        version: "1.0.3",
+        version: "1.1.1",
       },
     ],
     supportedFrameworks: ["fastapi", "express", "nextjs"],

helpers/types.ts

@@ -24,7 +24,8 @@ export type TemplateType =
   | "community"
   | "llamapack"
   | "multiagent"
-  | "reflex";
+  | "reflex"
+  | "llamaindexserver";
 export type TemplateFramework = "nextjs" | "express" | "fastapi";
 export type TemplateUI = "html" | "shadcn";
 export type TemplateVectorDB =
@@ -55,7 +56,8 @@ export type TemplateUseCase =
   | "deep_research"
   | "form_filling"
   | "extractor"
-  | "contract_review";
+  | "contract_review"
+  | "agentic_rag";
 // Config for both file and folder
 export type FileSourceConfig =
   | {

helpers/typescript.ts

@@ -8,42 +8,104 @@ import { templatesDir } from "./dir";
 import { PackageManager } from "./get-pkg-manager";
 import { InstallTemplateArgs, ModelProvider, TemplateVectorDB } from "./types";
 
-/**
- * Install a LlamaIndex internal template to a given `root` directory.
- */
-export const installTSTemplate = async ({
-  appName,
+const installLlamaIndexServerTemplate = async ({
+  root,
+  useCase,
+  vectorDb,
+}: Pick<InstallTemplateArgs, "root" | "useCase" | "vectorDb">) => {
+  if (!useCase) {
+    console.log(
+      red(
+        `There is no use case selected. Please pick a use case to use via --use-case flag.`,
+      ),
+    );
+    process.exit(1);
+  }
+
+  if (!vectorDb) {
+    console.log(
+      red(
+        `There is no vector db selected. Please pick a vector db to use via --vector-db flag.`,
+      ),
+    );
+    process.exit(1);
+  }
+
+  await copy("workflow.ts", path.join(root, "src", "app"), {
+    parents: true,
+    cwd: path.join(
+      templatesDir,
+      "components",
+      "workflows",
+      "typescript",
+      useCase,
+    ),
+  });
+
+  // copy workflow UI components to output/components folder
+  await copy("*", path.join(root, "components"), {
+    parents: true,
+    cwd: path.join(templatesDir, "components", "ui", "workflows", useCase),
+  });
+
+  if (vectorDb === "llamacloud") {
+    await copy("generate.ts", path.join(root, "src"), {
+      parents: true,
+      cwd: path.join(
+        templatesDir,
+        "components",
+        "vectordbs",
+        "llamaindexserver",
+        "llamacloud",
+        "typescript",
+      ),
+    });
+
+    await copy("index.ts", path.join(root, "src", "app"), {
+      parents: true,
+      cwd: path.join(
+        templatesDir,
+        "components",
+        "vectordbs",
+        "llamaindexserver",
+        "llamacloud",
+        "typescript",
+      ),
+      rename: () => "data.ts",
+    });
+  }
+  // Copy README.md
+  await copy("README-template.md", path.join(root), {
+    parents: true,
+    cwd: path.join(
+      templatesDir,
+      "components",
+      "workflows",
+      "typescript",
+      useCase,
+    ),
+    rename: assetRelocator,
+  });
+};
+
+const installLegacyTSTemplate = async ({
   root,
-  packageManager,
-  isOnline,
   template,
+  backend,
   framework,
   ui,
   vectorDb,
-  postInstallAction,
-  backend,
   observability,
   tools,
   dataSources,
   useLlamaParse,
   useCase,
   modelConfig,
-}: InstallTemplateArgs & { backend: boolean }) => {
-  console.log(bold(`Using ${packageManager}.`));
-
-  /**
-   * Copy the template files to the target directory.
-   */
-  console.log("\nInitializing project with template:", template, "\n");
-  const templatePath = path.join(templatesDir, "types", "streaming", framework);
-  const copySource = ["**"];
-
-  await copy(copySource, root, {
-    parents: true,
-    cwd: templatePath,
-    rename: assetRelocator,
-  });
-
+  relativeEngineDestPath,
+}: InstallTemplateArgs & {
+  backend: boolean;
+  relativeEngineDestPath: string;
+}) => {
   /**
    * If next.js is used, update its configuration if necessary
    */
@@ -98,10 +160,6 @@ export const installTSTemplate = async ({
   }
 
   const compPath = path.join(templatesDir, "components");
-  const relativeEngineDestPath =
-    framework === "nextjs"
-      ? path.join("app", "api", "chat")
-      : path.join("src", "controllers");
   const enginePath = path.join(root, relativeEngineDestPath, "engine");
 
   // copy llamaindex code for TS templates
@@ -236,6 +294,75 @@ export const installTSTemplate = async ({
     await fs.rm(path.join(root, "app", "api"), { recursive: true });
     await fs.rm(path.join(root, "config"), { recursive: true, force: true });
   }
+};
+
+/**
+ * Install a LlamaIndex internal template to a given `root` directory.
+ */
+export const installTSTemplate = async ({
+  appName,
+  root,
+  packageManager,
+  isOnline,
+  template,
+  framework,
+  ui,
+  vectorDb,
+  postInstallAction,
+  backend,
+  observability,
+  tools,
+  dataSources,
+  useLlamaParse,
+  useCase,
+  modelConfig,
+}: InstallTemplateArgs & { backend: boolean }) => {
+  console.log(bold(`Using ${packageManager}.`));
+
+  /**
+   * Copy the template files to the target directory.
+   */
+  console.log("\nInitializing project with template:", template, "\n");
+  const templatePath = path.join(templatesDir, "types", template, framework);
+  const copySource = ["**"];
+
+  await copy(copySource, root, {
+    parents: true,
+    cwd: templatePath,
+    rename: assetRelocator,
+  });
+
+  const relativeEngineDestPath =
+    framework === "nextjs"
+      ? path.join("app", "api", "chat")
+      : path.join("src", "controllers");
+
+  if (template === "llamaindexserver") {
+    await installLlamaIndexServerTemplate({
+      root,
+      useCase,
+      vectorDb,
+    });
+  } else {
+    await installLegacyTSTemplate({
+      appName,
+      root,
+      packageManager,
+      isOnline,
+      template,
+      backend,
+      framework,
+      ui,
+      vectorDb,
+      observability,
+      tools,
+      dataSources,
+      useLlamaParse,
+      useCase,
+      modelConfig,
+      relativeEngineDestPath,
+    });
+  }
 
   const packageJson = await updatePackageJson({
     root,
@@ -248,6 +375,7 @@ export const installTSTemplate = async ({
     vectorDb,
     backend,
     modelConfig,
+    template,
   });
 
   if (
@@ -262,27 +390,27 @@ const providerDependencies: {
   [key in ModelProvider]?: Record<string, string>;
 } = {
   openai: {
-    "@llamaindex/openai": "^0.1.52",
+    "@llamaindex/openai": "^0.2.0",
   },
   gemini: {
-    "@llamaindex/google": "^0.0.7",
+    "@llamaindex/google": "^0.2.0",
   },
   ollama: {
-    "@llamaindex/ollama": "^0.0.40",
+    "@llamaindex/ollama": "^0.1.0",
   },
   mistral: {
-    "@llamaindex/mistral": "^0.0.5",
+    "@llamaindex/mistral": "^0.2.0",
   },
   "azure-openai": {
-    "@llamaindex/openai": "^0.1.52",
+    "@llamaindex/openai": "^0.2.0",
   },
   groq: {
-    "@llamaindex/groq": "^0.0.51",
-    "@llamaindex/huggingface": "^0.0.36", // groq uses huggingface as default embedding model
+    "@llamaindex/groq": "^0.0.61",
+    "@llamaindex/huggingface": "^0.1.0", // groq uses huggingface as default embedding model
   },
   anthropic: {
-    "@llamaindex/anthropic": "^0.1.0",
-    "@llamaindex/huggingface": "^0.0.36", // anthropic uses huggingface as default embedding model
+    "@llamaindex/anthropic": "^0.3.0",
+    "@llamaindex/huggingface": "^0.1.0", // anthropic uses huggingface as default embedding model
   },
 };
 
@@ -331,6 +459,7 @@ async function updatePackageJson({
   vectorDb,
   backend,
   modelConfig,
+  template,
 }: Pick<
   InstallTemplateArgs,
   | "root"
@@ -341,6 +470,7 @@ async function updatePackageJson({
   | "observability"
   | "vectorDb"
   | "modelConfig"
+  | "template"
 > & {
   relativeEngineDestPath: string;
   backend: boolean;
@@ -352,7 +482,7 @@ async function updatePackageJson({
   packageJson.name = appName;
   packageJson.version = "0.1.0";
 
-  if (relativeEngineDestPath) {
+  if (relativeEngineDestPath && template !== "llamaindexserver") {
     // TODO: move script to {root}/scripts for all frameworks
     // add generate script if using context engine
     packageJson.scripts = {

llama-index-server/README.md

@@ -0,0 +1,141 @@
+# LlamaIndex Server
+
+LlamaIndexServer is a FastAPI-based application that allows you to quickly launch your [LlamaIndex Workflows](https://docs.llamaindex.ai/en/stable/module_guides/workflow/#workflows) and [Agent Workflows](https://docs.llamaindex.ai/en/stable/understanding/agent/multi_agent/) as an API server with an optional chat UI. It provides a complete environment for running LlamaIndex workflows with both API endpoints and a user interface for interaction.
+
+## Features
+
+- Serving a workflow as a chatbot
+- Built on FastAPI for high performance and easy API development
+- Optional built-in chat UI with extendable UI components
+- Prebuilt development code
+
+## Installation
+
+```bash
+pip install llama-index-server
+```
+
+## Quick Start
+
+```python
+# main.py
+from llama_index.core.agent.workflow import AgentWorkflow
+from llama_index.core.workflow import Workflow
+from llama_index.core.tools import FunctionTool
+from llama_index.server import LlamaIndexServer
+
+
+# Define a factory function that returns a Workflow or AgentWorkflow
+def create_workflow() -> Workflow:
+    def fetch_weather(city: str) -> str:
+        return f"The weather in {city} is sunny"
+
+    return AgentWorkflow.from_tools(
+        tools=[
+            FunctionTool.from_defaults(
+                fn=fetch_weather,
+            )
+        ]
+    )
+
+
+# Create an API server for the workflow
+app = LlamaIndexServer(
+    workflow_factory=create_workflow,  # Supports Workflow or AgentWorkflow
+    env="dev",  # Enable development mode
+    ui_config={ # Configure the chat UI, optional
+        "app_title": "Weather Bot",
+        "starter_questions": ["What is the weather in LA?", "Will it rain in SF?"],
+    },
+    verbose=True
+)
+```
+
+## Running the Server
+
+- In the same directory as `main.py`, run the following command to start the server:
+
+  ```bash
+  fastapi dev
+  ```
+
+- Making a request to the server:
+
+  ```bash
+  curl -X POST "http://localhost:8000/api/chat" -H "Content-Type: application/json" -d '{"message": "What is the weather in Tokyo?"}'
+  ```
+
+- See the API documentation at `http://localhost:8000/docs`
+- Access the chat UI at `http://localhost:8000/` (Make sure you set the `env="dev"` or `include_ui=True` in the server configuration)
+
+## Configuration Options
+
+The LlamaIndexServer accepts the following configuration parameters:
+
+- `workflow_factory`: A callable that creates a workflow instance for each request
+- `logger`: Optional logger instance (defaults to uvicorn logger)
+- `use_default_routers`: Whether to include default routers (chat, static file serving)
+- `env`: Environment setting ('dev' enables CORS and UI by default)
+- `ui_config`: UI configuration as a dictionary or UIConfig object with options:
+  - `enabled`: Whether to enable the chat UI (default: True)
+  - `app_title`: The title of the chat application (default: "LlamaIndex Server")
+  - `starter_questions`: List of starter questions for the chat UI (default: None)
+  - `ui_path`: Path for downloaded UI static files (default: ".ui")
+  - `component_dir`: The directory for custom UI components rendering events emitted by the workflow. The default is None, which does not render custom UI components.
+  - `llamacloud_index_selector`: Whether to show the LlamaCloud index selector in the chat UI (default: False). Requires `LLAMA_CLOUD_API_KEY` to be set.
+- `verbose`: Enable verbose logging
+- `api_prefix`: API route prefix (default: "/api")
+- `server_url`: The deployment URL of the server (default is None)
+
+## Default Routers and Features
+
+### Chat Router
+
+The server includes a default chat router at `/api/chat` for handling chat interactions.
+
+### Static File Serving
+
+- The server automatically mounts the `data` and `output` folders at `{server_url}{api_prefix}/files/data` (default: `/api/files/data`) and `{server_url}{api_prefix}/files/output` (default: `/api/files/output`) respectively.
+- Your workflows can use both folders to store and access files. As a convention, the `data` folder is used for documents that are ingested and the `output` folder is used for documents that are generated by the workflow.
+- The example workflows from `create-llama` (see below) are following this pattern.
+
+### Chat UI
+
+When enabled, the server provides a chat interface at the root path (`/`) with:
+
+- Configurable starter questions
+- Real-time chat interface
+- API endpoint integration
+
+### Custom UI Components
+
+You can add custom UI components for your workflow by providing `component_dir` config and adding custom .jsx or .tsx files to the directory.
+See [Custom UI Components](docs/custom_ui_component.md) for more details.
+
+## Development Mode
+
+In development mode (`env="dev"`), the server:
+
+- Enables CORS for all origins
+- Automatically includes the chat UI
+- Provides more verbose logging
+
+## API Endpoints
+
+The server provides the following default endpoints:
+
+- `/api/chat`: Chat interaction endpoint
+- `/api/files/data/*`: Access to data directory files
+- `/api/files/output/*`: Access to output directory files
+
+## Best Practices
+
+1. Always provide a workflow factory that creates fresh workflow instances
+2. Use environment variables for sensitive configuration
+3. Enable verbose logging during development
+4. Configure CORS appropriately for your deployment environment
+5. Use starter questions to guide users in the chat UI
+
+## Getting Started with a New Project
+
+Want to start a new project with LlamaIndexServer? Check out our [create-llama](https://github.com/run-llama/create-llama) tool to quickly generate a new project with LlamaIndexServer.

llama-index-server/docs/custom_ui_component.md

@@ -0,0 +1,103 @@
+# Custom UI Components
+
+The LlamaIndex server provides support for rendering workflow events using custom UI components, allowing you to extend and customize the chat interface.
+
+## Overview
+
+Custom UI components are a powerful feature that enables you to:
+
+- Add custom interface elements to the chat UI using React JSX or TSX files
+- Extend the default chat interface functionality
+- Create specialized visualizations or interactions
+
+## Configuration
+
+### Workflow events
+
+To display custom UI components, your workflow needs to emit `UIEvent` events with data that conforms to the data model of your custom UI component.
+
+```python
+from llama_index.server import UIEvent
+from pydantic import BaseModel, Field
+from typing import Literal, Any
+
+# Define a Pydantic model for your event data
+class DeepResearchEventData(BaseModel):
+    id: str = Field(description="The unique identifier for the event")
+    type: Literal["retrieval", "analysis"] = Field(description="DeepResearch has two main stages: retrieval and analysis")
+    status: Literal["pending", "completed", "failed"] = Field(description="The current status of the event")
+    content: str = Field(description="The textual content of the event")
+
+
+# In your workflow, emit the data model with UIEvent
+ctx.write_event_to_stream(
+    UIEvent(
+        type="deep_research_event",
+        data=DeepResearchEventData(
+            id="123",
+            type="retrieval",
+            status="pending",
+            content="Retrieving data...",
+        ),
+    )
+)
+```
+
+### Server Setup
+
+1. Initialize the LlamaIndex server with a component directory:
+
+```python
+server = LlamaIndexServer(
+    workflow_factory=your_workflow,
+    ui_config={
+        "component_dir": "path/to/components",
+    },
+    include_ui=True
+)
+```
+
+2. Add the custom component code to the directory following the naming pattern:
+
+   - File Extension: `.jsx` and `.tsx` for React components
+   - File Name: Should match the event type from your workflow (e.g., `deep_research_event.jsx` for handling `deep_research_event` type that you defined in your workflow). If there are TSX and JSX files with the same name, the TSX file will be used.
+   - Component Name: Export a default React component named `Component` that receives props from the event data
+
+   Example component structure:
+
+   ```jsx
+   function Component({ events }) {
+       // Your component logic here
+       return (
+           // Your UI code here
+       );
+   }
+   ```
+
+### Generate UI Component
+
+We provide a `generate_event_component` function that uses LLMs to automatically generate UI components for your workflow events.
+
+```python
+from llama_index.server.gen_ui import generate_event_component
+from llama_index.llms.openai import OpenAI
+
+# Generate a component using the event class you defined in your workflow
+from your_workflow import DeepResearchEvent
+ui_code = await generate_event_component(
+    event_cls=DeepResearchEvent,
+    llm=OpenAI(model="gpt-4.1"), # Default LLM is Claude 3.7 Sonnet if not provided
+)
+
+# Alternatively, generate from your workflow file
+ui_code = await generate_event_component(
+    workflow_file="your_workflow.py",
+)
+print(ui_code)
+
+# Save the generated code to a file for use in your project
+with open("deep_research_event.jsx", "w") as f:
+    f.write(ui_code)
+```
+
+> **Tip:** For optimal results, add descriptive documentation to each field in your event data class. This helps the LLM better understand your data structure and generate more appropriate UI components. We also recommend using GPT 4.1, Claude 3.7 Sonnet and Gemini 2.5 Pro for better results.

llama-index-server/llama_index/server/__init__.py

@@ -0,0 +1,4 @@
+from .api.models import UIEvent
+from .server import LlamaIndexServer, UIConfig
+
+__all__ = ["LlamaIndexServer", "UIConfig", "UIEvent"]

llama-index-server/llama_index/server/api/callbacks/__init__.py

@@ -0,0 +1,13 @@
+from llama_index.server.api.callbacks.base import EventCallback
+from llama_index.server.api.callbacks.llamacloud import LlamaCloudFileDownload
+from llama_index.server.api.callbacks.source_nodes import SourceNodesFromToolCall
+from llama_index.server.api.callbacks.suggest_next_questions import (
+    SuggestNextQuestions,
+)
+
+__all__ = [
+    "EventCallback",
+    "SourceNodesFromToolCall",
+    "SuggestNextQuestions",
+    "LlamaCloudFileDownload",
+]

llama-index-server/llama_index/server/api/callbacks/base.py

@@ -0,0 +1,31 @@
+import logging
+from abc import ABC, abstractmethod
+from typing import Any
+
+logger = logging.getLogger("uvicorn")
+
+
+class EventCallback(ABC):
+    """
+    Base class for event callbacks during event streaming.
+    """
+
+    async def run(self, event: Any) -> Any:
+        """
+        Called for each event in the stream.
+        Default behavior: pass through the event unchanged.
+        """
+        return event
+
+    async def on_complete(self, final_response: str) -> Any:
+        """
+        Called when the stream is complete.
+        Default behavior: return None.
+        """
+        return None
+
+    @abstractmethod
+    def from_default(self, *args: Any, **kwargs: Any) -> "EventCallback":
+        """
+        Create a new instance of the processor from default values.
+        """

llama-index-server/llama_index/server/api/callbacks/llamacloud.py

@@ -0,0 +1,39 @@
+import logging
+from typing import Any, List
+
+from fastapi import BackgroundTasks
+from llama_index.core.schema import NodeWithScore
+from llama_index.server.api.callbacks.base import EventCallback
+from llama_index.server.services.llamacloud.file import LlamaCloudFileService
+
+logger = logging.getLogger("uvicorn")
+
+
+class LlamaCloudFileDownload(EventCallback):
+    """
+    Processor for handling LlamaCloud file downloads from source nodes.
+    """
+
+    def __init__(self, background_tasks: BackgroundTasks) -> None:
+        self.background_tasks = background_tasks
+
+    async def run(self, event: Any) -> Any:
+        if hasattr(event, "to_response"):
+            event_response = event.to_response()
+            if event_response.get("type") == "sources" and hasattr(event, "nodes"):
+                await self._process_response_nodes(event.nodes)
+        return event
+
+    async def _process_response_nodes(self, source_nodes: List[NodeWithScore]) -> None:
+        try:
+            LlamaCloudFileService.download_files_from_nodes(
+                source_nodes, self.background_tasks
+            )
+        except ImportError:
+            pass
+
+    @classmethod
+    def from_default(
+        cls, background_tasks: BackgroundTasks
+    ) -> "LlamaCloudFileDownload":
+        return cls(background_tasks=background_tasks)

llama-index-server/llama_index/server/api/callbacks/source_nodes.py

@@ -0,0 +1,32 @@
+from typing import Any
+
+from llama_index.core.agent.workflow.workflow_events import ToolCallResult
+from llama_index.server.api.callbacks.base import EventCallback
+from llama_index.server.api.models import SourceNodesEvent
+
+
+class SourceNodesFromToolCall(EventCallback):
+    """
+    Extract source nodes from the query tool output.
+
+    Args:
+        query_tool_name: The name of the tool that queries the index.
+                         default is "query_index"
+    """
+
+    def __init__(self, query_tool_name: str = "query_index"):
+        self.query_tool_name = query_tool_name
+
+    def transform_tool_call_result(self, event: ToolCallResult) -> SourceNodesEvent:
+        source_nodes = event.tool_output.raw_output.source_nodes
+        return SourceNodesEvent(nodes=source_nodes)
+
+    async def run(self, event: Any) -> Any:
+        if isinstance(event, ToolCallResult):
+            if event.tool_name == self.query_tool_name:
+                return event, self.transform_tool_call_result(event)
+        return event
+
+    @classmethod
+    def from_default(cls, *args: Any, **kwargs: Any) -> "SourceNodesFromToolCall":
+        return cls()

llama-index-server/llama_index/server/api/callbacks/stream_handler.py

@@ -0,0 +1,69 @@
+import logging
+from typing import Any, AsyncGenerator, List, Optional
+
+from llama_index.core.workflow.handler import WorkflowHandler
+from llama_index.server.api.callbacks.base import EventCallback
+
+logger = logging.getLogger("uvicorn")
+
+
+class StreamHandler:
+    """
+    Streams events from a workflow handler through a chain of callbacks.
+    """
+
+    def __init__(
+        self,
+        workflow_handler: WorkflowHandler,
+        callbacks: Optional[List[EventCallback]] = None,
+    ):
+        self.workflow_handler = workflow_handler
+        self.callbacks = callbacks or []
+        self.accumulated_text = ""
+
+    async def cancel_run(self) -> None:
+        """Cancel the workflow handler."""
+        await self.workflow_handler.cancel_run()
+
+    async def stream_events(self) -> AsyncGenerator[Any, None]:
+        """Stream events through the processor chain."""
+        try:
+            async for event in self.workflow_handler.stream_events():
+                events_to_process = [event]
+                for callback in self.callbacks:
+                    next_events: list[Any] = []
+                    for evt in events_to_process:
+                        callback_output = await callback.run(evt)
+                        if isinstance(callback_output, (list, tuple)):
+                            next_events.extend(callback_output)
+                        elif callback_output is not None:
+                            next_events.append(callback_output)
+                    events_to_process = next_events
+
+                # Yield all processed events
+                for evt in events_to_process:
+                    yield evt
+
+            # After all events are processed, call on_complete for each callback
+            for callback in self.callbacks:
+                result = await callback.on_complete(self.accumulated_text)
+                if result:
+                    yield result
+
+        except Exception:
+            # Make sure to cancel the workflow on error
+            await self.workflow_handler.cancel_run()
+            raise
+
+    def accumulate_text(self, text: str) -> None:
+        """Accumulate text from the workflow handler."""
+        self.accumulated_text += text
+
+    @classmethod
+    def from_default(
+        cls,
+        handler: WorkflowHandler,
+        callbacks: Optional[List[EventCallback]] = None,
+    ) -> "StreamHandler":
+        """Create a new instance with the given workflow handler and callbacks."""
+        return cls(workflow_handler=handler, callbacks=callbacks)

llama-index-server/llama_index/server/api/callbacks/suggest_next_questions.py

@@ -0,0 +1,45 @@
+import logging
+from typing import Any, Optional
+
+from llama_index.server.api.callbacks.base import EventCallback
+from llama_index.server.api.models import ChatRequest
+from llama_index.server.services.suggest_next_question import (
+    SuggestNextQuestionsService,
+)
+
+logger = logging.getLogger("uvicorn")
+
+
+class SuggestNextQuestions(EventCallback):
+    """Processor for generating next question suggestions."""
+
+    def __init__(
+        self, chat_request: ChatRequest, logger: Optional[logging.Logger] = None
+    ):
+        self.chat_request = chat_request
+        self.accumulated_text = ""
+        if logger:
+            self.logger = logger
+        else:
+            self.logger = logging.getLogger("uvicorn")
+
+    async def on_complete(self, final_response: str) -> Any:
+        if final_response == "":
+            self.logger.warning(
+                "SuggestNextQuestions is enabled but final response is empty, make sure your content generator accumulates text"
+            )
+            return None
+
+        questions = await SuggestNextQuestionsService.run(
+            self.chat_request.messages, final_response
+        )
+        if questions:
+            return {
+                "type": "suggested_questions",
+                "data": questions,
+            }
+        return None
+
+    @classmethod
+    def from_default(cls, chat_request: ChatRequest) -> "SuggestNextQuestions":
+        return cls(chat_request=chat_request)

llama-index-server/llama_index/server/api/models.py

@@ -0,0 +1,153 @@
+import logging
+import os
+from enum import Enum
+from typing import Any, Dict, List, Optional
+
+from llama_index.core.schema import NodeWithScore
+from llama_index.core.types import ChatMessage, MessageRole
+from llama_index.core.workflow import Event
+from llama_index.server.settings import server_settings
+from pydantic import BaseModel, Field, field_validator
+
+logger = logging.getLogger("uvicorn")
+
+
+class ChatConfig(BaseModel):
+    next_question_suggestions: bool = Field(
+        default=True,
+        description="Whether to suggest next questions",
+    )
+
+
+class ChatAPIMessage(BaseModel):
+    role: MessageRole
+    content: str
+
+    def to_llamaindex_message(self) -> ChatMessage:
+        return ChatMessage(role=self.role, content=self.content)
+
+
+class ChatRequest(BaseModel):
+    messages: List[ChatAPIMessage]
+    data: Optional[Any] = None
+    config: Optional[ChatConfig] = ChatConfig()
+
+    @field_validator("messages")
+    def validate_messages(cls, v: List[ChatAPIMessage]) -> List[ChatAPIMessage]:
+        if v[-1].role != MessageRole.USER:
+            raise ValueError("Last message must be from user")
+        return v
+
+
+class AgentRunEventType(Enum):
+    TEXT = "text"
+    PROGRESS = "progress"
+
+
+class AgentRunEvent(Event):
+    name: str
+    msg: str
+    event_type: AgentRunEventType = AgentRunEventType.TEXT
+    data: Optional[dict] = None
+
+    def to_response(self) -> dict:
+        return {
+            "type": "agent",
+            "data": {
+                "agent": self.name,
+                "type": self.event_type.value,
+                "text": self.msg,
+                "data": self.data,
+            },
+        }
+
+
+class SourceNodesEvent(Event):
+    nodes: List[NodeWithScore]
+
+    def to_response(self) -> dict:
+        return {
+            "type": "sources",
+            "data": {
+                "nodes": [
+                    SourceNodes.from_source_node(node).model_dump()
+                    for node in self.nodes
+                ]
+            },
+        }
+
+
+class SourceNodes(BaseModel):
+    id: str
+    metadata: Dict[str, Any]
+    score: Optional[float]
+    text: str
+    url: Optional[str]
+
+    @classmethod
+    def from_source_node(cls, source_node: NodeWithScore) -> "SourceNodes":
+        metadata = source_node.node.metadata
+        url = cls.get_url_from_metadata(metadata)
+
+        return cls(
+            id=source_node.node.node_id,
+            metadata=metadata,
+            score=source_node.score,
+            text=source_node.node.text,  # type: ignore
+            url=url,
+        )
+
+    @classmethod
+    def get_url_from_metadata(
+        cls,
+        metadata: Dict[str, Any],
+        data_dir: Optional[str] = None,
+    ) -> Optional[str]:
+        url_prefix = server_settings.file_server_url_prefix
+        if data_dir is None:
+            data_dir = "data"
+        file_name = metadata.get("file_name")
+
+        if file_name and url_prefix:
+            # file_name exists and file server is configured
+            pipeline_id = metadata.get("pipeline_id")
+            if pipeline_id:
+                # file is from LlamaCloud
+                file_name = f"{pipeline_id}${file_name}"
+                return f"{url_prefix}/output/llamacloud/{file_name}"
+            is_private = metadata.get("private", "false") == "true"
+            if is_private:
+                # file is a private upload
+                return f"{url_prefix}/output/uploaded/{file_name}"
+            # file is from calling the 'generate' script
+            # Get the relative path of file_path to data_dir
+            file_path = metadata.get("file_path")
+            data_dir = os.path.abspath(data_dir)
+            if file_path and data_dir:
+                relative_path = os.path.relpath(file_path, data_dir)
+                return f"{url_prefix}/data/{relative_path}"
+        # fallback to URL in metadata (e.g. for websites)
+        return metadata.get("URL")
+
+    @classmethod
+    def from_source_nodes(
+        cls, source_nodes: List[NodeWithScore]
+    ) -> List["SourceNodes"]:
+        return [cls.from_source_node(node) for node in source_nodes]
+
+
+class ComponentDefinition(BaseModel):
+    type: str
+    code: str
+    filename: str
+
+
+class UIEvent(Event):
+    type: str
+    data: BaseModel
+
+    def to_response(self) -> dict:
+        return {
+            "type": self.type,
+            "data": self.data.model_dump(),
+        }

llama-index-server/llama_index/server/api/routers/__init__.py

@@ -0,0 +1,4 @@
+from llama_index.server.api.routers.chat import chat_router
+from llama_index.server.api.routers.ui import custom_components_router
+
+__all__ = ["chat_router", "custom_components_router"]

llama-index-server/llama_index/server/api/routers/chat.py

@@ -0,0 +1,144 @@
+import asyncio
+import inspect
+import logging
+import os
+from typing import AsyncGenerator, Callable, Union
+
+from fastapi import APIRouter, BackgroundTasks, HTTPException
+from fastapi.responses import StreamingResponse
+
+from llama_index.core.agent.workflow.workflow_events import AgentStream
+from llama_index.core.workflow import StopEvent, Workflow
+from llama_index.server.api.callbacks import (
+    SourceNodesFromToolCall,
+    SuggestNextQuestions,
+)
+from llama_index.server.api.callbacks.base import EventCallback
+from llama_index.server.api.callbacks.llamacloud import LlamaCloudFileDownload
+from llama_index.server.api.callbacks.stream_handler import StreamHandler
+from llama_index.server.api.models import ChatRequest
+from llama_index.server.api.utils.vercel_stream import VercelStreamResponse
+from llama_index.server.services.llamacloud import LlamaCloudFileService
+
+
+def chat_router(
+    workflow_factory: Callable[..., Workflow],
+    logger: logging.Logger,
+) -> APIRouter:
+    router = APIRouter(prefix="/chat")
+
+    @router.post("")
+    async def chat(
+        request: ChatRequest,
+        background_tasks: BackgroundTasks,
+    ) -> StreamingResponse:
+        try:
+            user_message = request.messages[-1].to_llamaindex_message()
+            chat_history = [
+                message.to_llamaindex_message() for message in request.messages[:-1]
+            ]
+            # detect if the workflow factory has chat_request as a parameter
+            factory_sig = inspect.signature(workflow_factory)
+            if "chat_request" in factory_sig.parameters:
+                workflow = workflow_factory(chat_request=request)
+            else:
+                workflow = workflow_factory()
+            workflow_handler = workflow.run(
+                user_msg=user_message.content,
+                chat_history=chat_history,
+            )
+
+            callbacks: list[EventCallback] = [
+                SourceNodesFromToolCall(),
+                LlamaCloudFileDownload(background_tasks),
+            ]
+            if request.config and request.config.next_question_suggestions:
+                callbacks.append(SuggestNextQuestions(request))
+            stream_handler = StreamHandler(
+                workflow_handler=workflow_handler,
+                callbacks=callbacks,
+            )
+
+            return VercelStreamResponse(
+                content_generator=_stream_content(stream_handler, request, logger),
+            )
+        except Exception as e:
+            logger.error(e)
+            raise HTTPException(status_code=500, detail=str(e))
+
+    if LlamaCloudFileService.is_configured():
+
+        @router.get("/config/llamacloud")
+        async def chat_llama_cloud_config() -> dict:
+            if not os.getenv("LLAMA_CLOUD_API_KEY"):
+                raise HTTPException(
+                    status_code=500, detail="LlamaCloud API KEY is not configured"
+                )
+            projects = LlamaCloudFileService.get_all_projects_with_pipelines()
+            pipeline = os.getenv("LLAMA_CLOUD_INDEX_NAME")
+            project = os.getenv("LLAMA_CLOUD_PROJECT_NAME")
+            pipeline_config = None
+            if pipeline and project:
+                pipeline_config = {
+                    "pipeline": pipeline,
+                    "project": project,
+                }
+            return {
+                "projects": projects,
+                "pipeline": pipeline_config,
+            }
+
+    return router
+
+
+async def _stream_content(
+    handler: StreamHandler,
+    request: ChatRequest,
+    logger: logging.Logger,
+) -> AsyncGenerator[str, None]:
+    async def _text_stream(
+        event: Union[AgentStream, StopEvent],
+    ) -> AsyncGenerator[str, None]:
+        if isinstance(event, AgentStream):
+            # Normally, if the stream is a tool call, the delta is always empty
+            # so it's not a text stream.
+            if len(event.tool_calls) == 0:
+                yield event.delta
+        elif isinstance(event, StopEvent):
+            if isinstance(event.result, str):
+                yield event.result
+            elif isinstance(event.result, AsyncGenerator):
+                async for chunk in event.result:
+                    if isinstance(chunk, str):
+                        yield chunk
+                    elif hasattr(chunk, "delta") and chunk.delta:
+                        yield chunk.delta
+
+    stream_started = False
+    try:
+        async for event in handler.stream_events():
+            if not stream_started:
+                # Start the stream with an empty message
+                stream_started = True
+                yield VercelStreamResponse.convert_text("")
+
+            # Handle different types of events
+            if isinstance(event, (AgentStream, StopEvent)):
+                async for chunk in _text_stream(event):
+                    handler.accumulate_text(chunk)
+                    yield VercelStreamResponse.convert_text(chunk)
+            elif isinstance(event, dict):
+                yield VercelStreamResponse.convert_data(event)
+            elif hasattr(event, "to_response"):
+                event_response = event.to_response()
+                yield VercelStreamResponse.convert_data(event_response)
+            else:
+                yield VercelStreamResponse.convert_data(event.model_dump())
+
+    except asyncio.CancelledError:
+        logger.warning("Client cancelled the request!")
+        await handler.cancel_run()
+    except Exception as e:
+        logger.error(f"Error in stream response: {e}")
+        yield VercelStreamResponse.convert_error(str(e))
+        await handler.cancel_run()

llama-index-server/llama_index/server/api/routers/ui.py

@@ -0,0 +1,20 @@
+import logging
+from typing import List
+
+from fastapi import APIRouter
+from llama_index.server.api.models import ComponentDefinition
+from llama_index.server.services.custom_ui import CustomUI
+
+
+def custom_components_router(
+    component_dir: str,
+    logger: logging.Logger,
+) -> APIRouter:
+    router = APIRouter(prefix="/components")
+
+    @router.get("")
+    async def components() -> List[ComponentDefinition]:
+        custom_ui = CustomUI(component_dir=component_dir, logger=logger)
+        return custom_ui.get_components()
+
+    return router

llama-index-server/llama_index/server/api/utils/vercel_stream.py

@@ -0,0 +1,44 @@
+import json
+import logging
+from typing import Any, AsyncGenerator, Union
+
+from fastapi.responses import StreamingResponse
+
+logger = logging.getLogger("uvicorn")
+
+
+class VercelStreamResponse(StreamingResponse):
+    """
+    Converts preprocessed events into Vercel-compatible streaming response format.
+    """
+
+    TEXT_PREFIX = "0:"
+    DATA_PREFIX = "8:"
+    ERROR_PREFIX = "3:"
+
+    def __init__(
+        self,
+        content_generator: AsyncGenerator[str, None],
+        *args: Any,
+        **kwargs: Any,
+    ):
+        super().__init__(content_generator, *args, **kwargs)
+
+    @classmethod
+    def convert_text(cls, token: str) -> str:
+        """Convert text event to Vercel format."""
+        # Escape newlines and double quotes to avoid breaking the stream
+        token = json.dumps(token)
+        return f"{cls.TEXT_PREFIX}{token}\n"
+
+    @classmethod
+    def convert_data(cls, data: Union[dict, str]) -> str:
+        """Convert data event to Vercel format."""
+        data_str = json.dumps(data) if isinstance(data, dict) else data
+        return f"{cls.DATA_PREFIX}[{data_str}]\n"
+
+    @classmethod
+    def convert_error(cls, error: str) -> str:
+        """Convert error event to Vercel format."""
+        error_str = json.dumps(error)
+        return f"{cls.ERROR_PREFIX}{error_str}\n"

llama-index-server/llama_index/server/chat_ui.py

@@ -0,0 +1,55 @@
+import logging
+import shutil
+from pathlib import Path
+from typing import Optional
+
+import requests
+
+CHAT_UI_VERSION = "0.1.2"
+
+
+def download_chat_ui(
+    logger: Optional[logging.Logger] = None, target_path: str = ".ui"
+) -> None:
+    if logger is None:
+        logger = logging.getLogger("uvicorn")
+    path = Path(target_path)
+    temp_dir = _download_package(_get_download_link(CHAT_UI_VERSION))
+    _copy_ui_files(temp_dir, path)
+    logger.info("Chat UI downloaded and copied to static folder")
+
+
+def _get_download_link(version: str) -> str:
+    """Get the download link for the chat UI from the npm registry."""
+    return f"https://registry.npmjs.org/@llamaindex/server/-/server-{version}.tgz"
+
+
+def _download_package(url: str) -> Path:
+    """Download tar.gz file and extract all files into a temporary directory."""
+    import io
+    import tarfile
+    import tempfile
+
+    response = requests.get(url, headers={"User-Agent": "Mozilla/5.0"})
+    content = response.content
+
+    temp_dir = Path(tempfile.mkdtemp())
+
+    with tarfile.open(fileobj=io.BytesIO(content), mode="r:gz") as tar:
+        tar.extractall(path=temp_dir)
+
+    return temp_dir
+
+
+def _copy_ui_files(temp_dir: Path, target_path: Path) -> None:
+    """Copy files from the .next directory to the static directory."""
+    target_path.mkdir(parents=True, exist_ok=True)
+    next_dir = temp_dir / "package/dist/static"
+
+    if next_dir.exists():
+        for item in next_dir.iterdir():
+            dest = target_path / item.name
+            if item.is_dir():
+                shutil.copytree(item, dest, dirs_exist_ok=True)
+            else:
+                shutil.copy2(item, dest)

llama-index-server/llama_index/server/gen_ui/__init__.py

@@ -0,0 +1,4 @@
+from .main import generate_event_component
+from .parse_workflow_code import get_workflow_event_schemas
+
+__all__ = ["generate_event_component", "get_workflow_event_schemas"]

llama-index-server/llama_index/server/gen_ui/main.py

@@ -0,0 +1,424 @@
+import re
+from typing import Any, Dict, List, Optional, Type
+
+from llama_index.core.llms import LLM
+from llama_index.core.prompts import PromptTemplate
+from llama_index.core.workflow import (
+    Context,
+    Event,
+    StartEvent,
+    StopEvent,
+    Workflow,
+    step,
+)
+from llama_index.server.gen_ui.parse_workflow_code import get_workflow_event_schemas
+from pydantic import BaseModel
+from rich.console import Console
+from rich.live import Live
+from rich.panel import Panel
+
+
+class PlanningEvent(Event):
+    """
+    Event for planning the UI.
+    """
+
+    events: List[Dict[str, Any]]
+
+
+class WriteAggregationEvent(Event):
+    """
+    Event for aggregating events.
+    """
+
+    events: List[Dict[str, Any]]
+    ui_description: str
+
+
+class WriteUIComponentEvent(Event):
+    """
+    Event for writing UI component.
+    """
+
+    events: List[Dict[str, Any]]
+    aggregation_function: Optional[str]
+    ui_description: str
+
+
+class RefineGeneratedCodeEvent(Event):
+    """
+    Refine the generated code.
+    """
+
+    generated_code: str
+    aggregation_function_context: Optional[str]
+    events: List[Dict[str, Any]]
+
+
+class ExtractEventSchemaEvent(Event):
+    """
+    Extract the event schema from the event.
+    """
+
+    events: List[Any]
+
+
+class AggregatePrediction(BaseModel):
+    """
+    Prediction for aggregating events or not.
+    If need_aggregation is True, the aggregation_function will be provided.
+    """
+
+    need_aggregation: bool
+    aggregation_function: Optional[str]
+
+
+class GenUIWorkflow(Workflow):
+    """
+    Generate UI component for event from workflow.
+    """
+
+    code_structure: str = """
+        ```jsx
+            // Note: Only shadcn/ui and lucide-react and tailwind css are allowed.
+            // shadcn import pattern: import { ComponentName } from "@/components/ui/<component_path>";
+            // e.g: import { Card, CardHeader, CardTitle, CardContent } from "@/components/ui/card";
+            //      import { Button } from "@/components/ui/button";
+            //      import cn from "@/lib/utils"; // clsx is not supported
+
+            // export the component
+            export default function Component({ events }) {
+                // logic for aggregating events (if needed)
+                const aggregateEvents = () => {
+                    // code for aggregating events here
+                }
+            
+                // State handling
+                // e.g: const [state, setState] = useState({});
+
+                return (
+                    // UI code here
+                )
+            }
+        ```
+    """
+
+    def __init__(self, llm: LLM, **kwargs: Any):
+        super().__init__(**kwargs)
+        self.llm = llm
+        self.console = Console()
+        self._live: Optional[Live] = None
+        self._completed_steps: List[str] = []
+        self._current_step: Optional[str] = None
+
+    def update_status(self, message: str, completed: bool = False) -> None:
+        """Show completed and current steps in a panel."""
+        if completed:
+            if self._current_step:
+                self._completed_steps.append(self._current_step)
+            self._current_step = None
+        else:
+            self._current_step = message
+
+        if self._live is None:
+            self._live = Live("", console=self.console, refresh_per_second=4)
+            self._live.start()
+
+        # Build status display
+        status_lines = []
+        for completed_step in self._completed_steps:
+            status_lines.append(f"[green]✓[/green] {completed_step}")
+        if self._current_step:
+            status_lines.append(f"[yellow]⋯[/yellow] {self._current_step}")
+
+        self._live.update(Panel("\n".join(status_lines)))
+
+    @step
+    async def start(self, ctx: Context, ev: StartEvent) -> PlanningEvent:
+        events = ev.events
+        if not events:
+            raise ValueError(
+                "events is required, provide list of filtered events to generate UI components for"
+            )
+        await ctx.set("events", events)
+        self.update_status("Planning the UI")
+        return PlanningEvent(events=events)
+
+    @step
+    async def planning(self, ctx: Context, ev: PlanningEvent) -> WriteAggregationEvent:
+        prompt_template = """
+            # Your role
+            You are a designer who is designing a UI for given events that are emitted from a backend workflow.
+            Here are the events that you need to work on: {events}
+
+            # Task
+            Your task is to analyze the event schema and data and provide a description that how the UI would look like.
+            The UI should be beautiful, no monotonous, and visually pleasing.
+            Focus on the elements and the layout, don't ask too much on the styles (transition, dark mode, responsive, etc...).
+
+            e.g: Assume that the backend produce list of events with animal name, action, and status.
+            ```
+            A card-based layout displaying animal actions:
+            - Each card shows an animal's image at the top
+            - Below the image: animal name as the card title
+            - Action details in the card body with an icon (eating 🍖, sleeping 😴, playing 🎾)
+            - Status badge in the corner showing if action is ongoing/completed
+            - Expandable section for additional details
+            - Soft color scheme based on action type
+            ```
+            Don't be verbose, just return the description for the UI based on the event schema and data.
+        """
+        response = await self.llm.acomplete(
+            PromptTemplate(prompt_template).format(events=ev.events),
+            formatted=True,
+        )
+        await ctx.set("ui_description", response.text)
+        self.update_status("Planning the UI", completed=True)
+        # Update the planning description to the console
+        self.console.print(
+            Panel(
+                response.text,
+                title="UI Description",
+                border_style="cyan",
+            )
+        )
+        self.update_status("Generating aggregation function")
+        return WriteAggregationEvent(
+            events=ev.events,
+            ui_description=response.text,
+        )
+
+    @step
+    async def generate_event_aggregations(
+        self, ctx: Context, ev: WriteAggregationEvent
+    ) -> WriteUIComponentEvent:
+        prompt_template = """
+            # Your role
+            You are a frontend developer who is developing a React component for given events that are emitted from a backend workflow.
+            Here are the events that you need to work on: {events}
+            Here is the description of the UI:
+            ```
+                {ui_description}
+            ```
+
+            # Task
+            Based on the description of the UI and the list of events, write the aggregation function that will be used to aggregate the events.
+            Take into account that the list of events grows with time. At the beginning, there is only one event in the list, and events are incrementally added. 
+            To render the events in a visually pleasing way, try to aggregate them by their attributes and render the aggregates instead of just rendering a list of all events.
+            Don't add computation to the aggregation function, just group the events by their attributes.
+            Make sure that the aggregation should reflect the description of the UI and the grouped events are not duplicated, make it as simple as possible to avoid unnecessary issues.
+
+            # Answer with the following format:
+            ```jsx
+            const aggregateEvents = () => {
+                // code for aggregating events here if needed otherwise let the jsx code block empty
+            }
+            ```
+            """
+
+        response = await self.llm.acomplete(
+            PromptTemplate(prompt_template).format(
+                events=ev.events,
+                ui_description=ev.ui_description,
+            ),
+            formatted=True,
+        )
+        await ctx.set("aggregation_context", response.text)
+
+        self.update_status("Generating aggregation function", completed=True)
+        self.update_status("Generating UI components")
+        return WriteUIComponentEvent(
+            events=ev.events,
+            aggregation_function=response.text,
+            ui_description=ev.ui_description,
+        )
+
+    @step
+    async def write_ui_component(
+        self, ctx: Context, ev: WriteUIComponentEvent
+    ) -> RefineGeneratedCodeEvent:
+        prompt_template = """
+            # Your role
+            You are a frontend developer who is developing a React component using shadcn/ui (@/components/ui/<component_name>) and lucide-react for the UI.
+            You are given a list of events and other context.
+            Your task is to write a beautiful UI for the events that will be included in a chat UI.
+
+            # Context:
+            Here are the events that you need to work on: {events}
+            {aggregation_function_context}
+            Here is the description of the UI:
+            ```
+                {ui_description}
+            ```
+
+            # Requirements:
+            - Write beautiful UI components for the events using shadcn/ui and lucide-react.
+            - The component text/label should be specified for each event type.
+
+            # Instructions:
+            ## Event and schema notice
+            - Based on the provided list of events, determine their types and attributes.
+            - It's normal that the schema is applied to all events, but the events might completely different which some of schema attributes aren't used.
+            - You should make the component visually distinct for each event type.
+              e.g: A simple cat schema
+                ```{"type": "cat", "action": ["jump", "run", "meow"], "jump": {"height": 10, "distance": 20}, "run": {"distance": 100}}```
+                You should display the jump, run and meow actions in different ways. don't try to render "height" for the "run" and "meow" action.
+
+            ## UI notice
+            - Use shadcn/ui and lucide-react and tailwind CSS for the UI.
+            - Be careful on state handling, make sure the update should be updated in the state and there is no duplicate state.
+            - For a long content, consider to use markdown along with dropdown to show the full content.
+                e.g:
+                ```jsx
+                import { Markdown } from "@llamaindex/chat-ui/widgets";
+                <Markdown content={content} />
+                ```
+            - Try to make the component placement not monotonous, consider use row/column/flex/grid layout.
+            """
+
+        aggregation_function_context = (
+            f"\nBefore rendering the events, we're using the following aggregation function: {ev.aggregation_function}"
+            if ev.aggregation_function
+            else ""
+        )
+
+        prompt = PromptTemplate(prompt_template).format(
+            events=ev.events,
+            aggregation_function_context=aggregation_function_context,
+            code_structure=self.code_structure,
+            ui_description=ev.ui_description,
+        )
+        response = await self.llm.acomplete(prompt, formatted=True)
+
+        self.update_status("Generating UI components", completed=True)
+        self.update_status("Refining generated code")
+        return RefineGeneratedCodeEvent(
+            generated_code=response.text,
+            events=ev.events,
+            aggregation_function_context=aggregation_function_context,
+        )
+
+    @step
+    async def refine_code(
+        self, ctx: Context, ev: RefineGeneratedCodeEvent
+    ) -> StopEvent:
+        prompt_template = """
+            # Your role
+            You are a frontend developer who is developing a React component for given events that are emitted from a backend workflow.
+            Your task is to assemble the pieces of code into a complete code segment that follows the specified code structure.
+
+            # Context:
+            ## Here is the generated code:
+            {generated_code}
+
+            {aggregation_function_context}
+
+            ## The generated code should follow the following structure:
+            {code_structure}
+
+            # Requirements:
+            - Refine the code if needed to ensure there are no potential bugs. 
+            - Be careful on code placement, make sure it doesn't call any undefined code.
+            - Make sure the import statements are correct. 
+              e.g: import { Button, Card, Accordion } from "@/components/ui" is correct because Button, Card are defined in different shadcn/ui components.
+              -> correction: import { Button } from "@/components/ui/button";
+                             import { Card } from "@/components/ui/card";
+            - Don't be verbose, only return the code, wrap it in ```jsx <code>```
+        """
+        prompt = PromptTemplate(prompt_template).format(
+            generated_code=ev.generated_code,
+            code_structure=self.code_structure,
+            aggregation_function_context=ev.aggregation_function_context,
+        )
+
+        response = await self.llm.acomplete(prompt, formatted=True)
+
+        # Extract code from response, handling case where code block is missing
+        code_match = re.search(r"```jsx(.*)```", response.text, re.DOTALL)
+        if code_match is None:
+            # If no code block found, use full response
+            code = response.text
+        else:
+            code = code_match.group(1).strip()
+
+        self.update_status("Refining generated code", completed=True)
+        if self._live is not None:
+            self._live.stop()
+            self._live = None
+
+        return StopEvent(
+            result=code,
+        )
+
+
+async def generate_event_component(
+    workflow_file: Optional[str] = None,
+    event_cls: Optional[Type[BaseModel]] = None,
+    llm: Optional[LLM] = None,
+) -> str:
+    """
+    Generate UI component for events from workflow.
+    Either workflow_file or event_cls must be provided.
+
+    Args:
+        workflow_file: The path to the workflow file that contains the event to generate UI for. e.g: `app/workflow.py`.
+        event_cls: A Pydantic class to generate UI for. e.g: `DeepResearchEvent`.
+        llm: The LLM to use for the generation. Default is Anthropic's Claude 3.7 Sonnet.
+             We recommend using these LLMs:
+                - Anthropic's Claude 3.7 Sonnet
+                - OpenAI's GPT-4.1
+                - Google Gemini 2.5 Pro
+    Returns:
+        The generated UI component code.
+    """
+    if workflow_file is None and event_cls is None:
+        raise ValueError(
+            "Either workflow_file or event_cls must be provided. Please provide one of them."
+        )
+    if workflow_file is not None and event_cls is not None:
+        raise ValueError(
+            "Only one of workflow_file or event_cls can be provided. Please provide only one of them."
+        )
+    if llm is None:
+        from llama_index.llms.anthropic import Anthropic
+
+        llm = Anthropic(model="claude-3-7-sonnet-latest", max_tokens=8192)
+
+    console = Console()
+
+    # Get event schemas
+    if workflow_file is not None:
+        # Get event schemas from the input file
+        console.rule("[bold blue]Analyzing Events[/bold blue]")
+        event_schemas = get_workflow_event_schemas(workflow_file)
+        if len(event_schemas) == 0:
+            console.print(
+                Panel(
+                    "[red]No events found that are used with write_event_to_stream[/red]",
+                    title="❌ Error",
+                    border_style="red",
+                )
+            )
+            raise RuntimeError(
+                "No events found that are used with write_event_to_stream. Please check the workflow file."
+            )
+    elif event_cls is not None:
+        event_schemas = [
+            {"type": event_cls.__name__, "schema": event_cls.model_json_schema()}
+        ]
+
+    # Generate UI component from event schemas
+    console.rule("[bold blue]Generate UI Components[/bold blue]")
+
+    workflow = GenUIWorkflow(llm=llm, timeout=500.0)
+    code = await workflow.run(events=event_schemas)
+
+    console.print(
+        Panel(
+            "[green]UI component has been generated successfully![/green]\n",
+            title="✨ Complete",
+            border_style="green",
+        )
+    )
+
+    return code

llama-index-server/llama_index/server/gen_ui/parse_workflow_code.py

@@ -0,0 +1,93 @@
+import ast
+import importlib
+import inspect
+import os
+import sys
+from typing import Any, Dict, List
+
+
+class EventAnalyzer(ast.NodeVisitor):
+    """
+    Parse the workflow code to find UIEvent instances passed to write_event_to_stream.
+    """
+
+    def __init__(self) -> None:
+        self.found_ui_event = False
+
+    def visit_Call(self, node: ast.Call) -> None:
+        # Check for ctx.write_event_to_stream call with UIEvent arg
+        if (
+            isinstance(node.func, ast.Attribute)
+            and isinstance(node.func.value, ast.Name)
+            and node.func.attr == "write_event_to_stream"
+            and node.args
+            and isinstance(node.args[0], ast.Call)
+            and isinstance(node.args[0].func, ast.Name)
+            and node.args[0].func.id == "UIEvent"
+        ):
+            self.found_ui_event = True
+
+        self.generic_visit(node)
+
+
+def get_workflow_event_schemas(file_path: str) -> List[Dict[str, Any]]:
+    """
+    Find UIEvent instances passed to write_event_to_stream and return their data type schema.
+    """
+    # Get absolute path for module importing
+    abs_file_path = os.path.abspath(file_path)
+    project_root = os.path.dirname(os.path.dirname(abs_file_path))
+
+    # Convert file path to module name
+    rel_path = os.path.relpath(abs_file_path, project_root)
+    module_name = rel_path.replace(os.sep, ".").replace(".py", "")
+
+    # Temporarily modify sys.path to allow imports
+    original_path = list(sys.path)
+    if project_root not in sys.path:
+        sys.path.insert(0, project_root)
+
+    try:
+        # Import the module
+        module = importlib.import_module(module_name)
+        importlib.reload(module)
+    except ImportError as e:
+        print(f"Error importing module {module_name}: {e}")
+        sys.path = original_path
+        return []
+    finally:
+        # Restore original path
+        if project_root in sys.path and project_root not in original_path:
+            sys.path.remove(project_root)
+
+    # Parse the file to check for UIEvent usage
+    try:
+        with open(file_path, "r") as f:
+            tree = ast.parse(f.read())
+    except (FileNotFoundError, SyntaxError) as e:
+        print(f"Error parsing {file_path}: {e}")
+        return []
+
+    # Check if UIEvent is passed to write_event_to_stream
+    analyzer = EventAnalyzer()
+    analyzer.visit(tree)
+
+    schema_list = []
+
+    # Only proceed if UIEvent was found and the module has the class
+    if analyzer.found_ui_event and hasattr(module, "UIEvent"):
+        # Look for class names containing "EventData" in the module
+        for name, obj in inspect.getmembers(module):
+            if (
+                inspect.isclass(obj)
+                and name.endswith("EventData")
+                and hasattr(obj, "model_json_schema")
+            ):
+                try:
+                    schema = obj.model_json_schema()
+                    if schema:
+                        schema_list.append(schema)
+                except Exception:
+                    pass
+
+    return schema_list

llama-index-server/llama_index/server/server.py

@@ -0,0 +1,230 @@
+import json
+import logging
+import os
+from typing import Any, Callable, Optional, Union
+
+from fastapi import FastAPI
+from fastapi.middleware.cors import CORSMiddleware
+from fastapi.staticfiles import StaticFiles
+from llama_index.core.workflow import Workflow
+from llama_index.server.api.routers import chat_router, custom_components_router
+from llama_index.server.chat_ui import download_chat_ui
+from llama_index.server.settings import server_settings
+from pydantic import BaseModel, Field
+
+
+class UIConfig(BaseModel):
+    enabled: bool = Field(default=True, description="Whether to enable the chat UI")
+    app_title: str = Field(
+        default="LlamaIndex Server", description="The title of the chat UI"
+    )
+    starter_questions: Optional[list[str]] = Field(
+        default=None, description="The starter questions for the chat UI"
+    )
+    llamacloud_index_selector: bool = Field(
+        default=False,
+        description="Whether to show the LlamaCloud index selector in the chat UI (need to set the LLAMA_CLOUD_API_KEY environment variable)",
+    )
+    ui_path: str = Field(
+        default=".ui", description="The path that stores static files for the chat UI"
+    )
+    component_dir: Optional[str] = Field(
+        default=None, description="The directory to custom UI components code"
+    )
+
+    def get_config_content(self) -> str:
+        return json.dumps(
+            {
+                "CHAT_API": f"{server_settings.api_url}/chat",
+                "STARTER_QUESTIONS": self.starter_questions or [],
+                "LLAMA_CLOUD_API": f"{server_settings.api_url}/chat/config/llamacloud"
+                if self.llamacloud_index_selector and os.getenv("LLAMA_CLOUD_API_KEY")
+                else None,
+                "APP_TITLE": self.app_title,
+                "COMPONENTS_API": f"{server_settings.api_url}/components"
+                if self.component_dir
+                else None,
+            },
+            indent=2,
+        )
+
+
+class LlamaIndexServer(FastAPI):
+    workflow_factory: Callable[..., Workflow]
+    verbose: bool = False
+    ui_config: UIConfig
+
+    def __init__(
+        self,
+        workflow_factory: Callable[..., Workflow],
+        logger: Optional[logging.Logger] = None,
+        use_default_routers: Optional[bool] = True,
+        env: Optional[str] = None,
+        ui_config: Optional[Union[UIConfig, dict]] = None,
+        server_url: Optional[str] = None,
+        api_prefix: Optional[str] = None,
+        verbose: bool = False,
+        *args: Any,
+        **kwargs: Any,
+    ):
+        """
+        Initialize the LlamaIndexServer.
+
+        Args:
+            workflow_factory: A factory function that creates a workflow instance for each request.
+            logger: The logger to use.
+            use_default_routers: Whether to use the default routers (chat, mount `data` and `output` directories).
+            env: The environment to run the server in.
+            ui_config: The configuration for the chat UI.
+            server_url: The URL of the server.
+            api_prefix: The prefix for the API endpoints.
+            verbose: Whether to show verbose logs.
+        """
+        super().__init__(*args, **kwargs)
+
+        self.workflow_factory = workflow_factory
+        self.logger = logger or logging.getLogger("uvicorn")
+        self.verbose = verbose
+        self.use_default_routers = use_default_routers or True
+        if ui_config is None:
+            self.ui_config = UIConfig()
+        elif isinstance(ui_config, dict):
+            self.ui_config = UIConfig(**ui_config)
+        else:
+            self.ui_config = ui_config
+
+        # Update the settings
+        if server_url:
+            server_settings.set_url(server_url)
+        if api_prefix:
+            server_settings.set_api_prefix(api_prefix)
+
+        if self.use_default_routers:
+            self.add_default_routers()
+
+        if str(env).lower() == "dev":
+            self.allow_cors("*")
+            if self.ui_config.enabled is None:
+                self.ui_config.enabled = True
+
+        if self.ui_config.enabled is None:
+            self.ui_config.enabled = False
+
+        if self.ui_config.enabled:
+            self.mount_ui()
+
+    # Default routers
+    def add_default_routers(self) -> None:
+        self.add_chat_router()
+        self.mount_data_dir()
+        self.mount_output_dir()
+
+    def add_chat_router(self) -> None:
+        """
+        Add the chat router.
+        """
+        self.include_router(
+            chat_router(
+                self.workflow_factory,
+                self.logger,
+            ),
+            prefix=server_settings.api_prefix,
+        )
+
+    def add_components_router(self) -> None:
+        """
+        Add the UI router.
+        """
+        if self.ui_config.component_dir is None:
+            raise ValueError("component_dir must be specified to add components router")
+
+        self.include_router(
+            custom_components_router(self.ui_config.component_dir, self.logger),
+            prefix=server_settings.api_prefix,
+        )
+
+    def mount_ui(self) -> None:
+        """
+        Mount the UI.
+        """
+        # Check if the static folder exists
+        if self.ui_config.enabled:
+            # Component dir
+            if self.ui_config.component_dir:
+                if not os.path.exists(self.ui_config.component_dir):
+                    os.makedirs(self.ui_config.component_dir)
+                self.add_components_router()
+            # UI static files
+            if not os.path.exists(self.ui_config.ui_path):
+                os.makedirs(self.ui_config.ui_path)
+                self.logger.warning(
+                    f"UI files not found, downloading UI to {self.ui_config.ui_path}"
+                )
+                download_chat_ui(logger=self.logger, target_path=self.ui_config.ui_path)
+            self._mount_static_files(
+                directory=self.ui_config.ui_path, path="/", html=True
+            )
+            self._override_ui_config()
+
+    def _override_ui_config(self) -> None:
+        """
+        Override the UI config by writing a complete configuration file.
+        """
+        try:
+            config_path = os.path.join(self.ui_config.ui_path, "config.js")
+            if not os.path.exists(config_path):
+                self.logger.error("Config file not found")
+                return
+            config_content = (
+                f"window.LLAMAINDEX = {self.ui_config.get_config_content()};"
+            )
+            with open(config_path, "w") as f:
+                f.write(config_content)
+        except Exception as e:
+            self.logger.error(f"Error overriding UI config: {e}")
+
+    def mount_data_dir(self, data_dir: str = "data") -> None:
+        """
+        Mount the data directory.
+        """
+        self._mount_static_files(
+            directory=data_dir,
+            path=f"{server_settings.api_prefix}/files/data",
+            html=True,
+        )
+
+    def mount_output_dir(self, output_dir: str = "output") -> None:
+        """
+        Mount the output directory.
+        """
+        self._mount_static_files(
+            directory=output_dir,
+            path=f"{server_settings.api_prefix}/files/output",
+            html=True,
+        )
+
+    def _mount_static_files(
+        self, directory: str, path: str, html: bool = False
+    ) -> None:
+        """
+        Mount static files from a directory if it exists.
+        """
+        if os.path.exists(directory):
+            self.logger.info(f"Mounting static files '{directory}' at '{path}'")
+            self.mount(
+                path,
+                StaticFiles(directory=directory, check_dir=False, html=html),
+                name=f"{directory}-static",
+            )
+
+    def allow_cors(self, origin: str = "*") -> None:
+        """
+        Allow CORS for a specific origin.
+        """
+        self.add_middleware(
+            CORSMiddleware,
+            allow_origins=[origin],
+            allow_credentials=True,
+            allow_methods=["*"],
+            allow_headers=["*"],
+        )

llama-index-server/llama_index/server/services/custom_ui.py

@@ -0,0 +1,81 @@
+import logging
+import os
+from typing import List, Optional
+
+from llama_index.server.api.models import ComponentDefinition
+
+
+class CustomUI:
+    def __init__(
+        self, component_dir: str, logger: Optional[logging.Logger] = None
+    ) -> None:
+        self.component_dir = component_dir
+        self.logger = logger or logging.getLogger(__name__)
+
+    def get_components(self) -> List[ComponentDefinition]:
+        """
+        List all js files in the component directory and return a list of ComponentDefinition objects.
+        Ignores files that fail to load and logs the error.
+        TSX files take precedence over JSX files when duplicate component names are found.
+        """
+        components_dict: dict[str, ComponentDefinition] = {}
+        if not os.path.exists(self.component_dir):
+            self.logger.warning(
+                f"Component directory {self.component_dir} does not exist"
+            )
+            return []
+        try:
+            for file in os.listdir(self.component_dir):
+                if not file.endswith((".jsx", ".tsx")):
+                    continue
+
+                component_name = file.split(".")[0]
+                file_path = os.path.join(self.component_dir, file)
+                file_ext = os.path.splitext(file)[1]
+
+                try:
+                    with open(file_path, "r", encoding="utf-8") as f:
+                        code = f.read()
+                        new_component = ComponentDefinition(
+                            type=component_name,
+                            code=code,
+                            filename=file,
+                        )
+
+                        if component_name in components_dict:
+                            existing_ext = os.path.splitext(
+                                components_dict[component_name].filename
+                            )[1]
+
+                            # If existing is TSX and new is JSX, skip and warn
+                            if existing_ext == ".tsx" and file_ext == ".jsx":
+                                self.logger.warning(
+                                    f"Skipping duplicate JSX component {file} as TSX version already exists"
+                                )
+                                continue
+
+                            # If both are same extension, warn and skip
+                            if existing_ext == file_ext:
+                                self.logger.warning(
+                                    f"Skipping duplicate component {file} with same extension"
+                                )
+                                continue
+
+                            # If existing is JSX and new is TSX, replace and warn
+                            if existing_ext == ".jsx" and file_ext == ".tsx":
+                                self.logger.warning(
+                                    f"Replacing JSX component {components_dict[component_name].filename} with TSX version {file}"
+                                )
+                                components_dict[component_name] = new_component
+                                continue
+
+                        components_dict[component_name] = new_component
+
+                except Exception as e:
+                    self.logger.error(f"Failed to load component {file}: {str(e)}")
+                    continue
+
+        except Exception as e:
+            self.logger.error(f"Error reading component directory: {str(e)}")
+
+        return list(components_dict.values())

llama-index-server/llama_index/server/services/file.py

@@ -0,0 +1,117 @@
+import logging
+import os
+import re
+import uuid
+from pathlib import Path
+from typing import List, Optional, Union
+
+from llama_index.server.settings import server_settings
+from pydantic import BaseModel, Field
+
+logger = logging.getLogger(__name__)
+
+PRIVATE_STORE_PATH = str(Path("output", "uploaded"))
+TOOL_STORE_PATH = str(Path("output", "tools"))
+LLAMA_CLOUD_STORE_PATH = str(Path("output", "llamacloud"))
+
+
+class DocumentFile(BaseModel):
+    id: str
+    name: str  # Stored file name
+    type: Optional[str] = None
+    size: Optional[int] = None
+    url: Optional[str] = None
+    path: Optional[str] = Field(
+        None,
+        description="The stored file path. Used internally in the server.",
+        exclude=True,
+    )
+    refs: Optional[List[str]] = Field(
+        None, description="The document ids in the index."
+    )
+
+
+class FileService:
+    """
+    To store the files uploaded by the user.
+    """
+
+    @classmethod
+    def save_file(
+        cls,
+        content: Union[bytes, str],
+        file_name: str,
+        save_dir: Optional[str] = None,
+    ) -> DocumentFile:
+        """
+        Save the content to a file in the local file server (accessible via URL).
+
+        Args:
+            content (bytes | str): The content to save, either bytes or string.
+            file_name (str): The original name of the file.
+            save_dir (Optional[str]): The relative path from the current working directory. Defaults to the `output/uploaded` directory.
+
+        Returns:
+            The metadata of the saved file.
+        """
+        if save_dir is None:
+            save_dir = os.path.join("output", "uploaded")
+
+        file_id = str(uuid.uuid4())
+        name, extension = os.path.splitext(file_name)
+        extension = extension.lstrip(".")
+        sanitized_name = _sanitize_file_name(name)
+        if extension == "":
+            raise ValueError("File is not supported!")
+        new_file_name = f"{sanitized_name}_{file_id}.{extension}"
+
+        file_path = os.path.join(save_dir, new_file_name)
+
+        if isinstance(content, str):
+            content = content.encode()
+
+        try:
+            os.makedirs(os.path.dirname(file_path), exist_ok=True)
+            with open(file_path, "wb") as file:
+                file.write(content)
+        except PermissionError as e:
+            logger.error(f"Permission denied when writing to file {file_path}: {e!s}")
+            raise
+        except OSError as e:
+            logger.error(f"IO error occurred when writing to file {file_path}: {e!s}")
+            raise
+        except Exception as e:
+            logger.error(f"Unexpected error when writing to file {file_path}: {e!s}")
+            raise
+
+        logger.info(f"Saved file to {file_path}")
+
+        file_size = os.path.getsize(file_path)
+        file_url = (
+            f"{server_settings.file_server_url_prefix}/{save_dir}/{new_file_name}"
+        )
+        return DocumentFile(
+            id=file_id,
+            name=new_file_name,
+            type=extension,
+            size=file_size,
+            path=file_path,
+            url=file_url,
+            refs=None,
+        )
+
+    @classmethod
+    def get_file_url(cls, file_name: str, save_dir: Optional[str] = None) -> str:
+        """
+        Get the URL of a file.
+        """
+        if save_dir is None:
+            save_dir = os.path.join("output", "uploaded")
+        return f"{server_settings.file_server_url_prefix}/{save_dir}/{file_name}"
+
+
+def _sanitize_file_name(file_name: str) -> str:
+    """
+    Sanitize the file name by replacing all non-alphanumeric characters with underscores.
+    """
+    return re.sub(r"[^a-zA-Z0-9.]", "_", file_name)

llama-index-server/llama_index/server/services/llamacloud/__init__.py

@@ -0,0 +1,11 @@
+from .file import LlamaCloudFileService
+from .generate import load_to_llamacloud
+from .index import LlamaCloudIndex, get_client, get_index
+
+__all__ = [
+    "LlamaCloudFileService",
+    "LlamaCloudIndex",
+    "get_client",
+    "get_index",
+    "load_to_llamacloud",
+]

llama-index-server/llama_index/server/services/llamacloud/file.py

@@ -0,0 +1,184 @@
+import logging
+import os
+import time
+import typing
+from io import BytesIO
+from typing import Any, Dict, List, Optional, Set, Tuple, Union
+
+import requests
+from fastapi import BackgroundTasks
+from llama_cloud import ManagedIngestionStatus, PipelineFileCreateCustomMetadataValue
+from llama_index.core.schema import NodeWithScore
+from llama_index.server.api.models import SourceNodes
+from llama_index.server.services.llamacloud.index import get_client
+from pydantic import BaseModel
+
+logger = logging.getLogger("uvicorn")
+
+
+class LlamaCloudFile(BaseModel):
+    file_name: str
+    pipeline_id: str
+
+    def __eq__(self, other: Any) -> bool:
+        if not isinstance(other, LlamaCloudFile):
+            return NotImplemented
+        return (
+            self.file_name == other.file_name and self.pipeline_id == other.pipeline_id
+        )
+
+    def __hash__(self) -> int:
+        return hash((self.file_name, self.pipeline_id))
+
+
+class LlamaCloudFileService:
+    LOCAL_STORE_PATH = "output/llamacloud"
+    DOWNLOAD_FILE_NAME_TPL = "{pipeline_id}${filename}"
+
+    @classmethod
+    def get_all_projects_with_pipelines(cls) -> List[Dict[str, Any]]:
+        try:
+            client = get_client()
+            projects = client.projects.list_projects()
+            pipelines = client.pipelines.search_pipelines()
+            return [
+                {
+                    **(project.dict()),
+                    "pipelines": [
+                        {"id": p.id, "name": p.name}
+                        for p in pipelines
+                        if p.project_id == project.id
+                    ],
+                }
+                for project in projects
+            ]
+        except Exception as error:
+            logger.error(f"Error listing projects and pipelines: {error}")
+            return []
+
+    @classmethod
+    def add_file_to_pipeline(
+        cls,
+        project_id: str,
+        pipeline_id: str,
+        upload_file: Union[typing.IO, Tuple[str, BytesIO]],
+        custom_metadata: Optional[Dict[str, PipelineFileCreateCustomMetadataValue]],
+        wait_for_processing: bool = True,
+    ) -> str:
+        client = get_client()
+        file = client.files.upload_file(project_id=project_id, upload_file=upload_file)
+        file_id = file.id
+        files = [
+            {
+                "file_id": file_id,
+                "custom_metadata": {"file_id": file_id, **(custom_metadata or {})},
+            }
+        ]
+        files = client.pipelines.add_files_to_pipeline_api(pipeline_id, request=files)
+
+        if not wait_for_processing:
+            return file_id
+
+        # Wait 2s for the file to be processed
+        max_attempts = 20
+        attempt = 0
+        while attempt < max_attempts:
+            result = client.pipelines.get_pipeline_file_status(
+                file_id=file_id, pipeline_id=pipeline_id
+            )
+            if result.status == ManagedIngestionStatus.ERROR:
+                raise Exception(f"File processing failed: {str(result)}")
+            if result.status == ManagedIngestionStatus.SUCCESS:
+                # File is ingested - return the file id
+                return file_id
+            attempt += 1
+            time.sleep(0.1)  # Sleep for 100ms
+        raise Exception(
+            f"File processing did not complete after {max_attempts} attempts."
+        )
+
+    @classmethod
+    def download_pipeline_file(
+        cls,
+        file: LlamaCloudFile,
+        force_download: bool = False,
+    ) -> None:
+        client = get_client()
+        file_name = file.file_name
+        pipeline_id = file.pipeline_id
+
+        # Check is the file already exists
+        downloaded_file_path = cls._get_file_path(file_name, pipeline_id)
+        if os.path.exists(downloaded_file_path) and not force_download:
+            logger.debug(f"File {file_name} already exists in local storage")
+            return
+        try:
+            logger.info(f"Downloading file {file_name} for pipeline {pipeline_id}")
+            files = client.pipelines.list_pipeline_files(pipeline_id)
+            if not files or not isinstance(files, list):
+                raise Exception("No files found in LlamaCloud")
+            for file_entry in files:
+                if file_entry.name == file_name:
+                    file_id = file_entry.file_id
+                    project_id = file_entry.project_id
+                    file_detail = client.files.read_file_content(
+                        file_id, project_id=project_id
+                    )
+                    cls._download_file(file_detail.url, downloaded_file_path)
+                    break
+        except Exception as error:
+            logger.info(f"Error fetching file from LlamaCloud: {error}")
+
+    @classmethod
+    def download_files_from_nodes(
+        cls, nodes: List[NodeWithScore], background_tasks: BackgroundTasks
+    ) -> None:
+        files = cls._get_files_to_download(nodes)
+        for file in files:
+            logger.info(f"Adding download of {file.file_name} to background tasks")
+            background_tasks.add_task(cls.download_pipeline_file, file)
+
+    @classmethod
+    def _get_files_to_download(cls, nodes: List[NodeWithScore]) -> Set[LlamaCloudFile]:
+        source_nodes = SourceNodes.from_source_nodes(nodes)
+        llama_cloud_files = [
+            LlamaCloudFile(
+                file_name=node.metadata.get("file_name"),  # type: ignore
+                pipeline_id=node.metadata.get("pipeline_id"),  # type: ignore
+            )
+            for node in source_nodes
+            if (
+                node.metadata.get("pipeline_id") is not None
+                and node.metadata.get("file_name") is not None
+            )
+        ]
+        # Remove duplicates and return
+        return set(llama_cloud_files)
+
+    @classmethod
+    def _get_file_name(cls, name: str, pipeline_id: str) -> str:
+        return cls.DOWNLOAD_FILE_NAME_TPL.format(pipeline_id=pipeline_id, filename=name)
+
+    @classmethod
+    def _get_file_path(cls, name: str, pipeline_id: str) -> str:
+        return os.path.join(cls.LOCAL_STORE_PATH, cls._get_file_name(name, pipeline_id))
+
+    @classmethod
+    def _download_file(cls, url: str, local_file_path: str) -> None:
+        logger.info(f"Saving file to {local_file_path}")
+        # Create directory if it doesn't exist
+        os.makedirs(cls.LOCAL_STORE_PATH, exist_ok=True)
+        # Download the file
+        with requests.get(url, stream=True) as r:
+            r.raise_for_status()
+            with open(local_file_path, "wb") as f:
+                for chunk in r.iter_content(chunk_size=8192):
+                    f.write(chunk)
+        logger.info("File downloaded successfully")
+
+    @classmethod
+    def is_configured(cls) -> bool:
+        try:
+            return os.environ.get("LLAMA_CLOUD_API_KEY") is not None
+        except Exception:
+            return False

llama-index-server/llama_index/server/services/llamacloud/generate.py

@@ -0,0 +1,56 @@
+import logging
+from typing import Optional
+
+from tqdm import tqdm
+
+from llama_index.core.readers import SimpleDirectoryReader
+from llama_index.indices.managed.llama_cloud import LlamaCloudIndex
+from llama_index.server.services.llamacloud.file import LlamaCloudFileService
+
+
+def load_to_llamacloud(
+    index: LlamaCloudIndex,
+    data_dir: Optional[str] = None,
+    recursive: Optional[bool] = None,
+    logger: Optional[logging.Logger] = None,
+) -> None:
+    if logger is None:
+        logging.basicConfig(level=logging.INFO)
+        logger = logging.getLogger()
+
+    logger.info("Generate index for the provided data")
+
+    # use SimpleDirectoryReader to retrieve the files to process
+    reader = SimpleDirectoryReader(
+        data_dir or "data",
+        recursive=recursive or True,
+    )
+    files_to_process = reader.input_files
+
+    # add each file to the LlamaCloud pipeline
+    error_files = []
+    for input_file in tqdm(
+        files_to_process,
+        desc="Processing files",
+        unit="file",
+    ):
+        with open(input_file, "rb") as f:
+            logger.debug(
+                f"Adding file {input_file} to pipeline {index.name} in project {index.project_name}"
+            )
+            try:
+                LlamaCloudFileService.add_file_to_pipeline(
+                    index.project.id,
+                    index.pipeline.id,
+                    f,
+                    custom_metadata={},
+                    wait_for_processing=False,
+                )
+            except Exception as e:
+                error_files.append(input_file)
+                logger.error(f"Error adding file {input_file}: {e}")
+
+    if error_files:
+        logger.error(f"Failed to add the following files: {error_files}")
+
+    logger.info("Finished generating the index")

llama-index-server/llama_index/server/services/llamacloud/index.py

@@ -0,0 +1,164 @@
+import logging
+import os
+from typing import TYPE_CHECKING, Any, Optional
+
+from llama_cloud import PipelineType
+from llama_index.core.callbacks import CallbackManager
+from llama_index.core.ingestion.api_utils import (
+    get_client as llama_cloud_get_client,
+)
+from llama_index.core.settings import Settings
+from llama_index.indices.managed.llama_cloud import LlamaCloudIndex
+from llama_index.server.api.models import ChatRequest
+from pydantic import BaseModel, Field, field_validator
+
+if TYPE_CHECKING:
+    from llama_cloud.client import LlamaCloud
+
+logger = logging.getLogger("uvicorn")
+
+
+class LlamaCloudConfig(BaseModel):
+    # Private attributes
+    api_key: str = Field(
+        exclude=True,  # Exclude from the model representation
+    )
+    base_url: Optional[str] = Field(
+        exclude=True,
+    )
+    organization_id: Optional[str] = Field(
+        exclude=True,
+    )
+    # Configuration attributes, can be set by the user
+    pipeline: str = Field(
+        description="The name of the pipeline to use",
+    )
+    project: str = Field(
+        description="The name of the LlamaCloud project",
+    )
+
+    def __init__(self, **kwargs: Any) -> None:
+        if "api_key" not in kwargs:
+            kwargs["api_key"] = os.getenv("LLAMA_CLOUD_API_KEY")
+        if "base_url" not in kwargs:
+            kwargs["base_url"] = os.getenv("LLAMA_CLOUD_BASE_URL")
+        if "organization_id" not in kwargs:
+            kwargs["organization_id"] = os.getenv("LLAMA_CLOUD_ORGANIZATION_ID")
+        if "pipeline" not in kwargs:
+            kwargs["pipeline"] = os.getenv("LLAMA_CLOUD_INDEX_NAME")
+        if "project" not in kwargs:
+            kwargs["project"] = os.getenv("LLAMA_CLOUD_PROJECT_NAME")
+        super().__init__(**kwargs)
+
+    # Validate and throw error if the env variables are not set before starting the app
+    @field_validator("pipeline", "project", "api_key", mode="before")
+    @classmethod
+    def validate_fields(cls, value: Any) -> Any:
+        if value is None:
+            raise ValueError(
+                "Please set LLAMA_CLOUD_INDEX_NAME, LLAMA_CLOUD_PROJECT_NAME and LLAMA_CLOUD_API_KEY"
+                " to your environment variables or config them in .env file"
+            )
+        return value
+
+    def to_client_kwargs(self) -> dict:
+        return {
+            "api_key": self.api_key,
+            "base_url": self.base_url,
+        }
+
+
+class IndexConfig(BaseModel):
+    llama_cloud_pipeline_config: LlamaCloudConfig = Field(
+        default_factory=lambda: LlamaCloudConfig(),
+        alias="llamaCloudPipeline",
+    )
+    callback_manager: Optional[CallbackManager] = Field(
+        default=None,
+    )
+
+    def to_index_kwargs(self) -> dict:
+        return {
+            "name": self.llama_cloud_pipeline_config.pipeline,
+            "project_name": self.llama_cloud_pipeline_config.project,
+            "api_key": self.llama_cloud_pipeline_config.api_key,
+            "base_url": self.llama_cloud_pipeline_config.base_url,
+            "organization_id": self.llama_cloud_pipeline_config.organization_id,
+            "callback_manager": self.callback_manager,
+        }
+
+    @classmethod
+    def from_default(cls, chat_request: Optional[ChatRequest] = None) -> "IndexConfig":
+        default_config = cls()
+        if chat_request is not None and chat_request.data is not None:
+            llamacloud_config = chat_request.data.get("llamaCloudPipeline")
+            if llamacloud_config is not None:
+                default_config.llama_cloud_pipeline_config.pipeline = llamacloud_config[
+                    "pipeline"
+                ]
+                default_config.llama_cloud_pipeline_config.project = llamacloud_config[
+                    "project"
+                ]
+        return default_config
+
+
+def get_index(
+    chat_request: Optional[ChatRequest] = None,
+    create_if_missing: bool = False,
+) -> Optional[LlamaCloudIndex]:
+    config = IndexConfig.from_default(chat_request)
+    # Check whether the index exists
+    try:
+        index = LlamaCloudIndex(**config.to_index_kwargs())
+        return index
+    except ValueError:
+        logger.warning("Index not found")
+        if create_if_missing:
+            logger.info("Creating index")
+            _create_index(config)
+            return LlamaCloudIndex(**config.to_index_kwargs())
+        return None
+
+
+def get_client() -> "LlamaCloud":
+    config = LlamaCloudConfig()
+    return llama_cloud_get_client(**config.to_client_kwargs())
+
+
+def _create_index(
+    config: IndexConfig,
+) -> None:
+    client = get_client()
+    pipeline_name = config.llama_cloud_pipeline_config.pipeline
+
+    pipelines = client.pipelines.search_pipelines(
+        pipeline_name=pipeline_name,
+        pipeline_type=PipelineType.MANAGED.value,
+    )
+    if len(pipelines) == 0:
+        from llama_index.embeddings.openai import OpenAIEmbedding
+
+        if not isinstance(Settings.embed_model, OpenAIEmbedding):
+            raise ValueError(
+                "Creating a new pipeline with a non-OpenAI embedding model is not supported."
+            )
+        client.pipelines.upsert_pipeline(
+            request={
+                "name": pipeline_name,
+                "embedding_config": {
+                    "type": "OPENAI_EMBEDDING",
+                    "component": {
+                        "api_key": os.getenv("OPENAI_API_KEY"),  # editable
+                        "model_name": Settings.embed_model.model_name
+                        or "text-embedding-3-small",
+                    },
+                },
+                "transform_config": {
+                    "mode": "auto",
+                    "config": {
+                        "chunk_size": Settings.chunk_size,  # editable
+                        "chunk_overlap": Settings.chunk_overlap,  # editable
+                    },
+                },
+            },
+        )

llama-index-server/llama_index/server/services/suggest_next_question.py

@@ -0,0 +1,95 @@
+import logging
+import os
+import re
+from typing import List, Optional, Union
+
+from llama_index.core.prompts import PromptTemplate
+from llama_index.core.settings import Settings
+from llama_index.server.api.models import ChatAPIMessage
+
+logger = logging.getLogger("uvicorn")
+
+
+class SuggestNextQuestionsService:
+    """
+    Suggest the next questions that user might ask based on the conversation history.
+    """
+
+    prompt = PromptTemplate(
+        r"""
+You're a helpful assistant! Your task is to suggest the next questions that user might interested in to keep the conversation going.
+Here is the conversation history
+---------------------
+{conversation}
+---------------------
+Given the conversation history, please give me 3 questions that user might ask next!
+Your answer should be wrapped in three sticks without any index numbers and follows the following format:
+\`\`\`
+<question 1>
+<question 2>
+<question 3>
+\`\`\`
+"""
+    )
+
+    @classmethod
+    def get_configured_prompt(cls) -> PromptTemplate:
+        prompt = os.getenv("NEXT_QUESTION_PROMPT", None)
+        if not prompt:
+            return cls.prompt
+        return PromptTemplate(prompt)
+
+    @classmethod
+    async def suggest_next_questions_all_messages(
+        cls,
+        messages: List[ChatAPIMessage],
+    ) -> Optional[List[str]]:
+        """
+        Suggest the next questions that user might ask based on the conversation history.
+        """
+        prompt_template = cls.get_configured_prompt()
+
+        try:
+            # Reduce the cost by only using the last two messages
+            last_user_message = None
+            last_assistant_message = None
+            for message in reversed(messages):
+                if message.role == "user":
+                    last_user_message = f"User: {message.content}"
+                elif message.role == "assistant":
+                    last_assistant_message = f"Assistant: {message.content}"
+                if last_user_message and last_assistant_message:
+                    break
+            conversation: str = f"{last_user_message}\n{last_assistant_message}"
+
+            # Call the LLM and parse questions from the output
+            prompt = prompt_template.format(conversation=conversation)
+            output = await Settings.llm.acomplete(prompt)
+            return cls._extract_questions(output.text)
+
+        except Exception as e:
+            logger.error(f"Error when generating next question: {e}")
+            return None
+
+    @classmethod
+    def _extract_questions(cls, text: str) -> Union[List[str], None]:
+        content_match = re.search(r"```(.*?)```", text, re.DOTALL)
+        content = content_match.group(1) if content_match else None
+        if not content:
+            return None
+        return [q.strip() for q in content.split("\n") if q.strip()]
+
+    @classmethod
+    async def run(
+        cls,
+        chat_history: List[ChatAPIMessage],
+        response: str,
+    ) -> Optional[List[str]]:
+        """
+        Suggest the next questions that user might ask based on the chat history and the last response.
+        """
+        messages = [
+            *chat_history,
+            ChatAPIMessage(role="assistant", content=response),  # type: ignore
+        ]
+        return await cls.suggest_next_questions_all_messages(messages)

llama-index-server/llama_index/server/settings.py

@@ -0,0 +1,47 @@
+from pydantic import Field, validator
+from pydantic_settings import BaseSettings
+
+
+class ServerSettings(BaseSettings):
+    url: str = Field(
+        default="",
+        description="The deployment URL of the server, to be referenced by tools and file services",
+    )
+    api_prefix: str = Field(
+        default="/api",
+        description="The prefix for the API endpoints",
+    )
+
+    @property
+    def file_server_url_prefix(self) -> str:
+        return f"{self.url}{self.api_prefix}/files"
+
+    @property
+    def api_url(self) -> str:
+        return f"{self.url}{self.api_prefix}"
+
+    @validator("url")
+    def validate_url(cls, v: str) -> str:
+        if v.endswith("/"):
+            raise ValueError("URL must not end with a '/'")
+        return v
+
+    @validator("api_prefix")
+    def validate_api_prefix(cls, v: str) -> str:
+        if not v.startswith("/"):
+            raise ValueError("API prefix must start with a '/'")
+        return v
+
+    def set_url(self, v: str) -> None:
+        self.url = v
+        self.validate_url(v)  # type: ignore
+
+    def set_api_prefix(self, v: str) -> None:
+        self.api_prefix = v
+        self.validate_api_prefix(v)  # type: ignore
+
+    class Config:
+        env_file_encoding = "utf-8"
+
+
+server_settings = ServerSettings()

llama-index-server/llama_index/server/tools/document_generator.py

@@ -0,0 +1,242 @@
+import logging
+import os
+import re
+from enum import Enum
+from io import BytesIO
+
+from llama_index.core.tools.function_tool import FunctionTool
+
+OUTPUT_DIR = "output/tools"
+
+
+class DocumentType(Enum):
+    PDF = "pdf"
+    HTML = "html"
+
+
+COMMON_STYLES = """
+body {
+    font-family: Arial, sans-serif;
+    line-height: 1.3;
+    color: #333;
+}
+h1, h2, h3, h4, h5, h6 {
+    margin-top: 1em;
+    margin-bottom: 0.5em;
+}
+p {
+    margin-bottom: 0.7em;
+}
+code {
+    background-color: #f4f4f4;
+    padding: 2px 4px;
+    border-radius: 4px;
+}
+pre {
+    background-color: #f4f4f4;
+    padding: 10px;
+    border-radius: 4px;
+    overflow-x: auto;
+}
+table {
+    border-collapse: collapse;
+    width: 100%;
+    margin-bottom: 1em;
+}
+th, td {
+    border: 1px solid #ddd;
+    padding: 8px;
+    text-align: left;
+}
+th {
+    background-color: #f2f2f2;
+    font-weight: bold;
+}
+"""
+
+HTML_SPECIFIC_STYLES = """
+body {
+    max-width: 800px;
+    margin: 0 auto;
+    padding: 20px;
+}
+"""
+
+PDF_SPECIFIC_STYLES = """
+@page {
+    size: letter;
+    margin: 2cm;
+}
+body {
+    font-size: 11pt;
+}
+h1 { font-size: 18pt; }
+h2 { font-size: 16pt; }
+h3 { font-size: 14pt; }
+h4, h5, h6 { font-size: 12pt; }
+pre, code {
+    font-family: Courier, monospace;
+    font-size: 0.9em;
+}
+"""
+
+HTML_TEMPLATE = """
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <style>
+        {common_styles}
+        {specific_styles}
+    </style>
+</head>
+<body>
+    {content}
+</body>
+</html>
+"""
+
+
+class DocumentGenerator:
+    def __init__(self, file_server_url_prefix: str):
+        if not file_server_url_prefix:
+            raise ValueError("file_server_url_prefix is required")
+        self.file_server_url_prefix = file_server_url_prefix
+
+    @classmethod
+    def _generate_html_content(cls, original_content: str) -> str:
+        """
+        Generate HTML content from the original markdown content.
+        """
+        try:
+            import markdown  # type: ignore
+        except ImportError:
+            raise ImportError(
+                "Failed to import required modules. Please install markdown."
+            )
+
+        # Convert markdown to HTML with fenced code and table extensions
+        return markdown.markdown(original_content, extensions=["fenced_code", "tables"])
+
+    @classmethod
+    def _generate_pdf(cls, html_content: str) -> BytesIO:
+        """
+        Generate a PDF from the HTML content.
+        """
+        try:
+            from xhtml2pdf import pisa
+        except ImportError:
+            raise ImportError(
+                "Failed to import required modules. Please install xhtml2pdf."
+            )
+
+        pdf_html = HTML_TEMPLATE.format(
+            common_styles=COMMON_STYLES,
+            specific_styles=PDF_SPECIFIC_STYLES,
+            content=html_content,
+        )
+
+        buffer = BytesIO()
+        pdf = pisa.pisaDocument(
+            BytesIO(pdf_html.encode("UTF-8")), buffer, encoding="UTF-8"
+        )
+
+        if pdf.err:
+            logging.error(f"PDF generation failed: {pdf.err}")
+            raise ValueError("PDF generation failed")
+
+        buffer.seek(0)
+        return buffer
+
+    @classmethod
+    def _generate_html(cls, html_content: str) -> str:
+        """
+        Generate a complete HTML document with the given HTML content.
+        """
+        return HTML_TEMPLATE.format(
+            common_styles=COMMON_STYLES,
+            specific_styles=HTML_SPECIFIC_STYLES,
+            content=html_content,
+        )
+
+    def generate_document(
+        self, original_content: str, document_type: str, file_name: str
+    ) -> str:
+        """
+        To generate document as PDF or HTML file.
+        Parameters:
+            original_content: str (markdown style)
+            document_type: str (pdf or html) specify the type of the file format based on the use case
+            file_name: str (name of the document file) must be a valid file name, no extensions needed
+        Returns:
+            str (URL to the document file): A file URL ready to serve.
+        """
+        try:
+            doc_type = DocumentType(document_type.lower())
+        except ValueError:
+            raise ValueError(
+                f"Invalid document type: {document_type}. Must be 'pdf' or 'html'."
+            )
+        # Always generate html content first
+        html_content = self._generate_html_content(original_content)
+
+        # Based on the type of document, generate the corresponding file
+        if doc_type == DocumentType.PDF:
+            content = self._generate_pdf(html_content)
+            file_extension = "pdf"
+        elif doc_type == DocumentType.HTML:
+            content = BytesIO(self._generate_html(html_content).encode("utf-8"))
+            file_extension = "html"
+        else:
+            raise ValueError(f"Unexpected document type: {document_type}")
+
+        file_name = self._validate_file_name(file_name)
+        file_path = os.path.join(OUTPUT_DIR, f"{file_name}.{file_extension}")
+
+        self._write_to_file(content, file_path)
+
+        return (
+            f"{self.file_server_url_prefix}/{OUTPUT_DIR}/{file_name}.{file_extension}"
+        )
+
+    @staticmethod
+    def _write_to_file(content: BytesIO, file_path: str) -> None:
+        """
+        Write the content to a file.
+        """
+        try:
+            os.makedirs(os.path.dirname(file_path), exist_ok=True)
+            with open(file_path, "wb") as file:
+                file.write(content.getvalue())
+        except Exception:
+            raise
+
+    @staticmethod
+    def _validate_file_name(file_name: str) -> str:
+        """
+        Validate the file name.
+        """
+        # Don't allow directory traversal
+        if os.path.isabs(file_name):
+            raise ValueError("File name is not allowed.")
+        # Don't allow special characters
+        if re.match(r"^[a-zA-Z0-9_.-]+$", file_name):
+            return file_name
+        else:
+            raise ValueError("File name is not allowed to contain special characters.")
+
+    @classmethod
+    def _validate_packages(cls) -> None:
+        try:
+            import markdown  # noqa: F401
+            import xhtml2pdf  # noqa: F401
+        except ImportError:
+            raise ImportError(
+                "Failed to import required modules. Please install markdown and xhtml2pdf "
+                "using `pip install markdown xhtml2pdf`"
+            )
+
+    def to_tool(self) -> FunctionTool:
+        self._validate_packages()
+        return FunctionTool.from_defaults(self.generate_document)

llama-index-server/llama_index/server/tools/index/__init__.py

@@ -0,0 +1,3 @@
+from .query import get_query_engine_tool
+
+__all__ = ["get_query_engine_tool"]

llama-index-server/llama_index/server/tools/index/query.py

@@ -0,0 +1,49 @@
+import os
+from typing import Any, Optional
+
+from llama_index.core.base.base_query_engine import BaseQueryEngine
+from llama_index.core.tools.query_engine import QueryEngineTool
+from llama_index.core.indices.base import BaseIndex
+
+
+def create_query_engine(index: BaseIndex, **kwargs: Any) -> BaseQueryEngine:
+    """
+    Create a query engine for the given index.
+
+    Args:
+        index: The index to create a query engine for.
+        params (optional): Additional parameters for the query engine, e.g: similarity_top_k
+    """
+    top_k = int(os.getenv("TOP_K", 0))
+    if top_k != 0 and kwargs.get("filters") is None:
+        kwargs["similarity_top_k"] = top_k
+
+    return index.as_query_engine(**kwargs)
+
+
+def get_query_engine_tool(
+    index: BaseIndex,
+    name: Optional[str] = None,
+    description: Optional[str] = None,
+    **kwargs: Any,
+) -> QueryEngineTool:
+    """
+    Get a query engine tool for the given index.
+
+    Args:
+        index: The index to create a query engine for.
+        name (optional): The name of the tool.
+        description (optional): The description of the tool.
+    """
+    if name is None:
+        name = "query_index"
+    if description is None:
+        description = (
+            "Use this tool to retrieve information about the text corpus from an index."
+        )
+    query_engine = create_query_engine(index, **kwargs)
+    return QueryEngineTool.from_defaults(
+        query_engine=query_engine,
+        name=name,
+        description=description,
+    )

llama-index-server/llama_index/server/tools/index/utils.py

@@ -0,0 +1,13 @@
+from datetime import timedelta
+
+from cachetools import TTLCache, cached  # type: ignore
+
+from llama_index.core.storage import StorageContext
+
+
+@cached(
+    TTLCache(maxsize=10, ttl=timedelta(minutes=5).total_seconds()),
+    key=lambda *args, **kwargs: "global_storage_context",
+)
+def get_storage_context(persist_dir: str) -> StorageContext:
+    return StorageContext.from_defaults(persist_dir=persist_dir)

llama-index-server/llama_index/server/tools/interpreter.py

@@ -0,0 +1,216 @@
+import base64
+import logging
+import os
+import uuid
+from typing import Any, List, Optional
+
+from llama_index.core.tools import FunctionTool
+from llama_index.server.services.file import DocumentFile, FileService
+from pydantic import BaseModel
+
+logger = logging.getLogger("uvicorn")
+
+
+class InterpreterExtraResult(BaseModel):
+    type: str
+    content: Optional[str] = None
+    filename: Optional[str] = None
+    url: Optional[str] = None
+
+
+class E2BToolOutput(BaseModel):
+    is_error: bool
+    logs: "Logs"  # type: ignore # noqa: F821
+    error_message: Optional[str] = None
+    results: List[InterpreterExtraResult] = []
+    retry_count: int = 0
+
+
+class E2BCodeInterpreter:
+    output_dir = "output/tools"
+    uploaded_files_dir = "output/uploaded"
+    interpreter: Optional["Sandbox"] = None  # type: ignore # noqa: F821
+
+    def __init__(
+        self,
+        api_key: str,
+        output_dir: Optional[str] = None,
+        uploaded_files_dir: Optional[str] = None,
+    ):
+        """
+        Args:
+            api_key: The API key for the E2B Code Interpreter.
+            output_dir: The directory for the output files. Default is `output/tools`.
+            uploaded_files_dir: The directory for the files to be uploaded to the sandbox. Default is `output/uploaded`.
+        """
+        self._validate_package()
+        if not api_key:
+            raise ValueError(
+                "api_key is required to run code interpreter. Get it here: https://e2b.dev/docs/getting-started/api-key"
+            )
+        self.api_key = api_key
+        self.output_dir = output_dir or "output/tools"
+        self.uploaded_files_dir = uploaded_files_dir or "output/uploaded"
+
+    @classmethod
+    def _validate_package(cls) -> None:
+        try:
+            from e2b_code_interpreter import Sandbox  # noqa: F401
+            from e2b_code_interpreter.models import Logs  # noqa: F401
+        except ImportError:
+            raise ImportError(
+                "e2b_code_interpreter is not installed. Please install it using `pip install e2b-code-interpreter`."
+            )
+
+    def __del__(self) -> None:
+        """
+        Kill the interpreter when the tool is no longer in use.
+        """
+        if self.interpreter is not None:
+            self.interpreter.kill()
+
+    def _init_interpreter(self, sandbox_files: List[str] = []) -> None:
+        """
+        Lazily initialize the interpreter.
+        """
+        from e2b_code_interpreter import Sandbox
+
+        logger.info(f"Initializing interpreter with {len(sandbox_files)} files")
+        self.interpreter = Sandbox(api_key=self.api_key)
+        if len(sandbox_files) > 0:
+            for file_path in sandbox_files:
+                file_name = os.path.basename(file_path)
+                local_file_path = os.path.join(self.uploaded_files_dir, file_name)
+                with open(local_file_path, "rb") as f:
+                    content = f.read()
+                    if self.interpreter and self.interpreter.files:
+                        self.interpreter.files.write(file_path, content)
+            logger.info(f"Uploaded {len(sandbox_files)} files to sandbox")
+
+    def _save_to_disk(self, base64_data: str, ext: str) -> DocumentFile:
+        buffer = base64.b64decode(base64_data)
+
+        # Output from e2b doesn't have a name. Create a random name for it.
+        filename = f"e2b_file_{uuid.uuid4()}.{ext}"
+
+        return FileService.save_file(
+            buffer, file_name=filename, save_dir=self.output_dir
+        )
+
+    def _parse_result(self, result: Any) -> List[InterpreterExtraResult]:
+        """
+        The result could include multiple formats (e.g. png, svg, etc.) but encoded in base64
+        We save each result to disk and return saved file metadata (extension, filename, url).
+        """
+        if not result:
+            return []
+
+        output = []
+
+        try:
+            formats = result.formats()
+            results = [result[format] for format in formats]
+
+            for ext, data in zip(formats, results):
+                if ext in ["png", "svg", "jpeg", "pdf"]:
+                    document_file = self._save_to_disk(data, ext)
+                    output.append(
+                        InterpreterExtraResult(
+                            type=ext,
+                            filename=document_file.name,
+                            url=document_file.url,
+                        )
+                    )
+                else:
+                    # Try serialize data to string
+                    try:
+                        data = str(data)
+                    except Exception as e:
+                        data = f"Error when serializing data: {e}"
+                    output.append(
+                        InterpreterExtraResult(
+                            type=ext,
+                            content=data,
+                        )
+                    )
+        except Exception as error:
+            logger.exception(error, exc_info=True)
+            logger.error("Error when parsing output from E2b interpreter tool", error)
+
+        return output
+
+    def interpret(
+        self,
+        code: str,
+        sandbox_files: List[str] = [],
+        retry_count: int = 0,
+    ) -> E2BToolOutput:
+        """
+        Execute Python code in a Jupyter notebook cell. The tool will return the result, stdout, stderr, display_data, and error.
+        If the code needs to use a file, ALWAYS pass the file path in the sandbox_files argument.
+        You have a maximum of 3 retries to get the code to run successfully.
+
+        Parameters:
+            code (str): The Python code to be executed in a single cell.
+            sandbox_files (List[str]): List of local file paths to be used by the code. The tool will throw an error if a file is not found.
+            retry_count (int): Number of times the tool has been retried.
+        """
+        from e2b_code_interpreter.models import Logs
+
+        if retry_count > 2:
+            return E2BToolOutput(
+                is_error=True,
+                logs=Logs(
+                    stdout="",
+                    stderr="",
+                    display_data="",
+                    error="",
+                ),
+                error_message="Failed to execute the code after 3 retries. Explain the error to the user and suggest a fix.",
+                retry_count=retry_count,
+            )
+
+        if self.interpreter is None:
+            self._init_interpreter(sandbox_files)
+
+        if self.interpreter:
+            logger.info(
+                f"\n{'=' * 50}\n> Running following AI-generated code:\n{code}\n{'=' * 50}"
+            )
+            exec = self.interpreter.run_code(code)
+
+            if exec.error:
+                error_message = f"The code failed to execute successfully. Error: {exec.error}. Try to fix the code and run again."
+                logger.error(error_message)
+                # Calling the generated code caused an error. Kill the interpreter and return the error to the LLM so it can try to fix the error
+                try:
+                    self.interpreter.kill()  # type: ignore
+                except Exception:
+                    pass
+                finally:
+                    self.interpreter = None
+                output = E2BToolOutput(
+                    is_error=True,
+                    logs=exec.logs,
+                    results=[],
+                    error_message=error_message,
+                    retry_count=retry_count + 1,
+                )
+            else:
+                if len(exec.results) == 0:
+                    output = E2BToolOutput(is_error=False, logs=exec.logs, results=[])
+                else:
+                    results = self._parse_result(exec.results[0])
+                    output = E2BToolOutput(
+                        is_error=False,
+                        logs=exec.logs,
+                        results=results,
+                        retry_count=retry_count + 1,
+                    )
+            return output
+        else:
+            raise ValueError("Interpreter is not initialized.")
+
+    def to_tool(self) -> FunctionTool:
+        self._validate_package()
+        return FunctionTool.from_defaults(self.interpret)

llama-index-server/llama_index/server/utils/agent_tool.py

@@ -0,0 +1,253 @@
+import logging
+import uuid
+from abc import ABC, abstractmethod
+from typing import Any, AsyncGenerator, Optional
+
+from pydantic import BaseModel, ConfigDict
+
+from llama_index.core.base.llms.types import ChatMessage, ChatResponse
+from llama_index.core.llms.function_calling import FunctionCallingLLM
+from llama_index.core.tools import (
+    BaseTool,
+    FunctionTool,
+    ToolOutput,
+    ToolSelection,
+)
+from llama_index.core.workflow import Context
+from llama_index.server.api.models import AgentRunEvent, AgentRunEventType
+from llama_index.core.agent.workflow.workflow_events import ToolCall, ToolCallResult
+
+logger = logging.getLogger("uvicorn")
+
+
+class ToolCallOutput(BaseModel):
+    tool_call_id: str
+    tool_output: ToolOutput
+
+
+class ContextAwareTool(FunctionTool, ABC):
+    @abstractmethod
+    async def acall(self, ctx: Context, input: Any) -> ToolOutput:  # type: ignore
+        pass
+
+
+class ChatWithToolsResponse(BaseModel):
+    """
+    A tool call response from chat_with_tools.
+    """
+
+    tool_calls: Optional[list[ToolSelection]]
+    tool_call_message: Optional[ChatMessage]
+    generator: Optional[AsyncGenerator[ChatResponse | None, None]]
+
+    model_config = ConfigDict(arbitrary_types_allowed=True)
+
+    def is_calling_different_tools(self) -> bool:
+        tool_names = {tool_call.tool_name for tool_call in self.tool_calls or []}
+        return len(tool_names) > 1
+
+    def has_tool_calls(self) -> bool:
+        return self.tool_calls is not None and len(self.tool_calls) > 0
+
+    def tool_name(self) -> str:
+        if not self.has_tool_calls():
+            raise ValueError("No tool calls")
+        if self.is_calling_different_tools():
+            raise ValueError("Calling different tools")
+        return self.tool_calls[0].tool_name  # type: ignore
+
+    async def full_response(self) -> str:
+        assert self.generator is not None
+        full_response = ""
+        async for chunk in self.generator:
+            content = chunk.delta  # type: ignore
+            if content:
+                full_response += content
+        return full_response
+
+
+async def chat_with_tools(  # type: ignore
+    llm: FunctionCallingLLM,
+    tools: list[BaseTool],
+    chat_history: list[ChatMessage],
+) -> ChatWithToolsResponse:
+    """
+    Request LLM to call tools or not.
+    This function doesn't change the memory.
+    """
+    generator = _tool_call_generator(llm, tools, chat_history)
+    is_tool_call = await generator.__anext__()
+    if is_tool_call:
+        # Last chunk is the full response
+        # Wait for the last chunk
+        full_response = None
+        async for chunk in generator:
+            full_response = chunk
+        assert isinstance(full_response, ChatResponse)
+        return ChatWithToolsResponse(
+            tool_calls=llm.get_tool_calls_from_response(full_response),
+            tool_call_message=full_response.message,
+            generator=None,
+        )
+    else:
+        return ChatWithToolsResponse(
+            tool_calls=None,
+            tool_call_message=None,
+            generator=generator,  # type: ignore
+        )
+
+
+async def call_tools(
+    ctx: Context,
+    agent_name: str,
+    tools: list[BaseTool],
+    tool_calls: list[ToolSelection],
+    emit_agent_events: bool = True,
+) -> list[ToolCallOutput]:
+    """
+    Call tools and return the tool call responses.
+    """
+    if len(tool_calls) == 0:
+        return []
+    tools_by_name = {tool.metadata.get_name(): tool for tool in tools}
+    if len(tool_calls) == 1:
+        if emit_agent_events:
+            ctx.write_event_to_stream(
+                AgentRunEvent(
+                    name=agent_name,
+                    msg=f"{tool_calls[0].tool_name}: {tool_calls[0].tool_kwargs}",
+                )
+            )
+        return [
+            await call_tool(ctx, tools_by_name[tool_calls[0].tool_name], tool_calls[0])
+        ]
+    # Multiple tool calls, show progress
+    tool_call_outputs: list[ToolCallOutput] = []
+
+    progress_id = str(uuid.uuid4())
+    total_steps = len(tool_calls)
+    if emit_agent_events:
+        ctx.write_event_to_stream(
+            AgentRunEvent(
+                name=agent_name,
+                msg=f"Making {total_steps} tool calls",
+            )
+        )
+    for i, tool_call in enumerate(tool_calls):
+        tool = tools_by_name.get(tool_call.tool_name)
+        if not tool:
+            tool_call_outputs.append(
+                ToolCallOutput(
+                    tool_call_id=tool_call.tool_id,
+                    tool_output=ToolOutput(
+                        is_error=True,
+                        content=f"Tool {tool_call.tool_name} does not exist",
+                        tool_name=tool_call.tool_name,
+                        raw_input=tool_call.tool_kwargs,
+                        raw_output={
+                            "error": f"Tool {tool_call.tool_name} does not exist",
+                        },
+                    ),
+                )
+            )
+            continue
+
+        tool_call_output = await call_tool(
+            ctx,
+            tool,
+            tool_call,
+        )
+        if emit_agent_events:
+            ctx.write_event_to_stream(
+                AgentRunEvent(
+                    name=agent_name,
+                    msg=f"{tool_call.tool_name}: {tool_call.tool_kwargs}",
+                    event_type=AgentRunEventType.PROGRESS,
+                    data={
+                        "id": progress_id,
+                        "total": total_steps,
+                        "current": i,
+                    },
+                )
+            )
+        tool_call_outputs.append(tool_call_output)
+    return tool_call_outputs
+
+
+async def call_tool(
+    ctx: Context,
+    tool: BaseTool,
+    tool_call: ToolSelection,
+) -> ToolCallOutput:
+    ctx.write_event_to_stream(
+        ToolCall(
+            tool_name=tool_call.tool_name,
+            tool_id=tool_call.tool_id,
+            tool_kwargs=tool_call.tool_kwargs,
+        )
+    )
+    try:
+        if isinstance(tool, ContextAwareTool):
+            if ctx is None:
+                raise ValueError("Context is required for context aware tool")
+            # inject context for calling an context aware tool
+            output = await tool.acall(ctx=ctx, **tool_call.tool_kwargs)
+        else:
+            output = await tool.acall(**tool_call.tool_kwargs)  # type: ignore
+    except Exception as e:
+        logger.error(f"Got error in tool {tool_call.tool_name}: {e!s}")
+        output = ToolOutput(
+            is_error=True,
+            content=f"Error: {e!s}",
+            tool_name=tool.metadata.get_name(),
+            raw_input=tool_call.tool_kwargs,
+            raw_output={
+                "error": str(e),
+            },
+        )
+    ctx.write_event_to_stream(
+        ToolCallResult(
+            tool_name=tool_call.tool_name,
+            tool_kwargs=tool_call.tool_kwargs,
+            tool_id=tool_call.tool_id,
+            tool_output=output,
+            return_direct=False,
+        )
+    )
+    return ToolCallOutput(
+        tool_call_id=tool_call.tool_id,
+        tool_output=output,
+    )
+
+
+async def _tool_call_generator(
+    llm: FunctionCallingLLM,
+    tools: list[BaseTool],
+    chat_history: list[ChatMessage],
+) -> AsyncGenerator[ChatResponse | bool, None]:
+    response_stream = await llm.astream_chat_with_tools(
+        tools,
+        chat_history=chat_history,
+        allow_parallel_tool_calls=False,
+    )
+
+    full_response = None
+    yielded_indicator = False
+    async for chunk in response_stream:
+        if "tool_calls" not in chunk.message.additional_kwargs:
+            # Yield a boolean to indicate whether the response is a tool call
+            if not yielded_indicator:
+                yield False
+                yielded_indicator = True
+
+            # if not a tool call, yield the chunks!
+            yield chunk  # type: ignore
+        elif not yielded_indicator:
+            # Yield the indicator for a tool call
+            yield True
+            yielded_indicator = True
+
+        full_response = chunk
+
+    if full_response:
+        yield full_response  # type: ignore

llama-index-server/pyproject.toml

@@ -0,0 +1,65 @@
+[build-system]
+build-backend = "poetry.core.masonry.api"
+requires = ["poetry-core"]
+
+[tool.codespell]
+check-filenames = true
+check-hidden = true
+# Feel free to un-skip examples, and experimental, you will just need to
+# work through many typos (--write-changes and --interactive will help)
+skip = "*.csv,*.html,*.json,*.jsonl,*.pdf,*.txt,*.ipynb"
+
+[tool.mypy]
+disallow_untyped_defs = true
+# Remove venv skip when integrated with pre-commit
+exclude = ["_static", "build", "examples", "notebooks", "venv"]
+ignore_missing_imports = true
+namespace_packages = true
+explicit_package_bases = true
+python_version = "3.10"
+
+[tool.poetry]
+authors = ["Your Name <you@example.com>"]
+description = "llama-index fastapi server"
+exclude = ["**/BUILD"]
+license = "MIT"
+name = "llama-index-server"
+packages = [{include = "llama_index/"}]
+readme = "README.md"
+version = "0.1.12"
+
+[tool.poetry.dependencies]
+python = ">=3.9,<4.0"
+fastapi = {extras = ["standard"], version = "^0.115.11"}
+cachetools = "^5.5.2"
+requests = "^2.32.3"
+pydantic-settings = "^2.8.1"
+llama-index-core = "^0.12.28"
+llama-index-readers-file = "^0.4.6"
+llama-index-indices-managed-llama-cloud = "0.6.3"
+
+[tool.poetry.group.dev.dependencies]
+black = {extras = ["jupyter"], version = "<=23.9.1,>=23.7.0"}
+codespell = {extras = ["toml"], version = ">=v2.2.6"}
+e2b-code-interpreter = "^1.1.1"
+ipython = "8.10.0"
+jupyter = "^1.0.0"
+markdown = "^3.7"
+mypy = "1.15.0"
+pre-commit = "3.2.0"
+pylint = "2.15.10"
+pytest = "^8.3.5"
+pytest-asyncio = "^0.25.3"
+pytest-mock = "3.11.1"
+ruff = "0.0.292"
+tree-sitter-languages = "^1.8.0"
+types-Deprecated = ">=0.1.0"
+types-PyYAML = "^6.0.12.12"
+types-protobuf = "^4.24.0.4"
+types-redis = "4.5.5.0"
+types-requests = "2.28.11.8"  # TODO: unpin when mypy>0.991
+types-setuptools = "67.1.0.0"
+xhtml2pdf = "^0.2.17"
+pytest-cov = "^6.0.0"
+llama-cloud = "^0.1.17"
+

llama-index-server/tests/api/test_chat_api.py

@@ -0,0 +1,149 @@
+import logging
+from unittest.mock import AsyncMock, MagicMock
+
+import pytest
+from fastapi import FastAPI
+from httpx import ASGITransport, AsyncClient
+
+from llama_index.core.workflow import StopEvent, Workflow
+from llama_index.core.workflow.handler import WorkflowHandler
+from llama_index.server.api.models import ChatAPIMessage, ChatRequest
+from llama_index.server.api.routers.chat import chat_router
+
+
+@pytest.fixture()
+def logger():
+    return logging.getLogger("test")
+
+
+@pytest.fixture()
+def chat_request():
+    """Create a simple chat request with one user message."""
+    return ChatRequest(
+        messages=[ChatAPIMessage(role="user", content="Hello, how are you?")]
+    )
+
+
+@pytest.fixture()
+def mock_workflow():
+    """Create a mock workflow that returns a simple response."""
+    workflow = MagicMock(spec=Workflow)
+    handler = AsyncMock(spec=WorkflowHandler)
+
+    # Setup the handler to stream a simple response event
+    async def mock_stream_events():
+        yield StopEvent(result="I'm doing well, thank you for asking!")
+
+    handler.stream_events.return_value = mock_stream_events()
+    workflow.run.return_value = handler
+
+    return workflow
+
+
+@pytest.fixture()
+def workflow_factory(mock_workflow):
+    """Create a factory function that returns our mock workflow."""
+
+    def factory(verbose=False):
+        return mock_workflow
+
+    return factory
+
+
+@pytest.mark.asyncio()
+async def test_chat_router(chat_request, workflow_factory, logger):
+    """Test that the chat router handles a request correctly."""
+    # Create a FastAPI app and mount our router
+    app = FastAPI()
+    router = chat_router(workflow_factory, logger)
+    app.include_router(router)
+
+    # Make a request to the chat endpoint
+    async with AsyncClient(
+        transport=ASGITransport(app=app), base_url="http://test"
+    ) as client:
+        response = await client.post("/chat", json=chat_request.model_dump())
+
+        # Check response status
+        assert response.status_code == 200
+
+        # For streaming responses we don't check the content-type header directly
+        # Instead, check that we get the expected content in the response body
+
+        # The response is a stream, so we need to collect the chunks
+        content = response.content.decode()
+
+        # Verify content structure follows expected format
+        assert "0:" in content  # Text prefix for VercelStreamResponse
+        # Verify if the response contains the expected message
+        assert "I'm doing well" in content
+
+        # Verify the mock workflow was called correctly
+        mock_workflow = workflow_factory()
+        mock_workflow.run.assert_called_once()
+
+        # Verify the workflow was called with the correct arguments
+        call_args = mock_workflow.run.call_args[1]
+        assert call_args["user_msg"] == "Hello, how are you?"
+        assert isinstance(call_args["chat_history"], list)
+        assert len(call_args["chat_history"]) == 0  # No history for first message
+
+
+@pytest.mark.asyncio()
+async def test_chat_with_agent_workflow(logger):
+    """Test that the chat router works with a workflow that mimics an agent workflow."""
+    # Create a simple workflow that mimics an agent workflow
+    mock_workflow = MagicMock(spec=Workflow)
+    handler = AsyncMock(spec=WorkflowHandler)
+
+    # Setup the handler to stream a simple response about weather
+    async def mock_stream_events():
+        yield StopEvent(
+            result="The weather in New York is sunny. I used the weather tool to get this information."
+        )
+
+    handler.stream_events.return_value = mock_stream_events()
+    mock_workflow.run.return_value = handler
+
+    # Create a factory function that returns our mock workflow
+    def workflow_factory(verbose=False):
+        return mock_workflow
+
+    # Create a FastAPI app and mount our router
+    app = FastAPI()
+    router = chat_router(workflow_factory, logger)
+    app.include_router(router)
+
+    # Create a chat request asking about weather
+    chat_request = ChatRequest(
+        messages=[
+            ChatAPIMessage(role="user", content="What's the weather in New York?")
+        ]
+    )
+
+    # Make a request to the chat endpoint
+    async with AsyncClient(
+        transport=ASGITransport(app=app), base_url="http://test"
+    ) as client:
+        response = await client.post("/chat", json=chat_request.model_dump())
+
+        # Check response status
+        assert response.status_code == 200
+
+        # The response is a stream, so we need to collect the chunks
+        content = response.content.decode()
+
+        # Verify content structure follows expected format
+        assert "0:" in content  # Text prefix for VercelStreamResponse
+
+        # Verify the response content contains expected keywords
+        assert "weather" in content and "New York" in content and "sunny" in content
+
+        # Verify the mock workflow was called correctly
+        mock_workflow.run.assert_called_once()
+
+        # Verify the workflow was called with the correct arguments
+        call_args = mock_workflow.run.call_args[1]
+        assert call_args["user_msg"] == "What's the weather in New York?"
+        assert isinstance(call_args["chat_history"], list)
+        assert len(call_args["chat_history"]) == 0  # No history for first message

llama-index-server/tests/api/test_event_stream.py

@@ -0,0 +1,249 @@
+import asyncio
+import logging
+from unittest.mock import AsyncMock, MagicMock
+
+import pytest
+
+from llama_index.core.agent.workflow.workflow_events import AgentStream
+from llama_index.core.workflow import StopEvent
+from llama_index.core.workflow.handler import WorkflowHandler
+from llama_index.server.api.models import ChatAPIMessage, ChatRequest
+from llama_index.server.api.routers.chat import _stream_content
+from llama_index.server.api.utils.vercel_stream import VercelStreamResponse
+
+
+@pytest.fixture()
+def logger():
+    return logging.getLogger("test")
+
+
+@pytest.fixture()
+def chat_request():
+    return ChatRequest(messages=[ChatAPIMessage(role="user", content="test message")])
+
+
+@pytest.fixture()
+def mock_workflow_handler():
+    handler = AsyncMock(spec=WorkflowHandler)
+    handler.accumulate_text = MagicMock()
+    return handler
+
+
+class TestEventStream:
+    @pytest.mark.asyncio()
+    async def test_stream_content_with_agent_stream(
+        self, mock_workflow_handler, chat_request, logger
+    ):
+        # Setup
+        mock_workflow_handler.stream_events.return_value = (
+            self._mock_agent_stream_events()
+        )
+
+        # Execute
+        result = [
+            chunk
+            async for chunk in _stream_content(
+                mock_workflow_handler, chat_request, logger
+            )
+        ]
+
+        # Assert
+        assert len(result) == 3  # Empty start + 2 text chunks
+        assert result[0] == VercelStreamResponse.convert_text("")
+        assert result[1] == VercelStreamResponse.convert_text("Hello")
+        assert result[2] == VercelStreamResponse.convert_text(" World")
+
+    @pytest.mark.asyncio()
+    async def test_stream_content_with_stop_event_string(
+        self, mock_workflow_handler, chat_request, logger
+    ):
+        # Setup
+        mock_workflow_handler.stream_events.return_value = (
+            self._mock_stop_event_string()
+        )
+
+        # Execute
+        result = [
+            chunk
+            async for chunk in _stream_content(
+                mock_workflow_handler, chat_request, logger
+            )
+        ]
+
+        # Assert
+        assert len(result) == 2  # Empty start + result string
+        assert result[0] == VercelStreamResponse.convert_text("")
+        assert result[1] == VercelStreamResponse.convert_text("Final answer")
+
+    @pytest.mark.asyncio()
+    async def test_stream_content_with_stop_event_delta_objects(
+        self, mock_workflow_handler, chat_request, logger
+    ):
+        # Setup
+        mock_workflow_handler.stream_events.return_value = (
+            self._mock_stop_event_delta_objects()
+        )
+
+        # Execute
+        result = [
+            chunk
+            async for chunk in _stream_content(
+                mock_workflow_handler, chat_request, logger
+            )
+        ]
+
+        # Assert
+        assert len(result) == 3  # Empty start + 2 delta chunks
+        assert result[0] == VercelStreamResponse.convert_text("")
+        assert result[1] == VercelStreamResponse.convert_text("Delta 1")
+        assert result[2] == VercelStreamResponse.convert_text("Delta 2")
+
+    @pytest.mark.asyncio()
+    async def test_stream_content_with_event_with_to_response(
+        self, mock_workflow_handler, chat_request, logger
+    ):
+        # Setup
+        mock_workflow_handler.stream_events.return_value = (
+            self._mock_event_with_to_response()
+        )
+
+        # Execute
+        result = [
+            chunk
+            async for chunk in _stream_content(
+                mock_workflow_handler, chat_request, logger
+            )
+        ]
+
+        # Assert
+        assert len(result) == 2  # Empty start + event with to_response
+        assert result[0] == VercelStreamResponse.convert_text("")
+        assert result[1] == VercelStreamResponse.convert_data({"event_type": "test"})
+
+    @pytest.mark.asyncio()
+    async def test_stream_content_with_event_with_model_dump(
+        self, mock_workflow_handler, chat_request, logger
+    ):
+        # Setup
+        mock_workflow_handler.stream_events.return_value = (
+            self._mock_event_with_model_dump()
+        )
+
+        # Execute
+        result = [
+            chunk
+            async for chunk in _stream_content(
+                mock_workflow_handler, chat_request, logger
+            )
+        ]
+
+        # Assert
+        assert len(result) == 2  # Empty start + event with model_dump
+        assert result[0] == VercelStreamResponse.convert_text("")
+        assert result[1] == VercelStreamResponse.convert_data(None)
+
+    @pytest.mark.asyncio()
+    async def test_stream_content_with_cancelled_error(
+        self, mock_workflow_handler, chat_request, logger
+    ):
+        # Setup
+        mock_workflow_handler.stream_events.side_effect = asyncio.CancelledError()
+        logger.warning = MagicMock()
+
+        # Execute
+        result = [
+            chunk
+            async for chunk in _stream_content(
+                mock_workflow_handler, chat_request, logger
+            )
+        ]
+
+        # Assert
+        assert len(result) == 0
+        mock_workflow_handler.cancel_run.assert_called_once()
+        logger.warning.assert_called_once()
+
+    @pytest.mark.asyncio()
+    async def test_stream_content_with_exception(
+        self, mock_workflow_handler, chat_request, logger
+    ):
+        # Setup
+        error_message = "Test error"
+        mock_workflow_handler.stream_events.side_effect = Exception(error_message)
+        logger.error = MagicMock()
+
+        # Execute
+        result = [
+            chunk
+            async for chunk in _stream_content(
+                mock_workflow_handler, chat_request, logger
+            )
+        ]
+
+        # Assert
+        assert len(result) == 1
+        assert result[0] == VercelStreamResponse.convert_error(error_message)
+        mock_workflow_handler.cancel_run.assert_called_once()
+        logger.error.assert_called_once()
+
+    async def _mock_agent_stream_events(self):
+        yield AgentStream(
+            delta="Hello", response="", current_agent_name="", tool_calls=[], raw=""
+        )
+        yield AgentStream(
+            delta=" World", response="", current_agent_name="", tool_calls=[], raw=""
+        )
+
+    async def _mock_agent_stream_with_empty_deltas(self):
+        yield AgentStream(
+            delta="   ",  # Empty delta with spaces - should be filtered
+            response="",
+            current_agent_name="",
+            tool_calls=[],
+            raw="",
+        )
+        yield AgentStream(
+            delta="Valid delta",
+            response="",
+            current_agent_name="",
+            tool_calls=[],
+            raw="",
+        )
+        yield AgentStream(
+            delta="\n",  # Newline-only delta - should be filtered
+            response="",
+            current_agent_name="",
+            tool_calls=[],
+            raw="",
+        )
+
+    async def _mock_stop_event_string(self):
+        yield StopEvent(result="Final answer")
+
+    async def _mock_stop_event_delta_objects(self):
+        async def generator():
+            # Create proper objects with delta attribute that can be serialized
+            class ObjectWithDelta:
+                def __init__(self, delta_value) -> None:
+                    self.delta = delta_value
+
+            yield ObjectWithDelta("Delta 1")
+            yield ObjectWithDelta("Delta 2")
+
+        yield StopEvent(result=generator())
+
+    async def _mock_dict_event(self):
+        yield {"key": "value"}
+
+    async def _mock_event_with_to_response(self):
+        event = MagicMock()
+        event.to_response.return_value = {"event_type": "test"}
+        yield event
+
+    async def _mock_event_with_model_dump(self):
+        event = MagicMock()
+        event.model_dump.return_value = {"name": "test_event"}
+        # Override to_response to return None - this means convert_data(None) will be called
+        event.to_response = MagicMock(return_value=None)
+        # The model_dump value is ignored when to_response returns None
+        yield event

llama-index-server/tests/services/test_file_service.py

@@ -0,0 +1,205 @@
+import os
+import uuid
+from unittest.mock import mock_open, patch
+
+import pytest
+from llama_index.server.services.file import FileService, _sanitize_file_name
+
+
+class TestFileService:
+    def test_sanitize_file_name(self):
+        # Test with normal alphanumeric name
+        assert _sanitize_file_name("test123") == "test123"
+
+        # Test with spaces
+        assert _sanitize_file_name("test file") == "test_file"
+
+        # Test with special characters
+        assert _sanitize_file_name("test@file!name") == "test_file_name"
+
+        # Test with path-like characters
+        assert _sanitize_file_name("test/file/name") == "test_file_name"
+
+        # Test with dots (should be preserved)
+        assert _sanitize_file_name("test.file.name") == "test.file.name"
+
+    @patch("uuid.uuid4")
+    @patch("os.path.getsize")
+    @patch("builtins.open", new_callable=mock_open)
+    @patch("os.makedirs")
+    def test_save_file_string_content(
+        self, mock_makedirs, mock_file_open, mock_getsize, mock_uuid
+    ):
+        # Setup
+        test_uuid = "12345678-1234-5678-1234-567812345678"
+        mock_uuid.return_value = uuid.UUID(test_uuid)
+        mock_getsize.return_value = 11  # Length of "Hello World"
+
+        # Execute
+        result = FileService.save_file(
+            content="Hello World", file_name="test.txt", save_dir="test_dir"
+        )
+
+        # Assert
+        expected_path = os.path.join("test_dir", f"test_{test_uuid}.txt")
+        mock_makedirs.assert_called_once_with(
+            os.path.dirname(expected_path), exist_ok=True
+        )
+        mock_file_open.assert_called_once_with(expected_path, "wb")
+        mock_file_open().write.assert_called_once_with(b"Hello World")
+
+        assert result.id == test_uuid
+        assert result.name == f"test_{test_uuid}.txt"
+        assert result.type == "txt"
+        assert result.size == 11
+        assert result.path == expected_path
+        assert result.url.endswith(expected_path.replace(os.path.sep, "/"))
+        assert result.refs is None
+
+    @patch("uuid.uuid4")
+    @patch("os.path.getsize")
+    @patch("builtins.open", new_callable=mock_open)
+    @patch("os.makedirs")
+    def test_save_file_bytes_content(
+        self, mock_makedirs, mock_file_open, mock_getsize, mock_uuid
+    ):
+        # Setup
+        test_uuid = "12345678-1234-5678-1234-567812345678"
+        mock_uuid.return_value = uuid.UUID(test_uuid)
+        mock_getsize.return_value = 11  # Length of "Hello World"
+
+        # Execute
+        result = FileService.save_file(
+            content=b"Hello World", file_name="test.txt", save_dir="test_dir"
+        )
+
+        # Assert
+        expected_path = os.path.join("test_dir", f"test_{test_uuid}.txt")
+        mock_makedirs.assert_called_once_with(
+            os.path.dirname(expected_path), exist_ok=True
+        )
+        mock_file_open.assert_called_once_with(expected_path, "wb")
+        mock_file_open().write.assert_called_once_with(b"Hello World")
+        assert result.path == expected_path
+        assert result.type == "txt"
+
+    @patch("uuid.uuid4")
+    @patch("os.path.getsize")
+    @patch("builtins.open", new_callable=mock_open)
+    @patch("os.makedirs")
+    def test_save_file_with_special_characters(
+        self, mock_makedirs, mock_file_open, mock_getsize, mock_uuid
+    ):
+        # Setup
+        test_uuid = "12345678-1234-5678-1234-567812345678"
+        mock_uuid.return_value = uuid.UUID(test_uuid)
+        mock_getsize.return_value = 11
+
+        # Execute
+        result = FileService.save_file(
+            content="Hello World", file_name="test@file!.txt", save_dir="test_dir"
+        )
+
+        # Assert
+        expected_path = os.path.join("test_dir", f"test_file__{test_uuid}.txt")
+        mock_makedirs.assert_called_once_with(
+            os.path.dirname(expected_path), exist_ok=True
+        )
+        mock_file_open.assert_called_once_with(expected_path, "wb")
+        assert result.path == expected_path
+        assert result.name == f"test_file__{test_uuid}.txt"
+
+    @patch("uuid.uuid4")
+    @patch("os.path.getsize")
+    @patch("builtins.open", new_callable=mock_open)
+    @patch("os.makedirs")
+    def test_save_file_default_directory(
+        self, mock_makedirs, mock_file_open, mock_getsize, mock_uuid
+    ):
+        # Setup
+        test_uuid = "12345678-1234-5678-1234-567812345678"
+        mock_uuid.return_value = uuid.UUID(test_uuid)
+        mock_getsize.return_value = 11
+
+        # Execute
+        result = FileService.save_file(content="Hello World", file_name="test.txt")
+
+        # Assert
+        expected_path = os.path.join("output", "uploaded", f"test_{test_uuid}.txt")
+        mock_makedirs.assert_called_once_with(
+            os.path.dirname(expected_path), exist_ok=True
+        )
+        assert result.path == expected_path
+
+    @patch("uuid.uuid4")
+    @patch("os.getenv")
+    @patch("os.path.getsize")
+    @patch("builtins.open", new_callable=mock_open)
+    @patch("os.makedirs")
+    def test_save_file_custom_url_prefix(
+        self, mock_makedirs, mock_file_open, mock_getsize, mock_getenv, mock_uuid
+    ):
+        # Setup
+        test_uuid = "12345678-1234-5678-1234-567812345678"
+        mock_uuid.return_value = uuid.UUID(test_uuid)
+        mock_getsize.return_value = 11
+        mock_getenv.return_value = "/api/files"
+
+        # Execute
+        result = FileService.save_file(
+            content="Hello World", file_name="test.txt", save_dir="test_dir"
+        )
+
+        # Assert
+        expected_path = os.path.join("test_dir", f"test_{test_uuid}.txt")
+        mock_makedirs.assert_called_once_with(
+            os.path.dirname(expected_path), exist_ok=True
+        )
+        mock_file_open.assert_called_once_with(expected_path, "wb")
+        assert result.path == expected_path
+        # URL paths must use forward slashes, even on Windows
+        expected_url = f"/api/files/test_dir/test_{test_uuid}.txt"
+        assert result.url == expected_url
+
+    def test_save_file_no_extension(self):
+        # Test that saving a file without extension raises ValueError
+        with pytest.raises(ValueError, match="File is not supported!"):
+            FileService.save_file(
+                content="Hello World", file_name="test", save_dir="test_dir"
+            )
+
+    @patch("uuid.uuid4")
+    @patch("os.path.getsize")
+    @patch("builtins.open")
+    @patch("os.makedirs")
+    def test_save_file_permission_error(
+        self, mock_makedirs, mock_file_open, mock_getsize, mock_uuid
+    ):
+        # Setup
+        test_uuid = "12345678-1234-5678-1234-567812345678"
+        mock_uuid.return_value = uuid.UUID(test_uuid)
+        mock_file_open.side_effect = PermissionError("Permission denied")
+
+        # Execute and Assert
+        with pytest.raises(PermissionError):
+            FileService.save_file(
+                content="Hello World", file_name="test.txt", save_dir="test_dir"
+            )
+
+    @patch("uuid.uuid4")
+    @patch("os.path.getsize")
+    @patch("builtins.open")
+    @patch("os.makedirs")
+    def test_save_file_io_error(
+        self, mock_makedirs, mock_file_open, mock_getsize, mock_uuid
+    ):
+        # Setup
+        test_uuid = "12345678-1234-5678-1234-567812345678"
+        mock_uuid.return_value = uuid.UUID(test_uuid)
+        mock_file_open.side_effect = OSError("IO Error")
+
+        # Execute and Assert
+        with pytest.raises(IOError):
+            FileService.save_file(
+                content="Hello World", file_name="test.txt", save_dir="test_dir"
+            )

llama-index-server/tests/test_llamaindex_server.py

@@ -0,0 +1,298 @@
+import json
+import os
+import shutil
+
+import pytest
+from httpx import ASGITransport, AsyncClient
+from llama_index.core.agent.workflow import AgentWorkflow
+from llama_index.core.llms import MockLLM
+from llama_index.server import LlamaIndexServer, UIConfig
+
+
+def fetch_weather(city: str) -> str:
+    """Fetch the weather for a given city."""
+    return f"The weather in {city} is sunny."
+
+
+def _agent_workflow() -> AgentWorkflow:
+    # Use MockLLM instead of default OpenAI
+    mock_llm = MockLLM()
+    return AgentWorkflow.from_tools_or_functions(
+        tools_or_functions=[fetch_weather],
+        verbose=True,
+        llm=mock_llm,
+    )
+
+
+@pytest.fixture()
+def server() -> LlamaIndexServer:
+    """Fixture to create a LlamaIndexServer instance."""
+    return LlamaIndexServer(
+        workflow_factory=_agent_workflow,
+        verbose=True,
+        use_default_routers=True,
+        mount_ui=False,
+        env="dev",
+    )
+
+
+@pytest.mark.asyncio()
+async def test_server_has_chat_route(server: LlamaIndexServer) -> None:
+    """Test that the server has the chat API route."""
+    chat_route_exists = any("/api/chat" in str(route) for route in server.routes)
+    assert chat_route_exists, "Chat API route not found in server routes"
+
+
+@pytest.mark.asyncio()
+async def test_server_swagger_docs(server: LlamaIndexServer) -> None:
+    """Test that the server serves Swagger UI docs."""
+    async with AsyncClient(
+        transport=ASGITransport(app=server), base_url="http://test"
+    ) as ac:
+        response = await ac.get("/docs")
+        assert response.status_code == 200
+        assert "text/html" in response.headers["content-type"]
+        assert "Swagger UI" in response.text
+
+
+@pytest.mark.asyncio()
+async def test_ui_is_downloaded(server: LlamaIndexServer) -> None:
+    """
+    Test if the UI is downloaded and mounted correctly.
+    """
+    # Clean up any existing static directory first
+    if os.path.exists(".ui"):
+        shutil.rmtree(".ui")
+
+    # Create a new server with UI enabled
+    ui_config = UIConfig(
+        enabled=True,
+        app_title="Test UI",
+        starter_questions=["What's the weather like?"],
+    )
+    ui_server = LlamaIndexServer(
+        workflow_factory=_agent_workflow,
+        verbose=True,
+        use_default_routers=True,
+        env="dev",
+        ui_config=ui_config,
+    )
+
+    # Verify that static directory was created with index.html
+    assert os.path.exists("./.ui"), "Static directory was not created"
+    assert os.path.isdir("./.ui"), "Static path is not a directory"
+    assert os.path.exists("./.ui/index.html"), "index.html was not downloaded"
+
+    # Check if the config.js was created with correct content
+    config_path = os.path.join(".ui", "config.js")
+    assert os.path.exists(config_path), "config.js was not created"
+
+    with open(config_path, "r") as f:
+        config_content = f.read()
+        assert "window.LLAMAINDEX =" in config_content
+        config_json = json.loads(
+            config_content.replace("window.LLAMAINDEX = ", "").rstrip(";")
+        )
+        assert config_json["CHAT_API"] == "/api/chat"
+        assert config_json["STARTER_QUESTIONS"] == ["What's the weather like?"]
+        assert config_json["LLAMA_CLOUD_API"] is None
+        assert config_json["APP_TITLE"] == "Test UI"
+
+    # Check if the UI is mounted and accessible
+    async with AsyncClient(
+        transport=ASGITransport(app=ui_server), base_url="http://test"
+    ) as ac:
+        response = await ac.get("/")
+        assert response.status_code == 200
+        assert "text/html" in response.headers["content-type"]
+
+    # Clean up after test
+    shutil.rmtree("./.ui")
+
+
+@pytest.mark.asyncio()
+async def test_ui_is_accessible(server: LlamaIndexServer) -> None:
+    """
+    Test if the UI is accessible.
+    """
+    # Manually trigger UI mounting
+    server.mount_ui()
+
+    async with AsyncClient(
+        transport=ASGITransport(app=server), base_url="http://test"
+    ) as ac:
+        response = await ac.get("/")
+        assert response.status_code == 200
+        assert "text/html" in response.headers["content-type"]
+
+
+@pytest.mark.asyncio()
+async def test_ui_config_customization() -> None:
+    """
+    Test if UI configuration can be customized.
+    """
+    custom_config = UIConfig(
+        enabled=True,
+        app_title="Custom App",
+        starter_questions=["Question 1", "Question 2"],
+        ui_path=".custom_ui",
+    )
+
+    server = LlamaIndexServer(
+        workflow_factory=_agent_workflow, verbose=True, ui_config=custom_config
+    )
+
+    assert server.ui_config.app_title == "Custom App"
+    assert server.ui_config.starter_questions == ["Question 1", "Question 2"]
+    assert server.ui_config.ui_path == ".custom_ui"
+
+    # Clean up if directory was created
+    if os.path.exists(".custom_ui"):
+        shutil.rmtree(".custom_ui")
+
+
+@pytest.mark.asyncio()
+async def test_ui_config_from_dict() -> None:
+    """
+    Test if UI configuration can be initialized from a dictionary.
+    """
+    ui_config_dict = {
+        "enabled": True,
+        "app_title": "Dict Config App",
+        "starter_questions": ["Dict Q1", "Dict Q2"],
+        "ui_path": ".dict_ui",
+    }
+
+    server = LlamaIndexServer(
+        workflow_factory=_agent_workflow,
+        verbose=True,
+        ui_config=ui_config_dict,
+    )
+
+    # Verify the config was properly converted to UIConfig object
+    assert isinstance(server.ui_config, UIConfig)
+    assert server.ui_config.app_title == "Dict Config App"
+    assert server.ui_config.starter_questions == ["Dict Q1", "Dict Q2"]
+    assert server.ui_config.ui_path == ".dict_ui"
+
+    # Verify the config.js is created with correct content
+    server.mount_ui()
+    config_path = os.path.join(".dict_ui", "config.js")
+    assert os.path.exists(config_path), "config.js was not created"
+
+    with open(config_path, "r") as f:
+        config_content = f.read()
+        assert "window.LLAMAINDEX =" in config_content
+        config_json = json.loads(
+            config_content.replace("window.LLAMAINDEX = ", "").rstrip(";")
+        )
+        assert config_json["APP_TITLE"] == "Dict Config App"
+        assert config_json["STARTER_QUESTIONS"] == ["Dict Q1", "Dict Q2"]
+        assert config_json["CHAT_API"] == "/api/chat"
+        assert config_json["LLAMA_CLOUD_API"] is None
+
+    # Clean up
+    if os.path.exists(".dict_ui"):
+        shutil.rmtree(".dict_ui")
+
+
+async def test_component_dir_creation(server: LlamaIndexServer) -> None:
+    """
+    Test if the component directory is created when specified and doesn't exist.
+    """
+    import os
+    import shutil
+
+    test_component_dir = "./test_components"
+
+    # Clean up any existing directory
+    if os.path.exists(test_component_dir):
+        shutil.rmtree(test_component_dir)
+
+    # Create server with component directory
+    _ = LlamaIndexServer(
+        workflow_factory=_agent_workflow,
+        verbose=True,
+        ui_config={
+            "component_dir": test_component_dir,
+            "include_ui": True,
+        },
+    )
+
+    # Verify directory was created
+    assert os.path.exists(test_component_dir), "Component directory was not created"
+    assert os.path.isdir(test_component_dir), "Component path is not a directory"
+
+    # Clean up after test
+    shutil.rmtree(test_component_dir)
+
+
+@pytest.mark.asyncio()
+async def test_component_router_addition(server: LlamaIndexServer, tmp_path) -> None:
+    """
+    Test if the component router is added when component directory is specified.
+    """
+    test_component_dir = tmp_path / "test_components"
+
+    # Create server with component directory
+    component_server = LlamaIndexServer(
+        workflow_factory=_agent_workflow,
+        verbose=True,
+        ui_config={
+            "component_dir": str(test_component_dir),
+            "include_ui": True,
+        },
+    )
+
+    # Verify component route exists
+    component_route_exists = any(
+        route.path == "/api/components" for route in component_server.routes
+    )
+    assert component_route_exists, "Component API route not found in server routes"
+
+
+@pytest.mark.asyncio()
+async def test_ui_config_includes_components_api(
+    server: LlamaIndexServer, tmp_path
+) -> None:
+    """
+    Test if the UI config includes components API when component directory is set.
+    """
+    test_component_dir = tmp_path / "test_components"
+
+    # Create server with component directory
+    component_server = LlamaIndexServer(
+        workflow_factory=_agent_workflow,
+        verbose=True,
+        ui_config={
+            "component_dir": str(test_component_dir),
+            "include_ui": True,
+        },
+    )
+
+    # Check if components API is in UI config
+    ui_config = component_server.ui_config
+    assert "COMPONENTS_API" in ui_config.get_config_content(), (
+        "Components API not found in UI config"
+    )
+
+
+@pytest.mark.asyncio()
+async def test_component_router_requires_component_dir(
+    server: LlamaIndexServer,
+) -> None:
+    """
+    Test that adding components router without component_dir raises an error.
+    """
+    server_without_component_dir = LlamaIndexServer(
+        workflow_factory=_agent_workflow,
+        verbose=True,
+        ui_config={
+            "include_ui": True,
+        },
+    )
+
+    with pytest.raises(
+        ValueError, match="component_dir must be specified to add components router"
+    ):
+        server_without_component_dir.add_components_router()

llama-index-server/tests/tools/test_document_generator.py

@@ -0,0 +1,89 @@
+from io import BytesIO
+from unittest.mock import MagicMock, patch
+
+import pytest
+from llama_index.server.tools.document_generator import (
+    OUTPUT_DIR,
+    DocumentGenerator,
+)
+
+
+class TestDocumentGenerator:
+    def test_validate_file_name(self) -> None:
+        # Valid names
+        assert (
+            DocumentGenerator("/api/files")._validate_file_name("valid-name")
+            == "valid-name"
+        )
+
+        # Invalid names
+        with pytest.raises(ValueError):
+            DocumentGenerator("/api/files")._validate_file_name("/invalid/path")
+
+    @patch("os.makedirs")
+    @patch("builtins.open")
+    def test_write_to_file(self, mock_open, mock_makedirs):  # type: ignore
+        content = BytesIO(b"test")
+        DocumentGenerator("/api/files")._write_to_file(content, "path/file.txt")
+
+        mock_makedirs.assert_called_once()
+        mock_open.assert_called_once()
+        mock_open.return_value.__enter__.return_value.write.assert_called_once_with(
+            b"test"
+        )
+
+    @patch("markdown.markdown")
+    def test_html_generation(self, mock_markdown):  # type: ignore
+        mock_markdown.return_value = "<h1>Test</h1>"
+
+        # Test HTML content generation
+        assert (
+            DocumentGenerator("/api/files")._generate_html_content("# Test")
+            == "<h1>Test</h1>"
+        )
+
+        # Test full HTML generation
+        html = DocumentGenerator("/api/files")._generate_html("<h1>Test</h1>")
+        assert "<!DOCTYPE html>" in html
+        assert "<h1>Test</h1>" in html
+
+    @patch("xhtml2pdf.pisa.pisaDocument")
+    def test_pdf_generation(self, mock_pisa):  # type: ignore
+        # Success case
+        mock_pisa.return_value = MagicMock(err=None)
+        assert isinstance(
+            DocumentGenerator("/api/files")._generate_pdf("test"), BytesIO
+        )
+
+        # Error case
+        mock_pisa.return_value = MagicMock(err="Error")
+        with pytest.raises(ValueError):
+            DocumentGenerator("/api/files")._generate_pdf("test")
+
+    @patch.multiple(
+        DocumentGenerator,
+        _generate_html_content=MagicMock(return_value="<h1>Test</h1>"),
+        _generate_html=MagicMock(
+            return_value="<html><body><h1>Test</h1></body></html>"
+        ),
+        _generate_pdf=MagicMock(return_value=BytesIO(b"pdf")),
+        _write_to_file=MagicMock(),
+    )
+    def test_generate_document(self):  # type: ignore
+        # HTML generation
+        url = DocumentGenerator("/api/files").generate_document(
+            "# Test", "html", "test-doc"
+        )
+        assert url == f"/api/files/{OUTPUT_DIR}/test-doc.html"
+
+        # PDF generation
+        url = DocumentGenerator("/api/files").generate_document(
+            "# Test", "pdf", "test-doc"
+        )
+        assert url == f"/api/files/{OUTPUT_DIR}/test-doc.pdf"
+
+        # Invalid type
+        with pytest.raises(ValueError):
+            DocumentGenerator("/api/files").generate_document(
+                "# Test", "invalid", "test-doc"
+            )

llama-index-server/tests/tools/test_interpreter.py

@@ -0,0 +1,65 @@
+from unittest.mock import MagicMock
+
+import pytest
+from e2b_code_interpreter.models import Execution, Logs
+from llama_index.server.tools.interpreter import E2BCodeInterpreter
+
+
+class TestE2BCodeInterpreter:
+    @pytest.fixture()
+    def sandbox(self):  # type: ignore
+        """Create a mock Sandbox with no API key requirement."""
+        mock_sandbox = MagicMock()
+        mock_sandbox.files = MagicMock()
+        mock_sandbox.files.write = MagicMock()
+        mock_sandbox.run_code = MagicMock()
+        return mock_sandbox
+
+    @pytest.fixture()
+    def code_interpreter(self, sandbox):  # type: ignore
+        """Create E2BCodeInterpreter that uses the mock Sandbox."""
+        interpreter = E2BCodeInterpreter(api_key="dummy_key")
+        interpreter.interpreter = sandbox
+        return interpreter
+
+    def test_interpret_success(self, code_interpreter, sandbox) -> None:  # type: ignore
+        """Test successful code execution."""
+        # Mock execution result
+        mock_execution = Execution()
+        mock_execution.error = None
+        mock_execution.results = []
+        mock_execution.logs = Logs(
+            stdout="stdout", stderr="", display_data="", error=""
+        )
+        sandbox.run_code.return_value = mock_execution
+
+        # Run the code
+        result = code_interpreter.interpret("print('hello')")
+
+        # Verify
+        sandbox.run_code.assert_called_once_with("print('hello')")
+        assert result.is_error is False
+        assert result.logs == mock_execution.logs
+
+    def test_interpret_error(self, code_interpreter, sandbox) -> None:  # type: ignore
+        """Test error in code execution."""
+        # Mock execution result with error
+        mock_execution = Execution()
+        mock_execution.error = "Test error"
+        mock_execution.logs = Logs(
+            stdout="", stderr="error", display_data="", error="Test error"
+        )
+        sandbox.run_code.return_value = mock_execution
+
+        # Run the code
+        result = code_interpreter.interpret("bad code")
+
+        # Verify
+        assert result.is_error is True
+        assert "Error: Test error" in result.error_message
+        sandbox.kill.assert_called_once()
+
+    def test_to_tool(self, code_interpreter) -> None:  # type: ignore
+        """Test tool conversion."""
+        tool = code_interpreter.to_tool()
+        assert tool.fn == code_interpreter.interpret

package.json

@@ -1,6 +1,6 @@
 {
   "name": "create-llama",
-  "version": "0.4.0",
+  "version": "0.5.5",
   "description": "Create LlamaIndex-powered apps with one command",
   "keywords": [
     "rag",

questions/index.ts

@@ -16,5 +16,6 @@ export const askQuestions = async (
     await askProQuestions(args);
     return args as unknown as QuestionResults;
   }
-  return await askSimpleQuestions(args);
+  const results = await askSimpleQuestions(args);
+  return results;
 };

questions/simple.ts

@@ -1,25 +1,12 @@
 import prompts from "prompts";
-import {
-  AI_REPORTS,
-  EXAMPLE_10K_SEC_FILES,
-  EXAMPLE_FILE,
-  EXAMPLE_GDPR,
-} from "../helpers/datasources";
+import { EXAMPLE_10K_SEC_FILES, EXAMPLE_FILE } from "../helpers/datasources";
 import { askModelConfig } from "../helpers/providers";
 import { getTools } from "../helpers/tools";
 import { ModelConfig, TemplateFramework } from "../helpers/types";
 import { PureQuestionArgs, QuestionResults } from "./types";
 import { askPostInstallAction, questionHandlers } from "./utils";
 
-type AppType =
-  | "rag"
-  | "code_artifact"
-  | "financial_report_agent"
-  | "form_filling"
-  | "extractor"
-  | "contract_review"
-  | "data_scientist"
-  | "deep_research";
+type AppType = "agentic_rag" | "financial_report" | "deep_research";
 
 type SimpleAnswers = {
   appType: AppType;
@@ -35,53 +22,22 @@ export const askSimpleQuestions = async (
     {
       type: "select",
       name: "appType",
-      message: "What app do you want to build?",
-      hint: "🤖: Agent, 🔀: Workflow",
+      message: "What use case do you want to build?",
       choices: [
         {
-          title: "🤖 Agentic RAG",
-          value: "rag",
+          title: "Agentic RAG",
+          value: "agentic_rag",
           description:
             "Chatbot that answers questions based on provided documents.",
         },
         {
-          title: "🤖 Data Scientist",
-          value: "data_scientist",
+          title: "Financial Report",
+          value: "financial_report",
           description:
             "Agent that analyzes data and generates visualizations by using a code interpreter.",
         },
         {
-          title: "🤖 Code Artifact Agent",
-          value: "code_artifact",
-          description:
-            "Agent that writes code, runs it in a sandbox, and shows the output in the chat UI.",
-        },
-        {
-          title: "🤖 Information Extractor",
-          value: "extractor",
-          description:
-            "Extracts information from documents and returns it as a structured JSON object.",
-        },
-        {
-          title: "🔀 Financial Report Generator",
-          value: "financial_report_agent",
-          description:
-            "Generates a financial report by analyzing the provided 10-K SEC data. Uses a code interpreter to create charts or to conduct further analysis.",
-        },
-        {
-          title: "🔀 Financial 10k SEC Form Filler",
-          value: "form_filling",
-          description:
-            "Extracts information from 10k SEC data and uses it to fill out a CSV form.",
-        },
-        {
-          title: "🔀 Contract Reviewer",
-          value: "contract_review",
-          description:
-            "Extracts and reviews contracts to ensure compliance with GDPR regulations",
-        },
-        {
-          title: "🔀 Deep Researcher",
+          title: "Deep Research",
           value: "deep_research",
           description:
             "Researches and analyzes provided documents from multiple perspectives, generating a comprehensive report with citations to support key findings and insights.",
@@ -93,13 +49,10 @@ export const askSimpleQuestions = async (
 
   let language: TemplateFramework = "fastapi";
   let llamaCloudKey = args.llamaCloudKey;
+
   let useLlamaCloud = false;
 
-  if (
-    appType !== "extractor" &&
-    appType !== "contract_review" &&
-    appType !== "deep_research"
-  ) {
+  if (appType !== "extractor" && appType !== "contract_review") {
     const { language: newLanguage } = await prompts(
       {
         type: "select",
@@ -170,80 +123,36 @@ const convertAnswers = async (
   };
   const lookup: Record<
     AppType,
-    Pick<
-      QuestionResults,
-      "template" | "tools" | "frontend" | "dataSources" | "useCase"
-    > & {
+    Pick<QuestionResults, "template" | "tools" | "dataSources" | "useCase"> & {
       modelConfig?: ModelConfig;
     }
   > = {
-    rag: {
-      template: "streaming",
-      tools: getTools(["weather"]),
-      frontend: true,
+    agentic_rag: {
+      template: "llamaindexserver",
       dataSources: [EXAMPLE_FILE],
     },
-    data_scientist: {
-      template: "streaming",
+    financial_report: {
+      template: "llamaindexserver",
+      dataSources: EXAMPLE_10K_SEC_FILES,
       tools: getTools(["interpreter", "document_generator"]),
-      frontend: true,
-      dataSources: [],
       modelConfig: MODEL_GPT4o,
     },
-    code_artifact: {
-      template: "streaming",
-      tools: getTools(["artifact"]),
-      frontend: true,
-      dataSources: [],
-      modelConfig: MODEL_GPT4o,
-    },
-    financial_report_agent: {
-      template: "multiagent",
-      useCase: "financial_report",
-      tools: getTools(["document_generator", "interpreter"]),
-      dataSources: EXAMPLE_10K_SEC_FILES,
-      frontend: true,
-      modelConfig: MODEL_GPT4o,
-    },
-    form_filling: {
-      template: "multiagent",
-      useCase: "form_filling",
-      tools: getTools(["form_filling"]),
-      dataSources: EXAMPLE_10K_SEC_FILES,
-      frontend: true,
-      modelConfig: MODEL_GPT4o,
-    },
-    extractor: {
-      template: "reflex",
-      useCase: "extractor",
-      tools: [],
-      frontend: false,
-      dataSources: [EXAMPLE_FILE],
-    },
-    contract_review: {
-      template: "reflex",
-      useCase: "contract_review",
-      tools: [],
-      frontend: false,
-      dataSources: [EXAMPLE_GDPR],
-    },
     deep_research: {
-      template: "multiagent",
-      useCase: "deep_research",
+      template: "llamaindexserver",
+      dataSources: EXAMPLE_10K_SEC_FILES,
       tools: [],
-      frontend: true,
-      dataSources: [AI_REPORTS],
+      modelConfig: MODEL_GPT4o,
     },
   };
+
   const results = lookup[answers.appType];
   return {
     framework: answers.language,
+    useCase: answers.appType,
     ui: "shadcn",
     llamaCloudKey: answers.llamaCloudKey,
     useLlamaParse: answers.useLlamaCloud,
-    llamapack: "",
     vectorDb: answers.useLlamaCloud ? "llamacloud" : "none",
-    observability: "none",
     ...results,
     modelConfig:
       results.modelConfig ??
@@ -252,6 +161,6 @@ const convertAnswers = async (
         askModels: args.askModels ?? false,
         framework: answers.language,
       })),
-    frontend: answers.language === "nextjs" ? false : results.frontend,
+    frontend: true,
   };
 };

templates/components/ui/workflows/deep_research/ui_event.jsx

@@ -0,0 +1,409 @@
+export default function DeepResearchComponent({ events }) {
+  // Aggregate events by type and track their state progression
+  const aggregateEvents = () => {
+    const retrieveEvents = events.filter((e) => e.event === "retrieve");
+    const analyzeEvents = events.filter((e) => e.event === "analyze");
+    const answerEvents = events.filter((e) => e.event === "answer");
+
+    // Get the latest state for retrieve and analyze events
+    const retrieveState =
+      retrieveEvents.length > 0
+        ? retrieveEvents[retrieveEvents.length - 1].state
+        : null;
+    const analyzeState =
+      analyzeEvents.length > 0
+        ? analyzeEvents[analyzeEvents.length - 1].state
+        : null;
+
+    // Group answer events by their ID
+    const answerGroups = {};
+    for (const event of answerEvents) {
+      if (!event.id) continue;
+
+      if (!answerGroups[event.id]) {
+        answerGroups[event.id] = {
+          id: event.id,
+          question: event.question,
+          answer: event.answer,
+          states: [],
+        };
+      }
+
+      const lastState =
+        answerGroups[event.id].states[answerGroups[event.id].states.length - 1];
+      if (lastState !== event.state) {
+        answerGroups[event.id].states.push(event.state);
+      }
+
+      if (event.answer) {
+        answerGroups[event.id].answer = event.answer;
+      }
+    }
+
+    return {
+      retrieveState,
+      analyzeState,
+      answerGroups: Object.values(answerGroups),
+    };
+  };
+
+  const { retrieveState, analyzeState, answerGroups } = aggregateEvents();
+
+  // Styles
+  const styles = {
+    container: {
+      maxWidth: "900px",
+      margin: "0 auto",
+      padding: "20px",
+      backgroundColor: "#FFFFFF",
+      borderRadius: "8px",
+      boxShadow: "0 2px 8px rgba(0, 0, 0, 0.15)",
+    },
+    header: {
+      fontSize: "24px",
+      fontWeight: "bold",
+      textAlign: "center",
+      marginBottom: "20px",
+      color: "#333",
+    },
+    cardsContainer: {
+      display: "flex",
+      gap: "16px",
+      marginBottom: "30px",
+      flexWrap: "wrap",
+    },
+    card: {
+      flex: "1",
+      minWidth: "250px",
+      backgroundColor: "#FFFFFF",
+      borderRadius: "8px",
+      boxShadow: "0 2px 4px rgba(0, 0, 0, 0.1)",
+      padding: "16px",
+      border: "1px solid #E5E7EB",
+      transition: "all 0.3s ease",
+    },
+    cardHeader: {
+      display: "flex",
+      alignItems: "center",
+      marginBottom: "12px",
+    },
+    cardTitle: {
+      fontSize: "18px",
+      fontWeight: "bold",
+      marginLeft: "8px",
+    },
+    cardContent: {
+      display: "flex",
+      justifyContent: "space-between",
+      alignItems: "center",
+    },
+    badge: {
+      padding: "4px 8px",
+      borderRadius: "9999px",
+      fontSize: "12px",
+      fontWeight: "bold",
+      flexShrink: 0,
+    },
+    questionsList: {
+      marginTop: "30px",
+    },
+    questionItem: {
+      backgroundColor: "#F9FAFB",
+      border: "1px solid #E5E7EB",
+      borderRadius: "8px",
+      marginBottom: "12px",
+      overflow: "hidden",
+    },
+    questionHeader: {
+      padding: "12px 16px",
+      cursor: "pointer",
+      display: "flex",
+      justifyContent: "space-between",
+      alignItems: "flex-start",
+      userSelect: "none",
+    },
+    questionTitle: {
+      fontWeight: "medium",
+      marginLeft: "12px",
+    },
+    answerContainer: {
+      padding: "0",
+      backgroundColor: "#F3F4F6",
+      overflow: "hidden",
+      maxHeight: "0",
+      transition: "max-height 0.3s ease-out",
+      whiteSpace: "pre-line",
+      fontSize: "13px",
+    },
+    answerContent: {
+      padding: "16px",
+    },
+    answerContainerExpanded: {
+      maxHeight: "1000px", // Adjust based on content
+    },
+    arrow: {
+      marginLeft: "8px",
+      transition: "transform 0.3s ease",
+      display: "inline-block",
+    },
+    loadingContainer: {
+      display: "flex",
+      justifyContent: "center",
+      alignItems: "center",
+      padding: "24px",
+      color: "#6B7280",
+    },
+    stateIconContainer: {
+      display: "flex",
+      alignItems: "center",
+      gap: "8px",
+    },
+  };
+
+  // Helper function to get color for a state
+  const getStateColor = (state) => {
+    switch (state) {
+      case "inprogress":
+        return "#FCD34D";
+      case "done":
+        return "#34D399";
+      case "pending":
+        return "#9CA3AF";
+      default:
+        return "#D1D5DB";
+    }
+  };
+
+  // Helper function to get badge styles based on state
+  const getBadgeStyles = (state) => {
+    const colors = {
+      inprogress: { background: "#FEF3C7", color: "#92400E" },
+      done: { background: "#D1FAE5", color: "#065F46" },
+      pending: { background: "#F3F4F6", color: "#1F2937" },
+    };
+    const stateColors = colors[state] || colors.pending;
+    return {
+      ...styles.badge,
+      backgroundColor: stateColors.background,
+      color: stateColors.color,
+    };
+  };
+
+  // Helper function to render state icon
+  const renderStateIcon = (state) => {
+    const color = getStateColor(state);
+    if (state === "inprogress") {
+      return (
+        <div style={{ color, animation: "spin 1s linear infinite" }}>⟳</div>
+      );
+    } else if (state === "done") {
+      return <div style={{ color }}>✓</div>;
+    } else if (state === "pending") {
+      return <div style={{ color }}>⏱</div>;
+    }
+    return <div style={{ color }}>?</div>;
+  };
+
+  // State for toggling question answers
+  const [expandedQuestions, setExpandedQuestions] = React.useState({});
+
+  const toggleQuestion = (questionId) => {
+    setExpandedQuestions((prev) => ({
+      ...prev,
+      [questionId]: !prev[questionId],
+    }));
+  };
+
+  return (
+    <div style={styles.container}>
+      <h1 style={styles.header}>Research Progress</h1>
+
+      {/* Status Cards */}
+      <div style={styles.cardsContainer}>
+        {/* Retrieve Card */}
+        <div
+          style={{
+            ...styles.card,
+            borderColor:
+              retrieveState === "done"
+                ? "#A7F3D0"
+                : retrieveState === "inprogress"
+                  ? "#FDE68A"
+                  : "#E5E7EB",
+          }}
+        >
+          <div style={styles.cardHeader}>
+            <span style={{ color: "#8B5CF6" }}>🔍</span>
+            <div style={styles.cardTitle}>Data Retrieval</div>
+          </div>
+          <div style={styles.cardContent}>
+            <span style={{ fontSize: "14px", color: "#6B7280" }}>Status:</span>
+            <div style={styles.stateIconContainer}>
+              {retrieveState && (
+                <span style={getBadgeStyles(retrieveState)}>
+                  {retrieveState === "inprogress"
+                    ? "In Progress"
+                    : retrieveState === "done"
+                      ? "Completed"
+                      : "Pending"}
+                </span>
+              )}
+              {renderStateIcon(retrieveState)}
+            </div>
+          </div>
+        </div>
+
+        {/* Analyze Card */}
+        <div
+          style={{
+            ...styles.card,
+            borderColor:
+              analyzeState === "done"
+                ? "#A7F3D0"
+                : analyzeState === "inprogress"
+                  ? "#FDE68A"
+                  : "#E5E7EB",
+          }}
+        >
+          <div style={styles.cardHeader}>
+            <span style={{ color: "#06B6D4" }}>📊</span>
+            <div style={styles.cardTitle}>Data Analysis</div>
+          </div>
+          <div style={styles.cardContent}>
+            <span style={{ fontSize: "14px", color: "#6B7280" }}>Status:</span>
+            <div style={styles.stateIconContainer}>
+              {analyzeState && (
+                <span style={getBadgeStyles(analyzeState)}>
+                  {analyzeState === "inprogress"
+                    ? "In Progress"
+                    : analyzeState === "done"
+                      ? "Completed"
+                      : "Pending"}
+                </span>
+              )}
+              {renderStateIcon(analyzeState)}
+            </div>
+          </div>
+        </div>
+
+        {/* Questions Card */}
+        <div style={styles.card}>
+          <div style={styles.cardHeader}>
+            <span style={{ color: "#10B981" }}>💬</span>
+            <div style={styles.cardTitle}>Questions</div>
+          </div>
+          <div style={styles.cardContent}>
+            <span style={{ fontSize: "14px", color: "#6B7280" }}>Status:</span>
+            <div style={{ display: "flex", alignItems: "center", gap: "8px" }}>
+              <span
+                style={{
+                  ...styles.badge,
+                  backgroundColor: "#F3F4F6",
+                  color: "#1F2937",
+                }}
+              >
+                {answerGroups.length} Questions
+              </span>
+              <span
+                style={{
+                  fontSize: "14px",
+                  fontWeight: "500",
+                  color: "#059669",
+                }}
+              >
+                {answerGroups.filter((g) => g.states.includes("done")).length}{" "}
+                Answered
+              </span>
+            </div>
+          </div>
+        </div>
+      </div>
+
+      {/* Questions List */}
+      {answerGroups.length > 0 && (
+        <div style={styles.questionsList}>
+          <h2
+            style={{
+              fontSize: "20px",
+              fontWeight: "600",
+              marginBottom: "16px",
+            }}
+          >
+            Research Questions
+          </h2>
+          {answerGroups.map((group, index) => {
+            const latestState = group.states[group.states.length - 1];
+            const questionId = group.id || `question-${index}`;
+            const isExpanded = expandedQuestions[questionId];
+
+            const answerWithoutCitation = group.answer?.replace(
+              /\[citation:[a-f0-9-]+\]/g,
+              "",
+            );
+
+            return (
+              <div key={questionId} style={styles.questionItem}>
+                <div
+                  style={styles.questionHeader}
+                  onClick={() => toggleQuestion(questionId)}
+                >
+                  <div
+                    style={{
+                      display: "flex",
+                      alignItems: "flex-start",
+                      gap: "12px",
+                      fontSize: "14px",
+                    }}
+                  >
+                    {renderStateIcon(latestState)}
+                    <span style={styles.questionTitle}>{group.question}</span>
+                    <span
+                      style={{
+                        ...styles.arrow,
+                        transform: isExpanded
+                          ? "rotate(180deg)"
+                          : "rotate(0deg)",
+                      }}
+                    >
+                      ▼
+                    </span>
+                  </div>
+                  <span style={getBadgeStyles(latestState)}>
+                    {latestState === "inprogress"
+                      ? "In Progress"
+                      : latestState === "done"
+                        ? "Answered"
+                        : "Pending"}
+                  </span>
+                </div>
+                <div
+                  style={{
+                    ...styles.answerContainer,
+                    ...(isExpanded ? styles.answerContainerExpanded : {}),
+                  }}
+                >
+                  {answerWithoutCitation ? (
+                    <div style={styles.answerContent}>
+                      {answerWithoutCitation}
+                    </div>
+                  ) : (
+                    <div
+                      style={{
+                        ...styles.loadingContainer,
+                        ...styles.answerContent,
+                      }}
+                    >
+                      <span style={{ marginLeft: "8px" }}>
+                        Generating answer...
+                      </span>
+                    </div>
+                  )}
+                </div>
+              </div>
+            );
+          })}
+        </div>
+      )}
+    </div>
+  );
+}

templates/components/vectordbs/llamaindexserver/llamacloud/python/generate.py

@@ -0,0 +1,31 @@
+# flake8: noqa: E402
+
+from dotenv import load_dotenv
+
+load_dotenv()
+
+import logging
+
+from app.index import get_index
+from app.settings import init_settings
+from llama_index.server.services.llamacloud.generate import (
+    load_to_llamacloud,
+)
+
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger()
+
+
+def generate_datasource():
+    init_settings()
+    logger.info("Generate index for the provided data")
+
+    index = get_index(create_if_missing=True)
+    if index is None:
+        raise ValueError("Index not found and could not be created")
+
+    load_to_llamacloud(index, logger=logger)
+
+
+if __name__ == "__main__":
+    generate_datasource()

templates/components/vectordbs/llamaindexserver/llamacloud/python/index.py

@@ -0,0 +1,7 @@
+from llama_index.server.services.llamacloud import (
+    LlamaCloudIndex,
+    get_client,
+    get_index,
+)
+
+__all__ = ["LlamaCloudIndex", "get_client", "get_index"]

templates/components/vectordbs/llamaindexserver/llamacloud/typescript/generate.ts

@@ -0,0 +1,100 @@
+import * as dotenv from "dotenv";
+import "dotenv/config";
+import * as fs from "fs/promises";
+import { LLamaCloudFileService } from "llamaindex";
+import * as path from "path";
+import { getIndex } from "./app/data";
+import { initSettings } from "./app/settings";
+
+dotenv.config();
+
+const REQUIRED_ENV_VARS = [
+  "LLAMA_CLOUD_INDEX_NAME",
+  "LLAMA_CLOUD_PROJECT_NAME",
+  "LLAMA_CLOUD_API_KEY",
+];
+
+export function checkRequiredEnvVars() {
+  const missingEnvVars = REQUIRED_ENV_VARS.filter((envVar) => {
+    return !process.env[envVar];
+  });
+
+  if (missingEnvVars.length > 0) {
+    console.log(
+      `The following environment variables are required but missing: ${missingEnvVars.join(
+        ", ",
+      )}`,
+    );
+    throw new Error(
+      `Missing environment variables: ${missingEnvVars.join(", ")}`,
+    );
+  }
+}
+
+async function* walk(dir: string): AsyncGenerator<string> {
+  const directory = await fs.opendir(dir);
+
+  for await (const dirent of directory) {
+    const entryPath = path.join(dir, dirent.name);
+
+    if (dirent.isDirectory()) {
+      yield* walk(entryPath); // Recursively walk through directories
+    } else if (dirent.isFile()) {
+      yield entryPath; // Yield file paths
+    }
+  }
+}
+
+async function loadAndIndex() {
+  const index = await getIndex();
+  // ensure the index is available or create a new one
+  await index.ensureIndex({
+    verbose: true,
+    embedding: {
+      type: "OPENAI_EMBEDDING",
+      component: {
+        api_key: process.env.OPENAI_API_KEY,
+        model_name: "text-embedding-3-small",
+      },
+    },
+  });
+
+  const projectId = await index.getProjectId();
+  const pipelineId = await index.getPipelineId();
+
+  // walk through the data directory and upload each file to LlamaCloud
+  for await (const filePath of walk("data")) {
+    const buffer = await fs.readFile(filePath);
+    const filename = path.basename(filePath);
+    try {
+      await LLamaCloudFileService.addFileToPipeline(
+        projectId,
+        pipelineId,
+        new File([buffer], filename),
+      );
+    } catch (error) {
+      if (
+        error instanceof ReferenceError &&
+        error.message.includes("File is not defined")
+      ) {
+        throw new Error(
+          "File class is not supported in the current Node.js version. Please use Node.js 20 or higher.",
+        );
+      }
+      throw error;
+    }
+  }
+
+  console.log(`Successfully uploaded documents to LlamaCloud!`);
+}
+
+(async () => {
+  try {
+    checkRequiredEnvVars();
+    initSettings();
+    await loadAndIndex();
+    console.log("Finished generating storage.");
+  } catch (error) {
+    console.error("Error generating storage.", error);
+  }
+})();

templates/components/vectordbs/llamaindexserver/llamacloud/typescript/index.ts

@@ -0,0 +1,28 @@
+import { LlamaCloudIndex } from "llamaindex/cloud/LlamaCloudIndex";
+
+type LlamaCloudDataSourceParams = {
+  llamaCloudPipeline?: {
+    project: string;
+    pipeline: string;
+  };
+};
+
+export async function getIndex(params?: LlamaCloudDataSourceParams) {
+  const { project, pipeline } = params?.llamaCloudPipeline ?? {};
+  const projectName = project ?? process.env.LLAMA_CLOUD_PROJECT_NAME;
+  const pipelineName = pipeline ?? process.env.LLAMA_CLOUD_INDEX_NAME;
+  const apiKey = process.env.LLAMA_CLOUD_API_KEY;
+  if (!projectName || !pipelineName || !apiKey) {
+    throw new Error(
+      "LlamaCloud is not configured. Please set project, pipeline, and api key in the params or as environment variables.",
+    );
+  }
+  const index = new LlamaCloudIndex({
+    organizationId: process.env.LLAMA_CLOUD_ORGANIZATION_ID,
+    name: pipelineName,
+    projectName,
+    apiKey,
+    baseUrl: process.env.LLAMA_CLOUD_BASE_URL,
+  });
+  return index;
+}

templates/components/vectordbs/typescript/astra/generate.ts

@@ -37,8 +37,12 @@ async function loadAndIndex() {
 }
 
 (async () => {
-  checkRequiredEnvVars();
-  initSettings();
-  await loadAndIndex();
-  console.log("Finished generating storage.");
+  try {
+    checkRequiredEnvVars();
+    initSettings();
+    await loadAndIndex();
+    console.log("Finished generating storage.");
+  } catch (error) {
+    console.error("Error generating storage.", error);
+  }
 })();

templates/components/vectordbs/typescript/chroma/generate.ts

@@ -30,8 +30,12 @@ async function loadAndIndex() {
 }
 
 (async () => {
-  checkRequiredEnvVars();
-  initSettings();
-  await loadAndIndex();
-  console.log("Finished generating storage.");
+  try {
+    checkRequiredEnvVars();
+    initSettings();
+    await loadAndIndex();
+    console.log("Finished generating storage.");
+  } catch (error) {
+    console.error("Error generating storage.", error);
+  }
 })();

templates/components/vectordbs/typescript/llamacloud/generate.ts

@@ -57,8 +57,12 @@ async function loadAndIndex() {
 }
 
 (async () => {
-  checkRequiredEnvVars();
-  initSettings();
-  await loadAndIndex();
-  console.log("Finished generating storage.");
+  try {
+    checkRequiredEnvVars();
+    initSettings();
+    await loadAndIndex();
+    console.log("Finished generating storage.");
+  } catch (error) {
+    console.error("Error generating storage.", error);
+  }
 })();

templates/components/vectordbs/typescript/milvus/generate.ts

@@ -29,8 +29,12 @@ async function loadAndIndex() {
 }
 
 (async () => {
-  checkRequiredEnvVars();
-  initSettings();
-  await loadAndIndex();
-  console.log("Finished generating storage.");
+  try {
+    checkRequiredEnvVars();
+    initSettings();
+    await loadAndIndex();
+    console.log("Finished generating storage.");
+  } catch (error) {
+    console.error("Error generating storage.", error);
+  }
 })();

templates/components/vectordbs/typescript/mongo/generate.ts

@@ -45,8 +45,12 @@ async function loadAndIndex() {
 }
 
 (async () => {
-  checkRequiredEnvVars();
-  initSettings();
-  await loadAndIndex();
-  console.log("Finished generating storage.");
+  try {
+    checkRequiredEnvVars();
+    initSettings();
+    await loadAndIndex();
+    console.log("Finished generating storage.");
+  } catch (error) {
+    console.error("Error generating storage.", error);
+  }
 })();

templates/components/vectordbs/typescript/none/generate.ts

@@ -37,7 +37,11 @@ async function generateDatasource() {
 }
 
 (async () => {
-  initSettings();
-  await generateDatasource();
-  console.log("Finished generating storage.");
+  try {
+    initSettings();
+    await generateDatasource();
+    console.log("Finished generating storage.");
+  } catch (error) {
+    console.error("Error generating storage.", error);
+  }
 })();

templates/components/vectordbs/typescript/pg/generate.ts

@@ -35,9 +35,12 @@ async function loadAndIndex() {
 }
 
 (async () => {
-  checkRequiredEnvVars();
-  initSettings();
-  await loadAndIndex();
-  console.log("Finished generating storage.");
-  process.exit(0);
+  try {
+    checkRequiredEnvVars();
+    initSettings();
+    await loadAndIndex();
+    console.log("Finished generating storage.");
+  } catch (error) {
+    console.error("Error generating storage.", error);
+  }
 })();

templates/components/vectordbs/typescript/pinecone/generate.ts

@@ -25,8 +25,12 @@ async function loadAndIndex() {
 }
 
 (async () => {
-  checkRequiredEnvVars();
-  initSettings();
-  await loadAndIndex();
-  console.log("Finished generating storage.");
+  try {
+    checkRequiredEnvVars();
+    initSettings();
+    await loadAndIndex();
+    console.log("Finished generating storage.");
+  } catch (error) {
+    console.error("Error generating storage.", error);
+  }
 })();

templates/components/vectordbs/typescript/qdrant/generate.ts

@@ -30,8 +30,12 @@ async function loadAndIndex() {
 }
 
 (async () => {
-  checkRequiredEnvVars();
-  initSettings();
-  await loadAndIndex();
-  console.log("Finished generating storage.");
+  try {
+    checkRequiredEnvVars();
+    initSettings();
+    await loadAndIndex();
+    console.log("Finished generating storage.");
+  } catch (error) {
+    console.error("Error generating storage.", error);
+  }
 })();

templates/components/vectordbs/typescript/weaviate/generate.ts

@@ -23,8 +23,12 @@ async function loadAndIndex() {
 }
 
 (async () => {
-  checkRequiredEnvVars();
-  initSettings();
-  await loadAndIndex();
-  console.log("Finished generating storage.");
+  try {
+    checkRequiredEnvVars();
+    initSettings();
+    await loadAndIndex();
+    console.log("Finished generating storage.");
+  } catch (error) {
+    console.error("Error generating storage.", error);
+  }
 })();

templates/components/workflows/python/agentic_rag/README-template.md

@@ -0,0 +1,59 @@
+This is a [LlamaIndex](https://www.llamaindex.ai/) simple agentic RAG project using [Agent Workflows](https://docs.llamaindex.ai/en/stable/examples/agent/agent_workflow_basic/).
+
+## Getting Started
+
+First, setup the environment with poetry:
+
+> **_Note:_** This step is not needed if you are using the dev-container.
+
+```shell
+poetry install
+```
+
+Then check the parameters that have been pre-configured in the `.env` file in this directory.
+Make sure you have set the `OPENAI_API_KEY` for the LLM.
+
+Second, generate the embeddings of the documents in the `./data` directory:
+
+```shell
+poetry run generate
+```
+
+Third, run the development server:
+
+```shell
+poetry run dev
+```
+
+Then open [http://localhost:8000](http://localhost:8000) with your browser to start the chat UI.
+
+To start the app optimized for **production**, run:
+
+```
+poetry run prod
+```
+
+## Configure LLM and Embedding Model
+
+You can configure [LLM model](https://docs.llamaindex.ai/en/stable/module_guides/models/llms) and [embedding model](https://docs.llamaindex.ai/en/stable/module_guides/models/embeddings) in [settings.py](app/settings.py).
+
+## Use Case
+
+We have prepared an [example workflow](./app/workflow.py) for the agentic RAG use case, where you can ask questions about the example documents in the [./data](./data) directory.
+
+You can start by sending an request on the [chat UI](http://localhost:8000) or you can test the `/api/chat` endpoint with the following curl request:
+
+```
+curl --location 'localhost:8000/api/chat' \
+--header 'Content-Type: application/json' \
+--data '{ "messages": [{ "role": "user", "content": "What standards for a letter exist?" }] }'
+```
+
+## Learn More
+
+To learn more about LlamaIndex, take a look at the following resources:
+
+- [LlamaIndex Documentation](https://docs.llamaindex.ai) - learn about LlamaIndex.
+- [Workflows Introduction](https://docs.llamaindex.ai/en/stable/understanding/workflows/) - learn about LlamaIndex workflows.
+
+You can check out [the LlamaIndex GitHub repository](https://github.com/run-llama/llama_index) - your feedback and contributions are welcome!

templates/components/workflows/python/agentic_rag/workflow.py

@@ -0,0 +1,22 @@
+from typing import Optional
+
+from app.index import get_index
+from llama_index.core.agent.workflow import AgentWorkflow
+from llama_index.core.settings import Settings
+from llama_index.llms.openai import OpenAI
+from llama_index.server.api.models import ChatRequest
+from llama_index.server.tools.index import get_query_engine_tool
+
+
+def create_workflow(chat_request: Optional[ChatRequest] = None) -> AgentWorkflow:
+    index = get_index(chat_request=chat_request)
+    if index is None:
+        raise RuntimeError(
+            "Index not found! Please run `poetry run generate` to index the data first."
+        )
+    query_tool = get_query_engine_tool(index=index)
+    return AgentWorkflow.from_tools_or_functions(
+        tools_or_functions=[query_tool],
+        llm=Settings.llm or OpenAI(model="gpt-4o-mini"),
+        system_prompt="You are a helpful assistant.",
+    )

templates/components/workflows/python/deep_research/README-template.md

@@ -0,0 +1,69 @@
+This is a [LlamaIndex](https://www.llamaindex.ai/) multi-agents project using [Workflows](https://docs.llamaindex.ai/en/stable/understanding/workflows/).
+
+## Getting Started
+
+First, setup the environment with poetry:
+
+> **_Note:_** This step is not needed if you are using the dev-container.
+
+```shell
+poetry install
+```
+
+Then check the parameters that have been pre-configured in the `.env` file in this directory.
+Make sure you have set the `OPENAI_API_KEY` for the LLM.
+
+Second, generate the embeddings of the documents in the `./data` directory:
+
+```shell
+poetry run generate
+```
+
+Third, run the development server:
+
+```shell
+poetry run dev
+```
+
+Then open [http://localhost:8000](http://localhost:8000) with your browser to start the chat UI.
+
+To start the app optimized for **production**, run:
+
+```
+poetry run prod
+```
+
+## Configure LLM and Embedding Model
+
+You can configure [LLM model](https://docs.llamaindex.ai/en/stable/module_guides/models/llms) and [embedding model](https://docs.llamaindex.ai/en/stable/module_guides/models/embeddings) in [settings.py](app/settings.py).
+
+## Use Case
+
+We have prepared an [example workflow](./app/workflow.py) for the deep research use case, where you can ask questions about the example documents in the [./data](./data) directory.
+
+You can start by sending an request on the [chat UI](http://localhost:8000) or you can test the `/api/chat` endpoint with the following curl request:
+
+```
+curl --location 'localhost:8000/api/chat' \
+--header 'Content-Type: application/json' \
+--data '{ "messages": [{ "role": "user", "content": "Create a report comparing the finances of Apple and Tesla" }] }'
+```
+
+## Customize the UI
+
+To customize the UI, you can start by modifying the [./components/ui_event.jsx](./components/ui_event.jsx) file.
+
+You can also generate a new code for the workflow using LLM by running the following command:
+
+```
+poetry run generate:ui
+```
+
+## Learn More
+
+To learn more about LlamaIndex, take a look at the following resources:
+
+- [LlamaIndex Documentation](https://docs.llamaindex.ai) - learn about LlamaIndex.
+- [Workflows Introduction](https://docs.llamaindex.ai/en/stable/understanding/workflows/) - learn about LlamaIndex workflows.
+
+You can check out [the LlamaIndex GitHub repository](https://github.com/run-llama/llama_index) - your feedback and contributions are welcome!

templates/components/workflows/python/deep_research/workflow.py

@@ -0,0 +1,542 @@
+import logging
+import os
+import uuid
+from typing import List, Literal, Optional
+
+from app.index import get_index
+from llama_index.core.base.llms.types import (
+    CompletionResponse,
+    CompletionResponseAsyncGen,
+)
+from llama_index.core.indices.base import BaseIndex
+from llama_index.core.memory import ChatMemoryBuffer
+from llama_index.core.memory.simple_composable_memory import SimpleComposableMemory
+from llama_index.core.prompts import PromptTemplate
+from llama_index.core.schema import MetadataMode, Node, NodeWithScore
+from llama_index.core.settings import Settings
+from llama_index.core.types import ChatMessage, MessageRole
+from llama_index.core.workflow import (
+    Context,
+    Event,
+    StartEvent,
+    StopEvent,
+    Workflow,
+    step,
+)
+from llama_index.server.api.models import ChatRequest, SourceNodesEvent, UIEvent
+from pydantic import BaseModel, Field
+
+logger = logging.getLogger("uvicorn")
+logger.setLevel(logging.INFO)
+
+
+def create_workflow(chat_request: Optional[ChatRequest] = None) -> Workflow:
+    index = get_index(chat_request=chat_request)
+    if index is None:
+        raise ValueError(
+            "Index is not found. Try run generation script to create the index first."
+        )
+
+    return DeepResearchWorkflow(
+        index=index,
+        timeout=120.0,
+    )
+
+
+# Workflow events
+class PlanResearchEvent(Event):
+    pass
+
+
+class ResearchEvent(Event):
+    question_id: str
+    question: str
+    context_nodes: List[NodeWithScore]
+
+
+class CollectAnswersEvent(Event):
+    question_id: str
+    question: str
+    answer: str
+
+
+class ReportEvent(Event):
+    pass
+
+
+# Events that are streamed to the frontend and rendered there
+class UIEventData(BaseModel):
+    """
+    Events for DeepResearch workflow which has 3 main stages:
+    - Retrieve: Retrieve information from the knowledge base.
+    - Analyze: Analyze the retrieved information and provide list of questions for answering.
+    - Answer: Answering the provided questions. There are multiple answer events, each with its own id that is used to display the answer for a particular question.
+    """
+
+    id: Optional[str] = Field(default=None, description="The id of the event")
+    event: Literal["retrieve", "analyze", "answer"] = Field(
+        default="retrieve", description="The event type"
+    )
+    state: Literal["pending", "inprogress", "done", "error"] = Field(
+        default="pending", description="The state of the event"
+    )
+    question: Optional[str] = Field(
+        default=None,
+        description="Used by answer event to display the question",
+    )
+    answer: Optional[str] = Field(
+        default=None,
+        description="Used by answer event to display the answer of the question",
+    )
+
+
+class DeepResearchWorkflow(Workflow):
+    """
+    A workflow to research and analyze documents from multiple perspectives and write a comprehensive report.
+
+    Requirements:
+    - An indexed documents containing the knowledge base related to the topic
+
+    Steps:
+    1. Retrieve information from the knowledge base
+    2. Analyze the retrieved information and provide questions for answering
+    3. Answer the questions
+    4. Write the report based on the research results
+    """
+
+    memory: SimpleComposableMemory
+    context_nodes: List[Node]
+    index: BaseIndex
+    user_request: str
+    stream: bool = True
+
+    def __init__(
+        self,
+        index: BaseIndex,
+        **kwargs,
+    ):
+        super().__init__(**kwargs)
+        self.index = index
+        self.context_nodes = []
+        self.memory = SimpleComposableMemory.from_defaults(
+            primary_memory=ChatMemoryBuffer.from_defaults(),
+        )
+
+    @step
+    async def retrieve(self, ctx: Context, ev: StartEvent) -> PlanResearchEvent:
+        """
+        Initiate the workflow: memory, tools, agent
+        """
+        self.stream = ev.get("stream", True)
+        self.user_request = ev.get("user_msg")
+        chat_history = ev.get("chat_history")
+        if chat_history is not None:
+            self.memory.put_messages(chat_history)
+
+        await ctx.set("total_questions", 0)
+
+        # Add user message to memory
+        self.memory.put_messages(
+            messages=[
+                ChatMessage(
+                    role=MessageRole.USER,
+                    content=self.user_request,
+                )
+            ]
+        )
+        ctx.write_event_to_stream(
+            UIEvent(
+                type="ui_event",
+                data=UIEventData(
+                    event="retrieve",
+                    state="inprogress",
+                ),
+            )
+        )
+        retriever = self.index.as_retriever(
+            similarity_top_k=int(os.getenv("TOP_K", 10)),
+        )
+        nodes = retriever.retrieve(self.user_request)
+        self.context_nodes.extend(nodes)  # type: ignore
+        ctx.write_event_to_stream(
+            UIEvent(
+                type="ui_event",
+                data=UIEventData(
+                    event="retrieve",
+                    state="done",
+                ),
+            )
+        )
+        # Send source nodes to the stream
+        # Use SourceNodesEvent to display source nodes in the UI.
+        ctx.write_event_to_stream(
+            SourceNodesEvent(
+                nodes=nodes,
+            )
+        )
+        return PlanResearchEvent()
+
+    @step
+    async def analyze(
+        self, ctx: Context, ev: PlanResearchEvent
+    ) -> ResearchEvent | ReportEvent | StopEvent:
+        """
+        Analyze the retrieved information
+        """
+        logger.info("Analyzing the retrieved information")
+        ctx.write_event_to_stream(
+            UIEvent(
+                type="ui_event",
+                data=UIEventData(
+                    event="analyze",
+                    state="inprogress",
+                ),
+            )
+        )
+        total_questions = await ctx.get("total_questions")
+        res = await plan_research(
+            memory=self.memory,
+            context_nodes=self.context_nodes,
+            user_request=self.user_request,
+            total_questions=total_questions,
+        )
+        if res.decision == "cancel":
+            ctx.write_event_to_stream(
+                UIEvent(
+                    type="ui_event",
+                    data=UIEventData(
+                        event="analyze",
+                        state="done",
+                    ),
+                )
+            )
+            return StopEvent(
+                result=res.cancel_reason,
+            )
+        elif res.decision == "write":
+            # Writing a report without any research context is not allowed.
+            # It's a LLM hallucination.
+            if total_questions == 0:
+                ctx.write_event_to_stream(
+                    UIEvent(
+                        type="ui_event",
+                        data=UIEventData(
+                            event="analyze",
+                            state="done",
+                        ),
+                    )
+                )
+                return StopEvent(
+                    result="Sorry, I have a problem when analyzing the retrieved information. Please try again.",
+                )
+
+            self.memory.put(
+                message=ChatMessage(
+                    role=MessageRole.ASSISTANT,
+                    content="No more idea to analyze. We should report the answers.",
+                )
+            )
+            ctx.send_event(ReportEvent())
+        else:
+            total_questions += len(res.research_questions)
+            await ctx.set("total_questions", total_questions)  # For tracking
+            await ctx.set(
+                "waiting_questions", len(res.research_questions)
+            )  # For waiting questions to be answered
+            self.memory.put(
+                message=ChatMessage(
+                    role=MessageRole.ASSISTANT,
+                    content="We need to find answers to the following questions:\n"
+                    + "\n".join(res.research_questions),
+                )
+            )
+            for question in res.research_questions:
+                question_id = str(uuid.uuid4())
+                ctx.write_event_to_stream(
+                    UIEvent(
+                        type="ui_event",
+                        data=UIEventData(
+                            event="answer",
+                            state="pending",
+                            id=question_id,
+                            question=question,
+                            answer=None,
+                        ),
+                    )
+                )
+                ctx.send_event(
+                    ResearchEvent(
+                        question_id=question_id,
+                        question=question,
+                        context_nodes=self.context_nodes,
+                    )
+                )
+        ctx.write_event_to_stream(
+            UIEvent(
+                type="ui_event",
+                data=UIEventData(
+                    event="analyze",
+                    state="done",
+                ),
+            )
+        )
+        return None
+
+    @step(num_workers=2)
+    async def answer(self, ctx: Context, ev: ResearchEvent) -> CollectAnswersEvent:
+        """
+        Answer the question
+        """
+        ctx.write_event_to_stream(
+            UIEvent(
+                type="ui_event",
+                data=UIEventData(
+                    event="answer",
+                    state="inprogress",
+                    id=ev.question_id,
+                    question=ev.question,
+                ),
+            )
+        )
+        try:
+            answer = await research(
+                context_nodes=ev.context_nodes,
+                question=ev.question,
+            )
+        except Exception as e:
+            logger.error(f"Error answering question {ev.question}: {e}")
+            answer = f"Got error when answering the question: {ev.question}"
+        ctx.write_event_to_stream(
+            UIEvent(
+                type="ui_event",
+                data=UIEventData(
+                    event="answer",
+                    state="done",
+                    id=ev.question_id,
+                    question=ev.question,
+                    answer=answer,
+                ),
+            )
+        )
+
+        return CollectAnswersEvent(
+            question_id=ev.question_id,
+            question=ev.question,
+            answer=answer,
+        )
+
+    @step
+    async def collect_answers(
+        self, ctx: Context, ev: CollectAnswersEvent
+    ) -> PlanResearchEvent:
+        """
+        Collect answers to all questions
+        """
+        num_questions = await ctx.get("waiting_questions")
+        results = ctx.collect_events(
+            ev,
+            expected=[CollectAnswersEvent] * num_questions,
+        )
+        if results is None:
+            return None
+        for result in results:
+            self.memory.put(
+                message=ChatMessage(
+                    role=MessageRole.ASSISTANT,
+                    content=f"<Question>{result.question}</Question>\n<Answer>{result.answer}</Answer>",
+                )
+            )
+        await ctx.set("waiting_questions", 0)
+        self.memory.put(
+            message=ChatMessage(
+                role=MessageRole.ASSISTANT,
+                content="Researched all the questions. Now, i need to analyze if it's ready to write a report or need to research more.",
+            )
+        )
+        return PlanResearchEvent()
+
+    @step
+    async def report(self, ctx: Context, ev: ReportEvent) -> StopEvent:
+        """
+        Report the answers
+        """
+        res = await write_report(
+            memory=self.memory,
+            user_request=self.user_request,
+            stream=self.stream,
+        )
+        return StopEvent(
+            result=res,
+        )
+
+
+class AnalysisDecision(BaseModel):
+    decision: Literal["research", "write", "cancel"] = Field(
+        description="Whether to continue research, write a report, or cancel the research after several retries"
+    )
+    research_questions: Optional[List[str]] = Field(
+        description="""
+        If the decision is to research, provide a list of questions to research that related to the user request.
+        Maximum 3 questions. Set to null or empty if writing a report or cancel the research.
+        """,
+        default_factory=list,
+    )
+    cancel_reason: Optional[str] = Field(
+        description="The reason for cancellation if the decision is to cancel research.",
+        default=None,
+    )
+
+
+async def plan_research(
+    memory: SimpleComposableMemory,
+    context_nodes: List[Node],
+    user_request: str,
+    total_questions: int,
+) -> AnalysisDecision:
+    analyze_prompt = """
+      You are a professor who is guiding a researcher to research a specific request/problem.
+      Your task is to decide on a research plan for the researcher.
+
+      The possible actions are:
+      + Provide a list of questions for the researcher to investigate, with the purpose of clarifying the request.
+      + Write a report if the researcher has already gathered enough research on the topic and can resolve the initial request.
+      + Cancel the research if most of the answers from researchers indicate there is insufficient information to research the request. Do not attempt more than 3 research iterations or too many questions.
+
+      The workflow should be:
+      + Always begin by providing some initial questions for the researcher to investigate.
+      + Analyze the provided answers against the initial topic/request. If the answers are insufficient to resolve the initial request, provide additional questions for the researcher to investigate.
+      + If the answers are sufficient to resolve the initial request, instruct the researcher to write a report.
+
+      Here are the context: 
+      <Collected information>
+      {context_str}
+      </Collected information>
+
+      <Conversation context>
+      {conversation_context}
+      </Conversation context>
+
+      {enhanced_prompt}
+
+      Now, provide your decision in the required format for this user request:
+      <User request>
+      {user_request}
+      </User request>
+      """
+    # Manually craft the prompt to avoid LLM hallucination
+    enhanced_prompt = ""
+    if total_questions == 0:
+        # Avoid writing a report without any research context
+        enhanced_prompt = """
+        
+        The student has no questions to research. Let start by asking some questions.
+        """
+    elif total_questions > 6:
+        # Avoid asking too many questions (when the data is not ready for writing a report)
+        enhanced_prompt = f"""
+
+        The student has researched {total_questions} questions. Should cancel the research if the context is not enough to write a report.
+        """
+
+    conversation_context = "\n".join(
+        [f"{message.role}: {message.content}" for message in memory.get_all()]
+    )
+    context_str = "\n".join(
+        [node.get_content(metadata_mode=MetadataMode.LLM) for node in context_nodes]
+    )
+    res = await Settings.llm.astructured_predict(
+        output_cls=AnalysisDecision,
+        prompt=PromptTemplate(template=analyze_prompt),
+        user_request=user_request,
+        context_str=context_str,
+        conversation_context=conversation_context,
+        enhanced_prompt=enhanced_prompt,
+    )
+    return res
+
+
+async def research(
+    question: str,
+    context_nodes: List[NodeWithScore],
+) -> str:
+    prompt = """
+    You are a researcher who is in the process of answering the question.
+    The purpose is to answer the question based on the collected information, without using prior knowledge or making up any new information.
+    Always add citations to the sentence/point/paragraph using the id of the provided content.
+    The citation should follow this format: [citation:id] where id is the id of the content.
+    
+    E.g:
+    If we have a context like this:
+    <Citation id='abc-xyz'>
+    Baby llama is called cria
+    </Citation id='abc-xyz'>
+
+    And your answer uses the content, then the citation should be:
+    - Baby llama is called cria [citation:abc-xyz]
+
+    Here is the provided context for the question:
+    <Collected information>
+    {context_str}
+    </Collected information>`
+
+    No prior knowledge, just use the provided context to answer the question: {question}
+    """
+    context_str = "\n".join(
+        [_get_text_node_content_for_citation(node) for node in context_nodes]
+    )
+    res = await Settings.llm.acomplete(
+        prompt=prompt.format(question=question, context_str=context_str),
+    )
+    return res.text
+
+
+async def write_report(
+    memory: SimpleComposableMemory,
+    user_request: str,
+    stream: bool = False,
+) -> CompletionResponse | CompletionResponseAsyncGen:
+    report_prompt = """
+    You are a researcher writing a report based on a user request and the research context.
+    You have researched various perspectives related to the user request.
+    The report should provide a comprehensive outline covering all important points from the researched perspectives.
+    Create a well-structured outline for the research report that covers all the answers.
+
+    # IMPORTANT when writing in markdown format:
+    + Use tables or figures where appropriate to enhance presentation.
+    + Preserve all citation syntax (the `[citation:id]()` parts in the provided context). Keep these citations in the final report - no separate reference section is needed.
+    + Do not add links, a table of contents, or a references section to the report.
+
+    <User request>
+    {user_request}
+    </User request>
+
+    <Research context>
+    {research_context}
+    </Research context>
+
+    Now, write a report addressing the user request based on the research provided following the format and guidelines above.
+    """
+    research_context = "\n".join(
+        [f"{message.role}: {message.content}" for message in memory.get_all()]
+    )
+
+    llm_complete_func = (
+        Settings.llm.astream_complete if stream else Settings.llm.acomplete
+    )
+
+    res = await llm_complete_func(
+        prompt=report_prompt.format(
+            user_request=user_request,
+            research_context=research_context,
+        ),
+    )
+    return res
+
+
+def _get_text_node_content_for_citation(node: NodeWithScore) -> str:
+    """
+    Construct node content for LLM with citation flag.
+    """
+    node_id = node.node.node_id
+    content = f"<Citation id='{node_id}'>\n{node.get_content(metadata_mode=MetadataMode.LLM)}</Citation id='{node_id}'>"
+    return content

templates/components/workflows/python/financial_report/README-template.md

@@ -0,0 +1,59 @@
+This is a [LlamaIndex](https://www.llamaindex.ai/) multi-agents project using [Workflows](https://docs.llamaindex.ai/en/stable/understanding/workflows/).
+
+## Getting Started
+
+First, setup the environment with poetry:
+
+> **_Note:_** This step is not needed if you are using the dev-container.
+
+```shell
+poetry install
+```
+
+Then check the parameters that have been pre-configured in the `.env` file in this directory.
+Make sure you have set the `OPENAI_API_KEY` for the LLM and the `E2B_API_KEY` for the code interpreter. You can get the E2B API key from [here](https://e2b.dev).
+
+Second, generate the embeddings of the documents in the `./data` directory:
+
+```shell
+poetry run generate
+```
+
+Third, run the development server:
+
+```shell
+poetry run dev
+```
+
+Then open [http://localhost:8000](http://localhost:8000) with your browser to start the chat UI.
+
+To start the app optimized for **production**, run:
+
+```
+poetry run prod
+```
+
+## Configure LLM and Embedding Model
+
+You can configure [LLM model](https://docs.llamaindex.ai/en/stable/module_guides/models/llms) and [embedding model](https://docs.llamaindex.ai/en/stable/module_guides/models/embeddings) in [settings.py](app/settings.py).
+
+## Use Case
+
+We have prepared an [example workflow](./app/workflow.py) for the financial report use case, where you can ask questions about the example documents in the [./data](./data) directory.
+
+You can start by sending an request on the [chat UI](http://localhost:8000) or you can test the `/api/chat` endpoint with the following curl request:
+
+```
+curl --location 'localhost:8000/api/chat' \
+--header 'Content-Type: application/json' \
+--data '{ "messages": [{ "role": "user", "content": "Create a report comparing the finances of Apple and Tesla" }] }'
+```
+
+## Learn More
+
+To learn more about LlamaIndex, take a look at the following resources:
+
+- [LlamaIndex Documentation](https://docs.llamaindex.ai) - learn about LlamaIndex.
+- [Workflows Introduction](https://docs.llamaindex.ai/en/stable/understanding/workflows/) - learn about LlamaIndex workflows.
+
+You can check out [the LlamaIndex GitHub repository](https://github.com/run-llama/llama_index) - your feedback and contributions are welcome!

templates/components/workflows/python/financial_report/workflow.py

@@ -0,0 +1,333 @@
+import os
+from typing import List, Optional
+
+from app.index import get_index
+from llama_index.core import Settings
+from llama_index.core.base.llms.types import ChatMessage, MessageRole
+from llama_index.core.llms.function_calling import FunctionCallingLLM
+from llama_index.core.memory import ChatMemoryBuffer
+from llama_index.core.tools import FunctionTool, QueryEngineTool, ToolSelection
+from llama_index.core.workflow import (
+    Context,
+    Event,
+    StartEvent,
+    StopEvent,
+    Workflow,
+    step,
+)
+from llama_index.server.api.models import AgentRunEvent, ChatRequest
+from llama_index.server.settings import server_settings
+from llama_index.server.tools.document_generator import DocumentGenerator
+from llama_index.server.tools.index import get_query_engine_tool
+from llama_index.server.tools.interpreter import E2BCodeInterpreter
+from llama_index.server.utils.agent_tool import (
+    call_tools,
+    chat_with_tools,
+)
+
+
+def create_workflow(chat_request: Optional[ChatRequest] = None) -> Workflow:
+    index = get_index(chat_request=chat_request)
+    if index is None:
+        raise ValueError(
+            "Index is not found. Try run generation script to create the index first."
+        )
+    query_engine_tool = get_query_engine_tool(index=index)
+    e2b_api_key = os.getenv("E2B_API_KEY")
+    if e2b_api_key is None:
+        raise ValueError(
+            "E2B_API_KEY is required to use the code interpreter tool. Please check README.md to know how to get the key."
+        )
+    code_interpreter_tool = E2BCodeInterpreter(api_key=e2b_api_key).to_tool()
+    document_generator_tool = DocumentGenerator(
+        file_server_url_prefix=server_settings.file_server_url_prefix,
+    ).to_tool()
+
+    return FinancialReportWorkflow(
+        query_engine_tool=query_engine_tool,
+        code_interpreter_tool=code_interpreter_tool,
+        document_generator_tool=document_generator_tool,
+        timeout=180,
+    )
+
+
+class InputEvent(Event):
+    input: List[ChatMessage]
+    response: bool = False
+
+
+class ResearchEvent(Event):
+    input: list[ToolSelection]
+
+
+class AnalyzeEvent(Event):
+    input: list[ToolSelection] | ChatMessage
+
+
+class ReportEvent(Event):
+    input: list[ToolSelection]
+
+
+class FinancialReportWorkflow(Workflow):
+    """
+    A workflow to generate a financial report using indexed documents.
+
+    Requirements:
+    - Indexed documents containing financial data and a query engine tool to search them
+    - A code interpreter tool to analyze data and generate reports
+    - A document generator tool to create report files
+
+    Steps:
+    1. LLM Input: The LLM determines the next step based on function calling.
+       For example, if the model requests the query engine tool, it returns a ResearchEvent;
+       if it requests document generation, it returns a ReportEvent.
+    2. Research: Uses the query engine to find relevant chunks from indexed documents.
+       After gathering information, it requests analysis (step 3).
+    3. Analyze: Uses a custom prompt to analyze research results and can call the code
+       interpreter tool for visualization or calculation. Returns results to the LLM.
+    4. Report: Uses the document generator tool to create a report. Returns results to the LLM.
+    """
+
+    _default_system_prompt = """
+    You are a financial analyst who are given a set of tools to help you.
+    It's good to using appropriate tools for the user request and always use the information from the tools, don't make up anything yourself.
+    For the query engine tool, you should break down the user request into a list of queries and call the tool with the queries.
+    """
+    stream: bool = True
+
+    def __init__(
+        self,
+        query_engine_tool: QueryEngineTool,
+        code_interpreter_tool: FunctionTool,
+        document_generator_tool: FunctionTool,
+        llm: Optional[FunctionCallingLLM] = None,
+        system_prompt: Optional[str] = None,
+        **kwargs,
+    ):
+        super().__init__(**kwargs)
+        self.system_prompt = system_prompt or self._default_system_prompt
+        self.query_engine_tool = query_engine_tool
+        self.code_interpreter_tool = code_interpreter_tool
+        self.document_generator_tool = document_generator_tool
+        assert query_engine_tool is not None, (
+            "Query engine tool is not found. Try run generation script or upload a document file first."
+        )
+        assert code_interpreter_tool is not None, "Code interpreter tool is required"
+        assert document_generator_tool is not None, (
+            "Document generator tool is required"
+        )
+        self.tools = [
+            self.query_engine_tool,
+            self.code_interpreter_tool,
+            self.document_generator_tool,
+        ]
+        self.llm: FunctionCallingLLM = llm or Settings.llm  # type: ignore
+        assert isinstance(self.llm, FunctionCallingLLM)
+        self.memory = ChatMemoryBuffer.from_defaults(llm=self.llm)
+
+    @step()
+    async def prepare_chat_history(self, ctx: Context, ev: StartEvent) -> InputEvent:
+        self.stream = ev.get("stream", True)
+        user_msg = ev.get("user_msg")
+        chat_history = ev.get("chat_history")
+
+        if chat_history is not None:
+            self.memory.put_messages(chat_history)
+
+        # Add user message to memory
+        self.memory.put(ChatMessage(role=MessageRole.USER, content=user_msg))
+
+        if self.system_prompt:
+            system_msg = ChatMessage(
+                role=MessageRole.SYSTEM, content=self.system_prompt
+            )
+            self.memory.put(system_msg)
+
+        return InputEvent(input=self.memory.get())
+
+    @step()
+    async def handle_llm_input(  # type: ignore
+        self,
+        ctx: Context,
+        ev: InputEvent,
+    ) -> ResearchEvent | AnalyzeEvent | ReportEvent | StopEvent:
+        """
+        Handle an LLM input and decide the next step.
+        """
+        # Always use the latest chat history from the input
+        chat_history: list[ChatMessage] = ev.input
+        # Get tool calls
+        response = await chat_with_tools(
+            self.llm,
+            self.tools,  # type: ignore
+            chat_history,
+        )
+        if not response.has_tool_calls():
+            if self.stream:
+                return StopEvent(result=response.generator)
+            else:
+                return StopEvent(result=await response.full_response())
+        # calling different tools at the same time is not supported at the moment
+        # add an error message to tell the AI to process step by step
+        if response.is_calling_different_tools():
+            self.memory.put(
+                ChatMessage(
+                    role=MessageRole.ASSISTANT,
+                    content="Cannot call different tools at the same time. Try calling one tool at a time.",
+                )
+            )
+            return InputEvent(input=self.memory.get())  # type: ignore
+        self.memory.put(response.tool_call_message)
+        match response.tool_name():
+            case self.code_interpreter_tool.metadata.name:
+                return AnalyzeEvent(input=response.tool_calls)
+            case self.document_generator_tool.metadata.name:
+                return ReportEvent(input=response.tool_calls)
+            case self.query_engine_tool.metadata.name:
+                return ResearchEvent(input=response.tool_calls)
+            case _:
+                raise ValueError(f"Unknown tool: {response.tool_name()}")
+
+    @step()
+    async def research(self, ctx: Context, ev: ResearchEvent) -> AnalyzeEvent:
+        """
+        Do a research to gather information for the user's request.
+        A researcher should have these tools: query engine, search engine, etc.
+        """
+        ctx.write_event_to_stream(
+            AgentRunEvent(
+                name="Researcher",
+                msg="Starting research",
+            )
+        )
+        tool_calls = ev.input
+
+        tool_call_outputs = await call_tools(
+            ctx=ctx,
+            agent_name="Researcher",
+            tools=[self.query_engine_tool],
+            tool_calls=tool_calls,
+        )
+        for tool_call_output in tool_call_outputs:
+            self.memory.put(
+                ChatMessage(
+                    role=MessageRole.TOOL,
+                    content=tool_call_output.tool_output.content,
+                    additional_kwargs={
+                        "name": tool_call_output.tool_output.tool_name,
+                        "tool_call_id": tool_call_output.tool_call_id,
+                    },
+                )
+            )
+        return AnalyzeEvent(
+            input=ChatMessage(
+                role=MessageRole.ASSISTANT,
+                content="Researcher: I've finished the research. Please analyze the result.",
+            ),
+        )
+
+    @step()
+    async def analyze(self, ctx: Context, ev: AnalyzeEvent) -> InputEvent:
+        """
+        Analyze the research result.
+        """
+        ctx.write_event_to_stream(
+            AgentRunEvent(
+                name="Analyst",
+                msg="Starting analysis",
+            )
+        )
+        event_requested_by_workflow_llm = isinstance(ev.input, list)
+        # Requested by the workflow LLM Input step, it's a tool call
+        if event_requested_by_workflow_llm:
+            # Set the tool calls
+            tool_calls = ev.input
+        else:
+            # Otherwise, it's triggered by the research step
+            # Use a custom prompt and independent memory for the analyst agent
+            analysis_prompt = """
+            You are a financial analyst, you are given a research result and a set of tools to help you.
+            Always use the given information, don't make up anything yourself. If there is not enough information, you can asking for more information.
+            If you have enough numerical information, it's good to include some charts/visualizations to the report so you can use the code interpreter tool to generate a report.
+            """
+            # This is handled by analyst agent
+            # Clone the shared memory to avoid conflicting with the workflow.
+            chat_history = self.memory.get()
+            chat_history.append(
+                ChatMessage(role=MessageRole.SYSTEM, content=analysis_prompt)
+            )
+            chat_history.append(ev.input)  # type: ignore
+            # Check if the analyst agent needs to call tools
+            response = await chat_with_tools(
+                self.llm,
+                [self.code_interpreter_tool],
+                chat_history,
+            )
+            if not response.has_tool_calls():
+                # If no tool call, fallback analyst message to the workflow
+                msg_content = await response.full_response()
+                analyst_msg = ChatMessage(
+                    role=MessageRole.ASSISTANT,
+                    content=f"Analyst: "
+                    f"\nHere is the analysis result: {msg_content}"
+                    "\nUse it for other steps or response the content to the user.",
+                )
+                self.memory.put(analyst_msg)
+                return InputEvent(input=self.memory.get())
+            else:
+                tool_calls = response.tool_calls
+                self.memory.put(response.tool_call_message)
+
+        # Call tools
+        tool_call_outputs = await call_tools(
+            ctx=ctx,
+            agent_name="Analyst",
+            tools=[self.code_interpreter_tool],
+            tool_calls=tool_calls,  # type: ignore
+        )
+        for tool_call_output in tool_call_outputs:
+            self.memory.put(
+                ChatMessage(
+                    role=MessageRole.TOOL,
+                    content=tool_call_output.tool_output.content,
+                    additional_kwargs={
+                        "name": tool_call_output.tool_output.tool_name,
+                        "tool_call_id": tool_call_output.tool_call_id,
+                    },
+                )
+            )
+
+        # Fallback to the input with the latest chat history
+        return InputEvent(input=self.memory.get())
+
+    @step()
+    async def report(self, ctx: Context, ev: ReportEvent) -> InputEvent:
+        """
+        Generate a report based on the analysis result.
+        """
+        ctx.write_event_to_stream(
+            AgentRunEvent(
+                name="Reporter",
+                msg="Starting report generation",
+            )
+        )
+        tool_calls = ev.input
+        tool_call_outputs = await call_tools(
+            ctx=ctx,
+            agent_name="Reporter",
+            tools=[self.document_generator_tool],
+            tool_calls=tool_calls,
+        )
+        for tool_call_output in tool_call_outputs:
+            self.memory.put(
+                ChatMessage(
+                    role=MessageRole.TOOL,
+                    content=tool_call_output.tool_output.content,
+                    additional_kwargs={
+                        "name": tool_call_output.tool_output.tool_name,
+                        "tool_call_id": tool_call_output.tool_call_id,
+                    },
+                )
+            )
+        # After the tool calls, fallback to the input with the latest chat history
+        return InputEvent(input=self.memory.get())

templates/components/workflows/typescript/agentic_rag/README-template.md

@@ -0,0 +1,52 @@
+This is a [LlamaIndex](https://www.llamaindex.ai/) project bootstrapped with [`create-llama`](https://github.com/run-llama/LlamaIndexTS/tree/main/packages/create-llama).
+
+## Getting Started
+
+First, install the dependencies:
+
+```
+npm install
+```
+
+Then check the parameters that have been pre-configured in the `.env` file in this directory.
+Make sure you have set the `OPENAI_API_KEY` for the LLM.
+
+Second, generate the embeddings of the example documents in the `./data` directory:
+
+```
+npm run generate
+```
+
+Third, run the development server:
+
+```
+npm run dev
+```
+
+Open [http://localhost:3000](http://localhost:3000) with your browser to see the chat UI.
+
+## Configure LLM and Embedding Model
+
+You can configure [LLM model](https://ts.llamaindex.ai/docs/llamaindex/modules/llms) and [embedding model](https://ts.llamaindex.ai/docs/llamaindex/modules/embeddings) in the [settings file](src/app/settings.ts).
+
+## Use Case
+
+We have prepared an [example workflow](./src/app/workflow.ts) for the agentic RAG use case, where you can ask questions about the example documents in the [./data](./data) directory.
+
+You can start by sending an request on the [chat UI](http://localhost:3000) or you can test the `/api/chat` endpoint with the following curl request:
+
+```shell
+curl --location 'localhost:3000/api/chat' \
+--header 'Content-Type: application/json' \
+--data '{ "messages": [{ "role": "user", "content": "What standards for a letter exist?" }] }'
+```
+
+## Learn More
+
+To learn more about LlamaIndex, take a look at the following resources:
+
+- [LlamaIndex Documentation](https://docs.llamaindex.ai) - learn about LlamaIndex (Python features).
+- [LlamaIndexTS Documentation](https://ts.llamaindex.ai/docs/llamaindex) - learn about LlamaIndex (Typescript features).
+- [Agent Workflows Introduction](https://ts.llamaindex.ai/docs/llamaindex/modules/agent_workflow) - learn about LlamaIndexTS Agent Workflows.
+
+You can check out [the LlamaIndexTS GitHub repository](https://github.com/run-llama/LlamaIndexTS) - your feedback and contributions are welcome!

templates/components/workflows/typescript/agentic_rag/workflow.ts

@@ -0,0 +1,16 @@
+import { agent } from "llamaindex";
+import { getIndex } from "./data";
+
+export const workflowFactory = async (reqBody: any) => {
+  const index = await getIndex(reqBody?.data);
+
+  const queryEngineTool = index.queryTool({
+    metadata: {
+      name: "query_document",
+      description: `This tool can retrieve information about Apple and Tesla financial data`,
+    },
+    includeSourceNodes: true,
+  });
+
+  return agent({ tools: [queryEngineTool] });
+};

templates/components/workflows/typescript/deep_research/README-template.md

@@ -0,0 +1,56 @@
+This is a [LlamaIndex](https://www.llamaindex.ai/) project bootstrapped with [`create-llama`](https://github.com/run-llama/LlamaIndexTS/tree/main/packages/create-llama).
+
+## Getting Started
+
+First, install the dependencies:
+
+```
+npm install
+```
+
+Then check the parameters that have been pre-configured in the `.env` file in this directory.
+Make sure you have set the `OPENAI_API_KEY` for the LLM.
+
+Second, generate the embeddings of the example documents in the `./data` directory:
+
+```
+npm run generate
+```
+
+Third, run the development server:
+
+```
+npm run dev
+```
+
+Open [http://localhost:3000](http://localhost:3000) with your browser to see the chat UI.
+
+## Configure LLM and Embedding Model
+
+You can configure [LLM model](https://ts.llamaindex.ai/docs/llamaindex/modules/llms) and [embedding model](https://ts.llamaindex.ai/docs/llamaindex/modules/embeddings) in the [settings file](src/app/settings.ts).
+
+## Custom UI Components
+
+For Deep Research, we have a custom component located in `components/deep_research_event.jsx`. This is used to display the results of the deep research workflow in a more user-friendly way
+
+## Use Case
+
+We have prepared an [example workflow](./src/app/workflow.ts) for the Deep Research use case, where you can request a detailed answer about the example documents in the [./data](./data) directory.
+
+You can start by sending an request on the [chat UI](http://localhost:3000) or you can test the `/api/chat` endpoint with the following curl request:
+
+```shell
+curl --location 'localhost:3000/api/chat' \
+--header 'Content-Type: application/json' \
+--data '{ "messages": [{ "role": "user", "content": "Compare the financial performance of Apple and Tesla" }] }'
+```
+
+## Learn More
+
+To learn more about LlamaIndex, take a look at the following resources:
+
+- [LlamaIndex Documentation](https://docs.llamaindex.ai) - learn about LlamaIndex (Python features).
+- [LlamaIndexTS Documentation](https://ts.llamaindex.ai/docs/llamaindex) - learn about LlamaIndex (Typescript features).
+- [Workflows Introduction](https://ts.llamaindex.ai/docs/llamaindex/modules/workflows) - learn about LlamaIndexTS workflows.
+
+You can check out [the LlamaIndexTS GitHub repository](https://github.com/run-llama/LlamaIndexTS) - your feedback and contributions are welcome!

templates/components/workflows/typescript/deep_research/workflow.ts

@@ -0,0 +1,434 @@
+import { toSourceEvent, toStreamGenerator } from "@llamaindex/server";
+import {
+  AgentInputData,
+  AgentWorkflowContext,
+  ChatMemoryBuffer,
+  ChatResponseChunk,
+  HandlerContext,
+  LlamaCloudIndex,
+  Metadata,
+  MetadataMode,
+  NodeWithScore,
+  PromptTemplate,
+  Settings,
+  StartEvent,
+  StopEvent as StopEventBase,
+  ToolCallLLM,
+  VectorStoreIndex,
+  Workflow,
+  WorkflowEvent,
+} from "llamaindex";
+import { randomUUID } from "node:crypto";
+import { z } from "zod";
+import { getIndex } from "./data";
+
+// workflow factory
+export const workflowFactory = async (reqBody: any) => {
+  const index = await getIndex(reqBody?.data);
+  return new DeepResearchWorkflow(index);
+};
+
+// workflow configs
+const MAX_QUESTIONS = 6; // max number of questions to research, research will stop when this number is reached
+const TIMEOUT = 360; // timeout in seconds
+const TOP_K = 10; // number of nodes to retrieve from the vector store
+
+const createPlanResearchPrompt = new PromptTemplate({
+  template: `
+You are a professor who is guiding a researcher to research a specific request/problem.
+Your task is to decide on a research plan for the researcher.
+
+The possible actions are:
++ Provide a list of questions for the researcher to investigate, with the purpose of clarifying the request.
++ Write a report if the researcher has already gathered enough research on the topic and can resolve the initial request.
++ Cancel the research if most of the answers from researchers indicate there is insufficient information to research the request. Do not attempt more than 3 research iterations or too many questions.
+
+The workflow should be:
++ Always begin by providing some initial questions for the researcher to investigate.
++ Analyze the provided answers against the initial topic/request. If the answers are insufficient to resolve the initial request, provide additional questions for the researcher to investigate.
++ If the answers are sufficient to resolve the initial request, instruct the researcher to write a report.
+
+Here are the context: 
+<Collected information>
+{context_str}
+</Collected information>
+
+<Conversation context>
+{conversation_context}
+</Conversation context>
+
+{enhanced_prompt}
+
+Now, provide your decision in the required format for this user request:
+<User request>
+{user_request}
+</User request>
+`,
+  templateVars: [
+    "context_str",
+    "conversation_context",
+    "enhanced_prompt",
+    "user_request",
+  ],
+});
+
+const researchPrompt = new PromptTemplate({
+  template: `
+You are a researcher who is in the process of answering the question.
+The purpose is to answer the question based on the collected information, without using prior knowledge or making up any new information.
+Always add citations to the sentence/point/paragraph using the id of the provided content.
+The citation should follow this format: [citation:id] where id is the id of the content.
+
+E.g:
+If we have a context like this:
+<Citation id='abc-xyz'>
+Baby llama is called cria
+</Citation id='abc-xyz'>
+
+And your answer uses the content, then the citation should be:
+- Baby llama is called cria [citation:abc-xyz]
+
+ Here is the provided context for the question:
+<Collected information>
+{context_str}
+</Collected information>
+
+No prior knowledge, just use the provided context to answer the question: {question}
+`,
+  templateVars: ["context_str", "question"],
+});
+
+const WRITE_REPORT_PROMPT = `
+You are a researcher writing a report based on a user request and the research context.
+You have researched various perspectives related to the user request.
+The report should provide a comprehensive outline covering all important points from the researched perspectives.
+Create a well-structured outline for the research report that covers all the answers.
+
+# IMPORTANT when writing in markdown format:
++ Use tables or figures where appropriate to enhance presentation.
++ Preserve all citation syntax (the \`[citation:id]()\` parts in the provided context). Keep these citations in the final report - no separate reference section is needed.
++ Do not add links, a table of contents, or a references section to the report.
+`;
+
+// workflow events
+type ResearchQuestion = { questionId: string; question: string };
+type ResearchResult = ResearchQuestion & { answer: string };
+
+class PlanResearchEvent extends WorkflowEvent<{}> {}
+class ResearchEvent extends WorkflowEvent<ResearchQuestion[]> {}
+class ReportEvent extends WorkflowEvent<{}> {}
+class StopEvent extends StopEventBase<AsyncGenerator<ChatResponseChunk>> {}
+
+type DeepResearchEventData = {
+  event: "retrieve" | "analyze" | "answer";
+  state: "pending" | "inprogress" | "done" | "error";
+  id?: string;
+  question?: string;
+  answer?: string;
+};
+
+class DeepResearchEvent extends WorkflowEvent<{
+  type: "ui_event";
+  data: DeepResearchEventData;
+}> {}
+
+// workflow definition
+class DeepResearchWorkflow extends Workflow<
+  AgentWorkflowContext,
+  AgentInputData,
+  string
+> {
+  #llm = Settings.llm as ToolCallLLM;
+  #index?: VectorStoreIndex | LlamaCloudIndex;
+
+  userRequest: string = "";
+  totalQuestions: number = 0;
+  contextNodes: NodeWithScore<Metadata>[] = [];
+  memory: ChatMemoryBuffer = new ChatMemoryBuffer({ llm: Settings.llm });
+
+  constructor(index: VectorStoreIndex | LlamaCloudIndex) {
+    super({ timeout: TIMEOUT });
+    this.#index = index;
+    this.addWorkflowSteps();
+  }
+
+  addWorkflowSteps() {
+    this.addStep(
+      {
+        inputs: [StartEvent<AgentInputData>],
+        outputs: [PlanResearchEvent],
+      },
+      this.handleStartWorkflow,
+    );
+    this.addStep(
+      {
+        inputs: [PlanResearchEvent],
+        outputs: [ResearchEvent, ReportEvent, StopEvent],
+      },
+      this.handlePlanResearch,
+    );
+    this.addStep(
+      {
+        inputs: [ResearchEvent],
+        outputs: [PlanResearchEvent],
+      },
+      this.handleResearch,
+    );
+    this.addStep(
+      {
+        inputs: [ReportEvent],
+        outputs: [StopEvent],
+      },
+      this.handleReport,
+    );
+  }
+
+  async initWorkflow(data: AgentInputData) {
+    const { userInput, chatHistory = [] } = data;
+    if (!userInput) throw new Error("Invalid input");
+
+    this.userRequest = userInput;
+
+    await this.memory.set(chatHistory);
+    await this.memory.put({ role: "user", content: userInput });
+  }
+
+  handleStartWorkflow = async (
+    ctx: HandlerContext<AgentWorkflowContext>,
+    ev: StartEvent<AgentInputData>,
+  ): Promise<PlanResearchEvent> => {
+    await this.initWorkflow(ev.data);
+
+    ctx.sendEvent(
+      new DeepResearchEvent({
+        type: "ui_event",
+        data: { event: "retrieve", state: "inprogress" },
+      }),
+    );
+
+    const retrievedNodes = await this.retriever.retrieve(this.userRequest);
+
+    ctx.sendEvent(toSourceEvent(retrievedNodes));
+
+    ctx.sendEvent(
+      new DeepResearchEvent({
+        type: "ui_event",
+        data: { event: "retrieve", state: "done" },
+      }),
+    );
+
+    this.contextNodes = retrievedNodes;
+
+    return new PlanResearchEvent({});
+  };
+
+  handlePlanResearch = async (
+    ctx: HandlerContext<AgentWorkflowContext>,
+    ev: PlanResearchEvent,
+  ): Promise<ResearchEvent | ReportEvent | StopEvent> => {
+    ctx.sendEvent(
+      new DeepResearchEvent({
+        type: "ui_event",
+        data: { event: "analyze", state: "inprogress" },
+      }),
+    );
+
+    const { decision, researchQuestions, cancelReason } =
+      await this.createResearchPlan();
+
+    // Stop workflow due to decision from LLM
+    if (decision === "cancel") {
+      ctx.sendEvent(
+        new DeepResearchEvent({
+          type: "ui_event",
+          data: { event: "analyze", state: "done" },
+        }),
+      );
+      return new StopEvent(
+        toStreamGenerator(
+          cancelReason ?? "Research cancelled without any reason.",
+        ),
+      );
+    }
+
+    // Trigger research from generated questions
+    if (decision === "research") {
+      this.memory.put({
+        role: "assistant",
+        content:
+          "We need to find answers to the following questions:\n" +
+          researchQuestions.join("\n"),
+      });
+
+      researchQuestions.forEach(({ questionId: id, question }) => {
+        ctx.sendEvent(
+          new DeepResearchEvent({
+            type: "ui_event",
+            data: { event: "answer", state: "pending", id, question },
+          }),
+        );
+      });
+
+      return new ResearchEvent(researchQuestions);
+    }
+
+    // Resarch done, start writing report
+    this.memory.put({
+      role: "assistant",
+      content: "No more idea to analyze. We should report the answers.",
+    });
+
+    ctx.sendEvent(
+      new DeepResearchEvent({
+        type: "ui_event",
+        data: { event: "analyze", state: "done" },
+      }),
+    );
+
+    return new ReportEvent({});
+  };
+
+  handleResearch = async (
+    ctx: HandlerContext<AgentWorkflowContext>,
+    ev: ResearchEvent,
+  ): Promise<PlanResearchEvent> => {
+    const researchQuestions = ev.data;
+
+    // Answer questions in parallel
+    const researchResults: ResearchResult[] = await Promise.all(
+      researchQuestions.map(async ({ questionId: id, question }) => {
+        ctx.sendEvent(
+          new DeepResearchEvent({
+            type: "ui_event",
+            data: { event: "answer", state: "inprogress", id, question },
+          }),
+        );
+
+        const answer = await this.answerQuestion(question);
+
+        ctx.sendEvent(
+          new DeepResearchEvent({
+            type: "ui_event",
+            data: { event: "answer", state: "done", id, question, answer },
+          }),
+        );
+
+        return { questionId: id, question, answer };
+      }),
+    );
+
+    // Save answers to memory
+    researchResults.forEach(({ question, answer }) => {
+      this.memory.put({
+        role: "assistant",
+        content: `<Question>${question}</Question>\n<Answer>${answer}</Answer>`,
+      });
+    });
+
+    this.memory.put({
+      role: "assistant",
+      content:
+        "Researched all the questions. Now, I need to analyze if it's ready to write a report or need to research more.",
+    });
+
+    this.totalQuestions += researchResults.length;
+
+    return new PlanResearchEvent({});
+  };
+
+  handleReport = async (
+    ctx: HandlerContext<AgentWorkflowContext>,
+    ev: ReportEvent,
+  ): Promise<StopEvent> => {
+    const chatHistory = await this.memory.getAllMessages();
+
+    const messages = chatHistory.concat([
+      {
+        role: "system",
+        content: WRITE_REPORT_PROMPT,
+      },
+      {
+        role: "user",
+        content:
+          "Write a report addressing the user request based on the research provided the context",
+      },
+    ]);
+
+    const stream = await this.llm.chat({ messages, stream: true });
+
+    return new StopEvent(toStreamGenerator(stream));
+  };
+
+  get llm() {
+    if (!this.#llm.supportToolCall) throw new Error("LLM is not a ToolCallLLM");
+    return this.#llm;
+  }
+
+  get retriever() {
+    if (!this.#index) throw new Error("Index is not initialized");
+    return this.#index.asRetriever({ similarityTopK: TOP_K });
+  }
+
+  get contextStr() {
+    return this.contextNodes
+      .map((node) => {
+        const nodeId = node.node.id_;
+        const nodeContent = node.node.getContent(MetadataMode.NONE);
+        return `<Citation id='${nodeId}'>\n${nodeContent}</Citation id='${nodeId}'>`;
+      })
+      .join("\n");
+  }
+
+  get enhancedPrompt() {
+    if (this.totalQuestions === 0) {
+      return "The student has no questions to research. Let start by asking some questions.";
+    }
+
+    if (this.totalQuestions > MAX_QUESTIONS) {
+      return `The student has researched ${this.totalQuestions} questions. Should cancel the research if the context is not enough to write a report.`;
+    }
+
+    return "";
+  }
+
+  async createResearchPlan() {
+    const chatHistory = await this.memory.getMessages();
+
+    const conversationContext = chatHistory
+      .map((message) => `${message.role}: ${message.content}`)
+      .join("\n");
+
+    const prompt = createPlanResearchPrompt.format({
+      context_str: this.contextStr,
+      conversation_context: conversationContext,
+      enhanced_prompt: this.enhancedPrompt,
+      user_request: this.userRequest,
+    });
+
+    const responseFormat = z.object({
+      decision: z.enum(["research", "write", "cancel"]),
+      researchQuestions: z.array(z.string()),
+      cancelReason: z.string().optional(),
+    });
+
+    const result = await this.llm.complete({ prompt, responseFormat });
+    const plan = JSON.parse(result.text) as z.infer<typeof responseFormat>;
+
+    return {
+      ...plan,
+      researchQuestions: plan.researchQuestions.map((question) => ({
+        questionId: randomUUID(),
+        question,
+      })),
+    };
+  }
+
+  async answerQuestion(question: string) {
+    const prompt = researchPrompt.format({
+      context_str: this.contextStr,
+      question,
+    });
+    const result = await this.llm.complete({ prompt });
+    return result.text;
+  }
+}

templates/components/workflows/typescript/financial_report/README-template.md

@@ -0,0 +1,52 @@
+This is a [LlamaIndex](https://www.llamaindex.ai/) project bootstrapped with [`create-llama`](https://github.com/run-llama/LlamaIndexTS/tree/main/packages/create-llama).
+
+## Getting Started
+
+First, install the dependencies:
+
+```
+npm install
+```
+
+Then check the parameters that have been pre-configured in the `.env` file in this directory.
+Make sure you have set the `OPENAI_API_KEY` for the LLM and the `E2B_API_KEY` for the code interpreter. You can get the E2B API key from [here](https://e2b.dev).
+
+Second, generate the embeddings of the example documents in the `./data` directory:
+
+```
+npm run generate
+```
+
+Third, run the development server:
+
+```
+npm run dev
+```
+
+Open [http://localhost:3000](http://localhost:3000) with your browser to see the chat UI.
+
+## Configure LLM and Embedding Model
+
+You can configure [LLM model](https://ts.llamaindex.ai/docs/llamaindex/modules/llms) and [embedding model](https://ts.llamaindex.ai/docs/llamaindex/modules/embeddings) in the [settings file](src/app/settings.ts).
+
+## Use Case
+
+We have prepared an [example workflow](./src/app/workflow.ts) for the financial report generation use case, where you can request for the app to generate a financial report using the example documents in the [./data](./data).
+
+You can start by sending an request on the [chat UI](http://localhost:3000) or you can test the `/api/chat` endpoint with the following curl request:
+
+```shell
+curl --location 'localhost:3000/api/chat' \
+--header 'Content-Type: application/json' \
+--data '{ "messages": [{ "role": "user", "content": "Generate a financial report that compares the financial performance of Apple and Tesla" }] }'
+```
+
+## Learn More
+
+To learn more about LlamaIndex, take a look at the following resources:
+
+- [LlamaIndex Documentation](https://docs.llamaindex.ai) - learn about LlamaIndex (Python features).
+- [LlamaIndexTS Documentation](https://ts.llamaindex.ai/docs/llamaindex) - learn about LlamaIndex (Typescript features).
+- [Workflows Introduction](https://ts.llamaindex.ai/docs/llamaindex/modules/workflows) - learn about LlamaIndexTS workflows.
+
+You can check out [the LlamaIndexTS GitHub repository](https://github.com/run-llama/LlamaIndexTS) - your feedback and contributions are welcome!

templates/components/workflows/typescript/financial_report/workflow.ts

@@ -0,0 +1,396 @@
+import { toAgentRunEvent, toSourceEvent } from "@llamaindex/server";
+import {
+  callTools,
+  chatWithTools,
+  documentGenerator,
+  interpreter,
+} from "@llamaindex/tools";
+import {
+  AgentInputData,
+  AgentWorkflowContext,
+  BaseToolWithCall,
+  ChatMemoryBuffer,
+  ChatMessage,
+  ChatResponseChunk,
+  HandlerContext,
+  Metadata,
+  NodeWithScore,
+  Settings,
+  StartEvent,
+  StopEvent,
+  ToolCall,
+  ToolCallLLM,
+  Workflow,
+  WorkflowEvent,
+} from "llamaindex";
+import { getIndex } from "./data";
+
+const TIMEOUT = 360 * 1000;
+
+export async function workflowFactory(reqBody: any) {
+  const index = await getIndex(reqBody?.data);
+
+  const queryEngineTool = index.queryTool({
+    metadata: {
+      name: "query_document",
+      description: `This tool can retrieve information about Apple and Tesla financial data`,
+    },
+    includeSourceNodes: true,
+  });
+
+  if (!process.env.E2B_API_KEY) {
+    throw new Error("E2B_API_KEY is required to use the code interpreter tool");
+  }
+
+  const codeInterpreterTool = interpreter({
+    apiKey: process.env.E2B_API_KEY!,
+  });
+  const documentGeneratorTool = documentGenerator();
+
+  return new FinancialReportWorkflow({
+    queryEngineTool,
+    codeInterpreterTool,
+    documentGeneratorTool,
+    timeout: TIMEOUT,
+  });
+}
+
+// Create a custom event type
+class InputEvent extends WorkflowEvent<{ input: ChatMessage[] }> {}
+
+class ResearchEvent extends WorkflowEvent<{
+  toolCalls: ToolCall[];
+}> {}
+
+class AnalyzeEvent extends WorkflowEvent<{
+  input: ChatMessage | ToolCall[];
+}> {}
+
+class ReportGenerationEvent extends WorkflowEvent<{
+  toolCalls: ToolCall[];
+}> {}
+
+const DEFAULT_SYSTEM_PROMPT = `
+You are a financial analyst who are given a set of tools to help you.
+It's good to using appropriate tools for the user request and always use the information from the tools, don't make up anything yourself.
+For the query engine tool, you should break down the user request into a list of queries and call the tool with the queries.
+`;
+
+class FinancialReportWorkflow extends Workflow<
+  AgentWorkflowContext,
+  AgentInputData,
+  string
+> {
+  llm: ToolCallLLM;
+  memory: ChatMemoryBuffer;
+  queryEngineTool: BaseToolWithCall;
+  codeInterpreterTool: BaseToolWithCall;
+  documentGeneratorTool: BaseToolWithCall;
+  systemPrompt?: string;
+
+  constructor(options: {
+    queryEngineTool: BaseToolWithCall;
+    codeInterpreterTool: BaseToolWithCall;
+    documentGeneratorTool: BaseToolWithCall;
+    systemPrompt?: string;
+    verbose?: boolean;
+    timeout?: number;
+  }) {
+    super({
+      verbose: options?.verbose ?? false,
+      timeout: options?.timeout ?? 360,
+    });
+
+    this.llm = Settings.llm as ToolCallLLM;
+    if (!this.llm.supportToolCall) {
+      throw new Error("LLM is not a ToolCallLLM");
+    }
+    this.systemPrompt = options.systemPrompt ?? DEFAULT_SYSTEM_PROMPT;
+    this.queryEngineTool = options.queryEngineTool;
+    this.codeInterpreterTool = options.codeInterpreterTool;
+
+    this.documentGeneratorTool = options.documentGeneratorTool;
+    this.memory = new ChatMemoryBuffer({ llm: this.llm, chatHistory: [] });
+
+    // Add steps
+    this.addStep(
+      {
+        inputs: [StartEvent<AgentInputData>],
+        outputs: [InputEvent],
+      },
+      this.prepareChatHistory,
+    );
+
+    this.addStep(
+      {
+        inputs: [InputEvent],
+        outputs: [
+          InputEvent,
+          ResearchEvent,
+          AnalyzeEvent,
+          ReportGenerationEvent,
+          StopEvent,
+        ],
+      },
+      this.handleLLMInput,
+    );
+
+    this.addStep(
+      {
+        inputs: [ResearchEvent],
+        outputs: [AnalyzeEvent],
+      },
+      this.handleResearch,
+    );
+
+    this.addStep(
+      {
+        inputs: [AnalyzeEvent],
+        outputs: [InputEvent],
+      },
+      this.handleAnalyze,
+    );
+
+    this.addStep(
+      {
+        inputs: [ReportGenerationEvent],
+        outputs: [InputEvent],
+      },
+      this.handleReportGeneration,
+    );
+  }
+
+  prepareChatHistory = async (
+    ctx: HandlerContext<AgentWorkflowContext>,
+    ev: StartEvent<AgentInputData>,
+  ): Promise<InputEvent> => {
+    const { userInput, chatHistory = [] } = ev.data;
+    if (!userInput) throw new Error("Invalid input");
+
+    this.memory.set(chatHistory);
+
+    if (this.systemPrompt) {
+      this.memory.put({ role: "system", content: this.systemPrompt });
+    }
+
+    this.memory.put({ role: "user", content: userInput });
+
+    const messages = await this.memory.getMessages();
+    return new InputEvent({ input: messages });
+  };
+
+  handleLLMInput = async (
+    ctx: HandlerContext<AgentWorkflowContext>,
+    ev: InputEvent,
+  ): Promise<
+    | InputEvent
+    | ResearchEvent
+    | AnalyzeEvent
+    | ReportGenerationEvent
+    | StopEvent<AsyncGenerator<ChatResponseChunk, any, any> | undefined>
+  > => {
+    const chatHistory = ev.data.input;
+
+    const tools = [
+      this.codeInterpreterTool,
+      this.documentGeneratorTool,
+      this.queryEngineTool,
+    ];
+
+    const toolCallResponse = await chatWithTools(this.llm, tools, chatHistory);
+
+    if (!toolCallResponse.hasToolCall()) {
+      return new StopEvent(toolCallResponse.responseGenerator);
+    }
+
+    if (toolCallResponse.hasMultipleTools()) {
+      this.memory.put({
+        role: "assistant",
+        content:
+          "Calling different tools is not allowed. Please only use multiple calls of the same tool.",
+      });
+      const chatHistory = await this.memory.getMessages();
+      return new InputEvent({ input: chatHistory });
+    }
+
+    // Put the LLM tool call message into the memory
+    // And trigger the next step according to the tool call
+    if (toolCallResponse.toolCallMessage) {
+      this.memory.put(toolCallResponse.toolCallMessage);
+    }
+    const toolName = toolCallResponse.getToolNames()[0];
+    switch (toolName) {
+      case this.codeInterpreterTool.metadata.name:
+        return new AnalyzeEvent({
+          input: toolCallResponse.toolCalls,
+        });
+      case this.documentGeneratorTool.metadata.name:
+        return new ReportGenerationEvent({
+          toolCalls: toolCallResponse.toolCalls,
+        });
+      default:
+        if (this.queryEngineTool.metadata.name === toolName) {
+          return new ResearchEvent({
+            toolCalls: toolCallResponse.toolCalls,
+          });
+        }
+        throw new Error(`Unknown tool: ${toolName}`);
+    }
+  };
+
+  handleResearch = async (
+    ctx: HandlerContext<AgentWorkflowContext>,
+    ev: ResearchEvent,
+  ): Promise<AnalyzeEvent> => {
+    ctx.sendEvent(
+      toAgentRunEvent({
+        agent: "Researcher",
+        text: "Researching data",
+        type: "text",
+      }),
+    );
+
+    const { toolCalls } = ev.data;
+
+    const toolMsgs = await callTools({
+      tools: [this.queryEngineTool],
+      toolCalls,
+      writeEvent: (text, step) => {
+        ctx.sendEvent(
+          toAgentRunEvent({
+            agent: "Researcher",
+            text,
+            type: toolCalls.length > 1 ? "progress" : "text",
+            current: step,
+            total: toolCalls.length,
+          }),
+        );
+      },
+    });
+    for (const toolMsg of toolMsgs) {
+      this.memory.put(toolMsg);
+    }
+
+    const sourcesNodes: NodeWithScore<Metadata>[] = toolMsgs
+      .map((msg) => (msg.options as any)?.toolResult?.result?.sourceNodes)
+      .flat()
+      .filter(Boolean);
+
+    if (sourcesNodes.length > 0) {
+      ctx.sendEvent(toSourceEvent(sourcesNodes));
+    }
+
+    return new AnalyzeEvent({
+      input: {
+        role: "assistant",
+        content:
+          "I have finished researching the data, please analyze the data.",
+      },
+    });
+  };
+
+  /**
+   * Analyze a research result or a tool call for code interpreter from the LLM
+   */
+  handleAnalyze = async (
+    ctx: HandlerContext<AgentWorkflowContext>,
+    ev: AnalyzeEvent,
+  ): Promise<InputEvent> => {
+    ctx.sendEvent(
+      toAgentRunEvent({
+        agent: "Analyst",
+        text: "Analyzing data",
+        type: "text",
+      }),
+    );
+    // Request by workflow LLM, input is a list of tool calls
+    let toolCalls: ToolCall[] = [];
+    if (Array.isArray(ev.data.input)) {
+      toolCalls = ev.data.input;
+    } else {
+      // Requested by Researcher, input is a ChatMessage
+      // We start new LLM chat specifically for analyzing the data
+      const analysisPrompt = `
+      You are an expert in analyzing financial data.
+      You are given a set of financial data to analyze. Your task is to analyze the financial data and return a report.
+      Your response should include a detailed analysis of the financial data, including any trends, patterns, or insights that you find.
+      Construct the analysis in textual format; including tables would be great!
+      Don't need to synthesize the data, just analyze and provide your findings.
+      `;
+
+      // Clone the current chat history
+      // Add the analysis system prompt and the message from the researcher
+      const chatHistory = await this.memory.getMessages();
+      const newChatHistory = [
+        ...chatHistory,
+        { role: "system", content: analysisPrompt },
+        ev.data.input,
+      ];
+      const toolCallResponse = await chatWithTools(
+        this.llm,
+        [this.codeInterpreterTool],
+        newChatHistory as ChatMessage[],
+      );
+
+      if (!toolCallResponse.hasToolCall()) {
+        this.memory.put(await toolCallResponse.asFullResponse());
+        const chatHistory = await this.memory.getMessages();
+        return new InputEvent({ input: chatHistory });
+      } else {
+        this.memory.put(toolCallResponse.toolCallMessage!);
+        toolCalls = toolCallResponse.toolCalls;
+      }
+    }
+
+    // Call the tools
+    const toolMsgs = await callTools({
+      tools: [this.codeInterpreterTool],
+      toolCalls,
+      writeEvent: (text, step) => {
+        ctx.sendEvent(
+          toAgentRunEvent({
+            agent: "Analyst",
+            text,
+            type: toolCalls.length > 1 ? "progress" : "text",
+            current: step,
+            total: toolCalls.length,
+          }),
+        );
+      },
+    });
+    for (const toolMsg of toolMsgs) {
+      this.memory.put(toolMsg);
+    }
+
+    const chatHistory = await this.memory.getMessages();
+    return new InputEvent({ input: chatHistory });
+  };
+
+  handleReportGeneration = async (
+    ctx: HandlerContext<AgentWorkflowContext>,
+    ev: ReportGenerationEvent,
+  ): Promise<InputEvent> => {
+    const { toolCalls } = ev.data;
+
+    const toolMsgs = await callTools({
+      tools: [this.documentGeneratorTool],
+      toolCalls,
+      writeEvent: (text, step) => {
+        ctx.sendEvent(
+          toAgentRunEvent({
+            agent: "Reporter",
+            text,
+            type: toolCalls.length > 1 ? "progress" : "text",
+            current: step,
+            total: toolCalls.length,
+          }),
+        );
+      },
+    });
+    for (const toolMsg of toolMsgs) {
+      this.memory.put(toolMsg);
+    }
+    const chatHistory = await this.memory.getMessages();
+    return new InputEvent({ input: chatHistory });
+  };
+}

templates/types/llamaindexserver/fastapi/app/index.py

@@ -0,0 +1,23 @@
+import logging
+import os
+from typing import Optional
+
+from llama_index.core.indices import load_index_from_storage
+from llama_index.server.api.models import ChatRequest
+from llama_index.server.tools.index.utils import get_storage_context
+
+logger = logging.getLogger("uvicorn")
+
+STORAGE_DIR = "storage"
+
+
+def get_index(chat_request: Optional[ChatRequest] = None):
+    # check if storage already exists
+    if not os.path.exists(STORAGE_DIR):
+        return None
+    # load the existing index
+    logger.info(f"Loading index from {STORAGE_DIR}...")
+    storage_context = get_storage_context(STORAGE_DIR)
+    index = load_index_from_storage(storage_context)
+    logger.info(f"Finished loading index from {STORAGE_DIR}")
+    return index

templates/types/llamaindexserver/fastapi/app/settings.py

@@ -0,0 +1,8 @@
+from llama_index.core import Settings
+from llama_index.embeddings.openai import OpenAIEmbedding
+from llama_index.llms.openai import OpenAI
+
+
+def init_settings():
+    Settings.llm = OpenAI(model="gpt-4o-mini")
+    Settings.embed_model = OpenAIEmbedding(model="text-embedding-3-small")

templates/types/llamaindexserver/fastapi/generate.py

@@ -0,0 +1,63 @@
+import logging
+import os
+
+from dotenv import load_dotenv
+from llama_index.llms.openai import OpenAI
+
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger()
+
+
+def generate_index():
+    """
+    Index the documents in the data directory.
+    """
+    from app.index import STORAGE_DIR
+    from app.settings import init_settings
+    from llama_index.core.indices import (
+        VectorStoreIndex,
+    )
+    from llama_index.core.readers import SimpleDirectoryReader
+
+    load_dotenv()
+    init_settings()
+
+    logger.info("Creating new index")
+    # load the documents and create the index
+    reader = SimpleDirectoryReader(
+        os.environ.get("DATA_DIR", "data"),
+        recursive=True,
+    )
+    documents = reader.load_data()
+    index = VectorStoreIndex.from_documents(
+        documents,
+        show_progress=True,
+    )
+    # store it for later
+    index.storage_context.persist(STORAGE_DIR)
+    logger.info(f"Finished creating new index. Stored in {STORAGE_DIR}")
+
+
+def generate_ui_for_workflow():
+    """
+    Generate UI for UIEventData event in app/workflow.py
+    """
+    import asyncio
+
+    from main import COMPONENT_DIR
+
+    # To generate UI components for additional event types,
+    # import the corresponding data model (e.g., MyCustomEventData)
+    # and run the generate_ui_for_workflow function with the imported model.
+    # Make sure the output filename of the generated UI component matches the event type (here `ui_event`)
+    try:
+        from app.workflow import UIEventData
+    except ImportError:
+        raise ImportError("Couldn't generate UI component for the current workflow.")
+    from llama_index.server.gen_ui import generate_event_component
+
+    # works also well with Claude 3.7 Sonnet or Gemini Pro 2.5
+    llm = OpenAI(model="gpt-4.1")
+    code = asyncio.run(generate_event_component(event_cls=UIEventData, llm=llm))
+    with open(f"{COMPONENT_DIR}/ui_event.jsx", "w") as f:
+        f.write(code)

templates/types/llamaindexserver/fastapi/gitignore

@@ -0,0 +1,5 @@
+__pycache__
+storage
+.env
+output
+.ui/

templates/types/llamaindexserver/fastapi/main.py

@@ -0,0 +1,46 @@
+import logging
+import os
+import subprocess
+
+from app.settings import init_settings
+from app.workflow import create_workflow
+from dotenv import load_dotenv
+from llama_index.server import LlamaIndexServer, UIConfig
+
+logger = logging.getLogger("uvicorn")
+
+# A path to a directory where the customized UI code is stored
+COMPONENT_DIR = "components"
+
+
+def create_app():
+    env = os.environ.get("APP_ENV")
+
+    app = LlamaIndexServer(
+        workflow_factory=create_workflow,  # A factory function that creates a new workflow for each request
+        ui_config=UIConfig(
+            component_dir=COMPONENT_DIR,
+            app_title="Chat App",
+        ),
+        env=env,
+        logger=logger,
+    )
+    # You can also add custom routes to the app
+    app.add_api_route("/api/health", lambda: {"message": "OK"}, status_code=200)
+    return app
+
+
+load_dotenv()
+init_settings()
+app = create_app()
+
+
+def run(env: str):
+    os.environ["APP_ENV"] = env
+    app_host = os.getenv("APP_HOST", "0.0.0.0")
+    app_port = os.getenv("APP_PORT", "8000")
+
+    if env == "dev":
+        subprocess.run(["fastapi", "dev", "--host", app_host, "--port", app_port])
+    else:
+        subprocess.run(["fastapi", "run", "--host", app_host, "--port", app_port])

templates/types/llamaindexserver/fastapi/pyproject.toml

@@ -0,0 +1,49 @@
+[tool]
+[tool.poetry]
+name = "app"
+version = "0.1.0"
+description = ""
+authors = ["Marcus Schiesser <mail@marcusschiesser.de>"]
+readme = "README.md"
+
+[tool.poetry.scripts]
+"generate" = "generate:generate_index"
+"generate:index" = "generate:generate_index"
+"generate:ui" = "generate:generate_ui_for_workflow"
+dev = "main:run('dev')"
+prod = "main:run('prod')"
+
+[tool.poetry.dependencies]
+python = ">=3.11,<3.14"
+python-dotenv = "^1.0.0"
+pydantic = "<2.10"
+aiostream = "^0.5.2"
+llama-index-core = "^0.12.28"
+llama-index-server = "^0.1.12"
+
+[tool.poetry.group.dev.dependencies]
+mypy = "^1.8.0"
+pytest = "^8.3.5"
+pytest-asyncio = "^0.25.3"
+
+[tool.mypy]
+python_version = "3.11"
+plugins = "pydantic.mypy"
+exclude = ["tests", "venv", ".venv", "output", "config"]
+check_untyped_defs = true
+warn_unused_ignores = false
+show_error_codes = true
+namespace_packages = true
+ignore_missing_imports = true
+follow_imports = "silent"
+implicit_optional = true
+strict_optional = false
+disable_error_code = ["return-value", "assignment"]
+
+[[tool.mypy.overrides]]
+module = "app.*"
+ignore_missing_imports = false
+
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"

templates/types/llamaindexserver/nextjs/gitignore

@@ -0,0 +1,40 @@
+# See https://help.github.com/articles/ignoring-files/ for more about ignoring files.
+
+# dependencies
+/node_modules
+/.pnp
+.pnp.js
+
+# testing
+/coverage
+
+# next.js
+/.next/
+/out/
+
+# production
+/build
+
+# misc
+.DS_Store
+*.pem
+
+# debug
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+
+# local env files
+.env*.local
+
+# vercel
+.vercel
+
+# typescript
+*.tsbuildinfo
+next-env.d.ts
+
+output/
+storage/
+
+.env

templates/types/llamaindexserver/nextjs/package.json

@@ -0,0 +1,23 @@
+{
+  "name": "llamaindex-server",
+  "version": "0.1.0",
+  "scripts": {
+    "generate": "tsx src/generate.ts",
+    "dev": "tsx watch src/index.ts",
+    "start": "tsx src/index.ts"
+  },
+  "dependencies": {
+    "@llamaindex/openai": "0.2.0",
+    "@llamaindex/readers": "^2.0.0",
+    "@llamaindex/server": "0.1.0",
+    "@llamaindex/tools": "0.0.4",
+    "dotenv": "^16.4.7",
+    "llamaindex": "0.9.17",
+    "zod": "^3.23.8"
+  },
+  "devDependencies": {
+    "@types/node": "^20.10.3",
+    "tsx": "^4.7.2",
+    "typescript": "^5.3.2"
+  }
+}

templates/types/llamaindexserver/nextjs/src/.prettierrc.json

@@ -0,0 +1,3 @@
+{
+  "trailingComma": "all"
+}

templates/types/llamaindexserver/nextjs/src/app/data.ts

@@ -0,0 +1,22 @@
+import {
+  SimpleDocumentStore,
+  storageContextFromDefaults,
+  VectorStoreIndex,
+} from "llamaindex";
+
+export async function getIndex(params?: any) {
+  const storageContext = await storageContextFromDefaults({
+    persistDir: "storage",
+  });
+
+  const numberOfDocs = Object.keys(
+    (storageContext.docStore as SimpleDocumentStore).toDict(),
+  ).length;
+  if (numberOfDocs === 0) {
+    throw new Error(
+      "Index not found. Please run `pnpm run generate` to generate the embeddings of the documents",
+    );
+  }
+
+  return await VectorStoreIndex.init({ storageContext });
+}