Release 0.1.37 (#239)

Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>

.github/workflows/e2e.yml

@@ -18,6 +18,8 @@ jobs:
         node-version: [18, 20]
         python-version: ["3.11"]
         os: [macos-latest, windows-latest, ubuntu-22.04]
+        frameworks: ["nextjs", "express", "fastapi"]
+        datasources: ["--no-files", "--example-file"]
     defaults:
       run:
         shell: bash
@@ -63,6 +65,8 @@ jobs:
         env:
           OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
           LLAMA_CLOUD_API_KEY: ${{ secrets.LLAMA_CLOUD_API_KEY }}
+          FRAMEWORK: ${{ matrix.frameworks }}
+          DATASOURCE: ${{ matrix.datasources }}
         working-directory: .
 
       - uses: actions/upload-artifact@v3

.github/workflows/lint_on_push_or_pull.yml

@@ -30,3 +30,13 @@ jobs:
 
       - name: Run Prettier
         run: pnpm run format
+
+      - name: Run Python format check
+        uses: chartboost/ruff-action@v1
+        with:
+          args: "format --check"
+
+      - name: Run Python lint
+        uses: chartboost/ruff-action@v1
+        with:
+          args: "check"

CHANGELOG.md

@@ -1,5 +1,101 @@
 # create-llama
 
+## 0.1.37
+
+### Patch Changes
+
+- 9fd832c: Add in-text citation references
+
+## 0.1.36
+
+### Patch Changes
+
+- 2b7a5d8: Fix: private file upload not working in Python without LlamaCloud
+
+## 0.1.35
+
+### Patch Changes
+
+- 81ef7f0: Use LlamaCloud pipeline for data ingestion (private file uploads and generate script)
+
+## 0.1.34
+
+### Patch Changes
+
+- c49a5e1: Add error handling for generating the next question
+- c49a5e1: Fix wrong api key variable in Azure OpenAI provider
+
+## 0.1.33
+
+### Patch Changes
+
+- d746c75: Add Weaviate vector store (Typescript)
+
+## 0.1.32
+
+### Patch Changes
+
+- 3ec5163: Add Weaviate vector database support (Python)
+
+## 0.1.31
+
+### Patch Changes
+
+- 04a9c71: Cluster nodes by document
+
+## 0.1.30
+
+### Patch Changes
+
+- 09e3022: Add support for LlamaTrace (Python)
+- c06ec4f: Fix imports for MongoDB
+- b6dd7a9: Always send chat data when submit message
+
+## 0.1.29
+
+### Patch Changes
+
+- 8890e27: Let user change indexes in LlamaCloud projects
+
+## 0.1.28
+
+### Patch Changes
+
+- 9a09e8c: Fix Vercel deployment
+
+## 0.1.27
+
+### Patch Changes
+
+- c5c7eee: Make components reusable for chat-llamaindex
+
+## 0.1.26
+
+### Patch Changes
+
+- f43399c: Add metadatafilters to context chat engine (Typescript)
+
+## 0.1.25
+
+### Patch Changes
+
+- c67daeb: fix: missing set private to false for default generate.py
+
+## 0.1.24
+
+### Patch Changes
+
+- 43474a5: Configure LlamaCloud organization ID for Python
+- cf11b23: Add Azure code interpreter for Python and TS
+- fd9fb42: Add Azure OpenAI as model provider
+- 5c13646: Fix starter questions not working in python backend
+
+## 0.1.23
+
+### Patch Changes
+
+- 6bd76fb: Add template for structured extraction
+
 ## 0.1.22
 
 ### Patch Changes

README.md

@@ -2,8 +2,6 @@
 
 The easiest way to get started with [LlamaIndex](https://www.llamaindex.ai/) is by using `create-llama`. This CLI tool enables you to quickly start building a new LlamaIndex application, with everything set up for you.
 
-https://github.com/user-attachments/assets/dd3edc36-4453-4416-91c2-d24326c6c167
-
 ## Get started
 
 Just run
@@ -12,7 +10,11 @@ Just run
 npx create-llama@latest
 ```
 
-to get started, or see below for more options. Once your app is generated, run
+to get started, or watch this video for a demo session:
+
+https://github.com/user-attachments/assets/dd3edc36-4453-4416-91c2-d24326c6c167
+
+Once your app is generated, run
 
 ```bash
 npm run dev
@@ -22,16 +24,20 @@ to start the development server. You can then visit [http://localhost:3000](http
 
 ## What you'll get
 
-- A Next.js-powered front-end using components from [shadcn/ui](https://ui.shadcn.com/). The app is set up as a chat interface that can answer questions about your data (see below)
+- A Next.js-powered front-end using components from [shadcn/ui](https://ui.shadcn.com/). The app is set up as a chat interface that can answer questions about your data or interact with your agent
 - Your choice of 3 back-ends:
   - **Next.js**: if you select this option, you’ll have a full-stack Next.js application that you can deploy to a host like [Vercel](https://vercel.com/) in just a few clicks. This uses [LlamaIndex.TS](https://www.npmjs.com/package/llamaindex), our TypeScript library.
   - **Express**: if you want a more traditional Node.js application you can generate an Express backend. This also uses LlamaIndex.TS.
-  - **Python FastAPI**: if you select this option, you’ll get a backend powered by the [llama-index python package](https://pypi.org/project/llama-index/), which you can deploy to a service like Render or fly.io.
+  - **Python FastAPI**: if you select this option, you’ll get a backend powered by the [llama-index Python package](https://pypi.org/project/llama-index/), which you can deploy to a service like Render or fly.io.
 - The back-end has two endpoints (one streaming, the other one non-streaming) that allow you to send the state of your chat and receive additional responses
 - You add arbitrary data sources to your chat, like local files, websites, or data retrieved from a database.
 - Turn your chat into an AI agent by adding tools (functions called by the LLM).
 - The app uses OpenAI by default, so you'll need an OpenAI API key, or you can customize it to use any of the dozens of LLMs we support.
 
+Here's how it looks like:
+
+https://github.com/user-attachments/assets/d57af1a1-d99b-4e9c-98d9-4cbd1327eff8
+
 ## Using your data
 
 You can supply your own data; the app will index it and answer questions. Your generated app will have a folder called `data` (If you're using Express or Python and generate a frontend, it will be `./backend/data`).
@@ -88,7 +94,7 @@ Need to install the following packages:
   create-llama@latest
 Ok to proceed? (y) y
 ✔ What is your project named? … my-app
-✔ Which template would you like to use? › Chat
+✔ Which template would you like to use? › Agentic RAG (single agent)
 ✔ Which framework would you like to use? › NextJS
 ✔ Would you like to set up observability? › No
 ✔ Please provide your OpenAI API key (leave blank to skip): …
@@ -96,6 +102,7 @@ Ok to proceed? (y) y
 ✔ Would you like to add another data source? › No
 ✔ Would you like to use LlamaParse (improved parser for RAG - requires API key)? … no / yes
 ✔ Would you like to use a vector database? › No, just store the data in the file system
+✔ Would you like to build an agent using tools? If so, select the tools here, otherwise just press enter › Weather
 ? How would you like to proceed? › - Use arrow-keys. Return to submit.
    Just generate code (~1 sec)
 ❯  Start in VSCode (~1 sec)

create-app.ts

@@ -9,7 +9,7 @@ import { makeDir } from "./helpers/make-dir";
 
 import fs from "fs";
 import terminalLink from "terminal-link";
-import type { InstallTemplateArgs } from "./helpers";
+import type { InstallTemplateArgs, TemplateObservability } from "./helpers";
 import { installTemplate } from "./helpers";
 import { writeDevcontainer } from "./helpers/devcontainer";
 import { templatesDir } from "./helpers/dir";
@@ -142,14 +142,7 @@ export async function createApp({
     )} and learn how to get started.`,
   );
 
-  if (args.observability === "opentelemetry") {
-    console.log(
-      `\n${yellow("Observability")}: Visit the ${terminalLink(
-        "documentation",
-        "https://traceloop.com/docs/openllmetry/integrations",
-      )} to set up the environment variables and start seeing execution traces.`,
-    );
-  }
+  outputObservability(args.observability);
 
   if (
     dataSources.some((dataSource) => dataSource.type === "file") &&
@@ -167,3 +160,24 @@ export async function createApp({
 
   console.log();
 }
+
+function outputObservability(observability?: TemplateObservability) {
+  switch (observability) {
+    case "traceloop":
+      console.log(
+        `\n${yellow("Observability")}: Visit the ${terminalLink(
+          "documentation",
+          "https://traceloop.com/docs/openllmetry/integrations",
+        )} to set up the environment variables and start seeing execution traces.`,
+      );
+      break;
+    case "llamatrace":
+      console.log(
+        `\n${yellow("Observability")}: LlamaTrace has been configured for your project. Visit the ${terminalLink(
+          "LlamaTrace dashboard",
+          "https://llamatrace.com/login",
+        )} to view your traces and monitor your application.`,
+      );
+      break;
+  }
+}

e2e/basic.spec.ts

@@ -11,128 +11,110 @@ import type {
 } from "../helpers";
 import { createTestDir, runCreateLlama, type AppType } from "./utils";
 
-const templateTypes: TemplateType[] = ["streaming"];
-const templateFrameworks: TemplateFramework[] = [
-  "nextjs",
-  "express",
-  "fastapi",
-];
-const dataSources: string[] = ["--no-files", "--llamacloud"];
-const templateUIs: TemplateUI[] = ["shadcn", "html"];
-const templatePostInstallActions: TemplatePostInstallAction[] = [
-  "none",
-  "runApp",
-];
+const templateType: TemplateType = "streaming";
+const templateFramework: TemplateFramework = process.env.FRAMEWORK
+  ? (process.env.FRAMEWORK as TemplateFramework)
+  : "fastapi";
+const dataSource: string = process.env.DATASOURCE
+  ? process.env.DATASOURCE
+  : "--example-file";
+const templateUI: TemplateUI = "shadcn";
+const templatePostInstallAction: TemplatePostInstallAction = "runApp";
 
 const llamaCloudProjectName = "create-llama";
 const llamaCloudIndexName = "e2e-test";
 
-for (const templateType of templateTypes) {
-  for (const templateFramework of templateFrameworks) {
-    for (const dataSource of dataSources) {
-      for (const templateUI of templateUIs) {
-        for (const templatePostInstallAction of templatePostInstallActions) {
-          const appType: AppType =
-            templateFramework === "nextjs" ? "" : "--frontend";
-          const userMessage =
-            dataSource !== "--no-files"
-              ? "Physical standard for letters"
-              : "Hello";
-          test.describe(`try create-llama ${templateType} ${templateFramework} ${dataSource} ${templateUI} ${appType} ${templatePostInstallAction}`, async () => {
-            let port: number;
-            let externalPort: number;
-            let cwd: string;
-            let name: string;
-            let appProcess: ChildProcess;
-            // Only test without using vector db for now
-            const vectorDb = "none";
+const appType: AppType = templateFramework === "nextjs" ? "" : "--frontend";
+const userMessage =
+  dataSource !== "--no-files" ? "Physical standard for letters" : "Hello";
+test.describe(`try create-llama ${templateType} ${templateFramework} ${dataSource} ${templateUI} ${appType} ${templatePostInstallAction}`, async () => {
+  let port: number;
+  let externalPort: number;
+  let cwd: string;
+  let name: string;
+  let appProcess: ChildProcess;
+  // Only test without using vector db for now
+  const vectorDb = "none";
 
-            test.beforeAll(async () => {
-              port = Math.floor(Math.random() * 10000) + 10000;
-              externalPort = port + 1;
-              cwd = await createTestDir();
-              const result = await runCreateLlama(
-                cwd,
-                templateType,
-                templateFramework,
-                dataSource,
-                templateUI,
-                vectorDb,
-                appType,
-                port,
-                externalPort,
-                templatePostInstallAction,
-                llamaCloudProjectName,
-                llamaCloudIndexName,
-              );
-              name = result.projectName;
-              appProcess = result.appProcess;
-            });
+  test.beforeAll(async () => {
+    port = Math.floor(Math.random() * 10000) + 10000;
+    externalPort = port + 1;
+    cwd = await createTestDir();
+    const result = await runCreateLlama(
+      cwd,
+      templateType,
+      templateFramework,
+      dataSource,
+      templateUI,
+      vectorDb,
+      appType,
+      port,
+      externalPort,
+      templatePostInstallAction,
+      llamaCloudProjectName,
+      llamaCloudIndexName,
+    );
+    name = result.projectName;
+    appProcess = result.appProcess;
+  });
 
-            test("App folder should exist", async () => {
-              const dirExists = fs.existsSync(path.join(cwd, name));
-              expect(dirExists).toBeTruthy();
-            });
-            test("Frontend should have a title", async ({ page }) => {
-              test.skip(templatePostInstallAction !== "runApp");
-              await page.goto(`http://localhost:${port}`);
-              await expect(page.getByText("Built by LlamaIndex")).toBeVisible();
-            });
+  test("App folder should exist", async () => {
+    const dirExists = fs.existsSync(path.join(cwd, name));
+    expect(dirExists).toBeTruthy();
+  });
+  test("Frontend should have a title", async ({ page }) => {
+    test.skip(templatePostInstallAction !== "runApp");
+    await page.goto(`http://localhost:${port}`);
+    await expect(page.getByText("Built by LlamaIndex")).toBeVisible();
+  });
 
-            test("Frontend should be able to submit a message and receive a response", async ({
-              page,
-            }) => {
-              test.skip(templatePostInstallAction !== "runApp");
-              await page.goto(`http://localhost:${port}`);
-              await page.fill("form input", userMessage);
-              const [response] = await Promise.all([
-                page.waitForResponse(
-                  (res) => {
-                    return (
-                      res.url().includes("/api/chat") && res.status() === 200
-                    );
-                  },
-                  {
-                    timeout: 1000 * 60,
-                  },
-                ),
-                page.click("form button[type=submit]"),
-              ]);
-              const text = await response.text();
-              console.log("AI response when submitting message: ", text);
-              expect(response.ok()).toBeTruthy();
-            });
+  test("Frontend should be able to submit a message and receive a response", async ({
+    page,
+  }) => {
+    test.skip(templatePostInstallAction !== "runApp");
+    await page.goto(`http://localhost:${port}`);
+    await page.fill("form input", userMessage);
+    const [response] = await Promise.all([
+      page.waitForResponse(
+        (res) => {
+          return res.url().includes("/api/chat") && res.status() === 200;
+        },
+        {
+          timeout: 1000 * 60,
+        },
+      ),
+      page.click("form button[type=submit]"),
+    ]);
+    const text = await response.text();
+    console.log("AI response when submitting message: ", text);
+    expect(response.ok()).toBeTruthy();
+  });
 
-            test("Backend frameworks should response when calling non-streaming chat API", async ({
-              request,
-            }) => {
-              test.skip(templatePostInstallAction !== "runApp");
-              test.skip(templateFramework === "nextjs");
-              const response = await request.post(
-                `http://localhost:${externalPort}/api/chat/request`,
-                {
-                  data: {
-                    messages: [
-                      {
-                        role: "user",
-                        content: userMessage,
-                      },
-                    ],
-                  },
-                },
-              );
-              const text = await response.text();
-              console.log("AI response when calling API: ", text);
-              expect(response.ok()).toBeTruthy();
-            });
+  test("Backend frameworks should response when calling non-streaming chat API", async ({
+    request,
+  }) => {
+    test.skip(templatePostInstallAction !== "runApp");
+    test.skip(templateFramework === "nextjs");
+    const response = await request.post(
+      `http://localhost:${externalPort}/api/chat/request`,
+      {
+        data: {
+          messages: [
+            {
+              role: "user",
+              content: userMessage,
+            },
+          ],
+        },
+      },
+    );
+    const text = await response.text();
+    console.log("AI response when calling API: ", text);
+    expect(response.ok()).toBeTruthy();
+  });
 
-            // clean processes
-            test.afterAll(async () => {
-              appProcess?.kill();
-            });
-          });
-        }
-      }
-    }
-  }
-}
+  // clean processes
+  test.afterAll(async () => {
+    appProcess?.kill();
+  });
+});

e2e/utils.ts

@@ -18,48 +18,6 @@ export type CreateLlamaResult = {
   appProcess: ChildProcess;
 };
 
-// eslint-disable-next-line max-params
-export async function checkAppHasStarted(
-  frontend: boolean,
-  framework: TemplateFramework,
-  port: number,
-  externalPort: number,
-  timeout: number,
-) {
-  if (frontend) {
-    await Promise.all([
-      waitPort({
-        host: "localhost",
-        port: port,
-        timeout,
-      }),
-      waitPort({
-        host: "localhost",
-        port: externalPort,
-        timeout,
-      }),
-    ]).catch((err) => {
-      console.error(err);
-      throw err;
-    });
-  } else {
-    let wPort: number;
-    if (framework === "nextjs") {
-      wPort = port;
-    } else {
-      wPort = externalPort;
-    }
-    await waitPort({
-      host: "localhost",
-      port: wPort,
-      timeout,
-    }).catch((err) => {
-      console.error(err);
-      throw err;
-    });
-  }
-}
-
 // eslint-disable-next-line max-params
 export async function runCreateLlama(
   cwd: string,
@@ -142,25 +100,10 @@ export async function runCreateLlama(
       templateFramework,
       port,
       externalPort,
-      1000 * 60 * 5,
     );
   } else {
-    // wait create-llama to exit
-    // we don't test install dependencies for now, so just set timeout for 10 seconds
-    await new Promise((resolve, reject) => {
-      const timeout = setTimeout(() => {
-        reject(new Error("create-llama timeout error"));
-      }, 1000 * 10);
-      appProcess.on("exit", (code) => {
-        if (code !== 0 && code !== null) {
-          clearTimeout(timeout);
-          reject(new Error("create-llama command was failed!"));
-        } else {
-          clearTimeout(timeout);
-          resolve(undefined);
-        }
-      });
-    });
+    // wait 10 seconds for create-llama to exit
+    await waitForProcess(appProcess, 1000 * 10);
   }
 
   return {
@@ -174,3 +117,53 @@ export async function createTestDir() {
   await mkdir(cwd, { recursive: true });
   return cwd;
 }
+
+// eslint-disable-next-line max-params
+async function checkAppHasStarted(
+  frontend: boolean,
+  framework: TemplateFramework,
+  port: number,
+  externalPort: number,
+) {
+  const portsToWait = frontend
+    ? [port, externalPort]
+    : [framework === "nextjs" ? port : externalPort];
+  await waitPorts(portsToWait);
+}
+
+async function waitPorts(ports: number[]): Promise<void> {
+  const waitForPort = async (port: number): Promise<void> => {
+    await waitPort({
+      host: "localhost",
+      port: port,
+      // wait max. 5 mins for start up of app
+      timeout: 1000 * 60 * 5,
+    });
+  };
+  try {
+    await Promise.all(ports.map(waitForPort));
+  } catch (err) {
+    console.error(err);
+    throw err;
+  }
+}
+
+async function waitForProcess(
+  process: ChildProcess,
+  timeoutMs: number,
+): Promise<void> {
+  return new Promise((resolve, reject) => {
+    const timeout = setTimeout(() => {
+      reject(new Error("Process timeout error"));
+    }, timeoutMs);
+
+    process.on("exit", (code) => {
+      clearTimeout(timeout);
+      if (code !== 0 && code !== null) {
+        reject(new Error("Process exited with non-zero code"));
+      } else {
+        resolve();
+      }
+    });
+  });
+}

helpers/env-variables.ts

@@ -2,9 +2,11 @@ import fs from "fs/promises";
 import path from "path";
 import { TOOL_SYSTEM_PROMPT_ENV_VAR, Tool } from "./tools";
 import {
+  InstallTemplateArgs,
   ModelConfig,
   TemplateDataSource,
   TemplateFramework,
+  TemplateObservability,
   TemplateType,
   TemplateVectorDB,
 } from "./types";
@@ -155,6 +157,22 @@ const getVectorDBEnvs = (
             "The base URL for the LlamaCloud API. Only change this for non-production environments",
           value: "https://api.cloud.llamaindex.ai",
         },
+        {
+          name: "LLAMA_CLOUD_ORGANIZATION_ID",
+          description:
+            "The organization ID for the LlamaCloud project (uses default organization if not specified - Python only)",
+        },
+        ...(framework === "nextjs"
+          ? // activate index selector per default (not needed for non-NextJS backends as it's handled by createFrontendEnvFile)
+            [
+              {
+                name: "NEXT_PUBLIC_USE_LLAMACLOUD",
+                description:
+                  "Let's the user change indexes in LlamaCloud projects",
+                value: "true",
+              },
+            ]
+          : []),
       ];
     case "chroma":
       const envs = [
@@ -181,6 +199,23 @@ Otherwise, use CHROMA_HOST and CHROMA_PORT config above`,
         });
       }
       return envs;
+    case "weaviate":
+      return [
+        {
+          name: "WEAVIATE_CLUSTER_URL",
+          description:
+            "The URL of the Weaviate cloud cluster, see: https://weaviate.io/developers/wcs/connect",
+        },
+        {
+          name: "WEAVIATE_API_KEY",
+          description: "The API key for the Weaviate cloud cluster",
+        },
+        {
+          name: "WEAVIATE_INDEX_NAME",
+          description:
+            "(Optional) The collection name to use, default is LlamaIndex if not specified",
+        },
+      ];
     default:
       return [];
   }
@@ -274,6 +309,33 @@ const getModelEnvs = (modelConfig: ModelConfig): EnvVar[] => {
           },
         ]
       : []),
+    ...(modelConfig.provider === "azure-openai"
+      ? [
+          {
+            name: "AZURE_OPENAI_API_KEY",
+            description: "The Azure OpenAI key to use.",
+            value: modelConfig.apiKey,
+          },
+          {
+            name: "AZURE_OPENAI_ENDPOINT",
+            description: "The Azure OpenAI endpoint to use.",
+          },
+          {
+            name: "AZURE_OPENAI_API_VERSION",
+            description: "The Azure OpenAI API version to use.",
+          },
+          {
+            name: "AZURE_OPENAI_LLM_DEPLOYMENT",
+            description:
+              "The Azure OpenAI deployment to use for LLM deployment.",
+          },
+          {
+            name: "AZURE_OPENAI_EMBEDDING_DEPLOYMENT",
+            description:
+              "The Azure OpenAI deployment to use for embedding deployment.",
+          },
+        ]
+      : []),
     ...(modelConfig.provider === "t-systems"
       ? [
           {
@@ -362,7 +424,11 @@ const getToolEnvs = (tools?: Tool[]): EnvVar[] => {
   return toolEnvs;
 };
 
-const getSystemPromptEnv = (tools?: Tool[]): EnvVar => {
+const getSystemPromptEnv = (
+  tools?: Tool[],
+  dataSources?: TemplateDataSource[],
+  framework?: TemplateFramework,
+): EnvVar[] => {
   const defaultSystemPrompt =
     "You are a helpful assistant who helps users with their questions.";
 
@@ -381,11 +447,49 @@ const getSystemPromptEnv = (tools?: Tool[]): EnvVar => {
     ? `\"${toolSystemPrompt}\"`
     : defaultSystemPrompt;
 
-  return {
-    name: "SYSTEM_PROMPT",
-    description: "The system prompt for the AI model.",
-    value: systemPrompt,
-  };
+  const systemPromptEnv = [
+    {
+      name: "SYSTEM_PROMPT",
+      description: "The system prompt for the AI model.",
+      value: systemPrompt,
+    },
+  ];
+
+  // Citation only works with FastAPI along with the chat engine and data source provided for now.
+  if (
+    framework === "fastapi" &&
+    tools?.length == 0 &&
+    (dataSources?.length ?? 0 > 0)
+  ) {
+    const citationPrompt = `'You have provided information from a knowledge base that has been passed to you in nodes of information.
+Each node has useful metadata such as node ID, file name, page, etc.
+Please add the citation to the data node for each sentence or paragraph that you reference in the provided information.
+The citation format is: . [citation:<node_id>]()
+Where the <node_id> is the unique identifier of the data node.
+
+Example:
+We have two nodes:
+  node_id: xyz
+  file_name: llama.pdf
+  
+  node_id: abc
+  file_name: animal.pdf
+
+User question: Tell me a fun fact about Llama.
+Your answer:
+A baby llama is called "Cria" [citation:xyz]().
+It often live in desert [citation:abc]().
+It\\'s cute animal.
+'`;
+    systemPromptEnv.push({
+      name: "SYSTEM_CITATION_PROMPT",
+      description:
+        "An additional system prompt to add citation when responding to user questions.",
+      value: citationPrompt,
+    });
+  }
+
+  return systemPromptEnv;
 };
 
 const getTemplateEnvs = (template?: TemplateType): EnvVar[] => {
@@ -418,18 +522,35 @@ const getTemplateEnvs = (template?: TemplateType): EnvVar[] => {
   }
 };
 
+const getObservabilityEnvs = (
+  observability?: TemplateObservability,
+): EnvVar[] => {
+  if (observability === "llamatrace") {
+    return [
+      {
+        name: "PHOENIX_API_KEY",
+        description:
+          "API key for LlamaTrace observability. Retrieve from https://llamatrace.com/login",
+      },
+    ];
+  }
+  return [];
+};
+
 export const createBackendEnvFile = async (
   root: string,
-  opts: {
-    llamaCloudKey?: string;
-    vectorDb?: TemplateVectorDB;
-    modelConfig: ModelConfig;
-    framework: TemplateFramework;
-    dataSources?: TemplateDataSource[];
-    template?: TemplateType;
-    port?: number;
-    tools?: Tool[];
-  },
+  opts: Pick<
+    InstallTemplateArgs,
+    | "llamaCloudKey"
+    | "vectorDb"
+    | "modelConfig"
+    | "framework"
+    | "dataSources"
+    | "template"
+    | "externalPort"
+    | "tools"
+    | "observability"
+  >,
 ) => {
   // Init env values
   const envFileName = ".env";
@@ -439,17 +560,15 @@ export const createBackendEnvFile = async (
       description: `The Llama Cloud API key.`,
       value: opts.llamaCloudKey,
     },
-    // Add model environment variables
+    // Add environment variables of each component
     ...getModelEnvs(opts.modelConfig),
-    // Add engine environment variables
     ...getEngineEnvs(),
-    // Add vector database environment variables
     ...getVectorDBEnvs(opts.vectorDb, opts.framework),
-    ...getFrameworkEnvs(opts.framework, opts.port),
+    ...getFrameworkEnvs(opts.framework, opts.externalPort),
     ...getToolEnvs(opts.tools),
-    // Add template environment variables
     ...getTemplateEnvs(opts.template),
-    getSystemPromptEnv(opts.tools),
+    ...getObservabilityEnvs(opts.observability),
+    ...getSystemPromptEnv(opts.tools, opts.dataSources, opts.framework),
   ];
   // Render and write env file
   const content = renderEnvVar(envVars);
@@ -461,6 +580,7 @@ export const createFrontendEnvFile = async (
   root: string,
   opts: {
     customApiPath?: string;
+    vectorDb?: TemplateVectorDB;
   },
 ) => {
   const defaultFrontendEnvs = [
@@ -471,6 +591,11 @@ export const createFrontendEnvFile = async (
         ? opts.customApiPath
         : "http://localhost:8000/api/chat",
     },
+    {
+      name: "NEXT_PUBLIC_USE_LLAMACLOUD",
+      description: "Let's the user change indexes in LlamaCloud projects",
+      value: opts.vectorDb === "llamacloud" ? "true" : "false",
+    },
   ];
   const content = renderEnvVar(defaultFrontendEnvs);
   await fs.writeFile(path.join(root, ".env"), content);

helpers/index.ts

@@ -142,12 +142,15 @@ export const installTemplate = async (
 
   if (props.framework === "fastapi") {
     await installPythonTemplate(props);
-    // write loaders configuration (currently Python only)
-    await writeLoadersConfig(
-      props.root,
-      props.dataSources,
-      props.useLlamaParse,
-    );
+    if (props.vectorDb !== "llamacloud") {
+      // write loaders configuration (currently Python only)
+      // not needed for LlamaCloud as it has its own loaders
+      await writeLoadersConfig(
+        props.root,
+        props.dataSources,
+        props.useLlamaParse,
+      );
+    }
   } else {
     await installTSTemplate(props);
   }
@@ -163,17 +166,12 @@ export const installTemplate = async (
     // This is a backend, so we need to copy the test data and create the env file.
 
     // Copy the environment file to the target directory.
-    if (props.template === "streaming" || props.template === "multiagent") {
-      await createBackendEnvFile(props.root, {
-        modelConfig: props.modelConfig,
-        llamaCloudKey: props.llamaCloudKey,
-        vectorDb: props.vectorDb,
-        framework: props.framework,
-        dataSources: props.dataSources,
-        port: props.externalPort,
-        tools: props.tools,
-        template: props.template,
-      });
+    if (
+      props.template === "streaming" ||
+      props.template === "multiagent" ||
+      props.template === "extractor"
+    ) {
+      await createBackendEnvFile(props.root, props);
     }
 
     if (props.dataSources.length > 0) {
@@ -205,6 +203,7 @@ export const installTemplate = async (
     // this is a frontend for a full-stack app, create .env file with model information
     await createFrontendEnvFile(props.root, {
       customApiPath: props.customApiPath,
+      vectorDb: props.vectorDb,
     });
   }
 };

helpers/providers/azure.ts

@@ -0,0 +1,118 @@
+import ciInfo from "ci-info";
+import prompts from "prompts";
+import { ModelConfigParams, ModelConfigQuestionsParams } from ".";
+import { questionHandlers } from "../../questions";
+
+const ALL_AZURE_OPENAI_CHAT_MODELS: Record<string, { openAIModel: string }> = {
+  "gpt-35-turbo": { openAIModel: "gpt-3.5-turbo" },
+  "gpt-35-turbo-16k": {
+    openAIModel: "gpt-3.5-turbo-16k",
+  },
+  "gpt-4o": { openAIModel: "gpt-4o" },
+  "gpt-4o-mini": { openAIModel: "gpt-4o-mini" },
+  "gpt-4": { openAIModel: "gpt-4" },
+  "gpt-4-32k": { openAIModel: "gpt-4-32k" },
+  "gpt-4-turbo": {
+    openAIModel: "gpt-4-turbo",
+  },
+  "gpt-4-turbo-2024-04-09": {
+    openAIModel: "gpt-4-turbo",
+  },
+  "gpt-4-vision-preview": {
+    openAIModel: "gpt-4-vision-preview",
+  },
+  "gpt-4-1106-preview": {
+    openAIModel: "gpt-4-1106-preview",
+  },
+  "gpt-4o-2024-05-13": {
+    openAIModel: "gpt-4o-2024-05-13",
+  },
+  "gpt-4o-mini-2024-07-18": {
+    openAIModel: "gpt-4o-mini-2024-07-18",
+  },
+};
+
+const ALL_AZURE_OPENAI_EMBEDDING_MODELS: Record<
+  string,
+  {
+    dimensions: number;
+    openAIModel: string;
+  }
+> = {
+  "text-embedding-3-small": {
+    dimensions: 1536,
+    openAIModel: "text-embedding-3-small",
+  },
+  "text-embedding-3-large": {
+    dimensions: 3072,
+    openAIModel: "text-embedding-3-large",
+  },
+};
+
+const DEFAULT_MODEL = "gpt-4o";
+const DEFAULT_EMBEDDING_MODEL = "text-embedding-3-large";
+
+export async function askAzureQuestions({
+  openAiKey,
+  askModels,
+}: ModelConfigQuestionsParams): Promise<ModelConfigParams> {
+  const config: ModelConfigParams = {
+    apiKey: openAiKey || process.env.AZURE_OPENAI_KEY,
+    model: DEFAULT_MODEL,
+    embeddingModel: DEFAULT_EMBEDDING_MODEL,
+    dimensions: getDimensions(DEFAULT_EMBEDDING_MODEL),
+    isConfigured(): boolean {
+      // the Azure model provider can't be fully configured as endpoint and deployment names have to be configured with env variables
+      return false;
+    },
+  };
+
+  // use default model values in CI or if user should not be asked
+  const useDefaults = ciInfo.isCI || !askModels;
+  if (!useDefaults) {
+    const { model } = await prompts(
+      {
+        type: "select",
+        name: "model",
+        message: "Which LLM model would you like to use?",
+        choices: getAvailableModelChoices(),
+        initial: 0,
+      },
+      questionHandlers,
+    );
+    config.model = model;
+
+    const { embeddingModel } = await prompts(
+      {
+        type: "select",
+        name: "embeddingModel",
+        message: "Which embedding model would you like to use?",
+        choices: getAvailableEmbeddingModelChoices(),
+        initial: 0,
+      },
+      questionHandlers,
+    );
+    config.embeddingModel = embeddingModel;
+    config.dimensions = getDimensions(embeddingModel);
+  }
+
+  return config;
+}
+
+function getAvailableModelChoices() {
+  return Object.keys(ALL_AZURE_OPENAI_CHAT_MODELS).map((key) => ({
+    title: key,
+    value: key,
+  }));
+}
+
+function getAvailableEmbeddingModelChoices() {
+  return Object.keys(ALL_AZURE_OPENAI_EMBEDDING_MODELS).map((key) => ({
+    title: key,
+    value: key,
+  }));
+}
+
+function getDimensions(modelName: string) {
+  return ALL_AZURE_OPENAI_EMBEDDING_MODELS[modelName].dimensions;
+}

helpers/providers/index.ts

@@ -3,6 +3,7 @@ import prompts from "prompts";
 import { questionHandlers } from "../../questions";
 import { ModelConfig, ModelProvider, TemplateFramework } from "../types";
 import { askAnthropicQuestions } from "./anthropic";
+import { askAzureQuestions } from "./azure";
 import { askGeminiQuestions } from "./gemini";
 import { askGroqQuestions } from "./groq";
 import { askLLMHubQuestions } from "./llmhub";
@@ -34,6 +35,7 @@ export async function askModelConfig({
       { title: "Anthropic", value: "anthropic" },
       { title: "Gemini", value: "gemini" },
       { title: "Mistral", value: "mistral" },
+      { title: "AzureOpenAI", value: "azure-openai" },
     ];
 
     if (framework === "fastapi") {
@@ -69,6 +71,9 @@ export async function askModelConfig({
     case "mistral":
       modelConfig = await askMistralQuestions({ askModels });
       break;
+    case "azure-openai":
+      modelConfig = await askAzureQuestions({ askModels });
+      break;
     case "t-systems":
       modelConfig = await askLLMHubQuestions({ askModels });
       break;

helpers/python.ts

@@ -84,6 +84,13 @@ const getAdditionalDependencies = (
       });
       break;
     }
+    case "weaviate": {
+      dependencies.push({
+        name: "llama-index-vector-stores-weaviate",
+        version: "^1.0.2",
+      });
+      break;
+    }
   }
 
   // Add data source dependencies
@@ -121,7 +128,7 @@ const getAdditionalDependencies = (
         case "llamacloud":
           dependencies.push({
             name: "llama-index-indices-managed-llama-cloud",
-            version: "^0.2.5",
+            version: "^0.2.7",
           });
           break;
       }
@@ -193,6 +200,16 @@ const getAdditionalDependencies = (
         version: "0.1.4",
       });
       break;
+    case "azure-openai":
+      dependencies.push({
+        name: "llama-index-llms-azure-openai",
+        version: "0.1.10",
+      });
+      dependencies.push({
+        name: "llama-index-embeddings-azure-openai",
+        version: "0.1.11",
+      });
+      break;
     case "t-systems":
       dependencies.push({
         name: "llama-index-agent-openai",
@@ -333,12 +350,15 @@ export const installPythonTemplate = async ({
     cwd: path.join(compPath, "vectordbs", "python", vectorDb ?? "none"),
   });
 
-  // Copy all loaders to enginePath
-  const loaderPath = path.join(enginePath, "loaders");
-  await copy("**", loaderPath, {
-    parents: true,
-    cwd: path.join(compPath, "loaders", "python"),
-  });
+  if (vectorDb !== "llamacloud") {
+    // Copy all loaders to enginePath
+    // Not needed for LlamaCloud as it has its own loaders
+    const loaderPath = path.join(enginePath, "loaders");
+    await copy("**", loaderPath, {
+      parents: true,
+      cwd: path.join(compPath, "loaders", "python"),
+    });
+  }
 
   // Copy settings.py to app
   await copy("**", path.join(root, "app"), {
@@ -370,18 +390,27 @@ export const installPythonTemplate = async ({
     tools,
   );
 
-  if (observability === "opentelemetry") {
-    addOnDependencies.push({
-      name: "traceloop-sdk",
-      version: "^0.15.11",
-    });
+  if (observability && observability !== "none") {
+    if (observability === "traceloop") {
+      addOnDependencies.push({
+        name: "traceloop-sdk",
+        version: "^0.15.11",
+      });
+    }
+
+    if (observability === "llamatrace") {
+      addOnDependencies.push({
+        name: "llama-index-callbacks-arize-phoenix",
+        version: "^0.1.6",
+      });
+    }
 
     const templateObservabilityPath = path.join(
       templatesDir,
       "components",
       "observability",
       "python",
-      "opentelemetry",
+      observability,
     );
     await copy("**", path.join(root, "app"), {
       cwd: templateObservabilityPath,

helpers/tools.ts

@@ -188,6 +188,35 @@ For better results, you can specify the region parameter to get results from a s
       },
     ],
   },
+  {
+    display: "Azure Code Interpreter",
+    name: "azure_code_interpreter.AzureCodeInterpreterToolSpec",
+    supportedFrameworks: ["fastapi", "nextjs", "express"],
+    type: ToolType.LLAMAHUB,
+    dependencies: [
+      {
+        name: "llama-index-tools-azure-code-interpreter",
+        version: "0.2.0",
+      },
+    ],
+    envVars: [
+      {
+        name: "AZURE_POOL_MANAGEMENT_ENDPOINT",
+        description:
+          "Please follow this guideline to create and get the pool management endpoint: https://learn.microsoft.com/azure/container-apps/sessions?tabs=azure-cli",
+      },
+      {
+        name: TOOL_SYSTEM_PROMPT_ENV_VAR,
+        description: "System prompt for Azure code interpreter tool.",
+        value: `-You are a Python interpreter that can run any python code in a secure environment.
+- The python code runs in a Jupyter notebook. Every time you call the 'interpreter' tool, the python code is executed in a separate cell. 
+- You are given tasks to complete and you run python code to solve them.
+- It's okay to make multiple calls to interpreter tool. If you get an error or the result is not what you expected, you can call the tool again. Don't give up too soon!
+- Plot visualizations using matplotlib or any other visualization library directly in the notebook.
+- You can install any pip package (if it exists) by running a cell with pip install.`,
+      },
+    ],
+  },
 ];
 
 export const getTool = (toolName: string): Tool | undefined => {

helpers/types.ts

@@ -8,6 +8,7 @@ export type ModelProvider =
   | "anthropic"
   | "gemini"
   | "mistral"
+  | "azure-openai"
   | "t-systems";
 export type ModelConfig = {
   provider: ModelProvider;
@@ -18,6 +19,7 @@ export type ModelConfig = {
   isConfigured(): boolean;
 };
 export type TemplateType =
+  | "extractor"
   | "streaming"
   | "community"
   | "llamapack"
@@ -33,7 +35,8 @@ export type TemplateVectorDB =
   | "astra"
   | "qdrant"
   | "chroma"
-  | "llamacloud";
+  | "llamacloud"
+  | "weaviate";
 export type TemplatePostInstallAction =
   | "none"
   | "VSCode"
@@ -44,7 +47,7 @@ export type TemplateDataSource = {
   config: TemplateDataSourceConfig;
 };
 export type TemplateDataSourceType = "file" | "web" | "db" | "llamacloud";
-export type TemplateObservability = "none" | "opentelemetry";
+export type TemplateObservability = "none" | "traceloop" | "llamatrace";
 // Config for both file and folder
 export type FileSourceConfig = {
   path: string;

helpers/typescript.ts

@@ -70,7 +70,7 @@ export const installTSTemplate = async ({
     );
 
     const webpackConfigOtelFile = path.join(root, "webpack.config.o11y.mjs");
-    if (observability === "opentelemetry") {
+    if (observability === "traceloop") {
       const webpackConfigDefaultFile = path.join(root, "webpack.config.mjs");
       await fs.rm(webpackConfigDefaultFile);
       await fs.rename(webpackConfigOtelFile, webpackConfigDefaultFile);
@@ -248,7 +248,7 @@ async function updatePackageJson({
     };
   }
 
-  if (observability === "opentelemetry") {
+  if (observability === "traceloop") {
     packageJson.dependencies = {
       ...packageJson.dependencies,
       "@traceloop/node-server-sdk": "^0.5.19",

package.json

@@ -1,6 +1,6 @@
 {
   "name": "create-llama",
-  "version": "0.1.22",
+  "version": "0.1.37",
   "description": "Create LlamaIndex-powered apps with one command",
   "keywords": [
     "rag",
@@ -9,7 +9,7 @@
   ],
   "repository": {
     "type": "git",
-    "url": "https://github.com/run-llama/LlamaIndexTS",
+    "url": "https://github.com/run-llama/create-llama",
     "directory": "packages/create-llama"
   },
   "license": "MIT",

questions.ts

@@ -103,6 +103,7 @@ const getVectorDbChoices = (framework: TemplateFramework) => {
     { title: "Astra", value: "astra" },
     { title: "Qdrant", value: "qdrant" },
     { title: "ChromaDB", value: "chroma" },
+    { title: "Weaviate", value: "weaviate" },
   ];
 
   const vectordbLang = framework === "fastapi" ? "python" : "typescript";
@@ -342,6 +343,7 @@ export const askQuestions = async (
               title: "Multi-agent app (using llama-agents)",
               value: "multiagent",
             },
+            { title: "Structured Extractor", value: "extractor" },
             {
               title: `Community template from ${styledRepo}`,
               value: "community",
@@ -405,7 +407,7 @@ export const askQuestions = async (
     return; // early return - no further questions needed for llamapack projects
   }
 
-  if (program.template === "multiagent") {
+  if (program.template === "multiagent" || program.template === "extractor") {
     // TODO: multi-agents currently only supports FastAPI
     program.framework = preferences.framework = "fastapi";
   }
@@ -485,7 +487,10 @@ export const askQuestions = async (
           message: "Would you like to set up observability?",
           choices: [
             { title: "No", value: "none" },
-            { title: "OpenTelemetry", value: "opentelemetry" },
+            ...(program.framework === "fastapi"
+              ? [{ title: "LlamaTrace", value: "llamatrace" }]
+              : []),
+            { title: "Traceloop", value: "traceloop" },
           ],
           initial: 0,
         },
@@ -647,24 +652,27 @@ export const askQuestions = async (
     // default to use LlamaParse if using LlamaCloud
     program.useLlamaParse = preferences.useLlamaParse = true;
   } else {
-    if (program.dataSources.some((ds) => ds.type === "file")) {
-      if (ciInfo.isCI) {
-        program.useLlamaParse = getPrefOrDefault("useLlamaParse");
-      } else {
-        const { useLlamaParse } = await prompts(
-          {
-            type: "toggle",
-            name: "useLlamaParse",
-            message:
-              "Would you like to use LlamaParse (improved parser for RAG - requires API key)?",
-            initial: false,
-            active: "yes",
-            inactive: "no",
-          },
-          questionHandlers,
-        );
-        program.useLlamaParse = useLlamaParse;
-        preferences.useLlamaParse = useLlamaParse;
+    if (program.useLlamaParse === undefined) {
+      // if already set useLlamaParse, don't ask again
+      if (program.dataSources.some((ds) => ds.type === "file")) {
+        if (ciInfo.isCI) {
+          program.useLlamaParse = getPrefOrDefault("useLlamaParse");
+        } else {
+          const { useLlamaParse } = await prompts(
+            {
+              type: "toggle",
+              name: "useLlamaParse",
+              message:
+                "Would you like to use LlamaParse (improved parser for RAG - requires API key)?",
+              initial: false,
+              active: "yes",
+              inactive: "no",
+            },
+            questionHandlers,
+          );
+          program.useLlamaParse = useLlamaParse;
+          preferences.useLlamaParse = useLlamaParse;
+        }
       }
     }
   }

templates/components/engines/python/agent/__init__.py

@@ -6,7 +6,7 @@ from app.engine.tools import ToolFactory
 from app.engine.index import get_index
 
 
-def get_chat_engine(filters=None):
+def get_chat_engine(filters=None, params=None):
     system_prompt = os.getenv("SYSTEM_PROMPT")
     top_k = os.getenv("TOP_K", "3")
     tools = []

templates/components/engines/python/agent/tools/__init__.py

@@ -1,8 +1,6 @@
 import os
 import yaml
-import json
 import importlib
-from cachetools import cached, LRUCache
 from llama_index.core.tools.tool_spec.base import BaseToolSpec
 from llama_index.core.tools.function_tool import FunctionTool
 
@@ -13,7 +11,6 @@ class ToolType:
 
 
 class ToolFactory:
-
     TOOL_SOURCE_PACKAGE_MAP = {
         ToolType.LLAMAHUB: "llama_index.tools",
         ToolType.LOCAL: "app.engine.tools",

templates/components/engines/python/agent/tools/interpreter.py

@@ -3,7 +3,7 @@ import logging
 import base64
 import uuid
 from pydantic import BaseModel
-from typing import List, Tuple, Dict, Optional
+from typing import List, Dict, Optional
 from llama_index.core.tools import FunctionTool
 from e2b_code_interpreter import CodeInterpreter
 from e2b_code_interpreter.models import Logs
@@ -26,7 +26,6 @@ class E2BToolOutput(BaseModel):
 
 
 class E2BCodeInterpreter:
-
     output_dir = "output/tool"
 
     def __init__(self, api_key: str = None):

templates/components/engines/python/chat/__init__.py

@@ -1,13 +1,22 @@
 import os
+
 from app.engine.index import get_index
+from app.engine.node_postprocessors import NodeCitationProcessor
 from fastapi import HTTPException
+from llama_index.core.chat_engine import CondensePlusContextChatEngine
 
 
-def get_chat_engine(filters=None):
+def get_chat_engine(filters=None, params=None):
     system_prompt = os.getenv("SYSTEM_PROMPT")
-    top_k = os.getenv("TOP_K", 3)
+    citation_prompt = os.getenv("SYSTEM_CITATION_PROMPT", None)
+    top_k = int(os.getenv("TOP_K", 3))
 
-    index = get_index()
+    node_postprocessors = []
+    if citation_prompt:
+        node_postprocessors = [NodeCitationProcessor()]
+        system_prompt = f"{system_prompt}\n{citation_prompt}"
+
+    index = get_index(params)
     if index is None:
         raise HTTPException(
             status_code=500,
@@ -16,9 +25,13 @@ def get_chat_engine(filters=None):
             ),
         )
 
-    return index.as_chat_engine(
-        similarity_top_k=int(top_k),
-        system_prompt=system_prompt,
-        chat_mode="condense_plus_context",
+    retriever = index.as_retriever(
+        similarity_top_k=top_k,
         filters=filters,
     )
+
+    return CondensePlusContextChatEngine.from_defaults(
+        system_prompt=system_prompt,
+        retriever=retriever,
+        node_postprocessors=node_postprocessors,
+    )

templates/components/engines/python/chat/node_postprocessors.py

@@ -0,0 +1,21 @@
+from typing import List, Optional
+
+from llama_index.core import QueryBundle
+from llama_index.core.postprocessor.types import BaseNodePostprocessor
+from llama_index.core.schema import NodeWithScore
+
+
+class NodeCitationProcessor(BaseNodePostprocessor):
+    """
+    Append node_id into metadata for citation purpose.
+    Config SYSTEM_CITATION_PROMPT in your runtime environment variable to enable this feature.
+    """
+
+    def _postprocess_nodes(
+        self,
+        nodes: List[NodeWithScore],
+        query_bundle: Optional[QueryBundle] = None,
+    ) -> List[NodeWithScore]:
+        for node_score in nodes:
+            node_score.node.metadata["node_id"] = node_score.node.node_id
+        return nodes

templates/components/engines/typescript/agent/chat.ts

@@ -1,21 +1,16 @@
-import {
-  BaseToolWithCall,
-  MetadataFilter,
-  MetadataFilters,
-  OpenAIAgent,
-  QueryEngineTool,
-} from "llamaindex";
+import { BaseToolWithCall, OpenAIAgent, QueryEngineTool } from "llamaindex";
 import fs from "node:fs/promises";
 import path from "node:path";
 import { getDataSource } from "./index";
+import { generateFilters } from "./queryFilter";
 import { createTools } from "./tools";
 
-export async function createChatEngine(documentIds?: string[]) {
+export async function createChatEngine(documentIds?: string[], params?: any) {
   const tools: BaseToolWithCall[] = [];
 
   // Add a query engine tool if we have a data source
   // Delete this code if you don't have a data source
-  const index = await getDataSource();
+  const index = await getDataSource(params);
   if (index) {
     tools.push(
       new QueryEngineTool({
@@ -47,27 +42,3 @@ export async function createChatEngine(documentIds?: string[]) {
     systemPrompt: process.env.SYSTEM_PROMPT,
   });
 }
-
-function generateFilters(documentIds: string[]): MetadataFilters | undefined {
-  // public documents don't have the "private" field or it's set to "false"
-  const publicDocumentsFilter: MetadataFilter = {
-    key: "private",
-    value: ["true"],
-    operator: "nin",
-  };
-
-  // if no documentIds are provided, only retrieve information from public documents
-  if (!documentIds.length) return { filters: [publicDocumentsFilter] };
-
-  const privateDocumentsFilter: MetadataFilter = {
-    key: "doc_id",
-    value: documentIds,
-    operator: "in",
-  };
-
-  // if documentIds are provided, retrieve information from public and private documents
-  return {
-    filters: [publicDocumentsFilter, privateDocumentsFilter],
-    condition: "or",
-  };
-}

templates/components/engines/typescript/chat/chat.ts

@@ -1,8 +1,9 @@
 import { ContextChatEngine, Settings } from "llamaindex";
 import { getDataSource } from "./index";
+import { generateFilters } from "./queryFilter";
 
-export async function createChatEngine(documentIds?: string[]) {
-  const index = await getDataSource();
+export async function createChatEngine(documentIds?: string[], params?: any) {
+  const index = await getDataSource(params);
   if (!index) {
     throw new Error(
       `StorageContext is empty - call 'npm run generate' to generate the storage first`,
@@ -10,12 +11,12 @@ export async function createChatEngine(documentIds?: string[]) {
   }
   const retriever = index.asRetriever({
     similarityTopK: process.env.TOP_K ? parseInt(process.env.TOP_K) : 3,
+    filters: generateFilters(documentIds || []),
   });
 
   return new ContextChatEngine({
     chatModel: Settings.llm,
     retriever,
-    // disable as a custom system prompt disables the generated context
-    // systemPrompt: process.env.SYSTEM_PROMPT,
+    systemPrompt: process.env.SYSTEM_PROMPT,
   });
 }

templates/components/llamaindex/typescript/documents/pipeline.ts

@@ -1,28 +1,16 @@
 import {
-  BaseNode,
   Document,
   IngestionPipeline,
-  Metadata,
   Settings,
   SimpleNodeParser,
-  storageContextFromDefaults,
   VectorStoreIndex,
 } from "llamaindex";
 import { LlamaCloudIndex } from "llamaindex/cloud/LlamaCloudIndex";
-import { getDataSource } from "../../engine";
-
-export async function runPipeline(documents: Document[], filename: string) {
-  const currentIndex = await getDataSource();
-
-  // Update documents with metadata
-  for (const document of documents) {
-    document.metadata = {
-      ...document.metadata,
-      file_name: filename,
-      private: "true", // to separate from other public documents
-    };
-  }
 
+export async function runPipeline(
+  currentIndex: VectorStoreIndex | LlamaCloudIndex,
+  documents: Document[],
+) {
   if (currentIndex instanceof LlamaCloudIndex) {
     // LlamaCloudIndex processes the documents automatically
     // so we don't need ingestion pipeline, just insert the documents directly
@@ -41,25 +29,10 @@ export async function runPipeline(documents: Document[], filename: string) {
       ],
     });
     const nodes = await pipeline.run({ documents });
-    await addNodesToVectorStore(nodes, currentIndex);
+    await currentIndex.insertNodes(nodes);
+    currentIndex.storageContext.docStore.persist();
+    console.log("Added nodes to the vector store.");
   }
 
   return documents.map((document) => document.id_);
 }
-
-async function addNodesToVectorStore(
-  nodes: BaseNode<Metadata>[],
-  currentIndex: VectorStoreIndex | null,
-) {
-  if (currentIndex) {
-    await currentIndex.insertNodes(nodes);
-  } else {
-    // Not using vectordb and haven't generated local index yet
-    const storageContext = await storageContextFromDefaults({
-      persistDir: "./cache",
-    });
-    currentIndex = await VectorStoreIndex.init({ nodes, storageContext });
-  }
-  currentIndex.storageContext.docStore.persist();
-  console.log("Added nodes to the vector store.");
-}

templates/components/llamaindex/typescript/documents/upload.ts

@@ -1,11 +1,26 @@
+import { VectorStoreIndex } from "llamaindex";
+import { LlamaCloudIndex } from "llamaindex/cloud/LlamaCloudIndex";
 import { loadDocuments, saveDocument } from "./helper";
 import { runPipeline } from "./pipeline";
 
-export async function uploadDocument(raw: string): Promise<string[]> {
+export async function uploadDocument(
+  index: VectorStoreIndex | LlamaCloudIndex,
+  raw: string,
+): Promise<string[]> {
   const [header, content] = raw.split(",");
   const mimeType = header.replace("data:", "").replace(";base64", "");
   const fileBuffer = Buffer.from(content, "base64");
   const documents = await loadDocuments(fileBuffer, mimeType);
   const { filename } = await saveDocument(fileBuffer, mimeType);
-  return await runPipeline(documents, filename);
+
+  // Update documents with metadata
+  for (const document of documents) {
+    document.metadata = {
+      ...document.metadata,
+      file_name: filename,
+      private: "true", // to separate private uploads from public documents
+    };
+  }
+
+  return await runPipeline(index, documents);
 }

templates/components/llamaindex/typescript/streaming/events.ts

@@ -2,6 +2,7 @@ import { StreamData } from "ai";
 import {
   CallbackManager,
   Metadata,
+  MetadataMode,
   NodeWithScore,
   ToolCall,
   ToolOutput,
@@ -15,10 +16,11 @@ export function appendSourceData(
   if (!sourceNodes?.length) return;
   try {
     const nodes = sourceNodes.map((node) => ({
-      ...node.node.toMutableJSON(),
+      metadata: node.node.metadata,
       id: node.node.id_,
       score: node.score ?? null,
       url: getNodeUrl(node.node.metadata),
+      text: node.node.getContent(MetadataMode.NONE),
     }));
     data.appendMessageAnnotation({
       type: "sources",

templates/components/llamaindex/typescript/streaming/service.ts

@@ -7,19 +7,51 @@ const LLAMA_CLOUD_OUTPUT_DIR = "output/llamacloud";
 const LLAMA_CLOUD_BASE_URL = "https://cloud.llamaindex.ai/api/v1";
 const FILE_DELIMITER = "$"; // delimiter between pipelineId and filename
 
-interface LlamaCloudFile {
+type LlamaCloudFile = {
   name: string;
   file_id: string;
   project_id: string;
-}
+};
+
+type LLamaCloudProject = {
+  id: string;
+  organization_id: string;
+  name: string;
+  is_default: boolean;
+};
+
+type LLamaCloudPipeline = {
+  id: string;
+  name: string;
+  project_id: string;
+};
 
 export class LLamaCloudFileService {
+  private static readonly headers = {
+    Accept: "application/json",
+    Authorization: `Bearer ${process.env.LLAMA_CLOUD_API_KEY}`,
+  };
+
+  public static async getAllProjectsWithPipelines() {
+    try {
+      const projects = await LLamaCloudFileService.getAllProjects();
+      const pipelines = await LLamaCloudFileService.getAllPipelines();
+      return projects.map((project) => ({
+        ...project,
+        pipelines: pipelines.filter((p) => p.project_id === project.id),
+      }));
+    } catch (error) {
+      console.error("Error listing projects and pipelines:", error);
+      return [];
+    }
+  }
+
   public static async downloadFiles(nodes: NodeWithScore<Metadata>[]) {
-    const files = this.nodesToDownloadFiles(nodes);
+    const files = LLamaCloudFileService.nodesToDownloadFiles(nodes);
     if (!files.length) return;
     console.log("Downloading files from LlamaCloud...");
     for (const file of files) {
-      await this.downloadFile(file.pipelineId, file.fileName);
+      await LLamaCloudFileService.downloadFile(file.pipelineId, file.fileName);
     }
   }
 
@@ -59,13 +91,19 @@ export class LLamaCloudFileService {
 
   private static async downloadFile(pipelineId: string, fileName: string) {
     try {
-      const downloadedName = this.toDownloadedName(pipelineId, fileName);
+      const downloadedName = LLamaCloudFileService.toDownloadedName(
+        pipelineId,
+        fileName,
+      );
       const downloadedPath = path.join(LLAMA_CLOUD_OUTPUT_DIR, downloadedName);
 
       // Check if file already exists
       if (fs.existsSync(downloadedPath)) return;
 
-      const urlToDownload = await this.getFileUrlByName(pipelineId, fileName);
+      const urlToDownload = await LLamaCloudFileService.getFileUrlByName(
+        pipelineId,
+        fileName,
+      );
       if (!urlToDownload) throw new Error("File not found in LlamaCloud");
 
       const file = fs.createWriteStream(downloadedPath);
@@ -93,10 +131,13 @@ export class LLamaCloudFileService {
     pipelineId: string,
     name: string,
   ): Promise<string | null> {
-    const files = await this.getAllFiles(pipelineId);
+    const files = await LLamaCloudFileService.getAllFiles(pipelineId);
     const file = files.find((file) => file.name === name);
     if (!file) return null;
-    return await this.getFileUrlById(file.project_id, file.file_id);
+    return await LLamaCloudFileService.getFileUrlById(
+      file.project_id,
+      file.file_id,
+    );
   }
 
   private static async getFileUrlById(
@@ -104,11 +145,10 @@ export class LLamaCloudFileService {
     fileId: string,
   ): Promise<string> {
     const url = `${LLAMA_CLOUD_BASE_URL}/files/${fileId}/content?project_id=${projectId}`;
-    const headers = {
-      Accept: "application/json",
-      Authorization: `Bearer ${process.env.LLAMA_CLOUD_API_KEY}`,
-    };
-    const response = await fetch(url, { method: "GET", headers });
+    const response = await fetch(url, {
+      method: "GET",
+      headers: LLamaCloudFileService.headers,
+    });
     const data = (await response.json()) as { url: string };
     return data.url;
   }
@@ -117,12 +157,31 @@ export class LLamaCloudFileService {
     pipelineId: string,
   ): Promise<LlamaCloudFile[]> {
     const url = `${LLAMA_CLOUD_BASE_URL}/pipelines/${pipelineId}/files`;
-    const headers = {
-      Accept: "application/json",
-      Authorization: `Bearer ${process.env.LLAMA_CLOUD_API_KEY}`,
-    };
-    const response = await fetch(url, { method: "GET", headers });
+    const response = await fetch(url, {
+      method: "GET",
+      headers: LLamaCloudFileService.headers,
+    });
     const data = await response.json();
     return data;
   }
+
+  private static async getAllProjects(): Promise<LLamaCloudProject[]> {
+    const url = `${LLAMA_CLOUD_BASE_URL}/projects`;
+    const response = await fetch(url, {
+      method: "GET",
+      headers: LLamaCloudFileService.headers,
+    });
+    const data = (await response.json()) as LLamaCloudProject[];
+    return data;
+  }
+
+  private static async getAllPipelines(): Promise<LLamaCloudPipeline[]> {
+    const url = `${LLAMA_CLOUD_BASE_URL}/pipelines`;
+    const response = await fetch(url, {
+      method: "GET",
+      headers: LLamaCloudFileService.headers,
+    });
+    const data = (await response.json()) as LLamaCloudPipeline[];
+    return data;
+  }
 }

templates/components/llamaindex/typescript/streaming/suggestion.ts

@@ -33,8 +33,8 @@ export async function generateNextQuestions(
     const questions = extractQuestions(response.text);
     return questions;
   } catch (error) {
-    console.error("Error: ", error);
-    throw error;
+    console.error("Error when generating the next questions: ", error);
+    return [];
   }
 }
 

templates/components/loaders/python/db.py

@@ -1,8 +1,6 @@
-import os
 import logging
 from typing import List
-from pydantic import BaseModel, validator
-from llama_index.core.indices.vector_store import VectorStoreIndex
+from pydantic import BaseModel
 
 logger = logging.getLogger(__name__)
 

templates/components/loaders/python/web.py

@@ -1,5 +1,3 @@
-import os
-import json
 from pydantic import BaseModel, Field
 
 

templates/components/loaders/typescript/file/loader.ts

@@ -10,7 +10,15 @@ export function getExtractors() {
 }
 
 export async function getDocuments() {
-  return await new SimpleDirectoryReader().loadData({
+  const documents = await new SimpleDirectoryReader().loadData({
     directoryPath: DATA_DIR,
   });
+  // Set private=false to mark the document as public (required for filtering)
+  for (const document of documents) {
+    document.metadata = {
+      ...document.metadata,
+      private: "false",
+    };
+  }
+  return documents;
 }

templates/components/loaders/typescript/llama_parse/loader.ts

@@ -23,8 +23,16 @@ export function getExtractors() {
 export async function getDocuments() {
   const reader = new SimpleDirectoryReader();
   const extractors = getExtractors();
-  return await reader.loadData({
+  const documents = await reader.loadData({
     directoryPath: DATA_DIR,
     fileExtToReader: extractors,
   });
+  // Set private=false to mark the document as public (required for filtering)
+  for (const document of documents) {
+    document.metadata = {
+      ...document.metadata,
+      private: "false",
+    };
+  }
+  return documents;
 }

templates/components/observability/python/llamatrace/observability.py

@@ -0,0 +1,12 @@
+import llama_index.core
+import os
+
+
+def init_observability():
+    PHOENIX_API_KEY = os.getenv("PHOENIX_API_KEY")
+    if not PHOENIX_API_KEY:
+        raise ValueError("PHOENIX_API_KEY environment variable is not set")
+    os.environ["OTEL_EXPORTER_OTLP_HEADERS"] = f"api_key={PHOENIX_API_KEY}"
+    llama_index.core.set_global_handler(
+        "arize_phoenix", endpoint="https://llamatrace.com/v1/traces"
+    )

templates/components/sample-projects/llamapack/pyproject.toml

@@ -6,7 +6,7 @@ authors = ["Marcus Schiesser <mail@marcusschiesser.de>"]
 readme = "README.md"
 
 [tool.poetry.dependencies]
-python = "^3.11,<3.12"
+python = "^3.11,<4.0"
 llama-index = "^0.10.6"
 llama-index-readers-file = "^0.1.3"
 python-dotenv = "^1.0.0"

templates/components/settings/python/llmhub.py

@@ -6,11 +6,13 @@ import os
 DEFAULT_MODEL = "gpt-3.5-turbo"
 DEFAULT_EMBEDDING_MODEL = "text-embedding-3-large"
 
+
 class TSIEmbedding(OpenAIEmbedding):
     def __init__(self, **kwargs):
         super().__init__(**kwargs)
         self._query_engine = self._text_engine = self.model_name
 
+
 def llm_config_from_env() -> Dict:
     from llama_index.core.constants import DEFAULT_TEMPERATURE
 
@@ -32,7 +34,7 @@ def llm_config_from_env() -> Dict:
 
 def embedding_config_from_env() -> Dict:
     from llama_index.core.constants import DEFAULT_EMBEDDING_DIM
-    
+
     model = os.getenv("EMBEDDING_MODEL", DEFAULT_EMBEDDING_MODEL)
     dimension = os.getenv("EMBEDDING_DIM", DEFAULT_EMBEDDING_DIM)
     api_key = os.getenv("T_SYSTEMS_LLMHUB_API_KEY")
@@ -46,6 +48,7 @@ def embedding_config_from_env() -> Dict:
     }
     return config
 
+
 def init_llmhub():
     from llama_index.llms.openai_like import OpenAILike
 
@@ -58,4 +61,4 @@ def init_llmhub():
         is_chat_model=True,
         is_function_calling_model=False,
         context_window=4096,
-    )
+    )

templates/components/settings/python/settings.py

@@ -75,27 +75,33 @@ def init_azure_openai():
     from llama_index.embeddings.azure_openai import AzureOpenAIEmbedding
     from llama_index.llms.azure_openai import AzureOpenAI
 
-    llm_deployment = os.getenv("AZURE_OPENAI_LLM_DEPLOYMENT")
-    embedding_deployment = os.getenv("AZURE_OPENAI_EMBEDDING_DEPLOYMENT")
+    llm_deployment = os.environ["AZURE_OPENAI_LLM_DEPLOYMENT"]
+    embedding_deployment = os.environ["AZURE_OPENAI_EMBEDDING_DEPLOYMENT"]
     max_tokens = os.getenv("LLM_MAX_TOKENS")
-    api_key = os.getenv("AZURE_OPENAI_API_KEY")
-    llm_config = {
-        "api_key": api_key,
-        "deployment_name": llm_deployment,
-        "model": os.getenv("MODEL"),
-        "temperature": float(os.getenv("LLM_TEMPERATURE", DEFAULT_TEMPERATURE)),
-        "max_tokens": int(max_tokens) if max_tokens is not None else None,
-    }
-    Settings.llm = AzureOpenAI(**llm_config)
-
+    temperature = os.getenv("LLM_TEMPERATURE", DEFAULT_TEMPERATURE)
     dimensions = os.getenv("EMBEDDING_DIM")
-    embedding_config = {
-        "api_key": api_key,
-        "deployment_name": embedding_deployment,
-        "model": os.getenv("EMBEDDING_MODEL"),
-        "dimensions": int(dimensions) if dimensions is not None else None,
+
+    azure_config = {
+        "api_key": os.environ["AZURE_OPENAI_API_KEY"],
+        "azure_endpoint": os.environ["AZURE_OPENAI_ENDPOINT"],
+        "api_version": os.getenv("AZURE_OPENAI_API_VERSION")
+        or os.getenv("OPENAI_API_VERSION"),
     }
-    Settings.embed_model = AzureOpenAIEmbedding(**embedding_config)
+
+    Settings.llm = AzureOpenAI(
+        model=os.getenv("MODEL"),
+        max_tokens=int(max_tokens) if max_tokens is not None else None,
+        temperature=float(temperature),
+        deployment_name=llm_deployment,
+        **azure_config,
+    )
+
+    Settings.embed_model = AzureOpenAIEmbedding(
+        model=os.getenv("EMBEDDING_MODEL"),
+        dimensions=int(dimensions) if dimensions is not None else None,
+        deployment_name=embedding_deployment,
+        **azure_config,
+    )
 
 
 def init_fastembed():
@@ -108,7 +114,7 @@ def init_fastembed():
         # Small and multilingual
         "all-MiniLM-L6-v2": "sentence-transformers/all-MiniLM-L6-v2",
         # Large and multilingual
-        "paraphrase-multilingual-mpnet-base-v2": "sentence-transformers/paraphrase-multilingual-mpnet-base-v2",   # noqa: E501
+        "paraphrase-multilingual-mpnet-base-v2": "sentence-transformers/paraphrase-multilingual-mpnet-base-v2",  # noqa: E501
     }
 
     # This will download the model automatically if it is not already downloaded
@@ -116,6 +122,7 @@ def init_fastembed():
         model_name=embed_model_map[os.getenv("EMBEDDING_MODEL")]
     )
 
+
 def init_groq():
     from llama_index.llms.groq import Groq
 
@@ -125,7 +132,6 @@ def init_groq():
         "mixtral-8x7b": "mixtral-8x7b-32768",
     }
 
-
     Settings.llm = Groq(model=model_map[os.getenv("MODEL")])
     # Groq does not provide embeddings, so we use FastEmbed instead
     init_fastembed()

templates/components/ui/html/chat/hooks/use-config.ts

@@ -1,31 +1,24 @@
 "use client";
 
-import { useEffect, useMemo, useState } from "react";
-
 export interface ChatConfig {
   backend?: string;
-  starterQuestions?: string[];
+}
+
+function getBackendOrigin(): string {
+  const chatAPI = process.env.NEXT_PUBLIC_CHAT_API;
+  if (chatAPI) {
+    return new URL(chatAPI).origin;
+  } else {
+    if (typeof window !== "undefined") {
+      // Use BASE_URL from window.ENV
+      return (window as any).ENV?.BASE_URL || "";
+    }
+    return "";
+  }
 }
 
 export function useClientConfig(): ChatConfig {
-  const chatAPI = process.env.NEXT_PUBLIC_CHAT_API;
-  const [config, setConfig] = useState<ChatConfig>();
-
-  const backendOrigin = useMemo(() => {
-    return chatAPI ? new URL(chatAPI).origin : "";
-  }, [chatAPI]);
-
-  const configAPI = `${backendOrigin}/api/chat/config`;
-
-  useEffect(() => {
-    fetch(configAPI)
-      .then((response) => response.json())
-      .then((data) => setConfig({ ...data, chatAPI }))
-      .catch((error) => console.error("Error fetching config", error));
-  }, [chatAPI, configAPI]);
-
   return {
-    backend: backendOrigin,
-    starterQuestions: config?.starterQuestions,
+    backend: getBackendOrigin(),
   };
 }

templates/components/vectordbs/python/llamacloud/generate.py

@@ -1,46 +1,47 @@
+# flake8: noqa: E402
 from dotenv import load_dotenv
 
+from app.engine.index import get_index
+
 load_dotenv()
 
-import os
 import logging
-from app.settings import init_settings
-from app.engine.loaders import get_documents
-from llama_index.indices.managed.llama_cloud import LlamaCloudIndex
-
+from llama_index.core.readers import SimpleDirectoryReader
+from app.engine.service import LLamaCloudFileService
 
 logging.basicConfig(level=logging.INFO)
 logger = logging.getLogger()
 
 
 def generate_datasource():
-    init_settings()
     logger.info("Generate index for the provided data")
 
-    name = os.getenv("LLAMA_CLOUD_INDEX_NAME")
-    project_name = os.getenv("LLAMA_CLOUD_PROJECT_NAME")
-    api_key = os.getenv("LLAMA_CLOUD_API_KEY")
-    base_url = os.getenv("LLAMA_CLOUD_BASE_URL")
+    index = get_index()
+    project_id = index._get_project_id()
+    pipeline_id = index._get_pipeline_id()
 
-    if name is None or project_name is None or api_key is None:
-        raise ValueError(
-            "Please set LLAMA_CLOUD_INDEX_NAME, LLAMA_CLOUD_PROJECT_NAME and LLAMA_CLOUD_API_KEY"
-            " to your environment variables or config them in .env file"
-        )
-
-    documents = get_documents()
-
-    # Set private=false to mark the document as public (required for filtering)
-    for doc in documents:
-        doc.metadata["private"] = "false"
-
-    LlamaCloudIndex.from_documents(
-        documents=documents,
-        name=name,
-        project_name=project_name,
-        api_key=api_key,
-        base_url=base_url,
+    # use SimpleDirectoryReader to retrieve the files to process
+    reader = SimpleDirectoryReader(
+        "data",
+        recursive=True,
     )
+    files_to_process = reader.input_files
+
+    # add each file to the LlamaCloud pipeline
+    for input_file in files_to_process:
+        with open(input_file, "rb") as f:
+            logger.info(
+                f"Adding file {input_file} to pipeline {index.name} in project {index.project_name}"
+            )
+            LLamaCloudFileService.add_file_to_pipeline(
+                project_id,
+                pipeline_id,
+                f,
+                custom_metadata={
+                    # Set private=false to mark the document as public (required for filtering)
+                    "private": "false",
+                },
+            )
 
     logger.info("Finished generating the index")
 

templates/components/vectordbs/python/llamacloud/index.py

@@ -1,16 +1,28 @@
 import logging
 import os
 from llama_index.indices.managed.llama_cloud import LlamaCloudIndex
-
+from llama_index.core.ingestion.api_utils import (
+    get_client as llama_cloud_get_client,
+)
 
 logger = logging.getLogger("uvicorn")
 
 
-def get_index():
-    name = os.getenv("LLAMA_CLOUD_INDEX_NAME")
-    project_name = os.getenv("LLAMA_CLOUD_PROJECT_NAME")
+def get_client():
+    return llama_cloud_get_client(
+        os.getenv("LLAMA_CLOUD_API_KEY"),
+        os.getenv("LLAMA_CLOUD_BASE_URL"),
+    )
+
+
+def get_index(params=None):
+    configParams = params or {}
+    pipelineConfig = configParams.get("llamaCloudPipeline", {})
+    name = pipelineConfig.get("pipeline", os.getenv("LLAMA_CLOUD_INDEX_NAME"))
+    project_name = pipelineConfig.get("project", os.getenv("LLAMA_CLOUD_PROJECT_NAME"))
     api_key = os.getenv("LLAMA_CLOUD_API_KEY")
     base_url = os.getenv("LLAMA_CLOUD_BASE_URL")
+    organization_id = os.getenv("LLAMA_CLOUD_ORGANIZATION_ID")
 
     if name is None or project_name is None or api_key is None:
         raise ValueError(
@@ -23,6 +35,7 @@ def get_index():
         project_name=project_name,
         api_key=api_key,
         base_url=base_url,
+        organization_id=organization_id,
     )
 
     return index

templates/components/vectordbs/python/llamacloud/query_filter.py

@@ -0,0 +1,35 @@
+from llama_index.core.vector_stores.types import MetadataFilter, MetadataFilters
+
+
+def generate_filters(doc_ids):
+    """
+    Generate public/private document filters based on the doc_ids and the vector store.
+    """
+    # Using "nin" filter to include the documents don't have the "private" key because they're uploaded in LlamaCloud UI
+    public_doc_filter = MetadataFilter(
+        key="private",
+        value=["true"],
+        operator="nin",  # type: ignore
+    )
+    selected_doc_filter = MetadataFilter(
+        key="file_id",  # Note: LLamaCloud uses "file_id" to reference private document ids as "doc_id" is a restricted field in LlamaCloud
+        value=doc_ids,
+        operator="in",  # type: ignore
+    )
+    if len(doc_ids) > 0:
+        # If doc_ids are provided, we will select both public and selected documents
+        filters = MetadataFilters(
+            filters=[
+                public_doc_filter,
+                selected_doc_filter,
+            ],
+            condition="or",  # type: ignore
+        )
+    else:
+        filters = MetadataFilters(
+            filters=[
+                public_doc_filter,
+            ]
+        )
+
+    return filters

templates/components/vectordbs/python/llamacloud/service.py

@@ -0,0 +1,173 @@
+from io import BytesIO
+import logging
+import os
+import time
+from typing import Any, Dict, List, Optional, Set, Tuple, Union
+import typing
+
+from fastapi import BackgroundTasks
+from llama_cloud import ManagedIngestionStatus, PipelineFileCreateCustomMetadataValue
+from pydantic import BaseModel
+import requests
+from app.api.routers.models import SourceNodes
+from app.engine.index import get_client
+from llama_index.core.schema import NodeWithScore
+
+
+logger = logging.getLogger("uvicorn")
+
+
+class LlamaCloudFile(BaseModel):
+    file_name: str
+    pipeline_id: str
+
+    def __eq__(self, other):
+        if not isinstance(other, LlamaCloudFile):
+            return NotImplemented
+        return (
+            self.file_name == other.file_name and self.pipeline_id == other.pipeline_id
+        )
+
+    def __hash__(self):
+        return hash((self.file_name, self.pipeline_id))
+
+
+class LLamaCloudFileService:
+    LOCAL_STORE_PATH = "output/llamacloud"
+    DOWNLOAD_FILE_NAME_TPL = "{pipeline_id}${filename}"
+
+    @classmethod
+    def get_all_projects_with_pipelines(cls) -> List[Dict[str, Any]]:
+        try:
+            client = get_client()
+            projects = client.projects.list_projects()
+            pipelines = client.pipelines.search_pipelines()
+            return [
+                {
+                    **(project.dict()),
+                    "pipelines": [
+                        {"id": p.id, "name": p.name}
+                        for p in pipelines
+                        if p.project_id == project.id
+                    ],
+                }
+                for project in projects
+            ]
+        except Exception as error:
+            logger.error(f"Error listing projects and pipelines: {error}")
+            return []
+
+    @classmethod
+    def add_file_to_pipeline(
+        cls,
+        project_id: str,
+        pipeline_id: str,
+        upload_file: Union[typing.IO, Tuple[str, BytesIO]],
+        custom_metadata: Optional[Dict[str, PipelineFileCreateCustomMetadataValue]],
+    ) -> str:
+        client = get_client()
+        file = client.files.upload_file(project_id=project_id, upload_file=upload_file)
+        files = [
+            {
+                "file_id": file.id,
+                "custom_metadata": {"file_id": file.id, **(custom_metadata or {})},
+            }
+        ]
+        files = client.pipelines.add_files_to_pipeline(pipeline_id, request=files)
+
+        # Wait 2s for the file to be processed
+        max_attempts = 20
+        attempt = 0
+        while attempt < max_attempts:
+            result = client.pipelines.get_pipeline_file_status(pipeline_id, file.id)
+            if result.status == ManagedIngestionStatus.ERROR:
+                raise Exception(f"File processing failed: {str(result)}")
+            if result.status == ManagedIngestionStatus.SUCCESS:
+                # File is ingested - return the file id
+                return file.id
+            attempt += 1
+            time.sleep(0.1)  # Sleep for 100ms
+        raise Exception(
+            f"File processing did not complete after {max_attempts} attempts."
+        )
+
+    @classmethod
+    def download_pipeline_file(
+        cls,
+        file: LlamaCloudFile,
+        force_download: bool = False,
+    ):
+        client = get_client()
+        file_name = file.file_name
+        pipeline_id = file.pipeline_id
+
+        # Check is the file already exists
+        downloaded_file_path = cls._get_file_path(file_name, pipeline_id)
+        if os.path.exists(downloaded_file_path) and not force_download:
+            logger.debug(f"File {file_name} already exists in local storage")
+            return
+        try:
+            logger.info(f"Downloading file {file_name} for pipeline {pipeline_id}")
+            files = client.pipelines.list_pipeline_files(pipeline_id)
+            if not files or not isinstance(files, list):
+                raise Exception("No files found in LlamaCloud")
+            for file_entry in files:
+                if file_entry.name == file_name:
+                    file_id = file_entry.file_id
+                    project_id = file_entry.project_id
+                    file_detail = client.files.read_file_content(
+                        file_id, project_id=project_id
+                    )
+                    cls._download_file(file_detail.url, downloaded_file_path)
+                    break
+        except Exception as error:
+            logger.info(f"Error fetching file from LlamaCloud: {error}")
+
+    @classmethod
+    def download_files_from_nodes(
+        cls, nodes: List[NodeWithScore], background_tasks: BackgroundTasks
+    ):
+        files = cls._get_files_to_download(nodes)
+        for file in files:
+            logger.info(f"Adding download of {file.file_name} to background tasks")
+            background_tasks.add_task(
+                LLamaCloudFileService.download_pipeline_file, file
+            )
+
+    @classmethod
+    def _get_files_to_download(cls, nodes: List[NodeWithScore]) -> Set[LlamaCloudFile]:
+        source_nodes = SourceNodes.from_source_nodes(nodes)
+        llama_cloud_files = [
+            LlamaCloudFile(
+                file_name=node.metadata.get("file_name"),
+                pipeline_id=node.metadata.get("pipeline_id"),
+            )
+            for node in source_nodes
+            if (
+                node.metadata.get("pipeline_id") is not None
+                and node.metadata.get("file_name") is not None
+            )
+        ]
+        # Remove duplicates and return
+        return set(llama_cloud_files)
+
+    @classmethod
+    def _get_file_name(cls, name: str, pipeline_id: str) -> str:
+        return cls.DOWNLOAD_FILE_NAME_TPL.format(pipeline_id=pipeline_id, filename=name)
+
+    @classmethod
+    def _get_file_path(cls, name: str, pipeline_id: str) -> str:
+        return os.path.join(cls.LOCAL_STORE_PATH, cls._get_file_name(name, pipeline_id))
+
+    @classmethod
+    def _download_file(cls, url: str, local_file_path: str):
+        logger.info(f"Saving file to {local_file_path}")
+        # Create directory if it doesn't exist
+        os.makedirs(cls.LOCAL_STORE_PATH, exist_ok=True)
+        # Download the file
+        with requests.get(url, stream=True) as r:
+            r.raise_for_status()
+            with open(local_file_path, "wb") as f:
+                for chunk in r.iter_content(chunk_size=8192):
+                    f.write(chunk)
+        logger.info("File downloaded successfully")

templates/components/vectordbs/python/none/generate.py

@@ -1,3 +1,4 @@
+# flake8: noqa: E402
 from dotenv import load_dotenv
 
 load_dotenv()

templates/components/vectordbs/python/none/index.py

@@ -17,7 +17,7 @@ def get_storage_context(persist_dir: str) -> StorageContext:
     return StorageContext.from_defaults(persist_dir=persist_dir)
 
 
-def get_index():
+def get_index(params=None):
     storage_dir = os.getenv("STORAGE_DIR", "storage")
     # check if storage already exists
     if not os.path.exists(storage_dir):

templates/components/vectordbs/python/weaviate/query_filter.py

@@ -0,0 +1,36 @@
+from llama_index.core.vector_stores.types import MetadataFilter, MetadataFilters
+
+
+def generate_filters(doc_ids):
+    """
+    Generate public/private document filters based on the doc_ids and the vector store.
+    """
+    public_doc_filter = MetadataFilter(
+        key="private",
+        value="true",
+        operator="!=",  # type: ignore
+    )
+    # Weaviate doesn't support "in" filter right now, so use "any" instead - it has the same behavior.
+    # TODO: Use "in" operator, once Weaviate supports it
+    selected_doc_filter = MetadataFilter(
+        key="doc_id",
+        value=doc_ids,
+        operator="any",  # type: ignore
+    )
+    if len(doc_ids) > 0:
+        # If doc_ids are provided, we will select both public and selected documents
+        filters = MetadataFilters(
+            filters=[
+                public_doc_filter,
+                selected_doc_filter,
+            ],
+            condition="or",  # type: ignore
+        )
+    else:
+        filters = MetadataFilters(
+            filters=[
+                public_doc_filter,
+            ]
+        )
+
+    return filters

templates/components/vectordbs/python/weaviate/vectordb.py

@@ -0,0 +1,35 @@
+import os
+
+import weaviate
+from llama_index.vector_stores.weaviate import WeaviateVectorStore
+
+DEFAULT_INDEX_NAME = "LlamaIndex"
+
+
+def _create_weaviate_client():
+    cluster_url = os.getenv("WEAVIATE_CLUSTER_URL")
+    api_key = os.getenv("WEAVIATE_API_KEY")
+    if not cluster_url or not api_key:
+        raise ValueError(
+            "Environment variables: WEAVIATE_CLUSTER_URL and WEAVIATE_API_KEY are required."
+        )
+    auth_credentials = weaviate.auth.AuthApiKey(api_key)
+    client = weaviate.connect_to_weaviate_cloud(cluster_url, auth_credentials)
+    return client
+
+
+# Global variable to store the Weaviate client
+client = None
+
+
+def get_vector_store():
+    global client
+    if client is None:
+        client = _create_weaviate_client()
+
+    index_name = os.getenv("WEAVIATE_INDEX_NAME", DEFAULT_INDEX_NAME)
+    vector_store = WeaviateVectorStore(
+        weaviate_client=client,
+        index_name=index_name,
+    )
+    return vector_store

templates/components/vectordbs/typescript/astra/index.ts

@@ -3,7 +3,7 @@ import { VectorStoreIndex } from "llamaindex";
 import { AstraDBVectorStore } from "llamaindex/storage/vectorStore/AstraDBVectorStore";
 import { checkRequiredEnvVars } from "./shared";
 
-export async function getDataSource() {
+export async function getDataSource(params?: any) {
   checkRequiredEnvVars();
   const store = new AstraDBVectorStore();
   await store.connect(process.env.ASTRA_DB_COLLECTION!);

templates/components/vectordbs/typescript/chroma/index.ts

@@ -3,7 +3,7 @@ import { VectorStoreIndex } from "llamaindex";
 import { ChromaVectorStore } from "llamaindex/storage/vectorStore/ChromaVectorStore";
 import { checkRequiredEnvVars } from "./shared";
 
-export async function getDataSource() {
+export async function getDataSource(params?: any) {
   checkRequiredEnvVars();
   const chromaUri = `http://${process.env.CHROMA_HOST}:${process.env.CHROMA_PORT}`;
 

templates/components/vectordbs/typescript/llamacloud/generate.ts

@@ -9,13 +9,7 @@ dotenv.config();
 
 async function loadAndIndex() {
   const documents = await getDocuments();
-  // Set private=false to mark the document as public (required for filtering)
-  for (const document of documents) {
-    document.metadata = {
-      ...document.metadata,
-      private: "false",
-    };
-  }
+
   await getDataSource();
   await LlamaCloudIndex.fromDocuments({
     documents,

templates/components/vectordbs/typescript/llamacloud/index.ts

@@ -1,12 +1,26 @@
 import { LlamaCloudIndex } from "llamaindex/cloud/LlamaCloudIndex";
-import { checkRequiredEnvVars } from "./shared";
 
-export async function getDataSource() {
-  checkRequiredEnvVars();
+type LlamaCloudDataSourceParams = {
+  llamaCloudPipeline?: {
+    project: string;
+    pipeline: string;
+  };
+};
+
+export async function getDataSource(params?: LlamaCloudDataSourceParams) {
+  const { project, pipeline } = params?.llamaCloudPipeline ?? {};
+  const projectName = project ?? process.env.LLAMA_CLOUD_PROJECT_NAME;
+  const pipelineName = pipeline ?? process.env.LLAMA_CLOUD_INDEX_NAME;
+  const apiKey = process.env.LLAMA_CLOUD_API_KEY;
+  if (!projectName || !pipelineName || !apiKey) {
+    throw new Error(
+      "Set project, pipeline, and api key in the params or as environment variables.",
+    );
+  }
   const index = new LlamaCloudIndex({
-    name: process.env.LLAMA_CLOUD_INDEX_NAME!,
-    projectName: process.env.LLAMA_CLOUD_PROJECT_NAME!,
-    apiKey: process.env.LLAMA_CLOUD_API_KEY,
+    name: pipelineName,
+    projectName,
+    apiKey,
     baseUrl: process.env.LLAMA_CLOUD_BASE_URL,
   });
   return index;

templates/components/vectordbs/typescript/llamacloud/queryFilter.ts

@@ -0,0 +1,25 @@
+import { MetadataFilter, MetadataFilters } from "llamaindex";
+
+export function generateFilters(documentIds: string[]): MetadataFilters {
+  // public documents don't have the "private" field or it's set to "false"
+  const publicDocumentsFilter: MetadataFilter = {
+    key: "private",
+    value: ["true"],
+    operator: "nin",
+  };
+
+  // if no documentIds are provided, only retrieve information from public documents
+  if (!documentIds.length) return { filters: [publicDocumentsFilter] };
+
+  const privateDocumentsFilter: MetadataFilter = {
+    key: "doc_id",
+    value: documentIds,
+    operator: "in",
+  };
+
+  // if documentIds are provided, retrieve information from public and private documents
+  return {
+    filters: [publicDocumentsFilter, privateDocumentsFilter],
+    condition: "or",
+  };
+}

templates/components/vectordbs/typescript/milvus/index.ts

@@ -2,7 +2,7 @@ import { VectorStoreIndex } from "llamaindex";
 import { MilvusVectorStore } from "llamaindex/storage/vectorStore/MilvusVectorStore";
 import { checkRequiredEnvVars, getMilvusClient } from "./shared";
 
-export async function getDataSource() {
+export async function getDataSource(params?: any) {
   checkRequiredEnvVars();
   const milvusClient = getMilvusClient();
   const store = new MilvusVectorStore({ milvusClient });

templates/components/vectordbs/typescript/milvus/shared.ts

@@ -4,7 +4,6 @@ const REQUIRED_ENV_VARS = [
   "MILVUS_ADDRESS",
   "MILVUS_USERNAME",
   "MILVUS_PASSWORD",
-  "MILVUS_COLLECTION",
 ];
 
 export function getMilvusClient() {
@@ -19,8 +18,13 @@ export function getMilvusClient() {
   });
 }
 
-export function checkRequiredEnvVars() {
-  const missingEnvVars = REQUIRED_ENV_VARS.filter((envVar) => {
+export function checkRequiredEnvVars(opts?: { checkCollectionEnv?: boolean }) {
+  const shouldCheckCollectionEnv = opts?.checkCollectionEnv ?? true; // default to true
+  const requiredEnvVars = [...REQUIRED_ENV_VARS]; // create a copy of the array
+  if (shouldCheckCollectionEnv) {
+    requiredEnvVars.push("MILVUS_COLLECTION");
+  }
+  const missingEnvVars = requiredEnvVars.filter((envVar) => {
     return !process.env[envVar];
   });
 

templates/components/vectordbs/typescript/mongo/generate.ts

@@ -1,7 +1,10 @@
 /* eslint-disable turbo/no-undeclared-env-vars */
 import * as dotenv from "dotenv";
-import { VectorStoreIndex, storageContextFromDefaults } from "llamaindex";
-import { MongoDBAtlasVectorSearch } from "llamaindex/storage/vectorStore/MongoDBAtlasVectorSearch";
+import {
+  MongoDBAtlasVectorSearch,
+  VectorStoreIndex,
+  storageContextFromDefaults,
+} from "llamaindex";
 import { MongoClient } from "mongodb";
 import { getDocuments } from "./loader";
 import { initSettings } from "./settings";

templates/components/vectordbs/typescript/mongo/index.ts

@@ -1,10 +1,9 @@
 /* eslint-disable turbo/no-undeclared-env-vars */
-import { VectorStoreIndex } from "llamaindex";
-import { MongoDBAtlasVectorSearch } from "llamaindex/storage/vectorStore/MongoDBAtlasVectorSearch";
+import { MongoDBAtlasVectorSearch, VectorStoreIndex } from "llamaindex";
 import { MongoClient } from "mongodb";
 import { checkRequiredEnvVars } from "./shared";
 
-export async function getDataSource() {
+export async function getDataSource(params?: any) {
   checkRequiredEnvVars();
   const client = new MongoClient(process.env.MONGO_URI!);
   const store = new MongoDBAtlasVectorSearch({

templates/components/vectordbs/typescript/none/generate.ts

@@ -25,10 +25,6 @@ async function generateDatasource() {
       persistDir: STORAGE_CACHE_DIR,
     });
     const documents = await getDocuments();
-    //  Set private=false to mark the document as public (required for filtering)
-    documents.forEach((doc) => {
-      doc.metadata["private"] = "false";
-    });
 
     await VectorStoreIndex.fromDocuments(documents, {
       storageContext,

templates/components/vectordbs/typescript/none/index.ts

@@ -2,7 +2,7 @@ import { SimpleDocumentStore, VectorStoreIndex } from "llamaindex";
 import { storageContextFromDefaults } from "llamaindex/storage/StorageContext";
 import { STORAGE_CACHE_DIR } from "./shared";
 
-export async function getDataSource() {
+export async function getDataSource(params?: any) {
   const storageContext = await storageContextFromDefaults({
     persistDir: `${STORAGE_CACHE_DIR}`,
   });

templates/components/vectordbs/typescript/pg/index.ts

@@ -7,7 +7,7 @@ import {
   checkRequiredEnvVars,
 } from "./shared";
 
-export async function getDataSource() {
+export async function getDataSource(params?: any) {
   checkRequiredEnvVars();
   const pgvs = new PGVectorStore({
     connectionString: process.env.PG_CONNECTION_STRING,

templates/components/vectordbs/typescript/pinecone/index.ts

@@ -3,7 +3,7 @@ import { VectorStoreIndex } from "llamaindex";
 import { PineconeVectorStore } from "llamaindex/storage/vectorStore/PineconeVectorStore";
 import { checkRequiredEnvVars } from "./shared";
 
-export async function getDataSource() {
+export async function getDataSource(params?: any) {
   checkRequiredEnvVars();
   const store = new PineconeVectorStore();
   return await VectorStoreIndex.fromVectorStore(store);

templates/components/vectordbs/typescript/qdrant/index.ts

@@ -5,7 +5,7 @@ import { checkRequiredEnvVars, getQdrantClient } from "./shared";
 
 dotenv.config();
 
-export async function getDataSource() {
+export async function getDataSource(params?: any) {
   checkRequiredEnvVars();
   const collectionName = process.env.QDRANT_COLLECTION;
   const store = new QdrantVectorStore({

templates/components/vectordbs/typescript/weaviate/generate.ts

@@ -0,0 +1,33 @@
+/* eslint-disable turbo/no-undeclared-env-vars */
+import * as dotenv from "dotenv";
+import {
+  VectorStoreIndex,
+  WeaviateVectorStore,
+  storageContextFromDefaults,
+} from "llamaindex";
+import { getDocuments } from "./loader";
+import { initSettings } from "./settings";
+import { DEFAULT_INDEX_NAME, checkRequiredEnvVars } from "./shared";
+dotenv.config();
+
+async function loadAndIndex() {
+  const indexName = process.env.WEAVIATE_INDEX_NAME || DEFAULT_INDEX_NAME;
+
+  // load objects from storage and convert them into LlamaIndex Document objects
+  const documents = await getDocuments();
+
+  const vectorStore = new WeaviateVectorStore({ indexName });
+
+  const storageContext = await storageContextFromDefaults({ vectorStore });
+  await VectorStoreIndex.fromDocuments(documents, {
+    storageContext: storageContext,
+  });
+  console.log(`Successfully upload embeddings to Weaviate index ${indexName}.`);
+}
+
+(async () => {
+  checkRequiredEnvVars();
+  initSettings();
+  await loadAndIndex();
+  console.log("Finished generating storage.");
+})();

templates/components/vectordbs/typescript/weaviate/index.ts

@@ -0,0 +1,14 @@
+import * as dotenv from "dotenv";
+import { VectorStoreIndex } from "llamaindex";
+import { WeaviateVectorStore } from "llamaindex/storage/vectorStore/WeaviateVectorStore";
+import { checkRequiredEnvVars, DEFAULT_INDEX_NAME } from "./shared";
+
+dotenv.config();
+
+export async function getDataSource(params?: any) {
+  checkRequiredEnvVars();
+  const indexName = process.env.WEAVIATE_INDEX_NAME || DEFAULT_INDEX_NAME;
+  const store = new WeaviateVectorStore({ indexName });
+
+  return await VectorStoreIndex.fromVectorStore(store);
+}

templates/components/vectordbs/typescript/weaviate/queryFilter.ts

@@ -0,0 +1,26 @@
+import { MetadataFilter, MetadataFilters } from "llamaindex";
+
+export function generateFilters(documentIds: string[]): MetadataFilters {
+  // filter all documents have the private metadata key set to true
+  const publicDocumentsFilter: MetadataFilter = {
+    key: "private",
+    value: "true",
+    operator: "!=",
+  };
+
+  // if no documentIds are provided, only retrieve information from public documents
+  if (!documentIds.length) return { filters: [publicDocumentsFilter] };
+
+  // Weaviate uses 'any' instead of 'in' for the operator
+  const privateDocumentsFilter: MetadataFilter = {
+    key: "doc_id",
+    value: documentIds,
+    operator: "any",
+  };
+
+  // if documentIds are provided, retrieve information from public and private documents
+  return {
+    filters: [publicDocumentsFilter, privateDocumentsFilter],
+    condition: "or",
+  };
+}

templates/components/vectordbs/typescript/weaviate/shared.ts

@@ -0,0 +1,20 @@
+const REQUIRED_ENV_VARS = ["WEAVIATE_CLUSTER_URL", "WEAVIATE_API_KEY"];
+
+export const DEFAULT_INDEX_NAME = "LlamaIndex";
+
+export function checkRequiredEnvVars() {
+  const missingEnvVars = REQUIRED_ENV_VARS.filter((envVar) => {
+    return !process.env[envVar];
+  });
+
+  if (missingEnvVars.length > 0) {
+    console.log(
+      `The following environment variables are required but missing: ${missingEnvVars.join(
+        ", ",
+      )}`,
+    );
+    throw new Error(
+      `Missing environment variables: ${missingEnvVars.join(", ")}`,
+    );
+  }
+}

templates/types/extractor/fastapi/README-template.md

@@ -0,0 +1,68 @@
+This is a [LlamaIndex](https://www.llamaindex.ai/) project using [FastAPI](https://fastapi.tiangolo.com/) bootstrapped with [`create-llama`](https://github.com/run-llama/LlamaIndexTS/tree/main/packages/create-llama) featuring [structured extraction](https://docs.llamaindex.ai/en/stable/examples/structured_outputs/structured_outputs/?h=structured+output).
+
+## Getting Started
+
+First, setup the environment with poetry:
+
+> **_Note:_** This step is not needed if you are using the dev-container.
+
+```shell
+poetry install
+poetry shell
+```
+
+Then check the parameters that have been pre-configured in the `.env` file in this directory. (E.g. you might need to configure an `OPENAI_API_KEY` if you're using OpenAI as model provider).
+
+Second, generate the embeddings of the documents in the `./data` directory (if this folder exists - otherwise, skip this step):
+
+```shell
+poetry run generate
+```
+
+Third, run the API in one command:
+
+```shell
+poetry run python main.py
+```
+
+The example provides the `/api/extractor/query` API endpoint.
+
+This query endpoint returns structured data in the format of the [Output](./app/api/routers/output.py) class. Modify this class to change the output format.
+
+You can test the endpoint with the following curl request:
+
+```shell
+curl --location 'localhost:8000/api/extractor/query' \
+--header 'Content-Type: application/json' \
+--data '{ "query": "What is the maximum weight for a parcel?" }'
+```
+
+Which will return a response that the RAG pipeline is confident about the answer.
+
+Try
+
+```shell
+curl --location 'localhost:8000/api/extractor/query' \
+--header 'Content-Type: application/json' \
+--data '{ "query": "What is the weather today?" }'
+```
+
+To retrieve a response with low confidence since the question is not related to the provided document in the `./data` directory.
+
+You can start editing the API endpoint by modifying [`extractor.py`](./app/api/routers/extractor.py). The endpoints auto-update as you save the file.
+
+Open [http://localhost:8000/docs](http://localhost:8000/docs) with your browser to see the Swagger UI of the API.
+
+The API allows CORS for all origins to simplify development. You can change this behavior by setting the `ENVIRONMENT` environment variable to `prod`:
+
+```
+ENVIRONMENT=prod python main.py
+```
+
+## Learn More
+
+To learn more about LlamaIndex, take a look at the following resources:
+
+- [LlamaIndex Documentation](https://docs.llamaindex.ai) - learn about LlamaIndex.
+
+You can check out [the LlamaIndex GitHub repository](https://github.com/run-llama/llama_index) - your feedback and contributions are welcome!

templates/types/extractor/fastapi/app/api/routers/extractor.py

@@ -0,0 +1,58 @@
+import logging
+import os
+
+from fastapi import APIRouter, HTTPException
+from llama_index.core.settings import Settings
+from pydantic import BaseModel
+
+from app.api.routers.output import Output
+from app.engine.index import get_index
+
+extractor_router = r = APIRouter()
+
+logger = logging.getLogger("uvicorn")
+
+
+class RequestData(BaseModel):
+    query: str
+
+    class Config:
+        json_schema_extra = {
+            "examples": [
+                {"query": "What's the maximum weight for a parcel?"},
+            ],
+        }
+
+
+@r.post("/query")
+async def query_request(
+    data: RequestData,
+):
+    # Create a query engine using that returns responses in the format of the Output class
+    query_engine = get_query_engine(Output)
+
+    response = await query_engine.aquery(data.query)
+
+    output_data = response.response.dict()
+    return Output(**output_data)
+
+
+def get_query_engine(output_cls: BaseModel):
+    top_k = os.getenv("TOP_K", 3)
+
+    index = get_index()
+    if index is None:
+        raise HTTPException(
+            status_code=500,
+            detail=str(
+                "StorageContext is empty - call 'poetry run generate' to generate the storage first"
+            ),
+        )
+
+    sllm = Settings.llm.as_structured_llm(output_cls)
+
+    return index.as_query_engine(
+        similarity_top_k=int(top_k),
+        llm=sllm,
+        response_mode="tree_summarize",
+    )

templates/types/extractor/fastapi/app/api/routers/output.py

@@ -0,0 +1,32 @@
+import logging
+from llama_index.core.schema import BaseModel, Field
+from typing import List
+
+logger = logging.getLogger("uvicorn")
+
+
+class Output(BaseModel):
+    response: str = Field(..., description="The answer to the question.")
+    page_numbers: List[int] = Field(
+        ...,
+        description="The page numbers of the sources used to answer this question. Do not include a page number if the context is irrelevant.",
+    )
+    confidence: float = Field(
+        ...,
+        ge=0,
+        le=1,
+        description="Confidence value between 0-1 of the correctness of the result.",
+    )
+    confidence_explanation: str = Field(
+        ..., description="Explanation for the confidence score"
+    )
+
+    class Config:
+        json_schema_extra = {
+            "example": {
+                "response": "This is an example answer.",
+                "page_numbers": [1, 2, 3],
+                "confidence": 0.85,
+                "confidence_explanation": "This is an explanation for the confidence score.",
+            }
+        }

templates/types/extractor/fastapi/main.py

@@ -0,0 +1,46 @@
+# flake8: noqa: E402
+from dotenv import load_dotenv
+
+load_dotenv()
+
+import logging
+import os
+import uvicorn
+from fastapi import FastAPI
+from fastapi.middleware.cors import CORSMiddleware
+from fastapi.responses import RedirectResponse
+from app.api.routers.extractor import extractor_router
+from app.settings import init_settings
+
+
+app = FastAPI()
+
+init_settings()
+
+environment = os.getenv("ENVIRONMENT", "dev")  # Default to 'development' if not set
+logger = logging.getLogger("uvicorn")
+
+if environment == "dev":
+    logger.warning("Running in development mode - allowing CORS for all origins")
+    app.add_middleware(
+        CORSMiddleware,
+        allow_origins=["*"],
+        allow_credentials=True,
+        allow_methods=["*"],
+        allow_headers=["*"],
+    )
+
+    # Redirect to documentation page when accessing base URL
+    @app.get("/")
+    async def redirect_to_docs():
+        return RedirectResponse(url="/docs")
+
+
+app.include_router(extractor_router, prefix="/api/extractor")
+
+if __name__ == "__main__":
+    app_host = os.getenv("APP_HOST", "0.0.0.0")
+    app_port = int(os.getenv("APP_PORT", "8000"))
+    reload = True if environment == "dev" else False
+
+    uvicorn.run(app="main:app", host=app_host, port=app_port, reload=reload)

templates/types/extractor/fastapi/pyproject.toml

@@ -0,0 +1,21 @@
+[tool.poetry]
+name = "app"
+version = "0.1.0"
+description = ""
+authors = ["Marcus Schiesser <mail@marcusschiesser.de>"]
+readme = "README.md"
+
+[tool.poetry.scripts]
+generate = "app.engine.generate:generate_datasource"
+
+[tool.poetry.dependencies]
+python = "^3.11,<4.0"
+fastapi = "^0.109.1"
+uvicorn = { extras = ["standard"], version = "^0.23.2" }
+python-dotenv = "^1.0.0"
+llama-index = "^0.10.58"
+cachetools = "^5.3.3"
+
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"

templates/types/multiagent/fastapi/app/utils.py

@@ -5,4 +5,4 @@ def load_from_env(var: str, throw_error: bool = True) -> str:
     res = os.getenv(var)
     if res is None and throw_error:
         raise ValueError(f"Missing environment variable: {var}")
-    return res
+    return res

templates/types/multiagent/fastapi/main.py

@@ -1,3 +1,4 @@
+# flake8: noqa: E402
 from dotenv import load_dotenv
 from app.settings import init_settings
 

templates/types/streaming/express/package.json

@@ -20,7 +20,7 @@
     "dotenv": "^16.3.1",
     "duck-duck-scrape": "^2.2.5",
     "express": "^4.18.2",
-    "llamaindex": "0.5.7",
+    "llamaindex": "0.5.17",
     "pdf2json": "3.0.5",
     "ajv": "^8.12.0",
     "@e2b/code-interpreter": "^0.0.5",

templates/types/streaming/express/src/controllers/chat-config.controller.ts

@@ -1,4 +1,5 @@
 import { Request, Response } from "express";
+import { LLamaCloudFileService } from "./llamaindex/streaming/service";
 
 export const chatConfig = async (_req: Request, res: Response) => {
   let starterQuestions = undefined;
@@ -12,3 +13,14 @@ export const chatConfig = async (_req: Request, res: Response) => {
     starterQuestions,
   });
 };
+
+export const chatLlamaCloudConfig = async (_req: Request, res: Response) => {
+  const config = {
+    projects: await LLamaCloudFileService.getAllProjectsWithPipelines(),
+    pipeline: {
+      pipeline: process.env.LLAMA_CLOUD_INDEX_NAME,
+      project: process.env.LLAMA_CLOUD_PROJECT_NAME,
+    },
+  };
+  return res.status(200).json(config);
+};

templates/types/streaming/express/src/controllers/chat-upload.controller.ts

@@ -1,12 +1,14 @@
 import { Request, Response } from "express";
+import { getDataSource } from "./engine";
 import { uploadDocument } from "./llamaindex/documents/upload";
 
 export const chatUpload = async (req: Request, res: Response) => {
-  const { base64 }: { base64: string } = req.body;
+  const { base64, params }: { base64: string; params?: any } = req.body;
   if (!base64) {
     return res.status(400).json({
       error: "base64 is required in the request body",
     });
   }
-  return res.status(200).json(await uploadDocument(base64));
+  const index = await getDataSource(params);
+  return res.status(200).json(await uploadDocument(index, base64));
 };

templates/types/streaming/express/src/controllers/chat.controller.ts

@@ -17,7 +17,7 @@ export const chat = async (req: Request, res: Response) => {
   const vercelStreamData = new StreamData();
   const streamTimeout = createStreamTimeout(vercelStreamData);
   try {
-    const { messages }: { messages: Message[] } = req.body;
+    const { messages, data }: { messages: Message[]; data?: any } = req.body;
     const userMessage = messages.pop();
     if (!messages || !userMessage || userMessage.role !== "user") {
       return res.status(400).json({
@@ -46,7 +46,7 @@ export const chat = async (req: Request, res: Response) => {
       },
     );
     const ids = retrieveDocumentIds(allAnnotations);
-    const chatEngine = await createChatEngine(ids);
+    const chatEngine = await createChatEngine(ids, data);
 
     // Convert message content from Vercel/AI format to LlamaIndex/OpenAI format
     const userMessageContent = convertMessageContent(

templates/types/streaming/express/src/controllers/engine/queryFilter.ts

@@ -0,0 +1,25 @@
+import { MetadataFilter, MetadataFilters } from "llamaindex";
+
+export function generateFilters(documentIds: string[]): MetadataFilters {
+  // filter all documents have the private metadata key set to true
+  const publicDocumentsFilter: MetadataFilter = {
+    key: "private",
+    value: "true",
+    operator: "!=",
+  };
+
+  // if no documentIds are provided, only retrieve information from public documents
+  if (!documentIds.length) return { filters: [publicDocumentsFilter] };
+
+  const privateDocumentsFilter: MetadataFilter = {
+    key: "doc_id",
+    value: documentIds,
+    operator: "in",
+  };
+
+  // if documentIds are provided, retrieve information from public and private documents
+  return {
+    filters: [publicDocumentsFilter, privateDocumentsFilter],
+    condition: "or",
+  };
+}

templates/types/streaming/express/src/controllers/engine/settings.ts

@@ -45,6 +45,9 @@ export const initSettings = async () => {
     case "mistral":
       initMistralAI();
       break;
+    case "azure-openai":
+      initAzureOpenAI();
+      break;
     default:
       initOpenAI();
       break;
@@ -68,6 +71,53 @@ function initOpenAI() {
   });
 }
 
+function initAzureOpenAI() {
+  // Map Azure OpenAI model names to OpenAI model names (only for TS)
+  const AZURE_OPENAI_MODEL_MAP: Record<string, string> = {
+    "gpt-35-turbo": "gpt-3.5-turbo",
+    "gpt-35-turbo-16k": "gpt-3.5-turbo-16k",
+    "gpt-4o": "gpt-4o",
+    "gpt-4": "gpt-4",
+    "gpt-4-32k": "gpt-4-32k",
+    "gpt-4-turbo": "gpt-4-turbo",
+    "gpt-4-turbo-2024-04-09": "gpt-4-turbo",
+    "gpt-4-vision-preview": "gpt-4-vision-preview",
+    "gpt-4-1106-preview": "gpt-4-1106-preview",
+    "gpt-4o-2024-05-13": "gpt-4o-2024-05-13",
+  };
+
+  const azureConfig = {
+    apiKey: process.env.AZURE_OPENAI_KEY,
+    endpoint: process.env.AZURE_OPENAI_ENDPOINT,
+    apiVersion:
+      process.env.AZURE_OPENAI_API_VERSION || process.env.OPENAI_API_VERSION,
+  };
+
+  Settings.llm = new OpenAI({
+    model:
+      AZURE_OPENAI_MODEL_MAP[process.env.MODEL ?? "gpt-35-turbo"] ??
+      "gpt-3.5-turbo",
+    maxTokens: process.env.LLM_MAX_TOKENS
+      ? Number(process.env.LLM_MAX_TOKENS)
+      : undefined,
+    azure: {
+      ...azureConfig,
+      deployment: process.env.AZURE_OPENAI_LLM_DEPLOYMENT,
+    },
+  });
+
+  Settings.embedModel = new OpenAIEmbedding({
+    model: process.env.EMBEDDING_MODEL,
+    dimensions: process.env.EMBEDDING_DIM
+      ? parseInt(process.env.EMBEDDING_DIM)
+      : undefined,
+    azure: {
+      ...azureConfig,
+      deployment: process.env.AZURE_OPENAI_EMBEDDING_DEPLOYMENT,
+    },
+  });
+}
+
 function initOllama() {
   const config = {
     host: process.env.OLLAMA_BASE_URL ?? "http://127.0.0.1:11434",

templates/types/streaming/express/src/routes/chat.route.ts

@@ -1,5 +1,8 @@
 import express, { Router } from "express";
-import { chatConfig } from "../controllers/chat-config.controller";
+import {
+  chatConfig,
+  chatLlamaCloudConfig,
+} from "../controllers/chat-config.controller";
 import { chatRequest } from "../controllers/chat-request.controller";
 import { chatUpload } from "../controllers/chat-upload.controller";
 import { chat } from "../controllers/chat.controller";
@@ -11,6 +14,7 @@ initSettings();
 llmRouter.route("/").post(chat);
 llmRouter.route("/request").post(chatRequest);
 llmRouter.route("/config").get(chatConfig);
+llmRouter.route("/config/llamacloud").get(chatLlamaCloudConfig);
 llmRouter.route("/upload").post(chatUpload);
 
 export default llmRouter;

templates/types/streaming/express/tsconfig.json

@@ -5,7 +5,7 @@
     "forceConsistentCasingInFileNames": true,
     "strict": true,
     "skipLibCheck": true,
-    "moduleResolution": "node",
+    "moduleResolution": "bundler",
     "paths": {
       "@/*": ["./*"]
     }

templates/types/streaming/fastapi/app/api/routers/chat.py

@@ -1,50 +1,32 @@
 import logging
-import os
 from typing import List
 
 from fastapi import APIRouter, BackgroundTasks, Depends, HTTPException, Request, status
 from llama_index.core.chat_engine.types import BaseChatEngine, NodeWithScore
 from llama_index.core.llms import MessageRole
-from llama_index.core.vector_stores.types import MetadataFilter, MetadataFilters
 
 from app.api.routers.events import EventCallbackHandler
 from app.api.routers.models import (
-    ChatConfig,
     ChatData,
     Message,
     Result,
     SourceNodes,
 )
 from app.api.routers.vercel_response import VercelStreamResponse
-from app.api.services.llama_cloud import LLamaCloudFileService
 from app.engine import get_chat_engine
+from app.engine.query_filter import generate_filters
 
 chat_router = r = APIRouter()
 
 logger = logging.getLogger("uvicorn")
 
 
-def process_response_nodes(
-    nodes: List[NodeWithScore],
-    background_tasks: BackgroundTasks,
-):
-    """
-    Start background tasks on the source nodes if needed.
-    """
-    files_to_download = SourceNodes.get_download_files(nodes)
-    for file in files_to_download:
-        background_tasks.add_task(
-            LLamaCloudFileService.download_llamacloud_pipeline_file, file
-        )
-
-
 # streaming endpoint - delete if not needed
 @r.post("")
 async def chat(
     request: Request,
     data: ChatData,
     background_tasks: BackgroundTasks,
-    chat_engine: BaseChatEngine = Depends(get_chat_engine),
 ):
     try:
         last_message_content = data.get_last_message_content()
@@ -52,8 +34,11 @@ async def chat(
 
         doc_ids = data.get_chat_document_ids()
         filters = generate_filters(doc_ids)
-        logger.info("Creating chat engine with filters", filters.dict())
-        chat_engine = get_chat_engine(filters=filters)
+        params = data.data or {}
+        logger.info(
+            f"Creating chat engine with filters: {str(filters)}",
+        )
+        chat_engine = get_chat_engine(filters=filters, params=params)
 
         event_handler = EventCallbackHandler()
         chat_engine.callback_manager.handlers.append(event_handler)  # type: ignore
@@ -70,38 +55,6 @@ async def chat(
         ) from e
 
 
-def generate_filters(doc_ids):
-    if len(doc_ids) > 0:
-        filters = MetadataFilters(
-            filters=[
-                MetadataFilter(
-                    key="private",
-                    value=["true"],
-                    operator="nin",  # type: ignore
-                ),
-                MetadataFilter(
-                    key="doc_id",
-                    value=doc_ids,
-                    operator="in",  # type: ignore
-                ),
-            ],
-            condition="or",  # type: ignore
-        )
-    else:
-        filters = MetadataFilters(
-            # Use the "NIN" - "not in" operator to include all public documents (don't have the private key set)
-            filters=[
-                MetadataFilter(
-                    key="private",
-                    value=["true"],
-                    operator="nin",  # type: ignore
-                ),
-            ]
-        )
-
-    return filters
-
-
 # non-streaming endpoint - delete if not needed
 @r.post("/request")
 async def chat_request(
@@ -118,10 +71,15 @@ async def chat_request(
     )
 
 
-@r.get("/config")
-async def chat_config() -> ChatConfig:
-    starter_questions = None
-    conversation_starters = os.getenv("CONVERSATION_STARTERS")
-    if conversation_starters and conversation_starters.strip():
-        starter_questions = conversation_starters.strip().split("\n")
-    return ChatConfig(starter_questions=starter_questions)
+def process_response_nodes(
+    nodes: List[NodeWithScore],
+    background_tasks: BackgroundTasks,
+):
+    try:
+        # Start background tasks to download documents from LlamaCloud if needed
+        from app.engine.service import LLamaCloudFileService
+
+        LLamaCloudFileService.download_files_from_nodes(nodes, background_tasks)
+    except ImportError:
+        logger.debug("LlamaCloud is not configured. Skipping post processing of nodes")
+        pass

templates/types/streaming/fastapi/app/api/routers/chat_config.py

@@ -0,0 +1,48 @@
+import logging
+import os
+
+from fastapi import APIRouter
+
+from app.api.routers.models import ChatConfig
+
+
+config_router = r = APIRouter()
+
+logger = logging.getLogger("uvicorn")
+
+
+@r.get("")
+async def chat_config() -> ChatConfig:
+    starter_questions = None
+    conversation_starters = os.getenv("CONVERSATION_STARTERS")
+    if conversation_starters and conversation_starters.strip():
+        starter_questions = conversation_starters.strip().split("\n")
+    return ChatConfig(starter_questions=starter_questions)
+
+
+try:
+    from app.engine.service import LLamaCloudFileService
+
+    logger.info("LlamaCloud is configured. Adding /config/llamacloud route.")
+
+    @r.get("/llamacloud")
+    async def chat_llama_cloud_config():
+        projects = LLamaCloudFileService.get_all_projects_with_pipelines()
+        pipeline = os.getenv("LLAMA_CLOUD_INDEX_NAME")
+        project = os.getenv("LLAMA_CLOUD_PROJECT_NAME")
+        pipeline_config = None
+        if pipeline and project:
+            pipeline_config = {
+                "pipeline": pipeline,
+                "project": project,
+            }
+        return {
+            "projects": projects,
+            "pipeline": pipeline_config,
+        }
+
+except ImportError:
+    logger.debug(
+        "LlamaCloud is not configured. Skipping adding /config/llamacloud route."
+    )
+    pass

templates/types/streaming/fastapi/app/api/routers/models.py

@@ -1,6 +1,6 @@
 import logging
 import os
-from typing import Any, Dict, List, Literal, Optional, Set
+from typing import Any, Dict, List, Literal, Optional
 
 from llama_index.core.llms import ChatMessage, MessageRole
 from llama_index.core.schema import NodeWithScore
@@ -75,6 +75,7 @@ class Message(BaseModel):
 
 class ChatData(BaseModel):
     messages: List[Message]
+    data: Any = None
 
     class Config:
         json_schema_extra = {
@@ -146,21 +147,6 @@ class ChatData(BaseModel):
         return list(set(document_ids))
 
 
-class LlamaCloudFile(BaseModel):
-    file_name: str
-    pipeline_id: str
-
-    def __eq__(self, other):
-        if not isinstance(other, LlamaCloudFile):
-            return NotImplemented
-        return (
-            self.file_name == other.file_name and self.pipeline_id == other.pipeline_id
-        )
-
-    def __hash__(self):
-        return hash((self.file_name, self.pipeline_id))
-
-
 class SourceNodes(BaseModel):
     id: str
     metadata: Dict[str, Any]
@@ -192,8 +178,8 @@ class SourceNodes(BaseModel):
         if file_name and url_prefix:
             # file_name exists and file server is configured
             pipeline_id = metadata.get("pipeline_id")
-            if pipeline_id and metadata.get("private") is None:
-                # file is from LlamaCloud and was not ingested locally
+            if pipeline_id:
+                # file is from LlamaCloud
                 file_name = f"{pipeline_id}${file_name}"
                 return f"{url_prefix}/output/llamacloud/{file_name}"
             is_private = metadata.get("private", "false") == "true"
@@ -208,25 +194,6 @@ class SourceNodes(BaseModel):
     def from_source_nodes(cls, source_nodes: List[NodeWithScore]):
         return [cls.from_source_node(node) for node in source_nodes]
 
-    @staticmethod
-    def get_download_files(nodes: List[NodeWithScore]) -> Set[LlamaCloudFile]:
-        source_nodes = SourceNodes.from_source_nodes(nodes)
-        llama_cloud_files = [
-            LlamaCloudFile(
-                file_name=node.metadata.get("file_name"),
-                pipeline_id=node.metadata.get("pipeline_id"),
-            )
-            for node in source_nodes
-            if (
-                node.metadata.get("private")
-                is None  # Only download files are from LlamaCloud and were not ingested locally
-                and node.metadata.get("pipeline_id") is not None
-                and node.metadata.get("file_name") is not None
-            )
-        ]
-        # Remove duplicates and return
-        return set(llama_cloud_files)
-
 
 class Result(BaseModel):
     result: Message
@@ -237,6 +204,7 @@ class ChatConfig(BaseModel):
     starter_questions: Optional[List[str]] = Field(
         default=None,
         description="List of starter questions",
+        serialization_alias="starterQuestions",
     )
 
     class Config:
@@ -248,4 +216,3 @@ class ChatConfig(BaseModel):
                 ]
             }
         }
-        alias_generator = to_camel

templates/types/streaming/fastapi/app/api/routers/upload.py

@@ -1,5 +1,5 @@
 import logging
-from typing import List
+from typing import List, Any
 
 from fastapi import APIRouter, HTTPException
 from pydantic import BaseModel
@@ -13,13 +13,17 @@ logger = logging.getLogger("uvicorn")
 
 class FileUploadRequest(BaseModel):
     base64: str
+    filename: str
+    params: Any = None
 
 
 @r.post("")
 def upload_file(request: FileUploadRequest) -> List[str]:
     try:
         logger.info("Processing file")
-        return PrivateFileService.process_file(request.base64)
+        return PrivateFileService.process_file(
+            request.filename, request.base64, request.params
+        )
     except Exception as e:
         logger.error(f"Error processing file: {e}", exc_info=True)
         raise HTTPException(status_code=500, detail="Error processing file")

templates/types/streaming/fastapi/app/api/routers/vercel_response.py

@@ -80,7 +80,7 @@ class VercelStreamResponse(StreamingResponse):
                     "type": "sources",
                     "data": {
                         "nodes": [
-                            SourceNodes.from_source_node(node).dict()
+                            SourceNodes.from_source_node(node).model_dump()
                             for node in response.source_nodes
                         ]
                     },

templates/types/streaming/fastapi/app/api/services/file.py

@@ -1,9 +1,10 @@
 import base64
 import mimetypes
 import os
+from io import BytesIO
 from pathlib import Path
-from typing import Dict, List
-from uuid import uuid4
+from typing import Any, List, Tuple
+
 
 from app.engine.index import get_index
 from llama_index.core import VectorStoreIndex
@@ -11,9 +12,6 @@ from llama_index.core.ingestion import IngestionPipeline
 from llama_index.core.readers.file.base import (
     _try_loading_included_file_formats as get_file_loaders_map,
 )
-from llama_index.core.readers.file.base import (
-    default_file_metadata_func,
-)
 from llama_index.core.schema import Document
 from llama_index.indices.managed.llama_cloud.base import LlamaCloudIndex
 from llama_index.readers.file import FlatReader
@@ -41,7 +39,7 @@ class PrivateFileService:
     PRIVATE_STORE_PATH = "output/uploaded"
 
     @staticmethod
-    def preprocess_base64_file(base64_content: str) -> tuple:
+    def preprocess_base64_file(base64_content: str) -> Tuple[bytes, str | None]:
         header, data = base64_content.split(",", 1)
         mime_type = header.split(";")[0].split(":", 1)[1]
         extension = mimetypes.guess_extension(mime_type)
@@ -49,12 +47,9 @@ class PrivateFileService:
         return base64.b64decode(data), extension
 
     @staticmethod
-    def store_and_parse_file(file_data, extension) -> List[Document]:
+    def store_and_parse_file(file_name, file_data, extension) -> List[Document]:
         # Store file to the private directory
         os.makedirs(PrivateFileService.PRIVATE_STORE_PATH, exist_ok=True)
-
-        # random file name
-        file_name = f"{uuid4().hex}{extension}"
         file_path = Path(os.path.join(PrivateFileService.PRIVATE_STORE_PATH, file_name))
 
         # write file
@@ -78,25 +73,36 @@ class PrivateFileService:
         return documents
 
     @staticmethod
-    def process_file(base64_content: str) -> List[str]:
+    def process_file(file_name: str, base64_content: str, params: Any) -> List[str]:
         file_data, extension = PrivateFileService.preprocess_base64_file(base64_content)
-        documents = PrivateFileService.store_and_parse_file(file_data, extension)
-
-        # Only process nodes, no store the index
-        pipeline = IngestionPipeline()
-        nodes = pipeline.run(documents=documents)
 
         # Add the nodes to the index and persist it
-        current_index = get_index()
+        current_index = get_index(params)
 
         # Insert the documents into the index
         if isinstance(current_index, LlamaCloudIndex):
-            # LlamaCloudIndex is a managed index so we don't need to process the nodes
-            # just insert the documents
-            for doc in documents:
-                current_index.insert(doc)
+            from app.engine.service import LLamaCloudFileService
+
+            project_id = current_index._get_project_id()
+            pipeline_id = current_index._get_pipeline_id()
+            # LlamaCloudIndex is a managed index so we can directly use the files
+            upload_file = (file_name, BytesIO(file_data))
+            return [
+                LLamaCloudFileService.add_file_to_pipeline(
+                    project_id,
+                    pipeline_id,
+                    upload_file,
+                    custom_metadata={
+                        # Set private=true to mark the document as private user docs (required for filtering)
+                        "private": "true",
+                    },
+                )
+            ]
         else:
-            # Only process nodes, no store the index
+            # First process documents into nodes
+            documents = PrivateFileService.store_and_parse_file(
+                file_name, file_data, extension
+            )
             pipeline = IngestionPipeline()
             nodes = pipeline.run(documents=documents)
 
@@ -109,5 +115,5 @@ class PrivateFileService:
                 persist_dir=os.environ.get("STORAGE_DIR", "storage")
             )
 
-        # Return the document ids
-        return [doc.doc_id for doc in documents]
+            # Return the document ids
+            return [doc.doc_id for doc in documents]

templates/types/streaming/fastapi/app/api/services/llama_cloud.py

@@ -1,88 +0,0 @@
-import logging
-import os
-from typing import Any, Dict, List, Optional
-
-import requests
-from app.api.routers.models import LlamaCloudFile
-
-logger = logging.getLogger("uvicorn")
-
-
-class LLamaCloudFileService:
-    LLAMA_CLOUD_URL = "https://cloud.llamaindex.ai/api/v1"
-    LOCAL_STORE_PATH = "output/llamacloud"
-
-    DOWNLOAD_FILE_NAME_TPL = "{pipeline_id}${filename}"
-
-    @classmethod
-    def _get_files(cls, pipeline_id: str) -> List[Dict[str, Any]]:
-        url = f"{cls.LLAMA_CLOUD_URL}/pipelines/{pipeline_id}/files"
-        return cls._make_request(url)
-
-    @classmethod
-    def _get_file_detail(cls, project_id: str, file_id: str) -> Dict[str, Any]:
-        url = f"{cls.LLAMA_CLOUD_URL}/files/{file_id}/content?project_id={project_id}"
-        return cls._make_request(url)
-
-    @classmethod
-    def _download_file(cls, url: str, local_file_path: str):
-        logger.info(f"Downloading file to {local_file_path}")
-        # Create directory if it doesn't exist
-        os.makedirs(cls.LOCAL_STORE_PATH, exist_ok=True)
-        # Download the file
-        with requests.get(url, stream=True) as r:
-            r.raise_for_status()
-            with open(local_file_path, "wb") as f:
-                for chunk in r.iter_content(chunk_size=8192):
-                    f.write(chunk)
-        logger.info("File downloaded successfully")
-
-    @classmethod
-    def download_llamacloud_pipeline_file(
-        cls,
-        file: LlamaCloudFile,
-        force_download: bool = False,
-    ):
-        file_name = file.file_name
-        pipeline_id = file.pipeline_id
-
-        # Check is the file already exists
-        downloaded_file_path = cls.get_file_path(file_name, pipeline_id)
-        if os.path.exists(downloaded_file_path) and not force_download:
-            logger.debug(f"File {file_name} already exists in local storage")
-            return
-        try:
-            logger.info(f"Downloading file {file_name} for pipeline {pipeline_id}")
-            files = cls._get_files(pipeline_id)
-            if not files or not isinstance(files, list):
-                raise Exception("No files found in LlamaCloud")
-            for file_entry in files:
-                if file_entry["name"] == file_name:
-                    file_id = file_entry["file_id"]
-                    project_id = file_entry["project_id"]
-                    file_detail = cls._get_file_detail(project_id, file_id)
-                    cls._download_file(file_detail["url"], downloaded_file_path)
-                    break
-        except Exception as error:
-            logger.info(f"Error fetching file from LlamaCloud: {error}")
-
-    @classmethod
-    def get_file_name(cls, name: str, pipeline_id: str) -> str:
-        return cls.DOWNLOAD_FILE_NAME_TPL.format(pipeline_id=pipeline_id, filename=name)
-
-    @classmethod
-    def get_file_path(cls, name: str, pipeline_id: str) -> str:
-        return os.path.join(cls.LOCAL_STORE_PATH, cls.get_file_name(name, pipeline_id))
-
-    @staticmethod
-    def _make_request(
-        url: str, data=None, headers: Optional[Dict] = None, method: str = "get"
-    ):
-        if headers is None:
-            headers = {
-                "Accept": "application/json",
-                "Authorization": f'Bearer {os.getenv("LLAMA_CLOUD_API_KEY")}',
-            }
-        response = requests.request(method, url, headers=headers, data=data)
-        response.raise_for_status()
-        return response.json()

templates/types/streaming/fastapi/app/api/services/suggestion.py

@@ -1,3 +1,4 @@
+import logging
 from typing import List
 
 from app.api.routers.models import Message
@@ -9,11 +10,14 @@ NEXT_QUESTIONS_SUGGESTION_PROMPT = PromptTemplate(
     "You're a helpful assistant! Your task is to suggest the next question that user might ask. "
     "\nHere is the conversation history"
     "\n---------------------\n{conversation}\n---------------------"
-    "Given the conversation history, please give me $number_of_questions questions that you might ask next!"
+    "Given the conversation history, please give me {number_of_questions} questions that you might ask next!"
 )
 N_QUESTION_TO_GENERATE = 3
 
 
+logger = logging.getLogger("uvicorn")
+
+
 class NextQuestions(BaseModel):
     """A list of questions that user might ask next"""
 
@@ -26,23 +30,31 @@ class NextQuestionSuggestion:
         messages: List[Message],
         number_of_questions: int = N_QUESTION_TO_GENERATE,
     ) -> List[str]:
-        # Reduce the cost by only using the last two messages
-        last_user_message = None
-        last_assistant_message = None
-        for message in reversed(messages):
-            if message.role == "user":
-                last_user_message = f"User: {message.content}"
-            elif message.role == "assistant":
-                last_assistant_message = f"Assistant: {message.content}"
-            if last_user_message and last_assistant_message:
-                break
-        conversation: str = f"{last_user_message}\n{last_assistant_message}"
+        """
+        Suggest the next questions that user might ask based on the conversation history
+        Return as empty list if there is an error
+        """
+        try:
+            # Reduce the cost by only using the last two messages
+            last_user_message = None
+            last_assistant_message = None
+            for message in reversed(messages):
+                if message.role == "user":
+                    last_user_message = f"User: {message.content}"
+                elif message.role == "assistant":
+                    last_assistant_message = f"Assistant: {message.content}"
+                if last_user_message and last_assistant_message:
+                    break
+            conversation: str = f"{last_user_message}\n{last_assistant_message}"
 
-        output: NextQuestions = await Settings.llm.astructured_predict(
-            NextQuestions,
-            prompt=NEXT_QUESTIONS_SUGGESTION_PROMPT,
-            conversation=conversation,
-            nun_questions=number_of_questions,
-        )
+            output: NextQuestions = await Settings.llm.astructured_predict(
+                NextQuestions,
+                prompt=NEXT_QUESTIONS_SUGGESTION_PROMPT,
+                conversation=conversation,
+                number_of_questions=number_of_questions,
+            )
 
-        return output.questions
+            return output.questions
+        except Exception as e:
+            logger.error(f"Error when generating next question: {e}")
+            return []

templates/types/streaming/fastapi/app/engine/generate.py

@@ -1,3 +1,4 @@
+# flake8: noqa: E402
 from dotenv import load_dotenv
 
 load_dotenv()
@@ -21,7 +22,6 @@ STORAGE_DIR = os.getenv("STORAGE_DIR", "storage")
 
 
 def get_doc_store():
-
     # If the storage directory is there, load the document store from it.
     # If not, set up an in-memory document store since we can't load from a directory that doesn't exist.
     if os.path.exists(STORAGE_DIR):
@@ -64,6 +64,9 @@ def generate_datasource():
 
     # Get the stores and documents or create new ones
     documents = get_documents()
+    # Set private=false to mark the document as public (required for filtering)
+    for doc in documents:
+        doc.metadata["private"] = "false"
     docstore = get_doc_store()
     vector_store = get_vector_store()
 

templates/types/streaming/fastapi/app/engine/index.py

@@ -6,7 +6,7 @@ from app.engine.vectordb import get_vector_store
 logger = logging.getLogger("uvicorn")
 
 
-def get_index():
+def get_index(params=None):
     logger.info("Connecting vector store...")
     store = get_vector_store()
     # Load the index from the vector store

templates/types/streaming/fastapi/app/engine/query_filter.py

@@ -0,0 +1,34 @@
+from llama_index.core.vector_stores.types import MetadataFilter, MetadataFilters
+
+
+def generate_filters(doc_ids):
+    """
+    Generate public/private document filters based on the doc_ids and the vector store.
+    """
+    public_doc_filter = MetadataFilter(
+        key="private",
+        value="true",
+        operator="!=",  # type: ignore
+    )
+    selected_doc_filter = MetadataFilter(
+        key="doc_id",
+        value=doc_ids,
+        operator="in",  # type: ignore
+    )
+    if len(doc_ids) > 0:
+        # If doc_ids are provided, we will select both public and selected documents
+        filters = MetadataFilters(
+            filters=[
+                public_doc_filter,
+                selected_doc_filter,
+            ],
+            condition="or",  # type: ignore
+        )
+    else:
+        filters = MetadataFilters(
+            filters=[
+                public_doc_filter,
+            ]
+        )
+
+    return filters

templates/types/streaming/fastapi/app/llmhub.py

@@ -1,61 +0,0 @@
-from llama_index.embeddings.openai import OpenAIEmbedding
-from llama_index.core.settings import Settings
-from typing import Dict
-import os
-
-DEFAULT_MODEL = "gpt-3.5-turbo"
-DEFAULT_EMBEDDING_MODEL = "text-embedding-3-large"
-
-class TSIEmbedding(OpenAIEmbedding):
-    def __init__(self, **kwargs):
-        super().__init__(**kwargs)
-        self._query_engine = self._text_engine = self.model_name
-
-def llm_config_from_env() -> Dict:
-    from llama_index.core.constants import DEFAULT_TEMPERATURE
-
-    model = os.getenv("MODEL", DEFAULT_MODEL)
-    temperature = os.getenv("LLM_TEMPERATURE", DEFAULT_TEMPERATURE)
-    max_tokens = os.getenv("LLM_MAX_TOKENS")
-    api_key = os.getenv("T_SYSTEMS_LLMHUB_API_KEY")
-    api_base = os.getenv("T_SYSTEMS_LLMHUB_BASE_URL")
-
-    config = {
-        "model": model,
-        "api_key": api_key,
-        "api_base": api_base,
-        "temperature": float(temperature),
-        "max_tokens": int(max_tokens) if max_tokens is not None else None,
-    }
-    return config
-
-
-def embedding_config_from_env() -> Dict:
-    from llama_index.core.constants import DEFAULT_EMBEDDING_DIM
-    
-    model = os.getenv("EMBEDDING_MODEL", DEFAULT_EMBEDDING_MODEL)
-    dimension = os.getenv("EMBEDDING_DIM", DEFAULT_EMBEDDING_DIM)
-    api_key = os.getenv("T_SYSTEMS_LLMHUB_API_KEY")
-    api_base = os.getenv("T_SYSTEMS_LLMHUB_BASE_URL")
-
-    config = {
-        "model_name": model,
-        "dimension": int(dimension) if dimension is not None else None,
-        "api_key": api_key,
-        "api_base": api_base,
-    }
-    return config
-
-def init_llmhub():
-    from llama_index.llms.openai_like import OpenAILike
-
-    llm_configs = llm_config_from_env()
-    embedding_configs = embedding_config_from_env()
-
-    Settings.embed_model = TSIEmbedding(**embedding_configs)
-    Settings.llm = OpenAILike(
-        **llm_configs,
-        is_chat_model=True,
-        is_function_calling_model=False,
-        context_window=4096,
-    )

templates/types/streaming/fastapi/main.py

@@ -1,20 +1,22 @@
+# flake8: noqa: E402
 from dotenv import load_dotenv
 
 load_dotenv()
 
 import logging
 import os
+
 import uvicorn
+from app.api.routers.chat import chat_router
+from app.api.routers.chat_config import config_router
+from app.api.routers.upload import file_upload_router
+from app.observability import init_observability
+from app.settings import init_settings
 from fastapi import FastAPI
 from fastapi.middleware.cors import CORSMiddleware
 from fastapi.responses import RedirectResponse
-from app.api.routers.chat import chat_router
-from app.api.routers.upload import file_upload_router
-from app.settings import init_settings
-from app.observability import init_observability
 from fastapi.staticfiles import StaticFiles
 
-
 app = FastAPI()
 
 init_settings()
@@ -54,6 +56,7 @@ mount_static_files("data", "/api/files/data")
 mount_static_files("output", "/api/files/output")
 
 app.include_router(chat_router, prefix="/api/chat")
+app.include_router(config_router, prefix="/api/chat/config")
 app.include_router(file_upload_router, prefix="/api/chat/upload")
 
 if __name__ == "__main__":

templates/types/streaming/fastapi/pyproject.toml

@@ -9,12 +9,12 @@ readme = "README.md"
 generate = "app.engine.generate:generate_datasource"
 
 [tool.poetry.dependencies]
-python = "^3.11,<3.12"
+python = "^3.11,<4.0"
 fastapi = "^0.109.1"
 uvicorn = { extras = ["standard"], version = "^0.23.2" }
 python-dotenv = "^1.0.0"
 aiostream = "^0.5.2"
-llama-index = "0.10.55"
+llama-index = "0.10.58"
 cachetools = "^5.3.3"
 
 [build-system]

templates/types/streaming/nextjs/app/api/chat/config/llamacloud/route.ts

@@ -0,0 +1,16 @@
+import { NextResponse } from "next/server";
+import { LLamaCloudFileService } from "../../llamaindex/streaming/service";
+
+/**
+ * This API is to get config from the backend envs and expose them to the frontend
+ */
+export async function GET() {
+  const config = {
+    projects: await LLamaCloudFileService.getAllProjectsWithPipelines(),
+    pipeline: {
+      pipeline: process.env.LLAMA_CLOUD_INDEX_NAME,
+      project: process.env.LLAMA_CLOUD_PROJECT_NAME,
+    },
+  };
+  return NextResponse.json(config, { status: 200 });
+}

templates/types/streaming/nextjs/app/api/chat/engine/queryFilter.ts

@@ -0,0 +1,25 @@
+import { MetadataFilter, MetadataFilters } from "llamaindex";
+
+export function generateFilters(documentIds: string[]): MetadataFilters {
+  // filter all documents have the private metadata key set to true
+  const publicDocumentsFilter: MetadataFilter = {
+    key: "private",
+    value: "true",
+    operator: "!=",
+  };
+
+  // if no documentIds are provided, only retrieve information from public documents
+  if (!documentIds.length) return { filters: [publicDocumentsFilter] };
+
+  const privateDocumentsFilter: MetadataFilter = {
+    key: "doc_id",
+    value: documentIds,
+    operator: "in",
+  };
+
+  // if documentIds are provided, retrieve information from public and private documents
+  return {
+    filters: [publicDocumentsFilter, privateDocumentsFilter],
+    condition: "or",
+  };
+}