refactor: import from llamaindex edge for typescript templates and vectordbs

create-app.ts

@@ -41,7 +41,7 @@ export async function createApp({
   vectorDb,
   externalPort,
   postInstallAction,
-  dataSources,
+  dataSource,
   tools,
   observability,
 }: InstallAppArgs): Promise<void> {
@@ -89,7 +89,7 @@ export async function createApp({
     vectorDb,
     externalPort,
     postInstallAction,
-    dataSources,
+    dataSource,
     tools,
     observability,
   };

helpers/env-variables.ts

@@ -5,7 +5,6 @@ import {
   TemplateDataSource,
   TemplateFramework,
   TemplateVectorDB,
-  WebSourceConfig,
 } from "./types";
 
 type EnvVar = {
@@ -100,32 +99,26 @@ const getVectorDBEnvs = (vectorDb: TemplateVectorDB) => {
   }
 };
 
-const getDataSourceEnvs = (dataSources: TemplateDataSource[]) => {
-  const envs = [];
-  for (const source of dataSources) {
-    switch (source.type) {
-      case "web":
-        const config = source.config as WebSourceConfig;
-        envs.push(
-          {
-            name: "BASE_URL",
-            description: "The base URL to start web scraping.",
-            value: config.baseUrl,
-          },
-          {
-            name: "URL_PREFIX",
-            description: "The prefix of the URL to start web scraping.",
-            value: config.baseUrl,
-          },
-          {
-            name: "MAX_DEPTH",
-            description: "The maximum depth to scrape.",
-            value: config.depth?.toString(),
-          },
-        );
-    }
+const getDataSourceEnvs = (dataSource: TemplateDataSource) => {
+  switch (dataSource.type) {
+    case "web":
+      return [
+        {
+          name: "BASE_URL",
+          description: "The base URL to start web scraping.",
+        },
+        {
+          name: "URL_PREFIX",
+          description: "The prefix of the URL to start web scraping.",
+        },
+        {
+          name: "MAX_DEPTH",
+          description: "The maximum depth to scrape.",
+        },
+      ];
+    default:
+      return [];
   }
-  return envs;
 };
 
 export const createBackendEnvFile = async (
@@ -137,7 +130,7 @@ export const createBackendEnvFile = async (
     model?: string;
     embeddingModel?: string;
     framework?: TemplateFramework;
-    dataSources?: TemplateDataSource[];
+    dataSource?: TemplateDataSource;
     port?: number;
   },
 ) => {
@@ -159,7 +152,7 @@ export const createBackendEnvFile = async (
     // Add vector database environment variables
     ...(opts.vectorDb ? getVectorDBEnvs(opts.vectorDb) : []),
     // Add data source environment variables
-    ...(opts.dataSources ? getDataSourceEnvs(opts.dataSources) : []),
+    ...(opts.dataSource ? getDataSourceEnvs(opts.dataSource) : []),
   ];
   let envVars: EnvVar[] = [];
   if (opts.framework === "fastapi") {
@@ -211,9 +204,7 @@ We have provided context information below.
 Given this information, please answer the question: {query_str}
 "`,
         },
-        opts?.dataSources?.some(
-          (ds) => (ds.config as FileSourceConfig).useLlamaParse,
-        )
+        (opts?.dataSource?.config as FileSourceConfig).useLlamaParse
           ? {
               name: "LLAMA_CLOUD_API_KEY",
               description: `The Llama Cloud API key.`,

helpers/index.ts

@@ -27,8 +27,8 @@ async function generateContextData(
   packageManager?: PackageManager,
   openAiKey?: string,
   vectorDb?: TemplateVectorDB,
+  dataSource?: TemplateDataSource,
   llamaCloudKey?: string,
-  useLlamaParse?: boolean,
 ) {
   if (packageManager) {
     const runGenerate = `${cyan(
@@ -37,7 +37,8 @@ async function generateContextData(
         : `${packageManager} run generate`,
     )}`;
     const openAiKeyConfigured = openAiKey || process.env["OPENAI_API_KEY"];
-    const llamaCloudKeyConfigured = useLlamaParse
+    const llamaCloudKeyConfigured = (dataSource?.config as FileSourceConfig)
+      ?.useLlamaParse
       ? llamaCloudKey || process.env["LLAMA_CLOUD_API_KEY"]
       : true;
     const hasVectorDb = vectorDb && vectorDb !== "none";
@@ -81,19 +82,18 @@ const copyContextData = async (
   dataSource?: TemplateDataSource,
 ) => {
   const destPath = path.join(root, "data");
+
   const dataSourceConfig = dataSource?.config as FileSourceConfig;
 
   // Copy file
   if (dataSource?.type === "file") {
-    if (dataSourceConfig.paths) {
+    if (dataSourceConfig.path) {
+      console.log(`\nCopying file to ${cyan(destPath)}\n`);
       await fs.mkdir(destPath, { recursive: true });
-      console.log(
-        "Copying data from files:",
-        dataSourceConfig.paths.toString(),
+      await fs.copyFile(
+        dataSourceConfig.path,
+        path.join(destPath, path.basename(dataSourceConfig.path)),
       );
-      for (const p of dataSourceConfig.paths) {
-        await fs.copyFile(p, path.join(destPath, path.basename(p)));
-      }
     } else {
       console.log("Missing file path in config");
       process.exit(1);
@@ -103,20 +103,13 @@ const copyContextData = async (
 
   // Copy folder
   if (dataSource?.type === "folder") {
-    // Example data does not have path config, set the default path
-    const srcPaths = dataSourceConfig.paths ?? [
-      path.join(templatesDir, "components", "data"),
-    ];
-    console.log("Copying data from folders: ", srcPaths);
-    for (const p of srcPaths) {
-      const folderName = path.basename(p);
-      const destFolderPath = path.join(destPath, folderName);
-      await fs.mkdir(destFolderPath, { recursive: true });
-      await copy("**", destFolderPath, {
-        parents: true,
-        cwd: p,
-      });
-    }
+    const srcPath =
+      dataSourceConfig.path ?? path.join(templatesDir, "components", "data");
+    console.log(`\nCopying data to ${cyan(destPath)}\n`);
+    await copy("**", destPath, {
+      parents: true,
+      cwd: srcPath,
+    });
     return;
   }
 };
@@ -167,17 +160,12 @@ export const installTemplate = async (
       model: props.model,
       embeddingModel: props.embeddingModel,
       framework: props.framework,
-      dataSources: props.dataSources,
+      dataSource: props.dataSource,
       port: props.externalPort,
     });
 
     if (props.engine === "context") {
-      console.log("\nGenerating context data...\n");
-      props.dataSources.forEach(async (ds) => {
-        if (ds.type === "file" || ds.type === "folder") {
-          await copyContextData(props.root, ds);
-        }
-      });
+      await copyContextData(props.root, props.dataSource);
       if (
         props.postInstallAction === "runApp" ||
         props.postInstallAction === "dependencies"
@@ -187,12 +175,8 @@ export const installTemplate = async (
           props.packageManager,
           props.openAiKey,
           props.vectorDb,
+          props.dataSource,
           props.llamaCloudKey,
-          props.dataSources.some(
-            (ds) =>
-              (ds.type === "file" || ds.type === "folder") &&
-              (ds.config as FileSourceConfig).useLlamaParse,
-          ),
         );
       }
     }

helpers/python.ts

@@ -175,7 +175,7 @@ export const installPythonTemplate = async ({
   framework,
   engine,
   vectorDb,
-  dataSources,
+  dataSource,
   tools,
   postInstallAction,
 }: Pick<
@@ -185,7 +185,7 @@ export const installPythonTemplate = async ({
   | "template"
   | "engine"
   | "vectorDb"
-  | "dataSources"
+  | "dataSource"
   | "tools"
   | "postInstallAction"
 >) => {
@@ -250,54 +250,27 @@ export const installPythonTemplate = async ({
       });
     }
 
-    if (dataSources.length > 0 || dataSources[0].type !== "none") {
-      // Copy loader.py file to enginePath
-      await copy("loader.py", enginePath, {
-        parents: true,
-        cwd: path.join(compPath, "loaders", "python"),
-      });
-
-      // Copy data source loaders
-      const loaderPath = path.join(enginePath, "loaders");
-      for (const source of dataSources) {
-        const sourceType = source.type;
-        if (sourceType === "file" || sourceType === "folder") {
-          const sourceConfig = source.config as FileSourceConfig;
-          const loaderFolder = sourceConfig.useLlamaParse
-            ? "llama_parse"
-            : "file";
-          await copy("**", loaderPath, {
-            parents: true,
-            cwd: path.join(compPath, "loaders", "python", loaderFolder),
-          });
-        } else {
-          await copy("**", loaderPath, {
-            parents: true,
-            cwd: path.join(compPath, "loaders", "python", sourceType),
-          });
-        }
+    const dataSourceType = dataSource?.type;
+    if (dataSourceType !== undefined && dataSourceType !== "none") {
+      let loaderFolder: string;
+      if (dataSourceType === "file" || dataSourceType === "folder") {
+        const dataSourceConfig = dataSource?.config as FileSourceConfig;
+        loaderFolder = dataSourceConfig.useLlamaParse ? "llama_parse" : "file";
+      } else {
+        loaderFolder = dataSourceType;
       }
+      await copy("**", enginePath, {
+        parents: true,
+        cwd: path.join(compPath, "loaders", "python", loaderFolder),
+      });
     }
-
-    // const dataSourceType = dataSource?.type;
-    // if (dataSourceType !== undefined && dataSourceType !== "none") {
-    //   let loaderFolder: string;
-    //   if (dataSourceType === "file" || dataSourceType === "folder") {
-    //     const dataSourceConfig = dataSource?.config as FileSourceConfig;
-    //     loaderFolder = dataSourceConfig.useLlamaParse ? "llama_parse" : "file";
-    //   } else {
-    //     loaderFolder = dataSourceType;
-    //   }
-    //   await copy("**", enginePath, {
-    //     parents: true,
-    //     cwd: path.join(compPath, "loaders", "python", loaderFolder),
-    //   });
-    // }
   }
 
-  const addOnDependencies = dataSources
-    .map((ds) => getAdditionalDependencies(vectorDb, ds, tools))
-    .flat();
+  const addOnDependencies = getAdditionalDependencies(
+    vectorDb,
+    dataSource,
+    tools,
+  );
   await addDependencies(root, addOnDependencies);
 
   if (postInstallAction === "runApp" || postInstallAction === "dependencies") {

helpers/types.ts

@@ -19,7 +19,7 @@ export type TemplateDataSourceType = "none" | "file" | "folder" | "web";
 export type TemplateObservability = "none" | "opentelemetry";
 // Config for both file and folder
 export type FileSourceConfig = {
-  paths?: string[];
+  path?: string;
   useLlamaParse?: boolean;
 };
 export type WebSourceConfig = {
@@ -44,7 +44,7 @@ export interface InstallTemplateArgs {
   framework: TemplateFramework;
   engine: TemplateEngine;
   ui: TemplateUI;
-  dataSources: TemplateDataSource[];
+  dataSource?: TemplateDataSource;
   eslint: boolean;
   customApiPath?: string;
   openAiKey?: string;

index.ts

@@ -303,7 +303,7 @@ async function run(): Promise<void> {
     vectorDb: program.vectorDb,
     externalPort: program.externalPort,
     postInstallAction: program.postInstallAction,
-    dataSources: program.dataSources,
+    dataSource: program.dataSource,
     tools: program.tools,
     observability: program.observability,
   });

questions.ts

@@ -9,7 +9,6 @@ import prompts from "prompts";
 import { InstallAppArgs } from "./create-app";
 import {
   FileSourceConfig,
-  TemplateDataSource,
   TemplateDataSourceType,
   TemplateFramework,
 } from "./helpers";
@@ -41,34 +40,31 @@ const MACOS_FILE_SELECTION_SCRIPT = `
 osascript -l JavaScript -e '
   a = Application.currentApplication();
   a.includeStandardAdditions = true;
-  a.chooseFile({ withPrompt: "Please select files to process:", multipleSelectionsAllowed: true }).map(file => file.toString())
+  a.chooseFile({ withPrompt: "Please select a file to process:" }).toString()
 '`;
 const MACOS_FOLDER_SELECTION_SCRIPT = `
 osascript -l JavaScript -e '
   a = Application.currentApplication();
   a.includeStandardAdditions = true;
-  a.chooseFolder({ withPrompt: "Please select folders to process:", multipleSelectionsAllowed: true }).map(folder => folder.toString())
+  a.chooseFolder({ withPrompt: "Please select a folder to process:" }).toString()
 '`;
 const WINDOWS_FILE_SELECTION_SCRIPT = `
 Add-Type -AssemblyName System.Windows.Forms
 $openFileDialog = New-Object System.Windows.Forms.OpenFileDialog
 $openFileDialog.InitialDirectory = [Environment]::GetFolderPath('Desktop')
-$openFileDialog.Multiselect = $true
 $result = $openFileDialog.ShowDialog()
 if ($result -eq 'OK') {
-  $openFileDialog.FileNames
+  $openFileDialog.FileName
 }
 `;
 const WINDOWS_FOLDER_SELECTION_SCRIPT = `
 Add-Type -AssemblyName System.windows.forms
 $folderBrowser = New-Object System.Windows.Forms.FolderBrowserDialog
-$folderBrowser.SelectedPath = [Environment]::GetFolderPath('Desktop')
-$folderBrowser.Description = "Please select folders to process:"
-$folderBrowser.ShowNewFolderButton = $true
-$folderBrowser.RootFolder = [System.Environment+SpecialFolder]::Desktop
-$folderBrowser.SelectedPath = [System.IO.Path]::GetFullPath($folderBrowser.SelectedPath)
-$folderBrowser.ShowDialog() | Out-Null
-$folderBrowser.SelectedPath, $folderBrowser.SelectedPaths
+$dialogResult = $folderBrowser.ShowDialog()
+if ($dialogResult -eq [System.Windows.Forms.DialogResult]::OK)
+{
+    $folderBrowser.SelectedPath
+}
 `;
 
 const defaults: QuestionArgs = {
@@ -85,7 +81,10 @@ const defaults: QuestionArgs = {
   communityProjectConfig: undefined,
   llamapack: "",
   postInstallAction: "dependencies",
-  dataSources: [],
+  dataSource: {
+    type: "none",
+    config: {},
+  },
   tools: [],
 };
 
@@ -123,53 +122,30 @@ const getVectorDbChoices = (framework: TemplateFramework) => {
   return displayedChoices;
 };
 
-export const getDataSourceChoices = (
-  framework: TemplateFramework,
-  selectedDataSource: TemplateDataSource[],
-) => {
-  const choices = [];
-  if (selectedDataSource.length > 0) {
-    choices.push({
-      title: "No",
-      value: "none",
-    });
-  }
-  if (selectedDataSource === undefined || selectedDataSource.length === 0) {
-    choices.push({
+const getDataSourceChoices = (framework: TemplateFramework) => {
+  const choices = [
+    {
       title: "No data, just a simple chat",
-      value: "none",
+      value: "simple",
+    },
+    { title: "Use an example PDF", value: "exampleFile" },
+  ];
+  if (process.platform === "win32" || process.platform === "darwin") {
+    choices.push({
+      title: `Use a local file (${supportedContextFileTypes.join(", ")})`,
+      value: "localFile",
     });
     choices.push({
-      title: "Use an example PDF",
-      value: "exampleFile",
+      title: `Use a local folder`,
+      value: "localFolder",
     });
   }
-
-  if (!selectedDataSource.some((ds) => ds.type === "file")) {
-    choices.push({
-      title: `Use local files (${supportedContextFileTypes.join(", ")})`,
-      value: "file",
-    });
-  }
-
-  if (!selectedDataSource.some((ds) => ds.type === "folder")) {
-    choices.push({
-      title: "Use local folder",
-      value: "folder",
-    });
-  }
-
-  if (
-    !selectedDataSource.some((ds) => ds.type === "web") &&
-    (process.platform === "win32" || process.platform === "darwin") &&
-    framework === "fastapi"
-  ) {
+  if (framework === "fastapi") {
     choices.push({
       title: "Use website content (requires Chrome)",
       value: "web",
     });
   }
-
   return choices;
 };
 
@@ -197,15 +173,9 @@ const selectLocalContextData = async (type: TemplateDataSourceType) => {
         process.exit(1);
     }
     selectedPath = execSync(execScript, execOpts).toString().trim();
-    const paths =
-      process.platform === "win32"
-        ? selectedPath.split("\r\n")
-        : selectedPath.split(", ");
-    for (const p of paths) {
-      if (
-        type == "file" &&
-        !supportedContextFileTypes.includes(path.extname(p))
-      ) {
+    if (type === "file") {
+      const fileType = path.extname(selectedPath);
+      if (!supportedContextFileTypes.includes(fileType)) {
         console.log(
           red(
             `Please select a supported file type: ${supportedContextFileTypes}`,
@@ -214,7 +184,7 @@ const selectLocalContextData = async (type: TemplateDataSourceType) => {
         process.exit(1);
       }
     }
-    return paths;
+    return selectedPath;
   } catch (error) {
     console.log(
       red(
@@ -339,11 +309,9 @@ export const askQuestions = async (
         const openAiKeyConfigured =
           program.openAiKey || process.env["OPENAI_API_KEY"];
         // If using LlamaParse, require LlamaCloud API key
-        const useLlamaParse = program.dataSources.some(
-          (ds) =>
-            ds.type === "file" && (ds.config as FileSourceConfig).useLlamaParse,
-        );
-        const llamaCloudKeyConfigured = useLlamaParse
+        const llamaCloudKeyConfigured = (
+          program.dataSource?.config as FileSourceConfig
+        )?.useLlamaParse
           ? program.llamaCloudKey || process.env["LLAMA_CLOUD_API_KEY"]
           : true;
         const hasVectorDb = program.vectorDb && program.vectorDb !== "none";
@@ -646,149 +614,124 @@ export const askQuestions = async (
       console.log("File or folder not found");
       process.exit(1);
     } else {
-      program.dataSources = [
-        {
-          type: fs.lstatSync(program.files).isDirectory() ? "folder" : "file",
-          config: {
-            paths: program.files.split(","),
-          },
+      program.dataSource = {
+        type: fs.lstatSync(program.files).isDirectory() ? "folder" : "file",
+        config: {
+          path: program.files,
         },
-      ];
+      };
     }
   }
 
-  // Asking for data source
   if (!program.engine) {
-    program.dataSources = getPrefOrDefault("dataSources");
     if (ciInfo.isCI) {
       program.engine = getPrefOrDefault("engine");
     } else {
-      for (let i = 0; i < 2; i++) {
-        const { selectedSource } = await prompts(
-          {
-            type: "select",
-            name: "selectedSource",
-            message:
-              i === 0
-                ? "Which data source would you like to use?"
-                : "Would you like to add another data source?",
-            choices: getDataSourceChoices(
-              program.framework,
-              program.dataSources,
-            ),
-            initial: 0,
-          },
-          handlers,
-        );
-
-        // Asking for data source config
-        // Select None data source, No need to config and asking for another data source
-        if (selectedSource === "none") {
-          if (selectedSource.length === 0) {
-            program.dataSources = [
-              {
-                type: "none",
-                config: {},
-              },
-            ];
-          }
-          break;
-        }
-
-        const dataSource = {
-          type: selectedSource === "exampleFile" ? "folder" : selectedSource,
-          config: {},
-        };
-
-        // Select local file or folder
-        if (selectedSource === "file" || selectedSource === "folder") {
-          const selectedPaths = await selectLocalContextData(selectedSource);
-          dataSource.config = {
-            paths: selectedPaths,
-          };
-        }
-
-        // Selected web data source
-        else if (selectedSource === "web") {
-          let { baseUrl } = await prompts(
-            {
-              type: "text",
-              name: "baseUrl",
-              message: "Please provide base URL of the website:",
-              initial: "https://www.llamaindex.ai",
-            },
-            handlers,
-          );
-          try {
-            if (!baseUrl.includes("://")) {
-              baseUrl = `https://${baseUrl}`;
-            }
-            const checkUrl = new URL(baseUrl);
-            if (
-              checkUrl.protocol !== "https:" &&
-              checkUrl.protocol !== "http:"
-            ) {
-              throw new Error("Invalid protocol");
-            }
-          } catch (error) {
-            console.log(
-              red(
-                "Invalid URL provided! Please provide a valid URL (e.g. https://www.llamaindex.ai)",
-              ),
-            );
-            process.exit(1);
-          }
-
-          dataSource.config = {
-            baseUrl: baseUrl,
-            depth: 1,
-          };
-        }
-        program.dataSources.push(dataSource);
-
-        // No need to ask for another data source if user selected example data
-        if (selectedSource === "exampleFile") {
-          break;
-        }
-      }
-
-      if (
-        program.dataSources.length === 0 ||
-        program.dataSources[0].type === "none"
-      ) {
-        program.engine = "simple";
-      } else {
-        program.engine = "context";
-      }
-    }
-  }
-
-  // Asking for LlamaParse
-  // Is user selected pdf file or is there a folder data source
-  if (!program.llamaParse && program.engine === "context") {
-    const askingLlamaParse = program.dataSources.some(
-      (ds) =>
-        ds.type === "folder" ||
-        (ds.type === "file" &&
-          (ds.config as FileSourceConfig).paths?.some(
-            (p) => path.extname(p) === ".pdf",
-          )),
-    );
-    if (askingLlamaParse) {
-      const { useLlamaParse } = await prompts(
+      const { dataSource } = await prompts(
         {
-          type: "toggle",
-          name: "useLlamaParse",
-          message:
-            "Would you like to use LlamaParse (improved parser for RAG - requires API key)?",
-          initial: true,
-          active: "yes",
-          inactive: "no",
+          type: "select",
+          name: "dataSource",
+          message: "Which data source would you like to use?",
+          choices: getDataSourceChoices(program.framework),
+          initial: 1,
         },
         handlers,
       );
+      // Initialize with default config
+      program.dataSource = getPrefOrDefault("dataSource");
+      if (program.dataSource) {
+        switch (dataSource) {
+          case "simple":
+            program.engine = "simple";
+            program.dataSource = { type: "none", config: {} };
+            break;
+          case "exampleFile":
+            program.engine = "context";
+            // Treat example as a folder data source with no config
+            program.dataSource = { type: "folder", config: {} };
+            break;
+          case "localFile":
+            program.engine = "context";
+            program.dataSource = {
+              type: "file",
+              config: {
+                path: await selectLocalContextData("file"),
+              },
+            };
+            break;
+          case "localFolder":
+            program.engine = "context";
+            program.dataSource = {
+              type: "folder",
+              config: {
+                path: await selectLocalContextData("folder"),
+              },
+            };
+            break;
+          case "web":
+            program.engine = "context";
+            program.dataSource.type = "web";
+            break;
+        }
+      }
+    }
+  } else if (!program.dataSource) {
+    // Handle a case when engine is specified but dataSource is not
+    if (program.engine === "context") {
+      program.dataSource = {
+        type: "folder",
+        config: {},
+      };
+    } else if (program.engine === "simple") {
+      program.dataSource = {
+        type: "none",
+        config: {},
+      };
+    }
+  }
+
+  if (
+    (program.dataSource?.type === "file" ||
+      program.dataSource?.type === "folder") &&
+    program.framework === "fastapi"
+  ) {
+    if (ciInfo.isCI) {
+      program.llamaCloudKey = getPrefOrDefault("llamaCloudKey");
+    } else {
+      const dataSourceConfig = program.dataSource.config as FileSourceConfig;
+      dataSourceConfig.useLlamaParse = program.llamaParse;
+
+      // Is pdf file selected as data source or is it a folder data source
+      const askingLlamaParse =
+        dataSourceConfig.useLlamaParse === undefined &&
+        (program.dataSource.type === "folder"
+          ? true
+          : dataSourceConfig.path &&
+            path.extname(dataSourceConfig.path) === ".pdf");
+
+      // Ask if user wants to use LlamaParse
+      if (askingLlamaParse) {
+        const { useLlamaParse } = await prompts(
+          {
+            type: "toggle",
+            name: "useLlamaParse",
+            message:
+              "Would you like to use LlamaParse (improved parser for RAG - requires API key)?",
+            initial: true,
+            active: "yes",
+            inactive: "no",
+          },
+          handlers,
+        );
+        dataSourceConfig.useLlamaParse = useLlamaParse;
+        program.dataSource.config = dataSourceConfig;
+      }
+
       // Ask for LlamaCloud API key
-      if (useLlamaParse && program.llamaCloudKey === undefined) {
+      if (
+        dataSourceConfig.useLlamaParse &&
+        program.llamaCloudKey === undefined
+      ) {
         const { llamaCloudKey } = await prompts(
           {
             type: "text",
@@ -800,15 +743,41 @@ export const askQuestions = async (
         );
         program.llamaCloudKey = llamaCloudKey;
       }
-      // TODO: Consider separate llamaParse to another config
-      program.dataSources.forEach((dataSource) => {
-        if (dataSource.type === "file" || dataSource.type === "folder") {
-          (dataSource.config as FileSourceConfig).useLlamaParse = useLlamaParse;
-        }
-      });
     }
   }
 
+  if (program.dataSource?.type === "web" && program.framework === "fastapi") {
+    let { baseUrl } = await prompts(
+      {
+        type: "text",
+        name: "baseUrl",
+        message: "Please provide base URL of the website:",
+        initial: "https://www.llamaindex.ai",
+      },
+      handlers,
+    );
+    try {
+      if (!baseUrl.includes("://")) {
+        baseUrl = `https://${baseUrl}`;
+      }
+      const checkUrl = new URL(baseUrl);
+      if (checkUrl.protocol !== "https:" && checkUrl.protocol !== "http:") {
+        throw new Error("Invalid protocol");
+      }
+    } catch (error) {
+      console.log(
+        red(
+          "Invalid URL provided! Please provide a valid URL (e.g. https://www.llamaindex.ai)",
+        ),
+      );
+      process.exit(1);
+    }
+    program.dataSource.config = {
+      baseUrl: baseUrl,
+      depth: 1,
+    };
+  }
+
   if (program.engine !== "simple" && !program.vectorDb) {
     if (ciInfo.isCI) {
       program.vectorDb = getPrefOrDefault("vectorDb");

templates/components/loaders/python/loader.py

@@ -1,12 +0,0 @@
-import os
-import importlib
-
-
-def get_documents():
-    # For each file in .loaders, import the module and call the get_documents function
-    for loader in os.listdir(os.path.join(os.path.dirname(__file__), "loaders")):
-        if loader.endswith(".py"):
-            loader = loader[:-3]
-            module = importlib.import_module(f"app.engine.loaders.{loader}")
-            documents = module.get_documents()
-            yield documents

templates/components/observability/typescript/opentelemetry/index.ts

@@ -1,5 +1,5 @@
+import * as LlamaIndex from "@llamaindex/edge";
 import * as traceloop from "@traceloop/node-server-sdk";
-import * as LlamaIndex from "llamaindex";
 
 export const initObservability = () => {
   traceloop.initialize({

templates/components/vectordbs/python/none/generate.py

@@ -2,10 +2,10 @@ from dotenv import load_dotenv
 
 load_dotenv()
 
-import os
 import logging
-from llama_index.core.indices import VectorStoreIndex
-from llama_index.core.storage import StorageContext
+from llama_index.core.indices import (
+    VectorStoreIndex,
+)
 from app.engine.constants import STORAGE_DIR
 from app.engine.loader import get_documents
 from app.settings import init_settings
@@ -16,17 +16,15 @@ logger = logging.getLogger()
 
 
 def generate_datasource():
-    storage_context = StorageContext.from_defaults()
-
-    docs = []
-
-    for doc in get_documents():
-        storage_context.docstore.add_documents(doc)
-        docs.extend(doc)
-
-    index = VectorStoreIndex.from_documents(docs, storage_context=storage_context)
-    index.storage_context.persist(persist_dir=STORAGE_DIR)
-    logger.info(f"Generated index at {STORAGE_DIR}")
+    logger.info("Creating new index")
+    # load the documents and create the index
+    documents = get_documents()
+    index = VectorStoreIndex.from_documents(
+        documents,
+    )
+    # store it for later
+    index.storage_context.persist(STORAGE_DIR)
+    logger.info(f"Finished creating new index. Stored in {STORAGE_DIR}")
 
 
 if __name__ == "__main__":

templates/components/vectordbs/typescript/milvus/generate.mjs

@@ -1,11 +1,10 @@
 /* eslint-disable turbo/no-undeclared-env-vars */
+import { VectorStoreIndex } from "@llamaindex/edge";
+import { SimpleDirectoryReader } from "@llamaindex/edge/readers/SimpleDirectoryReader";
+import { storageContextFromDefaults } from "@llamaindex/edge/storage/StorageContext";
+import { MilvusVectorStore } from "@llamaindex/edge/storage/vectorStore/MilvusVectorStore";
 import * as dotenv from "dotenv";
-import {
-  MilvusVectorStore,
-  SimpleDirectoryReader,
-  VectorStoreIndex,
-  storageContextFromDefaults,
-} from "llamaindex";
+
 import {
   STORAGE_DIR,
   checkRequiredEnvVars,

templates/components/vectordbs/typescript/milvus/index.ts

@@ -1,14 +1,14 @@
 import {
   ContextChatEngine,
   LLM,
-  MilvusVectorStore,
-  serviceContextFromDefaults,
   VectorStoreIndex,
-} from "llamaindex";
+  serviceContextFromDefaults,
+} from "@llamaindex/edge";
+import { MilvusVectorStore } from "@llamaindex/edge/storage/vectorStore/MilvusVectorStore";
 import {
-  checkRequiredEnvVars,
   CHUNK_OVERLAP,
   CHUNK_SIZE,
+  checkRequiredEnvVars,
   getMilvusClient,
 } from "./shared.mjs";
 

templates/components/vectordbs/typescript/mongo/generate.mjs

@@ -1,11 +1,9 @@
 /* eslint-disable turbo/no-undeclared-env-vars */
+import { VectorStoreIndex } from "@llamaindex/edge";
+import { SimpleDirectoryReader } from "@llamaindex/edge/readers/SimpleDirectoryReader";
+import { storageContextFromDefaults } from "@llamaindex/edge/storage/StorageContext";
+import { MongoDBAtlasVectorSearch } from "@llamaindex/edge/storage/vectorStore/MongoDBAtlasVectorSearch";
 import * as dotenv from "dotenv";
-import {
-  MongoDBAtlasVectorSearch,
-  SimpleDirectoryReader,
-  VectorStoreIndex,
-  storageContextFromDefaults,
-} from "llamaindex";
 import { MongoClient } from "mongodb";
 import { STORAGE_DIR, checkRequiredEnvVars } from "./shared.mjs";
 

templates/components/vectordbs/typescript/mongo/index.ts

@@ -2,12 +2,12 @@
 import {
   ContextChatEngine,
   LLM,
-  MongoDBAtlasVectorSearch,
-  serviceContextFromDefaults,
   VectorStoreIndex,
-} from "llamaindex";
+  serviceContextFromDefaults,
+} from "@llamaindex/edge";
+import { MongoDBAtlasVectorSearch } from "@llamaindex/edge/storage/vectorStore/MongoDBAtlasVectorSearch";
 import { MongoClient } from "mongodb";
-import { checkRequiredEnvVars, CHUNK_OVERLAP, CHUNK_SIZE } from "./shared.mjs";
+import { CHUNK_OVERLAP, CHUNK_SIZE, checkRequiredEnvVars } from "./shared.mjs";
 
 async function getDataSource(llm: LLM) {
   checkRequiredEnvVars();

templates/components/vectordbs/typescript/none/generate.mjs

@@ -1,9 +1,6 @@
-import {
-  serviceContextFromDefaults,
-  SimpleDirectoryReader,
-  storageContextFromDefaults,
-  VectorStoreIndex,
-} from "llamaindex";
+import { VectorStoreIndex, serviceContextFromDefaults } from "@llamaindex/edge";
+import { SimpleDirectoryReader } from "@llamaindex/edge/readers/SimpleDirectoryReader";
+import { storageContextFromDefaults } from "@llamaindex/edge/storage/StorageContext";
 
 import * as dotenv from "dotenv";
 

templates/components/vectordbs/typescript/none/index.ts

@@ -1,11 +1,11 @@
 import {
   ContextChatEngine,
   LLM,
-  serviceContextFromDefaults,
-  SimpleDocumentStore,
-  storageContextFromDefaults,
   VectorStoreIndex,
-} from "llamaindex";
+  serviceContextFromDefaults,
+} from "@llamaindex/edge";
+import { storageContextFromDefaults } from "@llamaindex/edge/storage/StorageContext";
+import { SimpleDocumentStore } from "@llamaindex/edge/storage/docStore/SimpleDocumentStore";
 import { CHUNK_OVERLAP, CHUNK_SIZE, STORAGE_CACHE_DIR } from "./constants.mjs";
 
 async function getDataSource(llm: LLM) {

templates/components/vectordbs/typescript/pg/generate.mjs

@@ -1,11 +1,9 @@
 /* eslint-disable turbo/no-undeclared-env-vars */
+import { VectorStoreIndex } from "@llamaindex/edge";
+import { SimpleDirectoryReader } from "@llamaindex/edge/readers/SimpleDirectoryReader";
+import { storageContextFromDefaults } from "@llamaindex/edge/storage/StorageContext";
+import { PGVectorStore } from "@llamaindex/edge/storage/vectorStore/PGVectorStore";
 import * as dotenv from "dotenv";
-import {
-  PGVectorStore,
-  SimpleDirectoryReader,
-  VectorStoreIndex,
-  storageContextFromDefaults,
-} from "llamaindex";
 import {
   PGVECTOR_SCHEMA,
   PGVECTOR_TABLE,

templates/components/vectordbs/typescript/pg/index.ts

@@ -2,10 +2,10 @@
 import {
   ContextChatEngine,
   LLM,
-  PGVectorStore,
   VectorStoreIndex,
   serviceContextFromDefaults,
-} from "llamaindex";
+} from "@llamaindex/edge";
+import { PGVectorStore } from "@llamaindex/edge/storage/vectorStore/PGVectorStore";
 import {
   CHUNK_OVERLAP,
   CHUNK_SIZE,

templates/components/vectordbs/typescript/pinecone/generate.mjs

@@ -1,11 +1,9 @@
 /* eslint-disable turbo/no-undeclared-env-vars */
+import { VectorStoreIndex } from "@llamaindex/edge";
+import { SimpleDirectoryReader } from "@llamaindex/edge/readers/SimpleDirectoryReader";
+import { storageContextFromDefaults } from "@llamaindex/edge/storage/StorageContext";
+import { PineconeVectorStore } from "@llamaindex/edge/storage/vectorStore/PineconeVectorStore";
 import * as dotenv from "dotenv";
-import {
-  PineconeVectorStore,
-  SimpleDirectoryReader,
-  VectorStoreIndex,
-  storageContextFromDefaults,
-} from "llamaindex";
 import { STORAGE_DIR, checkRequiredEnvVars } from "./shared.mjs";
 
 dotenv.config();

templates/components/vectordbs/typescript/pinecone/index.ts

@@ -2,10 +2,10 @@
 import {
   ContextChatEngine,
   LLM,
-  PineconeVectorStore,
   VectorStoreIndex,
   serviceContextFromDefaults,
-} from "llamaindex";
+} from "@llamaindex/edge";
+import { PineconeVectorStore } from "@llamaindex/edge/storage/vectorStore/PineconeVectorStore";
 import { CHUNK_OVERLAP, CHUNK_SIZE, checkRequiredEnvVars } from "./shared.mjs";
 
 async function getDataSource(llm: LLM) {

templates/types/simple/express/package.json

@@ -12,7 +12,7 @@
     "cors": "^2.8.5",
     "dotenv": "^16.3.1",
     "express": "^4.18.2",
-    "llamaindex": "latest"
+    "@llamaindex/edge": "^0.2.1"
   },
   "devDependencies": {
     "@types/cors": "^2.8.17",

templates/types/simple/express/src/controllers/chat.controller.ts

@@ -1,5 +1,5 @@
+import { ChatMessage, MessageContent, OpenAI } from "@llamaindex/edge";
 import { Request, Response } from "express";
-import { ChatMessage, MessageContent, OpenAI } from "llamaindex";
 import { createChatEngine } from "./engine";
 
 const convertMessageContent = (

templates/types/simple/express/src/controllers/engine/index.ts

@@ -1,4 +1,4 @@
-import { LLM, SimpleChatEngine } from "llamaindex";
+import { LLM, SimpleChatEngine } from "@llamaindex/edge";
 
 export async function createChatEngine(llm: LLM) {
   return new SimpleChatEngine({

templates/types/streaming/express/src/controllers/chat.controller.ts

@@ -1,6 +1,6 @@
+import { ChatMessage, MessageContent, OpenAI } from "@llamaindex/edge";
 import { streamToResponse } from "ai";
 import { Request, Response } from "express";
-import { ChatMessage, MessageContent, OpenAI } from "llamaindex";
 import { createChatEngine } from "./engine";
 import { LlamaIndexStream } from "./llamaindex-stream";
 

templates/types/streaming/express/src/controllers/engine/index.ts

@@ -1,4 +1,4 @@
-import { LLM, SimpleChatEngine } from "llamaindex";
+import { LLM, SimpleChatEngine } from "@llamaindex/edge";
 
 export async function createChatEngine(llm: LLM) {
   return new SimpleChatEngine({

templates/types/streaming/express/src/controllers/llamaindex-stream.ts

@@ -1,3 +1,4 @@
+import { Response } from "@llamaindex/edge";
 import {
   JSONValue,
   createCallbacksTransformer,
@@ -6,7 +7,6 @@ import {
   trimStartOfStreamHelper,
   type AIStreamCallbacksAndOptions,
 } from "ai";
-import { Response } from "llamaindex";
 
 type ParserOptions = {
   image_url?: string;

templates/types/streaming/nextjs/app/api/chat/engine/index.ts

@@ -1,4 +1,4 @@
-import { LLM, SimpleChatEngine } from "llamaindex";
+import { LLM, SimpleChatEngine } from "@llamaindex/edge";
 
 export async function createChatEngine(llm: LLM) {
   return new SimpleChatEngine({

templates/types/streaming/nextjs/app/api/chat/llamaindex-stream.ts

@@ -1,3 +1,4 @@
+import { Response } from "@llamaindex/edge";
 import {
   JSONValue,
   createCallbacksTransformer,
@@ -6,7 +7,6 @@ import {
   trimStartOfStreamHelper,
   type AIStreamCallbacksAndOptions,
 } from "ai";
-import { Response } from "llamaindex";
 
 type ParserOptions = {
   image_url?: string;

templates/types/streaming/nextjs/app/api/chat/route.ts

@@ -1,13 +1,13 @@
 import { initObservability } from "@/app/observability";
+import { ChatMessage, MessageContent, OpenAI } from "@llamaindex/edge";
 import { StreamingTextResponse } from "ai";
-import { ChatMessage, MessageContent, OpenAI } from "llamaindex";
 import { NextRequest, NextResponse } from "next/server";
 import { createChatEngine } from "./engine";
 import { LlamaIndexStream } from "./llamaindex-stream";
 
 initObservability();
 
-export const runtime = "nodejs";
+export const runtime = "edge";
 export const dynamic = "force-dynamic";
 
 const convertMessageContent = (

templates/types/streaming/nextjs/package.json

@@ -13,7 +13,7 @@
     "class-variance-authority": "^0.7.0",
     "clsx": "^1.2.1",
     "dotenv": "^16.3.1",
-    "llamaindex": "latest",
+    "@llamaindex/edge": "^0.2.1",
     "lucide-react": "^0.294.0",
     "next": "^14.0.3",
     "react": "^18.2.0",