update paths for macos

.changeset/config.json

@@ -1,7 +1,7 @@
 {
   "$schema": "https://unpkg.com/@changesets/config@3.0.0/schema.json",
   "changelog": "@changesets/cli/changelog",
-  "commit": false,
+  "commit": true,
   "fixed": [],
   "linked": [],
   "access": "public",

.changeset/shaggy-ads-fix.md

@@ -0,0 +1,5 @@
+---
+"create-llama": patch
+---
+
+Add fetching llm and embedding models from server

.changeset/short-ducks-drum.md

@@ -1,5 +0,0 @@
----
-"create-llama": patch
----
-
-Use ingestion pipeline for Python

.changeset/spicy-moose-allow.md

@@ -1,5 +0,0 @@
----
-"create-llama": patch
----
-
-Display events (e.g. retrieving nodes) per chat message

.changeset/tasty-hairs-wait.md

@@ -0,0 +1,5 @@
+---
+"create-llama": patch
+---
+
+Add Milvus vector database

.github/workflows/e2e.yml

@@ -24,46 +24,39 @@ jobs:
     runs-on: ${{ matrix.os }}
     steps:
       - uses: actions/checkout@v4
-
       - name: Set up python ${{ matrix.python-version }}
         uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python-version }}
-
       - name: Install Poetry
         uses: snok/install-poetry@v1
         with:
           version: ${{ env.POETRY_VERSION }}
-
-      - uses: pnpm/action-setup@v3
-
+      - uses: pnpm/action-setup@v2
       - name: Setup Node.js ${{ matrix.node-version }}
         uses: actions/setup-node@v4
         with:
           node-version: ${{ matrix.node-version }}
           cache: "pnpm"
-
       - name: Install dependencies
         run: pnpm install
-
       - name: Install Playwright Browsers
         run: pnpm exec playwright install --with-deps
         working-directory: .
-
       - name: Build create-llama
         run: pnpm run build
         working-directory: .
-
-      - name: Install
-        run: pnpm run pack-install
+      - name: Pack
+        run: pnpm pack --pack-destination ./output
+        working-directory: .
+      - name: Extract Pack
+        run: tar -xvzf ./output/*.tgz -C ./output
         working-directory: .
-
       - name: Run Playwright tests
-        run: pnpm run e2e
+        run: pnpm exec playwright test
         env:
           OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
         working-directory: .
-
       - uses: actions/upload-artifact@v3
         if: always()
         with:

.github/workflows/lint_on_push_or_pull.yml

@@ -13,20 +13,17 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
-
-      - uses: pnpm/action-setup@v3
-
+      - uses: pnpm/action-setup@v2
+        with:
+          version: latest
       - name: Setup Node.js
         uses: actions/setup-node@v4
         with:
           node-version-file: ".nvmrc"
           cache: "pnpm"
-
       - name: Install dependencies
         run: pnpm install
-
       - name: Run lint
         run: pnpm run lint
-
       - name: Run Prettier
         run: pnpm run format

.github/workflows/publish_github.yml

@@ -1,36 +0,0 @@
-name: Publish to GitHub Releases
-
-on:
-  push:
-    tags:
-      - "v*"
-
-jobs:
-  build-and-publish:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout Repo
-        uses: actions/checkout@v4
-
-      - uses: pnpm/action-setup@v3
-
-      - name: Setup Node.js
-        uses: actions/setup-node@v4
-        with:
-          node-version-file: ".nvmrc"
-          cache: "pnpm"
-
-      - name: Install dependencies
-        run: pnpm install
-
-      - name: Build tarball
-        run: |
-          pnpm pack
-
-      - name: Create release
-        uses: ncipollo/release-action@v1
-        with:
-          artifacts: "create-llama-*.tgz"
-          name: Release ${{ github.ref }}
-          bodyFile: "CHANGELOG.md"
-          token: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/release.yml

@@ -1,55 +0,0 @@
-name: Release
-
-on:
-  push:
-    branches:
-      - main
-
-concurrency: ${{ github.workflow }}-${{ github.ref }}
-
-jobs:
-  release:
-    name: Release
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout Repo
-        uses: actions/checkout@v4
-
-      - uses: pnpm/action-setup@v3
-
-      - name: Setup Node.js
-        uses: actions/setup-node@v4
-        with:
-          node-version-file: ".nvmrc"
-          cache: "pnpm"
-
-      - name: Install dependencies
-        run: pnpm install
-
-      - name: Add auth token to .npmrc file
-        run: |
-          cat << EOF >> ".npmrc"
-            //registry.npmjs.org/:_authToken=$NPM_TOKEN
-          EOF
-        env:
-          NPM_TOKEN: ${{ secrets.NPM_TOKEN }}
-
-      - name: Get changeset status
-        id: get-changeset-status
-        run: |
-          pnpm changeset status --output .changeset/status.json
-          new_version=$(jq -r '.releases[0].newVersion' < .changeset/status.json)
-          rm -v .changeset/status.json
-          echo "new-version=${new_version}" >> "$GITHUB_OUTPUT"
-
-      - name: Create Release Pull Request or Publish to npm
-        id: changesets
-        uses: changesets/action@v1
-        with:
-          commit: Release ${{ steps.get-changeset-status.outputs.new-version }}
-          title: Release ${{ steps.get-changeset-status.outputs.new-version }}
-          # build package and call changeset publish
-          publish: pnpm release
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          NPM_TOKEN: ${{ secrets.NPM_TOKEN }}

.gitignore

@@ -45,6 +45,3 @@ e2e/cache
 
 # intellij
 **/.idea
-
-# build artifacts
-create-llama-*.tgz

CHANGELOG.md

@@ -1,53 +1,5 @@
 # create-llama
 
-## 0.1.0
-
-### Minor Changes
-
-- f1c3e8d: Add Llama3 and Phi3 support using Ollama
-
-### Patch Changes
-
-- a0dec80: Use `gpt-4-turbo` model as default. Upgrade Python llama-index to 0.10.28
-- 753229d: Remove asking for AI models and use defaults instead (OpenAIs GPT-4 Vision Preview and Embeddings v3). Use `--ask-models` CLI parameter to select models.
-- 1d78202: Add observability for Python
-- 6acccd2: Use poetry run generate to generate embeddings for FastAPI
-- 9efcffe: Use Settings object for LlamaIndex configuration
-- 418bf9b: refactor: use tsx instead of ts-node
-- 1be69a5: Add Qdrant support
-
-## 0.0.32
-
-### Patch Changes
-
-- 625ed4d: Support Astra VectorDB
-- 922e0ce: Remove UI question (use shadcn as default). Use `html` UI by calling create-llama with --ui html parameter
-- ce2f24d: Update loaders and tools config to yaml format (for Python)
-- e8db041: Let user select multiple datasources (URLs, files and folders)
-- c06d4af: Add nodes to the response (Python)
-- 29b17ee: Allow using agents without any data source
-- 665c26c: Add redirect to documentation page when accessing the base URL (FastAPI)
-- 78ded9e: Add Dockerfile templates for Typescript and Python
-- 99e758f: Merge non-streaming and streaming template to one
-- b3f2685: Add support for agent generation for Typescript
-- 2739714: Use a database (MySQL or PostgreSQL) as a data source
-
-## 0.0.31
-
-### Patch Changes
-
-- 56faee0: Added windows e2e tests
-- 60ed8fe: Added missing environment variable config for URL data source
-- 60ed8fe: Fixed tool usage by freezing llama-index package versions
-
-## 0.0.30
-
-### Patch Changes
-
-- 3af6328: Add support for llamaparse using Typescript
-- dd92b91: Add fetching llm and embedding models from server
-- bac1b43: Add Milvus vector database
-
 ## 0.0.29
 
 ### Patch Changes

CONTRIBUTING.md

@@ -1,73 +0,0 @@
-# Contributing
-
-## Getting Started
-
-Install NodeJS. Preferably v18 using nvm or n.
-
-Inside the `create-llama` directory:
-
-```
-npm i -g pnpm
-pnpm install
-```
-
-Note: we use pnpm in this repo, which has a lot of the same functionality and CLI options as npm but it does do some things better, like caching.
-
-### Building
-
-When we publish to NPM we will have a [ncc](https://github.com/vercel/ncc) compiled version of the tool. To run the build command, run
-
-```
-pnpm run build
-```
-
-### Test cases
-
-We are using a set of e2e tests to ensure that the tool works as expected.
-
-We're using [playwright](https://playwright.dev/) to run the tests.
-To install it, call:
-
-```
-pnpm exec playwright install --with-deps
-```
-
-Then you can create a global `create-llama` command (used by the e2e tests) that is linked to your local dev environment (if you update the build, you don't need to re-link):
-
-```
-pnpm link --global
-```
-
-And then finally run the tests:
-
-```
-pnpm run e2e
-```
-
-To write new test cases write them in [e2e](/e2e)
-
-## Changeset
-
-We use [changesets](https://github.com/changesets/changesets) for managing versions and changelogs. To create a new changeset, run:
-
-```
-pnpm changeset
-```
-
-Please send a descriptive changeset for each PR.
-
-## Publishing (maintainers only)
-
-To publish a new version of the library, first create a new version:
-
-```shell
-pnpm new-version
-```
-
-If everything looks good, commit the generated files and release the new version:
-
-```shell
-pnpm release
-git push # push to the main branch
-git push --tags
-```

README.md

@@ -18,19 +18,25 @@ to start the development server. You can then visit [http://localhost:3000](http
 
 ## What you'll get
 
-- A Next.js-powered front-end using components from [shadcn/ui](https://ui.shadcn.com/). The app is set up as a chat interface that can answer questions about your data (see below)
+- A Next.js-powered front-end. The app is set up as a chat interface that can answer questions about your data (see below)
+  - You can style it with HTML and CSS, or you can optionally use components from [shadcn/ui](https://ui.shadcn.com/)
 - Your choice of 3 back-ends:
-  - **Next.js**: if you select this option, you’ll have a full-stack Next.js application that you can deploy to a host like [Vercel](https://vercel.com/) in just a few clicks. This uses [LlamaIndex.TS](https://www.npmjs.com/package/llamaindex), our TypeScript library.
+  - **Next.js**: if you select this option, you’ll have a full stack Next.js application that you can deploy to a host like [Vercel](https://vercel.com/) in just a few clicks. This uses [LlamaIndex.TS](https://www.npmjs.com/package/llamaindex), our TypeScript library.
   - **Express**: if you want a more traditional Node.js application you can generate an Express backend. This also uses LlamaIndex.TS.
-  - **Python FastAPI**: if you select this option, you’ll get a backend powered by the [llama-index python package](https://pypi.org/project/llama-index/), which you can deploy to a service like Render or fly.io.
-- The back-end has two endpoints (one streaming, the other one non-streaming) that allow you to send the state of your chat and receive additional responses
-- You add arbitrary data sources to your chat, like local files, websites, or data retrieved from a database.
-- Turn your chat into an AI agent by adding tools (functions called by the LLM).
+  - **Python FastAPI**: if you select this option you’ll get a backend powered by the [llama-index python package](https://pypi.org/project/llama-index/), which you can deploy to a service like Render or fly.io.
+- The back-end has a single endpoint that allows you to send the state of your chat and receive additional responses
+- You can choose whether you want a streaming or non-streaming back-end (if you're not sure, we recommend streaming)
+- You can choose whether you want to use `ContextChatEngine` or `SimpleChatEngine`
+  - `SimpleChatEngine` will just talk to the LLM directly without using your data
+  - `ContextChatEngine` will use your data to answer questions (see below).
 - The app uses OpenAI by default, so you'll need an OpenAI API key, or you can customize it to use any of the dozens of LLMs we support.
 
 ## Using your data
 
-You can supply your own data; the app will index it and answer questions. Your generated app will have a folder called `data` (If you're using Express or Python and generate a frontend, it will be `./backend/data`).
+If you've enabled `ContextChatEngine`, you can supply your own data and the app will index it and answer questions. Your generated app will have a folder called `data`:
+
+- With the Next.js backend this is `./data`
+- With the Express or Python backend this is in `./backend/data`
 
 The app will ingest any supported files you put in this directory. Your Next.js and Express apps use LlamaIndex.TS so they will be able to ingest any PDF, text, CSV, Markdown, Word and HTML files. The Python backend can read even more types, including video and audio files.
 
@@ -40,28 +46,22 @@ Before you can use your data, you need to index it. If you're using the Next.js
 npm run generate
 ```
 
-Then re-start your app. Remember you'll need to re-run `generate` if you add new files to your `data` folder.
+Then re-start your app. Remember you'll need to re-run `generate` if you add new files to your `data` folder. If you're using the Python backend, you can trigger indexing of your data by deleting the `./storage` folder and re-starting the app.
 
-If you're using the Python backend, you can trigger indexing of your data by calling:
+## Don't want a front-end?
 
-```bash
-poetry run generate
-```
+It's optional! If you've selected the Python or Express back-ends, just delete the `frontend` folder and you'll get an API without any front-end code.
 
-## Want a front-end?
+## Customizing the LLM
 
-Optionally generate a frontend if you've selected the Python or Express back-ends. If you do so, `create-llama` will generate two folders: `frontend`, for your Next.js-based frontend code, and `backend` containing your API.
+By default the app will use OpenAI's gpt-3.5-turbo model. If you want to use GPT-4, you can modify this by editing a file:
 
-## Customizing the AI models
-
-The app will default to OpenAI's `gpt-4-turbo` LLM and `text-embedding-3-large` embedding model.
-
-If you want to use different OpenAI models, add the `--ask-models` CLI parameter.
+- In the Next.js backend, edit `./app/api/chat/route.ts` and replace `gpt-3.5-turbo` with `gpt-4`
+- In the Express backend, edit `./backend/src/controllers/chat.controller.ts` and likewise replace `gpt-3.5-turbo` with `gpt-4`
+- In the Python backend, edit `./backend/app/utils/index.py` and once again replace `gpt-3.5-turbo` with `gpt-4`
 
 You can also replace OpenAI with one of our [dozens of other supported LLMs](https://docs.llamaindex.ai/en/stable/module_guides/models/llms/modules.html).
 
-To do so, you have to manually change the generated code (edit the `settings.ts` file for Typescript projects or the `settings.py` file for Python projects)
-
 ## Example
 
 The simplest thing to do is run `create-llama` in interactive mode:
@@ -84,19 +84,13 @@ Need to install the following packages:
   create-llama@latest
 Ok to proceed? (y) y
 ✔ What is your project named? … my-app
-✔ Which template would you like to use? › Chat
+✔ Which template would you like to use? › Chat with streaming
 ✔ Which framework would you like to use? › NextJS
-✔ Would you like to set up observability? › No
+✔ Which UI would you like to use? › Just HTML
+✔ Which chat engine would you like to use? › ContextChatEngine
 ✔ Please provide your OpenAI API key (leave blank to skip): …
-✔ Which data source would you like to use? › Use an example PDF
-✔ Would you like to add another data source? › No
-✔ Would you like to use LlamaParse (improved parser for RAG - requires API key)? … no / yes
-✔ Would you like to use a vector database? › No, just store the data in the file system
-? How would you like to proceed? › - Use arrow-keys. Return to submit.
-   Just generate code (~1 sec)
-❯  Start in VSCode (~1 sec)
-   Generate code and install dependencies (~2 min)
-   Generate code, install dependencies, and run the app (~2 min)
+✔ Would you like to use ESLint? … No / Yes
+Creating a new LlamaIndex app in /home/my-app.
 ```
 
 ### Running non-interactively

create-app.ts

@@ -26,12 +26,16 @@ export type InstallAppArgs = Omit<
 export async function createApp({
   template,
   framework,
+  engine,
   ui,
   appPath,
   packageManager,
+  eslint,
   frontend,
-  modelConfig,
+  openAiKey,
   llamaCloudKey,
+  model,
+  embeddingModel,
   communityProjectConfig,
   llamapack,
   vectorDb,
@@ -39,7 +43,6 @@ export async function createApp({
   postInstallAction,
   dataSources,
   tools,
-  useLlamaParse,
   observability,
 }: InstallAppArgs): Promise<void> {
   const root = path.resolve(appPath);
@@ -72,11 +75,15 @@ export async function createApp({
     root,
     template,
     framework,
+    engine,
     ui,
     packageManager,
     isOnline,
-    modelConfig,
+    eslint,
+    openAiKey,
     llamaCloudKey,
+    model,
+    embeddingModel,
     communityProjectConfig,
     llamapack,
     vectorDb,
@@ -84,7 +91,6 @@ export async function createApp({
     postInstallAction,
     dataSources,
     tools,
-    useLlamaParse,
     observability,
   };
 
@@ -121,13 +127,11 @@ export async function createApp({
   }
 
   if (toolsRequireConfig(tools)) {
-    const configFile =
-      framework === "fastapi" ? "config/tools.yaml" : "config/tools.json";
     console.log(
       yellow(
         `You have selected tools that require configuration. Please configure them in the ${terminalLink(
-          configFile,
-          `file://${root}/${configFile}`,
+          "tools_config.json",
+          `file://${root}/tools_config.json`,
         )} file.`,
       ),
     );

e2e/basic.spec.ts

@@ -4,6 +4,7 @@ import { ChildProcess } from "child_process";
 import fs from "fs";
 import path from "path";
 import type {
+  TemplateEngine,
   TemplateFramework,
   TemplatePostInstallAction,
   TemplateType,
@@ -11,13 +12,13 @@ import type {
 } from "../helpers";
 import { createTestDir, runCreateLlama, type AppType } from "./utils";
 
-const templateTypes: TemplateType[] = ["streaming"];
+const templateTypes: TemplateType[] = ["streaming", "simple"];
 const templateFrameworks: TemplateFramework[] = [
   "nextjs",
   "express",
   "fastapi",
 ];
-const dataSources: string[] = ["--no-files", "--example-file"];
+const templateEngines: TemplateEngine[] = ["simple", "context"];
 const templateUIs: TemplateUI[] = ["shadcn", "html"];
 const templatePostInstallActions: TemplatePostInstallAction[] = [
   "none",
@@ -26,12 +27,24 @@ const templatePostInstallActions: TemplatePostInstallAction[] = [
 
 for (const templateType of templateTypes) {
   for (const templateFramework of templateFrameworks) {
-    for (const dataSource of dataSources) {
+    for (const templateEngine of templateEngines) {
       for (const templateUI of templateUIs) {
         for (const templatePostInstallAction of templatePostInstallActions) {
+          if (templateFramework === "nextjs" && templateType === "simple") {
+            // nextjs doesn't support simple templates - skip tests
+            continue;
+          }
           const appType: AppType =
-            templateFramework === "nextjs" ? "" : "--frontend";
-          test.describe(`try create-llama ${templateType} ${templateFramework} ${dataSource} ${templateUI} ${appType} ${templatePostInstallAction}`, async () => {
+            templateFramework === "express" || templateFramework === "fastapi"
+              ? templateType === "simple"
+                ? "--no-frontend" // simple templates don't have frontends
+                : "--frontend"
+              : "";
+          if (appType === "--no-frontend" && templateUI !== "html") {
+            // if there's no frontend, don't iterate over UIs
+            continue;
+          }
+          test.describe(`try create-llama ${templateType} ${templateFramework} ${templateEngine} ${templateUI} ${appType} ${templatePostInstallAction}`, async () => {
             let port: number;
             let externalPort: number;
             let cwd: string;
@@ -48,7 +61,7 @@ for (const templateType of templateTypes) {
                 cwd,
                 templateType,
                 templateFramework,
-                dataSource,
+                templateEngine,
                 templateUI,
                 vectorDb,
                 appType,
@@ -66,6 +79,7 @@ for (const templateType of templateTypes) {
             });
             test("Frontend should have a title", async ({ page }) => {
               test.skip(templatePostInstallAction !== "runApp");
+              test.skip(appType === "--no-frontend");
               await page.goto(`http://localhost:${port}`);
               await expect(page.getByText("Built by LlamaIndex")).toBeVisible();
             });
@@ -74,6 +88,7 @@ for (const templateType of templateTypes) {
               page,
             }) => {
               test.skip(templatePostInstallAction !== "runApp");
+              test.skip(appType === "--no-frontend");
               await page.goto(`http://localhost:${port}`);
               await page.fill("form input", "hello");
               const [response] = await Promise.all([
@@ -94,13 +109,14 @@ for (const templateType of templateTypes) {
               expect(response.ok()).toBeTruthy();
             });
 
-            test("Backend frameworks should response when calling non-streaming chat API", async ({
+            test("Backend should response when calling API", async ({
               request,
             }) => {
               test.skip(templatePostInstallAction !== "runApp");
-              test.skip(templateFramework === "nextjs");
+              test.skip(appType !== "--no-frontend");
+              const backendPort = appType === "" ? port : externalPort;
               const response = await request.post(
-                `http://localhost:${externalPort}/api/chat/request`,
+                `http://localhost:${backendPort}/api/chat`,
                 {
                   data: {
                     messages: [

e2e/utils.ts

@@ -4,6 +4,7 @@ import { mkdir } from "node:fs/promises";
 import * as path from "path";
 import waitPort from "wait-port";
 import {
+  TemplateEngine,
   TemplateFramework,
   TemplatePostInstallAction,
   TemplateType,
@@ -12,7 +13,8 @@ import {
 } from "../helpers";
 
 export type AppType = "--frontend" | "--no-frontend" | "";
-
+const MODEL = "gpt-3.5-turbo";
+const EMBEDDING_MODEL = "text-embedding-ada-002";
 export type CreateLlamaResult = {
   projectName: string;
   appProcess: ChildProcess;
@@ -65,7 +67,7 @@ export async function runCreateLlama(
   cwd: string,
   templateType: TemplateType,
   templateFramework: TemplateFramework,
-  dataSource: string,
+  templateEngine: TemplateEngine,
   templateUI: TemplateUI,
   vectorDb: TemplateVectorDB,
   appType: AppType,
@@ -73,32 +75,45 @@ export async function runCreateLlama(
   externalPort: number,
   postInstallAction: TemplatePostInstallAction,
 ): Promise<CreateLlamaResult> {
-  if (!process.env.OPENAI_API_KEY) {
-    throw new Error("Setting OPENAI_API_KEY is mandatory to run tests");
-  }
+  const createLlama = path.join(
+    __dirname,
+    "..",
+    "output",
+    "package",
+    "dist",
+    "index.js",
+  );
+
   const name = [
     templateType,
     templateFramework,
-    dataSource,
+    templateEngine,
     templateUI,
     appType,
   ].join("-");
   const command = [
-    "create-llama",
+    "node",
+    createLlama,
     name,
     "--template",
     templateType,
     "--framework",
     templateFramework,
-    dataSource,
+    "--engine",
+    templateEngine,
     "--ui",
     templateUI,
     "--vector-db",
     vectorDb,
+    "--model",
+    MODEL,
+    "--embedding-model",
+    EMBEDDING_MODEL,
     "--open-ai-key",
-    process.env.OPENAI_API_KEY,
+    process.env.OPENAI_API_KEY || "testKey",
     appType,
-    "--use-pnpm",
+    "--eslint",
+    "--use-npm",
     "--port",
     port,
     "--external-port",

helpers/copy.ts

@@ -48,21 +48,3 @@ export const copy = async (
     }),
   );
 };
-
-export const assetRelocator = (name: string) => {
-  switch (name) {
-    case "gitignore":
-    case "npmrc":
-    case "eslintrc.json": {
-      return `.${name}`;
-    }
-    // README.md is ignored by webpack-asset-relocator-loader used by ncc:
-    // https://github.com/vercel/webpack-asset-relocator-loader/blob/e9308683d47ff507253e37c9bcbb99474603192b/src/asset-relocator.js#L227
-    case "README-template.md": {
-      return "README.md";
-    }
-    default: {
-      return name;
-    }
-  }
-};

helpers/datasources.ts

@@ -1,109 +0,0 @@
-import fs from "fs/promises";
-import path from "path";
-import yaml, { Document } from "yaml";
-import { templatesDir } from "./dir";
-import { DbSourceConfig, TemplateDataSource, WebSourceConfig } from "./types";
-
-export const EXAMPLE_FILE: TemplateDataSource = {
-  type: "file",
-  config: {
-    path: path.join(templatesDir, "components", "data", "101.pdf"),
-  },
-};
-
-export function getDataSources(
-  files?: string,
-  exampleFile?: boolean,
-): TemplateDataSource[] | undefined {
-  let dataSources: TemplateDataSource[] | undefined = undefined;
-  if (files) {
-    // If user specified files option, then the program should use context engine
-    dataSources = files.split(",").map((filePath) => ({
-      type: "file",
-      config: {
-        path: filePath,
-      },
-    }));
-  }
-  if (exampleFile) {
-    dataSources = [...(dataSources ? dataSources : []), EXAMPLE_FILE];
-  }
-  return dataSources;
-}
-
-export async function writeLoadersConfig(
-  root: string,
-  dataSources: TemplateDataSource[],
-  useLlamaParse?: boolean,
-) {
-  if (dataSources.length === 0) return; // no datasources, no config needed
-  const loaderConfig = new Document({});
-  // Web loader config
-  if (dataSources.some((ds) => ds.type === "web")) {
-    const webLoaderConfig = new Document({});
-
-    // Create config for browser driver arguments
-    const driverArgNodeValue = webLoaderConfig.createNode([
-      "--no-sandbox",
-      "--disable-dev-shm-usage",
-    ]);
-    driverArgNodeValue.commentBefore =
-      " The arguments to pass to the webdriver. E.g.: add --headless to run in headless mode";
-    webLoaderConfig.set("driver_arguments", driverArgNodeValue);
-
-    // Create config for urls
-    const urlConfigs = dataSources
-      .filter((ds) => ds.type === "web")
-      .map((ds) => {
-        const dsConfig = ds.config as WebSourceConfig;
-        return {
-          base_url: dsConfig.baseUrl,
-          prefix: dsConfig.prefix,
-          depth: dsConfig.depth,
-        };
-      });
-    const urlConfigNode = webLoaderConfig.createNode(urlConfigs);
-    urlConfigNode.commentBefore = ` base_url: The URL to start crawling with
- prefix: Only crawl URLs matching the specified prefix
- depth: The maximum depth for BFS traversal
- You can add more websites by adding more entries (don't forget the - prefix from YAML)`;
-    webLoaderConfig.set("urls", urlConfigNode);
-
-    // Add web config to the loaders config
-    loaderConfig.set("web", webLoaderConfig);
-  }
-
-  // File loader config
-  if (dataSources.some((ds) => ds.type === "file")) {
-    // Add documentation to web loader config
-    const node = loaderConfig.createNode({
-      use_llama_parse: useLlamaParse,
-    });
-    node.commentBefore = ` use_llama_parse: Use LlamaParse if \`true\`. Needs a \`LLAMA_CLOUD_API_KEY\` from https://cloud.llamaindex.ai set as environment variable`;
-    loaderConfig.set("file", node);
-  }
-
-  // DB loader config
-  const dbLoaders = dataSources.filter((ds) => ds.type === "db");
-  if (dbLoaders.length > 0) {
-    const dbLoaderConfig = new Document({});
-    const configEntries = dbLoaders.map((ds) => {
-      const dsConfig = ds.config as DbSourceConfig;
-      return {
-        uri: dsConfig.uri,
-        queries: [dsConfig.queries],
-      };
-    });
-
-    const node = dbLoaderConfig.createNode(configEntries);
-    node.commentBefore = ` The configuration for the database loader, only supports MySQL and PostgreSQL databases for now.
- uri: The URI for the database. E.g.: mysql+pymysql://user:password@localhost:3306/db or postgresql+psycopg2://user:password@localhost:5432/db
- query: The query to fetch data from the database. E.g.: SELECT * FROM table`;
-    loaderConfig.set("db", node);
-  }
-
-  // Write loaders config
-  const loaderConfigPath = path.join(root, "config", "loaders.yaml");
-  await fs.mkdir(path.join(root, "config"), { recursive: true });
-  await fs.writeFile(loaderConfigPath, yaml.stringify(loaderConfig));
-}

helpers/devcontainer.ts

@@ -46,16 +46,13 @@ export const writeDevcontainer = async (
   framework: TemplateFramework,
   frontend: boolean,
 ) => {
-  const devcontainerDir = path.join(root, ".devcontainer");
-  if (fs.existsSync(devcontainerDir)) {
-    console.log("Template already has a .devcontainer. Using it.");
-    return;
-  }
+  console.log("Adding .devcontainer");
   const devcontainerContent = renderDevcontainerContent(
     templatesDir,
     framework,
     frontend,
   );
+  const devcontainerDir = path.join(root, ".devcontainer");
   fs.mkdirSync(devcontainerDir);
   await fs.promises.writeFile(
     path.join(devcontainerDir, "devcontainer.json"),

helpers/env-variables.ts

@@ -1,10 +1,11 @@
 import fs from "fs/promises";
 import path from "path";
 import {
-  ModelConfig,
+  FileSourceConfig,
   TemplateDataSource,
   TemplateFramework,
   TemplateVectorDB,
+  WebSourceConfig,
 } from "./types";
 
 type EnvVar = {
@@ -29,10 +30,7 @@ const renderEnvVar = (envVars: EnvVar[]): string => {
   );
 };
 
-const getVectorDBEnvs = (vectorDb?: TemplateVectorDB): EnvVar[] => {
-  if (!vectorDb) {
-    return [];
-  }
+const getVectorDBEnvs = (vectorDb: TemplateVectorDB) => {
   switch (vectorDb) {
     case "mongo":
       return [
@@ -97,107 +95,113 @@ const getVectorDBEnvs = (vectorDb?: TemplateVectorDB): EnvVar[] => {
           description: "The password to access the Milvus server.",
         },
       ];
-    case "astra":
-      return [
-        {
-          name: "ASTRA_DB_APPLICATION_TOKEN",
-          description: "The generated app token for your Astra database",
-        },
-        {
-          name: "ASTRA_DB_ENDPOINT",
-          description: "The API endpoint for your Astra database",
-        },
-        {
-          name: "ASTRA_DB_COLLECTION",
-          description: "The name of the collection in your Astra database",
-        },
-      ];
-    case "qdrant":
-      return [
-        {
-          name: "QDRANT_URL",
-          description:
-            "The qualified REST URL of the Qdrant server. Eg: http://localhost:6333",
-        },
-        {
-          name: "QDRANT_COLLECTION",
-          description: "The name of Qdrant collection to use.",
-        },
-        {
-          name: "QDRANT_API_KEY",
-          description:
-            "Optional API key for authenticating requests to Qdrant.",
-        },
-      ];
     default:
       return [];
   }
 };
 
-const getModelEnvs = (modelConfig: ModelConfig): EnvVar[] => {
-  return [
-    {
-      name: "MODEL_PROVIDER",
-      description: "The provider for the AI models to use.",
-      value: modelConfig.provider,
-    },
-    {
-      name: "MODEL",
-      description: "The name of LLM model to use.",
-      value: modelConfig.model,
-    },
-    {
-      name: "EMBEDDING_MODEL",
-      description: "Name of the embedding model to use.",
-      value: modelConfig.embeddingModel,
-    },
-    {
-      name: "EMBEDDING_DIM",
-      description: "Dimension of the embedding model to use.",
-      value: modelConfig.dimensions.toString(),
-    },
-    ...(modelConfig.provider === "openai"
-      ? [
+const getDataSourceEnvs = (dataSources: TemplateDataSource[]) => {
+  const envs = [];
+  for (const source of dataSources) {
+    switch (source.type) {
+      case "web":
+        const config = source.config as WebSourceConfig;
+        envs.push(
           {
-            name: "OPENAI_API_KEY",
-            description: "The OpenAI API key to use.",
-            value: modelConfig.apiKey,
+            name: "BASE_URL",
+            description: "The base URL to start web scraping.",
+            value: config.baseUrl,
           },
           {
-            name: "LLM_TEMPERATURE",
-            description: "Temperature for sampling from the model.",
+            name: "URL_PREFIX",
+            description: "The prefix of the URL to start web scraping.",
+            value: config.baseUrl,
           },
           {
-            name: "LLM_MAX_TOKENS",
-            description: "Maximum number of tokens to generate.",
+            name: "MAX_DEPTH",
+            description: "The maximum depth to scrape.",
+            value: config.depth?.toString(),
           },
-        ]
-      : []),
-  ];
+        );
+    }
+  }
+  return envs;
 };
 
-const getFrameworkEnvs = (
-  framework?: TemplateFramework,
-  port?: number,
-): EnvVar[] => {
-  if (framework !== "fastapi") {
-    return [];
-  }
-  return [
+export const createBackendEnvFile = async (
+  root: string,
+  opts: {
+    openAiKey?: string;
+    llamaCloudKey?: string;
+    vectorDb?: TemplateVectorDB;
+    model?: string;
+    embeddingModel?: string;
+    framework?: TemplateFramework;
+    dataSources?: TemplateDataSource[];
+    port?: number;
+  },
+) => {
+  // Init env values
+  const envFileName = ".env";
+  const defaultEnvs = [
     {
-      name: "APP_HOST",
-      description: "The address to start the backend app.",
-      value: "0.0.0.0",
+      render: true,
+      name: "MODEL",
+      description: "The name of LLM model to use.",
+      value: opts.model || "gpt-3.5-turbo",
     },
     {
-      name: "APP_PORT",
-      description: "The port to start the backend app.",
-      value: port?.toString() || "8000",
+      render: true,
+      name: "OPENAI_API_KEY",
+      description: "The OpenAI API key to use.",
+      value: opts.openAiKey,
     },
-    // TODO: Once LlamaIndexTS supports string templates, move this to `getEngineEnvs`
-    {
-      name: "SYSTEM_PROMPT",
-      description: `Custom system prompt.
+    // Add vector database environment variables
+    ...(opts.vectorDb ? getVectorDBEnvs(opts.vectorDb) : []),
+    // Add data source environment variables
+    ...(opts.dataSources ? getDataSourceEnvs(opts.dataSources) : []),
+  ];
+  let envVars: EnvVar[] = [];
+  if (opts.framework === "fastapi") {
+    envVars = [
+      ...defaultEnvs,
+      ...[
+        {
+          name: "APP_HOST",
+          description: "The address to start the backend app.",
+          value: "0.0.0.0",
+        },
+        {
+          name: "APP_PORT",
+          description: "The port to start the backend app.",
+          value: opts.port?.toString() || "8000",
+        },
+        {
+          name: "EMBEDDING_MODEL",
+          description: "Name of the embedding model to use.",
+          value: opts.embeddingModel,
+        },
+        {
+          name: "EMBEDDING_DIM",
+          description: "Dimension of the embedding model to use.",
+        },
+        {
+          name: "LLM_TEMPERATURE",
+          description: "Temperature for sampling from the model.",
+        },
+        {
+          name: "LLM_MAX_TOKENS",
+          description: "Maximum number of tokens to generate.",
+        },
+        {
+          name: "TOP_K",
+          description:
+            "The number of similar embeddings to return when retrieving documents.",
+          value: "3",
+        },
+        {
+          name: "SYSTEM_PROMPT",
+          description: `Custom system prompt.
 Example:
 SYSTEM_PROMPT="
 We have provided context information below.
@@ -206,48 +210,33 @@ We have provided context information below.
 ---------------------
 Given this information, please answer the question: {query_str}
 "`,
-    },
-  ];
-};
-
-const getEngineEnvs = (): EnvVar[] => {
-  return [
-    {
-      name: "TOP_K",
-      description:
-        "The number of similar embeddings to return when retrieving documents.",
-      value: "3",
-    },
-  ];
-};
-
-export const createBackendEnvFile = async (
-  root: string,
-  opts: {
-    llamaCloudKey?: string;
-    vectorDb?: TemplateVectorDB;
-    modelConfig: ModelConfig;
-    framework?: TemplateFramework;
-    dataSources?: TemplateDataSource[];
-    port?: number;
-  },
-) => {
-  // Init env values
-  const envFileName = ".env";
-  const envVars: EnvVar[] = [
-    {
-      name: "LLAMA_CLOUD_API_KEY",
-      description: `The Llama Cloud API key.`,
-      value: opts.llamaCloudKey,
-    },
-    // Add model environment variables
-    ...getModelEnvs(opts.modelConfig),
-    // Add engine environment variables
-    ...getEngineEnvs(),
-    // Add vector database environment variables
-    ...getVectorDBEnvs(opts.vectorDb),
-    ...getFrameworkEnvs(opts.framework, opts.port),
-  ];
+        },
+        opts?.dataSources?.some(
+          (ds) => (ds.config as FileSourceConfig).useLlamaParse,
+        )
+          ? {
+              name: "LLAMA_CLOUD_API_KEY",
+              description: `The Llama Cloud API key.`,
+              value: opts.llamaCloudKey,
+            }
+          : {},
+      ],
+    ];
+  } else {
+    envVars = [
+      ...defaultEnvs,
+      ...[
+        opts.framework === "nextjs"
+          ? {
+              name: "NEXT_PUBLIC_MODEL",
+              description:
+                "The LLM model to use (hardcode to front-end artifact).",
+              value: opts.model || "gpt-3.5-turbo",
+            }
+          : {},
+      ],
+    ];
+  }
   // Render and write env file
   const content = renderEnvVar(envVars);
   await fs.writeFile(path.join(root, envFileName), content);
@@ -258,9 +247,20 @@ export const createFrontendEnvFile = async (
   root: string,
   opts: {
     customApiPath?: string;
+    model?: string;
   },
 ) => {
   const defaultFrontendEnvs = [
+    {
+      name: "MODEL",
+      description: "The OpenAI model to use.",
+      value: opts.model,
+    },
+    {
+      name: "NEXT_PUBLIC_MODEL",
+      description: "The OpenAI model to use (hardcode to front-end artifact).",
+      value: opts.model,
+    },
     {
       name: "NEXT_PUBLIC_CHAT_API",
       description: "The backend API for chat endpoint.",

helpers/index.ts

@@ -1,22 +1,20 @@
+import { copy } from "./copy";
 import { callPackageManager } from "./install";
 
+import fs from "fs/promises";
 import path from "path";
 import { cyan } from "picocolors";
 
-import fsExtra from "fs-extra";
-import { writeLoadersConfig } from "./datasources";
+import { templatesDir } from "./dir";
 import { createBackendEnvFile, createFrontendEnvFile } from "./env-variables";
 import { PackageManager } from "./get-pkg-manager";
 import { installLlamapackProject } from "./llama-pack";
 import { isHavingPoetryLockFile, tryPoetryRun } from "./poetry";
-import { isModelConfigured } from "./providers";
 import { installPythonTemplate } from "./python";
 import { downloadAndExtractRepo } from "./repo";
-import { ConfigFileType, writeToolsConfig } from "./tools";
 import {
   FileSourceConfig,
   InstallTemplateArgs,
-  ModelConfig,
   TemplateDataSource,
   TemplateFramework,
   TemplateVectorDB,
@@ -26,8 +24,8 @@ import { installTSTemplate } from "./typescript";
 // eslint-disable-next-line max-params
 async function generateContextData(
   framework: TemplateFramework,
-  modelConfig: ModelConfig,
   packageManager?: PackageManager,
+  openAiKey?: string,
   vectorDb?: TemplateVectorDB,
   llamaCloudKey?: string,
   useLlamaParse?: boolean,
@@ -35,37 +33,40 @@ async function generateContextData(
   if (packageManager) {
     const runGenerate = `${cyan(
       framework === "fastapi"
-        ? "poetry run generate"
+        ? "poetry run python app/engine/generate.py"
         : `${packageManager} run generate`,
     )}`;
-    const modelConfigured = isModelConfigured(modelConfig);
+    const openAiKeyConfigured = openAiKey || process.env["OPENAI_API_KEY"];
     const llamaCloudKeyConfigured = useLlamaParse
       ? llamaCloudKey || process.env["LLAMA_CLOUD_API_KEY"]
       : true;
     const hasVectorDb = vectorDb && vectorDb !== "none";
-    if (modelConfigured && llamaCloudKeyConfigured && !hasVectorDb) {
-      // If all the required environment variables are set, run the generate script
-      if (framework === "fastapi") {
-        if (isHavingPoetryLockFile()) {
-          console.log(`Running ${runGenerate} to generate the context data.`);
-          const result = tryPoetryRun("poetry run generate");
-          if (!result) {
-            console.log(`Failed to run ${runGenerate}.`);
-            process.exit(1);
-          }
-          console.log(`Generated context data`);
-          return;
+    if (framework === "fastapi") {
+      if (
+        openAiKeyConfigured &&
+        llamaCloudKeyConfigured &&
+        !hasVectorDb &&
+        isHavingPoetryLockFile()
+      ) {
+        console.log(`Running ${runGenerate} to generate the context data.`);
+        const result = tryPoetryRun("python app/engine/generate.py");
+        if (!result) {
+          console.log(`Failed to run ${runGenerate}.`);
+          process.exit(1);
         }
-      } else {
+        console.log(`Generated context data`);
+        return;
+      }
+    } else {
+      if (openAiKeyConfigured && vectorDb === "none") {
         console.log(`Running ${runGenerate} to generate the context data.`);
         await callPackageManager(packageManager, true, ["run", "generate"]);
         return;
       }
     }
 
-    // generate the message of what to do to run the generate script manually
     const settings = [];
-    if (!modelConfigured) settings.push("your model provider API key");
+    if (!openAiKeyConfigured) settings.push("your OpenAI key");
     if (!llamaCloudKeyConfigured) settings.push("your Llama Cloud key");
     if (hasVectorDb) settings.push("your Vector DB environment variables");
     const settingsMessage =
@@ -77,16 +78,46 @@ async function generateContextData(
 
 const copyContextData = async (
   root: string,
-  dataSources: TemplateDataSource[],
+  dataSource?: TemplateDataSource,
 ) => {
-  for (const dataSource of dataSources) {
-    const dataSourceConfig = dataSource?.config as FileSourceConfig;
-    // Copy local data
-    const dataPath = dataSourceConfig.path;
+  const destPath = path.join(root, "data");
+  const dataSourceConfig = dataSource?.config as FileSourceConfig;
 
-    const destPath = path.join(root, "data", path.basename(dataPath));
-    console.log("Copying data from path:", dataPath);
-    await fsExtra.copy(dataPath, destPath);
+  // Copy file
+  if (dataSource?.type === "file") {
+    if (dataSourceConfig.paths) {
+      await fs.mkdir(destPath, { recursive: true });
+      console.log(
+        "Copying data from files:",
+        dataSourceConfig.paths.toString(),
+      );
+      for (const p of dataSourceConfig.paths) {
+        await fs.copyFile(p, path.join(destPath, path.basename(p)));
+      }
+    } else {
+      console.log("Missing file path in config");
+      process.exit(1);
+    }
+    return;
+  }
+
+  // Copy folder
+  if (dataSource?.type === "folder") {
+    // Example data does not have path config, set the default path
+    const srcPaths = dataSourceConfig.paths ?? [
+      path.join(templatesDir, "components", "data"),
+    ];
+    console.log("Copying data from folders: ", srcPaths);
+    for (const p of srcPaths) {
+      const folderName = path.basename(p);
+      const destFolderPath = path.join(destPath, folderName);
+      await fs.mkdir(destFolderPath, { recursive: true });
+      await copy("**", destFolderPath, {
+        parents: true,
+        cwd: p,
+      });
+    }
+    return;
   }
 };
 
@@ -121,59 +152,54 @@ export const installTemplate = async (
 
   if (props.framework === "fastapi") {
     await installPythonTemplate(props);
-    // write loaders configuration (currently Python only)
-    await writeLoadersConfig(
-      props.root,
-      props.dataSources,
-      props.useLlamaParse,
-    );
   } else {
     await installTSTemplate(props);
   }
 
-  // write tools configuration
-  await writeToolsConfig(
-    props.root,
-    props.tools,
-    props.framework === "fastapi" ? ConfigFileType.YAML : ConfigFileType.JSON,
-  );
-
   if (props.backend) {
     // This is a backend, so we need to copy the test data and create the env file.
 
     // Copy the environment file to the target directory.
     await createBackendEnvFile(props.root, {
-      modelConfig: props.modelConfig,
+      openAiKey: props.openAiKey,
       llamaCloudKey: props.llamaCloudKey,
       vectorDb: props.vectorDb,
+      model: props.model,
+      embeddingModel: props.embeddingModel,
       framework: props.framework,
       dataSources: props.dataSources,
       port: props.externalPort,
     });
 
-    if (props.dataSources.length > 0) {
+    if (props.engine === "context") {
       console.log("\nGenerating context data...\n");
-      await copyContextData(
-        props.root,
-        props.dataSources.filter((ds) => ds.type === "file"),
-      );
+      props.dataSources.forEach(async (ds) => {
+        if (ds.type === "file" || ds.type === "folder") {
+          await copyContextData(props.root, ds);
+        }
+      });
       if (
         props.postInstallAction === "runApp" ||
         props.postInstallAction === "dependencies"
       ) {
         await generateContextData(
           props.framework,
-          props.modelConfig,
           props.packageManager,
+          props.openAiKey,
           props.vectorDb,
           props.llamaCloudKey,
-          props.useLlamaParse,
+          props.dataSources.some(
+            (ds) =>
+              (ds.type === "file" || ds.type === "folder") &&
+              (ds.config as FileSourceConfig).useLlamaParse,
+          ),
         );
       }
     }
   } else {
     // this is a frontend for a full-stack app, create .env file with model information
-    await createFrontendEnvFile(props.root, {
+    createFrontendEnvFile(props.root, {
+      model: props.model,
       customApiPath: props.customApiPath,
     });
   }

helpers/providers/index.ts

@@ -1,66 +0,0 @@
-import ciInfo from "ci-info";
-import prompts from "prompts";
-import { questionHandlers } from "../../questions";
-import { ModelConfig, ModelProvider } from "../types";
-import { askOllamaQuestions } from "./ollama";
-import { askOpenAIQuestions, isOpenAIConfigured } from "./openai";
-
-const DEFAULT_MODEL_PROVIDER = "openai";
-
-export type ModelConfigQuestionsParams = {
-  openAiKey?: string;
-  askModels: boolean;
-};
-
-export type ModelConfigParams = Omit<ModelConfig, "provider">;
-
-export async function askModelConfig({
-  askModels,
-  openAiKey,
-}: ModelConfigQuestionsParams): Promise<ModelConfig> {
-  let modelProvider: ModelProvider = DEFAULT_MODEL_PROVIDER;
-  if (askModels && !ciInfo.isCI) {
-    const { provider } = await prompts(
-      {
-        type: "select",
-        name: "provider",
-        message: "Which model provider would you like to use",
-        choices: [
-          {
-            title: "OpenAI",
-            value: "openai",
-          },
-          { title: "Ollama", value: "ollama" },
-        ],
-        initial: 0,
-      },
-      questionHandlers,
-    );
-    modelProvider = provider;
-  }
-
-  let modelConfig: ModelConfigParams;
-  switch (modelProvider) {
-    case "ollama":
-      modelConfig = await askOllamaQuestions({ askModels });
-      break;
-    default:
-      modelConfig = await askOpenAIQuestions({
-        openAiKey,
-        askModels,
-      });
-  }
-  return {
-    ...modelConfig,
-    provider: modelProvider,
-  };
-}
-
-export function isModelConfigured(modelConfig: ModelConfig): boolean {
-  switch (modelConfig.provider) {
-    case "openai":
-      return isOpenAIConfigured(modelConfig);
-    default:
-      return true;
-  }
-}

helpers/providers/ollama.ts

@@ -1,92 +0,0 @@
-import ciInfo from "ci-info";
-import ollama, { type ModelResponse } from "ollama";
-import { red } from "picocolors";
-import prompts from "prompts";
-import { ModelConfigParams } from ".";
-import { questionHandlers, toChoice } from "../../questions";
-
-type ModelData = {
-  dimensions: number;
-};
-const MODELS = ["llama3:8b", "wizardlm2:7b", "gemma:7b", "phi3"];
-const DEFAULT_MODEL = MODELS[0];
-// TODO: get embedding vector dimensions from the ollama sdk (currently not supported)
-const EMBEDDING_MODELS: Record<string, ModelData> = {
-  "nomic-embed-text": { dimensions: 768 },
-  "mxbai-embed-large": { dimensions: 1024 },
-  "all-minilm": { dimensions: 384 },
-};
-const DEFAULT_EMBEDDING_MODEL: string = Object.keys(EMBEDDING_MODELS)[0];
-
-type OllamaQuestionsParams = {
-  askModels: boolean;
-};
-
-export async function askOllamaQuestions({
-  askModels,
-}: OllamaQuestionsParams): Promise<ModelConfigParams> {
-  const config: ModelConfigParams = {
-    model: DEFAULT_MODEL,
-    embeddingModel: DEFAULT_EMBEDDING_MODEL,
-    dimensions: EMBEDDING_MODELS[DEFAULT_EMBEDDING_MODEL].dimensions,
-  };
-
-  // use default model values in CI or if user should not be asked
-  const useDefaults = ciInfo.isCI || !askModels;
-  if (!useDefaults) {
-    const { model } = await prompts(
-      {
-        type: "select",
-        name: "model",
-        message: "Which LLM model would you like to use?",
-        choices: MODELS.map(toChoice),
-        initial: 0,
-      },
-      questionHandlers,
-    );
-    await ensureModel(model);
-    config.model = model;
-
-    const { embeddingModel } = await prompts(
-      {
-        type: "select",
-        name: "embeddingModel",
-        message: "Which embedding model would you like to use?",
-        choices: Object.keys(EMBEDDING_MODELS).map(toChoice),
-        initial: 0,
-      },
-      questionHandlers,
-    );
-    await ensureModel(embeddingModel);
-    config.embeddingModel = embeddingModel;
-    config.dimensions = EMBEDDING_MODELS[embeddingModel].dimensions;
-  }
-
-  return config;
-}
-
-async function ensureModel(modelName: string) {
-  try {
-    if (modelName.split(":").length === 1) {
-      // model doesn't have a version suffix, use latest
-      modelName = modelName + ":latest";
-    }
-    const { models } = await ollama.list();
-    const found =
-      models.find((model: ModelResponse) => model.name === modelName) !==
-      undefined;
-    if (!found) {
-      console.log(
-        red(
-          `Model ${modelName} was not pulled yet. Call 'ollama pull ${modelName}' and try again.`,
-        ),
-      );
-      process.exit(1);
-    }
-  } catch (error) {
-    console.log(
-      red("Listing Ollama models failed. Is 'ollama' running? " + error),
-    );
-    process.exit(1);
-  }
-}

helpers/providers/openai.ts

@@ -1,146 +0,0 @@
-import ciInfo from "ci-info";
-import got from "got";
-import ora from "ora";
-import { red } from "picocolors";
-import prompts from "prompts";
-import { ModelConfigParams, ModelConfigQuestionsParams } from ".";
-import { questionHandlers } from "../../questions";
-
-const OPENAI_API_URL = "https://api.openai.com/v1";
-
-const DEFAULT_MODEL = "gpt-4-turbo";
-const DEFAULT_EMBEDDING_MODEL = "text-embedding-3-large";
-
-export async function askOpenAIQuestions({
-  openAiKey,
-  askModels,
-}: ModelConfigQuestionsParams): Promise<ModelConfigParams> {
-  const config: ModelConfigParams = {
-    apiKey: openAiKey,
-    model: DEFAULT_MODEL,
-    embeddingModel: DEFAULT_EMBEDDING_MODEL,
-    dimensions: getDimensions(DEFAULT_EMBEDDING_MODEL),
-  };
-
-  if (!config.apiKey) {
-    const { key } = await prompts(
-      {
-        type: "text",
-        name: "key",
-        message: askModels
-          ? "Please provide your OpenAI API key (or leave blank to use OPENAI_API_KEY env variable):"
-          : "Please provide your OpenAI API key (leave blank to skip):",
-        validate: (value: string) => {
-          console.log(value);
-          if (askModels && !value) {
-            if (process.env.OPENAI_API_KEY) {
-              return true;
-            }
-            return "OPENAI_API_KEY env variable is not set - key is required";
-          }
-          return true;
-        },
-      },
-      questionHandlers,
-    );
-    config.apiKey = key || process.env.OPENAI_API_KEY;
-  }
-
-  // use default model values in CI or if user should not be asked
-  const useDefaults = ciInfo.isCI || !askModels;
-  if (!useDefaults) {
-    const { model } = await prompts(
-      {
-        type: "select",
-        name: "model",
-        message: "Which LLM model would you like to use?",
-        choices: await getAvailableModelChoices(false, config.apiKey),
-        initial: 0,
-      },
-      questionHandlers,
-    );
-    config.model = model;
-
-    const { embeddingModel } = await prompts(
-      {
-        type: "select",
-        name: "embeddingModel",
-        message: "Which embedding model would you like to use?",
-        choices: await getAvailableModelChoices(true, config.apiKey),
-        initial: 0,
-      },
-      questionHandlers,
-    );
-    config.embeddingModel = embeddingModel;
-    config.dimensions = getDimensions(embeddingModel);
-  }
-
-  return config;
-}
-
-export function isOpenAIConfigured(params: ModelConfigParams): boolean {
-  if (params.apiKey) {
-    return true;
-  }
-  if (process.env["OPENAI_API_KEY"]) {
-    return true;
-  }
-  return false;
-}
-
-async function getAvailableModelChoices(
-  selectEmbedding: boolean,
-  apiKey?: string,
-) {
-  if (!apiKey) {
-    throw new Error("need OpenAI key to retrieve model choices");
-  }
-  const isLLMModel = (modelId: string) => {
-    return modelId.startsWith("gpt");
-  };
-
-  const isEmbeddingModel = (modelId: string) => {
-    return modelId.includes("embedding");
-  };
-
-  const spinner = ora("Fetching available models").start();
-  try {
-    const response = await got(`${OPENAI_API_URL}/models`, {
-      headers: {
-        Authorization: "Bearer " + apiKey,
-      },
-      timeout: 5000,
-      responseType: "json",
-    });
-    const data: any = await response.body;
-    spinner.stop();
-    return data.data
-      .filter((model: any) =>
-        selectEmbedding ? isEmbeddingModel(model.id) : isLLMModel(model.id),
-      )
-      .map((el: any) => {
-        return {
-          title: el.id,
-          value: el.id,
-        };
-      });
-  } catch (error) {
-    spinner.stop();
-    if ((error as any).response?.statusCode === 401) {
-      console.log(
-        red(
-          "Invalid OpenAI API key provided! Please provide a valid key and try again!",
-        ),
-      );
-    } else {
-      console.log(red("Request failed: " + error));
-    }
-    process.exit(1);
-  }
-}
-
-function getDimensions(modelName: string) {
-  // at 2024-04-24 all OpenAI embedding models support 1536 dimensions except
-  // "text-embedding-3-large", see https://openai.com/blog/new-embedding-models-and-api-updates
-  return modelName === "text-embedding-3-large" ? 1024 : 1536;
-}

helpers/python.ts

@@ -3,14 +3,13 @@ import path from "path";
 import { cyan, red } from "picocolors";
 import { parse, stringify } from "smol-toml";
 import terminalLink from "terminal-link";
-
-import { assetRelocator, copy } from "./copy";
+import { copy } from "./copy";
 import { templatesDir } from "./dir";
 import { isPoetryAvailable, tryPoetryInstall } from "./poetry";
 import { Tool } from "./tools";
 import {
+  FileSourceConfig,
   InstallTemplateArgs,
-  ModelConfig,
   TemplateDataSource,
   TemplateVectorDB,
 } from "./types";
@@ -22,7 +21,6 @@ interface Dependency {
 }
 
 const getAdditionalDependencies = (
-  modelConfig: ModelConfig,
   vectorDb?: TemplateVectorDB,
   dataSource?: TemplateDataSource,
   tools?: Tool[],
@@ -56,51 +54,23 @@ const getAdditionalDependencies = (
         name: "llama-index-vector-stores-milvus",
         version: "^0.1.6",
       });
-      dependencies.push({
-        name: "pymilvus",
-        version: "2.3.7",
-      });
-      break;
-    }
-    case "astra": {
-      dependencies.push({
-        name: "llama-index-vector-stores-astra-db",
-        version: "^0.1.5",
-      });
       break;
     }
   }
 
   // Add data source dependencies
   const dataSourceType = dataSource?.type;
-  switch (dataSourceType) {
-    case "file":
-      dependencies.push({
-        name: "docx2txt",
-        version: "^0.8",
-      });
-      break;
-    case "web":
-      dependencies.push({
-        name: "llama-index-readers-web",
-        version: "^0.1.6",
-      });
-      break;
-    case "db":
-      dependencies.push({
-        name: "llama-index-readers-database",
-        version: "^0.1.3",
-      });
-      dependencies.push({
-        name: "pymysql",
-        version: "^1.1.0",
-        extras: ["rsa"],
-      });
-      dependencies.push({
-        name: "psycopg2",
-        version: "^2.9.9",
-      });
-      break;
+  if (dataSourceType === "file" || dataSourceType === "folder") {
+    // llama-index-readers-file (pdf, excel, csv) is already included in llama_index package
+    dependencies.push({
+      name: "docx2txt",
+      version: "^0.8",
+    });
+  } else if (dataSourceType === "web") {
+    dependencies.push({
+      name: "llama-index-readers-web",
+      version: "^0.1.6",
+    });
   }
 
   // Add tools dependencies
@@ -110,25 +80,6 @@ const getAdditionalDependencies = (
     });
   });
 
-  switch (modelConfig.provider) {
-    case "ollama":
-      dependencies.push({
-        name: "llama-index-llms-ollama",
-        version: "0.1.2",
-      });
-      dependencies.push({
-        name: "llama-index-embeddings-ollama",
-        version: "0.1.2",
-      });
-      break;
-    case "openai":
-      dependencies.push({
-        name: "llama-index-agent-openai",
-        version: "0.2.2",
-      });
-      break;
-  }
-
   return dependencies;
 };
 
@@ -222,92 +173,134 @@ export const installPythonTemplate = async ({
   root,
   template,
   framework,
+  engine,
   vectorDb,
   dataSources,
   tools,
   postInstallAction,
-  observability,
-  modelConfig,
 }: Pick<
   InstallTemplateArgs,
   | "root"
   | "framework"
   | "template"
+  | "engine"
   | "vectorDb"
   | "dataSources"
   | "tools"
   | "postInstallAction"
-  | "observability"
-  | "modelConfig"
 >) => {
   console.log("\nInitializing Python project with template:", template, "\n");
   const templatePath = path.join(templatesDir, "types", template, framework);
   await copy("**", root, {
     parents: true,
     cwd: templatePath,
-    rename: assetRelocator,
+    rename(name) {
+      switch (name) {
+        case "gitignore": {
+          return `.${name}`;
+        }
+        // README.md is ignored by webpack-asset-relocator-loader used by ncc:
+        // https://github.com/vercel/webpack-asset-relocator-loader/blob/e9308683d47ff507253e37c9bcbb99474603192b/src/asset-relocator.js#L227
+        case "README-template.md": {
+          return "README.md";
+        }
+        default: {
+          return name;
+        }
+      }
+    },
   });
 
-  const compPath = path.join(templatesDir, "components");
-  const enginePath = path.join(root, "app", "engine");
+  if (engine === "context") {
+    const enginePath = path.join(root, "app", "engine");
+    const compPath = path.join(templatesDir, "components");
 
-  // Copy selected vector DB
-  await copy("**", enginePath, {
-    parents: true,
-    cwd: path.join(compPath, "vectordbs", "python", vectorDb ?? "none"),
-  });
+    const vectorDbDirName = vectorDb ?? "none";
+    const VectorDBPath = path.join(
+      compPath,
+      "vectordbs",
+      "python",
+      vectorDbDirName,
+    );
+    await copy("**", enginePath, {
+      parents: true,
+      cwd: VectorDBPath,
+    });
 
-  // Copy all loaders to enginePath
-  const loaderPath = path.join(enginePath, "loaders");
-  await copy("**", loaderPath, {
-    parents: true,
-    cwd: path.join(compPath, "loaders", "python"),
-  });
+    // Copy engine code
+    if (tools !== undefined && tools.length > 0) {
+      await copy("**", enginePath, {
+        parents: true,
+        cwd: path.join(compPath, "engines", "python", "agent"),
+      });
+      // Write tools_config.json
+      const configContent: Record<string, any> = {};
+      tools.forEach((tool) => {
+        configContent[tool.name] = tool.config ?? {};
+      });
+      const configFilePath = path.join(root, "tools_config.json");
+      await fs.writeFile(
+        configFilePath,
+        JSON.stringify(configContent, null, 2),
+      );
+    } else {
+      await copy("**", enginePath, {
+        parents: true,
+        cwd: path.join(compPath, "engines", "python", "chat"),
+      });
+    }
 
-  // Select and copy engine code based on data sources and tools
-  let engine;
-  tools = tools ?? [];
-  if (dataSources.length > 0 && tools.length === 0) {
-    console.log("\nNo tools selected - use optimized context chat engine\n");
-    engine = "chat";
-  } else {
-    engine = "agent";
+    if (dataSources.length > 0 || dataSources[0].type !== "none") {
+      // Copy loader.py file to enginePath
+      await copy("loader.py", enginePath, {
+        parents: true,
+        cwd: path.join(compPath, "loaders", "python"),
+      });
+
+      // Copy data source loaders
+      const loaderPath = path.join(enginePath, "loaders");
+      for (const source of dataSources) {
+        const sourceType = source.type;
+        if (sourceType === "file" || sourceType === "folder") {
+          const sourceConfig = source.config as FileSourceConfig;
+          const loaderFolder = sourceConfig.useLlamaParse
+            ? "llama_parse"
+            : "file";
+          await copy("**", loaderPath, {
+            parents: true,
+            cwd: path.join(compPath, "loaders", "python", loaderFolder),
+          });
+        } else {
+          await copy("**", loaderPath, {
+            parents: true,
+            cwd: path.join(compPath, "loaders", "python", sourceType),
+          });
+        }
+      }
+    }
+
+    // const dataSourceType = dataSource?.type;
+    // if (dataSourceType !== undefined && dataSourceType !== "none") {
+    //   let loaderFolder: string;
+    //   if (dataSourceType === "file" || dataSourceType === "folder") {
+    //     const dataSourceConfig = dataSource?.config as FileSourceConfig;
+    //     loaderFolder = dataSourceConfig.useLlamaParse ? "llama_parse" : "file";
+    //   } else {
+    //     loaderFolder = dataSourceType;
+    //   }
+    //   await copy("**", enginePath, {
+    //     parents: true,
+    //     cwd: path.join(compPath, "loaders", "python", loaderFolder),
+    //   });
+    // }
   }
-  await copy("**", enginePath, {
-    parents: true,
-    cwd: path.join(compPath, "engines", "python", engine),
-  });
 
   const addOnDependencies = dataSources
-    .map((ds) => getAdditionalDependencies(modelConfig, vectorDb, ds, tools))
+    .map((ds) => getAdditionalDependencies(vectorDb, ds, tools))
     .flat();
-
-  if (observability === "opentelemetry") {
-    addOnDependencies.push({
-      name: "traceloop-sdk",
-      version: "^0.15.11",
-    });
-
-    const templateObservabilityPath = path.join(
-      templatesDir,
-      "components",
-      "observability",
-      "python",
-      "opentelemetry",
-    );
-    await copy("**", path.join(root, "app"), {
-      cwd: templateObservabilityPath,
-    });
-  }
-
   await addDependencies(root, addOnDependencies);
 
   if (postInstallAction === "runApp" || postInstallAction === "dependencies") {
     installPythonDependencies();
   }
-
-  // Copy deployment files for python
-  await copy("**", root, {
-    cwd: path.join(compPath, "deployments", "python"),
-  });
 };

helpers/tools.ts

@@ -1,18 +1,11 @@
-import fs from "fs/promises";
-import path from "path";
 import { red } from "picocolors";
-import yaml from "yaml";
-import { makeDir } from "./make-dir";
-import { TemplateFramework } from "./types";
 
 export type Tool = {
   display: string;
   name: string;
   config?: Record<string, any>;
   dependencies?: ToolDependencies[];
-  supportedFrameworks?: Array<TemplateFramework>;
 };
-
 export type ToolDependencies = {
   name: string;
   version?: string;
@@ -34,7 +27,6 @@ export const supportedTools: Tool[] = [
         version: "0.1.2",
       },
     ],
-    supportedFrameworks: ["fastapi"],
   },
   {
     display: "Wikipedia",
@@ -45,7 +37,6 @@ export const supportedTools: Tool[] = [
         version: "0.1.2",
       },
     ],
-    supportedFrameworks: ["fastapi", "express", "nextjs"],
   },
 ];
 
@@ -78,33 +69,3 @@ export const toolsRequireConfig = (tools?: Tool[]): boolean => {
   }
   return false;
 };
-
-export enum ConfigFileType {
-  YAML = "yaml",
-  JSON = "json",
-}
-
-export const writeToolsConfig = async (
-  root: string,
-  tools: Tool[] = [],
-  type: ConfigFileType = ConfigFileType.YAML,
-) => {
-  if (tools.length === 0) return; // no tools selected, no config need
-  const configContent: Record<string, any> = {};
-  tools.forEach((tool) => {
-    configContent[tool.name] = tool.config ?? {};
-  });
-  const configPath = path.join(root, "config");
-  await makeDir(configPath);
-  if (type === ConfigFileType.YAML) {
-    await fs.writeFile(
-      path.join(configPath, "tools.yaml"),
-      yaml.stringify(configContent),
-    );
-  } else {
-    await fs.writeFile(
-      path.join(configPath, "tools.json"),
-      JSON.stringify(configContent, null, 2),
-    );
-  }
-};

helpers/types.ts

@@ -1,25 +1,11 @@
 import { PackageManager } from "../helpers/get-pkg-manager";
 import { Tool } from "./tools";
 
-export type ModelProvider = "openai" | "ollama";
-export type ModelConfig = {
-  provider: ModelProvider;
-  apiKey?: string;
-  model: string;
-  embeddingModel: string;
-  dimensions: number;
-};
-export type TemplateType = "streaming" | "community" | "llamapack";
+export type TemplateType = "simple" | "streaming" | "community" | "llamapack";
 export type TemplateFramework = "nextjs" | "express" | "fastapi";
+export type TemplateEngine = "simple" | "context";
 export type TemplateUI = "html" | "shadcn";
-export type TemplateVectorDB =
-  | "none"
-  | "mongo"
-  | "pg"
-  | "pinecone"
-  | "milvus"
-  | "astra"
-  | "qdrant";
+export type TemplateVectorDB = "none" | "mongo" | "pg" | "pinecone" | "milvus";
 export type TemplatePostInstallAction =
   | "none"
   | "VSCode"
@@ -29,26 +15,18 @@ export type TemplateDataSource = {
   type: TemplateDataSourceType;
   config: TemplateDataSourceConfig;
 };
-export type TemplateDataSourceType = "file" | "web" | "db";
+export type TemplateDataSourceType = "none" | "file" | "folder" | "web";
 export type TemplateObservability = "none" | "opentelemetry";
 // Config for both file and folder
 export type FileSourceConfig = {
-  path: string;
+  paths?: string[];
+  useLlamaParse?: boolean;
 };
 export type WebSourceConfig = {
   baseUrl?: string;
-  prefix?: string;
   depth?: number;
 };
-export type DbSourceConfig = {
-  uri?: string;
-  queries?: string;
-};
-
-export type TemplateDataSourceConfig =
-  | FileSourceConfig
-  | WebSourceConfig
-  | DbSourceConfig;
+export type TemplateDataSourceConfig = FileSourceConfig | WebSourceConfig;
 
 export type CommunityProjectConfig = {
   owner: string;
@@ -64,12 +42,15 @@ export interface InstallTemplateArgs {
   isOnline: boolean;
   template: TemplateType;
   framework: TemplateFramework;
+  engine: TemplateEngine;
   ui: TemplateUI;
   dataSources: TemplateDataSource[];
+  eslint: boolean;
   customApiPath?: string;
-  modelConfig: ModelConfig;
+  openAiKey?: string;
   llamaCloudKey?: string;
-  useLlamaParse?: boolean;
+  model: string;
+  embeddingModel: string;
   communityProjectConfig?: CommunityProjectConfig;
   llamapack?: string;
   vectorDb?: TemplateVectorDB;

helpers/typescript.ts

@@ -2,12 +2,50 @@ import fs from "fs/promises";
 import os from "os";
 import path from "path";
 import { bold, cyan } from "picocolors";
-import { assetRelocator, copy } from "../helpers/copy";
+import { copy } from "../helpers/copy";
 import { callPackageManager } from "../helpers/install";
 import { templatesDir } from "./dir";
 import { PackageManager } from "./get-pkg-manager";
 import { InstallTemplateArgs } from "./types";
 
+const rename = (name: string) => {
+  switch (name) {
+    case "gitignore":
+    case "eslintrc.json": {
+      return `.${name}`;
+    }
+    // README.md is ignored by webpack-asset-relocator-loader used by ncc:
+    // https://github.com/vercel/webpack-asset-relocator-loader/blob/e9308683d47ff507253e37c9bcbb99474603192b/src/asset-relocator.js#L227
+    case "README-template.md": {
+      return "README.md";
+    }
+    default: {
+      return name;
+    }
+  }
+};
+
+export const installTSDependencies = async (
+  packageJson: any,
+  packageManager: PackageManager,
+  isOnline: boolean,
+): Promise<void> => {
+  console.log("\nInstalling dependencies:");
+  for (const dependency in packageJson.dependencies)
+    console.log(`- ${cyan(dependency)}`);
+
+  console.log("\nInstalling devDependencies:");
+  for (const dependency in packageJson.devDependencies)
+    console.log(`- ${cyan(dependency)}`);
+
+  console.log();
+
+  await callPackageManager(packageManager, isOnline).catch((error) => {
+    console.error("Failed to install TS dependencies. Exiting...");
+    process.exit(1);
+  });
+};
+
 /**
  * Install a LlamaIndex internal template to a given `root` directory.
  */
@@ -18,14 +56,14 @@ export const installTSTemplate = async ({
   isOnline,
   template,
   framework,
+  engine,
   ui,
+  eslint,
+  customApiPath,
   vectorDb,
   postInstallAction,
   backend,
   observability,
-  tools,
-  dataSources,
-  useLlamaParse,
 }: InstallTemplateArgs & { backend: boolean }) => {
   console.log(bold(`Using ${packageManager}.`));
 
@@ -35,11 +73,12 @@ export const installTSTemplate = async ({
   console.log("\nInitializing project with template:", template, "\n");
   const templatePath = path.join(templatesDir, "types", template, framework);
   const copySource = ["**"];
+  if (!eslint) copySource.push("!eslintrc.json");
 
   await copy(copySource, root, {
     parents: true,
     cwd: templatePath,
-    rename: assetRelocator,
+    rename,
   });
 
   /**
@@ -79,7 +118,6 @@ export const installTSTemplate = async ({
     }
   }
 
-  // copy observability component
   if (observability && observability !== "none") {
     const chosenObservabilityPath = path.join(
       templatesDir,
@@ -97,40 +135,36 @@ export const installTSTemplate = async ({
     );
   }
 
+  /**
+   * Copy the selected chat engine files to the target directory and reference it.
+   */
+  let relativeEngineDestPath;
   const compPath = path.join(templatesDir, "components");
-  const relativeEngineDestPath =
-    framework === "nextjs"
-      ? path.join("app", "api", "chat")
-      : path.join("src", "controllers");
-  const enginePath = path.join(root, relativeEngineDestPath, "engine");
+  if (engine && (framework === "express" || framework === "nextjs")) {
+    console.log("\nUsing chat engine:", engine, "\n");
 
-  // copy vector db component
-  console.log("\nUsing vector DB:", vectorDb, "\n");
-  await copy("**", enginePath, {
-    parents: true,
-    cwd: path.join(compPath, "vectordbs", "typescript", vectorDb ?? "none"),
-  });
+    let vectorDBFolder: string = engine;
 
-  // copy loader component (TS only supports llama_parse and file for now)
-  const loaderFolder = useLlamaParse ? "llama_parse" : "file";
-  await copy("**", enginePath, {
-    parents: true,
-    cwd: path.join(compPath, "loaders", "typescript", loaderFolder),
-  });
+    if (engine !== "simple" && vectorDb) {
+      console.log("\nUsing vector DB:", vectorDb, "\n");
+      vectorDBFolder = vectorDb;
+    }
 
-  // Select and copy engine code based on data sources and tools
-  let engine;
-  tools = tools ?? [];
-  if (dataSources.length > 0 && tools.length === 0) {
-    console.log("\nNo tools selected - use optimized context chat engine\n");
-    engine = "chat";
-  } else {
-    engine = "agent";
+    const VectorDBPath = path.join(
+      compPath,
+      "vectordbs",
+      "typescript",
+      vectorDBFolder,
+    );
+    relativeEngineDestPath =
+      framework === "nextjs"
+        ? path.join("app", "api", "chat")
+        : path.join("src", "controllers");
+    await copy("**", path.join(root, relativeEngineDestPath, "engine"), {
+      parents: true,
+      cwd: VectorDBPath,
+    });
   }
-  await copy("**", enginePath, {
-    parents: true,
-    cwd: path.join(compPath, "engines", "typescript", engine),
-  });
 
   /**
    * Copy the selected UI files to the target directory and reference it.
@@ -145,54 +179,13 @@ export const installTSTemplate = async ({
     await copy("**", destUiPath, {
       parents: true,
       cwd: uiPath,
-      rename: assetRelocator,
+      rename,
     });
   }
 
-  /** Modify frontend code to use custom API path */
-  if (framework === "nextjs" && !backend) {
-    console.log(
-      "\nUsing external API for frontend, removing API code and configuration\n",
-    );
-    // remove the default api folder and config folder
-    await fs.rm(path.join(root, "app", "api"), { recursive: true });
-    await fs.rm(path.join(root, "config"), { recursive: true, force: true });
-  }
-
-  const packageJson = await updatePackageJson({
-    root,
-    appName,
-    dataSources,
-    relativeEngineDestPath,
-    framework,
-    ui,
-    observability,
-  });
-
-  if (postInstallAction === "runApp" || postInstallAction === "dependencies") {
-    await installTSDependencies(packageJson, packageManager, isOnline);
-  }
-
-  // Copy deployment files for typescript
-  await copy("**", root, {
-    cwd: path.join(compPath, "deployments", "typescript"),
-  });
-};
-
-async function updatePackageJson({
-  root,
-  appName,
-  dataSources,
-  relativeEngineDestPath,
-  framework,
-  ui,
-  observability,
-}: Pick<
-  InstallTemplateArgs,
-  "root" | "appName" | "dataSources" | "framework" | "ui" | "observability"
-> & {
-  relativeEngineDestPath: string;
-}): Promise<any> {
+  /**
+   * Update the package.json scripts.
+   */
   const packageJsonFile = path.join(root, "package.json");
   const packageJson: any = JSON.parse(
     await fs.readFile(packageJsonFile, "utf8"),
@@ -200,15 +193,26 @@ async function updatePackageJson({
   packageJson.name = appName;
   packageJson.version = "0.1.0";
 
-  if (relativeEngineDestPath) {
-    // TODO: move script to {root}/scripts for all frameworks
+  if (framework === "nextjs" && customApiPath) {
+    console.log(
+      "\nUsing external API with custom API path:",
+      customApiPath,
+      "\n",
+    );
+    // remove the default api folder
+    const apiPath = path.join(root, "app", "api");
+    await fs.rm(apiPath, { recursive: true });
+    // modify the dev script to use the custom api path
+  }
+
+  if (engine === "context" && relativeEngineDestPath) {
     // add generate script if using context engine
     packageJson.scripts = {
       ...packageJson.scripts,
-      generate: `tsx ${path.join(
+      generate: `node ${path.join(
         relativeEngineDestPath,
         "engine",
-        "generate.ts",
+        "generate.mjs",
       )}`,
     };
   }
@@ -248,31 +252,20 @@ async function updatePackageJson({
     };
   }
 
+  if (!eslint) {
+    // Remove packages starting with "eslint" from devDependencies
+    packageJson.devDependencies = Object.fromEntries(
+      Object.entries(packageJson.devDependencies).filter(
+        ([key]) => !key.startsWith("eslint"),
+      ),
+    );
+  }
   await fs.writeFile(
     packageJsonFile,
     JSON.stringify(packageJson, null, 2) + os.EOL,
   );
 
-  return packageJson;
-}
-
-async function installTSDependencies(
-  packageJson: any,
-  packageManager: PackageManager,
-  isOnline: boolean,
-): Promise<void> {
-  console.log("\nInstalling dependencies:");
-  for (const dependency in packageJson.dependencies)
-    console.log(`- ${cyan(dependency)}`);
-
-  console.log("\nInstalling devDependencies:");
-  for (const dependency in packageJson.devDependencies)
-    console.log(`- ${cyan(dependency)}`);
-
-  console.log();
-
-  await callPackageManager(packageManager, isOnline).catch((error) => {
-    console.error("Failed to install TS dependencies. Exiting...");
-    process.exit(1);
-  });
-}
+  if (postInstallAction === "runApp" || postInstallAction === "dependencies") {
+    await installTSDependencies(packageJson, packageManager, isOnline);
+  }
+};

index.ts

@@ -1,3 +1,4 @@
+#!/usr/bin/env node
 /* eslint-disable import/no-extraneous-dependencies */
 import { execSync } from "child_process";
 import Commander from "commander";
@@ -9,7 +10,6 @@ import prompts from "prompts";
 import terminalLink from "terminal-link";
 import checkForUpdate from "update-check";
 import { createApp } from "./create-app";
-import { getDataSources } from "./helpers/datasources";
 import { getPkgManager } from "./helpers/get-pkg-manager";
 import { isFolderEmpty } from "./helpers/is-folder-empty";
 import { runApp } from "./helpers/run-app";
@@ -32,6 +32,13 @@ const program = new Commander.Command(packageJson.name)
   .action((name) => {
     projectPath = name;
   })
+  .option(
+    "--eslint",
+    `
+
+  Initialize with eslint config.
+`,
+  )
   .option(
     "--use-npm",
     `
@@ -65,6 +72,13 @@ const program = new Commander.Command(packageJson.name)
     `
 
   Select a template to bootstrap the application with.
+`,
+  )
+  .option(
+    "--engine <engine>",
+    `
+
+  Select a chat engine to bootstrap the application with.
 `,
   )
   .option(
@@ -78,14 +92,7 @@ const program = new Commander.Command(packageJson.name)
     "--files <path>",
     `
   
-  Specify the path to a local file or folder for chatting.
-`,
-  )
-  .option(
-    "--example-file",
-    `
-
-  Select to use an example PDF as data source.
+    Specify the path to a local file or folder for chatting.
 `,
   )
   .option(
@@ -107,6 +114,19 @@ const program = new Commander.Command(packageJson.name)
     `
 
   Whether to generate a frontend for your backend.
+`,
+  )
+  .option(
+    "--model <model>",
+    `
+
+  Select OpenAI model to use. E.g. gpt-3.5-turbo.
+`,
+  )
+  .option(
+    "--embedding-model <embeddingModel>",
+    `
+  Select OpenAI embedding model to use. E.g. text-embedding-ada-002.
 `,
   )
   .option(
@@ -145,38 +165,33 @@ const program = new Commander.Command(packageJson.name)
 `,
   )
   .option(
-    "--use-llama-parse",
+    "--llama-parse",
     `
-
-  Enable LlamaParse.
+    Enable LlamaParse.
 `,
   )
   .option(
     "--llama-cloud-key <key>",
     `
-  
   Provide a LlamaCloud API key.
 `,
   )
   .option(
-    "--observability <observability>",
-    `
-    
-  Specify observability tools to use. Eg: none, opentelemetry
-`,
+    "--list-server-models",
+    "Fetch available LLM and embedding models from OpenAI API.",
   )
   .option(
-    "--ask-models",
-    `
-
-  Select LLM and embedding models.
-`,
+    "--observability <observability>",
+    "Specify observability tools to use. Eg: none, opentelemetry",
   )
   .allowUnknownOption()
   .parse(process.argv);
 if (process.argv.includes("--no-frontend")) {
   program.frontend = false;
 }
+if (process.argv.includes("--no-eslint")) {
+  program.eslint = false;
+}
 if (process.argv.includes("--tools")) {
   if (program.tools === "none") {
     program.tools = [];
@@ -185,13 +200,7 @@ if (process.argv.includes("--tools")) {
   }
 }
 if (process.argv.includes("--no-llama-parse")) {
-  program.useLlamaParse = false;
-}
-program.askModels = process.argv.includes("--ask-models");
-if (process.argv.includes("--no-files")) {
-  program.dataSources = [];
-} else {
-  program.dataSources = getDataSources(program.files, program.exampleFile);
+  program.llamaParse = false;
 }
 
 const packageManager = !!program.useNpm
@@ -274,21 +283,21 @@ async function run(): Promise<void> {
   }
 
   const preferences = (conf.get("preferences") || {}) as QuestionArgs;
-  await askQuestions(
-    program as unknown as QuestionArgs,
-    preferences,
-    program.openAiKey,
-  );
+  await askQuestions(program as unknown as QuestionArgs, preferences);
 
   await createApp({
     template: program.template,
     framework: program.framework,
+    engine: program.engine,
     ui: program.ui,
     appPath: resolvedProjectPath,
     packageManager,
+    eslint: program.eslint,
     frontend: program.frontend,
-    modelConfig: program.modelConfig,
+    openAiKey: program.openAiKey,
     llamaCloudKey: program.llamaCloudKey,
+    model: program.model,
+    embeddingModel: program.embeddingModel,
     communityProjectConfig: program.communityProjectConfig,
     llamapack: program.llamapack,
     vectorDb: program.vectorDb,
@@ -296,7 +305,6 @@ async function run(): Promise<void> {
     postInstallAction: program.postInstallAction,
     dataSources: program.dataSources,
     tools: program.tools,
-    useLlamaParse: program.useLlamaParse,
     observability: program.observability,
   });
   conf.set("preferences", preferences);

package.json

@@ -1,12 +1,12 @@
 {
   "name": "create-llama",
-  "version": "0.1.0",
-  "description": "Create LlamaIndex-powered apps with one command",
+  "version": "0.0.29",
   "keywords": [
     "rag",
     "llamaindex",
     "next.js"
   ],
+  "description": "Create LlamaIndex-powered apps with one command",
   "repository": {
     "type": "git",
     "url": "https://github.com/run-llama/LlamaIndexTS",
@@ -20,30 +20,27 @@
     "dist"
   ],
   "scripts": {
-    "build": "bash ./scripts/build.sh",
-    "build:ncc": "pnpm run clean && ncc build ./index.ts -o ./dist/ --minify --no-cache --no-source-map-register",
     "clean": "rimraf --glob ./dist ./templates/**/__pycache__ ./templates/**/node_modules ./templates/**/poetry.lock",
-    "dev": "ncc build ./index.ts -w -o dist/",
-    "e2e": "playwright test",
     "format": "prettier --ignore-unknown --cache --check .",
     "format:write": "prettier --ignore-unknown --write .",
+    "dev": "ncc build ./index.ts -w -o dist/",
+    "build": "npm run clean && ncc build ./index.ts -o ./dist/ --minify --no-cache --no-source-map-register",
     "lint": "eslint . --ignore-pattern dist --ignore-pattern e2e/cache",
-    "new-snapshot": "pnpm run build && changeset version --snapshot",
-    "new-version": "pnpm run build && changeset version",
-    "pack-install": "bash ./scripts/pack.sh",
+    "e2e": "playwright test",
     "prepare": "husky",
     "release": "pnpm run build && changeset publish",
-    "release-snapshot": "pnpm run build && changeset publish --tag snapshot"
+    "new-version": "pnpm run build && changeset version"
   },
-  "dependencies": {
+  "devDependencies": {
+    "@playwright/test": "^1.41.1",
     "@types/async-retry": "1.4.2",
     "@types/ci-info": "2.0.0",
     "@types/cross-spawn": "6.0.0",
-    "@types/fs-extra": "11.0.4",
     "@types/node": "^20.11.7",
     "@types/prompts": "2.0.1",
     "@types/tar": "6.1.5",
     "@types/validate-npm-package-name": "3.0.0",
+    "@vercel/ncc": "0.38.1",
     "async-retry": "1.3.1",
     "async-sema": "3.0.1",
     "ci-info": "github:watson/ci-info#f43f6a1cefff47fb361c88cf4b943fdbcaafe540",
@@ -51,34 +48,27 @@
     "conf": "10.2.0",
     "cross-spawn": "7.0.3",
     "fast-glob": "3.3.1",
-    "fs-extra": "11.2.0",
     "got": "10.7.0",
-    "ollama": "^0.5.0",
-    "ora": "^8.0.1",
     "picocolors": "1.0.0",
     "prompts": "2.1.0",
+    "rimraf": "^5.0.5",
     "smol-toml": "^1.1.4",
     "tar": "6.1.15",
     "terminal-link": "^3.0.0",
     "update-check": "1.5.4",
     "validate-npm-package-name": "3.0.0",
-    "yaml": "2.4.1"
-  },
-  "devDependencies": {
+    "wait-port": "^1.1.0",
     "@changesets/cli": "^2.27.1",
-    "@playwright/test": "^1.41.1",
-    "@vercel/ncc": "0.38.1",
     "eslint": "^8.56.0",
-    "eslint-config-prettier": "^8.10.0",
     "husky": "^9.0.10",
     "prettier": "^3.2.5",
     "prettier-plugin-organize-imports": "^3.2.4",
-    "rimraf": "^5.0.5",
     "typescript": "^5.3.3",
-    "wait-port": "^1.1.0"
+    "eslint-config-prettier": "^8.10.0",
+    "ora": "^8.0.1"
   },
-  "packageManager": "pnpm@9.0.5",
   "engines": {
     "node": ">=16.14.0"
-  }
+  },
+  "packageManager": "pnpm@8.15.1"
 }

questions.ts

@@ -1,28 +1,33 @@
 import { execSync } from "child_process";
 import ciInfo from "ci-info";
 import fs from "fs";
+import got from "got";
+import ora from "ora";
 import path from "path";
 import { blue, green, red } from "picocolors";
 import prompts from "prompts";
 import { InstallAppArgs } from "./create-app";
 import {
+  FileSourceConfig,
   TemplateDataSource,
   TemplateDataSourceType,
   TemplateFramework,
 } from "./helpers";
 import { COMMUNITY_OWNER, COMMUNITY_REPO } from "./helpers/constant";
-import { EXAMPLE_FILE } from "./helpers/datasources";
 import { templatesDir } from "./helpers/dir";
 import { getAvailableLlamapackOptions } from "./helpers/llama-pack";
-import { askModelConfig, isModelConfigured } from "./helpers/providers";
 import { getProjectOptions } from "./helpers/repo";
 import { supportedTools, toolsRequireConfig } from "./helpers/tools";
 
+const OPENAI_API_URL = "https://api.openai.com/v1";
+
 export type QuestionArgs = Omit<
   InstallAppArgs,
   "appPath" | "packageManager"
 > & {
-  askModels?: boolean;
+  files?: string;
+  llamaParse?: boolean;
+  listServerModels?: boolean;
 };
 const supportedContextFileTypes = [
   ".pdf",
@@ -57,20 +62,26 @@ if ($result -eq 'OK') {
 const WINDOWS_FOLDER_SELECTION_SCRIPT = `
 Add-Type -AssemblyName System.windows.forms
 $folderBrowser = New-Object System.Windows.Forms.FolderBrowserDialog
-$dialogResult = $folderBrowser.ShowDialog()
-if ($dialogResult -eq [System.Windows.Forms.DialogResult]::OK)
-{
-    $folderBrowser.SelectedPath
-}
+$folderBrowser.SelectedPath = [Environment]::GetFolderPath('Desktop')
+$folderBrowser.Description = "Please select folders to process:"
+$folderBrowser.ShowNewFolderButton = $true
+$folderBrowser.RootFolder = [System.Environment+SpecialFolder]::Desktop
+$folderBrowser.SelectedPath = [System.IO.Path]::GetFullPath($folderBrowser.SelectedPath)
+$folderBrowser.ShowDialog() | Out-Null
+$folderBrowser.SelectedPath, $folderBrowser.SelectedPaths
 `;
 
-const defaults: Omit<QuestionArgs, "modelConfig"> = {
+const defaults: QuestionArgs = {
   template: "streaming",
   framework: "nextjs",
-  ui: "shadcn",
+  engine: "simple",
+  ui: "html",
+  eslint: true,
   frontend: false,
+  openAiKey: "",
   llamaCloudKey: "",
-  useLlamaParse: false,
+  model: "gpt-3.5-turbo",
+  embeddingModel: "text-embedding-ada-002",
   communityProjectConfig: undefined,
   llamapack: "",
   postInstallAction: "dependencies",
@@ -78,7 +89,7 @@ const defaults: Omit<QuestionArgs, "modelConfig"> = {
   tools: [],
 };
 
-export const questionHandlers = {
+const handlers = {
   onCancel: () => {
     console.error("Exiting.");
     process.exit(1);
@@ -95,8 +106,6 @@ const getVectorDbChoices = (framework: TemplateFramework) => {
     { title: "PostgreSQL", value: "pg" },
     { title: "Pinecone", value: "pinecone" },
     { title: "Milvus", value: "milvus" },
-    { title: "Astra", value: "astra" },
-    { title: "Qdrant", value: "qdrant" },
   ];
 
   const vectordbLang = framework === "fastapi" ? "python" : "typescript";
@@ -122,12 +131,12 @@ export const getDataSourceChoices = (
   if (selectedDataSource.length > 0) {
     choices.push({
       title: "No",
-      value: "no",
+      value: "none",
     });
   }
   if (selectedDataSource === undefined || selectedDataSource.length === 0) {
     choices.push({
-      title: "No data, just a simple chat or agent",
+      title: "No data, just a simple chat",
       value: "none",
     });
     choices.push({
@@ -136,30 +145,31 @@ export const getDataSourceChoices = (
     });
   }
 
-  choices.push(
-    {
+  if (!selectedDataSource.some((ds) => ds.type === "file")) {
+    choices.push({
       title: `Use local files (${supportedContextFileTypes.join(", ")})`,
       value: "file",
-    },
-    {
-      title:
-        process.platform === "win32"
-          ? "Use a local folder"
-          : "Use local folders",
-      value: "folder",
-    },
-  );
+    });
+  }
 
-  if (framework === "fastapi") {
+  if (!selectedDataSource.some((ds) => ds.type === "folder")) {
+    choices.push({
+      title: "Use local folder",
+      value: "folder",
+    });
+  }
+
+  if (
+    !selectedDataSource.some((ds) => ds.type === "web") &&
+    (process.platform === "win32" || process.platform === "darwin") &&
+    framework === "fastapi"
+  ) {
     choices.push({
       title: "Use website content (requires Chrome)",
       value: "web",
     });
-    choices.push({
-      title: "Use data from a database (Mysql, PostgreSQL)",
-      value: "db",
-    });
   }
+
   return choices;
 };
 
@@ -191,10 +201,9 @@ const selectLocalContextData = async (type: TemplateDataSourceType) => {
       process.platform === "win32"
         ? selectedPath.split("\r\n")
         : selectedPath.split(", ");
-
     for (const p of paths) {
       if (
-        fs.statSync(p).isFile() &&
+        type == "file" &&
         !supportedContextFileTypes.includes(path.extname(p))
       ) {
         console.log(
@@ -226,15 +235,85 @@ export const onPromptState = (state: any) => {
   }
 };
 
+const getAvailableModelChoices = async (
+  selectEmbedding: boolean,
+  apiKey?: string,
+  listServerModels?: boolean,
+) => {
+  const defaultLLMModels = [
+    "gpt-3.5-turbo-0125",
+    "gpt-4-turbo-preview",
+    "gpt-4",
+    "gpt-4-vision-preview",
+  ];
+  const defaultEmbeddingModels = [
+    "text-embedding-ada-002",
+    "text-embedding-3-small",
+    "text-embedding-3-large",
+  ];
+
+  const isLLMModels = (model_id: string) => {
+    return model_id.startsWith("gpt");
+  };
+
+  const isEmbeddingModel = (model_id: string) => {
+    return (
+      model_id.includes("embedding") ||
+      defaultEmbeddingModels.includes(model_id)
+    );
+  };
+
+  if (apiKey && listServerModels) {
+    const spinner = ora("Fetching available models").start();
+    try {
+      const response = await got(`${OPENAI_API_URL}/models`, {
+        headers: {
+          Authorization: "Bearer " + apiKey,
+        },
+        timeout: 5000,
+        responseType: "json",
+      });
+      const data: any = await response.body;
+      spinner.stop();
+      return data.data
+        .filter((model: any) =>
+          selectEmbedding ? isEmbeddingModel(model.id) : isLLMModels(model.id),
+        )
+        .map((el: any) => {
+          return {
+            title: el.id,
+            value: el.id,
+          };
+        });
+    } catch (error) {
+      spinner.stop();
+      if ((error as any).response?.statusCode === 401) {
+        console.log(
+          red(
+            "Invalid OpenAI API key provided! Please provide a valid key and try again!",
+          ),
+        );
+      } else {
+        console.log(red("Request failed: " + error));
+      }
+      process.exit(1);
+    }
+  } else {
+    const data = selectEmbedding ? defaultEmbeddingModels : defaultLLMModels;
+    return data.map((model) => ({
+      title: model,
+      value: model,
+    }));
+  }
+};
+
 export const askQuestions = async (
   program: QuestionArgs,
   preferences: QuestionArgs,
-  openAiKey?: string,
 ) => {
-  const getPrefOrDefault = <K extends keyof Omit<QuestionArgs, "modelConfig">>(
+  const getPrefOrDefault = <K extends keyof QuestionArgs>(
     field: K,
-  ): Omit<QuestionArgs, "modelConfig">[K] =>
-    preferences[field] ?? defaults[field];
+  ): QuestionArgs[K] => preferences[field] ?? defaults[field];
 
   // Ask for next action after installation
   async function askPostInstallAction() {
@@ -257,16 +336,21 @@ export const askQuestions = async (
           },
         ];
 
-        const modelConfigured = isModelConfigured(program.modelConfig);
+        const openAiKeyConfigured =
+          program.openAiKey || process.env["OPENAI_API_KEY"];
         // If using LlamaParse, require LlamaCloud API key
-        const llamaCloudKeyConfigured = program.useLlamaParse
+        const useLlamaParse = program.dataSources.some(
+          (ds) =>
+            ds.type === "file" && (ds.config as FileSourceConfig).useLlamaParse,
+        );
+        const llamaCloudKeyConfigured = useLlamaParse
           ? program.llamaCloudKey || process.env["LLAMA_CLOUD_API_KEY"]
           : true;
         const hasVectorDb = program.vectorDb && program.vectorDb !== "none";
         // Can run the app if all tools do not require configuration
         if (
           !hasVectorDb &&
-          modelConfigured &&
+          openAiKeyConfigured &&
           llamaCloudKeyConfigured &&
           !toolsRequireConfig(program.tools) &&
           !program.llamapack
@@ -286,7 +370,7 @@ export const askQuestions = async (
             choices: actionChoices,
             initial: 1,
           },
-          questionHandlers,
+          handlers,
         );
 
         program.postInstallAction = action;
@@ -307,7 +391,8 @@ export const askQuestions = async (
           name: "template",
           message: "Which template would you like to use?",
           choices: [
-            { title: "Chat", value: "streaming" },
+            { title: "Chat without streaming", value: "simple" },
+            { title: "Chat with streaming", value: "streaming" },
             {
               title: `Community template from ${styledRepo}`,
               value: "community",
@@ -317,9 +402,9 @@ export const askQuestions = async (
               value: "llamapack",
             },
           ],
-          initial: 0,
+          initial: 1,
         },
-        questionHandlers,
+        handlers,
       );
       program.template = template;
       preferences.template = template;
@@ -342,7 +427,7 @@ export const askQuestions = async (
         })),
         initial: 0,
       },
-      questionHandlers,
+      handlers,
     );
     const projectConfig = JSON.parse(communityProjectConfig);
     program.communityProjectConfig = projectConfig;
@@ -363,7 +448,7 @@ export const askQuestions = async (
         })),
         initial: 0,
       },
-      questionHandlers,
+      handlers,
     );
     program.llamapack = llamapack;
     preferences.llamapack = llamapack;
@@ -376,10 +461,13 @@ export const askQuestions = async (
       program.framework = getPrefOrDefault("framework");
     } else {
       const choices = [
-        { title: "NextJS", value: "nextjs" },
         { title: "Express", value: "express" },
         { title: "FastAPI (Python)", value: "fastapi" },
       ];
+      if (program.template === "streaming") {
+        // allow NextJS only for streaming template
+        choices.unshift({ title: "NextJS", value: "nextjs" });
+      }
 
       const { framework } = await prompts(
         {
@@ -389,14 +477,17 @@ export const askQuestions = async (
           choices,
           initial: 0,
         },
-        questionHandlers,
+        handlers,
       );
       program.framework = framework;
       preferences.framework = framework;
     }
   }
 
-  if (program.framework === "express" || program.framework === "fastapi") {
+  if (
+    program.template === "streaming" &&
+    (program.framework === "express" || program.framework === "fastapi")
+  ) {
     // if a backend-only framework is selected, ask whether we should create a frontend
     // (only for streaming backends)
     if (program.frontend === undefined) {
@@ -430,185 +521,272 @@ export const askQuestions = async (
 
   if (program.framework === "nextjs" || program.frontend) {
     if (!program.ui) {
-      program.ui = defaults.ui;
-    }
-  }
-
-  if (!program.observability) {
-    if (ciInfo.isCI) {
-      program.observability = getPrefOrDefault("observability");
-    } else {
-      const { observability } = await prompts(
-        {
-          type: "select",
-          name: "observability",
-          message: "Would you like to set up observability?",
-          choices: [
-            { title: "No", value: "none" },
-            { title: "OpenTelemetry", value: "opentelemetry" },
-          ],
-          initial: 0,
-        },
-        questionHandlers,
-      );
-
-      program.observability = observability;
-      preferences.observability = observability;
-    }
-  }
-
-  if (!program.modelConfig) {
-    const modelConfig = await askModelConfig({
-      openAiKey,
-      askModels: program.askModels ?? false,
-    });
-    program.modelConfig = modelConfig;
-    preferences.modelConfig = modelConfig;
-  }
-
-  if (!program.dataSources) {
-    if (ciInfo.isCI) {
-      program.dataSources = getPrefOrDefault("dataSources");
-    } else {
-      program.dataSources = [];
-      // continue asking user for data sources if none are initially provided
-      while (true) {
-        const firstQuestion = program.dataSources.length === 0;
-        const { selectedSource } = await prompts(
+      if (ciInfo.isCI) {
+        program.ui = getPrefOrDefault("ui");
+      } else {
+        const { ui } = await prompts(
           {
             type: "select",
-            name: "selectedSource",
-            message: firstQuestion
-              ? "Which data source would you like to use?"
-              : "Would you like to add another data source?",
-            choices: getDataSourceChoices(
-              program.framework,
-              program.dataSources,
-            ),
-            initial: firstQuestion ? 1 : 0,
+            name: "ui",
+            message: "Which UI would you like to use?",
+            choices: [
+              { title: "Just HTML", value: "html" },
+              { title: "Shadcn", value: "shadcn" },
+            ],
+            initial: 0,
           },
-          questionHandlers,
+          handlers,
         );
-
-        if (selectedSource === "no" || selectedSource === "none") {
-          // user doesn't want another data source or any data source
-          break;
-        }
-        switch (selectedSource) {
-          case "exampleFile": {
-            program.dataSources.push(EXAMPLE_FILE);
-            break;
-          }
-          case "file":
-          case "folder": {
-            const selectedPaths = await selectLocalContextData(selectedSource);
-            for (const p of selectedPaths) {
-              program.dataSources.push({
-                type: "file",
-                config: {
-                  path: p,
-                },
-              });
-            }
-            break;
-          }
-          case "web": {
-            const { baseUrl } = await prompts(
-              {
-                type: "text",
-                name: "baseUrl",
-                message: "Please provide base URL of the website: ",
-                initial: "https://www.llamaindex.ai",
-                validate: (value: string) => {
-                  if (!value.includes("://")) {
-                    value = `https://${value}`;
-                  }
-                  const urlObj = new URL(value);
-                  if (
-                    urlObj.protocol !== "https:" &&
-                    urlObj.protocol !== "http:"
-                  ) {
-                    return `URL=${value} has invalid protocol, only allow http or https`;
-                  }
-                  return true;
-                },
-              },
-              questionHandlers,
-            );
-
-            program.dataSources.push({
-              type: "web",
-              config: {
-                baseUrl,
-                prefix: baseUrl,
-                depth: 1,
-              },
-            });
-            break;
-          }
-          case "db": {
-            const dbPrompts: prompts.PromptObject<string>[] = [
-              {
-                type: "text",
-                name: "uri",
-                message:
-                  "Please enter the connection string (URI) for the database.",
-                initial: "mysql+pymysql://user:pass@localhost:3306/mydb",
-                validate: (value: string) => {
-                  if (!value) {
-                    return "Please provide a valid connection string";
-                  } else if (
-                    !(
-                      value.startsWith("mysql+pymysql://") ||
-                      value.startsWith("postgresql+psycopg://")
-                    )
-                  ) {
-                    return "The connection string must start with 'mysql+pymysql://' for MySQL or 'postgresql+psycopg://' for PostgreSQL";
-                  }
-                  return true;
-                },
-              },
-              // Only ask for a query, user can provide more complex queries in the config file later
-              {
-                type: (prev) => (prev ? "text" : null),
-                name: "queries",
-                message: "Please enter the SQL query to fetch data:",
-                initial: "SELECT * FROM mytable",
-              },
-            ];
-            program.dataSources.push({
-              type: "db",
-              config: await prompts(dbPrompts, questionHandlers),
-            });
-          }
-        }
+        program.ui = ui;
+        preferences.ui = ui;
       }
     }
   }
 
-  // Asking for LlamaParse if user selected file or folder data source
-  if (
-    program.dataSources.some((ds) => ds.type === "file") &&
-    program.useLlamaParse === undefined
-  ) {
+  if (program.framework === "express" || program.framework === "nextjs") {
+    if (!program.observability) {
+      if (ciInfo.isCI) {
+        program.observability = getPrefOrDefault("observability");
+      } else {
+        const { observability } = await prompts(
+          {
+            type: "select",
+            name: "observability",
+            message: "Would you like to set up observability?",
+            choices: [
+              { title: "No", value: "none" },
+              { title: "OpenTelemetry", value: "opentelemetry" },
+            ],
+            initial: 0,
+          },
+          handlers,
+        );
+
+        program.observability = observability;
+        preferences.observability = observability;
+      }
+    }
+  }
+
+  if (!program.openAiKey) {
+    const { key } = await prompts(
+      {
+        type: "text",
+        name: "key",
+        message: program.listServerModels
+          ? "Please provide your OpenAI API key (or reuse OPENAI_API_KEY env variable):"
+          : "Please provide your OpenAI API key (leave blank to skip):",
+        validate: (value: string) => {
+          if (program.listServerModels && !value) {
+            if (process.env.OPENAI_API_KEY) {
+              return true;
+            }
+            return "OpenAI API key is required";
+          }
+          return true;
+        },
+      },
+      handlers,
+    );
+
+    program.openAiKey = key || process.env.OPENAI_API_KEY;
+    preferences.openAiKey = key || process.env.OPENAI_API_KEY;
+  }
+
+  if (!program.model) {
     if (ciInfo.isCI) {
-      program.useLlamaParse = getPrefOrDefault("useLlamaParse");
-      program.llamaCloudKey = getPrefOrDefault("llamaCloudKey");
+      program.model = getPrefOrDefault("model");
     } else {
+      const { model } = await prompts(
+        {
+          type: "select",
+          name: "model",
+          message: "Which model would you like to use?",
+          choices: await getAvailableModelChoices(
+            false,
+            program.openAiKey,
+            program.listServerModels,
+          ),
+          initial: 0,
+        },
+        handlers,
+      );
+      program.model = model;
+      preferences.model = model;
+    }
+  }
+
+  if (!program.embeddingModel && program.framework === "fastapi") {
+    if (ciInfo.isCI) {
+      program.embeddingModel = getPrefOrDefault("embeddingModel");
+    } else {
+      const { embeddingModel } = await prompts(
+        {
+          type: "select",
+          name: "embeddingModel",
+          message: "Which embedding model would you like to use?",
+          choices: await getAvailableModelChoices(
+            true,
+            program.openAiKey,
+            program.listServerModels,
+          ),
+          initial: 0,
+        },
+        handlers,
+      );
+      program.embeddingModel = embeddingModel;
+      preferences.embeddingModel = embeddingModel;
+    }
+  }
+
+  if (program.files) {
+    // If user specified files option, then the program should use context engine
+    program.engine == "context";
+    if (!fs.existsSync(program.files)) {
+      console.log("File or folder not found");
+      process.exit(1);
+    } else {
+      program.dataSources = [
+        {
+          type: fs.lstatSync(program.files).isDirectory() ? "folder" : "file",
+          config: {
+            paths: program.files.split(","),
+          },
+        },
+      ];
+    }
+  }
+
+  // Asking for data source
+  if (!program.engine) {
+    program.dataSources = getPrefOrDefault("dataSources");
+    if (ciInfo.isCI) {
+      program.engine = getPrefOrDefault("engine");
+    } else {
+      for (let i = 0; i < 2; i++) {
+        const { selectedSource } = await prompts(
+          {
+            type: "select",
+            name: "selectedSource",
+            message:
+              i === 0
+                ? "Which data source would you like to use?"
+                : "Would you like to add another data source?",
+            choices: getDataSourceChoices(
+              program.framework,
+              program.dataSources,
+            ),
+            initial: 0,
+          },
+          handlers,
+        );
+
+        // Asking for data source config
+        // Select None data source, No need to config and asking for another data source
+        if (selectedSource === "none") {
+          if (selectedSource.length === 0) {
+            program.dataSources = [
+              {
+                type: "none",
+                config: {},
+              },
+            ];
+          }
+          break;
+        }
+
+        const dataSource = {
+          type: selectedSource === "exampleFile" ? "folder" : selectedSource,
+          config: {},
+        };
+
+        // Select local file or folder
+        if (selectedSource === "file" || selectedSource === "folder") {
+          const selectedPaths = await selectLocalContextData(selectedSource);
+          dataSource.config = {
+            paths: selectedPaths,
+          };
+        }
+
+        // Selected web data source
+        else if (selectedSource === "web") {
+          let { baseUrl } = await prompts(
+            {
+              type: "text",
+              name: "baseUrl",
+              message: "Please provide base URL of the website:",
+              initial: "https://www.llamaindex.ai",
+            },
+            handlers,
+          );
+          try {
+            if (!baseUrl.includes("://")) {
+              baseUrl = `https://${baseUrl}`;
+            }
+            const checkUrl = new URL(baseUrl);
+            if (
+              checkUrl.protocol !== "https:" &&
+              checkUrl.protocol !== "http:"
+            ) {
+              throw new Error("Invalid protocol");
+            }
+          } catch (error) {
+            console.log(
+              red(
+                "Invalid URL provided! Please provide a valid URL (e.g. https://www.llamaindex.ai)",
+              ),
+            );
+            process.exit(1);
+          }
+
+          dataSource.config = {
+            baseUrl: baseUrl,
+            depth: 1,
+          };
+        }
+        program.dataSources.push(dataSource);
+
+        // No need to ask for another data source if user selected example data
+        if (selectedSource === "exampleFile") {
+          break;
+        }
+      }
+
+      if (
+        program.dataSources.length === 0 ||
+        program.dataSources[0].type === "none"
+      ) {
+        program.engine = "simple";
+      } else {
+        program.engine = "context";
+      }
+    }
+  }
+
+  // Asking for LlamaParse
+  // Is user selected pdf file or is there a folder data source
+  if (!program.llamaParse && program.engine === "context") {
+    const askingLlamaParse = program.dataSources.some(
+      (ds) =>
+        ds.type === "folder" ||
+        (ds.type === "file" &&
+          (ds.config as FileSourceConfig).paths?.some(
+            (p) => path.extname(p) === ".pdf",
+          )),
+    );
+    if (askingLlamaParse) {
       const { useLlamaParse } = await prompts(
         {
           type: "toggle",
           name: "useLlamaParse",
           message:
             "Would you like to use LlamaParse (improved parser for RAG - requires API key)?",
-          initial: false,
+          initial: true,
           active: "yes",
           inactive: "no",
         },
-        questionHandlers,
+        handlers,
       );
-      program.useLlamaParse = useLlamaParse;
-
       // Ask for LlamaCloud API key
       if (useLlamaParse && program.llamaCloudKey === undefined) {
         const { llamaCloudKey } = await prompts(
@@ -618,14 +796,20 @@ export const askQuestions = async (
             message:
               "Please provide your LlamaIndex Cloud API key (leave blank to skip):",
           },
-          questionHandlers,
+          handlers,
         );
         program.llamaCloudKey = llamaCloudKey;
       }
+      // TODO: Consider separate llamaParse to another config
+      program.dataSources.forEach((dataSource) => {
+        if (dataSource.type === "file" || dataSource.type === "folder") {
+          (dataSource.config as FileSourceConfig).useLlamaParse = useLlamaParse;
+        }
+      });
     }
   }
 
-  if (program.dataSources.length > 0 && !program.vectorDb) {
+  if (program.engine !== "simple" && !program.vectorDb) {
     if (ciInfo.isCI) {
       program.vectorDb = getPrefOrDefault("vectorDb");
     } else {
@@ -637,21 +821,22 @@ export const askQuestions = async (
           choices: getVectorDbChoices(program.framework),
           initial: 0,
         },
-        questionHandlers,
+        handlers,
       );
       program.vectorDb = vectorDb;
       preferences.vectorDb = vectorDb;
     }
   }
 
-  if (!program.tools) {
+  if (
+    !program.tools &&
+    program.framework === "fastapi" &&
+    program.engine === "context"
+  ) {
     if (ciInfo.isCI) {
       program.tools = getPrefOrDefault("tools");
     } else {
-      const options = supportedTools.filter((t) =>
-        t.supportedFrameworks?.includes(program.framework),
-      );
-      const toolChoices = options.map((tool) => ({
+      const toolChoices = supportedTools.map((tool) => ({
         title: tool.display,
         value: tool.name,
       }));
@@ -670,9 +855,30 @@ export const askQuestions = async (
     }
   }
 
-  await askPostInstallAction();
-};
+  if (program.framework !== "fastapi" && program.eslint === undefined) {
+    if (ciInfo.isCI) {
+      program.eslint = getPrefOrDefault("eslint");
+    } else {
+      const styledEslint = blue("ESLint");
+      const { eslint } = await prompts({
+        onState: onPromptState,
+        type: "toggle",
+        name: "eslint",
+        message: `Would you like to use ${styledEslint}?`,
+        initial: getPrefOrDefault("eslint"),
+        active: "Yes",
+        inactive: "No",
+      });
+      program.eslint = Boolean(eslint);
+      preferences.eslint = Boolean(eslint);
+    }
+  }
 
-export const toChoice = (value: string) => {
-  return { title: value, value };
+  await askPostInstallAction();
+
+  // TODO: consider using zod to validate the input (doesn't work like this as not every option is required)
+  // templateUISchema.parse(program.ui);
+  // templateEngineSchema.parse(program.engine);
+  // templateFrameworkSchema.parse(program.framework);
+  // templateTypeSchema.parse(program.template);``
 };

scripts/build.sh

@@ -1,12 +0,0 @@
-#!/usr/bin/env bash
-
-# build dist/index.js file
-pnpm run build:ncc
-
-# add shebang to the top of dist/index.js
-# XXX: Windows needs a space after `node` to work correctly
-# Note: ncc can handle shebang but it didn't work with Windows in our tests
-echo '#!/usr/bin/env node ' | cat - dist/index.js >temp && mv temp dist/index.js
-
-# make dist/index.js executable
-chmod +x dist/index.js

scripts/pack.sh

@@ -1,3 +0,0 @@
-#!/usr/bin/env bash
-
-pnpm pack && npm install -g $(pwd)/$(ls ./*.tgz | head -1)

templates/components/deployments/python/Dockerfile

@@ -1,26 +0,0 @@
-FROM python:3.11 as build
-
-WORKDIR /app
-
-ENV PYTHONPATH=/app
-
-# Install Poetry
-RUN curl -sSL https://install.python-poetry.org | POETRY_HOME=/opt/poetry python && \
-    cd /usr/local/bin && \
-    ln -s /opt/poetry/bin/poetry && \
-    poetry config virtualenvs.create false
-
-# Install Chromium for web loader
-# Can disable this if you don't use the web loader to reduce the image size
-RUN apt update && apt install -y chromium chromium-driver
-
-# Install dependencies
-COPY ./pyproject.toml ./poetry.lock* /app/
-RUN poetry install --no-root --no-cache --only main
-
-# ====================================
-FROM build as release
-
-COPY . .
-
-CMD ["python", "main.py"]

templates/components/deployments/typescript/Dockerfile

@@ -1,16 +0,0 @@
-FROM node:20-alpine as build
-
-WORKDIR /app
-
-# Install dependencies
-COPY package.json package-lock.* ./
-RUN npm install
-
-# Build the application
-COPY . .
-RUN npm run build
-
-# ====================================
-FROM build as release
-
-CMD ["npm", "run", "start"]

templates/components/engines/python/agent/__init__.py

@@ -11,12 +11,11 @@ def get_chat_engine():
     top_k = os.getenv("TOP_K", "3")
     tools = []
 
-    # Add query tool if index exists
+    # Add query tool
     index = get_index()
-    if index is not None:
-        query_engine = index.as_query_engine(similarity_top_k=int(top_k))
-        query_engine_tool = QueryEngineTool.from_defaults(query_engine=query_engine)
-        tools.append(query_engine_tool)
+    query_engine = index.as_query_engine(similarity_top_k=int(top_k))
+    query_engine_tool = QueryEngineTool.from_defaults(query_engine=query_engine)
+    tools.append(query_engine_tool)
 
     # Add additional tools
     tools += ToolFactory.from_env()

templates/components/engines/python/agent/tools.py

@@ -1,5 +1,4 @@
-import os
-import yaml
+import json
 import importlib
 
 from llama_index.core.tools.tool_spec.base import BaseToolSpec
@@ -27,9 +26,8 @@ class ToolFactory:
     @staticmethod
     def from_env() -> list[FunctionTool]:
         tools = []
-        if os.path.exists("config/tools.yaml"):
-            with open("config/tools.yaml", "r") as f:
-                tool_configs = yaml.safe_load(f)
-                for name, config in tool_configs.items():
-                    tools += ToolFactory.create_tool(name, **config)
+        with open("tools_config.json", "r") as f:
+            tool_configs = json.load(f)
+            for name, config in tool_configs.items():
+                tools += ToolFactory.create_tool(name, **config)
         return tools

templates/components/engines/python/chat/__init__.py

@@ -1,22 +1,12 @@
 import os
 from app.engine.index import get_index
-from fastapi import HTTPException
 
 
 def get_chat_engine():
     system_prompt = os.getenv("SYSTEM_PROMPT")
     top_k = os.getenv("TOP_K", 3)
 
-    index = get_index()
-    if index is None:
-        raise HTTPException(
-            status_code=500,
-            detail=str(
-                "StorageContext is empty - call 'poetry run generate' to generate the storage first"
-            ),
-        )
-
-    return index.as_chat_engine(
+    return get_index().as_chat_engine(
         similarity_top_k=int(top_k),
         system_prompt=system_prompt,
         chat_mode="condense_plus_context",

templates/components/engines/typescript/agent/chat.ts

@@ -1,37 +0,0 @@
-import { BaseTool, OpenAIAgent, QueryEngineTool } from "llamaindex";
-import { ToolsFactory } from "llamaindex/tools/ToolsFactory";
-import fs from "node:fs/promises";
-import path from "node:path";
-import { getDataSource } from "./index";
-import { STORAGE_CACHE_DIR } from "./shared";
-
-export async function createChatEngine() {
-  let tools: BaseTool[] = [];
-
-  // Add a query engine tool if we have a data source
-  // Delete this code if you don't have a data source
-  const index = await getDataSource();
-  if (index) {
-    tools.push(
-      new QueryEngineTool({
-        queryEngine: index.asQueryEngine(),
-        metadata: {
-          name: "data_query_engine",
-          description: `A query engine for documents in storage folder: ${STORAGE_CACHE_DIR}`,
-        },
-      }),
-    );
-  }
-
-  try {
-    // add tools from config file if it exists
-    const config = JSON.parse(
-      await fs.readFile(path.join("config", "tools.json"), "utf8"),
-    );
-    tools = tools.concat(await ToolsFactory.createTools(config));
-  } catch {}
-
-  return new OpenAIAgent({
-    tools,
-  });
-}

templates/components/engines/typescript/chat/chat.ts

@@ -1,20 +0,0 @@
-import { ContextChatEngine, Settings } from "llamaindex";
-import { getDataSource } from "./index";
-
-export async function createChatEngine() {
-  const index = await getDataSource();
-  if (!index) {
-    throw new Error(
-      `StorageContext is empty - call 'npm run generate' to generate the storage first`,
-    );
-  }
-  const retriever = index.asRetriever();
-  retriever.similarityTopK = process.env.TOP_K
-    ? parseInt(process.env.TOP_K)
-    : 3;
-
-  return new ContextChatEngine({
-    chatModel: Settings.llm,
-    retriever,
-  });
-}

templates/components/loaders/python/__init__.py

@@ -1,39 +0,0 @@
-import os
-import yaml
-import importlib
-import logging
-from typing import Dict
-from app.engine.loaders.file import FileLoaderConfig, get_file_documents
-from app.engine.loaders.web import WebLoaderConfig, get_web_documents
-from app.engine.loaders.db import DBLoaderConfig, get_db_documents
-
-logger = logging.getLogger(__name__)
-
-
-def load_configs():
-    with open("config/loaders.yaml") as f:
-        configs = yaml.safe_load(f)
-    return configs
-
-
-def get_documents():
-    documents = []
-    config = load_configs()
-    for loader_type, loader_config in config.items():
-        logger.info(
-            f"Loading documents from loader: {loader_type}, config: {loader_config}"
-        )
-        match loader_type:
-            case "file":
-                document = get_file_documents(FileLoaderConfig(**loader_config))
-            case "web":
-                document = get_web_documents(WebLoaderConfig(**loader_config))
-            case "db":
-                document = get_db_documents(
-                    configs=[DBLoaderConfig(**cfg) for cfg in loader_config]
-                )
-            case _:
-                raise ValueError(f"Invalid loader type: {loader_type}")
-        documents.extend(document)
-
-    return documents

templates/components/loaders/python/db.py

@@ -1,26 +0,0 @@
-import os
-import logging
-from typing import List
-from pydantic import BaseModel, validator
-from llama_index.core.indices.vector_store import VectorStoreIndex
-
-logger = logging.getLogger(__name__)
-
-
-class DBLoaderConfig(BaseModel):
-    uri: str
-    queries: List[str]
-
-
-def get_db_documents(configs: list[DBLoaderConfig]):
-    from llama_index.readers.database import DatabaseReader
-
-    docs = []
-    for entry in configs:
-        loader = DatabaseReader(uri=entry.uri)
-        for query in entry.queries:
-            logger.info(f"Loading data from database with query: {query}")
-            documents = loader.load_data(query=query)
-            docs.extend(documents)
-
-    return documents

templates/components/loaders/python/file.py

@@ -1,34 +0,0 @@
-import os
-from llama_parse import LlamaParse
-from pydantic import BaseModel, validator
-
-
-class FileLoaderConfig(BaseModel):
-    data_dir: str = "data"
-    use_llama_parse: bool = False
-
-    @validator("data_dir")
-    def data_dir_must_exist(cls, v):
-        if not os.path.isdir(v):
-            raise ValueError(f"Directory '{v}' does not exist")
-        return v
-
-
-def llama_parse_parser():
-    if os.getenv("LLAMA_CLOUD_API_KEY") is None:
-        raise ValueError(
-            "LLAMA_CLOUD_API_KEY environment variable is not set. "
-            "Please set it in .env file or in your shell environment then run again!"
-        )
-    parser = LlamaParse(result_type="markdown", verbose=True, language="en")
-    return parser
-
-
-def get_file_documents(config: FileLoaderConfig):
-    from llama_index.core.readers import SimpleDirectoryReader
-
-    reader = SimpleDirectoryReader(config.data_dir, recursive=True, filename_as_id=True)
-    if config.use_llama_parse:
-        parser = llama_parse_parser()
-        reader.file_extractor = {".pdf": parser}
-    return reader.load_data()

templates/components/loaders/python/file/file.py

@@ -0,0 +1,7 @@
+from llama_index.core.readers import SimpleDirectoryReader
+
+DATA_DIR = "data"  # directory containing the documents
+
+
+def get_documents():
+    return SimpleDirectoryReader(DATA_DIR).load_data()

templates/components/loaders/python/llama_parse/file.py

@@ -0,0 +1,17 @@
+import os
+from llama_parse import LlamaParse
+from llama_index.core import SimpleDirectoryReader
+
+DATA_DIR = "data"  # directory containing the documents
+
+
+def get_documents():
+    if os.getenv("LLAMA_CLOUD_API_KEY") is None:
+        raise ValueError(
+            "LLAMA_CLOUD_API_KEY environment variable is not set. "
+            "Please set it in .env file or in your shell environment then run again!"
+        )
+    parser = LlamaParse(result_type="markdown", verbose=True, language="en")
+
+    reader = SimpleDirectoryReader(DATA_DIR, file_extractor={".pdf": parser})
+    return reader.load_data()

templates/components/loaders/python/loader.py

@@ -0,0 +1,12 @@
+import os
+import importlib
+
+
+def get_documents():
+    # For each file in .loaders, import the module and call the get_documents function
+    for loader in os.listdir(os.path.join(os.path.dirname(__file__), "loaders")):
+        if loader.endswith(".py"):
+            loader = loader[:-3]
+            module = importlib.import_module(f"app.engine.loaders.{loader}")
+            documents = module.get_documents()
+            yield documents

templates/components/loaders/python/web.py

@@ -1,36 +0,0 @@
-import os
-import json
-from pydantic import BaseModel, Field
-
-
-class CrawlUrl(BaseModel):
-    base_url: str
-    prefix: str
-    max_depth: int = Field(default=1, ge=0)
-
-
-class WebLoaderConfig(BaseModel):
-    driver_arguments: list[str] = Field(default=None)
-    urls: list[CrawlUrl]
-
-
-def get_web_documents(config: WebLoaderConfig):
-    from llama_index.readers.web import WholeSiteReader
-    from selenium import webdriver
-    from selenium.webdriver.chrome.options import Options
-
-    options = Options()
-    driver_arguments = config.driver_arguments or []
-    for arg in driver_arguments:
-        options.add_argument(arg)
-
-    docs = []
-    for url in config.urls:
-        scraper = WholeSiteReader(
-            prefix=url.prefix,
-            max_depth=url.max_depth,
-            driver=webdriver.Chrome(options=options),
-        )
-        docs.extend(scraper.load_data(url.base_url))
-
-    return docs

templates/components/loaders/python/web/web.py

@@ -0,0 +1,13 @@
+import os
+from llama_index.readers.web import WholeSiteReader
+
+
+def get_documents():
+    # Initialize the scraper with a prefix URL and maximum depth
+    scraper = WholeSiteReader(
+        prefix=os.environ.get("URL_PREFIX"), max_depth=int(os.environ.get("MAX_DEPTH"))
+    )
+    # Start scraping from a base URL
+    documents = scraper.load_data(base_url=os.environ.get("BASE_URL"))
+
+    return documents

templates/components/loaders/typescript/file/loader.ts

@@ -1,9 +0,0 @@
-import { SimpleDirectoryReader } from "llamaindex";
-
-export const DATA_DIR = "./data";
-
-export async function getDocuments() {
-  return await new SimpleDirectoryReader().loadData({
-    directoryPath: DATA_DIR,
-  });
-}

templates/components/loaders/typescript/llama_parse/loader.ts

@@ -1,19 +0,0 @@
-import {
-  FILE_EXT_TO_READER,
-  LlamaParseReader,
-  SimpleDirectoryReader,
-} from "llamaindex";
-
-export const DATA_DIR = "./data";
-
-export async function getDocuments() {
-  const reader = new SimpleDirectoryReader();
-  // Load PDFs using LlamaParseReader
-  return await reader.loadData({
-    directoryPath: DATA_DIR,
-    fileExtToReader: {
-      ...FILE_EXT_TO_READER,
-      pdf: new LlamaParseReader({ resultType: "markdown" }),
-    },
-  });
-}

templates/components/observability/python/opentelemetry/observability.py

@@ -1,5 +0,0 @@
-from traceloop.sdk import Traceloop
-
-
-def init_observability():
-    Traceloop.init()

templates/components/ui/html/chat/chat-item.tsx

@@ -7,7 +7,7 @@ export default function ChatItem(message: Message) {
   return (
     <div className="flex items-start gap-4 pt-5">
       <ChatAvatar {...message} />
-      <p className="break-words whitespace-pre-wrap">{message.content}</p>
+      <p className="break-words">{message.content}</p>
     </div>
   );
 }

templates/components/vectordbs/python/astra/vectordb.py

@@ -1,12 +0,0 @@
-import os
-from llama_index.vector_stores.astra_db import AstraDBVectorStore
-
-
-def get_vector_store():
-    store = AstraDBVectorStore(
-        token=os.environ["ASTRA_DB_APPLICATION_TOKEN"],
-        api_endpoint=os.environ["ASTRA_DB_ENDPOINT"],
-        collection_name=os.environ["ASTRA_DB_COLLECTION"],
-        embedding_dimension=int(os.environ["EMBEDDING_DIM"]),
-    )
-    return store

templates/components/vectordbs/python/milvus/generate.py

@@ -0,0 +1,39 @@
+from dotenv import load_dotenv
+
+load_dotenv()
+
+import os
+import logging
+from llama_index.core.storage import StorageContext
+from llama_index.core.indices import VectorStoreIndex
+from llama_index.vector_stores.milvus import MilvusVectorStore
+from app.settings import init_settings
+from app.engine.loader import get_documents
+
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger()
+
+
+def generate_datasource():
+    logger.info("Creating new index")
+    # load the documents and create the index
+    documents = get_documents()
+    store = MilvusVectorStore(
+        uri=os.environ["MILVUS_ADDRESS"],
+        user=os.getenv("MILVUS_USER"),
+        password=os.getenv("MILVUS_PASSWORD"),
+        collection_name=os.getenv("MILVUS_COLLECTION"),
+        dim=int(os.getenv("MILVUS_DIMENSION", "1536")),
+    )
+    storage_context = StorageContext.from_defaults(vector_store=store)
+    VectorStoreIndex.from_documents(
+        documents,
+        storage_context=storage_context,
+        show_progress=True,  # this will show you a progress bar as the embeddings are created
+    )
+    logger.info(f"Successfully created embeddings in the Milvus")
+
+
+if __name__ == "__main__":
+    init_settings()
+    generate_datasource()

templates/components/vectordbs/python/milvus/index.py

@@ -0,0 +1,22 @@
+import logging
+import os
+
+from llama_index.core.indices import VectorStoreIndex
+from llama_index.vector_stores.milvus import MilvusVectorStore
+
+
+logger = logging.getLogger("uvicorn")
+
+
+def get_index():
+    logger.info("Connecting to index from Milvus...")
+    store = MilvusVectorStore(
+        uri=os.getenv("MILVUS_ADDRESS"),
+        user=os.getenv("MILVUS_USER"),
+        password=os.getenv("MILVUS_PASSWORD"),
+        collection_name=os.getenv("MILVUS_COLLECTION"),
+        dim=int(os.getenv("EMBEDDING_DIM", "1536")),
+    )
+    index = VectorStoreIndex.from_vector_store(store)
+    logger.info("Finished connecting to index from Milvus.")
+    return index

templates/components/vectordbs/python/milvus/vectordb.py

@@ -1,13 +0,0 @@
-import os
-from llama_index.vector_stores.milvus import MilvusVectorStore
-
-
-def get_vector_store():
-    store = MilvusVectorStore(
-        uri=os.environ["MILVUS_ADDRESS"],
-        user=os.getenv("MILVUS_USERNAME"),
-        password=os.getenv("MILVUS_PASSWORD"),
-        collection_name=os.getenv("MILVUS_COLLECTION"),
-        dim=int(os.getenv("EMBEDDING_DIM")),
-    )
-    return store

templates/components/vectordbs/python/mongo/generate.py

@@ -0,0 +1,43 @@
+from dotenv import load_dotenv
+
+load_dotenv()
+
+import os
+import logging
+from llama_index.core.storage import StorageContext
+from llama_index.core.indices import VectorStoreIndex
+from llama_index.vector_stores.mongodb import MongoDBAtlasVectorSearch
+from app.settings import init_settings
+from app.engine.loader import get_documents
+
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger()
+
+
+def generate_datasource():
+    logger.info("Creating new index")
+    # load the documents and create the index
+    documents = get_documents()
+    store = MongoDBAtlasVectorSearch(
+        db_name=os.environ["MONGODB_DATABASE"],
+        collection_name=os.environ["MONGODB_VECTORS"],
+        index_name=os.environ["MONGODB_VECTOR_INDEX"],
+    )
+    storage_context = StorageContext.from_defaults(vector_store=store)
+    VectorStoreIndex.from_documents(
+        documents,
+        storage_context=storage_context,
+        show_progress=True,  # this will show you a progress bar as the embeddings are created
+    )
+    logger.info(
+        f"Successfully created embeddings in the MongoDB collection {os.environ['MONGODB_VECTORS']}"
+    )
+    logger.info(
+        """IMPORTANT: You can't query your index yet because you need to create a vector search index in MongoDB's UI now.
+See https://github.com/run-llama/mongodb-demo/tree/main?tab=readme-ov-file#create-a-vector-search-index"""
+    )
+
+
+if __name__ == "__main__":
+    init_settings()
+    generate_datasource()

templates/components/vectordbs/python/mongo/index.py

@@ -0,0 +1,20 @@
+import logging
+import os
+
+from llama_index.core.indices import VectorStoreIndex
+from llama_index.vector_stores.mongodb import MongoDBAtlasVectorSearch
+
+
+logger = logging.getLogger("uvicorn")
+
+
+def get_index():
+    logger.info("Connecting to index from MongoDB...")
+    store = MongoDBAtlasVectorSearch(
+        db_name=os.environ["MONGODB_DATABASE"],
+        collection_name=os.environ["MONGODB_VECTORS"],
+        index_name=os.environ["MONGODB_VECTOR_INDEX"],
+    )
+    index = VectorStoreIndex.from_vector_store(store)
+    logger.info("Finished connecting to index from MongoDB.")
+    return index

templates/components/vectordbs/python/mongo/vectordb.py

@@ -1,11 +0,0 @@
-import os
-from llama_index.vector_stores.mongodb import MongoDBAtlasVectorSearch
-
-
-def get_vector_store():
-    store = MongoDBAtlasVectorSearch(
-        db_name=os.environ["MONGODB_DATABASE"],
-        collection_name=os.environ["MONGODB_VECTORS"],
-        index_name=os.environ["MONGODB_VECTOR_INDEX"],
-    )
-    return store

templates/components/vectordbs/python/none/constants.py

@@ -0,0 +1 @@
+STORAGE_DIR = "storage"  # directory to cache the generated index

templates/components/vectordbs/python/none/generate.py

@@ -0,0 +1,34 @@
+from dotenv import load_dotenv
+
+load_dotenv()
+
+import os
+import logging
+from llama_index.core.indices import VectorStoreIndex
+from llama_index.core.storage import StorageContext
+from app.engine.constants import STORAGE_DIR
+from app.engine.loader import get_documents
+from app.settings import init_settings
+
+
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger()
+
+
+def generate_datasource():
+    storage_context = StorageContext.from_defaults()
+
+    docs = []
+
+    for doc in get_documents():
+        storage_context.docstore.add_documents(doc)
+        docs.extend(doc)
+
+    index = VectorStoreIndex.from_documents(docs, storage_context=storage_context)
+    index.storage_context.persist(persist_dir=STORAGE_DIR)
+    logger.info(f"Generated index at {STORAGE_DIR}")
+
+
+if __name__ == "__main__":
+    init_settings()
+    generate_datasource()

templates/components/vectordbs/python/none/index.py

@@ -0,0 +1,23 @@
+import logging
+import os
+
+from app.engine.constants import STORAGE_DIR
+from llama_index.core.storage import StorageContext
+from llama_index.core.indices import load_index_from_storage
+
+logger = logging.getLogger("uvicorn")
+
+
+def get_index():
+    # check if storage already exists
+    if not os.path.exists(STORAGE_DIR):
+        raise Exception(
+            "StorageContext is empty - call 'python app/engine/generate.py' to generate the storage first"
+        )
+
+    # load the existing index
+    logger.info(f"Loading index from {STORAGE_DIR}...")
+    storage_context = StorageContext.from_defaults(persist_dir=STORAGE_DIR)
+    index = load_index_from_storage(storage_context)
+    logger.info(f"Finished loading index from {STORAGE_DIR}")
+    return index

templates/components/vectordbs/python/none/vectordb.py

@@ -1,16 +0,0 @@
-import os
-
-from llama_index.core.vector_stores import SimpleVectorStore
-from app.constants import STORAGE_DIR
-
-
-def get_vector_store():
-    if not os.path.exists(STORAGE_DIR):
-        # Vector store hasn't been persisted before, create a new one
-        vector_store = SimpleVectorStore()
-    else:
-        # Vector store has already been persisted before at STORAGE_DIR - load it
-        vector_store = SimpleVectorStore.from_persist_dir(
-            STORAGE_DIR, namespace="default"
-        )
-    return vector_store

templates/components/vectordbs/python/pg/constants.py

@@ -0,0 +1,2 @@
+PGVECTOR_SCHEMA = "public"
+PGVECTOR_TABLE = "llamaindex_embedding"

templates/components/vectordbs/python/pg/generate.py

@@ -0,0 +1,35 @@
+from dotenv import load_dotenv
+
+load_dotenv()
+
+import logging
+from llama_index.core.indices import VectorStoreIndex
+from llama_index.core.storage import StorageContext
+
+from app.engine.loader import get_documents
+from app.settings import init_settings
+from app.engine.utils import init_pg_vector_store_from_env
+
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger()
+
+
+def generate_datasource():
+    logger.info("Creating new index")
+    # load the documents and create the index
+    documents = get_documents()
+    store = init_pg_vector_store_from_env()
+    storage_context = StorageContext.from_defaults(vector_store=store)
+    VectorStoreIndex.from_documents(
+        documents,
+        storage_context=storage_context,
+        show_progress=True,  # this will show you a progress bar as the embeddings are created
+    )
+    logger.info(
+        f"Successfully created embeddings in the PG vector store, schema={store.schema_name} table={store.table_name}"
+    )
+
+
+if __name__ == "__main__":
+    init_settings()
+    generate_datasource()

templates/components/vectordbs/python/pg/index.py

@@ -0,0 +1,13 @@
+import logging
+from llama_index.core.indices.vector_store import VectorStoreIndex
+from app.engine.utils import init_pg_vector_store_from_env
+
+logger = logging.getLogger("uvicorn")
+
+
+def get_index():
+    logger.info("Connecting to index from PGVector...")
+    store = init_pg_vector_store_from_env()
+    index = VectorStoreIndex.from_vector_store(store)
+    logger.info("Finished connecting to index from PGVector.")
+    return index

templates/components/vectordbs/python/pg/utils.py

@@ -1,13 +1,10 @@
 import os
 from llama_index.vector_stores.postgres import PGVectorStore
 from urllib.parse import urlparse
-
-STORAGE_DIR = "storage"
-PGVECTOR_SCHEMA = "public"
-PGVECTOR_TABLE = "llamaindex_embedding"
+from app.engine.constants import PGVECTOR_SCHEMA, PGVECTOR_TABLE
 
 
-def get_vector_store():
+def init_pg_vector_store_from_env():
     original_conn_string = os.environ.get("PG_CONNECTION_STRING")
     if original_conn_string is None or original_conn_string == "":
         raise ValueError("PG_CONNECTION_STRING environment variable is not set.")
@@ -27,5 +24,4 @@ def get_vector_store():
         async_connection_string=async_conn_string,
         schema_name=PGVECTOR_SCHEMA,
         table_name=PGVECTOR_TABLE,
-        embed_dim=int(os.environ.get("EMBEDDING_DIM", 768)),
     )

templates/components/vectordbs/python/pinecone/generate.py

@@ -0,0 +1,39 @@
+from dotenv import load_dotenv
+
+load_dotenv()
+
+import os
+import logging
+from llama_index.core.storage import StorageContext
+from llama_index.core.indices import VectorStoreIndex
+from llama_index.vector_stores.pinecone import PineconeVectorStore
+from app.settings import init_settings
+from app.engine.loader import get_documents
+
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger()
+
+
+def generate_datasource():
+    logger.info("Creating new index")
+    # load the documents and create the index
+    documents = get_documents()
+    store = PineconeVectorStore(
+        api_key=os.environ["PINECONE_API_KEY"],
+        index_name=os.environ["PINECONE_INDEX_NAME"],
+        environment=os.environ["PINECONE_ENVIRONMENT"],
+    )
+    storage_context = StorageContext.from_defaults(vector_store=store)
+    VectorStoreIndex.from_documents(
+        documents,
+        storage_context=storage_context,
+        show_progress=True,  # this will show you a progress bar as the embeddings are created
+    )
+    logger.info(
+        f"Successfully created embeddings and save to your Pinecone index {os.environ['PINECONE_INDEX_NAME']}"
+    )
+
+
+if __name__ == "__main__":
+    init_settings()
+    generate_datasource()

templates/components/vectordbs/python/pinecone/index.py

@@ -0,0 +1,20 @@
+import logging
+import os
+
+from llama_index.core.indices import VectorStoreIndex
+from llama_index.vector_stores.pinecone import PineconeVectorStore
+
+
+logger = logging.getLogger("uvicorn")
+
+
+def get_index():
+    logger.info("Connecting to index from Pinecone...")
+    store = PineconeVectorStore(
+        api_key=os.environ["PINECONE_API_KEY"],
+        index_name=os.environ["PINECONE_INDEX_NAME"],
+        environment=os.environ["PINECONE_ENVIRONMENT"],
+    )
+    index = VectorStoreIndex.from_vector_store(store)
+    logger.info("Finished connecting to index from Pinecone.")
+    return index

templates/components/vectordbs/python/pinecone/vectordb.py

@@ -1,11 +0,0 @@
-import os
-from llama_index.vector_stores.pinecone import PineconeVectorStore
-
-
-def get_vector_store():
-    store = PineconeVectorStore(
-        api_key=os.environ["PINECONE_API_KEY"],
-        index_name=os.environ["PINECONE_INDEX_NAME"],
-        environment=os.environ["PINECONE_ENVIRONMENT"],
-    )
-    return store

templates/components/vectordbs/python/qdrant/vectordb.py

@@ -1,11 +0,0 @@
-import os
-from llama_index.vector_stores.qdrant import QdrantVectorStore
-
-
-def get_vector_store():
-    store = QdrantVectorStore(
-        collection_name=os.getenv("QDRANT_COLLECTION"),
-        url=os.getenv("QDRANT_URL"),
-        api_key=os.getenv("QDRANT_API_KEY"),
-    )
-    return store

templates/components/vectordbs/typescript/astra/generate.ts

@@ -1,43 +0,0 @@
-/* eslint-disable turbo/no-undeclared-env-vars */
-import * as dotenv from "dotenv";
-import {
-  AstraDBVectorStore,
-  VectorStoreIndex,
-  storageContextFromDefaults,
-} from "llamaindex";
-import { getDocuments } from "./loader";
-import { initSettings } from "./settings";
-import { checkRequiredEnvVars } from "./shared";
-
-dotenv.config();
-
-async function loadAndIndex() {
-  // load objects from storage and convert them into LlamaIndex Document objects
-  const documents = await getDocuments();
-
-  // create vector store and a collection
-  const collectionName = process.env.ASTRA_DB_COLLECTION!;
-  const vectorStore = new AstraDBVectorStore();
-  await vectorStore.create(collectionName, {
-    vector: {
-      dimension: parseInt(process.env.EMBEDDING_DIM!),
-      metric: "cosine",
-    },
-  });
-  await vectorStore.connect(collectionName);
-
-  // create index from documents and store them in Astra
-  console.log("Start creating embeddings...");
-  const storageContext = await storageContextFromDefaults({ vectorStore });
-  await VectorStoreIndex.fromDocuments(documents, { storageContext });
-  console.log(
-    "Successfully created embeddings and save to your Astra database.",
-  );
-}
-
-(async () => {
-  checkRequiredEnvVars();
-  initSettings();
-  await loadAndIndex();
-  console.log("Finished generating storage.");
-})();

templates/components/vectordbs/typescript/astra/index.ts

@@ -1,10 +0,0 @@
-/* eslint-disable turbo/no-undeclared-env-vars */
-import { AstraDBVectorStore, VectorStoreIndex } from "llamaindex";
-import { checkRequiredEnvVars } from "./shared";
-
-export async function getDataSource() {
-  checkRequiredEnvVars();
-  const store = new AstraDBVectorStore();
-  await store.connect(process.env.ASTRA_DB_COLLECTION!);
-  return await VectorStoreIndex.fromVectorStore(store);
-}

templates/components/vectordbs/typescript/astra/shared.ts

@@ -1,23 +0,0 @@
-const REQUIRED_ENV_VARS = [
-  "ASTRA_DB_APPLICATION_TOKEN",
-  "ASTRA_DB_ENDPOINT",
-  "ASTRA_DB_COLLECTION",
-  "EMBEDDING_DIM",
-];
-
-export function checkRequiredEnvVars() {
-  const missingEnvVars = REQUIRED_ENV_VARS.filter((envVar) => {
-    return !process.env[envVar];
-  });
-
-  if (missingEnvVars.length > 0) {
-    console.log(
-      `The following environment variables are required but missing: ${missingEnvVars.join(
-        ", ",
-      )}`,
-    );
-    throw new Error(
-      `Missing environment variables: ${missingEnvVars.join(", ")}`,
-    );
-  }
-}

templates/components/vectordbs/typescript/milvus/generate.mjs

@@ -2,12 +2,15 @@
 import * as dotenv from "dotenv";
 import {
   MilvusVectorStore,
+  SimpleDirectoryReader,
   VectorStoreIndex,
   storageContextFromDefaults,
 } from "llamaindex";
-import { getDocuments } from "./loader";
-import { initSettings } from "./settings";
-import { checkRequiredEnvVars, getMilvusClient } from "./shared";
+import {
+  STORAGE_DIR,
+  checkRequiredEnvVars,
+  getMilvusClient,
+} from "./shared.mjs";
 
 dotenv.config();
 
@@ -15,7 +18,9 @@ const collectionName = process.env.MILVUS_COLLECTION;
 
 async function loadAndIndex() {
   // load objects from storage and convert them into LlamaIndex Document objects
-  const documents = await getDocuments();
+  const documents = await new SimpleDirectoryReader().loadData({
+    directoryPath: STORAGE_DIR,
+  });
 
   // Connect to Milvus
   const milvusClient = getMilvusClient();
@@ -33,7 +38,6 @@ async function loadAndIndex() {
 
 (async () => {
   checkRequiredEnvVars();
-  initSettings();
   await loadAndIndex();
   console.log("Finished generating storage.");
 })();

templates/components/vectordbs/typescript/milvus/index.ts

@@ -1,10 +1,35 @@
-import { MilvusVectorStore, VectorStoreIndex } from "llamaindex";
-import { checkRequiredEnvVars, getMilvusClient } from "./shared";
+import {
+  ContextChatEngine,
+  LLM,
+  MilvusVectorStore,
+  serviceContextFromDefaults,
+  VectorStoreIndex,
+} from "llamaindex";
+import {
+  checkRequiredEnvVars,
+  CHUNK_OVERLAP,
+  CHUNK_SIZE,
+  getMilvusClient,
+} from "./shared.mjs";
 
-export async function getDataSource() {
+async function getDataSource(llm: LLM) {
   checkRequiredEnvVars();
+  const serviceContext = serviceContextFromDefaults({
+    llm,
+    chunkSize: CHUNK_SIZE,
+    chunkOverlap: CHUNK_OVERLAP,
+  });
   const milvusClient = getMilvusClient();
   const store = new MilvusVectorStore({ milvusClient });
 
-  return await VectorStoreIndex.fromVectorStore(store);
+  return await VectorStoreIndex.fromVectorStore(store, serviceContext);
+}
+
+export async function createChatEngine(llm: LLM) {
+  const index = await getDataSource(llm);
+  const retriever = index.asRetriever({ similarityTopK: 3 });
+  return new ContextChatEngine({
+    chatModel: llm,
+    retriever,
+  });
 }

templates/components/vectordbs/typescript/milvus/shared.mjs

@@ -1,5 +1,9 @@
 import { MilvusClient } from "@zilliz/milvus2-sdk-node";
 
+export const STORAGE_DIR = "./data";
+export const CHUNK_SIZE = 512;
+export const CHUNK_OVERLAP = 20;
+
 const REQUIRED_ENV_VARS = [
   "MILVUS_ADDRESS",
   "MILVUS_USERNAME",
@@ -13,7 +17,7 @@ export function getMilvusClient() {
     throw new Error("MILVUS_ADDRESS environment variable is required");
   }
   return new MilvusClient({
-    address: process.env.MILVUS_ADDRESS!,
+    address: process.env.MILVUS_ADDRESS,
     username: process.env.MILVUS_USERNAME,
     password: process.env.MILVUS_PASSWORD,
   });

templates/components/vectordbs/typescript/mongo/generate.mjs

@@ -2,19 +2,18 @@
 import * as dotenv from "dotenv";
 import {
   MongoDBAtlasVectorSearch,
+  SimpleDirectoryReader,
   VectorStoreIndex,
   storageContextFromDefaults,
 } from "llamaindex";
 import { MongoClient } from "mongodb";
-import { getDocuments } from "./loader";
-import { initSettings } from "./settings";
-import { checkRequiredEnvVars } from "./shared";
+import { STORAGE_DIR, checkRequiredEnvVars } from "./shared.mjs";
 
 dotenv.config();
 
-const mongoUri = process.env.MONGO_URI!;
-const databaseName = process.env.MONGODB_DATABASE!;
-const vectorCollectionName = process.env.MONGODB_VECTORS!;
+const mongoUri = process.env.MONGO_URI;
+const databaseName = process.env.MONGODB_DATABASE;
+const vectorCollectionName = process.env.MONGODB_VECTORS;
 const indexName = process.env.MONGODB_VECTOR_INDEX;
 
 async function loadAndIndex() {
@@ -22,7 +21,9 @@ async function loadAndIndex() {
   const client = new MongoClient(mongoUri);
 
   // load objects from storage and convert them into LlamaIndex Document objects
-  const documents = await getDocuments();
+  const documents = await new SimpleDirectoryReader().loadData({
+    directoryPath: STORAGE_DIR,
+  });
 
   // create Atlas as a vector store
   const vectorStore = new MongoDBAtlasVectorSearch({
@@ -43,7 +44,6 @@ async function loadAndIndex() {
 
 (async () => {
   checkRequiredEnvVars();
-  initSettings();
   await loadAndIndex();
   console.log("Finished generating storage.");
 })();

templates/components/vectordbs/typescript/mongo/index.ts

@@ -1,17 +1,37 @@
 /* eslint-disable turbo/no-undeclared-env-vars */
-import { MongoDBAtlasVectorSearch, VectorStoreIndex } from "llamaindex";
+import {
+  ContextChatEngine,
+  LLM,
+  MongoDBAtlasVectorSearch,
+  serviceContextFromDefaults,
+  VectorStoreIndex,
+} from "llamaindex";
 import { MongoClient } from "mongodb";
-import { checkRequiredEnvVars } from "./shared";
+import { checkRequiredEnvVars, CHUNK_OVERLAP, CHUNK_SIZE } from "./shared.mjs";
 
-export async function getDataSource() {
+async function getDataSource(llm: LLM) {
   checkRequiredEnvVars();
   const client = new MongoClient(process.env.MONGO_URI!);
+  const serviceContext = serviceContextFromDefaults({
+    llm,
+    chunkSize: CHUNK_SIZE,
+    chunkOverlap: CHUNK_OVERLAP,
+  });
   const store = new MongoDBAtlasVectorSearch({
     mongodbClient: client,
-    dbName: process.env.MONGODB_DATABASE!,
-    collectionName: process.env.MONGODB_VECTORS!,
+    dbName: process.env.MONGODB_DATABASE,
+    collectionName: process.env.MONGODB_VECTORS,
     indexName: process.env.MONGODB_VECTOR_INDEX,
   });
 
-  return await VectorStoreIndex.fromVectorStore(store);
+  return await VectorStoreIndex.fromVectorStore(store, serviceContext);
+}
+
+export async function createChatEngine(llm: LLM) {
+  const index = await getDataSource(llm);
+  const retriever = index.asRetriever({ similarityTopK: 3 });
+  return new ContextChatEngine({
+    chatModel: llm,
+    retriever,
+  });
 }

templates/components/vectordbs/typescript/mongo/shared.mjs

@@ -1,3 +1,7 @@
+export const STORAGE_DIR = "./data";
+export const CHUNK_SIZE = 512;
+export const CHUNK_OVERLAP = 20;
+
 const REQUIRED_ENV_VARS = [
   "MONGO_URI",
   "MONGODB_DATABASE",

templates/components/vectordbs/typescript/none/constants.mjs

@@ -0,0 +1,4 @@
+export const STORAGE_DIR = "./data";
+export const STORAGE_CACHE_DIR = "./cache";
+export const CHUNK_SIZE = 512;
+export const CHUNK_OVERLAP = 20;

templates/components/vectordbs/typescript/none/generate.mjs

@@ -1,38 +1,53 @@
-import { VectorStoreIndex, storageContextFromDefaults } from "llamaindex";
+import {
+  serviceContextFromDefaults,
+  SimpleDirectoryReader,
+  storageContextFromDefaults,
+  VectorStoreIndex,
+} from "llamaindex";
 
 import * as dotenv from "dotenv";
 
-import { getDocuments } from "./loader";
-import { initSettings } from "./settings";
-import { STORAGE_CACHE_DIR } from "./shared";
+import {
+  CHUNK_OVERLAP,
+  CHUNK_SIZE,
+  STORAGE_CACHE_DIR,
+  STORAGE_DIR,
+} from "./constants.mjs";
 
 // Load environment variables from local .env file
 dotenv.config();
 
-async function getRuntime(func: any) {
+async function getRuntime(func) {
   const start = Date.now();
   await func();
   const end = Date.now();
   return end - start;
 }
 
-async function generateDatasource() {
+async function generateDatasource(serviceContext) {
   console.log(`Generating storage context...`);
   // Split documents, create embeddings and store them in the storage context
   const ms = await getRuntime(async () => {
     const storageContext = await storageContextFromDefaults({
       persistDir: STORAGE_CACHE_DIR,
     });
-    const documents = await getDocuments();
+    const documents = await new SimpleDirectoryReader().loadData({
+      directoryPath: STORAGE_DIR,
+    });
     await VectorStoreIndex.fromDocuments(documents, {
       storageContext,
+      serviceContext,
     });
   });
   console.log(`Storage context successfully generated in ${ms / 1000}s.`);
 }
 
 (async () => {
-  initSettings();
-  await generateDatasource();
+  const serviceContext = serviceContextFromDefaults({
+    chunkSize: CHUNK_SIZE,
+    chunkOverlap: CHUNK_OVERLAP,
+  });
+
+  await generateDatasource(serviceContext);
   console.log("Finished generating storage.");
 })();

templates/components/vectordbs/typescript/none/index.ts

@@ -1,11 +1,19 @@
 import {
+  ContextChatEngine,
+  LLM,
+  serviceContextFromDefaults,
   SimpleDocumentStore,
   storageContextFromDefaults,
   VectorStoreIndex,
 } from "llamaindex";
-import { STORAGE_CACHE_DIR } from "./shared";
+import { CHUNK_OVERLAP, CHUNK_SIZE, STORAGE_CACHE_DIR } from "./constants.mjs";
 
-export async function getDataSource() {
+async function getDataSource(llm: LLM) {
+  const serviceContext = serviceContextFromDefaults({
+    llm,
+    chunkSize: CHUNK_SIZE,
+    chunkOverlap: CHUNK_OVERLAP,
+  });
   const storageContext = await storageContextFromDefaults({
     persistDir: `${STORAGE_CACHE_DIR}`,
   });
@@ -14,9 +22,23 @@ export async function getDataSource() {
     (storageContext.docStore as SimpleDocumentStore).toDict(),
   ).length;
   if (numberOfDocs === 0) {
-    return null;
+    throw new Error(
+      `StorageContext is empty - call 'npm run generate' to generate the storage first`,
+    );
   }
   return await VectorStoreIndex.init({
     storageContext,
+    serviceContext,
+  });
+}
+
+export async function createChatEngine(llm: LLM) {
+  const index = await getDataSource(llm);
+  const retriever = index.asRetriever();
+  retriever.similarityTopK = 3;
+
+  return new ContextChatEngine({
+    chatModel: llm,
+    retriever,
   });
 }

templates/components/vectordbs/typescript/none/shared.ts

@@ -1 +0,0 @@
-export const STORAGE_CACHE_DIR = "./cache";

templates/components/vectordbs/typescript/pg/generate.mjs

@@ -2,23 +2,24 @@
 import * as dotenv from "dotenv";
 import {
   PGVectorStore,
+  SimpleDirectoryReader,
   VectorStoreIndex,
   storageContextFromDefaults,
 } from "llamaindex";
-import { getDocuments } from "./loader";
-import { initSettings } from "./settings";
 import {
-  PGVECTOR_COLLECTION,
   PGVECTOR_SCHEMA,
   PGVECTOR_TABLE,
+  STORAGE_DIR,
   checkRequiredEnvVars,
-} from "./shared";
+} from "./shared.mjs";
 
 dotenv.config();
 
 async function loadAndIndex() {
   // load objects from storage and convert them into LlamaIndex Document objects
-  const documents = await getDocuments();
+  const documents = await new SimpleDirectoryReader().loadData({
+    directoryPath: STORAGE_DIR,
+  });
 
   // create postgres vector store
   const vectorStore = new PGVectorStore({
@@ -26,7 +27,7 @@ async function loadAndIndex() {
     schemaName: PGVECTOR_SCHEMA,
     tableName: PGVECTOR_TABLE,
   });
-  vectorStore.setCollection(PGVECTOR_COLLECTION);
+  vectorStore.setCollection(STORAGE_DIR);
   vectorStore.clearCollection();
 
   // create index from all the Documents
@@ -38,7 +39,6 @@ async function loadAndIndex() {
 
 (async () => {
   checkRequiredEnvVars();
-  initSettings();
   await loadAndIndex();
   console.log("Finished generating storage.");
   process.exit(0);

templates/components/vectordbs/typescript/pg/index.ts

@@ -1,17 +1,39 @@
 /* eslint-disable turbo/no-undeclared-env-vars */
-import { PGVectorStore, VectorStoreIndex } from "llamaindex";
 import {
+  ContextChatEngine,
+  LLM,
+  PGVectorStore,
+  VectorStoreIndex,
+  serviceContextFromDefaults,
+} from "llamaindex";
+import {
+  CHUNK_OVERLAP,
+  CHUNK_SIZE,
   PGVECTOR_SCHEMA,
   PGVECTOR_TABLE,
   checkRequiredEnvVars,
-} from "./shared";
+} from "./shared.mjs";
 
-export async function getDataSource() {
+async function getDataSource(llm: LLM) {
   checkRequiredEnvVars();
   const pgvs = new PGVectorStore({
     connectionString: process.env.PG_CONNECTION_STRING,
     schemaName: PGVECTOR_SCHEMA,
     tableName: PGVECTOR_TABLE,
   });
-  return await VectorStoreIndex.fromVectorStore(pgvs);
+  const serviceContext = serviceContextFromDefaults({
+    llm,
+    chunkSize: CHUNK_SIZE,
+    chunkOverlap: CHUNK_OVERLAP,
+  });
+  return await VectorStoreIndex.fromVectorStore(pgvs, serviceContext);
+}
+
+export async function createChatEngine(llm: LLM) {
+  const index = await getDataSource(llm);
+  const retriever = index.asRetriever({ similarityTopK: 3 });
+  return new ContextChatEngine({
+    chatModel: llm,
+    retriever,
+  });
 }

templates/components/vectordbs/typescript/pg/shared.mjs

@@ -1,8 +1,10 @@
-export const PGVECTOR_COLLECTION = "data";
+export const STORAGE_DIR = "./data";
+export const CHUNK_SIZE = 512;
+export const CHUNK_OVERLAP = 20;
 export const PGVECTOR_SCHEMA = "public";
 export const PGVECTOR_TABLE = "llamaindex_embedding";
 
-const REQUIRED_ENV_VARS = ["PG_CONNECTION_STRING"];
+const REQUIRED_ENV_VARS = ["PG_CONNECTION_STRING", "OPENAI_API_KEY"];
 
 export function checkRequiredEnvVars() {
   const missingEnvVars = REQUIRED_ENV_VARS.filter((envVar) => {

templates/components/vectordbs/typescript/pinecone/generate.mjs

@@ -2,18 +2,19 @@
 import * as dotenv from "dotenv";
 import {
   PineconeVectorStore,
+  SimpleDirectoryReader,
   VectorStoreIndex,
   storageContextFromDefaults,
 } from "llamaindex";
-import { getDocuments } from "./loader";
-import { initSettings } from "./settings";
-import { checkRequiredEnvVars } from "./shared";
+import { STORAGE_DIR, checkRequiredEnvVars } from "./shared.mjs";
 
 dotenv.config();
 
 async function loadAndIndex() {
   // load objects from storage and convert them into LlamaIndex Document objects
-  const documents = await getDocuments();
+  const documents = await new SimpleDirectoryReader().loadData({
+    directoryPath: STORAGE_DIR,
+  });
 
   // create vector store
   const vectorStore = new PineconeVectorStore();
@@ -29,7 +30,6 @@ async function loadAndIndex() {
 
 (async () => {
   checkRequiredEnvVars();
-  initSettings();
   await loadAndIndex();
   console.log("Finished generating storage.");
 })();

templates/components/vectordbs/typescript/pinecone/index.ts

@@ -1,9 +1,29 @@
 /* eslint-disable turbo/no-undeclared-env-vars */
-import { PineconeVectorStore, VectorStoreIndex } from "llamaindex";
-import { checkRequiredEnvVars } from "./shared";
+import {
+  ContextChatEngine,
+  LLM,
+  PineconeVectorStore,
+  VectorStoreIndex,
+  serviceContextFromDefaults,
+} from "llamaindex";
+import { CHUNK_OVERLAP, CHUNK_SIZE, checkRequiredEnvVars } from "./shared.mjs";
 
-export async function getDataSource() {
+async function getDataSource(llm: LLM) {
   checkRequiredEnvVars();
+  const serviceContext = serviceContextFromDefaults({
+    llm,
+    chunkSize: CHUNK_SIZE,
+    chunkOverlap: CHUNK_OVERLAP,
+  });
   const store = new PineconeVectorStore();
-  return await VectorStoreIndex.fromVectorStore(store);
+  return await VectorStoreIndex.fromVectorStore(store, serviceContext);
+}
+
+export async function createChatEngine(llm: LLM) {
+  const index = await getDataSource(llm);
+  const retriever = index.asRetriever({ similarityTopK: 5 });
+  return new ContextChatEngine({
+    chatModel: llm,
+    retriever,
+  });
 }

templates/components/vectordbs/typescript/pinecone/shared.mjs

@@ -1,3 +1,7 @@
+export const STORAGE_DIR = "./data";
+export const CHUNK_SIZE = 512;
+export const CHUNK_OVERLAP = 20;
+
 const REQUIRED_ENV_VARS = ["PINECONE_ENVIRONMENT", "PINECONE_API_KEY"];
 
 export function checkRequiredEnvVars() {

templates/components/vectordbs/typescript/qdrant/generate.ts

@@ -1,37 +0,0 @@
-/* eslint-disable turbo/no-undeclared-env-vars */
-import * as dotenv from "dotenv";
-import {
-  QdrantVectorStore,
-  VectorStoreIndex,
-  storageContextFromDefaults,
-} from "llamaindex";
-import { getDocuments } from "./loader";
-import { initSettings } from "./settings";
-import { checkRequiredEnvVars, getQdrantClient } from "./shared";
-
-dotenv.config();
-
-const collectionName = process.env.QDRANT_COLLECTION;
-
-async function loadAndIndex() {
-  // load objects from storage and convert them into LlamaIndex Document objects
-  const documents = await getDocuments();
-
-  // Connect to Qdrant
-  const vectorStore = new QdrantVectorStore(collectionName, getQdrantClient());
-
-  const storageContext = await storageContextFromDefaults({ vectorStore });
-  await VectorStoreIndex.fromDocuments(documents, {
-    storageContext: storageContext,
-  });
-  console.log(
-    `Successfully upload embeddings to Qdrant collection ${collectionName}.`,
-  );
-}
-
-(async () => {
-  checkRequiredEnvVars();
-  initSettings();
-  await loadAndIndex();
-  console.log("Finished generating storage.");
-})();

templates/components/vectordbs/typescript/qdrant/index.ts

@@ -1,13 +0,0 @@
-import * as dotenv from "dotenv";
-import { QdrantVectorStore, VectorStoreIndex } from "llamaindex";
-import { checkRequiredEnvVars, getQdrantClient } from "./shared";
-
-dotenv.config();
-
-export async function getDataSource() {
-  checkRequiredEnvVars();
-  const collectionName = process.env.QDRANT_COLLECTION;
-  const store = new QdrantVectorStore(collectionName, getQdrantClient());
-
-  return await VectorStoreIndex.fromVectorStore(store);
-}

templates/components/vectordbs/typescript/qdrant/shared.ts

@@ -1,32 +0,0 @@
-import { QdrantClient } from "@qdrant/js-client-rest";
-
-const REQUIRED_ENV_VARS = ["QDRANT_URL", "QDRANT_COLLECTION"]; // QDRANT_API_KEY is optional
-
-export function getQdrantClient() {
-  const url = process.env.QDRANT_URL;
-  if (!url) {
-    throw new Error("QDRANT_URL environment variable is required");
-  }
-  const apiKey = process.env?.QDRANT_API_KEY;
-  return new QdrantClient({
-    url,
-    apiKey,
-  });
-}
-
-export function checkRequiredEnvVars() {
-  const missingEnvVars = REQUIRED_ENV_VARS.filter((envVar) => {
-    return !process.env[envVar];
-  });
-
-  if (missingEnvVars.length > 0) {
-    console.log(
-      `The following environment variables are required but missing: ${missingEnvVars.join(
-        ", ",
-      )}`,
-    );
-    throw new Error(
-      `Missing environment variables: ${missingEnvVars.join(", ")}`,
-    );
-  }
-}

templates/types/simple/express/README-template.md

@@ -0,0 +1,50 @@
+This is a [LlamaIndex](https://www.llamaindex.ai/) project using [Express](https://expressjs.com/) bootstrapped with [`create-llama`](https://github.com/run-llama/LlamaIndexTS/tree/main/packages/create-llama).
+
+## Getting Started
+
+First, install the dependencies:
+
+```
+npm install
+```
+
+Second, run the development server:
+
+```
+npm run dev
+```
+
+Then call the express API endpoint `/api/chat` to see the result:
+
+```
+curl --location 'localhost:8000/api/chat' \
+--header 'Content-Type: application/json' \
+--data '{ "messages": [{ "role": "user", "content": "Hello" }] }'
+```
+
+You can start editing the API by modifying `src/controllers/chat.controller.ts`. The endpoint auto-updates as you save the file.
+
+## Production
+
+First, build the project:
+
+```
+npm run build
+```
+
+You can then run the production server:
+
+```
+NODE_ENV=production npm run start
+```
+
+> Note that the `NODE_ENV` environment variable is set to `production`. This disables CORS for all origins.
+
+## Learn More
+
+To learn more about LlamaIndex, take a look at the following resources:
+
+- [LlamaIndex Documentation](https://docs.llamaindex.ai) - learn about LlamaIndex (Python features).
+- [LlamaIndexTS Documentation](https://ts.llamaindex.ai) - learn about LlamaIndex (Typescript features).
+
+You can check out [the LlamaIndexTS GitHub repository](https://github.com/run-llama/LlamaIndexTS) - your feedback and contributions are welcome!