better log

---------
Co-authored-by: Marcus Schiesser <mail@marcusschiesser.de>

.changeset/config.json

@@ -1,7 +1,7 @@
 {
   "$schema": "https://unpkg.com/@changesets/config@3.0.0/schema.json",
   "changelog": "@changesets/cli/changelog",
-  "commit": true,
+  "commit": false,
   "fixed": [],
   "linked": [],
   "access": "public",

.changeset/short-ducks-drum.md

@@ -0,0 +1,5 @@
+---
+"create-llama": patch
+---
+
+Use ingestion pipeline for Python

.changeset/spicy-moose-allow.md

@@ -0,0 +1,5 @@
+---
+"create-llama": patch
+---
+
+Display events (e.g. retrieving nodes) per chat message

.github/workflows/e2e.yml

@@ -24,36 +24,46 @@ jobs:
     runs-on: ${{ matrix.os }}
     steps:
       - uses: actions/checkout@v4
+
       - name: Set up python ${{ matrix.python-version }}
         uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python-version }}
+
       - name: Install Poetry
         uses: snok/install-poetry@v1
         with:
           version: ${{ env.POETRY_VERSION }}
-      - uses: pnpm/action-setup@v2
+
+      - uses: pnpm/action-setup@v3
+
       - name: Setup Node.js ${{ matrix.node-version }}
         uses: actions/setup-node@v4
         with:
           node-version: ${{ matrix.node-version }}
           cache: "pnpm"
+
       - name: Install dependencies
         run: pnpm install
+
       - name: Install Playwright Browsers
         run: pnpm exec playwright install --with-deps
         working-directory: .
+
       - name: Build create-llama
         run: pnpm run build
         working-directory: .
+
       - name: Install
         run: pnpm run pack-install
         working-directory: .
+
       - name: Run Playwright tests
         run: pnpm run e2e
         env:
           OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
         working-directory: .
+
       - uses: actions/upload-artifact@v3
         if: always()
         with:

.github/workflows/lint_on_push_or_pull.yml

@@ -13,17 +13,20 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
-      - uses: pnpm/action-setup@v2
-        with:
-          version: latest
+
+      - uses: pnpm/action-setup@v3
+
       - name: Setup Node.js
         uses: actions/setup-node@v4
         with:
           node-version-file: ".nvmrc"
           cache: "pnpm"
+
       - name: Install dependencies
         run: pnpm install
+
       - name: Run lint
         run: pnpm run lint
+
       - name: Run Prettier
         run: pnpm run format

.github/workflows/publish_github.yml

@@ -0,0 +1,36 @@
+name: Publish to GitHub Releases
+
+on:
+  push:
+    tags:
+      - "v*"
+
+jobs:
+  build-and-publish:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout Repo
+        uses: actions/checkout@v4
+
+      - uses: pnpm/action-setup@v3
+
+      - name: Setup Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version-file: ".nvmrc"
+          cache: "pnpm"
+
+      - name: Install dependencies
+        run: pnpm install
+
+      - name: Build tarball
+        run: |
+          pnpm pack
+
+      - name: Create release
+        uses: ncipollo/release-action@v1
+        with:
+          artifacts: "create-llama-*.tgz"
+          name: Release ${{ github.ref }}
+          bodyFile: "CHANGELOG.md"
+          token: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/release.yml

@@ -0,0 +1,55 @@
+name: Release
+
+on:
+  push:
+    branches:
+      - main
+
+concurrency: ${{ github.workflow }}-${{ github.ref }}
+
+jobs:
+  release:
+    name: Release
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout Repo
+        uses: actions/checkout@v4
+
+      - uses: pnpm/action-setup@v3
+
+      - name: Setup Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version-file: ".nvmrc"
+          cache: "pnpm"
+
+      - name: Install dependencies
+        run: pnpm install
+
+      - name: Add auth token to .npmrc file
+        run: |
+          cat << EOF >> ".npmrc"
+            //registry.npmjs.org/:_authToken=$NPM_TOKEN
+          EOF
+        env:
+          NPM_TOKEN: ${{ secrets.NPM_TOKEN }}
+
+      - name: Get changeset status
+        id: get-changeset-status
+        run: |
+          pnpm changeset status --output .changeset/status.json
+          new_version=$(jq -r '.releases[0].newVersion' < .changeset/status.json)
+          rm -v .changeset/status.json
+          echo "new-version=${new_version}" >> "$GITHUB_OUTPUT"
+
+      - name: Create Release Pull Request or Publish to npm
+        id: changesets
+        uses: changesets/action@v1
+        with:
+          commit: Release ${{ steps.get-changeset-status.outputs.new-version }}
+          title: Release ${{ steps.get-changeset-status.outputs.new-version }}
+          # build package and call changeset publish
+          publish: pnpm release
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          NPM_TOKEN: ${{ secrets.NPM_TOKEN }}

CHANGELOG.md

@@ -1,5 +1,21 @@
 # create-llama
 
+## 0.1.0
+
+### Minor Changes
+
+- f1c3e8d: Add Llama3 and Phi3 support using Ollama
+
+### Patch Changes
+
+- a0dec80: Use `gpt-4-turbo` model as default. Upgrade Python llama-index to 0.10.28
+- 753229d: Remove asking for AI models and use defaults instead (OpenAIs GPT-4 Vision Preview and Embeddings v3). Use `--ask-models` CLI parameter to select models.
+- 1d78202: Add observability for Python
+- 6acccd2: Use poetry run generate to generate embeddings for FastAPI
+- 9efcffe: Use Settings object for LlamaIndex configuration
+- 418bf9b: refactor: use tsx instead of ts-node
+- 1be69a5: Add Qdrant support
+
 ## 0.0.32
 
 ### Patch Changes

CONTRIBUTING.md

@@ -7,7 +7,7 @@ Install NodeJS. Preferably v18 using nvm or n.
 Inside the `create-llama` directory:
 
 ```
-npm i -g pnpm ts-node
+npm i -g pnpm
 pnpm install
 ```
 

README.md

@@ -18,25 +18,19 @@ to start the development server. You can then visit [http://localhost:3000](http
 
 ## What you'll get
 
-- A Next.js-powered front-end. The app is set up as a chat interface that can answer questions about your data (see below)
-  - You can style it with HTML and CSS, or you can optionally use components from [shadcn/ui](https://ui.shadcn.com/)
+- A Next.js-powered front-end using components from [shadcn/ui](https://ui.shadcn.com/). The app is set up as a chat interface that can answer questions about your data (see below)
 - Your choice of 3 back-ends:
-  - **Next.js**: if you select this option, you’ll have a full stack Next.js application that you can deploy to a host like [Vercel](https://vercel.com/) in just a few clicks. This uses [LlamaIndex.TS](https://www.npmjs.com/package/llamaindex), our TypeScript library.
+  - **Next.js**: if you select this option, you’ll have a full-stack Next.js application that you can deploy to a host like [Vercel](https://vercel.com/) in just a few clicks. This uses [LlamaIndex.TS](https://www.npmjs.com/package/llamaindex), our TypeScript library.
   - **Express**: if you want a more traditional Node.js application you can generate an Express backend. This also uses LlamaIndex.TS.
-  - **Python FastAPI**: if you select this option you’ll get a backend powered by the [llama-index python package](https://pypi.org/project/llama-index/), which you can deploy to a service like Render or fly.io.
-- The back-end has a single endpoint that allows you to send the state of your chat and receive additional responses
-- You can choose whether you want a streaming or non-streaming back-end (if you're not sure, we recommend streaming)
-- You can choose whether you want to use `ContextChatEngine` or `SimpleChatEngine`
-  - `SimpleChatEngine` will just talk to the LLM directly without using your data
-  - `ContextChatEngine` will use your data to answer questions (see below).
+  - **Python FastAPI**: if you select this option, you’ll get a backend powered by the [llama-index python package](https://pypi.org/project/llama-index/), which you can deploy to a service like Render or fly.io.
+- The back-end has two endpoints (one streaming, the other one non-streaming) that allow you to send the state of your chat and receive additional responses
+- You add arbitrary data sources to your chat, like local files, websites, or data retrieved from a database.
+- Turn your chat into an AI agent by adding tools (functions called by the LLM).
 - The app uses OpenAI by default, so you'll need an OpenAI API key, or you can customize it to use any of the dozens of LLMs we support.
 
 ## Using your data
 
-If you've enabled `ContextChatEngine`, you can supply your own data and the app will index it and answer questions. Your generated app will have a folder called `data`:
-
-- With the Next.js backend this is `./data`
-- With the Express or Python backend this is in `./backend/data`
+You can supply your own data; the app will index it and answer questions. Your generated app will have a folder called `data` (If you're using Express or Python and generate a frontend, it will be `./backend/data`).
 
 The app will ingest any supported files you put in this directory. Your Next.js and Express apps use LlamaIndex.TS so they will be able to ingest any PDF, text, CSV, Markdown, Word and HTML files. The Python backend can read even more types, including video and audio files.
 
@@ -46,22 +40,28 @@ Before you can use your data, you need to index it. If you're using the Next.js
 npm run generate
 ```
 
-Then re-start your app. Remember you'll need to re-run `generate` if you add new files to your `data` folder. If you're using the Python backend, you can trigger indexing of your data by deleting the `./storage` folder and re-starting the app.
+Then re-start your app. Remember you'll need to re-run `generate` if you add new files to your `data` folder.
 
-## Don't want a front-end?
+If you're using the Python backend, you can trigger indexing of your data by calling:
 
-It's optional! If you've selected the Python or Express back-ends, just delete the `frontend` folder and you'll get an API without any front-end code.
+```bash
+poetry run generate
+```
 
-## Customizing the LLM
+## Want a front-end?
 
-By default the app will use OpenAI's gpt-3.5-turbo model. If you want to use GPT-4, you can modify this by editing a file:
+Optionally generate a frontend if you've selected the Python or Express back-ends. If you do so, `create-llama` will generate two folders: `frontend`, for your Next.js-based frontend code, and `backend` containing your API.
 
-- In the Next.js backend, edit `./app/api/chat/route.ts` and replace `gpt-3.5-turbo` with `gpt-4`
-- In the Express backend, edit `./backend/src/controllers/chat.controller.ts` and likewise replace `gpt-3.5-turbo` with `gpt-4`
-- In the Python backend, edit `./backend/app/utils/index.py` and once again replace `gpt-3.5-turbo` with `gpt-4`
+## Customizing the AI models
+
+The app will default to OpenAI's `gpt-4-turbo` LLM and `text-embedding-3-large` embedding model.
+
+If you want to use different OpenAI models, add the `--ask-models` CLI parameter.
 
 You can also replace OpenAI with one of our [dozens of other supported LLMs](https://docs.llamaindex.ai/en/stable/module_guides/models/llms/modules.html).
 
+To do so, you have to manually change the generated code (edit the `settings.ts` file for Typescript projects or the `settings.py` file for Python projects)
+
 ## Example
 
 The simplest thing to do is run `create-llama` in interactive mode:
@@ -84,13 +84,19 @@ Need to install the following packages:
   create-llama@latest
 Ok to proceed? (y) y
 ✔ What is your project named? … my-app
-✔ Which template would you like to use? › Chat with streaming
+✔ Which template would you like to use? › Chat
 ✔ Which framework would you like to use? › NextJS
-✔ Which UI would you like to use? › Just HTML
-✔ Which chat engine would you like to use? › ContextChatEngine
+✔ Would you like to set up observability? › No
 ✔ Please provide your OpenAI API key (leave blank to skip): …
-✔ Would you like to use ESLint? … No / Yes
-Creating a new LlamaIndex app in /home/my-app.
+✔ Which data source would you like to use? › Use an example PDF
+✔ Would you like to add another data source? › No
+✔ Would you like to use LlamaParse (improved parser for RAG - requires API key)? … no / yes
+✔ Would you like to use a vector database? › No, just store the data in the file system
+? How would you like to proceed? › - Use arrow-keys. Return to submit.
+   Just generate code (~1 sec)
+❯  Start in VSCode (~1 sec)
+   Generate code and install dependencies (~2 min)
+   Generate code, install dependencies, and run the app (~2 min)
 ```
 
 ### Running non-interactively

create-app.ts

@@ -30,10 +30,8 @@ export async function createApp({
   appPath,
   packageManager,
   frontend,
-  openAiKey,
+  modelConfig,
   llamaCloudKey,
-  model,
-  embeddingModel,
   communityProjectConfig,
   llamapack,
   vectorDb,
@@ -77,10 +75,8 @@ export async function createApp({
     ui,
     packageManager,
     isOnline,
-    openAiKey,
+    modelConfig,
     llamaCloudKey,
-    model,
-    embeddingModel,
     communityProjectConfig,
     llamapack,
     vectorDb,

e2e/utils.ts

@@ -12,8 +12,7 @@ import {
 } from "../helpers";
 
 export type AppType = "--frontend" | "--no-frontend" | "";
-const MODEL = "gpt-3.5-turbo";
-const EMBEDDING_MODEL = "text-embedding-ada-002";
+
 export type CreateLlamaResult = {
   projectName: string;
   appProcess: ChildProcess;
@@ -96,10 +95,6 @@ export async function runCreateLlama(
     templateUI,
     "--vector-db",
     vectorDb,
-    "--model",
-    MODEL,
-    "--embedding-model",
-    EMBEDDING_MODEL,
     "--open-ai-key",
     process.env.OPENAI_API_KEY,
     appType,

helpers/copy.ts

@@ -52,6 +52,7 @@ export const copy = async (
 export const assetRelocator = (name: string) => {
   switch (name) {
     case "gitignore":
+    case "npmrc":
     case "eslintrc.json": {
       return `.${name}`;
     }

helpers/env-variables.ts

@@ -1,6 +1,7 @@
 import fs from "fs/promises";
 import path from "path";
 import {
+  ModelConfig,
   TemplateDataSource,
   TemplateFramework,
   TemplateVectorDB,
@@ -28,7 +29,10 @@ const renderEnvVar = (envVars: EnvVar[]): string => {
   );
 };
 
-const getVectorDBEnvs = (vectorDb: TemplateVectorDB) => {
+const getVectorDBEnvs = (vectorDb?: TemplateVectorDB): EnvVar[] => {
+  if (!vectorDb) {
+    return [];
+  }
   switch (vectorDb) {
     case "mongo":
       return [
@@ -108,88 +112,92 @@ const getVectorDBEnvs = (vectorDb: TemplateVectorDB) => {
           description: "The name of the collection in your Astra database",
         },
       ];
+    case "qdrant":
+      return [
+        {
+          name: "QDRANT_URL",
+          description:
+            "The qualified REST URL of the Qdrant server. Eg: http://localhost:6333",
+        },
+        {
+          name: "QDRANT_COLLECTION",
+          description: "The name of Qdrant collection to use.",
+        },
+        {
+          name: "QDRANT_API_KEY",
+          description:
+            "Optional API key for authenticating requests to Qdrant.",
+        },
+      ];
     default:
       return [];
   }
 };
 
-export const createBackendEnvFile = async (
-  root: string,
-  opts: {
-    openAiKey?: string;
-    llamaCloudKey?: string;
-    vectorDb?: TemplateVectorDB;
-    model?: string;
-    embeddingModel?: string;
-    framework?: TemplateFramework;
-    dataSources?: TemplateDataSource[];
-    port?: number;
-  },
-) => {
-  // Init env values
-  const envFileName = ".env";
-  const defaultEnvs = [
+const getModelEnvs = (modelConfig: ModelConfig): EnvVar[] => {
+  return [
+    {
+      name: "MODEL_PROVIDER",
+      description: "The provider for the AI models to use.",
+      value: modelConfig.provider,
+    },
     {
-      render: true,
       name: "MODEL",
       description: "The name of LLM model to use.",
-      value: opts.model || "gpt-3.5-turbo",
+      value: modelConfig.model,
     },
     {
-      render: true,
-      name: "OPENAI_API_KEY",
-      description: "The OpenAI API key to use.",
-      value: opts.openAiKey,
+      name: "EMBEDDING_MODEL",
+      description: "Name of the embedding model to use.",
+      value: modelConfig.embeddingModel,
     },
     {
-      name: "LLAMA_CLOUD_API_KEY",
-      description: `The Llama Cloud API key.`,
-      value: opts.llamaCloudKey,
+      name: "EMBEDDING_DIM",
+      description: "Dimension of the embedding model to use.",
+      value: modelConfig.dimensions.toString(),
     },
-    // Add vector database environment variables
-    ...(opts.vectorDb ? getVectorDBEnvs(opts.vectorDb) : []),
+    ...(modelConfig.provider === "openai"
+      ? [
+          {
+            name: "OPENAI_API_KEY",
+            description: "The OpenAI API key to use.",
+            value: modelConfig.apiKey,
+          },
+          {
+            name: "LLM_TEMPERATURE",
+            description: "Temperature for sampling from the model.",
+          },
+          {
+            name: "LLM_MAX_TOKENS",
+            description: "Maximum number of tokens to generate.",
+          },
+        ]
+      : []),
   ];
-  let envVars: EnvVar[] = [];
-  if (opts.framework === "fastapi") {
-    envVars = [
-      ...defaultEnvs,
-      ...[
-        {
-          name: "APP_HOST",
-          description: "The address to start the backend app.",
-          value: "0.0.0.0",
-        },
-        {
-          name: "APP_PORT",
-          description: "The port to start the backend app.",
-          value: opts.port?.toString() || "8000",
-        },
-        {
-          name: "EMBEDDING_MODEL",
-          description: "Name of the embedding model to use.",
-          value: opts.embeddingModel,
-        },
-        {
-          name: "EMBEDDING_DIM",
-          description: "Dimension of the embedding model to use.",
-        },
-        {
-          name: "LLM_TEMPERATURE",
-          description: "Temperature for sampling from the model.",
-        },
-        {
-          name: "LLM_MAX_TOKENS",
-          description: "Maximum number of tokens to generate.",
-        },
-        {
-          name: "TOP_K",
-          description:
-            "The number of similar embeddings to return when retrieving documents.",
-          value: "3",
-        },
-        {
-          name: "SYSTEM_PROMPT",
-          description: `Custom system prompt.
+};
+
+const getFrameworkEnvs = (
+  framework?: TemplateFramework,
+  port?: number,
+): EnvVar[] => {
+  if (framework !== "fastapi") {
+    return [];
+  }
+  return [
+    {
+      name: "APP_HOST",
+      description: "The address to start the backend app.",
+      value: "0.0.0.0",
+    },
+    {
+      name: "APP_PORT",
+      description: "The port to start the backend app.",
+      value: port?.toString() || "8000",
+    },
+    // TODO: Once LlamaIndexTS supports string templates, move this to `getEngineEnvs`
+    {
+      name: "SYSTEM_PROMPT",
+      description: `Custom system prompt.
 Example:
 SYSTEM_PROMPT="
 We have provided context information below.
@@ -198,24 +206,48 @@ We have provided context information below.
 ---------------------
 Given this information, please answer the question: {query_str}
 "`,
-        },
-      ],
-    ];
-  } else {
-    envVars = [
-      ...defaultEnvs,
-      ...[
-        opts.framework === "nextjs"
-          ? {
-              name: "NEXT_PUBLIC_MODEL",
-              description:
-                "The LLM model to use (hardcode to front-end artifact).",
-              value: opts.model || "gpt-3.5-turbo",
-            }
-          : {},
-      ],
-    ];
-  }
+    },
+  ];
+};
+
+const getEngineEnvs = (): EnvVar[] => {
+  return [
+    {
+      name: "TOP_K",
+      description:
+        "The number of similar embeddings to return when retrieving documents.",
+      value: "3",
+    },
+  ];
+};
+
+export const createBackendEnvFile = async (
+  root: string,
+  opts: {
+    llamaCloudKey?: string;
+    vectorDb?: TemplateVectorDB;
+    modelConfig: ModelConfig;
+    framework?: TemplateFramework;
+    dataSources?: TemplateDataSource[];
+    port?: number;
+  },
+) => {
+  // Init env values
+  const envFileName = ".env";
+  const envVars: EnvVar[] = [
+    {
+      name: "LLAMA_CLOUD_API_KEY",
+      description: `The Llama Cloud API key.`,
+      value: opts.llamaCloudKey,
+    },
+    // Add model environment variables
+    ...getModelEnvs(opts.modelConfig),
+    // Add engine environment variables
+    ...getEngineEnvs(),
+    // Add vector database environment variables
+    ...getVectorDBEnvs(opts.vectorDb),
+    ...getFrameworkEnvs(opts.framework, opts.port),
+  ];
   // Render and write env file
   const content = renderEnvVar(envVars);
   await fs.writeFile(path.join(root, envFileName), content);
@@ -226,20 +258,9 @@ export const createFrontendEnvFile = async (
   root: string,
   opts: {
     customApiPath?: string;
-    model?: string;
   },
 ) => {
   const defaultFrontendEnvs = [
-    {
-      name: "MODEL",
-      description: "The OpenAI model to use.",
-      value: opts.model,
-    },
-    {
-      name: "NEXT_PUBLIC_MODEL",
-      description: "The OpenAI model to use (hardcode to front-end artifact).",
-      value: opts.model,
-    },
     {
       name: "NEXT_PUBLIC_CHAT_API",
       description: "The backend API for chat endpoint.",

helpers/index.ts

@@ -9,12 +9,14 @@ import { createBackendEnvFile, createFrontendEnvFile } from "./env-variables";
 import { PackageManager } from "./get-pkg-manager";
 import { installLlamapackProject } from "./llama-pack";
 import { isHavingPoetryLockFile, tryPoetryRun } from "./poetry";
+import { isModelConfigured } from "./providers";
 import { installPythonTemplate } from "./python";
 import { downloadAndExtractRepo } from "./repo";
 import { ConfigFileType, writeToolsConfig } from "./tools";
 import {
   FileSourceConfig,
   InstallTemplateArgs,
+  ModelConfig,
   TemplateDataSource,
   TemplateFramework,
   TemplateVectorDB,
@@ -24,8 +26,8 @@ import { installTSTemplate } from "./typescript";
 // eslint-disable-next-line max-params
 async function generateContextData(
   framework: TemplateFramework,
+  modelConfig: ModelConfig,
   packageManager?: PackageManager,
-  openAiKey?: string,
   vectorDb?: TemplateVectorDB,
   llamaCloudKey?: string,
   useLlamaParse?: boolean,
@@ -33,20 +35,20 @@ async function generateContextData(
   if (packageManager) {
     const runGenerate = `${cyan(
       framework === "fastapi"
-        ? "poetry run python app/engine/generate.py"
+        ? "poetry run generate"
         : `${packageManager} run generate`,
     )}`;
-    const openAiKeyConfigured = openAiKey || process.env["OPENAI_API_KEY"];
+    const modelConfigured = isModelConfigured(modelConfig);
     const llamaCloudKeyConfigured = useLlamaParse
       ? llamaCloudKey || process.env["LLAMA_CLOUD_API_KEY"]
       : true;
     const hasVectorDb = vectorDb && vectorDb !== "none";
-    if (openAiKeyConfigured && llamaCloudKeyConfigured && !hasVectorDb) {
+    if (modelConfigured && llamaCloudKeyConfigured && !hasVectorDb) {
       // If all the required environment variables are set, run the generate script
       if (framework === "fastapi") {
         if (isHavingPoetryLockFile()) {
           console.log(`Running ${runGenerate} to generate the context data.`);
-          const result = tryPoetryRun("python app/engine/generate.py");
+          const result = tryPoetryRun("poetry run generate");
           if (!result) {
             console.log(`Failed to run ${runGenerate}.`);
             process.exit(1);
@@ -63,7 +65,7 @@ async function generateContextData(
 
     // generate the message of what to do to run the generate script manually
     const settings = [];
-    if (!openAiKeyConfigured) settings.push("your OpenAI key");
+    if (!modelConfigured) settings.push("your model provider API key");
     if (!llamaCloudKeyConfigured) settings.push("your Llama Cloud key");
     if (hasVectorDb) settings.push("your Vector DB environment variables");
     const settingsMessage =
@@ -141,11 +143,9 @@ export const installTemplate = async (
 
     // Copy the environment file to the target directory.
     await createBackendEnvFile(props.root, {
-      openAiKey: props.openAiKey,
+      modelConfig: props.modelConfig,
       llamaCloudKey: props.llamaCloudKey,
       vectorDb: props.vectorDb,
-      model: props.model,
-      embeddingModel: props.embeddingModel,
       framework: props.framework,
       dataSources: props.dataSources,
       port: props.externalPort,
@@ -163,8 +163,8 @@ export const installTemplate = async (
       ) {
         await generateContextData(
           props.framework,
+          props.modelConfig,
           props.packageManager,
-          props.openAiKey,
           props.vectorDb,
           props.llamaCloudKey,
           props.useLlamaParse,
@@ -174,7 +174,6 @@ export const installTemplate = async (
   } else {
     // this is a frontend for a full-stack app, create .env file with model information
     await createFrontendEnvFile(props.root, {
-      model: props.model,
       customApiPath: props.customApiPath,
     });
   }

helpers/providers/index.ts

@@ -0,0 +1,66 @@
+import ciInfo from "ci-info";
+import prompts from "prompts";
+import { questionHandlers } from "../../questions";
+import { ModelConfig, ModelProvider } from "../types";
+import { askOllamaQuestions } from "./ollama";
+import { askOpenAIQuestions, isOpenAIConfigured } from "./openai";
+
+const DEFAULT_MODEL_PROVIDER = "openai";
+
+export type ModelConfigQuestionsParams = {
+  openAiKey?: string;
+  askModels: boolean;
+};
+
+export type ModelConfigParams = Omit<ModelConfig, "provider">;
+
+export async function askModelConfig({
+  askModels,
+  openAiKey,
+}: ModelConfigQuestionsParams): Promise<ModelConfig> {
+  let modelProvider: ModelProvider = DEFAULT_MODEL_PROVIDER;
+  if (askModels && !ciInfo.isCI) {
+    const { provider } = await prompts(
+      {
+        type: "select",
+        name: "provider",
+        message: "Which model provider would you like to use",
+        choices: [
+          {
+            title: "OpenAI",
+            value: "openai",
+          },
+          { title: "Ollama", value: "ollama" },
+        ],
+        initial: 0,
+      },
+      questionHandlers,
+    );
+    modelProvider = provider;
+  }
+
+  let modelConfig: ModelConfigParams;
+  switch (modelProvider) {
+    case "ollama":
+      modelConfig = await askOllamaQuestions({ askModels });
+      break;
+    default:
+      modelConfig = await askOpenAIQuestions({
+        openAiKey,
+        askModels,
+      });
+  }
+  return {
+    ...modelConfig,
+    provider: modelProvider,
+  };
+}
+
+export function isModelConfigured(modelConfig: ModelConfig): boolean {
+  switch (modelConfig.provider) {
+    case "openai":
+      return isOpenAIConfigured(modelConfig);
+    default:
+      return true;
+  }
+}

helpers/providers/ollama.ts

@@ -0,0 +1,92 @@
+import ciInfo from "ci-info";
+import ollama, { type ModelResponse } from "ollama";
+import { red } from "picocolors";
+import prompts from "prompts";
+import { ModelConfigParams } from ".";
+import { questionHandlers, toChoice } from "../../questions";
+
+type ModelData = {
+  dimensions: number;
+};
+const MODELS = ["llama3:8b", "wizardlm2:7b", "gemma:7b", "phi3"];
+const DEFAULT_MODEL = MODELS[0];
+// TODO: get embedding vector dimensions from the ollama sdk (currently not supported)
+const EMBEDDING_MODELS: Record<string, ModelData> = {
+  "nomic-embed-text": { dimensions: 768 },
+  "mxbai-embed-large": { dimensions: 1024 },
+  "all-minilm": { dimensions: 384 },
+};
+const DEFAULT_EMBEDDING_MODEL: string = Object.keys(EMBEDDING_MODELS)[0];
+
+type OllamaQuestionsParams = {
+  askModels: boolean;
+};
+
+export async function askOllamaQuestions({
+  askModels,
+}: OllamaQuestionsParams): Promise<ModelConfigParams> {
+  const config: ModelConfigParams = {
+    model: DEFAULT_MODEL,
+    embeddingModel: DEFAULT_EMBEDDING_MODEL,
+    dimensions: EMBEDDING_MODELS[DEFAULT_EMBEDDING_MODEL].dimensions,
+  };
+
+  // use default model values in CI or if user should not be asked
+  const useDefaults = ciInfo.isCI || !askModels;
+  if (!useDefaults) {
+    const { model } = await prompts(
+      {
+        type: "select",
+        name: "model",
+        message: "Which LLM model would you like to use?",
+        choices: MODELS.map(toChoice),
+        initial: 0,
+      },
+      questionHandlers,
+    );
+    await ensureModel(model);
+    config.model = model;
+
+    const { embeddingModel } = await prompts(
+      {
+        type: "select",
+        name: "embeddingModel",
+        message: "Which embedding model would you like to use?",
+        choices: Object.keys(EMBEDDING_MODELS).map(toChoice),
+        initial: 0,
+      },
+      questionHandlers,
+    );
+    await ensureModel(embeddingModel);
+    config.embeddingModel = embeddingModel;
+    config.dimensions = EMBEDDING_MODELS[embeddingModel].dimensions;
+  }
+
+  return config;
+}
+
+async function ensureModel(modelName: string) {
+  try {
+    if (modelName.split(":").length === 1) {
+      // model doesn't have a version suffix, use latest
+      modelName = modelName + ":latest";
+    }
+    const { models } = await ollama.list();
+    const found =
+      models.find((model: ModelResponse) => model.name === modelName) !==
+      undefined;
+    if (!found) {
+      console.log(
+        red(
+          `Model ${modelName} was not pulled yet. Call 'ollama pull ${modelName}' and try again.`,
+        ),
+      );
+      process.exit(1);
+    }
+  } catch (error) {
+    console.log(
+      red("Listing Ollama models failed. Is 'ollama' running? " + error),
+    );
+    process.exit(1);
+  }
+}

helpers/providers/openai.ts

@@ -0,0 +1,146 @@
+import ciInfo from "ci-info";
+import got from "got";
+import ora from "ora";
+import { red } from "picocolors";
+import prompts from "prompts";
+import { ModelConfigParams, ModelConfigQuestionsParams } from ".";
+import { questionHandlers } from "../../questions";
+
+const OPENAI_API_URL = "https://api.openai.com/v1";
+
+const DEFAULT_MODEL = "gpt-4-turbo";
+const DEFAULT_EMBEDDING_MODEL = "text-embedding-3-large";
+
+export async function askOpenAIQuestions({
+  openAiKey,
+  askModels,
+}: ModelConfigQuestionsParams): Promise<ModelConfigParams> {
+  const config: ModelConfigParams = {
+    apiKey: openAiKey,
+    model: DEFAULT_MODEL,
+    embeddingModel: DEFAULT_EMBEDDING_MODEL,
+    dimensions: getDimensions(DEFAULT_EMBEDDING_MODEL),
+  };
+
+  if (!config.apiKey) {
+    const { key } = await prompts(
+      {
+        type: "text",
+        name: "key",
+        message: askModels
+          ? "Please provide your OpenAI API key (or leave blank to use OPENAI_API_KEY env variable):"
+          : "Please provide your OpenAI API key (leave blank to skip):",
+        validate: (value: string) => {
+          console.log(value);
+          if (askModels && !value) {
+            if (process.env.OPENAI_API_KEY) {
+              return true;
+            }
+            return "OPENAI_API_KEY env variable is not set - key is required";
+          }
+          return true;
+        },
+      },
+      questionHandlers,
+    );
+    config.apiKey = key || process.env.OPENAI_API_KEY;
+  }
+
+  // use default model values in CI or if user should not be asked
+  const useDefaults = ciInfo.isCI || !askModels;
+  if (!useDefaults) {
+    const { model } = await prompts(
+      {
+        type: "select",
+        name: "model",
+        message: "Which LLM model would you like to use?",
+        choices: await getAvailableModelChoices(false, config.apiKey),
+        initial: 0,
+      },
+      questionHandlers,
+    );
+    config.model = model;
+
+    const { embeddingModel } = await prompts(
+      {
+        type: "select",
+        name: "embeddingModel",
+        message: "Which embedding model would you like to use?",
+        choices: await getAvailableModelChoices(true, config.apiKey),
+        initial: 0,
+      },
+      questionHandlers,
+    );
+    config.embeddingModel = embeddingModel;
+    config.dimensions = getDimensions(embeddingModel);
+  }
+
+  return config;
+}
+
+export function isOpenAIConfigured(params: ModelConfigParams): boolean {
+  if (params.apiKey) {
+    return true;
+  }
+  if (process.env["OPENAI_API_KEY"]) {
+    return true;
+  }
+  return false;
+}
+
+async function getAvailableModelChoices(
+  selectEmbedding: boolean,
+  apiKey?: string,
+) {
+  if (!apiKey) {
+    throw new Error("need OpenAI key to retrieve model choices");
+  }
+  const isLLMModel = (modelId: string) => {
+    return modelId.startsWith("gpt");
+  };
+
+  const isEmbeddingModel = (modelId: string) => {
+    return modelId.includes("embedding");
+  };
+
+  const spinner = ora("Fetching available models").start();
+  try {
+    const response = await got(`${OPENAI_API_URL}/models`, {
+      headers: {
+        Authorization: "Bearer " + apiKey,
+      },
+      timeout: 5000,
+      responseType: "json",
+    });
+    const data: any = await response.body;
+    spinner.stop();
+    return data.data
+      .filter((model: any) =>
+        selectEmbedding ? isEmbeddingModel(model.id) : isLLMModel(model.id),
+      )
+      .map((el: any) => {
+        return {
+          title: el.id,
+          value: el.id,
+        };
+      });
+  } catch (error) {
+    spinner.stop();
+    if ((error as any).response?.statusCode === 401) {
+      console.log(
+        red(
+          "Invalid OpenAI API key provided! Please provide a valid key and try again!",
+        ),
+      );
+    } else {
+      console.log(red("Request failed: " + error));
+    }
+    process.exit(1);
+  }
+}
+
+function getDimensions(modelName: string) {
+  // at 2024-04-24 all OpenAI embedding models support 1536 dimensions except
+  // "text-embedding-3-large", see https://openai.com/blog/new-embedding-models-and-api-updates
+  return modelName === "text-embedding-3-large" ? 1024 : 1536;
+}

helpers/python.ts

@@ -10,6 +10,7 @@ import { isPoetryAvailable, tryPoetryInstall } from "./poetry";
 import { Tool } from "./tools";
 import {
   InstallTemplateArgs,
+  ModelConfig,
   TemplateDataSource,
   TemplateVectorDB,
 } from "./types";
@@ -21,6 +22,7 @@ interface Dependency {
 }
 
 const getAdditionalDependencies = (
+  modelConfig: ModelConfig,
   vectorDb?: TemplateVectorDB,
   dataSource?: TemplateDataSource,
   tools?: Tool[],
@@ -108,6 +110,25 @@ const getAdditionalDependencies = (
     });
   });
 
+  switch (modelConfig.provider) {
+    case "ollama":
+      dependencies.push({
+        name: "llama-index-llms-ollama",
+        version: "0.1.2",
+      });
+      dependencies.push({
+        name: "llama-index-embeddings-ollama",
+        version: "0.1.2",
+      });
+      break;
+    case "openai":
+      dependencies.push({
+        name: "llama-index-agent-openai",
+        version: "0.2.2",
+      });
+      break;
+  }
+
   return dependencies;
 };
 
@@ -205,7 +226,8 @@ export const installPythonTemplate = async ({
   dataSources,
   tools,
   postInstallAction,
-  useLlamaParse,
+  observability,
+  modelConfig,
 }: Pick<
   InstallTemplateArgs,
   | "root"
@@ -214,8 +236,9 @@ export const installPythonTemplate = async ({
   | "vectorDb"
   | "dataSources"
   | "tools"
-  | "useLlamaParse"
   | "postInstallAction"
+  | "observability"
+  | "modelConfig"
 >) => {
   console.log("\nInitializing Python project with template:", template, "\n");
   const templatePath = path.join(templatesDir, "types", template, framework);
@@ -256,8 +279,27 @@ export const installPythonTemplate = async ({
   });
 
   const addOnDependencies = dataSources
-    .map((ds) => getAdditionalDependencies(vectorDb, ds, tools))
+    .map((ds) => getAdditionalDependencies(modelConfig, vectorDb, ds, tools))
     .flat();
+
+  if (observability === "opentelemetry") {
+    addOnDependencies.push({
+      name: "traceloop-sdk",
+      version: "^0.15.11",
+    });
+
+    const templateObservabilityPath = path.join(
+      templatesDir,
+      "components",
+      "observability",
+      "python",
+      "opentelemetry",
+    );
+    await copy("**", path.join(root, "app"), {
+      cwd: templateObservabilityPath,
+    });
+  }
+
   await addDependencies(root, addOnDependencies);
 
   if (postInstallAction === "runApp" || postInstallAction === "dependencies") {

helpers/types.ts

@@ -1,6 +1,14 @@
 import { PackageManager } from "../helpers/get-pkg-manager";
 import { Tool } from "./tools";
 
+export type ModelProvider = "openai" | "ollama";
+export type ModelConfig = {
+  provider: ModelProvider;
+  apiKey?: string;
+  model: string;
+  embeddingModel: string;
+  dimensions: number;
+};
 export type TemplateType = "streaming" | "community" | "llamapack";
 export type TemplateFramework = "nextjs" | "express" | "fastapi";
 export type TemplateUI = "html" | "shadcn";
@@ -10,7 +18,8 @@ export type TemplateVectorDB =
   | "pg"
   | "pinecone"
   | "milvus"
-  | "astra";
+  | "astra"
+  | "qdrant";
 export type TemplatePostInstallAction =
   | "none"
   | "VSCode"
@@ -58,11 +67,9 @@ export interface InstallTemplateArgs {
   ui: TemplateUI;
   dataSources: TemplateDataSource[];
   customApiPath?: string;
-  openAiKey?: string;
+  modelConfig: ModelConfig;
   llamaCloudKey?: string;
   useLlamaParse?: boolean;
-  model: string;
-  embeddingModel: string;
   communityProjectConfig?: CommunityProjectConfig;
   llamapack?: string;
   vectorDb?: TemplateVectorDB;

helpers/typescript.ts

@@ -200,12 +200,12 @@ async function updatePackageJson({
   packageJson.name = appName;
   packageJson.version = "0.1.0";
 
-  if (dataSources.length > 0 && relativeEngineDestPath) {
+  if (relativeEngineDestPath) {
     // TODO: move script to {root}/scripts for all frameworks
     // add generate script if using context engine
     packageJson.scripts = {
       ...packageJson.scripts,
-      generate: `ts-node ${path.join(
+      generate: `tsx ${path.join(
         relativeEngineDestPath,
         "engine",
         "generate.ts",

index.ts

@@ -78,7 +78,7 @@ const program = new Commander.Command(packageJson.name)
     "--files <path>",
     `
   
-    Specify the path to a local file or folder for chatting.
+  Specify the path to a local file or folder for chatting.
 `,
   )
   .option(
@@ -107,19 +107,6 @@ const program = new Commander.Command(packageJson.name)
     `
 
   Whether to generate a frontend for your backend.
-`,
-  )
-  .option(
-    "--model <model>",
-    `
-
-  Select OpenAI model to use. E.g. gpt-3.5-turbo.
-`,
-  )
-  .option(
-    "--embedding-model <embeddingModel>",
-    `
-  Select OpenAI embedding model to use. E.g. text-embedding-ada-002.
 `,
   )
   .option(
@@ -160,22 +147,30 @@ const program = new Commander.Command(packageJson.name)
   .option(
     "--use-llama-parse",
     `
-    Enable LlamaParse.
+
+  Enable LlamaParse.
 `,
   )
   .option(
     "--llama-cloud-key <key>",
     `
+  
   Provide a LlamaCloud API key.
 `,
   )
   .option(
-    "--list-server-models",
-    "Fetch available LLM and embedding models from OpenAI API.",
+    "--observability <observability>",
+    `
+    
+  Specify observability tools to use. Eg: none, opentelemetry
+`,
   )
   .option(
-    "--observability <observability>",
-    "Specify observability tools to use. Eg: none, opentelemetry",
+    "--ask-models",
+    `
+
+  Select LLM and embedding models.
+`,
   )
   .allowUnknownOption()
   .parse(process.argv);
@@ -192,6 +187,7 @@ if (process.argv.includes("--tools")) {
 if (process.argv.includes("--no-llama-parse")) {
   program.useLlamaParse = false;
 }
+program.askModels = process.argv.includes("--ask-models");
 if (process.argv.includes("--no-files")) {
   program.dataSources = [];
 } else {
@@ -278,7 +274,11 @@ async function run(): Promise<void> {
   }
 
   const preferences = (conf.get("preferences") || {}) as QuestionArgs;
-  await askQuestions(program as unknown as QuestionArgs, preferences);
+  await askQuestions(
+    program as unknown as QuestionArgs,
+    preferences,
+    program.openAiKey,
+  );
 
   await createApp({
     template: program.template,
@@ -287,10 +287,8 @@ async function run(): Promise<void> {
     appPath: resolvedProjectPath,
     packageManager,
     frontend: program.frontend,
-    openAiKey: program.openAiKey,
+    modelConfig: program.modelConfig,
     llamaCloudKey: program.llamaCloudKey,
-    model: program.model,
-    embeddingModel: program.embeddingModel,
     communityProjectConfig: program.communityProjectConfig,
     llamapack: program.llamapack,
     vectorDb: program.vectorDb,

package.json

@@ -1,12 +1,12 @@
 {
   "name": "create-llama",
-  "version": "0.0.32",
+  "version": "0.1.0",
+  "description": "Create LlamaIndex-powered apps with one command",
   "keywords": [
     "rag",
     "llamaindex",
     "next.js"
   ],
-  "description": "Create LlamaIndex-powered apps with one command",
   "repository": {
     "type": "git",
     "url": "https://github.com/run-llama/LlamaIndexTS",
@@ -20,32 +20,30 @@
     "dist"
   ],
   "scripts": {
-    "clean": "rimraf --glob ./dist ./templates/**/__pycache__ ./templates/**/node_modules ./templates/**/poetry.lock",
-    "format": "prettier --ignore-unknown --cache --check .",
-    "format:write": "prettier --ignore-unknown --write .",
-    "dev": "ncc build ./index.ts -w -o dist/",
     "build": "bash ./scripts/build.sh",
     "build:ncc": "pnpm run clean && ncc build ./index.ts -o ./dist/ --minify --no-cache --no-source-map-register",
-    "lint": "eslint . --ignore-pattern dist --ignore-pattern e2e/cache",
+    "clean": "rimraf --glob ./dist ./templates/**/__pycache__ ./templates/**/node_modules ./templates/**/poetry.lock",
+    "dev": "ncc build ./index.ts -w -o dist/",
     "e2e": "playwright test",
+    "format": "prettier --ignore-unknown --cache --check .",
+    "format:write": "prettier --ignore-unknown --write .",
+    "lint": "eslint . --ignore-pattern dist --ignore-pattern e2e/cache",
+    "new-snapshot": "pnpm run build && changeset version --snapshot",
+    "new-version": "pnpm run build && changeset version",
+    "pack-install": "bash ./scripts/pack.sh",
     "prepare": "husky",
     "release": "pnpm run build && changeset publish",
-    "new-version": "pnpm run build && changeset version",
-    "release-snapshot": "pnpm run build && changeset publish --tag snapshot",
-    "new-snapshot": "pnpm run build && changeset version --snapshot",
-    "pack-install": "bash ./scripts/pack.sh"
+    "release-snapshot": "pnpm run build && changeset publish --tag snapshot"
   },
-  "devDependencies": {
-    "@playwright/test": "^1.41.1",
+  "dependencies": {
     "@types/async-retry": "1.4.2",
     "@types/ci-info": "2.0.0",
     "@types/cross-spawn": "6.0.0",
+    "@types/fs-extra": "11.0.4",
     "@types/node": "^20.11.7",
     "@types/prompts": "2.0.1",
     "@types/tar": "6.1.5",
     "@types/validate-npm-package-name": "3.0.0",
-    "@types/fs-extra": "11.0.4",
-    "@vercel/ncc": "0.38.1",
     "async-retry": "1.3.1",
     "async-sema": "3.0.1",
     "ci-info": "github:watson/ci-info#f43f6a1cefff47fb361c88cf4b943fdbcaafe540",
@@ -53,29 +51,34 @@
     "conf": "10.2.0",
     "cross-spawn": "7.0.3",
     "fast-glob": "3.3.1",
+    "fs-extra": "11.2.0",
     "got": "10.7.0",
+    "ollama": "^0.5.0",
+    "ora": "^8.0.1",
     "picocolors": "1.0.0",
     "prompts": "2.1.0",
-    "rimraf": "^5.0.5",
     "smol-toml": "^1.1.4",
     "tar": "6.1.15",
     "terminal-link": "^3.0.0",
     "update-check": "1.5.4",
     "validate-npm-package-name": "3.0.0",
-    "wait-port": "^1.1.0",
+    "yaml": "2.4.1"
+  },
+  "devDependencies": {
     "@changesets/cli": "^2.27.1",
+    "@playwright/test": "^1.41.1",
+    "@vercel/ncc": "0.38.1",
     "eslint": "^8.56.0",
+    "eslint-config-prettier": "^8.10.0",
     "husky": "^9.0.10",
     "prettier": "^3.2.5",
     "prettier-plugin-organize-imports": "^3.2.4",
+    "rimraf": "^5.0.5",
     "typescript": "^5.3.3",
-    "eslint-config-prettier": "^8.10.0",
-    "ora": "^8.0.1",
-    "fs-extra": "11.2.0",
-    "yaml": "2.4.1"
+    "wait-port": "^1.1.0"
   },
+  "packageManager": "pnpm@9.0.5",
   "engines": {
     "node": ">=16.14.0"
-  },
-  "packageManager": "pnpm@8.15.1"
+  }
 }

questions.ts

@@ -1,8 +1,6 @@
 import { execSync } from "child_process";
 import ciInfo from "ci-info";
 import fs from "fs";
-import got from "got";
-import ora from "ora";
 import path from "path";
 import { blue, green, red } from "picocolors";
 import prompts from "prompts";
@@ -16,16 +14,15 @@ import { COMMUNITY_OWNER, COMMUNITY_REPO } from "./helpers/constant";
 import { EXAMPLE_FILE } from "./helpers/datasources";
 import { templatesDir } from "./helpers/dir";
 import { getAvailableLlamapackOptions } from "./helpers/llama-pack";
+import { askModelConfig, isModelConfigured } from "./helpers/providers";
 import { getProjectOptions } from "./helpers/repo";
 import { supportedTools, toolsRequireConfig } from "./helpers/tools";
 
-const OPENAI_API_URL = "https://api.openai.com/v1";
-
 export type QuestionArgs = Omit<
   InstallAppArgs,
   "appPath" | "packageManager"
 > & {
-  listServerModels?: boolean;
+  askModels?: boolean;
 };
 const supportedContextFileTypes = [
   ".pdf",
@@ -67,16 +64,13 @@ if ($dialogResult -eq [System.Windows.Forms.DialogResult]::OK)
 }
 `;
 
-const defaults: QuestionArgs = {
+const defaults: Omit<QuestionArgs, "modelConfig"> = {
   template: "streaming",
   framework: "nextjs",
   ui: "shadcn",
   frontend: false,
-  openAiKey: "",
   llamaCloudKey: "",
   useLlamaParse: false,
-  model: "gpt-3.5-turbo",
-  embeddingModel: "text-embedding-ada-002",
   communityProjectConfig: undefined,
   llamapack: "",
   postInstallAction: "dependencies",
@@ -84,7 +78,7 @@ const defaults: QuestionArgs = {
   tools: [],
 };
 
-const handlers = {
+export const questionHandlers = {
   onCancel: () => {
     console.error("Exiting.");
     process.exit(1);
@@ -102,6 +96,7 @@ const getVectorDbChoices = (framework: TemplateFramework) => {
     { title: "Pinecone", value: "pinecone" },
     { title: "Milvus", value: "milvus" },
     { title: "Astra", value: "astra" },
+    { title: "Qdrant", value: "qdrant" },
   ];
 
   const vectordbLang = framework === "fastapi" ? "python" : "typescript";
@@ -231,85 +226,15 @@ export const onPromptState = (state: any) => {
   }
 };
 
-const getAvailableModelChoices = async (
-  selectEmbedding: boolean,
-  apiKey?: string,
-  listServerModels?: boolean,
-) => {
-  const defaultLLMModels = [
-    "gpt-3.5-turbo-0125",
-    "gpt-4-turbo-preview",
-    "gpt-4",
-    "gpt-4-vision-preview",
-  ];
-  const defaultEmbeddingModels = [
-    "text-embedding-ada-002",
-    "text-embedding-3-small",
-    "text-embedding-3-large",
-  ];
-
-  const isLLMModels = (model_id: string) => {
-    return model_id.startsWith("gpt");
-  };
-
-  const isEmbeddingModel = (model_id: string) => {
-    return (
-      model_id.includes("embedding") ||
-      defaultEmbeddingModels.includes(model_id)
-    );
-  };
-
-  if (apiKey && listServerModels) {
-    const spinner = ora("Fetching available models").start();
-    try {
-      const response = await got(`${OPENAI_API_URL}/models`, {
-        headers: {
-          Authorization: "Bearer " + apiKey,
-        },
-        timeout: 5000,
-        responseType: "json",
-      });
-      const data: any = await response.body;
-      spinner.stop();
-      return data.data
-        .filter((model: any) =>
-          selectEmbedding ? isEmbeddingModel(model.id) : isLLMModels(model.id),
-        )
-        .map((el: any) => {
-          return {
-            title: el.id,
-            value: el.id,
-          };
-        });
-    } catch (error) {
-      spinner.stop();
-      if ((error as any).response?.statusCode === 401) {
-        console.log(
-          red(
-            "Invalid OpenAI API key provided! Please provide a valid key and try again!",
-          ),
-        );
-      } else {
-        console.log(red("Request failed: " + error));
-      }
-      process.exit(1);
-    }
-  } else {
-    const data = selectEmbedding ? defaultEmbeddingModels : defaultLLMModels;
-    return data.map((model) => ({
-      title: model,
-      value: model,
-    }));
-  }
-};
-
 export const askQuestions = async (
   program: QuestionArgs,
   preferences: QuestionArgs,
+  openAiKey?: string,
 ) => {
-  const getPrefOrDefault = <K extends keyof QuestionArgs>(
+  const getPrefOrDefault = <K extends keyof Omit<QuestionArgs, "modelConfig">>(
     field: K,
-  ): QuestionArgs[K] => preferences[field] ?? defaults[field];
+  ): Omit<QuestionArgs, "modelConfig">[K] =>
+    preferences[field] ?? defaults[field];
 
   // Ask for next action after installation
   async function askPostInstallAction() {
@@ -332,8 +257,7 @@ export const askQuestions = async (
           },
         ];
 
-        const openAiKeyConfigured =
-          program.openAiKey || process.env["OPENAI_API_KEY"];
+        const modelConfigured = isModelConfigured(program.modelConfig);
         // If using LlamaParse, require LlamaCloud API key
         const llamaCloudKeyConfigured = program.useLlamaParse
           ? program.llamaCloudKey || process.env["LLAMA_CLOUD_API_KEY"]
@@ -342,7 +266,7 @@ export const askQuestions = async (
         // Can run the app if all tools do not require configuration
         if (
           !hasVectorDb &&
-          openAiKeyConfigured &&
+          modelConfigured &&
           llamaCloudKeyConfigured &&
           !toolsRequireConfig(program.tools) &&
           !program.llamapack
@@ -362,7 +286,7 @@ export const askQuestions = async (
             choices: actionChoices,
             initial: 1,
           },
-          handlers,
+          questionHandlers,
         );
 
         program.postInstallAction = action;
@@ -395,7 +319,7 @@ export const askQuestions = async (
           ],
           initial: 0,
         },
-        handlers,
+        questionHandlers,
       );
       program.template = template;
       preferences.template = template;
@@ -418,7 +342,7 @@ export const askQuestions = async (
         })),
         initial: 0,
       },
-      handlers,
+      questionHandlers,
     );
     const projectConfig = JSON.parse(communityProjectConfig);
     program.communityProjectConfig = projectConfig;
@@ -439,7 +363,7 @@ export const askQuestions = async (
         })),
         initial: 0,
       },
-      handlers,
+      questionHandlers,
     );
     program.llamapack = llamapack;
     preferences.llamapack = llamapack;
@@ -465,7 +389,7 @@ export const askQuestions = async (
           choices,
           initial: 0,
         },
-        handlers,
+        questionHandlers,
       );
       program.framework = framework;
       preferences.framework = framework;
@@ -510,100 +434,36 @@ export const askQuestions = async (
     }
   }
 
-  if (program.framework === "express" || program.framework === "nextjs") {
-    if (!program.observability) {
-      if (ciInfo.isCI) {
-        program.observability = getPrefOrDefault("observability");
-      } else {
-        const { observability } = await prompts(
-          {
-            type: "select",
-            name: "observability",
-            message: "Would you like to set up observability?",
-            choices: [
-              { title: "No", value: "none" },
-              { title: "OpenTelemetry", value: "opentelemetry" },
-            ],
-            initial: 0,
-          },
-          handlers,
-        );
-
-        program.observability = observability;
-        preferences.observability = observability;
-      }
-    }
-  }
-
-  if (!program.openAiKey) {
-    const { key } = await prompts(
-      {
-        type: "text",
-        name: "key",
-        message: program.listServerModels
-          ? "Please provide your OpenAI API key (or reuse OPENAI_API_KEY env variable):"
-          : "Please provide your OpenAI API key (leave blank to skip):",
-        validate: (value: string) => {
-          if (program.listServerModels && !value) {
-            if (process.env.OPENAI_API_KEY) {
-              return true;
-            }
-            return "OpenAI API key is required";
-          }
-          return true;
-        },
-      },
-      handlers,
-    );
-
-    program.openAiKey = key || process.env.OPENAI_API_KEY;
-    preferences.openAiKey = key || process.env.OPENAI_API_KEY;
-  }
-
-  if (!program.model) {
+  if (!program.observability) {
     if (ciInfo.isCI) {
-      program.model = getPrefOrDefault("model");
+      program.observability = getPrefOrDefault("observability");
     } else {
-      const { model } = await prompts(
+      const { observability } = await prompts(
         {
           type: "select",
-          name: "model",
-          message: "Which model would you like to use?",
-          choices: await getAvailableModelChoices(
-            false,
-            program.openAiKey,
-            program.listServerModels,
-          ),
+          name: "observability",
+          message: "Would you like to set up observability?",
+          choices: [
+            { title: "No", value: "none" },
+            { title: "OpenTelemetry", value: "opentelemetry" },
+          ],
           initial: 0,
         },
-        handlers,
+        questionHandlers,
       );
-      program.model = model;
-      preferences.model = model;
+
+      program.observability = observability;
+      preferences.observability = observability;
     }
   }
 
-  if (!program.embeddingModel && program.framework === "fastapi") {
-    if (ciInfo.isCI) {
-      program.embeddingModel = getPrefOrDefault("embeddingModel");
-    } else {
-      const { embeddingModel } = await prompts(
-        {
-          type: "select",
-          name: "embeddingModel",
-          message: "Which embedding model would you like to use?",
-          choices: await getAvailableModelChoices(
-            true,
-            program.openAiKey,
-            program.listServerModels,
-          ),
-          initial: 0,
-        },
-        handlers,
-      );
-      program.embeddingModel = embeddingModel;
-      preferences.embeddingModel = embeddingModel;
-    }
+  if (!program.modelConfig) {
+    const modelConfig = await askModelConfig({
+      openAiKey,
+      askModels: program.askModels ?? false,
+    });
+    program.modelConfig = modelConfig;
+    preferences.modelConfig = modelConfig;
   }
 
   if (!program.dataSources) {
@@ -627,7 +487,7 @@ export const askQuestions = async (
             ),
             initial: firstQuestion ? 1 : 0,
           },
-          handlers,
+          questionHandlers,
         );
 
         if (selectedSource === "no" || selectedSource === "none") {
@@ -673,7 +533,7 @@ export const askQuestions = async (
                   return true;
                 },
               },
-              handlers,
+              questionHandlers,
             );
 
             program.dataSources.push({
@@ -718,7 +578,7 @@ export const askQuestions = async (
             ];
             program.dataSources.push({
               type: "db",
-              config: await prompts(dbPrompts, handlers),
+              config: await prompts(dbPrompts, questionHandlers),
             });
           }
         }
@@ -745,7 +605,7 @@ export const askQuestions = async (
           active: "yes",
           inactive: "no",
         },
-        handlers,
+        questionHandlers,
       );
       program.useLlamaParse = useLlamaParse;
 
@@ -758,7 +618,7 @@ export const askQuestions = async (
             message:
               "Please provide your LlamaIndex Cloud API key (leave blank to skip):",
           },
-          handlers,
+          questionHandlers,
         );
         program.llamaCloudKey = llamaCloudKey;
       }
@@ -777,7 +637,7 @@ export const askQuestions = async (
           choices: getVectorDbChoices(program.framework),
           initial: 0,
         },
-        handlers,
+        questionHandlers,
       );
       program.vectorDb = vectorDb;
       preferences.vectorDb = vectorDb;
@@ -812,3 +672,7 @@ export const askQuestions = async (
 
   await askPostInstallAction();
 };
+
+export const toChoice = (value: string) => {
+  return { title: value, value };
+};

templates/components/engines/python/chat/__init__.py

@@ -1,5 +1,6 @@
 import os
 from app.engine.index import get_index
+from fastapi import HTTPException
 
 
 def get_chat_engine():
@@ -8,8 +9,11 @@ def get_chat_engine():
 
     index = get_index()
     if index is None:
-        raise Exception(
-            "StorageContext is empty - call 'python app/engine/generate.py' to generate the storage first"
+        raise HTTPException(
+            status_code=500,
+            detail=str(
+                "StorageContext is empty - call 'poetry run generate' to generate the storage first"
+            ),
         )
 
     return index.as_chat_engine(

templates/components/engines/typescript/agent/chat.ts

@@ -1,21 +1,16 @@
-import {
-  BaseTool,
-  OpenAI,
-  OpenAIAgent,
-  QueryEngineTool,
-  ToolFactory,
-} from "llamaindex";
+import { BaseTool, OpenAIAgent, QueryEngineTool } from "llamaindex";
+import { ToolsFactory } from "llamaindex/tools/ToolsFactory";
 import fs from "node:fs/promises";
 import path from "node:path";
 import { getDataSource } from "./index";
 import { STORAGE_CACHE_DIR } from "./shared";
 
-export async function createChatEngine(llm: OpenAI) {
+export async function createChatEngine() {
   let tools: BaseTool[] = [];
 
   // Add a query engine tool if we have a data source
   // Delete this code if you don't have a data source
-  const index = await getDataSource(llm);
+  const index = await getDataSource();
   if (index) {
     tools.push(
       new QueryEngineTool({
@@ -33,12 +28,10 @@ export async function createChatEngine(llm: OpenAI) {
     const config = JSON.parse(
       await fs.readFile(path.join("config", "tools.json"), "utf8"),
     );
-    tools = tools.concat(await ToolFactory.createTools(config));
+    tools = tools.concat(await ToolsFactory.createTools(config));
   } catch {}
 
   return new OpenAIAgent({
     tools,
-    llm,
-    verbose: true,
   });
 }

templates/components/engines/typescript/chat/chat.ts

@@ -1,18 +1,20 @@
-import { ContextChatEngine, LLM } from "llamaindex";
+import { ContextChatEngine, Settings } from "llamaindex";
 import { getDataSource } from "./index";
 
-export async function createChatEngine(llm: LLM) {
-  const index = await getDataSource(llm);
+export async function createChatEngine() {
+  const index = await getDataSource();
   if (!index) {
     throw new Error(
       `StorageContext is empty - call 'npm run generate' to generate the storage first`,
     );
   }
   const retriever = index.asRetriever();
-  retriever.similarityTopK = 3;
+  retriever.similarityTopK = process.env.TOP_K
+    ? parseInt(process.env.TOP_K)
+    : 3;
 
   return new ContextChatEngine({
-    chatModel: llm,
+    chatModel: Settings.llm,
     retriever,
   });
 }

templates/components/loaders/python/file.py

@@ -27,10 +27,7 @@ def llama_parse_parser():
 def get_file_documents(config: FileLoaderConfig):
     from llama_index.core.readers import SimpleDirectoryReader
 
-    reader = SimpleDirectoryReader(
-        config.data_dir,
-        recursive=True,
-    )
+    reader = SimpleDirectoryReader(config.data_dir, recursive=True, filename_as_id=True)
     if config.use_llama_parse:
         parser = llama_parse_parser()
         reader.file_extractor = {".pdf": parser}

templates/components/observability/python/opentelemetry/observability.py

@@ -0,0 +1,5 @@
+from traceloop.sdk import Traceloop
+
+
+def init_observability():
+    Traceloop.init()

templates/components/vectordbs/python/astra/generate.py

@@ -1,37 +0,0 @@
-from dotenv import load_dotenv
-
-load_dotenv()
-
-import os
-import logging
-from llama_index.core.storage import StorageContext
-from llama_index.core.indices import VectorStoreIndex
-from llama_index.vector_stores.astra_db import AstraDBVectorStore
-from app.settings import init_settings
-from app.engine.loaders import get_documents
-
-logging.basicConfig(level=logging.INFO)
-logger = logging.getLogger()
-
-
-def generate_datasource():
-    logger.info("Creating new index")
-    documents = get_documents()
-    store = AstraDBVectorStore(
-        token=os.environ["ASTRA_DB_APPLICATION_TOKEN"],
-        api_endpoint=os.environ["ASTRA_DB_ENDPOINT"],
-        collection_name=os.environ["ASTRA_DB_COLLECTION"],
-        embedding_dimension=1536,
-    )
-    storage_context = StorageContext.from_defaults(vector_store=store)
-    VectorStoreIndex.from_documents(
-        documents,
-        storage_context=storage_context,
-        show_progress=True,  # this will show you a progress bar as the embeddings are created
-    )
-    logger.info(f"Successfully created embeddings in the AstraDB")
-
-
-if __name__ == "__main__":
-    init_settings()
-    generate_datasource()

templates/components/vectordbs/python/astra/index.py

@@ -1,21 +0,0 @@
-import logging
-import os
-
-from llama_index.core.indices import VectorStoreIndex
-from llama_index.vector_stores.astra_db import AstraDBVectorStore
-
-
-logger = logging.getLogger("uvicorn")
-
-
-def get_index():
-    logger.info("Connecting to index from AstraDB...")
-    store = AstraDBVectorStore(
-        token=os.environ["ASTRA_DB_APPLICATION_TOKEN"],
-        api_endpoint=os.environ["ASTRA_DB_ENDPOINT"],
-        collection_name=os.environ["ASTRA_DB_COLLECTION"],
-        embedding_dimension=1536,
-    )
-    index = VectorStoreIndex.from_vector_store(store)
-    logger.info("Finished connecting to index from AstraDB.")
-    return index

templates/components/vectordbs/python/astra/vectordb.py

@@ -0,0 +1,12 @@
+import os
+from llama_index.vector_stores.astra_db import AstraDBVectorStore
+
+
+def get_vector_store():
+    store = AstraDBVectorStore(
+        token=os.environ["ASTRA_DB_APPLICATION_TOKEN"],
+        api_endpoint=os.environ["ASTRA_DB_ENDPOINT"],
+        collection_name=os.environ["ASTRA_DB_COLLECTION"],
+        embedding_dimension=int(os.environ["EMBEDDING_DIM"]),
+    )
+    return store

templates/components/vectordbs/python/milvus/generate.py

@@ -1,39 +0,0 @@
-from dotenv import load_dotenv
-
-load_dotenv()
-
-import os
-import logging
-from llama_index.core.storage import StorageContext
-from llama_index.core.indices import VectorStoreIndex
-from llama_index.vector_stores.milvus import MilvusVectorStore
-from app.settings import init_settings
-from app.engine.loaders import get_documents
-
-logging.basicConfig(level=logging.INFO)
-logger = logging.getLogger()
-
-
-def generate_datasource():
-    logger.info("Creating new index")
-    # load the documents and create the index
-    documents = get_documents()
-    store = MilvusVectorStore(
-        uri=os.environ["MILVUS_ADDRESS"],
-        user=os.getenv("MILVUS_USERNAME"),
-        password=os.getenv("MILVUS_PASSWORD"),
-        collection_name=os.getenv("MILVUS_COLLECTION"),
-        dim=int(os.getenv("MILVUS_DIMENSION", "1536")),
-    )
-    storage_context = StorageContext.from_defaults(vector_store=store)
-    VectorStoreIndex.from_documents(
-        documents,
-        storage_context=storage_context,
-        show_progress=True,  # this will show you a progress bar as the embeddings are created
-    )
-    logger.info(f"Successfully created embeddings in the Milvus")
-
-
-if __name__ == "__main__":
-    init_settings()
-    generate_datasource()

templates/components/vectordbs/python/milvus/index.py

@@ -1,22 +0,0 @@
-import logging
-import os
-
-from llama_index.core.indices import VectorStoreIndex
-from llama_index.vector_stores.milvus import MilvusVectorStore
-
-
-logger = logging.getLogger("uvicorn")
-
-
-def get_index():
-    logger.info("Connecting to index from Milvus...")
-    store = MilvusVectorStore(
-        uri=os.getenv("MILVUS_ADDRESS"),
-        user=os.getenv("MILVUS_USERNAME"),
-        password=os.getenv("MILVUS_PASSWORD"),
-        collection_name=os.getenv("MILVUS_COLLECTION"),
-        dim=int(os.getenv("EMBEDDING_DIM", "1536")),
-    )
-    index = VectorStoreIndex.from_vector_store(store)
-    logger.info("Finished connecting to index from Milvus.")
-    return index

templates/components/vectordbs/python/milvus/vectordb.py

@@ -0,0 +1,13 @@
+import os
+from llama_index.vector_stores.milvus import MilvusVectorStore
+
+
+def get_vector_store():
+    store = MilvusVectorStore(
+        uri=os.environ["MILVUS_ADDRESS"],
+        user=os.getenv("MILVUS_USERNAME"),
+        password=os.getenv("MILVUS_PASSWORD"),
+        collection_name=os.getenv("MILVUS_COLLECTION"),
+        dim=int(os.getenv("EMBEDDING_DIM")),
+    )
+    return store

templates/components/vectordbs/python/mongo/generate.py

@@ -1,43 +0,0 @@
-from dotenv import load_dotenv
-
-load_dotenv()
-
-import os
-import logging
-from llama_index.core.storage import StorageContext
-from llama_index.core.indices import VectorStoreIndex
-from llama_index.vector_stores.mongodb import MongoDBAtlasVectorSearch
-from app.settings import init_settings
-from app.engine.loaders import get_documents
-
-logging.basicConfig(level=logging.INFO)
-logger = logging.getLogger()
-
-
-def generate_datasource():
-    logger.info("Creating new index")
-    # load the documents and create the index
-    documents = get_documents()
-    store = MongoDBAtlasVectorSearch(
-        db_name=os.environ["MONGODB_DATABASE"],
-        collection_name=os.environ["MONGODB_VECTORS"],
-        index_name=os.environ["MONGODB_VECTOR_INDEX"],
-    )
-    storage_context = StorageContext.from_defaults(vector_store=store)
-    VectorStoreIndex.from_documents(
-        documents,
-        storage_context=storage_context,
-        show_progress=True,  # this will show you a progress bar as the embeddings are created
-    )
-    logger.info(
-        f"Successfully created embeddings in the MongoDB collection {os.environ['MONGODB_VECTORS']}"
-    )
-    logger.info(
-        """IMPORTANT: You can't query your index yet because you need to create a vector search index in MongoDB's UI now.
-See https://github.com/run-llama/mongodb-demo/tree/main?tab=readme-ov-file#create-a-vector-search-index"""
-    )
-
-
-if __name__ == "__main__":
-    init_settings()
-    generate_datasource()

templates/components/vectordbs/python/mongo/index.py

@@ -1,20 +0,0 @@
-import logging
-import os
-
-from llama_index.core.indices import VectorStoreIndex
-from llama_index.vector_stores.mongodb import MongoDBAtlasVectorSearch
-
-
-logger = logging.getLogger("uvicorn")
-
-
-def get_index():
-    logger.info("Connecting to index from MongoDB...")
-    store = MongoDBAtlasVectorSearch(
-        db_name=os.environ["MONGODB_DATABASE"],
-        collection_name=os.environ["MONGODB_VECTORS"],
-        index_name=os.environ["MONGODB_VECTOR_INDEX"],
-    )
-    index = VectorStoreIndex.from_vector_store(store)
-    logger.info("Finished connecting to index from MongoDB.")
-    return index

templates/components/vectordbs/python/mongo/vectordb.py

@@ -0,0 +1,11 @@
+import os
+from llama_index.vector_stores.mongodb import MongoDBAtlasVectorSearch
+
+
+def get_vector_store():
+    store = MongoDBAtlasVectorSearch(
+        db_name=os.environ["MONGODB_DATABASE"],
+        collection_name=os.environ["MONGODB_VECTORS"],
+        index_name=os.environ["MONGODB_VECTOR_INDEX"],
+    )
+    return store

templates/components/vectordbs/python/none/constants.py

@@ -1 +0,0 @@
-STORAGE_DIR = "storage"  # directory to cache the generated index

templates/components/vectordbs/python/none/generate.py

@@ -1,32 +0,0 @@
-from dotenv import load_dotenv
-
-load_dotenv()
-
-import logging
-from llama_index.core.indices import (
-    VectorStoreIndex,
-)
-from app.engine.constants import STORAGE_DIR
-from app.engine.loaders import get_documents
-from app.settings import init_settings
-
-
-logging.basicConfig(level=logging.INFO)
-logger = logging.getLogger()
-
-
-def generate_datasource():
-    logger.info("Creating new index")
-    # load the documents and create the index
-    documents = get_documents()
-    index = VectorStoreIndex.from_documents(
-        documents,
-    )
-    # store it for later
-    index.storage_context.persist(STORAGE_DIR)
-    logger.info(f"Finished creating new index. Stored in {STORAGE_DIR}")
-
-
-if __name__ == "__main__":
-    init_settings()
-    generate_datasource()

templates/components/vectordbs/python/none/index.py

@@ -1,20 +0,0 @@
-import logging
-import os
-
-from app.engine.constants import STORAGE_DIR
-from llama_index.core.storage import StorageContext
-from llama_index.core.indices import load_index_from_storage
-
-logger = logging.getLogger("uvicorn")
-
-
-def get_index():
-    # check if storage already exists
-    if not os.path.exists(STORAGE_DIR):
-        return None
-    # load the existing index
-    logger.info(f"Loading index from {STORAGE_DIR}...")
-    storage_context = StorageContext.from_defaults(persist_dir=STORAGE_DIR)
-    index = load_index_from_storage(storage_context)
-    logger.info(f"Finished loading index from {STORAGE_DIR}")
-    return index

templates/components/vectordbs/python/none/vectordb.py

@@ -0,0 +1,16 @@
+import os
+
+from llama_index.core.vector_stores import SimpleVectorStore
+from app.constants import STORAGE_DIR
+
+
+def get_vector_store():
+    if not os.path.exists(STORAGE_DIR):
+        # Vector store hasn't been persisted before, create a new one
+        vector_store = SimpleVectorStore()
+    else:
+        # Vector store has already been persisted before at STORAGE_DIR - load it
+        vector_store = SimpleVectorStore.from_persist_dir(
+            STORAGE_DIR, namespace="default"
+        )
+    return vector_store

templates/components/vectordbs/python/pg/constants.py

@@ -1,2 +0,0 @@
-PGVECTOR_SCHEMA = "public"
-PGVECTOR_TABLE = "llamaindex_embedding"

templates/components/vectordbs/python/pg/generate.py

@@ -1,35 +0,0 @@
-from dotenv import load_dotenv
-
-load_dotenv()
-
-import logging
-from llama_index.core.indices import VectorStoreIndex
-from llama_index.core.storage import StorageContext
-
-from app.engine.loaders import get_documents
-from app.settings import init_settings
-from app.engine.utils import init_pg_vector_store_from_env
-
-logging.basicConfig(level=logging.INFO)
-logger = logging.getLogger()
-
-
-def generate_datasource():
-    logger.info("Creating new index")
-    # load the documents and create the index
-    documents = get_documents()
-    store = init_pg_vector_store_from_env()
-    storage_context = StorageContext.from_defaults(vector_store=store)
-    VectorStoreIndex.from_documents(
-        documents,
-        storage_context=storage_context,
-        show_progress=True,  # this will show you a progress bar as the embeddings are created
-    )
-    logger.info(
-        f"Successfully created embeddings in the PG vector store, schema={store.schema_name} table={store.table_name}"
-    )
-
-
-if __name__ == "__main__":
-    init_settings()
-    generate_datasource()

templates/components/vectordbs/python/pg/index.py

@@ -1,13 +0,0 @@
-import logging
-from llama_index.core.indices.vector_store import VectorStoreIndex
-from app.engine.utils import init_pg_vector_store_from_env
-
-logger = logging.getLogger("uvicorn")
-
-
-def get_index():
-    logger.info("Connecting to index from PGVector...")
-    store = init_pg_vector_store_from_env()
-    index = VectorStoreIndex.from_vector_store(store)
-    logger.info("Finished connecting to index from PGVector.")
-    return index

templates/components/vectordbs/python/pg/vectordb.py

@@ -1,10 +1,13 @@
 import os
 from llama_index.vector_stores.postgres import PGVectorStore
 from urllib.parse import urlparse
-from app.engine.constants import PGVECTOR_SCHEMA, PGVECTOR_TABLE
+
+STORAGE_DIR = "storage"
+PGVECTOR_SCHEMA = "public"
+PGVECTOR_TABLE = "llamaindex_embedding"
 
 
-def init_pg_vector_store_from_env():
+def get_vector_store():
     original_conn_string = os.environ.get("PG_CONNECTION_STRING")
     if original_conn_string is None or original_conn_string == "":
         raise ValueError("PG_CONNECTION_STRING environment variable is not set.")
@@ -24,4 +27,5 @@ def init_pg_vector_store_from_env():
         async_connection_string=async_conn_string,
         schema_name=PGVECTOR_SCHEMA,
         table_name=PGVECTOR_TABLE,
+        embed_dim=int(os.environ.get("EMBEDDING_DIM", 768)),
     )

templates/components/vectordbs/python/pinecone/generate.py

@@ -1,39 +0,0 @@
-from dotenv import load_dotenv
-
-load_dotenv()
-
-import os
-import logging
-from llama_index.core.storage import StorageContext
-from llama_index.core.indices import VectorStoreIndex
-from llama_index.vector_stores.pinecone import PineconeVectorStore
-from app.settings import init_settings
-from app.engine.loaders import get_documents
-
-logging.basicConfig(level=logging.INFO)
-logger = logging.getLogger()
-
-
-def generate_datasource():
-    logger.info("Creating new index")
-    # load the documents and create the index
-    documents = get_documents()
-    store = PineconeVectorStore(
-        api_key=os.environ["PINECONE_API_KEY"],
-        index_name=os.environ["PINECONE_INDEX_NAME"],
-        environment=os.environ["PINECONE_ENVIRONMENT"],
-    )
-    storage_context = StorageContext.from_defaults(vector_store=store)
-    VectorStoreIndex.from_documents(
-        documents,
-        storage_context=storage_context,
-        show_progress=True,  # this will show you a progress bar as the embeddings are created
-    )
-    logger.info(
-        f"Successfully created embeddings and save to your Pinecone index {os.environ['PINECONE_INDEX_NAME']}"
-    )
-
-
-if __name__ == "__main__":
-    init_settings()
-    generate_datasource()

templates/components/vectordbs/python/pinecone/index.py

@@ -1,20 +0,0 @@
-import logging
-import os
-
-from llama_index.core.indices import VectorStoreIndex
-from llama_index.vector_stores.pinecone import PineconeVectorStore
-
-
-logger = logging.getLogger("uvicorn")
-
-
-def get_index():
-    logger.info("Connecting to index from Pinecone...")
-    store = PineconeVectorStore(
-        api_key=os.environ["PINECONE_API_KEY"],
-        index_name=os.environ["PINECONE_INDEX_NAME"],
-        environment=os.environ["PINECONE_ENVIRONMENT"],
-    )
-    index = VectorStoreIndex.from_vector_store(store)
-    logger.info("Finished connecting to index from Pinecone.")
-    return index

templates/components/vectordbs/python/pinecone/vectordb.py

@@ -0,0 +1,11 @@
+import os
+from llama_index.vector_stores.pinecone import PineconeVectorStore
+
+
+def get_vector_store():
+    store = PineconeVectorStore(
+        api_key=os.environ["PINECONE_API_KEY"],
+        index_name=os.environ["PINECONE_INDEX_NAME"],
+        environment=os.environ["PINECONE_ENVIRONMENT"],
+    )
+    return store

templates/components/vectordbs/python/qdrant/vectordb.py

@@ -0,0 +1,11 @@
+import os
+from llama_index.vector_stores.qdrant import QdrantVectorStore
+
+
+def get_vector_store():
+    store = QdrantVectorStore(
+        collection_name=os.getenv("QDRANT_COLLECTION"),
+        url=os.getenv("QDRANT_URL"),
+        api_key=os.getenv("QDRANT_API_KEY"),
+    )
+    return store

templates/components/vectordbs/typescript/astra/generate.ts

@@ -6,6 +6,7 @@ import {
   storageContextFromDefaults,
 } from "llamaindex";
 import { getDocuments } from "./loader";
+import { initSettings } from "./settings";
 import { checkRequiredEnvVars } from "./shared";
 
 dotenv.config();
@@ -18,7 +19,10 @@ async function loadAndIndex() {
   const collectionName = process.env.ASTRA_DB_COLLECTION!;
   const vectorStore = new AstraDBVectorStore();
   await vectorStore.create(collectionName, {
-    vector: { dimension: 1536, metric: "cosine" },
+    vector: {
+      dimension: parseInt(process.env.EMBEDDING_DIM!),
+      metric: "cosine",
+    },
   });
   await vectorStore.connect(collectionName);
 
@@ -33,6 +37,7 @@ async function loadAndIndex() {
 
 (async () => {
   checkRequiredEnvVars();
+  initSettings();
   await loadAndIndex();
   console.log("Finished generating storage.");
 })();

templates/components/vectordbs/typescript/astra/index.ts

@@ -1,20 +1,10 @@
 /* eslint-disable turbo/no-undeclared-env-vars */
-import {
-  AstraDBVectorStore,
-  LLM,
-  VectorStoreIndex,
-  serviceContextFromDefaults,
-} from "llamaindex";
-import { CHUNK_OVERLAP, CHUNK_SIZE, checkRequiredEnvVars } from "./shared";
+import { AstraDBVectorStore, VectorStoreIndex } from "llamaindex";
+import { checkRequiredEnvVars } from "./shared";
 
-export async function getDataSource(llm: LLM) {
+export async function getDataSource() {
   checkRequiredEnvVars();
-  const serviceContext = serviceContextFromDefaults({
-    llm,
-    chunkSize: CHUNK_SIZE,
-    chunkOverlap: CHUNK_OVERLAP,
-  });
   const store = new AstraDBVectorStore();
   await store.connect(process.env.ASTRA_DB_COLLECTION!);
-  return await VectorStoreIndex.fromVectorStore(store, serviceContext);
+  return await VectorStoreIndex.fromVectorStore(store);
 }

templates/components/vectordbs/typescript/astra/shared.ts

@@ -1,10 +1,8 @@
-export const CHUNK_SIZE = 512;
-export const CHUNK_OVERLAP = 20;
-
 const REQUIRED_ENV_VARS = [
   "ASTRA_DB_APPLICATION_TOKEN",
   "ASTRA_DB_ENDPOINT",
   "ASTRA_DB_COLLECTION",
+  "EMBEDDING_DIM",
 ];
 
 export function checkRequiredEnvVars() {

templates/components/vectordbs/typescript/milvus/generate.ts

@@ -6,6 +6,7 @@ import {
   storageContextFromDefaults,
 } from "llamaindex";
 import { getDocuments } from "./loader";
+import { initSettings } from "./settings";
 import { checkRequiredEnvVars, getMilvusClient } from "./shared";
 
 dotenv.config();
@@ -32,6 +33,7 @@ async function loadAndIndex() {
 
 (async () => {
   checkRequiredEnvVars();
+  initSettings();
   await loadAndIndex();
   console.log("Finished generating storage.");
 })();

templates/components/vectordbs/typescript/milvus/index.ts

@@ -1,25 +1,10 @@
-import {
-  LLM,
-  MilvusVectorStore,
-  serviceContextFromDefaults,
-  VectorStoreIndex,
-} from "llamaindex";
-import {
-  checkRequiredEnvVars,
-  CHUNK_OVERLAP,
-  CHUNK_SIZE,
-  getMilvusClient,
-} from "./shared";
+import { MilvusVectorStore, VectorStoreIndex } from "llamaindex";
+import { checkRequiredEnvVars, getMilvusClient } from "./shared";
 
-export async function getDataSource(llm: LLM) {
+export async function getDataSource() {
   checkRequiredEnvVars();
-  const serviceContext = serviceContextFromDefaults({
-    llm,
-    chunkSize: CHUNK_SIZE,
-    chunkOverlap: CHUNK_OVERLAP,
-  });
   const milvusClient = getMilvusClient();
   const store = new MilvusVectorStore({ milvusClient });
 
-  return await VectorStoreIndex.fromVectorStore(store, serviceContext);
+  return await VectorStoreIndex.fromVectorStore(store);
 }

templates/components/vectordbs/typescript/milvus/shared.ts

@@ -1,8 +1,5 @@
 import { MilvusClient } from "@zilliz/milvus2-sdk-node";
 
-export const CHUNK_SIZE = 512;
-export const CHUNK_OVERLAP = 20;
-
 const REQUIRED_ENV_VARS = [
   "MILVUS_ADDRESS",
   "MILVUS_USERNAME",

templates/components/vectordbs/typescript/mongo/generate.ts

@@ -7,6 +7,7 @@ import {
 } from "llamaindex";
 import { MongoClient } from "mongodb";
 import { getDocuments } from "./loader";
+import { initSettings } from "./settings";
 import { checkRequiredEnvVars } from "./shared";
 
 dotenv.config();
@@ -42,6 +43,7 @@ async function loadAndIndex() {
 
 (async () => {
   checkRequiredEnvVars();
+  initSettings();
   await loadAndIndex();
   console.log("Finished generating storage.");
 })();

templates/components/vectordbs/typescript/mongo/index.ts

@@ -1,21 +1,11 @@
 /* eslint-disable turbo/no-undeclared-env-vars */
-import {
-  LLM,
-  MongoDBAtlasVectorSearch,
-  serviceContextFromDefaults,
-  VectorStoreIndex,
-} from "llamaindex";
+import { MongoDBAtlasVectorSearch, VectorStoreIndex } from "llamaindex";
 import { MongoClient } from "mongodb";
-import { checkRequiredEnvVars, CHUNK_OVERLAP, CHUNK_SIZE } from "./shared";
+import { checkRequiredEnvVars } from "./shared";
 
-export async function getDataSource(llm: LLM) {
+export async function getDataSource() {
   checkRequiredEnvVars();
   const client = new MongoClient(process.env.MONGO_URI!);
-  const serviceContext = serviceContextFromDefaults({
-    llm,
-    chunkSize: CHUNK_SIZE,
-    chunkOverlap: CHUNK_OVERLAP,
-  });
   const store = new MongoDBAtlasVectorSearch({
     mongodbClient: client,
     dbName: process.env.MONGODB_DATABASE!,
@@ -23,5 +13,5 @@ export async function getDataSource(llm: LLM) {
     indexName: process.env.MONGODB_VECTOR_INDEX,
   });
 
-  return await VectorStoreIndex.fromVectorStore(store, serviceContext);
+  return await VectorStoreIndex.fromVectorStore(store);
 }

templates/components/vectordbs/typescript/mongo/shared.ts

@@ -1,6 +1,3 @@
-export const CHUNK_SIZE = 512;
-export const CHUNK_OVERLAP = 20;
-
 const REQUIRED_ENV_VARS = [
   "MONGO_URI",
   "MONGODB_DATABASE",

templates/components/vectordbs/typescript/none/generate.ts

@@ -1,14 +1,10 @@
-import {
-  ServiceContext,
-  serviceContextFromDefaults,
-  storageContextFromDefaults,
-  VectorStoreIndex,
-} from "llamaindex";
+import { VectorStoreIndex, storageContextFromDefaults } from "llamaindex";
 
 import * as dotenv from "dotenv";
 
 import { getDocuments } from "./loader";
-import { CHUNK_OVERLAP, CHUNK_SIZE, STORAGE_CACHE_DIR } from "./shared";
+import { initSettings } from "./settings";
+import { STORAGE_CACHE_DIR } from "./shared";
 
 // Load environment variables from local .env file
 dotenv.config();
@@ -20,7 +16,7 @@ async function getRuntime(func: any) {
   return end - start;
 }
 
-async function generateDatasource(serviceContext: ServiceContext) {
+async function generateDatasource() {
   console.log(`Generating storage context...`);
   // Split documents, create embeddings and store them in the storage context
   const ms = await getRuntime(async () => {
@@ -30,18 +26,13 @@ async function generateDatasource(serviceContext: ServiceContext) {
     const documents = await getDocuments();
     await VectorStoreIndex.fromDocuments(documents, {
       storageContext,
-      serviceContext,
     });
   });
   console.log(`Storage context successfully generated in ${ms / 1000}s.`);
 }
 
 (async () => {
-  const serviceContext = serviceContextFromDefaults({
-    chunkSize: CHUNK_SIZE,
-    chunkOverlap: CHUNK_OVERLAP,
-  });
-
-  await generateDatasource(serviceContext);
+  initSettings();
+  await generateDatasource();
   console.log("Finished generating storage.");
 })();

templates/components/vectordbs/typescript/none/index.ts

@@ -1,18 +1,11 @@
 import {
-  LLM,
-  serviceContextFromDefaults,
   SimpleDocumentStore,
   storageContextFromDefaults,
   VectorStoreIndex,
 } from "llamaindex";
-import { CHUNK_OVERLAP, CHUNK_SIZE, STORAGE_CACHE_DIR } from "./shared";
+import { STORAGE_CACHE_DIR } from "./shared";
 
-export async function getDataSource(llm: LLM) {
-  const serviceContext = serviceContextFromDefaults({
-    llm,
-    chunkSize: CHUNK_SIZE,
-    chunkOverlap: CHUNK_OVERLAP,
-  });
+export async function getDataSource() {
   const storageContext = await storageContextFromDefaults({
     persistDir: `${STORAGE_CACHE_DIR}`,
   });
@@ -25,6 +18,5 @@ export async function getDataSource(llm: LLM) {
   }
   return await VectorStoreIndex.init({
     storageContext,
-    serviceContext,
   });
 }

templates/components/vectordbs/typescript/none/shared.ts

@@ -1,3 +1 @@
 export const STORAGE_CACHE_DIR = "./cache";
-export const CHUNK_SIZE = 512;
-export const CHUNK_OVERLAP = 20;

templates/components/vectordbs/typescript/pg/generate.ts

@@ -6,6 +6,7 @@ import {
   storageContextFromDefaults,
 } from "llamaindex";
 import { getDocuments } from "./loader";
+import { initSettings } from "./settings";
 import {
   PGVECTOR_COLLECTION,
   PGVECTOR_SCHEMA,
@@ -37,6 +38,7 @@ async function loadAndIndex() {
 
 (async () => {
   checkRequiredEnvVars();
+  initSettings();
   await loadAndIndex();
   console.log("Finished generating storage.");
   process.exit(0);

templates/components/vectordbs/typescript/pg/index.ts

@@ -1,29 +1,17 @@
 /* eslint-disable turbo/no-undeclared-env-vars */
+import { PGVectorStore, VectorStoreIndex } from "llamaindex";
 import {
-  LLM,
-  PGVectorStore,
-  VectorStoreIndex,
-  serviceContextFromDefaults,
-} from "llamaindex";
-import {
-  CHUNK_OVERLAP,
-  CHUNK_SIZE,
   PGVECTOR_SCHEMA,
   PGVECTOR_TABLE,
   checkRequiredEnvVars,
 } from "./shared";
 
-export async function getDataSource(llm: LLM) {
+export async function getDataSource() {
   checkRequiredEnvVars();
   const pgvs = new PGVectorStore({
     connectionString: process.env.PG_CONNECTION_STRING,
     schemaName: PGVECTOR_SCHEMA,
     tableName: PGVECTOR_TABLE,
   });
-  const serviceContext = serviceContextFromDefaults({
-    llm,
-    chunkSize: CHUNK_SIZE,
-    chunkOverlap: CHUNK_OVERLAP,
-  });
-  return await VectorStoreIndex.fromVectorStore(pgvs, serviceContext);
+  return await VectorStoreIndex.fromVectorStore(pgvs);
 }

templates/components/vectordbs/typescript/pg/shared.ts

@@ -1,10 +1,8 @@
 export const PGVECTOR_COLLECTION = "data";
-export const CHUNK_SIZE = 512;
-export const CHUNK_OVERLAP = 20;
 export const PGVECTOR_SCHEMA = "public";
 export const PGVECTOR_TABLE = "llamaindex_embedding";
 
-const REQUIRED_ENV_VARS = ["PG_CONNECTION_STRING", "OPENAI_API_KEY"];
+const REQUIRED_ENV_VARS = ["PG_CONNECTION_STRING"];
 
 export function checkRequiredEnvVars() {
   const missingEnvVars = REQUIRED_ENV_VARS.filter((envVar) => {

templates/components/vectordbs/typescript/pinecone/generate.ts

@@ -6,6 +6,7 @@ import {
   storageContextFromDefaults,
 } from "llamaindex";
 import { getDocuments } from "./loader";
+import { initSettings } from "./settings";
 import { checkRequiredEnvVars } from "./shared";
 
 dotenv.config();
@@ -28,6 +29,7 @@ async function loadAndIndex() {
 
 (async () => {
   checkRequiredEnvVars();
+  initSettings();
   await loadAndIndex();
   console.log("Finished generating storage.");
 })();

templates/components/vectordbs/typescript/pinecone/index.ts

@@ -1,19 +1,9 @@
 /* eslint-disable turbo/no-undeclared-env-vars */
-import {
-  LLM,
-  PineconeVectorStore,
-  VectorStoreIndex,
-  serviceContextFromDefaults,
-} from "llamaindex";
-import { CHUNK_OVERLAP, CHUNK_SIZE, checkRequiredEnvVars } from "./shared";
+import { PineconeVectorStore, VectorStoreIndex } from "llamaindex";
+import { checkRequiredEnvVars } from "./shared";
 
-export async function getDataSource(llm: LLM) {
+export async function getDataSource() {
   checkRequiredEnvVars();
-  const serviceContext = serviceContextFromDefaults({
-    llm,
-    chunkSize: CHUNK_SIZE,
-    chunkOverlap: CHUNK_OVERLAP,
-  });
   const store = new PineconeVectorStore();
-  return await VectorStoreIndex.fromVectorStore(store, serviceContext);
+  return await VectorStoreIndex.fromVectorStore(store);
 }

templates/components/vectordbs/typescript/pinecone/shared.ts

@@ -1,6 +1,3 @@
-export const CHUNK_SIZE = 512;
-export const CHUNK_OVERLAP = 20;
-
 const REQUIRED_ENV_VARS = ["PINECONE_ENVIRONMENT", "PINECONE_API_KEY"];
 
 export function checkRequiredEnvVars() {

templates/components/vectordbs/typescript/qdrant/generate.ts

@@ -0,0 +1,37 @@
+/* eslint-disable turbo/no-undeclared-env-vars */
+import * as dotenv from "dotenv";
+import {
+  QdrantVectorStore,
+  VectorStoreIndex,
+  storageContextFromDefaults,
+} from "llamaindex";
+import { getDocuments } from "./loader";
+import { initSettings } from "./settings";
+import { checkRequiredEnvVars, getQdrantClient } from "./shared";
+
+dotenv.config();
+
+const collectionName = process.env.QDRANT_COLLECTION;
+
+async function loadAndIndex() {
+  // load objects from storage and convert them into LlamaIndex Document objects
+  const documents = await getDocuments();
+
+  // Connect to Qdrant
+  const vectorStore = new QdrantVectorStore(collectionName, getQdrantClient());
+
+  const storageContext = await storageContextFromDefaults({ vectorStore });
+  await VectorStoreIndex.fromDocuments(documents, {
+    storageContext: storageContext,
+  });
+  console.log(
+    `Successfully upload embeddings to Qdrant collection ${collectionName}.`,
+  );
+}
+
+(async () => {
+  checkRequiredEnvVars();
+  initSettings();
+  await loadAndIndex();
+  console.log("Finished generating storage.");
+})();

templates/components/vectordbs/typescript/qdrant/index.ts

@@ -0,0 +1,13 @@
+import * as dotenv from "dotenv";
+import { QdrantVectorStore, VectorStoreIndex } from "llamaindex";
+import { checkRequiredEnvVars, getQdrantClient } from "./shared";
+
+dotenv.config();
+
+export async function getDataSource() {
+  checkRequiredEnvVars();
+  const collectionName = process.env.QDRANT_COLLECTION;
+  const store = new QdrantVectorStore(collectionName, getQdrantClient());
+
+  return await VectorStoreIndex.fromVectorStore(store);
+}

templates/components/vectordbs/typescript/qdrant/shared.ts

@@ -0,0 +1,32 @@
+import { QdrantClient } from "@qdrant/js-client-rest";
+
+const REQUIRED_ENV_VARS = ["QDRANT_URL", "QDRANT_COLLECTION"]; // QDRANT_API_KEY is optional
+
+export function getQdrantClient() {
+  const url = process.env.QDRANT_URL;
+  if (!url) {
+    throw new Error("QDRANT_URL environment variable is required");
+  }
+  const apiKey = process.env?.QDRANT_API_KEY;
+  return new QdrantClient({
+    url,
+    apiKey,
+  });
+}
+
+export function checkRequiredEnvVars() {
+  const missingEnvVars = REQUIRED_ENV_VARS.filter((envVar) => {
+    return !process.env[envVar];
+  });
+
+  if (missingEnvVars.length > 0) {
+    console.log(
+      `The following environment variables are required but missing: ${missingEnvVars.join(
+        ", ",
+      )}`,
+    );
+    throw new Error(
+      `Missing environment variables: ${missingEnvVars.join(", ")}`,
+    );
+  }
+}

templates/types/streaming/express/npmrc

@@ -0,0 +1 @@
+node-linker=hoisted

templates/types/streaming/express/package.json

@@ -10,11 +10,11 @@
     "dev": "concurrently \"tsup index.ts --format esm --dts --watch\" \"nodemon -q dist/index.mjs\""
   },
   "dependencies": {
-    "ai": "^2.2.25",
+    "ai": "^3.0.21",
     "cors": "^2.8.5",
     "dotenv": "^16.3.1",
     "express": "^4.18.2",
-    "llamaindex": "latest"
+    "llamaindex": "0.2.10"
   },
   "devDependencies": {
     "@types/cors": "^2.8.16",
@@ -22,12 +22,12 @@
     "@types/node": "^20.9.5",
     "concurrently": "^8.2.2",
     "eslint": "^8.54.0",
+    "eslint-config-prettier": "^8.10.0",
     "nodemon": "^3.0.1",
-    "tsup": "^8.0.1",
-    "typescript": "^5.3.2",
     "prettier": "^3.2.5",
     "prettier-plugin-organize-imports": "^3.2.4",
-    "eslint-config-prettier": "^8.10.0",
-    "ts-node": "^10.9.2"
+    "tsx": "^4.7.2",
+    "tsup": "^8.0.1",
+    "typescript": "^5.3.2"
   }
 }

templates/types/streaming/express/src/controllers/chat-request.controller.ts

@@ -1,5 +1,5 @@
 import { Request, Response } from "express";
-import { ChatMessage, MessageContent, OpenAI } from "llamaindex";
+import { ChatMessage, MessageContent } from "llamaindex";
 import { createChatEngine } from "./engine/chat";
 
 const convertMessageContent = (
@@ -32,10 +32,6 @@ export const chatRequest = async (req: Request, res: Response) => {
       });
     }
 
-    const llm = new OpenAI({
-      model: process.env.MODEL || "gpt-3.5-turbo",
-    });
-
     // Convert message content from Vercel/AI format to LlamaIndex/OpenAI format
     // Note: The non-streaming template does not need the Vercel/AI format, we're still using it for consistency with the streaming template
     const userMessageContent = convertMessageContent(
@@ -43,7 +39,7 @@ export const chatRequest = async (req: Request, res: Response) => {
       data?.imageUrl,
     );
 
-    const chatEngine = await createChatEngine(llm);
+    const chatEngine = await createChatEngine();
 
     // Calling LlamaIndex's ChatEngine to get a response
     const response = await chatEngine.chat({
@@ -61,7 +57,7 @@ export const chatRequest = async (req: Request, res: Response) => {
   } catch (error) {
     console.error("[LlamaIndex]", error);
     return res.status(500).json({
-      error: (error as Error).message,
+      detail: (error as Error).message,
     });
   }
 };

templates/types/streaming/express/src/controllers/chat.controller.ts

@@ -1,8 +1,9 @@
-import { streamToResponse } from "ai";
+import { Message, StreamData, streamToResponse } from "ai";
 import { Request, Response } from "express";
-import { ChatMessage, MessageContent, OpenAI } from "llamaindex";
+import { ChatMessage, MessageContent, Settings } from "llamaindex";
 import { createChatEngine } from "./engine/chat";
 import { LlamaIndexStream } from "./llamaindex-stream";
+import { appendEventData } from "./stream-helper";
 
 const convertMessageContent = (
   textMessage: string,
@@ -25,7 +26,7 @@ const convertMessageContent = (
 
 export const chat = async (req: Request, res: Response) => {
   try {
-    const { messages, data }: { messages: ChatMessage[]; data: any } = req.body;
+    const { messages, data }: { messages: Message[]; data: any } = req.body;
     const userMessage = messages.pop();
     if (!messages || !userMessage || userMessage.role !== "user") {
       return res.status(400).json({
@@ -34,11 +35,7 @@ export const chat = async (req: Request, res: Response) => {
       });
     }
 
-    const llm = new OpenAI({
-      model: (process.env.MODEL as any) || "gpt-3.5-turbo",
-    });
-
-    const chatEngine = await createChatEngine(llm);
+    const chatEngine = await createChatEngine();
 
     // Convert message content from Vercel/AI format to LlamaIndex/OpenAI format
     const userMessageContent = convertMessageContent(
@@ -46,22 +43,38 @@ export const chat = async (req: Request, res: Response) => {
       data?.imageUrl,
     );
 
+    // Init Vercel AI StreamData
+    const vercelStreamData = new StreamData();
+    appendEventData(
+      vercelStreamData,
+      `Retrieving context for query: '${userMessage.content}'`,
+    );
+
+    // Setup callback for streaming data before chatting
+    Settings.callbackManager.on("retrieve", (data) => {
+      const { nodes } = data.detail;
+      appendEventData(
+        vercelStreamData,
+        `Retrieved ${nodes.length} sources to use as context for the query`,
+      );
+    });
+
     // Calling LlamaIndex's ChatEngine to get a streamed response
     const response = await chatEngine.chat({
       message: userMessageContent,
-      chatHistory: messages,
+      chatHistory: messages as ChatMessage[],
       stream: true,
     });
 
     // Return a stream, which can be consumed by the Vercel/AI client
-    const { stream, data: streamData } = LlamaIndexStream(response, {
+    const { stream } = LlamaIndexStream(response, vercelStreamData, {
       parserOptions: {
         image_url: data?.imageUrl,
       },
     });
 
     // Pipe LlamaIndexStream to response
-    const processedStream = stream.pipeThrough(streamData.stream);
+    const processedStream = stream.pipeThrough(vercelStreamData.stream);
     return streamToResponse(processedStream, res, {
       headers: {
         // response MUST have the `X-Experimental-Stream-Data: 'true'` header
@@ -75,7 +88,7 @@ export const chat = async (req: Request, res: Response) => {
   } catch (error) {
     console.error("[LlamaIndex]", error);
     return res.status(500).json({
-      error: (error as Error).message,
+      detail: (error as Error).message,
     });
   }
 };

templates/types/streaming/express/src/controllers/engine/chat.ts

@@ -1,7 +1,7 @@
-import { LLM, SimpleChatEngine } from "llamaindex";
+import { Settings, SimpleChatEngine } from "llamaindex";
 
-export async function createChatEngine(llm: LLM) {
+export async function createChatEngine() {
   return new SimpleChatEngine({
-    llm,
+    llm: Settings.llm,
   });
 }

templates/types/streaming/express/src/controllers/engine/settings.ts

@@ -0,0 +1,52 @@
+import {
+  Ollama,
+  OllamaEmbedding,
+  OpenAI,
+  OpenAIEmbedding,
+  Settings,
+} from "llamaindex";
+
+const CHUNK_SIZE = 512;
+const CHUNK_OVERLAP = 20;
+
+export const initSettings = async () => {
+  // HINT: you can delete the initialization code for unused model providers
+  console.log(`Using '${process.env.MODEL_PROVIDER}' model provider`);
+  switch (process.env.MODEL_PROVIDER) {
+    case "ollama":
+      initOllama();
+      break;
+    default:
+      initOpenAI();
+      break;
+  }
+  Settings.chunkSize = CHUNK_SIZE;
+  Settings.chunkOverlap = CHUNK_OVERLAP;
+};
+
+function initOpenAI() {
+  Settings.llm = new OpenAI({
+    model: process.env.MODEL ?? "gpt-3.5-turbo",
+    maxTokens: 512,
+  });
+  Settings.embedModel = new OpenAIEmbedding({
+    model: process.env.EMBEDDING_MODEL,
+    dimensions: process.env.EMBEDDING_DIM
+      ? parseInt(process.env.EMBEDDING_DIM)
+      : undefined,
+  });
+}
+
+function initOllama() {
+  if (!process.env.MODEL || !process.env.EMBEDDING_MODEL) {
+    throw new Error(
+      "Using Ollama as model provider, 'MODEL' and 'EMBEDDING_MODEL' env variables must be set.",
+    );
+  }
+  Settings.llm = new Ollama({
+    model: process.env.MODEL ?? "",
+  });
+  Settings.embedModel = new OllamaEmbedding({
+    model: process.env.EMBEDDING_MODEL ?? "",
+  });
+}

templates/types/streaming/express/src/controllers/llamaindex-stream.ts

@@ -1,12 +1,17 @@
 import {
-  JSONValue,
+  StreamData,
   createCallbacksTransformer,
   createStreamDataTransformer,
-  experimental_StreamData,
   trimStartOfStreamHelper,
   type AIStreamCallbacksAndOptions,
 } from "ai";
-import { Response, StreamingAgentChatResponse } from "llamaindex";
+import {
+  Metadata,
+  NodeWithScore,
+  Response,
+  StreamingAgentChatResponse,
+} from "llamaindex";
+import { appendImageData, appendSourceData } from "./stream-helper";
 
 type ParserOptions = {
   image_url?: string;
@@ -14,35 +19,30 @@ type ParserOptions = {
 
 function createParser(
   res: AsyncIterable<Response>,
-  data: experimental_StreamData,
+  data: StreamData,
   opts?: ParserOptions,
 ) {
   const it = res[Symbol.asyncIterator]();
   const trimStartOfStream = trimStartOfStreamHelper();
+
+  let sourceNodes: NodeWithScore<Metadata>[] | undefined;
   return new ReadableStream<string>({
     start() {
-      // if image_url is provided, send it via the data stream
-      if (opts?.image_url) {
-        const message: JSONValue = {
-          type: "image_url",
-          image_url: {
-            url: opts.image_url,
-          },
-        };
-        data.append(message);
-      } else {
-        data.append({}); // send an empty image response for the user's message
-      }
+      appendImageData(data, opts?.image_url);
     },
     async pull(controller): Promise<void> {
       const { value, done } = await it.next();
       if (done) {
+        appendSourceData(data, sourceNodes);
         controller.close();
-        data.append({}); // send an empty image response for the assistant's message
         data.close();
         return;
       }
 
+      if (!sourceNodes) {
+        // get source nodes from the first response
+        sourceNodes = value.sourceNodes;
+      }
       const text = trimStartOfStream(value.response ?? "");
       if (text) {
         controller.enqueue(text);
@@ -53,12 +53,12 @@ function createParser(
 
 export function LlamaIndexStream(
   response: StreamingAgentChatResponse | AsyncIterable<Response>,
+  data: StreamData,
   opts?: {
     callbacks?: AIStreamCallbacksAndOptions;
     parserOptions?: ParserOptions;
   },
-): { stream: ReadableStream; data: experimental_StreamData } {
-  const data = new experimental_StreamData();
+): { stream: ReadableStream; data: StreamData } {
   const res =
     response instanceof StreamingAgentChatResponse
       ? response.response
@@ -66,7 +66,7 @@ export function LlamaIndexStream(
   return {
     stream: createParser(res, data, opts?.parserOptions)
       .pipeThrough(createCallbacksTransformer(opts?.callbacks))
-      .pipeThrough(createStreamDataTransformer(true)),
+      .pipeThrough(createStreamDataTransformer()),
     data,
   };
 }

templates/types/streaming/express/src/controllers/stream-helper.ts

@@ -0,0 +1,39 @@
+import { StreamData } from "ai";
+import { Metadata, NodeWithScore } from "llamaindex";
+
+export function appendImageData(data: StreamData, imageUrl?: string) {
+  if (!imageUrl) return;
+  data.appendMessageAnnotation({
+    type: "image",
+    data: {
+      url: imageUrl,
+    },
+  });
+}
+
+export function appendSourceData(
+  data: StreamData,
+  sourceNodes?: NodeWithScore<Metadata>[],
+) {
+  if (!sourceNodes?.length) return;
+  data.appendMessageAnnotation({
+    type: "sources",
+    data: {
+      nodes: sourceNodes.map((node) => ({
+        ...node.node.toMutableJSON(),
+        id: node.node.id_,
+        score: node.score ?? null,
+      })),
+    },
+  });
+}
+
+export function appendEventData(data: StreamData, title?: string) {
+  if (!title) return;
+  data.appendMessageAnnotation({
+    type: "events",
+    data: {
+      title,
+    },
+  });
+}

templates/types/streaming/express/src/routes/chat.route.ts

@@ -1,9 +1,11 @@
 import express, { Router } from "express";
 import { chatRequest } from "../controllers/chat-request.controller";
 import { chat } from "../controllers/chat.controller";
+import { initSettings } from "../controllers/engine/settings";
 
 const llmRouter: Router = express.Router();
 
+initSettings();
 llmRouter.route("/").post(chat);
 llmRouter.route("/request").post(chatRequest);
 

templates/types/streaming/express/tsconfig.json

@@ -9,10 +9,5 @@
     "paths": {
       "@/*": ["./*"]
     }
-  },
-  "ts-node": {
-    "compilerOptions": {
-      "module": "commonjs"
-    }
   }
 }

templates/types/streaming/fastapi/README-template.md

@@ -11,20 +11,14 @@ poetry install
 poetry shell
 ```
 
-By default, we use the OpenAI LLM (though you can customize, see `app/settings.py`). As a result, you need to specify an `OPENAI_API_KEY` in an .env file in this directory.
-
-Example `.env` file:
-
-```
-OPENAI_API_KEY=<openai_api_key>
-```
+Then check the parameters that have been pre-configured in the `.env` file in this directory. (E.g. you might need to configure an `OPENAI_API_KEY` if you're using OpenAI as model provider).
 
 If you are using any tools or data sources, you can update their config files in the `config` folder.
 
 Second, generate the embeddings of the documents in the `./data` directory (if this folder exists - otherwise, skip this step):
 
 ```
-python app/engine/generate.py
+poetry run generate
 ```
 
 Third, run the development server:
@@ -84,7 +78,7 @@ docker run \
   -v $(pwd)/data:/app/data \ # Use your local folder to read the data
   -v $(pwd)/storage:/app/storage \ # Use your file system to store the vector database
   <your_backend_image_name> \
-  python app/engine/generate.py
+  poetry run generate
 ```
 
 3. Start the API:

templates/types/streaming/fastapi/app/api/routers/chat.py

@@ -1,13 +1,16 @@
 from pydantic import BaseModel
 from typing import List, Any, Optional, Dict, Tuple
 from fastapi import APIRouter, Depends, HTTPException, Request, status
-from fastapi.responses import StreamingResponse
 from llama_index.core.chat_engine.types import (
     BaseChatEngine,
+    StreamingAgentChatResponse,
 )
 from llama_index.core.schema import NodeWithScore
 from llama_index.core.llms import ChatMessage, MessageRole
 from app.engine import get_chat_engine
+from app.api.routers.vercel_response import VercelStreamResponse
+from app.api.routers.messaging import EventCallbackHandler
+from aiostream import stream
 
 chat_router = r = APIRouter()
 
@@ -37,6 +40,7 @@ class _SourceNodes(BaseModel):
     id: str
     metadata: Dict[str, Any]
     score: Optional[float]
+    text: str
 
     @classmethod
     def from_source_node(cls, source_node: NodeWithScore):
@@ -44,6 +48,7 @@ class _SourceNodes(BaseModel):
             id=source_node.node.node_id,
             metadata=source_node.node.metadata,
             score=source_node.score,
+            text=source_node.node.text,  # type: ignore
         )
 
     @classmethod
@@ -89,15 +94,49 @@ async def chat(
 ):
     last_message_content, messages = await parse_chat_data(data)
 
+    event_handler = EventCallbackHandler()
+    chat_engine.callback_manager.handlers.append(event_handler)  # type: ignore
     response = await chat_engine.astream_chat(last_message_content, messages)
 
-    async def event_generator():
-        async for token in response.async_response_gen():
-            if await request.is_disconnected():
-                break
-            yield token
+    async def content_generator():
+        # Yield the text response
+        async def _text_generator():
+            async for token in response.async_response_gen():
+                yield VercelStreamResponse.convert_text(token)
+            # the text_generator is the leading stream, once it's finished, also finish the event stream
+            event_handler.is_done = True
 
-    return StreamingResponse(event_generator(), media_type="text/plain")
+        # Yield the events from the event handler
+        async def _event_generator():
+            async for event in event_handler.async_event_gen():
+                yield VercelStreamResponse.convert_data(
+                    {
+                        "type": "events",
+                        "data": {"title": event.get_title()},
+                    }
+                )
+
+        combine = stream.merge(_text_generator(), _event_generator())
+        async with combine.stream() as streamer:
+            async for item in streamer:
+                if await request.is_disconnected():
+                    break
+                yield item
+
+        # Yield the source nodes
+        yield VercelStreamResponse.convert_data(
+            {
+                "type": "sources",
+                "data": {
+                    "nodes": [
+                        _SourceNodes.from_source_node(node).dict()
+                        for node in response.source_nodes
+                    ]
+                },
+            }
+        )
+
+    return VercelStreamResponse(content=content_generator())
 
 
 # non-streaming endpoint - delete if not needed

templates/types/streaming/fastapi/app/api/routers/messaging.py

@@ -0,0 +1,86 @@
+import asyncio
+from typing import AsyncGenerator, Dict, Any, List, Optional
+
+from llama_index.core.callbacks.base import BaseCallbackHandler
+from llama_index.core.callbacks.schema import CBEventType
+from pydantic import BaseModel
+
+
+class CallbackEvent(BaseModel):
+    event_type: CBEventType
+    payload: Optional[Dict[str, Any]] = None
+    event_id: str = ""
+
+    def get_title(self) -> str | None:
+        # Return as None for the unhandled event types
+        # to avoid showing them in the UI
+        match self.event_type:
+            case "retrieve":
+                if self.payload:
+                    nodes = self.payload.get("nodes")
+                    if nodes:
+                        return f"Retrieved {len(nodes)} sources to use as context for the query"
+                    else:
+                        return f"Retrieving context for query: '{self.payload.get('query_str')}'"
+                else:
+                    return None
+            case _:
+                return None
+
+
+class EventCallbackHandler(BaseCallbackHandler):
+    _aqueue: asyncio.Queue
+    is_done: bool = False
+
+    def __init__(
+        self,
+    ):
+        """Initialize the base callback handler."""
+        ignored_events = [
+            CBEventType.CHUNKING,
+            CBEventType.NODE_PARSING,
+            CBEventType.EMBEDDING,
+            CBEventType.LLM,
+            CBEventType.TEMPLATING,
+        ]
+        super().__init__(ignored_events, ignored_events)
+        self._aqueue = asyncio.Queue()
+
+    def on_event_start(
+        self,
+        event_type: CBEventType,
+        payload: Optional[Dict[str, Any]] = None,
+        event_id: str = "",
+        **kwargs: Any,
+    ) -> str:
+        event = CallbackEvent(event_id=event_id, event_type=event_type, payload=payload)
+        if event.get_title() is not None:
+            self._aqueue.put_nowait(event)
+
+    def on_event_end(
+        self,
+        event_type: CBEventType,
+        payload: Optional[Dict[str, Any]] = None,
+        event_id: str = "",
+        **kwargs: Any,
+    ) -> None:
+        event = CallbackEvent(event_id=event_id, event_type=event_type, payload=payload)
+        if event.get_title() is not None:
+            self._aqueue.put_nowait(event)
+
+    def start_trace(self, trace_id: Optional[str] = None) -> None:
+        """No-op."""
+
+    def end_trace(
+        self,
+        trace_id: Optional[str] = None,
+        trace_map: Optional[Dict[str, List[str]]] = None,
+    ) -> None:
+        """No-op."""
+
+    async def async_event_gen(self) -> AsyncGenerator[CallbackEvent, None]:
+        while not self._aqueue.empty() or not self.is_done:
+            try:
+                yield await asyncio.wait_for(self._aqueue.get(), timeout=0.1)
+            except asyncio.TimeoutError:
+                pass

templates/types/streaming/fastapi/app/api/routers/vercel_response.py

@@ -0,0 +1,29 @@
+import json
+from typing import Any
+from fastapi.responses import StreamingResponse
+
+
+class VercelStreamResponse(StreamingResponse):
+    """
+    Class to convert the response from the chat engine to the streaming format expected by Vercel
+    """
+
+    TEXT_PREFIX = "0:"
+    DATA_PREFIX = "8:"
+
+    @classmethod
+    def convert_text(cls, token: str):
+        # Escape newlines and double quotes to avoid breaking the stream
+        token = json.dumps(token)
+        return f"{cls.TEXT_PREFIX}{token}\n"
+
+    @classmethod
+    def convert_data(cls, data: dict):
+        data_str = json.dumps(data)
+        return f"{cls.DATA_PREFIX}[{data_str}]\n"
+
+    def __init__(self, content: Any, **kwargs):
+        super().__init__(
+            content=content,
+            **kwargs,
+        )

templates/types/streaming/fastapi/app/constants.py

@@ -0,0 +1 @@
+STORAGE_DIR = "storage"  # directory to save the stores to (document store and if used, the `SimpleVectorStore`)

templates/types/streaming/fastapi/app/engine/generate.py

@@ -0,0 +1,96 @@
+from dotenv import load_dotenv
+
+load_dotenv()
+
+import os
+import logging
+from llama_index.core.settings import Settings
+from llama_index.core.ingestion import IngestionPipeline
+from llama_index.core.node_parser import SentenceSplitter
+from llama_index.core.vector_stores import SimpleVectorStore
+from llama_index.core.storage.docstore import SimpleDocumentStore
+from llama_index.core.storage import StorageContext
+from llama_index.core import VectorStoreIndex
+from app.constants import STORAGE_DIR
+from app.settings import init_settings
+from app.engine.loaders import get_documents
+from app.engine.vectordb import get_vector_store
+
+
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger()
+
+
+def get_doc_store():
+    if not os.path.exists(STORAGE_DIR):
+        docstore = SimpleDocumentStore()
+        return docstore
+    else:
+        return SimpleDocumentStore.from_persist_dir(STORAGE_DIR)
+
+
+def run_ingestion_pipeline(docstore, vector_store, documents):
+    # Create ingestion pipeline
+    ingestion_pipeline = IngestionPipeline(
+        transformations=[
+            SentenceSplitter(
+                chunk_size=Settings.chunk_size,
+                chunk_overlap=Settings.chunk_overlap,
+            ),
+            Settings.embed_model,
+        ],
+        docstore=docstore,
+        docstore_strategy="upserts_and_delete",
+    )
+
+    # llama_index having an typing issue when passing vector_store to IngestionPipeline
+    # so we need to set it manually after initialization
+    ingestion_pipeline.vector_store = vector_store
+
+    # Run the ingestion pipeline and store the results
+    nodes = ingestion_pipeline.run(show_progress=True, documents=documents)
+
+    return nodes
+
+
+def persist_storage(docstore, vector_store, nodes):
+    storage_context = StorageContext.from_defaults(
+        docstore=docstore,
+        vector_store=vector_store,
+    )
+    # SimpleVectorStore does not include index by default
+    # so we need to create the index manually
+    # can be removed if using other vector store
+    if isinstance(vector_store, SimpleVectorStore):
+        VectorStoreIndex(
+            nodes=nodes,
+            storage_context=storage_context,
+            store_nodes_override=True,  # Need enable this to store the nodes and index's id
+        )
+    storage_context.persist(STORAGE_DIR)
+
+
+def generate_datasource():
+    init_settings()
+    logger.info("Generate index for the provided data")
+
+    # Get the stores and documents or create new ones
+    documents = get_documents()
+    docstore = get_doc_store()
+    vector_store = get_vector_store()
+
+    # Run the ingestion pipeline
+    nodes = run_ingestion_pipeline(
+        docstore=docstore,
+        vector_store=vector_store,
+        documents=documents,
+    )
+
+    # Build the index and persist storage
+    persist_storage(docstore, vector_store, nodes)
+
+    logger.info("Finished generating the index")
+
+
+if __name__ == "__main__":
+    generate_datasource()

templates/types/streaming/fastapi/app/engine/index.py

@@ -0,0 +1,27 @@
+import logging
+from llama_index.core import load_index_from_storage
+from llama_index.core.storage import StorageContext
+from llama_index.core.indices.vector_store import VectorStoreIndex
+from llama_index.core.vector_stores.simple import SimpleVectorStore
+from app.constants import STORAGE_DIR
+from app.engine.vectordb import get_vector_store
+
+logger = logging.getLogger("uvicorn")
+
+
+def get_index():
+    logger.info("Loading the index...")
+    store = get_vector_store()
+    # If the store is a SimpleVectorStore, we need to load the index from the storage
+    if isinstance(store, SimpleVectorStore):
+        index = load_index_from_storage(
+            StorageContext.from_defaults(
+                vector_store=store,
+                persist_dir=STORAGE_DIR,
+            )
+        )
+    else:
+        index = VectorStoreIndex.from_vector_store(store)
+
+    logger.info("Loaded index successfully.")
+    return index

templates/types/streaming/fastapi/app/observability.py

@@ -0,0 +1,2 @@
+def init_observability():
+    pass

templates/types/streaming/fastapi/app/settings.py

@@ -1,41 +1,44 @@
 import os
 from typing import Dict
 from llama_index.core.settings import Settings
-from llama_index.llms.openai import OpenAI
-from llama_index.embeddings.openai import OpenAIEmbedding
-
-
-def llm_config_from_env() -> Dict:
-    from llama_index.core.constants import DEFAULT_TEMPERATURE
-
-    model = os.getenv("MODEL")
-    temperature = os.getenv("LLM_TEMPERATURE", DEFAULT_TEMPERATURE)
-    max_tokens = os.getenv("LLM_MAX_TOKENS")
-
-    config = {
-        "model": model,
-        "temperature": float(temperature),
-        "max_tokens": int(max_tokens) if max_tokens is not None else None,
-    }
-    return config
-
-
-def embedding_config_from_env() -> Dict:
-    model = os.getenv("EMBEDDING_MODEL")
-    dimension = os.getenv("EMBEDDING_DIM")
-
-    config = {
-        "model": model,
-        "dimension": int(dimension) if dimension is not None else None,
-    }
-    return config
 
 
 def init_settings():
-    llm_configs = llm_config_from_env()
-    embedding_configs = embedding_config_from_env()
-
-    Settings.llm = OpenAI(**llm_configs)
-    Settings.embed_model = OpenAIEmbedding(**embedding_configs)
+    model_provider = os.getenv("MODEL_PROVIDER")
+    if model_provider == "openai":
+        init_openai()
+    elif model_provider == "ollama":
+        init_ollama()
+    else:
+        raise ValueError(f"Invalid model provider: {model_provider}")
     Settings.chunk_size = int(os.getenv("CHUNK_SIZE", "1024"))
     Settings.chunk_overlap = int(os.getenv("CHUNK_OVERLAP", "20"))
+
+
+def init_ollama():
+    from llama_index.llms.ollama import Ollama
+    from llama_index.embeddings.ollama import OllamaEmbedding
+
+    Settings.embed_model = OllamaEmbedding(model_name=os.getenv("EMBEDDING_MODEL"))
+    Settings.llm = Ollama(model=os.getenv("MODEL"))
+
+
+def init_openai():
+    from llama_index.llms.openai import OpenAI
+    from llama_index.embeddings.openai import OpenAIEmbedding
+    from llama_index.core.constants import DEFAULT_TEMPERATURE
+
+    max_tokens = os.getenv("LLM_MAX_TOKENS")
+    config = {
+        "model": os.getenv("MODEL"),
+        "temperature": float(os.getenv("LLM_TEMPERATURE", DEFAULT_TEMPERATURE)),
+        "max_tokens": int(max_tokens) if max_tokens is not None else None,
+    }
+    Settings.llm = OpenAI(**config)
+
+    dimensions = os.getenv("EMBEDDING_DIM")
+    config = {
+        "model": os.getenv("EMBEDDING_MODEL"),
+        "dimensions": int(dimensions) if dimensions is not None else None,
+    }
+    Settings.embed_model = OpenAIEmbedding(**config)

templates/types/streaming/fastapi/main.py

@@ -10,11 +10,13 @@ from fastapi.middleware.cors import CORSMiddleware
 from fastapi.responses import RedirectResponse
 from app.api.routers.chat import chat_router
 from app.settings import init_settings
+from app.observability import init_observability
 
 
 app = FastAPI()
 
 init_settings()
+init_observability()
 
 environment = os.getenv("ENVIRONMENT", "dev")  # Default to 'development' if not set
 

templates/types/streaming/fastapi/pyproject.toml

@@ -5,14 +5,17 @@ description = ""
 authors = ["Marcus Schiesser <mail@marcusschiesser.de>"]
 readme = "README.md"
 
+[tool.poetry.scripts]
+generate = "app.engine.generate:generate_datasource"
+
 [tool.poetry.dependencies]
 python = "^3.11,<3.12"
 fastapi = "^0.109.1"
 uvicorn = { extras = ["standard"], version = "^0.23.2" }
 python-dotenv = "^1.0.0"
-llama-index = "0.10.15"
-llama-index-core = "0.10.15"
-llama-index-agent-openai = "0.1.5"
+aiostream = "^0.5.2"
+llama-index = "0.10.28"
+llama-index-core = "0.10.28"
 
 [build-system]
 requires = ["poetry-core"]

templates/types/streaming/nextjs/app/api/chat/engine/chat.ts

@@ -1,7 +1,7 @@
-import { LLM, SimpleChatEngine } from "llamaindex";
+import { Settings, SimpleChatEngine } from "llamaindex";
 
-export async function createChatEngine(llm: LLM) {
+export async function createChatEngine() {
   return new SimpleChatEngine({
-    llm,
+    llm: Settings.llm,
   });
 }