Release 0.1.28 (#206)

Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>

CHANGELOG.md

@@ -1,5 +1,44 @@
 # create-llama
 
+## 0.1.28
+
+### Patch Changes
+
+- 9a09e8c: Fix Vercel deployment
+
+## 0.1.27
+
+### Patch Changes
+
+- c5c7eee: Make components reusable for chat-llamaindex
+
+## 0.1.26
+
+### Patch Changes
+
+- f43399c: Add metadatafilters to context chat engine (Typescript)
+
+## 0.1.25
+
+### Patch Changes
+
+- c67daeb: fix: missing set private to false for default generate.py
+
+## 0.1.24
+
+### Patch Changes
+
+- 43474a5: Configure LlamaCloud organization ID for Python
+- cf11b23: Add Azure code interpreter for Python and TS
+- fd9fb42: Add Azure OpenAI as model provider
+- 5c13646: Fix starter questions not working in python backend
+
+## 0.1.23
+
+### Patch Changes
+
+- 6bd76fb: Add template for structured extraction
+
 ## 0.1.22
 
 ### Patch Changes

README.md

@@ -2,8 +2,6 @@
 
 The easiest way to get started with [LlamaIndex](https://www.llamaindex.ai/) is by using `create-llama`. This CLI tool enables you to quickly start building a new LlamaIndex application, with everything set up for you.
 
-https://github.com/user-attachments/assets/dd3edc36-4453-4416-91c2-d24326c6c167
-
 ## Get started
 
 Just run
@@ -12,7 +10,11 @@ Just run
 npx create-llama@latest
 ```
 
-to get started, or see below for more options. Once your app is generated, run
+to get started, or watch this video for a demo session:
+
+https://github.com/user-attachments/assets/dd3edc36-4453-4416-91c2-d24326c6c167
+
+Once your app is generated, run
 
 ```bash
 npm run dev
@@ -22,16 +24,20 @@ to start the development server. You can then visit [http://localhost:3000](http
 
 ## What you'll get
 
-- A Next.js-powered front-end using components from [shadcn/ui](https://ui.shadcn.com/). The app is set up as a chat interface that can answer questions about your data (see below)
+- A Next.js-powered front-end using components from [shadcn/ui](https://ui.shadcn.com/). The app is set up as a chat interface that can answer questions about your data or interact with your agent
 - Your choice of 3 back-ends:
   - **Next.js**: if you select this option, you’ll have a full-stack Next.js application that you can deploy to a host like [Vercel](https://vercel.com/) in just a few clicks. This uses [LlamaIndex.TS](https://www.npmjs.com/package/llamaindex), our TypeScript library.
   - **Express**: if you want a more traditional Node.js application you can generate an Express backend. This also uses LlamaIndex.TS.
-  - **Python FastAPI**: if you select this option, you’ll get a backend powered by the [llama-index python package](https://pypi.org/project/llama-index/), which you can deploy to a service like Render or fly.io.
+  - **Python FastAPI**: if you select this option, you’ll get a backend powered by the [llama-index Python package](https://pypi.org/project/llama-index/), which you can deploy to a service like Render or fly.io.
 - The back-end has two endpoints (one streaming, the other one non-streaming) that allow you to send the state of your chat and receive additional responses
 - You add arbitrary data sources to your chat, like local files, websites, or data retrieved from a database.
 - Turn your chat into an AI agent by adding tools (functions called by the LLM).
 - The app uses OpenAI by default, so you'll need an OpenAI API key, or you can customize it to use any of the dozens of LLMs we support.
 
+Here's how it looks like:
+
+https://github.com/user-attachments/assets/d57af1a1-d99b-4e9c-98d9-4cbd1327eff8
+
 ## Using your data
 
 You can supply your own data; the app will index it and answer questions. Your generated app will have a folder called `data` (If you're using Express or Python and generate a frontend, it will be `./backend/data`).
@@ -88,7 +94,7 @@ Need to install the following packages:
   create-llama@latest
 Ok to proceed? (y) y
 ✔ What is your project named? … my-app
-✔ Which template would you like to use? › Chat
+✔ Which template would you like to use? › Agentic RAG (single agent)
 ✔ Which framework would you like to use? › NextJS
 ✔ Would you like to set up observability? › No
 ✔ Please provide your OpenAI API key (leave blank to skip): …
@@ -96,6 +102,7 @@ Ok to proceed? (y) y
 ✔ Would you like to add another data source? › No
 ✔ Would you like to use LlamaParse (improved parser for RAG - requires API key)? … no / yes
 ✔ Would you like to use a vector database? › No, just store the data in the file system
+✔ Would you like to build an agent using tools? If so, select the tools here, otherwise just press enter › Weather
 ? How would you like to proceed? › - Use arrow-keys. Return to submit.
    Just generate code (~1 sec)
 ❯  Start in VSCode (~1 sec)

helpers/env-variables.ts

@@ -155,6 +155,11 @@ const getVectorDBEnvs = (
             "The base URL for the LlamaCloud API. Only change this for non-production environments",
           value: "https://api.cloud.llamaindex.ai",
         },
+        {
+          name: "LLAMA_CLOUD_ORGANIZATION_ID",
+          description:
+            "The organization ID for the LlamaCloud project (uses default organization if not specified - Python only)",
+        },
       ];
     case "chroma":
       const envs = [
@@ -274,6 +279,33 @@ const getModelEnvs = (modelConfig: ModelConfig): EnvVar[] => {
           },
         ]
       : []),
+    ...(modelConfig.provider === "azure-openai"
+      ? [
+          {
+            name: "AZURE_OPENAI_KEY",
+            description: "The Azure OpenAI key to use.",
+            value: modelConfig.apiKey,
+          },
+          {
+            name: "AZURE_OPENAI_ENDPOINT",
+            description: "The Azure OpenAI endpoint to use.",
+          },
+          {
+            name: "AZURE_OPENAI_API_VERSION",
+            description: "The Azure OpenAI API version to use.",
+          },
+          {
+            name: "AZURE_OPENAI_LLM_DEPLOYMENT",
+            description:
+              "The Azure OpenAI deployment to use for LLM deployment.",
+          },
+          {
+            name: "AZURE_OPENAI_EMBEDDING_DEPLOYMENT",
+            description:
+              "The Azure OpenAI deployment to use for embedding deployment.",
+          },
+        ]
+      : []),
     ...(modelConfig.provider === "t-systems"
       ? [
           {

helpers/index.ts

@@ -163,7 +163,11 @@ export const installTemplate = async (
     // This is a backend, so we need to copy the test data and create the env file.
 
     // Copy the environment file to the target directory.
-    if (props.template === "streaming" || props.template === "multiagent") {
+    if (
+      props.template === "streaming" ||
+      props.template === "multiagent" ||
+      props.template === "extractor"
+    ) {
       await createBackendEnvFile(props.root, {
         modelConfig: props.modelConfig,
         llamaCloudKey: props.llamaCloudKey,

helpers/providers/azure.ts

@@ -0,0 +1,118 @@
+import ciInfo from "ci-info";
+import prompts from "prompts";
+import { ModelConfigParams, ModelConfigQuestionsParams } from ".";
+import { questionHandlers } from "../../questions";
+
+const ALL_AZURE_OPENAI_CHAT_MODELS: Record<string, { openAIModel: string }> = {
+  "gpt-35-turbo": { openAIModel: "gpt-3.5-turbo" },
+  "gpt-35-turbo-16k": {
+    openAIModel: "gpt-3.5-turbo-16k",
+  },
+  "gpt-4o": { openAIModel: "gpt-4o" },
+  "gpt-4": { openAIModel: "gpt-4" },
+  "gpt-4-32k": { openAIModel: "gpt-4-32k" },
+  "gpt-4-turbo": {
+    openAIModel: "gpt-4-turbo",
+  },
+  "gpt-4-turbo-2024-04-09": {
+    openAIModel: "gpt-4-turbo",
+  },
+  "gpt-4-vision-preview": {
+    openAIModel: "gpt-4-vision-preview",
+  },
+  "gpt-4-1106-preview": {
+    openAIModel: "gpt-4-1106-preview",
+  },
+  "gpt-4o-2024-05-13": {
+    openAIModel: "gpt-4o-2024-05-13",
+  },
+};
+
+const ALL_AZURE_OPENAI_EMBEDDING_MODELS: Record<
+  string,
+  {
+    dimensions: number;
+    openAIModel: string;
+  }
+> = {
+  "text-embedding-ada-002": {
+    dimensions: 1536,
+    openAIModel: "text-embedding-ada-002",
+  },
+  "text-embedding-3-small": {
+    dimensions: 1536,
+    openAIModel: "text-embedding-3-small",
+  },
+  "text-embedding-3-large": {
+    dimensions: 3072,
+    openAIModel: "text-embedding-3-large",
+  },
+};
+
+const DEFAULT_MODEL = "gpt-4o";
+const DEFAULT_EMBEDDING_MODEL = "text-embedding-3-large";
+
+export async function askAzureQuestions({
+  openAiKey,
+  askModels,
+}: ModelConfigQuestionsParams): Promise<ModelConfigParams> {
+  const config: ModelConfigParams = {
+    apiKey: openAiKey || process.env.AZURE_OPENAI_KEY,
+    model: DEFAULT_MODEL,
+    embeddingModel: DEFAULT_EMBEDDING_MODEL,
+    dimensions: getDimensions(DEFAULT_EMBEDDING_MODEL),
+    isConfigured(): boolean {
+      // the Azure model provider can't be fully configured as endpoint and deployment names have to be configured with env variables
+      return false;
+    },
+  };
+
+  // use default model values in CI or if user should not be asked
+  const useDefaults = ciInfo.isCI || !askModels;
+  if (!useDefaults) {
+    const { model } = await prompts(
+      {
+        type: "select",
+        name: "model",
+        message: "Which LLM model would you like to use?",
+        choices: getAvailableModelChoices(),
+        initial: 0,
+      },
+      questionHandlers,
+    );
+    config.model = model;
+
+    const { embeddingModel } = await prompts(
+      {
+        type: "select",
+        name: "embeddingModel",
+        message: "Which embedding model would you like to use?",
+        choices: getAvailableEmbeddingModelChoices(),
+        initial: 0,
+      },
+      questionHandlers,
+    );
+    config.embeddingModel = embeddingModel;
+    config.dimensions = getDimensions(embeddingModel);
+  }
+
+  return config;
+}
+
+function getAvailableModelChoices() {
+  return Object.keys(ALL_AZURE_OPENAI_CHAT_MODELS).map((key) => ({
+    title: key,
+    value: key,
+  }));
+}
+
+function getAvailableEmbeddingModelChoices() {
+  return Object.keys(ALL_AZURE_OPENAI_EMBEDDING_MODELS).map((key) => ({
+    title: key,
+    value: key,
+  }));
+}
+
+function getDimensions(modelName: string) {
+  return ALL_AZURE_OPENAI_EMBEDDING_MODELS[modelName].dimensions;
+}

helpers/providers/index.ts

@@ -3,6 +3,7 @@ import prompts from "prompts";
 import { questionHandlers } from "../../questions";
 import { ModelConfig, ModelProvider, TemplateFramework } from "../types";
 import { askAnthropicQuestions } from "./anthropic";
+import { askAzureQuestions } from "./azure";
 import { askGeminiQuestions } from "./gemini";
 import { askGroqQuestions } from "./groq";
 import { askLLMHubQuestions } from "./llmhub";
@@ -34,6 +35,7 @@ export async function askModelConfig({
       { title: "Anthropic", value: "anthropic" },
       { title: "Gemini", value: "gemini" },
       { title: "Mistral", value: "mistral" },
+      { title: "AzureOpenAI", value: "azure-openai" },
     ];
 
     if (framework === "fastapi") {
@@ -69,6 +71,9 @@ export async function askModelConfig({
     case "mistral":
       modelConfig = await askMistralQuestions({ askModels });
       break;
+    case "azure-openai":
+      modelConfig = await askAzureQuestions({ askModels });
+      break;
     case "t-systems":
       modelConfig = await askLLMHubQuestions({ askModels });
       break;

helpers/python.ts

@@ -121,7 +121,7 @@ const getAdditionalDependencies = (
         case "llamacloud":
           dependencies.push({
             name: "llama-index-indices-managed-llama-cloud",
-            version: "^0.2.5",
+            version: "^0.2.7",
           });
           break;
       }
@@ -193,6 +193,16 @@ const getAdditionalDependencies = (
         version: "0.1.4",
       });
       break;
+    case "azure-openai":
+      dependencies.push({
+        name: "llama-index-llms-azure-openai",
+        version: "0.1.10",
+      });
+      dependencies.push({
+        name: "llama-index-embeddings-azure-openai",
+        version: "0.1.11",
+      });
+      break;
     case "t-systems":
       dependencies.push({
         name: "llama-index-agent-openai",

helpers/tools.ts

@@ -188,6 +188,35 @@ For better results, you can specify the region parameter to get results from a s
       },
     ],
   },
+  {
+    display: "Azure Code Interpreter",
+    name: "azure_code_interpreter.AzureCodeInterpreterToolSpec",
+    supportedFrameworks: ["fastapi", "nextjs", "express"],
+    type: ToolType.LLAMAHUB,
+    dependencies: [
+      {
+        name: "llama-index-tools-azure-code-interpreter",
+        version: "0.2.0",
+      },
+    ],
+    envVars: [
+      {
+        name: "AZURE_POOL_MANAGEMENT_ENDPOINT",
+        description:
+          "Please follow this guideline to create and get the pool management endpoint: https://learn.microsoft.com/azure/container-apps/sessions?tabs=azure-cli",
+      },
+      {
+        name: TOOL_SYSTEM_PROMPT_ENV_VAR,
+        description: "System prompt for Azure code interpreter tool.",
+        value: `-You are a Python interpreter that can run any python code in a secure environment.
+- The python code runs in a Jupyter notebook. Every time you call the 'interpreter' tool, the python code is executed in a separate cell. 
+- You are given tasks to complete and you run python code to solve them.
+- It's okay to make multiple calls to interpreter tool. If you get an error or the result is not what you expected, you can call the tool again. Don't give up too soon!
+- Plot visualizations using matplotlib or any other visualization library directly in the notebook.
+- You can install any pip package (if it exists) by running a cell with pip install.`,
+      },
+    ],
+  },
 ];
 
 export const getTool = (toolName: string): Tool | undefined => {

helpers/types.ts

@@ -8,6 +8,7 @@ export type ModelProvider =
   | "anthropic"
   | "gemini"
   | "mistral"
+  | "azure-openai"
   | "t-systems";
 export type ModelConfig = {
   provider: ModelProvider;
@@ -18,6 +19,7 @@ export type ModelConfig = {
   isConfigured(): boolean;
 };
 export type TemplateType =
+  | "extractor"
   | "streaming"
   | "community"
   | "llamapack"

package.json

@@ -1,6 +1,6 @@
 {
   "name": "create-llama",
-  "version": "0.1.22",
+  "version": "0.1.28",
   "description": "Create LlamaIndex-powered apps with one command",
   "keywords": [
     "rag",

questions.ts

@@ -342,6 +342,7 @@ export const askQuestions = async (
               title: "Multi-agent app (using llama-agents)",
               value: "multiagent",
             },
+            { title: "Structured Extractor", value: "extractor" },
             {
               title: `Community template from ${styledRepo}`,
               value: "community",
@@ -405,7 +406,7 @@ export const askQuestions = async (
     return; // early return - no further questions needed for llamapack projects
   }
 
-  if (program.template === "multiagent") {
+  if (program.template === "multiagent" || program.template === "extractor") {
     // TODO: multi-agents currently only supports FastAPI
     program.framework = preferences.framework = "fastapi";
   }
@@ -647,24 +648,27 @@ export const askQuestions = async (
     // default to use LlamaParse if using LlamaCloud
     program.useLlamaParse = preferences.useLlamaParse = true;
   } else {
-    if (program.dataSources.some((ds) => ds.type === "file")) {
-      if (ciInfo.isCI) {
-        program.useLlamaParse = getPrefOrDefault("useLlamaParse");
-      } else {
-        const { useLlamaParse } = await prompts(
-          {
-            type: "toggle",
-            name: "useLlamaParse",
-            message:
-              "Would you like to use LlamaParse (improved parser for RAG - requires API key)?",
-            initial: false,
-            active: "yes",
-            inactive: "no",
-          },
-          questionHandlers,
-        );
-        program.useLlamaParse = useLlamaParse;
-        preferences.useLlamaParse = useLlamaParse;
+    if (program.useLlamaParse === undefined) {
+      // if already set useLlamaParse, don't ask again
+      if (program.dataSources.some((ds) => ds.type === "file")) {
+        if (ciInfo.isCI) {
+          program.useLlamaParse = getPrefOrDefault("useLlamaParse");
+        } else {
+          const { useLlamaParse } = await prompts(
+            {
+              type: "toggle",
+              name: "useLlamaParse",
+              message:
+                "Would you like to use LlamaParse (improved parser for RAG - requires API key)?",
+              initial: false,
+              active: "yes",
+              inactive: "no",
+            },
+            questionHandlers,
+          );
+          program.useLlamaParse = useLlamaParse;
+          preferences.useLlamaParse = useLlamaParse;
+        }
       }
     }
   }

templates/components/engines/typescript/agent/chat.ts

@@ -48,7 +48,7 @@ export async function createChatEngine(documentIds?: string[]) {
   });
 }
 
-function generateFilters(documentIds: string[]): MetadataFilters | undefined {
+export function generateFilters(documentIds: string[]): MetadataFilters {
   // public documents don't have the "private" field or it's set to "false"
   const publicDocumentsFilter: MetadataFilter = {
     key: "private",

templates/components/engines/typescript/chat/chat.ts

@@ -1,4 +1,9 @@
-import { ContextChatEngine, Settings } from "llamaindex";
+import {
+  ContextChatEngine,
+  MetadataFilter,
+  MetadataFilters,
+  Settings,
+} from "llamaindex";
 import { getDataSource } from "./index";
 
 export async function createChatEngine(documentIds?: string[]) {
@@ -10,12 +15,36 @@ export async function createChatEngine(documentIds?: string[]) {
   }
   const retriever = index.asRetriever({
     similarityTopK: process.env.TOP_K ? parseInt(process.env.TOP_K) : 3,
+    filters: generateFilters(documentIds || []),
   });
 
   return new ContextChatEngine({
     chatModel: Settings.llm,
     retriever,
-    // disable as a custom system prompt disables the generated context
-    // systemPrompt: process.env.SYSTEM_PROMPT,
+    systemPrompt: process.env.SYSTEM_PROMPT,
   });
 }
+
+export function generateFilters(documentIds: string[]): MetadataFilters {
+  // public documents don't have the "private" field or it's set to "false"
+  const publicDocumentsFilter: MetadataFilter = {
+    key: "private",
+    value: ["true"],
+    operator: "nin",
+  };
+
+  // if no documentIds are provided, only retrieve information from public documents
+  if (!documentIds.length) return { filters: [publicDocumentsFilter] };
+
+  const privateDocumentsFilter: MetadataFilter = {
+    key: "doc_id",
+    value: documentIds,
+    operator: "in",
+  };
+
+  // if documentIds are provided, retrieve information from public and private documents
+  return {
+    filters: [publicDocumentsFilter, privateDocumentsFilter],
+    condition: "or",
+  };
+}

templates/components/llamaindex/typescript/documents/pipeline.ts

@@ -1,28 +1,16 @@
 import {
-  BaseNode,
   Document,
   IngestionPipeline,
-  Metadata,
   Settings,
   SimpleNodeParser,
-  storageContextFromDefaults,
   VectorStoreIndex,
 } from "llamaindex";
 import { LlamaCloudIndex } from "llamaindex/cloud/LlamaCloudIndex";
-import { getDataSource } from "../../engine";
-
-export async function runPipeline(documents: Document[], filename: string) {
-  const currentIndex = await getDataSource();
-
-  // Update documents with metadata
-  for (const document of documents) {
-    document.metadata = {
-      ...document.metadata,
-      file_name: filename,
-      private: "true", // to separate from other public documents
-    };
-  }
 
+export async function runPipeline(
+  currentIndex: VectorStoreIndex | LlamaCloudIndex,
+  documents: Document[],
+) {
   if (currentIndex instanceof LlamaCloudIndex) {
     // LlamaCloudIndex processes the documents automatically
     // so we don't need ingestion pipeline, just insert the documents directly
@@ -41,25 +29,10 @@ export async function runPipeline(documents: Document[], filename: string) {
       ],
     });
     const nodes = await pipeline.run({ documents });
-    await addNodesToVectorStore(nodes, currentIndex);
+    await currentIndex.insertNodes(nodes);
+    currentIndex.storageContext.docStore.persist();
+    console.log("Added nodes to the vector store.");
   }
 
   return documents.map((document) => document.id_);
 }
-
-async function addNodesToVectorStore(
-  nodes: BaseNode<Metadata>[],
-  currentIndex: VectorStoreIndex | null,
-) {
-  if (currentIndex) {
-    await currentIndex.insertNodes(nodes);
-  } else {
-    // Not using vectordb and haven't generated local index yet
-    const storageContext = await storageContextFromDefaults({
-      persistDir: "./cache",
-    });
-    currentIndex = await VectorStoreIndex.init({ nodes, storageContext });
-  }
-  currentIndex.storageContext.docStore.persist();
-  console.log("Added nodes to the vector store.");
-}

templates/components/llamaindex/typescript/documents/upload.ts

@@ -1,11 +1,26 @@
+import { VectorStoreIndex } from "llamaindex";
+import { LlamaCloudIndex } from "llamaindex/cloud/LlamaCloudIndex";
 import { loadDocuments, saveDocument } from "./helper";
 import { runPipeline } from "./pipeline";
 
-export async function uploadDocument(raw: string): Promise<string[]> {
+export async function uploadDocument(
+  index: VectorStoreIndex | LlamaCloudIndex,
+  raw: string,
+): Promise<string[]> {
   const [header, content] = raw.split(",");
   const mimeType = header.replace("data:", "").replace(";base64", "");
   const fileBuffer = Buffer.from(content, "base64");
   const documents = await loadDocuments(fileBuffer, mimeType);
   const { filename } = await saveDocument(fileBuffer, mimeType);
-  return await runPipeline(documents, filename);
+
+  // Update documents with metadata
+  for (const document of documents) {
+    document.metadata = {
+      ...document.metadata,
+      file_name: filename,
+      private: "true", // to separate private uploads from public documents
+    };
+  }
+
+  return await runPipeline(index, documents);
 }

templates/components/settings/python/settings.py

@@ -75,27 +75,33 @@ def init_azure_openai():
     from llama_index.embeddings.azure_openai import AzureOpenAIEmbedding
     from llama_index.llms.azure_openai import AzureOpenAI
 
-    llm_deployment = os.getenv("AZURE_OPENAI_LLM_DEPLOYMENT")
-    embedding_deployment = os.getenv("AZURE_OPENAI_EMBEDDING_DEPLOYMENT")
+    llm_deployment = os.environ["AZURE_OPENAI_LLM_DEPLOYMENT"]
+    embedding_deployment = os.environ["AZURE_OPENAI_EMBEDDING_DEPLOYMENT"]
     max_tokens = os.getenv("LLM_MAX_TOKENS")
-    api_key = os.getenv("AZURE_OPENAI_API_KEY")
-    llm_config = {
-        "api_key": api_key,
-        "deployment_name": llm_deployment,
-        "model": os.getenv("MODEL"),
-        "temperature": float(os.getenv("LLM_TEMPERATURE", DEFAULT_TEMPERATURE)),
-        "max_tokens": int(max_tokens) if max_tokens is not None else None,
-    }
-    Settings.llm = AzureOpenAI(**llm_config)
-
+    temperature = os.getenv("LLM_TEMPERATURE", DEFAULT_TEMPERATURE)
     dimensions = os.getenv("EMBEDDING_DIM")
-    embedding_config = {
-        "api_key": api_key,
-        "deployment_name": embedding_deployment,
-        "model": os.getenv("EMBEDDING_MODEL"),
-        "dimensions": int(dimensions) if dimensions is not None else None,
+
+    azure_config = {
+        "api_key": os.environ["AZURE_OPENAI_KEY"],
+        "azure_endpoint": os.environ["AZURE_OPENAI_ENDPOINT"],
+        "api_version": os.getenv("AZURE_OPENAI_API_VERSION")
+        or os.getenv("OPENAI_API_VERSION"),
     }
-    Settings.embed_model = AzureOpenAIEmbedding(**embedding_config)
+
+    Settings.llm = AzureOpenAI(
+        model=os.getenv("MODEL"),
+        max_tokens=int(max_tokens) if max_tokens is not None else None,
+        temperature=float(temperature),
+        deployment_name=llm_deployment,
+        **azure_config,
+    )
+
+    Settings.embed_model = AzureOpenAIEmbedding(
+        model=os.getenv("EMBEDDING_MODEL"),
+        dimensions=int(dimensions) if dimensions is not None else None,
+        deployment_name=embedding_deployment,
+        **azure_config,
+    )
 
 
 def init_fastembed():
@@ -108,7 +114,7 @@ def init_fastembed():
         # Small and multilingual
         "all-MiniLM-L6-v2": "sentence-transformers/all-MiniLM-L6-v2",
         # Large and multilingual
-        "paraphrase-multilingual-mpnet-base-v2": "sentence-transformers/paraphrase-multilingual-mpnet-base-v2",   # noqa: E501
+        "paraphrase-multilingual-mpnet-base-v2": "sentence-transformers/paraphrase-multilingual-mpnet-base-v2",  # noqa: E501
     }
 
     # This will download the model automatically if it is not already downloaded
@@ -116,6 +122,7 @@ def init_fastembed():
         model_name=embed_model_map[os.getenv("EMBEDDING_MODEL")]
     )
 
+
 def init_groq():
     from llama_index.llms.groq import Groq
 
@@ -125,7 +132,6 @@ def init_groq():
         "mixtral-8x7b": "mixtral-8x7b-32768",
     }
 
-
     Settings.llm = Groq(model=model_map[os.getenv("MODEL")])
     # Groq does not provide embeddings, so we use FastEmbed instead
     init_fastembed()

templates/components/vectordbs/python/llamacloud/generate.py

@@ -21,6 +21,7 @@ def generate_datasource():
     project_name = os.getenv("LLAMA_CLOUD_PROJECT_NAME")
     api_key = os.getenv("LLAMA_CLOUD_API_KEY")
     base_url = os.getenv("LLAMA_CLOUD_BASE_URL")
+    organization_id = os.getenv("LLAMA_CLOUD_ORGANIZATION_ID")
 
     if name is None or project_name is None or api_key is None:
         raise ValueError(
@@ -40,6 +41,7 @@ def generate_datasource():
         project_name=project_name,
         api_key=api_key,
         base_url=base_url,
+        organization_id=organization_id
     )
 
     logger.info("Finished generating the index")

templates/components/vectordbs/python/llamacloud/index.py

@@ -11,6 +11,7 @@ def get_index():
     project_name = os.getenv("LLAMA_CLOUD_PROJECT_NAME")
     api_key = os.getenv("LLAMA_CLOUD_API_KEY")
     base_url = os.getenv("LLAMA_CLOUD_BASE_URL")
+    organization_id = os.getenv("LLAMA_CLOUD_ORGANIZATION_ID")
 
     if name is None or project_name is None or api_key is None:
         raise ValueError(
@@ -23,6 +24,7 @@ def get_index():
         project_name=project_name,
         api_key=api_key,
         base_url=base_url,
+        organization_id=organization_id
     )
 
     return index

templates/components/vectordbs/typescript/milvus/shared.ts

@@ -4,7 +4,6 @@ const REQUIRED_ENV_VARS = [
   "MILVUS_ADDRESS",
   "MILVUS_USERNAME",
   "MILVUS_PASSWORD",
-  "MILVUS_COLLECTION",
 ];
 
 export function getMilvusClient() {
@@ -19,8 +18,13 @@ export function getMilvusClient() {
   });
 }
 
-export function checkRequiredEnvVars() {
-  const missingEnvVars = REQUIRED_ENV_VARS.filter((envVar) => {
+export function checkRequiredEnvVars(opts?: { checkCollectionEnv?: boolean }) {
+  const shouldCheckCollectionEnv = opts?.checkCollectionEnv ?? true; // default to true
+  const requiredEnvVars = [...REQUIRED_ENV_VARS]; // create a copy of the array
+  if (shouldCheckCollectionEnv) {
+    requiredEnvVars.push("MILVUS_COLLECTION");
+  }
+  const missingEnvVars = requiredEnvVars.filter((envVar) => {
     return !process.env[envVar];
   });
 

templates/types/extractor/fastapi/README-template.md

@@ -0,0 +1,68 @@
+This is a [LlamaIndex](https://www.llamaindex.ai/) project using [FastAPI](https://fastapi.tiangolo.com/) bootstrapped with [`create-llama`](https://github.com/run-llama/LlamaIndexTS/tree/main/packages/create-llama) featuring [structured extraction](https://docs.llamaindex.ai/en/stable/examples/structured_outputs/structured_outputs/?h=structured+output).
+
+## Getting Started
+
+First, setup the environment with poetry:
+
+> **_Note:_** This step is not needed if you are using the dev-container.
+
+```shell
+poetry install
+poetry shell
+```
+
+Then check the parameters that have been pre-configured in the `.env` file in this directory. (E.g. you might need to configure an `OPENAI_API_KEY` if you're using OpenAI as model provider).
+
+Second, generate the embeddings of the documents in the `./data` directory (if this folder exists - otherwise, skip this step):
+
+```shell
+poetry run generate
+```
+
+Third, run the API in one command:
+
+```shell
+poetry run python main.py
+```
+
+The example provides the `/api/extractor/query` API endpoint.
+
+This query endpoint returns structured data in the format of the [Output](./app/api/routers/output.py) class. Modify this class to change the output format.
+
+You can test the endpoint with the following curl request:
+
+```shell
+curl --location 'localhost:8000/api/extractor/query' \
+--header 'Content-Type: application/json' \
+--data '{ "query": "What is the maximum weight for a parcel?" }'
+```
+
+Which will return a response that the RAG pipeline is confident about the answer.
+
+Try
+
+```shell
+curl --location 'localhost:8000/api/extractor/query' \
+--header 'Content-Type: application/json' \
+--data '{ "query": "What is the weather today?" }'
+```
+
+To retrieve a response with low confidence since the question is not related to the provided document in the `./data` directory.
+
+You can start editing the API endpoint by modifying [`extractor.py`](./app/api/routers/extractor.py). The endpoints auto-update as you save the file.
+
+Open [http://localhost:8000/docs](http://localhost:8000/docs) with your browser to see the Swagger UI of the API.
+
+The API allows CORS for all origins to simplify development. You can change this behavior by setting the `ENVIRONMENT` environment variable to `prod`:
+
+```
+ENVIRONMENT=prod python main.py
+```
+
+## Learn More
+
+To learn more about LlamaIndex, take a look at the following resources:
+
+- [LlamaIndex Documentation](https://docs.llamaindex.ai) - learn about LlamaIndex.
+
+You can check out [the LlamaIndex GitHub repository](https://github.com/run-llama/llama_index) - your feedback and contributions are welcome!

templates/types/extractor/fastapi/app/api/routers/extractor.py

@@ -0,0 +1,58 @@
+import logging
+import os
+
+from fastapi import APIRouter, HTTPException
+from llama_index.core.settings import Settings
+from pydantic import BaseModel
+
+from app.api.routers.output import Output
+from app.engine.index import get_index
+
+extractor_router = r = APIRouter()
+
+logger = logging.getLogger("uvicorn")
+
+
+class RequestData(BaseModel):
+    query: str
+
+    class Config:
+        json_schema_extra = {
+            "examples": [
+                {"query": "What's the maximum weight for a parcel?"},
+            ],
+        }
+
+
+@r.post("/query")
+async def query_request(
+    data: RequestData,
+):
+    # Create a query engine using that returns responses in the format of the Output class
+    query_engine = get_query_engine(Output)
+
+    response = await query_engine.aquery(data.query)
+
+    output_data = response.response.dict()
+    return Output(**output_data)
+
+
+def get_query_engine(output_cls: BaseModel):
+    top_k = os.getenv("TOP_K", 3)
+
+    index = get_index()
+    if index is None:
+        raise HTTPException(
+            status_code=500,
+            detail=str(
+                "StorageContext is empty - call 'poetry run generate' to generate the storage first"
+            ),
+        )
+
+    sllm = Settings.llm.as_structured_llm(output_cls)
+
+    return index.as_query_engine(
+        similarity_top_k=int(top_k),
+        llm=sllm,
+        response_mode="tree_summarize",
+    )

templates/types/extractor/fastapi/app/api/routers/output.py

@@ -0,0 +1,32 @@
+import logging
+from llama_index.core.schema import BaseModel, Field
+from typing import List
+
+logger = logging.getLogger("uvicorn")
+
+
+class Output(BaseModel):
+    response: str = Field(..., description="The answer to the question.")
+    page_numbers: List[int] = Field(
+        ...,
+        description="The page numbers of the sources used to answer this question. Do not include a page number if the context is irrelevant.",
+    )
+    confidence: float = Field(
+        ...,
+        ge=0,
+        le=1,
+        description="Confidence value between 0-1 of the correctness of the result.",
+    )
+    confidence_explanation: str = Field(
+        ..., description="Explanation for the confidence score"
+    )
+
+    class Config:
+        json_schema_extra = {
+            "example": {
+                "response": "This is an example answer.",
+                "page_numbers": [1, 2, 3],
+                "confidence": 0.85,
+                "confidence_explanation": "This is an explanation for the confidence score.",
+            }
+        }

templates/types/extractor/fastapi/main.py

@@ -0,0 +1,45 @@
+from dotenv import load_dotenv
+
+load_dotenv()
+
+import logging
+import os
+import uvicorn
+from fastapi import FastAPI
+from fastapi.middleware.cors import CORSMiddleware
+from fastapi.responses import RedirectResponse
+from app.api.routers.extractor import extractor_router
+from app.settings import init_settings
+
+
+app = FastAPI()
+
+init_settings()
+
+environment = os.getenv("ENVIRONMENT", "dev")  # Default to 'development' if not set
+logger = logging.getLogger("uvicorn")
+
+if environment == "dev":
+    logger.warning("Running in development mode - allowing CORS for all origins")
+    app.add_middleware(
+        CORSMiddleware,
+        allow_origins=["*"],
+        allow_credentials=True,
+        allow_methods=["*"],
+        allow_headers=["*"],
+    )
+
+    # Redirect to documentation page when accessing base URL
+    @app.get("/")
+    async def redirect_to_docs():
+        return RedirectResponse(url="/docs")
+
+
+app.include_router(extractor_router, prefix="/api/extractor")
+
+if __name__ == "__main__":
+    app_host = os.getenv("APP_HOST", "0.0.0.0")
+    app_port = int(os.getenv("APP_PORT", "8000"))
+    reload = True if environment == "dev" else False
+
+    uvicorn.run(app="main:app", host=app_host, port=app_port, reload=reload)

templates/types/extractor/fastapi/pyproject.toml

@@ -0,0 +1,21 @@
+[tool.poetry]
+name = "app"
+version = "0.1.0"
+description = ""
+authors = ["Marcus Schiesser <mail@marcusschiesser.de>"]
+readme = "README.md"
+
+[tool.poetry.scripts]
+generate = "app.engine.generate:generate_datasource"
+
+[tool.poetry.dependencies]
+python = "^3.11,<3.12"
+fastapi = "^0.109.1"
+uvicorn = { extras = ["standard"], version = "^0.23.2" }
+python-dotenv = "^1.0.0"
+llama-index = "^0.10.58"
+cachetools = "^5.3.3"
+
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"

templates/types/streaming/express/package.json

@@ -20,7 +20,7 @@
     "dotenv": "^16.3.1",
     "duck-duck-scrape": "^2.2.5",
     "express": "^4.18.2",
-    "llamaindex": "0.5.7",
+    "llamaindex": "0.5.12",
     "pdf2json": "3.0.5",
     "ajv": "^8.12.0",
     "@e2b/code-interpreter": "^0.0.5",

templates/types/streaming/express/src/controllers/chat-upload.controller.ts

@@ -1,4 +1,5 @@
 import { Request, Response } from "express";
+import { getDataSource } from "./engine";
 import { uploadDocument } from "./llamaindex/documents/upload";
 
 export const chatUpload = async (req: Request, res: Response) => {
@@ -8,5 +9,6 @@ export const chatUpload = async (req: Request, res: Response) => {
       error: "base64 is required in the request body",
     });
   }
-  return res.status(200).json(await uploadDocument(base64));
+  const index = await getDataSource();
+  return res.status(200).json(await uploadDocument(index, base64));
 };

templates/types/streaming/express/src/controllers/engine/settings.ts

@@ -45,6 +45,9 @@ export const initSettings = async () => {
     case "mistral":
       initMistralAI();
       break;
+    case "azure-openai":
+      initAzureOpenAI();
+      break;
     default:
       initOpenAI();
       break;
@@ -68,6 +71,53 @@ function initOpenAI() {
   });
 }
 
+function initAzureOpenAI() {
+  // Map Azure OpenAI model names to OpenAI model names (only for TS)
+  const AZURE_OPENAI_MODEL_MAP: Record<string, string> = {
+    "gpt-35-turbo": "gpt-3.5-turbo",
+    "gpt-35-turbo-16k": "gpt-3.5-turbo-16k",
+    "gpt-4o": "gpt-4o",
+    "gpt-4": "gpt-4",
+    "gpt-4-32k": "gpt-4-32k",
+    "gpt-4-turbo": "gpt-4-turbo",
+    "gpt-4-turbo-2024-04-09": "gpt-4-turbo",
+    "gpt-4-vision-preview": "gpt-4-vision-preview",
+    "gpt-4-1106-preview": "gpt-4-1106-preview",
+    "gpt-4o-2024-05-13": "gpt-4o-2024-05-13",
+  };
+
+  const azureConfig = {
+    apiKey: process.env.AZURE_OPENAI_KEY,
+    endpoint: process.env.AZURE_OPENAI_ENDPOINT,
+    apiVersion:
+      process.env.AZURE_OPENAI_API_VERSION || process.env.OPENAI_API_VERSION,
+  };
+
+  Settings.llm = new OpenAI({
+    model:
+      AZURE_OPENAI_MODEL_MAP[process.env.MODEL ?? "gpt-35-turbo"] ??
+      "gpt-3.5-turbo",
+    maxTokens: process.env.LLM_MAX_TOKENS
+      ? Number(process.env.LLM_MAX_TOKENS)
+      : undefined,
+    azure: {
+      ...azureConfig,
+      deployment: process.env.AZURE_OPENAI_LLM_DEPLOYMENT,
+    },
+  });
+
+  Settings.embedModel = new OpenAIEmbedding({
+    model: process.env.EMBEDDING_MODEL,
+    dimensions: process.env.EMBEDDING_DIM
+      ? parseInt(process.env.EMBEDDING_DIM)
+      : undefined,
+    azure: {
+      ...azureConfig,
+      deployment: process.env.AZURE_OPENAI_EMBEDDING_DEPLOYMENT,
+    },
+  });
+}
+
 function initOllama() {
   const config = {
     host: process.env.OLLAMA_BASE_URL ?? "http://127.0.0.1:11434",

templates/types/streaming/fastapi/app/api/routers/models.py

@@ -237,6 +237,7 @@ class ChatConfig(BaseModel):
     starter_questions: Optional[List[str]] = Field(
         default=None,
         description="List of starter questions",
+        serialization_alias="starterQuestions"
     )
 
     class Config:
@@ -248,4 +249,3 @@ class ChatConfig(BaseModel):
                 ]
             }
         }
-        alias_generator = to_camel

templates/types/streaming/fastapi/app/engine/generate.py

@@ -64,6 +64,9 @@ def generate_datasource():
 
     # Get the stores and documents or create new ones
     documents = get_documents()
+    # Set private=false to mark the document as public (required for filtering)
+    for doc in documents:
+        doc.metadata["private"] = "false"
     docstore = get_doc_store()
     vector_store = get_vector_store()
 

templates/types/streaming/fastapi/app/llmhub.py

@@ -1,61 +0,0 @@
-from llama_index.embeddings.openai import OpenAIEmbedding
-from llama_index.core.settings import Settings
-from typing import Dict
-import os
-
-DEFAULT_MODEL = "gpt-3.5-turbo"
-DEFAULT_EMBEDDING_MODEL = "text-embedding-3-large"
-
-class TSIEmbedding(OpenAIEmbedding):
-    def __init__(self, **kwargs):
-        super().__init__(**kwargs)
-        self._query_engine = self._text_engine = self.model_name
-
-def llm_config_from_env() -> Dict:
-    from llama_index.core.constants import DEFAULT_TEMPERATURE
-
-    model = os.getenv("MODEL", DEFAULT_MODEL)
-    temperature = os.getenv("LLM_TEMPERATURE", DEFAULT_TEMPERATURE)
-    max_tokens = os.getenv("LLM_MAX_TOKENS")
-    api_key = os.getenv("T_SYSTEMS_LLMHUB_API_KEY")
-    api_base = os.getenv("T_SYSTEMS_LLMHUB_BASE_URL")
-
-    config = {
-        "model": model,
-        "api_key": api_key,
-        "api_base": api_base,
-        "temperature": float(temperature),
-        "max_tokens": int(max_tokens) if max_tokens is not None else None,
-    }
-    return config
-
-
-def embedding_config_from_env() -> Dict:
-    from llama_index.core.constants import DEFAULT_EMBEDDING_DIM
-    
-    model = os.getenv("EMBEDDING_MODEL", DEFAULT_EMBEDDING_MODEL)
-    dimension = os.getenv("EMBEDDING_DIM", DEFAULT_EMBEDDING_DIM)
-    api_key = os.getenv("T_SYSTEMS_LLMHUB_API_KEY")
-    api_base = os.getenv("T_SYSTEMS_LLMHUB_BASE_URL")
-
-    config = {
-        "model_name": model,
-        "dimension": int(dimension) if dimension is not None else None,
-        "api_key": api_key,
-        "api_base": api_base,
-    }
-    return config
-
-def init_llmhub():
-    from llama_index.llms.openai_like import OpenAILike
-
-    llm_configs = llm_config_from_env()
-    embedding_configs = embedding_config_from_env()
-
-    Settings.embed_model = TSIEmbedding(**embedding_configs)
-    Settings.llm = OpenAILike(
-        **llm_configs,
-        is_chat_model=True,
-        is_function_calling_model=False,
-        context_window=4096,
-    )

templates/types/streaming/fastapi/pyproject.toml

@@ -14,7 +14,7 @@ fastapi = "^0.109.1"
 uvicorn = { extras = ["standard"], version = "^0.23.2" }
 python-dotenv = "^1.0.0"
 aiostream = "^0.5.2"
-llama-index = "0.10.55"
+llama-index = "0.10.58"
 cachetools = "^5.3.3"
 
 [build-system]

templates/types/streaming/nextjs/app/api/chat/engine/settings.ts

@@ -45,6 +45,9 @@ export const initSettings = async () => {
     case "mistral":
       initMistralAI();
       break;
+    case "azure-openai":
+      initAzureOpenAI();
+      break;
     default:
       initOpenAI();
       break;
@@ -55,7 +58,7 @@ export const initSettings = async () => {
 
 function initOpenAI() {
   Settings.llm = new OpenAI({
-    model: process.env.MODEL ?? "gpt-3.5-turbo",
+    model: process.env.MODEL ?? "gpt-4o-mini",
     maxTokens: process.env.LLM_MAX_TOKENS
       ? Number(process.env.LLM_MAX_TOKENS)
       : undefined,
@@ -68,6 +71,53 @@ function initOpenAI() {
   });
 }
 
+function initAzureOpenAI() {
+  // Map Azure OpenAI model names to OpenAI model names (only for TS)
+  const AZURE_OPENAI_MODEL_MAP: Record<string, string> = {
+    "gpt-35-turbo": "gpt-3.5-turbo",
+    "gpt-35-turbo-16k": "gpt-3.5-turbo-16k",
+    "gpt-4o": "gpt-4o",
+    "gpt-4": "gpt-4",
+    "gpt-4-32k": "gpt-4-32k",
+    "gpt-4-turbo": "gpt-4-turbo",
+    "gpt-4-turbo-2024-04-09": "gpt-4-turbo",
+    "gpt-4-vision-preview": "gpt-4-vision-preview",
+    "gpt-4-1106-preview": "gpt-4-1106-preview",
+    "gpt-4o-2024-05-13": "gpt-4o-2024-05-13",
+  };
+
+  const azureConfig = {
+    apiKey: process.env.AZURE_OPENAI_KEY,
+    endpoint: process.env.AZURE_OPENAI_ENDPOINT,
+    apiVersion:
+      process.env.AZURE_OPENAI_API_VERSION || process.env.OPENAI_API_VERSION,
+  };
+
+  Settings.llm = new OpenAI({
+    model:
+      AZURE_OPENAI_MODEL_MAP[process.env.MODEL ?? "gpt-35-turbo"] ??
+      "gpt-3.5-turbo",
+    maxTokens: process.env.LLM_MAX_TOKENS
+      ? Number(process.env.LLM_MAX_TOKENS)
+      : undefined,
+    azure: {
+      ...azureConfig,
+      deployment: process.env.AZURE_OPENAI_LLM_DEPLOYMENT,
+    },
+  });
+
+  Settings.embedModel = new OpenAIEmbedding({
+    model: process.env.EMBEDDING_MODEL,
+    dimensions: process.env.EMBEDDING_DIM
+      ? parseInt(process.env.EMBEDDING_DIM)
+      : undefined,
+    azure: {
+      ...azureConfig,
+      deployment: process.env.AZURE_OPENAI_EMBEDDING_DEPLOYMENT,
+    },
+  });
+}
+
 function initOllama() {
   const config = {
     host: process.env.OLLAMA_BASE_URL ?? "http://127.0.0.1:11434",

templates/types/streaming/nextjs/app/api/chat/upload/route.ts

@@ -1,4 +1,5 @@
 import { NextRequest, NextResponse } from "next/server";
+import { getDataSource } from "../engine";
 import { initSettings } from "../engine/settings";
 import { uploadDocument } from "../llamaindex/documents/upload";
 
@@ -16,7 +17,13 @@ export async function POST(request: NextRequest) {
         { status: 400 },
       );
     }
-    return NextResponse.json(await uploadDocument(base64));
+    const index = await getDataSource();
+    if (!index) {
+      throw new Error(
+        `StorageContext is empty - call 'npm run generate' to generate the storage first`,
+      );
+    }
+    return NextResponse.json(await uploadDocument(index, base64));
   } catch (error) {
     console.error("[Upload API]", error);
     return NextResponse.json(

templates/types/streaming/nextjs/app/components/ui/chat/chat-input.tsx

@@ -21,7 +21,9 @@ export default function ChatInput(
     | "messages"
     | "setInput"
     | "append"
-  >,
+  > & {
+    requestParams?: any;
+  },
 ) {
   const {
     imageUrl,
@@ -64,7 +66,7 @@ export default function ChatInput(
       return;
     }
     try {
-      await uploadFile(file);
+      await uploadFile(file, props.requestParams);
       props.onFileUpload?.(file);
     } catch (error: any) {
       props.onFileError?.(error.message);

templates/types/streaming/nextjs/app/components/ui/chat/hooks/use-file.ts

@@ -48,7 +48,10 @@ export function useFile() {
     files.length && setFiles([]);
   };
 
-  const uploadContent = async (base64: string): Promise<string[]> => {
+  const uploadContent = async (
+    base64: string,
+    requestParams: any = {},
+  ): Promise<string[]> => {
     const uploadAPI = `${backend}/api/chat/upload`;
     const response = await fetch(uploadAPI, {
       method: "POST",
@@ -57,6 +60,7 @@ export function useFile() {
       },
       body: JSON.stringify({
         base64,
+        ...requestParams,
       }),
     });
     if (!response.ok) throw new Error("Failed to upload document.");
@@ -98,7 +102,7 @@ export function useFile() {
     return content;
   };
 
-  const uploadFile = async (file: File) => {
+  const uploadFile = async (file: File, requestParams: any = {}) => {
     if (file.type.startsWith("image/")) {
       const base64 = await readContent({ file, asUrl: true });
       return setImageUrl(base64);
@@ -125,7 +129,7 @@ export function useFile() {
       }
       default: {
         const base64 = await readContent({ file, asUrl: true });
-        const ids = await uploadContent(base64);
+        const ids = await uploadContent(base64, requestParams);
         return addDoc({
           ...newDoc,
           content: {

templates/types/streaming/nextjs/next.config.json

@@ -1,7 +1,8 @@
 {
   "experimental": {
     "outputFileTracingIncludes": {
-      "/*": ["./cache/**/*"]
+      "/*": ["./cache/**/*"],
+      "/api/**/*": ["./node_modules/**/*.wasm"]
     }
   }
 }

templates/types/streaming/nextjs/package.json

@@ -24,7 +24,7 @@
     "duck-duck-scrape": "^2.2.5",
     "formdata-node": "^6.0.3",
     "got": "^14.4.1",
-    "llamaindex": "0.5.7",
+    "llamaindex": "0.5.12",
     "lucide-react": "^0.294.0",
     "next": "^14.2.4",
     "react": "^18.2.0",