use --watch

.github/workflows/e2e.yml

@@ -17,7 +17,7 @@ jobs:
       matrix:
         node-version: [18, 20]
         python-version: ["3.11"]
-        os: [macos-latest, windows-latest, ubuntu-22.04]
+        os: [macos-latest, windows-latest]
     defaults:
       run:
         shell: bash

CHANGELOG.md

@@ -1,41 +1,5 @@
 # create-llama
 
-## 0.1.14
-
-### Patch Changes
-
-- f2c3389: chore: update to llamaindex 0.4.3
-- 5093b37: Remove non-working file selectors for Linux
-
-## 0.1.13
-
-### Patch Changes
-
-- b3c969d: Add image generator tool
-
-## 0.1.12
-
-### Patch Changes
-
-- aa69014: Fix NextJS for TS 5.2
-
-## 0.1.11
-
-### Patch Changes
-
-- 48b96ff: Add DuckDuckGo search tool
-- 9c9decb: Reuse function tool instances and improve e2b interpreter tool for Python
-- 02ed277: Add Groq as a model provider
-- 0748f2e: Remove hard-coded Gemini supported models
-
-## 0.1.10
-
-### Patch Changes
-
-- 9112d08: Add OpenAPI tool for Typescript
-- 8f03f8d: Add OLLAMA_REQUEST_TIMEOUT variable to config Ollama timeout (Python)
-- 8f03f8d: Apply nest_asyncio for llama parse
-
 ## 0.1.9
 
 ### Patch Changes

create-app.ts

@@ -151,19 +151,5 @@ export async function createApp({
     );
   }
 
-  if (
-    dataSources.some((dataSource) => dataSource.type === "file") &&
-    process.platform === "linux"
-  ) {
-    console.log(
-      yellow(
-        `You can add your own data files to ${terminalLink(
-          "data",
-          `file://${root}/data`,
-        )} folder manually.`,
-      ),
-    );
-  }
-
   console.log();
 }

helpers/env-variables.ts

@@ -215,15 +215,6 @@ const getModelEnvs = (modelConfig: ModelConfig): EnvVar[] => {
           },
         ]
       : []),
-    ...(modelConfig.provider === "groq"
-      ? [
-          {
-            name: "GROQ_API_KEY",
-            description: "The Groq API key to use.",
-            value: modelConfig.apiKey,
-          },
-        ]
-      : []),
     ...(modelConfig.provider === "gemini"
       ? [
           {

helpers/providers/groq.ts

@@ -1,99 +0,0 @@
-import ciInfo from "ci-info";
-import prompts from "prompts";
-import { ModelConfigParams } from ".";
-import { questionHandlers, toChoice } from "../../questions";
-
-const MODELS = ["llama3-8b", "llama3-70b", "mixtral-8x7b"];
-const DEFAULT_MODEL = MODELS[0];
-
-// Use huggingface embedding models for now as Groq doesn't support embedding models
-enum HuggingFaceEmbeddingModelType {
-  XENOVA_ALL_MINILM_L6_V2 = "all-MiniLM-L6-v2",
-  XENOVA_ALL_MPNET_BASE_V2 = "all-mpnet-base-v2",
-}
-type ModelData = {
-  dimensions: number;
-};
-const EMBEDDING_MODELS: Record<HuggingFaceEmbeddingModelType, ModelData> = {
-  [HuggingFaceEmbeddingModelType.XENOVA_ALL_MINILM_L6_V2]: {
-    dimensions: 384,
-  },
-  [HuggingFaceEmbeddingModelType.XENOVA_ALL_MPNET_BASE_V2]: {
-    dimensions: 768,
-  },
-};
-const DEFAULT_EMBEDDING_MODEL = Object.keys(EMBEDDING_MODELS)[0];
-const DEFAULT_DIMENSIONS = Object.values(EMBEDDING_MODELS)[0].dimensions;
-
-type GroqQuestionsParams = {
-  apiKey?: string;
-  askModels: boolean;
-};
-
-export async function askGroqQuestions({
-  askModels,
-  apiKey,
-}: GroqQuestionsParams): Promise<ModelConfigParams> {
-  const config: ModelConfigParams = {
-    apiKey,
-    model: DEFAULT_MODEL,
-    embeddingModel: DEFAULT_EMBEDDING_MODEL,
-    dimensions: DEFAULT_DIMENSIONS,
-    isConfigured(): boolean {
-      if (config.apiKey) {
-        return true;
-      }
-      if (process.env["GROQ_API_KEY"]) {
-        return true;
-      }
-      return false;
-    },
-  };
-
-  if (!config.apiKey) {
-    const { key } = await prompts(
-      {
-        type: "text",
-        name: "key",
-        message:
-          "Please provide your Groq API key (or leave blank to use GROQ_API_KEY env variable):",
-      },
-      questionHandlers,
-    );
-    config.apiKey = key || process.env.GROQ_API_KEY;
-  }
-
-  // use default model values in CI or if user should not be asked
-  const useDefaults = ciInfo.isCI || !askModels;
-  if (!useDefaults) {
-    const { model } = await prompts(
-      {
-        type: "select",
-        name: "model",
-        message: "Which LLM model would you like to use?",
-        choices: MODELS.map(toChoice),
-        initial: 0,
-      },
-      questionHandlers,
-    );
-    config.model = model;
-
-    const { embeddingModel } = await prompts(
-      {
-        type: "select",
-        name: "embeddingModel",
-        message: "Which embedding model would you like to use?",
-        choices: Object.keys(EMBEDDING_MODELS).map(toChoice),
-        initial: 0,
-      },
-      questionHandlers,
-    );
-    config.embeddingModel = embeddingModel;
-    config.dimensions =
-      EMBEDDING_MODELS[
-        embeddingModel as HuggingFaceEmbeddingModelType
-      ].dimensions;
-  }
-
-  return config;
-}

helpers/providers/index.ts

@@ -4,7 +4,6 @@ import { questionHandlers } from "../../questions";
 import { ModelConfig, ModelProvider } from "../types";
 import { askAnthropicQuestions } from "./anthropic";
 import { askGeminiQuestions } from "./gemini";
-import { askGroqQuestions } from "./groq";
 import { askOllamaQuestions } from "./ollama";
 import { askOpenAIQuestions } from "./openai";
 
@@ -33,7 +32,6 @@ export async function askModelConfig({
             title: "OpenAI",
             value: "openai",
           },
-          { title: "Groq", value: "groq" },
           { title: "Ollama", value: "ollama" },
           { title: "Anthropic", value: "anthropic" },
           { title: "Gemini", value: "gemini" },
@@ -50,9 +48,6 @@ export async function askModelConfig({
     case "ollama":
       modelConfig = await askOllamaQuestions({ askModels });
       break;
-    case "groq":
-      modelConfig = await askGroqQuestions({ askModels });
-      break;
     case "anthropic":
       modelConfig = await askAnthropicQuestions({ askModels });
       break;

helpers/tools.ts

@@ -54,29 +54,6 @@ export const supportedTools: Tool[] = [
       },
     ],
   },
-  {
-    // For python app, we will use a local DuckDuckGo search tool (instead of DuckDuckGo search tool in LlamaHub)
-    // to get the same results as the TS app.
-    display: "DuckDuckGo Search",
-    name: "duckduckgo",
-    dependencies: [
-      {
-        name: "duckduckgo-search",
-        version: "6.1.7",
-      },
-    ],
-    supportedFrameworks: ["fastapi", "nextjs", "express"],
-    type: ToolType.LOCAL,
-    envVars: [
-      {
-        name: TOOL_SYSTEM_PROMPT_ENV_VAR,
-        description: "System prompt for DuckDuckGo search tool.",
-        value: `You are a DuckDuckGo search agent. 
-You can use the duckduckgo search tool to get information from the web to answer user questions.
-For better results, you can specify the region parameter to get results from a specific region but it's optional.`,
-      },
-    ],
-  },
   {
     display: "Wikipedia",
     name: "wikipedia.WikipediaToolSpec",
@@ -130,12 +107,13 @@ For better results, you can specify the region parameter to get results from a s
       {
         name: TOOL_SYSTEM_PROMPT_ENV_VAR,
         description: "System prompt for code interpreter tool.",
-        value: `-You are a Python interpreter that can run any python code in a secure environment.
-- The python code runs in a Jupyter notebook. Every time you call the 'interpreter' tool, the python code is executed in a separate cell. 
-- You are given tasks to complete and you run python code to solve them.
-- It's okay to make multiple calls to interpreter tool. If you get an error or the result is not what you expected, you can call the tool again. Don't give up too soon!
-- Plot visualizations using matplotlib or any other visualization library directly in the notebook.
-- You can install any pip package (if it exists) by running a cell with pip install.`,
+        value: `You are a Python interpreter.
+        - You are given tasks to complete and you run python code to solve them.
+        - The python code runs in a Jupyter notebook. Every time you call \`interpreter\` tool, the python code is executed in a separate cell. It's okay to make multiple calls to \`interpreter\`.
+        - Display visualizations using matplotlib or any other visualization library directly in the notebook. Shouldn't save the visualizations to a file, just return the base64 encoded data.
+        - You can install any pip package (if it exists) if you need to but the usual packages for data analysis are already preinstalled.
+        - You can run any python code you want in a secure environment.
+        - Use absolute url from result to display images or any other media.`,
       },
     ],
   },
@@ -159,7 +137,7 @@ For better results, you can specify the region parameter to get results from a s
     config: {
       openapi_uri: "The URL or file path of the OpenAPI schema",
     },
-    supportedFrameworks: ["fastapi", "express", "nextjs"],
+    supportedFrameworks: ["fastapi"],
     type: ToolType.LOCAL,
     envVars: [
       {
@@ -170,24 +148,6 @@ For better results, you can specify the region parameter to get results from a s
       },
     ],
   },
-  {
-    display: "Image Generator",
-    name: "img_gen",
-    supportedFrameworks: ["fastapi", "express", "nextjs"],
-    type: ToolType.LOCAL,
-    envVars: [
-      {
-        name: "STABILITY_API_KEY",
-        description:
-          "STABILITY_API_KEY key is required to run image generator. Get it here: https://platform.stability.ai/account/keys",
-      },
-      {
-        name: TOOL_SYSTEM_PROMPT_ENV_VAR,
-        description: "System prompt for image generator tool.",
-        value: `You are an image generator agent. You help users to generate images using the Stability API.`,
-      },
-    ],
-  },
 ];
 
 export const getTool = (toolName: string): Tool | undefined => {

helpers/types.ts

@@ -1,12 +1,7 @@
 import { PackageManager } from "../helpers/get-pkg-manager";
 import { Tool } from "./tools";
 
-export type ModelProvider =
-  | "openai"
-  | "groq"
-  | "ollama"
-  | "anthropic"
-  | "gemini";
+export type ModelProvider = "openai" | "ollama" | "anthropic" | "gemini";
 export type ModelConfig = {
   provider: ModelProvider;
   apiKey?: string;

package.json

@@ -1,6 +1,6 @@
 {
   "name": "create-llama",
-  "version": "0.1.14",
+  "version": "0.1.9",
   "description": "Create LlamaIndex-powered apps with one command",
   "keywords": [
     "rag",

questions.ts

@@ -136,30 +136,24 @@ export const getDataSourceChoices = (
       value: "none",
     });
     choices.push({
-      title:
-        process.platform !== "linux"
-          ? "Use an example PDF"
-          : "Use an example PDF (you can add your own data files later)",
+      title: "Use an example PDF",
       value: "exampleFile",
     });
   }
 
-  // Linux has many distros so we won't support file/folder picker for now
-  if (process.platform !== "linux") {
-    choices.push(
-      {
-        title: `Use local files (${supportedContextFileTypes.join(", ")})`,
-        value: "file",
-      },
-      {
-        title:
-          process.platform === "win32"
-            ? "Use a local folder"
-            : "Use local folders",
-        value: "folder",
-      },
-    );
-  }
+  choices.push(
+    {
+      title: `Use local files (${supportedContextFileTypes.join(", ")})`,
+      value: "file",
+    },
+    {
+      title:
+        process.platform === "win32"
+          ? "Use a local folder"
+          : "Use local folders",
+      value: "folder",
+    },
+  );
 
   if (framework === "fastapi") {
     choices.push({
@@ -662,7 +656,7 @@ export const askQuestions = async (
         t.supportedFrameworks?.includes(program.framework),
       );
       const toolChoices = options.map((tool) => ({
-        title: `${tool.display}${toolRequiresConfig(tool) ? " (needs configuration)" : ""}`,
+        title: `${tool.display}${toolRequiresConfig(tool) ? "" : " (no config needed)"}`,
         value: tool.name,
       }));
       const { toolsName } = await prompts({

templates/components/engines/python/agent/tools/__init__.py

@@ -19,6 +19,15 @@ class ToolFactory:
         ToolType.LOCAL: "app.engine.tools",
     }
 
+    @staticmethod
+    @cached(
+        LRUCache(maxsize=100),
+        key=lambda tool_type, tool_name, config: (
+            tool_type,
+            tool_name,
+            json.dumps(config, sort_keys=True),
+        ),
+    )
     def load_tools(tool_type: str, tool_name: str, config: dict) -> list[FunctionTool]:
         source_package = ToolFactory.TOOL_SOURCE_PACKAGE_MAP[tool_type]
         try:
@@ -31,7 +40,7 @@ class ToolFactory:
                 return tool_spec.to_tool_list()
             else:
                 module = importlib.import_module(f"{source_package}.{tool_name}")
-                tools = module.get_tools(**config)
+                tools = getattr(module, "tools")
                 if not all(isinstance(tool, FunctionTool) for tool in tools):
                     raise ValueError(
                         f"The module {module} does not contain valid tools"

templates/components/engines/python/agent/tools/duckduckgo.py

@@ -1,36 +0,0 @@
-from llama_index.core.tools.function_tool import FunctionTool
-
-
-def duckduckgo_search(
-    query: str,
-    region: str = "wt-wt",
-    max_results: int = 10,
-):
-    """
-    Use this function to search for any query in DuckDuckGo.
-    Args:
-        query (str): The query to search in DuckDuckGo.
-        region Optional(str): The region to be used for the search in [country-language] convention, ex us-en, uk-en, ru-ru, etc...
-        max_results Optional(int): The maximum number of results to be returned. Default is 10.
-    """
-    try:
-        from duckduckgo_search import DDGS
-    except ImportError:
-        raise ImportError(
-            "duckduckgo_search package is required to use this function."
-            "Please install it by running: `poetry add duckduckgo_search` or `pip install duckduckgo_search`"
-        )
-
-    params = {
-        "keywords": query,
-        "region": region,
-        "max_results": max_results,
-    }
-    results = []
-    with DDGS() as ddg:
-        results = list(ddg.text(**params))
-    return results
-
-
-def get_tools(**kwargs):
-    return [FunctionTool.from_defaults(duckduckgo_search)]

templates/components/engines/python/agent/tools/img_gen.py

@@ -1,108 +0,0 @@
-import os
-import uuid
-import logging
-import requests
-from typing import Optional
-from pydantic import BaseModel, Field
-from llama_index.core.tools import FunctionTool
-
-logger = logging.getLogger(__name__)
-
-
-class ImageGeneratorToolOutput(BaseModel):
-    is_success: bool = Field(
-        ...,
-        description="Whether the image generation was successful.",
-    )
-    image_url: Optional[str] = Field(
-        None,
-        description="The URL of the generated image.",
-    )
-    error_message: Optional[str] = Field(
-        None,
-        description="The error message if the image generation failed.",
-    )
-
-
-class ImageGeneratorTool:
-    _IMG_OUTPUT_FORMAT = "webp"
-    _IMG_OUTPUT_DIR = "tool-output"
-    _IMG_GEN_API = "https://api.stability.ai/v2beta/stable-image/generate/core"
-
-    def __init__(self, api_key: str = None):
-        if not api_key:
-            api_key = os.getenv("STABILITY_API_KEY")
-        self._api_key = api_key
-        self.fileserver_url_prefix = os.getenv("FILESERVER_URL_PREFIX")
-        if self._api_key is None:
-            raise ValueError(
-                "STABILITY_API_KEY key is required to run image generator. Get it here: https://platform.stability.ai/account/keys"
-            )
-        if self.fileserver_url_prefix is None:
-            raise ValueError("FILESERVER_URL_PREFIX is required.")
-
-    def _prepare_output_dir(self):
-        """
-        Create the output directory if it doesn't exist
-        """
-        if not os.path.exists(self._IMG_OUTPUT_DIR):
-            os.makedirs(self._IMG_OUTPUT_DIR, exist_ok=True)
-
-    def _save_image(self, image_data: bytes):
-        self._prepare_output_dir()
-        filename = f"{uuid.uuid4()}.{self._IMG_OUTPUT_FORMAT}"
-        output_path = os.path.join(self._IMG_OUTPUT_DIR, filename)
-        with open(output_path, "wb") as f:
-            f.write(image_data)
-        url = f"{os.getenv('FILESERVER_URL_PREFIX')}/{self._IMG_OUTPUT_DIR}/{filename}"
-        logger.info(f"Saved image to {output_path}.\nURL: {url}")
-        return url
-
-    def _call_stability_api(self, prompt: str):
-        headers = {
-            "authorization": f"Bearer {self._api_key}",
-            "accept": "image/*",
-        }
-        data = {
-            "prompt": prompt,
-            "output_format": self._IMG_OUTPUT_FORMAT,
-        }
-
-        response = requests.post(
-            self._IMG_GEN_API,
-            headers=headers,
-            files={"none": ""},
-            data=data,
-        )
-        response.raise_for_status()
-
-        return response
-
-    def generate_image(self, prompt: str) -> ImageGeneratorToolOutput:
-        """
-        Use this tool to generate an image based on the prompt.
-        Args:
-            prompt (str): The prompt to generate the image from.
-        """
-
-        try:
-            # Call the Stability API
-            response = self._call_stability_api(prompt)
-
-            # Save the image and get the URL
-            image_url = self._save_image(response.content)
-
-            return ImageGeneratorToolOutput(
-                is_success=True,
-                image_url=image_url,
-            )
-        except Exception as e:
-            logger.exception(e, exc_info=True)
-            return ImageGeneratorToolOutput(
-                is_success=False,
-                error_message=str(e),
-            )
-
-
-def get_tools(**kwargs):
-    return [FunctionTool.from_defaults(ImageGeneratorTool(**kwargs).generate_image)]

templates/components/engines/python/agent/tools/interpreter.py

@@ -29,24 +29,9 @@ class E2BCodeInterpreter:
 
     output_dir = "tool-output"
 
-    def __init__(self, api_key: str = None):
-        if api_key is None:
-            api_key = os.getenv("E2B_API_KEY")
-        filesever_url_prefix = os.getenv("FILESERVER_URL_PREFIX")
-        if not api_key:
-            raise ValueError(
-                "E2B_API_KEY key is required to run code interpreter. Get it here: https://e2b.dev/docs/getting-started/api-key"
-            )
-        if not filesever_url_prefix:
-            raise ValueError(
-                "FILESERVER_URL_PREFIX is required to display file output from sandbox"
-            )
-
+    def __init__(self, api_key: str, filesever_url_prefix: str):
+        self.api_key = api_key
         self.filesever_url_prefix = filesever_url_prefix
-        self.interpreter = CodeInterpreter(api_key=api_key)
-
-    def __del__(self):
-        self.interpreter.close()
 
     def get_output_path(self, filename: str) -> str:
         # if output directory doesn't exist, create it
@@ -116,28 +101,50 @@ class E2BCodeInterpreter:
         return output
 
     def interpret(self, code: str) -> E2BToolOutput:
-        """
-        Execute python code in a Jupyter notebook cell, the toll will return result, stdout, stderr, display_data, and error.
+        with CodeInterpreter(api_key=self.api_key) as interpreter:
+            logger.info(
+                f"\n{'='*50}\n> Running following AI-generated code:\n{code}\n{'='*50}"
+            )
+            exec = interpreter.notebook.exec_cell(code)
 
-        Parameters:
-            code (str): The python code to be executed in a single cell.
-        """
-        logger.info(
-            f"\n{'='*50}\n> Running following AI-generated code:\n{code}\n{'='*50}"
-        )
-        exec = self.interpreter.notebook.exec_cell(code)
-
-        if exec.error:
-            logger.error("Error when executing code", exec.error)
-            output = E2BToolOutput(is_error=True, logs=exec.logs, results=[])
-        else:
-            if len(exec.results) == 0:
-                output = E2BToolOutput(is_error=False, logs=exec.logs, results=[])
+            if exec.error:
+                logger.error("Error when executing code", exec.error)
+                output = E2BToolOutput(is_error=True, logs=exec.logs, results=[])
             else:
-                results = self.parse_result(exec.results[0])
-                output = E2BToolOutput(is_error=False, logs=exec.logs, results=results)
-        return output
+                if len(exec.results) == 0:
+                    output = E2BToolOutput(is_error=False, logs=exec.logs, results=[])
+                else:
+                    results = self.parse_result(exec.results[0])
+                    output = E2BToolOutput(
+                        is_error=False, logs=exec.logs, results=results
+                    )
+            return output
 
 
-def get_tools(**kwargs):
-    return [FunctionTool.from_defaults(E2BCodeInterpreter(**kwargs).interpret)]
+def code_interpret(code: str) -> Dict:
+    """
+    Execute python code in a Jupyter notebook cell and return any result, stdout, stderr, display_data, and error.
+
+    Parameters:
+        code (str): The python code to be executed in a single cell.
+    """
+    api_key = os.getenv("E2B_API_KEY")
+    filesever_url_prefix = os.getenv("FILESERVER_URL_PREFIX")
+    if not api_key:
+        raise ValueError(
+            "E2B_API_KEY key is required to run code interpreter. Get it here: https://e2b.dev/docs/getting-started/api-key"
+        )
+    if not filesever_url_prefix:
+        raise ValueError(
+            "FILESERVER_URL_PREFIX is required to display file output from sandbox"
+        )
+
+    interpreter = E2BCodeInterpreter(
+        api_key=api_key, filesever_url_prefix=filesever_url_prefix
+    )
+    output = interpreter.interpret(code)
+    return output.dict()
+
+
+# Specify as functions tools to be loaded by the ToolFactory
+tools = [FunctionTool.from_defaults(code_interpret)]

templates/components/engines/python/agent/tools/openapi_action.py

@@ -12,17 +12,10 @@ class OpenAPIActionToolSpec(OpenAPIToolSpec, RequestsToolSpec):
     """
 
     spec_functions = OpenAPIToolSpec.spec_functions + RequestsToolSpec.spec_functions
-    # Cached parsed specs by URI
-    _specs: Dict[str, Tuple[Dict, List[str]]] = {}
 
-    def __init__(self, openapi_uri: str, domain_headers: dict = None, **kwargs):
-        if domain_headers is None:
-            domain_headers = {}
-        if openapi_uri not in self._specs:
-            openapi_spec, servers = self._load_openapi_spec(openapi_uri)
-            self._specs[openapi_uri] = (openapi_spec, servers)
-        else:
-            openapi_spec, servers = self._specs[openapi_uri]
+    def __init__(self, openapi_uri: str, domain_headers: dict = {}, **kwargs):
+        # Load the OpenAPI spec
+        openapi_spec, servers = self.load_openapi_spec(openapi_uri)
 
         # Add the servers to the domain headers if they are not already present
         for server in servers:
@@ -33,7 +26,7 @@ class OpenAPIActionToolSpec(OpenAPIToolSpec, RequestsToolSpec):
         RequestsToolSpec.__init__(self, domain_headers)
 
     @staticmethod
-    def _load_openapi_spec(uri: str) -> Tuple[Dict, List[str]]:
+    def load_openapi_spec(uri: str) -> Tuple[Dict, List[str]]:
         """
         Load an OpenAPI spec from a URI.
 

templates/components/engines/python/agent/tools/weather.py

@@ -69,5 +69,4 @@ class OpenMeteoWeather:
         return response.json()
 
 
-def get_tools(**kwargs):
-    return [FunctionTool.from_defaults(OpenMeteoWeather.get_weather_information)]
+tools = [FunctionTool.from_defaults(OpenMeteoWeather.get_weather_information)]

templates/components/engines/typescript/agent/tools/duckduckgo.ts

@@ -1,61 +0,0 @@
-import { JSONSchemaType } from "ajv";
-import { search } from "duck-duck-scrape";
-import { BaseTool, ToolMetadata } from "llamaindex";
-
-export type DuckDuckGoParameter = {
-  query: string;
-  region?: string;
-};
-
-export type DuckDuckGoToolParams = {
-  metadata?: ToolMetadata<JSONSchemaType<DuckDuckGoParameter>>;
-};
-
-const DEFAULT_META_DATA: ToolMetadata<JSONSchemaType<DuckDuckGoParameter>> = {
-  name: "duckduckgo",
-  description: "Use this function to search for any query in DuckDuckGo.",
-  parameters: {
-    type: "object",
-    properties: {
-      query: {
-        type: "string",
-        description: "The query to search in DuckDuckGo.",
-      },
-      region: {
-        type: "string",
-        description:
-          "Optional, The region to be used for the search in [country-language] convention, ex us-en, uk-en, ru-ru, etc...",
-        nullable: true,
-      },
-    },
-    required: ["query"],
-  },
-};
-
-type DuckDuckGoSearchResult = {
-  title: string;
-  description: string;
-  url: string;
-};
-
-export class DuckDuckGoSearchTool implements BaseTool<DuckDuckGoParameter> {
-  metadata: ToolMetadata<JSONSchemaType<DuckDuckGoParameter>>;
-
-  constructor(params: DuckDuckGoToolParams) {
-    this.metadata = params.metadata ?? DEFAULT_META_DATA;
-  }
-
-  async call(input: DuckDuckGoParameter) {
-    const { query, region } = input;
-    const options = region ? { region } : {};
-    const searchResults = await search(query, options);
-
-    return searchResults.results.map((result) => {
-      return {
-        title: result.title,
-        description: result.description,
-        url: result.url,
-      } as DuckDuckGoSearchResult;
-    });
-  }
-}

templates/components/engines/typescript/agent/tools/img-gen.ts

@@ -1,112 +0,0 @@
-import type { JSONSchemaType } from "ajv";
-import { FormData } from "formdata-node";
-import fs from "fs";
-import got from "got";
-import { BaseTool, ToolMetadata } from "llamaindex";
-import path from "node:path";
-import { Readable } from "stream";
-
-export type ImgGeneratorParameter = {
-  prompt: string;
-};
-
-export type ImgGeneratorToolParams = {
-  metadata?: ToolMetadata<JSONSchemaType<ImgGeneratorParameter>>;
-};
-
-export type ImgGeneratorToolOutput = {
-  isSuccess: boolean;
-  imageUrl?: string;
-  errorMessage?: string;
-};
-
-const DEFAULT_META_DATA: ToolMetadata<JSONSchemaType<ImgGeneratorParameter>> = {
-  name: "image_generator",
-  description: `Use this function to generate an image based on the prompt.`,
-  parameters: {
-    type: "object",
-    properties: {
-      prompt: {
-        type: "string",
-        description: "The prompt to generate the image",
-      },
-    },
-    required: ["prompt"],
-  },
-};
-
-export class ImgGeneratorTool implements BaseTool<ImgGeneratorParameter> {
-  readonly IMG_OUTPUT_FORMAT = "webp";
-  readonly IMG_OUTPUT_DIR = "tool-output";
-  readonly IMG_GEN_API =
-    "https://api.stability.ai/v2beta/stable-image/generate/core";
-
-  metadata: ToolMetadata<JSONSchemaType<ImgGeneratorParameter>>;
-
-  constructor(params?: ImgGeneratorToolParams) {
-    this.checkRequiredEnvVars();
-    this.metadata = params?.metadata || DEFAULT_META_DATA;
-  }
-
-  async call(input: ImgGeneratorParameter): Promise<ImgGeneratorToolOutput> {
-    return await this.generateImage(input.prompt);
-  }
-
-  private generateImage = async (
-    prompt: string,
-  ): Promise<ImgGeneratorToolOutput> => {
-    try {
-      const buffer = await this.promptToImgBuffer(prompt);
-      const imageUrl = this.saveImage(buffer);
-      return { isSuccess: true, imageUrl };
-    } catch (error) {
-      console.error(error);
-      return {
-        isSuccess: false,
-        errorMessage: "Failed to generate image. Please try again.",
-      };
-    }
-  };
-
-  private promptToImgBuffer = async (prompt: string) => {
-    const form = new FormData();
-    form.append("prompt", prompt);
-    form.append("output_format", this.IMG_OUTPUT_FORMAT);
-    const buffer = await got
-      .post(this.IMG_GEN_API, {
-        // Not sure why it shows an type error when passing form to body
-        // Although I follow document: https://github.com/sindresorhus/got/blob/main/documentation/2-options.md#body
-        // Tt still works fine, so I make casting to unknown to avoid the typescript warning
-        // Found a similar issue: https://github.com/sindresorhus/got/discussions/1877
-        body: form as unknown as Buffer | Readable | string,
-        headers: {
-          Authorization: `Bearer ${process.env.STABILITY_API_KEY}`,
-          Accept: "image/*",
-        },
-      })
-      .buffer();
-    return buffer;
-  };
-
-  private saveImage = (buffer: Buffer) => {
-    const filename = `${crypto.randomUUID()}.${this.IMG_OUTPUT_FORMAT}`;
-    const outputPath = path.join(this.IMG_OUTPUT_DIR, filename);
-    fs.writeFileSync(outputPath, buffer);
-    const url = `${process.env.FILESERVER_URL_PREFIX}/${this.IMG_OUTPUT_DIR}/${filename}`;
-    console.log(`Saved image to ${outputPath}.\nURL: ${url}`);
-    return url;
-  };
-
-  private checkRequiredEnvVars = () => {
-    if (!process.env.STABILITY_API_KEY) {
-      throw new Error(
-        "STABILITY_API_KEY key is required to run image generator. Get it here: https://platform.stability.ai/account/keys",
-      );
-    }
-    if (!process.env.FILESERVER_URL_PREFIX) {
-      throw new Error(
-        "FILESERVER_URL_PREFIX is required to display file output after generation",
-      );
-    }
-  };
-}

templates/components/engines/typescript/agent/tools/index.ts

@@ -1,61 +1,42 @@
 import { BaseToolWithCall } from "llamaindex";
 import { ToolsFactory } from "llamaindex/tools/ToolsFactory";
-import { DuckDuckGoSearchTool, DuckDuckGoToolParams } from "./duckduckgo";
-import { ImgGeneratorTool, ImgGeneratorToolParams } from "./img-gen";
 import { InterpreterTool, InterpreterToolParams } from "./interpreter";
-import { OpenAPIActionTool } from "./openapi-action";
 import { WeatherTool, WeatherToolParams } from "./weather";
 
-type ToolCreator = (config: unknown) => Promise<BaseToolWithCall[]>;
+type ToolCreator = (config: unknown) => BaseToolWithCall;
 
 export async function createTools(toolConfig: {
   local: Record<string, unknown>;
   llamahub: any;
 }): Promise<BaseToolWithCall[]> {
   // add local tools from the 'tools' folder (if configured)
-  const tools = await createLocalTools(toolConfig.local);
+  const tools = createLocalTools(toolConfig.local);
   // add tools from LlamaIndexTS (if configured)
   tools.push(...(await ToolsFactory.createTools(toolConfig.llamahub)));
   return tools;
 }
 
 const toolFactory: Record<string, ToolCreator> = {
-  weather: async (config: unknown) => {
-    return [new WeatherTool(config as WeatherToolParams)];
+  weather: (config: unknown) => {
+    return new WeatherTool(config as WeatherToolParams);
   },
-  interpreter: async (config: unknown) => {
-    return [new InterpreterTool(config as InterpreterToolParams)];
-  },
-  "openapi_action.OpenAPIActionToolSpec": async (config: unknown) => {
-    const { openapi_uri, domain_headers } = config as {
-      openapi_uri: string;
-      domain_headers: Record<string, Record<string, string>>;
-    };
-    const openAPIActionTool = new OpenAPIActionTool(
-      openapi_uri,
-      domain_headers,
-    );
-    return await openAPIActionTool.toToolFunctions();
-  },
-  duckduckgo: async (config: unknown) => {
-    return [new DuckDuckGoSearchTool(config as DuckDuckGoToolParams)];
-  },
-  img_gen: async (config: unknown) => {
-    return [new ImgGeneratorTool(config as ImgGeneratorToolParams)];
+  interpreter: (config: unknown) => {
+    return new InterpreterTool(config as InterpreterToolParams);
   },
 };
 
-async function createLocalTools(
+function createLocalTools(
   localConfig: Record<string, unknown>,
-): Promise<BaseToolWithCall[]> {
+): BaseToolWithCall[] {
   const tools: BaseToolWithCall[] = [];
 
-  for (const [key, toolConfig] of Object.entries(localConfig)) {
+  Object.keys(localConfig).forEach((key) => {
     if (key in toolFactory) {
-      const newTools = await toolFactory[key](toolConfig);
-      tools.push(...newTools);
+      const toolConfig = localConfig[key];
+      const tool = toolFactory[key](toolConfig);
+      tools.push(tool);
     }
-  }
+  });
 
   return tools;
 }

templates/components/engines/typescript/agent/tools/openapi-action.ts

@@ -1,164 +0,0 @@
-import SwaggerParser from "@apidevtools/swagger-parser";
-import { JSONSchemaType } from "ajv";
-import got from "got";
-import { FunctionTool, JSONValue, ToolMetadata } from "llamaindex";
-
-interface DomainHeaders {
-  [key: string]: { [header: string]: string };
-}
-
-type Input = {
-  url: string;
-  params: object;
-};
-
-type APIInfo = {
-  description: string;
-  title: string;
-};
-
-export class OpenAPIActionTool {
-  // cache the loaded specs by URL
-  private static specs: Record<string, any> = {};
-
-  private readonly INVALID_URL_PROMPT =
-    "This url did not include a hostname or scheme. Please determine the complete URL and try again.";
-
-  private createLoadSpecMetaData = (info: APIInfo) => {
-    return {
-      name: "load_openapi_spec",
-      description: `Use this to retrieve the OpenAPI spec for the API named ${info.title} with the following description: ${info.description}. Call it before making any requests to the API.`,
-    };
-  };
-
-  private readonly createMethodCallMetaData = (
-    method: "POST" | "PATCH" | "GET",
-    info: APIInfo,
-  ) => {
-    return {
-      name: `${method.toLowerCase()}_request`,
-      description: `Use this to call the ${method} method on the API named ${info.title}`,
-      parameters: {
-        type: "object",
-        properties: {
-          url: {
-            type: "string",
-            description: `The url to make the ${method} request against`,
-          },
-          params: {
-            type: "object",
-            description:
-              method === "GET"
-                ? "the URL parameters to provide with the get request"
-                : `the key-value pairs to provide with the ${method} request`,
-          },
-        },
-        required: ["url"],
-      },
-    } as ToolMetadata<JSONSchemaType<Input>>;
-  };
-
-  constructor(
-    public openapi_uri: string,
-    public domainHeaders: DomainHeaders = {},
-  ) {}
-
-  async loadOpenapiSpec(url: string): Promise<any> {
-    const api = await SwaggerParser.validate(url);
-    return {
-      servers: "servers" in api ? api.servers : "",
-      info: { description: api.info.description, title: api.info.title },
-      endpoints: api.paths,
-    };
-  }
-
-  async getRequest(input: Input): Promise<JSONValue> {
-    if (!this.validUrl(input.url)) {
-      return this.INVALID_URL_PROMPT;
-    }
-    try {
-      const data = await got
-        .get(input.url, {
-          headers: this.getHeadersForUrl(input.url),
-          searchParams: input.params as URLSearchParams,
-        })
-        .json();
-      return data as JSONValue;
-    } catch (error) {
-      return error as JSONValue;
-    }
-  }
-
-  async postRequest(input: Input): Promise<JSONValue> {
-    if (!this.validUrl(input.url)) {
-      return this.INVALID_URL_PROMPT;
-    }
-    try {
-      const res = await got.post(input.url, {
-        headers: this.getHeadersForUrl(input.url),
-        json: input.params,
-      });
-      return res.body as JSONValue;
-    } catch (error) {
-      return error as JSONValue;
-    }
-  }
-
-  async patchRequest(input: Input): Promise<JSONValue> {
-    if (!this.validUrl(input.url)) {
-      return this.INVALID_URL_PROMPT;
-    }
-    try {
-      const res = await got.patch(input.url, {
-        headers: this.getHeadersForUrl(input.url),
-        json: input.params,
-      });
-      return res.body as JSONValue;
-    } catch (error) {
-      return error as JSONValue;
-    }
-  }
-
-  public async toToolFunctions() {
-    if (!OpenAPIActionTool.specs[this.openapi_uri]) {
-      console.log(`Loading spec for URL: ${this.openapi_uri}`);
-      const spec = await this.loadOpenapiSpec(this.openapi_uri);
-      OpenAPIActionTool.specs[this.openapi_uri] = spec;
-    }
-    const spec = OpenAPIActionTool.specs[this.openapi_uri];
-    // TODO: read endpoints with parameters from spec and create one tool for each endpoint
-    // For now, we just create a tool for each HTTP method which does not work well for passing parameters
-    return [
-      FunctionTool.from(() => {
-        return spec;
-      }, this.createLoadSpecMetaData(spec.info)),
-      FunctionTool.from(
-        this.getRequest.bind(this),
-        this.createMethodCallMetaData("GET", spec.info),
-      ),
-      FunctionTool.from(
-        this.postRequest.bind(this),
-        this.createMethodCallMetaData("POST", spec.info),
-      ),
-      FunctionTool.from(
-        this.patchRequest.bind(this),
-        this.createMethodCallMetaData("PATCH", spec.info),
-      ),
-    ];
-  }
-
-  private validUrl(url: string): boolean {
-    const parsed = new URL(url);
-    return !!parsed.protocol && !!parsed.hostname;
-  }
-
-  private getDomain(url: string): string {
-    const parsed = new URL(url);
-    return parsed.hostname;
-  }
-
-  private getHeadersForUrl(url: string): { [header: string]: string } {
-    const domain = this.getDomain(url);
-    return this.domainHeaders[domain] || {};
-  }
-}

templates/components/loaders/python/file.py

@@ -23,12 +23,7 @@ def llama_parse_parser():
             "LLAMA_CLOUD_API_KEY environment variable is not set. "
             "Please set it in .env file or in your shell environment then run again!"
         )
-    parser = LlamaParse(
-        result_type="markdown",
-        verbose=True,
-        language="en",
-        ignore_errors=False,
-    )
+    parser = LlamaParse(result_type="markdown", verbose=True, language="en")
     return parser
 
 
@@ -37,19 +32,15 @@ def get_file_documents(config: FileLoaderConfig):
 
     try:
         reader = SimpleDirectoryReader(
-            config.data_dir, recursive=True, filename_as_id=True, raise_on_error=True
+            config.data_dir,
+            recursive=True,
+            filename_as_id=True,
         )
         if config.use_llama_parse:
-            # LlamaParse is async first,
-            # so we need to use nest_asyncio to run it in sync mode
-            import nest_asyncio
-
-            nest_asyncio.apply()
-
             parser = llama_parse_parser()
             reader.file_extractor = {".pdf": parser}
         return reader.load_data()
-    except Exception as e:
+    except ValueError as e:
         import sys, traceback
 
         # Catch the error if the data dir is empty

templates/components/ui/html/chat/chat-messages.tsx

@@ -24,7 +24,6 @@ export default function ChatMessages({
   ) => Promise<string | null | undefined>;
 }) {
   const scrollableChatContainerRef = useRef<HTMLDivElement>(null);
-  const lastMessage = messages[messages.length - 1];
 
   const scrollToBottom = () => {
     if (scrollableChatContainerRef.current) {
@@ -35,14 +34,14 @@ export default function ChatMessages({
 
   useEffect(() => {
     scrollToBottom();
-  }, [messages.length, lastMessage]);
+  }, [messages.length]);
 
   return (
-    <div
-      className="flex-1 w-full max-w-5xl p-4 bg-white rounded-xl shadow-xl overflow-auto"
-      ref={scrollableChatContainerRef}
-    >
-      <div className="flex flex-col gap-5 divide-y">
+    <div className="w-full max-w-5xl p-4 bg-white rounded-xl shadow-xl">
+      <div
+        className="flex flex-col gap-5 divide-y h-[50vh] overflow-auto"
+        ref={scrollableChatContainerRef}
+      >
         {messages.map((m: Message) => (
           <ChatItem key={m.id} {...m} />
         ))}

templates/types/streaming/express/eslintrc.json

@@ -3,8 +3,5 @@
   "rules": {
     "max-params": ["error", 4],
     "prefer-const": "error"
-  },
-  "parserOptions": {
-    "sourceType": "module"
   }
 }

templates/types/streaming/express/package.json

@@ -1,32 +1,23 @@
 {
   "name": "llama-index-express-streaming",
   "version": "1.0.0",
-  "exports": "./index.js",
-  "types": "./index.d.ts",
-  "type": "module",
-  "engines": {
-    "node": ">=18"
-  },
+  "main": "dist/index.js",
   "scripts": {
     "format": "prettier --ignore-unknown --cache --check .",
     "format:write": "prettier --ignore-unknown --write .",
-    "build": "tsup index.ts --format esm --dts",
+    "build": "tsup index.ts --format cjs --dts",
     "start": "node dist/index.js",
-    "dev": "concurrently \"tsup index.ts --format esm --dts --watch\" \"nodemon --watch dist/index.js\""
+    "dev": "concurrently \"tsup index.ts --format cjs --dts --watch\" \"nodemon --watch dist/index.js\""
   },
   "dependencies": {
     "ai": "^3.0.21",
     "cors": "^2.8.5",
     "dotenv": "^16.3.1",
-    "duck-duck-scrape": "^2.2.5",
     "express": "^4.18.2",
-    "llamaindex": "0.4.3",
+    "llamaindex": "0.3.16",
     "pdf2json": "3.0.5",
     "ajv": "^8.12.0",
-    "@e2b/code-interpreter": "^0.0.5",
-    "got": "^14.4.1",
-    "@apidevtools/swagger-parser": "^10.1.0",
-    "formdata-node": "^6.0.3"
+    "@e2b/code-interpreter": "^0.0.5"
   },
   "devDependencies": {
     "@types/cors": "^2.8.16",

templates/types/streaming/express/src/controllers/engine/settings.ts

@@ -4,7 +4,6 @@ import {
   GEMINI_MODEL,
   Gemini,
   GeminiEmbedding,
-  Groq,
   OpenAI,
   OpenAIEmbedding,
   Settings,
@@ -29,9 +28,6 @@ export const initSettings = async () => {
     case "ollama":
       initOllama();
       break;
-    case "groq":
-      initGroq();
-      break;
     case "anthropic":
       initAnthropic();
       break;
@@ -89,27 +85,6 @@ function initAnthropic() {
   });
 }
 
-function initGroq() {
-  const embedModelMap: Record<string, string> = {
-    "all-MiniLM-L6-v2": "Xenova/all-MiniLM-L6-v2",
-    "all-mpnet-base-v2": "Xenova/all-mpnet-base-v2",
-  };
-
-  const modelMap: Record<string, string> = {
-    "llama3-8b": "llama3-8b-8192",
-    "llama3-70b": "llama3-70b-8192",
-    "mixtral-8x7b": "mixtral-8x7b-32768",
-  };
-
-  Settings.llm = new Groq({
-    model: modelMap[process.env.MODEL!],
-  });
-
-  Settings.embedModel = new HuggingFaceEmbedding({
-    modelType: embedModelMap[process.env.EMBEDDING_MODEL!],
-  });
-}
-
 function initGemini() {
   Settings.llm = new Gemini({
     model: process.env.MODEL as GEMINI_MODEL,

templates/types/streaming/express/src/controllers/llamaindex-stream.ts

@@ -7,12 +7,20 @@ import {
   type AIStreamCallbacksAndOptions,
 } from "ai";
 import {
-  EngineResponse,
   MessageContent,
   MessageContentDetail,
+  Metadata,
+  NodeWithScore,
+  Response,
+  ToolCallLLMMessageOptions,
 } from "llamaindex";
 
-import { CsvFile } from "./stream-helper";
+import { AgentStreamChatResponse } from "llamaindex/agent/base";
+import { CsvFile, appendSourceData } from "./stream-helper";
+
+type LlamaIndexResponse =
+  | AgentStreamChatResponse<ToolCallLLMMessageOptions>
+  | Response;
 
 export const convertMessageContent = (
   content: string,
@@ -78,19 +86,39 @@ const convertAnnotations = (
   return content;
 };
 
-function createParser(res: AsyncIterable<EngineResponse>, data: StreamData) {
+function createParser(
+  res: AsyncIterable<LlamaIndexResponse>,
+  data: StreamData,
+) {
   const it = res[Symbol.asyncIterator]();
   const trimStartOfStream = trimStartOfStreamHelper();
 
+  let sourceNodes: NodeWithScore<Metadata>[] | undefined;
   return new ReadableStream<string>({
     async pull(controller): Promise<void> {
       const { value, done } = await it.next();
       if (done) {
+        if (sourceNodes) {
+          appendSourceData(data, sourceNodes);
+        }
         controller.close();
         data.close();
         return;
       }
-      const text = trimStartOfStream(value.delta ?? "");
+
+      let delta;
+      if (value instanceof Response) {
+        // handle Response type
+        if (value.sourceNodes) {
+          // get source nodes from the first response
+          sourceNodes = value.sourceNodes;
+        }
+        delta = value.response ?? "";
+      } else {
+        // handle other types
+        delta = value.response.delta;
+      }
+      const text = trimStartOfStream(delta ?? "");
       if (text) {
         controller.enqueue(text);
       }
@@ -99,7 +127,7 @@ function createParser(res: AsyncIterable<EngineResponse>, data: StreamData) {
 }
 
 export function LlamaIndexStream(
-  response: AsyncIterable<EngineResponse>,
+  response: AsyncIterable<LlamaIndexResponse>,
   data: StreamData,
   opts?: {
     callbacks?: AIStreamCallbacksAndOptions;

templates/types/streaming/express/src/controllers/stream-helper.ts

@@ -84,9 +84,8 @@ export function createStreamTimeout(stream: StreamData) {
 export function createCallbackManager(stream: StreamData) {
   const callbackManager = new CallbackManager();
 
-  callbackManager.on("retrieve-end", (data) => {
-    const { nodes, query } = data.detail.payload;
-    appendSourceData(stream, nodes);
+  callbackManager.on("retrieve", (data) => {
+    const { nodes, query } = data.detail;
     appendEventData(stream, `Retrieving context for query: '${query}'`);
     appendEventData(
       stream,

templates/types/streaming/fastapi/app/settings.py

@@ -1,6 +1,5 @@
 import os
 from typing import Dict
-
 from llama_index.core.settings import Settings
 
 
@@ -9,8 +8,6 @@ def init_settings():
     match model_provider:
         case "openai":
             init_openai()
-        case "groq":
-            init_groq()
         case "ollama":
             init_ollama()
         case "anthropic":
@@ -26,26 +23,21 @@ def init_settings():
 
 
 def init_ollama():
+    from llama_index.llms.ollama import Ollama
     from llama_index.embeddings.ollama import OllamaEmbedding
-    from llama_index.llms.ollama.base import DEFAULT_REQUEST_TIMEOUT, Ollama
 
     base_url = os.getenv("OLLAMA_BASE_URL") or "http://127.0.0.1:11434"
-    request_timeout = float(
-        os.getenv("OLLAMA_REQUEST_TIMEOUT", DEFAULT_REQUEST_TIMEOUT)
-    )
     Settings.embed_model = OllamaEmbedding(
         base_url=base_url,
         model_name=os.getenv("EMBEDDING_MODEL"),
     )
-    Settings.llm = Ollama(
-        base_url=base_url, model=os.getenv("MODEL"), request_timeout=request_timeout
-    )
+    Settings.llm = Ollama(base_url=base_url, model=os.getenv("MODEL"))
 
 
 def init_openai():
-    from llama_index.core.constants import DEFAULT_TEMPERATURE
-    from llama_index.embeddings.openai import OpenAIEmbedding
     from llama_index.llms.openai import OpenAI
+    from llama_index.embeddings.openai import OpenAIEmbedding
+    from llama_index.core.constants import DEFAULT_TEMPERATURE
 
     max_tokens = os.getenv("LLM_MAX_TOKENS")
     config = {
@@ -64,9 +56,9 @@ def init_openai():
 
 
 def init_azure_openai():
-    from llama_index.core.constants import DEFAULT_TEMPERATURE
-    from llama_index.embeddings.azure_openai import AzureOpenAIEmbedding
     from llama_index.llms.azure_openai import AzureOpenAI
+    from llama_index.embeddings.azure_openai import AzureOpenAIEmbedding
+    from llama_index.core.constants import DEFAULT_TEMPERATURE
 
     llm_deployment = os.getenv("AZURE_OPENAI_LLM_DEPLOYMENT")
     embedding_deployment = os.getenv("AZURE_OPENAI_EMBEDDING_DEPLOYMENT")
@@ -91,30 +83,9 @@ def init_azure_openai():
     Settings.embed_model = AzureOpenAIEmbedding(**embedding_config)
 
 
-def init_groq():
-    from llama_index.embeddings.huggingface import HuggingFaceEmbedding
-    from llama_index.llms.groq import Groq
-
-    model_map: Dict[str, str] = {
-        "llama3-8b": "llama3-8b-8192",
-        "llama3-70b": "llama3-70b-8192",
-        "mixtral-8x7b": "mixtral-8x7b-32768",
-    }
-
-    embed_model_map: Dict[str, str] = {
-        "all-MiniLM-L6-v2": "sentence-transformers/all-MiniLM-L6-v2",
-        "all-mpnet-base-v2": "sentence-transformers/all-mpnet-base-v2",
-    }
-
-    Settings.llm = Groq(model=model_map[os.getenv("MODEL")])
-    Settings.embed_model = HuggingFaceEmbedding(
-        model_name=embed_model_map[os.getenv("EMBEDDING_MODEL")]
-    )
-
-
 def init_anthropic():
-    from llama_index.embeddings.huggingface import HuggingFaceEmbedding
     from llama_index.llms.anthropic import Anthropic
+    from llama_index.embeddings.huggingface import HuggingFaceEmbedding
 
     model_map: Dict[str, str] = {
         "claude-3-opus": "claude-3-opus-20240229",
@@ -136,11 +107,21 @@ def init_anthropic():
 
 
 def init_gemini():
-    from llama_index.embeddings.gemini import GeminiEmbedding
     from llama_index.llms.gemini import Gemini
+    from llama_index.embeddings.gemini import GeminiEmbedding
 
-    model_name = f"models/{os.getenv('MODEL')}"
-    embed_model_name = f"models/{os.getenv('EMBEDDING_MODEL')}"
+    model_map: Dict[str, str] = {
+        "gemini-1.5-pro-latest": "models/gemini-1.5-pro-latest",
+        "gemini-pro": "models/gemini-pro",
+        "gemini-pro-vision": "models/gemini-pro-vision",
+    }
 
-    Settings.llm = Gemini(model=model_name)
-    Settings.embed_model = GeminiEmbedding(model_name=embed_model_name)
+    embed_model_map: Dict[str, str] = {
+        "embedding-001": "models/embedding-001",
+        "text-embedding-004": "models/text-embedding-004",
+    }
+
+    Settings.llm = Gemini(model=model_map[os.getenv("MODEL")])
+    Settings.embed_model = GeminiEmbedding(
+        model_name=embed_model_map[os.getenv("EMBEDDING_MODEL")]
+    )

templates/types/streaming/nextjs/app/api/chat/engine/settings.ts

@@ -4,7 +4,6 @@ import {
   GEMINI_MODEL,
   Gemini,
   GeminiEmbedding,
-  Groq,
   OpenAI,
   OpenAIEmbedding,
   Settings,
@@ -29,9 +28,6 @@ export const initSettings = async () => {
     case "ollama":
       initOllama();
       break;
-    case "groq":
-      initGroq();
-      break;
     case "anthropic":
       initAnthropic();
       break;
@@ -75,27 +71,6 @@ function initOllama() {
   });
 }
 
-function initGroq() {
-  const embedModelMap: Record<string, string> = {
-    "all-MiniLM-L6-v2": "Xenova/all-MiniLM-L6-v2",
-    "all-mpnet-base-v2": "Xenova/all-mpnet-base-v2",
-  };
-
-  const modelMap: Record<string, string> = {
-    "llama3-8b": "llama3-8b-8192",
-    "llama3-70b": "llama3-70b-8192",
-    "mixtral-8x7b": "mixtral-8x7b-32768",
-  };
-
-  Settings.llm = new Groq({
-    model: modelMap[process.env.MODEL!],
-  });
-
-  Settings.embedModel = new HuggingFaceEmbedding({
-    modelType: embedModelMap[process.env.EMBEDDING_MODEL!],
-  });
-}
-
 function initAnthropic() {
   const embedModelMap: Record<string, string> = {
     "all-MiniLM-L6-v2": "Xenova/all-MiniLM-L6-v2",

templates/types/streaming/nextjs/app/api/chat/llamaindex-stream.ts

@@ -7,12 +7,20 @@ import {
   type AIStreamCallbacksAndOptions,
 } from "ai";
 import {
-  EngineResponse,
   MessageContent,
   MessageContentDetail,
+  Metadata,
+  NodeWithScore,
+  Response,
+  ToolCallLLMMessageOptions,
 } from "llamaindex";
 
-import { CsvFile } from "./stream-helper";
+import { AgentStreamChatResponse } from "llamaindex/agent/base";
+import { CsvFile, appendSourceData } from "./stream-helper";
+
+type LlamaIndexResponse =
+  | AgentStreamChatResponse<ToolCallLLMMessageOptions>
+  | Response;
 
 export const convertMessageContent = (
   content: string,
@@ -78,19 +86,39 @@ const convertAnnotations = (
   return content;
 };
 
-function createParser(res: AsyncIterable<EngineResponse>, data: StreamData) {
+function createParser(
+  res: AsyncIterable<LlamaIndexResponse>,
+  data: StreamData,
+) {
   const it = res[Symbol.asyncIterator]();
   const trimStartOfStream = trimStartOfStreamHelper();
 
+  let sourceNodes: NodeWithScore<Metadata>[] | undefined;
   return new ReadableStream<string>({
     async pull(controller): Promise<void> {
       const { value, done } = await it.next();
       if (done) {
+        if (sourceNodes) {
+          appendSourceData(data, sourceNodes);
+        }
         controller.close();
         data.close();
         return;
       }
-      const text = trimStartOfStream(value.delta ?? "");
+
+      let delta;
+      if (value instanceof Response) {
+        // handle Response type
+        if (value.sourceNodes) {
+          // get source nodes from the first response
+          sourceNodes = value.sourceNodes;
+        }
+        delta = value.response ?? "";
+      } else {
+        // handle other types
+        delta = value.response.delta;
+      }
+      const text = trimStartOfStream(delta ?? "");
       if (text) {
         controller.enqueue(text);
       }
@@ -99,7 +127,7 @@ function createParser(res: AsyncIterable<EngineResponse>, data: StreamData) {
 }
 
 export function LlamaIndexStream(
-  response: AsyncIterable<EngineResponse>,
+  response: AsyncIterable<LlamaIndexResponse>,
   data: StreamData,
   opts?: {
     callbacks?: AIStreamCallbacksAndOptions;

templates/types/streaming/nextjs/app/api/chat/stream-helper.ts

@@ -84,9 +84,8 @@ export function createStreamTimeout(stream: StreamData) {
 export function createCallbackManager(stream: StreamData) {
   const callbackManager = new CallbackManager();
 
-  callbackManager.on("retrieve-end", (data) => {
-    const { nodes, query } = data.detail.payload;
-    appendSourceData(stream, nodes);
+  callbackManager.on("retrieve", (data) => {
+    const { nodes, query } = data.detail;
     appendEventData(stream, `Retrieving context for query: '${query}'`);
     appendEventData(
       stream,

templates/types/streaming/nextjs/app/components/chat-section.tsx

@@ -2,7 +2,7 @@
 
 import { useChat } from "ai/react";
 import { ChatInput, ChatMessages } from "./ui/chat";
-import { useClientConfig } from "./ui/chat/hooks/use-config";
+import { useClientConfig } from "./ui/chat/use-config";
 
 export default function ChatSection() {
   const { chatAPI } = useClientConfig();
@@ -29,7 +29,7 @@ export default function ChatSection() {
   });
 
   return (
-    <div className="space-y-4 w-full h-full flex flex-col">
+    <div className="space-y-4 max-w-5xl w-full">
       <ChatMessages
         messages={messages}
         isLoading={isLoading}

templates/types/streaming/nextjs/app/components/ui/chat/chat-events.tsx

@@ -1,12 +1,12 @@
 import { ChevronDown, ChevronRight, Loader2 } from "lucide-react";
 import { useState } from "react";
-import { Button } from "../../button";
+import { Button } from "../button";
 import {
   Collapsible,
   CollapsibleContent,
   CollapsibleTrigger,
-} from "../../collapsible";
-import { EventData } from "../index";
+} from "../collapsible";
+import { EventData } from "./index";
 
 export function ChatEvents({
   data,

templates/types/streaming/nextjs/app/components/ui/chat/chat-image.tsx

@@ -1,5 +1,5 @@
 import Image from "next/image";
-import { type ImageData } from "../index";
+import { type ImageData } from "./index";
 
 export function ChatImage({ data }: { data: ImageData }) {
   return (

templates/types/streaming/nextjs/app/components/ui/chat/chat-input.tsx

@@ -8,7 +8,7 @@ import { Input } from "../input";
 import UploadCsvPreview from "../upload-csv-preview";
 import UploadImagePreview from "../upload-image-preview";
 import { ChatHandler } from "./chat.interface";
-import { useCsv } from "./hooks/use-csv";
+import { useCsv } from "./use-csv";
 
 export default function ChatInput(
   props: Pick<
@@ -81,27 +81,23 @@ export default function ChatInput(
 
   const onRemovePreviewImage = () => setImageUrl(null);
 
-  const readContent = async (file: File): Promise<string> => {
-    const content = await new Promise<string>((resolve, reject) => {
+  const handleUploadImageFile = async (file: File) => {
+    const base64 = await new Promise<string>((resolve, reject) => {
       const reader = new FileReader();
-      if (file.type.startsWith("image/")) {
-        reader.readAsDataURL(file);
-      } else {
-        reader.readAsText(file);
-      }
+      reader.readAsDataURL(file);
       reader.onload = () => resolve(reader.result as string);
       reader.onerror = (error) => reject(error);
     });
-    return content;
-  };
-
-  const handleUploadImageFile = async (file: File) => {
-    const base64 = await readContent(file);
     setImageUrl(base64);
   };
 
   const handleUploadCsvFile = async (file: File) => {
-    const content = await readContent(file);
+    const content = await new Promise<string>((resolve, reject) => {
+      const reader = new FileReader();
+      reader.readAsText(file);
+      reader.onload = () => resolve(reader.result as string);
+      reader.onerror = (error) => reject(error);
+    });
     const isSuccess = upload({
       id: uuidv4(),
       content,
@@ -134,7 +130,7 @@ export default function ChatInput(
   return (
     <form
       onSubmit={onSubmit}
-      className="rounded-xl bg-white p-4 shadow-xl space-y-4 shrink-0"
+      className="rounded-xl bg-white p-4 shadow-xl space-y-4"
     >
       {imageUrl && (
         <UploadImagePreview url={imageUrl} onRemove={onRemovePreviewImage} />

templates/types/streaming/nextjs/app/components/ui/chat/chat-message.tsx

@@ -2,8 +2,13 @@ import { Check, Copy } from "lucide-react";
 
 import { Message } from "ai";
 import { Fragment } from "react";
-import { Button } from "../../button";
-import { useCopyToClipboard } from "../hooks/use-copy-to-clipboard";
+import { Button } from "../button";
+import ChatAvatar from "./chat-avatar";
+import { ChatEvents } from "./chat-events";
+import { ChatImage } from "./chat-image";
+import { ChatSources } from "./chat-sources";
+import ChatTools from "./chat-tools";
+import CsvContent from "./csv-content";
 import {
   CsvData,
   EventData,
@@ -13,14 +18,9 @@ import {
   SourceData,
   ToolData,
   getAnnotationData,
-} from "../index";
-import ChatAvatar from "./chat-avatar";
-import { ChatEvents } from "./chat-events";
-import { ChatImage } from "./chat-image";
-import { ChatSources } from "./chat-sources";
-import ChatTools from "./chat-tools";
-import CsvContent from "./csv-content";
+} from "./index";
 import Markdown from "./markdown";
+import { useCopyToClipboard } from "./use-copy-to-clipboard";
 
 type ContentDisplayConfig = {
   order: number;

templates/types/streaming/nextjs/app/components/ui/chat/chat-messages.tsx

@@ -5,7 +5,7 @@ import { Button } from "../button";
 import ChatActions from "./chat-actions";
 import ChatMessage from "./chat-message";
 import { ChatHandler } from "./chat.interface";
-import { useClientConfig } from "./hooks/use-config";
+import { useClientConfig } from "./use-config";
 
 export default function ChatMessages(
   props: Pick<
@@ -41,11 +41,11 @@ export default function ChatMessages(
   }, [messageLength, lastMessage]);
 
   return (
-    <div
-      className="flex-1 w-full rounded-xl bg-white p-4 shadow-xl relative overflow-y-auto"
-      ref={scrollableChatContainerRef}
-    >
-      <div className="flex flex-col gap-5 divide-y">
+    <div className="w-full rounded-xl bg-white p-4 shadow-xl pb-0 relative">
+      <div
+        className="flex h-[50vh] flex-col gap-5 divide-y overflow-y-auto pb-4"
+        ref={scrollableChatContainerRef}
+      >
         {props.messages.map((m, i) => {
           const isLoadingMessage = i === messageLength - 1 && props.isLoading;
           return (
@@ -62,16 +62,14 @@ export default function ChatMessages(
           </div>
         )}
       </div>
-      {(showReload || showStop) && (
-        <div className="flex justify-end py-4">
-          <ChatActions
-            reload={props.reload}
-            stop={props.stop}
-            showReload={showReload}
-            showStop={showStop}
-          />
-        </div>
-      )}
+      <div className="flex justify-end py-4">
+        <ChatActions
+          reload={props.reload}
+          stop={props.stop}
+          showReload={showReload}
+          showStop={showStop}
+        />
+      </div>
       {!messageLength && starterQuestions?.length && props.append && (
         <div className="absolute bottom-6 left-0 w-full">
           <div className="grid grid-cols-2 gap-2 mx-20">

templates/types/streaming/nextjs/app/components/ui/chat/chat-sources.tsx

@@ -1,14 +1,10 @@
 import { Check, Copy } from "lucide-react";
 import { useMemo } from "react";
-import { Button } from "../../button";
-import {
-  HoverCard,
-  HoverCardContent,
-  HoverCardTrigger,
-} from "../../hover-card";
-import { useCopyToClipboard } from "../hooks/use-copy-to-clipboard";
-import { SourceData } from "../index";
-import PdfDialog from "../widgets/PdfDialog";
+import { Button } from "../button";
+import { HoverCard, HoverCardContent, HoverCardTrigger } from "../hover-card";
+import { SourceData } from "./index";
+import { useCopyToClipboard } from "./use-copy-to-clipboard";
+import PdfDialog from "./widgets/PdfDialog";
 
 const SCORE_THRESHOLD = 0.3;
 

templates/types/streaming/nextjs/app/components/ui/chat/chat-tools.tsx

@@ -1,5 +1,5 @@
-import { ToolData } from "../index";
-import { WeatherCard, WeatherData } from "../widgets/WeatherCard";
+import { ToolData } from "./index";
+import { WeatherCard, WeatherData } from "./widgets/WeatherCard";
 
 // TODO: If needed, add displaying more tool outputs here
 export default function ChatTools({ data }: { data: ToolData }) {

templates/types/streaming/nextjs/app/components/ui/chat/codeblock.tsx

@@ -5,8 +5,8 @@ import { FC, memo } from "react";
 import { Prism, SyntaxHighlighterProps } from "react-syntax-highlighter";
 import { coldarkDark } from "react-syntax-highlighter/dist/cjs/styles/prism";
 
-import { Button } from "../../button";
-import { useCopyToClipboard } from "../hooks/use-copy-to-clipboard";
+import { Button } from "../button";
+import { useCopyToClipboard } from "./use-copy-to-clipboard";
 
 // TODO: Remove this when @type/react-syntax-highlighter is updated
 const SyntaxHighlighter = Prism as unknown as FC<SyntaxHighlighterProps>;

templates/types/streaming/nextjs/app/components/ui/chat/csv-content.tsx

@@ -1,5 +1,5 @@
-import UploadCsvPreview from "../../upload-csv-preview";
-import { CsvData } from "../index";
+import { CsvData } from ".";
+import UploadCsvPreview from "../upload-csv-preview";
 
 export default function CsvContent({ data }: { data: CsvData }) {
   if (!data.csvFiles.length) return null;

templates/types/streaming/nextjs/app/components/ui/chat/markdown.tsx

@@ -17,12 +17,12 @@ const MemoizedReactMarkdown: FC<Options> = memo(
 const preprocessLaTeX = (content: string) => {
   // Replace block-level LaTeX delimiters \[ \] with $$ $$
   const blockProcessedContent = content.replace(
-    /\\\[([\s\S]*?)\\\]/g,
+    /\\\[(.*?)\\\]/gs,
     (_, equation) => `$$${equation}$$`,
   );
   // Replace inline LaTeX delimiters \( \) with $ $
   const inlineProcessedContent = blockProcessedContent.replace(
-    /\\\[([\s\S]*?)\\\]/g,
+    /\\\((.*?)\\\)/gs,
     (_, equation) => `$${equation}$`,
   );
   return inlineProcessedContent;

templates/types/streaming/nextjs/app/components/ui/chat/use-csv.ts

@@ -1,7 +1,7 @@
 "use client";
 
 import { useState } from "react";
-import { CsvFile } from "../index";
+import { CsvFile } from ".";
 
 export function useCsv() {
   const [files, setFiles] = useState<CsvFile[]>([]);

templates/types/streaming/nextjs/app/globals.css

@@ -74,11 +74,8 @@
   * {
     @apply border-border;
   }
-  html {
-    @apply h-full;
-  }
   body {
-    @apply bg-background text-foreground h-full;
+    @apply bg-background text-foreground;
     font-feature-settings:
       "rlig" 1,
       "calt" 1;

templates/types/streaming/nextjs/app/page.tsx

@@ -3,13 +3,9 @@ import ChatSection from "./components/chat-section";
 
 export default function Home() {
   return (
-    <main className="h-full w-full flex justify-center items-center background-gradient">
-      <div className="space-y-2 lg:space-y-10 w-[90%] lg:w-[60rem]">
-        <Header />
-        <div className="h-[65vh] flex">
-          <ChatSection />
-        </div>
-      </div>
+    <main className="flex min-h-screen flex-col items-center gap-10 p-24 background-gradient">
+      <Header />
+      <ChatSection />
     </main>
   );
 }

templates/types/streaming/nextjs/package.json

@@ -10,9 +10,6 @@
     "lint": "next lint"
   },
   "dependencies": {
-    "@apidevtools/swagger-parser": "^10.1.0",
-    "@e2b/code-interpreter": "^0.0.5",
-    "@llamaindex/pdf-viewer": "^1.1.1",
     "@radix-ui/react-collapsible": "^1.0.3",
     "@radix-ui/react-hover-card": "^1.0.7",
     "@radix-ui/react-slot": "^1.0.2",
@@ -21,44 +18,42 @@
     "class-variance-authority": "^0.7.0",
     "clsx": "^2.1.1",
     "dotenv": "^16.3.1",
-    "duck-duck-scrape": "^2.2.5",
-    "formdata-node": "^6.0.3",
-    "got": "^14.4.1",
-    "llamaindex": "0.4.3",
+    "llamaindex": "0.3.16",
     "lucide-react": "^0.294.0",
-    "next": "^14.2.4",
+    "next": "^14.0.3",
     "pdf2json": "3.0.5",
     "react": "^18.2.0",
     "react-dom": "^18.2.0",
     "react-markdown": "^8.0.7",
     "react-syntax-highlighter": "^15.5.0",
-    "rehype-katex": "^7.0.0",
     "remark": "^14.0.3",
     "remark-code-import": "^1.2.0",
     "remark-gfm": "^3.0.1",
     "remark-math": "^5.1.1",
+    "rehype-katex": "^7.0.0",
     "supports-color": "^8.1.1",
     "tailwind-merge": "^2.1.0",
-    "tiktoken": "^1.0.15",
-    "uuid": "^9.0.1",
-    "vaul": "^0.9.1"
+    "vaul": "^0.9.1",
+    "@llamaindex/pdf-viewer": "^1.1.1",
+    "@e2b/code-interpreter": "^0.0.5",
+    "uuid": "^9.0.1"
   },
   "devDependencies": {
     "@types/node": "^20.10.3",
     "@types/react": "^18.2.42",
     "@types/react-dom": "^18.2.17",
     "@types/react-syntax-highlighter": "^15.5.11",
-    "@types/uuid": "^9.0.8",
     "autoprefixer": "^10.4.16",
     "cross-env": "^7.0.3",
     "eslint": "^8.55.0",
-    "eslint-config-next": "^14.2.4",
+    "eslint-config-next": "^14.0.3",
     "eslint-config-prettier": "^8.10.0",
     "postcss": "^8.4.32",
     "prettier": "^3.2.5",
     "prettier-plugin-organize-imports": "^3.2.4",
     "tailwindcss": "^3.3.6",
     "tsx": "^4.7.2",
-    "typescript": "^5.3.2"
+    "typescript": "^5.3.2",
+    "@types/uuid": "^9.0.8"
   }
 }