format code and fix missing default EMBEDDING_DIM

* feat: display conversation starter from frontend env

* use nextjs config api

* update to /api/chat/config

* add config api for express

* add api config for fast api

* Create ten-badgers-learn.md

* remove default conversation staters

* check empty string

* update pydantic docs

* refactor: move NEXT_PUBLIC_CHAT_API to use config

* use config to get chatAPI

* refactor: rename useClientConfig

.changeset/bright-turkeys-melt.md

@@ -0,0 +1,5 @@
+---
+"create-llama": patch
+---
+
+Add CSV upload

.changeset/itchy-ads-travel.md

@@ -0,0 +1,5 @@
+---
+"create-llama": patch
+---
+
+Add E2B code interpreter tool for FastAPI

.changeset/quiet-cars-clap.md

@@ -0,0 +1,5 @@
+---
+"create-llama": patch
+---
+
+Add OpenAPI action tool for FastAPI

helpers/env-variables.ts

@@ -185,6 +185,10 @@ const getModelEnvs = (modelConfig: ModelConfig): EnvVar[] => {
       description: "Dimension of the embedding model to use.",
       value: modelConfig.dimensions.toString(),
     },
+    {
+      name: "CONVERSATION_STARTERS",
+      description: "The questions to help users get started (multi-line).",
+    },
     ...(modelConfig.provider === "openai"
       ? [
           {
@@ -276,6 +280,12 @@ const getEngineEnvs = (): EnvVar[] => {
         "The number of similar embeddings to return when retrieving documents.",
       value: "3",
     },
+    {
+      name: "STREAM_TIMEOUT",
+      description:
+        "The time in milliseconds to wait for the stream to return a response.",
+      value: "60000",
+    },
   ];
 };
 

helpers/index.ts

@@ -171,6 +171,11 @@ export const installTemplate = async (
         );
       }
     }
+
+    // Create tool-output directory
+    if (props.tools && props.tools.length > 0) {
+      await fsExtra.mkdir(path.join(props.root, "tool-output"));
+    }
   } else {
     // this is a frontend for a full-stack app, create .env file with model information
     await createFrontendEnvFile(props.root, {

helpers/python.ts

@@ -144,7 +144,7 @@ const getAdditionalDependencies = (
     case "openai":
       dependencies.push({
         name: "llama-index-agent-openai",
-        version: "0.2.2",
+        version: "0.2.6",
       });
       break;
     case "anthropic":
@@ -160,7 +160,7 @@ const getAdditionalDependencies = (
     case "gemini":
       dependencies.push({
         name: "llama-index-llms-gemini",
-        version: "0.1.7",
+        version: "0.1.10",
       });
       dependencies.push({
         name: "llama-index-embeddings-gemini",

helpers/tools.ts

@@ -90,8 +90,13 @@ export const supportedTools: Tool[] = [
   {
     display: "Code Interpreter",
     name: "interpreter",
-    dependencies: [],
-    supportedFrameworks: ["express", "nextjs"],
+    dependencies: [
+      {
+        name: "e2b_code_interpreter",
+        version: "0.0.7",
+      },
+    ],
+    supportedFrameworks: ["fastapi", "express", "nextjs"],
     type: ToolType.LOCAL,
     envVars: [
       {
@@ -112,6 +117,37 @@ export const supportedTools: Tool[] = [
       },
     ],
   },
+  {
+    display: "OpenAPI action",
+    name: "openapi_action.OpenAPIActionToolSpec",
+    dependencies: [
+      {
+        name: "llama-index-tools-openapi",
+        version: "0.1.3",
+      },
+      {
+        name: "jsonschema",
+        version: "^4.22.0",
+      },
+      {
+        name: "llama-index-tools-requests",
+        version: "0.1.3",
+      },
+    ],
+    config: {
+      openapi_uri: "The URL or file path of the OpenAPI schema",
+    },
+    supportedFrameworks: ["fastapi"],
+    type: ToolType.LOCAL,
+    envVars: [
+      {
+        name: TOOL_SYSTEM_PROMPT_ENV_VAR,
+        description: "System prompt for openapi action tool.",
+        value:
+          "You are an OpenAPI action agent. You help users to make requests to the provided OpenAPI schema.",
+      },
+    ],
+  },
 ];
 
 export const getTool = (toolName: string): Tool | undefined => {
@@ -137,9 +173,15 @@ export const getTools = (toolsName: string[]): Tool[] => {
   return tools;
 };
 
+export const toolRequiresConfig = (tool: Tool): boolean => {
+  const hasConfig = Object.keys(tool.config || {}).length > 0;
+  const hasEmptyEnvVar = tool.envVars?.some((envVar) => !envVar.value) ?? false;
+  return hasConfig || hasEmptyEnvVar;
+};
+
 export const toolsRequireConfig = (tools?: Tool[]): boolean => {
   if (tools) {
-    return tools?.some((tool) => Object.keys(tool.config || {}).length > 0);
+    return tools?.some(toolRequiresConfig);
   }
   return false;
 };
@@ -154,7 +196,6 @@ export const writeToolsConfig = async (
   tools: Tool[] = [],
   type: ConfigFileType = ConfigFileType.YAML,
 ) => {
-  if (tools.length === 0) return; // no tools selected, no config need
   const configContent: {
     [key in ToolType]: Record<string, any>;
   } = {

questions.ts

@@ -16,7 +16,11 @@ import { templatesDir } from "./helpers/dir";
 import { getAvailableLlamapackOptions } from "./helpers/llama-pack";
 import { askModelConfig } from "./helpers/providers";
 import { getProjectOptions } from "./helpers/repo";
-import { supportedTools, toolsRequireConfig } from "./helpers/tools";
+import {
+  supportedTools,
+  toolRequiresConfig,
+  toolsRequireConfig,
+} from "./helpers/tools";
 
 export type QuestionArgs = Omit<
   InstallAppArgs,
@@ -652,7 +656,7 @@ export const askQuestions = async (
         t.supportedFrameworks?.includes(program.framework),
       );
       const toolChoices = options.map((tool) => ({
-        title: tool.display,
+        title: `${tool.display}${toolRequiresConfig(tool) ? "" : " (no config needed)"}`,
         value: tool.name,
       }));
       const { toolsName } = await prompts({

templates/components/engines/python/agent/tools/__init__.py

@@ -1,7 +1,8 @@
 import os
 import yaml
+import json
 import importlib
-
+from cachetools import cached, LRUCache
 from llama_index.core.tools.tool_spec.base import BaseToolSpec
 from llama_index.core.tools.function_tool import FunctionTool
 
@@ -12,13 +13,20 @@ class ToolType:
 
 
 class ToolFactory:
-
     TOOL_SOURCE_PACKAGE_MAP = {
         ToolType.LLAMAHUB: "llama_index.tools",
         ToolType.LOCAL: "app.engine.tools",
     }
 
     @staticmethod
+    @cached(
+        LRUCache(maxsize=100),
+        key=lambda tool_type, tool_name, config: (
+            tool_type,
+            tool_name,
+            json.dumps(config, sort_keys=True),
+        ),
+    )
     def load_tools(tool_type: str, tool_name: str, config: dict) -> list[FunctionTool]:
         source_package = ToolFactory.TOOL_SOURCE_PACKAGE_MAP[tool_type]
         try:

templates/components/engines/python/agent/tools/interpreter.py

@@ -0,0 +1,156 @@
+import os
+import logging
+import base64
+import uuid
+from pydantic import BaseModel
+from typing import List, Tuple, Dict, Optional
+from llama_index.core.tools import FunctionTool
+from e2b_code_interpreter import CodeInterpreter
+from e2b_code_interpreter.models import Logs
+
+
+logger = logging.getLogger(__name__)
+
+
+class InterpreterExtraResult(BaseModel):
+    type: str
+    content: Optional[str] = None
+    filename: Optional[str] = None
+    url: Optional[str] = None
+
+
+class E2BToolOutput(BaseModel):
+    is_error: bool
+    logs: Logs
+    results: List[InterpreterExtraResult | str] = []
+
+
+class E2BCodeInterpreter:
+    output_dir = "tool-output"
+
+    def __init__(self, api_key: str, filesever_url_prefix: str):
+        self.api_key = api_key
+        self.filesever_url_prefix = filesever_url_prefix
+
+    def get_output_path(self, filename: str) -> str:
+        # if output directory doesn't exist, create it
+        if not os.path.exists(self.output_dir):
+            os.makedirs(self.output_dir, exist_ok=True)
+        return os.path.join(self.output_dir, filename)
+
+    def save_to_disk(self, base64_data: str, ext: str) -> Dict:
+        filename = f"{uuid.uuid4()}.{ext}"  # generate a unique filename
+        buffer = base64.b64decode(base64_data)
+        output_path = self.get_output_path(filename)
+
+        try:
+            with open(output_path, "wb") as file:
+                file.write(buffer)
+        except IOError as e:
+            logger.error(f"Failed to write to file {output_path}: {str(e)}")
+            raise e
+
+        logger.info(f"Saved file to {output_path}")
+
+        return {
+            "outputPath": output_path,
+            "filename": filename,
+        }
+
+    def get_file_url(self, filename: str) -> str:
+        return f"{self.filesever_url_prefix}/{self.output_dir}/{filename}"
+
+    def parse_result(self, result) -> List[InterpreterExtraResult]:
+        """
+        The result format could be either a base64 string (png, svg, etc.) or a raw text (text, html, markdown,...)
+        If it's base64, we save each result to disk and return saved file metadata (extension, filename, url),
+        otherwise just return the raw text content
+        """
+        if not result:
+            return []
+
+        output = []
+
+        try:
+            formats = result.formats()
+            data_list = [result[format] for format in formats]
+
+            for ext, data in zip(formats, data_list):
+                match ext:
+                    case "png" | "jpeg" | "svg":
+                        result = self.save_to_disk(data, ext)
+                        filename = result["filename"]
+                        output.append(
+                            InterpreterExtraResult(
+                                type=ext,
+                                filename=filename,
+                                url=self.get_file_url(filename),
+                            )
+                        )
+                        break
+                    case "text" | "html" | "markdown":
+                        output.append(InterpreterExtraResult(type=ext, content=data))
+        except Exception as error:
+            logger.error("Error when saving data to disk", error)
+
+        return output
+
+    def interpret(self, code: str, file_path: Optional[str] = None) -> E2BToolOutput:
+        with CodeInterpreter(api_key=self.api_key) as interpreter:
+            # Upload file to E2B sandbox
+            if file_path is not None:
+                with open(file_path, "rb") as f:
+                    remote_path = interpreter.upload_file(f)
+
+            # Execute the code to analyze the file
+            logger.info(
+                f"\n{'='*50}\n> Running following AI-generated code:\n{code}\n{'='*50}"
+            )
+            exec = interpreter.notebook.exec_cell(code)
+
+            if exec.error:
+                logger.error(
+                    f"Error when executing code in E2B sandbox: {exec.error} {exec.logs}"
+                )
+                output = E2BToolOutput(is_error=True, logs=exec.logs, results=[])
+            else:
+                if len(exec.results) == 0:
+                    output = E2BToolOutput(is_error=False, logs=exec.logs, results=[])
+                else:
+                    results = self.parse_result(exec.results[0])
+                    output = E2BToolOutput(
+                        is_error=False, logs=exec.logs, results=results
+                    )
+            return output
+
+
+def code_interpret(code: str, local_file_path: str) -> Dict:
+    """
+    Use this tool to analyze the provided data in a sandbox environment.
+    The tool will:
+        1. Upload the provided file from local to the sandbox. The uploaded file path will be /home/user/{filename}
+        2. Execute python code in a Jupyter notebook cell to analyze the uploaded file in the sandbox.
+        3. Get the result from the code in stdout, stderr, display_data, and error.
+    You must to provide the code and the provided file path to run this tool.
+    Your code should read the file from the sandbox path /home/user/{filename}.
+    """
+    api_key = os.getenv("E2B_API_KEY")
+    filesever_url_prefix = os.getenv("FILESERVER_URL_PREFIX")
+    if not api_key:
+        raise ValueError(
+            "E2B_API_KEY key is required to run code interpreter. Get it here: https://e2b.dev/docs/getting-started/api-key"
+        )
+    if not filesever_url_prefix:
+        raise ValueError(
+            "FILESERVER_URL_PREFIX is required to display file output from sandbox"
+        )
+
+    interpreter = E2BCodeInterpreter(
+        api_key=api_key, filesever_url_prefix=filesever_url_prefix
+    )
+    output = interpreter.interpret(code, local_file_path)
+    return output.dict()
+
+
+# Specify as functions tools to be loaded by the ToolFactory
+tools = [FunctionTool.from_defaults(code_interpret)]

templates/components/engines/python/agent/tools/openapi_action.py

@@ -0,0 +1,71 @@
+from typing import Dict, List, Tuple
+from llama_index.tools.openapi import OpenAPIToolSpec
+from llama_index.tools.requests import RequestsToolSpec
+
+
+class OpenAPIActionToolSpec(OpenAPIToolSpec, RequestsToolSpec):
+    """
+    A combination of OpenAPI and Requests tool specs that can parse OpenAPI specs and make requests.
+
+    openapi_uri: str: The file path or URL to the OpenAPI spec.
+    domain_headers: dict: Whitelist domains and the headers to use.
+    """
+
+    spec_functions = OpenAPIToolSpec.spec_functions + RequestsToolSpec.spec_functions
+
+    def __init__(self, openapi_uri: str, domain_headers: dict = {}, **kwargs):
+        # Load the OpenAPI spec
+        openapi_spec, servers = self.load_openapi_spec(openapi_uri)
+
+        # Add the servers to the domain headers if they are not already present
+        for server in servers:
+            if server not in domain_headers:
+                domain_headers[server] = {}
+
+        OpenAPIToolSpec.__init__(self, spec=openapi_spec)
+        RequestsToolSpec.__init__(self, domain_headers)
+
+    @staticmethod
+    def load_openapi_spec(uri: str) -> Tuple[Dict, List[str]]:
+        """
+        Load an OpenAPI spec from a URI.
+
+        Args:
+            uri (str): A file path or URL to the OpenAPI spec.
+
+        Returns:
+            List[Document]: A list of Document objects.
+        """
+        import yaml
+        from urllib.parse import urlparse
+
+        if uri.startswith("http"):
+            import requests
+
+            response = requests.get(uri)
+            if response.status_code != 200:
+                raise ValueError(
+                    "Could not initialize OpenAPIActionToolSpec: "
+                    f"Failed to load OpenAPI spec from {uri}, status code: {response.status_code}"
+                )
+            spec = yaml.safe_load(response.text)
+        elif uri.startswith("file"):
+            filepath = urlparse(uri).path
+            with open(filepath, "r") as file:
+                spec = yaml.safe_load(file)
+        else:
+            raise ValueError(
+                "Could not initialize OpenAPIActionToolSpec: Invalid OpenAPI URI provided. "
+                "Only HTTP and file path are supported."
+            )
+        # Add the servers to the whitelist
+        try:
+            servers = [
+                urlparse(server["url"]).netloc for server in spec.get("servers", [])
+            ]
+        except KeyError as e:
+            raise ValueError(
+                "Could not initialize OpenAPIActionToolSpec: Invalid OpenAPI spec provided. "
+                "Could not get `servers` from the spec."
+            ) from e
+        return spec, servers

templates/components/engines/typescript/agent/tools/interpreter.ts

@@ -7,6 +7,7 @@ import path from "node:path";
 
 export type InterpreterParameter = {
   code: string;
+  localFilePath: string;
 };
 
 export type InterpreterToolParams = {
@@ -15,7 +16,7 @@ export type InterpreterToolParams = {
   fileServerURLPrefix?: string;
 };
 
-export type InterpreterToolOuput = {
+export type InterpreterToolOutput = {
   isError: boolean;
   logs: Logs;
   extraResult: InterpreterExtraResult[];
@@ -34,14 +35,21 @@ type InterpreterExtraType =
 
 export type InterpreterExtraResult = {
   type: InterpreterExtraType;
+  content?: string;
   filename: string;
   url: string;
 };
 
 const DEFAULT_META_DATA: ToolMetadata<JSONSchemaType<InterpreterParameter>> = {
   name: "interpreter",
-  description:
-    "Execute python code in a Jupyter notebook cell and return any result, stdout, stderr, display_data, and error.",
+  description: `Use this tool to analyze the provided data in a sandbox environment.
+    The tool will:
+        1. Upload the provided file from local to the sandbox. The uploaded file path will be /home/user/{filename}
+        2. Execute python code in a Jupyter notebook cell to analyze the uploaded file in the sandbox.
+        3. Get the result from the code in stdout, stderr, display_data, and error.
+    You must to provide the code and the provided file path to run this tool.
+    Your code should read the file from the sandbox path /home/user/{filename}.
+    `,
   parameters: {
     type: "object",
     properties: {
@@ -49,6 +57,10 @@ const DEFAULT_META_DATA: ToolMetadata<JSONSchemaType<InterpreterParameter>> = {
         type: "string",
         description: "The python code to execute in a single cell.",
       },
+      localFilePath: {
+        type: "string",
+        description: "The local file path to upload to the sandbox.",
+      },
     },
     required: ["code"],
   },
@@ -88,15 +100,26 @@ export class InterpreterTool implements BaseTool<InterpreterParameter> {
     return this.codeInterpreter;
   }
 
-  public async codeInterpret(code: string): Promise<InterpreterToolOuput> {
+  public async codeInterpret(
+    code: string,
+    localFilePath: string,
+  ): Promise<InterpreterToolOutput> {
+    const interpreter = await this.initInterpreter();
+    // Upload file to sandbox
+    console.log(`Uploading file ${localFilePath} to sandbox`);
+    const fileBuffer = fs.readFileSync(localFilePath);
+    const fileName = path.basename(localFilePath);
+    await interpreter.uploadFile(fileBuffer, fileName);
+    console.log(`Uploaded file ${fileName} to sandbox`);
+
+    // Execute code in sandbox
     console.log(
       `\n${"=".repeat(50)}\n> Running following AI-generated code:\n${code}\n${"=".repeat(50)}`,
     );
-    const interpreter = await this.initInterpreter();
     const exec = await interpreter.notebook.execCell(code);
     if (exec.error) console.error("[Code Interpreter error]", exec.error);
     const extraResult = await this.getExtraResult(exec.results[0]);
-    const result: InterpreterToolOuput = {
+    const result: InterpreterToolOutput = {
       isError: !!exec.error,
       logs: exec.logs,
       extraResult,
@@ -104,8 +127,8 @@ export class InterpreterTool implements BaseTool<InterpreterParameter> {
     return result;
   }
 
-  async call(input: InterpreterParameter): Promise<InterpreterToolOuput> {
-    const result = await this.codeInterpret(input.code);
+  async call(input: InterpreterParameter): Promise<InterpreterToolOutput> {
+    const result = await this.codeInterpret(input.code, input.localFilePath);
     await this.codeInterpreter?.close();
     return result;
   }
@@ -119,18 +142,26 @@ export class InterpreterTool implements BaseTool<InterpreterParameter> {
     try {
       const formats = res.formats(); // formats available for the result. Eg: ['png', ...]
       const base64DataArr = formats.map((f) => res[f as keyof Result]); // get base64 data for each format
+      console.log("data", base64DataArr);
 
       // save base64 data to file and return the url
       for (let i = 0; i < formats.length; i++) {
         const ext = formats[i];
         const base64Data = base64DataArr[i];
-        if (ext && base64Data) {
+        if (ext === "png" && base64Data) {
           const { filename } = this.saveToDisk(base64Data, ext);
           output.push({
             type: ext as InterpreterExtraType,
             filename,
             url: this.getFileUrl(filename),
           });
+        } else {
+          output.push({
+            type: ext as InterpreterExtraType,
+            content: base64Data,
+            filename: `output.${ext}`,
+            url: "",
+          });
         }
       }
     } catch (error) {

templates/components/ui/html/chat/chat-input.tsx

@@ -1,5 +1,7 @@
 "use client";
 
+import { Message } from "./chat-messages";
+
 export interface ChatInputProps {
   /** The current value of the input */
   input?: string;
@@ -12,7 +14,7 @@ export interface ChatInputProps {
   /** Form submission handler to automatically reset input and append a user message  */
   handleSubmit: (e: React.FormEvent<HTMLFormElement>) => void;
   isLoading: boolean;
-  multiModal?: boolean;
+  messages: Message[];
 }
 
 export default function ChatInput(props: ChatInputProps) {

templates/components/ui/html/chat/chat-messages.tsx

@@ -19,6 +19,9 @@ export default function ChatMessages({
   isLoading?: boolean;
   stop?: () => void;
   reload?: () => void;
+  append?: (
+    message: Message | Omit<Message, "id">,
+  ) => Promise<string | null | undefined>;
 }) {
   const scrollableChatContainerRef = useRef<HTMLDivElement>(null);
 

templates/components/ui/html/chat/use-config.ts

@@ -0,0 +1,30 @@
+"use client";
+
+import { useEffect, useMemo, useState } from "react";
+
+export interface ChatConfig {
+  chatAPI?: string;
+  starterQuestions?: string[];
+}
+
+export function useClientConfig() {
+  const API_ROUTE = "/api/chat/config";
+  const chatAPI = process.env.NEXT_PUBLIC_CHAT_API;
+  const [config, setConfig] = useState<ChatConfig>({
+    chatAPI,
+  });
+
+  const configAPI = useMemo(() => {
+    const backendOrigin = chatAPI ? new URL(chatAPI).origin : "";
+    return `${backendOrigin}${API_ROUTE}`;
+  }, [chatAPI]);
+
+  useEffect(() => {
+    fetch(configAPI)
+      .then((response) => response.json())
+      .then((data) => setConfig({ ...data, chatAPI }))
+      .catch((error) => console.error("Error fetching config", error));
+  }, [chatAPI, configAPI]);
+
+  return config;
+}

templates/components/vectordbs/python/milvus/vectordb.py

@@ -1,4 +1,5 @@
 import os
+
 from llama_index.vector_stores.milvus import MilvusVectorStore
 
 
@@ -15,6 +16,6 @@ def get_vector_store():
         user=os.getenv("MILVUS_USERNAME"),
         password=os.getenv("MILVUS_PASSWORD"),
         collection_name=collection,
-        dim=int(os.getenv("EMBEDDING_DIM")),
+        dim=int(os.getenv("EMBEDDING_DIM", 768)),
     )
     return store

templates/types/streaming/express/src/controllers/chat-config.controller.ts

@@ -0,0 +1,14 @@
+import { Request, Response } from "express";
+
+export const chatConfig = async (_req: Request, res: Response) => {
+  let starterQuestions = undefined;
+  if (
+    process.env.CONVERSATION_STARTERS &&
+    process.env.CONVERSATION_STARTERS.trim()
+  ) {
+    starterQuestions = process.env.CONVERSATION_STARTERS.trim().split("\n");
+  }
+  return res.status(200).json({
+    starterQuestions,
+  });
+};

templates/types/streaming/express/src/controllers/chat.controller.ts

@@ -1,32 +1,23 @@
 import { Message, StreamData, streamToResponse } from "ai";
 import { Request, Response } from "express";
-import { ChatMessage, MessageContent, Settings } from "llamaindex";
+import { ChatMessage, Settings } from "llamaindex";
 import { createChatEngine } from "./engine/chat";
-import { LlamaIndexStream } from "./llamaindex-stream";
-import { createCallbackManager } from "./stream-helper";
-
-const convertMessageContent = (
-  textMessage: string,
-  imageUrl: string | undefined,
-): MessageContent => {
-  if (!imageUrl) return textMessage;
-  return [
-    {
-      type: "text",
-      text: textMessage,
-    },
-    {
-      type: "image_url",
-      image_url: {
-        url: imageUrl,
-      },
-    },
-  ];
-};
+import {
+  DataParserOptions,
+  LlamaIndexStream,
+  convertMessageContent,
+} from "./llamaindex-stream";
+import { createCallbackManager, createStreamTimeout } from "./stream-helper";
 
 export const chat = async (req: Request, res: Response) => {
+  // Init Vercel AI StreamData and timeout
+  const vercelStreamData = new StreamData();
+  const streamTimeout = createStreamTimeout(vercelStreamData);
   try {
-    const { messages, data }: { messages: Message[]; data: any } = req.body;
+    const {
+      messages,
+      data,
+    }: { messages: Message[]; data: DataParserOptions | undefined } = req.body;
     const userMessage = messages.pop();
     if (!messages || !userMessage || userMessage.role !== "user") {
       return res.status(400).json({
@@ -38,13 +29,7 @@ export const chat = async (req: Request, res: Response) => {
     const chatEngine = await createChatEngine();
 
     // Convert message content from Vercel/AI format to LlamaIndex/OpenAI format
-    const userMessageContent = convertMessageContent(
-      userMessage.content,
-      data?.imageUrl,
-    );
-
-    // Init Vercel AI StreamData
-    const vercelStreamData = new StreamData();
+    const userMessageContent = convertMessageContent(userMessage.content, data);
 
     // Setup callbacks
     const callbackManager = createCallbackManager(vercelStreamData);
@@ -61,7 +46,8 @@ export const chat = async (req: Request, res: Response) => {
     // Return a stream, which can be consumed by the Vercel/AI client
     const stream = LlamaIndexStream(response, vercelStreamData, {
       parserOptions: {
-        image_url: data?.imageUrl,
+        imageUrl: data?.imageUrl,
+        csvFiles: data?.csvFiles,
       },
     });
 
@@ -71,5 +57,7 @@ export const chat = async (req: Request, res: Response) => {
     return res.status(500).json({
       detail: (error as Error).message,
     });
+  } finally {
+    clearTimeout(streamTimeout);
   }
 };

templates/types/streaming/express/src/controllers/llamaindex-stream.ts

@@ -6,6 +6,7 @@ import {
   type AIStreamCallbacksAndOptions,
 } from "ai";
 import {
+  MessageContent,
   Metadata,
   NodeWithScore,
   Response,
@@ -13,20 +14,61 @@ import {
 } from "llamaindex";
 
 import { AgentStreamChatResponse } from "llamaindex/agent/base";
-import { appendImageData, appendSourceData } from "./stream-helper";
+import {
+  CsvFile,
+  appendCsvData,
+  appendImageData,
+  appendSourceData,
+} from "./stream-helper";
 
 type LlamaIndexResponse =
   | AgentStreamChatResponse<ToolCallLLMMessageOptions>
   | Response;
 
-type ParserOptions = {
-  image_url?: string;
+export type DataParserOptions = {
+  imageUrl?: string;
+  csvFiles?: CsvFile[];
+};
+
+export const convertMessageContent = (
+  textMessage: string,
+  additionalData?: DataParserOptions,
+): MessageContent => {
+  if (!additionalData) return textMessage;
+  const content: MessageContent = [
+    {
+      type: "text",
+      text: textMessage,
+    },
+  ];
+  if (additionalData?.imageUrl) {
+    content.push({
+      type: "image_url",
+      image_url: {
+        url: additionalData?.imageUrl,
+      },
+    });
+  }
+
+  if (additionalData?.csvFiles?.length) {
+    const rawContents = additionalData.csvFiles.map((csv) => {
+      return "```csv\n" + csv.content + "\n```";
+    });
+    const csvContent =
+      "Use data from following CSV raw contents:\n" + rawContents.join("\n\n");
+    content.push({
+      type: "text",
+      text: `${csvContent}\n\n${textMessage}`,
+    });
+  }
+
+  return content;
 };
 
 function createParser(
   res: AsyncIterable<LlamaIndexResponse>,
   data: StreamData,
-  opts?: ParserOptions,
+  opts?: DataParserOptions,
 ) {
   const it = res[Symbol.asyncIterator]();
   const trimStartOfStream = trimStartOfStreamHelper();
@@ -34,7 +76,8 @@ function createParser(
   let sourceNodes: NodeWithScore<Metadata>[] | undefined;
   return new ReadableStream<string>({
     start() {
-      appendImageData(data, opts?.image_url);
+      appendImageData(data, opts?.imageUrl);
+      appendCsvData(data, opts?.csvFiles);
     },
     async pull(controller): Promise<void> {
       const { value, done } = await it.next();
@@ -72,7 +115,7 @@ export function LlamaIndexStream(
   data: StreamData,
   opts?: {
     callbacks?: AIStreamCallbacksAndOptions;
-    parserOptions?: ParserOptions;
+    parserOptions?: DataParserOptions;
   },
 ): ReadableStream<Uint8Array> {
   return createParser(response, data, opts?.parserOptions)

templates/types/streaming/express/src/controllers/stream-helper.ts

@@ -17,6 +17,22 @@ export function appendImageData(data: StreamData, imageUrl?: string) {
   });
 }
 
+function getNodeUrl(metadata: Metadata) {
+  const url = metadata["URL"];
+  if (url) return url;
+  const fileName = metadata["file_name"];
+  if (!process.env.FILESERVER_URL_PREFIX) {
+    console.warn(
+      "FILESERVER_URL_PREFIX is not set. File URLs will not be generated.",
+    );
+    return undefined;
+  }
+  if (fileName) {
+    return `${process.env.FILESERVER_URL_PREFIX}/data/${fileName}`;
+  }
+  return undefined;
+}
+
 export function appendSourceData(
   data: StreamData,
   sourceNodes?: NodeWithScore<Metadata>[],
@@ -29,6 +45,7 @@ export function appendSourceData(
         ...node.node.toMutableJSON(),
         id: node.node.id_,
         score: node.score ?? null,
+        url: getNodeUrl(node.node.metadata),
       })),
     },
   });
@@ -65,6 +82,15 @@ export function appendToolData(
   });
 }
 
+export function createStreamTimeout(stream: StreamData) {
+  const timeout = Number(process.env.STREAM_TIMEOUT ?? 1000 * 60 * 5); // default to 5 minutes
+  const t = setTimeout(() => {
+    appendEventData(stream, `Stream timed out after ${timeout / 1000} seconds`);
+    stream.close();
+  }, timeout);
+  return t;
+}
+
 export function createCallbackManager(stream: StreamData) {
   const callbackManager = new CallbackManager();
 
@@ -95,3 +121,20 @@ export function createCallbackManager(stream: StreamData) {
 
   return callbackManager;
 }
+
+export type CsvFile = {
+  content: string;
+  filename: string;
+  filesize: number;
+  id: string;
+};
+
+export function appendCsvData(data: StreamData, csvFiles?: CsvFile[]) {
+  if (!csvFiles) return;
+  data.appendMessageAnnotation({
+    type: "csv",
+    data: {
+      csvFiles,
+    },
+  });
+}

templates/types/streaming/express/src/routes/chat.route.ts

@@ -1,4 +1,5 @@
 import express, { Router } from "express";
+import { chatConfig } from "../controllers/chat-config.controller";
 import { chatRequest } from "../controllers/chat-request.controller";
 import { chat } from "../controllers/chat.controller";
 import { initSettings } from "../controllers/engine/settings";
@@ -8,5 +9,6 @@ const llmRouter: Router = express.Router();
 initSettings();
 llmRouter.route("/").post(chat);
 llmRouter.route("/request").post(chatRequest);
+llmRouter.route("/config").get(chatConfig);
 
 export default llmRouter;

templates/types/streaming/fastapi/app/api/routers/chat.py

@@ -1,101 +1,53 @@
-from pydantic import BaseModel
-from typing import List, Any, Optional, Dict, Tuple
+import os
+import logging
+
+from aiostream import stream
 from fastapi import APIRouter, Depends, HTTPException, Request, status
 from llama_index.core.chat_engine.types import BaseChatEngine
-from llama_index.core.schema import NodeWithScore
-from llama_index.core.llms import ChatMessage, MessageRole
+from llama_index.core.llms import MessageRole
 from app.engine import get_chat_engine
 from app.api.routers.vercel_response import VercelStreamResponse
-from app.api.routers.messaging import EventCallbackHandler
-from aiostream import stream
+from app.api.routers.events import EventCallbackHandler
+from app.api.routers.models import (
+    ChatData,
+    ChatConfig,
+    SourceNodes,
+    Result,
+    Message,
+)
 
 chat_router = r = APIRouter()
 
-
-class _Message(BaseModel):
-    role: MessageRole
-    content: str
-
-
-class _ChatData(BaseModel):
-    messages: List[_Message]
-
-    class Config:
-        json_schema_extra = {
-            "example": {
-                "messages": [
-                    {
-                        "role": "user",
-                        "content": "What standards for letters exist?",
-                    }
-                ]
-            }
-        }
-
-
-class _SourceNodes(BaseModel):
-    id: str
-    metadata: Dict[str, Any]
-    score: Optional[float]
-    text: str
-
-    @classmethod
-    def from_source_node(cls, source_node: NodeWithScore):
-        return cls(
-            id=source_node.node.node_id,
-            metadata=source_node.node.metadata,
-            score=source_node.score,
-            text=source_node.node.text,  # type: ignore
-        )
-
-    @classmethod
-    def from_source_nodes(cls, source_nodes: List[NodeWithScore]):
-        return [cls.from_source_node(node) for node in source_nodes]
-
-
-class _Result(BaseModel):
-    result: _Message
-    nodes: List[_SourceNodes]
-
-
-async def parse_chat_data(data: _ChatData) -> Tuple[str, List[ChatMessage]]:
-    # check preconditions and get last message
-    if len(data.messages) == 0:
-        raise HTTPException(
-            status_code=status.HTTP_400_BAD_REQUEST,
-            detail="No messages provided",
-        )
-    last_message = data.messages.pop()
-    if last_message.role != MessageRole.USER:
-        raise HTTPException(
-            status_code=status.HTTP_400_BAD_REQUEST,
-            detail="Last message must be from user",
-        )
-    # convert messages coming from the request to type ChatMessage
-    messages = [
-        ChatMessage(
-            role=m.role,
-            content=m.content,
-        )
-        for m in data.messages
-    ]
-    return last_message.content, messages
+logger = logging.getLogger("uvicorn")
 
 
 # streaming endpoint - delete if not needed
 @r.post("")
 async def chat(
     request: Request,
-    data: _ChatData,
+    data: ChatData,
     chat_engine: BaseChatEngine = Depends(get_chat_engine),
 ):
-    last_message_content, messages = await parse_chat_data(data)
+    last_message_content = data.get_last_message_content()
+    messages = data.get_history_messages()
 
     event_handler = EventCallbackHandler()
     chat_engine.callback_manager.handlers.append(event_handler)  # type: ignore
-    response = await chat_engine.astream_chat(last_message_content, messages)
+    try:
+        response = await chat_engine.astream_chat(last_message_content, messages)
+    except Exception as e:
+        logger.exception("Error in chat engine", exc_info=True)
+        raise HTTPException(
+            status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+            detail=f"Error in chat engine: {e}",
+        ) from e
 
     async def content_generator():
+        # Yield the additional data
+        if data.data is not None:
+            for data_response in data.get_additional_data_response():
+                yield VercelStreamResponse.convert_data(data_response)
+
         # Yield the text response
         async def _text_generator():
             async for token in response.async_response_gen():
@@ -123,7 +75,7 @@ async def chat(
                 "type": "sources",
                 "data": {
                     "nodes": [
-                        _SourceNodes.from_source_node(node).dict()
+                        SourceNodes.from_source_node(node).dict()
                         for node in response.source_nodes
                     ]
                 },
@@ -136,13 +88,23 @@ async def chat(
 # non-streaming endpoint - delete if not needed
 @r.post("/request")
 async def chat_request(
-    data: _ChatData,
+    data: ChatData,
     chat_engine: BaseChatEngine = Depends(get_chat_engine),
-) -> _Result:
-    last_message_content, messages = await parse_chat_data(data)
+) -> Result:
+    last_message_content = data.get_last_message_content()
+    messages = data.get_history_messages()
 
     response = await chat_engine.achat(last_message_content, messages)
-    return _Result(
-        result=_Message(role=MessageRole.ASSISTANT, content=response.response),
-        nodes=_SourceNodes.from_source_nodes(response.source_nodes),
+    return Result(
+        result=Message(role=MessageRole.ASSISTANT, content=response.response),
+        nodes=SourceNodes.from_source_nodes(response.source_nodes),
     )
+
+
+@r.get("/config")
+async def chat_config() -> ChatConfig:
+    starter_questions = None
+    conversation_starters = os.getenv("CONVERSATION_STARTERS")
+    if conversation_starters and conversation_starters.strip():
+        starter_questions = conversation_starters.strip().split("\n")
+    return ChatConfig(starterQuestions=starter_questions)

templates/types/streaming/fastapi/app/api/routers/events.py

@@ -1,5 +1,6 @@
 import json
 import asyncio
+import logging
 from typing import AsyncGenerator, Dict, Any, List, Optional
 from llama_index.core.callbacks.base import BaseCallbackHandler
 from llama_index.core.callbacks.schema import CBEventType
@@ -7,6 +8,9 @@ from llama_index.core.tools.types import ToolOutput
 from pydantic import BaseModel
 
 
+logger = logging.getLogger(__name__)
+
+
 class CallbackEvent(BaseModel):
     event_type: CBEventType
     payload: Optional[Dict[str, Any]] = None
@@ -72,15 +76,19 @@ class CallbackEvent(BaseModel):
                     }
 
     def to_response(self):
-        match self.event_type:
-            case "retrieve":
-                return self.get_retrieval_message()
-            case "function_call":
-                return self.get_tool_message()
-            case "agent_step":
-                return self.get_agent_tool_response()
-            case _:
-                return None
+        try:
+            match self.event_type:
+                case "retrieve":
+                    return self.get_retrieval_message()
+                case "function_call":
+                    return self.get_tool_message()
+                case "agent_step":
+                    return self.get_agent_tool_response()
+                case _:
+                    return None
+        except Exception as e:
+            logger.error(f"Error in converting event to response: {e}")
+            return None
 
 
 class EventCallbackHandler(BaseCallbackHandler):

templates/types/streaming/fastapi/app/api/routers/models.py

@@ -0,0 +1,206 @@
+import os
+import logging
+import tempfile
+from pydantic import BaseModel, Field, validator
+from pydantic.alias_generators import to_camel
+from typing import List, Any, Optional, Dict
+from llama_index.core.schema import NodeWithScore
+from llama_index.core.llms import ChatMessage, MessageRole
+
+
+logger = logging.getLogger("uvicorn")
+
+
+class Message(BaseModel):
+    role: MessageRole
+    content: str
+
+
+class CsvFile(BaseModel):
+    content: str
+    filename: str
+    filesize: int
+    id: str
+    type: str
+    local_file_path: Optional[str] = None
+
+    def __init__(self, **data):
+        super().__init__(**data)
+
+        # Write the content to a temporary file
+        saved_path = self.write_to_temp_file(self.content)
+        self.local_file_path = saved_path
+
+    @staticmethod
+    def write_to_temp_file(file_content: str) -> str:
+        """
+        Write the content to a temporary file and return the file path
+        """
+        csv_file = tempfile.NamedTemporaryFile(mode="w", delete=False, suffix=".csv")
+        csv_file.write(file_content)
+        file_path = csv_file.name
+        return file_path
+
+    def __del__(self):
+        # Remove the temporary file once the object is deleted
+        if self.local_file_path:
+            os.remove(self.local_file_path)
+
+
+class DataParserOptions(BaseModel):
+    csv_files: List[CsvFile] | None = Field(
+        default=None,
+        description="List of CSV files",
+    )
+
+    class Config:
+        json_schema_extra = {
+            "example": {
+                "csvFiles": [
+                    {
+                        "content": "Name, Age\nAlice, 25\nBob, 30",
+                        "filename": "example.csv",
+                        "filesize": 123,
+                        "id": "123",
+                        "type": "text/csv",
+                    }
+                ]
+            }
+        }
+        alias_generator = to_camel
+
+    def to_raw_content(self) -> str:
+        if self.csv_files is not None and len(self.csv_files) > 0:
+            saved_path = self.csv_files[0].local_file_path
+            saved_file_name = os.path.basename(saved_path)
+            content = self.csv_files[0].content
+            csv_meta = {
+                "local_file_path": saved_path,
+                "example_data": content[: min(200, len(content))],
+                "sandbox_file_path": f"/home/user/{saved_file_name}",
+            }
+
+            return f"Provided CSV file metadata:\n{csv_meta}"
+
+    def to_response_data(self) -> list[dict] | None:
+        output = []
+        if self.csv_files is not None and len(self.csv_files) > 0:
+            output.append(
+                {
+                    "type": "csv",
+                    "data": {
+                        "csvFiles": [csv_file.dict() for csv_file in self.csv_files]
+                    },
+                }
+            )
+        return output if len(output) > 0 else None
+
+
+class ChatData(BaseModel):
+    data: DataParserOptions | None = Field(
+        default=None,
+    )
+    messages: List[Message]
+
+    class Config:
+        json_schema_extra = {
+            "example": {
+                "messages": [
+                    {
+                        "role": "user",
+                        "content": "What standards for letters exist?",
+                    }
+                ]
+            }
+        }
+
+    @validator("messages")
+    def messages_must_not_be_empty(cls, v):
+        if len(v) == 0:
+            raise ValueError("Messages must not be empty")
+        return v
+
+    def get_last_message_content(self) -> str:
+        """
+        Get the content of the last message along with the data content if available
+        """
+        message_content = self.messages[-1].content
+        if self.data:
+            message_content += "\n" + self.data.to_raw_content()
+        return message_content
+
+    def get_history_messages(self) -> List[Message]:
+        """
+        Get the history messages
+        """
+        return [
+            ChatMessage(role=message.role, content=message.content)
+            for message in self.messages[:-1]
+        ]
+
+    def get_additional_data_response(self) -> list[dict] | None:
+        """
+        Get the additional data
+        """
+        return self.data.to_response_data()
+
+    def is_last_message_from_user(self) -> bool:
+        return self.messages[-1].role == MessageRole.USER
+
+
+class SourceNodes(BaseModel):
+    id: str
+    metadata: Dict[str, Any]
+    score: Optional[float]
+    text: str
+    url: Optional[str]
+
+    @classmethod
+    def from_source_node(cls, source_node: NodeWithScore):
+        metadata = source_node.node.metadata
+        url = metadata.get("URL")
+
+        if not url:
+            file_name = metadata.get("file_name")
+            url_prefix = os.getenv("FILESERVER_URL_PREFIX")
+            if not url_prefix:
+                logger.warning(
+                    "Warning: FILESERVER_URL_PREFIX not set in environment variables"
+                )
+            if file_name and url_prefix:
+                url = f"{url_prefix}/data/{file_name}"
+
+        return cls(
+            id=source_node.node.node_id,
+            metadata=metadata,
+            score=source_node.score,
+            text=source_node.node.text,  # type: ignore
+            url=url,
+        )
+
+    @classmethod
+    def from_source_nodes(cls, source_nodes: List[NodeWithScore]):
+        return [cls.from_source_node(node) for node in source_nodes]
+
+
+class Result(BaseModel):
+    result: Message
+    nodes: List[SourceNodes]
+
+
+class ChatConfig(BaseModel):
+    starter_questions: Optional[List[str]] = Field(
+        default=None,
+        description="List of starter questions",
+    )
+
+    class Config:
+        json_schema_extra = {
+            "example": {
+                "starterQuestions": [
+                    "What standards for letters exist?",
+                    "What are the requirements for a letter to be considered a letter?",
+                ]
+            }
+        }
+        alias_generator = to_camel

templates/types/streaming/fastapi/app/engine/generate.py

@@ -21,7 +21,6 @@ STORAGE_DIR = os.getenv("STORAGE_DIR", "storage")
 
 
 def get_doc_store():
-
     # If the storage directory is there, load the document store from it.
     # If not, set up an in-memory document store since we can't load from a directory that doesn't exist.
     if os.path.exists(STORAGE_DIR):

templates/types/streaming/fastapi/app/settings.py

@@ -5,16 +5,19 @@ from llama_index.core.settings import Settings
 
 def init_settings():
     model_provider = os.getenv("MODEL_PROVIDER")
-    if model_provider == "openai":
-        init_openai()
-    elif model_provider == "ollama":
-        init_ollama()
-    elif model_provider == "anthropic":
-        init_anthropic()
-    elif model_provider == "gemini":
-        init_gemini()
-    else:
-        raise ValueError(f"Invalid model provider: {model_provider}")
+    match model_provider:
+        case "openai":
+            init_openai()
+        case "ollama":
+            init_ollama()
+        case "anthropic":
+            init_anthropic()
+        case "gemini":
+            init_gemini()
+        case "azure-openai":
+            init_azure_openai()
+        case _:
+            raise ValueError(f"Invalid model provider: {model_provider}")
     Settings.chunk_size = int(os.getenv("CHUNK_SIZE", "1024"))
     Settings.chunk_overlap = int(os.getenv("CHUNK_OVERLAP", "20"))
 
@@ -52,6 +55,34 @@ def init_openai():
     Settings.embed_model = OpenAIEmbedding(**config)
 
 
+def init_azure_openai():
+    from llama_index.llms.azure_openai import AzureOpenAI
+    from llama_index.embeddings.azure_openai import AzureOpenAIEmbedding
+    from llama_index.core.constants import DEFAULT_TEMPERATURE
+
+    llm_deployment = os.getenv("AZURE_OPENAI_LLM_DEPLOYMENT")
+    embedding_deployment = os.getenv("AZURE_OPENAI_EMBEDDING_DEPLOYMENT")
+    max_tokens = os.getenv("LLM_MAX_TOKENS")
+    api_key = os.getenv("AZURE_OPENAI_API_KEY")
+    llm_config = {
+        "api_key": api_key,
+        "deployment_name": llm_deployment,
+        "model": os.getenv("MODEL"),
+        "temperature": float(os.getenv("LLM_TEMPERATURE", DEFAULT_TEMPERATURE)),
+        "max_tokens": int(max_tokens) if max_tokens is not None else None,
+    }
+    Settings.llm = AzureOpenAI(**llm_config)
+
+    dimensions = os.getenv("EMBEDDING_DIM")
+    embedding_config = {
+        "api_key": api_key,
+        "deployment_name": embedding_deployment,
+        "model": os.getenv("EMBEDDING_MODEL"),
+        "dimensions": int(dimensions) if dimensions is not None else None,
+    }
+    Settings.embed_model = AzureOpenAIEmbedding(**embedding_config)
+
+
 def init_anthropic():
     from llama_index.llms.anthropic import Anthropic
     from llama_index.embeddings.huggingface import HuggingFaceEmbedding

templates/types/streaming/fastapi/main.py

@@ -37,8 +37,17 @@ if environment == "dev":
     async def redirect_to_docs():
         return RedirectResponse(url="/docs")
 
-if os.path.exists("data"):
-    app.mount("/api/files/data", StaticFiles(directory="data"), name="data-static")
+
+def mount_static_files(directory, path):
+    if os.path.exists(directory):
+        app.mount(path, StaticFiles(directory=directory), name=f"{directory}-static")
+
+
+# Mount the data files to serve the file viewer
+mount_static_files("data", "/api/files/data")
+# Mount the output files from tools
+mount_static_files("tool-output", "/api/files/tool-output")
+
 app.include_router(chat_router, prefix="/api/chat")
 
 

templates/types/streaming/fastapi/pyproject.toml

@@ -14,8 +14,8 @@ fastapi = "^0.109.1"
 uvicorn = { extras = ["standard"], version = "^0.23.2" }
 python-dotenv = "^1.0.0"
 aiostream = "^0.5.2"
-llama-index = "0.10.28"
-llama-index-core = "0.10.28"
+llama-index = "0.10.41"
+llama-index-core = "0.10.41"
 cachetools = "^5.3.3"
 
 [build-system]

templates/types/streaming/nextjs/app/api/chat/config/route.ts

@@ -0,0 +1,11 @@
+import { NextResponse } from "next/server";
+
+/**
+ * This API is to get config from the backend envs and expose them to the frontend
+ */
+export async function GET() {
+  const config = {
+    starterQuestions: process.env.CONVERSATION_STARTERS?.trim().split("\n"),
+  };
+  return NextResponse.json(config, { status: 200 });
+}

templates/types/streaming/nextjs/app/api/chat/llamaindex-stream.ts

@@ -6,6 +6,7 @@ import {
   type AIStreamCallbacksAndOptions,
 } from "ai";
 import {
+  MessageContent,
   Metadata,
   NodeWithScore,
   Response,
@@ -13,20 +14,71 @@ import {
 } from "llamaindex";
 
 import { AgentStreamChatResponse } from "llamaindex/agent/base";
-import { appendImageData, appendSourceData } from "./stream-helper";
+import path from "path";
+import {
+  CsvFile,
+  appendCsvData,
+  appendImageData,
+  appendSourceData,
+  writeTempCsvFiles,
+} from "./stream-helper";
 
 type LlamaIndexResponse =
   | AgentStreamChatResponse<ToolCallLLMMessageOptions>
   | Response;
 
-type ParserOptions = {
-  image_url?: string;
+export type DataParserOptions = {
+  imageUrl?: string;
+  csvFiles?: CsvFile[];
+};
+
+export const convertMessageContent = (
+  textMessage: string,
+  additionalData?: DataParserOptions,
+): MessageContent => {
+  if (!additionalData) return textMessage;
+  const content: MessageContent = [
+    {
+      type: "text",
+      text: textMessage,
+    },
+  ];
+  if (additionalData?.imageUrl) {
+    content.push({
+      type: "image_url",
+      image_url: {
+        url: additionalData?.imageUrl,
+      },
+    });
+  }
+
+  if (additionalData?.csvFiles?.length) {
+    const tmpFile = writeTempCsvFiles(additionalData.csvFiles);
+    // Get a few lines of the CSV file as sample content
+    const sampleContent = additionalData.csvFiles
+      .map((csv) => csv.content.split("\n").slice(1, 4).join("\n"))
+      .join("\n\n");
+    const metadata = {
+      localFilePath: tmpFile.name,
+      sampleContent: sampleContent,
+      sandboxFilePath: `/home/user/${path.basename(tmpFile.name)}`,
+    };
+    const csvContent =
+      "Provided CSV file metadata:\n" + JSON.stringify(metadata, null, 2);
+    console.log(csvContent);
+    content.push({
+      type: "text",
+      text: `${textMessage}\n\n${csvContent}`,
+    });
+  }
+
+  return content;
 };
 
 function createParser(
   res: AsyncIterable<LlamaIndexResponse>,
   data: StreamData,
-  opts?: ParserOptions,
+  opts?: DataParserOptions,
 ) {
   const it = res[Symbol.asyncIterator]();
   const trimStartOfStream = trimStartOfStreamHelper();
@@ -34,7 +86,8 @@ function createParser(
   let sourceNodes: NodeWithScore<Metadata>[] | undefined;
   return new ReadableStream<string>({
     start() {
-      appendImageData(data, opts?.image_url);
+      appendImageData(data, opts?.imageUrl);
+      appendCsvData(data, opts?.csvFiles);
     },
     async pull(controller): Promise<void> {
       const { value, done } = await it.next();
@@ -72,7 +125,7 @@ export function LlamaIndexStream(
   data: StreamData,
   opts?: {
     callbacks?: AIStreamCallbacksAndOptions;
-    parserOptions?: ParserOptions;
+    parserOptions?: DataParserOptions;
   },
 ): ReadableStream<Uint8Array> {
   return createParser(response, data, opts?.parserOptions)

templates/types/streaming/nextjs/app/api/chat/route.ts

@@ -1,11 +1,15 @@
 import { initObservability } from "@/app/observability";
 import { Message, StreamData, StreamingTextResponse } from "ai";
-import { ChatMessage, MessageContent, Settings } from "llamaindex";
+import { ChatMessage, Settings } from "llamaindex";
 import { NextRequest, NextResponse } from "next/server";
 import { createChatEngine } from "./engine/chat";
 import { initSettings } from "./engine/settings";
-import { LlamaIndexStream } from "./llamaindex-stream";
-import { createCallbackManager } from "./stream-helper";
+import {
+  DataParserOptions,
+  LlamaIndexStream,
+  convertMessageContent,
+} from "./llamaindex-stream";
+import { createCallbackManager, createStreamTimeout } from "./stream-helper";
 
 initObservability();
 initSettings();
@@ -13,29 +17,17 @@ initSettings();
 export const runtime = "nodejs";
 export const dynamic = "force-dynamic";
 
-const convertMessageContent = (
-  textMessage: string,
-  imageUrl: string | undefined,
-): MessageContent => {
-  if (!imageUrl) return textMessage;
-  return [
-    {
-      type: "text",
-      text: textMessage,
-    },
-    {
-      type: "image_url",
-      image_url: {
-        url: imageUrl,
-      },
-    },
-  ];
-};
-
 export async function POST(request: NextRequest) {
+  // Init Vercel AI StreamData and timeout
+  const vercelStreamData = new StreamData();
+  const streamTimeout = createStreamTimeout(vercelStreamData);
+
   try {
     const body = await request.json();
-    const { messages, data }: { messages: Message[]; data: any } = body;
+    const {
+      messages,
+      data,
+    }: { messages: Message[]; data: DataParserOptions | undefined } = body;
     const userMessage = messages.pop();
     if (!messages || !userMessage || userMessage.role !== "user") {
       return NextResponse.json(
@@ -50,13 +42,7 @@ export async function POST(request: NextRequest) {
     const chatEngine = await createChatEngine();
 
     // Convert message content from Vercel/AI format to LlamaIndex/OpenAI format
-    const userMessageContent = convertMessageContent(
-      userMessage.content,
-      data?.imageUrl,
-    );
-
-    // Init Vercel AI StreamData
-    const vercelStreamData = new StreamData();
+    const userMessageContent = convertMessageContent(userMessage.content, data);
 
     // Setup callbacks
     const callbackManager = createCallbackManager(vercelStreamData);
@@ -73,7 +59,8 @@ export async function POST(request: NextRequest) {
     // Transform LlamaIndex stream to Vercel/AI format
     const stream = LlamaIndexStream(response, vercelStreamData, {
       parserOptions: {
-        image_url: data?.imageUrl,
+        imageUrl: data?.imageUrl,
+        csvFiles: data?.csvFiles,
       },
     });
 
@@ -89,5 +76,7 @@ export async function POST(request: NextRequest) {
         status: 500,
       },
     );
+  } finally {
+    clearTimeout(streamTimeout);
   }
 }

templates/types/streaming/nextjs/app/api/chat/stream-helper.ts

@@ -1,4 +1,5 @@
 import { StreamData } from "ai";
+import fs from "fs";
 import {
   CallbackManager,
   Metadata,
@@ -6,6 +7,7 @@ import {
   ToolCall,
   ToolOutput,
 } from "llamaindex";
+import tmp from "tmp";
 
 export function appendImageData(data: StreamData, imageUrl?: string) {
   if (!imageUrl) return;
@@ -17,6 +19,22 @@ export function appendImageData(data: StreamData, imageUrl?: string) {
   });
 }
 
+function getNodeUrl(metadata: Metadata) {
+  const url = metadata["URL"];
+  if (url) return url;
+  const fileName = metadata["file_name"];
+  if (!process.env.FILESERVER_URL_PREFIX) {
+    console.warn(
+      "FILESERVER_URL_PREFIX is not set. File URLs will not be generated.",
+    );
+    return undefined;
+  }
+  if (fileName) {
+    return `${process.env.FILESERVER_URL_PREFIX}/data/${fileName}`;
+  }
+  return undefined;
+}
+
 export function appendSourceData(
   data: StreamData,
   sourceNodes?: NodeWithScore<Metadata>[],
@@ -29,6 +47,7 @@ export function appendSourceData(
         ...node.node.toMutableJSON(),
         id: node.node.id_,
         score: node.score ?? null,
+        url: getNodeUrl(node.node.metadata),
       })),
     },
   });
@@ -65,6 +84,15 @@ export function appendToolData(
   });
 }
 
+export function createStreamTimeout(stream: StreamData) {
+  const timeout = Number(process.env.STREAM_TIMEOUT ?? 1000 * 60 * 5); // default to 5 minutes
+  const t = setTimeout(() => {
+    appendEventData(stream, `Stream timed out after ${timeout / 1000} seconds`);
+    stream.close();
+  }, timeout);
+  return t;
+}
+
 export function createCallbackManager(stream: StreamData) {
   const callbackManager = new CallbackManager();
 
@@ -95,3 +123,28 @@ export function createCallbackManager(stream: StreamData) {
 
   return callbackManager;
 }
+
+export type CsvFile = {
+  content: string;
+  filename: string;
+  filesize: number;
+  id: string;
+  localFilePath: string;
+};
+
+export function appendCsvData(data: StreamData, csvFiles?: CsvFile[]) {
+  if (!csvFiles) return;
+  data.appendMessageAnnotation({
+    type: "csv",
+    data: {
+      csvFiles,
+    },
+  });
+}
+
+export function writeTempCsvFiles(csvFiles: CsvFile[]) {
+  const csvFile = csvFiles[0];
+  const tmpFile = tmp.fileSync({ postfix: ".csv" });
+  fs.writeFileSync(tmpFile.name, csvFile.content);
+  return tmpFile;
+}

templates/types/streaming/nextjs/app/components/chat-section.tsx

@@ -2,8 +2,10 @@
 
 import { useChat } from "ai/react";
 import { ChatInput, ChatMessages } from "./ui/chat";
+import { useClientConfig } from "./ui/chat/use-config";
 
 export default function ChatSection() {
+  const { chatAPI } = useClientConfig();
   const {
     messages,
     input,
@@ -12,8 +14,9 @@ export default function ChatSection() {
     handleInputChange,
     reload,
     stop,
+    append,
   } = useChat({
-    api: process.env.NEXT_PUBLIC_CHAT_API,
+    api: chatAPI,
     headers: {
       "Content-Type": "application/json", // using JSON because of vercel/ai 2.2.26
     },
@@ -31,13 +34,14 @@ export default function ChatSection() {
         isLoading={isLoading}
         reload={reload}
         stop={stop}
+        append={append}
       />
       <ChatInput
         input={input}
         handleSubmit={handleSubmit}
         handleInputChange={handleInputChange}
         isLoading={isLoading}
-        multiModal={true}
+        messages={messages}
       />
     </div>
   );

templates/types/streaming/nextjs/app/components/ui/chat/chat-input.tsx

@@ -1,9 +1,13 @@
+import { Loader2 } from "lucide-react";
 import { useState } from "react";
+import { v4 as uuidv4 } from "uuid";
 import { Button } from "../button";
 import FileUploader from "../file-uploader";
 import { Input } from "../input";
+import UploadCsvPreview from "../upload-csv-preview";
 import UploadImagePreview from "../upload-image-preview";
 import { ChatHandler } from "./chat.interface";
+import { useCsv } from "./use-csv";
 
 export default function ChatInput(
   props: Pick<
@@ -14,11 +18,13 @@ export default function ChatInput(
     | "onFileError"
     | "handleSubmit"
     | "handleInputChange"
-  > & {
-    multiModal?: boolean;
-  },
+    | "messages"
+  >,
 ) {
   const [imageUrl, setImageUrl] = useState<string | null>(null);
+  const { files, uploadNew, removeFile, resetUploadedFiles } = useCsv(
+    props.messages,
+  );
 
   const onSubmit = (e: React.FormEvent<HTMLFormElement>) => {
     if (imageUrl) {
@@ -28,6 +34,15 @@ export default function ChatInput(
       setImageUrl(null);
       return;
     }
+
+    if (files.length > 0) {
+      props.handleSubmit(e, {
+        data: { csvFiles: files },
+      });
+      resetUploadedFiles();
+      return;
+    }
+
     props.handleSubmit(e);
   };
 
@@ -43,11 +58,32 @@ export default function ChatInput(
     setImageUrl(base64);
   };
 
+  const handleUploadCsvFile = async (file: File) => {
+    const content = await new Promise<string>((resolve, reject) => {
+      const reader = new FileReader();
+      reader.readAsText(file);
+      reader.onload = () => resolve(reader.result as string);
+      reader.onerror = (error) => reject(error);
+    });
+    const isSuccess = uploadNew({
+      id: uuidv4(),
+      content,
+      filename: file.name,
+      filesize: file.size,
+    });
+    if (!isSuccess) {
+      alert("File already exists in the list.");
+    }
+  };
+
   const handleUploadFile = async (file: File) => {
     try {
-      if (props.multiModal && file.type.startsWith("image/")) {
+      if (file.type.startsWith("image/")) {
         return await handleUploadImageFile(file);
       }
+      if (file.type === "text/csv") {
+        return await handleUploadCsvFile(file);
+      }
       props.onFileUpload?.(file);
     } catch (error: any) {
       props.onFileError?.(error.message);
@@ -62,6 +98,30 @@ export default function ChatInput(
       {imageUrl && (
         <UploadImagePreview url={imageUrl} onRemove={onRemovePreviewImage} />
       )}
+      {files.length > 0 && (
+        <div className="flex gap-4 w-full overflow-auto py-2">
+          {props.isLoading ? (
+            <div className="flex gap-2 items-center">
+              <Loader2 className="h-4 w-4 animate-spin" />{" "}
+              <span>Handling csv files...</span>
+            </div>
+          ) : (
+            <>
+              {files.map((csv) => {
+                return (
+                  <UploadCsvPreview
+                    key={csv.id}
+                    filename={csv.filename}
+                    filesize={csv.filesize}
+                    onRemove={() => removeFile(csv)}
+                    isNew={csv.type === "new_upload"}
+                  />
+                );
+              })}
+            </>
+          )}
+        </div>
+      )}
       <div className="flex w-full items-start justify-between gap-4 ">
         <Input
           autoFocus

templates/types/streaming/nextjs/app/components/ui/chat/chat-message.tsx

@@ -8,14 +8,16 @@ import { ChatEvents } from "./chat-events";
 import { ChatImage } from "./chat-image";
 import { ChatSources } from "./chat-sources";
 import ChatTools from "./chat-tools";
+import CsvContent from "./csv-content";
 import {
-  AnnotationData,
+  CsvData,
   EventData,
   ImageData,
   MessageAnnotation,
   MessageAnnotationType,
   SourceData,
   ToolData,
+  getAnnotationData,
 } from "./index";
 import Markdown from "./markdown";
 import { useCopyToClipboard } from "./use-copy-to-clipboard";
@@ -25,13 +27,6 @@ type ContentDisplayConfig = {
   component: JSX.Element | null;
 };
 
-function getAnnotationData<T extends AnnotationData>(
-  annotations: MessageAnnotation[],
-  type: MessageAnnotationType,
-): T[] {
-  return annotations.filter((a) => a.type === type).map((a) => a.data as T);
-}
-
 function ChatMessageContent({
   message,
   isLoading,
@@ -46,6 +41,10 @@ function ChatMessageContent({
     annotations,
     MessageAnnotationType.IMAGE,
   );
+  const csvData = getAnnotationData<CsvData>(
+    annotations,
+    MessageAnnotationType.CSV,
+  );
   const eventData = getAnnotationData<EventData>(
     annotations,
     MessageAnnotationType.EVENTS,
@@ -61,16 +60,20 @@ function ChatMessageContent({
 
   const contents: ContentDisplayConfig[] = [
     {
-      order: -3,
+      order: -4,
       component: imageData[0] ? <ChatImage data={imageData[0]} /> : null,
     },
     {
-      order: -2,
+      order: -3,
       component:
         eventData.length > 0 ? (
           <ChatEvents isLoading={isLoading} data={eventData} />
         ) : null,
     },
+    {
+      order: -2,
+      component: csvData[0] ? <CsvContent data={csvData[0]} /> : null,
+    },
     {
       order: -1,
       component: toolData[0] ? <ChatTools data={toolData[0]} /> : null,

templates/types/streaming/nextjs/app/components/ui/chat/chat-messages.tsx

@@ -1,13 +1,19 @@
 import { Loader2 } from "lucide-react";
 import { useEffect, useRef } from "react";
 
+import { Button } from "../button";
 import ChatActions from "./chat-actions";
 import ChatMessage from "./chat-message";
 import { ChatHandler } from "./chat.interface";
+import { useClientConfig } from "./use-config";
 
 export default function ChatMessages(
-  props: Pick<ChatHandler, "messages" | "isLoading" | "reload" | "stop">,
+  props: Pick<
+    ChatHandler,
+    "messages" | "isLoading" | "reload" | "stop" | "append"
+  >,
 ) {
+  const { starterQuestions } = useClientConfig();
   const scrollableChatContainerRef = useRef<HTMLDivElement>(null);
   const messageLength = props.messages.length;
   const lastMessage = props.messages[messageLength - 1];
@@ -35,7 +41,7 @@ export default function ChatMessages(
   }, [messageLength, lastMessage]);
 
   return (
-    <div className="w-full rounded-xl bg-white p-4 shadow-xl pb-0">
+    <div className="w-full rounded-xl bg-white p-4 shadow-xl pb-0 relative">
       <div
         className="flex h-[50vh] flex-col gap-5 divide-y overflow-y-auto pb-4"
         ref={scrollableChatContainerRef}
@@ -64,6 +70,23 @@ export default function ChatMessages(
           showStop={showStop}
         />
       </div>
+      {!messageLength && starterQuestions?.length && props.append && (
+        <div className="absolute bottom-6 left-0 w-full">
+          <div className="grid grid-cols-2 gap-2 mx-20">
+            {starterQuestions.map((question, i) => (
+              <Button
+                variant="outline"
+                key={i}
+                onClick={() =>
+                  props.append!({ role: "user", content: question })
+                }
+              >
+                {question}
+              </Button>
+            ))}
+          </div>
+        </div>
+      )}
     </div>
   );
 }

templates/types/streaming/nextjs/app/components/ui/chat/chat-sources.tsx

@@ -2,12 +2,10 @@ import { Check, Copy } from "lucide-react";
 import { useMemo } from "react";
 import { Button } from "../button";
 import { HoverCard, HoverCardContent, HoverCardTrigger } from "../hover-card";
-import { getStaticFileDataUrl } from "../lib/url";
-import { SourceData, SourceNode } from "./index";
+import { SourceData } from "./index";
 import { useCopyToClipboard } from "./use-copy-to-clipboard";
 import PdfDialog from "./widgets/PdfDialog";
 
-const DATA_SOURCE_FOLDER = "data";
 const SCORE_THRESHOLD = 0.3;
 
 function SourceNumberButton({ index }: { index: number }) {
@@ -18,46 +16,11 @@ function SourceNumberButton({ index }: { index: number }) {
   );
 }
 
-enum NODE_TYPE {
-  URL,
-  FILE,
-  UNKNOWN,
-}
-
 type NodeInfo = {
   id: string;
-  type: NODE_TYPE;
-  path?: string;
   url?: string;
 };
 
-function getNodeInfo(node: SourceNode): NodeInfo {
-  if (typeof node.metadata["URL"] === "string") {
-    const url = node.metadata["URL"];
-    return {
-      id: node.id,
-      type: NODE_TYPE.URL,
-      path: url,
-      url,
-    };
-  }
-  if (typeof node.metadata["file_path"] === "string") {
-    const fileName = node.metadata["file_name"] as string;
-    const filePath = `${DATA_SOURCE_FOLDER}/${fileName}`;
-    return {
-      id: node.id,
-      type: NODE_TYPE.FILE,
-      path: node.metadata["file_path"],
-      url: getStaticFileDataUrl(filePath),
-    };
-  }
-
-  return {
-    id: node.id,
-    type: NODE_TYPE.UNKNOWN,
-  };
-}
-
 export function ChatSources({ data }: { data: SourceData }) {
   const sources: NodeInfo[] = useMemo(() => {
     // aggregate nodes by url or file_path (get the highest one by score)
@@ -67,8 +30,11 @@ export function ChatSources({ data }: { data: SourceData }) {
       .filter((node) => (node.score ?? 1) > SCORE_THRESHOLD)
       .sort((a, b) => (b.score ?? 1) - (a.score ?? 1))
       .forEach((node) => {
-        const nodeInfo = getNodeInfo(node);
-        const key = nodeInfo.path ?? nodeInfo.id; // use id as key for UNKNOWN type
+        const nodeInfo = {
+          id: node.id,
+          url: node.url,
+        };
+        const key = nodeInfo.url ?? nodeInfo.id; // use id as key for UNKNOWN type
         if (!nodesByPath[key]) {
           nodesByPath[key] = nodeInfo;
         }
@@ -84,13 +50,12 @@ export function ChatSources({ data }: { data: SourceData }) {
       <span className="font-semibold">Sources:</span>
       <div className="inline-flex gap-1 items-center">
         {sources.map((nodeInfo: NodeInfo, index: number) => {
-          if (nodeInfo.path?.endsWith(".pdf")) {
+          if (nodeInfo.url?.endsWith(".pdf")) {
             return (
               <PdfDialog
                 key={nodeInfo.id}
                 documentId={nodeInfo.id}
                 url={nodeInfo.url!}
-                path={nodeInfo.path}
                 trigger={<SourceNumberButton index={index} />}
               />
             );
@@ -116,16 +81,16 @@ export function ChatSources({ data }: { data: SourceData }) {
 function NodeInfo({ nodeInfo }: { nodeInfo: NodeInfo }) {
   const { isCopied, copyToClipboard } = useCopyToClipboard({ timeout: 1000 });
 
-  if (nodeInfo.type !== NODE_TYPE.UNKNOWN) {
+  if (nodeInfo.url) {
     // this is a node generated by the web loader or file loader,
     // add a link to view its URL and a button to copy the URL to the clipboard
     return (
       <div className="flex items-center my-2">
         <a className="hover:text-blue-900" href={nodeInfo.url} target="_blank">
-          <span>{nodeInfo.path}</span>
+          <span>{nodeInfo.url}</span>
         </a>
         <Button
-          onClick={() => copyToClipboard(nodeInfo.path!)}
+          onClick={() => copyToClipboard(nodeInfo.url!)}
           size="icon"
           variant="ghost"
           className="h-12 w-12 shrink-0"

templates/types/streaming/nextjs/app/components/ui/chat/chat.interface.ts

@@ -15,4 +15,7 @@ export interface ChatHandler {
   stop?: () => void;
   onFileUpload?: (file: File) => Promise<void>;
   onFileError?: (errMsg: string) => void;
+  append?: (
+    message: Message | Omit<Message, "id">,
+  ) => Promise<string | null | undefined>;
 }

templates/types/streaming/nextjs/app/components/ui/chat/csv-content.tsx

@@ -0,0 +1,16 @@
+import { CsvData } from ".";
+import CsvDialog from "./widgets/CsvDialog";
+
+export default function CsvContent({ data }: { data: CsvData }) {
+  if (!data.csvFiles.length) return null;
+  return (
+    <div>
+      <p className="font-semibold mb-2">Using data from following CSV files:</p>
+      <div className="flex gap-2 items-center">
+        {data.csvFiles.map((csv, index) => (
+          <CsvDialog key={index} csv={csv} />
+        ))}
+      </div>
+    </div>
+  );
+}

templates/types/streaming/nextjs/app/components/ui/chat/index.ts

@@ -6,6 +6,7 @@ export { type ChatHandler } from "./chat.interface";
 export { ChatInput, ChatMessages };
 
 export enum MessageAnnotationType {
+  CSV = "csv",
   IMAGE = "image",
   SOURCES = "sources",
   EVENTS = "events",
@@ -16,11 +17,23 @@ export type ImageData = {
   url: string;
 };
 
+export type CsvFile = {
+  content: string;
+  filename: string;
+  filesize: number;
+  id: string;
+};
+
+export type CsvData = {
+  csvFiles: CsvFile[];
+};
+
 export type SourceNode = {
   id: string;
   metadata: Record<string, unknown>;
   score?: number;
   text: string;
+  url?: string;
 };
 
 export type SourceData = {
@@ -46,9 +59,21 @@ export type ToolData = {
   };
 };
 
-export type AnnotationData = ImageData | SourceData | EventData | ToolData;
+export type AnnotationData =
+  | ImageData
+  | CsvData
+  | SourceData
+  | EventData
+  | ToolData;
 
 export type MessageAnnotation = {
   type: MessageAnnotationType;
   data: AnnotationData;
 };
+
+export function getAnnotationData<T extends AnnotationData>(
+  annotations: MessageAnnotation[],
+  type: MessageAnnotationType,
+): T[] {
+  return annotations.filter((a) => a.type === type).map((a) => a.data as T);
+}

templates/types/streaming/nextjs/app/components/ui/chat/use-config.ts

@@ -0,0 +1,30 @@
+"use client";
+
+import { useEffect, useMemo, useState } from "react";
+
+export interface ChatConfig {
+  chatAPI?: string;
+  starterQuestions?: string[];
+}
+
+export function useClientConfig() {
+  const API_ROUTE = "/api/chat/config";
+  const chatAPI = process.env.NEXT_PUBLIC_CHAT_API;
+  const [config, setConfig] = useState<ChatConfig>({
+    chatAPI,
+  });
+
+  const configAPI = useMemo(() => {
+    const backendOrigin = chatAPI ? new URL(chatAPI).origin : "";
+    return `${backendOrigin}${API_ROUTE}`;
+  }, [chatAPI]);
+
+  useEffect(() => {
+    fetch(configAPI)
+      .then((response) => response.json())
+      .then((data) => setConfig({ ...data, chatAPI }))
+      .catch((error) => console.error("Error fetching config", error));
+  }, [chatAPI, configAPI]);
+
+  return config;
+}

templates/types/streaming/nextjs/app/components/ui/chat/use-csv.ts

@@ -0,0 +1,85 @@
+"use client";
+
+import { Message } from "ai";
+import { useEffect, useMemo, useState } from "react";
+import {
+  CsvData,
+  CsvFile,
+  MessageAnnotation,
+  MessageAnnotationType,
+  getAnnotationData,
+} from ".";
+
+interface FrontendCSVData extends CsvFile {
+  type: "available" | "new_upload";
+}
+
+export function useCsv(messages: Message[]) {
+  const [availableFiles, setAvailableFiles] = useState<FrontendCSVData[]>([]);
+  const [uploadedFiles, setUploadedFiles] = useState<FrontendCSVData[]>([]);
+
+  const files = useMemo(() => {
+    return [...availableFiles, ...uploadedFiles];
+  }, [availableFiles, uploadedFiles]);
+
+  useEffect(() => {
+    const items = getAvailableCsvFiles(messages);
+    setAvailableFiles(items.map((data) => ({ ...data, type: "available" })));
+  }, [messages]);
+
+  const csvEqual = (a: CsvFile, b: CsvFile) => {
+    if (a.id === b.id) return true;
+    if (a.filename === b.filename && a.filesize === b.filesize) return true;
+    return false;
+  };
+
+  // Get available csv files from annotations chat history
+  // returns the unique csv files by id
+  const getAvailableCsvFiles = (messages: Message[]): Array<CsvFile> => {
+    const docHash: Record<string, CsvFile> = {};
+    messages.forEach((message) => {
+      if (message.annotations) {
+        const csvData = getAnnotationData<CsvData>(
+          message.annotations as MessageAnnotation[],
+          MessageAnnotationType.CSV,
+        );
+        csvData.forEach((data) => {
+          data.csvFiles.forEach((file) => {
+            if (!docHash[file.id]) {
+              docHash[file.id] = file;
+            }
+          });
+        });
+      }
+    });
+    return Object.values(docHash);
+  };
+
+  const uploadNew = (file: CsvFile) => {
+    const existedCsv = files.find((f) => csvEqual(f, file));
+    if (!existedCsv) {
+      setUploadedFiles((prev) => [...prev, { ...file, type: "new_upload" }]);
+      return true;
+    }
+    return false;
+  };
+
+  const removeFile = (file: FrontendCSVData) => {
+    if (file.type === "new_upload") {
+      setUploadedFiles((prev) => prev.filter((f) => f.id !== file.id));
+    } else {
+      setAvailableFiles((prev) => prev.filter((f) => f.id !== file.id));
+    }
+  };
+
+  const resetUploadedFiles = () => {
+    setUploadedFiles([]);
+  };
+
+  return {
+    files,
+    uploadNew,
+    removeFile,
+    resetUploadedFiles,
+  };
+}

templates/types/streaming/nextjs/app/components/ui/chat/widgets/CsvDialog.tsx

@@ -0,0 +1,62 @@
+import Image from "next/image";
+import { CsvFile } from "..";
+import SheetIcon from "../../../ui/icons/sheet.svg";
+import { Button } from "../../button";
+import {
+  Drawer,
+  DrawerClose,
+  DrawerContent,
+  DrawerDescription,
+  DrawerHeader,
+  DrawerTitle,
+  DrawerTrigger,
+} from "../../drawer";
+
+export interface CsvDialogProps {
+  csv: CsvFile;
+}
+
+export default function CsvDialog(props: CsvDialogProps) {
+  const { filename, filesize, content } = props.csv;
+  const fileSizeInKB = Math.round((filesize / 1024) * 10) / 10;
+  return (
+    <Drawer direction="left">
+      <DrawerTrigger asChild>
+        <div
+          className="border-2 border-green-700 py-1.5 px-3 rounded-lg flex gap-2 items-center cursor-pointer text-sm hover:bg-green-700 hover:text-white transition-colors duration-200 ease-in-out"
+          key={filename}
+        >
+          <div className="h-4 w-4 shrink-0 rounded-md">
+            <Image
+              className="h-full w-auto"
+              priority
+              src={SheetIcon}
+              alt="SheetIcon"
+            />
+          </div>
+          <span>
+            {filename} - {fileSizeInKB} KB
+          </span>
+        </div>
+      </DrawerTrigger>
+      <DrawerContent className="w-3/5 mt-24 h-full max-h-[96%] ">
+        <DrawerHeader className="flex justify-between">
+          <div className="space-y-2">
+            <DrawerTitle>Csv Raw Content</DrawerTitle>
+            <DrawerDescription>
+              {filename} ({fileSizeInKB} KB)
+            </DrawerDescription>
+          </div>
+          <DrawerClose asChild>
+            <Button variant="outline">Close</Button>
+          </DrawerClose>
+        </DrawerHeader>
+        <div className="m-4 max-h-[80%] overflow-auto">
+          <pre className="bg-secondary rounded-md p-4 block text-sm">
+            {content}
+          </pre>
+        </div>
+      </DrawerContent>
+    </Drawer>
+  );
+}

templates/types/streaming/nextjs/app/components/ui/chat/widgets/PdfDialog.tsx

@@ -12,7 +12,6 @@ import {
 
 export interface PdfDialogProps {
   documentId: string;
-  path: string;
   url: string;
   trigger: React.ReactNode;
 }
@@ -26,13 +25,13 @@ export default function PdfDialog(props: PdfDialogProps) {
           <div className="space-y-2">
             <DrawerTitle>PDF Content</DrawerTitle>
             <DrawerDescription>
-              File path:{" "}
+              File URL:{" "}
               <a
                 className="hover:text-blue-900"
                 href={props.url}
                 target="_blank"
               >
-                {props.path}
+                {props.url}
               </a>
             </DrawerDescription>
           </div>

templates/types/streaming/nextjs/app/components/ui/icons/sheet.svg

@@ -0,0 +1,90 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<svg width="49px" height="67px" viewBox="0 0 49 67" version="1.1"
+    xmlns="http://www.w3.org/2000/svg"
+    xmlns:xlink="http://www.w3.org/1999/xlink">
+    <title>Sheets-icon</title>
+    <desc>Created with Sketch.</desc>
+    <defs>
+        <path d="M29.5833333,0 L4.4375,0 C1.996875,0 0,1.996875 0,4.4375 L0,60.6458333 C0,63.0864583 1.996875,65.0833333 4.4375,65.0833333 L42.8958333,65.0833333 C45.3364583,65.0833333 47.3333333,63.0864583 47.3333333,60.6458333 L47.3333333,17.75 L29.5833333,0 Z" id="path-1"></path>
+        <path d="M29.5833333,0 L4.4375,0 C1.996875,0 0,1.996875 0,4.4375 L0,60.6458333 C0,63.0864583 1.996875,65.0833333 4.4375,65.0833333 L42.8958333,65.0833333 C45.3364583,65.0833333 47.3333333,63.0864583 47.3333333,60.6458333 L47.3333333,17.75 L29.5833333,0 Z" id="path-3"></path>
+        <path d="M29.5833333,0 L4.4375,0 C1.996875,0 0,1.996875 0,4.4375 L0,60.6458333 C0,63.0864583 1.996875,65.0833333 4.4375,65.0833333 L42.8958333,65.0833333 C45.3364583,65.0833333 47.3333333,63.0864583 47.3333333,60.6458333 L47.3333333,17.75 L29.5833333,0 Z" id="path-5"></path>
+        <linearGradient x1="50.0053945%" y1="8.58610612%" x2="50.0053945%" y2="100.013939%" id="linearGradient-7">
+            <stop stop-color="#263238" stop-opacity="0.2" offset="0%"></stop>
+            <stop stop-color="#263238" stop-opacity="0.02" offset="100%"></stop>
+        </linearGradient>
+        <path d="M29.5833333,0 L4.4375,0 C1.996875,0 0,1.996875 0,4.4375 L0,60.6458333 C0,63.0864583 1.996875,65.0833333 4.4375,65.0833333 L42.8958333,65.0833333 C45.3364583,65.0833333 47.3333333,63.0864583 47.3333333,60.6458333 L47.3333333,17.75 L29.5833333,0 Z" id="path-8"></path>
+        <path d="M29.5833333,0 L4.4375,0 C1.996875,0 0,1.996875 0,4.4375 L0,60.6458333 C0,63.0864583 1.996875,65.0833333 4.4375,65.0833333 L42.8958333,65.0833333 C45.3364583,65.0833333 47.3333333,63.0864583 47.3333333,60.6458333 L47.3333333,17.75 L29.5833333,0 Z" id="path-10"></path>
+        <path d="M29.5833333,0 L4.4375,0 C1.996875,0 0,1.996875 0,4.4375 L0,60.6458333 C0,63.0864583 1.996875,65.0833333 4.4375,65.0833333 L42.8958333,65.0833333 C45.3364583,65.0833333 47.3333333,63.0864583 47.3333333,60.6458333 L47.3333333,17.75 L29.5833333,0 Z" id="path-12"></path>
+        <path d="M29.5833333,0 L4.4375,0 C1.996875,0 0,1.996875 0,4.4375 L0,60.6458333 C0,63.0864583 1.996875,65.0833333 4.4375,65.0833333 L42.8958333,65.0833333 C45.3364583,65.0833333 47.3333333,63.0864583 47.3333333,60.6458333 L47.3333333,17.75 L29.5833333,0 Z" id="path-14"></path>
+        <radialGradient cx="3.16804688%" cy="2.71744318%" fx="3.16804688%" fy="2.71744318%" r="161.248516%" gradientTransform="translate(0.031680,0.027174),scale(1.000000,0.727273),translate(-0.031680,-0.027174)" id="radialGradient-16">
+            <stop stop-color="#FFFFFF" stop-opacity="0.1" offset="0%"></stop>
+            <stop stop-color="#FFFFFF" stop-opacity="0" offset="100%"></stop>
+        </radialGradient>
+    </defs>
+    <g id="Page-1" stroke="none" stroke-width="1" fill="none" fill-rule="evenodd">
+        <g id="Consumer-Apps-Sheets-Large-VD-R8-" transform="translate(-451.000000, -451.000000)">
+            <g id="Hero" transform="translate(0.000000, 63.000000)">
+                <g id="Personal" transform="translate(277.000000, 299.000000)">
+                    <g id="Sheets-icon" transform="translate(174.833333, 89.958333)">
+                        <g id="Group">
+                            <g id="Clipped">
+                                <mask id="mask-2" fill="white">
+                                    <use xlink:href="#path-1"></use>
+                                </mask>
+                                <g id="SVGID_1_"></g>
+                                <path d="M29.5833333,0 L4.4375,0 C1.996875,0 0,1.996875 0,4.4375 L0,60.6458333 C0,63.0864583 1.996875,65.0833333 4.4375,65.0833333 L42.8958333,65.0833333 C45.3364583,65.0833333 47.3333333,63.0864583 47.3333333,60.6458333 L47.3333333,17.75 L36.9791667,10.3541667 L29.5833333,0 Z" id="Path" fill="#0F9D58" fill-rule="nonzero" mask="url(#mask-2)"></path>
+                            </g>
+                            <g id="Clipped">
+                                <mask id="mask-4" fill="white">
+                                    <use xlink:href="#path-3"></use>
+                                </mask>
+                                <g id="SVGID_1_"></g>
+                                <path d="M11.8333333,31.8020833 L11.8333333,53.25 L35.5,53.25 L35.5,31.8020833 L11.8333333,31.8020833 Z M22.1875,50.2916667 L14.7916667,50.2916667 L14.7916667,46.59375 L22.1875,46.59375 L22.1875,50.2916667 Z M22.1875,44.375 L14.7916667,44.375 L14.7916667,40.6770833 L22.1875,40.6770833 L22.1875,44.375 Z M22.1875,38.4583333 L14.7916667,38.4583333 L14.7916667,34.7604167 L22.1875,34.7604167 L22.1875,38.4583333 Z M32.5416667,50.2916667 L25.1458333,50.2916667 L25.1458333,46.59375 L32.5416667,46.59375 L32.5416667,50.2916667 Z M32.5416667,44.375 L25.1458333,44.375 L25.1458333,40.6770833 L32.5416667,40.6770833 L32.5416667,44.375 Z M32.5416667,38.4583333 L25.1458333,38.4583333 L25.1458333,34.7604167 L32.5416667,34.7604167 L32.5416667,38.4583333 Z" id="Shape" fill="#F1F1F1" fill-rule="nonzero" mask="url(#mask-4)"></path>
+                            </g>
+                            <g id="Clipped">
+                                <mask id="mask-6" fill="white">
+                                    <use xlink:href="#path-5"></use>
+                                </mask>
+                                <g id="SVGID_1_"></g>
+                                <polygon id="Path" fill="url(#linearGradient-7)" fill-rule="nonzero" mask="url(#mask-6)" points="30.8813021 16.4520313 47.3333333 32.9003646 47.3333333 17.75"></polygon>
+                            </g>
+                            <g id="Clipped">
+                                <mask id="mask-9" fill="white">
+                                    <use xlink:href="#path-8"></use>
+                                </mask>
+                                <g id="SVGID_1_"></g>
+                                <g id="Group" mask="url(#mask-9)">
+                                    <g transform="translate(26.625000, -2.958333)">
+                                        <path d="M2.95833333,2.95833333 L2.95833333,16.2708333 C2.95833333,18.7225521 4.94411458,20.7083333 7.39583333,20.7083333 L20.7083333,20.7083333 L2.95833333,2.95833333 Z" id="Path" fill="#87CEAC" fill-rule="nonzero"></path>
+                                    </g>
+                                </g>
+                            </g>
+                            <g id="Clipped">
+                                <mask id="mask-11" fill="white">
+                                    <use xlink:href="#path-10"></use>
+                                </mask>
+                                <g id="SVGID_1_"></g>
+                                <path d="M4.4375,0 C1.996875,0 0,1.996875 0,4.4375 L0,4.80729167 C0,2.36666667 1.996875,0.369791667 4.4375,0.369791667 L29.5833333,0.369791667 L29.5833333,0 L4.4375,0 Z" id="Path" fill-opacity="0.2" fill="#FFFFFF" fill-rule="nonzero" mask="url(#mask-11)"></path>
+                            </g>
+                            <g id="Clipped">
+                                <mask id="mask-13" fill="white">
+                                    <use xlink:href="#path-12"></use>
+                                </mask>
+                                <g id="SVGID_1_"></g>
+                                <path d="M42.8958333,64.7135417 L4.4375,64.7135417 C1.996875,64.7135417 0,62.7166667 0,60.2760417 L0,60.6458333 C0,63.0864583 1.996875,65.0833333 4.4375,65.0833333 L42.8958333,65.0833333 C45.3364583,65.0833333 47.3333333,63.0864583 47.3333333,60.6458333 L47.3333333,60.2760417 C47.3333333,62.7166667 45.3364583,64.7135417 42.8958333,64.7135417 Z" id="Path" fill-opacity="0.2" fill="#263238" fill-rule="nonzero" mask="url(#mask-13)"></path>
+                            </g>
+                            <g id="Clipped">
+                                <mask id="mask-15" fill="white">
+                                    <use xlink:href="#path-14"></use>
+                                </mask>
+                                <g id="SVGID_1_"></g>
+                                <path d="M34.0208333,17.75 C31.5691146,17.75 29.5833333,15.7642188 29.5833333,13.3125 L29.5833333,13.6822917 C29.5833333,16.1340104 31.5691146,18.1197917 34.0208333,18.1197917 L47.3333333,18.1197917 L47.3333333,17.75 L34.0208333,17.75 Z" id="Path" fill-opacity="0.1" fill="#263238" fill-rule="nonzero" mask="url(#mask-15)"></path>
+                            </g>
+                        </g>
+                        <path d="M29.5833333,0 L4.4375,0 C1.996875,0 0,1.996875 0,4.4375 L0,60.6458333 C0,63.0864583 1.996875,65.0833333 4.4375,65.0833333 L42.8958333,65.0833333 C45.3364583,65.0833333 47.3333333,63.0864583 47.3333333,60.6458333 L47.3333333,17.75 L29.5833333,0 Z" id="Path" fill="url(#radialGradient-16)" fill-rule="nonzero"></path>
+                    </g>
+                </g>
+            </g>
+        </g>
+    </g>
+</svg>

templates/types/streaming/nextjs/app/components/ui/lib/url.ts

@@ -1,11 +0,0 @@
-const staticFileAPI = "/api/files";
-
-export const getStaticFileDataUrl = (filePath: string) => {
-  const isUsingBackend = !!process.env.NEXT_PUBLIC_CHAT_API;
-  const fileUrl = `${staticFileAPI}/${filePath}`;
-  if (isUsingBackend) {
-    const backendOrigin = new URL(process.env.NEXT_PUBLIC_CHAT_API!).origin;
-    return `${backendOrigin}${fileUrl}`;
-  }
-  return fileUrl;
-};

templates/types/streaming/nextjs/app/components/ui/upload-csv-preview.tsx

@@ -0,0 +1,55 @@
+import { XCircleIcon } from "lucide-react";
+import Image from "next/image";
+import SheetIcon from "../ui/icons/sheet.svg";
+import { cn } from "./lib/utils";
+
+export default function UploadCsvPreview({
+  filename,
+  filesize,
+  onRemove,
+  isNew,
+}: {
+  filename: string;
+  filesize: number;
+  onRemove: () => void;
+  isNew?: boolean;
+}) {
+  const fileSizeInKB = Math.round((filesize / 1024) * 10) / 10;
+  return (
+    <div className="p-2 w-60 max-w-60 bg-secondary rounded-lg text-sm relative">
+      <div className="flex flex-row items-center gap-2">
+        <div className="relative h-10 w-10 shrink-0 overflow-hidden rounded-md">
+          <Image
+            className="h-full w-auto"
+            priority
+            src={SheetIcon}
+            alt="SheetIcon"
+          />
+        </div>
+        <div className="overflow-hidden">
+          <div className="truncate font-semibold">
+            {filename} ({fileSizeInKB} KB)
+          </div>
+          <div className="truncate text-token-text-tertiary flex items-center gap-2">
+            <span>Spreadsheet</span>
+            {isNew && (
+              <span className="px-2 py-0.5 bg-red-400 text-white text-xs rounded-2xl">
+                new
+              </span>
+            )}
+          </div>
+        </div>
+      </div>
+      <div
+        className={cn(
+          "absolute -top-2 -right-2 w-6 h-6 z-10 bg-gray-500 text-white rounded-full",
+        )}
+      >
+        <XCircleIcon
+          className="w-6 h-6 bg-gray-500 text-white rounded-full"
+          onClick={onRemove}
+        />
+      </div>
+    </div>
+  );
+}

templates/types/streaming/nextjs/package.json

@@ -35,7 +35,9 @@
     "tailwind-merge": "^2.1.0",
     "vaul": "^0.9.1",
     "@llamaindex/pdf-viewer": "^1.1.1",
-    "@e2b/code-interpreter": "^0.0.5"
+    "@e2b/code-interpreter": "^0.0.5",
+    "uuid": "^9.0.1",
+    "tmp": "^0.2.3"
   },
   "devDependencies": {
     "@types/node": "^20.10.3",
@@ -52,6 +54,7 @@
     "prettier-plugin-organize-imports": "^3.2.4",
     "tailwindcss": "^3.3.6",
     "tsx": "^4.7.2",
-    "typescript": "^5.3.2"
+    "typescript": "^5.3.2",
+    "@types/uuid": "^9.0.8"
   }
 }