use --watch

fix: ignore cache
fix: express server restart when cache changed
2026-07-19 15:03:35 -04:00 · 2024-06-10 12:41:48 +07:00 · 2024-06-10 12:13:27 +07:00 · 2024-06-10 11:44:59 +07:00 · 2024-06-07 22:56:34 +07:00 · 2024-06-07 17:40:39 +02:00
66 changed files with 2440 additions and 546 deletions
@@ -26,7 +26,7 @@ jobs:
      - uses: actions/checkout@v4

      - name: Set up python ${{ matrix.python-version }}
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}

@@ -1,5 +1,42 @@
 # create-llama

+## 0.1.9
+
+### Patch Changes
+
+- a42fa53: Add CSV upload
+- 563b51d: Fix Vercel streaming (python) to stream data events instantly
+- d60b3c5: Add E2B code interpreter tool for FastAPI
+- 956538e: Add OpenAPI action tool for FastAPI
+
+## 0.1.8
+
+### Patch Changes
+
+- cd50a33: Add interpreter tool for TS using e2b.dev
+
+## 0.1.7
+
+### Patch Changes
+
+- 260d37a: Add system prompt env variable for TS
+- bbd5b8d: Fix postgres connection leaking issue
+- bb53425: Support HTTP proxies by setting the GLOBAL_AGENT_HTTP_PROXY env variable
+- 69c2e16: Fix streaming for Express
+- 7873bfb: Update Ollama provider to run with the base URL from the environment variable
+
+## 0.1.6
+
+### Patch Changes
+
+- 56537a1: Display PDF files in source nodes
+
+## 0.1.5
+
+### Patch Changes
+
+- 84db798: feat: support display latex in chat markdown
+
 ## 0.1.4

 ### Patch Changes
@@ -1,5 +1,6 @@
 import fs from "fs/promises";
 import path from "path";
+import { TOOL_SYSTEM_PROMPT_ENV_VAR, Tool } from "./tools";
 import {
  ModelConfig,
  TemplateDataSource,
@@ -7,7 +8,7 @@ import {
  TemplateVectorDB,
 } from "./types";

-type EnvVar = {
+export type EnvVar = {
  name?: string;
  description?: string;
  value?: string;
@@ -184,6 +185,10 @@ const getModelEnvs = (modelConfig: ModelConfig): EnvVar[] => {
      description: "Dimension of the embedding model to use.",
      value: modelConfig.dimensions.toString(),
    },
+    {
+      name: "CONVERSATION_STARTERS",
+      description: "The questions to help users get started (multi-line).",
+    },
    ...(modelConfig.provider === "openai"
      ? [
          {
@@ -219,35 +224,52 @@ const getModelEnvs = (modelConfig: ModelConfig): EnvVar[] => {
          },
        ]
      : []),
+    ...(modelConfig.provider === "ollama"
+      ? [
+          {
+            name: "OLLAMA_BASE_URL",
+            description:
+              "The base URL for the Ollama API. Eg: http://127.0.0.1:11434",
+          },
+        ]
+      : []),
  ];
 };

 const getFrameworkEnvs = (
-  framework?: TemplateFramework,
+  framework: TemplateFramework,
  port?: number,
 ): EnvVar[] => {
-  if (framework !== "fastapi") {
-    return [];
-  }
-  return [
+  const sPort = port?.toString() || "8000";
+  const result: EnvVar[] = [
    {
-      name: "APP_HOST",
-      description: "The address to start the backend app.",
-      value: "0.0.0.0",
-    },
-    {
-      name: "APP_PORT",
-      description: "The port to start the backend app.",
-      value: port?.toString() || "8000",
-    },
-    // TODO: Once LlamaIndexTS supports string templates, move this to `getEngineEnvs`
-    {
-      name: "SYSTEM_PROMPT",
-      description: `Custom system prompt.
-Example:
-SYSTEM_PROMPT="You are a helpful assistant who helps users with their questions."`,
+      name: "FILESERVER_URL_PREFIX",
+      description:
+        "FILESERVER_URL_PREFIX is the URL prefix of the server storing the images generated by the interpreter.",
+      value:
+        framework === "nextjs"
+          ? // FIXME: if we are using nextjs, port should be 3000
+            "http://localhost:3000/api/files"
+          : `http://localhost:${sPort}/api/files`,
    },
  ];
+  if (framework === "fastapi") {
+    result.push(
+      ...[
+        {
+          name: "APP_HOST",
+          description: "The address to start the backend app.",
+          value: "0.0.0.0",
+        },
+        {
+          name: "APP_PORT",
+          description: "The port to start the backend app.",
+          value: sPort,
+        },
+      ],
+    );
+  }
+  return result;
 };

 const getEngineEnvs = (): EnvVar[] => {
@@ -258,18 +280,68 @@ const getEngineEnvs = (): EnvVar[] => {
        "The number of similar embeddings to return when retrieving documents.",
      value: "3",
    },
+    {
+      name: "STREAM_TIMEOUT",
+      description:
+        "The time in milliseconds to wait for the stream to return a response.",
+      value: "60000",
+    },
  ];
 };

+const getToolEnvs = (tools?: Tool[]): EnvVar[] => {
+  if (!tools?.length) return [];
+  const toolEnvs: EnvVar[] = [];
+  tools.forEach((tool) => {
+    if (tool.envVars?.length) {
+      toolEnvs.push(
+        // Don't include the system prompt env var here
+        // It should be handled separately by merging with the default system prompt
+        ...tool.envVars.filter(
+          (env) => env.name !== TOOL_SYSTEM_PROMPT_ENV_VAR,
+        ),
+      );
+    }
+  });
+  return toolEnvs;
+};
+
+const getSystemPromptEnv = (tools?: Tool[]): EnvVar => {
+  const defaultSystemPrompt =
+    "You are a helpful assistant who helps users with their questions.";
+
+  // build tool system prompt by merging all tool system prompts
+  let toolSystemPrompt = "";
+  tools?.forEach((tool) => {
+    const toolSystemPromptEnv = tool.envVars?.find(
+      (env) => env.name === TOOL_SYSTEM_PROMPT_ENV_VAR,
+    );
+    if (toolSystemPromptEnv) {
+      toolSystemPrompt += toolSystemPromptEnv.value + "\n";
+    }
+  });
+
+  const systemPrompt = toolSystemPrompt
+    ? `\"${toolSystemPrompt}\"`
+    : defaultSystemPrompt;
+
+  return {
+    name: "SYSTEM_PROMPT",
+    description: "The system prompt for the AI model.",
+    value: systemPrompt,
+  };
+};
+
 export const createBackendEnvFile = async (
  root: string,
  opts: {
    llamaCloudKey?: string;
    vectorDb?: TemplateVectorDB;
    modelConfig: ModelConfig;
-    framework?: TemplateFramework;
+    framework: TemplateFramework;
    dataSources?: TemplateDataSource[];
    port?: number;
+    tools?: Tool[];
  },
 ) => {
  // Init env values
@@ -287,6 +359,8 @@ export const createBackendEnvFile = async (
    // Add vector database environment variables
    ...getVectorDBEnvs(opts.vectorDb, opts.framework),
    ...getFrameworkEnvs(opts.framework, opts.port),
+    ...getToolEnvs(opts.tools),
+    getSystemPromptEnv(opts.tools),
  ];
  // Render and write env file
  const content = renderEnvVar(envVars);
@@ -148,6 +148,7 @@ export const installTemplate = async (
      framework: props.framework,
      dataSources: props.dataSources,
      port: props.externalPort,
+      tools: props.tools,
    });

    if (props.dataSources.length > 0) {
@@ -170,6 +171,11 @@ export const installTemplate = async (
        );
      }
    }
+
+    // Create tool-output directory
+    if (props.tools && props.tools.length > 0) {
+      await fsExtra.mkdir(path.join(props.root, "tool-output"));
+    }
  } else {
    // this is a frontend for a full-stack app, create .env file with model information
    await createFrontendEnvFile(props.root, {
@@ -0,0 +1,8 @@
+/* Function to conditionally load the global-agent/bootstrap module */
+export async function initializeGlobalAgent() {
+  if (process.env.GLOBAL_AGENT_HTTP_PROXY) {
+    /* Dynamically import global-agent/bootstrap */
+    await import("global-agent/bootstrap");
+    console.log("Proxy enabled via global-agent.");
+  }
+}
@@ -144,7 +144,7 @@ const getAdditionalDependencies = (
    case "openai":
      dependencies.push({
        name: "llama-index-agent-openai",
-        version: "0.2.2",
+        version: "0.2.6",
      });
      break;
    case "anthropic":
@@ -160,7 +160,7 @@ const getAdditionalDependencies = (
    case "gemini":
      dependencies.push({
        name: "llama-index-llms-gemini",
-        version: "0.1.7",
+        version: "0.1.10",
      });
      dependencies.push({
        name: "llama-index-embeddings-gemini",
@@ -2,9 +2,12 @@ import fs from "fs/promises";
 import path from "path";
 import { red } from "picocolors";
 import yaml from "yaml";
+import { EnvVar } from "./env-variables";
 import { makeDir } from "./make-dir";
 import { TemplateFramework } from "./types";

+export const TOOL_SYSTEM_PROMPT_ENV_VAR = "TOOL_SYSTEM_PROMPT";
+
 export enum ToolType {
  LLAMAHUB = "llamahub",
  LOCAL = "local",
@@ -17,6 +20,7 @@ export type Tool = {
  dependencies?: ToolDependencies[];
  supportedFrameworks?: Array<TemplateFramework>;
  type: ToolType;
+  envVars?: EnvVar[];
 };

 export type ToolDependencies = {
@@ -26,7 +30,7 @@ export type ToolDependencies = {

 export const supportedTools: Tool[] = [
  {
-    display: "Google Search (configuration required after installation)",
+    display: "Google Search",
    name: "google.GoogleSearchToolSpec",
    config: {
      engine:
@@ -42,6 +46,13 @@ export const supportedTools: Tool[] = [
    ],
    supportedFrameworks: ["fastapi"],
    type: ToolType.LLAMAHUB,
+    envVars: [
+      {
+        name: TOOL_SYSTEM_PROMPT_ENV_VAR,
+        description: "System prompt for google search tool.",
+        value: `You are a Google search agent. You help users to get information from Google search.`,
+      },
+    ],
  },
  {
    display: "Wikipedia",
@@ -54,6 +65,13 @@ export const supportedTools: Tool[] = [
    ],
    supportedFrameworks: ["fastapi", "express", "nextjs"],
    type: ToolType.LLAMAHUB,
+    envVars: [
+      {
+        name: TOOL_SYSTEM_PROMPT_ENV_VAR,
+        description: "System prompt for wiki tool.",
+        value: `You are a Wikipedia agent. You help users to get information from Wikipedia.`,
+      },
+    ],
  },
  {
    display: "Weather",
@@ -61,6 +79,74 @@ export const supportedTools: Tool[] = [
    dependencies: [],
    supportedFrameworks: ["fastapi", "express", "nextjs"],
    type: ToolType.LOCAL,
+    envVars: [
+      {
+        name: TOOL_SYSTEM_PROMPT_ENV_VAR,
+        description: "System prompt for weather tool.",
+        value: `You are a weather forecast agent. You help users to get the weather forecast for a given location.`,
+      },
+    ],
+  },
+  {
+    display: "Code Interpreter",
+    name: "interpreter",
+    dependencies: [
+      {
+        name: "e2b_code_interpreter",
+        version: "0.0.7",
+      },
+    ],
+    supportedFrameworks: ["fastapi", "express", "nextjs"],
+    type: ToolType.LOCAL,
+    envVars: [
+      {
+        name: "E2B_API_KEY",
+        description:
+          "E2B_API_KEY key is required to run code interpreter tool. Get it here: https://e2b.dev/docs/getting-started/api-key",
+      },
+      {
+        name: TOOL_SYSTEM_PROMPT_ENV_VAR,
+        description: "System prompt for code interpreter tool.",
+        value: `You are a Python interpreter.
+        - You are given tasks to complete and you run python code to solve them.
+        - The python code runs in a Jupyter notebook. Every time you call \`interpreter\` tool, the python code is executed in a separate cell. It's okay to make multiple calls to \`interpreter\`.
+        - Display visualizations using matplotlib or any other visualization library directly in the notebook. Shouldn't save the visualizations to a file, just return the base64 encoded data.
+        - You can install any pip package (if it exists) if you need to but the usual packages for data analysis are already preinstalled.
+        - You can run any python code you want in a secure environment.
+        - Use absolute url from result to display images or any other media.`,
+      },
+    ],
+  },
+  {
+    display: "OpenAPI action",
+    name: "openapi_action.OpenAPIActionToolSpec",
+    dependencies: [
+      {
+        name: "llama-index-tools-openapi",
+        version: "0.1.3",
+      },
+      {
+        name: "jsonschema",
+        version: "^4.22.0",
+      },
+      {
+        name: "llama-index-tools-requests",
+        version: "0.1.3",
+      },
+    ],
+    config: {
+      openapi_uri: "The URL or file path of the OpenAPI schema",
+    },
+    supportedFrameworks: ["fastapi"],
+    type: ToolType.LOCAL,
+    envVars: [
+      {
+        name: TOOL_SYSTEM_PROMPT_ENV_VAR,
+        description: "System prompt for openapi action tool.",
+        value:
+          "You are an OpenAPI action agent. You help users to make requests to the provided OpenAPI schema.",
+      },
+    ],
  },
 ];

@@ -87,9 +173,15 @@ export const getTools = (toolsName: string[]): Tool[] => {
  return tools;
 };

+export const toolRequiresConfig = (tool: Tool): boolean => {
+  const hasConfig = Object.keys(tool.config || {}).length > 0;
+  const hasEmptyEnvVar = tool.envVars?.some((envVar) => !envVar.value) ?? false;
+  return hasConfig || hasEmptyEnvVar;
+};
+
 export const toolsRequireConfig = (tools?: Tool[]): boolean => {
  if (tools) {
-    return tools?.some((tool) => Object.keys(tool.config || {}).length > 0);
+    return tools?.some(toolRequiresConfig);
  }
  return false;
 };
@@ -104,7 +196,6 @@ export const writeToolsConfig = async (
  tools: Tool[] = [],
  type: ConfigFileType = ConfigFileType.YAML,
 ) => {
-  if (tools.length === 0) return; // no tools selected, no config need
  const configContent: {
    [key in ToolType]: Record<string, any>;
  } = {
@@ -12,12 +12,16 @@ import { createApp } from "./create-app";
 import { getDataSources } from "./helpers/datasources";
 import { getPkgManager } from "./helpers/get-pkg-manager";
 import { isFolderEmpty } from "./helpers/is-folder-empty";
+import { initializeGlobalAgent } from "./helpers/proxy";
 import { runApp } from "./helpers/run-app";
 import { getTools } from "./helpers/tools";
 import { validateNpmName } from "./helpers/validate-pkg";
 import packageJson from "./package.json";
 import { QuestionArgs, askQuestions, onPromptState } from "./questions";

+// Run the initialization function
+initializeGlobalAgent();
+
 let projectPath: string = "";

 const handleSigTerm = () => process.exit(0);
@@ -1,6 +1,6 @@
 {
  "name": "create-llama",
-  "version": "0.1.4",
+  "version": "0.1.9",
  "description": "Create LlamaIndex-powered apps with one command",
  "keywords": [
    "rag",
@@ -52,6 +52,7 @@
    "cross-spawn": "7.0.3",
    "fast-glob": "3.3.1",
    "fs-extra": "11.2.0",
+    "global-agent": "^3.0.0",
    "got": "10.7.0",
    "ollama": "^0.5.0",
    "ora": "^8.0.1",
@@ -16,7 +16,11 @@ import { templatesDir } from "./helpers/dir";
 import { getAvailableLlamapackOptions } from "./helpers/llama-pack";
 import { askModelConfig } from "./helpers/providers";
 import { getProjectOptions } from "./helpers/repo";
-import { supportedTools, toolsRequireConfig } from "./helpers/tools";
+import {
+  supportedTools,
+  toolRequiresConfig,
+  toolsRequireConfig,
+} from "./helpers/tools";

 export type QuestionArgs = Omit<
  InstallAppArgs,
@@ -652,7 +656,7 @@ export const askQuestions = async (
        t.supportedFrameworks?.includes(program.framework),
      );
      const toolChoices = options.map((tool) => ({
-        title: tool.display,
+        title: `${tool.display}${toolRequiresConfig(tool) ? "" : " (no config needed)"}`,
        value: tool.name,
      }));
      const { toolsName } = await prompts({
@@ -1,7 +1,8 @@
 import os
 import yaml
+import json
 import importlib
-
+from cachetools import cached, LRUCache
 from llama_index.core.tools.tool_spec.base import BaseToolSpec
 from llama_index.core.tools.function_tool import FunctionTool

@@ -19,6 +20,14 @@ class ToolFactory:
    }

    @staticmethod
+    @cached(
+        LRUCache(maxsize=100),
+        key=lambda tool_type, tool_name, config: (
+            tool_type,
+            tool_name,
+            json.dumps(config, sort_keys=True),
+        ),
+    )
    def load_tools(tool_type: str, tool_name: str, config: dict) -> list[FunctionTool]:
        source_package = ToolFactory.TOOL_SOURCE_PACKAGE_MAP[tool_type]
        try:
@@ -0,0 +1,150 @@
+import os
+import logging
+import base64
+import uuid
+from pydantic import BaseModel
+from typing import List, Tuple, Dict, Optional
+from llama_index.core.tools import FunctionTool
+from e2b_code_interpreter import CodeInterpreter
+from e2b_code_interpreter.models import Logs
+
+
+logger = logging.getLogger(__name__)
+
+
+class InterpreterExtraResult(BaseModel):
+    type: str
+    content: Optional[str] = None
+    filename: Optional[str] = None
+    url: Optional[str] = None
+
+
+class E2BToolOutput(BaseModel):
+    is_error: bool
+    logs: Logs
+    results: List[InterpreterExtraResult] = []
+
+
+class E2BCodeInterpreter:
+
+    output_dir = "tool-output"
+
+    def __init__(self, api_key: str, filesever_url_prefix: str):
+        self.api_key = api_key
+        self.filesever_url_prefix = filesever_url_prefix
+
+    def get_output_path(self, filename: str) -> str:
+        # if output directory doesn't exist, create it
+        if not os.path.exists(self.output_dir):
+            os.makedirs(self.output_dir, exist_ok=True)
+        return os.path.join(self.output_dir, filename)
+
+    def save_to_disk(self, base64_data: str, ext: str) -> Dict:
+        filename = f"{uuid.uuid4()}.{ext}"  # generate a unique filename
+        buffer = base64.b64decode(base64_data)
+        output_path = self.get_output_path(filename)
+
+        try:
+            with open(output_path, "wb") as file:
+                file.write(buffer)
+        except IOError as e:
+            logger.error(f"Failed to write to file {output_path}: {str(e)}")
+            raise e
+
+        logger.info(f"Saved file to {output_path}")
+
+        return {
+            "outputPath": output_path,
+            "filename": filename,
+        }
+
+    def get_file_url(self, filename: str) -> str:
+        return f"{self.filesever_url_prefix}/{self.output_dir}/{filename}"
+
+    def parse_result(self, result) -> List[InterpreterExtraResult]:
+        """
+        The result could include multiple formats (e.g. png, svg, etc.) but encoded in base64
+        We save each result to disk and return saved file metadata (extension, filename, url)
+        """
+        if not result:
+            return []
+
+        output = []
+
+        try:
+            formats = result.formats()
+            results = [result[format] for format in formats]
+
+            for ext, data in zip(formats, results):
+                match ext:
+                    case "png" | "svg" | "jpeg" | "pdf":
+                        result = self.save_to_disk(data, ext)
+                        filename = result["filename"]
+                        output.append(
+                            InterpreterExtraResult(
+                                type=ext,
+                                filename=filename,
+                                url=self.get_file_url(filename),
+                            )
+                        )
+                    case _:
+                        output.append(
+                            InterpreterExtraResult(
+                                type=ext,
+                                content=data,
+                            )
+                        )
+        except Exception as error:
+            logger.exception(error, exc_info=True)
+            logger.error("Error when parsing output from E2b interpreter tool", error)
+
+        return output
+
+    def interpret(self, code: str) -> E2BToolOutput:
+        with CodeInterpreter(api_key=self.api_key) as interpreter:
+            logger.info(
+                f"\n{'='*50}\n> Running following AI-generated code:\n{code}\n{'='*50}"
+            )
+            exec = interpreter.notebook.exec_cell(code)
+
+            if exec.error:
+                logger.error("Error when executing code", exec.error)
+                output = E2BToolOutput(is_error=True, logs=exec.logs, results=[])
+            else:
+                if len(exec.results) == 0:
+                    output = E2BToolOutput(is_error=False, logs=exec.logs, results=[])
+                else:
+                    results = self.parse_result(exec.results[0])
+                    output = E2BToolOutput(
+                        is_error=False, logs=exec.logs, results=results
+                    )
+            return output
+
+
+def code_interpret(code: str) -> Dict:
+    """
+    Execute python code in a Jupyter notebook cell and return any result, stdout, stderr, display_data, and error.
+
+    Parameters:
+        code (str): The python code to be executed in a single cell.
+    """
+    api_key = os.getenv("E2B_API_KEY")
+    filesever_url_prefix = os.getenv("FILESERVER_URL_PREFIX")
+    if not api_key:
+        raise ValueError(
+            "E2B_API_KEY key is required to run code interpreter. Get it here: https://e2b.dev/docs/getting-started/api-key"
+        )
+    if not filesever_url_prefix:
+        raise ValueError(
+            "FILESERVER_URL_PREFIX is required to display file output from sandbox"
+        )
+
+    interpreter = E2BCodeInterpreter(
+        api_key=api_key, filesever_url_prefix=filesever_url_prefix
+    )
+    output = interpreter.interpret(code)
+    return output.dict()
+
+
+# Specify as functions tools to be loaded by the ToolFactory
+tools = [FunctionTool.from_defaults(code_interpret)]
@@ -0,0 +1,71 @@
+from typing import Dict, List, Tuple
+from llama_index.tools.openapi import OpenAPIToolSpec
+from llama_index.tools.requests import RequestsToolSpec
+
+
+class OpenAPIActionToolSpec(OpenAPIToolSpec, RequestsToolSpec):
+    """
+    A combination of OpenAPI and Requests tool specs that can parse OpenAPI specs and make requests.
+
+    openapi_uri: str: The file path or URL to the OpenAPI spec.
+    domain_headers: dict: Whitelist domains and the headers to use.
+    """
+
+    spec_functions = OpenAPIToolSpec.spec_functions + RequestsToolSpec.spec_functions
+
+    def __init__(self, openapi_uri: str, domain_headers: dict = {}, **kwargs):
+        # Load the OpenAPI spec
+        openapi_spec, servers = self.load_openapi_spec(openapi_uri)
+
+        # Add the servers to the domain headers if they are not already present
+        for server in servers:
+            if server not in domain_headers:
+                domain_headers[server] = {}
+
+        OpenAPIToolSpec.__init__(self, spec=openapi_spec)
+        RequestsToolSpec.__init__(self, domain_headers)
+
+    @staticmethod
+    def load_openapi_spec(uri: str) -> Tuple[Dict, List[str]]:
+        """
+        Load an OpenAPI spec from a URI.
+
+        Args:
+            uri (str): A file path or URL to the OpenAPI spec.
+
+        Returns:
+            List[Document]: A list of Document objects.
+        """
+        import yaml
+        from urllib.parse import urlparse
+
+        if uri.startswith("http"):
+            import requests
+
+            response = requests.get(uri)
+            if response.status_code != 200:
+                raise ValueError(
+                    "Could not initialize OpenAPIActionToolSpec: "
+                    f"Failed to load OpenAPI spec from {uri}, status code: {response.status_code}"
+                )
+            spec = yaml.safe_load(response.text)
+        elif uri.startswith("file"):
+            filepath = urlparse(uri).path
+            with open(filepath, "r") as file:
+                spec = yaml.safe_load(file)
+        else:
+            raise ValueError(
+                "Could not initialize OpenAPIActionToolSpec: Invalid OpenAPI URI provided. "
+                "Only HTTP and file path are supported."
+            )
+        # Add the servers to the whitelist
+        try:
+            servers = [
+                urlparse(server["url"]).netloc for server in spec.get("servers", [])
+            ]
+        except KeyError as e:
+            raise ValueError(
+                "Could not initialize OpenAPIActionToolSpec: Invalid OpenAPI spec provided. "
+                "Could not get `servers` from the spec."
+            ) from e
+        return spec, servers
@@ -1,10 +1,9 @@
 import { BaseToolWithCall, OpenAIAgent, QueryEngineTool } from "llamaindex";
-import { ToolsFactory } from "llamaindex/tools/ToolsFactory";
 import fs from "node:fs/promises";
 import path from "node:path";
 import { getDataSource } from "./index";
 import { STORAGE_CACHE_DIR } from "./shared";
-import { createLocalTools } from "./tools";
+import { createTools } from "./tools";

 export async function createChatEngine() {
  const tools: BaseToolWithCall[] = [];
@@ -24,22 +23,20 @@ export async function createChatEngine() {
    );
  }

+  const configFile = path.join("config", "tools.json");
+  let toolConfig: any;
  try {
    // add tools from config file if it exists
-    const config = JSON.parse(
-      await fs.readFile(path.join("config", "tools.json"), "utf8"),
-    );
-
-    // add local tools from the 'tools' folder (if configured)
-    const localTools = createLocalTools(config.local);
-    tools.push(...localTools);
-
-    // add tools from LlamaIndexTS (if configured)
-    const llamaTools = await ToolsFactory.createTools(config.llamahub);
-    tools.push(...llamaTools);
-  } catch {}
+    toolConfig = JSON.parse(await fs.readFile(configFile, "utf8"));
+  } catch (e) {
+    console.info(`Could not read ${configFile} file. Using no tools.`);
+  }
+  if (toolConfig) {
+    tools.push(...(await createTools(toolConfig)));
+  }

  return new OpenAIAgent({
    tools,
+    systemPrompt: process.env.SYSTEM_PROMPT,
  });
 }
@@ -1,15 +1,31 @@
 import { BaseToolWithCall } from "llamaindex";
+import { ToolsFactory } from "llamaindex/tools/ToolsFactory";
+import { InterpreterTool, InterpreterToolParams } from "./interpreter";
 import { WeatherTool, WeatherToolParams } from "./weather";

 type ToolCreator = (config: unknown) => BaseToolWithCall;

+export async function createTools(toolConfig: {
+  local: Record<string, unknown>;
+  llamahub: any;
+}): Promise<BaseToolWithCall[]> {
+  // add local tools from the 'tools' folder (if configured)
+  const tools = createLocalTools(toolConfig.local);
+  // add tools from LlamaIndexTS (if configured)
+  tools.push(...(await ToolsFactory.createTools(toolConfig.llamahub)));
+  return tools;
+}
+
 const toolFactory: Record<string, ToolCreator> = {
  weather: (config: unknown) => {
    return new WeatherTool(config as WeatherToolParams);
  },
+  interpreter: (config: unknown) => {
+    return new InterpreterTool(config as InterpreterToolParams);
+  },
 };

-export function createLocalTools(
+function createLocalTools(
  localConfig: Record<string, unknown>,
 ): BaseToolWithCall[] {
  const tools: BaseToolWithCall[] = [];
@@ -0,0 +1,189 @@
+import { CodeInterpreter, Logs, Result } from "@e2b/code-interpreter";
+import type { JSONSchemaType } from "ajv";
+import fs from "fs";
+import { BaseTool, ToolMetadata } from "llamaindex";
+import crypto from "node:crypto";
+import path from "node:path";
+
+export type InterpreterParameter = {
+  code: string;
+};
+
+export type InterpreterToolParams = {
+  metadata?: ToolMetadata<JSONSchemaType<InterpreterParameter>>;
+  apiKey?: string;
+  fileServerURLPrefix?: string;
+};
+
+export type InterpreterToolOutput = {
+  isError: boolean;
+  logs: Logs;
+  extraResult: InterpreterExtraResult[];
+};
+
+type InterpreterExtraType =
+  | "html"
+  | "markdown"
+  | "svg"
+  | "png"
+  | "jpeg"
+  | "pdf"
+  | "latex"
+  | "json"
+  | "javascript";
+
+export type InterpreterExtraResult = {
+  type: InterpreterExtraType;
+  content?: string;
+  filename?: string;
+  url?: string;
+};
+
+const DEFAULT_META_DATA: ToolMetadata<JSONSchemaType<InterpreterParameter>> = {
+  name: "interpreter",
+  description:
+    "Execute python code in a Jupyter notebook cell and return any result, stdout, stderr, display_data, and error.",
+  parameters: {
+    type: "object",
+    properties: {
+      code: {
+        type: "string",
+        description: "The python code to execute in a single cell.",
+      },
+    },
+    required: ["code"],
+  },
+};
+
+export class InterpreterTool implements BaseTool<InterpreterParameter> {
+  private readonly outputDir = "tool-output";
+  private apiKey?: string;
+  private fileServerURLPrefix?: string;
+  metadata: ToolMetadata<JSONSchemaType<InterpreterParameter>>;
+  codeInterpreter?: CodeInterpreter;
+
+  constructor(params?: InterpreterToolParams) {
+    this.metadata = params?.metadata || DEFAULT_META_DATA;
+    this.apiKey = params?.apiKey || process.env.E2B_API_KEY;
+    this.fileServerURLPrefix =
+      params?.fileServerURLPrefix || process.env.FILESERVER_URL_PREFIX;
+
+    if (!this.apiKey) {
+      throw new Error(
+        "E2B_API_KEY key is required to run code interpreter. Get it here: https://e2b.dev/docs/getting-started/api-key",
+      );
+    }
+    if (!this.fileServerURLPrefix) {
+      throw new Error(
+        "FILESERVER_URL_PREFIX is required to display file output from sandbox",
+      );
+    }
+  }
+
+  public async initInterpreter() {
+    if (!this.codeInterpreter) {
+      this.codeInterpreter = await CodeInterpreter.create({
+        apiKey: this.apiKey,
+      });
+    }
+    return this.codeInterpreter;
+  }
+
+  public async codeInterpret(code: string): Promise<InterpreterToolOutput> {
+    console.log(
+      `\n${"=".repeat(50)}\n> Running following AI-generated code:\n${code}\n${"=".repeat(50)}`,
+    );
+    const interpreter = await this.initInterpreter();
+    const exec = await interpreter.notebook.execCell(code);
+    if (exec.error) console.error("[Code Interpreter error]", exec.error);
+    const extraResult = await this.getExtraResult(exec.results[0]);
+    const result: InterpreterToolOutput = {
+      isError: !!exec.error,
+      logs: exec.logs,
+      extraResult,
+    };
+    return result;
+  }
+
+  async call(input: InterpreterParameter): Promise<InterpreterToolOutput> {
+    const result = await this.codeInterpret(input.code);
+    return result;
+  }
+
+  async close() {
+    await this.codeInterpreter?.close();
+  }
+
+  private async getExtraResult(
+    res?: Result,
+  ): Promise<InterpreterExtraResult[]> {
+    if (!res) return [];
+    const output: InterpreterExtraResult[] = [];
+
+    try {
+      const formats = res.formats(); // formats available for the result. Eg: ['png', ...]
+      const results = formats.map((f) => res[f as keyof Result]); // get base64 data for each format
+
+      // save base64 data to file and return the url
+      for (let i = 0; i < formats.length; i++) {
+        const ext = formats[i];
+        const data = results[i];
+        switch (ext) {
+          case "png":
+          case "jpeg":
+          case "svg":
+          case "pdf":
+            const { filename } = this.saveToDisk(data, ext);
+            output.push({
+              type: ext as InterpreterExtraType,
+              filename,
+              url: this.getFileUrl(filename),
+            });
+            break;
+          default:
+            output.push({
+              type: ext as InterpreterExtraType,
+              content: data,
+            });
+            break;
+        }
+      }
+    } catch (error) {
+      console.error("Error when parsing e2b response", error);
+    }
+
+    return output;
+  }
+
+  // Consider saving to cloud storage instead but it may cost more for you
+  // See: https://e2b.dev/docs/sandbox/api/filesystem#write-to-file
+  private saveToDisk(
+    base64Data: string,
+    ext: string,
+  ): {
+    outputPath: string;
+    filename: string;
+  } {
+    const filename = `${crypto.randomUUID()}.${ext}`; // generate a unique filename
+    const buffer = Buffer.from(base64Data, "base64");
+    const outputPath = this.getOutputPath(filename);
+    fs.writeFileSync(outputPath, buffer);
+    console.log(`Saved file to ${outputPath}`);
+    return {
+      outputPath,
+      filename,
+    };
+  }
+
+  private getOutputPath(filename: string): string {
+    // if outputDir doesn't exist, create it
+    if (!fs.existsSync(this.outputDir)) {
+      fs.mkdirSync(this.outputDir, { recursive: true });
+    }
+    return path.join(this.outputDir, filename);
+  }
+
+  private getFileUrl(filename: string): string {
+    return `${this.fileServerURLPrefix}/${this.outputDir}/${filename}`;
+  }
+}
@@ -16,5 +16,6 @@ export async function createChatEngine() {
  return new ContextChatEngine({
    chatModel: Settings.llm,
    retriever,
+    systemPrompt: process.env.SYSTEM_PROMPT,
  });
 }
@@ -1,5 +1,7 @@
 "use client";

+import { Message } from "./chat-messages";
+
 export interface ChatInputProps {
  /** The current value of the input */
  input?: string;
@@ -12,7 +14,8 @@ export interface ChatInputProps {
  /** Form submission handler to automatically reset input and append a user message  */
  handleSubmit: (e: React.FormEvent<HTMLFormElement>) => void;
  isLoading: boolean;
-  multiModal?: boolean;
+  messages: Message[];
+  setInput?: (input: string) => void;
 }

 export default function ChatInput(props: ChatInputProps) {
@@ -19,6 +19,9 @@ export default function ChatMessages({
  isLoading?: boolean;
  stop?: () => void;
  reload?: () => void;
+  append?: (
+    message: Message | Omit<Message, "id">,
+  ) => Promise<string | null | undefined>;
 }) {
  const scrollableChatContainerRef = useRef<HTMLDivElement>(null);

@@ -0,0 +1,30 @@
+"use client";
+
+import { useEffect, useMemo, useState } from "react";
+
+export interface ChatConfig {
+  chatAPI?: string;
+  starterQuestions?: string[];
+}
+
+export function useClientConfig() {
+  const API_ROUTE = "/api/chat/config";
+  const chatAPI = process.env.NEXT_PUBLIC_CHAT_API;
+  const [config, setConfig] = useState<ChatConfig>({
+    chatAPI,
+  });
+
+  const configAPI = useMemo(() => {
+    const backendOrigin = chatAPI ? new URL(chatAPI).origin : "";
+    return `${backendOrigin}${API_ROUTE}`;
+  }, [chatAPI]);
+
+  useEffect(() => {
+    fetch(configAPI)
+      .then((response) => response.json())
+      .then((data) => setConfig({ ...data, chatAPI }))
+      .catch((error) => console.error("Error fetching config", error));
+  }, [chatAPI, configAPI]);
+
+  return config;
+}
@@ -1,12 +1,22 @@
-import logging
 import os
+import logging
+from datetime import timedelta

+from cachetools import cached, TTLCache
 from llama_index.core.storage import StorageContext
 from llama_index.core.indices import load_index_from_storage

 logger = logging.getLogger("uvicorn")


+@cached(
+    TTLCache(maxsize=10, ttl=timedelta(minutes=5).total_seconds()),
+    key=lambda *args, **kwargs: "global_storage_context",
+)
+def get_storage_context(persist_dir: str) -> StorageContext:
+    return StorageContext.from_defaults(persist_dir=persist_dir)
+
+
 def get_index():
    storage_dir = os.getenv("STORAGE_DIR", "storage")
    # check if storage already exists
@@ -14,7 +24,7 @@ def get_index():
        return None
    # load the existing index
    logger.info(f"Loading index from {storage_dir}...")
-    storage_context = StorageContext.from_defaults(persist_dir=storage_dir)
+    storage_context = get_storage_context(storage_dir)
    index = load_index_from_storage(storage_context)
    logger.info(f"Finished loading index from {storage_dir}")
    return index
@@ -5,26 +5,33 @@ from urllib.parse import urlparse
 PGVECTOR_SCHEMA = "public"
 PGVECTOR_TABLE = "llamaindex_embedding"

+vector_store: PGVectorStore = None
+

 def get_vector_store():
-    original_conn_string = os.environ.get("PG_CONNECTION_STRING")
-    if original_conn_string is None or original_conn_string == "":
-        raise ValueError("PG_CONNECTION_STRING environment variable is not set.")
+    global vector_store

-    # The PGVectorStore requires both two connection strings, one for psycopg2 and one for asyncpg
-    # Update the configured scheme with the psycopg2 and asyncpg schemes
-    original_scheme = urlparse(original_conn_string).scheme + "://"
-    conn_string = original_conn_string.replace(
-        original_scheme, "postgresql+psycopg2://"
-    )
-    async_conn_string = original_conn_string.replace(
-        original_scheme, "postgresql+asyncpg://"
-    )
+    if vector_store is None:
+        original_conn_string = os.environ.get("PG_CONNECTION_STRING")
+        if original_conn_string is None or original_conn_string == "":
+            raise ValueError("PG_CONNECTION_STRING environment variable is not set.")

-    return PGVectorStore(
-        connection_string=conn_string,
-        async_connection_string=async_conn_string,
-        schema_name=PGVECTOR_SCHEMA,
-        table_name=PGVECTOR_TABLE,
-        embed_dim=int(os.environ.get("EMBEDDING_DIM", 1024)),
-    )
+        # The PGVectorStore requires both two connection strings, one for psycopg2 and one for asyncpg
+        # Update the configured scheme with the psycopg2 and asyncpg schemes
+        original_scheme = urlparse(original_conn_string).scheme + "://"
+        conn_string = original_conn_string.replace(
+            original_scheme, "postgresql+psycopg2://"
+        )
+        async_conn_string = original_conn_string.replace(
+            original_scheme, "postgresql+asyncpg://"
+        )
+
+        vector_store = PGVectorStore(
+            connection_string=conn_string,
+            async_connection_string=async_conn_string,
+            schema_name=PGVECTOR_SCHEMA,
+            table_name=PGVECTOR_TABLE,
+            embed_dim=int(os.environ.get("EMBEDDING_DIM", 1024)),
+        )
+
+    return vector_store
@@ -1,3 +1,5 @@
 # local env files
 .env
-node_modules/
+node_modules/
+
+tool-output/
@@ -31,6 +31,8 @@ if (isDevelopment) {
  console.warn("Production CORS origin not set, defaulting to no CORS.");
 }

+app.use("/api/files/data", express.static("data"));
+app.use("/api/files/tool-output", express.static("tool-output"));
 app.use(express.text());

 app.get("/", (req: Request, res: Response) => {
@@ -7,16 +7,17 @@
    "format:write": "prettier --ignore-unknown --write .",
    "build": "tsup index.ts --format cjs --dts",
    "start": "node dist/index.js",
-    "dev": "concurrently \"tsup index.ts --format cjs --dts --watch\" \"nodemon -q dist/index.js\""
+    "dev": "concurrently \"tsup index.ts --format cjs --dts --watch\" \"nodemon --watch dist/index.js\""
  },
  "dependencies": {
    "ai": "^3.0.21",
    "cors": "^2.8.5",
    "dotenv": "^16.3.1",
    "express": "^4.18.2",
-    "llamaindex": "0.3.9",
+    "llamaindex": "0.3.16",
    "pdf2json": "3.0.5",
-    "ajv": "^8.12.0"
+    "ajv": "^8.12.0",
+    "@e2b/code-interpreter": "^0.0.5"
  },
  "devDependencies": {
    "@types/cors": "^2.8.16",
@@ -0,0 +1,14 @@
+import { Request, Response } from "express";
+
+export const chatConfig = async (_req: Request, res: Response) => {
+  let starterQuestions = undefined;
+  if (
+    process.env.CONVERSATION_STARTERS &&
+    process.env.CONVERSATION_STARTERS.trim()
+  ) {
+    starterQuestions = process.env.CONVERSATION_STARTERS.trim().split("\n");
+  }
+  return res.status(200).json({
+    starterQuestions,
+  });
+};
@@ -1,32 +1,16 @@
 import { Message, StreamData, streamToResponse } from "ai";
 import { Request, Response } from "express";
-import { ChatMessage, MessageContent, Settings } from "llamaindex";
+import { ChatMessage, Settings } from "llamaindex";
 import { createChatEngine } from "./engine/chat";
-import { LlamaIndexStream } from "./llamaindex-stream";
-import { createCallbackManager } from "./stream-helper";
-
-const convertMessageContent = (
-  textMessage: string,
-  imageUrl: string | undefined,
-): MessageContent => {
-  if (!imageUrl) return textMessage;
-  return [
-    {
-      type: "text",
-      text: textMessage,
-    },
-    {
-      type: "image_url",
-      image_url: {
-        url: imageUrl,
-      },
-    },
-  ];
-};
+import { LlamaIndexStream, convertMessageContent } from "./llamaindex-stream";
+import { createCallbackManager, createStreamTimeout } from "./stream-helper";

 export const chat = async (req: Request, res: Response) => {
+  // Init Vercel AI StreamData and timeout
+  const vercelStreamData = new StreamData();
+  const streamTimeout = createStreamTimeout(vercelStreamData);
  try {
-    const { messages, data }: { messages: Message[]; data: any } = req.body;
+    const { messages }: { messages: Message[] } = req.body;
    const userMessage = messages.pop();
    if (!messages || !userMessage || userMessage.role !== "user") {
      return res.status(400).json({
@@ -37,15 +21,25 @@ export const chat = async (req: Request, res: Response) => {

    const chatEngine = await createChatEngine();

+    let annotations = userMessage.annotations;
+    if (!annotations) {
+      // the user didn't send any new annotations with the last message
+      // so use the annotations from the last user message that has annotations
+      // REASON: GPT4 doesn't consider MessageContentDetail from previous messages, only strings
+      annotations = messages
+        .slice()
+        .reverse()
+        .find(
+          (message) => message.role === "user" && message.annotations,
+        )?.annotations;
+    }
+
    // Convert message content from Vercel/AI format to LlamaIndex/OpenAI format
    const userMessageContent = convertMessageContent(
      userMessage.content,
-      data?.imageUrl,
+      annotations,
    );

-    // Init Vercel AI StreamData
-    const vercelStreamData = new StreamData();
-
    // Setup callbacks
    const callbackManager = createCallbackManager(vercelStreamData);

@@ -59,18 +53,15 @@ export const chat = async (req: Request, res: Response) => {
    });

    // Return a stream, which can be consumed by the Vercel/AI client
-    const stream = LlamaIndexStream(response, vercelStreamData, {
-      parserOptions: {
-        image_url: data?.imageUrl,
-      },
-    });
-    const processedStream = stream.pipeThrough(vercelStreamData.stream);
+    const stream = LlamaIndexStream(response, vercelStreamData);

-    return streamToResponse(processedStream, res);
+    return streamToResponse(stream, res, {}, vercelStreamData);
  } catch (error) {
    console.error("[LlamaIndex]", error);
    return res.status(500).json({
      detail: (error as Error).message,
    });
+  } finally {
+    clearTimeout(streamTimeout);
  }
 };
@@ -45,7 +45,9 @@ export const initSettings = async () => {
 function initOpenAI() {
  Settings.llm = new OpenAI({
    model: process.env.MODEL ?? "gpt-3.5-turbo",
-    maxTokens: 512,
+    maxTokens: process.env.LLM_MAX_TOKENS
+      ? Number(process.env.LLM_MAX_TOKENS)
+      : undefined,
  });
  Settings.embedModel = new OpenAIEmbedding({
    model: process.env.EMBEDDING_MODEL,
@@ -56,11 +58,17 @@ function initOpenAI() {
 }

 function initOllama() {
+  const config = {
+    host: process.env.OLLAMA_BASE_URL ?? "http://127.0.0.1:11434",
+  };
+
  Settings.llm = new Ollama({
    model: process.env.MODEL ?? "",
+    config,
  });
  Settings.embedModel = new OllamaEmbedding({
    model: process.env.EMBEDDING_MODEL ?? "",
+    config,
  });
 }

@@ -1,4 +1,5 @@
 import {
+  JSONValue,
  StreamData,
  createCallbacksTransformer,
  createStreamDataTransformer,
@@ -6,6 +7,8 @@ import {
  type AIStreamCallbacksAndOptions,
 } from "ai";
 import {
+  MessageContent,
+  MessageContentDetail,
  Metadata,
  NodeWithScore,
  Response,
@@ -13,29 +16,85 @@ import {
 } from "llamaindex";

 import { AgentStreamChatResponse } from "llamaindex/agent/base";
-import { appendImageData, appendSourceData } from "./stream-helper";
+import { CsvFile, appendSourceData } from "./stream-helper";

 type LlamaIndexResponse =
  | AgentStreamChatResponse<ToolCallLLMMessageOptions>
  | Response;

-type ParserOptions = {
-  image_url?: string;
+export const convertMessageContent = (
+  content: string,
+  annotations?: JSONValue[],
+): MessageContent => {
+  if (!annotations) return content;
+  return [
+    {
+      type: "text",
+      text: content,
+    },
+    ...convertAnnotations(annotations),
+  ];
+};
+
+const convertAnnotations = (
+  annotations: JSONValue[],
+): MessageContentDetail[] => {
+  const content: MessageContentDetail[] = [];
+  annotations.forEach((annotation: JSONValue) => {
+    // first skip invalid annotation
+    if (
+      !(
+        annotation &&
+        typeof annotation === "object" &&
+        "type" in annotation &&
+        "data" in annotation &&
+        annotation.data &&
+        typeof annotation.data === "object"
+      )
+    ) {
+      console.log(
+        "Client sent invalid annotation. Missing data and type",
+        annotation,
+      );
+      return;
+    }
+    const { type, data } = annotation;
+    // convert image
+    if (type === "image" && "url" in data && typeof data.url === "string") {
+      content.push({
+        type: "image_url",
+        image_url: {
+          url: data.url,
+        },
+      });
+    }
+    // convert CSV files to text
+    if (type === "csv" && "csvFiles" in data && Array.isArray(data.csvFiles)) {
+      const rawContents = data.csvFiles.map((csv) => {
+        return "```csv\n" + (csv as CsvFile).content + "\n```";
+      });
+      const csvContent =
+        "Use data from following CSV raw contents:\n" +
+        rawContents.join("\n\n");
+      content.push({
+        type: "text",
+        text: csvContent,
+      });
+    }
+  });
+
+  return content;
 };

 function createParser(
  res: AsyncIterable<LlamaIndexResponse>,
  data: StreamData,
-  opts?: ParserOptions,
 ) {
  const it = res[Symbol.asyncIterator]();
  const trimStartOfStream = trimStartOfStreamHelper();

  let sourceNodes: NodeWithScore<Metadata>[] | undefined;
  return new ReadableStream<string>({
-    start() {
-      appendImageData(data, opts?.image_url);
-    },
    async pull(controller): Promise<void> {
      const { value, done } = await it.next();
      if (done) {
@@ -72,10 +131,9 @@ export function LlamaIndexStream(
  data: StreamData,
  opts?: {
    callbacks?: AIStreamCallbacksAndOptions;
-    parserOptions?: ParserOptions;
  },
 ): ReadableStream<Uint8Array> {
-  return createParser(response, data, opts?.parserOptions)
+  return createParser(response, data)
    .pipeThrough(createCallbacksTransformer(opts?.callbacks))
    .pipeThrough(createStreamDataTransformer());
 }
@@ -7,14 +7,20 @@ import {
  ToolOutput,
 } from "llamaindex";

-export function appendImageData(data: StreamData, imageUrl?: string) {
-  if (!imageUrl) return;
-  data.appendMessageAnnotation({
-    type: "image",
-    data: {
-      url: imageUrl,
-    },
-  });
+function getNodeUrl(metadata: Metadata) {
+  const url = metadata["URL"];
+  if (url) return url;
+  const fileName = metadata["file_name"];
+  if (!process.env.FILESERVER_URL_PREFIX) {
+    console.warn(
+      "FILESERVER_URL_PREFIX is not set. File URLs will not be generated.",
+    );
+    return undefined;
+  }
+  if (fileName) {
+    return `${process.env.FILESERVER_URL_PREFIX}/data/${fileName}`;
+  }
+  return undefined;
 }

 export function appendSourceData(
@@ -29,6 +35,7 @@ export function appendSourceData(
        ...node.node.toMutableJSON(),
        id: node.node.id_,
        score: node.score ?? null,
+        url: getNodeUrl(node.node.metadata),
      })),
    },
  });
@@ -65,6 +72,15 @@ export function appendToolData(
  });
 }

+export function createStreamTimeout(stream: StreamData) {
+  const timeout = Number(process.env.STREAM_TIMEOUT ?? 1000 * 60 * 5); // default to 5 minutes
+  const t = setTimeout(() => {
+    appendEventData(stream, `Stream timed out after ${timeout / 1000} seconds`);
+    stream.close();
+  }, timeout);
+  return t;
+}
+
 export function createCallbackManager(stream: StreamData) {
  const callbackManager = new CallbackManager();

@@ -95,3 +111,10 @@ export function createCallbackManager(stream: StreamData) {

  return callbackManager;
 }
+
+export type CsvFile = {
+  content: string;
+  filename: string;
+  filesize: number;
+  id: string;
+};
@@ -1,4 +1,5 @@
 import express, { Router } from "express";
+import { chatConfig } from "../controllers/chat-config.controller";
 import { chatRequest } from "../controllers/chat-request.controller";
 import { chat } from "../controllers/chat.controller";
 import { initSettings } from "../controllers/engine/settings";
@@ -8,5 +9,6 @@ const llmRouter: Router = express.Router();
 initSettings();
 llmRouter.route("/").post(chat);
 llmRouter.route("/request").post(chatRequest);
+llmRouter.route("/config").get(chatConfig);

 export default llmRouter;
@@ -1,148 +1,114 @@
-from pydantic import BaseModel
-from typing import List, Any, Optional, Dict, Tuple
+import os
+import logging
+
+from aiostream import stream
 from fastapi import APIRouter, Depends, HTTPException, Request, status
 from llama_index.core.chat_engine.types import BaseChatEngine
-from llama_index.core.schema import NodeWithScore
-from llama_index.core.llms import ChatMessage, MessageRole
+from llama_index.core.llms import MessageRole
 from app.engine import get_chat_engine
 from app.api.routers.vercel_response import VercelStreamResponse
-from app.api.routers.messaging import EventCallbackHandler
-from aiostream import stream
+from app.api.routers.events import EventCallbackHandler
+from app.api.routers.models import (
+    ChatData,
+    ChatConfig,
+    SourceNodes,
+    Result,
+    Message,
+)

 chat_router = r = APIRouter()

-
-class _Message(BaseModel):
-    role: MessageRole
-    content: str
-
-
-class _ChatData(BaseModel):
-    messages: List[_Message]
-
-    class Config:
-        json_schema_extra = {
-            "example": {
-                "messages": [
-                    {
-                        "role": "user",
-                        "content": "What standards for letters exist?",
-                    }
-                ]
-            }
-        }
-
-
-class _SourceNodes(BaseModel):
-    id: str
-    metadata: Dict[str, Any]
-    score: Optional[float]
-    text: str
-
-    @classmethod
-    def from_source_node(cls, source_node: NodeWithScore):
-        return cls(
-            id=source_node.node.node_id,
-            metadata=source_node.node.metadata,
-            score=source_node.score,
-            text=source_node.node.text,  # type: ignore
-        )
-
-    @classmethod
-    def from_source_nodes(cls, source_nodes: List[NodeWithScore]):
-        return [cls.from_source_node(node) for node in source_nodes]
-
-
-class _Result(BaseModel):
-    result: _Message
-    nodes: List[_SourceNodes]
-
-
-async def parse_chat_data(data: _ChatData) -> Tuple[str, List[ChatMessage]]:
-    # check preconditions and get last message
-    if len(data.messages) == 0:
-        raise HTTPException(
-            status_code=status.HTTP_400_BAD_REQUEST,
-            detail="No messages provided",
-        )
-    last_message = data.messages.pop()
-    if last_message.role != MessageRole.USER:
-        raise HTTPException(
-            status_code=status.HTTP_400_BAD_REQUEST,
-            detail="Last message must be from user",
-        )
-    # convert messages coming from the request to type ChatMessage
-    messages = [
-        ChatMessage(
-            role=m.role,
-            content=m.content,
-        )
-        for m in data.messages
-    ]
-    return last_message.content, messages
+logger = logging.getLogger("uvicorn")


 # streaming endpoint - delete if not needed
@r.post("")
 async def chat(
    request: Request,
-    data: _ChatData,
+    data: ChatData,
    chat_engine: BaseChatEngine = Depends(get_chat_engine),
 ):
-    last_message_content, messages = await parse_chat_data(data)
+    try:
+        last_message_content = data.get_last_message_content()
+        messages = data.get_history_messages()

-    event_handler = EventCallbackHandler()
-    chat_engine.callback_manager.handlers.append(event_handler)  # type: ignore
-    response = await chat_engine.astream_chat(last_message_content, messages)
+        event_handler = EventCallbackHandler()
+        chat_engine.callback_manager.handlers.append(event_handler)  # type: ignore

-    async def content_generator():
-        # Yield the text response
-        async def _text_generator():
-            async for token in response.async_response_gen():
-                yield VercelStreamResponse.convert_text(token)
-            # the text_generator is the leading stream, once it's finished, also finish the event stream
-            event_handler.is_done = True
+        async def content_generator():
+            # Yield the text response
+            async def _chat_response_generator():
+                response = await chat_engine.astream_chat(
+                    last_message_content, messages
+                )
+                async for token in response.async_response_gen():
+                    yield VercelStreamResponse.convert_text(token)
+                # the text_generator is the leading stream, once it's finished, also finish the event stream
+                event_handler.is_done = True

-        # Yield the events from the event handler
-        async def _event_generator():
-            async for event in event_handler.async_event_gen():
-                event_response = event.to_response()
-                if event_response is not None:
-                    yield VercelStreamResponse.convert_data(event_response)
+                # Yield the source nodes
+                yield VercelStreamResponse.convert_data(
+                    {
+                        "type": "sources",
+                        "data": {
+                            "nodes": [
+                                SourceNodes.from_source_node(node).dict()
+                                for node in response.source_nodes
+                            ]
+                        },
+                    }
+                )

-        combine = stream.merge(_text_generator(), _event_generator())
-        async with combine.stream() as streamer:
-            async for item in streamer:
-                if await request.is_disconnected():
-                    break
-                yield item
+            # Yield the events from the event handler
+            async def _event_generator():
+                async for event in event_handler.async_event_gen():
+                    event_response = event.to_response()
+                    if event_response is not None:
+                        yield VercelStreamResponse.convert_data(event_response)

-        # Yield the source nodes
-        yield VercelStreamResponse.convert_data(
-            {
-                "type": "sources",
-                "data": {
-                    "nodes": [
-                        _SourceNodes.from_source_node(node).dict()
-                        for node in response.source_nodes
-                    ]
-                },
-            }
-        )
+            combine = stream.merge(_chat_response_generator(), _event_generator())
+            is_stream_started = False
+            async with combine.stream() as streamer:
+                async for output in streamer:
+                    if not is_stream_started:
+                        is_stream_started = True
+                        # Stream a blank message to start the stream
+                        yield VercelStreamResponse.convert_text("")

-    return VercelStreamResponse(content=content_generator())
+                    yield output
+
+                    if await request.is_disconnected():
+                        break
+
+        return VercelStreamResponse(content=content_generator())
+    except Exception as e:
+        logger.exception("Error in chat engine", exc_info=True)
+        raise HTTPException(
+            status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+            detail=f"Error in chat engine: {e}",
+        ) from e


 # non-streaming endpoint - delete if not needed
@r.post("/request")
 async def chat_request(
-    data: _ChatData,
+    data: ChatData,
    chat_engine: BaseChatEngine = Depends(get_chat_engine),
-) -> _Result:
-    last_message_content, messages = await parse_chat_data(data)
+) -> Result:
+    last_message_content = data.get_last_message_content()
+    messages = data.get_history_messages()

    response = await chat_engine.achat(last_message_content, messages)
-    return _Result(
-        result=_Message(role=MessageRole.ASSISTANT, content=response.response),
-        nodes=_SourceNodes.from_source_nodes(response.source_nodes),
+    return Result(
+        result=Message(role=MessageRole.ASSISTANT, content=response.response),
+        nodes=SourceNodes.from_source_nodes(response.source_nodes),
    )
+
+
+@r.get("/config")
+async def chat_config() -> ChatConfig:
+    starter_questions = None
+    conversation_starters = os.getenv("CONVERSATION_STARTERS")
+    if conversation_starters and conversation_starters.strip():
+        starter_questions = conversation_starters.strip().split("\n")
+    return ChatConfig(starterQuestions=starter_questions)
@@ -1,5 +1,6 @@
 import json
 import asyncio
+import logging
 from typing import AsyncGenerator, Dict, Any, List, Optional
 from llama_index.core.callbacks.base import BaseCallbackHandler
 from llama_index.core.callbacks.schema import CBEventType
@@ -7,6 +8,9 @@ from llama_index.core.tools.types import ToolOutput
 from pydantic import BaseModel


+logger = logging.getLogger(__name__)
+
+
 class CallbackEvent(BaseModel):
    event_type: CBEventType
    payload: Optional[Dict[str, Any]] = None
@@ -72,15 +76,19 @@ class CallbackEvent(BaseModel):
                    }

    def to_response(self):
-        match self.event_type:
-            case "retrieve":
-                return self.get_retrieval_message()
-            case "function_call":
-                return self.get_tool_message()
-            case "agent_step":
-                return self.get_agent_tool_response()
-            case _:
-                return None
+        try:
+            match self.event_type:
+                case "retrieve":
+                    return self.get_retrieval_message()
+                case "function_call":
+                    return self.get_tool_message()
+                case "agent_step":
+                    return self.get_agent_tool_response()
+                case _:
+                    return None
+        except Exception as e:
+            logger.error(f"Error in converting event to response: {e}")
+            return None


 class EventCallbackHandler(BaseCallbackHandler):
@@ -0,0 +1,170 @@
+import os
+import logging
+from pydantic import BaseModel, Field, validator
+from pydantic.alias_generators import to_camel
+from typing import List, Any, Optional, Dict
+from llama_index.core.schema import NodeWithScore
+from llama_index.core.llms import ChatMessage, MessageRole
+
+
+logger = logging.getLogger("uvicorn")
+
+
+class CsvFile(BaseModel):
+    content: str
+    filename: str
+    filesize: int
+    id: str
+
+
+class AnnotationData(BaseModel):
+    csv_files: List[CsvFile] | None = Field(
+        default=None,
+        description="List of CSV files",
+    )
+
+    class Config:
+        json_schema_extra = {
+            "example": {
+                "csvFiles": [
+                    {
+                        "content": "Name, Age\nAlice, 25\nBob, 30",
+                        "filename": "example.csv",
+                        "filesize": 123,
+                        "id": "123",
+                        "type": "text/csv",
+                    }
+                ]
+            }
+        }
+        alias_generator = to_camel
+
+
+class Annotation(BaseModel):
+    type: str
+    data: AnnotationData
+
+    def to_content(self) -> str:
+        if self.type == "csv":
+            csv_files = self.data.csv_files
+            if csv_files is not None and len(csv_files) > 0:
+                return "Use data from following CSV raw contents\n" + "\n".join(
+                    [f"```csv\n{csv_file.content}\n```" for csv_file in csv_files]
+                )
+        raise ValueError(f"Unsupported annotation type: {self.type}")
+
+
+class Message(BaseModel):
+    role: MessageRole
+    content: str
+    annotations: List[Annotation] | None = None
+
+
+class ChatData(BaseModel):
+    messages: List[Message]
+
+    class Config:
+        json_schema_extra = {
+            "example": {
+                "messages": [
+                    {
+                        "role": "user",
+                        "content": "What standards for letters exist?",
+                    }
+                ]
+            }
+        }
+
+    @validator("messages")
+    def messages_must_not_be_empty(cls, v):
+        if len(v) == 0:
+            raise ValueError("Messages must not be empty")
+        return v
+
+    def get_last_message_content(self) -> str:
+        """
+        Get the content of the last message along with the data content if available. Fallback to use data content from previous messages
+        """
+        if len(self.messages) == 0:
+            raise ValueError("There is not any message in the chat")
+        last_message = self.messages[-1]
+        message_content = last_message.content
+        for message in reversed(self.messages):
+            if message.role == MessageRole.USER and message.annotations is not None:
+                annotation_contents = (
+                    annotation.to_content() for annotation in message.annotations
+                )
+                annotation_text = "\n".join(annotation_contents)
+                message_content = f"{message_content}\n{annotation_text}"
+                break
+        return message_content
+
+    def get_history_messages(self) -> List[Message]:
+        """
+        Get the history messages
+        """
+        return [
+            ChatMessage(role=message.role, content=message.content)
+            for message in self.messages[:-1]
+        ]
+
+    def is_last_message_from_user(self) -> bool:
+        return self.messages[-1].role == MessageRole.USER
+
+
+class SourceNodes(BaseModel):
+    id: str
+    metadata: Dict[str, Any]
+    score: Optional[float]
+    text: str
+    url: Optional[str]
+
+    @classmethod
+    def from_source_node(cls, source_node: NodeWithScore):
+        metadata = source_node.node.metadata
+        url = metadata.get("URL")
+
+        if not url:
+            file_name = metadata.get("file_name")
+            url_prefix = os.getenv("FILESERVER_URL_PREFIX")
+            if not url_prefix:
+                logger.warning(
+                    "Warning: FILESERVER_URL_PREFIX not set in environment variables"
+                )
+            if file_name and url_prefix:
+                url = f"{url_prefix}/data/{file_name}"
+
+        return cls(
+            id=source_node.node.node_id,
+            metadata=metadata,
+            score=source_node.score,
+            text=source_node.node.text,  # type: ignore
+            url=url,
+        )
+
+    @classmethod
+    def from_source_nodes(cls, source_nodes: List[NodeWithScore]):
+        return [cls.from_source_node(node) for node in source_nodes]
+
+
+class Result(BaseModel):
+    result: Message
+    nodes: List[SourceNodes]
+
+
+class ChatConfig(BaseModel):
+    starter_questions: Optional[List[str]] = Field(
+        default=None,
+        description="List of starter questions",
+    )
+
+    class Config:
+        json_schema_extra = {
+            "example": {
+                "starterQuestions": [
+                    "What standards for letters exist?",
+                    "What are the requirements for a letter to be considered a letter?",
+                ]
+            }
+        }
+        alias_generator = to_camel
@@ -5,16 +5,19 @@ from llama_index.core.settings import Settings

 def init_settings():
    model_provider = os.getenv("MODEL_PROVIDER")
-    if model_provider == "openai":
-        init_openai()
-    elif model_provider == "ollama":
-        init_ollama()
-    elif model_provider == "anthropic":
-        init_anthropic()
-    elif model_provider == "gemini":
-        init_gemini()
-    else:
-        raise ValueError(f"Invalid model provider: {model_provider}")
+    match model_provider:
+        case "openai":
+            init_openai()
+        case "ollama":
+            init_ollama()
+        case "anthropic":
+            init_anthropic()
+        case "gemini":
+            init_gemini()
+        case "azure-openai":
+            init_azure_openai()
+        case _:
+            raise ValueError(f"Invalid model provider: {model_provider}")
    Settings.chunk_size = int(os.getenv("CHUNK_SIZE", "1024"))
    Settings.chunk_overlap = int(os.getenv("CHUNK_OVERLAP", "20"))

@@ -23,8 +26,12 @@ def init_ollama():
    from llama_index.llms.ollama import Ollama
    from llama_index.embeddings.ollama import OllamaEmbedding

-    Settings.embed_model = OllamaEmbedding(model_name=os.getenv("EMBEDDING_MODEL"))
-    Settings.llm = Ollama(model=os.getenv("MODEL"))
+    base_url = os.getenv("OLLAMA_BASE_URL") or "http://127.0.0.1:11434"
+    Settings.embed_model = OllamaEmbedding(
+        base_url=base_url,
+        model_name=os.getenv("EMBEDDING_MODEL"),
+    )
+    Settings.llm = Ollama(base_url=base_url, model=os.getenv("MODEL"))


 def init_openai():
@@ -48,6 +55,34 @@ def init_openai():
    Settings.embed_model = OpenAIEmbedding(**config)


+def init_azure_openai():
+    from llama_index.llms.azure_openai import AzureOpenAI
+    from llama_index.embeddings.azure_openai import AzureOpenAIEmbedding
+    from llama_index.core.constants import DEFAULT_TEMPERATURE
+
+    llm_deployment = os.getenv("AZURE_OPENAI_LLM_DEPLOYMENT")
+    embedding_deployment = os.getenv("AZURE_OPENAI_EMBEDDING_DEPLOYMENT")
+    max_tokens = os.getenv("LLM_MAX_TOKENS")
+    api_key = os.getenv("AZURE_OPENAI_API_KEY")
+    llm_config = {
+        "api_key": api_key,
+        "deployment_name": llm_deployment,
+        "model": os.getenv("MODEL"),
+        "temperature": float(os.getenv("LLM_TEMPERATURE", DEFAULT_TEMPERATURE)),
+        "max_tokens": int(max_tokens) if max_tokens is not None else None,
+    }
+    Settings.llm = AzureOpenAI(**llm_config)
+
+    dimensions = os.getenv("EMBEDDING_DIM")
+    embedding_config = {
+        "api_key": api_key,
+        "deployment_name": embedding_deployment,
+        "model": os.getenv("EMBEDDING_MODEL"),
+        "dimensions": int(dimensions) if dimensions is not None else None,
+    }
+    Settings.embed_model = AzureOpenAIEmbedding(**embedding_config)
+
+
 def init_anthropic():
    from llama_index.llms.anthropic import Anthropic
    from llama_index.embeddings.huggingface import HuggingFaceEmbedding
@@ -1,3 +1,3 @@
 __pycache__
 storage
-.env
+.env
@@ -11,6 +11,7 @@ from fastapi.responses import RedirectResponse
 from app.api.routers.chat import chat_router
 from app.settings import init_settings
 from app.observability import init_observability
+from fastapi.staticfiles import StaticFiles


 app = FastAPI()
@@ -20,7 +21,6 @@ init_observability()

 environment = os.getenv("ENVIRONMENT", "dev")  # Default to 'development' if not set

-
 if environment == "dev":
    logger = logging.getLogger("uvicorn")
    logger.warning("Running in development mode - allowing CORS for all origins")
@@ -38,6 +38,16 @@ if environment == "dev":
        return RedirectResponse(url="/docs")


+def mount_static_files(directory, path):
+    if os.path.exists(directory):
+        app.mount(path, StaticFiles(directory=directory), name=f"{directory}-static")
+
+
+# Mount the data files to serve the file viewer
+mount_static_files("data", "/api/files/data")
+# Mount the output files from tools
+mount_static_files("tool-output", "/api/files/tool-output")
+
 app.include_router(chat_router, prefix="/api/chat")


@@ -14,8 +14,9 @@ fastapi = "^0.109.1"
 uvicorn = { extras = ["standard"], version = "^0.23.2" }
 python-dotenv = "^1.0.0"
 aiostream = "^0.5.2"
-llama-index = "0.10.28"
-llama-index-core = "0.10.28"
+llama-index = "0.10.41"
+llama-index-core = "0.10.41"
+cachetools = "^5.3.3"

 [build-system]
 requires = ["poetry-core"]
@@ -0,0 +1,11 @@
+import { NextResponse } from "next/server";
+
+/**
+ * This API is to get config from the backend envs and expose them to the frontend
+ */
+export async function GET() {
+  const config = {
+    starterQuestions: process.env.CONVERSATION_STARTERS?.trim().split("\n"),
+  };
+  return NextResponse.json(config, { status: 200 });
+}
@@ -45,7 +45,9 @@ export const initSettings = async () => {
 function initOpenAI() {
  Settings.llm = new OpenAI({
    model: process.env.MODEL ?? "gpt-3.5-turbo",
-    maxTokens: 512,
+    maxTokens: process.env.LLM_MAX_TOKENS
+      ? Number(process.env.LLM_MAX_TOKENS)
+      : undefined,
  });
  Settings.embedModel = new OpenAIEmbedding({
    model: process.env.EMBEDDING_MODEL,
@@ -56,11 +58,16 @@ function initOpenAI() {
 }

 function initOllama() {
+  const config = {
+    host: process.env.OLLAMA_BASE_URL ?? "http://127.0.0.1:11434",
+  };
  Settings.llm = new Ollama({
    model: process.env.MODEL ?? "",
+    config,
  });
  Settings.embedModel = new OllamaEmbedding({
    model: process.env.EMBEDDING_MODEL ?? "",
+    config,
  });
 }

@@ -1,4 +1,5 @@
 import {
+  JSONValue,
  StreamData,
  createCallbacksTransformer,
  createStreamDataTransformer,
@@ -6,6 +7,8 @@ import {
  type AIStreamCallbacksAndOptions,
 } from "ai";
 import {
+  MessageContent,
+  MessageContentDetail,
  Metadata,
  NodeWithScore,
  Response,
@@ -13,29 +16,85 @@ import {
 } from "llamaindex";

 import { AgentStreamChatResponse } from "llamaindex/agent/base";
-import { appendImageData, appendSourceData } from "./stream-helper";
+import { CsvFile, appendSourceData } from "./stream-helper";

 type LlamaIndexResponse =
  | AgentStreamChatResponse<ToolCallLLMMessageOptions>
  | Response;

-type ParserOptions = {
-  image_url?: string;
+export const convertMessageContent = (
+  content: string,
+  annotations?: JSONValue[],
+): MessageContent => {
+  if (!annotations) return content;
+  return [
+    {
+      type: "text",
+      text: content,
+    },
+    ...convertAnnotations(annotations),
+  ];
+};
+
+const convertAnnotations = (
+  annotations: JSONValue[],
+): MessageContentDetail[] => {
+  const content: MessageContentDetail[] = [];
+  annotations.forEach((annotation: JSONValue) => {
+    // first skip invalid annotation
+    if (
+      !(
+        annotation &&
+        typeof annotation === "object" &&
+        "type" in annotation &&
+        "data" in annotation &&
+        annotation.data &&
+        typeof annotation.data === "object"
+      )
+    ) {
+      console.log(
+        "Client sent invalid annotation. Missing data and type",
+        annotation,
+      );
+      return;
+    }
+    const { type, data } = annotation;
+    // convert image
+    if (type === "image" && "url" in data && typeof data.url === "string") {
+      content.push({
+        type: "image_url",
+        image_url: {
+          url: data.url,
+        },
+      });
+    }
+    // convert CSV files to text
+    if (type === "csv" && "csvFiles" in data && Array.isArray(data.csvFiles)) {
+      const rawContents = data.csvFiles.map((csv) => {
+        return "```csv\n" + (csv as CsvFile).content + "\n```";
+      });
+      const csvContent =
+        "Use data from following CSV raw contents:\n" +
+        rawContents.join("\n\n");
+      content.push({
+        type: "text",
+        text: csvContent,
+      });
+    }
+  });
+
+  return content;
 };

 function createParser(
  res: AsyncIterable<LlamaIndexResponse>,
  data: StreamData,
-  opts?: ParserOptions,
 ) {
  const it = res[Symbol.asyncIterator]();
  const trimStartOfStream = trimStartOfStreamHelper();

  let sourceNodes: NodeWithScore<Metadata>[] | undefined;
  return new ReadableStream<string>({
-    start() {
-      appendImageData(data, opts?.image_url);
-    },
    async pull(controller): Promise<void> {
      const { value, done } = await it.next();
      if (done) {
@@ -72,10 +131,9 @@ export function LlamaIndexStream(
  data: StreamData,
  opts?: {
    callbacks?: AIStreamCallbacksAndOptions;
-    parserOptions?: ParserOptions;
  },
 ): ReadableStream<Uint8Array> {
-  return createParser(response, data, opts?.parserOptions)
+  return createParser(response, data)
    .pipeThrough(createCallbacksTransformer(opts?.callbacks))
    .pipeThrough(createStreamDataTransformer());
 }
@@ -1,11 +1,11 @@
 import { initObservability } from "@/app/observability";
 import { Message, StreamData, StreamingTextResponse } from "ai";
-import { ChatMessage, MessageContent, Settings } from "llamaindex";
+import { ChatMessage, Settings } from "llamaindex";
 import { NextRequest, NextResponse } from "next/server";
 import { createChatEngine } from "./engine/chat";
 import { initSettings } from "./engine/settings";
-import { LlamaIndexStream } from "./llamaindex-stream";
-import { createCallbackManager } from "./stream-helper";
+import { LlamaIndexStream, convertMessageContent } from "./llamaindex-stream";
+import { createCallbackManager, createStreamTimeout } from "./stream-helper";

 initObservability();
 initSettings();
@@ -13,29 +13,14 @@ initSettings();
 export const runtime = "nodejs";
 export const dynamic = "force-dynamic";

-const convertMessageContent = (
-  textMessage: string,
-  imageUrl: string | undefined,
-): MessageContent => {
-  if (!imageUrl) return textMessage;
-  return [
-    {
-      type: "text",
-      text: textMessage,
-    },
-    {
-      type: "image_url",
-      image_url: {
-        url: imageUrl,
-      },
-    },
-  ];
-};
-
 export async function POST(request: NextRequest) {
+  // Init Vercel AI StreamData and timeout
+  const vercelStreamData = new StreamData();
+  const streamTimeout = createStreamTimeout(vercelStreamData);
+
  try {
    const body = await request.json();
-    const { messages, data }: { messages: Message[]; data: any } = body;
+    const { messages }: { messages: Message[] } = body;
    const userMessage = messages.pop();
    if (!messages || !userMessage || userMessage.role !== "user") {
      return NextResponse.json(
@@ -49,15 +34,25 @@ export async function POST(request: NextRequest) {

    const chatEngine = await createChatEngine();

+    let annotations = userMessage.annotations;
+    if (!annotations) {
+      // the user didn't send any new annotations with the last message
+      // so use the annotations from the last user message that has annotations
+      // REASON: GPT4 doesn't consider MessageContentDetail from previous messages, only strings
+      annotations = messages
+        .slice()
+        .reverse()
+        .find(
+          (message) => message.role === "user" && message.annotations,
+        )?.annotations;
+    }
+
    // Convert message content from Vercel/AI format to LlamaIndex/OpenAI format
    const userMessageContent = convertMessageContent(
      userMessage.content,
-      data?.imageUrl,
+      annotations,
    );

-    // Init Vercel AI StreamData
-    const vercelStreamData = new StreamData();
-
    // Setup callbacks
    const callbackManager = createCallbackManager(vercelStreamData);

@@ -71,11 +66,7 @@ export async function POST(request: NextRequest) {
    });

    // Transform LlamaIndex stream to Vercel/AI format
-    const stream = LlamaIndexStream(response, vercelStreamData, {
-      parserOptions: {
-        image_url: data?.imageUrl,
-      },
-    });
+    const stream = LlamaIndexStream(response, vercelStreamData);

    // Return a StreamingTextResponse, which can be consumed by the Vercel/AI client
    return new StreamingTextResponse(stream, {}, vercelStreamData);
@@ -89,5 +80,7 @@ export async function POST(request: NextRequest) {
        status: 500,
      },
    );
+  } finally {
+    clearTimeout(streamTimeout);
  }
 }
@@ -7,14 +7,20 @@ import {
  ToolOutput,
 } from "llamaindex";

-export function appendImageData(data: StreamData, imageUrl?: string) {
-  if (!imageUrl) return;
-  data.appendMessageAnnotation({
-    type: "image",
-    data: {
-      url: imageUrl,
-    },
-  });
+function getNodeUrl(metadata: Metadata) {
+  const url = metadata["URL"];
+  if (url) return url;
+  const fileName = metadata["file_name"];
+  if (!process.env.FILESERVER_URL_PREFIX) {
+    console.warn(
+      "FILESERVER_URL_PREFIX is not set. File URLs will not be generated.",
+    );
+    return undefined;
+  }
+  if (fileName) {
+    return `${process.env.FILESERVER_URL_PREFIX}/data/${fileName}`;
+  }
+  return undefined;
 }

 export function appendSourceData(
@@ -29,6 +35,7 @@ export function appendSourceData(
        ...node.node.toMutableJSON(),
        id: node.node.id_,
        score: node.score ?? null,
+        url: getNodeUrl(node.node.metadata),
      })),
    },
  });
@@ -65,6 +72,15 @@ export function appendToolData(
  });
 }

+export function createStreamTimeout(stream: StreamData) {
+  const timeout = Number(process.env.STREAM_TIMEOUT ?? 1000 * 60 * 5); // default to 5 minutes
+  const t = setTimeout(() => {
+    appendEventData(stream, `Stream timed out after ${timeout / 1000} seconds`);
+    stream.close();
+  }, timeout);
+  return t;
+}
+
 export function createCallbackManager(stream: StreamData) {
  const callbackManager = new CallbackManager();

@@ -95,3 +111,10 @@ export function createCallbackManager(stream: StreamData) {

  return callbackManager;
 }
+
+export type CsvFile = {
+  content: string;
+  filename: string;
+  filesize: number;
+  id: string;
+};
@@ -0,0 +1,45 @@
+import { readFile } from "fs/promises";
+import { NextRequest, NextResponse } from "next/server";
+import path from "path";
+
+/**
+ * This API is to get file data from allowed folders
+ * It receives path slug and response file data like serve static file
+ */
+export async function GET(
+  _request: NextRequest,
+  { params }: { params: { slug: string[] } },
+) {
+  const slug = params.slug;
+
+  if (!slug) {
+    return NextResponse.json({ detail: "Missing file slug" }, { status: 400 });
+  }
+
+  if (slug.includes("..") || path.isAbsolute(path.join(...slug))) {
+    return NextResponse.json({ detail: "Invalid file path" }, { status: 400 });
+  }
+
+  const [folder, ...pathTofile] = params.slug; // data, file.pdf
+  const allowedFolders = ["data", "tool-output"];
+
+  if (!allowedFolders.includes(folder)) {
+    return NextResponse.json({ detail: "No permission" }, { status: 400 });
+  }
+
+  try {
+    const filePath = path.join(process.cwd(), folder, path.join(...pathTofile));
+    const blob = await readFile(filePath);
+
+    return new NextResponse(blob, {
+      status: 200,
+      statusText: "OK",
+      headers: {
+        "Content-Length": blob.byteLength.toString(),
+      },
+    });
+  } catch (error) {
+    console.error(error);
+    return NextResponse.json({ detail: "File not found" }, { status: 404 });
+  }
+}
@@ -2,8 +2,10 @@

 import { useChat } from "ai/react";
 import { ChatInput, ChatMessages } from "./ui/chat";
+import { useClientConfig } from "./ui/chat/use-config";

 export default function ChatSection() {
+  const { chatAPI } = useClientConfig();
  const {
    messages,
    input,
@@ -12,8 +14,10 @@ export default function ChatSection() {
    handleInputChange,
    reload,
    stop,
+    append,
+    setInput,
  } = useChat({
-    api: process.env.NEXT_PUBLIC_CHAT_API,
+    api: chatAPI,
    headers: {
      "Content-Type": "application/json", // using JSON because of vercel/ai 2.2.26
    },
@@ -31,13 +35,16 @@ export default function ChatSection() {
        isLoading={isLoading}
        reload={reload}
        stop={stop}
+        append={append}
      />
      <ChatInput
        input={input}
        handleSubmit={handleSubmit}
        handleInputChange={handleInputChange}
        isLoading={isLoading}
-        multiModal={true}
+        messages={messages}
+        append={append}
+        setInput={setInput}
      />
    </div>
  );
@@ -7,7 +7,7 @@ export default function Header() {
        Get started by editing&nbsp;
        <code className="font-mono font-bold">app/page.tsx</code>
      </p>
-      <div className="fixed bottom-0 left-0 flex h-48 w-full items-end justify-center bg-gradient-to-t from-white via-white dark:from-black dark:via-black lg:static lg:h-auto lg:w-auto lg:bg-none">
+      <div className="fixed bottom-0 left-0 mb-4 flex h-auto w-full items-end justify-center bg-gradient-to-t from-white via-white dark:from-black dark:via-black lg:static lg:w-auto lg:bg-none lg:mb-0">
        <a
          href="https://www.llamaindex.ai/"
          className="flex items-center justify-center font-nunito text-lg font-bold gap-2"
@@ -38,7 +38,9 @@ export function ChatEvents({
        <CollapsibleContent asChild>
          <div className="mt-4 text-sm space-y-2">
            {data.map((eventItem, index) => (
-              <div key={index}>{eventItem.title}</div>
+              <div className="whitespace-break-spaces" key={index}>
+                {eventItem.title}
+              </div>
            ))}
          </div>
        </CollapsibleContent>
@@ -1,9 +1,14 @@
+import { JSONValue } from "ai";
 import { useState } from "react";
+import { v4 as uuidv4 } from "uuid";
+import { MessageAnnotation, MessageAnnotationType } from ".";
 import { Button } from "../button";
 import FileUploader from "../file-uploader";
 import { Input } from "../input";
+import UploadCsvPreview from "../upload-csv-preview";
 import UploadImagePreview from "../upload-image-preview";
 import { ChatHandler } from "./chat.interface";
+import { useCsv } from "./use-csv";

 export default function ChatInput(
  props: Pick<
@@ -14,18 +19,61 @@ export default function ChatInput(
    | "onFileError"
    | "handleSubmit"
    | "handleInputChange"
-  > & {
-    multiModal?: boolean;
-  },
+    | "messages"
+    | "setInput"
+    | "append"
+  >,
 ) {
  const [imageUrl, setImageUrl] = useState<string | null>(null);
+  const { files: csvFiles, upload, remove, reset } = useCsv();
+
+  const getAnnotations = () => {
+    if (!imageUrl && csvFiles.length === 0) return undefined;
+    const annotations: MessageAnnotation[] = [];
+    if (imageUrl) {
+      annotations.push({
+        type: MessageAnnotationType.IMAGE,
+        data: { url: imageUrl },
+      });
+    }
+    if (csvFiles.length > 0) {
+      annotations.push({
+        type: MessageAnnotationType.CSV,
+        data: {
+          csvFiles: csvFiles.map((file) => ({
+            id: file.id,
+            content: file.content,
+            filename: file.filename,
+            filesize: file.filesize,
+          })),
+        },
+      });
+    }
+    return annotations as JSONValue[];
+  };
+
+  // default submit function does not handle including annotations in the message
+  // so we need to use append function to submit new message with annotations
+  const handleSubmitWithAnnotations = (
+    e: React.FormEvent<HTMLFormElement>,
+    annotations: JSONValue[] | undefined,
+  ) => {
+    e.preventDefault();
+    props.append!({
+      content: props.input,
+      role: "user",
+      createdAt: new Date(),
+      annotations,
+    });
+    props.setInput!("");
+  };

  const onSubmit = (e: React.FormEvent<HTMLFormElement>) => {
-    if (imageUrl) {
-      props.handleSubmit(e, {
-        data: { imageUrl: imageUrl },
-      });
-      setImageUrl(null);
+    const annotations = getAnnotations();
+    if (annotations) {
+      handleSubmitWithAnnotations(e, annotations);
+      imageUrl && setImageUrl(null);
+      csvFiles.length && reset();
      return;
    }
    props.handleSubmit(e);
@@ -43,11 +91,36 @@ export default function ChatInput(
    setImageUrl(base64);
  };

+  const handleUploadCsvFile = async (file: File) => {
+    const content = await new Promise<string>((resolve, reject) => {
+      const reader = new FileReader();
+      reader.readAsText(file);
+      reader.onload = () => resolve(reader.result as string);
+      reader.onerror = (error) => reject(error);
+    });
+    const isSuccess = upload({
+      id: uuidv4(),
+      content,
+      filename: file.name,
+      filesize: file.size,
+    });
+    if (!isSuccess) {
+      alert("File already exists in the list.");
+    }
+  };
+
  const handleUploadFile = async (file: File) => {
    try {
-      if (props.multiModal && file.type.startsWith("image/")) {
+      if (file.type.startsWith("image/")) {
        return await handleUploadImageFile(file);
      }
+      if (file.type === "text/csv") {
+        if (csvFiles.length > 0) {
+          alert("You can only upload one csv file at a time.");
+          return;
+        }
+        return await handleUploadCsvFile(file);
+      }
      props.onFileUpload?.(file);
    } catch (error: any) {
      props.onFileError?.(error.message);
@@ -62,6 +135,19 @@ export default function ChatInput(
      {imageUrl && (
        <UploadImagePreview url={imageUrl} onRemove={onRemovePreviewImage} />
      )}
+      {csvFiles.length > 0 && (
+        <div className="flex gap-4 w-full overflow-auto py-2">
+          {csvFiles.map((csv) => {
+            return (
+              <UploadCsvPreview
+                key={csv.id}
+                csv={csv}
+                onRemove={() => remove(csv)}
+              />
+            );
+          })}
+        </div>
+      )}
      <div className="flex w-full items-start justify-between gap-4 ">
        <Input
          autoFocus
@@ -75,7 +161,7 @@ export default function ChatInput(
          onFileUpload={handleUploadFile}
          onFileError={props.onFileError}
        />
-        <Button type="submit" disabled={props.isLoading}>
+        <Button type="submit" disabled={props.isLoading || !props.input.trim()}>
          Send message
        </Button>
      </div>
@@ -8,14 +8,16 @@ import { ChatEvents } from "./chat-events";
 import { ChatImage } from "./chat-image";
 import { ChatSources } from "./chat-sources";
 import ChatTools from "./chat-tools";
+import CsvContent from "./csv-content";
 import {
-  AnnotationData,
+  CsvData,
  EventData,
  ImageData,
  MessageAnnotation,
  MessageAnnotationType,
  SourceData,
  ToolData,
+  getAnnotationData,
 } from "./index";
 import Markdown from "./markdown";
 import { useCopyToClipboard } from "./use-copy-to-clipboard";
@@ -25,13 +27,6 @@ type ContentDisplayConfig = {
  component: JSX.Element | null;
 };

-function getAnnotationData<T extends AnnotationData>(
-  annotations: MessageAnnotation[],
-  type: MessageAnnotationType,
-): T[] {
-  return annotations.filter((a) => a.type === type).map((a) => a.data as T);
-}
-
 function ChatMessageContent({
  message,
  isLoading,
@@ -46,6 +41,10 @@ function ChatMessageContent({
    annotations,
    MessageAnnotationType.IMAGE,
  );
+  const csvData = getAnnotationData<CsvData>(
+    annotations,
+    MessageAnnotationType.CSV,
+  );
  const eventData = getAnnotationData<EventData>(
    annotations,
    MessageAnnotationType.EVENTS,
@@ -61,16 +60,20 @@ function ChatMessageContent({

  const contents: ContentDisplayConfig[] = [
    {
-      order: -3,
+      order: 1,
      component: imageData[0] ? <ChatImage data={imageData[0]} /> : null,
    },
    {
-      order: -2,
+      order: -3,
      component:
        eventData.length > 0 ? (
          <ChatEvents isLoading={isLoading} data={eventData} />
        ) : null,
    },
+    {
+      order: 2,
+      component: csvData[0] ? <CsvContent data={csvData[0]} /> : null,
+    },
    {
      order: -1,
      component: toolData[0] ? <ChatTools data={toolData[0]} /> : null,
@@ -80,7 +83,7 @@ function ChatMessageContent({
      component: <Markdown content={message.content} />,
    },
    {
-      order: 1,
+      order: 3,
      component: sourceData[0] ? <ChatSources data={sourceData[0]} /> : null,
    },
  ];
@@ -1,13 +1,19 @@
 import { Loader2 } from "lucide-react";
 import { useEffect, useRef } from "react";

+import { Button } from "../button";
 import ChatActions from "./chat-actions";
 import ChatMessage from "./chat-message";
 import { ChatHandler } from "./chat.interface";
+import { useClientConfig } from "./use-config";

 export default function ChatMessages(
-  props: Pick<ChatHandler, "messages" | "isLoading" | "reload" | "stop">,
+  props: Pick<
+    ChatHandler,
+    "messages" | "isLoading" | "reload" | "stop" | "append"
+  >,
 ) {
+  const { starterQuestions } = useClientConfig();
  const scrollableChatContainerRef = useRef<HTMLDivElement>(null);
  const messageLength = props.messages.length;
  const lastMessage = props.messages[messageLength - 1];
@@ -35,7 +41,7 @@ export default function ChatMessages(
  }, [messageLength, lastMessage]);

  return (
-    <div className="w-full rounded-xl bg-white p-4 shadow-xl pb-0">
+    <div className="w-full rounded-xl bg-white p-4 shadow-xl pb-0 relative">
      <div
        className="flex h-[50vh] flex-col gap-5 divide-y overflow-y-auto pb-4"
        ref={scrollableChatContainerRef}
@@ -64,6 +70,23 @@ export default function ChatMessages(
          showStop={showStop}
        />
      </div>
+      {!messageLength && starterQuestions?.length && props.append && (
+        <div className="absolute bottom-6 left-0 w-full">
+          <div className="grid grid-cols-2 gap-2 mx-20">
+            {starterQuestions.map((question, i) => (
+              <Button
+                variant="outline"
+                key={i}
+                onClick={() =>
+                  props.append!({ role: "user", content: question })
+                }
+              >
+                {question}
+              </Button>
+            ))}
+          </div>
+        </div>
+      )}
    </div>
  );
 }
@@ -1,20 +1,46 @@
-import { ArrowUpRightSquare, Check, Copy } from "lucide-react";
+import { Check, Copy } from "lucide-react";
 import { useMemo } from "react";
 import { Button } from "../button";
 import { HoverCard, HoverCardContent, HoverCardTrigger } from "../hover-card";
-import { SourceData, SourceNode } from "./index";
+import { SourceData } from "./index";
 import { useCopyToClipboard } from "./use-copy-to-clipboard";
+import PdfDialog from "./widgets/PdfDialog";

-const SCORE_THRESHOLD = 0.5;
+const SCORE_THRESHOLD = 0.3;
+
+function SourceNumberButton({ index }: { index: number }) {
+  return (
+    <div className="text-xs w-5 h-5 rounded-full bg-gray-100 mb-2 flex items-center justify-center hover:text-white hover:bg-primary hover:cursor-pointer">
+      {index + 1}
+    </div>
+  );
+}
+
+type NodeInfo = {
+  id: string;
+  url?: string;
+};

 export function ChatSources({ data }: { data: SourceData }) {
-  const sources = useMemo(() => {
-    return (
-      data.nodes
-        ?.filter((node) => Object.keys(node.metadata).length > 0)
-        ?.filter((node) => (node.score ?? 1) > SCORE_THRESHOLD)
-        .sort((a, b) => (b.score ?? 1) - (a.score ?? 1)) || []
-    );
+  const sources: NodeInfo[] = useMemo(() => {
+    // aggregate nodes by url or file_path (get the highest one by score)
+    const nodesByPath: { [path: string]: NodeInfo } = {};
+
+    data.nodes
+      .filter((node) => (node.score ?? 1) > SCORE_THRESHOLD)
+      .sort((a, b) => (b.score ?? 1) - (a.score ?? 1))
+      .forEach((node) => {
+        const nodeInfo = {
+          id: node.id,
+          url: node.url,
+        };
+        const key = nodeInfo.url ?? nodeInfo.id; // use id as key for UNKNOWN type
+        if (!nodesByPath[key]) {
+          nodesByPath[key] = nodeInfo;
+        }
+      });
+
+    return Object.values(nodesByPath);
  }, [data.nodes]);

  if (sources.length === 0) return null;
@@ -23,55 +49,51 @@ export function ChatSources({ data }: { data: SourceData }) {
    <div className="space-x-2 text-sm">
      <span className="font-semibold">Sources:</span>
      <div className="inline-flex gap-1 items-center">
-        {sources.map((node: SourceNode, index: number) => (
-          <div key={node.id}>
-            <HoverCard>
-              <HoverCardTrigger>
-                <div className="text-xs w-5 h-5 rounded-full bg-gray-100 mb-2 flex items-center justify-center hover:text-white hover:bg-primary hover:cursor-pointer">
-                  {index + 1}
-                </div>
-              </HoverCardTrigger>
-              <HoverCardContent>
-                <NodeInfo node={node} />
-              </HoverCardContent>
-            </HoverCard>
-          </div>
-        ))}
+        {sources.map((nodeInfo: NodeInfo, index: number) => {
+          if (nodeInfo.url?.endsWith(".pdf")) {
+            return (
+              <PdfDialog
+                key={nodeInfo.id}
+                documentId={nodeInfo.id}
+                url={nodeInfo.url!}
+                trigger={<SourceNumberButton index={index} />}
+              />
+            );
+          }
+          return (
+            <div key={nodeInfo.id}>
+              <HoverCard>
+                <HoverCardTrigger>
+                  <SourceNumberButton index={index} />
+                </HoverCardTrigger>
+                <HoverCardContent className="w-[320px]">
+                  <NodeInfo nodeInfo={nodeInfo} />
+                </HoverCardContent>
+              </HoverCard>
+            </div>
+          );
+        })}
      </div>
    </div>
  );
 }

-function NodeInfo({ node }: { node: SourceNode }) {
+function NodeInfo({ nodeInfo }: { nodeInfo: NodeInfo }) {
  const { isCopied, copyToClipboard } = useCopyToClipboard({ timeout: 1000 });

-  if (typeof node.metadata["URL"] === "string") {
-    // this is a node generated by the web loader, it contains an external URL
-    // add a link to view this URL
+  if (nodeInfo.url) {
+    // this is a node generated by the web loader or file loader,
+    // add a link to view its URL and a button to copy the URL to the clipboard
    return (
-      <a
-        className="space-x-2 flex items-center my-2 hover:text-blue-900"
-        href={node.metadata["URL"]}
-        target="_blank"
-      >
-        <span>{node.metadata["URL"]}</span>
-        <ArrowUpRightSquare className="w-4 h-4" />
-      </a>
-    );
-  }
-
-  if (typeof node.metadata["file_path"] === "string") {
-    // this is a node generated by the file loader, it contains file path
-    // add a button to copy the path to the clipboard
-    const filePath = node.metadata["file_path"];
-    return (
-      <div className="flex items-center px-2 py-1 justify-between my-2">
-        <span>{filePath}</span>
+      <div className="flex items-center my-2">
+        <a className="hover:text-blue-900" href={nodeInfo.url} target="_blank">
+          <span>{nodeInfo.url}</span>
+        </a>
        <Button
-          onClick={() => copyToClipboard(filePath)}
+          onClick={() => copyToClipboard(nodeInfo.url!)}
          size="icon"
          variant="ghost"
-          className="h-12 w-12"
+          className="h-12 w-12 shrink-0"
        >
          {isCopied ? (
            <Check className="h-4 w-4" />
@@ -84,7 +106,6 @@ function NodeInfo({ node }: { node: SourceNode }) {
  }

  // node generated by unknown loader, implement renderer by analyzing logged out metadata
-  console.log("Node metadata", node.metadata);
  return (
    <p>
      Sorry, unknown node type. Please add a new renderer in the NodeInfo
@@ -15,4 +15,11 @@ export interface ChatHandler {
  stop?: () => void;
  onFileUpload?: (file: File) => Promise<void>;
  onFileError?: (errMsg: string) => void;
+  setInput?: (input: string) => void;
+  append?: (
+    message: Message | Omit<Message, "id">,
+    ops?: {
+      data: any;
+    },
+  ) => Promise<string | null | undefined>;
 }
@@ -0,0 +1,13 @@
+import { CsvData } from ".";
+import UploadCsvPreview from "../upload-csv-preview";
+
+export default function CsvContent({ data }: { data: CsvData }) {
+  if (!data.csvFiles.length) return null;
+  return (
+    <div className="flex gap-2 items-center">
+      {data.csvFiles.map((csv, index) => (
+        <UploadCsvPreview key={index} csv={csv} />
+      ))}
+    </div>
+  );
+}
@@ -6,6 +6,7 @@ export { type ChatHandler } from "./chat.interface";
 export { ChatInput, ChatMessages };

 export enum MessageAnnotationType {
+  CSV = "csv",
  IMAGE = "image",
  SOURCES = "sources",
  EVENTS = "events",
@@ -16,11 +17,23 @@ export type ImageData = {
  url: string;
 };

+export type CsvFile = {
+  content: string;
+  filename: string;
+  filesize: number;
+  id: string;
+};
+
+export type CsvData = {
+  csvFiles: CsvFile[];
+};
+
 export type SourceNode = {
  id: string;
  metadata: Record<string, unknown>;
  score?: number;
  text: string;
+  url?: string;
 };

 export type SourceData = {
@@ -46,9 +59,21 @@ export type ToolData = {
  };
 };

-export type AnnotationData = ImageData | SourceData | EventData | ToolData;
+export type AnnotationData =
+  | ImageData
+  | CsvData
+  | SourceData
+  | EventData
+  | ToolData;

 export type MessageAnnotation = {
  type: MessageAnnotationType;
  data: AnnotationData;
 };
+
+export function getAnnotationData<T extends AnnotationData>(
+  annotations: MessageAnnotation[],
+  type: MessageAnnotationType,
+): T[] {
+  return annotations.filter((a) => a.type === type).map((a) => a.data as T);
+}
@@ -1,5 +1,7 @@
+import "katex/dist/katex.min.css";
 import { FC, memo } from "react";
 import ReactMarkdown, { Options } from "react-markdown";
+import rehypeKatex from "rehype-katex";
 import remarkGfm from "remark-gfm";
 import remarkMath from "remark-math";

@@ -12,11 +14,27 @@ const MemoizedReactMarkdown: FC<Options> = memo(
    prevProps.className === nextProps.className,
 );

+const preprocessLaTeX = (content: string) => {
+  // Replace block-level LaTeX delimiters \[ \] with $$ $$
+  const blockProcessedContent = content.replace(
+    /\\\[(.*?)\\\]/gs,
+    (_, equation) => `$$${equation}$$`,
+  );
+  // Replace inline LaTeX delimiters \( \) with $ $
+  const inlineProcessedContent = blockProcessedContent.replace(
+    /\\\((.*?)\\\)/gs,
+    (_, equation) => `$${equation}$`,
+  );
+  return inlineProcessedContent;
+};
+
 export default function Markdown({ content }: { content: string }) {
+  const processedContent = preprocessLaTeX(content);
  return (
    <MemoizedReactMarkdown
      className="prose dark:prose-invert prose-p:leading-relaxed prose-pre:p-0 break-words custom-markdown"
      remarkPlugins={[remarkGfm, remarkMath]}
+      rehypePlugins={[rehypeKatex as any]}
      components={{
        p({ children }) {
          return <p className="mb-2 last:mb-0">{children}</p>;
@@ -53,7 +71,7 @@ export default function Markdown({ content }: { content: string }) {
        },
      }}
    >
-      {content}
+      {processedContent}
    </MemoizedReactMarkdown>
  );
 }
@@ -0,0 +1,30 @@
+"use client";
+
+import { useEffect, useMemo, useState } from "react";
+
+export interface ChatConfig {
+  chatAPI?: string;
+  starterQuestions?: string[];
+}
+
+export function useClientConfig() {
+  const API_ROUTE = "/api/chat/config";
+  const chatAPI = process.env.NEXT_PUBLIC_CHAT_API;
+  const [config, setConfig] = useState<ChatConfig>({
+    chatAPI,
+  });
+
+  const configAPI = useMemo(() => {
+    const backendOrigin = chatAPI ? new URL(chatAPI).origin : "";
+    return `${backendOrigin}${API_ROUTE}`;
+  }, [chatAPI]);
+
+  useEffect(() => {
+    fetch(configAPI)
+      .then((response) => response.json())
+      .then((data) => setConfig({ ...data, chatAPI }))
+      .catch((error) => console.error("Error fetching config", error));
+  }, [chatAPI, configAPI]);
+
+  return config;
+}
@@ -0,0 +1,33 @@
+"use client";
+
+import { useState } from "react";
+import { CsvFile } from ".";
+
+export function useCsv() {
+  const [files, setFiles] = useState<CsvFile[]>([]);
+
+  const csvEqual = (a: CsvFile, b: CsvFile) => {
+    if (a.id === b.id) return true;
+    if (a.filename === b.filename && a.filesize === b.filesize) return true;
+    return false;
+  };
+
+  const upload = (file: CsvFile) => {
+    const existedCsv = files.find((f) => csvEqual(f, file));
+    if (!existedCsv) {
+      setFiles((prev) => [...prev, file]);
+      return true;
+    }
+    return false;
+  };
+
+  const remove = (file: CsvFile) => {
+    setFiles((prev) => prev.filter((f) => f.id !== file.id));
+  };
+
+  const reset = () => {
+    setFiles([]);
+  };
+
+  return { files, upload, remove, reset };
+}
@@ -0,0 +1,55 @@
+import { PDFViewer, PdfFocusProvider } from "@llamaindex/pdf-viewer";
+import { Button } from "../../button";
+import {
+  Drawer,
+  DrawerClose,
+  DrawerContent,
+  DrawerDescription,
+  DrawerHeader,
+  DrawerTitle,
+  DrawerTrigger,
+} from "../../drawer";
+
+export interface PdfDialogProps {
+  documentId: string;
+  url: string;
+  trigger: React.ReactNode;
+}
+
+export default function PdfDialog(props: PdfDialogProps) {
+  return (
+    <Drawer direction="left">
+      <DrawerTrigger>{props.trigger}</DrawerTrigger>
+      <DrawerContent className="w-3/5 mt-24 h-full max-h-[96%] ">
+        <DrawerHeader className="flex justify-between">
+          <div className="space-y-2">
+            <DrawerTitle>PDF Content</DrawerTitle>
+            <DrawerDescription>
+              File URL:{" "}
+              <a
+                className="hover:text-blue-900"
+                href={props.url}
+                target="_blank"
+              >
+                {props.url}
+              </a>
+            </DrawerDescription>
+          </div>
+          <DrawerClose asChild>
+            <Button variant="outline">Close</Button>
+          </DrawerClose>
+        </DrawerHeader>
+        <div className="m-4">
+          <PdfFocusProvider>
+            <PDFViewer
+              file={{
+                id: props.documentId,
+                url: props.url,
+              }}
+            />
+          </PdfFocusProvider>
+        </div>
+      </DrawerContent>
+    </Drawer>
+  );
+}
@@ -0,0 +1,118 @@
+"use client";
+
+import * as React from "react";
+import { Drawer as DrawerPrimitive } from "vaul";
+
+import { cn } from "./lib/utils";
+
+const Drawer = ({
+  shouldScaleBackground = true,
+  ...props
+}: React.ComponentProps<typeof DrawerPrimitive.Root>) => (
+  <DrawerPrimitive.Root
+    shouldScaleBackground={shouldScaleBackground}
+    {...props}
+  />
+);
+Drawer.displayName = "Drawer";
+
+const DrawerTrigger = DrawerPrimitive.Trigger;
+
+const DrawerPortal = DrawerPrimitive.Portal;
+
+const DrawerClose = DrawerPrimitive.Close;
+
+const DrawerOverlay = React.forwardRef<
+  React.ElementRef<typeof DrawerPrimitive.Overlay>,
+  React.ComponentPropsWithoutRef<typeof DrawerPrimitive.Overlay>
+>(({ className, ...props }, ref) => (
+  <DrawerPrimitive.Overlay
+    ref={ref}
+    className={cn("fixed inset-0 z-50 bg-black/80", className)}
+    {...props}
+  />
+));
+DrawerOverlay.displayName = DrawerPrimitive.Overlay.displayName;
+
+const DrawerContent = React.forwardRef<
+  React.ElementRef<typeof DrawerPrimitive.Content>,
+  React.ComponentPropsWithoutRef<typeof DrawerPrimitive.Content>
+>(({ className, children, ...props }, ref) => (
+  <DrawerPortal>
+    <DrawerOverlay />
+    <DrawerPrimitive.Content
+      ref={ref}
+      className={cn(
+        "fixed inset-x-0 bottom-0 z-50 mt-24 flex h-auto flex-col rounded-t-[10px] border bg-background",
+        className,
+      )}
+      {...props}
+    >
+      <div className="mx-auto mt-4 h-2 w-[100px] rounded-full bg-muted" />
+      {children}
+    </DrawerPrimitive.Content>
+  </DrawerPortal>
+));
+DrawerContent.displayName = "DrawerContent";
+
+const DrawerHeader = ({
+  className,
+  ...props
+}: React.HTMLAttributes<HTMLDivElement>) => (
+  <div
+    className={cn("grid gap-1.5 p-4 text-center sm:text-left", className)}
+    {...props}
+  />
+);
+DrawerHeader.displayName = "DrawerHeader";
+
+const DrawerFooter = ({
+  className,
+  ...props
+}: React.HTMLAttributes<HTMLDivElement>) => (
+  <div
+    className={cn("mt-auto flex flex-col gap-2 p-4", className)}
+    {...props}
+  />
+);
+DrawerFooter.displayName = "DrawerFooter";
+
+const DrawerTitle = React.forwardRef<
+  React.ElementRef<typeof DrawerPrimitive.Title>,
+  React.ComponentPropsWithoutRef<typeof DrawerPrimitive.Title>
+>(({ className, ...props }, ref) => (
+  <DrawerPrimitive.Title
+    ref={ref}
+    className={cn(
+      "text-lg font-semibold leading-none tracking-tight",
+      className,
+    )}
+    {...props}
+  />
+));
+DrawerTitle.displayName = DrawerPrimitive.Title.displayName;
+
+const DrawerDescription = React.forwardRef<
+  React.ElementRef<typeof DrawerPrimitive.Description>,
+  React.ComponentPropsWithoutRef<typeof DrawerPrimitive.Description>
+>(({ className, ...props }, ref) => (
+  <DrawerPrimitive.Description
+    ref={ref}
+    className={cn("text-sm text-muted-foreground", className)}
+    {...props}
+  />
+));
+DrawerDescription.displayName = DrawerPrimitive.Description.displayName;
+
+export {
+  Drawer,
+  DrawerClose,
+  DrawerContent,
+  DrawerDescription,
+  DrawerFooter,
+  DrawerHeader,
+  DrawerOverlay,
+  DrawerPortal,
+  DrawerTitle,
+  DrawerTrigger,
+};
@@ -0,0 +1,90 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<svg width="49px" height="67px" viewBox="0 0 49 67" version="1.1"
+    xmlns="http://www.w3.org/2000/svg"
+    xmlns:xlink="http://www.w3.org/1999/xlink">
+    <title>Sheets-icon</title>
+    <desc>Created with Sketch.</desc>
+    <defs>
+        <path d="M29.5833333,0 L4.4375,0 C1.996875,0 0,1.996875 0,4.4375 L0,60.6458333 C0,63.0864583 1.996875,65.0833333 4.4375,65.0833333 L42.8958333,65.0833333 C45.3364583,65.0833333 47.3333333,63.0864583 47.3333333,60.6458333 L47.3333333,17.75 L29.5833333,0 Z" id="path-1"></path>
+        <path d="M29.5833333,0 L4.4375,0 C1.996875,0 0,1.996875 0,4.4375 L0,60.6458333 C0,63.0864583 1.996875,65.0833333 4.4375,65.0833333 L42.8958333,65.0833333 C45.3364583,65.0833333 47.3333333,63.0864583 47.3333333,60.6458333 L47.3333333,17.75 L29.5833333,0 Z" id="path-3"></path>
+        <path d="M29.5833333,0 L4.4375,0 C1.996875,0 0,1.996875 0,4.4375 L0,60.6458333 C0,63.0864583 1.996875,65.0833333 4.4375,65.0833333 L42.8958333,65.0833333 C45.3364583,65.0833333 47.3333333,63.0864583 47.3333333,60.6458333 L47.3333333,17.75 L29.5833333,0 Z" id="path-5"></path>
+        <linearGradient x1="50.0053945%" y1="8.58610612%" x2="50.0053945%" y2="100.013939%" id="linearGradient-7">
+            <stop stop-color="#263238" stop-opacity="0.2" offset="0%"></stop>
+            <stop stop-color="#263238" stop-opacity="0.02" offset="100%"></stop>
+        </linearGradient>
+        <path d="M29.5833333,0 L4.4375,0 C1.996875,0 0,1.996875 0,4.4375 L0,60.6458333 C0,63.0864583 1.996875,65.0833333 4.4375,65.0833333 L42.8958333,65.0833333 C45.3364583,65.0833333 47.3333333,63.0864583 47.3333333,60.6458333 L47.3333333,17.75 L29.5833333,0 Z" id="path-8"></path>
+        <path d="M29.5833333,0 L4.4375,0 C1.996875,0 0,1.996875 0,4.4375 L0,60.6458333 C0,63.0864583 1.996875,65.0833333 4.4375,65.0833333 L42.8958333,65.0833333 C45.3364583,65.0833333 47.3333333,63.0864583 47.3333333,60.6458333 L47.3333333,17.75 L29.5833333,0 Z" id="path-10"></path>
+        <path d="M29.5833333,0 L4.4375,0 C1.996875,0 0,1.996875 0,4.4375 L0,60.6458333 C0,63.0864583 1.996875,65.0833333 4.4375,65.0833333 L42.8958333,65.0833333 C45.3364583,65.0833333 47.3333333,63.0864583 47.3333333,60.6458333 L47.3333333,17.75 L29.5833333,0 Z" id="path-12"></path>
+        <path d="M29.5833333,0 L4.4375,0 C1.996875,0 0,1.996875 0,4.4375 L0,60.6458333 C0,63.0864583 1.996875,65.0833333 4.4375,65.0833333 L42.8958333,65.0833333 C45.3364583,65.0833333 47.3333333,63.0864583 47.3333333,60.6458333 L47.3333333,17.75 L29.5833333,0 Z" id="path-14"></path>
+        <radialGradient cx="3.16804688%" cy="2.71744318%" fx="3.16804688%" fy="2.71744318%" r="161.248516%" gradientTransform="translate(0.031680,0.027174),scale(1.000000,0.727273),translate(-0.031680,-0.027174)" id="radialGradient-16">
+            <stop stop-color="#FFFFFF" stop-opacity="0.1" offset="0%"></stop>
+            <stop stop-color="#FFFFFF" stop-opacity="0" offset="100%"></stop>
+        </radialGradient>
+    </defs>
+    <g id="Page-1" stroke="none" stroke-width="1" fill="none" fill-rule="evenodd">
+        <g id="Consumer-Apps-Sheets-Large-VD-R8-" transform="translate(-451.000000, -451.000000)">
+            <g id="Hero" transform="translate(0.000000, 63.000000)">
+                <g id="Personal" transform="translate(277.000000, 299.000000)">
+                    <g id="Sheets-icon" transform="translate(174.833333, 89.958333)">
+                        <g id="Group">
+                            <g id="Clipped">
+                                <mask id="mask-2" fill="white">
+                                    <use xlink:href="#path-1"></use>
+                                </mask>
+                                <g id="SVGID_1_"></g>
+                                <path d="M29.5833333,0 L4.4375,0 C1.996875,0 0,1.996875 0,4.4375 L0,60.6458333 C0,63.0864583 1.996875,65.0833333 4.4375,65.0833333 L42.8958333,65.0833333 C45.3364583,65.0833333 47.3333333,63.0864583 47.3333333,60.6458333 L47.3333333,17.75 L36.9791667,10.3541667 L29.5833333,0 Z" id="Path" fill="#0F9D58" fill-rule="nonzero" mask="url(#mask-2)"></path>
+                            </g>
+                            <g id="Clipped">
+                                <mask id="mask-4" fill="white">
+                                    <use xlink:href="#path-3"></use>
+                                </mask>
+                                <g id="SVGID_1_"></g>
+                                <path d="M11.8333333,31.8020833 L11.8333333,53.25 L35.5,53.25 L35.5,31.8020833 L11.8333333,31.8020833 Z M22.1875,50.2916667 L14.7916667,50.2916667 L14.7916667,46.59375 L22.1875,46.59375 L22.1875,50.2916667 Z M22.1875,44.375 L14.7916667,44.375 L14.7916667,40.6770833 L22.1875,40.6770833 L22.1875,44.375 Z M22.1875,38.4583333 L14.7916667,38.4583333 L14.7916667,34.7604167 L22.1875,34.7604167 L22.1875,38.4583333 Z M32.5416667,50.2916667 L25.1458333,50.2916667 L25.1458333,46.59375 L32.5416667,46.59375 L32.5416667,50.2916667 Z M32.5416667,44.375 L25.1458333,44.375 L25.1458333,40.6770833 L32.5416667,40.6770833 L32.5416667,44.375 Z M32.5416667,38.4583333 L25.1458333,38.4583333 L25.1458333,34.7604167 L32.5416667,34.7604167 L32.5416667,38.4583333 Z" id="Shape" fill="#F1F1F1" fill-rule="nonzero" mask="url(#mask-4)"></path>
+                            </g>
+                            <g id="Clipped">
+                                <mask id="mask-6" fill="white">
+                                    <use xlink:href="#path-5"></use>
+                                </mask>
+                                <g id="SVGID_1_"></g>
+                                <polygon id="Path" fill="url(#linearGradient-7)" fill-rule="nonzero" mask="url(#mask-6)" points="30.8813021 16.4520313 47.3333333 32.9003646 47.3333333 17.75"></polygon>
+                            </g>
+                            <g id="Clipped">
+                                <mask id="mask-9" fill="white">
+                                    <use xlink:href="#path-8"></use>
+                                </mask>
+                                <g id="SVGID_1_"></g>
+                                <g id="Group" mask="url(#mask-9)">
+                                    <g transform="translate(26.625000, -2.958333)">
+                                        <path d="M2.95833333,2.95833333 L2.95833333,16.2708333 C2.95833333,18.7225521 4.94411458,20.7083333 7.39583333,20.7083333 L20.7083333,20.7083333 L2.95833333,2.95833333 Z" id="Path" fill="#87CEAC" fill-rule="nonzero"></path>
+                                    </g>
+                                </g>
+                            </g>
+                            <g id="Clipped">
+                                <mask id="mask-11" fill="white">
+                                    <use xlink:href="#path-10"></use>
+                                </mask>
+                                <g id="SVGID_1_"></g>
+                                <path d="M4.4375,0 C1.996875,0 0,1.996875 0,4.4375 L0,4.80729167 C0,2.36666667 1.996875,0.369791667 4.4375,0.369791667 L29.5833333,0.369791667 L29.5833333,0 L4.4375,0 Z" id="Path" fill-opacity="0.2" fill="#FFFFFF" fill-rule="nonzero" mask="url(#mask-11)"></path>
+                            </g>
+                            <g id="Clipped">
+                                <mask id="mask-13" fill="white">
+                                    <use xlink:href="#path-12"></use>
+                                </mask>
+                                <g id="SVGID_1_"></g>
+                                <path d="M42.8958333,64.7135417 L4.4375,64.7135417 C1.996875,64.7135417 0,62.7166667 0,60.2760417 L0,60.6458333 C0,63.0864583 1.996875,65.0833333 4.4375,65.0833333 L42.8958333,65.0833333 C45.3364583,65.0833333 47.3333333,63.0864583 47.3333333,60.6458333 L47.3333333,60.2760417 C47.3333333,62.7166667 45.3364583,64.7135417 42.8958333,64.7135417 Z" id="Path" fill-opacity="0.2" fill="#263238" fill-rule="nonzero" mask="url(#mask-13)"></path>
+                            </g>
+                            <g id="Clipped">
+                                <mask id="mask-15" fill="white">
+                                    <use xlink:href="#path-14"></use>
+                                </mask>
+                                <g id="SVGID_1_"></g>
+                                <path d="M34.0208333,17.75 C31.5691146,17.75 29.5833333,15.7642188 29.5833333,13.3125 L29.5833333,13.6822917 C29.5833333,16.1340104 31.5691146,18.1197917 34.0208333,18.1197917 L47.3333333,18.1197917 L47.3333333,17.75 L34.0208333,17.75 Z" id="Path" fill-opacity="0.1" fill="#263238" fill-rule="nonzero" mask="url(#mask-15)"></path>
+                            </g>
+                        </g>
+                        <path d="M29.5833333,0 L4.4375,0 C1.996875,0 0,1.996875 0,4.4375 L0,60.6458333 C0,63.0864583 1.996875,65.0833333 4.4375,65.0833333 L42.8958333,65.0833333 C45.3364583,65.0833333 47.3333333,63.0864583 47.3333333,60.6458333 L47.3333333,17.75 L29.5833333,0 Z" id="Path" fill="url(#radialGradient-16)" fill-rule="nonzero"></path>
+                    </g>
+                </g>
+            </g>
+        </g>
+    </g>
+</svg>
@@ -0,0 +1,93 @@
+import { XCircleIcon } from "lucide-react";
+import Image from "next/image";
+import SheetIcon from "../ui/icons/sheet.svg";
+import { Button } from "./button";
+import { CsvFile } from "./chat";
+import {
+  Drawer,
+  DrawerClose,
+  DrawerContent,
+  DrawerDescription,
+  DrawerHeader,
+  DrawerTitle,
+  DrawerTrigger,
+} from "./drawer";
+import { cn } from "./lib/utils";
+
+export interface UploadCsvPreviewProps {
+  csv: CsvFile;
+  onRemove?: () => void;
+}
+
+export default function UploadCsvPreview(props: UploadCsvPreviewProps) {
+  const { filename, filesize, content } = props.csv;
+  return (
+    <Drawer direction="left">
+      <DrawerTrigger asChild>
+        <div>
+          <CSVSummaryCard {...props} />
+        </div>
+      </DrawerTrigger>
+      <DrawerContent className="w-3/5 mt-24 h-full max-h-[96%] ">
+        <DrawerHeader className="flex justify-between">
+          <div className="space-y-2">
+            <DrawerTitle>Csv Raw Content</DrawerTitle>
+            <DrawerDescription>
+              {filename} ({inKB(filesize)} KB)
+            </DrawerDescription>
+          </div>
+          <DrawerClose asChild>
+            <Button variant="outline">Close</Button>
+          </DrawerClose>
+        </DrawerHeader>
+        <div className="m-4 max-h-[80%] overflow-auto">
+          <pre className="bg-secondary rounded-md p-4 block text-sm">
+            {content}
+          </pre>
+        </div>
+      </DrawerContent>
+    </Drawer>
+  );
+}
+
+function CSVSummaryCard(props: UploadCsvPreviewProps) {
+  const { onRemove, csv } = props;
+  return (
+    <div className="p-2 w-60 max-w-60 bg-secondary rounded-lg text-sm relative cursor-pointer">
+      <div className="flex flex-row items-center gap-2">
+        <div className="relative h-10 w-10 shrink-0 overflow-hidden rounded-md">
+          <Image
+            className="h-full w-auto"
+            priority
+            src={SheetIcon}
+            alt="SheetIcon"
+          />
+        </div>
+        <div className="overflow-hidden">
+          <div className="truncate font-semibold">
+            {csv.filename} ({inKB(csv.filesize)} KB)
+          </div>
+          <div className="truncate text-token-text-tertiary flex items-center gap-2">
+            <span>Spreadsheet</span>
+          </div>
+        </div>
+      </div>
+      {onRemove && (
+        <div
+          className={cn(
+            "absolute -top-2 -right-2 w-6 h-6 z-10 bg-gray-500 text-white rounded-full",
+          )}
+        >
+          <XCircleIcon
+            className="w-6 h-6 bg-gray-500 text-white rounded-full"
+            onClick={onRemove}
+          />
+        </div>
+      )}
+    </div>
+  );
+}
+
+function inKB(size: number) {
+  return Math.round((size / 1024) * 10) / 10;
+}
@@ -33,3 +33,5 @@ yarn-error.log*
 # typescript
 *.tsbuildinfo
 next-env.d.ts
+
+tool-output/
@@ -18,7 +18,7 @@
    "class-variance-authority": "^0.7.0",
    "clsx": "^2.1.1",
    "dotenv": "^16.3.1",
-    "llamaindex": "0.3.8",
+    "llamaindex": "0.3.16",
    "lucide-react": "^0.294.0",
    "next": "^14.0.3",
    "pdf2json": "3.0.5",
@@ -30,8 +30,13 @@
    "remark-code-import": "^1.2.0",
    "remark-gfm": "^3.0.1",
    "remark-math": "^5.1.1",
+    "rehype-katex": "^7.0.0",
    "supports-color": "^8.1.1",
-    "tailwind-merge": "^2.1.0"
+    "tailwind-merge": "^2.1.0",
+    "vaul": "^0.9.1",
+    "@llamaindex/pdf-viewer": "^1.1.1",
+    "@e2b/code-interpreter": "^0.0.5",
+    "uuid": "^9.0.1"
  },
  "devDependencies": {
    "@types/node": "^20.10.3",
@@ -48,6 +53,7 @@
    "prettier-plugin-organize-imports": "^3.2.4",
    "tailwindcss": "^3.3.6",
    "tsx": "^4.7.2",
-    "typescript": "^5.3.2"
+    "typescript": "^5.3.2",
+    "@types/uuid": "^9.0.8"
  }
 }
@@ -11,14 +11,16 @@
    "forceConsistentCasingInFileNames": true,
    "incremental": true,
    "outDir": "./lib",
-    "tsBuildInfoFile": "./lib/.tsbuildinfo"
+    "tsBuildInfoFile": "./lib/.tsbuildinfo",
+    "typeRoots": ["./types", "./node_modules/@types"]
  },
  "include": [
    "create-app.ts",
    "index.ts",
    "./helpers",
    "questions.ts",
-    "package.json"
+    "package.json",
+    "types/**/*"
  ],
  "exclude": ["dist"]
 }
@@ -0,0 +1 @@
+declare module "global-agent/bootstrap";
Author	SHA1	Message	Date
Thuc Pham	eda6b3be23	use --watch	2024-06-10 12:41:48 +07:00
Thuc Pham	65edc14c15	fix: ignore cache	2024-06-10 12:13:27 +07:00
Thuc Pham	c40757cbb7	fix: express server restart when cache changed	2024-06-10 11:44:59 +07:00
github-actions[bot]	d53b760fd0	Release 0.1.9 (#101 ) Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>	2024-06-07 22:56:34 +07:00
Marcus Schiesser	a880c7c016	chore: update llamaindex@0.3.16	2024-06-07 17:40:39 +02:00
Marcus Schiesser	7b116ce7f7	fix: allow subsequent tool calls	2024-06-07 17:35:23 +02:00
Marcus Schiesser	d1232fb1d5	fix: log interpreter tool error	2024-06-07 16:10:33 +02:00
Marcus Schiesser	bedf199236	fix: throw and show error if unsupported annotation (e.g. image) is uploaded	2024-06-07 15:30:31 +02:00
Marcus Schiesser	c1510bd3fa	fix: remove redundant config info	2024-06-07 14:37:08 +02:00
Huu Le	69b9ce76bf	refactor code (#119 )	2024-06-07 13:46:25 +02:00
Marcus Schiesser	9ced116e1a	refactor: use message annotations instead of sending data (#116 ) --------- Co-authored-by: Thuc Pham <51660321+thucpn@users.noreply.github.com> Co-authored-by: leehuwuj <leehuwuj@gmail.com>	2024-06-07 17:14:15 +07:00
Huu Le	fae9bcd65a	add raw text e2b tool output response (#115 )	2024-06-06 13:23:31 +02:00
Thuc Pham	2091fea2b4	feat: display attachments in user messages (#114 ) * use same csv card for message and upload box * do not send csv and image data back to client * fix: use LLM_MAX_TOKENS --------- Co-authored-by: leehuwuj <leehuwuj@gmail.com> Co-authored-by: Marcus Schiesser <mail@marcusschiesser.de>	2024-06-06 14:24:31 +07:00
Huu Le	563b51d76d	Fix: Vercel streaming (python) does not stream data events instantly (#111 )	2024-06-05 15:54:55 +07:00
Thuc Pham	88c88bf16d	fix: logo overlay text input because of hegiht (#112 )	2024-06-05 15:40:38 +07:00
Marcus Schiesser	cd6ebf7295	dx: add hint if tool config is needed	2024-06-04 12:20:52 +02:00
Marcus Schiesser	50b2ddbbf5	docs: updated changeset	2024-06-04 11:15:47 +02:00
Huu Le	5fe2d519d2	chore: Add Azure OpenAI model provider python (#110 )	2024-06-04 16:14:21 +07:00
Huu Le	09f1db3b5e	feat: Support uploading CSV files for FastAPI app (#109 )	2024-06-04 14:23:25 +07:00
Thuc Pham	cb3be7d1d4	feat: display conversation starter from backend env (#104 ) * feat: display conversation starter from frontend env * use nextjs config api * update to /api/chat/config * add config api for express * add api config for fast api * Create ten-badgers-learn.md * remove default conversation staters * check empty string * update pydantic docs * refactor: move NEXT_PUBLIC_CHAT_API to use config * use config to get chatAPI * refactor: rename useClientConfig	2024-06-01 09:57:17 +07:00
Thuc Pham	5474a1f182	feat: enhance csv upload feature (#105 ) * remove all multiModal props * hide uploaded csv files if choose a new one * feat: support multiple csv upload and reuse * rename type and make it scrollable	2024-06-01 09:37:46 +07:00
Huu Le	1148ddba53	bump llama-index-agent-openai version to 0.2.6 (#107 )	2024-05-31 13:46:35 +01:00
Huu Le	9e945ed355	bump llama_index and gemini version (#106 )	2024-05-31 15:12:14 +07:00
Thuc Pham	6342163df2	Merge pull request #103 from run-llama/feat/add-openapi-tool feat: Add OpenAPI Action tool	2024-05-30 15:33:36 +07:00
Thuc Pham	a42fa53a6b	feat: implement csv upload (#96 ) * feat: implement interpreter tool * build tool system prompt * refactor: use local file system, use absolute resource url * fix: typo * feat: implement csv upload * remove dead code * fix lint * update icon & fix code review * fix lint * Update .gitignore * Update pre-commit * add timeout for streaming * Create bright-turkeys-melt.md * remove multi modal prop * suggest csv resources from frontend annotation data * get resouces inside chat input * resolve conflict * update convert message content * fix lint * feat: limit display --------- Co-authored-by: Marcus Schiesser <mail@marcusschiesser.de>	2024-05-30 10:38:54 +07:00
leehuwuj	099f626586	use urlparse for file path	2024-05-30 10:05:00 +07:00
leehuwuj	956538eeb0	add changeset	2024-05-30 09:27:21 +07:00
leehuwuj	555f6b2905	refactor code	2024-05-30 09:25:56 +07:00
leehuwuj	d8bc271a21	add local tool that combine openapi and request tool	2024-05-30 09:11:21 +07:00
leehuwuj	f29561cde2	add cache to toolfactory load_tools	2024-05-29 10:40:40 +07:00
leehuwuj	442abae8ac	add openapi tool and http request tool	2024-05-29 08:40:16 +07:00
Huu Le	0ad2207684	Merge pull request #98 from run-llama/feat/construct-resource-url-from-backend feat: construct resource url from backend	2024-05-28 20:43:04 +07:00
Thuc Pham	bfde30deed	move logger to global scope	2024-05-28 18:42:46 +07:00
Thuc Pham	96fdb83abf	use logger warning	2024-05-28 18:33:53 +07:00
Huu Le	b7e0072c9c	chore: always generate tools config if user selects agent mode (#102 )	2024-05-28 14:35:36 +07:00
Thuc Pham	81bc340dda	add warning when no file server url prefix	2024-05-27 18:21:32 +07:00
Thuc Pham	ddf3aef7dc	remove node path	2024-05-27 18:20:27 +07:00
Thuc Pham	1f5a26f3a8	Merge pull request #100 from run-llama/feat/code-interpreter-python feat: add support for FastAPI in code interpreter tool	2024-05-27 16:58:32 +07:00
leehuwuj	05748bdf10	refactor code	2024-05-27 14:53:01 +07:00
leehuwuj	d60b3c5a96	refactor code and add changeset	2024-05-27 13:09:59 +07:00
leehuwuj	c3e9ed3df4	feat: add support for FastAPI in code interpreter tool	2024-05-27 12:37:49 +07:00
Thuc Pham	48188ca3f9	feat: construct resource url from backend	2024-05-24 14:40:44 +07:00
github-actions[bot]	1fde1dc585	Release 0.1.8 (#97 ) Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>	2024-05-23 22:15:51 +07:00
Thuc Pham	cd50a33d43	feat: implement interpreter tool (#94 ) --------- Co-authored-by: Marcus Schiesser <mail@marcusschiesser.de>	2024-05-23 21:49:18 +07:00
github-actions[bot]	ed114856d9	Release 0.1.7 (#93 ) Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>	2024-05-22 18:30:49 +07:00
Marcus Schiesser	69c2e16c82	fix: streaming for express	2024-05-22 13:04:35 +02:00
Marcus Schiesser	f5da6623cf	fix: update llamaindex, use 127.0.0.1 for ollama as default	2024-05-22 12:42:34 +02:00
Marcus Schiesser	0950cb90f2	fix: global-agent types	2024-05-22 11:50:34 +02:00
Mohammad Amir	bb53425b4b	Proxy support added via global agent (#76 )	2024-05-22 16:35:03 +07:00
Huu Le (Lee)	bbd5b8ddd6	fix: Reuse PG vector store to avoid recreating sqlalchemy engine (#95 )	2024-05-22 16:12:44 +07:00
Thuc Pham	260d37a3f1	feat(ts): add system prompt for chat engine (#92 )	2024-05-20 16:12:19 +07:00
Huu Le (Lee)	7873bfb030	chore: Add Ollama API base URL environment variable (#91 )	2024-05-17 17:01:06 +07:00
github-actions[bot]	0c7c41ee3b	Release 0.1.6 (#90 ) Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>	2024-05-16 19:08:40 +07:00
Thuc Pham	56537a1473	feat: host local files and add viewer for PDFs (#85 )	2024-05-16 18:06:26 +07:00
github-actions[bot]	d8dfc29edd	Release 0.1.5 (#89 ) Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>	2024-05-16 16:12:40 +07:00
Thuc Pham	84db798353	feat: support display latex in chat markdown (#88 )	2024-05-16 15:25:53 +07:00