feat: init project with create-llama Nextjs Agentic RAG template

.devcontainer/devcontainer.json

@@ -0,0 +1,46 @@
+{
+  "image": "mcr.microsoft.com/vscode/devcontainers/typescript-node:dev-20-bullseye",
+  "features": {
+    "ghcr.io/devcontainers-contrib/features/turborepo-npm:1": {},
+    "ghcr.io/devcontainers-contrib/features/typescript:2": {},
+    "ghcr.io/devcontainers/features/python:1": {
+      "version": "3.11",
+      "toolsToInstall": [
+        "flake8",
+        "black",
+        "mypy",
+        "poetry"
+      ]
+    }
+  },
+  "customizations": {
+    "codespaces": {
+      "openFiles": [
+        "README.md"
+      ]
+    },
+    "vscode": {
+      "extensions": [
+        "ms-vscode.typescript-language-features",
+        "esbenp.prettier-vscode",
+        "ms-python.python",
+        "ms-python.black-formatter",
+        "ms-python.vscode-flake8",
+        "ms-python.vscode-pylance"
+      ],
+      "settings": {
+        "python.formatting.provider": "black",
+        "python.languageServer": "Pylance",
+        "python.analysis.typeCheckingMode": "basic"
+      }
+    }
+  },
+  "containerEnv": {
+    "POETRY_VIRTUALENVS_CREATE": "false"
+  },
+  "forwardPorts": [
+    3000,
+    8000
+  ],
+  "postCreateCommand": "npm install"
+}

.eslintrc.json

@@ -0,0 +1,7 @@
+{
+  "extends": ["next/core-web-vitals", "prettier"],
+  "rules": {
+    "max-params": ["error", 4],
+    "prefer-const": "error"
+  }
+}

.gitignore

@@ -0,0 +1,38 @@
+# See https://help.github.com/articles/ignoring-files/ for more about ignoring files.
+
+# dependencies
+/node_modules
+/.pnp
+.pnp.js
+
+# testing
+/coverage
+
+# next.js
+/.next/
+/out/
+
+# production
+/build
+
+# misc
+.DS_Store
+*.pem
+
+# debug
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+
+# local env files
+.env*.local
+
+# vercel
+.vercel
+
+# typescript
+*.tsbuildinfo
+next-env.d.ts
+
+output/
+.env

Dockerfile

@@ -0,0 +1,16 @@
+FROM node:20-alpine as build
+
+WORKDIR /app
+
+# Install dependencies
+COPY package.json package-lock.* ./
+RUN npm install
+
+# Build the application
+COPY . .
+RUN npm run build
+
+# ====================================
+FROM build as release
+
+CMD ["npm", "run", "start"]

README.md

@@ -0,0 +1,71 @@
+This is a [LlamaIndex](https://www.llamaindex.ai/) project using [Next.js](https://nextjs.org/) bootstrapped with [`create-llama`](https://github.com/run-llama/LlamaIndexTS/tree/main/packages/create-llama).
+
+## Getting Started
+
+First, install the dependencies:
+
+```
+npm install
+```
+
+Second, generate the embeddings of the documents in the `./data` directory (if this folder exists - otherwise, skip this step):
+
+```
+npm run generate
+```
+
+Third, run the development server:
+
+```
+npm run dev
+```
+
+Open [http://localhost:3000](http://localhost:3000) with your browser to see the result.
+
+You can start editing the page by modifying `app/page.tsx`. The page auto-updates as you edit the file.
+
+This project uses [`next/font`](https://nextjs.org/docs/basic-features/font-optimization) to automatically optimize and load Inter, a custom Google Font.
+
+## Using Docker
+
+1. Build an image for the Next.js app:
+
+```
+docker build -t <your_app_image_name> .
+```
+
+2. Generate embeddings:
+
+Parse the data and generate the vector embeddings if the `./data` folder exists - otherwise, skip this step:
+
+```
+docker run \
+  --rm \
+  -v $(pwd)/.env:/app/.env \ # Use ENV variables and configuration from your file-system
+  -v $(pwd)/config:/app/config \
+  -v $(pwd)/data:/app/data \
+  -v $(pwd)/cache:/app/cache \ # Use your file system to store the vector database
+  <your_app_image_name> \
+  npm run generate
+```
+
+3. Start the app:
+
+```
+docker run \
+  --rm \
+  -v $(pwd)/.env:/app/.env \ # Use ENV variables and configuration from your file-system
+  -v $(pwd)/config:/app/config \
+  -v $(pwd)/cache:/app/cache \ # Use your file system to store gea vector database
+  -p 3000:3000 \
+  <your_app_image_name>
+```
+
+## Learn More
+
+To learn more about LlamaIndex, take a look at the following resources:
+
+- [LlamaIndex Documentation](https://docs.llamaindex.ai) - learn about LlamaIndex (Python features).
+- [LlamaIndexTS Documentation](https://ts.llamaindex.ai) - learn about LlamaIndex (Typescript features).
+
+You can check out [the LlamaIndexTS GitHub repository](https://github.com/run-llama/LlamaIndexTS) - your feedback and contributions are welcome!

app/api/chat/config/llamacloud/route.ts

@@ -0,0 +1,24 @@
+import { LLamaCloudFileService } from "llamaindex";
+import { NextResponse } from "next/server";
+
+/**
+ * This API is to get config from the backend envs and expose them to the frontend
+ */
+export async function GET() {
+  if (!process.env.LLAMA_CLOUD_API_KEY) {
+    return NextResponse.json(
+      {
+        error: "env variable LLAMA_CLOUD_API_KEY is required to use LlamaCloud",
+      },
+      { status: 500 },
+    );
+  }
+  const config = {
+    projects: await LLamaCloudFileService.getAllProjectsWithPipelines(),
+    pipeline: {
+      pipeline: process.env.LLAMA_CLOUD_INDEX_NAME,
+      project: process.env.LLAMA_CLOUD_PROJECT_NAME,
+    },
+  };
+  return NextResponse.json(config, { status: 200 });
+}

app/api/chat/config/route.ts

@@ -0,0 +1,11 @@
+import { NextResponse } from "next/server";
+
+/**
+ * This API is to get config from the backend envs and expose them to the frontend
+ */
+export async function GET() {
+  const config = {
+    starterQuestions: process.env.CONVERSATION_STARTERS?.trim().split("\n"),
+  };
+  return NextResponse.json(config, { status: 200 });
+}

app/api/chat/engine/chat.ts

@@ -0,0 +1,51 @@
+import {
+  BaseChatEngine,
+  BaseToolWithCall,
+  OpenAIAgent,
+  QueryEngineTool,
+} from "llamaindex";
+import fs from "node:fs/promises";
+import path from "node:path";
+import { getDataSource } from "./index";
+import { generateFilters } from "./queryFilter";
+import { createTools } from "./tools";
+
+export async function createChatEngine(documentIds?: string[], params?: any) {
+  const tools: BaseToolWithCall[] = [];
+
+  // Add a query engine tool if we have a data source
+  // Delete this code if you don't have a data source
+  const index = await getDataSource(params);
+  if (index) {
+    tools.push(
+      new QueryEngineTool({
+        queryEngine: index.asQueryEngine({
+          preFilters: generateFilters(documentIds || []),
+        }),
+        metadata: {
+          name: "data_query_engine",
+          description: `A query engine for documents from your data source.`,
+        },
+      }),
+    );
+  }
+
+  const configFile = path.join("config", "tools.json");
+  let toolConfig: any;
+  try {
+    // add tools from config file if it exists
+    toolConfig = JSON.parse(await fs.readFile(configFile, "utf8"));
+  } catch (e) {
+    console.info(`Could not read ${configFile} file. Using no tools.`);
+  }
+  if (toolConfig) {
+    tools.push(...(await createTools(toolConfig)));
+  }
+
+  const agent = new OpenAIAgent({
+    tools,
+    systemPrompt: process.env.SYSTEM_PROMPT,
+  }) as unknown as BaseChatEngine;
+
+  return agent;
+}

app/api/chat/engine/generate.ts

@@ -0,0 +1,40 @@
+import { VectorStoreIndex } from "llamaindex";
+import { storageContextFromDefaults } from "llamaindex/storage/StorageContext";
+
+import * as dotenv from "dotenv";
+
+import { getDocuments } from "./loader";
+import { initSettings } from "./settings";
+import { STORAGE_CACHE_DIR } from "./shared";
+
+// Load environment variables from local .env file
+dotenv.config();
+
+async function getRuntime(func: any) {
+  const start = Date.now();
+  await func();
+  const end = Date.now();
+  return end - start;
+}
+
+async function generateDatasource() {
+  console.log(`Generating storage context...`);
+  // Split documents, create embeddings and store them in the storage context
+  const ms = await getRuntime(async () => {
+    const storageContext = await storageContextFromDefaults({
+      persistDir: STORAGE_CACHE_DIR,
+    });
+    const documents = await getDocuments();
+
+    await VectorStoreIndex.fromDocuments(documents, {
+      storageContext,
+    });
+  });
+  console.log(`Storage context successfully generated in ${ms / 1000}s.`);
+}
+
+(async () => {
+  initSettings();
+  await generateDatasource();
+  console.log("Finished generating storage.");
+})();

app/api/chat/engine/index.ts

@@ -0,0 +1,19 @@
+import { SimpleDocumentStore, VectorStoreIndex } from "llamaindex";
+import { storageContextFromDefaults } from "llamaindex/storage/StorageContext";
+import { STORAGE_CACHE_DIR } from "./shared";
+
+export async function getDataSource(params?: any) {
+  const storageContext = await storageContextFromDefaults({
+    persistDir: `${STORAGE_CACHE_DIR}`,
+  });
+
+  const numberOfDocs = Object.keys(
+    (storageContext.docStore as SimpleDocumentStore).toDict(),
+  ).length;
+  if (numberOfDocs === 0) {
+    return null;
+  }
+  return await VectorStoreIndex.init({
+    storageContext,
+  });
+}

app/api/chat/engine/loader.ts

@@ -0,0 +1,24 @@
+import {
+  FILE_EXT_TO_READER,
+  SimpleDirectoryReader,
+} from "llamaindex/readers/SimpleDirectoryReader";
+
+export const DATA_DIR = "./data";
+
+export function getExtractors() {
+  return FILE_EXT_TO_READER;
+}
+
+export async function getDocuments() {
+  const documents = await new SimpleDirectoryReader().loadData({
+    directoryPath: DATA_DIR,
+  });
+  // Set private=false to mark the document as public (required for filtering)
+  for (const document of documents) {
+    document.metadata = {
+      ...document.metadata,
+      private: "false",
+    };
+  }
+  return documents;
+}

app/api/chat/engine/queryFilter.ts

@@ -0,0 +1,25 @@
+import { MetadataFilter, MetadataFilters } from "llamaindex";
+
+export function generateFilters(documentIds: string[]): MetadataFilters {
+  // filter all documents have the private metadata key set to true
+  const publicDocumentsFilter: MetadataFilter = {
+    key: "private",
+    value: "true",
+    operator: "!=",
+  };
+
+  // if no documentIds are provided, only retrieve information from public documents
+  if (!documentIds.length) return { filters: [publicDocumentsFilter] };
+
+  const privateDocumentsFilter: MetadataFilter = {
+    key: "doc_id",
+    value: documentIds,
+    operator: "in",
+  };
+
+  // if documentIds are provided, retrieve information from public and private documents
+  return {
+    filters: [publicDocumentsFilter, privateDocumentsFilter],
+    condition: "or",
+  };
+}

app/api/chat/engine/settings.ts

@@ -0,0 +1,179 @@
+import {
+  ALL_AVAILABLE_MISTRAL_MODELS,
+  Anthropic,
+  GEMINI_EMBEDDING_MODEL,
+  GEMINI_MODEL,
+  Gemini,
+  GeminiEmbedding,
+  Groq,
+  MistralAI,
+  MistralAIEmbedding,
+  MistralAIEmbeddingModelType,
+  OpenAI,
+  OpenAIEmbedding,
+  Settings,
+} from "llamaindex";
+import { HuggingFaceEmbedding } from "llamaindex/embeddings/HuggingFaceEmbedding";
+import { OllamaEmbedding } from "llamaindex/embeddings/OllamaEmbedding";
+import { ALL_AVAILABLE_ANTHROPIC_MODELS } from "llamaindex/llm/anthropic";
+import { Ollama } from "llamaindex/llm/ollama";
+
+const CHUNK_SIZE = 512;
+const CHUNK_OVERLAP = 20;
+
+export const initSettings = async () => {
+  // HINT: you can delete the initialization code for unused model providers
+  console.log(`Using '${process.env.MODEL_PROVIDER}' model provider`);
+
+  if (!process.env.MODEL || !process.env.EMBEDDING_MODEL) {
+    throw new Error("'MODEL' and 'EMBEDDING_MODEL' env variables must be set.");
+  }
+
+  switch (process.env.MODEL_PROVIDER) {
+    case "ollama":
+      initOllama();
+      break;
+    case "groq":
+      initGroq();
+      break;
+    case "anthropic":
+      initAnthropic();
+      break;
+    case "gemini":
+      initGemini();
+      break;
+    case "mistral":
+      initMistralAI();
+      break;
+    case "azure-openai":
+      initAzureOpenAI();
+      break;
+    default:
+      initOpenAI();
+      break;
+  }
+  Settings.chunkSize = CHUNK_SIZE;
+  Settings.chunkOverlap = CHUNK_OVERLAP;
+};
+
+function initOpenAI() {
+  Settings.llm = new OpenAI({
+    model: process.env.MODEL ?? "gpt-4o-mini",
+    maxTokens: process.env.LLM_MAX_TOKENS
+      ? Number(process.env.LLM_MAX_TOKENS)
+      : undefined,
+  });
+  Settings.embedModel = new OpenAIEmbedding({
+    model: process.env.EMBEDDING_MODEL,
+    dimensions: process.env.EMBEDDING_DIM
+      ? parseInt(process.env.EMBEDDING_DIM)
+      : undefined,
+  });
+}
+
+function initAzureOpenAI() {
+  // Map Azure OpenAI model names to OpenAI model names (only for TS)
+  const AZURE_OPENAI_MODEL_MAP: Record<string, string> = {
+    "gpt-35-turbo": "gpt-3.5-turbo",
+    "gpt-35-turbo-16k": "gpt-3.5-turbo-16k",
+    "gpt-4o": "gpt-4o",
+    "gpt-4": "gpt-4",
+    "gpt-4-32k": "gpt-4-32k",
+    "gpt-4-turbo": "gpt-4-turbo",
+    "gpt-4-turbo-2024-04-09": "gpt-4-turbo",
+    "gpt-4-vision-preview": "gpt-4-vision-preview",
+    "gpt-4-1106-preview": "gpt-4-1106-preview",
+    "gpt-4o-2024-05-13": "gpt-4o-2024-05-13",
+  };
+
+  const azureConfig = {
+    apiKey: process.env.AZURE_OPENAI_KEY,
+    endpoint: process.env.AZURE_OPENAI_ENDPOINT,
+    apiVersion:
+      process.env.AZURE_OPENAI_API_VERSION || process.env.OPENAI_API_VERSION,
+  };
+
+  Settings.llm = new OpenAI({
+    model:
+      AZURE_OPENAI_MODEL_MAP[process.env.MODEL ?? "gpt-35-turbo"] ??
+      "gpt-3.5-turbo",
+    maxTokens: process.env.LLM_MAX_TOKENS
+      ? Number(process.env.LLM_MAX_TOKENS)
+      : undefined,
+    azure: {
+      ...azureConfig,
+      deployment: process.env.AZURE_OPENAI_LLM_DEPLOYMENT,
+    },
+  });
+
+  Settings.embedModel = new OpenAIEmbedding({
+    model: process.env.EMBEDDING_MODEL,
+    dimensions: process.env.EMBEDDING_DIM
+      ? parseInt(process.env.EMBEDDING_DIM)
+      : undefined,
+    azure: {
+      ...azureConfig,
+      deployment: process.env.AZURE_OPENAI_EMBEDDING_DEPLOYMENT,
+    },
+  });
+}
+
+function initOllama() {
+  const config = {
+    host: process.env.OLLAMA_BASE_URL ?? "http://127.0.0.1:11434",
+  };
+  Settings.llm = new Ollama({
+    model: process.env.MODEL ?? "",
+    config,
+  });
+  Settings.embedModel = new OllamaEmbedding({
+    model: process.env.EMBEDDING_MODEL ?? "",
+    config,
+  });
+}
+
+function initGroq() {
+  const embedModelMap: Record<string, string> = {
+    "all-MiniLM-L6-v2": "Xenova/all-MiniLM-L6-v2",
+    "all-mpnet-base-v2": "Xenova/all-mpnet-base-v2",
+  };
+
+  Settings.llm = new Groq({
+    model: process.env.MODEL!,
+  });
+
+  Settings.embedModel = new HuggingFaceEmbedding({
+    modelType: embedModelMap[process.env.EMBEDDING_MODEL!],
+  });
+}
+
+function initAnthropic() {
+  const embedModelMap: Record<string, string> = {
+    "all-MiniLM-L6-v2": "Xenova/all-MiniLM-L6-v2",
+    "all-mpnet-base-v2": "Xenova/all-mpnet-base-v2",
+  };
+  Settings.llm = new Anthropic({
+    model: process.env.MODEL as keyof typeof ALL_AVAILABLE_ANTHROPIC_MODELS,
+  });
+  Settings.embedModel = new HuggingFaceEmbedding({
+    modelType: embedModelMap[process.env.EMBEDDING_MODEL!],
+  });
+}
+
+function initGemini() {
+  Settings.llm = new Gemini({
+    model: process.env.MODEL as GEMINI_MODEL,
+  });
+  Settings.embedModel = new GeminiEmbedding({
+    model: process.env.EMBEDDING_MODEL as GEMINI_EMBEDDING_MODEL,
+  });
+}
+
+function initMistralAI() {
+  Settings.llm = new MistralAI({
+    model: process.env.MODEL as keyof typeof ALL_AVAILABLE_MISTRAL_MODELS,
+  });
+  Settings.embedModel = new MistralAIEmbedding({
+    model: process.env.EMBEDDING_MODEL as MistralAIEmbeddingModelType,
+  });
+}

app/api/chat/engine/shared.ts

@@ -0,0 +1 @@
+export const STORAGE_CACHE_DIR = "./cache";

app/api/chat/engine/tools/code-generator.ts

@@ -0,0 +1,143 @@
+import type { JSONSchemaType } from "ajv";
+import {
+  BaseTool,
+  ChatMessage,
+  JSONValue,
+  Settings,
+  ToolMetadata,
+} from "llamaindex";
+
+// prompt based on https://github.com/e2b-dev/ai-artifacts
+const CODE_GENERATION_PROMPT = `You are a skilled software engineer. You do not make mistakes. Generate an artifact. You can install additional dependencies. You can use one of the following templates:\n
+
+1. code-interpreter-multilang: "Runs code as a Jupyter notebook cell. Strong data analysis angle. Can use complex visualisation to explain results.". File: script.py. Dependencies installed: python, jupyter, numpy, pandas, matplotlib, seaborn, plotly. Port: none.
+
+2. nextjs-developer: "A Next.js 13+ app that reloads automatically. Using the pages router.". File: pages/index.tsx. Dependencies installed: nextjs@14.2.5, typescript, @types/node, @types/react, @types/react-dom, postcss, tailwindcss, shadcn. Port: 3000.
+
+3. vue-developer: "A Vue.js 3+ app that reloads automatically. Only when asked specifically for a Vue app.". File: app.vue. Dependencies installed: vue@latest, nuxt@3.13.0, tailwindcss. Port: 3000.
+
+4. streamlit-developer: "A streamlit app that reloads automatically.". File: app.py. Dependencies installed: streamlit, pandas, numpy, matplotlib, request, seaborn, plotly. Port: 8501.
+
+5. gradio-developer: "A gradio app. Gradio Blocks/Interface should be called demo.". File: app.py. Dependencies installed: gradio, pandas, numpy, matplotlib, request, seaborn, plotly. Port: 7860.
+
+Provide detail information about the artifact you're about to generate in the following JSON format with the following keys:
+  
+commentary: Describe what you're about to do and the steps you want to take for generating the artifact in great detail.
+template: Name of the template used to generate the artifact.
+title: Short title of the artifact. Max 3 words.
+description: Short description of the artifact. Max 1 sentence.
+additional_dependencies: Additional dependencies required by the artifact. Do not include dependencies that are already included in the template.
+has_additional_dependencies: Detect if additional dependencies that are not included in the template are required by the artifact.
+install_dependencies_command: Command to install additional dependencies required by the artifact.
+port: Port number used by the resulted artifact. Null when no ports are exposed.
+file_path: Relative path to the file, including the file name.
+code: Code generated by the artifact. Only runnable code is allowed.
+
+Make sure to use the correct syntax for the programming language you're using. Make sure to generate only one code file. If you need to use CSS, make sure to include the CSS in the code file using Tailwind CSS syntax.
+`;
+
+// detail information to execute code
+export type CodeArtifact = {
+  commentary: string;
+  template: string;
+  title: string;
+  description: string;
+  additional_dependencies: string[];
+  has_additional_dependencies: boolean;
+  install_dependencies_command: string;
+  port: number | null;
+  file_path: string;
+  code: string;
+  files?: string[];
+};
+
+export type CodeGeneratorParameter = {
+  requirement: string;
+  oldCode?: string;
+  sandboxFiles?: string[];
+};
+
+export type CodeGeneratorToolParams = {
+  metadata?: ToolMetadata<JSONSchemaType<CodeGeneratorParameter>>;
+};
+
+const DEFAULT_META_DATA: ToolMetadata<JSONSchemaType<CodeGeneratorParameter>> =
+  {
+    name: "artifact",
+    description: `Generate a code artifact based on the input. Don't call this tool if the user has not asked for code generation. E.g. if the user asks to write a description or specification, don't call this tool.`,
+    parameters: {
+      type: "object",
+      properties: {
+        requirement: {
+          type: "string",
+          description: "The description of the application you want to build.",
+        },
+        oldCode: {
+          type: "string",
+          description: "The existing code to be modified",
+          nullable: true,
+        },
+        sandboxFiles: {
+          type: "array",
+          description:
+            "A list of sandbox file paths. Include these files if the code requires them.",
+          items: {
+            type: "string",
+          },
+          nullable: true,
+        },
+      },
+      required: ["requirement"],
+    },
+  };
+
+export class CodeGeneratorTool implements BaseTool<CodeGeneratorParameter> {
+  metadata: ToolMetadata<JSONSchemaType<CodeGeneratorParameter>>;
+
+  constructor(params?: CodeGeneratorToolParams) {
+    this.metadata = params?.metadata || DEFAULT_META_DATA;
+  }
+
+  async call(input: CodeGeneratorParameter) {
+    try {
+      const artifact = await this.generateArtifact(
+        input.requirement,
+        input.oldCode,
+      );
+      if (input.sandboxFiles) {
+        artifact.files = input.sandboxFiles;
+      }
+      return artifact as JSONValue;
+    } catch (error) {
+      return { isError: true };
+    }
+  }
+
+  // Generate artifact (code, environment, dependencies, etc.)
+  async generateArtifact(
+    query: string,
+    oldCode?: string,
+  ): Promise<CodeArtifact> {
+    const userMessage = `
+    ${query}
+    ${oldCode ? `The existing code is: \n\`\`\`${oldCode}\`\`\`` : ""}
+    `;
+    const messages: ChatMessage[] = [
+      { role: "system", content: CODE_GENERATION_PROMPT },
+      { role: "user", content: userMessage },
+    ];
+    try {
+      const response = await Settings.llm.chat({ messages });
+      const content = response.message.content.toString();
+      const jsonContent = content
+        .replace(/^```json\s*|\s*```$/g, "")
+        .replace(/^`+|`+$/g, "")
+        .trim();
+      const artifact = JSON.parse(jsonContent) as CodeArtifact;
+      return artifact;
+    } catch (error) {
+      console.log("Failed to generate artifact", error);
+      throw error;
+    }
+  }
+}

app/api/chat/engine/tools/document-generator.ts

@@ -0,0 +1,142 @@
+import { JSONSchemaType } from "ajv";
+import { BaseTool, ToolMetadata } from "llamaindex";
+import { marked } from "marked";
+import path from "node:path";
+import { saveDocument } from "../../llamaindex/documents/helper";
+
+const OUTPUT_DIR = "output/tools";
+
+type DocumentParameter = {
+  originalContent: string;
+  fileName: string;
+};
+
+const DEFAULT_METADATA: ToolMetadata<JSONSchemaType<DocumentParameter>> = {
+  name: "document_generator",
+  description:
+    "Generate HTML document from markdown content. Return a file url to the document",
+  parameters: {
+    type: "object",
+    properties: {
+      originalContent: {
+        type: "string",
+        description: "The original markdown content to convert.",
+      },
+      fileName: {
+        type: "string",
+        description: "The name of the document file (without extension).",
+      },
+    },
+    required: ["originalContent", "fileName"],
+  },
+};
+
+const COMMON_STYLES = `
+  body {
+    font-family: Arial, sans-serif;
+    line-height: 1.3;
+    color: #333;
+  }
+  h1, h2, h3, h4, h5, h6 {
+    margin-top: 1em;
+    margin-bottom: 0.5em;
+  }
+  p {
+    margin-bottom: 0.7em;
+  }
+  code {
+    background-color: #f4f4f4;
+    padding: 2px 4px;
+    border-radius: 4px;
+  }
+  pre {
+    background-color: #f4f4f4;
+    padding: 10px;
+    border-radius: 4px;
+    overflow-x: auto;
+  }
+  table {
+    border-collapse: collapse;
+    width: 100%;
+    margin-bottom: 1em;
+  }
+  th, td {
+    border: 1px solid #ddd;
+    padding: 8px;
+    text-align: left;
+  }
+  th {
+    background-color: #f2f2f2;
+    font-weight: bold;
+  }
+  img {
+    max-width: 90%;
+    height: auto;
+    display: block;
+    margin: 1em auto;
+    border-radius: 10px;
+  }
+`;
+
+const HTML_SPECIFIC_STYLES = `
+  body {
+    max-width: 800px;
+    margin: 0 auto;
+    padding: 20px;
+  }
+`;
+
+const HTML_TEMPLATE = `
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <style>
+        ${COMMON_STYLES}
+        ${HTML_SPECIFIC_STYLES}
+    </style>
+</head>
+<body>
+    {{content}}
+</body>
+</html>
+`;
+
+export interface DocumentGeneratorParams {
+  metadata?: ToolMetadata<JSONSchemaType<DocumentParameter>>;
+}
+
+export class DocumentGenerator implements BaseTool<DocumentParameter> {
+  metadata: ToolMetadata<JSONSchemaType<DocumentParameter>>;
+
+  constructor(params: DocumentGeneratorParams) {
+    this.metadata = params.metadata ?? DEFAULT_METADATA;
+  }
+
+  private static async generateHtmlContent(
+    originalContent: string,
+  ): Promise<string> {
+    return await marked(originalContent);
+  }
+
+  private static generateHtmlDocument(htmlContent: string): string {
+    return HTML_TEMPLATE.replace("{{content}}", htmlContent);
+  }
+
+  async call(input: DocumentParameter): Promise<string> {
+    const { originalContent, fileName } = input;
+
+    const htmlContent =
+      await DocumentGenerator.generateHtmlContent(originalContent);
+    const fileContent = DocumentGenerator.generateHtmlDocument(htmlContent);
+
+    const filePath = path.join(OUTPUT_DIR, `${fileName}.html`);
+
+    return `URL: ${await saveDocument(filePath, fileContent)}`;
+  }
+}
+
+export function getTools(): BaseTool[] {
+  return [new DocumentGenerator({})];
+}

app/api/chat/engine/tools/duckduckgo.ts

@@ -0,0 +1,78 @@
+import { JSONSchemaType } from "ajv";
+import { search } from "duck-duck-scrape";
+import { BaseTool, ToolMetadata } from "llamaindex";
+
+export type DuckDuckGoParameter = {
+  query: string;
+  region?: string;
+  maxResults?: number;
+};
+
+export type DuckDuckGoToolParams = {
+  metadata?: ToolMetadata<JSONSchemaType<DuckDuckGoParameter>>;
+};
+
+const DEFAULT_SEARCH_METADATA: ToolMetadata<
+  JSONSchemaType<DuckDuckGoParameter>
+> = {
+  name: "duckduckgo_search",
+  description:
+    "Use this function to search for information (only text) in the internet using DuckDuckGo.",
+  parameters: {
+    type: "object",
+    properties: {
+      query: {
+        type: "string",
+        description: "The query to search in DuckDuckGo.",
+      },
+      region: {
+        type: "string",
+        description:
+          "Optional, The region to be used for the search in [country-language] convention, ex us-en, uk-en, ru-ru, etc...",
+        nullable: true,
+      },
+      maxResults: {
+        type: "number",
+        description:
+          "Optional, The maximum number of results to be returned. Default is 10.",
+        nullable: true,
+      },
+    },
+    required: ["query"],
+  },
+};
+
+type DuckDuckGoSearchResult = {
+  title: string;
+  description: string;
+  url: string;
+};
+
+export class DuckDuckGoSearchTool implements BaseTool<DuckDuckGoParameter> {
+  metadata: ToolMetadata<JSONSchemaType<DuckDuckGoParameter>>;
+
+  constructor(params: DuckDuckGoToolParams) {
+    this.metadata = params.metadata ?? DEFAULT_SEARCH_METADATA;
+  }
+
+  async call(input: DuckDuckGoParameter) {
+    const { query, region, maxResults = 10 } = input;
+    const options = region ? { region } : {};
+    // Temporarily sleep to reduce overloading the DuckDuckGo
+    await new Promise((resolve) => setTimeout(resolve, 1000));
+
+    const searchResults = await search(query, options);
+
+    return searchResults.results.slice(0, maxResults).map((result) => {
+      return {
+        title: result.title,
+        description: result.description,
+        url: result.url,
+      } as DuckDuckGoSearchResult;
+    });
+  }
+}
+
+export function getTools() {
+  return [new DuckDuckGoSearchTool({})];
+}

app/api/chat/engine/tools/img-gen.ts

@@ -0,0 +1,112 @@
+import type { JSONSchemaType } from "ajv";
+import { FormData } from "formdata-node";
+import fs from "fs";
+import got from "got";
+import { BaseTool, ToolMetadata } from "llamaindex";
+import path from "node:path";
+import { Readable } from "stream";
+
+export type ImgGeneratorParameter = {
+  prompt: string;
+};
+
+export type ImgGeneratorToolParams = {
+  metadata?: ToolMetadata<JSONSchemaType<ImgGeneratorParameter>>;
+};
+
+export type ImgGeneratorToolOutput = {
+  isSuccess: boolean;
+  imageUrl?: string;
+  errorMessage?: string;
+};
+
+const DEFAULT_META_DATA: ToolMetadata<JSONSchemaType<ImgGeneratorParameter>> = {
+  name: "image_generator",
+  description: `Use this function to generate an image based on the prompt.`,
+  parameters: {
+    type: "object",
+    properties: {
+      prompt: {
+        type: "string",
+        description: "The prompt to generate the image",
+      },
+    },
+    required: ["prompt"],
+  },
+};
+
+export class ImgGeneratorTool implements BaseTool<ImgGeneratorParameter> {
+  readonly IMG_OUTPUT_FORMAT = "webp";
+  readonly IMG_OUTPUT_DIR = "output/tools";
+  readonly IMG_GEN_API =
+    "https://api.stability.ai/v2beta/stable-image/generate/core";
+
+  metadata: ToolMetadata<JSONSchemaType<ImgGeneratorParameter>>;
+
+  constructor(params?: ImgGeneratorToolParams) {
+    this.checkRequiredEnvVars();
+    this.metadata = params?.metadata || DEFAULT_META_DATA;
+  }
+
+  async call(input: ImgGeneratorParameter): Promise<ImgGeneratorToolOutput> {
+    return await this.generateImage(input.prompt);
+  }
+
+  private generateImage = async (
+    prompt: string,
+  ): Promise<ImgGeneratorToolOutput> => {
+    try {
+      const buffer = await this.promptToImgBuffer(prompt);
+      const imageUrl = this.saveImage(buffer);
+      return { isSuccess: true, imageUrl };
+    } catch (error) {
+      console.error(error);
+      return {
+        isSuccess: false,
+        errorMessage: "Failed to generate image. Please try again.",
+      };
+    }
+  };
+
+  private promptToImgBuffer = async (prompt: string) => {
+    const form = new FormData();
+    form.append("prompt", prompt);
+    form.append("output_format", this.IMG_OUTPUT_FORMAT);
+    const buffer = await got
+      .post(this.IMG_GEN_API, {
+        // Not sure why it shows an type error when passing form to body
+        // Although I follow document: https://github.com/sindresorhus/got/blob/main/documentation/2-options.md#body
+        // Tt still works fine, so I make casting to unknown to avoid the typescript warning
+        // Found a similar issue: https://github.com/sindresorhus/got/discussions/1877
+        body: form as unknown as Buffer | Readable | string,
+        headers: {
+          Authorization: `Bearer ${process.env.STABILITY_API_KEY}`,
+          Accept: "image/*",
+        },
+      })
+      .buffer();
+    return buffer;
+  };
+
+  private saveImage = (buffer: Buffer) => {
+    const filename = `${crypto.randomUUID()}.${this.IMG_OUTPUT_FORMAT}`;
+    const outputPath = path.join(this.IMG_OUTPUT_DIR, filename);
+    fs.writeFileSync(outputPath, buffer);
+    const url = `${process.env.FILESERVER_URL_PREFIX}/${this.IMG_OUTPUT_DIR}/${filename}`;
+    console.log(`Saved image to ${outputPath}.\nURL: ${url}`);
+    return url;
+  };
+
+  private checkRequiredEnvVars = () => {
+    if (!process.env.STABILITY_API_KEY) {
+      throw new Error(
+        "STABILITY_API_KEY key is required to run image generator. Get it here: https://platform.stability.ai/account/keys",
+      );
+    }
+    if (!process.env.FILESERVER_URL_PREFIX) {
+      throw new Error(
+        "FILESERVER_URL_PREFIX is required to display file output after generation",
+      );
+    }
+  };
+}

app/api/chat/engine/tools/index.ts

@@ -0,0 +1,72 @@
+import { BaseToolWithCall } from "llamaindex";
+import { ToolsFactory } from "llamaindex/tools/ToolsFactory";
+import { CodeGeneratorTool, CodeGeneratorToolParams } from "./code-generator";
+import {
+  DocumentGenerator,
+  DocumentGeneratorParams,
+} from "./document-generator";
+import { DuckDuckGoSearchTool, DuckDuckGoToolParams } from "./duckduckgo";
+import { ImgGeneratorTool, ImgGeneratorToolParams } from "./img-gen";
+import { InterpreterTool, InterpreterToolParams } from "./interpreter";
+import { OpenAPIActionTool } from "./openapi-action";
+import { WeatherTool, WeatherToolParams } from "./weather";
+
+type ToolCreator = (config: unknown) => Promise<BaseToolWithCall[]>;
+
+export async function createTools(toolConfig: {
+  local: Record<string, unknown>;
+  llamahub: any;
+}): Promise<BaseToolWithCall[]> {
+  // add local tools from the 'tools' folder (if configured)
+  const tools = await createLocalTools(toolConfig.local);
+  // add tools from LlamaIndexTS (if configured)
+  tools.push(...(await ToolsFactory.createTools(toolConfig.llamahub)));
+  return tools;
+}
+
+const toolFactory: Record<string, ToolCreator> = {
+  weather: async (config: unknown) => {
+    return [new WeatherTool(config as WeatherToolParams)];
+  },
+  interpreter: async (config: unknown) => {
+    return [new InterpreterTool(config as InterpreterToolParams)];
+  },
+  "openapi_action.OpenAPIActionToolSpec": async (config: unknown) => {
+    const { openapi_uri, domain_headers } = config as {
+      openapi_uri: string;
+      domain_headers: Record<string, Record<string, string>>;
+    };
+    const openAPIActionTool = new OpenAPIActionTool(
+      openapi_uri,
+      domain_headers,
+    );
+    return await openAPIActionTool.toToolFunctions();
+  },
+  duckduckgo: async (config: unknown) => {
+    return [new DuckDuckGoSearchTool(config as DuckDuckGoToolParams)];
+  },
+  img_gen: async (config: unknown) => {
+    return [new ImgGeneratorTool(config as ImgGeneratorToolParams)];
+  },
+  artifact: async (config: unknown) => {
+    return [new CodeGeneratorTool(config as CodeGeneratorToolParams)];
+  },
+  document_generator: async (config: unknown) => {
+    return [new DocumentGenerator(config as DocumentGeneratorParams)];
+  },
+};
+
+async function createLocalTools(
+  localConfig: Record<string, unknown>,
+): Promise<BaseToolWithCall[]> {
+  const tools: BaseToolWithCall[] = [];
+
+  for (const [key, toolConfig] of Object.entries(localConfig)) {
+    if (key in toolFactory) {
+      const newTools = await toolFactory[key](toolConfig);
+      tools.push(...newTools);
+    }
+  }
+
+  return tools;
+}

app/api/chat/engine/tools/interpreter.ts

@@ -0,0 +1,242 @@
+import { CodeInterpreter, Logs, Result } from "@e2b/code-interpreter";
+import type { JSONSchemaType } from "ajv";
+import fs from "fs";
+import { BaseTool, ToolMetadata } from "llamaindex";
+import crypto from "node:crypto";
+import path from "node:path";
+
+export type InterpreterParameter = {
+  code: string;
+  sandboxFiles?: string[];
+  retryCount?: number;
+};
+
+export type InterpreterToolParams = {
+  metadata?: ToolMetadata<JSONSchemaType<InterpreterParameter>>;
+  apiKey?: string;
+  fileServerURLPrefix?: string;
+};
+
+export type InterpreterToolOutput = {
+  isError: boolean;
+  logs: Logs;
+  text?: string;
+  extraResult: InterpreterExtraResult[];
+  retryCount?: number;
+};
+
+type InterpreterExtraType =
+  | "html"
+  | "markdown"
+  | "svg"
+  | "png"
+  | "jpeg"
+  | "pdf"
+  | "latex"
+  | "json"
+  | "javascript";
+
+export type InterpreterExtraResult = {
+  type: InterpreterExtraType;
+  content?: string;
+  filename?: string;
+  url?: string;
+};
+
+const DEFAULT_META_DATA: ToolMetadata<JSONSchemaType<InterpreterParameter>> = {
+  name: "interpreter",
+  description: `Execute python code in a Jupyter notebook cell and return any result, stdout, stderr, display_data, and error.
+If the code needs to use a file, ALWAYS pass the file path in the sandbox_files argument.
+You have a maximum of 3 retries to get the code to run successfully.
+`,
+  parameters: {
+    type: "object",
+    properties: {
+      code: {
+        type: "string",
+        description: "The python code to execute in a single cell.",
+      },
+      sandboxFiles: {
+        type: "array",
+        description:
+          "List of local file paths to be used by the code. The tool will throw an error if a file is not found.",
+        items: {
+          type: "string",
+        },
+        nullable: true,
+      },
+      retryCount: {
+        type: "number",
+        description: "The number of times the tool has been retried",
+        default: 0,
+        nullable: true,
+      },
+    },
+    required: ["code"],
+  },
+};
+
+export class InterpreterTool implements BaseTool<InterpreterParameter> {
+  private readonly outputDir = "output/tools";
+  private readonly uploadedFilesDir = "output/uploaded";
+  private apiKey?: string;
+  private fileServerURLPrefix?: string;
+  metadata: ToolMetadata<JSONSchemaType<InterpreterParameter>>;
+  codeInterpreter?: CodeInterpreter;
+
+  constructor(params?: InterpreterToolParams) {
+    this.metadata = params?.metadata || DEFAULT_META_DATA;
+    this.apiKey = params?.apiKey || process.env.E2B_API_KEY;
+    this.fileServerURLPrefix =
+      params?.fileServerURLPrefix || process.env.FILESERVER_URL_PREFIX;
+
+    if (!this.apiKey) {
+      throw new Error(
+        "E2B_API_KEY key is required to run code interpreter. Get it here: https://e2b.dev/docs/getting-started/api-key",
+      );
+    }
+    if (!this.fileServerURLPrefix) {
+      throw new Error(
+        "FILESERVER_URL_PREFIX is required to display file output from sandbox",
+      );
+    }
+  }
+
+  public async initInterpreter(input: InterpreterParameter) {
+    if (!this.codeInterpreter) {
+      this.codeInterpreter = await CodeInterpreter.create({
+        apiKey: this.apiKey,
+      });
+    }
+    // upload files to sandbox
+    if (input.sandboxFiles) {
+      console.log(`Uploading ${input.sandboxFiles.length} files to sandbox`);
+      for (const filePath of input.sandboxFiles) {
+        const fileName = path.basename(filePath);
+        const localFilePath = path.join(this.uploadedFilesDir, fileName);
+        const content = fs.readFileSync(localFilePath);
+        await this.codeInterpreter?.files.write(filePath, content);
+      }
+      console.log(`Uploaded ${input.sandboxFiles.length} files to sandbox`);
+    }
+    return this.codeInterpreter;
+  }
+
+  public async codeInterpret(
+    input: InterpreterParameter,
+  ): Promise<InterpreterToolOutput> {
+    console.log(
+      `Sandbox files: ${input.sandboxFiles}. Retry count: ${input.retryCount}`,
+    );
+
+    if (input.retryCount && input.retryCount >= 3) {
+      return {
+        isError: true,
+        logs: {
+          stdout: [],
+          stderr: [],
+        },
+        text: "Max retries reached",
+        extraResult: [],
+      };
+    }
+
+    console.log(
+      `\n${"=".repeat(50)}\n> Running following AI-generated code:\n${input.code}\n${"=".repeat(50)}`,
+    );
+    const interpreter = await this.initInterpreter(input);
+    const exec = await interpreter.notebook.execCell(input.code);
+    if (exec.error) console.error("[Code Interpreter error]", exec.error);
+    const extraResult = await this.getExtraResult(exec.results[0]);
+    const result: InterpreterToolOutput = {
+      isError: !!exec.error,
+      logs: exec.logs,
+      text: exec.text,
+      extraResult,
+      retryCount: input.retryCount ? input.retryCount + 1 : 1,
+    };
+    return result;
+  }
+
+  async call(input: InterpreterParameter): Promise<InterpreterToolOutput> {
+    const result = await this.codeInterpret(input);
+    return result;
+  }
+
+  async close() {
+    await this.codeInterpreter?.close();
+  }
+
+  private async getExtraResult(
+    res?: Result,
+  ): Promise<InterpreterExtraResult[]> {
+    if (!res) return [];
+    const output: InterpreterExtraResult[] = [];
+
+    try {
+      const formats = res.formats(); // formats available for the result. Eg: ['png', ...]
+      const results = formats.map((f) => res[f as keyof Result]); // get base64 data for each format
+
+      // save base64 data to file and return the url
+      for (let i = 0; i < formats.length; i++) {
+        const ext = formats[i];
+        const data = results[i];
+        switch (ext) {
+          case "png":
+          case "jpeg":
+          case "svg":
+          case "pdf":
+            const { filename } = this.saveToDisk(data, ext);
+            output.push({
+              type: ext as InterpreterExtraType,
+              filename,
+              url: this.getFileUrl(filename),
+            });
+            break;
+          default:
+            output.push({
+              type: ext as InterpreterExtraType,
+              content: data,
+            });
+            break;
+        }
+      }
+    } catch (error) {
+      console.error("Error when parsing e2b response", error);
+    }
+
+    return output;
+  }
+
+  // Consider saving to cloud storage instead but it may cost more for you
+  // See: https://e2b.dev/docs/sandbox/api/filesystem#write-to-file
+  private saveToDisk(
+    base64Data: string,
+    ext: string,
+  ): {
+    outputPath: string;
+    filename: string;
+  } {
+    const filename = `${crypto.randomUUID()}.${ext}`; // generate a unique filename
+    const buffer = Buffer.from(base64Data, "base64");
+    const outputPath = this.getOutputPath(filename);
+    fs.writeFileSync(outputPath, buffer);
+    console.log(`Saved file to ${outputPath}`);
+    return {
+      outputPath,
+      filename,
+    };
+  }
+
+  private getOutputPath(filename: string): string {
+    // if outputDir doesn't exist, create it
+    if (!fs.existsSync(this.outputDir)) {
+      fs.mkdirSync(this.outputDir, { recursive: true });
+    }
+    return path.join(this.outputDir, filename);
+  }
+
+  private getFileUrl(filename: string): string {
+    return `${this.fileServerURLPrefix}/${this.outputDir}/${filename}`;
+  }
+}

app/api/chat/engine/tools/openapi-action.ts

@@ -0,0 +1,164 @@
+import SwaggerParser from "@apidevtools/swagger-parser";
+import { JSONSchemaType } from "ajv";
+import got from "got";
+import { FunctionTool, JSONValue, ToolMetadata } from "llamaindex";
+
+interface DomainHeaders {
+  [key: string]: { [header: string]: string };
+}
+
+type Input = {
+  url: string;
+  params: object;
+};
+
+type APIInfo = {
+  description: string;
+  title: string;
+};
+
+export class OpenAPIActionTool {
+  // cache the loaded specs by URL
+  private static specs: Record<string, any> = {};
+
+  private readonly INVALID_URL_PROMPT =
+    "This url did not include a hostname or scheme. Please determine the complete URL and try again.";
+
+  private createLoadSpecMetaData = (info: APIInfo) => {
+    return {
+      name: "load_openapi_spec",
+      description: `Use this to retrieve the OpenAPI spec for the API named ${info.title} with the following description: ${info.description}. Call it before making any requests to the API.`,
+    };
+  };
+
+  private readonly createMethodCallMetaData = (
+    method: "POST" | "PATCH" | "GET",
+    info: APIInfo,
+  ) => {
+    return {
+      name: `${method.toLowerCase()}_request`,
+      description: `Use this to call the ${method} method on the API named ${info.title}`,
+      parameters: {
+        type: "object",
+        properties: {
+          url: {
+            type: "string",
+            description: `The url to make the ${method} request against`,
+          },
+          params: {
+            type: "object",
+            description:
+              method === "GET"
+                ? "the URL parameters to provide with the get request"
+                : `the key-value pairs to provide with the ${method} request`,
+          },
+        },
+        required: ["url"],
+      },
+    } as ToolMetadata<JSONSchemaType<Input>>;
+  };
+
+  constructor(
+    public openapi_uri: string,
+    public domainHeaders: DomainHeaders = {},
+  ) {}
+
+  async loadOpenapiSpec(url: string): Promise<any> {
+    const api = await SwaggerParser.validate(url);
+    return {
+      servers: "servers" in api ? api.servers : "",
+      info: { description: api.info.description, title: api.info.title },
+      endpoints: api.paths,
+    };
+  }
+
+  async getRequest(input: Input): Promise<JSONValue> {
+    if (!this.validUrl(input.url)) {
+      return this.INVALID_URL_PROMPT;
+    }
+    try {
+      const data = await got
+        .get(input.url, {
+          headers: this.getHeadersForUrl(input.url),
+          searchParams: input.params as URLSearchParams,
+        })
+        .json();
+      return data as JSONValue;
+    } catch (error) {
+      return error as JSONValue;
+    }
+  }
+
+  async postRequest(input: Input): Promise<JSONValue> {
+    if (!this.validUrl(input.url)) {
+      return this.INVALID_URL_PROMPT;
+    }
+    try {
+      const res = await got.post(input.url, {
+        headers: this.getHeadersForUrl(input.url),
+        json: input.params,
+      });
+      return res.body as JSONValue;
+    } catch (error) {
+      return error as JSONValue;
+    }
+  }
+
+  async patchRequest(input: Input): Promise<JSONValue> {
+    if (!this.validUrl(input.url)) {
+      return this.INVALID_URL_PROMPT;
+    }
+    try {
+      const res = await got.patch(input.url, {
+        headers: this.getHeadersForUrl(input.url),
+        json: input.params,
+      });
+      return res.body as JSONValue;
+    } catch (error) {
+      return error as JSONValue;
+    }
+  }
+
+  public async toToolFunctions() {
+    if (!OpenAPIActionTool.specs[this.openapi_uri]) {
+      console.log(`Loading spec for URL: ${this.openapi_uri}`);
+      const spec = await this.loadOpenapiSpec(this.openapi_uri);
+      OpenAPIActionTool.specs[this.openapi_uri] = spec;
+    }
+    const spec = OpenAPIActionTool.specs[this.openapi_uri];
+    // TODO: read endpoints with parameters from spec and create one tool for each endpoint
+    // For now, we just create a tool for each HTTP method which does not work well for passing parameters
+    return [
+      FunctionTool.from(() => {
+        return spec;
+      }, this.createLoadSpecMetaData(spec.info)),
+      FunctionTool.from(
+        this.getRequest.bind(this),
+        this.createMethodCallMetaData("GET", spec.info),
+      ),
+      FunctionTool.from(
+        this.postRequest.bind(this),
+        this.createMethodCallMetaData("POST", spec.info),
+      ),
+      FunctionTool.from(
+        this.patchRequest.bind(this),
+        this.createMethodCallMetaData("PATCH", spec.info),
+      ),
+    ];
+  }
+
+  private validUrl(url: string): boolean {
+    const parsed = new URL(url);
+    return !!parsed.protocol && !!parsed.hostname;
+  }
+
+  private getDomain(url: string): string {
+    const parsed = new URL(url);
+    return parsed.hostname;
+  }
+
+  private getHeadersForUrl(url: string): { [header: string]: string } {
+    const domain = this.getDomain(url);
+    return this.domainHeaders[domain] || {};
+  }
+}

app/api/chat/engine/tools/weather.ts

@@ -0,0 +1,81 @@
+import type { JSONSchemaType } from "ajv";
+import { BaseTool, ToolMetadata } from "llamaindex";
+
+interface GeoLocation {
+  id: string;
+  name: string;
+  latitude: number;
+  longitude: number;
+}
+
+export type WeatherParameter = {
+  location: string;
+};
+
+export type WeatherToolParams = {
+  metadata?: ToolMetadata<JSONSchemaType<WeatherParameter>>;
+};
+
+const DEFAULT_META_DATA: ToolMetadata<JSONSchemaType<WeatherParameter>> = {
+  name: "get_weather_information",
+  description: `
+    Use this function to get the weather of any given location.
+    Note that the weather code should follow WMO Weather interpretation codes (WW):
+    0: Clear sky
+    1, 2, 3: Mainly clear, partly cloudy, and overcast
+    45, 48: Fog and depositing rime fog
+    51, 53, 55: Drizzle: Light, moderate, and dense intensity
+    56, 57: Freezing Drizzle: Light and dense intensity
+    61, 63, 65: Rain: Slight, moderate and heavy intensity
+    66, 67: Freezing Rain: Light and heavy intensity
+    71, 73, 75: Snow fall: Slight, moderate, and heavy intensity
+    77: Snow grains
+    80, 81, 82: Rain showers: Slight, moderate, and violent
+    85, 86: Snow showers slight and heavy
+    95: Thunderstorm: Slight or moderate
+    96, 99: Thunderstorm with slight and heavy hail
+  `,
+  parameters: {
+    type: "object",
+    properties: {
+      location: {
+        type: "string",
+        description: "The location to get the weather information",
+      },
+    },
+    required: ["location"],
+  },
+};
+
+export class WeatherTool implements BaseTool<WeatherParameter> {
+  metadata: ToolMetadata<JSONSchemaType<WeatherParameter>>;
+
+  private getGeoLocation = async (location: string): Promise<GeoLocation> => {
+    const apiUrl = `https://geocoding-api.open-meteo.com/v1/search?name=${location}&count=10&language=en&format=json`;
+    const response = await fetch(apiUrl);
+    const data = await response.json();
+    const { id, name, latitude, longitude } = data.results[0];
+    return { id, name, latitude, longitude };
+  };
+
+  private getWeatherByLocation = async (location: string) => {
+    console.log(
+      "Calling open-meteo api to get weather information of location:",
+      location,
+    );
+    const { latitude, longitude } = await this.getGeoLocation(location);
+    const timezone = Intl.DateTimeFormat().resolvedOptions().timeZone;
+    const apiUrl = `https://api.open-meteo.com/v1/forecast?latitude=${latitude}&longitude=${longitude}&current=temperature_2m,weather_code&hourly=temperature_2m,weather_code&daily=weather_code&timezone=${timezone}`;
+    const response = await fetch(apiUrl);
+    const data = await response.json();
+    return data;
+  };
+
+  constructor(params?: WeatherToolParams) {
+    this.metadata = params?.metadata || DEFAULT_META_DATA;
+  }
+
+  async call(input: WeatherParameter) {
+    return await this.getWeatherByLocation(input.location);
+  }
+}

app/api/chat/llamaindex/documents/helper.ts

@@ -0,0 +1,108 @@
+import { Document } from "llamaindex";
+import crypto from "node:crypto";
+import fs from "node:fs";
+import path from "node:path";
+import { getExtractors } from "../../engine/loader";
+
+const MIME_TYPE_TO_EXT: Record<string, string> = {
+  "application/pdf": "pdf",
+  "text/plain": "txt",
+  "text/csv": "csv",
+  "application/vnd.openxmlformats-officedocument.wordprocessingml.document":
+    "docx",
+};
+
+const UPLOADED_FOLDER = "output/uploaded";
+
+export type FileMetadata = {
+  id: string;
+  name: string;
+  url: string;
+  refs: string[];
+};
+
+export async function storeAndParseFile(
+  filename: string,
+  fileBuffer: Buffer,
+  mimeType: string,
+): Promise<FileMetadata> {
+  const fileMetadata = await storeFile(filename, fileBuffer, mimeType);
+  const documents: Document[] = await parseFile(fileBuffer, filename, mimeType);
+  // Update document IDs in the file metadata
+  fileMetadata.refs = documents.map((document) => document.id_ as string);
+  return fileMetadata;
+}
+
+export async function storeFile(
+  filename: string,
+  fileBuffer: Buffer,
+  mimeType: string,
+) {
+  const fileExt = MIME_TYPE_TO_EXT[mimeType];
+  if (!fileExt) throw new Error(`Unsupported document type: ${mimeType}`);
+
+  const fileId = crypto.randomUUID();
+  const newFilename = `${fileId}_${sanitizeFileName(filename)}`;
+  const filepath = path.join(UPLOADED_FOLDER, newFilename);
+  const fileUrl = await saveDocument(filepath, fileBuffer);
+  return {
+    id: fileId,
+    name: newFilename,
+    url: fileUrl,
+    refs: [] as string[],
+  } as FileMetadata;
+}
+
+export async function parseFile(
+  fileBuffer: Buffer,
+  filename: string,
+  mimeType: string,
+) {
+  const documents = await loadDocuments(fileBuffer, mimeType);
+  for (const document of documents) {
+    document.metadata = {
+      ...document.metadata,
+      file_name: filename,
+      private: "true", // to separate private uploads from public documents
+    };
+  }
+  return documents;
+}
+
+async function loadDocuments(fileBuffer: Buffer, mimeType: string) {
+  const extractors = getExtractors();
+  const reader = extractors[MIME_TYPE_TO_EXT[mimeType]];
+
+  if (!reader) {
+    throw new Error(`Unsupported document type: ${mimeType}`);
+  }
+  console.log(`Processing uploaded document of type: ${mimeType}`);
+  return await reader.loadDataAsContent(fileBuffer);
+}
+
+// Save document to file server and return the file url
+export async function saveDocument(filepath: string, content: string | Buffer) {
+  if (path.isAbsolute(filepath)) {
+    throw new Error("Absolute file paths are not allowed.");
+  }
+  if (!process.env.FILESERVER_URL_PREFIX) {
+    throw new Error("FILESERVER_URL_PREFIX environment variable is not set.");
+  }
+
+  const dirPath = path.dirname(filepath);
+  await fs.promises.mkdir(dirPath, { recursive: true });
+
+  if (typeof content === "string") {
+    await fs.promises.writeFile(filepath, content, "utf-8");
+  } else {
+    await fs.promises.writeFile(filepath, content);
+  }
+
+  const fileurl = `${process.env.FILESERVER_URL_PREFIX}/${filepath}`;
+  console.log(`Saved document to ${filepath}. Reachable at URL: ${fileurl}`);
+  return fileurl;
+}
+
+function sanitizeFileName(fileName: string) {
+  return fileName.replace(/[^a-zA-Z0-9_.-]/g, "_");
+}

app/api/chat/llamaindex/documents/pipeline.ts

@@ -0,0 +1,38 @@
+import {
+  Document,
+  IngestionPipeline,
+  Settings,
+  SimpleNodeParser,
+  VectorStoreIndex,
+} from "llamaindex";
+
+export async function runPipeline(
+  currentIndex: VectorStoreIndex | null,
+  documents: Document[],
+) {
+  // Use ingestion pipeline to process the documents into nodes and add them to the vector store
+  const pipeline = new IngestionPipeline({
+    transformations: [
+      new SimpleNodeParser({
+        chunkSize: Settings.chunkSize,
+        chunkOverlap: Settings.chunkOverlap,
+      }),
+      Settings.embedModel,
+    ],
+  });
+  const nodes = await pipeline.run({ documents });
+  if (currentIndex) {
+    await currentIndex.insertNodes(nodes);
+    currentIndex.storageContext.docStore.persist();
+    console.log("Added nodes to the vector store.");
+    return documents.map((document) => document.id_);
+  } else {
+    // Initialize a new index with the documents
+    const newIndex = await VectorStoreIndex.fromDocuments(documents);
+    newIndex.storageContext.docStore.persist();
+    console.log(
+      "Got empty index, created new index with the uploaded documents",
+    );
+    return documents.map((document) => document.id_);
+  }
+}

app/api/chat/llamaindex/documents/upload.ts

@@ -0,0 +1,70 @@
+import { Document, LLamaCloudFileService, VectorStoreIndex } from "llamaindex";
+import { LlamaCloudIndex } from "llamaindex/cloud/LlamaCloudIndex";
+import fs from "node:fs/promises";
+import path from "node:path";
+import { FileMetadata, parseFile, storeFile } from "./helper";
+import { runPipeline } from "./pipeline";
+
+export async function uploadDocument(
+  index: VectorStoreIndex | LlamaCloudIndex | null,
+  filename: string,
+  raw: string,
+): Promise<FileMetadata> {
+  const [header, content] = raw.split(",");
+  const mimeType = header.replace("data:", "").replace(";base64", "");
+  const fileBuffer = Buffer.from(content, "base64");
+
+  // Store file
+  const fileMetadata = await storeFile(filename, fileBuffer, mimeType);
+
+  // If the file is csv and has codeExecutorTool, we don't need to index the file.
+  if (mimeType === "text/csv" && (await hasCodeExecutorTool())) {
+    return fileMetadata;
+  }
+
+  if (index instanceof LlamaCloudIndex) {
+    // trigger LlamaCloudIndex API to upload the file and run the pipeline
+    const projectId = await index.getProjectId();
+    const pipelineId = await index.getPipelineId();
+    try {
+      const documentId = await LLamaCloudFileService.addFileToPipeline(
+        projectId,
+        pipelineId,
+        new File([fileBuffer], filename, { type: mimeType }),
+        { private: "true" },
+      );
+      // Update file metadata with document IDs
+      fileMetadata.refs = [documentId];
+      return fileMetadata;
+    } catch (error) {
+      if (
+        error instanceof ReferenceError &&
+        error.message.includes("File is not defined")
+      ) {
+        throw new Error(
+          "File class is not supported in the current Node.js version. Please use Node.js 20 or higher.",
+        );
+      }
+      throw error;
+    }
+  }
+
+  // run the pipeline for other vector store indexes
+  const documents: Document[] = await parseFile(fileBuffer, filename, mimeType);
+  // Update file metadata with document IDs
+  fileMetadata.refs = documents.map((document) => document.id_ as string);
+  // Run the pipeline
+  await runPipeline(index, documents);
+  return fileMetadata;
+}
+
+const hasCodeExecutorTool = async () => {
+  const codeExecutorTools = ["interpreter", "artifact"];
+
+  const configFile = path.join("config", "tools.json");
+  const toolConfig = JSON.parse(await fs.readFile(configFile, "utf8"));
+
+  const localTools = toolConfig.local || {};
+  // Check if local tools contains codeExecutorTools
+  return codeExecutorTools.some((tool) => localTools[tool] !== undefined);
+};

app/api/chat/llamaindex/streaming/annotations.ts

@@ -0,0 +1,210 @@
+import { JSONValue, Message } from "ai";
+import { MessageContent, MessageContentDetail } from "llamaindex";
+
+export type DocumentFileType = "csv" | "pdf" | "txt" | "docx";
+
+export type UploadedFileMeta = {
+  id: string;
+  name: string;
+  url?: string;
+  refs?: string[];
+};
+
+export type DocumentFile = {
+  type: DocumentFileType;
+  url: string;
+  metadata: UploadedFileMeta;
+};
+
+type Annotation = {
+  type: string;
+  data: object;
+};
+
+export function isValidMessages(messages: Message[]): boolean {
+  const lastMessage =
+    messages && messages.length > 0 ? messages[messages.length - 1] : null;
+  return lastMessage !== null && lastMessage.role === "user";
+}
+
+export function retrieveDocumentIds(messages: Message[]): string[] {
+  // retrieve document Ids from the annotations of all messages (if any)
+  const documentFiles = retrieveDocumentFiles(messages);
+  return documentFiles.map((file) => file.metadata?.refs || []).flat();
+}
+
+export function retrieveDocumentFiles(messages: Message[]): DocumentFile[] {
+  const annotations = getAllAnnotations(messages);
+  if (annotations.length === 0) return [];
+
+  const files: DocumentFile[] = [];
+  for (const { type, data } of annotations) {
+    if (
+      type === "document_file" &&
+      "files" in data &&
+      Array.isArray(data.files)
+    ) {
+      files.push(...data.files);
+    }
+  }
+  return files;
+}
+
+export function retrieveMessageContent(messages: Message[]): MessageContent {
+  const userMessage = messages[messages.length - 1];
+  return [
+    {
+      type: "text",
+      text: userMessage.content,
+    },
+    ...retrieveLatestArtifact(messages),
+    ...convertAnnotations(messages),
+  ];
+}
+
+function getFileContent(file: DocumentFile): string {
+  const fileMetadata = file.metadata;
+  let defaultContent = `=====File: ${fileMetadata.name}=====\n`;
+  // Include file URL if it's available
+  const urlPrefix = process.env.FILESERVER_URL_PREFIX;
+  let urlContent = "";
+  if (urlPrefix) {
+    if (fileMetadata.url) {
+      urlContent = `File URL: ${fileMetadata.url}\n`;
+    } else {
+      urlContent = `File URL (instruction: do not update this file URL yourself): ${urlPrefix}/output/uploaded/${fileMetadata.name}\n`;
+    }
+  } else {
+    console.warn(
+      "Warning: FILESERVER_URL_PREFIX not set in environment variables. Can't use file server",
+    );
+  }
+  defaultContent += urlContent;
+
+  // Include document IDs if it's available
+  if (fileMetadata.refs) {
+    defaultContent += `Document IDs: ${fileMetadata.refs}\n`;
+  }
+  // Include sandbox file paths
+  const sandboxFilePath = `/tmp/${fileMetadata.name}`;
+  defaultContent += `Sandbox file path (instruction: only use sandbox path for artifact or code interpreter tool): ${sandboxFilePath}\n`;
+
+  return defaultContent;
+}
+
+function getAllAnnotations(messages: Message[]): Annotation[] {
+  return messages.flatMap((message) =>
+    (message.annotations ?? []).map((annotation) =>
+      getValidAnnotation(annotation),
+    ),
+  );
+}
+
+// get latest artifact from annotations to append to the user message
+function retrieveLatestArtifact(messages: Message[]): MessageContentDetail[] {
+  const annotations = getAllAnnotations(messages);
+  if (annotations.length === 0) return [];
+
+  for (const { type, data } of annotations.reverse()) {
+    if (
+      type === "tools" &&
+      "toolCall" in data &&
+      "toolOutput" in data &&
+      typeof data.toolCall === "object" &&
+      typeof data.toolOutput === "object" &&
+      data.toolCall !== null &&
+      data.toolOutput !== null &&
+      "name" in data.toolCall &&
+      data.toolCall.name === "artifact"
+    ) {
+      const toolOutput = data.toolOutput as { output?: { code?: string } };
+      if (toolOutput.output?.code) {
+        return [
+          {
+            type: "text",
+            text: `The existing code is:\n\`\`\`\n${toolOutput.output.code}\n\`\`\``,
+          },
+        ];
+      }
+    }
+  }
+  return [];
+}
+
+function convertAnnotations(messages: Message[]): MessageContentDetail[] {
+  // annotations from the last user message that has annotations
+  const annotations: Annotation[] =
+    messages
+      .slice()
+      .reverse()
+      .find((message) => message.role === "user" && message.annotations)
+      ?.annotations?.map(getValidAnnotation) || [];
+  if (annotations.length === 0) return [];
+
+  const content: MessageContentDetail[] = [];
+  annotations.forEach(({ type, data }) => {
+    // convert image
+    if (type === "image" && "url" in data && typeof data.url === "string") {
+      content.push({
+        type: "image_url",
+        image_url: {
+          url: data.url,
+        },
+      });
+    }
+    // convert the content of files to a text message
+    if (
+      type === "document_file" &&
+      "files" in data &&
+      Array.isArray(data.files)
+    ) {
+      const fileContent = data.files.map(getFileContent).join("\n");
+      content.push({
+        type: "text",
+        text: fileContent,
+      });
+    }
+  });
+
+  return content;
+}
+
+function getValidAnnotation(annotation: JSONValue): Annotation {
+  if (
+    !(
+      annotation &&
+      typeof annotation === "object" &&
+      "type" in annotation &&
+      typeof annotation.type === "string" &&
+      "data" in annotation &&
+      annotation.data &&
+      typeof annotation.data === "object"
+    )
+  ) {
+    throw new Error("Client sent invalid annotation. Missing data and type");
+  }
+  return { type: annotation.type, data: annotation.data };
+}
+
+// validate and get all annotations of a specific type or role from the frontend messages
+export function getAnnotations<
+  T extends Annotation["data"] = Annotation["data"],
+>(
+  messages: Message[],
+  options?: {
+    role?: Message["role"]; // message role
+    type?: Annotation["type"]; // annotation type
+  },
+): {
+  type: string;
+  data: T;
+}[] {
+  const messagesByRole = options?.role
+    ? messages.filter((msg) => msg.role === options?.role)
+    : messages;
+  const annotations = getAllAnnotations(messagesByRole);
+  const annotationsByType = options?.type
+    ? annotations.filter((a) => a.type === options.type)
+    : annotations;
+  return annotationsByType as { type: string; data: T }[];
+}

app/api/chat/llamaindex/streaming/events.ts

@@ -0,0 +1,182 @@
+import { StreamData } from "ai";
+import {
+  CallbackManager,
+  LLamaCloudFileService,
+  Metadata,
+  MetadataMode,
+  NodeWithScore,
+  ToolCall,
+  ToolOutput,
+} from "llamaindex";
+import path from "node:path";
+import { DATA_DIR } from "../../engine/loader";
+import { downloadFile } from "./file";
+
+const LLAMA_CLOUD_DOWNLOAD_FOLDER = "output/llamacloud";
+
+export function appendSourceData(
+  data: StreamData,
+  sourceNodes?: NodeWithScore<Metadata>[],
+) {
+  if (!sourceNodes?.length) return;
+  try {
+    const nodes = sourceNodes.map((node) => ({
+      metadata: node.node.metadata,
+      id: node.node.id_,
+      score: node.score ?? null,
+      url: getNodeUrl(node.node.metadata),
+      text: node.node.getContent(MetadataMode.NONE),
+    }));
+    data.appendMessageAnnotation({
+      type: "sources",
+      data: {
+        nodes,
+      },
+    });
+  } catch (error) {
+    console.error("Error appending source data:", error);
+  }
+}
+
+export function appendEventData(data: StreamData, title?: string) {
+  if (!title) return;
+  data.appendMessageAnnotation({
+    type: "events",
+    data: {
+      title,
+    },
+  });
+}
+
+export function appendToolData(
+  data: StreamData,
+  toolCall: ToolCall,
+  toolOutput: ToolOutput,
+) {
+  data.appendMessageAnnotation({
+    type: "tools",
+    data: {
+      toolCall: {
+        id: toolCall.id,
+        name: toolCall.name,
+        input: toolCall.input,
+      },
+      toolOutput: {
+        output: toolOutput.output,
+        isError: toolOutput.isError,
+      },
+    },
+  });
+}
+
+export function createCallbackManager(stream: StreamData) {
+  const callbackManager = new CallbackManager();
+
+  callbackManager.on("retrieve-end", (data) => {
+    const { nodes, query } = data.detail;
+    appendSourceData(stream, nodes);
+    appendEventData(stream, `Retrieving context for query: '${query.query}'`);
+    appendEventData(
+      stream,
+      `Retrieved ${nodes.length} sources to use as context for the query`,
+    );
+    downloadFilesFromNodes(nodes); // don't await to avoid blocking chat streaming
+  });
+
+  callbackManager.on("llm-tool-call", (event) => {
+    const { name, input } = event.detail.toolCall;
+    const inputString = Object.entries(input)
+      .map(([key, value]) => `${key}: ${value}`)
+      .join(", ");
+    appendEventData(
+      stream,
+      `Using tool: '${name}' with inputs: '${inputString}'`,
+    );
+  });
+
+  callbackManager.on("llm-tool-result", (event) => {
+    const { toolCall, toolResult } = event.detail;
+    appendToolData(stream, toolCall, toolResult);
+  });
+
+  return callbackManager;
+}
+
+function getNodeUrl(metadata: Metadata) {
+  if (!process.env.FILESERVER_URL_PREFIX) {
+    console.warn(
+      "FILESERVER_URL_PREFIX is not set. File URLs will not be generated.",
+    );
+  }
+  const fileName = metadata["file_name"];
+  if (fileName && process.env.FILESERVER_URL_PREFIX) {
+    // file_name exists and file server is configured
+    const pipelineId = metadata["pipeline_id"];
+    if (pipelineId) {
+      const name = toDownloadedName(pipelineId, fileName);
+      return `${process.env.FILESERVER_URL_PREFIX}/${LLAMA_CLOUD_DOWNLOAD_FOLDER}/${name}`;
+    }
+    const isPrivate = metadata["private"] === "true";
+    if (isPrivate) {
+      return `${process.env.FILESERVER_URL_PREFIX}/output/uploaded/${fileName}`;
+    }
+    const filePath = metadata["file_path"];
+    const dataDir = path.resolve(DATA_DIR);
+
+    if (filePath && dataDir) {
+      const relativePath = path.relative(dataDir, filePath);
+      return `${process.env.FILESERVER_URL_PREFIX}/data/${relativePath}`;
+    }
+  }
+  // fallback to URL in metadata (e.g. for websites)
+  return metadata["URL"];
+}
+
+async function downloadFilesFromNodes(nodes: NodeWithScore<Metadata>[]) {
+  try {
+    const files = nodesToLlamaCloudFiles(nodes);
+    for (const { pipelineId, fileName, downloadedName } of files) {
+      const downloadUrl = await LLamaCloudFileService.getFileUrl(
+        pipelineId,
+        fileName,
+      );
+      if (downloadUrl) {
+        await downloadFile(
+          downloadUrl,
+          downloadedName,
+          LLAMA_CLOUD_DOWNLOAD_FOLDER,
+        );
+      }
+    }
+  } catch (error) {
+    console.error("Error downloading files from nodes:", error);
+  }
+}
+
+function nodesToLlamaCloudFiles(nodes: NodeWithScore<Metadata>[]) {
+  const files: Array<{
+    pipelineId: string;
+    fileName: string;
+    downloadedName: string;
+  }> = [];
+  for (const node of nodes) {
+    const pipelineId = node.node.metadata["pipeline_id"];
+    const fileName = node.node.metadata["file_name"];
+    if (!pipelineId || !fileName) continue;
+    const isDuplicate = files.some(
+      (f) => f.pipelineId === pipelineId && f.fileName === fileName,
+    );
+    if (!isDuplicate) {
+      files.push({
+        pipelineId,
+        fileName,
+        downloadedName: toDownloadedName(pipelineId, fileName),
+      });
+    }
+  }
+  return files;
+}
+
+function toDownloadedName(pipelineId: string, fileName: string) {
+  return `${pipelineId}$${fileName}`;
+}

app/api/chat/llamaindex/streaming/file.ts

@@ -0,0 +1,35 @@
+import fs from "node:fs";
+import https from "node:https";
+import path from "node:path";
+
+export async function downloadFile(
+  urlToDownload: string,
+  filename: string,
+  folder = "output/uploaded",
+) {
+  try {
+    const downloadedPath = path.join(folder, filename);
+
+    // Check if file already exists
+    if (fs.existsSync(downloadedPath)) return;
+
+    const file = fs.createWriteStream(downloadedPath);
+    https
+      .get(urlToDownload, (response) => {
+        response.pipe(file);
+        file.on("finish", () => {
+          file.close(() => {
+            console.log("File downloaded successfully");
+          });
+        });
+      })
+      .on("error", (err) => {
+        fs.unlink(downloadedPath, () => {
+          console.error("Error downloading file:", err);
+          throw err;
+        });
+      });
+  } catch (error) {
+    throw new Error(`Error downloading file: ${error}`);
+  }
+}

app/api/chat/llamaindex/streaming/suggestion.ts

@@ -0,0 +1,43 @@
+import { ChatMessage, Settings } from "llamaindex";
+
+export async function generateNextQuestions(conversation: ChatMessage[]) {
+  const llm = Settings.llm;
+  const NEXT_QUESTION_PROMPT = process.env.NEXT_QUESTION_PROMPT;
+  if (!NEXT_QUESTION_PROMPT) {
+    return [];
+  }
+
+  // Format conversation
+  const conversationText = conversation
+    .map((message) => `${message.role}: ${message.content}`)
+    .join("\n");
+  const message = NEXT_QUESTION_PROMPT.replace(
+    "{conversation}",
+    conversationText,
+  );
+
+  try {
+    const response = await llm.complete({ prompt: message });
+    const questions = extractQuestions(response.text);
+    return questions;
+  } catch (error) {
+    console.error("Error when generating the next questions: ", error);
+    return [];
+  }
+}
+
+// TODO: instead of parsing the LLM's result we can use structured predict, once LITS supports it
+function extractQuestions(text: string): string[] {
+  // Extract the text inside the triple backticks
+  // @ts-ignore
+  const contentMatch = text.match(/```(.*?)```/s);
+  const content = contentMatch ? contentMatch[1] : "";
+
+  // Split the content by newlines to get each question
+  const questions = content
+    .split("\n")
+    .map((question) => question.trim())
+    .filter((question) => question !== "");
+
+  return questions;
+}

app/api/chat/route.ts

@@ -0,0 +1,90 @@
+import { initObservability } from "@/app/observability";
+import { LlamaIndexAdapter, Message, StreamData } from "ai";
+import { ChatMessage, Settings } from "llamaindex";
+import { NextRequest, NextResponse } from "next/server";
+import { createChatEngine } from "./engine/chat";
+import { initSettings } from "./engine/settings";
+import {
+  isValidMessages,
+  retrieveDocumentIds,
+  retrieveMessageContent,
+} from "./llamaindex/streaming/annotations";
+import { createCallbackManager } from "./llamaindex/streaming/events";
+import { generateNextQuestions } from "./llamaindex/streaming/suggestion";
+
+initObservability();
+initSettings();
+
+export const runtime = "nodejs";
+export const dynamic = "force-dynamic";
+
+export async function POST(request: NextRequest) {
+  // Init Vercel AI StreamData and timeout
+  const vercelStreamData = new StreamData();
+
+  try {
+    const body = await request.json();
+    const { messages, data }: { messages: Message[]; data?: any } = body;
+    if (!isValidMessages(messages)) {
+      return NextResponse.json(
+        {
+          error:
+            "messages are required in the request body and the last message must be from the user",
+        },
+        { status: 400 },
+      );
+    }
+
+    // retrieve document ids from the annotations of all messages (if any)
+    const ids = retrieveDocumentIds(messages);
+    // create chat engine with index using the document ids
+    const chatEngine = await createChatEngine(ids, data);
+
+    // retrieve user message content from Vercel/AI format
+    const userMessageContent = retrieveMessageContent(messages);
+
+    // Setup callbacks
+    const callbackManager = createCallbackManager(vercelStreamData);
+    const chatHistory: ChatMessage[] = messages as ChatMessage[];
+
+    // Calling LlamaIndex's ChatEngine to get a streamed response
+    const response = await Settings.withCallbackManager(callbackManager, () => {
+      return chatEngine.chat({
+        message: userMessageContent,
+        chatHistory,
+        stream: true,
+      });
+    });
+
+    const onFinal = (content: string) => {
+      chatHistory.push({ role: "assistant", content: content });
+      generateNextQuestions(chatHistory)
+        .then((questions: string[]) => {
+          if (questions.length > 0) {
+            vercelStreamData.appendMessageAnnotation({
+              type: "suggested_questions",
+              data: questions,
+            });
+          }
+        })
+        .finally(() => {
+          vercelStreamData.close();
+        });
+    };
+
+    return LlamaIndexAdapter.toDataStreamResponse(response, {
+      data: vercelStreamData,
+      callbacks: { onFinal },
+    });
+  } catch (error) {
+    console.error("[LlamaIndex]", error);
+    return NextResponse.json(
+      {
+        detail: (error as Error).message,
+      },
+      {
+        status: 500,
+      },
+    );
+  }
+}

app/api/chat/upload/route.ts

@@ -0,0 +1,34 @@
+import { NextRequest, NextResponse } from "next/server";
+import { getDataSource } from "../engine";
+import { initSettings } from "../engine/settings";
+import { uploadDocument } from "../llamaindex/documents/upload";
+
+initSettings();
+
+export const runtime = "nodejs";
+export const dynamic = "force-dynamic";
+
+export async function POST(request: NextRequest) {
+  try {
+    const {
+      filename,
+      base64,
+      params,
+    }: { filename: string; base64: string; params?: any } =
+      await request.json();
+    if (!base64 || !filename) {
+      return NextResponse.json(
+        { error: "base64 and filename is required in the request body" },
+        { status: 400 },
+      );
+    }
+    const index = await getDataSource(params);
+    return NextResponse.json(await uploadDocument(index, filename, base64));
+  } catch (error) {
+    console.error("[Upload API]", error);
+    return NextResponse.json(
+      { error: (error as Error).message },
+      { status: 500 },
+    );
+  }
+}

app/api/files/[...slug]/route.ts

@@ -0,0 +1,50 @@
+import { readFile } from "fs/promises";
+import { NextRequest, NextResponse } from "next/server";
+import path from "path";
+import { DATA_DIR } from "../../chat/engine/loader";
+
+/**
+ * This API is to get file data from allowed folders
+ * It receives path slug and response file data like serve static file
+ */
+export async function GET(
+  _request: NextRequest,
+  { params }: { params: { slug: string[] } },
+) {
+  const slug = params.slug;
+
+  if (!slug) {
+    return NextResponse.json({ detail: "Missing file slug" }, { status: 400 });
+  }
+
+  if (slug.includes("..") || path.isAbsolute(path.join(...slug))) {
+    return NextResponse.json({ detail: "Invalid file path" }, { status: 400 });
+  }
+
+  const [folder, ...pathTofile] = params.slug; // data, file.pdf
+  const allowedFolders = ["data", "output"];
+
+  if (!allowedFolders.includes(folder)) {
+    return NextResponse.json({ detail: "No permission" }, { status: 400 });
+  }
+
+  try {
+    const filePath = path.join(
+      process.cwd(),
+      folder === "data" ? DATA_DIR : folder,
+      path.join(...pathTofile),
+    );
+    const blob = await readFile(filePath);
+
+    return new NextResponse(blob, {
+      status: 200,
+      statusText: "OK",
+      headers: {
+        "Content-Length": blob.byteLength.toString(),
+      },
+    });
+  } catch (error) {
+    console.error(error);
+    return NextResponse.json({ detail: "File not found" }, { status: 404 });
+  }
+}

app/api/sandbox/route.ts

@@ -0,0 +1,157 @@
+/*
+ * Copyright 2023 FoundryLabs, Inc.
+ * Portions of this file are copied from the e2b project (https://github.com/e2b-dev/ai-artifacts)
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {
+  CodeInterpreter,
+  ExecutionError,
+  Result,
+  Sandbox,
+} from "@e2b/code-interpreter";
+import fs from "node:fs/promises";
+import path from "node:path";
+import { saveDocument } from "../chat/llamaindex/documents/helper";
+
+type CodeArtifact = {
+  commentary: string;
+  template: string;
+  title: string;
+  description: string;
+  additional_dependencies: string[];
+  has_additional_dependencies: boolean;
+  install_dependencies_command: string;
+  port: number | null;
+  file_path: string;
+  code: string;
+  files?: string[];
+};
+
+const sandboxTimeout = 10 * 60 * 1000; // 10 minute in ms
+
+export const maxDuration = 60;
+
+export type ExecutionResult = {
+  template: string;
+  stdout: string[];
+  stderr: string[];
+  runtimeError?: ExecutionError;
+  outputUrls: Array<{ url: string; filename: string }>;
+  url: string;
+};
+
+export async function POST(req: Request) {
+  const { artifact }: { artifact: CodeArtifact } = await req.json();
+
+  let sbx: Sandbox | CodeInterpreter | undefined = undefined;
+
+  // Create a interpreter or a sandbox
+  if (artifact.template === "code-interpreter-multilang") {
+    sbx = await CodeInterpreter.create({
+      metadata: { template: artifact.template },
+      timeoutMs: sandboxTimeout,
+    });
+    console.log("Created code interpreter", sbx.sandboxID);
+  } else {
+    sbx = await Sandbox.create(artifact.template, {
+      metadata: { template: artifact.template, userID: "default" },
+      timeoutMs: sandboxTimeout,
+    });
+    console.log("Created sandbox", sbx.sandboxID);
+  }
+
+  // Install packages
+  if (artifact.has_additional_dependencies) {
+    if (sbx instanceof CodeInterpreter) {
+      await sbx.notebook.execCell(artifact.install_dependencies_command);
+      console.log(
+        `Installed dependencies: ${artifact.additional_dependencies.join(", ")} in code interpreter ${sbx.sandboxID}`,
+      );
+    } else if (sbx instanceof Sandbox) {
+      await sbx.commands.run(artifact.install_dependencies_command);
+      console.log(
+        `Installed dependencies: ${artifact.additional_dependencies.join(", ")} in sandbox ${sbx.sandboxID}`,
+      );
+    }
+  }
+
+  // Copy files
+  if (artifact.files) {
+    artifact.files.forEach(async (sandboxFilePath) => {
+      const fileName = path.basename(sandboxFilePath);
+      const localFilePath = path.join("output", "uploaded", fileName);
+      const fileContent = await fs.readFile(localFilePath);
+
+      await sbx.files.write(sandboxFilePath, fileContent);
+      console.log(`Copied file to ${sandboxFilePath} in ${sbx.sandboxID}`);
+    });
+  }
+
+  // Copy code to fs
+  if (artifact.code && Array.isArray(artifact.code)) {
+    artifact.code.forEach(async (file) => {
+      await sbx.files.write(file.file_path, file.file_content);
+      console.log(`Copied file to ${file.file_path} in ${sbx.sandboxID}`);
+    });
+  } else {
+    await sbx.files.write(artifact.file_path, artifact.code);
+    console.log(`Copied file to ${artifact.file_path} in ${sbx.sandboxID}`);
+  }
+
+  // Execute code or return a URL to the running sandbox
+  if (artifact.template === "code-interpreter-multilang") {
+    const result = await (sbx as CodeInterpreter).notebook.execCell(
+      artifact.code || "",
+    );
+    await (sbx as CodeInterpreter).close();
+    const outputUrls = await downloadCellResults(result.results);
+    return new Response(
+      JSON.stringify({
+        template: artifact.template,
+        stdout: result.logs.stdout,
+        stderr: result.logs.stderr,
+        runtimeError: result.error,
+        outputUrls: outputUrls,
+      }),
+    );
+  } else {
+    return new Response(
+      JSON.stringify({
+        template: artifact.template,
+        url: `https://${sbx?.getHost(artifact.port || 80)}`,
+      }),
+    );
+  }
+}
+
+async function downloadCellResults(
+  cellResults?: Result[],
+): Promise<Array<{ url: string; filename: string }>> {
+  if (!cellResults) return [];
+  const results = await Promise.all(
+    cellResults.map(async (res) => {
+      const formats = res.formats(); // available formats in the result
+      const formatResults = await Promise.all(
+        formats.map(async (ext) => {
+          const filename = `${crypto.randomUUID()}.${ext}`;
+          const base64 = res[ext as keyof Result];
+          const buffer = Buffer.from(base64, "base64");
+          const fileurl = await saveDocument(filename, buffer);
+          return { url: fileurl, filename };
+        }),
+      );
+      return formatResults;
+    }),
+  );
+  return results.flat();
+}

app/components/chat-section.tsx

@@ -0,0 +1,57 @@
+"use client";
+
+import { useChat } from "ai/react";
+import { useState } from "react";
+import { ChatInput, ChatMessages } from "./ui/chat";
+import { useClientConfig } from "./ui/chat/hooks/use-config";
+
+export default function ChatSection() {
+  const { backend } = useClientConfig();
+  const [requestData, setRequestData] = useState<any>();
+  const {
+    messages,
+    input,
+    isLoading,
+    handleSubmit,
+    handleInputChange,
+    reload,
+    stop,
+    append,
+    setInput,
+  } = useChat({
+    body: { data: requestData },
+    api: `${backend}/api/chat`,
+    headers: {
+      "Content-Type": "application/json", // using JSON because of vercel/ai 2.2.26
+    },
+    onError: (error: unknown) => {
+      if (!(error instanceof Error)) throw error;
+      const message = JSON.parse(error.message);
+      alert(message.detail);
+    },
+    sendExtraMessageFields: true,
+  });
+
+  return (
+    <div className="space-y-4 w-full h-full flex flex-col">
+      <ChatMessages
+        messages={messages}
+        isLoading={isLoading}
+        reload={reload}
+        stop={stop}
+        append={append}
+      />
+      <ChatInput
+        input={input}
+        handleSubmit={handleSubmit}
+        handleInputChange={handleInputChange}
+        isLoading={isLoading}
+        messages={messages}
+        append={append}
+        setInput={setInput}
+        requestParams={{ params: requestData }}
+        setRequestData={setRequestData}
+      />
+    </div>
+  );
+}

app/components/header.tsx

@@ -0,0 +1,28 @@
+import Image from "next/image";
+
+export default function Header() {
+  return (
+    <div className="z-10 max-w-5xl w-full items-center justify-between font-mono text-sm lg:flex">
+      <p className="fixed left-0 top-0 flex w-full justify-center border-b border-gray-300 bg-gradient-to-b from-zinc-200 pb-6 pt-8 backdrop-blur-2xl dark:border-neutral-800 dark:bg-zinc-800/30 dark:from-inherit lg:static lg:w-auto  lg:rounded-xl lg:border lg:bg-gray-200 lg:p-4 lg:dark:bg-zinc-800/30">
+        Get started by editing&nbsp;
+        <code className="font-mono font-bold">app/page.tsx</code>
+      </p>
+      <div className="fixed bottom-0 left-0 mb-4 flex h-auto w-full items-end justify-center bg-gradient-to-t from-white via-white dark:from-black dark:via-black lg:static lg:w-auto lg:bg-none lg:mb-0">
+        <a
+          href="https://www.llamaindex.ai/"
+          className="flex items-center justify-center font-nunito text-lg font-bold gap-2"
+        >
+          <span>Built by LlamaIndex</span>
+          <Image
+            className="rounded-xl"
+            src="/llama.png"
+            alt="Llama Logo"
+            width={40}
+            height={40}
+            priority
+          />
+        </a>
+      </div>
+    </div>
+  );
+}

app/components/ui/README.md

@@ -0,0 +1 @@
+Using the chat component from https://github.com/marcusschiesser/ui (based on https://ui.shadcn.com/)

app/components/ui/button.tsx

@@ -0,0 +1,56 @@
+import { Slot } from "@radix-ui/react-slot";
+import { cva, type VariantProps } from "class-variance-authority";
+import * as React from "react";
+
+import { cn } from "./lib/utils";
+
+const buttonVariants = cva(
+  "inline-flex items-center justify-center whitespace-nowrap rounded-md text-sm font-medium ring-offset-background transition-colors focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring focus-visible:ring-offset-2 disabled:pointer-events-none disabled:opacity-50",
+  {
+    variants: {
+      variant: {
+        default: "bg-primary text-primary-foreground hover:bg-primary/90",
+        destructive:
+          "bg-destructive text-destructive-foreground hover:bg-destructive/90",
+        outline:
+          "border border-input bg-background hover:bg-accent hover:text-accent-foreground",
+        secondary:
+          "bg-secondary text-secondary-foreground hover:bg-secondary/80",
+        ghost: "hover:bg-accent hover:text-accent-foreground",
+        link: "text-primary underline-offset-4 hover:underline",
+      },
+      size: {
+        default: "h-10 px-4 py-2",
+        sm: "h-9 rounded-md px-3",
+        lg: "h-11 rounded-md px-8",
+        icon: "h-10 w-10",
+      },
+    },
+    defaultVariants: {
+      variant: "default",
+      size: "default",
+    },
+  },
+);
+
+export interface ButtonProps
+  extends React.ButtonHTMLAttributes<HTMLButtonElement>,
+    VariantProps<typeof buttonVariants> {
+  asChild?: boolean;
+}
+
+const Button = React.forwardRef<HTMLButtonElement, ButtonProps>(
+  ({ className, variant, size, asChild = false, ...props }, ref) => {
+    const Comp = asChild ? Slot : "button";
+    return (
+      <Comp
+        className={cn(buttonVariants({ variant, size, className }))}
+        ref={ref}
+        {...props}
+      />
+    );
+  },
+);
+Button.displayName = "Button";
+
+export { Button, buttonVariants };

app/components/ui/chat/chat-actions.tsx

@@ -0,0 +1,28 @@
+import { PauseCircle, RefreshCw } from "lucide-react";
+
+import { Button } from "../button";
+import { ChatHandler } from "./chat.interface";
+
+export default function ChatActions(
+  props: Pick<ChatHandler, "stop" | "reload"> & {
+    showReload?: boolean;
+    showStop?: boolean;
+  },
+) {
+  return (
+    <div className="space-x-4">
+      {props.showStop && (
+        <Button variant="outline" size="sm" onClick={props.stop}>
+          <PauseCircle className="mr-2 h-4 w-4" />
+          Stop generating
+        </Button>
+      )}
+      {props.showReload && (
+        <Button variant="outline" size="sm" onClick={props.reload}>
+          <RefreshCw className="mr-2 h-4 w-4" />
+          Regenerate
+        </Button>
+      )}
+    </div>
+  );
+}

app/components/ui/chat/chat-input.tsx

@@ -0,0 +1,136 @@
+import { JSONValue } from "ai";
+import React from "react";
+import { Button } from "../button";
+import { DocumentPreview } from "../document-preview";
+import FileUploader from "../file-uploader";
+import { Textarea } from "../textarea";
+import UploadImagePreview from "../upload-image-preview";
+import { ChatHandler } from "./chat.interface";
+import { useFile } from "./hooks/use-file";
+import { LlamaCloudSelector } from "./widgets/LlamaCloudSelector";
+
+const ALLOWED_EXTENSIONS = ["png", "jpg", "jpeg", "csv", "pdf", "txt", "docx"];
+
+export default function ChatInput(
+  props: Pick<
+    ChatHandler,
+    | "isLoading"
+    | "input"
+    | "onFileUpload"
+    | "onFileError"
+    | "handleSubmit"
+    | "handleInputChange"
+    | "messages"
+    | "setInput"
+    | "append"
+  > & {
+    requestParams?: any;
+    setRequestData?: React.Dispatch<any>;
+  },
+) {
+  const {
+    imageUrl,
+    setImageUrl,
+    uploadFile,
+    files,
+    removeDoc,
+    reset,
+    getAnnotations,
+  } = useFile();
+
+  // default submit function does not handle including annotations in the message
+  // so we need to use append function to submit new message with annotations
+  const handleSubmitWithAnnotations = (
+    e: React.FormEvent<HTMLFormElement>,
+    annotations: JSONValue[] | undefined,
+  ) => {
+    e.preventDefault();
+    props.append!({
+      content: props.input,
+      role: "user",
+      createdAt: new Date(),
+      annotations,
+    });
+    props.setInput!("");
+  };
+
+  const onSubmit = (e: React.FormEvent<HTMLFormElement>) => {
+    e.preventDefault();
+    const annotations = getAnnotations();
+    if (annotations.length) {
+      handleSubmitWithAnnotations(e, annotations);
+      return reset();
+    }
+    props.handleSubmit(e);
+  };
+
+  const handleUploadFile = async (file: File) => {
+    if (imageUrl || files.length > 0) {
+      alert("You can only upload one file at a time.");
+      return;
+    }
+    try {
+      await uploadFile(file, props.requestParams);
+      props.onFileUpload?.(file);
+    } catch (error: any) {
+      const onFileUploadError = props.onFileError || window.alert;
+      onFileUploadError(error.message);
+    }
+  };
+
+  const handleKeyDown = (e: React.KeyboardEvent<HTMLTextAreaElement>) => {
+    if (e.key === "Enter" && !e.shiftKey) {
+      e.preventDefault();
+      onSubmit(e as unknown as React.FormEvent<HTMLFormElement>);
+    }
+  };
+
+  return (
+    <form
+      onSubmit={onSubmit}
+      className="rounded-xl bg-white p-4 shadow-xl space-y-4 shrink-0"
+    >
+      {imageUrl && (
+        <UploadImagePreview url={imageUrl} onRemove={() => setImageUrl(null)} />
+      )}
+      {files.length > 0 && (
+        <div className="flex gap-4 w-full overflow-auto py-2">
+          {files.map((file, index) => (
+            <DocumentPreview
+              key={file.metadata?.id ?? `${file.filename}-${index}`}
+              file={file}
+              onRemove={() => removeDoc(file)}
+            />
+          ))}
+        </div>
+      )}
+      <div className="flex w-full items-start justify-between gap-4 ">
+        <Textarea
+          id="chat-input"
+          autoFocus
+          name="message"
+          placeholder="Type a message"
+          className="flex-1 min-h-0 h-[40px]"
+          value={props.input}
+          onChange={props.handleInputChange}
+          onKeyDown={handleKeyDown}
+        />
+        <FileUploader
+          onFileUpload={handleUploadFile}
+          onFileError={props.onFileError}
+          config={{
+            allowedExtensions: ALLOWED_EXTENSIONS,
+            disabled: props.isLoading,
+          }}
+        />
+        {process.env.NEXT_PUBLIC_USE_LLAMACLOUD === "true" &&
+          props.setRequestData && (
+            <LlamaCloudSelector setRequestData={props.setRequestData} />
+          )}
+        <Button type="submit" disabled={props.isLoading || !props.input.trim()}>
+          Send message
+        </Button>
+      </div>
+    </form>
+  );
+}

app/components/ui/chat/chat-message/chat-agent-events.tsx

@@ -0,0 +1,153 @@
+import { icons, LucideIcon } from "lucide-react";
+import { useMemo } from "react";
+import { Button } from "../../button";
+import {
+  Drawer,
+  DrawerClose,
+  DrawerContent,
+  DrawerHeader,
+  DrawerTitle,
+  DrawerTrigger,
+} from "../../drawer";
+import { AgentEventData } from "../index";
+import Markdown from "./markdown";
+
+const AgentIcons: Record<string, LucideIcon> = {
+  bot: icons.Bot,
+  researcher: icons.ScanSearch,
+  writer: icons.PenLine,
+  reviewer: icons.MessageCircle,
+  publisher: icons.BookCheck,
+};
+
+type MergedEvent = {
+  agent: string;
+  texts: string[];
+  icon: LucideIcon;
+};
+
+export function ChatAgentEvents({
+  data,
+  isFinished,
+}: {
+  data: AgentEventData[];
+  isFinished: boolean;
+}) {
+  const events = useMemo(() => mergeAdjacentEvents(data), [data]);
+  return (
+    <div className="pl-2">
+      <div className="text-sm space-y-4">
+        {events.map((eventItem, index) => (
+          <AgentEventContent
+            key={index}
+            event={eventItem}
+            isLast={index === events.length - 1}
+            isFinished={isFinished}
+          />
+        ))}
+      </div>
+    </div>
+  );
+}
+
+const MAX_TEXT_LENGTH = 150;
+
+function AgentEventContent({
+  event,
+  isLast,
+  isFinished,
+}: {
+  event: MergedEvent;
+  isLast: boolean;
+  isFinished: boolean;
+}) {
+  const { agent, texts } = event;
+  const AgentIcon = event.icon;
+  return (
+    <div className="flex gap-4 border-b pb-4 items-center fadein-agent">
+      <div className="w-[100px] flex flex-col items-center gap-2">
+        <div className="relative">
+          {isLast && !isFinished && (
+            <div className="absolute -top-0 -right-4">
+              <span className="relative flex h-3 w-3">
+                <span className="animate-ping absolute inline-flex h-full w-full rounded-full bg-sky-400 opacity-75"></span>
+                <span className="relative inline-flex rounded-full h-3 w-3 bg-sky-500"></span>
+              </span>
+            </div>
+          )}
+          <AgentIcon />
+        </div>
+        <span className="font-bold">{agent}</span>
+      </div>
+      <ul className="flex-1 list-decimal space-y-2">
+        {texts.map((text, index) => (
+          <li className="whitespace-break-spaces" key={index}>
+            {text.length <= MAX_TEXT_LENGTH && <span>{text}</span>}
+            {text.length > MAX_TEXT_LENGTH && (
+              <div>
+                <span>{text.slice(0, MAX_TEXT_LENGTH)}...</span>
+                <AgentEventDialog
+                  content={text}
+                  title={`Agent "${agent}" - Step: ${index + 1}`}
+                >
+                  <span className="font-semibold underline cursor-pointer ml-2">
+                    Show more
+                  </span>
+                </AgentEventDialog>
+              </div>
+            )}
+          </li>
+        ))}
+      </ul>
+    </div>
+  );
+}
+
+type AgentEventDialogProps = {
+  title: string;
+  content: string;
+  children: React.ReactNode;
+};
+
+function AgentEventDialog(props: AgentEventDialogProps) {
+  return (
+    <Drawer direction="left">
+      <DrawerTrigger asChild>{props.children}</DrawerTrigger>
+      <DrawerContent className="w-3/5 mt-24 h-full max-h-[96%] ">
+        <DrawerHeader className="flex justify-between">
+          <div className="space-y-2">
+            <DrawerTitle>{props.title}</DrawerTitle>
+          </div>
+          <DrawerClose asChild>
+            <Button variant="outline">Close</Button>
+          </DrawerClose>
+        </DrawerHeader>
+        <div className="m-4 overflow-auto">
+          <Markdown content={props.content} />
+        </div>
+      </DrawerContent>
+    </Drawer>
+  );
+}
+
+function mergeAdjacentEvents(events: AgentEventData[]): MergedEvent[] {
+  const mergedEvents: MergedEvent[] = [];
+
+  for (const event of events) {
+    const lastMergedEvent = mergedEvents[mergedEvents.length - 1];
+
+    if (lastMergedEvent && lastMergedEvent.agent === event.agent) {
+      // If the last event in mergedEvents has the same non-null agent, add the title to it
+      lastMergedEvent.texts.push(event.text);
+    } else {
+      // Otherwise, create a new merged event
+      mergedEvents.push({
+        agent: event.agent,
+        texts: [event.text],
+        icon: AgentIcons[event.agent] ?? icons.Bot,
+      });
+    }
+  }
+
+  return mergedEvents;
+}

app/components/ui/chat/chat-message/chat-avatar.tsx

@@ -0,0 +1,25 @@
+import { User2 } from "lucide-react";
+import Image from "next/image";
+
+export default function ChatAvatar({ role }: { role: string }) {
+  if (role === "user") {
+    return (
+      <div className="flex h-8 w-8 shrink-0 select-none items-center justify-center rounded-md border bg-background shadow">
+        <User2 className="h-4 w-4" />
+      </div>
+    );
+  }
+
+  return (
+    <div className="flex h-8 w-8 shrink-0 select-none items-center justify-center rounded-md border bg-black text-white shadow">
+      <Image
+        className="rounded-md"
+        src="/llama.png"
+        alt="Llama Logo"
+        width={24}
+        height={24}
+        priority
+      />
+    </div>
+  );
+}

app/components/ui/chat/chat-message/chat-events.tsx

@@ -0,0 +1,50 @@
+import { ChevronDown, ChevronRight, Loader2 } from "lucide-react";
+import { useState } from "react";
+import { Button } from "../../button";
+import {
+  Collapsible,
+  CollapsibleContent,
+  CollapsibleTrigger,
+} from "../../collapsible";
+import { EventData } from "../index";
+
+export function ChatEvents({
+  data,
+  isLoading,
+}: {
+  data: EventData[];
+  isLoading: boolean;
+}) {
+  const [isOpen, setIsOpen] = useState(false);
+
+  const buttonLabel = isOpen ? "Hide events" : "Show events";
+
+  const EventIcon = isOpen ? (
+    <ChevronDown className="h-4 w-4" />
+  ) : (
+    <ChevronRight className="h-4 w-4" />
+  );
+
+  return (
+    <div className="border-l-2 border-indigo-400 pl-2">
+      <Collapsible open={isOpen} onOpenChange={setIsOpen}>
+        <CollapsibleTrigger asChild>
+          <Button variant="secondary" className="space-x-2">
+            {isLoading ? <Loader2 className="h-4 w-4 animate-spin" /> : null}
+            <span>{buttonLabel}</span>
+            {EventIcon}
+          </Button>
+        </CollapsibleTrigger>
+        <CollapsibleContent asChild>
+          <div className="mt-4 text-sm space-y-2">
+            {data.map((eventItem, index) => (
+              <div className="whitespace-break-spaces" key={index}>
+                {eventItem.title}
+              </div>
+            ))}
+          </div>
+        </CollapsibleContent>
+      </Collapsible>
+    </div>
+  );
+}

app/components/ui/chat/chat-message/chat-files.tsx

@@ -0,0 +1,16 @@
+import { DocumentPreview } from "../../document-preview";
+import { DocumentFileData } from "../index";
+
+export function ChatFiles({ data }: { data: DocumentFileData }) {
+  if (!data.files.length) return null;
+  return (
+    <div className="flex gap-2 items-center">
+      {data.files.map((file, index) => (
+        <DocumentPreview
+          key={file.metadata?.id ?? `${file.filename}-${index}`}
+          file={file}
+        />
+      ))}
+    </div>
+  );
+}

app/components/ui/chat/chat-message/chat-image.tsx

@@ -0,0 +1,17 @@
+import Image from "next/image";
+import { type ImageData } from "../index";
+
+export function ChatImage({ data }: { data: ImageData }) {
+  return (
+    <div className="rounded-md max-w-[200px] shadow-md">
+      <Image
+        src={data.url}
+        width={0}
+        height={0}
+        sizes="100vw"
+        style={{ width: "100%", height: "auto" }}
+        alt=""
+      />
+    </div>
+  );
+}

app/components/ui/chat/chat-message/chat-sources.tsx

@@ -0,0 +1,173 @@
+import { Check, Copy } from "lucide-react";
+import { useMemo } from "react";
+import { Button } from "../../button";
+import { PreviewCard } from "../../document-preview";
+import {
+  HoverCard,
+  HoverCardContent,
+  HoverCardTrigger,
+} from "../../hover-card";
+import { cn } from "../../lib/utils";
+import { useCopyToClipboard } from "../hooks/use-copy-to-clipboard";
+import { DocumentFileType, SourceData, SourceNode } from "../index";
+import PdfDialog from "../widgets/PdfDialog";
+
+type Document = {
+  url: string;
+  sources: SourceNode[];
+};
+
+export function ChatSources({ data }: { data: SourceData }) {
+  const documents: Document[] = useMemo(() => {
+    // group nodes by document (a document must have a URL)
+    const nodesByUrl: Record<string, SourceNode[]> = {};
+    data.nodes.forEach((node) => {
+      const key = node.url;
+      nodesByUrl[key] ??= [];
+      nodesByUrl[key].push(node);
+    });
+
+    // convert to array of documents
+    return Object.entries(nodesByUrl).map(([url, sources]) => ({
+      url,
+      sources,
+    }));
+  }, [data.nodes]);
+
+  if (documents.length === 0) return null;
+
+  return (
+    <div className="space-y-2 text-sm">
+      <div className="font-semibold text-lg">Sources:</div>
+      <div className="flex gap-3 flex-wrap">
+        {documents.map((document) => {
+          return <DocumentInfo key={document.url} document={document} />;
+        })}
+      </div>
+    </div>
+  );
+}
+
+function SourceInfo({ node, index }: { node?: SourceNode; index: number }) {
+  if (!node) return <SourceNumberButton index={index} />;
+  return (
+    <HoverCard>
+      <HoverCardTrigger
+        className="cursor-default"
+        onClick={(e) => {
+          e.preventDefault();
+          e.stopPropagation();
+        }}
+      >
+        <SourceNumberButton
+          index={index}
+          className="hover:text-white hover:bg-primary"
+        />
+      </HoverCardTrigger>
+      <HoverCardContent className="w-[400px]">
+        <NodeInfo nodeInfo={node} />
+      </HoverCardContent>
+    </HoverCard>
+  );
+}
+
+export function SourceNumberButton({
+  index,
+  className,
+}: {
+  index: number;
+  className?: string;
+}) {
+  return (
+    <span
+      className={cn(
+        "text-xs w-5 h-5 rounded-full bg-gray-100 inline-flex items-center justify-center",
+        className,
+      )}
+    >
+      {index + 1}
+    </span>
+  );
+}
+
+export function DocumentInfo({
+  document,
+  className,
+}: {
+  document: Document;
+  className?: string;
+}) {
+  const { url, sources } = document;
+  // Extract filename from URL
+  const urlParts = url.split("/");
+  const fileName = urlParts.length > 0 ? urlParts[urlParts.length - 1] : url;
+  const fileExt = fileName?.split(".").pop() as DocumentFileType | undefined;
+
+  const previewFile = {
+    filename: fileName,
+    filetype: fileExt,
+  };
+
+  const DocumentDetail = (
+    <div className={`relative ${className}`}>
+      <PreviewCard className={"cursor-pointer"} file={previewFile} />
+      <div className="absolute bottom-2 right-2 space-x-2 flex">
+        {sources.map((node: SourceNode, index: number) => (
+          <div key={node.id}>
+            <SourceInfo node={node} index={index} />
+          </div>
+        ))}
+      </div>
+    </div>
+  );
+
+  if (url.endsWith(".pdf")) {
+    // open internal pdf dialog for pdf files when click document card
+    return <PdfDialog documentId={url} url={url} trigger={DocumentDetail} />;
+  }
+  // open external link when click document card for other file types
+  return <div onClick={() => window.open(url, "_blank")}>{DocumentDetail}</div>;
+}
+
+function NodeInfo({ nodeInfo }: { nodeInfo: SourceNode }) {
+  const { isCopied, copyToClipboard } = useCopyToClipboard({ timeout: 1000 });
+
+  const pageNumber =
+    // XXX: page_label is used in Python, but page_number is used by Typescript
+    (nodeInfo.metadata?.page_number as number) ??
+    (nodeInfo.metadata?.page_label as number) ??
+    null;
+
+  return (
+    <div className="space-y-4">
+      <div className="flex justify-between items-center">
+        <span className="font-semibold">
+          {pageNumber ? `On page ${pageNumber}:` : "Node content:"}
+        </span>
+        {nodeInfo.text && (
+          <Button
+            onClick={(e) => {
+              e.stopPropagation();
+              copyToClipboard(nodeInfo.text);
+            }}
+            size="icon"
+            variant="ghost"
+            className="h-12 w-12 shrink-0"
+          >
+            {isCopied ? (
+              <Check className="h-4 w-4" />
+            ) : (
+              <Copy className="h-4 w-4" />
+            )}
+          </Button>
+        )}
+      </div>
+
+      {nodeInfo.text && (
+        <pre className="max-h-[200px] overflow-auto whitespace-pre-line">
+          &ldquo;{nodeInfo.text}&rdquo;
+        </pre>
+      )}
+    </div>
+  );
+}

app/components/ui/chat/chat-message/chat-suggestedQuestions.tsx

@@ -0,0 +1,31 @@
+import { ChatHandler, SuggestedQuestionsData } from "..";
+
+export function SuggestedQuestions({
+  questions,
+  append,
+  isLastMessage,
+}: {
+  questions: SuggestedQuestionsData;
+  append: Pick<ChatHandler, "append">["append"];
+  isLastMessage: boolean;
+}) {
+  const showQuestions = isLastMessage && questions.length > 0;
+  return (
+    showQuestions &&
+    append !== undefined && (
+      <div className="flex flex-col space-y-2">
+        {questions.map((question, index) => (
+          <a
+            key={index}
+            onClick={() => {
+              append({ role: "user", content: question });
+            }}
+            className="text-sm italic hover:underline cursor-pointer"
+          >
+            {"->"} {question}
+          </a>
+        ))}
+      </div>
+    )
+  );
+}

app/components/ui/chat/chat-message/chat-tools.tsx

@@ -0,0 +1,40 @@
+import { ToolData } from "../index";
+import { Artifact, CodeArtifact } from "../widgets/Artifact";
+import { WeatherCard, WeatherData } from "../widgets/WeatherCard";
+
+// TODO: If needed, add displaying more tool outputs here
+export default function ChatTools({
+  data,
+  artifactVersion,
+}: {
+  data: ToolData;
+  artifactVersion?: number;
+}) {
+  if (!data) return null;
+  const { toolCall, toolOutput } = data;
+
+  if (toolOutput.isError) {
+    return (
+      <div className="border-l-2 border-red-400 pl-2">
+        There was an error when calling the tool {toolCall.name} with input:{" "}
+        <br />
+        {JSON.stringify(toolCall.input)}
+      </div>
+    );
+  }
+
+  switch (toolCall.name) {
+    case "get_weather_information":
+      const weatherData = toolOutput.output as unknown as WeatherData;
+      return <WeatherCard data={weatherData} />;
+    case "artifact":
+      return (
+        <Artifact
+          artifact={toolOutput.output as CodeArtifact}
+          version={artifactVersion}
+        />
+      );
+    default:
+      return null;
+  }
+}

app/components/ui/chat/chat-message/codeblock.tsx

@@ -0,0 +1,131 @@
+"use client";
+
+import hljs from "highlight.js";
+// instead of atom-one-dark theme, there are a lot of others: https://highlightjs.org/demo
+import "highlight.js/styles/atom-one-dark-reasonable.css";
+import { Check, Copy, Download } from "lucide-react";
+import { FC, memo, useEffect, useRef } from "react";
+import { Button } from "../../button";
+import { useCopyToClipboard } from "../hooks/use-copy-to-clipboard";
+
+interface Props {
+  language: string;
+  value: string;
+  className?: string;
+}
+
+interface languageMap {
+  [key: string]: string | undefined;
+}
+
+export const programmingLanguages: languageMap = {
+  javascript: ".js",
+  python: ".py",
+  java: ".java",
+  c: ".c",
+  cpp: ".cpp",
+  "c++": ".cpp",
+  "c#": ".cs",
+  ruby: ".rb",
+  php: ".php",
+  swift: ".swift",
+  "objective-c": ".m",
+  kotlin: ".kt",
+  typescript: ".ts",
+  go: ".go",
+  perl: ".pl",
+  rust: ".rs",
+  scala: ".scala",
+  haskell: ".hs",
+  lua: ".lua",
+  shell: ".sh",
+  sql: ".sql",
+  html: ".html",
+  css: ".css",
+  // add more file extensions here, make sure the key is same as language prop in CodeBlock.tsx component
+};
+
+export const generateRandomString = (length: number, lowercase = false) => {
+  const chars = "ABCDEFGHJKLMNPQRSTUVWXY3456789"; // excluding similar looking characters like Z, 2, I, 1, O, 0
+  let result = "";
+  for (let i = 0; i < length; i++) {
+    result += chars.charAt(Math.floor(Math.random() * chars.length));
+  }
+  return lowercase ? result.toLowerCase() : result;
+};
+
+const CodeBlock: FC<Props> = memo(({ language, value, className }) => {
+  const { isCopied, copyToClipboard } = useCopyToClipboard({ timeout: 2000 });
+  const codeRef = useRef<HTMLElement>(null);
+
+  useEffect(() => {
+    if (codeRef.current && codeRef.current.dataset.highlighted !== "yes") {
+      hljs.highlightElement(codeRef.current);
+    }
+  }, [language, value]);
+
+  const downloadAsFile = () => {
+    if (typeof window === "undefined") {
+      return;
+    }
+    const fileExtension = programmingLanguages[language] || ".file";
+    const suggestedFileName = `file-${generateRandomString(
+      3,
+      true,
+    )}${fileExtension}`;
+    const fileName = window.prompt("Enter file name", suggestedFileName);
+
+    if (!fileName) {
+      // User pressed cancel on prompt.
+      return;
+    }
+
+    const blob = new Blob([value], { type: "text/plain" });
+    const url = URL.createObjectURL(blob);
+    const link = document.createElement("a");
+    link.download = fileName;
+    link.href = url;
+    link.style.display = "none";
+    document.body.appendChild(link);
+    link.click();
+    document.body.removeChild(link);
+    URL.revokeObjectURL(url);
+  };
+
+  const onCopy = () => {
+    if (isCopied) return;
+    copyToClipboard(value);
+  };
+
+  return (
+    <div
+      className={`codeblock relative w-full bg-zinc-950 font-sans ${className}`}
+    >
+      <div className="flex w-full items-center justify-between bg-zinc-800 px-6 py-2 pr-4 text-zinc-100">
+        <span className="text-xs lowercase">{language}</span>
+        <div className="flex items-center space-x-1">
+          <Button variant="ghost" onClick={downloadAsFile} size="icon">
+            <Download />
+            <span className="sr-only">Download</span>
+          </Button>
+          <Button variant="ghost" size="icon" onClick={onCopy}>
+            {isCopied ? (
+              <Check className="h-4 w-4" />
+            ) : (
+              <Copy className="h-4 w-4" />
+            )}
+            <span className="sr-only">Copy code</span>
+          </Button>
+        </div>
+      </div>
+      <pre className="border border-zinc-700">
+        <code ref={codeRef} className={`language-${language} font-mono`}>
+          {value}
+        </code>
+      </pre>
+    </div>
+  );
+});
+CodeBlock.displayName = "CodeBlock";
+
+export { CodeBlock };

app/components/ui/chat/chat-message/index.tsx

@@ -0,0 +1,184 @@
+import { Check, Copy } from "lucide-react";
+
+import { Message } from "ai";
+import { Fragment } from "react";
+import { Button } from "../../button";
+import { useCopyToClipboard } from "../hooks/use-copy-to-clipboard";
+import {
+  AgentEventData,
+  ChatHandler,
+  DocumentFileData,
+  EventData,
+  ImageData,
+  MessageAnnotation,
+  MessageAnnotationType,
+  SuggestedQuestionsData,
+  ToolData,
+  getAnnotationData,
+  getSourceAnnotationData,
+} from "../index";
+import { ChatAgentEvents } from "./chat-agent-events";
+import ChatAvatar from "./chat-avatar";
+import { ChatEvents } from "./chat-events";
+import { ChatFiles } from "./chat-files";
+import { ChatImage } from "./chat-image";
+import { ChatSources } from "./chat-sources";
+import { SuggestedQuestions } from "./chat-suggestedQuestions";
+import ChatTools from "./chat-tools";
+import Markdown from "./markdown";
+
+type ContentDisplayConfig = {
+  order: number;
+  component: JSX.Element | null;
+};
+
+function ChatMessageContent({
+  message,
+  isLoading,
+  append,
+  isLastMessage,
+  artifactVersion,
+}: {
+  message: Message;
+  isLoading: boolean;
+  append: Pick<ChatHandler, "append">["append"];
+  isLastMessage: boolean;
+  artifactVersion: number | undefined;
+}) {
+  const annotations = message.annotations as MessageAnnotation[] | undefined;
+  if (!annotations?.length) return <Markdown content={message.content} />;
+
+  const imageData = getAnnotationData<ImageData>(
+    annotations,
+    MessageAnnotationType.IMAGE,
+  );
+  const contentFileData = getAnnotationData<DocumentFileData>(
+    annotations,
+    MessageAnnotationType.DOCUMENT_FILE,
+  );
+  const eventData = getAnnotationData<EventData>(
+    annotations,
+    MessageAnnotationType.EVENTS,
+  );
+  const agentEventData = getAnnotationData<AgentEventData>(
+    annotations,
+    MessageAnnotationType.AGENT_EVENTS,
+  );
+
+  const sourceData = getSourceAnnotationData(annotations);
+
+  const toolData = getAnnotationData<ToolData>(
+    annotations,
+    MessageAnnotationType.TOOLS,
+  );
+  const suggestedQuestionsData = getAnnotationData<SuggestedQuestionsData>(
+    annotations,
+    MessageAnnotationType.SUGGESTED_QUESTIONS,
+  );
+
+  const contents: ContentDisplayConfig[] = [
+    {
+      order: 1,
+      component: imageData[0] ? <ChatImage data={imageData[0]} /> : null,
+    },
+    {
+      order: -3,
+      component:
+        eventData.length > 0 ? (
+          <ChatEvents isLoading={isLoading} data={eventData} />
+        ) : null,
+    },
+    {
+      order: -2,
+      component:
+        agentEventData.length > 0 ? (
+          <ChatAgentEvents
+            data={agentEventData}
+            isFinished={!!message.content}
+          />
+        ) : null,
+    },
+    {
+      order: 2,
+      component: contentFileData[0] ? (
+        <ChatFiles data={contentFileData[0]} />
+      ) : null,
+    },
+    {
+      order: -1,
+      component: toolData[0] ? (
+        <ChatTools data={toolData[0]} artifactVersion={artifactVersion} />
+      ) : null,
+    },
+    {
+      order: 0,
+      component: <Markdown content={message.content} sources={sourceData[0]} />,
+    },
+    {
+      order: 3,
+      component: sourceData[0] ? <ChatSources data={sourceData[0]} /> : null,
+    },
+    {
+      order: 4,
+      component: suggestedQuestionsData[0] ? (
+        <SuggestedQuestions
+          questions={suggestedQuestionsData[0]}
+          append={append}
+          isLastMessage={isLastMessage}
+        />
+      ) : null,
+    },
+  ];
+
+  return (
+    <div className="flex-1 gap-4 flex flex-col">
+      {contents
+        .sort((a, b) => a.order - b.order)
+        .map((content, index) => (
+          <Fragment key={index}>{content.component}</Fragment>
+        ))}
+    </div>
+  );
+}
+
+export default function ChatMessage({
+  chatMessage,
+  isLoading,
+  append,
+  isLastMessage,
+  artifactVersion,
+}: {
+  chatMessage: Message;
+  isLoading: boolean;
+  append: Pick<ChatHandler, "append">["append"];
+  isLastMessage: boolean;
+  artifactVersion: number | undefined;
+}) {
+  const { isCopied, copyToClipboard } = useCopyToClipboard({ timeout: 2000 });
+  return (
+    <div className="flex items-start gap-4 pr-5 pt-5">
+      <ChatAvatar role={chatMessage.role} />
+      <div className="group flex flex-1 justify-between gap-2">
+        <ChatMessageContent
+          message={chatMessage}
+          isLoading={isLoading}
+          append={append}
+          isLastMessage={isLastMessage}
+          artifactVersion={artifactVersion}
+        />
+        <Button
+          onClick={() => copyToClipboard(chatMessage.content)}
+          size="icon"
+          variant="ghost"
+          className="h-8 w-8 opacity-0 group-hover:opacity-100"
+        >
+          {isCopied ? (
+            <Check className="h-4 w-4" />
+          ) : (
+            <Copy className="h-4 w-4" />
+          )}
+        </Button>
+      </div>
+    </div>
+  );
+}

app/components/ui/chat/chat-message/markdown.tsx

@@ -0,0 +1,170 @@
+import "katex/dist/katex.min.css";
+import { FC, memo } from "react";
+import ReactMarkdown, { Options } from "react-markdown";
+import rehypeKatex from "rehype-katex";
+import remarkGfm from "remark-gfm";
+import remarkMath from "remark-math";
+
+import { DOCUMENT_FILE_TYPES, DocumentFileType, SourceData } from "..";
+import { useClientConfig } from "../hooks/use-config";
+import { DocumentInfo, SourceNumberButton } from "./chat-sources";
+import { CodeBlock } from "./codeblock";
+
+const MemoizedReactMarkdown: FC<Options> = memo(
+  ReactMarkdown,
+  (prevProps, nextProps) =>
+    prevProps.children === nextProps.children &&
+    prevProps.className === nextProps.className,
+);
+
+const preprocessLaTeX = (content: string) => {
+  // Replace block-level LaTeX delimiters \[ \] with $$ $$
+  const blockProcessedContent = content.replace(
+    /\\\[([\s\S]*?)\\\]/g,
+    (_, equation) => `$$${equation}$$`,
+  );
+  // Replace inline LaTeX delimiters \( \) with $ $
+  const inlineProcessedContent = blockProcessedContent.replace(
+    /\\\[([\s\S]*?)\\\]/g,
+    (_, equation) => `$${equation}$`,
+  );
+  return inlineProcessedContent;
+};
+
+const preprocessMedia = (content: string) => {
+  // Remove `sandbox:` from the beginning of the URL
+  // to fix OpenAI's models issue appending `sandbox:` to the relative URL
+  return content.replace(/(sandbox|attachment|snt):/g, "");
+};
+
+/**
+ * Update the citation flag [citation:id]() to the new format [citation:index](url)
+ */
+const preprocessCitations = (content: string, sources?: SourceData) => {
+  if (sources) {
+    const citationRegex = /\[citation:(.+?)\]\(\)/g;
+    let match;
+    // Find all the citation references in the content
+    while ((match = citationRegex.exec(content)) !== null) {
+      const citationId = match[1];
+      // Find the source node with the id equal to the citation-id, also get the index of the source node
+      const sourceNode = sources.nodes.find((node) => node.id === citationId);
+      // If the source node is found, replace the citation reference with the new format
+      if (sourceNode !== undefined) {
+        content = content.replace(
+          match[0],
+          `[citation:${sources.nodes.indexOf(sourceNode)}]()`,
+        );
+      } else {
+        // If the source node is not found, remove the citation reference
+        content = content.replace(match[0], "");
+      }
+    }
+  }
+  return content;
+};
+
+const preprocessContent = (content: string, sources?: SourceData) => {
+  return preprocessCitations(
+    preprocessMedia(preprocessLaTeX(content)),
+    sources,
+  );
+};
+
+export default function Markdown({
+  content,
+  sources,
+}: {
+  content: string;
+  sources?: SourceData;
+}) {
+  const processedContent = preprocessContent(content, sources);
+  const { backend } = useClientConfig();
+
+  return (
+    <MemoizedReactMarkdown
+      className="prose dark:prose-invert prose-p:leading-relaxed prose-pre:p-0 break-words custom-markdown"
+      remarkPlugins={[remarkGfm, remarkMath]}
+      rehypePlugins={[rehypeKatex as any]}
+      components={{
+        p({ children }) {
+          return <div className="mb-2 last:mb-0">{children}</div>;
+        },
+        code({ node, inline, className, children, ...props }) {
+          if (children.length) {
+            if (children[0] == "▍") {
+              return (
+                <span className="mt-1 animate-pulse cursor-default">▍</span>
+              );
+            }
+
+            children[0] = (children[0] as string).replace("`▍`", "▍");
+          }
+
+          const match = /language-(\w+)/.exec(className || "");
+
+          if (inline) {
+            return (
+              <code className={className} {...props}>
+                {children}
+              </code>
+            );
+          }
+
+          return (
+            <CodeBlock
+              key={Math.random()}
+              language={(match && match[1]) || ""}
+              value={String(children).replace(/\n$/, "")}
+              className="mb-2"
+              {...props}
+            />
+          );
+        },
+        a({ href, children }) {
+          // If href starts with `{backend}/api/files`, then it's a local document and we use DocumenInfo for rendering
+          if (href?.startsWith(backend + "/api/files")) {
+            // Check if the file is document file type
+            const fileExtension = href.split(".").pop()?.toLowerCase();
+
+            if (
+              fileExtension &&
+              DOCUMENT_FILE_TYPES.includes(fileExtension as DocumentFileType)
+            ) {
+              return (
+                <DocumentInfo
+                  document={{
+                    url: new URL(decodeURIComponent(href)).href,
+                    sources: [],
+                  }}
+                  className="mb-2 mt-2"
+                />
+              );
+            }
+          }
+          // If a text link starts with 'citation:', then render it as a citation reference
+          if (
+            Array.isArray(children) &&
+            typeof children[0] === "string" &&
+            children[0].startsWith("citation:")
+          ) {
+            const index = Number(children[0].replace("citation:", ""));
+            if (!isNaN(index)) {
+              return <SourceNumberButton index={index} />;
+            } else {
+              // citation is not looked up yet, don't render anything
+              return <></>;
+            }
+          }
+          return (
+            <a href={href} target="_blank">
+              {children}
+            </a>
+          );
+        },
+      }}
+    >
+      {processedContent}
+    </MemoizedReactMarkdown>
+  );
+}

app/components/ui/chat/chat-messages.tsx

@@ -0,0 +1,136 @@
+import { Loader2 } from "lucide-react";
+import { useEffect, useMemo, useRef, useState } from "react";
+
+import { ToolData } from ".";
+import { Button } from "../button";
+import ChatActions from "./chat-actions";
+import ChatMessage from "./chat-message";
+import { ChatHandler } from "./chat.interface";
+import { useClientConfig } from "./hooks/use-config";
+
+export default function ChatMessages(
+  props: Pick<
+    ChatHandler,
+    "messages" | "isLoading" | "reload" | "stop" | "append"
+  >,
+) {
+  const { backend } = useClientConfig();
+  const [starterQuestions, setStarterQuestions] = useState<string[]>();
+
+  const scrollableChatContainerRef = useRef<HTMLDivElement>(null);
+  const messageLength = props.messages.length;
+  const lastMessage = props.messages[messageLength - 1];
+
+  const scrollToBottom = () => {
+    if (scrollableChatContainerRef.current) {
+      scrollableChatContainerRef.current.scrollTop =
+        scrollableChatContainerRef.current.scrollHeight;
+    }
+  };
+
+  const isLastMessageFromAssistant =
+    messageLength > 0 && lastMessage?.role !== "user";
+  const showReload =
+    props.reload && !props.isLoading && isLastMessageFromAssistant;
+  const showStop = props.stop && props.isLoading;
+
+  // `isPending` indicate
+  // that stream response is not yet received from the server,
+  // so we show a loading indicator to give a better UX.
+  const isPending = props.isLoading && !isLastMessageFromAssistant;
+
+  useEffect(() => {
+    scrollToBottom();
+  }, [messageLength, lastMessage]);
+
+  useEffect(() => {
+    if (!starterQuestions) {
+      fetch(`${backend}/api/chat/config`)
+        .then((response) => response.json())
+        .then((data) => {
+          if (data?.starterQuestions) {
+            setStarterQuestions(data.starterQuestions);
+          }
+        })
+        .catch((error) => console.error("Error fetching config", error));
+    }
+  }, [starterQuestions, backend]);
+
+  // build a map of message id to artifact version
+  const artifactVersionMap = useMemo(() => {
+    const map = new Map<string, number | undefined>();
+    let versionIndex = 1;
+    props.messages.forEach((m) => {
+      m.annotations?.forEach((annotation) => {
+        if (
+          typeof annotation === "object" &&
+          annotation != null &&
+          "type" in annotation &&
+          annotation.type === "tools"
+        ) {
+          const data = annotation.data as ToolData;
+          if (data?.toolCall?.name === "artifact") {
+            map.set(m.id, versionIndex);
+            versionIndex++;
+          }
+        }
+      });
+    });
+    return map;
+  }, [props.messages]);
+
+  return (
+    <div
+      className="flex-1 w-full rounded-xl bg-white p-4 shadow-xl relative overflow-y-auto"
+      ref={scrollableChatContainerRef}
+    >
+      <div className="flex flex-col gap-5 divide-y">
+        {props.messages.map((m, i) => {
+          const isLoadingMessage = i === messageLength - 1 && props.isLoading;
+          return (
+            <ChatMessage
+              key={m.id}
+              chatMessage={m}
+              isLoading={isLoadingMessage}
+              append={props.append!}
+              isLastMessage={i === messageLength - 1}
+              artifactVersion={artifactVersionMap.get(m.id)}
+            />
+          );
+        })}
+        {isPending && (
+          <div className="flex justify-center items-center pt-10">
+            <Loader2 className="h-4 w-4 animate-spin" />
+          </div>
+        )}
+      </div>
+      {(showReload || showStop) && (
+        <div className="flex justify-end py-4">
+          <ChatActions
+            reload={props.reload}
+            stop={props.stop}
+            showReload={showReload}
+            showStop={showStop}
+          />
+        </div>
+      )}
+      {!messageLength && starterQuestions?.length && props.append && (
+        <div className="absolute bottom-6 left-0 w-full">
+          <div className="grid grid-cols-2 gap-2 mx-20">
+            {starterQuestions.map((question, i) => (
+              <Button
+                variant="outline"
+                key={i}
+                onClick={() =>
+                  props.append!({ role: "user", content: question })
+                }
+              >
+                {question}
+              </Button>
+            ))}
+          </div>
+        </div>
+      )}
+    </div>
+  );
+}

app/components/ui/chat/chat.interface.ts

@@ -0,0 +1,25 @@
+import { Message } from "ai";
+
+export interface ChatHandler {
+  messages: Message[];
+  input: string;
+  isLoading: boolean;
+  handleSubmit: (
+    e: React.FormEvent<HTMLFormElement>,
+    ops?: {
+      data?: any;
+    },
+  ) => void;
+  handleInputChange: (e: React.ChangeEvent<HTMLTextAreaElement>) => void;
+  reload?: () => void;
+  stop?: () => void;
+  onFileUpload?: (file: File) => Promise<void>;
+  onFileError?: (errMsg: string) => void;
+  setInput?: (input: string) => void;
+  append?: (
+    message: Message | Omit<Message, "id">,
+    ops?: {
+      data: any;
+    },
+  ) => Promise<string | null | undefined>;
+}

app/components/ui/chat/hooks/use-config.ts

@@ -0,0 +1,24 @@
+"use client";
+
+export interface ChatConfig {
+  backend?: string;
+}
+
+function getBackendOrigin(): string {
+  const chatAPI = process.env.NEXT_PUBLIC_CHAT_API;
+  if (chatAPI) {
+    return new URL(chatAPI).origin;
+  } else {
+    if (typeof window !== "undefined") {
+      // Use BASE_URL from window.ENV
+      return (window as any).ENV?.BASE_URL || "";
+    }
+    return "";
+  }
+}
+
+export function useClientConfig(): ChatConfig {
+  return {
+    backend: getBackendOrigin(),
+  };
+}

app/components/ui/chat/hooks/use-copy-to-clipboard.tsx

@@ -0,0 +1,33 @@
+"use client";
+
+import * as React from "react";
+
+export interface useCopyToClipboardProps {
+  timeout?: number;
+}
+
+export function useCopyToClipboard({
+  timeout = 2000,
+}: useCopyToClipboardProps) {
+  const [isCopied, setIsCopied] = React.useState<Boolean>(false);
+
+  const copyToClipboard = (value: string) => {
+    if (typeof window === "undefined" || !navigator.clipboard?.writeText) {
+      return;
+    }
+
+    if (!value) {
+      return;
+    }
+
+    navigator.clipboard.writeText(value).then(() => {
+      setIsCopied(true);
+
+      setTimeout(() => {
+        setIsCopied(false);
+      }, timeout);
+    });
+  };
+
+  return { isCopied, copyToClipboard };
+}

app/components/ui/chat/hooks/use-file.ts

@@ -0,0 +1,136 @@
+"use client";
+
+import { JSONValue } from "llamaindex";
+import { useState } from "react";
+import {
+  DocumentFile,
+  DocumentFileType,
+  MessageAnnotation,
+  MessageAnnotationType,
+  UploadedFileMeta,
+} from "..";
+import { useClientConfig } from "./use-config";
+
+const docMineTypeMap: Record<string, DocumentFileType> = {
+  "text/csv": "csv",
+  "application/pdf": "pdf",
+  "text/plain": "txt",
+  "application/vnd.openxmlformats-officedocument.wordprocessingml.document":
+    "docx",
+};
+
+export function useFile() {
+  const { backend } = useClientConfig();
+  const [imageUrl, setImageUrl] = useState<string | null>(null);
+  const [files, setFiles] = useState<DocumentFile[]>([]);
+
+  const docEqual = (a: DocumentFile, b: DocumentFile) => {
+    if (a.metadata?.id === b.metadata?.id) return true;
+    if (a.filename === b.filename && a.filesize === b.filesize) return true;
+    return false;
+  };
+
+  const addDoc = (file: DocumentFile) => {
+    const existedFile = files.find((f) => docEqual(f, file));
+    if (!existedFile) {
+      setFiles((prev) => [...prev, file]);
+      return true;
+    }
+    return false;
+  };
+
+  const removeDoc = (file: DocumentFile) => {
+    setFiles((prev) =>
+      prev.filter((f) => f.metadata?.id !== file.metadata?.id),
+    );
+  };
+
+  const reset = () => {
+    imageUrl && setImageUrl(null);
+    files.length && setFiles([]);
+  };
+
+  const uploadContent = async (
+    file: File,
+    requestParams: any = {},
+  ): Promise<UploadedFileMeta> => {
+    const base64 = await readContent({ file, asUrl: true });
+    const uploadAPI = `${backend}/api/chat/upload`;
+    const response = await fetch(uploadAPI, {
+      method: "POST",
+      headers: {
+        "Content-Type": "application/json",
+      },
+      body: JSON.stringify({
+        ...requestParams,
+        base64,
+        filename: file.name,
+      }),
+    });
+    if (!response.ok) throw new Error("Failed to upload document.");
+    return (await response.json()) as UploadedFileMeta;
+  };
+
+  const getAnnotations = () => {
+    const annotations: MessageAnnotation[] = [];
+    if (imageUrl) {
+      annotations.push({
+        type: MessageAnnotationType.IMAGE,
+        data: { url: imageUrl },
+      });
+    }
+    if (files.length > 0) {
+      annotations.push({
+        type: MessageAnnotationType.DOCUMENT_FILE,
+        data: { files },
+      });
+    }
+    return annotations as JSONValue[];
+  };
+
+  const readContent = async (input: {
+    file: File;
+    asUrl?: boolean;
+  }): Promise<string> => {
+    const { file, asUrl } = input;
+    const content = await new Promise<string>((resolve, reject) => {
+      const reader = new FileReader();
+      if (asUrl) {
+        reader.readAsDataURL(file);
+      } else {
+        reader.readAsText(file);
+      }
+      reader.onload = () => resolve(reader.result as string);
+      reader.onerror = (error) => reject(error);
+    });
+    return content;
+  };
+
+  const uploadFile = async (file: File, requestParams: any = {}) => {
+    if (file.type.startsWith("image/")) {
+      const base64 = await readContent({ file, asUrl: true });
+      return setImageUrl(base64);
+    }
+
+    const filetype = docMineTypeMap[file.type];
+    if (!filetype) throw new Error("Unsupported document type.");
+    const uploadedFileMeta = await uploadContent(file, requestParams);
+    const newDoc: DocumentFile = {
+      filename: file.name,
+      filesize: file.size,
+      filetype,
+      metadata: uploadedFileMeta,
+    };
+    return addDoc(newDoc);
+  };
+
+  return {
+    imageUrl,
+    setImageUrl,
+    files,
+    removeDoc,
+    reset,
+    getAnnotations,
+    uploadFile,
+  };
+}

app/components/ui/chat/index.ts

@@ -0,0 +1,136 @@
+import { JSONValue } from "ai";
+import ChatInput from "./chat-input";
+import ChatMessages from "./chat-messages";
+
+export { type ChatHandler } from "./chat.interface";
+export { ChatInput, ChatMessages };
+
+export enum MessageAnnotationType {
+  IMAGE = "image",
+  DOCUMENT_FILE = "document_file",
+  SOURCES = "sources",
+  EVENTS = "events",
+  TOOLS = "tools",
+  SUGGESTED_QUESTIONS = "suggested_questions",
+  AGENT_EVENTS = "agent",
+}
+
+export type ImageData = {
+  url: string;
+};
+
+export type DocumentFileType = "csv" | "pdf" | "txt" | "docx";
+export const DOCUMENT_FILE_TYPES: DocumentFileType[] = [
+  "csv",
+  "pdf",
+  "txt",
+  "docx",
+];
+
+export type UploadedFileMeta = {
+  id: string;
+  name: string; // The uploaded file name in the backend (including uuid and sanitized)
+  url?: string;
+  refs?: string[];
+};
+
+export type DocumentFile = {
+  filename: string; // The original file name
+  filesize: number;
+  filetype: DocumentFileType;
+  metadata?: UploadedFileMeta; // undefined when the file is not uploaded yet
+};
+
+export type DocumentFileData = {
+  files: DocumentFile[];
+};
+
+export type SourceNode = {
+  id: string;
+  metadata: Record<string, unknown>;
+  score?: number;
+  text: string;
+  url: string;
+};
+
+export type SourceData = {
+  nodes: SourceNode[];
+};
+
+export type EventData = {
+  title: string;
+};
+
+export type AgentEventData = {
+  agent: string;
+  text: string;
+};
+
+export type ToolData = {
+  toolCall: {
+    id: string;
+    name: string;
+    input: {
+      [key: string]: JSONValue;
+    };
+  };
+  toolOutput: {
+    output: JSONValue;
+    isError: boolean;
+  };
+};
+
+export type SuggestedQuestionsData = string[];
+
+export type AnnotationData =
+  | ImageData
+  | DocumentFileData
+  | SourceData
+  | EventData
+  | AgentEventData
+  | ToolData
+  | SuggestedQuestionsData;
+
+export type MessageAnnotation = {
+  type: MessageAnnotationType;
+  data: AnnotationData;
+};
+
+const NODE_SCORE_THRESHOLD = 0.25;
+
+export function getAnnotationData<T extends AnnotationData>(
+  annotations: MessageAnnotation[],
+  type: MessageAnnotationType,
+): T[] {
+  return annotations.filter((a) => a.type === type).map((a) => a.data as T);
+}
+
+export function getSourceAnnotationData(
+  annotations: MessageAnnotation[],
+): SourceData[] {
+  const data = getAnnotationData<SourceData>(
+    annotations,
+    MessageAnnotationType.SOURCES,
+  );
+  if (data.length > 0) {
+    const sourceData = data[0] as SourceData;
+    if (sourceData.nodes) {
+      sourceData.nodes = preprocessSourceNodes(sourceData.nodes);
+    }
+  }
+  return data;
+}
+
+function preprocessSourceNodes(nodes: SourceNode[]): SourceNode[] {
+  // Filter source nodes has lower score
+  nodes = nodes
+    .filter((node) => (node.score ?? 1) > NODE_SCORE_THRESHOLD)
+    .filter((node) => node.url && node.url.trim() !== "")
+    .sort((a, b) => (b.score ?? 1) - (a.score ?? 1))
+    .map((node) => {
+      // remove trailing slash for node url if exists
+      node.url = node.url.replace(/\/$/, "");
+      return node;
+    });
+  return nodes;
+}

app/components/ui/chat/widgets/Artifact.tsx

@@ -0,0 +1,387 @@
+"use client";
+
+import { Check, ChevronDown, Code, Copy, Loader2 } from "lucide-react";
+import { useEffect, useRef, useState } from "react";
+import { Button, buttonVariants } from "../../button";
+import {
+  Collapsible,
+  CollapsibleContent,
+  CollapsibleTrigger,
+} from "../../collapsible";
+import { cn } from "../../lib/utils";
+import { Tabs, TabsContent, TabsList, TabsTrigger } from "../../tabs";
+import Markdown from "../chat-message/markdown";
+import { useClientConfig } from "../hooks/use-config";
+import { useCopyToClipboard } from "../hooks/use-copy-to-clipboard";
+
+// detail information to execute code
+export type CodeArtifact = {
+  commentary: string;
+  template: string;
+  title: string;
+  description: string;
+  additional_dependencies: string[];
+  has_additional_dependencies: boolean;
+  install_dependencies_command: string;
+  port: number | null;
+  file_path: string;
+  code: string;
+  files?: string[];
+};
+
+type ArtifactResult = {
+  template: string;
+  stdout: string[];
+  stderr: string[];
+  runtimeError?: { name: string; value: string; tracebackRaw: string[] };
+  outputUrls: Array<{ url: string; filename: string }>;
+  url: string;
+};
+
+export function Artifact({
+  artifact,
+  version,
+}: {
+  artifact: CodeArtifact | null;
+  version?: number;
+}) {
+  const [result, setResult] = useState<ArtifactResult | null>(null);
+  const [sandboxCreationError, setSandboxCreationError] = useState<string>();
+  const [sandboxCreating, setSandboxCreating] = useState(false);
+  const [openOutputPanel, setOpenOutputPanel] = useState(false);
+  const panelRef = useRef<HTMLDivElement>(null);
+  const { backend } = useClientConfig();
+
+  const handleOpenOutput = async () => {
+    setOpenOutputPanel(true);
+    openPanel();
+    panelRef.current?.classList.remove("hidden");
+  };
+
+  const fetchArtifactResult = async () => {
+    try {
+      setSandboxCreating(true);
+
+      const response = await fetch(`${backend}/api/sandbox`, {
+        method: "POST",
+        headers: {
+          "Content-Type": "application/json",
+        },
+        body: JSON.stringify({ artifact }),
+      });
+
+      if (!response.ok) {
+        throw new Error("Failure running code artifact");
+      }
+
+      const fetchedResult = await response.json();
+
+      setResult(fetchedResult);
+    } catch (error) {
+      console.error("Error fetching artifact result:", error);
+      setSandboxCreationError(
+        error instanceof Error
+          ? error.message
+          : "An unknown error occurred when executing code",
+      );
+    } finally {
+      setSandboxCreating(false);
+    }
+  };
+
+  useEffect(() => {
+    // auto trigger code execution
+    !result && fetchArtifactResult();
+    // eslint-disable-next-line react-hooks/exhaustive-deps
+  }, []);
+
+  if (!artifact || version === undefined) return null;
+
+  return (
+    <div>
+      <div
+        onClick={handleOpenOutput}
+        className={cn(
+          buttonVariants({ variant: "outline" }),
+          "h-auto cursor-pointer px-6 py-3 w-full flex gap-4 items-center justify-start border border-gray-200 rounded-md",
+        )}
+      >
+        <Code className="h-6 w-6" />
+        <div className="flex flex-col gap-1">
+          <h4 className="font-semibold m-0">
+            {artifact.title} v{version}
+          </h4>
+          <span className="text-xs text-gray-500">Click to open code</span>
+        </div>
+      </div>
+
+      {openOutputPanel && (
+        <div
+          className="w-[45vw] fixed top-0 right-0 h-screen z-50 artifact-panel animate-slideIn"
+          ref={panelRef}
+        >
+          <div className="flex justify-between items-center pl-5 pr-10 py-6 border-b">
+            <div className="space-y-2">
+              <h2 className="text-2xl font-bold m-0">{artifact?.title}</h2>
+              <span className="text-sm text-gray-500">Version: v{version}</span>
+            </div>
+            <Button
+              onClick={() => {
+                closePanel();
+                setOpenOutputPanel(false);
+              }}
+              variant="outline"
+            >
+              Close
+            </Button>
+          </div>
+
+          {sandboxCreating && (
+            <div className="flex justify-center items-center h-full">
+              <Loader2 className="h-6 w-6 animate-spin" />
+            </div>
+          )}
+          {sandboxCreationError && (
+            <div className="p-4 bg-red-100 text-red-800 rounded-md m-4">
+              <h3 className="font-bold mb-2 mt-0">
+                Error when creating Sandbox:
+              </h3>
+              <p className="font-semibold">{sandboxCreationError}</p>
+            </div>
+          )}
+          {result && (
+            <ArtifactOutput
+              artifact={artifact}
+              result={result}
+              version={version}
+            />
+          )}
+        </div>
+      )}
+    </div>
+  );
+}
+
+function ArtifactOutput({
+  artifact,
+  result,
+  version,
+}: {
+  artifact: CodeArtifact;
+  result: ArtifactResult;
+  version: number;
+}) {
+  const fileExtension = artifact.file_path.split(".").pop() || "";
+  const markdownCode = `\`\`\`${fileExtension}\n${artifact.code}\n\`\`\``;
+  const { url: sandboxUrl, outputUrls, runtimeError, stderr, stdout } = result;
+
+  return (
+    <Tabs defaultValue="code" className="h-full p-4 overflow-auto">
+      <TabsList className="grid grid-cols-2 max-w-[400px] mx-auto">
+        <TabsTrigger value="code">Code</TabsTrigger>
+        <TabsTrigger value="preview">Preview</TabsTrigger>
+      </TabsList>
+      <TabsContent value="code" className="h-[80%] mb-4 overflow-auto">
+        <div className="m-4 overflow-auto">
+          <Markdown content={markdownCode} />
+        </div>
+      </TabsContent>
+      <TabsContent
+        value="preview"
+        className="h-[80%] mb-4 overflow-auto mt-4 space-y-4"
+      >
+        {runtimeError && <RunTimeError runtimeError={runtimeError} />}
+        <ArtifactLogs stderr={stderr} stdout={stdout} />
+        {sandboxUrl && <CodeSandboxPreview url={sandboxUrl} />}
+        {outputUrls && <InterpreterOutput outputUrls={outputUrls} />}
+      </TabsContent>
+    </Tabs>
+  );
+}
+
+function RunTimeError({
+  runtimeError,
+}: {
+  runtimeError: { name: string; value: string; tracebackRaw?: string[] };
+}) {
+  const { isCopied, copyToClipboard } = useCopyToClipboard({ timeout: 1000 });
+  const contentToCopy = `Fix this error:\n${runtimeError.name}\n${runtimeError.value}\n${runtimeError.tracebackRaw?.join("\n")}`;
+  return (
+    <Collapsible className="bg-red-100 text-red-800 rounded-md py-2 px-4 space-y-4">
+      <CollapsibleTrigger className="font-bold w-full text-start flex items-center justify-between">
+        <span>Runtime Error:</span>
+        <ChevronDown className="w-4 h-4" />
+      </CollapsibleTrigger>
+      <CollapsibleContent className="text-sm flex gap-2">
+        <div className="flex flex-col gap-2">
+          <p className="font-semibold">{runtimeError.name}</p>
+          <p>{runtimeError.value}</p>
+          {runtimeError.tracebackRaw?.map((trace, index) => (
+            <pre key={index} className="whitespace-pre-wrap text-sm mb-2">
+              {trace}
+            </pre>
+          ))}
+        </div>
+        <Button
+          onClick={(e) => {
+            e.stopPropagation();
+            copyToClipboard(contentToCopy);
+          }}
+          size="icon"
+          variant="ghost"
+          className="h-12 w-12 shrink-0"
+        >
+          {isCopied ? (
+            <Check className="h-4 w-4" />
+          ) : (
+            <Copy className="h-4 w-4" />
+          )}
+        </Button>
+      </CollapsibleContent>
+    </Collapsible>
+  );
+}
+
+function CodeSandboxPreview({ url }: { url: string }) {
+  const [loading, setLoading] = useState(true);
+  const iframeRef = useRef<HTMLIFrameElement>(null);
+
+  useEffect(() => {
+    if (!loading && iframeRef.current) {
+      iframeRef.current.focus();
+    }
+  }, [loading]);
+
+  return (
+    <>
+      <iframe
+        key={url}
+        ref={iframeRef}
+        className="h-full w-full"
+        sandbox="allow-forms allow-scripts allow-same-origin"
+        loading="lazy"
+        src={url}
+        onLoad={() => setLoading(false)}
+      />
+      {loading && (
+        <div className="absolute top-1/2 left-1/2 transform -translate-x-1/2">
+          <Loader2 className="h-10 w-10 animate-spin" />
+        </div>
+      )}
+    </>
+  );
+}
+
+function InterpreterOutput({
+  outputUrls,
+}: {
+  outputUrls: Array<{ url: string; filename: string }>;
+}) {
+  return (
+    <ul className="flex flex-col gap-2 mt-4">
+      {outputUrls.map((url) => (
+        <li key={url.url}>
+          <div className="mt-4">
+            {isImageFile(url.filename) ? (
+              // eslint-disable-next-line @next/next/no-img-element
+              <img src={url.url} alt={url.filename} className="my-4 w-1/2" />
+            ) : (
+              <a
+                href={url.url}
+                target="_blank"
+                rel="noopener noreferrer"
+                className="text-blue-400 underline"
+              >
+                {url.filename}
+              </a>
+            )}
+          </div>
+        </li>
+      ))}
+    </ul>
+  );
+}
+
+function ArtifactLogs({
+  stderr,
+  stdout,
+}: {
+  stderr?: string[];
+  stdout?: string[];
+}) {
+  if (!stderr?.length && !stdout?.length) return null;
+
+  return (
+    <div className="flex flex-col gap-4">
+      {stdout && stdout.length > 0 && (
+        <Collapsible className="bg-green-100 text-green-800 rounded-md py-2 px-4 space-y-4">
+          <CollapsibleTrigger className="font-bold w-full text-start flex items-center justify-between">
+            <span>Output log:</span>
+            <ChevronDown className="w-4 h-4" />
+          </CollapsibleTrigger>
+          <CollapsibleContent className="text-sm">
+            <ArtifactLogItems logs={stdout} />
+          </CollapsibleContent>
+        </Collapsible>
+      )}
+      {stderr && stderr.length > 0 && (
+        <Collapsible className="bg-yellow-100 text-yellow-800 rounded-md py-2 px-4 space-y-4">
+          <CollapsibleTrigger className="font-bold w-full text-start flex items-center justify-between">
+            <span>Error log:</span>
+            <ChevronDown className="w-4 h-4" />
+          </CollapsibleTrigger>
+          <CollapsibleContent className="text-sm">
+            <ArtifactLogItems logs={stderr} />
+          </CollapsibleContent>
+        </Collapsible>
+      )}
+    </div>
+  );
+}
+
+function ArtifactLogItems({ logs }: { logs: string[] }) {
+  return (
+    <ul className="flex flex-col gap-2">
+      {logs.map((log, index) => (
+        <li key={index}>
+          <pre className="whitespace-pre-wrap text-sm">{log}</pre>
+        </li>
+      ))}
+    </ul>
+  );
+}
+
+function isImageFile(filename: string): boolean {
+  const imageExtensions = [".png", ".jpg", ".jpeg", ".gif", ".webp"];
+  return imageExtensions.some((ext) => filename.toLowerCase().endsWith(ext));
+}
+
+// this is just a hack to handle the layout when opening or closing the output panel
+// for real world application, you should use a global state management to control layout
+function openPanel() {
+  // hide all current artifact panel
+  const artifactPanels = document.querySelectorAll(".artifact-panel");
+  artifactPanels.forEach((panel) => {
+    panel.classList.add("hidden");
+  });
+
+  // make the main div width smaller to have space for the output panel
+  const mainDiv = document.querySelector("main");
+  mainDiv?.classList.remove("w-screen");
+  mainDiv?.classList.add("w-[55vw]");
+  mainDiv?.classList.add("px-8");
+}
+
+function closePanel() {
+  // reset the main div width
+  const mainDiv = document.querySelector("main");
+  mainDiv?.classList.remove("w-[55vw]");
+  mainDiv?.classList.remove("px-8");
+  mainDiv?.classList.add("w-screen");
+
+  // hide all current artifact panel
+  const artifactPanels = document.querySelectorAll(".artifact-panel");
+  artifactPanels.forEach((panel) => {
+    panel.classList.add("hidden");
+  });
+}

app/components/ui/chat/widgets/LlamaCloudSelector.tsx

@@ -0,0 +1,184 @@
+import { Loader2 } from "lucide-react";
+import { useCallback, useEffect, useState } from "react";
+import {
+  Select,
+  SelectContent,
+  SelectGroup,
+  SelectItem,
+  SelectLabel,
+  SelectTrigger,
+  SelectValue,
+} from "../../select";
+import { useClientConfig } from "../hooks/use-config";
+
+type LLamaCloudPipeline = {
+  id: string;
+  name: string;
+};
+
+type LLamaCloudProject = {
+  id: string;
+  organization_id: string;
+  name: string;
+  is_default: boolean;
+  pipelines: Array<LLamaCloudPipeline>;
+};
+
+type PipelineConfig = {
+  project: string; // project name
+  pipeline: string; // pipeline name
+};
+
+type LlamaCloudConfig = {
+  projects?: LLamaCloudProject[];
+  pipeline?: PipelineConfig;
+};
+
+export interface LlamaCloudSelectorProps {
+  setRequestData?: React.Dispatch<any>;
+  onSelect?: (pipeline: PipelineConfig | undefined) => void;
+  defaultPipeline?: PipelineConfig;
+  shouldCheckValid?: boolean;
+}
+
+export function LlamaCloudSelector({
+  setRequestData,
+  onSelect,
+  defaultPipeline,
+  shouldCheckValid = false,
+}: LlamaCloudSelectorProps) {
+  const { backend } = useClientConfig();
+  const [config, setConfig] = useState<LlamaCloudConfig>();
+
+  const updateRequestParams = useCallback(
+    (pipeline?: PipelineConfig) => {
+      if (setRequestData) {
+        setRequestData({
+          llamaCloudPipeline: pipeline,
+        });
+      } else {
+        onSelect?.(pipeline);
+      }
+    },
+    [onSelect, setRequestData],
+  );
+
+  useEffect(() => {
+    if (process.env.NEXT_PUBLIC_USE_LLAMACLOUD === "true" && !config) {
+      fetch(`${backend}/api/chat/config/llamacloud`)
+        .then((response) => {
+          if (!response.ok) {
+            return response.json().then((errorData) => {
+              window.alert(
+                `Error: ${JSON.stringify(errorData) || "Unknown error occurred"}`,
+              );
+            });
+          }
+          return response.json();
+        })
+        .then((data) => {
+          const pipeline = defaultPipeline ?? data.pipeline; // defaultPipeline will override pipeline in .env
+          setConfig({ ...data, pipeline });
+          updateRequestParams(pipeline);
+        })
+        .catch((error) => console.error("Error fetching config", error));
+    }
+  }, [backend, config, defaultPipeline, updateRequestParams]);
+
+  const setPipeline = (pipelineConfig?: PipelineConfig) => {
+    setConfig((prevConfig: any) => ({
+      ...prevConfig,
+      pipeline: pipelineConfig,
+    }));
+    updateRequestParams(pipelineConfig);
+  };
+
+  const handlePipelineSelect = async (value: string) => {
+    setPipeline(JSON.parse(value) as PipelineConfig);
+  };
+
+  if (!config) {
+    return (
+      <div className="flex justify-center items-center p-3">
+        <Loader2 className="h-4 w-4 animate-spin" />
+      </div>
+    );
+  }
+
+  if (shouldCheckValid && !isValid(config.projects, config.pipeline)) {
+    return (
+      <p className="text-red-500">
+        Invalid LlamaCloud configuration. Check console logs.
+      </p>
+    );
+  }
+  const { projects, pipeline } = config;
+
+  return (
+    <Select
+      onValueChange={handlePipelineSelect}
+      defaultValue={
+        isValid(projects, pipeline, false)
+          ? JSON.stringify(pipeline)
+          : undefined
+      }
+    >
+      <SelectTrigger className="w-[200px]">
+        <SelectValue placeholder="Select a pipeline" />
+      </SelectTrigger>
+      <SelectContent>
+        {projects!.map((project: LLamaCloudProject) => (
+          <SelectGroup key={project.id}>
+            <SelectLabel className="capitalize">
+              Project: {project.name}
+            </SelectLabel>
+            {project.pipelines.map((pipeline) => (
+              <SelectItem
+                key={pipeline.id}
+                className="last:border-b"
+                value={JSON.stringify({
+                  pipeline: pipeline.name,
+                  project: project.name,
+                })}
+              >
+                <span className="pl-2">{pipeline.name}</span>
+              </SelectItem>
+            ))}
+          </SelectGroup>
+        ))}
+      </SelectContent>
+    </Select>
+  );
+}
+
+function isValid(
+  projects: LLamaCloudProject[] | undefined,
+  pipeline: PipelineConfig | undefined,
+  logErrors: boolean = true,
+): boolean {
+  if (!projects?.length) return false;
+  if (!pipeline) return false;
+  const matchedProject = projects.find(
+    (project: LLamaCloudProject) => project.name === pipeline.project,
+  );
+  if (!matchedProject) {
+    if (logErrors) {
+      console.error(
+        `LlamaCloud project ${pipeline.project} not found. Check LLAMA_CLOUD_PROJECT_NAME variable`,
+      );
+    }
+    return false;
+  }
+  const pipelineExists = matchedProject.pipelines.some(
+    (p) => p.name === pipeline.pipeline,
+  );
+  if (!pipelineExists) {
+    if (logErrors) {
+      console.error(
+        `LlamaCloud pipeline ${pipeline.pipeline} not found. Check LLAMA_CLOUD_INDEX_NAME variable`,
+      );
+    }
+    return false;
+  }
+  return true;
+}

app/components/ui/chat/widgets/PdfDialog.tsx

@@ -0,0 +1,67 @@
+import dynamic from "next/dynamic";
+import { Button } from "../../button";
+import {
+  Drawer,
+  DrawerClose,
+  DrawerContent,
+  DrawerDescription,
+  DrawerHeader,
+  DrawerTitle,
+  DrawerTrigger,
+} from "../../drawer";
+
+export interface PdfDialogProps {
+  documentId: string;
+  url: string;
+  trigger: React.ReactNode;
+}
+
+// Dynamic imports for client-side rendering only
+const PDFViewer = dynamic(
+  () => import("@llamaindex/pdf-viewer").then((module) => module.PDFViewer),
+  { ssr: false },
+);
+
+const PdfFocusProvider = dynamic(
+  () =>
+    import("@llamaindex/pdf-viewer").then((module) => module.PdfFocusProvider),
+  { ssr: false },
+);
+
+export default function PdfDialog(props: PdfDialogProps) {
+  return (
+    <Drawer direction="left">
+      <DrawerTrigger asChild>{props.trigger}</DrawerTrigger>
+      <DrawerContent className="w-3/5 mt-24 h-full max-h-[96%] ">
+        <DrawerHeader className="flex justify-between">
+          <div className="space-y-2">
+            <DrawerTitle>PDF Content</DrawerTitle>
+            <DrawerDescription>
+              File URL:{" "}
+              <a
+                className="hover:text-blue-900"
+                href={props.url}
+                target="_blank"
+              >
+                {props.url}
+              </a>
+            </DrawerDescription>
+          </div>
+          <DrawerClose asChild>
+            <Button variant="outline">Close</Button>
+          </DrawerClose>
+        </DrawerHeader>
+        <div className="m-4">
+          <PdfFocusProvider>
+            <PDFViewer
+              file={{
+                id: props.documentId,
+                url: props.url,
+              }}
+            />
+          </PdfFocusProvider>
+        </div>
+      </DrawerContent>
+    </Drawer>
+  );
+}

app/components/ui/chat/widgets/WeatherCard.tsx

@@ -0,0 +1,213 @@
+export interface WeatherData {
+  latitude: number;
+  longitude: number;
+  generationtime_ms: number;
+  utc_offset_seconds: number;
+  timezone: string;
+  timezone_abbreviation: string;
+  elevation: number;
+  current_units: {
+    time: string;
+    interval: string;
+    temperature_2m: string;
+    weather_code: string;
+  };
+  current: {
+    time: string;
+    interval: number;
+    temperature_2m: number;
+    weather_code: number;
+  };
+  hourly_units: {
+    time: string;
+    temperature_2m: string;
+    weather_code: string;
+  };
+  hourly: {
+    time: string[];
+    temperature_2m: number[];
+    weather_code: number[];
+  };
+  daily_units: {
+    time: string;
+    weather_code: string;
+  };
+  daily: {
+    time: string[];
+    weather_code: number[];
+  };
+}
+
+// Follow WMO Weather interpretation codes (WW)
+const weatherCodeDisplayMap: Record<
+  string,
+  {
+    icon: JSX.Element;
+    status: string;
+  }
+> = {
+  "0": {
+    icon: <span>☀️</span>,
+    status: "Clear sky",
+  },
+  "1": {
+    icon: <span>🌤️</span>,
+    status: "Mainly clear",
+  },
+  "2": {
+    icon: <span>☁️</span>,
+    status: "Partly cloudy",
+  },
+  "3": {
+    icon: <span>☁️</span>,
+    status: "Overcast",
+  },
+  "45": {
+    icon: <span>🌫️</span>,
+    status: "Fog",
+  },
+  "48": {
+    icon: <span>🌫️</span>,
+    status: "Depositing rime fog",
+  },
+  "51": {
+    icon: <span>🌧️</span>,
+    status: "Drizzle",
+  },
+  "53": {
+    icon: <span>🌧️</span>,
+    status: "Drizzle",
+  },
+  "55": {
+    icon: <span>🌧️</span>,
+    status: "Drizzle",
+  },
+  "56": {
+    icon: <span>🌧️</span>,
+    status: "Freezing Drizzle",
+  },
+  "57": {
+    icon: <span>🌧️</span>,
+    status: "Freezing Drizzle",
+  },
+  "61": {
+    icon: <span>🌧️</span>,
+    status: "Rain",
+  },
+  "63": {
+    icon: <span>🌧️</span>,
+    status: "Rain",
+  },
+  "65": {
+    icon: <span>🌧️</span>,
+    status: "Rain",
+  },
+  "66": {
+    icon: <span>🌧️</span>,
+    status: "Freezing Rain",
+  },
+  "67": {
+    icon: <span>🌧️</span>,
+    status: "Freezing Rain",
+  },
+  "71": {
+    icon: <span>❄️</span>,
+    status: "Snow fall",
+  },
+  "73": {
+    icon: <span>❄️</span>,
+    status: "Snow fall",
+  },
+  "75": {
+    icon: <span>❄️</span>,
+    status: "Snow fall",
+  },
+  "77": {
+    icon: <span>❄️</span>,
+    status: "Snow grains",
+  },
+  "80": {
+    icon: <span>🌧️</span>,
+    status: "Rain showers",
+  },
+  "81": {
+    icon: <span>🌧️</span>,
+    status: "Rain showers",
+  },
+  "82": {
+    icon: <span>🌧️</span>,
+    status: "Rain showers",
+  },
+  "85": {
+    icon: <span>❄️</span>,
+    status: "Snow showers",
+  },
+  "86": {
+    icon: <span>❄️</span>,
+    status: "Snow showers",
+  },
+  "95": {
+    icon: <span>⛈️</span>,
+    status: "Thunderstorm",
+  },
+  "96": {
+    icon: <span>⛈️</span>,
+    status: "Thunderstorm",
+  },
+  "99": {
+    icon: <span>⛈️</span>,
+    status: "Thunderstorm",
+  },
+};
+
+const displayDay = (time: string) => {
+  return new Date(time).toLocaleDateString("en-US", {
+    weekday: "long",
+  });
+};
+
+export function WeatherCard({ data }: { data: WeatherData }) {
+  const currentDayString = new Date(data.current.time).toLocaleDateString(
+    "en-US",
+    {
+      weekday: "long",
+      month: "long",
+      day: "numeric",
+    },
+  );
+
+  return (
+    <div className="bg-[#61B9F2] rounded-2xl shadow-xl p-5 space-y-4 text-white w-fit">
+      <div className="flex justify-between">
+        <div className="space-y-2">
+          <div className="text-xl">{currentDayString}</div>
+          <div className="text-5xl font-semibold flex gap-4">
+            <span>
+              {data.current.temperature_2m} {data.current_units.temperature_2m}
+            </span>
+            {weatherCodeDisplayMap[data.current.weather_code].icon}
+          </div>
+        </div>
+        <span className="text-xl">
+          {weatherCodeDisplayMap[data.current.weather_code].status}
+        </span>
+      </div>
+      <div className="gap-2 grid grid-cols-6">
+        {data.daily.time.map((time, index) => {
+          if (index === 0) return null; // skip the current day
+          return (
+            <div key={time} className="flex flex-col items-center gap-4">
+              <span>{displayDay(time)}</span>
+              <div className="text-4xl">
+                {weatherCodeDisplayMap[data.daily.weather_code[index]].icon}
+              </div>
+              <span className="text-sm">
+                {weatherCodeDisplayMap[data.daily.weather_code[index]].status}
+              </span>
+            </div>
+          );
+        })}
+      </div>
+    </div>
+  );
+}

app/components/ui/collapsible.tsx

@@ -0,0 +1,11 @@
+"use client";
+
+import * as CollapsiblePrimitive from "@radix-ui/react-collapsible";
+
+const Collapsible = CollapsiblePrimitive.Root;
+
+const CollapsibleTrigger = CollapsiblePrimitive.CollapsibleTrigger;
+
+const CollapsibleContent = CollapsiblePrimitive.CollapsibleContent;
+
+export { Collapsible, CollapsibleContent, CollapsibleTrigger };

app/components/ui/document-preview.tsx

@@ -0,0 +1,129 @@
+import { XCircleIcon } from "lucide-react";
+import Image from "next/image";
+import DocxIcon from "../ui/icons/docx.svg";
+import PdfIcon from "../ui/icons/pdf.svg";
+import SheetIcon from "../ui/icons/sheet.svg";
+import TxtIcon from "../ui/icons/txt.svg";
+import { Button } from "./button";
+import { DocumentFile, DocumentFileType } from "./chat";
+import {
+  Drawer,
+  DrawerClose,
+  DrawerContent,
+  DrawerDescription,
+  DrawerHeader,
+  DrawerTitle,
+  DrawerTrigger,
+} from "./drawer";
+import { cn } from "./lib/utils";
+
+export interface DocumentPreviewProps {
+  file: DocumentFile;
+  onRemove?: () => void;
+}
+
+export function DocumentPreview(props: DocumentPreviewProps) {
+  const { filename, filesize, filetype, metadata } = props.file;
+
+  if (metadata?.refs?.length) {
+    return (
+      <div title={`Document IDs: ${metadata.refs.join(", ")}`}>
+        <PreviewCard {...props} />
+      </div>
+    );
+  }
+
+  return (
+    <Drawer direction="left">
+      <DrawerTrigger asChild>
+        <div>
+          <PreviewCard className="cursor-pointer" {...props} />
+        </div>
+      </DrawerTrigger>
+      <DrawerContent className="w-3/5 mt-24 h-full max-h-[96%] ">
+        <DrawerHeader className="flex justify-between">
+          <div className="space-y-2">
+            <DrawerTitle>{filetype.toUpperCase()} Raw Content</DrawerTitle>
+            <DrawerDescription>
+              {filename} ({inKB(filesize)} KB)
+            </DrawerDescription>
+          </div>
+          <DrawerClose asChild>
+            <Button variant="outline">Close</Button>
+          </DrawerClose>
+        </DrawerHeader>
+        <div className="m-4 max-h-[80%] overflow-auto">
+          {metadata?.refs?.length && (
+            <pre className="bg-secondary rounded-md p-4 block text-sm">
+              {metadata.refs.join(", ")}
+            </pre>
+          )}
+        </div>
+      </DrawerContent>
+    </Drawer>
+  );
+}
+
+export const FileIcon: Record<DocumentFileType, string> = {
+  csv: SheetIcon,
+  pdf: PdfIcon,
+  docx: DocxIcon,
+  txt: TxtIcon,
+};
+
+export function PreviewCard(props: {
+  file: {
+    filename: string;
+    filesize?: number;
+    filetype?: DocumentFileType;
+  };
+  onRemove?: () => void;
+  className?: string;
+}) {
+  const { onRemove, file, className } = props;
+  return (
+    <div
+      className={cn(
+        "p-2 w-60 max-w-60 bg-secondary rounded-lg text-sm relative",
+        className,
+      )}
+    >
+      <div className="flex flex-row items-center gap-2">
+        <div className="relative h-8 w-8 shrink-0 overflow-hidden rounded-md flex items-center justify-center">
+          <Image
+            className="h-full w-auto object-contain"
+            priority
+            src={FileIcon[file.filetype || "txt"]}
+            alt="Icon"
+          />
+        </div>
+        <div className="overflow-hidden">
+          <div className="truncate font-semibold">
+            {file.filename} {file.filesize ? `(${inKB(file.filesize)} KB)` : ""}
+          </div>
+          {file.filetype && (
+            <div className="truncate text-token-text-tertiary flex items-center gap-2">
+              <span>{file.filetype.toUpperCase()} File</span>
+            </div>
+          )}
+        </div>
+      </div>
+      {onRemove && (
+        <div
+          className={cn(
+            "absolute -top-2 -right-2 w-6 h-6 z-10 bg-gray-500 text-white rounded-full",
+          )}
+        >
+          <XCircleIcon
+            className="w-6 h-6 bg-gray-500 text-white rounded-full"
+            onClick={onRemove}
+          />
+        </div>
+      )}
+    </div>
+  );
+}
+
+function inKB(size: number) {
+  return Math.round((size / 1024) * 10) / 10;
+}

app/components/ui/drawer.tsx

@@ -0,0 +1,118 @@
+"use client";
+
+import * as React from "react";
+import { Drawer as DrawerPrimitive } from "vaul";
+
+import { cn } from "./lib/utils";
+
+const Drawer = ({
+  shouldScaleBackground = true,
+  ...props
+}: React.ComponentProps<typeof DrawerPrimitive.Root>) => (
+  <DrawerPrimitive.Root
+    shouldScaleBackground={shouldScaleBackground}
+    {...props}
+  />
+);
+Drawer.displayName = "Drawer";
+
+const DrawerTrigger = DrawerPrimitive.Trigger;
+
+const DrawerPortal = DrawerPrimitive.Portal;
+
+const DrawerClose = DrawerPrimitive.Close;
+
+const DrawerOverlay = React.forwardRef<
+  React.ElementRef<typeof DrawerPrimitive.Overlay>,
+  React.ComponentPropsWithoutRef<typeof DrawerPrimitive.Overlay>
+>(({ className, ...props }, ref) => (
+  <DrawerPrimitive.Overlay
+    ref={ref}
+    className={cn("fixed inset-0 z-50 bg-black/80", className)}
+    {...props}
+  />
+));
+DrawerOverlay.displayName = DrawerPrimitive.Overlay.displayName;
+
+const DrawerContent = React.forwardRef<
+  React.ElementRef<typeof DrawerPrimitive.Content>,
+  React.ComponentPropsWithoutRef<typeof DrawerPrimitive.Content>
+>(({ className, children, ...props }, ref) => (
+  <DrawerPortal>
+    <DrawerOverlay />
+    <DrawerPrimitive.Content
+      ref={ref}
+      className={cn(
+        "fixed inset-x-0 bottom-0 z-50 mt-24 flex h-auto flex-col rounded-t-[10px] border bg-background",
+        className,
+      )}
+      {...props}
+    >
+      <div className="mx-auto mt-4 h-2 w-[100px] rounded-full bg-muted" />
+      {children}
+    </DrawerPrimitive.Content>
+  </DrawerPortal>
+));
+DrawerContent.displayName = "DrawerContent";
+
+const DrawerHeader = ({
+  className,
+  ...props
+}: React.HTMLAttributes<HTMLDivElement>) => (
+  <div
+    className={cn("grid gap-1.5 p-4 text-center sm:text-left", className)}
+    {...props}
+  />
+);
+DrawerHeader.displayName = "DrawerHeader";
+
+const DrawerFooter = ({
+  className,
+  ...props
+}: React.HTMLAttributes<HTMLDivElement>) => (
+  <div
+    className={cn("mt-auto flex flex-col gap-2 p-4", className)}
+    {...props}
+  />
+);
+DrawerFooter.displayName = "DrawerFooter";
+
+const DrawerTitle = React.forwardRef<
+  React.ElementRef<typeof DrawerPrimitive.Title>,
+  React.ComponentPropsWithoutRef<typeof DrawerPrimitive.Title>
+>(({ className, ...props }, ref) => (
+  <DrawerPrimitive.Title
+    ref={ref}
+    className={cn(
+      "text-lg font-semibold leading-none tracking-tight",
+      className,
+    )}
+    {...props}
+  />
+));
+DrawerTitle.displayName = DrawerPrimitive.Title.displayName;
+
+const DrawerDescription = React.forwardRef<
+  React.ElementRef<typeof DrawerPrimitive.Description>,
+  React.ComponentPropsWithoutRef<typeof DrawerPrimitive.Description>
+>(({ className, ...props }, ref) => (
+  <DrawerPrimitive.Description
+    ref={ref}
+    className={cn("text-sm text-muted-foreground", className)}
+    {...props}
+  />
+));
+DrawerDescription.displayName = DrawerPrimitive.Description.displayName;
+
+export {
+  Drawer,
+  DrawerClose,
+  DrawerContent,
+  DrawerDescription,
+  DrawerFooter,
+  DrawerHeader,
+  DrawerOverlay,
+  DrawerPortal,
+  DrawerTitle,
+  DrawerTrigger,
+};

app/components/ui/file-uploader.tsx

@@ -0,0 +1,105 @@
+"use client";
+
+import { Loader2, Paperclip } from "lucide-react";
+import { ChangeEvent, useState } from "react";
+import { buttonVariants } from "./button";
+import { cn } from "./lib/utils";
+
+export interface FileUploaderProps {
+  config?: {
+    inputId?: string;
+    fileSizeLimit?: number;
+    allowedExtensions?: string[];
+    checkExtension?: (extension: string) => string | null;
+    disabled: boolean;
+  };
+  onFileUpload: (file: File) => Promise<void>;
+  onFileError?: (errMsg: string) => void;
+}
+
+const DEFAULT_INPUT_ID = "fileInput";
+const DEFAULT_FILE_SIZE_LIMIT = 1024 * 1024 * 50; // 50 MB
+
+export default function FileUploader({
+  config,
+  onFileUpload,
+  onFileError,
+}: FileUploaderProps) {
+  const [uploading, setUploading] = useState(false);
+
+  const inputId = config?.inputId || DEFAULT_INPUT_ID;
+  const fileSizeLimit = config?.fileSizeLimit || DEFAULT_FILE_SIZE_LIMIT;
+  const allowedExtensions = config?.allowedExtensions;
+  const defaultCheckExtension = (extension: string) => {
+    if (allowedExtensions && !allowedExtensions.includes(extension)) {
+      return `Invalid file type. Please select a file with one of these formats: ${allowedExtensions!.join(
+        ",",
+      )}`;
+    }
+    return null;
+  };
+  const checkExtension = config?.checkExtension ?? defaultCheckExtension;
+
+  const isFileSizeExceeded = (file: File) => {
+    return file.size > fileSizeLimit;
+  };
+
+  const resetInput = () => {
+    const fileInput = document.getElementById(inputId) as HTMLInputElement;
+    fileInput.value = "";
+  };
+
+  const onFileChange = async (e: ChangeEvent<HTMLInputElement>) => {
+    const file = e.target.files?.[0];
+    if (!file) return;
+
+    setUploading(true);
+    await handleUpload(file);
+    resetInput();
+    setUploading(false);
+  };
+
+  const handleUpload = async (file: File) => {
+    const onFileUploadError = onFileError || window.alert;
+    const fileExtension = file.name.split(".").pop() || "";
+    const extensionFileError = checkExtension(fileExtension);
+    if (extensionFileError) {
+      return onFileUploadError(extensionFileError);
+    }
+
+    if (isFileSizeExceeded(file)) {
+      return onFileUploadError(
+        `File size exceeded. Limit is ${fileSizeLimit / 1024 / 1024} MB`,
+      );
+    }
+
+    await onFileUpload(file);
+  };
+
+  return (
+    <div className="self-stretch">
+      <input
+        type="file"
+        id={inputId}
+        style={{ display: "none" }}
+        onChange={onFileChange}
+        accept={allowedExtensions?.join(",")}
+        disabled={config?.disabled || uploading}
+      />
+      <label
+        htmlFor={inputId}
+        className={cn(
+          buttonVariants({ variant: "secondary", size: "icon" }),
+          "cursor-pointer",
+          uploading && "opacity-50",
+        )}
+      >
+        {uploading ? (
+          <Loader2 className="h-4 w-4 animate-spin" />
+        ) : (
+          <Paperclip className="-rotate-45 w-4 h-4" />
+        )}
+      </label>
+    </div>
+  );
+}

app/components/ui/hover-card.tsx

@@ -0,0 +1,29 @@
+"use client";
+
+import * as HoverCardPrimitive from "@radix-ui/react-hover-card";
+import * as React from "react";
+
+import { cn } from "./lib/utils";
+
+const HoverCard = HoverCardPrimitive.Root;
+
+const HoverCardTrigger = HoverCardPrimitive.Trigger;
+
+const HoverCardContent = React.forwardRef<
+  React.ElementRef<typeof HoverCardPrimitive.Content>,
+  React.ComponentPropsWithoutRef<typeof HoverCardPrimitive.Content>
+>(({ className, align = "center", sideOffset = 4, ...props }, ref) => (
+  <HoverCardPrimitive.Content
+    ref={ref}
+    align={align}
+    sideOffset={sideOffset}
+    className={cn(
+      "z-50 w-64 rounded-md border bg-popover p-4 text-popover-foreground shadow-md outline-none data-[state=open]:animate-in data-[state=closed]:animate-out data-[state=closed]:fade-out-0 data-[state=open]:fade-in-0 data-[state=closed]:zoom-out-95 data-[state=open]:zoom-in-95 data-[side=bottom]:slide-in-from-top-2 data-[side=left]:slide-in-from-right-2 data-[side=right]:slide-in-from-left-2 data-[side=top]:slide-in-from-bottom-2",
+      className,
+    )}
+    {...props}
+  />
+));
+HoverCardContent.displayName = HoverCardPrimitive.Content.displayName;
+
+export { HoverCard, HoverCardContent, HoverCardTrigger };

app/components/ui/icons/docx.svg

@@ -0,0 +1,10 @@
+<?xml version="1.0" encoding="utf-8"?>
+
+<!-- Uploaded to: SVG Repo, www.svgrepo.com, Generator: SVG Repo Mixer Tools -->
+<svg width="800px" height="800px" viewBox="-4 0 64 64" xmlns="http://www.w3.org/2000/svg">
+
+<g fill-rule="evenodd">
+
+<path d="m5.11 0a5.07 5.07 0 0 0 -5.11 5v53.88a5.07 5.07 0 0 0 5.11 5.12h45.78a5.07 5.07 0 0 0 5.11-5.12v-38.6l-18.94-20.28z" fill="#107cad"/>
+
+<path d="m56 20.35v1h-12.82s-6.31-1.26-6.13-6.71c0 0 .21 5.71 6 5.71z" fill="#084968"/>

app/components/ui/icons/pdf.svg

@@ -0,0 +1,19 @@
+<?xml version="1.0" encoding="iso-8859-1"?>
+<!-- Uploaded to: SVG Repo, www.svgrepo.com, Generator: SVG Repo Mixer Tools -->
+<svg height="800px" width="800px" version="1.1" id="Layer_1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" 
+	 viewBox="0 0 309.267 309.267" xml:space="preserve">
+<g>
+	<path style="fill:#E2574C;" d="M38.658,0h164.23l87.049,86.711v203.227c0,10.679-8.659,19.329-19.329,19.329H38.658
+		c-10.67,0-19.329-8.65-19.329-19.329V19.329C19.329,8.65,27.989,0,38.658,0z"/>
+	<path style="fill:#B53629;" d="M289.658,86.981h-67.372c-10.67,0-19.329-8.659-19.329-19.329V0.193L289.658,86.981z"/>
+	<path style="fill:#FFFFFF;" d="M217.434,146.544c3.238,0,4.823-2.822,4.823-5.557c0-2.832-1.653-5.567-4.823-5.567h-18.44
+		c-3.605,0-5.615,2.986-5.615,6.282v45.317c0,4.04,2.3,6.282,5.412,6.282c3.093,0,5.403-2.242,5.403-6.282v-12.438h11.153
+		c3.46,0,5.19-2.832,5.19-5.644c0-2.754-1.73-5.49-5.19-5.49h-11.153v-16.903C204.194,146.544,217.434,146.544,217.434,146.544z
+		 M155.107,135.42h-13.492c-3.663,0-6.263,2.513-6.263,6.243v45.395c0,4.629,3.74,6.079,6.417,6.079h14.159
+		c16.758,0,27.824-11.027,27.824-28.047C183.743,147.095,173.325,135.42,155.107,135.42z M155.755,181.946h-8.225v-35.334h7.413
+		c11.221,0,16.101,7.529,16.101,17.918C171.044,174.253,166.25,181.946,155.755,181.946z M106.33,135.42H92.964
+		c-3.779,0-5.886,2.493-5.886,6.282v45.317c0,4.04,2.416,6.282,5.663,6.282s5.663-2.242,5.663-6.282v-13.231h8.379
+		c10.341,0,18.875-7.326,18.875-19.107C125.659,143.152,117.425,135.42,106.33,135.42z M106.108,163.158h-7.703v-17.097h7.703
+		c4.755,0,7.78,3.711,7.78,8.553C113.878,159.447,110.863,163.158,106.108,163.158z"/>
+</g>
+</svg>

app/components/ui/icons/sheet.svg

@@ -0,0 +1,90 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<svg width="49px" height="67px" viewBox="0 0 49 67" version="1.1"
+    xmlns="http://www.w3.org/2000/svg"
+    xmlns:xlink="http://www.w3.org/1999/xlink">
+    <title>Sheets-icon</title>
+    <desc>Created with Sketch.</desc>
+    <defs>
+        <path d="M29.5833333,0 L4.4375,0 C1.996875,0 0,1.996875 0,4.4375 L0,60.6458333 C0,63.0864583 1.996875,65.0833333 4.4375,65.0833333 L42.8958333,65.0833333 C45.3364583,65.0833333 47.3333333,63.0864583 47.3333333,60.6458333 L47.3333333,17.75 L29.5833333,0 Z" id="path-1"></path>
+        <path d="M29.5833333,0 L4.4375,0 C1.996875,0 0,1.996875 0,4.4375 L0,60.6458333 C0,63.0864583 1.996875,65.0833333 4.4375,65.0833333 L42.8958333,65.0833333 C45.3364583,65.0833333 47.3333333,63.0864583 47.3333333,60.6458333 L47.3333333,17.75 L29.5833333,0 Z" id="path-3"></path>
+        <path d="M29.5833333,0 L4.4375,0 C1.996875,0 0,1.996875 0,4.4375 L0,60.6458333 C0,63.0864583 1.996875,65.0833333 4.4375,65.0833333 L42.8958333,65.0833333 C45.3364583,65.0833333 47.3333333,63.0864583 47.3333333,60.6458333 L47.3333333,17.75 L29.5833333,0 Z" id="path-5"></path>
+        <linearGradient x1="50.0053945%" y1="8.58610612%" x2="50.0053945%" y2="100.013939%" id="linearGradient-7">
+            <stop stop-color="#263238" stop-opacity="0.2" offset="0%"></stop>
+            <stop stop-color="#263238" stop-opacity="0.02" offset="100%"></stop>
+        </linearGradient>
+        <path d="M29.5833333,0 L4.4375,0 C1.996875,0 0,1.996875 0,4.4375 L0,60.6458333 C0,63.0864583 1.996875,65.0833333 4.4375,65.0833333 L42.8958333,65.0833333 C45.3364583,65.0833333 47.3333333,63.0864583 47.3333333,60.6458333 L47.3333333,17.75 L29.5833333,0 Z" id="path-8"></path>
+        <path d="M29.5833333,0 L4.4375,0 C1.996875,0 0,1.996875 0,4.4375 L0,60.6458333 C0,63.0864583 1.996875,65.0833333 4.4375,65.0833333 L42.8958333,65.0833333 C45.3364583,65.0833333 47.3333333,63.0864583 47.3333333,60.6458333 L47.3333333,17.75 L29.5833333,0 Z" id="path-10"></path>
+        <path d="M29.5833333,0 L4.4375,0 C1.996875,0 0,1.996875 0,4.4375 L0,60.6458333 C0,63.0864583 1.996875,65.0833333 4.4375,65.0833333 L42.8958333,65.0833333 C45.3364583,65.0833333 47.3333333,63.0864583 47.3333333,60.6458333 L47.3333333,17.75 L29.5833333,0 Z" id="path-12"></path>
+        <path d="M29.5833333,0 L4.4375,0 C1.996875,0 0,1.996875 0,4.4375 L0,60.6458333 C0,63.0864583 1.996875,65.0833333 4.4375,65.0833333 L42.8958333,65.0833333 C45.3364583,65.0833333 47.3333333,63.0864583 47.3333333,60.6458333 L47.3333333,17.75 L29.5833333,0 Z" id="path-14"></path>
+        <radialGradient cx="3.16804688%" cy="2.71744318%" fx="3.16804688%" fy="2.71744318%" r="161.248516%" gradientTransform="translate(0.031680,0.027174),scale(1.000000,0.727273),translate(-0.031680,-0.027174)" id="radialGradient-16">
+            <stop stop-color="#FFFFFF" stop-opacity="0.1" offset="0%"></stop>
+            <stop stop-color="#FFFFFF" stop-opacity="0" offset="100%"></stop>
+        </radialGradient>
+    </defs>
+    <g id="Page-1" stroke="none" stroke-width="1" fill="none" fill-rule="evenodd">
+        <g id="Consumer-Apps-Sheets-Large-VD-R8-" transform="translate(-451.000000, -451.000000)">
+            <g id="Hero" transform="translate(0.000000, 63.000000)">
+                <g id="Personal" transform="translate(277.000000, 299.000000)">
+                    <g id="Sheets-icon" transform="translate(174.833333, 89.958333)">
+                        <g id="Group">
+                            <g id="Clipped">
+                                <mask id="mask-2" fill="white">
+                                    <use xlink:href="#path-1"></use>
+                                </mask>
+                                <g id="SVGID_1_"></g>
+                                <path d="M29.5833333,0 L4.4375,0 C1.996875,0 0,1.996875 0,4.4375 L0,60.6458333 C0,63.0864583 1.996875,65.0833333 4.4375,65.0833333 L42.8958333,65.0833333 C45.3364583,65.0833333 47.3333333,63.0864583 47.3333333,60.6458333 L47.3333333,17.75 L36.9791667,10.3541667 L29.5833333,0 Z" id="Path" fill="#0F9D58" fill-rule="nonzero" mask="url(#mask-2)"></path>
+                            </g>
+                            <g id="Clipped">
+                                <mask id="mask-4" fill="white">
+                                    <use xlink:href="#path-3"></use>
+                                </mask>
+                                <g id="SVGID_1_"></g>
+                                <path d="M11.8333333,31.8020833 L11.8333333,53.25 L35.5,53.25 L35.5,31.8020833 L11.8333333,31.8020833 Z M22.1875,50.2916667 L14.7916667,50.2916667 L14.7916667,46.59375 L22.1875,46.59375 L22.1875,50.2916667 Z M22.1875,44.375 L14.7916667,44.375 L14.7916667,40.6770833 L22.1875,40.6770833 L22.1875,44.375 Z M22.1875,38.4583333 L14.7916667,38.4583333 L14.7916667,34.7604167 L22.1875,34.7604167 L22.1875,38.4583333 Z M32.5416667,50.2916667 L25.1458333,50.2916667 L25.1458333,46.59375 L32.5416667,46.59375 L32.5416667,50.2916667 Z M32.5416667,44.375 L25.1458333,44.375 L25.1458333,40.6770833 L32.5416667,40.6770833 L32.5416667,44.375 Z M32.5416667,38.4583333 L25.1458333,38.4583333 L25.1458333,34.7604167 L32.5416667,34.7604167 L32.5416667,38.4583333 Z" id="Shape" fill="#F1F1F1" fill-rule="nonzero" mask="url(#mask-4)"></path>
+                            </g>
+                            <g id="Clipped">
+                                <mask id="mask-6" fill="white">
+                                    <use xlink:href="#path-5"></use>
+                                </mask>
+                                <g id="SVGID_1_"></g>
+                                <polygon id="Path" fill="url(#linearGradient-7)" fill-rule="nonzero" mask="url(#mask-6)" points="30.8813021 16.4520313 47.3333333 32.9003646 47.3333333 17.75"></polygon>
+                            </g>
+                            <g id="Clipped">
+                                <mask id="mask-9" fill="white">
+                                    <use xlink:href="#path-8"></use>
+                                </mask>
+                                <g id="SVGID_1_"></g>
+                                <g id="Group" mask="url(#mask-9)">
+                                    <g transform="translate(26.625000, -2.958333)">
+                                        <path d="M2.95833333,2.95833333 L2.95833333,16.2708333 C2.95833333,18.7225521 4.94411458,20.7083333 7.39583333,20.7083333 L20.7083333,20.7083333 L2.95833333,2.95833333 Z" id="Path" fill="#87CEAC" fill-rule="nonzero"></path>
+                                    </g>
+                                </g>
+                            </g>
+                            <g id="Clipped">
+                                <mask id="mask-11" fill="white">
+                                    <use xlink:href="#path-10"></use>
+                                </mask>
+                                <g id="SVGID_1_"></g>
+                                <path d="M4.4375,0 C1.996875,0 0,1.996875 0,4.4375 L0,4.80729167 C0,2.36666667 1.996875,0.369791667 4.4375,0.369791667 L29.5833333,0.369791667 L29.5833333,0 L4.4375,0 Z" id="Path" fill-opacity="0.2" fill="#FFFFFF" fill-rule="nonzero" mask="url(#mask-11)"></path>
+                            </g>
+                            <g id="Clipped">
+                                <mask id="mask-13" fill="white">
+                                    <use xlink:href="#path-12"></use>
+                                </mask>
+                                <g id="SVGID_1_"></g>
+                                <path d="M42.8958333,64.7135417 L4.4375,64.7135417 C1.996875,64.7135417 0,62.7166667 0,60.2760417 L0,60.6458333 C0,63.0864583 1.996875,65.0833333 4.4375,65.0833333 L42.8958333,65.0833333 C45.3364583,65.0833333 47.3333333,63.0864583 47.3333333,60.6458333 L47.3333333,60.2760417 C47.3333333,62.7166667 45.3364583,64.7135417 42.8958333,64.7135417 Z" id="Path" fill-opacity="0.2" fill="#263238" fill-rule="nonzero" mask="url(#mask-13)"></path>
+                            </g>
+                            <g id="Clipped">
+                                <mask id="mask-15" fill="white">
+                                    <use xlink:href="#path-14"></use>
+                                </mask>
+                                <g id="SVGID_1_"></g>
+                                <path d="M34.0208333,17.75 C31.5691146,17.75 29.5833333,15.7642188 29.5833333,13.3125 L29.5833333,13.6822917 C29.5833333,16.1340104 31.5691146,18.1197917 34.0208333,18.1197917 L47.3333333,18.1197917 L47.3333333,17.75 L34.0208333,17.75 Z" id="Path" fill-opacity="0.1" fill="#263238" fill-rule="nonzero" mask="url(#mask-15)"></path>
+                            </g>
+                        </g>
+                        <path d="M29.5833333,0 L4.4375,0 C1.996875,0 0,1.996875 0,4.4375 L0,60.6458333 C0,63.0864583 1.996875,65.0833333 4.4375,65.0833333 L42.8958333,65.0833333 C45.3364583,65.0833333 47.3333333,63.0864583 47.3333333,60.6458333 L47.3333333,17.75 L29.5833333,0 Z" id="Path" fill="url(#radialGradient-16)" fill-rule="nonzero"></path>
+                    </g>
+                </g>
+            </g>
+        </g>
+    </g>
+</svg>

app/components/ui/icons/txt.svg

@@ -0,0 +1,21 @@
+<?xml version="1.0" encoding="iso-8859-1"?>
+<!-- Uploaded to: SVG Repo, www.svgrepo.com, Generator: SVG Repo Mixer Tools -->
+<svg height="800px" width="800px" version="1.1" id="Layer_1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" 
+	 viewBox="0 0 512 512" xml:space="preserve">
+<path style="fill:#E2E5E7;" d="M128,0c-17.6,0-32,14.4-32,32v448c0,17.6,14.4,32,32,32h320c17.6,0,32-14.4,32-32V128L352,0H128z"/>
+<path style="fill:#B0B7BD;" d="M384,128h96L352,0v96C352,113.6,366.4,128,384,128z"/>
+<polygon style="fill:#CAD1D8;" points="480,224 384,128 480,128 "/>
+<path style="fill:#576D7E;" d="M416,416c0,8.8-7.2,16-16,16H48c-8.8,0-16-7.2-16-16V256c0-8.8,7.2-16,16-16h352c8.8,0,16,7.2,16,16
+	V416z"/>
+<g>
+	<path style="fill:#FFFFFF;" d="M132.784,311.472H110.4c-11.136,0-11.136-16.368,0-16.368h60.512c11.392,0,11.392,16.368,0,16.368
+		h-21.248v64.592c0,11.12-16.896,11.392-16.896,0v-64.592H132.784z"/>
+	<path style="fill:#FFFFFF;" d="M224.416,326.176l22.272-27.888c6.656-8.688,19.568,2.432,12.288,10.752
+		c-7.68,9.088-15.728,18.944-23.424,29.024l26.112,32.496c7.024,9.6-7.04,18.816-13.952,9.344l-23.536-30.192l-23.152,30.832
+		c-6.528,9.328-20.992-1.152-13.68-9.856l25.696-32.624c-8.048-10.096-15.856-19.936-23.664-29.024
+		c-8.064-9.6,6.912-19.44,12.784-10.48L224.416,326.176z"/>
+	<path style="fill:#FFFFFF;" d="M298.288,311.472H275.92c-11.136,0-11.136-16.368,0-16.368h60.496c11.392,0,11.392,16.368,0,16.368
+		h-21.232v64.592c0,11.12-16.896,11.392-16.896,0V311.472z"/>
+</g>
+<path style="fill:#CAD1D8;" d="M400,432H96v16h304c8.8,0,16-7.2,16-16v-16C416,424.8,408.8,432,400,432z"/>
+</svg>

app/components/ui/input.tsx

@@ -0,0 +1,25 @@
+import * as React from "react";
+
+import { cn } from "./lib/utils";
+
+export interface InputProps
+  extends React.InputHTMLAttributes<HTMLInputElement> {}
+
+const Input = React.forwardRef<HTMLInputElement, InputProps>(
+  ({ className, type, ...props }, ref) => {
+    return (
+      <input
+        type={type}
+        className={cn(
+          "flex h-10 w-full rounded-md border border-input bg-background px-3 py-2 text-sm ring-offset-background file:border-0 file:bg-transparent file:text-sm file:font-medium placeholder:text-muted-foreground focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring focus-visible:ring-offset-2 disabled:cursor-not-allowed disabled:opacity-50",
+          className,
+        )}
+        ref={ref}
+        {...props}
+      />
+    );
+  },
+);
+Input.displayName = "Input";
+
+export { Input };

app/components/ui/lib/utils.ts

@@ -0,0 +1,6 @@
+import { clsx, type ClassValue } from "clsx";
+import { twMerge } from "tailwind-merge";
+
+export function cn(...inputs: ClassValue[]) {
+  return twMerge(clsx(inputs));
+}

app/components/ui/select.tsx

@@ -0,0 +1,159 @@
+"use client";
+
+import * as SelectPrimitive from "@radix-ui/react-select";
+import { Check, ChevronDown, ChevronUp } from "lucide-react";
+import * as React from "react";
+import { cn } from "./lib/utils";
+
+const Select = SelectPrimitive.Root;
+
+const SelectGroup = SelectPrimitive.Group;
+
+const SelectValue = SelectPrimitive.Value;
+
+const SelectTrigger = React.forwardRef<
+  React.ElementRef<typeof SelectPrimitive.Trigger>,
+  React.ComponentPropsWithoutRef<typeof SelectPrimitive.Trigger>
+>(({ className, children, ...props }, ref) => (
+  <SelectPrimitive.Trigger
+    ref={ref}
+    className={cn(
+      "flex h-10 w-full items-center justify-between rounded-md border border-input bg-background px-3 py-2 text-sm ring-offset-background placeholder:text-muted-foreground focus:outline-none focus:ring-2 focus:ring-ring focus:ring-offset-2 disabled:cursor-not-allowed disabled:opacity-50 [&>span]:line-clamp-1",
+      className,
+    )}
+    {...props}
+  >
+    {children}
+    <SelectPrimitive.Icon asChild>
+      <ChevronDown className="h-4 w-4 opacity-50" />
+    </SelectPrimitive.Icon>
+  </SelectPrimitive.Trigger>
+));
+SelectTrigger.displayName = SelectPrimitive.Trigger.displayName;
+
+const SelectScrollUpButton = React.forwardRef<
+  React.ElementRef<typeof SelectPrimitive.ScrollUpButton>,
+  React.ComponentPropsWithoutRef<typeof SelectPrimitive.ScrollUpButton>
+>(({ className, ...props }, ref) => (
+  <SelectPrimitive.ScrollUpButton
+    ref={ref}
+    className={cn(
+      "flex cursor-default items-center justify-center py-1",
+      className,
+    )}
+    {...props}
+  >
+    <ChevronUp className="h-4 w-4" />
+  </SelectPrimitive.ScrollUpButton>
+));
+SelectScrollUpButton.displayName = SelectPrimitive.ScrollUpButton.displayName;
+
+const SelectScrollDownButton = React.forwardRef<
+  React.ElementRef<typeof SelectPrimitive.ScrollDownButton>,
+  React.ComponentPropsWithoutRef<typeof SelectPrimitive.ScrollDownButton>
+>(({ className, ...props }, ref) => (
+  <SelectPrimitive.ScrollDownButton
+    ref={ref}
+    className={cn(
+      "flex cursor-default items-center justify-center py-1",
+      className,
+    )}
+    {...props}
+  >
+    <ChevronDown className="h-4 w-4" />
+  </SelectPrimitive.ScrollDownButton>
+));
+SelectScrollDownButton.displayName =
+  SelectPrimitive.ScrollDownButton.displayName;
+
+const SelectContent = React.forwardRef<
+  React.ElementRef<typeof SelectPrimitive.Content>,
+  React.ComponentPropsWithoutRef<typeof SelectPrimitive.Content>
+>(({ className, children, position = "popper", ...props }, ref) => (
+  <SelectPrimitive.Portal>
+    <SelectPrimitive.Content
+      ref={ref}
+      className={cn(
+        "relative z-50 max-h-96 min-w-[8rem] overflow-hidden rounded-md border bg-popover text-popover-foreground shadow-md data-[state=open]:animate-in data-[state=closed]:animate-out data-[state=closed]:fade-out-0 data-[state=open]:fade-in-0 data-[state=closed]:zoom-out-95 data-[state=open]:zoom-in-95 data-[side=bottom]:slide-in-from-top-2 data-[side=left]:slide-in-from-right-2 data-[side=right]:slide-in-from-left-2 data-[side=top]:slide-in-from-bottom-2",
+        position === "popper" &&
+          "data-[side=bottom]:translate-y-1 data-[side=left]:-translate-x-1 data-[side=right]:translate-x-1 data-[side=top]:-translate-y-1",
+        className,
+      )}
+      position={position}
+      {...props}
+    >
+      <SelectScrollUpButton />
+      <SelectPrimitive.Viewport
+        className={cn(
+          "p-1",
+          position === "popper" &&
+            "h-[var(--radix-select-trigger-height)] w-full min-w-[var(--radix-select-trigger-width)]",
+        )}
+      >
+        {children}
+      </SelectPrimitive.Viewport>
+      <SelectScrollDownButton />
+    </SelectPrimitive.Content>
+  </SelectPrimitive.Portal>
+));
+SelectContent.displayName = SelectPrimitive.Content.displayName;
+
+const SelectLabel = React.forwardRef<
+  React.ElementRef<typeof SelectPrimitive.Label>,
+  React.ComponentPropsWithoutRef<typeof SelectPrimitive.Label>
+>(({ className, ...props }, ref) => (
+  <SelectPrimitive.Label
+    ref={ref}
+    className={cn("py-1.5 pl-8 pr-2 text-sm font-semibold", className)}
+    {...props}
+  />
+));
+SelectLabel.displayName = SelectPrimitive.Label.displayName;
+
+const SelectItem = React.forwardRef<
+  React.ElementRef<typeof SelectPrimitive.Item>,
+  React.ComponentPropsWithoutRef<typeof SelectPrimitive.Item>
+>(({ className, children, ...props }, ref) => (
+  <SelectPrimitive.Item
+    ref={ref}
+    className={cn(
+      "relative flex w-full cursor-default select-none items-center rounded-sm py-1.5 pl-8 pr-2 text-sm outline-none focus:bg-accent focus:text-accent-foreground data-[disabled]:pointer-events-none data-[disabled]:opacity-50",
+      className,
+    )}
+    {...props}
+  >
+    <span className="absolute left-2 flex h-3.5 w-3.5 items-center justify-center">
+      <SelectPrimitive.ItemIndicator>
+        <Check className="h-4 w-4" />
+      </SelectPrimitive.ItemIndicator>
+    </span>
+
+    <SelectPrimitive.ItemText>{children}</SelectPrimitive.ItemText>
+  </SelectPrimitive.Item>
+));
+SelectItem.displayName = SelectPrimitive.Item.displayName;
+
+const SelectSeparator = React.forwardRef<
+  React.ElementRef<typeof SelectPrimitive.Separator>,
+  React.ComponentPropsWithoutRef<typeof SelectPrimitive.Separator>
+>(({ className, ...props }, ref) => (
+  <SelectPrimitive.Separator
+    ref={ref}
+    className={cn("-mx-1 my-1 h-px bg-muted", className)}
+    {...props}
+  />
+));
+SelectSeparator.displayName = SelectPrimitive.Separator.displayName;
+
+export {
+  Select,
+  SelectContent,
+  SelectGroup,
+  SelectItem,
+  SelectLabel,
+  SelectScrollDownButton,
+  SelectScrollUpButton,
+  SelectSeparator,
+  SelectTrigger,
+  SelectValue,
+};

app/components/ui/tabs.tsx

@@ -0,0 +1,54 @@
+"use client";
+
+import * as TabsPrimitive from "@radix-ui/react-tabs";
+import * as React from "react";
+import { cn } from "./lib/utils";
+
+const Tabs = TabsPrimitive.Root;
+
+const TabsList = React.forwardRef<
+  React.ElementRef<typeof TabsPrimitive.List>,
+  React.ComponentPropsWithoutRef<typeof TabsPrimitive.List>
+>(({ className, ...props }, ref) => (
+  <TabsPrimitive.List
+    ref={ref}
+    className={cn(
+      "inline-flex h-9 items-center justify-center rounded-lg bg-muted p-1 text-muted-foreground",
+      className,
+    )}
+    {...props}
+  />
+));
+TabsList.displayName = TabsPrimitive.List.displayName;
+
+const TabsTrigger = React.forwardRef<
+  React.ElementRef<typeof TabsPrimitive.Trigger>,
+  React.ComponentPropsWithoutRef<typeof TabsPrimitive.Trigger>
+>(({ className, ...props }, ref) => (
+  <TabsPrimitive.Trigger
+    ref={ref}
+    className={cn(
+      "inline-flex items-center justify-center whitespace-nowrap rounded-md px-3 py-1 text-sm font-medium ring-offset-background transition-all focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring focus-visible:ring-offset-2 disabled:pointer-events-none disabled:opacity-50 data-[state=active]:bg-background data-[state=active]:text-foreground data-[state=active]:shadow",
+      className,
+    )}
+    {...props}
+  />
+));
+TabsTrigger.displayName = TabsPrimitive.Trigger.displayName;
+
+const TabsContent = React.forwardRef<
+  React.ElementRef<typeof TabsPrimitive.Content>,
+  React.ComponentPropsWithoutRef<typeof TabsPrimitive.Content>
+>(({ className, ...props }, ref) => (
+  <TabsPrimitive.Content
+    ref={ref}
+    className={cn(
+      "ring-offset-background focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring focus-visible:ring-offset-2",
+      className,
+    )}
+    {...props}
+  />
+));
+TabsContent.displayName = TabsPrimitive.Content.displayName;
+
+export { Tabs, TabsContent, TabsList, TabsTrigger };

app/components/ui/textarea.tsx

@@ -0,0 +1,23 @@
+import * as React from "react";
+import { cn } from "./lib/utils";
+
+export interface TextareaProps
+  extends React.TextareaHTMLAttributes<HTMLTextAreaElement> {}
+
+const Textarea = React.forwardRef<HTMLTextAreaElement, TextareaProps>(
+  ({ className, ...props }, ref) => {
+    return (
+      <textarea
+        className={cn(
+          "flex min-h-[80px] w-full rounded-md border border-input bg-background px-3 py-2 text-sm ring-offset-background placeholder:text-muted-foreground focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring focus-visible:ring-offset-2 disabled:cursor-not-allowed disabled:opacity-50",
+          className,
+        )}
+        ref={ref}
+        {...props}
+      />
+    );
+  },
+);
+Textarea.displayName = "Textarea";
+
+export { Textarea };

app/components/ui/upload-image-preview.tsx

@@ -0,0 +1,32 @@
+import { XCircleIcon } from "lucide-react";
+import Image from "next/image";
+import { cn } from "./lib/utils";
+
+export default function UploadImagePreview({
+  url,
+  onRemove,
+}: {
+  url: string;
+  onRemove: () => void;
+}) {
+  return (
+    <div className="relative w-20 h-20 group">
+      <Image
+        src={url}
+        alt="Uploaded image"
+        fill
+        className="object-cover w-full h-full rounded-xl hover:brightness-75"
+      />
+      <div
+        className={cn(
+          "absolute -top-2 -right-2 w-6 h-6 z-10 bg-gray-500 text-white rounded-full hidden group-hover:block",
+        )}
+      >
+        <XCircleIcon
+          className="w-6 h-6 bg-gray-500 text-white rounded-full"
+          onClick={onRemove}
+        />
+      </div>
+    </div>
+  );
+}

app/globals.css

@@ -0,0 +1,126 @@
+@tailwind base;
+@tailwind components;
+@tailwind utilities;
+
+@layer base {
+  :root {
+    --background: 0 0% 100%;
+    --foreground: 222.2 47.4% 11.2%;
+
+    --muted: 210 40% 96.1%;
+    --muted-foreground: 215.4 16.3% 46.9%;
+
+    --popover: 0 0% 100%;
+    --popover-foreground: 222.2 47.4% 11.2%;
+
+    --border: 214.3 31.8% 91.4%;
+    --input: 214.3 31.8% 91.4%;
+
+    --card: 0 0% 100%;
+    --card-foreground: 222.2 47.4% 11.2%;
+
+    --primary: 222.2 47.4% 11.2%;
+    --primary-foreground: 210 40% 98%;
+
+    --secondary: 210 40% 96.1%;
+    --secondary-foreground: 222.2 47.4% 11.2%;
+
+    --accent: 210 40% 96.1%;
+    --accent-foreground: 222.2 47.4% 11.2%;
+
+    --destructive: 0 100% 50%;
+    --destructive-foreground: 210 40% 98%;
+
+    --ring: 215 20.2% 65.1%;
+
+    --radius: 0.5rem;
+  }
+
+  .dark {
+    --background: 224 71% 4%;
+    --foreground: 213 31% 91%;
+
+    --muted: 223 47% 11%;
+    --muted-foreground: 215.4 16.3% 56.9%;
+
+    --accent: 216 34% 17%;
+    --accent-foreground: 210 40% 98%;
+
+    --popover: 224 71% 4%;
+    --popover-foreground: 215 20.2% 65.1%;
+
+    --border: 216 34% 17%;
+    --input: 216 34% 17%;
+
+    --card: 224 71% 4%;
+    --card-foreground: 213 31% 91%;
+
+    --primary: 210 40% 98%;
+    --primary-foreground: 222.2 47.4% 1.2%;
+
+    --secondary: 222.2 47.4% 11.2%;
+    --secondary-foreground: 210 40% 98%;
+
+    --destructive: 0 63% 31%;
+    --destructive-foreground: 210 40% 98%;
+
+    --ring: 216 34% 17%;
+
+    --radius: 0.5rem;
+  }
+}
+
+@layer base {
+  * {
+    @apply border-border;
+  }
+  html {
+    @apply h-full;
+  }
+  body {
+    @apply bg-background text-foreground h-full;
+    font-feature-settings:
+      "rlig" 1,
+      "calt" 1;
+  }
+  .background-gradient {
+    background-color: #fff;
+    background-image: radial-gradient(
+        at 21% 11%,
+        rgba(186, 186, 233, 0.53) 0,
+        transparent 50%
+      ),
+      radial-gradient(at 85% 0, hsla(46, 57%, 78%, 0.52) 0, transparent 50%),
+      radial-gradient(at 91% 36%, rgba(194, 213, 255, 0.68) 0, transparent 50%),
+      radial-gradient(at 8% 40%, rgba(251, 218, 239, 0.46) 0, transparent 50%);
+  }
+
+  @keyframes fadeIn {
+    0% {
+      opacity: 0;
+    }
+    100% {
+      opacity: 1;
+    }
+  }
+
+  .fadein-agent {
+    animation-name: fadeIn;
+    animation-duration: 1.5s;
+  }
+
+  @keyframes slideIn {
+    from {
+      transform: translateX(10%);
+      opacity: 0;
+    }
+    to {
+      transform: translateX(0);
+      opacity: 1;
+    }
+  }
+
+  .animate-slideIn {
+    animation: slideIn 0.5s ease-out;
+  }
+}

app/layout.tsx

@@ -0,0 +1,23 @@
+import type { Metadata } from "next";
+import { Inter } from "next/font/google";
+import "./globals.css";
+import "./markdown.css";
+
+const inter = Inter({ subsets: ["latin"] });
+
+export const metadata: Metadata = {
+  title: "Create Llama App",
+  description: "Generated by create-llama",
+};
+
+export default function RootLayout({
+  children,
+}: {
+  children: React.ReactNode;
+}) {
+  return (
+    <html lang="en">
+      <body className={inter.className}>{children}</body>
+    </html>
+  );
+}

app/markdown.css

@@ -0,0 +1,79 @@
+/* Custom CSS for chat message markdown */
+.custom-markdown ul {
+  list-style-type: disc;
+  margin-left: 20px;
+  margin-top: 20px;
+  margin-bottom: 20px;
+}
+
+.custom-markdown ol {
+  list-style-type: decimal;
+  margin-left: 20px;
+  margin-top: 20px;
+  margin-bottom: 20px;
+}
+
+.custom-markdown li {
+  margin-bottom: 5px;
+}
+
+.custom-markdown ol ol {
+  list-style: lower-alpha;
+}
+
+.custom-markdown ul ul,
+.custom-markdown ol ol {
+  margin-left: 20px;
+}
+
+.custom-markdown img {
+  border-radius: 8px;
+  box-shadow: 0 0 10px rgba(0, 0, 0, 0.1);
+  margin: 10px 0;
+}
+
+.custom-markdown a {
+  text-decoration: underline;
+  color: #007bff;
+}
+
+.custom-markdown h1,
+h2,
+h3,
+h4,
+h5,
+h6 {
+  font-weight: bold;
+  margin-bottom: 20px;
+  margin-top: 20px;
+}
+
+.custom-markdown h6 {
+  font-size: 16px;
+}
+
+.custom-markdown h5 {
+  font-size: 18px;
+}
+
+.custom-markdown h4 {
+  font-size: 20px;
+}
+
+.custom-markdown h3 {
+  font-size: 22px;
+}
+
+.custom-markdown h2 {
+  font-size: 24px;
+}
+
+.custom-markdown h1 {
+  font-size: 26px;
+}
+
+.custom-markdown hr {
+  border: 0;
+  border-top: 1px solid #e1e4e8;
+  margin: 20px 0;
+}

app/observability/index.ts

@@ -0,0 +1 @@
+export const initObservability = () => {};

app/page.tsx

@@ -0,0 +1,15 @@
+import Header from "@/app/components/header";
+import ChatSection from "./components/chat-section";
+
+export default function Home() {
+  return (
+    <main className="h-screen w-screen flex justify-center items-center background-gradient">
+      <div className="space-y-2 lg:space-y-10 w-[90%] lg:w-[60rem]">
+        <Header />
+        <div className="h-[65vh] flex">
+          <ChatSection />
+        </div>
+      </div>
+    </main>
+  );
+}

config/tools.json

@@ -0,0 +1,6 @@
+{
+  "local": {
+    "duckduckgo": {}
+  },
+  "llamahub": {}
+}

next.config.json

@@ -0,0 +1,17 @@
+{
+  "experimental": {
+    "outputFileTracingIncludes": {
+      "/*": [
+        "./cache/**/*"
+      ]
+    },
+    "outputFileTracingExcludes": {
+      "/api/files/*": [
+        ".next/**/*",
+        "node_modules/**/*",
+        "public/**/*",
+        "app/**/*"
+      ]
+    }
+  }
+}

next.config.mjs

@@ -0,0 +1,10 @@
+/** @type {import('next').NextConfig} */
+import fs from "fs";
+import withLlamaIndex from "llamaindex/next";
+import webpack from "./webpack.config.mjs";
+
+const nextConfig = JSON.parse(fs.readFileSync("./next.config.json", "utf-8"));
+nextConfig.webpack = webpack;
+
+// use withLlamaIndex to add necessary modifications for llamaindex library
+export default withLlamaIndex(nextConfig);

package.json

@@ -0,0 +1,67 @@
+{
+  "name": "rag",
+  "version": "0.1.0",
+  "scripts": {
+    "format": "prettier --ignore-unknown --cache --check .",
+    "format:write": "prettier --ignore-unknown --write .",
+    "dev": "next dev",
+    "build": "next build",
+    "start": "next start",
+    "lint": "next lint",
+    "generate": "tsx app\\api\\chat\\engine\\generate.ts"
+  },
+  "dependencies": {
+    "@apidevtools/swagger-parser": "^10.1.0",
+    "@e2b/code-interpreter": "0.0.9-beta.3",
+    "@llamaindex/core": "^0.2.6",
+    "@llamaindex/pdf-viewer": "^1.1.3",
+    "@radix-ui/react-collapsible": "^1.0.3",
+    "@radix-ui/react-hover-card": "^1.0.7",
+    "@radix-ui/react-select": "^2.1.1",
+    "@radix-ui/react-slot": "^1.0.2",
+    "@radix-ui/react-tabs": "^1.1.0",
+    "ai": "3.3.42",
+    "ajv": "^8.12.0",
+    "class-variance-authority": "^0.7.0",
+    "clsx": "^2.1.1",
+    "dotenv": "^16.3.1",
+    "duck-duck-scrape": "^2.2.5",
+    "formdata-node": "^6.0.3",
+    "got": "^14.4.1",
+    "llamaindex": "0.6.22",
+    "lucide-react": "^0.294.0",
+    "next": "^14.2.4",
+    "react": "^18.2.0",
+    "react-dom": "^18.2.0",
+    "react-markdown": "^8.0.7",
+    "rehype-katex": "^7.0.0",
+    "remark": "^14.0.3",
+    "remark-code-import": "^1.2.0",
+    "remark-gfm": "^3.0.1",
+    "remark-math": "^5.1.1",
+    "supports-color": "^8.1.1",
+    "tailwind-merge": "^2.1.0",
+    "tiktoken": "^1.0.15",
+    "uuid": "^9.0.1",
+    "vaul": "^0.9.1",
+    "marked": "^14.1.2",
+    "highlight.js": "^11.10.0"
+  },
+  "devDependencies": {
+    "@types/node": "^20.10.3",
+    "@types/react": "^18.2.42",
+    "@types/react-dom": "^18.2.17",
+    "@types/uuid": "^9.0.8",
+    "autoprefixer": "^10.4.16",
+    "cross-env": "^7.0.3",
+    "eslint": "^8.55.0",
+    "eslint-config-next": "^14.2.4",
+    "eslint-config-prettier": "^8.10.0",
+    "postcss": "^8.4.32",
+    "prettier": "^3.2.5",
+    "prettier-plugin-organize-imports": "^3.2.4",
+    "tailwindcss": "^3.3.6",
+    "tsx": "^4.7.2",
+    "typescript": "^5.3.2"
+  }
+}

postcss.config.js

@@ -0,0 +1,6 @@
+module.exports = {
+  plugins: {
+    tailwindcss: {},
+    autoprefixer: {},
+  },
+};

prettier.config.js

@@ -0,0 +1,3 @@
+module.exports = {
+  plugins: ["prettier-plugin-organize-imports"],
+};

tailwind.config.ts

@@ -0,0 +1,78 @@
+import type { Config } from "tailwindcss";
+import { fontFamily } from "tailwindcss/defaultTheme";
+
+const config: Config = {
+  darkMode: ["class"],
+  content: ["app/**/*.{ts,tsx}", "components/**/*.{ts,tsx}"],
+  theme: {
+    container: {
+      center: true,
+      padding: "2rem",
+      screens: {
+        "2xl": "1400px",
+      },
+    },
+    extend: {
+      colors: {
+        border: "hsl(var(--border))",
+        input: "hsl(var(--input))",
+        ring: "hsl(var(--ring))",
+        background: "hsl(var(--background))",
+        foreground: "hsl(var(--foreground))",
+        primary: {
+          DEFAULT: "hsl(var(--primary))",
+          foreground: "hsl(var(--primary-foreground))",
+        },
+        secondary: {
+          DEFAULT: "hsl(var(--secondary))",
+          foreground: "hsl(var(--secondary-foreground))",
+        },
+        destructive: {
+          DEFAULT: "hsl(var(--destructive) / <alpha-value>)",
+          foreground: "hsl(var(--destructive-foreground) / <alpha-value>)",
+        },
+        muted: {
+          DEFAULT: "hsl(var(--muted))",
+          foreground: "hsl(var(--muted-foreground))",
+        },
+        accent: {
+          DEFAULT: "hsl(var(--accent))",
+          foreground: "hsl(var(--accent-foreground))",
+        },
+        popover: {
+          DEFAULT: "hsl(var(--popover))",
+          foreground: "hsl(var(--popover-foreground))",
+        },
+        card: {
+          DEFAULT: "hsl(var(--card))",
+          foreground: "hsl(var(--card-foreground))",
+        },
+      },
+      borderRadius: {
+        xl: `calc(var(--radius) + 4px)`,
+        lg: `var(--radius)`,
+        md: `calc(var(--radius) - 2px)`,
+        sm: "calc(var(--radius) - 4px)",
+      },
+      fontFamily: {
+        sans: ["var(--font-sans)", ...fontFamily.sans],
+      },
+      keyframes: {
+        "accordion-down": {
+          from: { height: "0" },
+          to: { height: "var(--radix-accordion-content-height)" },
+        },
+        "accordion-up": {
+          from: { height: "var(--radix-accordion-content-height)" },
+          to: { height: "0" },
+        },
+      },
+      animation: {
+        "accordion-down": "accordion-down 0.2s ease-out",
+        "accordion-up": "accordion-up 0.2s ease-out",
+      },
+    },
+  },
+  plugins: [],
+};
+export default config;

tsconfig.json

@@ -0,0 +1,26 @@
+{
+  "compilerOptions": {
+    "lib": ["dom", "dom.iterable", "esnext"],
+    "allowJs": true,
+    "skipLibCheck": true,
+    "strict": true,
+    "noEmit": true,
+    "esModuleInterop": true,
+    "module": "esnext",
+    "moduleResolution": "bundler",
+    "resolveJsonModule": true,
+    "isolatedModules": true,
+    "jsx": "preserve",
+    "incremental": true,
+    "plugins": [
+      {
+        "name": "next"
+      }
+    ],
+    "paths": {
+      "@/*": ["./*"]
+    }
+  },
+  "include": ["next-env.d.ts", "**/*.ts", "**/*.tsx", ".next/types/**/*.ts"],
+  "exclude": ["node_modules"]
+}

webpack.config.mjs

@@ -0,0 +1,8 @@
+// webpack config must be a function in NextJS that is used to patch the default webpack config provided by NextJS, see https://nextjs.org/docs/pages/api-reference/next-config-js/webpack
+export default function webpack(config) {
+  config.resolve.fallback = {
+    aws4: false,
+  };
+
+  return config;
+}