Release 0.3.12 (#416)

Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>

CHANGELOG.md

@@ -1,5 +1,35 @@
 # create-llama
 
+## 0.3.12
+
+### Patch Changes
+
+- 6edea6a: Optimize generated workflow code for Python
+- 8431b78: Optimize Typescript multi-agent code
+- 8431b78: Add form filling use case (Typescript)
+
+## 0.3.11
+
+### Patch Changes
+
+- 2b8aaa8: Add support for local models via Hugging Face
+- b9570b2: Fix: use generic LLMAgent instead of OpenAIAgent (adds support for Gemini and Anthropic for Agentic RAG)
+- 1fe21f8: Fix the highlight.js issue with the Next.js static build
+- 00009ae: feat: import pdf css
+
+## 0.3.10
+
+### Patch Changes
+
+- 9172fed: feat: bump LITS 0.8.2
+- 78ccde7: feat: use llamaindex chat-ui for nextjs frontend
+
+## 0.3.9
+
+### Patch Changes
+
+- ed59927: Add form filling use case (Python)
+
 ## 0.3.8
 
 ### Patch Changes

e2e/shared/multiagent_template.spec.ts

@@ -18,7 +18,7 @@ const templateUI: TemplateUI = "shadcn";
 const templatePostInstallAction: TemplatePostInstallAction = "runApp";
 const appType: AppType = templateFramework === "nextjs" ? "" : "--frontend";
 const userMessage = "Write a blog post about physical standards for letters";
-const templateAgents = ["financial_report", "blog"];
+const templateAgents = ["financial_report", "blog", "form_filling"];
 
 for (const agents of templateAgents) {
   test.describe(`Test multiagent template ${agents} ${templateFramework} ${dataSource} ${templateUI} ${appType} ${templatePostInstallAction}`, async () => {
@@ -68,6 +68,10 @@ for (const agents of templateAgents) {
     test("Frontend should be able to submit a message and receive the start of a streamed response", async ({
       page,
     }) => {
+      test.skip(
+        agents === "financial_report" || agents === "form_filling",
+        "Skip chat tests for financial report and form filling.",
+      );
       await page.goto(`http://localhost:${port}`);
       await page.fill("form textarea", userMessage);
 

helpers/env-variables.ts

@@ -336,6 +336,20 @@ const getModelEnvs = (modelConfig: ModelConfig): EnvVar[] => {
           },
         ]
       : []),
+    ...(modelConfig.provider === "huggingface"
+      ? [
+          {
+            name: "EMBEDDING_BACKEND",
+            description:
+              "The backend to use for the Sentence Transformers embedding model, either 'torch', 'onnx', or 'openvino'. Defaults to 'onnx'.",
+          },
+          {
+            name: "EMBEDDING_TRUST_REMOTE_CODE",
+            description:
+              "Whether to trust remote code for the embedding model, required for some models with custom code.",
+          },
+        ]
+      : []),
     ...(modelConfig.provider === "t-systems"
       ? [
           {

helpers/providers/huggingface.ts

@@ -0,0 +1,68 @@
+import prompts from "prompts";
+import { ModelConfigParams } from ".";
+import { questionHandlers, toChoice } from "../../questions/utils";
+
+const MODELS = ["HuggingFaceH4/zephyr-7b-alpha"];
+type ModelData = {
+  dimensions: number;
+};
+const EMBEDDING_MODELS: Record<string, ModelData> = {
+  "BAAI/bge-small-en-v1.5": { dimensions: 384 },
+  "BAAI/bge-base-en-v1.5": { dimensions: 768 },
+  "BAAI/bge-large-en-v1.5": { dimensions: 1024 },
+  "sentence-transformers/all-MiniLM-L6-v2": { dimensions: 384 },
+  "sentence-transformers/all-mpnet-base-v2": { dimensions: 768 },
+  "intfloat/multilingual-e5-large": { dimensions: 1024 },
+  "mixedbread-ai/mxbai-embed-large-v1": { dimensions: 1024 },
+  "nomic-ai/nomic-embed-text-v1.5": { dimensions: 768 },
+};
+
+const DEFAULT_MODEL = MODELS[0];
+const DEFAULT_EMBEDDING_MODEL = Object.keys(EMBEDDING_MODELS)[0];
+const DEFAULT_DIMENSIONS = Object.values(EMBEDDING_MODELS)[0].dimensions;
+
+type HuggingfaceQuestionsParams = {
+  askModels: boolean;
+};
+
+export async function askHuggingfaceQuestions({
+  askModels,
+}: HuggingfaceQuestionsParams): Promise<ModelConfigParams> {
+  const config: ModelConfigParams = {
+    model: DEFAULT_MODEL,
+    embeddingModel: DEFAULT_EMBEDDING_MODEL,
+    dimensions: DEFAULT_DIMENSIONS,
+    isConfigured(): boolean {
+      return true;
+    },
+  };
+
+  if (askModels) {
+    const { model } = await prompts(
+      {
+        type: "select",
+        name: "model",
+        message: "Which Hugging Face model would you like to use?",
+        choices: MODELS.map(toChoice),
+        initial: 0,
+      },
+      questionHandlers,
+    );
+    config.model = model;
+
+    const { embeddingModel } = await prompts(
+      {
+        type: "select",
+        name: "embeddingModel",
+        message: "Which embedding model would you like to use?",
+        choices: Object.keys(EMBEDDING_MODELS).map(toChoice),
+        initial: 0,
+      },
+      questionHandlers,
+    );
+    config.embeddingModel = embeddingModel;
+    config.dimensions = EMBEDDING_MODELS[embeddingModel].dimensions;
+  }
+
+  return config;
+}

helpers/providers/index.ts

@@ -5,6 +5,7 @@ import { askAnthropicQuestions } from "./anthropic";
 import { askAzureQuestions } from "./azure";
 import { askGeminiQuestions } from "./gemini";
 import { askGroqQuestions } from "./groq";
+import { askHuggingfaceQuestions } from "./huggingface";
 import { askLLMHubQuestions } from "./llmhub";
 import { askMistralQuestions } from "./mistral";
 import { askOllamaQuestions } from "./ollama";
@@ -39,6 +40,7 @@ export async function askModelConfig({
 
     if (framework === "fastapi") {
       choices.push({ title: "T-Systems", value: "t-systems" });
+      choices.push({ title: "Huggingface", value: "huggingface" });
     }
     const { provider } = await prompts(
       {
@@ -76,6 +78,9 @@ export async function askModelConfig({
     case "t-systems":
       modelConfig = await askLLMHubQuestions({ askModels });
       break;
+    case "huggingface":
+      modelConfig = await askHuggingfaceQuestions({ askModels });
+      break;
     default:
       modelConfig = await askOpenAIQuestions({
         openAiKey,

helpers/python.ts

@@ -234,6 +234,21 @@ const getAdditionalDependencies = (
         version: "0.2.4",
       });
       break;
+    case "huggingface":
+      dependencies.push({
+        name: "llama-index-llms-huggingface",
+        version: "^0.3.5",
+      });
+      dependencies.push({
+        name: "llama-index-embeddings-huggingface",
+        version: "^0.3.1",
+      });
+      dependencies.push({
+        name: "optimum",
+        version: "^1.23.3",
+        extras: ["onnxruntime"],
+      });
+      break;
     case "t-systems":
       dependencies.push({
         name: "llama-index-agent-openai",

helpers/tools.ts

@@ -267,6 +267,22 @@ For better results, you can specify the region parameter to get results from a s
       },
     ],
   },
+  {
+    display: "Form Filling",
+    name: "form_filling",
+    supportedFrameworks: ["fastapi"],
+    type: ToolType.LOCAL,
+    dependencies: [
+      {
+        name: "pandas",
+        version: "^2.2.3",
+      },
+      {
+        name: "tabulate",
+        version: "^0.9.0",
+      },
+    ],
+  },
 ];
 
 export const getTool = (toolName: string): Tool | undefined => {

helpers/types.ts

@@ -9,6 +9,7 @@ export type ModelProvider =
   | "gemini"
   | "mistral"
   | "azure-openai"
+  | "huggingface"
   | "t-systems";
 export type ModelConfig = {
   provider: ModelProvider;
@@ -48,7 +49,7 @@ export type TemplateDataSource = {
 };
 export type TemplateDataSourceType = "file" | "web" | "db";
 export type TemplateObservability = "none" | "traceloop" | "llamatrace";
-export type TemplateAgents = "financial_report" | "blog";
+export type TemplateAgents = "financial_report" | "blog" | "form_filling";
 // Config for both file and folder
 export type FileSourceConfig =
   | {

helpers/typescript.ts

@@ -136,19 +136,22 @@ export const installTSTemplate = async ({
     // Copy agents use case code for multiagent template
     if (agents) {
       console.log("\nCopying agent:", agents, "\n");
+      const useCasePath = path.join(compPath, "agents", "typescript", agents);
+      const agentsCodePath = path.join(useCasePath, "workflow");
 
-      const agentsCodePath = path.join(
-        compPath,
-        "agents",
-        "typescript",
-        agents,
-      );
-
+      // Copy agent codes
       await copy("**", path.join(root, relativeEngineDestPath, "workflow"), {
         parents: true,
         cwd: agentsCodePath,
         rename: assetRelocator,
       });
+
+      // Copy use case files to project root
+      await copy("*.*", path.join(root), {
+        parents: true,
+        cwd: useCasePath,
+        rename: assetRelocator,
+      });
     } else {
       console.log(
         red(

package.json

@@ -1,6 +1,6 @@
 {
   "name": "create-llama",
-  "version": "0.3.8",
+  "version": "0.3.12",
   "description": "Create LlamaIndex-powered apps with one command",
   "keywords": [
     "rag",

questions/questions.ts

@@ -177,6 +177,34 @@ export const askProQuestions = async (program: QuestionArgs) => {
     program.observability = observability;
   }
 
+  // Ask agents
+  if (program.template === "multiagent" && !program.agents) {
+    const { agents } = await prompts(
+      {
+        type: "select",
+        name: "agents",
+        message: "Which agents would you like to use?",
+        choices: [
+          {
+            title: "Financial report (generate a financial report)",
+            value: "financial_report",
+          },
+          {
+            title: "Form filling (fill missing value in a CSV file)",
+            value: "form_filling",
+          },
+          {
+            title: "Blog writer (Write a blog post)",
+            value: "blog_writer",
+          },
+        ],
+        initial: 0,
+      },
+      questionHandlers,
+    );
+    program.agents = agents;
+  }
+
   if (!program.modelConfig) {
     const modelConfig = await askModelConfig({
       openAiKey: program.openAiKey,

questions/simple.ts

@@ -10,6 +10,7 @@ type AppType =
   | "rag"
   | "code_artifact"
   | "financial_report_agent"
+  | "form_filling"
   | "extractor"
   | "data_scientist";
 
@@ -35,8 +36,12 @@ export const askSimpleQuestions = async (
           title: "Financial Report Generator (using Workflows)",
           value: "financial_report_agent",
         },
+        {
+          title: "Form Filler (using Workflows)",
+          value: "form_filling",
+        },
         { title: "Code Artifact Agent", value: "code_artifact" },
-        { title: "Structured extraction", value: "extractor" },
+        { title: "Information Extractor", value: "extractor" },
       ],
     },
     questionHandlers,
@@ -152,6 +157,14 @@ const convertAnswers = async (
       frontend: true,
       modelConfig: MODEL_GPT4o,
     },
+    form_filling: {
+      template: "multiagent",
+      agents: "form_filling",
+      tools: getTools(["form_filling"]),
+      dataSources: EXAMPLE_10K_SEC_FILES,
+      frontend: true,
+      modelConfig: MODEL_GPT4o,
+    },
     extractor: {
       template: "extractor",
       tools: [],

templates/components/agents/python/blog/README-template.md

@@ -8,9 +8,9 @@ This example is using three agents to generate a blog post:
 
 There are three different methods how the agents can interact to reach their goal:
 
-1. [Choreography](./app/examples/choreography.py) - the agents decide themselves to delegate a task to another agent
-1. [Orchestrator](./app/examples/orchestrator.py) - a central orchestrator decides which agent should execute a task
-1. [Explicit Workflow](./app/examples/workflow.py) - a pre-defined workflow specific for the task is used to execute the tasks
+1. [Choreography](./app/agents/choreography.py) - the agents decide themselves to delegate a task to another agent
+1. [Orchestrator](./app/agents/orchestrator.py) - a central orchestrator decides which agent should execute a task
+1. [Explicit Workflow](./app/agents/workflow.py) - a pre-defined workflow specific for the task is used to execute the tasks
 
 ## Getting Started
 

templates/components/agents/python/blog/app/agents/publisher.py

@@ -11,11 +11,11 @@ def get_publisher_tools() -> Tuple[List[FunctionTool], str, str]:
     tools = []
     # Get configured tools from the tools.yaml file
     configured_tools = ToolFactory.from_env(map_result=True)
-    if "document_generator" in configured_tools.keys():
-        tools.extend(configured_tools["document_generator"])
+    if "generate_document" in configured_tools.keys():
+        tools.append(configured_tools["generate_document"])
         prompt_instructions = dedent("""
             Normally, reply the blog post content to the user directly. 
-            But if user requested to generate a file, use the document_generator tool to generate the file and reply the link to the file.
+            But if user requested to generate a file, use the generate_document tool to generate the file and reply the link to the file.
         """)
         description = "Expert in publishing the blog post, able to publish the blog post in PDF or HTML format."
     else:

templates/components/agents/python/blog/app/agents/researcher.py

@@ -42,11 +42,15 @@ def _get_research_tools(**kwargs) -> QueryEngineTool:
     query_engine_tool = _create_query_engine_tool(**kwargs)
     if query_engine_tool is not None:
         tools.append(query_engine_tool)
-    researcher_tool_names = ["duckduckgo", "wikipedia.WikipediaToolSpec"]
+    researcher_tool_names = [
+        "duckduckgo_search",
+        "duckduckgo_image_search",
+        "wikipedia.WikipediaToolSpec",
+    ]
     configured_tools = ToolFactory.from_env(map_result=True)
     for tool_name, tool in configured_tools.items():
         if tool_name in researcher_tool_names:
-            tools.extend(tool)
+            tools.append(tool)
     return tools
 
 

templates/components/agents/python/blog/app/workflows/__init__.py

@@ -0,0 +1,3 @@
+from .blog import create_workflow
+
+__all__ = ["create_workflow"]

templates/components/agents/python/blog/app/workflows/blog.py

@@ -4,17 +4,18 @@ from typing import List, Optional
 
 from app.agents.choreography import create_choreography
 from app.agents.orchestrator import create_orchestrator
-from app.agents.workflow import create_workflow
+from app.agents.workflow import create_workflow as create_blog_workflow
 from llama_index.core.chat_engine.types import ChatMessage
 from llama_index.core.workflow import Workflow
 
 logger = logging.getLogger("uvicorn")
 
 
-def get_chat_engine(
+def create_workflow(
     chat_history: Optional[List[ChatMessage]] = None, **kwargs
 ) -> Workflow:
-    # TODO: the EXAMPLE_TYPE could be passed as a chat config parameter?
+    # Chat filters are not supported yet
+    kwargs.pop("filters", None)
     agent_type = os.getenv("EXAMPLE_TYPE", "").lower()
     match agent_type:
         case "choreography":
@@ -22,7 +23,7 @@ def get_chat_engine(
         case "orchestrator":
             agent = create_orchestrator(chat_history, **kwargs)
         case _:
-            agent = create_workflow(chat_history, **kwargs)
+            agent = create_blog_workflow(chat_history, **kwargs)
 
     logger.info(f"Using agent pattern: {agent_type}")
 

templates/components/agents/python/blog/app/workflows/single.py

@@ -1,4 +1,5 @@
 from abc import abstractmethod
+from enum import Enum
 from typing import Any, AsyncGenerator, List, Optional
 
 from llama_index.core.llms import ChatMessage, ChatResponse
@@ -15,7 +16,7 @@ from llama_index.core.workflow import (
     Workflow,
     step,
 )
-from pydantic import BaseModel
+from pydantic import BaseModel, Field
 
 
 class InputEvent(Event):
@@ -26,17 +27,27 @@ class ToolCallEvent(Event):
     tool_calls: list[ToolSelection]
 
 
+class AgentRunEventType(Enum):
+    TEXT = "text"
+    PROGRESS = "progress"
+
+
 class AgentRunEvent(Event):
     name: str
-    _msg: str
+    msg: str
+    event_type: AgentRunEventType = Field(default=AgentRunEventType.TEXT)
+    data: Optional[dict] = None
 
-    @property
-    def msg(self):
-        return self._msg
-
-    @msg.setter
-    def msg(self, value):
-        self._msg = value
+    def to_response(self) -> dict:
+        return {
+            "type": "agent",
+            "data": {
+                "agent": self.name,
+                "type": self.event_type.value,
+                "text": self.msg,
+                "data": self.data,
+            },
+        }
 
 
 class AgentRunResult(BaseModel):

templates/components/agents/python/financial_report/README-template.md

@@ -33,7 +33,7 @@ curl --location 'localhost:8000/api/chat' \
 --data '{ "messages": [{ "role": "user", "content": "Create a report comparing the finances of Apple and Tesla" }] }'
 ```
 
-You can start editing the API by modifying `app/api/routers/chat.py` or `app/financial_report/workflow.py`. The API auto-updates as you save the files.
+You can start editing the API by modifying `app/api/routers/chat.py` or `app/workflows/financial_report.py`. The API auto-updates as you save the files.
 
 Open [http://localhost:8000/docs](http://localhost:8000/docs) with your browser to see the Swagger UI of the API.
 

templates/components/agents/python/financial_report/app/agents/analyst.py

@@ -1,47 +0,0 @@
-from textwrap import dedent
-from typing import List, Tuple
-
-from app.engine.tools import ToolFactory
-from app.workflows.single import FunctionCallingAgent
-from llama_index.core.chat_engine.types import ChatMessage
-from llama_index.core.tools import FunctionTool
-
-
-def _get_analyst_params() -> Tuple[List[type[FunctionTool]], str, str]:
-    tools = []
-    prompt_instructions = dedent(
-        """
-        You are an expert in analyzing financial data.
-        You are given a task and a set of financial data to analyze. Your task is to analyze the financial data and return a report.
-        Your response should include a detailed analysis of the financial data, including any trends, patterns, or insights that you find.
-        Construct the analysis in a textual format like tables would be great!
-        Don't need to synthesize the data, just analyze and provide your findings.
-        Always use the provided information, don't make up any information yourself.
-        """
-    )
-    description = "Expert in analyzing financial data"
-    configured_tools = ToolFactory.from_env(map_result=True)
-    # Check if the interpreter tool is configured
-    if "interpreter" in configured_tools.keys():
-        tools.extend(configured_tools["interpreter"])
-        prompt_instructions += dedent("""
-            You are able to visualize the financial data using code interpreter tool.
-            It's very useful to create and include visualizations to the report (make sure you include the right code and data for the visualization).
-            Never include any code into the report, just the visualization.
-        """)
-        description += (
-            ", able to visualize the financial data using code interpreter tool."
-        )
-    return tools, prompt_instructions, description
-
-
-def create_analyst(chat_history: List[ChatMessage]):
-    tools, prompt_instructions, description = _get_analyst_params()
-
-    return FunctionCallingAgent(
-        name="analyst",
-        tools=tools,
-        description=description,
-        system_prompt=dedent(prompt_instructions),
-        chat_history=chat_history,
-    )

templates/components/agents/python/financial_report/app/agents/reporter.py

@@ -1,44 +0,0 @@
-from textwrap import dedent
-from typing import List, Tuple
-
-from app.engine.tools import ToolFactory
-from app.workflows.single import FunctionCallingAgent
-from llama_index.core.chat_engine.types import ChatMessage
-from llama_index.core.tools import BaseTool
-
-
-def _get_reporter_params(
-    chat_history: List[ChatMessage],
-) -> Tuple[List[type[BaseTool]], str, str]:
-    tools: List[type[BaseTool]] = []
-    description = "Expert in representing a financial report"
-    prompt_instructions = dedent(
-        """
-        You are a report generation assistant tasked with producing a well-formatted report given parsed context.
-        Given a comprehensive analysis of the user request, your task is to synthesize the information and return a well-formatted report.
-
-        ## Instructions
-        You are responsible for representing the analysis in a well-formatted report. If tables or visualizations provided, add them to the right sections that are most relevant.
-        Use only the provided information to create the report. Do not make up any information yourself.
-        Finally, the report should be presented in markdown format.
-        """
-    )
-    configured_tools = ToolFactory.from_env(map_result=True)
-    if "document_generator" in configured_tools:  # type: ignore
-        tools.extend(configured_tools["document_generator"])  # type: ignore
-        prompt_instructions += (
-            "\nYou are also able to generate a file document (PDF/HTML) of the report."
-        )
-        description += " and generate a file document (PDF/HTML) of the report."
-    return tools, description, prompt_instructions
-
-
-def create_reporter(chat_history: List[ChatMessage]):
-    tools, description, prompt_instructions = _get_reporter_params(chat_history)
-    return FunctionCallingAgent(
-        name="reporter",
-        tools=tools,
-        description=description,
-        system_prompt=prompt_instructions,
-        chat_history=chat_history,
-    )

templates/components/agents/python/financial_report/app/agents/researcher.py

@@ -1,105 +0,0 @@
-import os
-from textwrap import dedent
-from typing import List, Optional
-
-from app.engine.index import IndexConfig, get_index
-from app.workflows.single import FunctionCallingAgent
-from llama_index.core.chat_engine.types import ChatMessage
-from llama_index.core.tools import BaseTool, QueryEngineTool, ToolMetadata
-from llama_index.indices.managed.llama_cloud import LlamaCloudIndex
-
-
-def _create_query_engine_tools(params=None) -> Optional[list[type[BaseTool]]]:
-    """
-    Provide an agent worker that can be used to query the index.
-    """
-    # Add query tool if index exists
-    index_config = IndexConfig(**(params or {}))
-    index = get_index(index_config)
-    if index is None:
-        return None
-
-    top_k = int(os.getenv("TOP_K", 5))
-
-    # Construct query engine tools
-    tools = []
-    # If index is LlamaCloudIndex, we need to add chunk and doc retriever tools
-    if isinstance(index, LlamaCloudIndex):
-        # Document retriever
-        doc_retriever = index.as_query_engine(
-            retriever_mode="files_via_content",
-            similarity_top_k=top_k,
-        )
-        chunk_retriever = index.as_query_engine(
-            retriever_mode="chunks",
-            similarity_top_k=top_k,
-        )
-        tools.append(
-            QueryEngineTool(
-                query_engine=doc_retriever,
-                metadata=ToolMetadata(
-                    name="document_retriever",
-                    description=dedent(
-                        """
-                        Document retriever that retrieves entire documents from the corpus.
-                        ONLY use for research questions that may require searching over entire research reports.
-                        Will be slower and more expensive than chunk-level retrieval but may be necessary.
-                        """
-                    ),
-                ),
-            )
-        )
-        tools.append(
-            QueryEngineTool(
-                query_engine=chunk_retriever,
-                metadata=ToolMetadata(
-                    name="chunk_retriever",
-                    description=dedent(
-                        """
-                        Retrieves a small set of relevant document chunks from the corpus.
-                        Use for research questions that want to look up specific facts from the knowledge corpus,
-                        and need entire documents.
-                        """
-                    ),
-                ),
-            )
-        )
-    else:
-        query_engine = index.as_query_engine(
-            **({"similarity_top_k": top_k} if top_k != 0 else {})
-        )
-        tools.append(
-            QueryEngineTool(
-                query_engine=query_engine,
-                metadata=ToolMetadata(
-                    name="retrieve_information",
-                    description="Use this tool to retrieve information about the text corpus from the index.",
-                ),
-            )
-        )
-    return tools
-
-
-def create_researcher(chat_history: List[ChatMessage], **kwargs):
-    """
-    Researcher is an agent that take responsibility for using tools to complete a given task.
-    """
-    tools = _create_query_engine_tools(**kwargs)
-
-    if tools is None:
-        raise ValueError("No tools found for researcher agent")
-
-    return FunctionCallingAgent(
-        name="researcher",
-        tools=tools,
-        description="expert in retrieving any unknown content from the corpus",
-        system_prompt=dedent(
-            """
-            You are a researcher agent. You are responsible for retrieving information from the corpus.
-            ## Instructions
-            + Don't synthesize the information, just return the whole retrieved information.
-            + Don't need to retrieve the information that is already provided in the chat history and response with: "There is no new information, please reuse the information from the conversation."
-            """
-        ),
-        chat_history=chat_history,
-    )

templates/components/agents/python/financial_report/app/agents/workflow.py

@@ -1,177 +0,0 @@
-from textwrap import dedent
-from typing import AsyncGenerator, List, Optional
-
-from app.agents.analyst import create_analyst
-from app.agents.reporter import create_reporter
-from app.agents.researcher import create_researcher
-from app.workflows.single import AgentRunEvent, AgentRunResult, FunctionCallingAgent
-from llama_index.core.chat_engine.types import ChatMessage
-from llama_index.core.prompts import PromptTemplate
-from llama_index.core.settings import Settings
-from llama_index.core.workflow import (
-    Context,
-    Event,
-    StartEvent,
-    StopEvent,
-    Workflow,
-    step,
-)
-
-
-def create_workflow(chat_history: Optional[List[ChatMessage]] = None, **kwargs):
-    researcher = create_researcher(
-        chat_history=chat_history,
-        **kwargs,
-    )
-
-    analyst = create_analyst(chat_history=chat_history)
-
-    reporter = create_reporter(chat_history=chat_history)
-
-    workflow = FinancialReportWorkflow(timeout=360, chat_history=chat_history)
-
-    workflow.add_workflows(
-        researcher=researcher,
-        analyst=analyst,
-        reporter=reporter,
-    )
-    return workflow
-
-
-class ResearchEvent(Event):
-    input: str
-
-
-class AnalyzeEvent(Event):
-    input: str
-
-
-class ReportEvent(Event):
-    input: str
-
-
-class FinancialReportWorkflow(Workflow):
-    def __init__(
-        self, timeout: int = 360, chat_history: Optional[List[ChatMessage]] = None
-    ):
-        super().__init__(timeout=timeout)
-        self.chat_history = chat_history or []
-
-    @step()
-    async def start(self, ctx: Context, ev: StartEvent) -> ResearchEvent | ReportEvent:
-        # set streaming
-        ctx.data["streaming"] = getattr(ev, "streaming", False)
-        # start the workflow with researching about a topic
-        ctx.data["task"] = ev.input
-        ctx.data["user_input"] = ev.input
-
-        # Decision-making process
-        decision = await self._decide_workflow(ev.input, self.chat_history)
-
-        if decision != "publish":
-            return ResearchEvent(input=f"Research for this task: {ev.input}")
-        else:
-            chat_history_str = "\n".join(
-                [f"{msg.role}: {msg.content}" for msg in self.chat_history]
-            )
-            return ReportEvent(
-                input=f"Create a report based on the chat history\n{chat_history_str}\n\n and task: {ev.input}"
-            )
-
-    async def _decide_workflow(
-        self, input: str, chat_history: List[ChatMessage]
-    ) -> str:
-        # TODO: Refactor this by using prompt generation
-        prompt_template = PromptTemplate(
-            dedent(
-                """
-                You are an expert in decision-making, helping people create financial reports for the provided data.
-                If the user doesn't need to add or update anything, respond with 'publish'.
-                Otherwise, respond with 'research'.
-
-                Here is the chat history:
-                {chat_history}
-
-                The current user request is:
-                {input}
-
-                Given the chat history and the new user request, decide whether to create a report based on existing information.
-                Decision (respond with either 'not_publish' or 'publish'):
-            """
-            )
-        )
-
-        chat_history_str = "\n".join(
-            [f"{msg.role}: {msg.content}" for msg in chat_history]
-        )
-        prompt = prompt_template.format(chat_history=chat_history_str, input=input)
-
-        output = await Settings.llm.acomplete(prompt)
-        decision = output.text.strip().lower()
-
-        return "publish" if decision == "publish" else "research"
-
-    @step()
-    async def research(
-        self, ctx: Context, ev: ResearchEvent, researcher: FunctionCallingAgent
-    ) -> AnalyzeEvent:
-        result: AgentRunResult = await self.run_agent(ctx, researcher, ev.input)
-        content = result.response.message.content
-        return AnalyzeEvent(
-            input=dedent(
-                f"""
-                Given the following research content:
-                {content}
-                Provide a comprehensive analysis of the data for the user's request: {ctx.data["task"]}
-                """
-            )
-        )
-
-    @step()
-    async def analyze(
-        self, ctx: Context, ev: AnalyzeEvent, analyst: FunctionCallingAgent
-    ) -> ReportEvent | StopEvent:
-        result: AgentRunResult = await self.run_agent(ctx, analyst, ev.input)
-        content = result.response.message.content
-        return ReportEvent(
-            input=dedent(
-                f"""
-                Given the following analysis:
-                {content}
-                Create a report for the user's request: {ctx.data["task"]}
-                """
-            )
-        )
-
-    @step()
-    async def report(
-        self, ctx: Context, ev: ReportEvent, reporter: FunctionCallingAgent
-    ) -> StopEvent:
-        try:
-            result: AgentRunResult = await self.run_agent(
-                ctx, reporter, ev.input, streaming=ctx.data["streaming"]
-            )
-            return StopEvent(result=result)
-        except Exception as e:
-            ctx.write_event_to_stream(
-                AgentRunEvent(
-                    name=reporter.name,
-                    msg=f"Error creating a report: {e}",
-                )
-            )
-            return StopEvent(result=None)
-
-    async def run_agent(
-        self,
-        ctx: Context,
-        agent: FunctionCallingAgent,
-        input: str,
-        streaming: bool = False,
-    ) -> AgentRunResult | AsyncGenerator:
-        handler = agent.run(input=input, streaming=streaming)
-        # bubble all events while running the executor to the planner
-        async for event in handler.stream_events():
-            # Don't write the StopEvent from sub task to the stream
-            if type(event) is not StopEvent:
-                ctx.write_event_to_stream(event)
-        return await handler

templates/components/agents/python/financial_report/app/engine/engine.py

@@ -1,12 +0,0 @@
-from typing import List, Optional
-
-from app.agents.workflow import create_workflow
-from llama_index.core.chat_engine.types import ChatMessage
-from llama_index.core.workflow import Workflow
-
-
-def get_chat_engine(
-    chat_history: Optional[List[ChatMessage]] = None, **kwargs
-) -> Workflow:
-    agent_workflow = create_workflow(chat_history, **kwargs)
-    return agent_workflow

templates/components/agents/python/financial_report/app/workflows/__init__.py

@@ -0,0 +1,3 @@
+from .financial_report import create_workflow
+
+__all__ = ["create_workflow"]

templates/components/agents/python/financial_report/app/workflows/financial_report.py

@@ -0,0 +1,298 @@
+import os
+from typing import Any, Dict, List, Optional
+
+from app.engine.index import IndexConfig, get_index
+from app.engine.tools import ToolFactory
+from app.workflows.events import AgentRunEvent
+from app.workflows.tools import (
+    call_tools,
+    chat_with_tools,
+)
+from llama_index.core import Settings
+from llama_index.core.base.llms.types import ChatMessage, MessageRole
+from llama_index.core.indices.vector_store import VectorStoreIndex
+from llama_index.core.llms.function_calling import FunctionCallingLLM
+from llama_index.core.memory import ChatMemoryBuffer
+from llama_index.core.tools import FunctionTool, QueryEngineTool, ToolSelection
+from llama_index.core.workflow import (
+    Context,
+    Event,
+    StartEvent,
+    StopEvent,
+    Workflow,
+    step,
+)
+
+
+def create_workflow(
+    chat_history: Optional[List[ChatMessage]] = None,
+    params: Optional[Dict[str, Any]] = None,
+    filters: Optional[List[Any]] = None,
+) -> Workflow:
+    index_config = IndexConfig(**params)
+    index: VectorStoreIndex = get_index(config=index_config)
+    if index is None:
+        query_engine_tool = None
+    else:
+        top_k = int(os.getenv("TOP_K", 10))
+        query_engine = index.as_query_engine(similarity_top_k=top_k, filters=filters)
+        query_engine_tool = QueryEngineTool.from_defaults(query_engine=query_engine)
+
+    configured_tools: Dict[str, FunctionTool] = ToolFactory.from_env(map_result=True)  # type: ignore
+    code_interpreter_tool = configured_tools.get("interpret")
+    document_generator_tool = configured_tools.get("generate_document")
+
+    return FinancialReportWorkflow(
+        query_engine_tool=query_engine_tool,
+        code_interpreter_tool=code_interpreter_tool,
+        document_generator_tool=document_generator_tool,
+        chat_history=chat_history,
+    )
+
+
+class InputEvent(Event):
+    input: List[ChatMessage]
+    response: bool = False
+
+
+class ResearchEvent(Event):
+    input: list[ToolSelection]
+
+
+class AnalyzeEvent(Event):
+    input: list[ToolSelection] | ChatMessage
+
+
+class ReportEvent(Event):
+    input: list[ToolSelection]
+
+
+class FinancialReportWorkflow(Workflow):
+    """
+    A workflow to generate a financial report using indexed documents.
+
+    Requirements:
+    - Indexed documents containing financial data and a query engine tool to search them
+    - A code interpreter tool to analyze data and generate reports
+    - A document generator tool to create report files
+
+    Steps:
+    1. LLM Input: The LLM determines the next step based on function calling.
+       For example, if the model requests the query engine tool, it returns a ResearchEvent;
+       if it requests document generation, it returns a ReportEvent.
+    2. Research: Uses the query engine to find relevant chunks from indexed documents.
+       After gathering information, it requests analysis (step 3).
+    3. Analyze: Uses a custom prompt to analyze research results and can call the code
+       interpreter tool for visualization or calculation. Returns results to the LLM.
+    4. Report: Uses the document generator tool to create a report. Returns results to the LLM.
+    """
+
+    _default_system_prompt = """
+    You are a financial analyst who are given a set of tools to help you.
+    It's good to using appropriate tools for the user request and always use the information from the tools, don't make up anything yourself.
+    For the query engine tool, you should break down the user request into a list of queries and call the tool with the queries.
+    """
+
+    def __init__(
+        self,
+        query_engine_tool: QueryEngineTool,
+        code_interpreter_tool: FunctionTool,
+        document_generator_tool: FunctionTool,
+        llm: Optional[FunctionCallingLLM] = None,
+        timeout: int = 360,
+        chat_history: Optional[List[ChatMessage]] = None,
+        system_prompt: Optional[str] = None,
+    ):
+        super().__init__(timeout=timeout)
+        self.system_prompt = system_prompt or self._default_system_prompt
+        self.chat_history = chat_history or []
+        self.query_engine_tool = query_engine_tool
+        self.code_interpreter_tool = code_interpreter_tool
+        self.document_generator_tool = document_generator_tool
+        assert (
+            query_engine_tool is not None
+        ), "Query engine tool is not found. Try run generation script or upload a document file first."
+        assert code_interpreter_tool is not None, "Code interpreter tool is required"
+        assert (
+            document_generator_tool is not None
+        ), "Document generator tool is required"
+        self.tools = [
+            self.query_engine_tool,
+            self.code_interpreter_tool,
+            self.document_generator_tool,
+        ]
+        self.llm: FunctionCallingLLM = llm or Settings.llm
+        assert isinstance(self.llm, FunctionCallingLLM)
+        self.memory = ChatMemoryBuffer.from_defaults(
+            llm=self.llm, chat_history=self.chat_history
+        )
+
+    @step()
+    async def prepare_chat_history(self, ctx: Context, ev: StartEvent) -> InputEvent:
+        ctx.data["input"] = ev.input
+
+        if self.system_prompt:
+            system_msg = ChatMessage(
+                role=MessageRole.SYSTEM, content=self.system_prompt
+            )
+            self.memory.put(system_msg)
+
+        # Add user input to memory
+        self.memory.put(ChatMessage(role=MessageRole.USER, content=ev.input))
+
+        return InputEvent(input=self.memory.get())
+
+    @step()
+    async def handle_llm_input(  # type: ignore
+        self,
+        ctx: Context,
+        ev: InputEvent,
+    ) -> ResearchEvent | AnalyzeEvent | ReportEvent | StopEvent:
+        """
+        Handle an LLM input and decide the next step.
+        """
+        # Always use the latest chat history from the input
+        chat_history: list[ChatMessage] = ev.input
+
+        # Get tool calls
+        response = await chat_with_tools(
+            self.llm,
+            self.tools,  # type: ignore
+            chat_history,
+        )
+        if not response.has_tool_calls():
+            # If no tool call, return the response generator
+            return StopEvent(result=response.generator)
+        # calling different tools at the same time is not supported at the moment
+        # add an error message to tell the AI to process step by step
+        if response.is_calling_different_tools():
+            self.memory.put(
+                ChatMessage(
+                    role=MessageRole.ASSISTANT,
+                    content="Cannot call different tools at the same time. Try calling one tool at a time.",
+                )
+            )
+            return InputEvent(input=self.memory.get())
+        self.memory.put(response.tool_call_message)
+        match response.tool_name():
+            case self.code_interpreter_tool.metadata.name:
+                return AnalyzeEvent(input=response.tool_calls)
+            case self.document_generator_tool.metadata.name:
+                return ReportEvent(input=response.tool_calls)
+            case self.query_engine_tool.metadata.name:
+                return ResearchEvent(input=response.tool_calls)
+            case _:
+                raise ValueError(f"Unknown tool: {response.tool_name()}")
+
+    @step()
+    async def research(self, ctx: Context, ev: ResearchEvent) -> AnalyzeEvent:
+        """
+        Do a research to gather information for the user's request.
+        A researcher should have these tools: query engine, search engine, etc.
+        """
+        ctx.write_event_to_stream(
+            AgentRunEvent(
+                name="Researcher",
+                msg="Starting research",
+            )
+        )
+        tool_calls = ev.input
+
+        tool_messages = await call_tools(
+            ctx=ctx,
+            agent_name="Researcher",
+            tools=[self.query_engine_tool],
+            tool_calls=tool_calls,
+        )
+        self.memory.put_messages(tool_messages)
+        return AnalyzeEvent(
+            input=ChatMessage(
+                role=MessageRole.ASSISTANT,
+                content="I've finished the research. Please analyze the result.",
+            ),
+        )
+
+    @step()
+    async def analyze(self, ctx: Context, ev: AnalyzeEvent) -> InputEvent:
+        """
+        Analyze the research result.
+        """
+        ctx.write_event_to_stream(
+            AgentRunEvent(
+                name="Analyst",
+                msg="Starting analysis",
+            )
+        )
+        event_requested_by_workflow_llm = isinstance(ev.input, list)
+        # Requested by the workflow LLM Input step, it's a tool call
+        if event_requested_by_workflow_llm:
+            # Set the tool calls
+            tool_calls = ev.input
+        else:
+            # Otherwise, it's triggered by the research step
+            # Use a custom prompt and independent memory for the analyst agent
+            analysis_prompt = """
+            You are a financial analyst, you are given a research result and a set of tools to help you.
+            Always use the given information, don't make up anything yourself. If there is not enough information, you can asking for more information.
+            If you have enough numerical information, it's good to include some charts/visualizations to the report so you can use the code interpreter tool to generate a report.
+            """
+            # This is handled by analyst agent
+            # Clone the shared memory to avoid conflicting with the workflow.
+            chat_history = self.memory.get()
+            chat_history.append(
+                ChatMessage(role=MessageRole.SYSTEM, content=analysis_prompt)
+            )
+            chat_history.append(ev.input)  # type: ignore
+            # Check if the analyst agent needs to call tools
+            response = await chat_with_tools(
+                self.llm,
+                [self.code_interpreter_tool],
+                chat_history,
+            )
+            if not response.has_tool_calls():
+                # If no tool call, fallback analyst message to the workflow
+                analyst_msg = ChatMessage(
+                    role=MessageRole.ASSISTANT,
+                    content=await response.full_response(),
+                )
+                self.memory.put(analyst_msg)
+                return InputEvent(input=self.memory.get())
+            else:
+                # Set the tool calls and the tool call message to the memory
+                tool_calls = response.tool_calls
+                self.memory.put(response.tool_call_message)
+
+        # Call tools
+        tool_messages = await call_tools(
+            ctx=ctx,
+            agent_name="Analyst",
+            tools=[self.code_interpreter_tool],
+            tool_calls=tool_calls,  # type: ignore
+        )
+        self.memory.put_messages(tool_messages)
+
+        # Fallback to the input with the latest chat history
+        return InputEvent(input=self.memory.get())
+
+    @step()
+    async def report(self, ctx: Context, ev: ReportEvent) -> InputEvent:
+        """
+        Generate a report based on the analysis result.
+        """
+        ctx.write_event_to_stream(
+            AgentRunEvent(
+                name="Reporter",
+                msg="Starting report generation",
+            )
+        )
+        tool_calls = ev.input
+        tool_messages = await call_tools(
+            ctx=ctx,
+            agent_name="Reporter",
+            tools=[self.document_generator_tool],
+            tool_calls=tool_calls,
+        )
+        self.memory.put_messages(tool_messages)
+
+        # After the tool calls, fallback to the input with the latest chat history
+        return InputEvent(input=self.memory.get())

templates/components/agents/python/form_filling/README-template.md

@@ -0,0 +1,59 @@
+This is a [LlamaIndex](https://www.llamaindex.ai/) multi-agents project using [Workflows](https://docs.llamaindex.ai/en/stable/understanding/workflows/).
+
+## Getting Started
+
+First, setup the environment with poetry:
+
+> **_Note:_** This step is not needed if you are using the dev-container.
+
+```shell
+poetry install
+```
+
+Then check the parameters that have been pre-configured in the `.env` file in this directory.
+Make sure you have the `OPENAI_API_KEY` set.
+
+Second, run the development server:
+
+```shell
+poetry run python main.py
+```
+
+## Use Case: Filling Financial CSV Template
+
+To reproduce the use case, start the [frontend](../frontend/README.md) and follow these steps in the frontend:
+
+1. Upload the Apple and Tesla financial reports from the [data](./data) directory. Just send an empty message.
+2. Upload the CSV file [sec_10k_template.csv](./sec_10k_template.csv) and send the message "Fill the missing cells in the CSV file".
+
+The agent will fill the missing cells by retrieving the information from the uploaded financial reports and return a new CSV file with the filled cells.
+
+### API endpoints
+
+The example provides one streaming API endpoint `/api/chat`.
+You can test the endpoint with the following curl request:
+
+```
+curl --location 'localhost:8000/api/chat' \
+--header 'Content-Type: application/json' \
+--data '{ "messages": [{ "role": "user", "content": "What can you do?" }] }'
+```
+
+You can start editing the API by modifying `app/api/routers/chat.py` or `app/workflows/form_filling.py`. The API auto-updates as you save the files.
+
+Open [http://localhost:8000/docs](http://localhost:8000/docs) with your browser to see the Swagger UI of the API.
+
+The API allows CORS for all origins to simplify development. You can change this behavior by setting the `ENVIRONMENT` environment variable to `prod`:
+
+```
+ENVIRONMENT=prod poetry run python main.py
+```
+
+## Learn More
+
+To learn more about LlamaIndex, take a look at the following resources:
+
+- [LlamaIndex Documentation](https://docs.llamaindex.ai) - learn about LlamaIndex.
+- [Workflows Introduction](https://docs.llamaindex.ai/en/stable/understanding/workflows/) - learn about LlamaIndex workflows.
+
+You can check out [the LlamaIndex GitHub repository](https://github.com/run-llama/llama_index) - your feedback and contributions are welcome!

templates/components/agents/python/form_filling/app/workflows/__init__.py

@@ -0,0 +1,3 @@
+from .form_filling import create_workflow
+
+__all__ = ["create_workflow"]

templates/components/agents/python/form_filling/app/workflows/form_filling.py

@@ -0,0 +1,241 @@
+import os
+from typing import Any, Dict, List, Optional
+
+from app.engine.index import IndexConfig, get_index
+from app.engine.tools import ToolFactory
+from app.workflows.events import AgentRunEvent
+from app.workflows.tools import (
+    call_tools,
+    chat_with_tools,
+)
+from llama_index.core import Settings
+from llama_index.core.base.llms.types import ChatMessage, MessageRole
+from llama_index.core.indices.vector_store import VectorStoreIndex
+from llama_index.core.llms.function_calling import FunctionCallingLLM
+from llama_index.core.memory import ChatMemoryBuffer
+from llama_index.core.tools import FunctionTool, QueryEngineTool, ToolSelection
+from llama_index.core.workflow import (
+    Context,
+    Event,
+    StartEvent,
+    StopEvent,
+    Workflow,
+    step,
+)
+
+
+def create_workflow(
+    chat_history: Optional[List[ChatMessage]] = None,
+    params: Optional[Dict[str, Any]] = None,
+    filters: Optional[List[Any]] = None,
+) -> Workflow:
+    if params is None:
+        params = {}
+    if filters is None:
+        filters = []
+    index_config = IndexConfig(**params)
+    index: VectorStoreIndex = get_index(config=index_config)
+    if index is None:
+        query_engine_tool = None
+    else:
+        top_k = int(os.getenv("TOP_K", 10))
+        query_engine = index.as_query_engine(similarity_top_k=top_k, filters=filters)
+        query_engine_tool = QueryEngineTool.from_defaults(query_engine=query_engine)
+
+    configured_tools = ToolFactory.from_env(map_result=True)
+    extractor_tool = configured_tools.get("extract_questions")  # type: ignore
+    filling_tool = configured_tools.get("fill_form")  # type: ignore
+
+    workflow = FormFillingWorkflow(
+        query_engine_tool=query_engine_tool,
+        extractor_tool=extractor_tool,  # type: ignore
+        filling_tool=filling_tool,  # type: ignore
+        chat_history=chat_history,
+    )
+
+    return workflow
+
+
+class InputEvent(Event):
+    input: List[ChatMessage]
+    response: bool = False
+
+
+class ExtractMissingCellsEvent(Event):
+    tool_calls: list[ToolSelection]
+
+
+class FindAnswersEvent(Event):
+    tool_calls: list[ToolSelection]
+
+
+class FillEvent(Event):
+    tool_calls: list[ToolSelection]
+
+
+class FormFillingWorkflow(Workflow):
+    """
+    A predefined workflow for filling missing cells in a CSV file.
+    Required tools:
+    - query_engine: A query engine to query for the answers to the questions.
+    - extract_question: Extract missing cells in a CSV file and generate questions to fill them.
+    - answer_question: Query for the answers to the questions.
+
+    Flow:
+    1. Extract missing cells in a CSV file and generate questions to fill them.
+    2. Query for the answers to the questions.
+    3. Fill the missing cells with the answers.
+    """
+
+    _default_system_prompt = """
+    You are a helpful assistant who helps fill missing cells in a CSV file.
+    Only extract missing cells from CSV files.
+    Only use provided data - never make up any information yourself. Fill N/A if an answer is not found.
+    If there is no query engine tool or the gathered information has many N/A values indicating the questions don't match the data, respond with a warning and ask the user to upload a different file or connect to a knowledge base.
+    """
+
+    def __init__(
+        self,
+        query_engine_tool: Optional[QueryEngineTool],
+        extractor_tool: FunctionTool,
+        filling_tool: FunctionTool,
+        llm: Optional[FunctionCallingLLM] = None,
+        timeout: int = 360,
+        chat_history: Optional[List[ChatMessage]] = None,
+        system_prompt: Optional[str] = None,
+    ):
+        super().__init__(timeout=timeout)
+        self.system_prompt = system_prompt or self._default_system_prompt
+        self.chat_history = chat_history or []
+        self.query_engine_tool = query_engine_tool
+        self.extractor_tool = extractor_tool
+        self.filling_tool = filling_tool
+        if self.extractor_tool is None or self.filling_tool is None:
+            raise ValueError("Extractor and filling tools are required.")
+        self.tools = [self.extractor_tool, self.filling_tool]
+        if self.query_engine_tool is not None:
+            self.tools.append(self.query_engine_tool)  # type: ignore
+        self.llm: FunctionCallingLLM = llm or Settings.llm
+        if not isinstance(self.llm, FunctionCallingLLM):
+            raise ValueError("FormFillingWorkflow only supports FunctionCallingLLM.")
+        self.memory = ChatMemoryBuffer.from_defaults(
+            llm=self.llm, chat_history=self.chat_history
+        )
+
+    @step()
+    async def start(self, ctx: Context, ev: StartEvent) -> InputEvent:
+        ctx.data["input"] = ev.input
+
+        if self.system_prompt:
+            system_msg = ChatMessage(
+                role=MessageRole.SYSTEM, content=self.system_prompt
+            )
+            self.memory.put(system_msg)
+
+        user_input = ev.input
+        user_msg = ChatMessage(role=MessageRole.USER, content=user_input)
+        self.memory.put(user_msg)
+
+        chat_history = self.memory.get()
+        return InputEvent(input=chat_history)
+
+    @step()
+    async def handle_llm_input(  # type: ignore
+        self,
+        ctx: Context,
+        ev: InputEvent,
+    ) -> ExtractMissingCellsEvent | FillEvent | StopEvent:
+        """
+        Handle an LLM input and decide the next step.
+        """
+        chat_history: list[ChatMessage] = ev.input
+        response = await chat_with_tools(
+            self.llm,
+            self.tools,
+            chat_history,
+        )
+        if not response.has_tool_calls():
+            return StopEvent(result=response.generator)
+        # calling different tools at the same time is not supported at the moment
+        # add an error message to tell the AI to process step by step
+        if response.is_calling_different_tools():
+            self.memory.put(
+                ChatMessage(
+                    role=MessageRole.ASSISTANT,
+                    content="Cannot call different tools at the same time. Try calling one tool at a time.",
+                )
+            )
+            return InputEvent(input=self.memory.get())
+        self.memory.put(response.tool_call_message)
+        match response.tool_name():
+            case self.extractor_tool.metadata.name:
+                return ExtractMissingCellsEvent(tool_calls=response.tool_calls)
+            case self.query_engine_tool.metadata.name:
+                return FindAnswersEvent(tool_calls=response.tool_calls)
+            case self.filling_tool.metadata.name:
+                return FillEvent(tool_calls=response.tool_calls)
+            case _:
+                raise ValueError(f"Unknown tool: {response.tool_name()}")
+
+    @step()
+    async def extract_missing_cells(
+        self, ctx: Context, ev: ExtractMissingCellsEvent
+    ) -> InputEvent | FindAnswersEvent:
+        """
+        Extract missing cells in a CSV file and generate questions to fill them.
+        """
+        ctx.write_event_to_stream(
+            AgentRunEvent(
+                name="Extractor",
+                msg="Extracting missing cells",
+            )
+        )
+        # Call the extract questions tool
+        tool_messages = await call_tools(
+            agent_name="Extractor",
+            tools=[self.extractor_tool],
+            ctx=ctx,
+            tool_calls=ev.tool_calls,
+        )
+        self.memory.put_messages(tool_messages)
+        return InputEvent(input=self.memory.get())
+
+    @step()
+    async def find_answers(self, ctx: Context, ev: FindAnswersEvent) -> InputEvent:
+        """
+        Call answer questions tool to query for the answers to the questions.
+        """
+        ctx.write_event_to_stream(
+            AgentRunEvent(
+                name="Researcher",
+                msg="Finding answers for missing cells",
+            )
+        )
+        tool_messages = await call_tools(
+            ctx=ctx,
+            agent_name="Researcher",
+            tools=[self.query_engine_tool],
+            tool_calls=ev.tool_calls,
+        )
+        self.memory.put_messages(tool_messages)
+        return InputEvent(input=self.memory.get())
+
+    @step()
+    async def fill_cells(self, ctx: Context, ev: FillEvent) -> InputEvent:
+        """
+        Call fill cells tool to fill the missing cells with the answers.
+        """
+        ctx.write_event_to_stream(
+            AgentRunEvent(
+                name="Processor",
+                msg="Filling missing cells",
+            )
+        )
+        tool_messages = await call_tools(
+            agent_name="Processor",
+            tools=[self.filling_tool],
+            ctx=ctx,
+            tool_calls=ev.tool_calls,
+        )
+        self.memory.put_messages(tool_messages)
+        return InputEvent(input=self.memory.get())

templates/components/agents/python/form_filling/sec_10k_template.csv

@@ -0,0 +1,17 @@
+Parameter,2023 Apple (AAPL),2023 Tesla (TSLA)
+Revenue,,
+Net Income,,
+Earnings Per Share (EPS),,
+Debt-to-Equity Ratio,,
+Current Ratio,,
+Gross Margin,,
+Operating Margin,,
+Net Profit Margin,,
+Inventory Turnover,,
+Accounts Receivable Turnover,,
+Capital Expenditure,,
+Research and Development Expense,,
+Market Cap,,
+Price to Earnings Ratio,,
+Dividend Yield,,
+Year-over-Year Growth Rate,,

templates/components/agents/typescript/blog/factory.ts

@@ -1,230 +0,0 @@
-import {
-  Context,
-  StartEvent,
-  StopEvent,
-  Workflow,
-  WorkflowEvent,
-} from "@llamaindex/core/workflow";
-import { Message } from "ai";
-import { ChatMessage, ChatResponseChunk, Settings } from "llamaindex";
-import { getAnnotations } from "../llamaindex/streaming/annotations";
-import {
-  createPublisher,
-  createResearcher,
-  createReviewer,
-  createWriter,
-} from "./agents";
-import { AgentInput, AgentRunEvent } from "./type";
-
-const TIMEOUT = 360 * 1000;
-const MAX_ATTEMPTS = 2;
-
-class ResearchEvent extends WorkflowEvent<{ input: string }> {}
-class WriteEvent extends WorkflowEvent<{
-  input: string;
-  isGood: boolean;
-}> {}
-class ReviewEvent extends WorkflowEvent<{ input: string }> {}
-class PublishEvent extends WorkflowEvent<{ input: string }> {}
-
-const prepareChatHistory = (chatHistory: Message[]): ChatMessage[] => {
-  // By default, the chat history only contains the assistant and user messages
-  // all the agents messages are stored in annotation data which is not visible to the LLM
-
-  const MAX_AGENT_MESSAGES = 10;
-  const agentAnnotations = getAnnotations<{ agent: string; text: string }>(
-    chatHistory,
-    { role: "assistant", type: "agent" },
-  ).slice(-MAX_AGENT_MESSAGES);
-
-  const agentMessages = agentAnnotations
-    .map(
-      (annotation) =>
-        `\n<${annotation.data.agent}>\n${annotation.data.text}\n</${annotation.data.agent}>`,
-    )
-    .join("\n");
-
-  const agentContent = agentMessages
-    ? "Here is the previous conversation of agents:\n" + agentMessages
-    : "";
-
-  if (agentContent) {
-    const agentMessage: ChatMessage = {
-      role: "assistant",
-      content: agentContent,
-    };
-    return [
-      ...chatHistory.slice(0, -1),
-      agentMessage,
-      chatHistory.slice(-1)[0],
-    ] as ChatMessage[];
-  }
-  return chatHistory as ChatMessage[];
-};
-
-export const createWorkflow = (messages: Message[], params?: any) => {
-  const chatHistoryWithAgentMessages = prepareChatHistory(messages);
-  const runAgent = async (
-    context: Context,
-    agent: Workflow,
-    input: AgentInput,
-  ) => {
-    const run = agent.run(new StartEvent({ input }));
-    for await (const event of agent.streamEvents()) {
-      if (event.data instanceof AgentRunEvent) {
-        context.writeEventToStream(event.data);
-      }
-    }
-    return await run;
-  };
-
-  const start = async (context: Context, ev: StartEvent) => {
-    context.set("task", ev.data.input);
-
-    const chatHistoryStr = chatHistoryWithAgentMessages
-      .map((msg) => `${msg.role}: ${msg.content}`)
-      .join("\n");
-
-    // Decision-making process
-    const decision = await decideWorkflow(ev.data.input, chatHistoryStr);
-
-    if (decision !== "publish") {
-      return new ResearchEvent({
-        input: `Research for this task: ${ev.data.input}`,
-      });
-    } else {
-      return new PublishEvent({
-        input: `Publish content based on the chat history\n${chatHistoryStr}\n\n and task: ${ev.data.input}`,
-      });
-    }
-  };
-
-  const decideWorkflow = async (task: string, chatHistoryStr: string) => {
-    const llm = Settings.llm;
-
-    const prompt = `You are an expert in decision-making, helping people write and publish blog posts.
-If the user is asking for a file or to publish content, respond with 'publish'.
-If the user requests to write or update a blog post, respond with 'not_publish'.
-
-Here is the chat history:
-${chatHistoryStr}
-
-The current user request is:
-${task}
-
-Given the chat history and the new user request, decide whether to publish based on existing information.
-Decision (respond with either 'not_publish' or 'publish'):`;
-
-    const output = await llm.complete({ prompt: prompt });
-    const decision = output.text.trim().toLowerCase();
-    return decision === "publish" ? "publish" : "research";
-  };
-
-  const research = async (context: Context, ev: ResearchEvent) => {
-    const researcher = await createResearcher(
-      chatHistoryWithAgentMessages,
-      params,
-    );
-    const researchRes = await runAgent(context, researcher, {
-      message: ev.data.input,
-    });
-    const researchResult = researchRes.data.result;
-    return new WriteEvent({
-      input: `Write a blog post given this task: ${context.get("task")} using this research content: ${researchResult}`,
-      isGood: false,
-    });
-  };
-
-  const write = async (context: Context, ev: WriteEvent) => {
-    const writer = createWriter(chatHistoryWithAgentMessages);
-
-    context.set("attempts", context.get("attempts", 0) + 1);
-    const tooManyAttempts = context.get("attempts") > MAX_ATTEMPTS;
-    if (tooManyAttempts) {
-      context.writeEventToStream(
-        new AgentRunEvent({
-          name: "writer",
-          msg: `Too many attempts (${MAX_ATTEMPTS}) to write the blog post. Proceeding with the current version.`,
-        }),
-      );
-    }
-
-    if (ev.data.isGood || tooManyAttempts) {
-      // the blog post is good or too many attempts
-      // stream the final content
-      const result = await runAgent(context, writer, {
-        message: `Based on the reviewer's feedback, refine the post and return only the final version of the post. Here's the current version: ${ev.data.input}`,
-        streaming: true,
-      });
-      return result as unknown as StopEvent<AsyncGenerator<ChatResponseChunk>>;
-    }
-
-    const writeRes = await runAgent(context, writer, {
-      message: ev.data.input,
-    });
-    const writeResult = writeRes.data.result;
-    context.set("result", writeResult); // store the last result
-    return new ReviewEvent({ input: writeResult });
-  };
-
-  const review = async (context: Context, ev: ReviewEvent) => {
-    const reviewer = createReviewer(chatHistoryWithAgentMessages);
-    const reviewRes = await reviewer.run(
-      new StartEvent<AgentInput>({ input: { message: ev.data.input } }),
-    );
-    const reviewResult = reviewRes.data.result;
-    const oldContent = context.get("result");
-    const postIsGood = reviewResult.toLowerCase().includes("post is good");
-    context.writeEventToStream(
-      new AgentRunEvent({
-        name: "reviewer",
-        msg: `The post is ${postIsGood ? "" : "not "}good enough for publishing. Sending back to the writer${
-          postIsGood ? " for publication." : "."
-        }`,
-      }),
-    );
-    if (postIsGood) {
-      return new WriteEvent({
-        input: "",
-        isGood: true,
-      });
-    }
-
-    return new WriteEvent({
-      input: `Improve the writing of a given blog post by using a given review.
-            Blog post:
-            \`\`\`
-            ${oldContent}
-            \`\`\`
-
-            Review:
-            \`\`\`
-            ${reviewResult}
-            \`\`\``,
-      isGood: false,
-    });
-  };
-
-  const publish = async (context: Context, ev: PublishEvent) => {
-    const publisher = await createPublisher(chatHistoryWithAgentMessages);
-
-    const publishResult = await runAgent(context, publisher, {
-      message: `${ev.data.input}`,
-      streaming: true,
-    });
-    return publishResult as unknown as StopEvent<
-      AsyncGenerator<ChatResponseChunk>
-    >;
-  };
-
-  const workflow = new Workflow({ timeout: TIMEOUT, validate: true });
-  workflow.addStep(StartEvent, start, {
-    outputs: [ResearchEvent, PublishEvent],
-  });
-  workflow.addStep(ResearchEvent, research, { outputs: WriteEvent });
-  workflow.addStep(WriteEvent, write, { outputs: [ReviewEvent, StopEvent] });
-  workflow.addStep(ReviewEvent, review, { outputs: WriteEvent });
-  workflow.addStep(PublishEvent, publish, { outputs: StopEvent });
-
-  return workflow;
-};

templates/components/agents/typescript/blog/tools.ts

@@ -1,54 +0,0 @@
-import fs from "fs/promises";
-import { BaseToolWithCall, QueryEngineTool } from "llamaindex";
-import path from "path";
-import { getDataSource } from "../engine";
-import { createTools } from "../engine/tools/index";
-
-export const getQueryEngineTool = async (
-  params?: any,
-): Promise<QueryEngineTool | null> => {
-  const index = await getDataSource(params);
-  if (!index) {
-    return null;
-  }
-
-  const topK = process.env.TOP_K ? parseInt(process.env.TOP_K) : undefined;
-  return new QueryEngineTool({
-    queryEngine: index.asQueryEngine({
-      similarityTopK: topK,
-    }),
-    metadata: {
-      name: "query_index",
-      description: `Use this tool to retrieve information about the text corpus from the index.`,
-    },
-  });
-};
-
-export const getAvailableTools = async () => {
-  const configFile = path.join("config", "tools.json");
-  let toolConfig: any;
-  const tools: BaseToolWithCall[] = [];
-  try {
-    toolConfig = JSON.parse(await fs.readFile(configFile, "utf8"));
-  } catch (e) {
-    console.info(`Could not read ${configFile} file. Using no tools.`);
-  }
-  if (toolConfig) {
-    tools.push(...(await createTools(toolConfig)));
-  }
-  const queryEngineTool = await getQueryEngineTool();
-  if (queryEngineTool) {
-    tools.push(queryEngineTool);
-  }
-
-  return tools;
-};
-
-export const lookupTools = async (
-  toolNames: string[],
-): Promise<BaseToolWithCall[]> => {
-  const availableTools = await getAvailableTools();
-  return availableTools.filter((tool) =>
-    toolNames.includes(tool.metadata.name),
-  );
-};

templates/components/agents/typescript/blog/workflow/agents.ts

@@ -1,19 +1,16 @@
 import { ChatMessage } from "llamaindex";
+import { getTool } from "../engine/tools";
 import { FunctionCallingAgent } from "./single-agent";
-import { getQueryEngineTool, lookupTools } from "./tools";
+import { getQueryEngineTools } from "./tools";
 
-export const createResearcher = async (
-  chatHistory: ChatMessage[],
-  params?: any,
-) => {
-  const queryEngineTool = await getQueryEngineTool(params);
-  const tools = (
-    await lookupTools([
-      "wikipedia_tool",
-      "duckduckgo_search",
-      "image_generator",
-    ])
-  ).concat(queryEngineTool ? [queryEngineTool] : []);
+export const createResearcher = async (chatHistory: ChatMessage[]) => {
+  const queryEngineTools = await getQueryEngineTools();
+  const tools = [
+    await getTool("wikipedia_tool"),
+    await getTool("duckduckgo_search"),
+    await getTool("image_generator"),
+    ...(queryEngineTools ? queryEngineTools : []),
+  ].filter((tool) => tool !== undefined);
 
   return new FunctionCallingAgent({
     name: "researcher",
@@ -81,17 +78,17 @@ Example:
 };
 
 export const createPublisher = async (chatHistory: ChatMessage[]) => {
-  const tools = await lookupTools(["document_generator"]);
+  const tool = await getTool("document_generator");
   let systemPrompt = `You are an expert in publishing blog posts. You are given a task to publish a blog post. 
 If the writer says that there was an error, you should reply with the error and not publish the post.`;
-  if (tools.length > 0) {
+  if (tool) {
     systemPrompt = `${systemPrompt}. 
 If the user requests to generate a file, use the document_generator tool to generate the file and reply with the link to the file.
 Otherwise, simply return the content of the post.`;
   }
   return new FunctionCallingAgent({
     name: "publisher",
-    tools: tools,
+    tools: tool ? [tool] : [],
     systemPrompt: systemPrompt,
     chatHistory,
   });

templates/components/agents/typescript/blog/workflow/factory.ts

@@ -0,0 +1,291 @@
+import {
+  HandlerContext,
+  StartEvent,
+  StopEvent,
+  Workflow,
+  WorkflowContext,
+  WorkflowEvent,
+} from "@llamaindex/workflow";
+import { ChatMessage, ChatResponseChunk, Settings } from "llamaindex";
+import {
+  createPublisher,
+  createResearcher,
+  createReviewer,
+  createWriter,
+} from "./agents";
+import {
+  FunctionCallingAgent,
+  FunctionCallingAgentInput,
+} from "./single-agent";
+import { AgentInput, AgentRunEvent } from "./type";
+
+const TIMEOUT = 360 * 1000;
+const MAX_ATTEMPTS = 2;
+
+class ResearchEvent extends WorkflowEvent<{ input: string }> {}
+class WriteEvent extends WorkflowEvent<{
+  input: string;
+  isGood: boolean;
+}> {}
+class ReviewEvent extends WorkflowEvent<{ input: string }> {}
+class PublishEvent extends WorkflowEvent<{ input: string }> {}
+
+type BlogContext = {
+  task: string;
+  attempts: number;
+  result: string;
+};
+
+export const createWorkflow = ({
+  chatHistory,
+  params,
+}: {
+  chatHistory: ChatMessage[];
+  params?: any;
+}) => {
+  const runAgent = async (
+    context: HandlerContext<BlogContext>,
+    agent: FunctionCallingAgent,
+    input: FunctionCallingAgentInput,
+  ) => {
+    const agentContext = agent.run(input, {
+      streaming: input.streaming ?? false,
+    });
+    for await (const event of agentContext) {
+      if (event instanceof AgentRunEvent) {
+        context.sendEvent(event);
+      }
+      if (event instanceof StopEvent) {
+        return event;
+      }
+    }
+    return null;
+  };
+
+  const start = async (
+    context: HandlerContext<BlogContext>,
+    ev: StartEvent<AgentInput>,
+  ) => {
+    const chatHistoryStr = chatHistory
+      .map((msg) => `${msg.role}: ${msg.content}`)
+      .join("\n");
+
+    // Decision-making process
+    const decision = await decideWorkflow(
+      ev.data.message.toString(),
+      chatHistoryStr,
+    );
+
+    if (decision !== "publish") {
+      return new ResearchEvent({
+        input: `Research for this task: ${JSON.stringify(context.data.task)}`,
+      });
+    } else {
+      return new PublishEvent({
+        input: `Publish content based on the chat history\n${chatHistoryStr}\n\n and task: ${context.data.task}`,
+      });
+    }
+  };
+
+  const decideWorkflow = async (task: string, chatHistoryStr: string) => {
+    const llm = Settings.llm;
+
+    const prompt = `You are an expert in decision-making, helping people write and publish blog posts.
+If the user is asking for a file or to publish content, respond with 'publish'.
+If the user requests to write or update a blog post, respond with 'not_publish'.
+
+Here is the chat history:
+${chatHistoryStr}
+
+The current user request is:
+${task}
+
+Given the chat history and the new user request, decide whether to publish based on existing information.
+Decision (respond with either 'not_publish' or 'publish'):`;
+
+    const output = await llm.complete({ prompt: prompt });
+    const decision = output.text.trim().toLowerCase();
+    return decision === "publish" ? "publish" : "research";
+  };
+
+  const research = async (
+    context: HandlerContext<BlogContext>,
+    ev: ResearchEvent,
+  ) => {
+    const researcher = await createResearcher(chatHistory);
+    const researchRes = await runAgent(context, researcher, {
+      displayName: "Researcher",
+      message: ev.data.input,
+    });
+    const researchResult = researchRes?.data;
+
+    return new WriteEvent({
+      input: `Write a blog post given this task: ${JSON.stringify(
+        context.data.task,
+      )} using this research content: ${researchResult}`,
+      isGood: false,
+    });
+  };
+
+  const write = async (
+    context: HandlerContext<BlogContext>,
+    ev: WriteEvent,
+  ) => {
+    const writer = createWriter(chatHistory);
+    context.data.attempts = context.data.attempts + 1;
+    const tooManyAttempts = context.data.attempts > MAX_ATTEMPTS;
+    if (tooManyAttempts) {
+      context.sendEvent(
+        new AgentRunEvent({
+          agent: "writer",
+          text: `Too many attempts (${MAX_ATTEMPTS}) to write the blog post. Proceeding with the current version.`,
+          type: "text",
+        }),
+      );
+    }
+
+    if (ev.data.isGood || tooManyAttempts) {
+      // the blog post is good or too many attempts
+      // stream the final content
+      const result = await runAgent(context, writer, {
+        message: `Based on the reviewer's feedback, refine the post and return only the final version of the post. Here's the current version: ${ev.data.input}`,
+        displayName: "Writer",
+        streaming: true,
+      });
+      return result as unknown as StopEvent<AsyncGenerator<ChatResponseChunk>>;
+    }
+
+    const writeRes = await runAgent(context, writer, {
+      message: ev.data.input,
+      displayName: "Writer",
+      streaming: false,
+    });
+    const writeResult = writeRes?.data;
+    context.data.result = writeResult; // store the last result
+
+    return new ReviewEvent({ input: writeResult });
+  };
+
+  const review = async (
+    context: HandlerContext<BlogContext>,
+    ev: ReviewEvent,
+  ) => {
+    const reviewer = createReviewer(chatHistory);
+    const reviewResult = (await runAgent(context, reviewer, {
+      message: ev.data.input,
+      displayName: "Reviewer",
+      streaming: false,
+    })) as unknown as StopEvent<string>;
+    const reviewResultStr = reviewResult.data;
+    const oldContent = context.data.result;
+    const postIsGood = reviewResultStr.toLowerCase().includes("post is good");
+    context.sendEvent(
+      new AgentRunEvent({
+        agent: "reviewer",
+        text: `The post is ${postIsGood ? "" : "not "}good enough for publishing. Sending back to the writer${
+          postIsGood ? " for publication." : "."
+        }`,
+        type: "text",
+      }),
+    );
+    if (postIsGood) {
+      return new WriteEvent({
+        input: "",
+        isGood: true,
+      });
+    }
+
+    return new WriteEvent({
+      input: `Improve the writing of a given blog post by using a given review.
+            Blog post:
+            \`\`\`
+            ${oldContent}
+            \`\`\`
+
+            Review:
+            \`\`\`
+            ${reviewResult}
+            \`\`\``,
+      isGood: false,
+    });
+  };
+
+  const publish = async (
+    context: HandlerContext<BlogContext>,
+    ev: PublishEvent,
+  ) => {
+    const publisher = await createPublisher(chatHistory);
+
+    const publishResult = await runAgent(context, publisher, {
+      message: `${ev.data.input}`,
+      displayName: "Publisher",
+      streaming: true,
+    });
+    return publishResult as unknown as StopEvent<
+      AsyncGenerator<ChatResponseChunk>
+    >;
+  };
+
+  const workflow: Workflow<
+    BlogContext,
+    AgentInput,
+    string | AsyncGenerator<boolean | ChatResponseChunk>
+  > = new Workflow();
+
+  workflow.addStep(
+    {
+      inputs: [StartEvent<AgentInput>],
+      outputs: [ResearchEvent, PublishEvent],
+    },
+    start,
+  );
+
+  workflow.addStep(
+    {
+      inputs: [ResearchEvent],
+      outputs: [WriteEvent],
+    },
+    research,
+  );
+
+  workflow.addStep(
+    {
+      inputs: [WriteEvent],
+      outputs: [ReviewEvent, StopEvent<AsyncGenerator<ChatResponseChunk>>],
+    },
+    write,
+  );
+
+  workflow.addStep(
+    {
+      inputs: [ReviewEvent],
+      outputs: [WriteEvent],
+    },
+    review,
+  );
+
+  workflow.addStep(
+    {
+      inputs: [PublishEvent],
+      outputs: [StopEvent],
+    },
+    publish,
+  );
+
+  // Overload run method to initialize the context
+  workflow.run = function (
+    input: AgentInput,
+  ): WorkflowContext<
+    AgentInput,
+    string | AsyncGenerator<boolean | ChatResponseChunk>,
+    BlogContext
+  > {
+    return Workflow.prototype.run.call(workflow, new StartEvent(input), {
+      task: input.message.toString(),
+      attempts: 0,
+      result: "",
+    });
+  };
+
+  return workflow;
+};

templates/components/agents/typescript/financial_report/README-template.md

@@ -0,0 +1,47 @@
+This is a [LlamaIndex](https://www.llamaindex.ai/) project using [Next.js](https://nextjs.org/) bootstrapped with [`create-llama`](https://github.com/run-llama/LlamaIndexTS/tree/main/packages/create-llama).
+
+## Getting Started
+
+First, install the dependencies:
+
+```
+npm install
+```
+
+Then check the parameters that have been pre-configured in the `.env` file in this directory.
+Make sure you have the `OPENAI_API_KEY` set.
+
+Second, generate the embeddings of the documents in the `./data` directory:
+
+```
+npm run generate
+```
+
+Third, run the development server:
+
+```
+npm run dev
+```
+
+Open [http://localhost:3000](http://localhost:3000) with your browser to see the chat UI.
+
+## Use Case: Filling Financial CSV Template
+
+You can start by sending an request on the chat UI to create a report comparing the finances of Apple and Tesla.
+Or you can test the `/api/chat` endpoint with the following curl request:
+
+```
+curl --location 'localhost:3000/api/chat' \
+--header 'Content-Type: application/json' \
+--data '{ "messages": [{ "role": "user", "content": "Create a report comparing the finances of Apple and Tesla" }] }'
+```
+
+## Learn More
+
+To learn more about LlamaIndex, take a look at the following resources:
+
+- [LlamaIndex Documentation](https://docs.llamaindex.ai) - learn about LlamaIndex (Python features).
+- [LlamaIndexTS Documentation](https://ts.llamaindex.ai/docs/llamaindex) - learn about LlamaIndex (Typescript features).
+- [Workflows Introduction](https://ts.llamaindex.ai/docs/llamaindex/guide/workflow) - learn about LlamaIndexTS workflows.
+
+You can check out [the LlamaIndexTS GitHub repository](https://github.com/run-llama/LlamaIndexTS) - your feedback and contributions are welcome!

templates/components/agents/typescript/financial_report/agents.ts

@@ -1,65 +0,0 @@
-import { ChatMessage } from "llamaindex";
-import { FunctionCallingAgent } from "./single-agent";
-import { getQueryEngineTools, lookupTools } from "./tools";
-
-export const createResearcher = async (
-  chatHistory: ChatMessage[],
-  params?: any,
-) => {
-  const queryEngineTools = await getQueryEngineTools(params);
-
-  if (!queryEngineTools) {
-    throw new Error("Query engine tool not found");
-  }
-
-  return new FunctionCallingAgent({
-    name: "researcher",
-    tools: queryEngineTools,
-    systemPrompt: `You are a researcher agent. You are responsible for retrieving information from the corpus.
-## Instructions:
-+ Don't synthesize the information, just return the whole retrieved information.
-+ Don't need to retrieve the information that is already provided in the chat history and respond with: "There is no new information, please reuse the information from the conversation."
-`,
-    chatHistory,
-  });
-};
-
-export const createAnalyst = async (chatHistory: ChatMessage[]) => {
-  let systemPrompt = `You are an expert in analyzing financial data.
-You are given a task and a set of financial data to analyze. Your task is to analyze the financial data and return a report.
-Your response should include a detailed analysis of the financial data, including any trends, patterns, or insights that you find.
-Construct the analysis in textual format; including tables would be great!
-Don't need to synthesize the data, just analyze and provide your findings.
-Always use the provided information, don't make up any information yourself.`;
-  const tools = await lookupTools(["interpreter"]);
-  if (tools.length > 0) {
-    systemPrompt = `${systemPrompt}
-You are able to visualize the financial data using code interpreter tool.
-It's very useful to create and include visualizations in the report. Never include any code in the report, just the visualization.`;
-  }
-  return new FunctionCallingAgent({
-    name: "analyst",
-    tools: tools,
-    chatHistory,
-  });
-};
-
-export const createReporter = async (chatHistory: ChatMessage[]) => {
-  const tools = await lookupTools(["document_generator"]);
-  let systemPrompt = `You are a report generation assistant tasked with producing a well-formatted report given parsed context.
-Given a comprehensive analysis of the user request, your task is to synthesize the information and return a well-formatted report.
-
-## Instructions
-You are responsible for representing the analysis in a well-formatted report. If tables or visualizations are provided, add them to the most relevant sections.
-Finally, the report should be presented in markdown format.`;
-  if (tools.length > 0) {
-    systemPrompt = `${systemPrompt}. 
-You are also able to generate an HTML file of the report.`;
-  }
-  return new FunctionCallingAgent({
-    name: "reporter",
-    tools: tools,
-    systemPrompt: systemPrompt,
-    chatHistory,
-  });
-};

templates/components/agents/typescript/financial_report/factory.ts

@@ -1,159 +0,0 @@
-import {
-  Context,
-  StartEvent,
-  StopEvent,
-  Workflow,
-  WorkflowEvent,
-} from "@llamaindex/core/workflow";
-import { Message } from "ai";
-import { ChatMessage, ChatResponseChunk, Settings } from "llamaindex";
-import { getAnnotations } from "../llamaindex/streaming/annotations";
-import { createAnalyst, createReporter, createResearcher } from "./agents";
-import { AgentInput, AgentRunEvent } from "./type";
-
-const TIMEOUT = 360 * 1000;
-const MAX_ATTEMPTS = 2;
-
-class ResearchEvent extends WorkflowEvent<{ input: string }> {}
-class AnalyzeEvent extends WorkflowEvent<{ input: string }> {}
-class ReportEvent extends WorkflowEvent<{ input: string }> {}
-
-const prepareChatHistory = (chatHistory: Message[]): ChatMessage[] => {
-  // By default, the chat history only contains the assistant and user messages
-  // all the agents messages are stored in annotation data which is not visible to the LLM
-
-  const MAX_AGENT_MESSAGES = 10;
-  const agentAnnotations = getAnnotations<{ agent: string; text: string }>(
-    chatHistory,
-    { role: "assistant", type: "agent" },
-  ).slice(-MAX_AGENT_MESSAGES);
-
-  const agentMessages = agentAnnotations
-    .map(
-      (annotation) =>
-        `\n<${annotation.data.agent}>\n${annotation.data.text}\n</${annotation.data.agent}>`,
-    )
-    .join("\n");
-
-  const agentContent = agentMessages
-    ? "Here is the previous conversation of agents:\n" + agentMessages
-    : "";
-
-  if (agentContent) {
-    const agentMessage: ChatMessage = {
-      role: "assistant",
-      content: agentContent,
-    };
-    return [
-      ...chatHistory.slice(0, -1),
-      agentMessage,
-      chatHistory.slice(-1)[0],
-    ] as ChatMessage[];
-  }
-  return chatHistory as ChatMessage[];
-};
-
-export const createWorkflow = (messages: Message[], params?: any) => {
-  const chatHistoryWithAgentMessages = prepareChatHistory(messages);
-  const runAgent = async (
-    context: Context,
-    agent: Workflow,
-    input: AgentInput,
-  ) => {
-    const run = agent.run(new StartEvent({ input }));
-    for await (const event of agent.streamEvents()) {
-      if (event.data instanceof AgentRunEvent) {
-        context.writeEventToStream(event.data);
-      }
-    }
-    return await run;
-  };
-
-  const start = async (context: Context, ev: StartEvent) => {
-    context.set("task", ev.data.input);
-
-    const chatHistoryStr = chatHistoryWithAgentMessages
-      .map((msg) => `${msg.role}: ${msg.content}`)
-      .join("\n");
-
-    // Decision-making process
-    const decision = await decideWorkflow(ev.data.input, chatHistoryStr);
-
-    if (decision !== "publish") {
-      return new ResearchEvent({
-        input: `Research for this task: ${ev.data.input}`,
-      });
-    } else {
-      return new ReportEvent({
-        input: `Publish content based on the chat history\n${chatHistoryStr}\n\n and task: ${ev.data.input}`,
-      });
-    }
-  };
-
-  const decideWorkflow = async (task: string, chatHistoryStr: string) => {
-    const llm = Settings.llm;
-
-    const prompt = `You are an expert in decision-making, helping people write and publish blog posts.
-If the user is asking for a file or to publish content, respond with 'publish'.
-If the user requests to write or update a blog post, respond with 'not_publish'.
-
-Here is the chat history:
-${chatHistoryStr}
-
-The current user request is:
-${task}
-
-Given the chat history and the new user request, decide whether to publish based on existing information.
-Decision (respond with either 'not_publish' or 'publish'):`;
-
-    const output = await llm.complete({ prompt: prompt });
-    const decision = output.text.trim().toLowerCase();
-    return decision === "publish" ? "publish" : "research";
-  };
-
-  const research = async (context: Context, ev: ResearchEvent) => {
-    const researcher = await createResearcher(
-      chatHistoryWithAgentMessages,
-      params,
-    );
-    const researchRes = await runAgent(context, researcher, {
-      message: ev.data.input,
-    });
-    const researchResult = researchRes.data.result;
-    return new AnalyzeEvent({
-      input: `Write a blog post given this task: ${context.get("task")} using this research content: ${researchResult}`,
-    });
-  };
-
-  const analyze = async (context: Context, ev: AnalyzeEvent) => {
-    const analyst = await createAnalyst(chatHistoryWithAgentMessages);
-    const analyzeRes = await runAgent(context, analyst, {
-      message: ev.data.input,
-    });
-    return new ReportEvent({
-      input: `Publish content based on the chat history\n${analyzeRes.data.result}\n\n and task: ${ev.data.input}`,
-    });
-  };
-
-  const report = async (context: Context, ev: ReportEvent) => {
-    const reporter = await createReporter(chatHistoryWithAgentMessages);
-
-    const reportResult = await runAgent(context, reporter, {
-      message: `${ev.data.input}`,
-      streaming: true,
-    });
-    return reportResult as unknown as StopEvent<
-      AsyncGenerator<ChatResponseChunk>
-    >;
-  };
-
-  const workflow = new Workflow({ timeout: TIMEOUT, validate: true });
-  workflow.addStep(StartEvent, start, {
-    outputs: [ResearchEvent, ReportEvent],
-  });
-  workflow.addStep(ResearchEvent, research, { outputs: AnalyzeEvent });
-  workflow.addStep(AnalyzeEvent, analyze, { outputs: ReportEvent });
-  workflow.addStep(ReportEvent, report, { outputs: StopEvent });
-
-  return workflow;
-};

templates/components/agents/typescript/financial_report/tools.ts

@@ -1,86 +0,0 @@
-import fs from "fs/promises";
-import { BaseToolWithCall, LlamaCloudIndex, QueryEngineTool } from "llamaindex";
-import path from "path";
-import { getDataSource } from "../engine";
-import { createTools } from "../engine/tools/index";
-
-export const getQueryEngineTools = async (
-  params?: any,
-): Promise<QueryEngineTool[] | null> => {
-  const topK = process.env.TOP_K ? parseInt(process.env.TOP_K) : undefined;
-
-  const index = await getDataSource(params);
-  if (!index) {
-    return null;
-  }
-  // index is LlamaCloudIndex use two query engine tools
-  if (index instanceof LlamaCloudIndex) {
-    return [
-      new QueryEngineTool({
-        queryEngine: index.asQueryEngine({
-          similarityTopK: topK,
-          retrieval_mode: "files_via_content",
-        }),
-        metadata: {
-          name: "document_retriever",
-          description: `Document retriever that retrieves entire documents from the corpus.
-  ONLY use for research questions that may require searching over entire research reports.
-  Will be slower and more expensive than chunk-level retrieval but may be necessary.`,
-        },
-      }),
-      new QueryEngineTool({
-        queryEngine: index.asQueryEngine({
-          similarityTopK: topK,
-          retrieval_mode: "chunks",
-        }),
-        metadata: {
-          name: "chunk_retriever",
-          description: `Retrieves a small set of relevant document chunks from the corpus.
-      Use for research questions that want to look up specific facts from the knowledge corpus,
-      and need entire documents.`,
-        },
-      }),
-    ];
-  } else {
-    return [
-      new QueryEngineTool({
-        queryEngine: (index as any).asQueryEngine({
-          similarityTopK: topK,
-        }),
-        metadata: {
-          name: "retriever",
-          description: `Use this tool to retrieve information about the text corpus from the index.`,
-        },
-      }),
-    ];
-  }
-};
-
-export const getAvailableTools = async () => {
-  const configFile = path.join("config", "tools.json");
-  let toolConfig: any;
-  const tools: BaseToolWithCall[] = [];
-  try {
-    toolConfig = JSON.parse(await fs.readFile(configFile, "utf8"));
-  } catch (e) {
-    console.info(`Could not read ${configFile} file. Using no tools.`);
-  }
-  if (toolConfig) {
-    tools.push(...(await createTools(toolConfig)));
-  }
-  const queryEngineTools = await getQueryEngineTools();
-  if (queryEngineTools) {
-    tools.push(...queryEngineTools);
-  }
-
-  return tools;
-};
-
-export const lookupTools = async (
-  toolNames: string[],
-): Promise<BaseToolWithCall[]> => {
-  const availableTools = await getAvailableTools();
-  return availableTools.filter((tool) =>
-    toolNames.includes(tool.metadata.name),
-  );
-};

templates/components/agents/typescript/financial_report/workflow/factory.ts

@@ -0,0 +1,20 @@
+import { ChatMessage, ToolCallLLM } from "llamaindex";
+import { getTool } from "../engine/tools";
+import { FinancialReportWorkflow } from "./fin-report";
+import { getQueryEngineTools } from "./tools";
+
+const TIMEOUT = 360 * 1000;
+
+export async function createWorkflow(options: {
+  chatHistory: ChatMessage[];
+  llm?: ToolCallLLM;
+}) {
+  return new FinancialReportWorkflow({
+    chatHistory: options.chatHistory,
+    queryEngineTools: (await getQueryEngineTools()) || [],
+    codeInterpreterTool: (await getTool("interpreter"))!,
+    documentGeneratorTool: (await getTool("document_generator"))!,
+    llm: options.llm,
+    timeout: TIMEOUT,
+  });
+}

templates/components/agents/typescript/financial_report/workflow/fin-report.ts

@@ -0,0 +1,322 @@
+import {
+  HandlerContext,
+  StartEvent,
+  StopEvent,
+  Workflow,
+  WorkflowEvent,
+} from "@llamaindex/workflow";
+import {
+  BaseToolWithCall,
+  ChatMemoryBuffer,
+  ChatMessage,
+  ChatResponseChunk,
+  Settings,
+  ToolCall,
+  ToolCallLLM,
+} from "llamaindex";
+import { callTools, chatWithTools } from "./tools";
+import { AgentInput, AgentRunEvent } from "./type";
+
+// Create a custom event type
+class InputEvent extends WorkflowEvent<{ input: ChatMessage[] }> {}
+
+class ResearchEvent extends WorkflowEvent<{
+  toolCalls: ToolCall[];
+}> {}
+
+class AnalyzeEvent extends WorkflowEvent<{
+  input: ChatMessage | ToolCall[];
+}> {}
+
+class ReportGenerationEvent extends WorkflowEvent<{
+  toolCalls: ToolCall[];
+}> {}
+
+const DEFAULT_SYSTEM_PROMPT = `
+You are a financial analyst who are given a set of tools to help you.
+It's good to using appropriate tools for the user request and always use the information from the tools, don't make up anything yourself.
+For the query engine tool, you should break down the user request into a list of queries and call the tool with the queries.
+`;
+
+export class FinancialReportWorkflow extends Workflow<
+  null,
+  AgentInput,
+  ChatResponseChunk
+> {
+  llm: ToolCallLLM;
+  memory: ChatMemoryBuffer;
+  queryEngineTools: BaseToolWithCall[];
+  codeInterpreterTool: BaseToolWithCall;
+  documentGeneratorTool: BaseToolWithCall;
+  systemPrompt?: string;
+
+  constructor(options: {
+    llm?: ToolCallLLM;
+    chatHistory: ChatMessage[];
+    queryEngineTools: BaseToolWithCall[];
+    codeInterpreterTool: BaseToolWithCall;
+    documentGeneratorTool: BaseToolWithCall;
+    systemPrompt?: string;
+    verbose?: boolean;
+    timeout?: number;
+  }) {
+    super({
+      verbose: options?.verbose ?? false,
+      timeout: options?.timeout ?? 360,
+    });
+
+    this.llm = options.llm ?? (Settings.llm as ToolCallLLM);
+    if (!(this.llm instanceof ToolCallLLM)) {
+      throw new Error("LLM is not a ToolCallLLM");
+    }
+    this.systemPrompt = options.systemPrompt ?? DEFAULT_SYSTEM_PROMPT;
+    this.queryEngineTools = options.queryEngineTools;
+    this.codeInterpreterTool = options.codeInterpreterTool;
+
+    this.documentGeneratorTool = options.documentGeneratorTool;
+    this.memory = new ChatMemoryBuffer({
+      llm: this.llm,
+      chatHistory: options.chatHistory,
+    });
+
+    // Add steps
+    this.addStep(
+      {
+        inputs: [StartEvent<AgentInput>],
+        outputs: [InputEvent],
+      },
+      this.prepareChatHistory,
+    );
+
+    this.addStep(
+      {
+        inputs: [InputEvent],
+        outputs: [
+          InputEvent,
+          ResearchEvent,
+          AnalyzeEvent,
+          ReportGenerationEvent,
+          StopEvent,
+        ],
+      },
+      this.handleLLMInput,
+    );
+
+    this.addStep(
+      {
+        inputs: [ResearchEvent],
+        outputs: [AnalyzeEvent],
+      },
+      this.handleResearch,
+    );
+
+    this.addStep(
+      {
+        inputs: [AnalyzeEvent],
+        outputs: [InputEvent],
+      },
+      this.handleAnalyze,
+    );
+
+    this.addStep(
+      {
+        inputs: [ReportGenerationEvent],
+        outputs: [InputEvent],
+      },
+      this.handleReportGeneration,
+    );
+  }
+
+  prepareChatHistory = async (
+    ctx: HandlerContext<null>,
+    ev: StartEvent<AgentInput>,
+  ): Promise<InputEvent> => {
+    const { message } = ev.data;
+
+    if (this.systemPrompt) {
+      this.memory.put({ role: "system", content: this.systemPrompt });
+    }
+    this.memory.put({ role: "user", content: message });
+
+    return new InputEvent({ input: this.memory.getMessages() });
+  };
+
+  handleLLMInput = async (
+    ctx: HandlerContext<null>,
+    ev: InputEvent,
+  ): Promise<
+    | InputEvent
+    | ResearchEvent
+    | AnalyzeEvent
+    | ReportGenerationEvent
+    | StopEvent
+  > => {
+    const chatHistory = ev.data.input;
+
+    const tools = [this.codeInterpreterTool, this.documentGeneratorTool];
+    if (this.queryEngineTools) {
+      tools.push(...this.queryEngineTools);
+    }
+
+    const toolCallResponse = await chatWithTools(this.llm, tools, chatHistory);
+
+    if (!toolCallResponse.hasToolCall()) {
+      return new StopEvent(toolCallResponse.responseGenerator);
+    }
+
+    if (toolCallResponse.hasMultipleTools()) {
+      this.memory.put({
+        role: "assistant",
+        content:
+          "Calling different tools is not allowed. Please only use multiple calls of the same tool.",
+      });
+      return new InputEvent({ input: this.memory.getMessages() });
+    }
+
+    // Put the LLM tool call message into the memory
+    // And trigger the next step according to the tool call
+    if (toolCallResponse.toolCallMessage) {
+      this.memory.put(toolCallResponse.toolCallMessage);
+    }
+    const toolName = toolCallResponse.getToolNames()[0];
+    switch (toolName) {
+      case this.codeInterpreterTool.metadata.name:
+        return new AnalyzeEvent({
+          input: toolCallResponse.toolCalls,
+        });
+      case this.documentGeneratorTool.metadata.name:
+        return new ReportGenerationEvent({
+          toolCalls: toolCallResponse.toolCalls,
+        });
+      default:
+        if (
+          this.queryEngineTools &&
+          this.queryEngineTools.some((tool) => tool.metadata.name === toolName)
+        ) {
+          return new ResearchEvent({
+            toolCalls: toolCallResponse.toolCalls,
+          });
+        }
+        throw new Error(`Unknown tool: ${toolName}`);
+    }
+  };
+
+  handleResearch = async (
+    ctx: HandlerContext<null>,
+    ev: ResearchEvent,
+  ): Promise<AnalyzeEvent> => {
+    ctx.sendEvent(
+      new AgentRunEvent({
+        agent: "Researcher",
+        text: "Researching data",
+        type: "text",
+      }),
+    );
+
+    const { toolCalls } = ev.data;
+
+    const toolMsgs = await callTools({
+      tools: this.queryEngineTools,
+      toolCalls,
+      ctx,
+      agentName: "Researcher",
+    });
+    for (const toolMsg of toolMsgs) {
+      this.memory.put(toolMsg);
+    }
+    return new AnalyzeEvent({
+      input: {
+        role: "assistant",
+        content:
+          "I have finished researching the data, please analyze the data.",
+      },
+    });
+  };
+
+  /**
+   * Analyze a research result or a tool call for code interpreter from the LLM
+   */
+  handleAnalyze = async (
+    ctx: HandlerContext<null>,
+    ev: AnalyzeEvent,
+  ): Promise<InputEvent> => {
+    ctx.sendEvent(
+      new AgentRunEvent({
+        agent: "Analyst",
+        text: `Starting analysis`,
+        type: "text",
+      }),
+    );
+    // Request by workflow LLM, input is a list of tool calls
+    let toolCalls: ToolCall[] = [];
+    if (Array.isArray(ev.data.input)) {
+      toolCalls = ev.data.input;
+    } else {
+      // Requested by Researcher, input is a ChatMessage
+      // We start new LLM chat specifically for analyzing the data
+      const analysisPrompt = `
+      You are an expert in analyzing financial data.
+      You are given a set of financial data to analyze. Your task is to analyze the financial data and return a report.
+      Your response should include a detailed analysis of the financial data, including any trends, patterns, or insights that you find.
+      Construct the analysis in textual format; including tables would be great!
+      Don't need to synthesize the data, just analyze and provide your findings.
+      `;
+
+      // Clone the current chat history
+      // Add the analysis system prompt and the message from the researcher
+      const newChatHistory = [
+        ...this.memory.getMessages(),
+        { role: "system", content: analysisPrompt },
+        ev.data.input,
+      ];
+      const toolCallResponse = await chatWithTools(
+        this.llm,
+        [this.codeInterpreterTool],
+        newChatHistory as ChatMessage[],
+      );
+
+      if (!toolCallResponse.hasToolCall()) {
+        this.memory.put(await toolCallResponse.asFullResponse());
+        return new InputEvent({
+          input: this.memory.getMessages(),
+        });
+      } else {
+        this.memory.put(toolCallResponse.toolCallMessage);
+        toolCalls = toolCallResponse.toolCalls;
+      }
+    }
+
+    // Call the tools
+    const toolMsgs = await callTools({
+      tools: [this.codeInterpreterTool],
+      toolCalls,
+      ctx,
+      agentName: "Analyst",
+    });
+    for (const toolMsg of toolMsgs) {
+      this.memory.put(toolMsg);
+    }
+
+    return new InputEvent({
+      input: this.memory.getMessages(),
+    });
+  };
+
+  handleReportGeneration = async (
+    ctx: HandlerContext<null>,
+    ev: ReportGenerationEvent,
+  ): Promise<InputEvent> => {
+    const { toolCalls } = ev.data;
+
+    const toolMsgs = await callTools({
+      tools: [this.documentGeneratorTool],
+      toolCalls,
+      ctx,
+      agentName: "Reporter",
+    });
+    for (const toolMsg of toolMsgs) {
+      this.memory.put(toolMsg);
+    }
+    return new InputEvent({ input: this.memory.getMessages() });
+  };
+}

templates/components/agents/typescript/form_filling/README-template.md

@@ -0,0 +1,37 @@
+This is a [LlamaIndex](https://www.llamaindex.ai/) project using [Next.js](https://nextjs.org/) bootstrapped with [`create-llama`](https://github.com/run-llama/LlamaIndexTS/tree/main/packages/create-llama).
+
+## Getting Started
+
+First, install the dependencies:
+
+```
+npm install
+```
+
+Then check the parameters that have been pre-configured in the `.env` file in this directory.
+Make sure you have the `OPENAI_API_KEY` set.
+
+Second, run the development server:
+
+```
+npm run dev
+```
+
+Open [http://localhost:3000](http://localhost:3000) with your browser to see the chat UI.
+
+## Use Case: Filling Financial CSV Template
+
+1. Upload the Apple and Tesla financial reports from the [data](./data) directory. Just send an empty message.
+2. Upload the CSV file [sec_10k_template.csv](./sec_10k_template.csv) and send the message "Fill the missing cells in the CSV file".
+
+The agent will fill the missing cells by retrieving the information from the uploaded financial reports and return a new CSV file with the filled cells.
+
+## Learn More
+
+To learn more about LlamaIndex, take a look at the following resources:
+
+- [LlamaIndex Documentation](https://docs.llamaindex.ai) - learn about LlamaIndex (Python features).
+- [LlamaIndexTS Documentation](https://ts.llamaindex.ai/docs/llamaindex) - learn about LlamaIndex (Typescript features).
+- [Workflows Introduction](https://ts.llamaindex.ai/docs/llamaindex/guide/workflow) - learn about LlamaIndexTS workflows.
+
+You can check out [the LlamaIndexTS GitHub repository](https://github.com/run-llama/LlamaIndexTS) - your feedback and contributions are welcome!

templates/components/agents/typescript/form_filling/sec_10k_template.csv

@@ -0,0 +1,17 @@
+Parameter,2023 Apple (AAPL),2023 Tesla (TSLA)
+Revenue,,
+Net Income,,
+Earnings Per Share (EPS),,
+Debt-to-Equity Ratio,,
+Current Ratio,,
+Gross Margin,,
+Operating Margin,,
+Net Profit Margin,,
+Inventory Turnover,,
+Accounts Receivable Turnover,,
+Capital Expenditure,,
+Research and Development Expense,,
+Market Cap,,
+Price to Earnings Ratio,,
+Dividend Yield,,
+Year-over-Year Growth Rate,,

templates/components/agents/typescript/form_filling/workflow/factory.ts

@@ -0,0 +1,20 @@
+import { ChatMessage, ToolCallLLM } from "llamaindex";
+import { getTool } from "../engine/tools";
+import { FormFillingWorkflow } from "./form-filling";
+import { getQueryEngineTools } from "./tools";
+
+const TIMEOUT = 360 * 1000;
+
+export async function createWorkflow(options: {
+  chatHistory: ChatMessage[];
+  llm?: ToolCallLLM;
+}) {
+  return new FormFillingWorkflow({
+    chatHistory: options.chatHistory,
+    queryEngineTools: (await getQueryEngineTools()) || [],
+    extractorTool: (await getTool("extract_missing_cells"))!,
+    fillMissingCellsTool: (await getTool("fill_missing_cells"))!,
+    llm: options.llm,
+    timeout: TIMEOUT,
+  });
+}

templates/components/agents/typescript/form_filling/workflow/form-filling.ts

@@ -0,0 +1,275 @@
+import {
+  HandlerContext,
+  StartEvent,
+  StopEvent,
+  Workflow,
+  WorkflowEvent,
+} from "@llamaindex/workflow";
+import {
+  BaseToolWithCall,
+  ChatMemoryBuffer,
+  ChatMessage,
+  ChatResponseChunk,
+  Settings,
+  ToolCall,
+  ToolCallLLM,
+} from "llamaindex";
+import { callTools, chatWithTools } from "./tools";
+import { AgentInput, AgentRunEvent } from "./type";
+
+// Create a custom event type
+class InputEvent extends WorkflowEvent<{ input: ChatMessage[] }> {}
+
+class ExtractMissingCellsEvent extends WorkflowEvent<{
+  toolCalls: ToolCall[];
+}> {}
+
+class FindAnswersEvent extends WorkflowEvent<{
+  toolCalls: ToolCall[];
+}> {}
+
+class FillMissingCellsEvent extends WorkflowEvent<{
+  toolCalls: ToolCall[];
+}> {}
+
+const DEFAULT_SYSTEM_PROMPT = `
+You are a helpful assistant who helps fill missing cells in a CSV file.
+Only use the information from the retriever tool - don't make up any information yourself. Fill N/A if an answer is not found.
+If there is no retriever tool or the gathered information has many N/A values indicating the questions don't match the data, respond with a warning and ask the user to upload a different file or connect to a knowledge base.
+You can make multiple tool calls at once but only call with the same tool.
+Only use the local file path for the tools.
+`;
+
+export class FormFillingWorkflow extends Workflow<
+  null,
+  AgentInput,
+  ChatResponseChunk
+> {
+  llm: ToolCallLLM;
+  memory: ChatMemoryBuffer;
+  extractorTool: BaseToolWithCall;
+  queryEngineTools?: BaseToolWithCall[];
+  fillMissingCellsTool: BaseToolWithCall;
+  systemPrompt?: string;
+
+  constructor(options: {
+    llm?: ToolCallLLM;
+    chatHistory: ChatMessage[];
+    extractorTool: BaseToolWithCall;
+    queryEngineTools?: BaseToolWithCall[];
+    fillMissingCellsTool: BaseToolWithCall;
+    systemPrompt?: string;
+    verbose?: boolean;
+    timeout?: number;
+  }) {
+    super({
+      verbose: options?.verbose ?? false,
+      timeout: options?.timeout ?? 360,
+    });
+
+    this.llm = options.llm ?? (Settings.llm as ToolCallLLM);
+    if (!(this.llm instanceof ToolCallLLM)) {
+      throw new Error("LLM is not a ToolCallLLM");
+    }
+    this.systemPrompt = options.systemPrompt ?? DEFAULT_SYSTEM_PROMPT;
+    this.extractorTool = options.extractorTool;
+    this.queryEngineTools = options.queryEngineTools;
+    this.fillMissingCellsTool = options.fillMissingCellsTool;
+
+    this.memory = new ChatMemoryBuffer({
+      llm: this.llm,
+      chatHistory: options.chatHistory,
+    });
+
+    // Add steps
+    this.addStep(
+      {
+        inputs: [StartEvent<AgentInput>],
+        outputs: [InputEvent],
+      },
+      this.prepareChatHistory,
+    );
+
+    this.addStep(
+      {
+        inputs: [InputEvent],
+        outputs: [
+          InputEvent,
+          ExtractMissingCellsEvent,
+          FindAnswersEvent,
+          FillMissingCellsEvent,
+          StopEvent,
+        ],
+      },
+      this.handleLLMInput,
+    );
+
+    this.addStep(
+      {
+        inputs: [ExtractMissingCellsEvent],
+        outputs: [InputEvent],
+      },
+      this.handleExtractMissingCells,
+    );
+
+    this.addStep(
+      {
+        inputs: [FindAnswersEvent],
+        outputs: [InputEvent],
+      },
+      this.handleFindAnswers,
+    );
+
+    this.addStep(
+      {
+        inputs: [FillMissingCellsEvent],
+        outputs: [InputEvent],
+      },
+      this.handleFillMissingCells,
+    );
+  }
+
+  prepareChatHistory = async (
+    ctx: HandlerContext<null>,
+    ev: StartEvent<AgentInput>,
+  ): Promise<InputEvent> => {
+    const { message } = ev.data;
+
+    if (this.systemPrompt) {
+      this.memory.put({ role: "system", content: this.systemPrompt });
+    }
+    this.memory.put({ role: "user", content: message });
+
+    return new InputEvent({ input: this.memory.getMessages() });
+  };
+
+  handleLLMInput = async (
+    ctx: HandlerContext<null>,
+    ev: InputEvent,
+  ): Promise<
+    | InputEvent
+    | ExtractMissingCellsEvent
+    | FindAnswersEvent
+    | FillMissingCellsEvent
+    | StopEvent
+  > => {
+    const chatHistory = ev.data.input;
+
+    const tools = [this.extractorTool, this.fillMissingCellsTool];
+    if (this.queryEngineTools) {
+      tools.push(...this.queryEngineTools);
+    }
+
+    const toolCallResponse = await chatWithTools(this.llm, tools, chatHistory);
+
+    if (!toolCallResponse.hasToolCall()) {
+      return new StopEvent(toolCallResponse.responseGenerator);
+    }
+
+    if (toolCallResponse.hasMultipleTools()) {
+      this.memory.put({
+        role: "assistant",
+        content:
+          "Calling different tools is not allowed. Please only use multiple calls of the same tool.",
+      });
+      return new InputEvent({ input: this.memory.getMessages() });
+    }
+
+    // Put the LLM tool call message into the memory
+    // And trigger the next step according to the tool call
+    if (toolCallResponse.toolCallMessage) {
+      this.memory.put(toolCallResponse.toolCallMessage);
+    }
+    const toolName = toolCallResponse.getToolNames()[0];
+    switch (toolName) {
+      case this.extractorTool.metadata.name:
+        return new ExtractMissingCellsEvent({
+          toolCalls: toolCallResponse.toolCalls,
+        });
+      case this.fillMissingCellsTool.metadata.name:
+        return new FillMissingCellsEvent({
+          toolCalls: toolCallResponse.toolCalls,
+        });
+      default:
+        if (
+          this.queryEngineTools &&
+          this.queryEngineTools.some((tool) => tool.metadata.name === toolName)
+        ) {
+          return new FindAnswersEvent({
+            toolCalls: toolCallResponse.toolCalls,
+          });
+        }
+        throw new Error(`Unknown tool: ${toolName}`);
+    }
+  };
+
+  handleExtractMissingCells = async (
+    ctx: HandlerContext<null>,
+    ev: ExtractMissingCellsEvent,
+  ): Promise<InputEvent> => {
+    ctx.sendEvent(
+      new AgentRunEvent({
+        agent: "CSVExtractor",
+        text: "Extracting missing cells",
+        type: "text",
+      }),
+    );
+    const { toolCalls } = ev.data;
+    const toolMsgs = await callTools({
+      tools: [this.extractorTool],
+      toolCalls,
+      ctx,
+      agentName: "CSVExtractor",
+    });
+    for (const toolMsg of toolMsgs) {
+      this.memory.put(toolMsg);
+    }
+    return new InputEvent({ input: this.memory.getMessages() });
+  };
+
+  handleFindAnswers = async (
+    ctx: HandlerContext<null>,
+    ev: FindAnswersEvent,
+  ): Promise<InputEvent> => {
+    const { toolCalls } = ev.data;
+    if (!this.queryEngineTools) {
+      throw new Error("Query engine tool is not available");
+    }
+    ctx.sendEvent(
+      new AgentRunEvent({
+        agent: "Researcher",
+        text: "Finding answers",
+        type: "text",
+      }),
+    );
+    const toolMsgs = await callTools({
+      tools: this.queryEngineTools,
+      toolCalls,
+      ctx,
+      agentName: "Researcher",
+    });
+
+    for (const toolMsg of toolMsgs) {
+      this.memory.put(toolMsg);
+    }
+    return new InputEvent({ input: this.memory.getMessages() });
+  };
+
+  handleFillMissingCells = async (
+    ctx: HandlerContext<null>,
+    ev: FillMissingCellsEvent,
+  ): Promise<InputEvent> => {
+    const { toolCalls } = ev.data;
+
+    const toolMsgs = await callTools({
+      tools: [this.fillMissingCellsTool],
+      toolCalls,
+      ctx,
+      agentName: "Processor",
+    });
+    for (const toolMsg of toolMsgs) {
+      this.memory.put(toolMsg);
+    }
+    return new InputEvent({ input: this.memory.getMessages() });
+  };
+}

templates/components/engines/python/agent/tools/__init__.py

@@ -56,7 +56,7 @@ class ToolFactory:
             A dictionary of tool names to lists of FunctionTools if map_result is True,
             otherwise a list of FunctionTools.
         """
-        tools: Union[Dict[str, List[FunctionTool]], List[FunctionTool]] = (
+        tools: Union[Dict[str, FunctionTool], List[FunctionTool]] = (
             {} if map_result else []
         )
 
@@ -69,7 +69,9 @@ class ToolFactory:
                             tool_type, tool_name, config
                         )
                         if map_result:
-                            tools[tool_name] = loaded_tools  # type: ignore
+                            tools.update(  # type: ignore
+                                {tool.metadata.name: tool for tool in loaded_tools}
+                            )
                         else:
                             tools.extend(loaded_tools)  # type: ignore
 

templates/components/engines/python/agent/tools/form_filling.py

@@ -0,0 +1,224 @@
+import logging
+import os
+import uuid
+from textwrap import dedent
+from typing import Optional
+
+import pandas as pd
+from app.services.file import FileService
+from llama_index.core import Settings
+from llama_index.core.prompts import PromptTemplate
+from llama_index.core.tools import FunctionTool
+from pydantic import BaseModel, Field
+
+logger = logging.getLogger(__name__)
+
+
+class MissingCell(BaseModel):
+    """
+    A missing cell in a table.
+    """
+
+    row_index: int = Field(description="The index of the row of the missing cell")
+    column_index: int = Field(description="The index of the column of the missing cell")
+    question_to_answer: str = Field(
+        description="The question to answer to fill the missing cell"
+    )
+
+
+class MissingCells(BaseModel):
+    """
+    A list of missing cells.
+    """
+
+    missing_cells: list[MissingCell] = Field(description="The missing cells")
+
+
+class CellValue(BaseModel):
+    row_index: int = Field(description="The row index of the cell")
+    column_index: int = Field(description="The column index of the cell")
+    value: str = Field(
+        description="The value of the cell. Should be a concise value (numerical value or specific value)"
+    )
+
+
+class FormFillingTool:
+    """
+    Fill out missing cells in a CSV file using information from the knowledge base.
+    """
+
+    save_dir: str = os.path.join("output", "tools")
+
+    # Default prompt for extracting questions
+    # Replace the default prompt with a custom prompt by setting the EXTRACT_QUESTIONS_PROMPT environment variable.
+    _default_extract_questions_prompt = dedent(
+        """
+        You are a data analyst. You are given a table with missing cells.
+        Your task is to identify the missing cells and the questions needed to fill them.
+        IMPORTANT: Column indices should be 0-based, where the first data column is index 1 
+        (index 0 is typically the row names/index column).
+
+        # Instructions:
+        - Understand the entire content of the table and the topics of the table.
+        - Identify the missing cells and the meaning of the data in the cells.
+        - For each missing cell, provide the row index and the correct column index (remember: first data column is 1).
+        - For each missing cell, provide the question needed to fill the cell (it's important to provide the question that is relevant to the topic of the table).
+        - Since the cell's value should be concise, the question should request a numerical answer or a specific value.
+
+        # Example:
+        # |    | Name | Age | City |
+        # |----|------|-----|------|
+        # | 0  | John |     | Paris|
+        # | 1  | Mary |     |      |
+        # | 2  |      | 30  |      |
+        #
+        # Your thoughts:
+        # - The table is about people's names, ages, and cities.
+        # - Row: 1, Column: 1 (Age column), Question: "How old is Mary? Please provide only the numerical answer."
+        # - Row: 1, Column: 2 (City column), Question: "In which city does Mary live? Please provide only the city name."
+        
+
+        Please provide your answer in the requested format.
+        # Here is your task:
+
+        - Table content:
+        {table_content}
+
+        - Your answer:
+        """
+    )
+
+    def extract_questions(
+        self,
+        file_path: Optional[str] = None,
+        file_content: Optional[str] = None,
+    ) -> dict:
+        """
+        Use this tool to extract missing cells in a CSV file and generate questions to fill them.
+        Pass either the path to the CSV file or the content of the CSV file.
+
+        Args:
+            file_path (Optional[str]): The local file path to the CSV file to extract missing cells from (Don't pass a sandbox path).
+            file_content (Optional[str]): The content of the CSV file to extract missing cells from.
+
+        Returns:
+            dict: A dictionary containing the missing cells and their corresponding questions.
+        """
+        extract_questions_prompt = os.getenv(
+            "EXTRACT_QUESTIONS_PROMPT", self._default_extract_questions_prompt
+        )
+        if file_path is None and file_content is None:
+            raise ValueError("Either `file_path` or `file_content` must be provided")
+
+        table_content = None
+
+        if file_path:
+            file_name, file_extension = self._get_file_name_and_extension(
+                file_path, file_content
+            )
+
+            try:
+                df = pd.read_csv(file_path)
+            except FileNotFoundError as e:
+                return {
+                    "error": str(e),
+                    "message": "Please check and update the file path and ensure it's a local path - not a sandbox path.",
+                }
+
+            table_content = df.to_markdown()
+            if table_content is None:
+                raise ValueError("Could not convert the table to markdown")
+        if file_content:
+            table_content = file_content
+
+        if table_content is None:
+            raise ValueError("Table content not found")
+
+        response: MissingCells = Settings.llm.structured_predict(
+            output_cls=MissingCells,
+            prompt=PromptTemplate(extract_questions_prompt),
+            table_content=table_content,
+        )
+        return response.model_dump()
+
+    def fill_form(
+        self,
+        cell_values: list[CellValue],
+        file_path: Optional[str] = None,
+        file_content: Optional[str] = None,
+    ) -> dict:
+        """
+        Use this tool to fill cell values into a CSV file.
+        Requires cell values to be used for filling out, as well as either the path to the CSV file or the content of the CSV file.
+
+        Args:
+            cell_values (list[CellValue]): The cell values used to fill out the CSV file (call `extract_questions` and query engine to construct the cell values).
+            file_path (Optional[str]): The local file path to the CSV file that should be filled out (not as sandbox path).
+            file_content (Optional[str]): The content of the CSV file that should be filled out.
+
+        Returns:
+            dict: A dictionary containing the content and metadata of the filled-out file.
+        """
+        file_name, file_extension = self._get_file_name_and_extension(
+            file_path, file_content
+        )
+        df = pd.read_csv(file_path)
+
+        # Fill the dataframe with the cell values
+        filled_df = df.copy()
+        for cell_value in cell_values:
+            if not isinstance(cell_value, CellValue):
+                cell_value = CellValue(**cell_value)
+            filled_df.iloc[cell_value.row_index, cell_value.column_index] = (
+                cell_value.value
+            )
+
+        # Save the filled table to a new CSV file
+        csv_content: str = filled_df.to_csv(index=False)
+        file_metadata = FileService.save_file(
+            content=csv_content,
+            file_name=f"{file_name}_filled.csv",
+            save_dir=self.save_dir,
+        )
+
+        new_content: str = filled_df.to_markdown()
+        result = {
+            "filled_content": new_content,
+            "filled_file": file_metadata,
+        }
+        return result
+
+    def _get_file_name_and_extension(
+        self, file_path: Optional[str], file_content: Optional[str]
+    ) -> tuple[str, str]:
+        if file_path is None and file_content is None:
+            raise ValueError("Either `file_path` or `file_content` must be provided")
+
+        if file_path is None:
+            file_name = str(uuid.uuid4())
+            file_extension = ".csv"
+        else:
+            file_name, file_extension = os.path.splitext(file_path)
+            if file_extension != ".csv":
+                raise ValueError("Form filling is only supported for CSV files")
+
+        return file_name, file_extension
+
+    def _save_output(self, file_name: str, output: str) -> dict:
+        """
+        Save the output to a file.
+        """
+        file_metadata = FileService.save_file(
+            content=output,
+            file_name=file_name,
+            save_dir=self.save_dir,
+        )
+        return file_metadata.model_dump()
+
+
+def get_tools(**kwargs):
+    tool = FormFillingTool()
+    return [
+        FunctionTool.from_defaults(tool.extract_questions),
+        FunctionTool.from_defaults(tool.fill_form),
+    ]

templates/components/engines/typescript/agent/chat.ts

@@ -1,7 +1,7 @@
 import {
   BaseChatEngine,
   BaseToolWithCall,
-  OpenAIAgent,
+  LLMAgent,
   QueryEngineTool,
 } from "llamaindex";
 import fs from "node:fs/promises";
@@ -42,7 +42,7 @@ export async function createChatEngine(documentIds?: string[], params?: any) {
     tools.push(...(await createTools(toolConfig)));
   }
 
-  const agent = new OpenAIAgent({
+  const agent = new LLMAgent({
     tools,
     systemPrompt: process.env.SYSTEM_PROMPT,
   }) as unknown as BaseChatEngine;

templates/components/engines/typescript/agent/tools/form-filling.ts

@@ -0,0 +1,296 @@
+import { JSONSchemaType } from "ajv";
+import fs from "fs";
+import { BaseTool, Settings, ToolMetadata } from "llamaindex";
+import Papa from "papaparse";
+import path from "path";
+import { saveDocument } from "../../llamaindex/documents/helper";
+
+type ExtractMissingCellsParameter = {
+  filePath: string;
+};
+
+export type MissingCell = {
+  rowIndex: number;
+  columnIndex: number;
+  question: string;
+};
+
+const CSV_EXTRACTION_PROMPT = `You are a data analyst. You are given a table with missing cells.
+Your task is to identify the missing cells and the questions needed to fill them.
+IMPORTANT: Column indices should be 0-based
+
+# Instructions:
+- Understand the entire content of the table and the topics of the table.
+- Identify the missing cells and the meaning of the data in the cells.
+- For each missing cell, provide the row index and the correct column index (remember: first data column is 1).
+- For each missing cell, provide the question needed to fill the cell (it's important to provide the question that is relevant to the topic of the table).
+- Since the cell's value should be concise, the question should request a numerical answer or a specific value.
+- Finally, only return the answer in JSON format with the following schema:
+{
+  "missing_cells": [
+    {
+      "rowIndex": number,
+      "columnIndex": number,
+      "question": string
+    }
+  ]
+}
+- If there are no missing cells, return an empty array.
+- The answer is only the JSON object, nothing else and don't wrap it inside markdown code block.
+
+# Example:
+# |    | Name | Age | City |
+# |----|------|-----|------|
+# | 0  | John |     | Paris|
+# | 1  | Mary |     |      |
+# | 2  |      | 30  |      |
+#
+# Your thoughts:
+# - The table is about people's names, ages, and cities.
+# - Row: 1, Column: 2 (Age column), Question: "How old is Mary? Please provide only the numerical answer."
+# - Row: 1, Column: 3 (City column), Question: "In which city does Mary live? Please provide only the city name."
+# Your answer:
+# {
+#   "missing_cells": [
+#     {
+#       "rowIndex": 1,
+#       "columnIndex": 2,
+#       "question": "How old is Mary? Please provide only the numerical answer."
+#     },
+#     {
+#       "rowIndex": 1,
+#       "columnIndex": 3,
+#       "question": "In which city does Mary live? Please provide only the city name."
+#     }
+#   ]
+# }
+
+
+# Here is your task:
+
+- Table content:
+{table_content}
+
+- Your answer:
+`;
+
+const DEFAULT_METADATA: ToolMetadata<
+  JSONSchemaType<ExtractMissingCellsParameter>
+> = {
+  name: "extract_missing_cells",
+  description: `Use this tool to extract missing cells in a CSV file and generate questions to fill them. This tool only works with local file path.`,
+  parameters: {
+    type: "object",
+    properties: {
+      filePath: {
+        type: "string",
+        description: "The local file path to the CSV file.",
+      },
+    },
+    required: ["filePath"],
+  },
+};
+
+export interface ExtractMissingCellsParams {
+  metadata?: ToolMetadata<JSONSchemaType<ExtractMissingCellsParameter>>;
+}
+
+export class ExtractMissingCellsTool
+  implements BaseTool<ExtractMissingCellsParameter>
+{
+  metadata: ToolMetadata<JSONSchemaType<ExtractMissingCellsParameter>>;
+  defaultExtractionPrompt: string;
+
+  constructor(params: ExtractMissingCellsParams) {
+    this.metadata = params.metadata ?? DEFAULT_METADATA;
+    this.defaultExtractionPrompt = CSV_EXTRACTION_PROMPT;
+  }
+
+  private readCsvFile(filePath: string): Promise<string[][]> {
+    return new Promise((resolve, reject) => {
+      fs.readFile(filePath, "utf8", (err, data) => {
+        if (err) {
+          reject(err);
+          return;
+        }
+
+        const parsedData = Papa.parse<string[]>(data, {
+          skipEmptyLines: false,
+        });
+
+        if (parsedData.errors.length) {
+          reject(parsedData.errors);
+          return;
+        }
+
+        // Ensure all rows have the same number of columns as the header
+        const maxColumns = parsedData.data[0].length;
+        const paddedRows = parsedData.data.map((row) => {
+          return [...row, ...Array(maxColumns - row.length).fill("")];
+        });
+
+        resolve(paddedRows);
+      });
+    });
+  }
+
+  private formatToMarkdownTable(data: string[][]): string {
+    if (data.length === 0) return "";
+
+    const maxColumns = data[0].length;
+
+    const headerRow = `| ${data[0].join(" | ")} |`;
+    const separatorRow = `| ${Array(maxColumns).fill("---").join(" | ")} |`;
+
+    const dataRows = data.slice(1).map((row) => {
+      return `| ${row.join(" | ")} |`;
+    });
+
+    return [headerRow, separatorRow, ...dataRows].join("\n");
+  }
+
+  async call(input: ExtractMissingCellsParameter): Promise<MissingCell[]> {
+    const { filePath } = input;
+    let tableContent: string[][];
+    try {
+      tableContent = await this.readCsvFile(filePath);
+    } catch (error) {
+      throw new Error(
+        `Failed to read CSV file. Make sure that you are reading a local file path (not a sandbox path).`,
+      );
+    }
+
+    const prompt = this.defaultExtractionPrompt.replace(
+      "{table_content}",
+      this.formatToMarkdownTable(tableContent),
+    );
+
+    const llm = Settings.llm;
+    const response = await llm.complete({
+      prompt,
+    });
+    const rawAnswer = response.text;
+    const parsedResponse = JSON.parse(rawAnswer) as {
+      missing_cells: MissingCell[];
+    };
+    if (!parsedResponse.missing_cells) {
+      throw new Error(
+        "The answer is not in the correct format. There should be a missing_cells array.",
+      );
+    }
+    const answer = parsedResponse.missing_cells;
+
+    return answer;
+  }
+}
+
+type FillMissingCellsParameter = {
+  filePath: string;
+  cells: {
+    rowIndex: number;
+    columnIndex: number;
+    answer: string;
+  }[];
+};
+
+const FILL_CELLS_METADATA: ToolMetadata<
+  JSONSchemaType<FillMissingCellsParameter>
+> = {
+  name: "fill_missing_cells",
+  description: `Use this tool to fill missing cells in a CSV file with provided answers. This tool only works with local file path.`,
+  parameters: {
+    type: "object",
+    properties: {
+      filePath: {
+        type: "string",
+        description: "The local file path to the CSV file.",
+      },
+      cells: {
+        type: "array",
+        items: {
+          type: "object",
+          properties: {
+            rowIndex: { type: "number" },
+            columnIndex: { type: "number" },
+            answer: { type: "string" },
+          },
+          required: ["rowIndex", "columnIndex", "answer"],
+        },
+        description: "Array of cells to fill with their answers",
+      },
+    },
+    required: ["filePath", "cells"],
+  },
+};
+
+export interface FillMissingCellsParams {
+  metadata?: ToolMetadata<JSONSchemaType<FillMissingCellsParameter>>;
+}
+
+export class FillMissingCellsTool
+  implements BaseTool<FillMissingCellsParameter>
+{
+  metadata: ToolMetadata<JSONSchemaType<FillMissingCellsParameter>>;
+
+  constructor(params: FillMissingCellsParams = {}) {
+    this.metadata = params.metadata ?? FILL_CELLS_METADATA;
+  }
+
+  async call(input: FillMissingCellsParameter): Promise<string> {
+    const { filePath, cells } = input;
+
+    // Read the CSV file
+    const fileContent = await new Promise<string>((resolve, reject) => {
+      fs.readFile(filePath, "utf8", (err, data) => {
+        if (err) {
+          reject(err);
+        } else {
+          resolve(data);
+        }
+      });
+    });
+
+    // Parse CSV with PapaParse
+    const parseResult = Papa.parse<string[]>(fileContent, {
+      header: false, // Ensure the header is not treated as a separate object
+      skipEmptyLines: false, // Ensure empty lines are not skipped
+    });
+
+    if (parseResult.errors.length) {
+      throw new Error(
+        "Failed to parse CSV file: " + parseResult.errors[0].message,
+      );
+    }
+
+    const rows = parseResult.data;
+
+    // Fill the cells with answers
+    for (const cell of cells) {
+      // Adjust rowIndex to start from 1 for data rows
+      const adjustedRowIndex = cell.rowIndex + 1;
+      if (
+        adjustedRowIndex < rows.length &&
+        cell.columnIndex < rows[adjustedRowIndex].length
+      ) {
+        rows[adjustedRowIndex][cell.columnIndex] = cell.answer;
+      }
+    }
+
+    // Convert back to CSV format
+    const updatedContent = Papa.unparse(rows, {
+      delimiter: parseResult.meta.delimiter,
+    });
+
+    // Use the helper function to write the file
+    const parsedPath = path.parse(filePath);
+    const newFileName = `${parsedPath.name}-filled${parsedPath.ext}`;
+    const newFilePath = path.join("output/tools", newFileName);
+
+    const newFileUrl = await saveDocument(newFilePath, updatedContent);
+
+    return (
+      "Successfully filled missing cells in the CSV file. File URL to show to the user: " +
+      newFileUrl
+    );
+  }
+}

templates/components/engines/typescript/agent/tools/index.ts

@@ -1,11 +1,19 @@
 import { BaseToolWithCall } from "llamaindex";
 import { ToolsFactory } from "llamaindex/tools/ToolsFactory";
+import fs from "node:fs/promises";
+import path from "node:path";
 import { CodeGeneratorTool, CodeGeneratorToolParams } from "./code-generator";
 import {
   DocumentGenerator,
   DocumentGeneratorParams,
 } from "./document-generator";
 import { DuckDuckGoSearchTool, DuckDuckGoToolParams } from "./duckduckgo";
+import {
+  ExtractMissingCellsParams,
+  ExtractMissingCellsTool,
+  FillMissingCellsParams,
+  FillMissingCellsTool,
+} from "./form-filling";
 import { ImgGeneratorTool, ImgGeneratorToolParams } from "./img-gen";
 import { InterpreterTool, InterpreterToolParams } from "./interpreter";
 import { OpenAPIActionTool } from "./openapi-action";
@@ -54,6 +62,12 @@ const toolFactory: Record<string, ToolCreator> = {
   document_generator: async (config: unknown) => {
     return [new DocumentGenerator(config as DocumentGeneratorParams)];
   },
+  form_filling: async (config: unknown) => {
+    return [
+      new ExtractMissingCellsTool(config as ExtractMissingCellsParams),
+      new FillMissingCellsTool(config as FillMissingCellsParams),
+    ];
+  },
 };
 
 async function createLocalTools(
@@ -70,3 +84,19 @@ async function createLocalTools(
 
   return tools;
 }
+
+export async function getConfiguredTools(
+  configPath?: string,
+): Promise<BaseToolWithCall[]> {
+  const configFile = path.join(configPath ?? "config", "tools.json");
+  const toolConfig = JSON.parse(await fs.readFile(configFile, "utf8"));
+  const tools = await createTools(toolConfig);
+  return tools;
+}
+
+export async function getTool(
+  toolName: string,
+): Promise<BaseToolWithCall | undefined> {
+  const tools = await getConfiguredTools();
+  return tools.find((tool) => tool.metadata.name === toolName);
+}

templates/components/llamaindex/typescript/documents/helper.ts

@@ -13,7 +13,7 @@ const MIME_TYPE_TO_EXT: Record<string, string> = {
     "docx",
 };
 
-const UPLOADED_FOLDER = "output/uploaded";
+export const UPLOADED_FOLDER = "output/uploaded";
 
 export async function storeAndParseFile(
   name: string,

templates/components/llamaindex/typescript/documents/upload.ts

@@ -1,7 +1,5 @@
 import { Document, LLamaCloudFileService, VectorStoreIndex } from "llamaindex";
 import { LlamaCloudIndex } from "llamaindex/cloud/LlamaCloudIndex";
-import fs from "node:fs/promises";
-import path from "node:path";
 import { DocumentFile } from "../streaming/annotations";
 import { parseFile, storeFile } from "./helper";
 import { runPipeline } from "./pipeline";
@@ -18,8 +16,8 @@ export async function uploadDocument(
   // Store file
   const fileMetadata = await storeFile(name, fileBuffer, mimeType);
 
-  // If the file is csv and has codeExecutorTool, we don't need to index the file.
-  if (mimeType === "text/csv" && (await hasCodeExecutorTool())) {
+  // Do not index csv files
+  if (mimeType === "text/csv") {
     return fileMetadata;
   }
   let documentIds: string[] = [];
@@ -49,7 +47,11 @@ export async function uploadDocument(
     }
   } else {
     // run the pipeline for other vector store indexes
-    const documents: Document[] = await parseFile(fileBuffer, name, mimeType);
+    const documents: Document[] = await parseFile(
+      fileBuffer,
+      fileMetadata.name,
+      mimeType,
+    );
     documentIds = await runPipeline(index, documents);
   }
 
@@ -57,14 +59,3 @@ export async function uploadDocument(
   fileMetadata.refs = documentIds;
   return fileMetadata;
 }
-
-const hasCodeExecutorTool = async () => {
-  const codeExecutorTools = ["interpreter", "artifact"];
-
-  const configFile = path.join("config", "tools.json");
-  const toolConfig = JSON.parse(await fs.readFile(configFile, "utf8"));
-
-  const localTools = toolConfig.local || {};
-  // Check if local tools contains codeExecutorTools
-  return codeExecutorTools.some((tool) => localTools[tool] !== undefined);
-};

templates/components/llamaindex/typescript/streaming/annotations.ts

@@ -1,5 +1,11 @@
 import { JSONValue, Message } from "ai";
-import { MessageContent, MessageContentDetail } from "llamaindex";
+import {
+  ChatMessage,
+  MessageContent,
+  MessageContentDetail,
+  MessageType,
+} from "llamaindex";
+import { UPLOADED_FOLDER } from "../documents/helper";
 
 export type DocumentFileType = "csv" | "pdf" | "txt" | "docx";
 
@@ -58,6 +64,45 @@ export function retrieveMessageContent(messages: Message[]): MessageContent {
   ];
 }
 
+export function convertToChatHistory(messages: Message[]): ChatMessage[] {
+  if (!messages || !Array.isArray(messages)) {
+    return [];
+  }
+  const agentHistory = retrieveAgentHistoryMessage(messages);
+  if (agentHistory) {
+    const previousMessages = messages.slice(0, -1);
+    return [...previousMessages, agentHistory].map((msg) => ({
+      role: msg.role as MessageType,
+      content: msg.content,
+    }));
+  }
+  return messages.map((msg) => ({
+    role: msg.role as MessageType,
+    content: msg.content,
+  }));
+}
+
+function retrieveAgentHistoryMessage(
+  messages: Message[],
+  maxAgentMessages = 10,
+): ChatMessage | null {
+  const agentAnnotations = getAnnotations<{ agent: string; text: string }>(
+    messages,
+    { role: "assistant", type: "agent" },
+  ).slice(-maxAgentMessages);
+
+  if (agentAnnotations.length > 0) {
+    const messageContent =
+      "Here is the previous conversation of agents:\n" +
+      agentAnnotations.map((annotation) => annotation.data.text).join("\n");
+    return {
+      role: "assistant",
+      content: messageContent,
+    };
+  }
+  return null;
+}
+
 function getFileContent(file: DocumentFile): string {
   let defaultContent = `=====File: ${file.name}=====\n`;
   // Include file URL if it's available
@@ -84,6 +129,10 @@ function getFileContent(file: DocumentFile): string {
   const sandboxFilePath = `/tmp/${file.name}`;
   defaultContent += `Sandbox file path (instruction: only use sandbox path for artifact or code interpreter tool): ${sandboxFilePath}\n`;
 
+  // Include local file path
+  const localFilePath = `${UPLOADED_FOLDER}/${file.name}`;
+  defaultContent += `Local file path (instruction: use for local tool that requires a local path): ${localFilePath}\n`;
+
   return defaultContent;
 }
 
@@ -127,13 +176,10 @@ function retrieveLatestArtifact(messages: Message[]): MessageContentDetail[] {
 }
 
 function convertAnnotations(messages: Message[]): MessageContentDetail[] {
-  // annotations from the last user message that has annotations
-  const annotations: Annotation[] =
-    messages
-      .slice()
-      .reverse()
-      .find((message) => message.role === "user" && message.annotations)
-      ?.annotations?.map(getValidAnnotation) || [];
+  // get all annotations from user messages
+  const annotations: Annotation[] = messages
+    .filter((message) => message.role === "user" && message.annotations)
+    .flatMap((message) => message.annotations?.map(getValidAnnotation) || []);
   if (annotations.length === 0) return [];
 
   const content: MessageContentDetail[] = [];

templates/components/loaders/typescript/file/loader.ts

@@ -1,7 +1,7 @@
 import {
   FILE_EXT_TO_READER,
   SimpleDirectoryReader,
-} from "llamaindex/readers/SimpleDirectoryReader";
+} from "llamaindex/readers/index";
 
 export const DATA_DIR = "./data";
 

templates/components/loaders/typescript/llama_parse/loader.ts

@@ -2,7 +2,7 @@ import { LlamaParseReader } from "llamaindex";
 import {
   FILE_EXT_TO_READER,
   SimpleDirectoryReader,
-} from "llamaindex/readers/SimpleDirectoryReader";
+} from "llamaindex/readers/index";
 
 export const DATA_DIR = "./data";
 

templates/components/multiagent/python/app/api/routers/chat.py

@@ -4,7 +4,8 @@ from app.api.routers.models import (
     ChatData,
 )
 from app.api.routers.vercel_response import VercelStreamResponse
-from app.engine.engine import get_chat_engine
+from app.engine.query_filter import generate_filters
+from app.workflows import create_workflow
 from fastapi import APIRouter, BackgroundTasks, HTTPException, Request, status
 
 chat_router = r = APIRouter()
@@ -22,19 +23,20 @@ async def chat(
         last_message_content = data.get_last_message_content()
         messages = data.get_history_messages(include_agent_messages=True)
 
-        # The chat API supports passing private document filters and chat params
-        # but agent workflow does not support them yet
-        # ignore chat params and use all documents for now
-        # TODO: generate filters based on doc_ids
+        doc_ids = data.get_chat_document_ids()
+        filters = generate_filters(doc_ids)
         params = data.data or {}
-        engine = get_chat_engine(chat_history=messages, params=params)
 
-        event_handler = engine.run(input=last_message_content, streaming=True)
+        workflow = create_workflow(
+            chat_history=messages, params=params, filters=filters
+        )
+
+        event_handler = workflow.run(input=last_message_content, streaming=True)
         return VercelStreamResponse(
             request=request,
             chat_data=data,
             event_handler=event_handler,
-            events=engine.stream_events(),
+            events=workflow.stream_events(),
         )
     except Exception as e:
         logger.exception("Error in chat engine", exc_info=True)

templates/components/multiagent/python/app/api/routers/vercel_response.py

@@ -1,12 +1,11 @@
 import asyncio
 import json
 import logging
-from typing import AsyncGenerator, List
+from typing import AsyncGenerator, Awaitable, List
 
 from aiostream import stream
 from app.api.routers.models import ChatData, Message
 from app.api.services.suggestion import NextQuestionSuggestion
-from app.workflows.single import AgentRunEvent, AgentRunResult
 from fastapi import Request
 from fastapi.responses import StreamingResponse
 
@@ -55,8 +54,8 @@ class VercelStreamResponse(StreamingResponse):
         self,
         request: Request,
         chat_data: ChatData,
-        event_handler: AgentRunResult | AsyncGenerator,
-        events: AsyncGenerator[AgentRunEvent, None],
+        event_handler: Awaitable,
+        events: AsyncGenerator,
         verbose: bool = True,
     ):
         # Yield the text response
@@ -64,15 +63,17 @@ class VercelStreamResponse(StreamingResponse):
             result = await event_handler
             final_response = ""
 
-            if isinstance(result, AgentRunResult):
-                for token in result.response.message.content:
-                    final_response += token
-                    yield self.convert_text(token)
-
             if isinstance(result, AsyncGenerator):
                 async for token in result:
-                    final_response += token.delta
+                    final_response += str(token.delta)
                     yield self.convert_text(token.delta)
+            else:
+                if hasattr(result, "response"):
+                    content = result.response.message.content
+                    if content:
+                        for token in content:
+                            final_response += str(token)
+                            yield self.convert_text(token)
 
             # Generate next questions if next question prompt is configured
             question_data = await self._generate_next_questions(
@@ -86,7 +87,7 @@ class VercelStreamResponse(StreamingResponse):
         # Yield the events from the event handler
         async def _event_generator():
             async for event in events:
-                event_response = self._event_to_response(event)
+                event_response = event.to_response()
                 if verbose:
                     logger.debug(event_response)
                 if event_response is not None:
@@ -95,13 +96,6 @@ class VercelStreamResponse(StreamingResponse):
         combine = stream.merge(_chat_response_generator(), _event_generator())
         return combine
 
-    @staticmethod
-    def _event_to_response(event: AgentRunEvent) -> dict:
-        return {
-            "type": "agent",
-            "data": {"agent": event.name, "text": event.msg},
-        }
-
     @classmethod
     def convert_text(cls, token: str):
         # Escape newlines and double quotes to avoid breaking the stream

templates/components/multiagent/python/app/workflows/events.py

@@ -0,0 +1,27 @@
+from enum import Enum
+from typing import Optional
+
+from llama_index.core.workflow import Event
+
+
+class AgentRunEventType(Enum):
+    TEXT = "text"
+    PROGRESS = "progress"
+
+
+class AgentRunEvent(Event):
+    name: str
+    msg: str
+    event_type: AgentRunEventType = AgentRunEventType.TEXT
+    data: Optional[dict] = None
+
+    def to_response(self) -> dict:
+        return {
+            "type": "agent",
+            "data": {
+                "agent": self.name,
+                "type": self.event_type.value,
+                "text": self.msg,
+                "data": self.data,
+            },
+        }

templates/components/multiagent/python/app/workflows/function_calling_agent.py

@@ -0,0 +1,121 @@
+from typing import Any, List, Optional
+
+from app.workflows.events import AgentRunEvent
+from app.workflows.tools import ToolCallResponse, call_tools, chat_with_tools
+from llama_index.core.base.llms.types import ChatMessage
+from llama_index.core.llms.function_calling import FunctionCallingLLM
+from llama_index.core.memory import ChatMemoryBuffer
+from llama_index.core.settings import Settings
+from llama_index.core.tools.types import BaseTool
+from llama_index.core.workflow import (
+    Context,
+    Event,
+    StartEvent,
+    StopEvent,
+    Workflow,
+    step,
+)
+
+
+class InputEvent(Event):
+    input: list[ChatMessage]
+
+
+class ToolCallEvent(Event):
+    input: ToolCallResponse
+
+
+class FunctionCallingAgent(Workflow):
+    """
+    A simple workflow to request LLM with tools independently.
+    You can share the previous chat history to provide the context for the LLM.
+    """
+
+    def __init__(
+        self,
+        *args: Any,
+        llm: FunctionCallingLLM | None = None,
+        chat_history: Optional[List[ChatMessage]] = None,
+        tools: List[BaseTool] | None = None,
+        system_prompt: str | None = None,
+        verbose: bool = False,
+        timeout: float = 360.0,
+        name: str,
+        write_events: bool = True,
+        **kwargs: Any,
+    ) -> None:
+        super().__init__(*args, verbose=verbose, timeout=timeout, **kwargs)  # type: ignore
+        self.tools = tools or []
+        self.name = name
+        self.write_events = write_events
+
+        if llm is None:
+            llm = Settings.llm
+        self.llm = llm
+        if not self.llm.metadata.is_function_calling_model:
+            raise ValueError("The provided LLM must support function calling.")
+
+        self.system_prompt = system_prompt
+
+        self.memory = ChatMemoryBuffer.from_defaults(
+            llm=self.llm, chat_history=chat_history
+        )
+        self.sources = []  # type: ignore
+
+    @step()
+    async def prepare_chat_history(self, ctx: Context, ev: StartEvent) -> InputEvent:
+        # clear sources
+        self.sources = []
+
+        # set streaming
+        ctx.data["streaming"] = getattr(ev, "streaming", False)
+
+        # set system prompt
+        if self.system_prompt is not None:
+            system_msg = ChatMessage(role="system", content=self.system_prompt)
+            self.memory.put(system_msg)
+
+        # get user input
+        user_input = ev.input
+        user_msg = ChatMessage(role="user", content=user_input)
+        self.memory.put(user_msg)
+
+        if self.write_events:
+            ctx.write_event_to_stream(
+                AgentRunEvent(name=self.name, msg=f"Start to work on: {user_input}")
+            )
+
+        return InputEvent(input=self.memory.get())
+
+    @step()
+    async def handle_llm_input(
+        self,
+        ctx: Context,
+        ev: InputEvent,
+    ) -> ToolCallEvent | StopEvent:
+        chat_history = ev.input
+
+        response = await chat_with_tools(
+            self.llm,
+            self.tools,
+            chat_history,
+        )
+        is_tool_call = isinstance(response, ToolCallResponse)
+        if not is_tool_call:
+            if ctx.data["streaming"]:
+                return StopEvent(result=response)
+            else:
+                full_response = ""
+                async for chunk in response.generator:
+                    full_response += chunk.message.content
+                return StopEvent(result=full_response)
+        return ToolCallEvent(input=response)
+
+    @step()
+    async def handle_tool_calls(self, ctx: Context, ev: ToolCallEvent) -> InputEvent:
+        tool_calls = ev.input.tool_calls
+        tool_call_message = ev.input.tool_call_message
+        self.memory.put(tool_call_message)
+        tool_messages = await call_tools(self.name, self.tools, ctx, tool_calls)
+        self.memory.put_messages(tool_messages)
+        return InputEvent(input=self.memory.get())

templates/components/multiagent/python/app/workflows/tools.py

@@ -0,0 +1,227 @@
+import logging
+import uuid
+from abc import ABC, abstractmethod
+from typing import Any, AsyncGenerator, Callable, Optional
+
+from app.workflows.events import AgentRunEvent, AgentRunEventType
+from llama_index.core.base.llms.types import ChatMessage, ChatResponse, MessageRole
+from llama_index.core.llms.function_calling import FunctionCallingLLM
+from llama_index.core.tools import (
+    BaseTool,
+    FunctionTool,
+    ToolOutput,
+    ToolSelection,
+)
+from llama_index.core.workflow import Context
+from pydantic import BaseModel, ConfigDict
+
+logger = logging.getLogger("uvicorn")
+
+
+class ContextAwareTool(FunctionTool, ABC):
+    @abstractmethod
+    async def acall(self, ctx: Context, input: Any) -> ToolOutput:  # type: ignore
+        pass
+
+
+class ChatWithToolsResponse(BaseModel):
+    """
+    A tool call response from chat_with_tools.
+    """
+
+    tool_calls: Optional[list[ToolSelection]]
+    tool_call_message: Optional[ChatMessage]
+    generator: Optional[AsyncGenerator[ChatResponse | None, None]]
+
+    model_config = ConfigDict(arbitrary_types_allowed=True)
+
+    def is_calling_different_tools(self) -> bool:
+        tool_names = {tool_call.tool_name for tool_call in self.tool_calls}
+        return len(tool_names) > 1
+
+    def has_tool_calls(self) -> bool:
+        return self.tool_calls is not None and len(self.tool_calls) > 0
+
+    def tool_name(self) -> str:
+        assert self.has_tool_calls()
+        assert not self.is_calling_different_tools()
+        return self.tool_calls[0].tool_name
+
+    async def full_response(self) -> str:
+        assert self.generator is not None
+        full_response = ""
+        async for chunk in self.generator:
+            full_response += chunk.message.content
+        return full_response
+
+
+async def chat_with_tools(  # type: ignore
+    llm: FunctionCallingLLM,
+    tools: list[BaseTool],
+    chat_history: list[ChatMessage],
+) -> ChatWithToolsResponse:
+    """
+    Request LLM to call tools or not.
+    This function doesn't change the memory.
+    """
+    generator = _tool_call_generator(llm, tools, chat_history)
+    is_tool_call = await generator.__anext__()
+    if is_tool_call:
+        # Last chunk is the full response
+        # Wait for the last chunk
+        full_response = None
+        async for chunk in generator:
+            full_response = chunk
+        assert isinstance(full_response, ChatResponse)
+        return ChatWithToolsResponse(
+            tool_calls=llm.get_tool_calls_from_response(full_response),
+            tool_call_message=full_response.message,
+            generator=None,
+        )
+    else:
+        return ChatWithToolsResponse(
+            tool_calls=None,
+            tool_call_message=None,
+            generator=generator,
+        )
+
+
+async def call_tools(
+    ctx: Context,
+    agent_name: str,
+    tools: list[BaseTool],
+    tool_calls: list[ToolSelection],
+    emit_agent_events: bool = True,
+) -> list[ChatMessage]:
+    if len(tool_calls) == 0:
+        return []
+
+    tools_by_name = {tool.metadata.get_name(): tool for tool in tools}
+    if len(tool_calls) == 1:
+        return [
+            await call_tool(
+                ctx,
+                tools_by_name[tool_calls[0].tool_name],
+                tool_calls[0],
+                lambda msg: ctx.write_event_to_stream(
+                    AgentRunEvent(
+                        name=agent_name,
+                        msg=msg,
+                    )
+                ),
+            )
+        ]
+    # Multiple tool calls, show progress
+    tool_msgs: list[ChatMessage] = []
+
+    progress_id = str(uuid.uuid4())
+    total_steps = len(tool_calls)
+    if emit_agent_events:
+        ctx.write_event_to_stream(
+            AgentRunEvent(
+                name=agent_name,
+                msg=f"Making {total_steps} tool calls",
+            )
+        )
+    for i, tool_call in enumerate(tool_calls):
+        tool = tools_by_name.get(tool_call.tool_name)
+        if not tool:
+            tool_msgs.append(
+                ChatMessage(
+                    role=MessageRole.ASSISTANT,
+                    content=f"Tool {tool_call.tool_name} does not exist",
+                )
+            )
+            continue
+        tool_msg = await call_tool(
+            ctx,
+            tool,
+            tool_call,
+            event_emitter=lambda msg: ctx.write_event_to_stream(
+                AgentRunEvent(
+                    name=agent_name,
+                    msg=msg,
+                    event_type=AgentRunEventType.PROGRESS,
+                    data={
+                        "id": progress_id,
+                        "total": total_steps,
+                        "current": i,
+                    },
+                )
+            ),
+        )
+        tool_msgs.append(tool_msg)
+    return tool_msgs
+
+
+async def call_tool(
+    ctx: Context,
+    tool: BaseTool,
+    tool_call: ToolSelection,
+    event_emitter: Optional[Callable[[str], None]],
+) -> ChatMessage:
+    if event_emitter:
+        event_emitter(
+            f"Calling tool {tool_call.tool_name}, {str(tool_call.tool_kwargs)}"
+        )
+    try:
+        if isinstance(tool, ContextAwareTool):
+            if ctx is None:
+                raise ValueError("Context is required for context aware tool")
+            # inject context for calling an context aware tool
+            response = await tool.acall(ctx=ctx, **tool_call.tool_kwargs)
+        else:
+            response = await tool.acall(**tool_call.tool_kwargs)  # type: ignore
+        return ChatMessage(
+            role=MessageRole.TOOL,
+            content=str(response.raw_output),
+            additional_kwargs={
+                "tool_call_id": tool_call.tool_id,
+                "name": tool.metadata.get_name(),
+            },
+        )
+    except Exception as e:
+        logger.error(f"Got error in tool {tool_call.tool_name}: {str(e)}")
+        if event_emitter:
+            event_emitter(f"Got error in tool {tool_call.tool_name}: {str(e)}")
+        return ChatMessage(
+            role=MessageRole.TOOL,
+            content=f"Error: {str(e)}",
+            additional_kwargs={
+                "tool_call_id": tool_call.tool_id,
+                "name": tool.metadata.get_name(),
+            },
+        )
+
+
+async def _tool_call_generator(
+    llm: FunctionCallingLLM,
+    tools: list[BaseTool],
+    chat_history: list[ChatMessage],
+) -> AsyncGenerator[ChatResponse | bool, None]:
+    response_stream = await llm.astream_chat_with_tools(
+        tools,
+        chat_history=chat_history,
+        allow_parallel_tool_calls=False,
+    )
+
+    full_response = None
+    yielded_indicator = False
+    async for chunk in response_stream:
+        if "tool_calls" not in chunk.message.additional_kwargs:
+            # Yield a boolean to indicate whether the response is a tool call
+            if not yielded_indicator:
+                yield False
+                yielded_indicator = True
+
+            # if not a tool call, yield the chunks!
+            yield chunk  # type: ignore
+        elif not yielded_indicator:
+            # Yield the indicator for a tool call
+            yield True
+            yielded_indicator = True
+
+        full_response = chunk
+
+    if full_response:
+        yield full_response  # type: ignore

templates/components/multiagent/typescript/express/chat.controller.ts

@@ -1,36 +1,34 @@
-import { StopEvent } from "@llamaindex/core/workflow";
 import { Message, streamToResponse } from "ai";
 import { Request, Response } from "express";
-import { ChatResponseChunk } from "llamaindex";
+import {
+  convertToChatHistory,
+  retrieveMessageContent,
+} from "./llamaindex/streaming/annotations";
 import { createWorkflow } from "./workflow/factory";
-import { toDataStream, workflowEventsToStreamData } from "./workflow/stream";
+import { createStreamFromWorkflowContext } from "./workflow/stream";
 
 export const chat = async (req: Request, res: Response) => {
   try {
-    const { messages, data }: { messages: Message[]; data?: any } = req.body;
-    const userMessage = messages.pop();
-    if (!messages || !userMessage || userMessage.role !== "user") {
+    const { messages }: { messages: Message[] } = req.body;
+    if (!messages || messages.length === 0) {
       return res.status(400).json({
-        error:
-          "messages are required in the request body and the last message must be from the user",
+        error: "messages are required in the request body",
       });
     }
+    const chatHistory = convertToChatHistory(messages);
+    const userMessageContent = retrieveMessageContent(messages);
 
-    const agent = createWorkflow(messages, data);
-    const result = agent.run<AsyncGenerator<ChatResponseChunk>>(
-      userMessage.content,
-    ) as unknown as Promise<StopEvent<AsyncGenerator<ChatResponseChunk>>>;
+    const workflow = await createWorkflow({ chatHistory });
 
-    // convert the workflow events to a vercel AI stream data object
-    const agentStreamData = await workflowEventsToStreamData(
-      agent.streamEvents(),
-    );
-    // convert the workflow result to a vercel AI content stream
-    const stream = toDataStream(result, {
-      onFinal: () => agentStreamData.close(),
+    const context = workflow.run({
+      message: userMessageContent,
+      streaming: true,
     });
 
-    return streamToResponse(stream, res, {}, agentStreamData);
+    const { stream, dataStream } =
+      await createStreamFromWorkflowContext(context);
+
+    return streamToResponse(stream, res, {}, dataStream);
   } catch (error) {
     console.error("[LlamaIndex]", error);
     return res.status(500).json({

templates/components/multiagent/typescript/nextjs/route.ts

@@ -1,11 +1,14 @@
 import { initObservability } from "@/app/observability";
-import { StopEvent } from "@llamaindex/core/workflow";
-import { Message, StreamingTextResponse } from "ai";
-import { ChatResponseChunk } from "llamaindex";
+import { StreamingTextResponse, type Message } from "ai";
 import { NextRequest, NextResponse } from "next/server";
 import { initSettings } from "./engine/settings";
+import {
+  convertToChatHistory,
+  isValidMessages,
+  retrieveMessageContent,
+} from "./llamaindex/streaming/annotations";
 import { createWorkflow } from "./workflow/factory";
-import { toDataStream, workflowEventsToStreamData } from "./workflow/stream";
+import { createStreamFromWorkflowContext } from "./workflow/stream";
 
 initObservability();
 initSettings();
@@ -16,9 +19,8 @@ export const dynamic = "force-dynamic";
 export async function POST(request: NextRequest) {
   try {
     const body = await request.json();
-    const { messages, data }: { messages: Message[]; data?: any } = body;
-    const userMessage = messages.pop();
-    if (!messages || !userMessage || userMessage.role !== "user") {
+    const { messages }: { messages: Message[]; data?: any } = body;
+    if (!isValidMessages(messages)) {
       return NextResponse.json(
         {
           error:
@@ -28,20 +30,20 @@ export async function POST(request: NextRequest) {
       );
     }
 
-    const agent = createWorkflow(messages, data);
-    // TODO: fix type in agent.run in LITS
-    const result = agent.run<AsyncGenerator<ChatResponseChunk>>(
-      userMessage.content,
-    ) as unknown as Promise<StopEvent<AsyncGenerator<ChatResponseChunk>>>;
-    // convert the workflow events to a vercel AI stream data object
-    const agentStreamData = await workflowEventsToStreamData(
-      agent.streamEvents(),
-    );
-    // convert the workflow result to a vercel AI content stream
-    const stream = toDataStream(result, {
-      onFinal: () => agentStreamData.close(),
+    const chatHistory = convertToChatHistory(messages);
+    const userMessageContent = retrieveMessageContent(messages);
+
+    const workflow = await createWorkflow({ chatHistory });
+
+    const context = workflow.run({
+      message: userMessageContent,
+      streaming: true,
     });
-    return new StreamingTextResponse(stream, {}, agentStreamData);
+    const { stream, dataStream } =
+      await createStreamFromWorkflowContext(context);
+
+    // Return the two streams in one response
+    return new StreamingTextResponse(stream, {}, dataStream);
   } catch (error) {
     console.error("[LlamaIndex]", error);
     return NextResponse.json(

templates/components/multiagent/typescript/workflow/single-agent.ts

@@ -1,22 +1,21 @@
 import {
-  Context,
+  HandlerContext,
   StartEvent,
   StopEvent,
   Workflow,
   WorkflowEvent,
-} from "@llamaindex/core/workflow";
+} from "@llamaindex/workflow";
 import {
   BaseToolWithCall,
   ChatMemoryBuffer,
   ChatMessage,
-  ChatResponse,
   ChatResponseChunk,
+  QueryEngineTool,
   Settings,
   ToolCall,
   ToolCallLLM,
-  ToolCallLLMMessageOptions,
-  callTool,
 } from "llamaindex";
+import { callTools, chatWithTools } from "./tools";
 import { AgentInput, AgentRunEvent } from "./type";
 
 class InputEvent extends WorkflowEvent<{
@@ -27,11 +26,23 @@ class ToolCallEvent extends WorkflowEvent<{
   toolCalls: ToolCall[];
 }> {}
 
-export class FunctionCallingAgent extends Workflow {
+type FunctionCallingAgentContextData = {
+  streaming: boolean;
+};
+
+export type FunctionCallingAgentInput = AgentInput & {
+  displayName: string;
+};
+
+export class FunctionCallingAgent extends Workflow<
+  FunctionCallingAgentContextData,
+  FunctionCallingAgentInput,
+  string | AsyncGenerator<boolean | ChatResponseChunk<object>>
+> {
   name: string;
   llm: ToolCallLLM;
   memory: ChatMemoryBuffer;
-  tools: BaseToolWithCall[];
+  tools: BaseToolWithCall[] | QueryEngineTool[];
   systemPrompt?: string;
   writeEvents: boolean;
   role?: string;
@@ -53,7 +64,9 @@ export class FunctionCallingAgent extends Workflow {
     });
     this.name = options?.name;
     this.llm = options.llm ?? (Settings.llm as ToolCallLLM);
-    this.checkToolCallSupport();
+    if (!(this.llm instanceof ToolCallLLM)) {
+      throw new Error("LLM is not a ToolCallLLM");
+    }
     this.memory = new ChatMemoryBuffer({
       llm: this.llm,
       chatHistory: options.chatHistory,
@@ -64,175 +77,103 @@ export class FunctionCallingAgent extends Workflow {
     this.role = options?.role;
 
     // add steps
-    this.addStep(StartEvent<AgentInput>, this.prepareChatHistory, {
-      outputs: InputEvent,
-    });
-    this.addStep(InputEvent, this.handleLLMInput, {
-      outputs: [ToolCallEvent, StopEvent],
-    });
-    this.addStep(ToolCallEvent, this.handleToolCalls, {
-      outputs: InputEvent,
-    });
+    this.addStep(
+      {
+        inputs: [StartEvent<AgentInput>],
+        outputs: [InputEvent],
+      },
+      this.prepareChatHistory,
+    );
+    this.addStep(
+      {
+        inputs: [InputEvent],
+        outputs: [ToolCallEvent, StopEvent],
+      },
+      this.handleLLMInput,
+    );
+    this.addStep(
+      {
+        inputs: [ToolCallEvent],
+        outputs: [InputEvent],
+      },
+      this.handleToolCalls,
+    );
   }
 
   private get chatHistory() {
     return this.memory.getMessages();
   }
 
-  private async prepareChatHistory(
-    ctx: Context,
+  prepareChatHistory = async (
+    ctx: HandlerContext<FunctionCallingAgentContextData>,
     ev: StartEvent<AgentInput>,
-  ): Promise<InputEvent> {
-    const { message, streaming } = ev.data.input;
-    ctx.set("streaming", streaming);
+  ): Promise<InputEvent> => {
+    const { message, streaming } = ev.data;
+    ctx.data.streaming = streaming ?? false;
     this.writeEvent(`Start to work on: ${message}`, ctx);
     if (this.systemPrompt) {
       this.memory.put({ role: "system", content: this.systemPrompt });
     }
     this.memory.put({ role: "user", content: message });
     return new InputEvent({ input: this.chatHistory });
-  }
+  };
 
-  private async handleLLMInput(
-    ctx: Context,
+  handleLLMInput = async (
+    ctx: HandlerContext<FunctionCallingAgentContextData>,
     ev: InputEvent,
-  ): Promise<StopEvent<string | AsyncGenerator> | ToolCallEvent> {
-    if (ctx.get("streaming")) {
-      return await this.handleLLMInputStream(ctx, ev);
+  ): Promise<StopEvent<string | AsyncGenerator> | ToolCallEvent> => {
+    const toolCallResponse = await chatWithTools(
+      this.llm,
+      this.tools,
+      this.chatHistory,
+    );
+    if (toolCallResponse.toolCallMessage) {
+      this.memory.put(toolCallResponse.toolCallMessage);
     }
 
-    const result = await this.llm.chat({
-      messages: this.chatHistory,
-      tools: this.tools,
-    });
-    this.memory.put(result.message);
-
-    const toolCalls = this.getToolCallsFromResponse(result);
-    if (toolCalls.length) {
-      return new ToolCallEvent({ toolCalls });
+    if (toolCallResponse.hasToolCall()) {
+      return new ToolCallEvent({ toolCalls: toolCallResponse.toolCalls });
     }
-    this.writeEvent("Finished task", ctx);
-    return new StopEvent({ result: result.message.content.toString() });
-  }
 
-  private async handleLLMInputStream(
-    context: Context,
-    ev: InputEvent,
-  ): Promise<StopEvent<AsyncGenerator> | ToolCallEvent> {
-    const { llm, tools, memory } = this;
-    const llmArgs = { messages: this.chatHistory, tools };
-
-    const responseGenerator = async function* () {
-      const responseStream = await llm.chat({ ...llmArgs, stream: true });
-
-      let fullResponse = null;
-      let yieldedIndicator = false;
-      for await (const chunk of responseStream) {
-        const hasToolCalls = chunk.options && "toolCall" in chunk.options;
-        if (!hasToolCalls) {
-          if (!yieldedIndicator) {
-            yield false;
-            yieldedIndicator = true;
-          }
-          yield chunk;
-        } else if (!yieldedIndicator) {
-          yield true;
-          yieldedIndicator = true;
-        }
-
-        fullResponse = chunk;
+    if (ctx.data.streaming) {
+      if (!toolCallResponse.responseGenerator) {
+        throw new Error("No streaming response");
       }
-
-      if (fullResponse?.options && Object.keys(fullResponse.options).length) {
-        memory.put({
-          role: "assistant",
-          content: "",
-          options: fullResponse.options,
-        });
-        yield fullResponse;
-      }
-    };
-
-    const generator = responseGenerator();
-    const isToolCall = await generator.next();
-    if (isToolCall.value) {
-      const fullResponse = await generator.next();
-      const toolCalls = this.getToolCallsFromResponse(
-        fullResponse.value as ChatResponseChunk<ToolCallLLMMessageOptions>,
-      );
-      return new ToolCallEvent({ toolCalls });
+      return new StopEvent(toolCallResponse.responseGenerator);
     }
 
-    this.writeEvent("Finished task", context);
-    return new StopEvent({ result: generator });
-  }
+    const fullResponse = await toolCallResponse.asFullResponse();
+    this.memory.put(fullResponse);
+    return new StopEvent(fullResponse.content.toString());
+  };
 
-  private async handleToolCalls(
-    ctx: Context,
+  handleToolCalls = async (
+    ctx: HandlerContext<FunctionCallingAgentContextData>,
     ev: ToolCallEvent,
-  ): Promise<InputEvent> {
+  ): Promise<InputEvent> => {
     const { toolCalls } = ev.data;
 
-    const toolMsgs: ChatMessage[] = [];
-
-    for (const call of toolCalls) {
-      const targetTool = this.tools.find(
-        (tool) => tool.metadata.name === call.name,
-      );
-      // TODO: make logger optional in callTool in framework
-      const toolOutput = await callTool(targetTool, call, {
-        log: () => {},
-        error: (...args: unknown[]) => {
-          console.error(`[Tool ${call.name} Error]:`, ...args);
-        },
-        warn: () => {},
-      });
-      toolMsgs.push({
-        content: JSON.stringify(toolOutput.output),
-        role: "user",
-        options: {
-          toolResult: {
-            result: toolOutput.output,
-            isError: toolOutput.isError,
-            id: call.id,
-          },
-        },
-      });
-    }
+    const toolMsgs = await callTools({
+      tools: this.tools,
+      toolCalls,
+      ctx,
+      agentName: this.name,
+    });
 
     for (const msg of toolMsgs) {
       this.memory.put(msg);
     }
 
     return new InputEvent({ input: this.memory.getMessages() });
-  }
+  };
 
-  private writeEvent(msg: string, context: Context) {
+  writeEvent = (
+    msg: string,
+    ctx: HandlerContext<FunctionCallingAgentContextData>,
+  ) => {
     if (!this.writeEvents) return;
-    context.writeEventToStream({
-      data: new AgentRunEvent({ name: this.name, msg }),
-    });
-  }
-
-  private checkToolCallSupport() {
-    const { supportToolCall } = this.llm as ToolCallLLM;
-    if (!supportToolCall) throw new Error("LLM does not support tool calls");
-  }
-
-  private getToolCallsFromResponse(
-    response:
-      | ChatResponse<ToolCallLLMMessageOptions>
-      | ChatResponseChunk<ToolCallLLMMessageOptions>,
-  ): ToolCall[] {
-    let options;
-    if ("message" in response) {
-      options = response.message.options;
-    } else {
-      options = response.options;
-    }
-    if (options && "toolCall" in options) {
-      return options.toolCall as ToolCall[];
-    }
-    return [];
-  }
+    ctx.sendEvent(
+      new AgentRunEvent({ agent: this.name, text: msg, type: "text" }),
+    );
+  };
 }

templates/components/multiagent/typescript/workflow/stream.ts

@@ -1,65 +1,77 @@
-import { StopEvent } from "@llamaindex/core/workflow";
 import {
-  createCallbacksTransformer,
-  createStreamDataTransformer,
+  StopEvent,
+  WorkflowContext,
+  WorkflowEvent,
+} from "@llamaindex/workflow";
+import {
   StreamData,
+  createStreamDataTransformer,
   trimStartOfStreamHelper,
-  type AIStreamCallbacksAndOptions,
 } from "ai";
 import { ChatResponseChunk } from "llamaindex";
 import { AgentRunEvent } from "./type";
 
-export function toDataStream(
-  result: Promise<StopEvent<AsyncGenerator<ChatResponseChunk>>>,
-  callbacks?: AIStreamCallbacksAndOptions,
-) {
-  return toReadableStream(result)
-    .pipeThrough(createCallbacksTransformer(callbacks))
-    .pipeThrough(createStreamDataTransformer());
-}
-
-function toReadableStream(
-  result: Promise<StopEvent<AsyncGenerator<ChatResponseChunk>>>,
-) {
+export async function createStreamFromWorkflowContext<Input, Output, Context>(
+  context: WorkflowContext<Input, Output, Context>,
+): Promise<{ stream: ReadableStream<string>; dataStream: StreamData }> {
   const trimStartOfStream = trimStartOfStreamHelper();
-  return new ReadableStream<string>({
-    start(controller) {
-      controller.enqueue(""); // Kickstart the stream
+  const dataStream = new StreamData();
+  const encoder = new TextEncoder();
+  let generator: AsyncGenerator<ChatResponseChunk> | undefined;
+
+  const closeStreams = (controller: ReadableStreamDefaultController) => {
+    controller.close();
+    dataStream.close();
+  };
+
+  const mainStream = new ReadableStream({
+    async start(controller) {
+      // Kickstart the stream by sending an empty string
+      controller.enqueue(encoder.encode(""));
     },
-    async pull(controller): Promise<void> {
-      const stopEvent = await result;
-      const generator = stopEvent.data.result;
-      const { value, done } = await generator.next();
+    async pull(controller) {
+      while (!generator) {
+        // get next event from workflow context
+        const { value: event, done } =
+          await context[Symbol.asyncIterator]().next();
+        if (done) {
+          closeStreams(controller);
+          return;
+        }
+        generator = handleEvent(event, dataStream);
+      }
+
+      const { value: chunk, done } = await generator.next();
       if (done) {
-        controller.close();
+        closeStreams(controller);
         return;
       }
-
-      const text = trimStartOfStream(value.delta ?? "");
-      if (text) controller.enqueue(text);
+      const text = trimStartOfStream(chunk.delta ?? "");
+      if (text) {
+        controller.enqueue(encoder.encode(text));
+      }
     },
   });
+
+  return {
+    stream: mainStream.pipeThrough(createStreamDataTransformer()),
+    dataStream,
+  };
 }
 
-export async function workflowEventsToStreamData(
-  events: AsyncIterable<AgentRunEvent>,
-): Promise<StreamData> {
-  const streamData = new StreamData();
-
-  (async () => {
-    for await (const event of events) {
-      if (event instanceof AgentRunEvent) {
-        const { name, msg } = event.data;
-        if ((streamData as any).isClosed) {
-          break;
-        }
-        streamData.appendMessageAnnotation({
-          type: "agent",
-          data: { agent: name, text: msg },
-        });
-      }
-    }
-  })();
-
-  return streamData;
+function handleEvent(
+  event: WorkflowEvent<any>,
+  dataStream: StreamData,
+): AsyncGenerator<ChatResponseChunk> | undefined {
+  // Handle for StopEvent
+  if (event instanceof StopEvent) {
+    return event.data as AsyncGenerator<ChatResponseChunk>;
+  }
+  // Handle for AgentRunEvent
+  if (event instanceof AgentRunEvent) {
+    dataStream.appendMessageAnnotation({
+      type: "agent",
+      data: event.data,
+    });
+  }
 }

templates/components/multiagent/typescript/workflow/tools.ts

@@ -0,0 +1,342 @@
+import { HandlerContext } from "@llamaindex/workflow";
+import {
+  BaseToolWithCall,
+  callTool,
+  ChatMessage,
+  ChatResponse,
+  ChatResponseChunk,
+  LlamaCloudIndex,
+  PartialToolCall,
+  QueryEngineTool,
+  ToolCall,
+  ToolCallLLM,
+  ToolCallLLMMessageOptions,
+} from "llamaindex";
+import crypto from "node:crypto";
+import { getDataSource } from "../engine";
+import { AgentRunEvent } from "./type";
+
+export const getQueryEngineTools = async (): Promise<
+  QueryEngineTool[] | null
+> => {
+  const topK = process.env.TOP_K ? parseInt(process.env.TOP_K) : undefined;
+
+  const index = await getDataSource();
+  if (!index) {
+    return null;
+  }
+  // index is LlamaCloudIndex use two query engine tools
+  if (index instanceof LlamaCloudIndex) {
+    return [
+      new QueryEngineTool({
+        queryEngine: index.asQueryEngine({
+          similarityTopK: topK,
+          retrieval_mode: "files_via_content",
+        }),
+        metadata: {
+          name: "document_retriever",
+          description: `Document retriever that retrieves entire documents from the corpus.
+  ONLY use for research questions that may require searching over entire research reports.
+  Will be slower and more expensive than chunk-level retrieval but may be necessary.`,
+        },
+      }),
+      new QueryEngineTool({
+        queryEngine: index.asQueryEngine({
+          similarityTopK: topK,
+          retrieval_mode: "chunks",
+        }),
+        metadata: {
+          name: "chunk_retriever",
+          description: `Retrieves a small set of relevant document chunks from the corpus.
+      Use for research questions that want to look up specific facts from the knowledge corpus,
+      and need entire documents.`,
+        },
+      }),
+    ];
+  } else {
+    return [
+      new QueryEngineTool({
+        queryEngine: index.asQueryEngine({
+          similarityTopK: topK,
+        }),
+        metadata: {
+          name: "retriever",
+          description: `Use this tool to retrieve information about the text corpus from the index.`,
+        },
+      }),
+    ];
+  }
+};
+
+/**
+ * Call multiple tools and return the tool messages
+ */
+export const callTools = async <T>({
+  tools,
+  toolCalls,
+  ctx,
+  agentName,
+  writeEvent = true,
+}: {
+  toolCalls: ToolCall[];
+  tools: BaseToolWithCall[];
+  ctx: HandlerContext<T>;
+  agentName: string;
+  writeEvent?: boolean;
+}): Promise<ChatMessage[]> => {
+  const toolMsgs: ChatMessage[] = [];
+  if (toolCalls.length === 0) {
+    return toolMsgs;
+  }
+  if (toolCalls.length === 1) {
+    const tool = tools.find((tool) => tool.metadata.name === toolCalls[0].name);
+    if (!tool) {
+      throw new Error(`Tool ${toolCalls[0].name} not found`);
+    }
+    return [
+      await callSingleTool(
+        tool,
+        toolCalls[0],
+        writeEvent
+          ? (msg: string) => {
+              ctx.sendEvent(
+                new AgentRunEvent({
+                  agent: agentName,
+                  text: msg,
+                  type: "text",
+                }),
+              );
+            }
+          : undefined,
+      ),
+    ];
+  }
+  // Multiple tool calls, show events in progress
+  const progressId = crypto.randomUUID();
+  const totalSteps = toolCalls.length;
+  let currentStep = 0;
+  for (const toolCall of toolCalls) {
+    const tool = tools.find((tool) => tool.metadata.name === toolCall.name);
+    if (!tool) {
+      throw new Error(`Tool ${toolCall.name} not found`);
+    }
+    const toolMsg = await callSingleTool(tool, toolCall, (msg: string) => {
+      ctx.sendEvent(
+        new AgentRunEvent({
+          agent: agentName,
+          text: msg,
+          type: "progress",
+          data: {
+            id: progressId,
+            total: totalSteps,
+            current: currentStep,
+          },
+        }),
+      );
+      currentStep++;
+    });
+    toolMsgs.push(toolMsg);
+  }
+  return toolMsgs;
+};
+
+export const callSingleTool = async (
+  tool: BaseToolWithCall,
+  toolCall: ToolCall,
+  eventEmitter?: (msg: string) => void,
+): Promise<ChatMessage> => {
+  if (eventEmitter) {
+    eventEmitter(
+      `Calling tool ${toolCall.name} with input: ${JSON.stringify(toolCall.input)}`,
+    );
+  }
+
+  const toolOutput = await callTool(tool, toolCall, {
+    log: () => {},
+    error: (...args: unknown[]) => {
+      console.error(`Tool ${toolCall.name} got error:`, ...args);
+      if (eventEmitter) {
+        eventEmitter(`Tool ${toolCall.name} got error: ${args.join(" ")}`);
+      }
+      return {
+        content: JSON.stringify({
+          error: args.join(" "),
+        }),
+        role: "user",
+        options: {
+          toolResult: {
+            id: toolCall.id,
+            result: JSON.stringify({
+              error: args.join(" "),
+            }),
+            isError: true,
+          },
+        },
+      };
+    },
+    warn: () => {},
+  });
+
+  return {
+    content: JSON.stringify(toolOutput.output),
+    role: "user",
+    options: {
+      toolResult: {
+        result: toolOutput.output,
+        isError: toolOutput.isError,
+        id: toolCall.id,
+      },
+    },
+  };
+};
+
+class ChatWithToolsResponse {
+  toolCalls: ToolCall[];
+  toolCallMessage?: ChatMessage;
+  responseGenerator?: AsyncGenerator<ChatResponseChunk>;
+
+  constructor(options: {
+    toolCalls: ToolCall[];
+    toolCallMessage?: ChatMessage;
+    responseGenerator?: AsyncGenerator<ChatResponseChunk>;
+  }) {
+    this.toolCalls = options.toolCalls;
+    this.toolCallMessage = options.toolCallMessage;
+    this.responseGenerator = options.responseGenerator;
+  }
+
+  hasMultipleTools() {
+    const uniqueToolNames = new Set(this.getToolNames());
+    return uniqueToolNames.size > 1;
+  }
+
+  hasToolCall() {
+    return this.toolCalls.length > 0;
+  }
+
+  getToolNames() {
+    return this.toolCalls.map((toolCall) => toolCall.name);
+  }
+
+  async asFullResponse(): Promise<ChatMessage> {
+    if (!this.responseGenerator) {
+      throw new Error("No response generator");
+    }
+    let fullResponse = "";
+    for await (const chunk of this.responseGenerator) {
+      fullResponse += chunk.delta;
+    }
+    return {
+      role: "assistant",
+      content: fullResponse,
+    };
+  }
+}
+
+export const chatWithTools = async (
+  llm: ToolCallLLM,
+  tools: BaseToolWithCall[],
+  messages: ChatMessage[],
+): Promise<ChatWithToolsResponse> => {
+  const responseGenerator = async function* (): AsyncGenerator<
+    boolean | ChatResponseChunk,
+    void,
+    unknown
+  > {
+    const responseStream = await llm.chat({ messages, tools, stream: true });
+
+    let fullResponse = null;
+    let yieldedIndicator = false;
+    const toolCallMap = new Map();
+    for await (const chunk of responseStream) {
+      const hasToolCalls = chunk.options && "toolCall" in chunk.options;
+      if (!hasToolCalls) {
+        if (!yieldedIndicator) {
+          yield false;
+          yieldedIndicator = true;
+        }
+        yield chunk;
+      } else if (!yieldedIndicator) {
+        yield true;
+        yieldedIndicator = true;
+      }
+
+      if (chunk.options && "toolCall" in chunk.options) {
+        for (const toolCall of chunk.options.toolCall as PartialToolCall[]) {
+          if (toolCall.id) {
+            toolCallMap.set(toolCall.id, toolCall);
+          }
+        }
+      }
+
+      if (
+        hasToolCalls &&
+        (chunk.raw as any)?.choices?.[0]?.finish_reason !== null
+      ) {
+        // Update the fullResponse with the tool calls
+        const toolCalls = Array.from(toolCallMap.values());
+        fullResponse = {
+          ...chunk,
+          options: {
+            ...chunk.options,
+            toolCall: toolCalls,
+          },
+        };
+      }
+    }
+
+    if (fullResponse) {
+      yield fullResponse;
+    }
+  };
+
+  const generator = responseGenerator();
+  const isToolCall = await generator.next();
+
+  if (isToolCall.value) {
+    // If it's a tool call, we need to wait for the full response
+    let fullResponse = null;
+    for await (const chunk of generator) {
+      fullResponse = chunk;
+    }
+
+    if (fullResponse) {
+      const responseChunk = fullResponse as ChatResponseChunk;
+      const toolCalls = getToolCallsFromResponse(responseChunk);
+      return new ChatWithToolsResponse({
+        toolCalls,
+        toolCallMessage: {
+          options: responseChunk.options,
+          role: "assistant",
+          content: "",
+        },
+      });
+    } else {
+      throw new Error("Cannot get tool calls from response");
+    }
+  }
+
+  return new ChatWithToolsResponse({
+    toolCalls: [],
+    responseGenerator: generator as AsyncGenerator<ChatResponseChunk>,
+  });
+};
+
+export const getToolCallsFromResponse = (
+  response:
+    | ChatResponse<ToolCallLLMMessageOptions>
+    | ChatResponseChunk<ToolCallLLMMessageOptions>,
+): ToolCall[] => {
+  let options;
+
+  if ("message" in response) {
+    options = response.message.options;
+  } else {
+    options = response.options;
+  }
+
+  if (options && "toolCall" in options) {
+    return options.toolCall as ToolCall[];
+  }
+  return [];
+};

templates/components/multiagent/typescript/workflow/type.ts

@@ -1,11 +1,24 @@
-import { WorkflowEvent } from "@llamaindex/core/workflow";
+import { WorkflowEvent } from "@llamaindex/workflow";
+import { MessageContent } from "llamaindex";
 
 export type AgentInput = {
-  message: string;
+  message: MessageContent;
   streaming?: boolean;
 };
 
+export type AgentRunEventType = "text" | "progress";
+
+export type ProgressEventData = {
+  id: string;
+  total: number;
+  current: number;
+};
+
+export type AgentRunEventData = ProgressEventData;
+
 export class AgentRunEvent extends WorkflowEvent<{
-  name: string;
-  msg: string;
+  agent: string;
+  text: string;
+  type: AgentRunEventType;
+  data?: AgentRunEventData;
 }> {}

templates/components/settings/python/settings.py

@@ -21,6 +21,8 @@ def init_settings():
             init_mistral()
         case "azure-openai":
             init_azure_openai()
+        case "huggingface":
+            init_huggingface()
         case "t-systems":
             from .llmhub import init_llmhub
 
@@ -138,6 +140,42 @@ def init_fastembed():
     )
 
 
+def init_huggingface_embedding():
+    try:
+        from llama_index.embeddings.huggingface import HuggingFaceEmbedding
+    except ImportError:
+        raise ImportError(
+            "Hugging Face support is not installed. Please install it with `poetry add llama-index-embeddings-huggingface`"
+        )
+
+    embedding_model = os.getenv("EMBEDDING_MODEL", "all-MiniLM-L6-v2")
+    backend = os.getenv("EMBEDDING_BACKEND", "onnx")  # "torch", "onnx", or "openvino"
+    trust_remote_code = (
+        os.getenv("EMBEDDING_TRUST_REMOTE_CODE", "false").lower() == "true"
+    )
+
+    Settings.embed_model = HuggingFaceEmbedding(
+        model_name=embedding_model,
+        trust_remote_code=trust_remote_code,
+        backend=backend,
+    )
+
+
+def init_huggingface():
+    try:
+        from llama_index.llms.huggingface import HuggingFaceLLM
+    except ImportError:
+        raise ImportError(
+            "Hugging Face support is not installed. Please install it with `poetry add llama-index-llms-huggingface` and `poetry add llama-index-embeddings-huggingface`"
+        )
+
+    Settings.llm = HuggingFaceLLM(
+        model_name=os.getenv("MODEL"),
+        tokenizer_name=os.getenv("MODEL"),
+    )
+    init_huggingface_embedding()
+
+
 def init_groq():
     try:
         from llama_index.llms.groq import Groq

templates/types/streaming/express/package.json

@@ -20,19 +20,22 @@
     "dotenv": "^16.3.1",
     "duck-duck-scrape": "^2.2.5",
     "express": "^4.18.2",
-    "llamaindex": "0.7.10",
+    "llamaindex": "0.8.2",
     "pdf2json": "3.0.5",
     "ajv": "^8.12.0",
     "@e2b/code-interpreter": "0.0.9-beta.3",
     "got": "^14.4.1",
     "@apidevtools/swagger-parser": "^10.1.0",
     "formdata-node": "^6.0.3",
-    "marked": "^14.1.2"
+    "marked": "^14.1.2",
+    "papaparse": "^5.4.1"
   },
   "devDependencies": {
     "@types/cors": "^2.8.16",
     "@types/express": "^4.17.21",
     "@types/node": "^20.9.5",
+    "@llamaindex/workflow": "^0.0.3",
+    "@types/papaparse": "^5.3.15",
     "concurrently": "^8.2.2",
     "eslint": "^8.54.0",
     "eslint-config-prettier": "^8.10.0",

templates/types/streaming/fastapi/app/api/routers/models.py

@@ -60,9 +60,11 @@ class AnnotationFileData(BaseModel):
         # Include document IDs if it's available
         if file.refs is not None:
             default_content += f"Document IDs: {file.refs}\n"
-        # Include sandbox file path
+        # file path
         sandbox_file_path = f"/tmp/{file.name}"
+        local_file_path = f"output/uploaded/{file.name}"
         default_content += f"Sandbox file path (instruction: only use sandbox path for artifact or code interpreter tool): {sandbox_file_path}\n"
+        default_content += f"Local file path (instruction: Use for local tools: form filling, extractor): {local_file_path}\n"
         return default_content
 
     def to_llm_content(self) -> Optional[str]:
@@ -128,24 +130,29 @@ class ChatData(BaseModel):
 
     def get_last_message_content(self) -> str:
         """
-        Get the content of the last message along with the data content if available.
-        Fallback to use data content from previous messages
+        Get the content of the last message along with the data content from all user messages
         """
         if len(self.messages) == 0:
             raise ValueError("There is not any message in the chat")
+
         last_message = self.messages[-1]
         message_content = last_message.content
-        for message in reversed(self.messages):
+
+        # Collect annotation contents from all user messages
+        all_annotation_contents: List[str] = []
+        for message in self.messages:
             if message.role == MessageRole.USER and message.annotations is not None:
                 annotation_contents = filter(
                     None,
                     [annotation.to_content() for annotation in message.annotations],
                 )
-                if not annotation_contents:
-                    continue
-                annotation_text = "\n".join(annotation_contents)
-                message_content = f"{message_content}\n{annotation_text}"
-                break
+                all_annotation_contents.extend(annotation_contents)
+
+        # Add all annotation contents if any exist
+        if len(all_annotation_contents) > 0:
+            annotation_text = "\n".join(all_annotation_contents)
+            message_content = f"{message_content}\n{annotation_text}"
+
         return message_content
 
     def _get_agent_messages(self, max_messages: int = 10) -> List[str]:

templates/types/streaming/fastapi/app/services/file.py

@@ -6,7 +6,7 @@ import re
 import uuid
 from io import BytesIO
 from pathlib import Path
-from typing import Dict, List, Optional, Tuple
+from typing import List, Optional, Tuple
 
 from llama_index.core import VectorStoreIndex
 from llama_index.core.ingestion import IngestionPipeline
@@ -14,7 +14,6 @@ from llama_index.core.readers.file.base import (
     _try_loading_included_file_formats as get_file_loaders_map,
 )
 from llama_index.core.schema import Document
-from llama_index.core.tools.function_tool import FunctionTool
 from llama_index.indices.managed.llama_cloud.base import LlamaCloudIndex
 from llama_index.readers.file import FlatReader
 from pydantic import BaseModel, Field
@@ -78,10 +77,8 @@ class FileService:
             save_dir=PRIVATE_STORE_PATH,
         )
 
-        tools = _get_available_tools()
-        code_executor_tools = ["interpreter", "artifact"]
-        # If the file is CSV and there is a code executor tool, we don't need to index.
-        if extension == "csv" and any(tool in tools for tool in code_executor_tools):
+        # Don't index csv files (they are handled by tools)
+        if extension == "csv":
             return document_file
         else:
             # Insert the file into the index and update document ids to the file metadata
@@ -283,18 +280,3 @@ def _default_file_loaders_map():
     default_loaders[".txt"] = FlatReader
     default_loaders[".csv"] = FlatReader
     return default_loaders
-
-
-def _get_available_tools() -> Dict[str, List[FunctionTool]]:
-    try:
-        from app.engine.tools import ToolFactory  # type: ignore
-    except ImportError:
-        logger.warning("ToolFactory not found, no tools will be available")
-        return {}
-
-    try:
-        tools = ToolFactory.from_env(map_result=True)
-        return tools  # type: ignore
-    except Exception as e:
-        logger.error(f"Error loading tools from environment: {str(e)}")
-        raise ValueError(f"Failed to get available tools: {str(e)}") from e

templates/types/streaming/nextjs/app/components/chat-section.tsx

@@ -1,57 +1,27 @@
 "use client";
 
+import { ChatSection as ChatSectionUI } from "@llamaindex/chat-ui";
+import "@llamaindex/chat-ui/styles/code.css";
+import "@llamaindex/chat-ui/styles/katex.css";
+import "@llamaindex/chat-ui/styles/pdf.css";
 import { useChat } from "ai/react";
-import { useState } from "react";
-import { ChatInput, ChatMessages } from "./ui/chat";
+import CustomChatInput from "./ui/chat/chat-input";
+import CustomChatMessages from "./ui/chat/chat-messages";
 import { useClientConfig } from "./ui/chat/hooks/use-config";
 
 export default function ChatSection() {
   const { backend } = useClientConfig();
-  const [requestData, setRequestData] = useState<any>();
-  const {
-    messages,
-    input,
-    isLoading,
-    handleSubmit,
-    handleInputChange,
-    reload,
-    stop,
-    append,
-    setInput,
-  } = useChat({
-    body: { data: requestData },
+  const handler = useChat({
     api: `${backend}/api/chat`,
-    headers: {
-      "Content-Type": "application/json", // using JSON because of vercel/ai 2.2.26
-    },
     onError: (error: unknown) => {
       if (!(error instanceof Error)) throw error;
-      const message = JSON.parse(error.message);
-      alert(message.detail);
+      alert(JSON.parse(error.message).detail);
     },
-    sendExtraMessageFields: true,
   });
-
   return (
-    <div className="space-y-4 w-full h-full flex flex-col">
-      <ChatMessages
-        messages={messages}
-        isLoading={isLoading}
-        reload={reload}
-        stop={stop}
-        append={append}
-      />
-      <ChatInput
-        input={input}
-        handleSubmit={handleSubmit}
-        handleInputChange={handleInputChange}
-        isLoading={isLoading}
-        messages={messages}
-        append={append}
-        setInput={setInput}
-        requestParams={{ params: requestData }}
-        setRequestData={setRequestData}
-      />
-    </div>
+    <ChatSectionUI handler={handler} className="w-full h-full">
+      <CustomChatMessages />
+      <CustomChatInput />
+    </ChatSectionUI>
   );
 }

templates/types/streaming/nextjs/app/components/ui/README-template.md

@@ -1 +0,0 @@
-Using the chat component from https://github.com/marcusschiesser/ui (based on https://ui.shadcn.com/)

templates/types/streaming/nextjs/app/components/ui/chat/chat-actions.tsx

@@ -1,28 +0,0 @@
-import { PauseCircle, RefreshCw } from "lucide-react";
-
-import { Button } from "../button";
-import { ChatHandler } from "./chat.interface";
-
-export default function ChatActions(
-  props: Pick<ChatHandler, "stop" | "reload"> & {
-    showReload?: boolean;
-    showStop?: boolean;
-  },
-) {
-  return (
-    <div className="space-x-4">
-      {props.showStop && (
-        <Button variant="outline" size="sm" onClick={props.stop}>
-          <PauseCircle className="mr-2 h-4 w-4" />
-          Stop generating
-        </Button>
-      )}
-      {props.showReload && (
-        <Button variant="outline" size="sm" onClick={props.reload}>
-          <RefreshCw className="mr-2 h-4 w-4" />
-          Regenerate
-        </Button>
-      )}
-    </div>
-  );
-}

templates/types/streaming/nextjs/app/components/ui/chat/chat-avatar.tsx

@@ -1,8 +1,10 @@
+import { useChatMessage } from "@llamaindex/chat-ui";
 import { User2 } from "lucide-react";
 import Image from "next/image";
 
-export default function ChatAvatar({ role }: { role: string }) {
-  if (role === "user") {
+export function ChatMessageAvatar() {
+  const { message } = useChatMessage();
+  if (message.role === "user") {
     return (
       <div className="flex h-8 w-8 shrink-0 select-none items-center justify-center rounded-md border bg-background shadow">
         <User2 className="h-4 w-4" />

templates/types/streaming/nextjs/app/components/ui/chat/chat-input.tsx

@@ -1,34 +1,13 @@
-import { JSONValue } from "ai";
-import React from "react";
-import { DocumentFile } from ".";
-import { Button } from "../button";
-import { DocumentPreview } from "../document-preview";
-import FileUploader from "../file-uploader";
-import { Textarea } from "../textarea";
-import UploadImagePreview from "../upload-image-preview";
-import { ChatHandler } from "./chat.interface";
-import { useFile } from "./hooks/use-file";
-import { LlamaCloudSelector } from "./widgets/LlamaCloudSelector";
+"use client";
 
-const ALLOWED_EXTENSIONS = ["png", "jpg", "jpeg", "csv", "pdf", "txt", "docx"];
+import { ChatInput, useChatUI, useFile } from "@llamaindex/chat-ui";
+import { DocumentPreview, ImagePreview } from "@llamaindex/chat-ui/widgets";
+import { LlamaCloudSelector } from "./custom/llama-cloud-selector";
+import { useClientConfig } from "./hooks/use-config";
 
-export default function ChatInput(
-  props: Pick<
-    ChatHandler,
-    | "isLoading"
-    | "input"
-    | "onFileUpload"
-    | "onFileError"
-    | "handleSubmit"
-    | "handleInputChange"
-    | "messages"
-    | "setInput"
-    | "append"
-  > & {
-    requestParams?: any;
-    setRequestData?: React.Dispatch<any>;
-  },
-) {
+export default function CustomChatInput() {
+  const { requestData, isLoading, input } = useChatUI();
+  const { backend } = useClientConfig();
   const {
     imageUrl,
     setImageUrl,
@@ -37,101 +16,65 @@ export default function ChatInput(
     removeDoc,
     reset,
     getAnnotations,
-  } = useFile();
-
-  // default submit function does not handle including annotations in the message
-  // so we need to use append function to submit new message with annotations
-  const handleSubmitWithAnnotations = (
-    e: React.FormEvent<HTMLFormElement>,
-    annotations: JSONValue[] | undefined,
-  ) => {
-    e.preventDefault();
-    props.append!({
-      content: props.input,
-      role: "user",
-      createdAt: new Date(),
-      annotations,
-    });
-    props.setInput!("");
-  };
-
-  const onSubmit = (e: React.FormEvent<HTMLFormElement>) => {
-    e.preventDefault();
-    const annotations = getAnnotations();
-    if (annotations.length) {
-      handleSubmitWithAnnotations(e, annotations);
-      return reset();
-    }
-    props.handleSubmit(e);
-  };
+  } = useFile({ uploadAPI: `${backend}/api/chat/upload` });
 
+  /**
+   * Handles file uploads. Overwrite to hook into the file upload behavior.
+   * @param file The file to upload
+   */
   const handleUploadFile = async (file: File) => {
+    // There's already an image uploaded, only allow one image at a time
     if (imageUrl) {
       alert("You can only upload one image at a time.");
       return;
     }
+
     try {
-      await uploadFile(file, props.requestParams);
-      props.onFileUpload?.(file);
+      // Upload the file and send with it the current request data
+      await uploadFile(file, requestData);
     } catch (error: any) {
-      const onFileUploadError = props.onFileError || window.alert;
-      onFileUploadError(error.message);
+      // Show error message if upload fails
+      alert(error.message);
     }
   };
 
-  const handleKeyDown = (e: React.KeyboardEvent<HTMLTextAreaElement>) => {
-    if (e.key === "Enter" && !e.shiftKey) {
-      e.preventDefault();
-      onSubmit(e as unknown as React.FormEvent<HTMLFormElement>);
-    }
-  };
+  // Get references to the upload files in message annotations format, see https://github.com/run-llama/chat-ui/blob/main/packages/chat-ui/src/hook/use-file.tsx#L56
+  const annotations = getAnnotations();
 
   return (
-    <form
-      onSubmit={onSubmit}
-      className="rounded-xl bg-white p-4 shadow-xl space-y-4 shrink-0"
+    <ChatInput
+      className="shadow-xl rounded-xl"
+      resetUploadedFiles={reset}
+      annotations={annotations}
     >
-      {imageUrl && (
-        <UploadImagePreview url={imageUrl} onRemove={() => setImageUrl(null)} />
-      )}
-      {files.length > 0 && (
-        <div className="flex gap-4 w-full overflow-auto py-2">
-          {files.map((file: DocumentFile) => (
-            <DocumentPreview
-              key={file.id}
-              file={file}
-              onRemove={() => removeDoc(file)}
-            />
-          ))}
-        </div>
-      )}
-      <div className="flex w-full items-start justify-between gap-4 ">
-        <Textarea
-          id="chat-input"
-          autoFocus
-          name="message"
-          placeholder="Type a message"
-          className="flex-1 min-h-0 h-[40px]"
-          value={props.input}
-          onChange={props.handleInputChange}
-          onKeyDown={handleKeyDown}
-        />
-        <FileUploader
-          onFileUpload={handleUploadFile}
-          onFileError={props.onFileError}
-          config={{
-            allowedExtensions: ALLOWED_EXTENSIONS,
-            disabled: props.isLoading,
-          }}
-        />
-        {process.env.NEXT_PUBLIC_USE_LLAMACLOUD === "true" &&
-          props.setRequestData && (
-            <LlamaCloudSelector setRequestData={props.setRequestData} />
-          )}
-        <Button type="submit" disabled={props.isLoading || !props.input.trim()}>
-          Send message
-        </Button>
+      <div>
+        {/* Image preview section */}
+        {imageUrl && (
+          <ImagePreview url={imageUrl} onRemove={() => setImageUrl(null)} />
+        )}
+        {/* Document previews section */}
+        {files.length > 0 && (
+          <div className="flex gap-4 w-full overflow-auto py-2">
+            {files.map((file) => (
+              <DocumentPreview
+                key={file.id}
+                file={file}
+                onRemove={() => removeDoc(file)}
+              />
+            ))}
+          </div>
+        )}
       </div>
-    </form>
+      <ChatInput.Form>
+        <ChatInput.Field />
+        <ChatInput.Upload onUpload={handleUploadFile} />
+        <LlamaCloudSelector />
+        <ChatInput.Submit
+          disabled={
+            isLoading || (!input.trim() && files.length === 0 && !imageUrl)
+          }
+        />
+      </ChatInput.Form>
+    </ChatInput>
   );
 }

templates/types/streaming/nextjs/app/components/ui/chat/chat-message-content.tsx

@@ -0,0 +1,30 @@
+import {
+  ChatMessage,
+  ContentPosition,
+  getSourceAnnotationData,
+  useChatMessage,
+} from "@llamaindex/chat-ui";
+import { Markdown } from "./custom/markdown";
+import { ToolAnnotations } from "./tools/chat-tools";
+
+export function ChatMessageContent() {
+  const { message } = useChatMessage();
+  const customContent = [
+    {
+      // override the default markdown component
+      position: ContentPosition.MARKDOWN,
+      component: (
+        <Markdown
+          content={message.content}
+          sources={getSourceAnnotationData(message.annotations)?.[0]}
+        />
+      ),
+    },
+    {
+      // add the tool annotations after events
+      position: ContentPosition.AFTER_EVENTS,
+      component: <ToolAnnotations message={message} />,
+    },
+  ];
+  return <ChatMessage.Content content={customContent} />;
+}

templates/types/streaming/nextjs/app/components/ui/chat/chat-message/chat-agent-events.tsx

@@ -1,153 +0,0 @@
-import { icons, LucideIcon } from "lucide-react";
-import { useMemo } from "react";
-import { Button } from "../../button";
-import {
-  Drawer,
-  DrawerClose,
-  DrawerContent,
-  DrawerHeader,
-  DrawerTitle,
-  DrawerTrigger,
-} from "../../drawer";
-import { AgentEventData } from "../index";
-import Markdown from "./markdown";
-
-const AgentIcons: Record<string, LucideIcon> = {
-  bot: icons.Bot,
-  researcher: icons.ScanSearch,
-  writer: icons.PenLine,
-  reviewer: icons.MessageCircle,
-  publisher: icons.BookCheck,
-};
-
-type MergedEvent = {
-  agent: string;
-  texts: string[];
-  icon: LucideIcon;
-};
-
-export function ChatAgentEvents({
-  data,
-  isFinished,
-}: {
-  data: AgentEventData[];
-  isFinished: boolean;
-}) {
-  const events = useMemo(() => mergeAdjacentEvents(data), [data]);
-  return (
-    <div className="pl-2">
-      <div className="text-sm space-y-4">
-        {events.map((eventItem, index) => (
-          <AgentEventContent
-            key={index}
-            event={eventItem}
-            isLast={index === events.length - 1}
-            isFinished={isFinished}
-          />
-        ))}
-      </div>
-    </div>
-  );
-}
-
-const MAX_TEXT_LENGTH = 150;
-
-function AgentEventContent({
-  event,
-  isLast,
-  isFinished,
-}: {
-  event: MergedEvent;
-  isLast: boolean;
-  isFinished: boolean;
-}) {
-  const { agent, texts } = event;
-  const AgentIcon = event.icon;
-  return (
-    <div className="flex gap-4 border-b pb-4 items-center fadein-agent">
-      <div className="w-[100px] flex flex-col items-center gap-2">
-        <div className="relative">
-          {isLast && !isFinished && (
-            <div className="absolute -top-0 -right-4">
-              <span className="relative flex h-3 w-3">
-                <span className="animate-ping absolute inline-flex h-full w-full rounded-full bg-sky-400 opacity-75"></span>
-                <span className="relative inline-flex rounded-full h-3 w-3 bg-sky-500"></span>
-              </span>
-            </div>
-          )}
-          <AgentIcon />
-        </div>
-        <span className="font-bold">{agent}</span>
-      </div>
-      <ul className="flex-1 list-decimal space-y-2">
-        {texts.map((text, index) => (
-          <li className="whitespace-break-spaces" key={index}>
-            {text.length <= MAX_TEXT_LENGTH && <span>{text}</span>}
-            {text.length > MAX_TEXT_LENGTH && (
-              <div>
-                <span>{text.slice(0, MAX_TEXT_LENGTH)}...</span>
-                <AgentEventDialog
-                  content={text}
-                  title={`Agent "${agent}" - Step: ${index + 1}`}
-                >
-                  <span className="font-semibold underline cursor-pointer ml-2">
-                    Show more
-                  </span>
-                </AgentEventDialog>
-              </div>
-            )}
-          </li>
-        ))}
-      </ul>
-    </div>
-  );
-}
-
-type AgentEventDialogProps = {
-  title: string;
-  content: string;
-  children: React.ReactNode;
-};
-
-function AgentEventDialog(props: AgentEventDialogProps) {
-  return (
-    <Drawer direction="left">
-      <DrawerTrigger asChild>{props.children}</DrawerTrigger>
-      <DrawerContent className="w-3/5 mt-24 h-full max-h-[96%] ">
-        <DrawerHeader className="flex justify-between">
-          <div className="space-y-2">
-            <DrawerTitle>{props.title}</DrawerTitle>
-          </div>
-          <DrawerClose asChild>
-            <Button variant="outline">Close</Button>
-          </DrawerClose>
-        </DrawerHeader>
-        <div className="m-4 overflow-auto">
-          <Markdown content={props.content} />
-        </div>
-      </DrawerContent>
-    </Drawer>
-  );
-}
-
-function mergeAdjacentEvents(events: AgentEventData[]): MergedEvent[] {
-  const mergedEvents: MergedEvent[] = [];
-
-  for (const event of events) {
-    const lastMergedEvent = mergedEvents[mergedEvents.length - 1];
-
-    if (lastMergedEvent && lastMergedEvent.agent === event.agent) {
-      // If the last event in mergedEvents has the same non-null agent, add the title to it
-      lastMergedEvent.texts.push(event.text);
-    } else {
-      // Otherwise, create a new merged event
-      mergedEvents.push({
-        agent: event.agent,
-        texts: [event.text],
-        icon: AgentIcons[event.agent] ?? icons.Bot,
-      });
-    }
-  }
-
-  return mergedEvents;
-}

templates/types/streaming/nextjs/app/components/ui/chat/chat-message/chat-events.tsx

@@ -1,50 +0,0 @@
-import { ChevronDown, ChevronRight, Loader2 } from "lucide-react";
-import { useState } from "react";
-import { Button } from "../../button";
-import {
-  Collapsible,
-  CollapsibleContent,
-  CollapsibleTrigger,
-} from "../../collapsible";
-import { EventData } from "../index";
-
-export function ChatEvents({
-  data,
-  isLoading,
-}: {
-  data: EventData[];
-  isLoading: boolean;
-}) {
-  const [isOpen, setIsOpen] = useState(false);
-
-  const buttonLabel = isOpen ? "Hide events" : "Show events";
-
-  const EventIcon = isOpen ? (
-    <ChevronDown className="h-4 w-4" />
-  ) : (
-    <ChevronRight className="h-4 w-4" />
-  );
-
-  return (
-    <div className="border-l-2 border-indigo-400 pl-2">
-      <Collapsible open={isOpen} onOpenChange={setIsOpen}>
-        <CollapsibleTrigger asChild>
-          <Button variant="secondary" className="space-x-2">
-            {isLoading ? <Loader2 className="h-4 w-4 animate-spin" /> : null}
-            <span>{buttonLabel}</span>
-            {EventIcon}
-          </Button>
-        </CollapsibleTrigger>
-        <CollapsibleContent asChild>
-          <div className="mt-4 text-sm space-y-2">
-            {data.map((eventItem, index) => (
-              <div className="whitespace-break-spaces" key={index}>
-                {eventItem.title}
-              </div>
-            ))}
-          </div>
-        </CollapsibleContent>
-      </Collapsible>
-    </div>
-  );
-}

templates/types/streaming/nextjs/app/components/ui/chat/chat-message/chat-files.tsx

@@ -1,13 +0,0 @@
-import { DocumentPreview } from "../../document-preview";
-import { DocumentFileData } from "../index";
-
-export function ChatFiles({ data }: { data: DocumentFileData }) {
-  if (!data.files.length) return null;
-  return (
-    <div className="flex gap-2 items-center">
-      {data.files.map((file, index) => (
-        <DocumentPreview key={file.id} file={file} />
-      ))}
-    </div>
-  );
-}

templates/types/streaming/nextjs/app/components/ui/chat/chat-message/chat-image.tsx

@@ -1,17 +0,0 @@
-import Image from "next/image";
-import { type ImageData } from "../index";
-
-export function ChatImage({ data }: { data: ImageData }) {
-  return (
-    <div className="rounded-md max-w-[200px] shadow-md">
-      <Image
-        src={data.url}
-        width={0}
-        height={0}
-        sizes="100vw"
-        style={{ width: "100%", height: "auto" }}
-        alt=""
-      />
-    </div>
-  );
-}

templates/types/streaming/nextjs/app/components/ui/chat/chat-message/chat-sources.tsx

@@ -1,173 +0,0 @@
-import { Check, Copy } from "lucide-react";
-import { useMemo } from "react";
-import { Button } from "../../button";
-import { PreviewCard } from "../../document-preview";
-import {
-  HoverCard,
-  HoverCardContent,
-  HoverCardTrigger,
-} from "../../hover-card";
-import { cn } from "../../lib/utils";
-import { useCopyToClipboard } from "../hooks/use-copy-to-clipboard";
-import { DocumentFileType, SourceData, SourceNode } from "../index";
-import PdfDialog from "../widgets/PdfDialog";
-
-type Document = {
-  url: string;
-  sources: SourceNode[];
-};
-
-export function ChatSources({ data }: { data: SourceData }) {
-  const documents: Document[] = useMemo(() => {
-    // group nodes by document (a document must have a URL)
-    const nodesByUrl: Record<string, SourceNode[]> = {};
-    data.nodes.forEach((node) => {
-      const key = node.url;
-      nodesByUrl[key] ??= [];
-      nodesByUrl[key].push(node);
-    });
-
-    // convert to array of documents
-    return Object.entries(nodesByUrl).map(([url, sources]) => ({
-      url,
-      sources,
-    }));
-  }, [data.nodes]);
-
-  if (documents.length === 0) return null;
-
-  return (
-    <div className="space-y-2 text-sm">
-      <div className="font-semibold text-lg">Sources:</div>
-      <div className="flex gap-3 flex-wrap">
-        {documents.map((document) => {
-          return <DocumentInfo key={document.url} document={document} />;
-        })}
-      </div>
-    </div>
-  );
-}
-
-function SourceInfo({ node, index }: { node?: SourceNode; index: number }) {
-  if (!node) return <SourceNumberButton index={index} />;
-  return (
-    <HoverCard>
-      <HoverCardTrigger
-        className="cursor-default"
-        onClick={(e) => {
-          e.preventDefault();
-          e.stopPropagation();
-        }}
-      >
-        <SourceNumberButton
-          index={index}
-          className="hover:text-white hover:bg-primary"
-        />
-      </HoverCardTrigger>
-      <HoverCardContent className="w-[400px]">
-        <NodeInfo nodeInfo={node} />
-      </HoverCardContent>
-    </HoverCard>
-  );
-}
-
-export function SourceNumberButton({
-  index,
-  className,
-}: {
-  index: number;
-  className?: string;
-}) {
-  return (
-    <span
-      className={cn(
-        "text-xs w-5 h-5 rounded-full bg-gray-100 inline-flex items-center justify-center",
-        className,
-      )}
-    >
-      {index + 1}
-    </span>
-  );
-}
-
-export function DocumentInfo({
-  document,
-  className,
-}: {
-  document: Document;
-  className?: string;
-}) {
-  const { url, sources } = document;
-  // Extract filename from URL
-  const urlParts = url.split("/");
-  const fileName = urlParts.length > 0 ? urlParts[urlParts.length - 1] : url;
-  const fileExt = fileName?.split(".").pop() as DocumentFileType | undefined;
-
-  const previewFile = {
-    name: fileName,
-    type: fileExt as DocumentFileType,
-  };
-
-  const DocumentDetail = (
-    <div className={`relative ${className}`}>
-      <PreviewCard className={"cursor-pointer"} file={previewFile} />
-      <div className="absolute bottom-2 right-2 space-x-2 flex">
-        {sources.map((node: SourceNode, index: number) => (
-          <div key={node.id}>
-            <SourceInfo node={node} index={index} />
-          </div>
-        ))}
-      </div>
-    </div>
-  );
-
-  if (url.endsWith(".pdf")) {
-    // open internal pdf dialog for pdf files when click document card
-    return <PdfDialog documentId={url} url={url} trigger={DocumentDetail} />;
-  }
-  // open external link when click document card for other file types
-  return <div onClick={() => window.open(url, "_blank")}>{DocumentDetail}</div>;
-}
-
-function NodeInfo({ nodeInfo }: { nodeInfo: SourceNode }) {
-  const { isCopied, copyToClipboard } = useCopyToClipboard({ timeout: 1000 });
-
-  const pageNumber =
-    // XXX: page_label is used in Python, but page_number is used by Typescript
-    (nodeInfo.metadata?.page_number as number) ??
-    (nodeInfo.metadata?.page_label as number) ??
-    null;
-
-  return (
-    <div className="space-y-4">
-      <div className="flex justify-between items-center">
-        <span className="font-semibold">
-          {pageNumber ? `On page ${pageNumber}:` : "Node content:"}
-        </span>
-        {nodeInfo.text && (
-          <Button
-            onClick={(e) => {
-              e.stopPropagation();
-              copyToClipboard(nodeInfo.text);
-            }}
-            size="icon"
-            variant="ghost"
-            className="h-12 w-12 shrink-0"
-          >
-            {isCopied ? (
-              <Check className="h-4 w-4" />
-            ) : (
-              <Copy className="h-4 w-4" />
-            )}
-          </Button>
-        )}
-      </div>
-
-      {nodeInfo.text && (
-        <pre className="max-h-[200px] overflow-auto whitespace-pre-line">
-          &ldquo;{nodeInfo.text}&rdquo;
-        </pre>
-      )}
-    </div>
-  );
-}

templates/types/streaming/nextjs/app/components/ui/chat/chat-message/chat-suggestedQuestions.tsx

@@ -1,31 +0,0 @@
-import { ChatHandler, SuggestedQuestionsData } from "..";
-
-export function SuggestedQuestions({
-  questions,
-  append,
-  isLastMessage,
-}: {
-  questions: SuggestedQuestionsData;
-  append: Pick<ChatHandler, "append">["append"];
-  isLastMessage: boolean;
-}) {
-  const showQuestions = isLastMessage && questions.length > 0;
-  return (
-    showQuestions &&
-    append !== undefined && (
-      <div className="flex flex-col space-y-2">
-        {questions.map((question, index) => (
-          <a
-            key={index}
-            onClick={() => {
-              append({ role: "user", content: question });
-            }}
-            className="text-sm italic hover:underline cursor-pointer"
-          >
-            {"->"} {question}
-          </a>
-        ))}
-      </div>
-    )
-  );
-}

templates/types/streaming/nextjs/app/components/ui/chat/chat-message/chat-tools.tsx

@@ -1,40 +0,0 @@
-import { ToolData } from "../index";
-import { Artifact, CodeArtifact } from "../widgets/Artifact";
-import { WeatherCard, WeatherData } from "../widgets/WeatherCard";
-
-// TODO: If needed, add displaying more tool outputs here
-export default function ChatTools({
-  data,
-  artifactVersion,
-}: {
-  data: ToolData;
-  artifactVersion?: number;
-}) {
-  if (!data) return null;
-  const { toolCall, toolOutput } = data;
-
-  if (toolOutput.isError) {
-    return (
-      <div className="border-l-2 border-red-400 pl-2">
-        There was an error when calling the tool {toolCall.name} with input:{" "}
-        <br />
-        {JSON.stringify(toolCall.input)}
-      </div>
-    );
-  }
-
-  switch (toolCall.name) {
-    case "get_weather_information":
-      const weatherData = toolOutput.output as unknown as WeatherData;
-      return <WeatherCard data={weatherData} />;
-    case "artifact":
-      return (
-        <Artifact
-          artifact={toolOutput.output as CodeArtifact}
-          version={artifactVersion}
-        />
-      );
-    default:
-      return null;
-  }
-}

templates/types/streaming/nextjs/app/components/ui/chat/chat-message/codeblock.tsx

@@ -1,131 +0,0 @@
-"use client";
-
-import hljs from "highlight.js";
-// instead of atom-one-dark theme, there are a lot of others: https://highlightjs.org/demo
-import "highlight.js/styles/atom-one-dark-reasonable.css";
-import { Check, Copy, Download } from "lucide-react";
-import { FC, memo, useEffect, useRef } from "react";
-import { Button } from "../../button";
-import { useCopyToClipboard } from "../hooks/use-copy-to-clipboard";
-
-interface Props {
-  language: string;
-  value: string;
-  className?: string;
-}
-
-interface languageMap {
-  [key: string]: string | undefined;
-}
-
-export const programmingLanguages: languageMap = {
-  javascript: ".js",
-  python: ".py",
-  java: ".java",
-  c: ".c",
-  cpp: ".cpp",
-  "c++": ".cpp",
-  "c#": ".cs",
-  ruby: ".rb",
-  php: ".php",
-  swift: ".swift",
-  "objective-c": ".m",
-  kotlin: ".kt",
-  typescript: ".ts",
-  go: ".go",
-  perl: ".pl",
-  rust: ".rs",
-  scala: ".scala",
-  haskell: ".hs",
-  lua: ".lua",
-  shell: ".sh",
-  sql: ".sql",
-  html: ".html",
-  css: ".css",
-  // add more file extensions here, make sure the key is same as language prop in CodeBlock.tsx component
-};
-
-export const generateRandomString = (length: number, lowercase = false) => {
-  const chars = "ABCDEFGHJKLMNPQRSTUVWXY3456789"; // excluding similar looking characters like Z, 2, I, 1, O, 0
-  let result = "";
-  for (let i = 0; i < length; i++) {
-    result += chars.charAt(Math.floor(Math.random() * chars.length));
-  }
-  return lowercase ? result.toLowerCase() : result;
-};
-
-const CodeBlock: FC<Props> = memo(({ language, value, className }) => {
-  const { isCopied, copyToClipboard } = useCopyToClipboard({ timeout: 2000 });
-  const codeRef = useRef<HTMLElement>(null);
-
-  useEffect(() => {
-    if (codeRef.current && codeRef.current.dataset.highlighted !== "yes") {
-      hljs.highlightElement(codeRef.current);
-    }
-  }, [language, value]);
-
-  const downloadAsFile = () => {
-    if (typeof window === "undefined") {
-      return;
-    }
-    const fileExtension = programmingLanguages[language] || ".file";
-    const suggestedFileName = `file-${generateRandomString(
-      3,
-      true,
-    )}${fileExtension}`;
-    const fileName = window.prompt("Enter file name", suggestedFileName);
-
-    if (!fileName) {
-      // User pressed cancel on prompt.
-      return;
-    }
-
-    const blob = new Blob([value], { type: "text/plain" });
-    const url = URL.createObjectURL(blob);
-    const link = document.createElement("a");
-    link.download = fileName;
-    link.href = url;
-    link.style.display = "none";
-    document.body.appendChild(link);
-    link.click();
-    document.body.removeChild(link);
-    URL.revokeObjectURL(url);
-  };
-
-  const onCopy = () => {
-    if (isCopied) return;
-    copyToClipboard(value);
-  };
-
-  return (
-    <div
-      className={`codeblock relative w-full bg-zinc-950 font-sans ${className}`}
-    >
-      <div className="flex w-full items-center justify-between bg-zinc-800 px-6 py-2 pr-4 text-zinc-100">
-        <span className="text-xs lowercase">{language}</span>
-        <div className="flex items-center space-x-1">
-          <Button variant="ghost" onClick={downloadAsFile} size="icon">
-            <Download />
-            <span className="sr-only">Download</span>
-          </Button>
-          <Button variant="ghost" size="icon" onClick={onCopy}>
-            {isCopied ? (
-              <Check className="h-4 w-4" />
-            ) : (
-              <Copy className="h-4 w-4" />
-            )}
-            <span className="sr-only">Copy code</span>
-          </Button>
-        </div>
-      </div>
-      <pre className="border border-zinc-700">
-        <code ref={codeRef} className={`language-${language} font-mono`}>
-          {value}
-        </code>
-      </pre>
-    </div>
-  );
-});
-CodeBlock.displayName = "CodeBlock";
-
-export { CodeBlock };

templates/types/streaming/nextjs/app/components/ui/chat/chat-message/index.tsx

@@ -1,184 +0,0 @@
-import { Check, Copy } from "lucide-react";
-
-import { Message } from "ai";
-import { Fragment } from "react";
-import { Button } from "../../button";
-import { useCopyToClipboard } from "../hooks/use-copy-to-clipboard";
-import {
-  AgentEventData,
-  ChatHandler,
-  DocumentFileData,
-  EventData,
-  ImageData,
-  MessageAnnotation,
-  MessageAnnotationType,
-  SuggestedQuestionsData,
-  ToolData,
-  getAnnotationData,
-  getSourceAnnotationData,
-} from "../index";
-import { ChatAgentEvents } from "./chat-agent-events";
-import ChatAvatar from "./chat-avatar";
-import { ChatEvents } from "./chat-events";
-import { ChatFiles } from "./chat-files";
-import { ChatImage } from "./chat-image";
-import { ChatSources } from "./chat-sources";
-import { SuggestedQuestions } from "./chat-suggestedQuestions";
-import ChatTools from "./chat-tools";
-import Markdown from "./markdown";
-
-type ContentDisplayConfig = {
-  order: number;
-  component: JSX.Element | null;
-};
-
-function ChatMessageContent({
-  message,
-  isLoading,
-  append,
-  isLastMessage,
-  artifactVersion,
-}: {
-  message: Message;
-  isLoading: boolean;
-  append: Pick<ChatHandler, "append">["append"];
-  isLastMessage: boolean;
-  artifactVersion: number | undefined;
-}) {
-  const annotations = message.annotations as MessageAnnotation[] | undefined;
-  if (!annotations?.length) return <Markdown content={message.content} />;
-
-  const imageData = getAnnotationData<ImageData>(
-    annotations,
-    MessageAnnotationType.IMAGE,
-  );
-  const contentFileData = getAnnotationData<DocumentFileData>(
-    annotations,
-    MessageAnnotationType.DOCUMENT_FILE,
-  );
-  const eventData = getAnnotationData<EventData>(
-    annotations,
-    MessageAnnotationType.EVENTS,
-  );
-  const agentEventData = getAnnotationData<AgentEventData>(
-    annotations,
-    MessageAnnotationType.AGENT_EVENTS,
-  );
-
-  const sourceData = getSourceAnnotationData(annotations);
-
-  const toolData = getAnnotationData<ToolData>(
-    annotations,
-    MessageAnnotationType.TOOLS,
-  );
-  const suggestedQuestionsData = getAnnotationData<SuggestedQuestionsData>(
-    annotations,
-    MessageAnnotationType.SUGGESTED_QUESTIONS,
-  );
-
-  const contents: ContentDisplayConfig[] = [
-    {
-      order: 1,
-      component: imageData[0] ? <ChatImage data={imageData[0]} /> : null,
-    },
-    {
-      order: -3,
-      component:
-        eventData.length > 0 ? (
-          <ChatEvents isLoading={isLoading} data={eventData} />
-        ) : null,
-    },
-    {
-      order: -2,
-      component:
-        agentEventData.length > 0 ? (
-          <ChatAgentEvents
-            data={agentEventData}
-            isFinished={!!message.content}
-          />
-        ) : null,
-    },
-    {
-      order: 2,
-      component: contentFileData[0] ? (
-        <ChatFiles data={contentFileData[0]} />
-      ) : null,
-    },
-    {
-      order: -1,
-      component: toolData[0] ? (
-        <ChatTools data={toolData[0]} artifactVersion={artifactVersion} />
-      ) : null,
-    },
-    {
-      order: 0,
-      component: <Markdown content={message.content} sources={sourceData[0]} />,
-    },
-    {
-      order: 3,
-      component: sourceData[0] ? <ChatSources data={sourceData[0]} /> : null,
-    },
-    {
-      order: 4,
-      component: suggestedQuestionsData[0] ? (
-        <SuggestedQuestions
-          questions={suggestedQuestionsData[0]}
-          append={append}
-          isLastMessage={isLastMessage}
-        />
-      ) : null,
-    },
-  ];
-
-  return (
-    <div className="flex-1 gap-4 flex flex-col">
-      {contents
-        .sort((a, b) => a.order - b.order)
-        .map((content, index) => (
-          <Fragment key={index}>{content.component}</Fragment>
-        ))}
-    </div>
-  );
-}
-
-export default function ChatMessage({
-  chatMessage,
-  isLoading,
-  append,
-  isLastMessage,
-  artifactVersion,
-}: {
-  chatMessage: Message;
-  isLoading: boolean;
-  append: Pick<ChatHandler, "append">["append"];
-  isLastMessage: boolean;
-  artifactVersion: number | undefined;
-}) {
-  const { isCopied, copyToClipboard } = useCopyToClipboard({ timeout: 2000 });
-  return (
-    <div className="flex items-start gap-4 pr-5 pt-5">
-      <ChatAvatar role={chatMessage.role} />
-      <div className="group flex flex-1 justify-between gap-2">
-        <ChatMessageContent
-          message={chatMessage}
-          isLoading={isLoading}
-          append={append}
-          isLastMessage={isLastMessage}
-          artifactVersion={artifactVersion}
-        />
-        <Button
-          onClick={() => copyToClipboard(chatMessage.content)}
-          size="icon"
-          variant="ghost"
-          className="h-8 w-8 opacity-0 group-hover:opacity-100"
-        >
-          {isCopied ? (
-            <Check className="h-4 w-4" />
-          ) : (
-            <Copy className="h-4 w-4" />
-          )}
-        </Button>
-      </div>
-    </div>
-  );
-}

templates/types/streaming/nextjs/app/components/ui/chat/chat-message/markdown.tsx

@@ -1,172 +0,0 @@
-import "katex/dist/katex.min.css";
-import { FC, memo } from "react";
-import ReactMarkdown, { Options } from "react-markdown";
-import rehypeKatex from "rehype-katex";
-import remarkGfm from "remark-gfm";
-import remarkMath from "remark-math";
-
-import { DOCUMENT_FILE_TYPES, DocumentFileType, SourceData } from "..";
-import { useClientConfig } from "../hooks/use-config";
-import { DocumentInfo, SourceNumberButton } from "./chat-sources";
-import { CodeBlock } from "./codeblock";
-
-const MemoizedReactMarkdown: FC<Options> = memo(
-  ReactMarkdown,
-  (prevProps, nextProps) =>
-    prevProps.children === nextProps.children &&
-    prevProps.className === nextProps.className,
-);
-
-const preprocessLaTeX = (content: string) => {
-  // Replace block-level LaTeX delimiters \[ \] with $$ $$
-  const blockProcessedContent = content.replace(
-    /\\\[([\s\S]*?)\\\]/g,
-    (_, equation) => `$$${equation}$$`,
-  );
-  // Replace inline LaTeX delimiters \( \) with $ $
-  const inlineProcessedContent = blockProcessedContent.replace(
-    /\\\[([\s\S]*?)\\\]/g,
-    (_, equation) => `$${equation}$`,
-  );
-  return inlineProcessedContent;
-};
-
-const preprocessMedia = (content: string) => {
-  // Remove `sandbox:` from the beginning of the URL
-  // to fix OpenAI's models issue appending `sandbox:` to the relative URL
-  return content.replace(/(sandbox|attachment|snt):/g, "");
-};
-
-/**
- * Update the citation flag [citation:id]() to the new format [citation:index](url)
- */
-const preprocessCitations = (content: string, sources?: SourceData) => {
-  if (sources) {
-    const citationRegex = /\[citation:(.+?)\]\(\)/g;
-    let match;
-    // Find all the citation references in the content
-    while ((match = citationRegex.exec(content)) !== null) {
-      const citationId = match[1];
-      // Find the source node with the id equal to the citation-id, also get the index of the source node
-      const sourceNode = sources.nodes.find((node) => node.id === citationId);
-      // If the source node is found, replace the citation reference with the new format
-      if (sourceNode !== undefined) {
-        content = content.replace(
-          match[0],
-          `[citation:${sources.nodes.indexOf(sourceNode)}]()`,
-        );
-      } else {
-        // If the source node is not found, remove the citation reference
-        content = content.replace(match[0], "");
-      }
-    }
-  }
-  return content;
-};
-
-const preprocessContent = (content: string, sources?: SourceData) => {
-  return preprocessCitations(
-    preprocessMedia(preprocessLaTeX(content)),
-    sources,
-  );
-};
-
-export default function Markdown({
-  content,
-  sources,
-}: {
-  content: string;
-  sources?: SourceData;
-}) {
-  const processedContent = preprocessContent(content, sources);
-  const { backend } = useClientConfig();
-
-  return (
-    <MemoizedReactMarkdown
-      className="prose dark:prose-invert prose-p:leading-relaxed prose-pre:p-0 break-words custom-markdown"
-      remarkPlugins={[remarkGfm, remarkMath]}
-      rehypePlugins={[rehypeKatex as any]}
-      components={{
-        p({ children }) {
-          return <div className="mb-2 last:mb-0">{children}</div>;
-        },
-        code({ node, inline, className, children, ...props }) {
-          if (children.length) {
-            if (children[0] == "▍") {
-              return (
-                <span className="mt-1 animate-pulse cursor-default">▍</span>
-              );
-            }
-
-            children[0] = (children[0] as string).replace("`▍`", "▍");
-          }
-
-          const match = /language-(\w+)/.exec(className || "");
-
-          if (inline) {
-            return (
-              <code className={className} {...props}>
-                {children}
-              </code>
-            );
-          }
-
-          return (
-            <CodeBlock
-              key={Math.random()}
-              language={(match && match[1]) || ""}
-              value={String(children).replace(/\n$/, "")}
-              className="mb-2"
-              {...props}
-            />
-          );
-        },
-        a({ href, children }) {
-          // If href starts with `{backend}/api/files`, then it's a local document and we use DocumenInfo for rendering
-          if (href?.startsWith(backend + "/api/files")) {
-            // Check if the file is document file type
-            const fileExtension = href.split(".").pop()?.toLowerCase();
-
-            if (
-              fileExtension &&
-              DOCUMENT_FILE_TYPES.includes(fileExtension as DocumentFileType)
-            ) {
-              return (
-                <DocumentInfo
-                  document={{
-                    url: backend
-                      ? new URL(decodeURIComponent(href)).href
-                      : href,
-                    sources: [],
-                  }}
-                  className="mb-2 mt-2"
-                />
-              );
-            }
-          }
-          // If a text link starts with 'citation:', then render it as a citation reference
-          if (
-            Array.isArray(children) &&
-            typeof children[0] === "string" &&
-            children[0].startsWith("citation:")
-          ) {
-            const index = Number(children[0].replace("citation:", ""));
-            if (!isNaN(index)) {
-              return <SourceNumberButton index={index} />;
-            } else {
-              // citation is not looked up yet, don't render anything
-              return <></>;
-            }
-          }
-          return (
-            <a href={href} target="_blank">
-              {children}
-            </a>
-          );
-        },
-      }}
-    >
-      {processedContent}
-    </MemoizedReactMarkdown>
-  );
-}

templates/types/streaming/nextjs/app/components/ui/chat/chat-messages.tsx

@@ -1,136 +1,30 @@
-import { Loader2 } from "lucide-react";
-import { useEffect, useMemo, useRef, useState } from "react";
+"use client";
 
-import { ToolData } from ".";
-import { Button } from "../button";
-import ChatActions from "./chat-actions";
-import ChatMessage from "./chat-message";
-import { ChatHandler } from "./chat.interface";
-import { useClientConfig } from "./hooks/use-config";
-
-export default function ChatMessages(
-  props: Pick<
-    ChatHandler,
-    "messages" | "isLoading" | "reload" | "stop" | "append"
-  >,
-) {
-  const { backend } = useClientConfig();
-  const [starterQuestions, setStarterQuestions] = useState<string[]>();
-
-  const scrollableChatContainerRef = useRef<HTMLDivElement>(null);
-  const messageLength = props.messages.length;
-  const lastMessage = props.messages[messageLength - 1];
-
-  const scrollToBottom = () => {
-    if (scrollableChatContainerRef.current) {
-      scrollableChatContainerRef.current.scrollTop =
-        scrollableChatContainerRef.current.scrollHeight;
-    }
-  };
-
-  const isLastMessageFromAssistant =
-    messageLength > 0 && lastMessage?.role !== "user";
-  const showReload =
-    props.reload && !props.isLoading && isLastMessageFromAssistant;
-  const showStop = props.stop && props.isLoading;
-
-  // `isPending` indicate
-  // that stream response is not yet received from the server,
-  // so we show a loading indicator to give a better UX.
-  const isPending = props.isLoading && !isLastMessageFromAssistant;
-
-  useEffect(() => {
-    scrollToBottom();
-  }, [messageLength, lastMessage]);
-
-  useEffect(() => {
-    if (!starterQuestions) {
-      fetch(`${backend}/api/chat/config`)
-        .then((response) => response.json())
-        .then((data) => {
-          if (data?.starterQuestions) {
-            setStarterQuestions(data.starterQuestions);
-          }
-        })
-        .catch((error) => console.error("Error fetching config", error));
-    }
-  }, [starterQuestions, backend]);
-
-  // build a map of message id to artifact version
-  const artifactVersionMap = useMemo(() => {
-    const map = new Map<string, number | undefined>();
-    let versionIndex = 1;
-    props.messages.forEach((m) => {
-      m.annotations?.forEach((annotation) => {
-        if (
-          typeof annotation === "object" &&
-          annotation != null &&
-          "type" in annotation &&
-          annotation.type === "tools"
-        ) {
-          const data = annotation.data as ToolData;
-          if (data?.toolCall?.name === "artifact") {
-            map.set(m.id, versionIndex);
-            versionIndex++;
-          }
-        }
-      });
-    });
-    return map;
-  }, [props.messages]);
+import { ChatMessage, ChatMessages, useChatUI } from "@llamaindex/chat-ui";
+import { ChatMessageAvatar } from "./chat-avatar";
+import { ChatMessageContent } from "./chat-message-content";
+import { ChatStarter } from "./chat-starter";
 
+export default function CustomChatMessages() {
+  const { messages } = useChatUI();
   return (
-    <div
-      className="flex-1 w-full rounded-xl bg-white p-4 shadow-xl relative overflow-y-auto"
-      ref={scrollableChatContainerRef}
-    >
-      <div className="flex flex-col gap-5 divide-y">
-        {props.messages.map((m, i) => {
-          const isLoadingMessage = i === messageLength - 1 && props.isLoading;
-          return (
-            <ChatMessage
-              key={m.id}
-              chatMessage={m}
-              isLoading={isLoadingMessage}
-              append={props.append!}
-              isLastMessage={i === messageLength - 1}
-              artifactVersion={artifactVersionMap.get(m.id)}
-            />
-          );
-        })}
-        {isPending && (
-          <div className="flex justify-center items-center pt-10">
-            <Loader2 className="h-4 w-4 animate-spin" />
-          </div>
-        )}
-      </div>
-      {(showReload || showStop) && (
-        <div className="flex justify-end py-4">
-          <ChatActions
-            reload={props.reload}
-            stop={props.stop}
-            showReload={showReload}
-            showStop={showStop}
-          />
-        </div>
-      )}
-      {!messageLength && starterQuestions?.length && props.append && (
-        <div className="absolute bottom-6 left-0 w-full">
-          <div className="grid grid-cols-2 gap-2 mx-20">
-            {starterQuestions.map((question, i) => (
-              <Button
-                variant="outline"
-                key={i}
-                onClick={() =>
-                  props.append!({ role: "user", content: question })
-                }
-              >
-                {question}
-              </Button>
-            ))}
-          </div>
-        </div>
-      )}
-    </div>
+    <ChatMessages className="shadow-xl rounded-xl">
+      <ChatMessages.List>
+        {messages.map((message, index) => (
+          <ChatMessage
+            key={message.id}
+            message={message}
+            isLast={index === messages.length - 1}
+          >
+            <ChatMessageAvatar />
+            <ChatMessageContent />
+            <ChatMessage.Actions />
+          </ChatMessage>
+        ))}
+        <ChatMessages.Loading />
+      </ChatMessages.List>
+      <ChatMessages.Actions />
+      <ChatStarter />
+    </ChatMessages>
   );
 }

templates/types/streaming/nextjs/app/components/ui/chat/chat-starter.tsx

@@ -0,0 +1,26 @@
+import { useChatUI } from "@llamaindex/chat-ui";
+import { StarterQuestions } from "@llamaindex/chat-ui/widgets";
+import { useEffect, useState } from "react";
+import { useClientConfig } from "./hooks/use-config";
+
+export function ChatStarter() {
+  const { append } = useChatUI();
+  const { backend } = useClientConfig();
+  const [starterQuestions, setStarterQuestions] = useState<string[]>();
+
+  useEffect(() => {
+    if (!starterQuestions) {
+      fetch(`${backend}/api/chat/config`)
+        .then((response) => response.json())
+        .then((data) => {
+          if (data?.starterQuestions) {
+            setStarterQuestions(data.starterQuestions);
+          }
+        })
+        .catch((error) => console.error("Error fetching config", error));
+    }
+  }, [starterQuestions, backend]);
+
+  if (!starterQuestions?.length) return null;
+  return <StarterQuestions append={append} questions={starterQuestions} />;
+}

templates/types/streaming/nextjs/app/components/ui/chat/chat.interface.ts

@@ -1,25 +0,0 @@
-import { Message } from "ai";
-
-export interface ChatHandler {
-  messages: Message[];
-  input: string;
-  isLoading: boolean;
-  handleSubmit: (
-    e: React.FormEvent<HTMLFormElement>,
-    ops?: {
-      data?: any;
-    },
-  ) => void;
-  handleInputChange: (e: React.ChangeEvent<HTMLTextAreaElement>) => void;
-  reload?: () => void;
-  stop?: () => void;
-  onFileUpload?: (file: File) => Promise<void>;
-  onFileError?: (errMsg: string) => void;
-  setInput?: (input: string) => void;
-  append?: (
-    message: Message | Omit<Message, "id">,
-    ops?: {
-      data: any;
-    },
-  ) => Promise<string | null | undefined>;
-}

templates/types/streaming/nextjs/app/components/ui/chat/custom/llama-cloud-selector.tsx

@@ -1,3 +1,4 @@
+import { useChatUI } from "@llamaindex/chat-ui";
 import { Loader2 } from "lucide-react";
 import { useCallback, useEffect, useState } from "react";
 import {
@@ -35,19 +36,18 @@ type LlamaCloudConfig = {
 };
 
 export interface LlamaCloudSelectorProps {
-  setRequestData?: React.Dispatch<any>;
   onSelect?: (pipeline: PipelineConfig | undefined) => void;
   defaultPipeline?: PipelineConfig;
   shouldCheckValid?: boolean;
 }
 
 export function LlamaCloudSelector({
-  setRequestData,
   onSelect,
   defaultPipeline,
   shouldCheckValid = false,
 }: LlamaCloudSelectorProps) {
   const { backend } = useClientConfig();
+  const { setRequestData } = useChatUI();
   const [config, setConfig] = useState<LlamaCloudConfig>();
 
   const updateRequestParams = useCallback(
@@ -97,6 +97,10 @@ export function LlamaCloudSelector({
     setPipeline(JSON.parse(value) as PipelineConfig);
   };
 
+  if (process.env.NEXT_PUBLIC_USE_LLAMACLOUD !== "true") {
+    return null;
+  }
+
   if (!config) {
     return (
       <div className="flex justify-center items-center p-3">

templates/types/streaming/nextjs/app/components/ui/chat/custom/markdown.tsx

@@ -0,0 +1,27 @@
+import { SourceData } from "@llamaindex/chat-ui";
+import { Markdown as MarkdownUI } from "@llamaindex/chat-ui/widgets";
+import { useClientConfig } from "../hooks/use-config";
+
+const preprocessMedia = (content: string) => {
+  // Remove `sandbox:` from the beginning of the URL before rendering markdown
+  // OpenAI models sometimes prepend `sandbox:` to relative URLs - this fixes it
+  return content.replace(/(sandbox|attachment|snt):/g, "");
+};
+
+export function Markdown({
+  content,
+  sources,
+}: {
+  content: string;
+  sources?: SourceData;
+}) {
+  const { backend } = useClientConfig();
+  const processedContent = preprocessMedia(content);
+  return (
+    <MarkdownUI
+      content={processedContent}
+      backend={backend}
+      sources={sources}
+    />
+  );
+}

templates/types/streaming/nextjs/app/components/ui/chat/hooks/use-file.ts

@@ -1,121 +0,0 @@
-"use client";
-
-import { JSONValue } from "llamaindex";
-import { useState } from "react";
-import {
-  DocumentFile,
-  DocumentFileType,
-  MessageAnnotation,
-  MessageAnnotationType,
-} from "..";
-import { useClientConfig } from "./use-config";
-
-const docMineTypeMap: Record<string, DocumentFileType> = {
-  "text/csv": "csv",
-  "application/pdf": "pdf",
-  "text/plain": "txt",
-  "application/vnd.openxmlformats-officedocument.wordprocessingml.document":
-    "docx",
-};
-
-export function useFile() {
-  const { backend } = useClientConfig();
-  const [imageUrl, setImageUrl] = useState<string | null>(null);
-  const [files, setFiles] = useState<DocumentFile[]>([]);
-
-  const addDoc = (file: DocumentFile) => {
-    const existedFile = files.find((f) => f.id === file.id);
-    if (!existedFile) {
-      setFiles((prev) => [...prev, file]);
-      return true;
-    }
-    return false;
-  };
-
-  const removeDoc = (file: DocumentFile) => {
-    setFiles((prev) => prev.filter((f) => f.id !== file.id));
-  };
-
-  const reset = () => {
-    imageUrl && setImageUrl(null);
-    files.length && setFiles([]);
-  };
-
-  const uploadContent = async (
-    file: File,
-    requestParams: any = {},
-  ): Promise<DocumentFile> => {
-    const base64 = await readContent({ file, asUrl: true });
-    const uploadAPI = `${backend}/api/chat/upload`;
-    const response = await fetch(uploadAPI, {
-      method: "POST",
-      headers: {
-        "Content-Type": "application/json",
-      },
-      body: JSON.stringify({
-        ...requestParams,
-        base64,
-        name: file.name,
-      }),
-    });
-    if (!response.ok) throw new Error("Failed to upload document.");
-    return (await response.json()) as DocumentFile;
-  };
-
-  const getAnnotations = () => {
-    const annotations: MessageAnnotation[] = [];
-    if (imageUrl) {
-      annotations.push({
-        type: MessageAnnotationType.IMAGE,
-        data: { url: imageUrl },
-      });
-    }
-    if (files.length > 0) {
-      annotations.push({
-        type: MessageAnnotationType.DOCUMENT_FILE,
-        data: { files },
-      });
-    }
-    return annotations as JSONValue[];
-  };
-
-  const readContent = async (input: {
-    file: File;
-    asUrl?: boolean;
-  }): Promise<string> => {
-    const { file, asUrl } = input;
-    const content = await new Promise<string>((resolve, reject) => {
-      const reader = new FileReader();
-      if (asUrl) {
-        reader.readAsDataURL(file);
-      } else {
-        reader.readAsText(file);
-      }
-      reader.onload = () => resolve(reader.result as string);
-      reader.onerror = (error) => reject(error);
-    });
-    return content;
-  };
-
-  const uploadFile = async (file: File, requestParams: any = {}) => {
-    if (file.type.startsWith("image/")) {
-      const base64 = await readContent({ file, asUrl: true });
-      return setImageUrl(base64);
-    }
-
-    const filetype = docMineTypeMap[file.type];
-    if (!filetype) throw new Error("Unsupported document type.");
-    const newDoc = await uploadContent(file, requestParams);
-    return addDoc(newDoc);
-  };
-
-  return {
-    imageUrl,
-    setImageUrl,
-    files,
-    removeDoc,
-    reset,
-    getAnnotations,
-    uploadFile,
-  };
-}

templates/types/streaming/nextjs/app/components/ui/chat/index.ts

@@ -1,131 +0,0 @@
-import { JSONValue } from "ai";
-import ChatInput from "./chat-input";
-import ChatMessages from "./chat-messages";
-
-export { type ChatHandler } from "./chat.interface";
-export { ChatInput, ChatMessages };
-
-export enum MessageAnnotationType {
-  IMAGE = "image",
-  DOCUMENT_FILE = "document_file",
-  SOURCES = "sources",
-  EVENTS = "events",
-  TOOLS = "tools",
-  SUGGESTED_QUESTIONS = "suggested_questions",
-  AGENT_EVENTS = "agent",
-}
-
-export type ImageData = {
-  url: string;
-};
-
-export type DocumentFileType = "csv" | "pdf" | "txt" | "docx";
-export const DOCUMENT_FILE_TYPES: DocumentFileType[] = [
-  "csv",
-  "pdf",
-  "txt",
-  "docx",
-];
-
-export type DocumentFile = {
-  id: string;
-  name: string; // The uploaded file name in the backend
-  size: number; // The file size in bytes
-  type: DocumentFileType;
-  url: string; // The URL of the uploaded file in the backend
-  refs?: string[]; // DocumentIDs of the uploaded file in the vector index
-};
-
-export type DocumentFileData = {
-  files: DocumentFile[];
-};
-
-export type SourceNode = {
-  id: string;
-  metadata: Record<string, unknown>;
-  score?: number;
-  text: string;
-  url: string;
-};
-
-export type SourceData = {
-  nodes: SourceNode[];
-};
-
-export type EventData = {
-  title: string;
-};
-
-export type AgentEventData = {
-  agent: string;
-  text: string;
-};
-
-export type ToolData = {
-  toolCall: {
-    id: string;
-    name: string;
-    input: {
-      [key: string]: JSONValue;
-    };
-  };
-  toolOutput: {
-    output: JSONValue;
-    isError: boolean;
-  };
-};
-
-export type SuggestedQuestionsData = string[];
-
-export type AnnotationData =
-  | ImageData
-  | DocumentFileData
-  | SourceData
-  | EventData
-  | AgentEventData
-  | ToolData
-  | SuggestedQuestionsData;
-
-export type MessageAnnotation = {
-  type: MessageAnnotationType;
-  data: AnnotationData;
-};
-
-const NODE_SCORE_THRESHOLD = 0.25;
-
-export function getAnnotationData<T extends AnnotationData>(
-  annotations: MessageAnnotation[],
-  type: MessageAnnotationType,
-): T[] {
-  return annotations.filter((a) => a.type === type).map((a) => a.data as T);
-}
-
-export function getSourceAnnotationData(
-  annotations: MessageAnnotation[],
-): SourceData[] {
-  const data = getAnnotationData<SourceData>(
-    annotations,
-    MessageAnnotationType.SOURCES,
-  );
-  if (data.length > 0) {
-    const sourceData = data[0] as SourceData;
-    if (sourceData.nodes) {
-      sourceData.nodes = preprocessSourceNodes(sourceData.nodes);
-    }
-  }
-  return data;
-}
-
-function preprocessSourceNodes(nodes: SourceNode[]): SourceNode[] {
-  // Filter source nodes has lower score
-  nodes = nodes
-    .filter((node) => (node.score ?? 1) > NODE_SCORE_THRESHOLD)
-    .filter((node) => node.url && node.url.trim() !== "")
-    .sort((a, b) => (b.score ?? 1) - (a.score ?? 1))
-    .map((node) => {
-      // remove trailing slash for node url if exists
-      node.url = node.url.replace(/\/$/, "");
-      return node;
-    });
-  return nodes;
-}

templates/types/streaming/nextjs/app/components/ui/chat/tools/artifact.tsx

@@ -10,7 +10,7 @@ import {
 } from "../../collapsible";
 import { cn } from "../../lib/utils";
 import { Tabs, TabsContent, TabsList, TabsTrigger } from "../../tabs";
-import Markdown from "../chat-message/markdown";
+import { Markdown } from "../custom/markdown";
 import { useClientConfig } from "../hooks/use-config";
 import { useCopyToClipboard } from "../hooks/use-copy-to-clipboard";
 
@@ -29,12 +29,17 @@ export type CodeArtifact = {
   files?: string[];
 };
 
+type OutputUrl = {
+  url: string;
+  filename: string;
+};
+
 type ArtifactResult = {
   template: string;
   stdout: string[];
   stderr: string[];
   runtimeError?: { name: string; value: string; tracebackRaw: string[] };
-  outputUrls: Array<{ url: string; filename: string }>;
+  outputUrls: OutputUrl[];
   url: string;
 };
 
@@ -272,11 +277,7 @@ function CodeSandboxPreview({ url }: { url: string }) {
   );
 }
 
-function InterpreterOutput({
-  outputUrls,
-}: {
-  outputUrls: Array<{ url: string; filename: string }>;
-}) {
+function InterpreterOutput({ outputUrls }: { outputUrls: OutputUrl[] }) {
   return (
     <ul className="flex flex-col gap-2 mt-4">
       {outputUrls.map((url) => (

templates/types/streaming/nextjs/app/components/ui/chat/tools/chat-tools.tsx

@@ -0,0 +1,89 @@
+import {
+  getAnnotationData,
+  MessageAnnotation,
+  useChatMessage,
+  useChatUI,
+} from "@llamaindex/chat-ui";
+import { JSONValue, Message } from "ai";
+import { useMemo } from "react";
+import { Artifact, CodeArtifact } from "./artifact";
+import { WeatherCard, WeatherData } from "./weather-card";
+
+export function ToolAnnotations({ message }: { message: Message }) {
+  const annotations = message.annotations as MessageAnnotation[] | undefined;
+  const toolData = annotations
+    ? (getAnnotationData(annotations, "tools") as unknown as ToolData[])
+    : null;
+  return toolData?.[0] ? <ChatTools data={toolData[0]} /> : null;
+}
+
+// TODO: Used to render outputs of tools. If needed, add more renderers here.
+function ChatTools({ data }: { data: ToolData }) {
+  const { messages } = useChatUI();
+  const { message } = useChatMessage();
+
+  // build a map of message id to artifact version
+  const artifactVersionMap = useMemo(() => {
+    const map = new Map<string, number | undefined>();
+    let versionIndex = 1;
+    messages.forEach((m) => {
+      m.annotations?.forEach((annotation: any) => {
+        if (
+          typeof annotation === "object" &&
+          annotation != null &&
+          "type" in annotation &&
+          annotation.type === "tools"
+        ) {
+          const data = annotation.data as ToolData;
+          if (data?.toolCall?.name === "artifact") {
+            map.set(m.id, versionIndex);
+            versionIndex++;
+          }
+        }
+      });
+    });
+    return map;
+  }, [messages]);
+
+  if (!data) return null;
+  const { toolCall, toolOutput } = data;
+
+  if (toolOutput.isError) {
+    return (
+      <div className="border-l-2 border-red-400 pl-2">
+        There was an error when calling the tool {toolCall.name} with input:{" "}
+        <br />
+        {JSON.stringify(toolCall.input)}
+      </div>
+    );
+  }
+
+  switch (toolCall.name) {
+    case "get_weather_information":
+      const weatherData = toolOutput.output as unknown as WeatherData;
+      return <WeatherCard data={weatherData} />;
+    case "artifact":
+      return (
+        <Artifact
+          artifact={toolOutput.output as CodeArtifact}
+          version={artifactVersionMap.get(message.id)}
+        />
+      );
+    default:
+      return null;
+  }
+}
+
+type ToolData = {
+  toolCall: {
+    id: string;
+    name: string;
+    input: {
+      [key: string]: JSONValue;
+    };
+  };
+  toolOutput: {
+    output: JSONValue;
+    isError: boolean;
+  };
+};

templates/types/streaming/nextjs/app/components/ui/chat/widgets/PdfDialog.tsx

@@ -1,67 +0,0 @@
-import dynamic from "next/dynamic";
-import { Button } from "../../button";
-import {
-  Drawer,
-  DrawerClose,
-  DrawerContent,
-  DrawerDescription,
-  DrawerHeader,
-  DrawerTitle,
-  DrawerTrigger,
-} from "../../drawer";
-
-export interface PdfDialogProps {
-  documentId: string;
-  url: string;
-  trigger: React.ReactNode;
-}
-
-// Dynamic imports for client-side rendering only
-const PDFViewer = dynamic(
-  () => import("@llamaindex/pdf-viewer").then((module) => module.PDFViewer),
-  { ssr: false },
-);
-
-const PdfFocusProvider = dynamic(
-  () =>
-    import("@llamaindex/pdf-viewer").then((module) => module.PdfFocusProvider),
-  { ssr: false },
-);
-
-export default function PdfDialog(props: PdfDialogProps) {
-  return (
-    <Drawer direction="left">
-      <DrawerTrigger asChild>{props.trigger}</DrawerTrigger>
-      <DrawerContent className="w-3/5 mt-24 h-full max-h-[96%] ">
-        <DrawerHeader className="flex justify-between">
-          <div className="space-y-2">
-            <DrawerTitle>PDF Content</DrawerTitle>
-            <DrawerDescription>
-              File URL:{" "}
-              <a
-                className="hover:text-blue-900"
-                href={props.url}
-                target="_blank"
-              >
-                {props.url}
-              </a>
-            </DrawerDescription>
-          </div>
-          <DrawerClose asChild>
-            <Button variant="outline">Close</Button>
-          </DrawerClose>
-        </DrawerHeader>
-        <div className="m-4">
-          <PdfFocusProvider>
-            <PDFViewer
-              file={{
-                id: props.documentId,
-                url: props.url,
-              }}
-            />
-          </PdfFocusProvider>
-        </div>
-      </DrawerContent>
-    </Drawer>
-  );
-}

templates/types/streaming/nextjs/app/components/ui/document-preview.tsx

@@ -1,129 +0,0 @@
-import { XCircleIcon } from "lucide-react";
-import Image from "next/image";
-import DocxIcon from "../ui/icons/docx.svg";
-import PdfIcon from "../ui/icons/pdf.svg";
-import SheetIcon from "../ui/icons/sheet.svg";
-import TxtIcon from "../ui/icons/txt.svg";
-import { Button } from "./button";
-import { DocumentFile, DocumentFileType } from "./chat";
-import {
-  Drawer,
-  DrawerClose,
-  DrawerContent,
-  DrawerDescription,
-  DrawerHeader,
-  DrawerTitle,
-  DrawerTrigger,
-} from "./drawer";
-import { cn } from "./lib/utils";
-
-export interface DocumentPreviewProps {
-  file: DocumentFile;
-  onRemove?: () => void;
-}
-
-export function DocumentPreview(props: DocumentPreviewProps) {
-  const { name, size, type, refs } = props.file;
-
-  if (refs?.length) {
-    return (
-      <div title={`Document IDs: ${refs.join(", ")}`}>
-        <PreviewCard {...props} />
-      </div>
-    );
-  }
-
-  return (
-    <Drawer direction="left">
-      <DrawerTrigger asChild>
-        <div>
-          <PreviewCard className="cursor-pointer" {...props} />
-        </div>
-      </DrawerTrigger>
-      <DrawerContent className="w-3/5 mt-24 h-full max-h-[96%] ">
-        <DrawerHeader className="flex justify-between">
-          <div className="space-y-2">
-            <DrawerTitle>{type.toUpperCase()} Raw Content</DrawerTitle>
-            <DrawerDescription>
-              {name} ({inKB(size)} KB)
-            </DrawerDescription>
-          </div>
-          <DrawerClose asChild>
-            <Button variant="outline">Close</Button>
-          </DrawerClose>
-        </DrawerHeader>
-        <div className="m-4 max-h-[80%] overflow-auto">
-          {refs?.length && (
-            <pre className="bg-secondary rounded-md p-4 block text-sm">
-              {refs.join(", ")}
-            </pre>
-          )}
-        </div>
-      </DrawerContent>
-    </Drawer>
-  );
-}
-
-export const FileIcon: Record<DocumentFileType, string> = {
-  csv: SheetIcon,
-  pdf: PdfIcon,
-  docx: DocxIcon,
-  txt: TxtIcon,
-};
-
-export function PreviewCard(props: {
-  file: {
-    name: string;
-    size?: number;
-    type: DocumentFileType;
-  };
-  onRemove?: () => void;
-  className?: string;
-}) {
-  const { onRemove, file, className } = props;
-  return (
-    <div
-      className={cn(
-        "p-2 w-60 max-w-60 bg-secondary rounded-lg text-sm relative",
-        className,
-      )}
-    >
-      <div className="flex flex-row items-center gap-2">
-        <div className="relative h-8 w-8 shrink-0 overflow-hidden rounded-md flex items-center justify-center">
-          <Image
-            className="h-full w-auto object-contain"
-            priority
-            src={FileIcon[file.type]}
-            alt="Icon"
-          />
-        </div>
-        <div className="overflow-hidden">
-          <div className="truncate font-semibold">
-            {file.name} {file.size ? `(${inKB(file.size)} KB)` : ""}
-          </div>
-          {file.type && (
-            <div className="truncate text-token-text-tertiary flex items-center gap-2">
-              <span>{file.type.toUpperCase()} File</span>
-            </div>
-          )}
-        </div>
-      </div>
-      {onRemove && (
-        <div
-          className={cn(
-            "absolute -top-2 -right-2 w-6 h-6 z-10 bg-gray-500 text-white rounded-full",
-          )}
-        >
-          <XCircleIcon
-            className="w-6 h-6 bg-gray-500 text-white rounded-full"
-            onClick={onRemove}
-          />
-        </div>
-      )}
-    </div>
-  );
-}
-
-function inKB(size: number) {
-  return Math.round((size / 1024) * 10) / 10;
-}

templates/types/streaming/nextjs/app/components/ui/file-uploader.tsx

@@ -1,105 +0,0 @@
-"use client";
-
-import { Loader2, Paperclip } from "lucide-react";
-import { ChangeEvent, useState } from "react";
-import { buttonVariants } from "./button";
-import { cn } from "./lib/utils";
-
-export interface FileUploaderProps {
-  config?: {
-    inputId?: string;
-    fileSizeLimit?: number;
-    allowedExtensions?: string[];
-    checkExtension?: (extension: string) => string | null;
-    disabled: boolean;
-  };
-  onFileUpload: (file: File) => Promise<void>;
-  onFileError?: (errMsg: string) => void;
-}
-
-const DEFAULT_INPUT_ID = "fileInput";
-const DEFAULT_FILE_SIZE_LIMIT = 1024 * 1024 * 50; // 50 MB
-
-export default function FileUploader({
-  config,
-  onFileUpload,
-  onFileError,
-}: FileUploaderProps) {
-  const [uploading, setUploading] = useState(false);
-
-  const inputId = config?.inputId || DEFAULT_INPUT_ID;
-  const fileSizeLimit = config?.fileSizeLimit || DEFAULT_FILE_SIZE_LIMIT;
-  const allowedExtensions = config?.allowedExtensions;
-  const defaultCheckExtension = (extension: string) => {
-    if (allowedExtensions && !allowedExtensions.includes(extension)) {
-      return `Invalid file type. Please select a file with one of these formats: ${allowedExtensions!.join(
-        ",",
-      )}`;
-    }
-    return null;
-  };
-  const checkExtension = config?.checkExtension ?? defaultCheckExtension;
-
-  const isFileSizeExceeded = (file: File) => {
-    return file.size > fileSizeLimit;
-  };
-
-  const resetInput = () => {
-    const fileInput = document.getElementById(inputId) as HTMLInputElement;
-    fileInput.value = "";
-  };
-
-  const onFileChange = async (e: ChangeEvent<HTMLInputElement>) => {
-    const file = e.target.files?.[0];
-    if (!file) return;
-
-    setUploading(true);
-    await handleUpload(file);
-    resetInput();
-    setUploading(false);
-  };
-
-  const handleUpload = async (file: File) => {
-    const onFileUploadError = onFileError || window.alert;
-    const fileExtension = file.name.split(".").pop() || "";
-    const extensionFileError = checkExtension(fileExtension);
-    if (extensionFileError) {
-      return onFileUploadError(extensionFileError);
-    }
-
-    if (isFileSizeExceeded(file)) {
-      return onFileUploadError(
-        `File size exceeded. Limit is ${fileSizeLimit / 1024 / 1024} MB`,
-      );
-    }
-
-    await onFileUpload(file);
-  };
-
-  return (
-    <div className="self-stretch">
-      <input
-        type="file"
-        id={inputId}
-        style={{ display: "none" }}
-        onChange={onFileChange}
-        accept={allowedExtensions?.join(",")}
-        disabled={config?.disabled || uploading}
-      />
-      <label
-        htmlFor={inputId}
-        className={cn(
-          buttonVariants({ variant: "secondary", size: "icon" }),
-          "cursor-pointer",
-          uploading && "opacity-50",
-        )}
-      >
-        {uploading ? (
-          <Loader2 className="h-4 w-4 animate-spin" />
-        ) : (
-          <Paperclip className="-rotate-45 w-4 h-4" />
-        )}
-      </label>
-    </div>
-  );
-}