Release 0.1.9 (#101)

Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>

.changeset/bright-turkeys-melt.md

@@ -1,5 +0,0 @@
----
-"create-llama": patch
----
-
-Add CSV upload

.changeset/itchy-ads-travel.md

@@ -1,5 +0,0 @@
----
-"create-llama": patch
----
-
-Add E2B code interpreter tool for FastAPI

.changeset/quiet-cars-clap.md

@@ -1,5 +0,0 @@
----
-"create-llama": patch
----
-
-Add OpenAPI action tool for FastAPI

CHANGELOG.md

@@ -1,5 +1,14 @@
 # create-llama
 
+## 0.1.9
+
+### Patch Changes
+
+- a42fa53: Add CSV upload
+- 563b51d: Fix Vercel streaming (python) to stream data events instantly
+- d60b3c5: Add E2B code interpreter tool for FastAPI
+- 956538e: Add OpenAPI action tool for FastAPI
+
 ## 0.1.8
 
 ### Patch Changes

helpers/tools.ts

@@ -30,7 +30,7 @@ export type ToolDependencies = {
 
 export const supportedTools: Tool[] = [
   {
-    display: "Google Search (configuration required after installation)",
+    display: "Google Search",
     name: "google.GoogleSearchToolSpec",
     config: {
       engine:

package.json

@@ -1,6 +1,6 @@
 {
   "name": "create-llama",
-  "version": "0.1.8",
+  "version": "0.1.9",
   "description": "Create LlamaIndex-powered apps with one command",
   "keywords": [
     "rag",

templates/components/engines/python/agent/tools/interpreter.py

@@ -3,7 +3,7 @@ import logging
 import base64
 import uuid
 from pydantic import BaseModel
-from typing import List, Tuple, Dict
+from typing import List, Tuple, Dict, Optional
 from llama_index.core.tools import FunctionTool
 from e2b_code_interpreter import CodeInterpreter
 from e2b_code_interpreter.models import Logs
@@ -14,8 +14,9 @@ logger = logging.getLogger(__name__)
 
 class InterpreterExtraResult(BaseModel):
     type: str
-    filename: str
-    url: str
+    content: Optional[str] = None
+    filename: Optional[str] = None
+    url: Optional[str] = None
 
 
 class E2BToolOutput(BaseModel):
@@ -72,19 +73,30 @@ class E2BCodeInterpreter:
 
         try:
             formats = result.formats()
-            base64_data_arr = [result[format] for format in formats]
+            results = [result[format] for format in formats]
 
-            for ext, base64_data in zip(formats, base64_data_arr):
-                if ext and base64_data:
-                    result = self.save_to_disk(base64_data, ext)
-                    filename = result["filename"]
-                    output.append(
-                        InterpreterExtraResult(
-                            type=ext, filename=filename, url=self.get_file_url(filename)
+            for ext, data in zip(formats, results):
+                match ext:
+                    case "png" | "svg" | "jpeg" | "pdf":
+                        result = self.save_to_disk(data, ext)
+                        filename = result["filename"]
+                        output.append(
+                            InterpreterExtraResult(
+                                type=ext,
+                                filename=filename,
+                                url=self.get_file_url(filename),
+                            )
+                        )
+                    case _:
+                        output.append(
+                            InterpreterExtraResult(
+                                type=ext,
+                                content=data,
+                            )
                         )
-                    )
         except Exception as error:
-            logger.error("Error when saving data to disk", error)
+            logger.exception(error, exc_info=True)
+            logger.error("Error when parsing output from E2b interpreter tool", error)
 
         return output
 
@@ -96,7 +108,8 @@ class E2BCodeInterpreter:
             exec = interpreter.notebook.exec_cell(code)
 
             if exec.error:
-                output = E2BToolOutput(is_error=True, logs=[exec.error])
+                logger.error("Error when executing code", exec.error)
+                output = E2BToolOutput(is_error=True, logs=exec.logs, results=[])
             else:
                 if len(exec.results) == 0:
                     output = E2BToolOutput(is_error=False, logs=exec.logs, results=[])
@@ -111,6 +124,9 @@ class E2BCodeInterpreter:
 def code_interpret(code: str) -> Dict:
     """
     Execute python code in a Jupyter notebook cell and return any result, stdout, stderr, display_data, and error.
+
+    Parameters:
+        code (str): The python code to be executed in a single cell.
     """
     api_key = os.getenv("E2B_API_KEY")
     filesever_url_prefix = os.getenv("FILESERVER_URL_PREFIX")

templates/components/engines/typescript/agent/tools/interpreter.ts

@@ -34,8 +34,9 @@ type InterpreterExtraType =
 
 export type InterpreterExtraResult = {
   type: InterpreterExtraType;
-  filename: string;
-  url: string;
+  content?: string;
+  filename?: string;
+  url?: string;
 };
 
 const DEFAULT_META_DATA: ToolMetadata<JSONSchemaType<InterpreterParameter>> = {
@@ -106,10 +107,13 @@ export class InterpreterTool implements BaseTool<InterpreterParameter> {
 
   async call(input: InterpreterParameter): Promise<InterpreterToolOutput> {
     const result = await this.codeInterpret(input.code);
-    await this.codeInterpreter?.close();
     return result;
   }
 
+  async close() {
+    await this.codeInterpreter?.close();
+  }
+
   private async getExtraResult(
     res?: Result,
   ): Promise<InterpreterExtraResult[]> {
@@ -118,23 +122,34 @@ export class InterpreterTool implements BaseTool<InterpreterParameter> {
 
     try {
       const formats = res.formats(); // formats available for the result. Eg: ['png', ...]
-      const base64DataArr = formats.map((f) => res[f as keyof Result]); // get base64 data for each format
+      const results = formats.map((f) => res[f as keyof Result]); // get base64 data for each format
 
       // save base64 data to file and return the url
       for (let i = 0; i < formats.length; i++) {
         const ext = formats[i];
-        const base64Data = base64DataArr[i];
-        if (ext && base64Data) {
-          const { filename } = this.saveToDisk(base64Data, ext);
-          output.push({
-            type: ext as InterpreterExtraType,
-            filename,
-            url: this.getFileUrl(filename),
-          });
+        const data = results[i];
+        switch (ext) {
+          case "png":
+          case "jpeg":
+          case "svg":
+          case "pdf":
+            const { filename } = this.saveToDisk(data, ext);
+            output.push({
+              type: ext as InterpreterExtraType,
+              filename,
+              url: this.getFileUrl(filename),
+            });
+            break;
+          default:
+            output.push({
+              type: ext as InterpreterExtraType,
+              content: data,
+            });
+            break;
         }
       }
     } catch (error) {
-      console.error("Error when saving data to disk", error);
+      console.error("Error when parsing e2b response", error);
     }
 
     return output;

templates/components/ui/html/chat/chat-input.tsx

@@ -15,6 +15,7 @@ export interface ChatInputProps {
   handleSubmit: (e: React.FormEvent<HTMLFormElement>) => void;
   isLoading: boolean;
   messages: Message[];
+  setInput?: (input: string) => void;
 }
 
 export default function ChatInput(props: ChatInputProps) {

templates/types/streaming/express/package.json

@@ -14,7 +14,7 @@
     "cors": "^2.8.5",
     "dotenv": "^16.3.1",
     "express": "^4.18.2",
-    "llamaindex": "0.3.13",
+    "llamaindex": "0.3.16",
     "pdf2json": "3.0.5",
     "ajv": "^8.12.0",
     "@e2b/code-interpreter": "^0.0.5"

templates/types/streaming/express/src/controllers/chat.controller.ts

@@ -2,11 +2,7 @@ import { Message, StreamData, streamToResponse } from "ai";
 import { Request, Response } from "express";
 import { ChatMessage, Settings } from "llamaindex";
 import { createChatEngine } from "./engine/chat";
-import {
-  DataParserOptions,
-  LlamaIndexStream,
-  convertMessageContent,
-} from "./llamaindex-stream";
+import { LlamaIndexStream, convertMessageContent } from "./llamaindex-stream";
 import { createCallbackManager, createStreamTimeout } from "./stream-helper";
 
 export const chat = async (req: Request, res: Response) => {
@@ -14,10 +10,7 @@ export const chat = async (req: Request, res: Response) => {
   const vercelStreamData = new StreamData();
   const streamTimeout = createStreamTimeout(vercelStreamData);
   try {
-    const {
-      messages,
-      data,
-    }: { messages: Message[]; data: DataParserOptions | undefined } = req.body;
+    const { messages }: { messages: Message[] } = req.body;
     const userMessage = messages.pop();
     if (!messages || !userMessage || userMessage.role !== "user") {
       return res.status(400).json({
@@ -28,8 +21,24 @@ export const chat = async (req: Request, res: Response) => {
 
     const chatEngine = await createChatEngine();
 
+    let annotations = userMessage.annotations;
+    if (!annotations) {
+      // the user didn't send any new annotations with the last message
+      // so use the annotations from the last user message that has annotations
+      // REASON: GPT4 doesn't consider MessageContentDetail from previous messages, only strings
+      annotations = messages
+        .slice()
+        .reverse()
+        .find(
+          (message) => message.role === "user" && message.annotations,
+        )?.annotations;
+    }
+
     // Convert message content from Vercel/AI format to LlamaIndex/OpenAI format
-    const userMessageContent = convertMessageContent(userMessage.content, data);
+    const userMessageContent = convertMessageContent(
+      userMessage.content,
+      annotations,
+    );
 
     // Setup callbacks
     const callbackManager = createCallbackManager(vercelStreamData);
@@ -44,12 +53,7 @@ export const chat = async (req: Request, res: Response) => {
     });
 
     // Return a stream, which can be consumed by the Vercel/AI client
-    const stream = LlamaIndexStream(response, vercelStreamData, {
-      parserOptions: {
-        imageUrl: data?.imageUrl,
-        csvFiles: data?.csvFiles,
-      },
-    });
+    const stream = LlamaIndexStream(response, vercelStreamData);
 
     return streamToResponse(stream, res, {}, vercelStreamData);
   } catch (error) {

templates/types/streaming/express/src/controllers/engine/settings.ts

@@ -45,7 +45,9 @@ export const initSettings = async () => {
 function initOpenAI() {
   Settings.llm = new OpenAI({
     model: process.env.MODEL ?? "gpt-3.5-turbo",
-    maxTokens: 512,
+    maxTokens: process.env.LLM_MAX_TOKENS
+      ? Number(process.env.LLM_MAX_TOKENS)
+      : undefined,
   });
   Settings.embedModel = new OpenAIEmbedding({
     model: process.env.EMBEDDING_MODEL,

templates/types/streaming/express/src/controllers/llamaindex-stream.ts

@@ -1,4 +1,5 @@
 import {
+  JSONValue,
   StreamData,
   createCallbacksTransformer,
   createStreamDataTransformer,
@@ -7,6 +8,7 @@ import {
 } from "ai";
 import {
   MessageContent,
+  MessageContentDetail,
   Metadata,
   NodeWithScore,
   Response,
@@ -14,53 +16,72 @@ import {
 } from "llamaindex";
 
 import { AgentStreamChatResponse } from "llamaindex/agent/base";
-import {
-  CsvFile,
-  appendCsvData,
-  appendImageData,
-  appendSourceData,
-} from "./stream-helper";
+import { CsvFile, appendSourceData } from "./stream-helper";
 
 type LlamaIndexResponse =
   | AgentStreamChatResponse<ToolCallLLMMessageOptions>
   | Response;
 
-export type DataParserOptions = {
-  imageUrl?: string;
-  csvFiles?: CsvFile[];
-};
-
 export const convertMessageContent = (
-  textMessage: string,
-  additionalData?: DataParserOptions,
+  content: string,
+  annotations?: JSONValue[],
 ): MessageContent => {
-  if (!additionalData) return textMessage;
-  const content: MessageContent = [
+  if (!annotations) return content;
+  return [
     {
       type: "text",
-      text: textMessage,
+      text: content,
     },
+    ...convertAnnotations(annotations),
   ];
-  if (additionalData?.imageUrl) {
-    content.push({
-      type: "image_url",
-      image_url: {
-        url: additionalData?.imageUrl,
-      },
-    });
-  }
+};
 
-  if (additionalData?.csvFiles?.length) {
-    const rawContents = additionalData.csvFiles.map((csv) => {
-      return "```csv\n" + csv.content + "\n```";
-    });
-    const csvContent =
-      "Use data from following CSV raw contents:\n" + rawContents.join("\n\n");
-    content.push({
-      type: "text",
-      text: `${csvContent}\n\n${textMessage}`,
-    });
-  }
+const convertAnnotations = (
+  annotations: JSONValue[],
+): MessageContentDetail[] => {
+  const content: MessageContentDetail[] = [];
+  annotations.forEach((annotation: JSONValue) => {
+    // first skip invalid annotation
+    if (
+      !(
+        annotation &&
+        typeof annotation === "object" &&
+        "type" in annotation &&
+        "data" in annotation &&
+        annotation.data &&
+        typeof annotation.data === "object"
+      )
+    ) {
+      console.log(
+        "Client sent invalid annotation. Missing data and type",
+        annotation,
+      );
+      return;
+    }
+    const { type, data } = annotation;
+    // convert image
+    if (type === "image" && "url" in data && typeof data.url === "string") {
+      content.push({
+        type: "image_url",
+        image_url: {
+          url: data.url,
+        },
+      });
+    }
+    // convert CSV files to text
+    if (type === "csv" && "csvFiles" in data && Array.isArray(data.csvFiles)) {
+      const rawContents = data.csvFiles.map((csv) => {
+        return "```csv\n" + (csv as CsvFile).content + "\n```";
+      });
+      const csvContent =
+        "Use data from following CSV raw contents:\n" +
+        rawContents.join("\n\n");
+      content.push({
+        type: "text",
+        text: csvContent,
+      });
+    }
+  });
 
   return content;
 };
@@ -68,17 +89,12 @@ export const convertMessageContent = (
 function createParser(
   res: AsyncIterable<LlamaIndexResponse>,
   data: StreamData,
-  opts?: DataParserOptions,
 ) {
   const it = res[Symbol.asyncIterator]();
   const trimStartOfStream = trimStartOfStreamHelper();
 
   let sourceNodes: NodeWithScore<Metadata>[] | undefined;
   return new ReadableStream<string>({
-    start() {
-      appendImageData(data, opts?.imageUrl);
-      appendCsvData(data, opts?.csvFiles);
-    },
     async pull(controller): Promise<void> {
       const { value, done } = await it.next();
       if (done) {
@@ -115,10 +131,9 @@ export function LlamaIndexStream(
   data: StreamData,
   opts?: {
     callbacks?: AIStreamCallbacksAndOptions;
-    parserOptions?: DataParserOptions;
   },
 ): ReadableStream<Uint8Array> {
-  return createParser(response, data, opts?.parserOptions)
+  return createParser(response, data)
     .pipeThrough(createCallbacksTransformer(opts?.callbacks))
     .pipeThrough(createStreamDataTransformer());
 }

templates/types/streaming/express/src/controllers/stream-helper.ts

@@ -7,16 +7,6 @@ import {
   ToolOutput,
 } from "llamaindex";
 
-export function appendImageData(data: StreamData, imageUrl?: string) {
-  if (!imageUrl) return;
-  data.appendMessageAnnotation({
-    type: "image",
-    data: {
-      url: imageUrl,
-    },
-  });
-}
-
 function getNodeUrl(metadata: Metadata) {
   const url = metadata["URL"];
   if (url) return url;
@@ -128,13 +118,3 @@ export type CsvFile = {
   filesize: number;
   id: string;
 };
-
-export function appendCsvData(data: StreamData, csvFiles?: CsvFile[]) {
-  if (!csvFiles) return;
-  data.appendMessageAnnotation({
-    type: "csv",
-    data: {
-      csvFiles,
-    },
-  });
-}

templates/types/streaming/fastapi/app/api/routers/chat.py

@@ -28,13 +28,59 @@ async def chat(
     data: ChatData,
     chat_engine: BaseChatEngine = Depends(get_chat_engine),
 ):
-    last_message_content = data.get_last_message_content()
-    messages = data.get_history_messages()
-
-    event_handler = EventCallbackHandler()
-    chat_engine.callback_manager.handlers.append(event_handler)  # type: ignore
     try:
-        response = await chat_engine.astream_chat(last_message_content, messages)
+        last_message_content = data.get_last_message_content()
+        messages = data.get_history_messages()
+
+        event_handler = EventCallbackHandler()
+        chat_engine.callback_manager.handlers.append(event_handler)  # type: ignore
+
+        async def content_generator():
+            # Yield the text response
+            async def _chat_response_generator():
+                response = await chat_engine.astream_chat(
+                    last_message_content, messages
+                )
+                async for token in response.async_response_gen():
+                    yield VercelStreamResponse.convert_text(token)
+                # the text_generator is the leading stream, once it's finished, also finish the event stream
+                event_handler.is_done = True
+
+                # Yield the source nodes
+                yield VercelStreamResponse.convert_data(
+                    {
+                        "type": "sources",
+                        "data": {
+                            "nodes": [
+                                SourceNodes.from_source_node(node).dict()
+                                for node in response.source_nodes
+                            ]
+                        },
+                    }
+                )
+
+            # Yield the events from the event handler
+            async def _event_generator():
+                async for event in event_handler.async_event_gen():
+                    event_response = event.to_response()
+                    if event_response is not None:
+                        yield VercelStreamResponse.convert_data(event_response)
+
+            combine = stream.merge(_chat_response_generator(), _event_generator())
+            is_stream_started = False
+            async with combine.stream() as streamer:
+                async for output in streamer:
+                    if not is_stream_started:
+                        is_stream_started = True
+                        # Stream a blank message to start the stream
+                        yield VercelStreamResponse.convert_text("")
+
+                    yield output
+
+                    if await request.is_disconnected():
+                        break
+
+        return VercelStreamResponse(content=content_generator())
     except Exception as e:
         logger.exception("Error in chat engine", exc_info=True)
         raise HTTPException(
@@ -42,48 +88,6 @@ async def chat(
             detail=f"Error in chat engine: {e}",
         ) from e
 
-    async def content_generator():
-        # Yield the additional data
-        if data.data is not None:
-            for data_response in data.get_additional_data_response():
-                yield VercelStreamResponse.convert_data(data_response)
-
-        # Yield the text response
-        async def _text_generator():
-            async for token in response.async_response_gen():
-                yield VercelStreamResponse.convert_text(token)
-            # the text_generator is the leading stream, once it's finished, also finish the event stream
-            event_handler.is_done = True
-
-        # Yield the events from the event handler
-        async def _event_generator():
-            async for event in event_handler.async_event_gen():
-                event_response = event.to_response()
-                if event_response is not None:
-                    yield VercelStreamResponse.convert_data(event_response)
-
-        combine = stream.merge(_text_generator(), _event_generator())
-        async with combine.stream() as streamer:
-            async for item in streamer:
-                if await request.is_disconnected():
-                    break
-                yield item
-
-        # Yield the source nodes
-        yield VercelStreamResponse.convert_data(
-            {
-                "type": "sources",
-                "data": {
-                    "nodes": [
-                        SourceNodes.from_source_node(node).dict()
-                        for node in response.source_nodes
-                    ]
-                },
-            }
-        )
-
-    return VercelStreamResponse(content=content_generator())
-
 
 # non-streaming endpoint - delete if not needed
 @r.post("/request")

templates/types/streaming/fastapi/app/api/routers/models.py

@@ -10,20 +10,14 @@ from llama_index.core.llms import ChatMessage, MessageRole
 logger = logging.getLogger("uvicorn")
 
 
-class Message(BaseModel):
-    role: MessageRole
-    content: str
-
-
 class CsvFile(BaseModel):
     content: str
     filename: str
     filesize: int
     id: str
-    type: str
 
 
-class DataParserOptions(BaseModel):
+class AnnotationData(BaseModel):
     csv_files: List[CsvFile] | None = Field(
         default=None,
         description="List of CSV files",
@@ -45,30 +39,28 @@ class DataParserOptions(BaseModel):
         }
         alias_generator = to_camel
 
-    def to_raw_content(self) -> str:
-        if self.csv_files is not None and len(self.csv_files) > 0:
-            return "Use data from following CSV raw contents" + "\n".join(
-                [f"```csv\n{csv_file.content}\n```" for csv_file in self.csv_files]
-            )
 
-    def to_response_data(self) -> list[dict] | None:
-        output = []
-        if self.csv_files is not None and len(self.csv_files) > 0:
-            output.append(
-                {
-                    "type": "csv",
-                    "data": {
-                        "csvFiles": [csv_file.dict() for csv_file in self.csv_files]
-                    },
-                }
-            )
-        return output if len(output) > 0 else None
+class Annotation(BaseModel):
+    type: str
+    data: AnnotationData
+
+    def to_content(self) -> str:
+        if self.type == "csv":
+            csv_files = self.data.csv_files
+            if csv_files is not None and len(csv_files) > 0:
+                return "Use data from following CSV raw contents\n" + "\n".join(
+                    [f"```csv\n{csv_file.content}\n```" for csv_file in csv_files]
+                )
+        raise ValueError(f"Unsupported annotation type: {self.type}")
+
+
+class Message(BaseModel):
+    role: MessageRole
+    content: str
+    annotations: List[Annotation] | None = None
 
 
 class ChatData(BaseModel):
-    data: DataParserOptions | None = Field(
-        default=None,
-    )
     messages: List[Message]
 
     class Config:
@@ -91,11 +83,20 @@ class ChatData(BaseModel):
 
     def get_last_message_content(self) -> str:
         """
-        Get the content of the last message along with the data content if available
+        Get the content of the last message along with the data content if available. Fallback to use data content from previous messages
         """
-        message_content = self.messages[-1].content
-        if self.data:
-            message_content += "\n" + self.data.to_raw_content()
+        if len(self.messages) == 0:
+            raise ValueError("There is not any message in the chat")
+        last_message = self.messages[-1]
+        message_content = last_message.content
+        for message in reversed(self.messages):
+            if message.role == MessageRole.USER and message.annotations is not None:
+                annotation_contents = (
+                    annotation.to_content() for annotation in message.annotations
+                )
+                annotation_text = "\n".join(annotation_contents)
+                message_content = f"{message_content}\n{annotation_text}"
+                break
         return message_content
 
     def get_history_messages(self) -> List[Message]:
@@ -107,12 +108,6 @@ class ChatData(BaseModel):
             for message in self.messages[:-1]
         ]
 
-    def get_additional_data_response(self) -> list[dict] | None:
-        """
-        Get the additional data
-        """
-        return self.data.to_response_data()
-
     def is_last_message_from_user(self) -> bool:
         return self.messages[-1].role == MessageRole.USER
 

templates/types/streaming/nextjs/app/api/chat/engine/settings.ts

@@ -45,7 +45,9 @@ export const initSettings = async () => {
 function initOpenAI() {
   Settings.llm = new OpenAI({
     model: process.env.MODEL ?? "gpt-3.5-turbo",
-    maxTokens: 512,
+    maxTokens: process.env.LLM_MAX_TOKENS
+      ? Number(process.env.LLM_MAX_TOKENS)
+      : undefined,
   });
   Settings.embedModel = new OpenAIEmbedding({
     model: process.env.EMBEDDING_MODEL,

templates/types/streaming/nextjs/app/api/chat/llamaindex-stream.ts

@@ -1,4 +1,5 @@
 import {
+  JSONValue,
   StreamData,
   createCallbacksTransformer,
   createStreamDataTransformer,
@@ -7,6 +8,7 @@ import {
 } from "ai";
 import {
   MessageContent,
+  MessageContentDetail,
   Metadata,
   NodeWithScore,
   Response,
@@ -14,53 +16,72 @@ import {
 } from "llamaindex";
 
 import { AgentStreamChatResponse } from "llamaindex/agent/base";
-import {
-  CsvFile,
-  appendCsvData,
-  appendImageData,
-  appendSourceData,
-} from "./stream-helper";
+import { CsvFile, appendSourceData } from "./stream-helper";
 
 type LlamaIndexResponse =
   | AgentStreamChatResponse<ToolCallLLMMessageOptions>
   | Response;
 
-export type DataParserOptions = {
-  imageUrl?: string;
-  csvFiles?: CsvFile[];
-};
-
 export const convertMessageContent = (
-  textMessage: string,
-  additionalData?: DataParserOptions,
+  content: string,
+  annotations?: JSONValue[],
 ): MessageContent => {
-  if (!additionalData) return textMessage;
-  const content: MessageContent = [
+  if (!annotations) return content;
+  return [
     {
       type: "text",
-      text: textMessage,
+      text: content,
     },
+    ...convertAnnotations(annotations),
   ];
-  if (additionalData?.imageUrl) {
-    content.push({
-      type: "image_url",
-      image_url: {
-        url: additionalData?.imageUrl,
-      },
-    });
-  }
+};
 
-  if (additionalData?.csvFiles?.length) {
-    const rawContents = additionalData.csvFiles.map((csv) => {
-      return "```csv\n" + csv.content + "\n```";
-    });
-    const csvContent =
-      "Use data from following CSV raw contents:\n" + rawContents.join("\n\n");
-    content.push({
-      type: "text",
-      text: `${csvContent}\n\n${textMessage}`,
-    });
-  }
+const convertAnnotations = (
+  annotations: JSONValue[],
+): MessageContentDetail[] => {
+  const content: MessageContentDetail[] = [];
+  annotations.forEach((annotation: JSONValue) => {
+    // first skip invalid annotation
+    if (
+      !(
+        annotation &&
+        typeof annotation === "object" &&
+        "type" in annotation &&
+        "data" in annotation &&
+        annotation.data &&
+        typeof annotation.data === "object"
+      )
+    ) {
+      console.log(
+        "Client sent invalid annotation. Missing data and type",
+        annotation,
+      );
+      return;
+    }
+    const { type, data } = annotation;
+    // convert image
+    if (type === "image" && "url" in data && typeof data.url === "string") {
+      content.push({
+        type: "image_url",
+        image_url: {
+          url: data.url,
+        },
+      });
+    }
+    // convert CSV files to text
+    if (type === "csv" && "csvFiles" in data && Array.isArray(data.csvFiles)) {
+      const rawContents = data.csvFiles.map((csv) => {
+        return "```csv\n" + (csv as CsvFile).content + "\n```";
+      });
+      const csvContent =
+        "Use data from following CSV raw contents:\n" +
+        rawContents.join("\n\n");
+      content.push({
+        type: "text",
+        text: csvContent,
+      });
+    }
+  });
 
   return content;
 };
@@ -68,17 +89,12 @@ export const convertMessageContent = (
 function createParser(
   res: AsyncIterable<LlamaIndexResponse>,
   data: StreamData,
-  opts?: DataParserOptions,
 ) {
   const it = res[Symbol.asyncIterator]();
   const trimStartOfStream = trimStartOfStreamHelper();
 
   let sourceNodes: NodeWithScore<Metadata>[] | undefined;
   return new ReadableStream<string>({
-    start() {
-      appendImageData(data, opts?.imageUrl);
-      appendCsvData(data, opts?.csvFiles);
-    },
     async pull(controller): Promise<void> {
       const { value, done } = await it.next();
       if (done) {
@@ -115,10 +131,9 @@ export function LlamaIndexStream(
   data: StreamData,
   opts?: {
     callbacks?: AIStreamCallbacksAndOptions;
-    parserOptions?: DataParserOptions;
   },
 ): ReadableStream<Uint8Array> {
-  return createParser(response, data, opts?.parserOptions)
+  return createParser(response, data)
     .pipeThrough(createCallbacksTransformer(opts?.callbacks))
     .pipeThrough(createStreamDataTransformer());
 }

templates/types/streaming/nextjs/app/api/chat/route.ts

@@ -4,11 +4,7 @@ import { ChatMessage, Settings } from "llamaindex";
 import { NextRequest, NextResponse } from "next/server";
 import { createChatEngine } from "./engine/chat";
 import { initSettings } from "./engine/settings";
-import {
-  DataParserOptions,
-  LlamaIndexStream,
-  convertMessageContent,
-} from "./llamaindex-stream";
+import { LlamaIndexStream, convertMessageContent } from "./llamaindex-stream";
 import { createCallbackManager, createStreamTimeout } from "./stream-helper";
 
 initObservability();
@@ -24,10 +20,7 @@ export async function POST(request: NextRequest) {
 
   try {
     const body = await request.json();
-    const {
-      messages,
-      data,
-    }: { messages: Message[]; data: DataParserOptions | undefined } = body;
+    const { messages }: { messages: Message[] } = body;
     const userMessage = messages.pop();
     if (!messages || !userMessage || userMessage.role !== "user") {
       return NextResponse.json(
@@ -41,8 +34,24 @@ export async function POST(request: NextRequest) {
 
     const chatEngine = await createChatEngine();
 
+    let annotations = userMessage.annotations;
+    if (!annotations) {
+      // the user didn't send any new annotations with the last message
+      // so use the annotations from the last user message that has annotations
+      // REASON: GPT4 doesn't consider MessageContentDetail from previous messages, only strings
+      annotations = messages
+        .slice()
+        .reverse()
+        .find(
+          (message) => message.role === "user" && message.annotations,
+        )?.annotations;
+    }
+
     // Convert message content from Vercel/AI format to LlamaIndex/OpenAI format
-    const userMessageContent = convertMessageContent(userMessage.content, data);
+    const userMessageContent = convertMessageContent(
+      userMessage.content,
+      annotations,
+    );
 
     // Setup callbacks
     const callbackManager = createCallbackManager(vercelStreamData);
@@ -57,12 +66,7 @@ export async function POST(request: NextRequest) {
     });
 
     // Transform LlamaIndex stream to Vercel/AI format
-    const stream = LlamaIndexStream(response, vercelStreamData, {
-      parserOptions: {
-        imageUrl: data?.imageUrl,
-        csvFiles: data?.csvFiles,
-      },
-    });
+    const stream = LlamaIndexStream(response, vercelStreamData);
 
     // Return a StreamingTextResponse, which can be consumed by the Vercel/AI client
     return new StreamingTextResponse(stream, {}, vercelStreamData);

templates/types/streaming/nextjs/app/api/chat/stream-helper.ts

@@ -7,16 +7,6 @@ import {
   ToolOutput,
 } from "llamaindex";
 
-export function appendImageData(data: StreamData, imageUrl?: string) {
-  if (!imageUrl) return;
-  data.appendMessageAnnotation({
-    type: "image",
-    data: {
-      url: imageUrl,
-    },
-  });
-}
-
 function getNodeUrl(metadata: Metadata) {
   const url = metadata["URL"];
   if (url) return url;
@@ -128,13 +118,3 @@ export type CsvFile = {
   filesize: number;
   id: string;
 };
-
-export function appendCsvData(data: StreamData, csvFiles?: CsvFile[]) {
-  if (!csvFiles) return;
-  data.appendMessageAnnotation({
-    type: "csv",
-    data: {
-      csvFiles,
-    },
-  });
-}

templates/types/streaming/nextjs/app/components/chat-section.tsx

@@ -15,6 +15,7 @@ export default function ChatSection() {
     reload,
     stop,
     append,
+    setInput,
   } = useChat({
     api: chatAPI,
     headers: {
@@ -42,6 +43,8 @@ export default function ChatSection() {
         handleInputChange={handleInputChange}
         isLoading={isLoading}
         messages={messages}
+        append={append}
+        setInput={setInput}
       />
     </div>
   );

templates/types/streaming/nextjs/app/components/header.tsx

@@ -7,7 +7,7 @@ export default function Header() {
         Get started by editing 
         <code className="font-mono font-bold">app/page.tsx</code>
       </p>
-      <div className="fixed bottom-0 left-0 flex h-48 w-full items-end justify-center bg-gradient-to-t from-white via-white dark:from-black dark:via-black lg:static lg:h-auto lg:w-auto lg:bg-none">
+      <div className="fixed bottom-0 left-0 mb-4 flex h-auto w-full items-end justify-center bg-gradient-to-t from-white via-white dark:from-black dark:via-black lg:static lg:w-auto lg:bg-none lg:mb-0">
         <a
           href="https://www.llamaindex.ai/"
           className="flex items-center justify-center font-nunito text-lg font-bold gap-2"

templates/types/streaming/nextjs/app/components/ui/chat/chat-input.tsx

@@ -1,6 +1,7 @@
-import { Loader2 } from "lucide-react";
+import { JSONValue } from "ai";
 import { useState } from "react";
 import { v4 as uuidv4 } from "uuid";
+import { MessageAnnotation, MessageAnnotationType } from ".";
 import { Button } from "../button";
 import FileUploader from "../file-uploader";
 import { Input } from "../input";
@@ -19,30 +20,62 @@ export default function ChatInput(
     | "handleSubmit"
     | "handleInputChange"
     | "messages"
+    | "setInput"
+    | "append"
   >,
 ) {
   const [imageUrl, setImageUrl] = useState<string | null>(null);
-  const { files, uploadNew, removeFile, resetUploadedFiles } = useCsv(
-    props.messages,
-  );
+  const { files: csvFiles, upload, remove, reset } = useCsv();
+
+  const getAnnotations = () => {
+    if (!imageUrl && csvFiles.length === 0) return undefined;
+    const annotations: MessageAnnotation[] = [];
+    if (imageUrl) {
+      annotations.push({
+        type: MessageAnnotationType.IMAGE,
+        data: { url: imageUrl },
+      });
+    }
+    if (csvFiles.length > 0) {
+      annotations.push({
+        type: MessageAnnotationType.CSV,
+        data: {
+          csvFiles: csvFiles.map((file) => ({
+            id: file.id,
+            content: file.content,
+            filename: file.filename,
+            filesize: file.filesize,
+          })),
+        },
+      });
+    }
+    return annotations as JSONValue[];
+  };
+
+  // default submit function does not handle including annotations in the message
+  // so we need to use append function to submit new message with annotations
+  const handleSubmitWithAnnotations = (
+    e: React.FormEvent<HTMLFormElement>,
+    annotations: JSONValue[] | undefined,
+  ) => {
+    e.preventDefault();
+    props.append!({
+      content: props.input,
+      role: "user",
+      createdAt: new Date(),
+      annotations,
+    });
+    props.setInput!("");
+  };
 
   const onSubmit = (e: React.FormEvent<HTMLFormElement>) => {
-    if (imageUrl) {
-      props.handleSubmit(e, {
-        data: { imageUrl: imageUrl },
-      });
-      setImageUrl(null);
+    const annotations = getAnnotations();
+    if (annotations) {
+      handleSubmitWithAnnotations(e, annotations);
+      imageUrl && setImageUrl(null);
+      csvFiles.length && reset();
       return;
     }
-
-    if (files.length > 0) {
-      props.handleSubmit(e, {
-        data: { csvFiles: files },
-      });
-      resetUploadedFiles();
-      return;
-    }
-
     props.handleSubmit(e);
   };
 
@@ -65,7 +98,7 @@ export default function ChatInput(
       reader.onload = () => resolve(reader.result as string);
       reader.onerror = (error) => reject(error);
     });
-    const isSuccess = uploadNew({
+    const isSuccess = upload({
       id: uuidv4(),
       content,
       filename: file.name,
@@ -82,6 +115,10 @@ export default function ChatInput(
         return await handleUploadImageFile(file);
       }
       if (file.type === "text/csv") {
+        if (csvFiles.length > 0) {
+          alert("You can only upload one csv file at a time.");
+          return;
+        }
         return await handleUploadCsvFile(file);
       }
       props.onFileUpload?.(file);
@@ -98,28 +135,17 @@ export default function ChatInput(
       {imageUrl && (
         <UploadImagePreview url={imageUrl} onRemove={onRemovePreviewImage} />
       )}
-      {files.length > 0 && (
+      {csvFiles.length > 0 && (
         <div className="flex gap-4 w-full overflow-auto py-2">
-          {props.isLoading ? (
-            <div className="flex gap-2 items-center">
-              <Loader2 className="h-4 w-4 animate-spin" />{" "}
-              <span>Handling csv files...</span>
-            </div>
-          ) : (
-            <>
-              {files.map((csv) => {
-                return (
-                  <UploadCsvPreview
-                    key={csv.id}
-                    filename={csv.filename}
-                    filesize={csv.filesize}
-                    onRemove={() => removeFile(csv)}
-                    isNew={csv.type === "new_upload"}
-                  />
-                );
-              })}
-            </>
-          )}
+          {csvFiles.map((csv) => {
+            return (
+              <UploadCsvPreview
+                key={csv.id}
+                csv={csv}
+                onRemove={() => remove(csv)}
+              />
+            );
+          })}
         </div>
       )}
       <div className="flex w-full items-start justify-between gap-4 ">
@@ -135,7 +161,7 @@ export default function ChatInput(
           onFileUpload={handleUploadFile}
           onFileError={props.onFileError}
         />
-        <Button type="submit" disabled={props.isLoading}>
+        <Button type="submit" disabled={props.isLoading || !props.input.trim()}>
           Send message
         </Button>
       </div>

templates/types/streaming/nextjs/app/components/ui/chat/chat-message.tsx

@@ -60,7 +60,7 @@ function ChatMessageContent({
 
   const contents: ContentDisplayConfig[] = [
     {
-      order: -4,
+      order: 1,
       component: imageData[0] ? <ChatImage data={imageData[0]} /> : null,
     },
     {
@@ -71,7 +71,7 @@ function ChatMessageContent({
         ) : null,
     },
     {
-      order: -2,
+      order: 2,
       component: csvData[0] ? <CsvContent data={csvData[0]} /> : null,
     },
     {
@@ -83,7 +83,7 @@ function ChatMessageContent({
       component: <Markdown content={message.content} />,
     },
     {
-      order: 1,
+      order: 3,
       component: sourceData[0] ? <ChatSources data={sourceData[0]} /> : null,
     },
   ];

templates/types/streaming/nextjs/app/components/ui/chat/chat.interface.ts

@@ -15,7 +15,11 @@ export interface ChatHandler {
   stop?: () => void;
   onFileUpload?: (file: File) => Promise<void>;
   onFileError?: (errMsg: string) => void;
+  setInput?: (input: string) => void;
   append?: (
     message: Message | Omit<Message, "id">,
+    ops?: {
+      data: any;
+    },
   ) => Promise<string | null | undefined>;
 }

templates/types/streaming/nextjs/app/components/ui/chat/csv-content.tsx

@@ -1,16 +1,13 @@
 import { CsvData } from ".";
-import CsvDialog from "./widgets/CsvDialog";
+import UploadCsvPreview from "../upload-csv-preview";
 
 export default function CsvContent({ data }: { data: CsvData }) {
   if (!data.csvFiles.length) return null;
   return (
-    <div>
-      <p className="font-semibold mb-2">Using data from following CSV files:</p>
-      <div className="flex gap-2 items-center">
-        {data.csvFiles.map((csv, index) => (
-          <CsvDialog key={index} csv={csv} />
-        ))}
-      </div>
+    <div className="flex gap-2 items-center">
+      {data.csvFiles.map((csv, index) => (
+        <UploadCsvPreview key={index} csv={csv} />
+      ))}
     </div>
   );
 }

templates/types/streaming/nextjs/app/components/ui/chat/use-csv.ts

@@ -1,31 +1,10 @@
 "use client";
 
-import { Message } from "ai";
-import { useEffect, useMemo, useState } from "react";
-import {
-  CsvData,
-  CsvFile,
-  MessageAnnotation,
-  MessageAnnotationType,
-  getAnnotationData,
-} from ".";
+import { useState } from "react";
+import { CsvFile } from ".";
 
-interface FrontendCSVData extends CsvFile {
-  type: "available" | "new_upload";
-}
-
-export function useCsv(messages: Message[]) {
-  const [availableFiles, setAvailableFiles] = useState<FrontendCSVData[]>([]);
-  const [uploadedFiles, setUploadedFiles] = useState<FrontendCSVData[]>([]);
-
-  const files = useMemo(() => {
-    return [...availableFiles, ...uploadedFiles];
-  }, [availableFiles, uploadedFiles]);
-
-  useEffect(() => {
-    const items = getAvailableCsvFiles(messages);
-    setAvailableFiles(items.map((data) => ({ ...data, type: "available" })));
-  }, [messages]);
+export function useCsv() {
+  const [files, setFiles] = useState<CsvFile[]>([]);
 
   const csvEqual = (a: CsvFile, b: CsvFile) => {
     if (a.id === b.id) return true;
@@ -33,53 +12,22 @@ export function useCsv(messages: Message[]) {
     return false;
   };
 
-  // Get available csv files from annotations chat history
-  // returns the unique csv files by id
-  const getAvailableCsvFiles = (messages: Message[]): Array<CsvFile> => {
-    const docHash: Record<string, CsvFile> = {};
-    messages.forEach((message) => {
-      if (message.annotations) {
-        const csvData = getAnnotationData<CsvData>(
-          message.annotations as MessageAnnotation[],
-          MessageAnnotationType.CSV,
-        );
-        csvData.forEach((data) => {
-          data.csvFiles.forEach((file) => {
-            if (!docHash[file.id]) {
-              docHash[file.id] = file;
-            }
-          });
-        });
-      }
-    });
-    return Object.values(docHash);
-  };
-
-  const uploadNew = (file: CsvFile) => {
+  const upload = (file: CsvFile) => {
     const existedCsv = files.find((f) => csvEqual(f, file));
     if (!existedCsv) {
-      setUploadedFiles((prev) => [...prev, { ...file, type: "new_upload" }]);
+      setFiles((prev) => [...prev, file]);
       return true;
     }
     return false;
   };
 
-  const removeFile = (file: FrontendCSVData) => {
-    if (file.type === "new_upload") {
-      setUploadedFiles((prev) => prev.filter((f) => f.id !== file.id));
-    } else {
-      setAvailableFiles((prev) => prev.filter((f) => f.id !== file.id));
-    }
+  const remove = (file: CsvFile) => {
+    setFiles((prev) => prev.filter((f) => f.id !== file.id));
   };
 
-  const resetUploadedFiles = () => {
-    setUploadedFiles([]);
+  const reset = () => {
+    setFiles([]);
   };
 
-  return {
-    files,
-    uploadNew,
-    removeFile,
-    resetUploadedFiles,
-  };
+  return { files, upload, remove, reset };
 }

templates/types/streaming/nextjs/app/components/ui/chat/widgets/CsvDialog.tsx

@@ -1,62 +0,0 @@
-import Image from "next/image";
-import { CsvFile } from "..";
-import SheetIcon from "../../../ui/icons/sheet.svg";
-import { Button } from "../../button";
-import {
-  Drawer,
-  DrawerClose,
-  DrawerContent,
-  DrawerDescription,
-  DrawerHeader,
-  DrawerTitle,
-  DrawerTrigger,
-} from "../../drawer";
-
-export interface CsvDialogProps {
-  csv: CsvFile;
-}
-
-export default function CsvDialog(props: CsvDialogProps) {
-  const { filename, filesize, content } = props.csv;
-  const fileSizeInKB = Math.round((filesize / 1024) * 10) / 10;
-  return (
-    <Drawer direction="left">
-      <DrawerTrigger asChild>
-        <div
-          className="border-2 border-green-700 py-1.5 px-3 rounded-lg flex gap-2 items-center cursor-pointer text-sm hover:bg-green-700 hover:text-white transition-colors duration-200 ease-in-out"
-          key={filename}
-        >
-          <div className="h-4 w-4 shrink-0 rounded-md">
-            <Image
-              className="h-full w-auto"
-              priority
-              src={SheetIcon}
-              alt="SheetIcon"
-            />
-          </div>
-          <span>
-            {filename} - {fileSizeInKB} KB
-          </span>
-        </div>
-      </DrawerTrigger>
-      <DrawerContent className="w-3/5 mt-24 h-full max-h-[96%] ">
-        <DrawerHeader className="flex justify-between">
-          <div className="space-y-2">
-            <DrawerTitle>Csv Raw Content</DrawerTitle>
-            <DrawerDescription>
-              {filename} ({fileSizeInKB} KB)
-            </DrawerDescription>
-          </div>
-          <DrawerClose asChild>
-            <Button variant="outline">Close</Button>
-          </DrawerClose>
-        </DrawerHeader>
-        <div className="m-4 max-h-[80%] overflow-auto">
-          <pre className="bg-secondary rounded-md p-4 block text-sm">
-            {content}
-          </pre>
-        </div>
-      </DrawerContent>
-    </Drawer>
-  );
-}

templates/types/streaming/nextjs/app/components/ui/upload-csv-preview.tsx

@@ -1,22 +1,59 @@
 import { XCircleIcon } from "lucide-react";
 import Image from "next/image";
 import SheetIcon from "../ui/icons/sheet.svg";
+import { Button } from "./button";
+import { CsvFile } from "./chat";
+import {
+  Drawer,
+  DrawerClose,
+  DrawerContent,
+  DrawerDescription,
+  DrawerHeader,
+  DrawerTitle,
+  DrawerTrigger,
+} from "./drawer";
 import { cn } from "./lib/utils";
 
-export default function UploadCsvPreview({
-  filename,
-  filesize,
-  onRemove,
-  isNew,
-}: {
-  filename: string;
-  filesize: number;
-  onRemove: () => void;
-  isNew?: boolean;
-}) {
-  const fileSizeInKB = Math.round((filesize / 1024) * 10) / 10;
+export interface UploadCsvPreviewProps {
+  csv: CsvFile;
+  onRemove?: () => void;
+}
+
+export default function UploadCsvPreview(props: UploadCsvPreviewProps) {
+  const { filename, filesize, content } = props.csv;
   return (
-    <div className="p-2 w-60 max-w-60 bg-secondary rounded-lg text-sm relative">
+    <Drawer direction="left">
+      <DrawerTrigger asChild>
+        <div>
+          <CSVSummaryCard {...props} />
+        </div>
+      </DrawerTrigger>
+      <DrawerContent className="w-3/5 mt-24 h-full max-h-[96%] ">
+        <DrawerHeader className="flex justify-between">
+          <div className="space-y-2">
+            <DrawerTitle>Csv Raw Content</DrawerTitle>
+            <DrawerDescription>
+              {filename} ({inKB(filesize)} KB)
+            </DrawerDescription>
+          </div>
+          <DrawerClose asChild>
+            <Button variant="outline">Close</Button>
+          </DrawerClose>
+        </DrawerHeader>
+        <div className="m-4 max-h-[80%] overflow-auto">
+          <pre className="bg-secondary rounded-md p-4 block text-sm">
+            {content}
+          </pre>
+        </div>
+      </DrawerContent>
+    </Drawer>
+  );
+}
+
+function CSVSummaryCard(props: UploadCsvPreviewProps) {
+  const { onRemove, csv } = props;
+  return (
+    <div className="p-2 w-60 max-w-60 bg-secondary rounded-lg text-sm relative cursor-pointer">
       <div className="flex flex-row items-center gap-2">
         <div className="relative h-10 w-10 shrink-0 overflow-hidden rounded-md">
           <Image
@@ -28,28 +65,29 @@ export default function UploadCsvPreview({
         </div>
         <div className="overflow-hidden">
           <div className="truncate font-semibold">
-            {filename} ({fileSizeInKB} KB)
+            {csv.filename} ({inKB(csv.filesize)} KB)
           </div>
           <div className="truncate text-token-text-tertiary flex items-center gap-2">
             <span>Spreadsheet</span>
-            {isNew && (
-              <span className="px-2 py-0.5 bg-red-400 text-white text-xs rounded-2xl">
-                new
-              </span>
-            )}
           </div>
         </div>
       </div>
-      <div
-        className={cn(
-          "absolute -top-2 -right-2 w-6 h-6 z-10 bg-gray-500 text-white rounded-full",
-        )}
-      >
-        <XCircleIcon
-          className="w-6 h-6 bg-gray-500 text-white rounded-full"
-          onClick={onRemove}
-        />
-      </div>
+      {onRemove && (
+        <div
+          className={cn(
+            "absolute -top-2 -right-2 w-6 h-6 z-10 bg-gray-500 text-white rounded-full",
+          )}
+        >
+          <XCircleIcon
+            className="w-6 h-6 bg-gray-500 text-white rounded-full"
+            onClick={onRemove}
+          />
+        </div>
+      )}
     </div>
   );
 }
+
+function inKB(size: number) {
+  return Math.round((size / 1024) * 10) / 10;
+}

templates/types/streaming/nextjs/package.json

@@ -18,7 +18,7 @@
     "class-variance-authority": "^0.7.0",
     "clsx": "^2.1.1",
     "dotenv": "^16.3.1",
-    "llamaindex": "0.3.13",
+    "llamaindex": "0.3.16",
     "lucide-react": "^0.294.0",
     "next": "^14.0.3",
     "pdf2json": "3.0.5",