Release 0.1.21 (#680)

Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>

packages/create-llama/CHANGELOG.md

@@ -1,5 +1,12 @@
 # create-llama
 
+## 0.5.21
+
+### Patch Changes
+
+- af9ad3c: feat: show document artifact after generating report
+- a543a27: feat: bump chat-ui with inline artifact
+
 ## 0.5.20
 
 ### Patch Changes

packages/create-llama/package.json

@@ -1,6 +1,6 @@
 {
   "name": "create-llama",
-  "version": "0.5.20",
+  "version": "0.5.21",
   "description": "Create LlamaIndex-powered apps with one command",
   "keywords": [
     "rag",

packages/create-llama/templates/components/use-cases/python/deep_research/workflow.py

@@ -23,7 +23,18 @@ from llama_index.core.workflow import (
     Workflow,
     step,
 )
-from llama_index.server.api.models import ChatRequest, SourceNodesEvent, UIEvent
+from llama_index.server.api.models import (
+    ArtifactEvent,
+    ArtifactType,
+    ChatRequest,
+    SourceNodesEvent,
+    UIEvent,
+    Artifact,
+    DocumentArtifactData,
+    DocumentArtifactSource,
+)
+import time
+from llama_index.server.utils.stream import write_response_to_stream
 from pydantic import BaseModel, Field
 
 logger = logging.getLogger("uvicorn")
@@ -365,8 +376,31 @@ class DeepResearchWorkflow(Workflow):
             user_request=self.user_request,
             stream=self.stream,
         )
+
+        final_response = await write_response_to_stream(res, ctx)
+
+        ctx.write_event_to_stream(
+            ArtifactEvent(
+                data=Artifact(
+                    type=ArtifactType.DOCUMENT,
+                    created_at=int(time.time()),
+                    data=DocumentArtifactData(
+                        title="DeepResearch Report",
+                        content=final_response,
+                        type="markdown",
+                        sources=[
+                            DocumentArtifactSource(
+                                id=node.id_,
+                            )
+                            for node in self.context_nodes
+                        ],
+                    ),
+                ),
+            )
+        )
+
         return StopEvent(
-            result=res,
+            result="",
         )
 
 

packages/create-llama/templates/components/use-cases/typescript/code_generator/src/app/workflow.ts

@@ -1,4 +1,4 @@
-import { extractLastArtifact } from "@llamaindex/server";
+import { artifactEvent, extractLastArtifact } from "@llamaindex/server";
 import { ChatMemoryBuffer, MessageContent, Settings } from "llamaindex";
 
 import {
@@ -52,19 +52,6 @@ const synthesizeAnswerEvent = workflowEvent<object>();
 
 const uiEvent = workflowEvent<UIEvent>();
 
-const artifactEvent = workflowEvent<{
-  type: "artifact";
-  data: {
-    type: "code";
-    created_at: number;
-    data: {
-      language: string;
-      file_name: string;
-      code: string;
-    };
-  };
-}>();
-
 export function workflowFactory(reqBody: any) {
   const llm = Settings.llm;
 

packages/create-llama/templates/components/use-cases/typescript/deep_research/src/app/workflow.ts

@@ -1,4 +1,4 @@
-import { toSourceEvent } from "@llamaindex/server";
+import { artifactEvent, toSourceEvent } from "@llamaindex/server";
 import {
   agentStreamEvent,
   createStatefulMiddleware,
@@ -339,6 +339,26 @@ export function getWorkflow(index: VectorStoreIndex | LlamaCloudIndex) {
         }),
       );
     }
+
+    // Open the generated report in Canvas
+    sendEvent(
+      artifactEvent.with({
+        type: "artifact",
+        data: {
+          type: "document",
+          created_at: Date.now(),
+          data: {
+            title: "DeepResearch Report",
+            content: response,
+            type: "markdown",
+            sources: state.contextNodes.map((node) => ({
+              id: node.node.id_,
+            })),
+          },
+        },
+      }),
+    );
+
     return stopAgentEvent.with({
       result: response,
     });

packages/create-llama/templates/components/use-cases/typescript/document_generator/src/app/workflow.ts

@@ -1,4 +1,4 @@
-import { extractLastArtifact } from "@llamaindex/server";
+import { artifactEvent, extractLastArtifact } from "@llamaindex/server";
 import { ChatMemoryBuffer, MessageContent, Settings } from "llamaindex";
 
 import {
@@ -55,19 +55,6 @@ const synthesizeAnswerEvent = workflowEvent<{
 
 const uiEvent = workflowEvent<UIEvent>();
 
-const artifactEvent = workflowEvent<{
-  type: "artifact";
-  data: {
-    type: "document";
-    created_at: number;
-    data: {
-      title: string;
-      content: string;
-      type: "markdown" | "html";
-    };
-  };
-}>();
-
 export function workflowFactory(reqBody: any) {
   const llm = Settings.llm;
 

packages/server/CHANGELOG.md

@@ -1,5 +1,13 @@
 # @llamaindex/server
 
+## 0.2.7
+
+### Patch Changes
+
+- af9ad3c: feat: show document artifact after generating report
+- a543a27: feat: bump chat-ui with inline artifact
+- 1ff6eaf: Add support for chat upload file
+
 ## 0.2.6
 
 ### Patch Changes

packages/server/README.md

@@ -60,6 +60,7 @@ The `LlamaIndexServer` accepts the following configuration options:
 - `workflow`: A callable function that creates a workflow instance for each request. See [Workflow factory contract](#workflow-factory-contract) for more details.
 - `uiConfig`: An object to configure the chat UI containing the following properties:
   - `starterQuestions`: List of starter questions for the chat UI (default: `[]`)
+  - `enableFileUpload`: Whether to enable file upload in the chat UI (default: `false`). See [Upload file example](./examples/private-file/README.md) for more details.
   - `componentsDir`: The directory for custom UI components rendering events emitted by the workflow. The default is undefined, which does not render custom UI components.
   - `layoutDir`: The directory for custom layout sections. The default value is `layout`. See [Custom Layout](#custom-layout) for more details.
   - `llamaCloudIndexSelector`: Whether to show the LlamaCloud index selector in the chat UI (requires `LLAMA_CLOUD_API_KEY` to be set in the environment variables) (default: `false`)
@@ -71,9 +72,18 @@ See all Nextjs Custom Server options [here](https://nextjs.org/docs/app/building
 
 ## Workflow factory contract
 
-The `workflow` provided will be called for each chat request to initialize a new workflow instance. The contract of the generated workflow must be the same as for the [Agent Workflow](https://ts.llamaindex.ai/docs/llamaindex/modules/agents/agent_workflow).
+The `workflow` provided will be called for each chat request to initialize a new workflow instance. For advanced use cases, you can define workflowFactory with a chatBody which include list of UI messages in the request body.
 
-This means that the workflow must handle a `startAgentEvent` event, which is the entry point of the workflow and contains the following information in it's `data` property:
+```typescript
+import { type Message } from "ai";
+import { agent } from "@llamaindex/workflow";
+
+const workflowFactory = (chatBody: { messages: Message[] }) => {
+  ...
+};
+```
+
+The contract of the generated workflow must be the same as for the [Agent Workflow](https://ts.llamaindex.ai/docs/llamaindex/modules/agents/agent_workflow). This means that the workflow must handle a `startAgentEvent` event, which is the entry point of the workflow and contains the following information in it's `data` property:
 
 ```typescript
 {

packages/server/examples/README.md

@@ -1,12 +1,38 @@
 # LlamaIndex Server Examples
 
-This directory contains examples of how to use the LlamaIndex Server.
+This directory provides example projects demonstrating how to use the LlamaIndex Server.
 
-## Running the examples
+## How to Run the Examples
 
-```bash
-export OPENAI_API_KEY=your_openai_api_key
-pnpm run dev
-```
+1. **Install dependencies**
 
-## Open browser at http://localhost:3000
+   In the root of this directory, run:
+
+   ```bash
+   pnpm install
+   ```
+
+2. **Set your OpenAI API key**
+
+   Export your OpenAI API key as an environment variable:
+
+   ```bash
+   export OPENAI_API_KEY=your_openai_api_key
+   ```
+
+3. **Start an example**
+
+   Replace `<example>` with the name of the example you want to run (e.g., `private-file`):
+
+   ```bash
+   pnpm nodemon --exec tsx <example>/index.ts
+   ```
+
+4. **Open the application in your browser**
+
+   Visit [http://localhost:3000](http://localhost:3000) to interact with the running example.
+
+## Notes
+
+- Make sure you have [pnpm](https://pnpm.io/) installed.
+- Each example may have its own specific instructions or requirements; check the individual example's index.ts for details.

packages/server/examples/agentic-rag/index.ts

@@ -1,12 +1,7 @@
+import { OpenAI, OpenAIEmbedding } from "@llamaindex/openai";
 import { LlamaIndexServer } from "@llamaindex/server";
 import { agent } from "@llamaindex/workflow";
-import {
-  Document,
-  OpenAI,
-  OpenAIEmbedding,
-  Settings,
-  VectorStoreIndex,
-} from "llamaindex";
+import { Document, Settings, VectorStoreIndex } from "llamaindex";
 
 Settings.llm = new OpenAI({
   model: "gpt-4o-mini",

packages/server/examples/code-gen/README.md

@@ -0,0 +1,22 @@
+This example demonstrates how to use the code generation workflow.
+
+```ts
+new LlamaIndexServer({
+  workflow: workflowFactory,
+  uiConfig: {
+    starterQuestions: [
+      "Generate a calculator app",
+      "Create a simple todo list app",
+    ],
+    componentsDir: "components",
+  },
+  port: 3000,
+}).start();
+```
+
+Export OpenAI API key and start the server in dev mode.
+
+```bash
+export OPENAI_API_KEY=<your-openai-api-key>
+npx nodemon --exec tsx index.ts
+```

packages/server/examples/code-gen/components/ui_event.jsx

@@ -0,0 +1,132 @@
+import { Badge } from "@/components/ui/badge";
+import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card";
+import { Progress } from "@/components/ui/progress";
+import { Skeleton } from "@/components/ui/skeleton";
+import { cn } from "@/lib/utils";
+import { Markdown } from "@llamaindex/chat-ui/widgets";
+import { ListChecks, Loader2, Wand2 } from "lucide-react";
+import { useEffect, useState } from "react";
+
+const STAGE_META = {
+  plan: {
+    icon: ListChecks,
+    badgeText: "Step 1/2: Planning",
+    gradient: "from-blue-100 via-blue-50 to-white",
+    progress: 33,
+    iconBg: "bg-blue-100 text-blue-600",
+    badge: "bg-blue-100 text-blue-700",
+  },
+  generate: {
+    icon: Wand2,
+    badgeText: "Step 2/2: Generating",
+    gradient: "from-violet-100 via-violet-50 to-white",
+    progress: 66,
+    iconBg: "bg-violet-100 text-violet-600",
+    badge: "bg-violet-100 text-violet-700",
+  },
+};
+
+function ArtifactWorkflowCard({ event }) {
+  const [visible, setVisible] = useState(event?.state !== "completed");
+  const [fade, setFade] = useState(false);
+
+  useEffect(() => {
+    if (event?.state === "completed") {
+      setVisible(false);
+    } else {
+      setVisible(true);
+      setFade(false);
+    }
+  }, [event?.state]);
+
+  if (!event || !visible) return null;
+
+  const { state, requirement } = event;
+  const meta = STAGE_META[state];
+
+  if (!meta) return null;
+
+  return (
+    <div className="flex min-h-[180px] w-full items-center justify-center py-2">
+      <Card
+        className={cn(
+          "w-full rounded-xl shadow-md transition-all duration-500",
+          "border-0",
+          fade && "pointer-events-none opacity-0",
+          `bg-gradient-to-br ${meta.gradient}`,
+        )}
+        style={{
+          boxShadow:
+            "0 2px 12px 0 rgba(80, 80, 120, 0.08), 0 1px 3px 0 rgba(80, 80, 120, 0.04)",
+        }}
+      >
+        <CardHeader className="flex flex-row items-center gap-2 px-3 pb-1 pt-2">
+          <div
+            className={cn(
+              "flex items-center justify-center rounded-full p-1",
+              meta.iconBg,
+            )}
+          >
+            <meta.icon className="h-5 w-5" />
+          </div>
+          <CardTitle className="flex items-center gap-2 text-base font-semibold">
+            <Badge className={cn("ml-1", meta.badge, "px-2 py-0.5 text-xs")}>
+              {meta.badgeText}
+            </Badge>
+          </CardTitle>
+        </CardHeader>
+        <CardContent className="px-3 py-1">
+          {state === "plan" && (
+            <div className="flex flex-col items-center gap-2 py-2">
+              <Loader2 className="mb-1 h-6 w-6 animate-spin text-blue-400" />
+              <div className="text-center text-sm font-medium text-blue-900">
+                Analyzing your request...
+              </div>
+              <Skeleton className="mt-1 h-3 w-1/2 rounded-full" />
+            </div>
+          )}
+          {state === "generate" && (
+            <div className="flex flex-col gap-2 py-2">
+              <div className="flex items-center gap-1">
+                <Loader2 className="h-4 w-4 animate-spin text-violet-400" />
+                <span className="text-sm font-medium text-violet-900">
+                  Working on the requirement:
+                </span>
+              </div>
+              <div className="max-h-24 overflow-auto rounded-lg border border-violet-200 bg-violet-50 px-2 py-1 text-xs">
+                {requirement ? (
+                  <Markdown content={requirement} />
+                ) : (
+                  <span className="italic text-violet-400">
+                    No requirements available yet.
+                  </span>
+                )}
+              </div>
+            </div>
+          )}
+        </CardContent>
+        <div className="px-3 pb-2 pt-1">
+          <Progress
+            value={meta.progress}
+            className={cn(
+              "h-1 rounded-full bg-gray-200",
+              state === "plan" && "bg-blue-200",
+              state === "generate" && "bg-violet-200",
+            )}
+          />
+        </div>
+      </Card>
+    </div>
+  );
+}
+
+export default function Component({ events }) {
+  const aggregateEvents = () => {
+    if (!events || events.length === 0) return null;
+    return events[events.length - 1];
+  };
+
+  const event = aggregateEvents();
+
+  return <ArtifactWorkflowCard event={event} />;
+}

packages/server/examples/code-gen/index.ts

@@ -0,0 +1,20 @@
+import { OpenAI } from "@llamaindex/openai";
+import { LlamaIndexServer } from "@llamaindex/server";
+import { Settings } from "llamaindex";
+import { workflowFactory } from "./src/app/workflow";
+
+Settings.llm = new OpenAI({
+  model: "gpt-4o-mini",
+});
+
+new LlamaIndexServer({
+  workflow: workflowFactory,
+  uiConfig: {
+    starterQuestions: [
+      "Generate a calculator app",
+      "Create a simple todo list app",
+    ],
+    componentsDir: "components",
+  },
+  port: 3000,
+}).start();

packages/server/examples/code-gen/src/app/workflow.ts

@@ -0,0 +1,337 @@
+import { artifactEvent, extractLastArtifact } from "@llamaindex/server";
+import { ChatMemoryBuffer, MessageContent, Settings } from "llamaindex";
+
+import {
+  agentStreamEvent,
+  createStatefulMiddleware,
+  createWorkflow,
+  startAgentEvent,
+  stopAgentEvent,
+  workflowEvent,
+} from "@llamaindex/workflow";
+
+import { z } from "zod";
+
+export const RequirementSchema = z.object({
+  next_step: z.enum(["answering", "coding"]),
+  language: z.string().nullable().optional(),
+  file_name: z.string().nullable().optional(),
+  requirement: z.string(),
+});
+
+export type Requirement = z.infer<typeof RequirementSchema>;
+
+export const UIEventSchema = z.object({
+  type: z.literal("ui_event"),
+  data: z.object({
+    state: z
+      .enum(["plan", "generate", "completed"])
+      .describe(
+        "The current state of the workflow: 'plan', 'generate', or 'completed'.",
+      ),
+    requirement: z
+      .string()
+      .optional()
+      .describe(
+        "An optional requirement creating or updating a code, if applicable.",
+      ),
+  }),
+});
+
+export type UIEvent = z.infer<typeof UIEventSchema>;
+const planEvent = workflowEvent<{
+  userInput: MessageContent;
+  context?: string | undefined;
+}>();
+
+const generateArtifactEvent = workflowEvent<{
+  requirement: Requirement;
+}>();
+
+const synthesizeAnswerEvent = workflowEvent<object>();
+
+const uiEvent = workflowEvent<UIEvent>();
+
+export function workflowFactory(reqBody: unknown) {
+  const llm = Settings.llm;
+
+  const { withState, getContext } = createStatefulMiddleware(() => {
+    return {
+      memory: new ChatMemoryBuffer({ llm }),
+      lastArtifact: extractLastArtifact(reqBody),
+    };
+  });
+  const workflow = withState(createWorkflow());
+
+  workflow.handle([startAgentEvent], async ({ data }) => {
+    const { userInput, chatHistory = [] } = data;
+    // Prepare chat history
+    const { state } = getContext();
+    // Put user input to the memory
+    if (!userInput) {
+      throw new Error("Missing user input to start the workflow");
+    }
+    state.memory.set(chatHistory);
+    state.memory.put({ role: "user", content: userInput });
+
+    return planEvent.with({
+      userInput: userInput,
+      context: state.lastArtifact
+        ? JSON.stringify(state.lastArtifact)
+        : undefined,
+    });
+  });
+
+  workflow.handle([planEvent], async ({ data: planData }) => {
+    const { sendEvent } = getContext();
+    const { state } = getContext();
+    sendEvent(
+      uiEvent.with({
+        type: "ui_event",
+        data: {
+          state: "plan",
+        },
+      }),
+    );
+    const user_msg = planData.userInput;
+    const context = planData.context
+      ? `## The context is: \n${planData.context}\n`
+      : "";
+    const prompt = `
+You are a product analyst responsible for analyzing the user's request and providing the next step for code or document generation.
+You are helping user with their code artifact. To update the code, you need to plan a coding step.
+
+Follow these instructions:
+1. Carefully analyze the conversation history and the user's request to determine what has been done and what the next step should be.
+2. The next step must be one of the following two options:
+    - "coding": To make the changes to the current code.
+    - "answering": If you don't need to update the current code or need clarification from the user.
+Important: Avoid telling the user to update the code themselves, you are the one who will update the code (by planning a coding step).
+3. If the next step is "coding", you may specify the language ("typescript" or "python") and file_name if known, otherwise set them to null. 
+4. The requirement must be provided clearly what is the user request and what need to be done for the next step in details
+    as precise and specific as possible, don't be stingy with in the requirement.
+5. If the next step is "answering", set language and file_name to null, and the requirement should describe what to answer or explain to the user.
+6. Be concise; only return the requirements for the next step.
+7. The requirements must be in the following format:
+    \`\`\`json
+    {
+        "next_step": "answering" | "coding",
+        "language": "typescript" | "python" | null,
+        "file_name": string | null,
+        "requirement": string
+    }
+    \`\`\`
+
+## Example 1:
+User request: Create a calculator app.
+You should return:
+\`\`\`json
+{
+    "next_step": "coding",
+    "language": "typescript",
+    "file_name": "calculator.tsx",
+    "requirement": "Generate code for a calculator app that has a simple UI with a display and button layout. The display should show the current input and the result. The buttons should include basic operators, numbers, clear, and equals. The calculation should work correctly."
+}
+\`\`\`
+
+## Example 2:
+User request: Explain how the game loop works.
+Context: You have already generated the code for a snake game.
+You should return:
+\`\`\`json
+{
+    "next_step": "answering",
+    "language": null,
+    "file_name": null,
+    "requirement": "The user is asking about the game loop. Explain how the game loop works."
+}
+\`\`\`
+
+${context}
+
+Now, plan the user's next step for this request:
+${user_msg}
+`;
+
+    const response = await llm.complete({
+      prompt,
+    });
+    // parse the response to Requirement
+    // 1. use regex to find the json block
+    const jsonBlock = response.text.match(/```json\s*([\s\S]*?)\s*```/);
+    if (!jsonBlock) {
+      throw new Error("No JSON block found in the response.");
+    }
+    const requirement = RequirementSchema.parse(JSON.parse(jsonBlock[1]));
+    state.memory.put({
+      role: "assistant",
+      content: `The plan for next step: \n${response.text}`,
+    });
+
+    if (requirement.next_step === "coding") {
+      return generateArtifactEvent.with({
+        requirement,
+      });
+    } else {
+      return synthesizeAnswerEvent.with({});
+    }
+  });
+
+  workflow.handle([generateArtifactEvent], async ({ data: planData }) => {
+    const { sendEvent } = getContext();
+    const { state } = getContext();
+
+    sendEvent(
+      uiEvent.with({
+        type: "ui_event",
+        data: {
+          state: "generate",
+          requirement: planData.requirement.requirement,
+        },
+      }),
+    );
+
+    const previousArtifact = state.lastArtifact
+      ? JSON.stringify(state.lastArtifact)
+      : "There is no previous artifact";
+    const requirementText = planData.requirement.requirement;
+
+    const prompt = `
+        You are a skilled developer who can help user with coding.
+        You are given a task to generate or update a code for a given requirement.
+
+        ## Follow these instructions:
+        **1. Carefully read the user's requirements.** 
+           If any details are ambiguous or missing, make reasonable assumptions and clearly reflect those in your output.
+           If the previous code is provided:
+           + Carefully analyze the code with the request to make the right changes.
+           + Avoid making a lot of changes from the previous code if the request is not to write the code from scratch again.
+        **2. For code requests:**
+           - If the user does not specify a framework or language, default to a React component using the Next.js framework.
+           - For Next.js, use Shadcn UI components, Typescript, @types/node, @types/react, @types/react-dom, PostCSS, and TailwindCSS.
+           The import pattern should be:
+           \`\`\`typescript
+           import { ComponentName } from "@/components/ui/component-name"
+           import { Markdown } from "@llamaindex/chat-ui"
+           import { cn } from "@/lib/utils"
+           \`\`\`
+           - Ensure the code is idiomatic, production-ready, and includes necessary imports.
+           - Only generate code relevant to the user's request—do not add extra boilerplate.
+        **3. Don't be verbose on response**
+           - No other text or comments only return the code which wrapped by \`\`\`language\`\`\` block.
+           - If the user's request is to update the code, only return the updated code.
+        **4. Only the following languages are allowed: "typescript", "python".**
+        **5. If there is no code to update, return the reason without any code block.**
+           
+        ## Example:
+        \`\`\`typescript
+        import React from "react";
+        import { Button } from "@/components/ui/button";
+        import { cn } from "@/lib/utils";
+
+        export default function MyComponent() {
+        return (
+           <div className="flex flex-col items-center justify-center h-screen">
+              <Button>Click me</Button>
+           </div>
+        );
+        }
+        \`\`\`
+
+        The previous code is:
+        {previousArtifact}
+
+        Now, i have to generate the code for the following requirement:
+        {requirement}
+      `
+      .replace("{previousArtifact}", previousArtifact)
+      .replace("{requirement}", requirementText);
+
+    const response = await llm.complete({
+      prompt,
+    });
+
+    // Extract the code from the response
+    const codeMatch = response.text.match(/```(\w+)([\s\S]*)```/);
+    if (!codeMatch) {
+      return synthesizeAnswerEvent.with({});
+    }
+
+    const code = codeMatch[2].trim();
+
+    // Put the generated code to the memory
+    state.memory.put({
+      role: "assistant",
+      content: `Updated the code: \n${response.text}`,
+    });
+
+    // To show the Canvas panel for the artifact
+    sendEvent(
+      artifactEvent.with({
+        type: "artifact",
+        data: {
+          type: "code",
+          created_at: Date.now(),
+          data: {
+            language: planData.requirement.language || "",
+            file_name: planData.requirement.file_name || "",
+            code,
+          },
+        },
+      }),
+    );
+
+    return synthesizeAnswerEvent.with({});
+  });
+
+  workflow.handle([synthesizeAnswerEvent], async () => {
+    const { sendEvent } = getContext();
+    const { state } = getContext();
+
+    const chatHistory = await state.memory.getMessages();
+    const messages = [
+      ...chatHistory,
+      {
+        role: "system" as const,
+        content: `
+        You are a helpful assistant who is responsible for explaining the work to the user.
+        Based on the conversation history, provide an answer to the user's question. 
+        The user has access to the code so avoid mentioning the whole code again in your response.
+      `,
+      },
+    ];
+
+    const responseStream = await llm.chat({
+      messages,
+      stream: true,
+    });
+
+    sendEvent(
+      uiEvent.with({
+        type: "ui_event",
+        data: {
+          state: "completed",
+        },
+      }),
+    );
+
+    let response = "";
+    for await (const chunk of responseStream) {
+      response += chunk.delta;
+      sendEvent(
+        agentStreamEvent.with({
+          delta: chunk.delta,
+          response: "",
+          currentAgentName: "assistant",
+          raw: chunk,
+        }),
+      );
+    }
+
+    return stopAgentEvent.with({
+      result: response,
+    });
+  });
+
+  return workflow;
+}

packages/server/examples/custom-layout/index.ts

@@ -1,8 +1,13 @@
+import { OpenAI } from "@llamaindex/openai";
 import { LlamaIndexServer } from "@llamaindex/server";
 import { agent } from "@llamaindex/workflow";
-import { tool } from "llamaindex";
+import { Settings, tool } from "llamaindex";
 import { z } from "zod";
 
+Settings.llm = new OpenAI({
+  model: "gpt-4o-mini",
+});
+
 const weatherAgent = agent({
   tools: [
     tool({

packages/server/examples/devmode/index.ts

@@ -1,6 +1,12 @@
+import { OpenAI } from "@llamaindex/openai";
 import { LlamaIndexServer } from "@llamaindex/server";
+import { Settings } from "llamaindex";
 import { workflowFactory } from "./src/app/workflow";
 
+Settings.llm = new OpenAI({
+  model: "gpt-4o-mini",
+});
+
 new LlamaIndexServer({
   workflow: workflowFactory,
   uiConfig: {

packages/server/examples/package.json

@@ -7,14 +7,13 @@
     "dev": "nodemon --exec tsx simple-workflow/calculator.ts"
   },
   "dependencies": {
-    "@llamaindex/openai": "^0.2.0",
-    "@llamaindex/readers": "^3.0.0",
+    "@llamaindex/openai": "~0.4.0",
+    "@llamaindex/readers": "~3.1.4",
     "@llamaindex/server": "workspace:*",
-    "@llamaindex/tools": "0.0.4",
-    "@llamaindex/workflow": "1.1.0",
+    "@llamaindex/tools": "~0.0.11",
     "dotenv": "^16.4.7",
-    "llamaindex": "0.10.2",
-    "zod": "^3.23.8"
+    "llamaindex": "~0.11.0",
+    "zod": "^3.24.2"
   },
   "devDependencies": {
     "@types/node": "^20.10.3",

packages/server/examples/private-file/README.md

@@ -0,0 +1,68 @@
+# Upload File Example
+
+This example shows how to use the uploaded file (private file) from the user in the workflow.
+
+## Prerequisites
+
+Please follow the setup instructions in the [examples README](../README.md).
+
+You will also need:
+
+- An OpenAI API key
+- The `enableFileUpload` option in the `uiConfig` is set to `true`.
+
+```typescript
+new LlamaIndexServer({
+  // ... other options
+  uiConfig: { enableFileUpload: true },
+}).start();
+```
+
+## How to get the uploaded files in your workflow:
+
+In LlamaIndexServer, the uploaded file is included in chat message annotations. You can easily get the uploaded files from chat messages using the [extractFileAttachments](https://github.com/llamaindex/llamaindex/blob/main/packages/server/src/utils/events.ts) function.
+
+```typescript
+import { type Message } from "ai";
+import { extractFileAttachments } from "@llamaindex/server";
+
+async function workflowFactory(reqBody: { messages: Message[] }) {
+  const attachments = extractFileAttachments(reqBody.messages);
+  // ...
+}
+```
+
+### AgentWorkflow
+
+If you are using AgentWorkflow, to provide file access to the agent, you can create a tool to read the file content. We recommend to use the `fileId` as the parameter of the tool instead of the `filePath` to avoid showing internal file path to the user. You can use the `getStoredFilePath` helper function to get the file path from the file id.
+
+```typescript
+import { getStoredFilePath, extractFileAttachments } from "@llamaindex/server";
+
+const readFileTool = tool(
+  ({ fileId }) => {
+    // Get the file path from the file id
+    const filePath = getStoredFilePath({ id: fileId });
+    return fsPromises.readFile(filePath, "utf8");
+  },
+  {
+    name: "read_file",
+    description: `Use this tool with the file id to read the file content. The available file are: [${attachments.map((file) => file.id).join(", ")}]`,
+    parameters: z.object({
+      fileId: z.string(),
+    }),
+  },
+);
+```
+
+**Tip:** You can either put the attachments file information to the tool description or agent's system prompt.
+
+Check: [agent-workflow.ts](./agent-workflow.ts) for the full example.
+
+### Custom Workflow
+
+In custom workflow, instead of defining a tool, you can use the helper functions (`extractFileAttachments` and `getStoredFilePath`) to work with file attachments in your workflow.
+
+Check: [custom-workflow.ts](./custom-workflow.ts) for the full example.
+
+> To run custom workflow example, update the `index.ts` file to use the `workflowFactory` from `custom-workflow.ts` instead of `agent-workflow.ts`.

packages/server/examples/private-file/agent-workflow.ts

@@ -0,0 +1,39 @@
+import { extractFileAttachments, getStoredFilePath } from "@llamaindex/server";
+import { agent } from "@llamaindex/workflow";
+import { type Message } from "ai";
+import { tool } from "llamaindex";
+import { promises as fsPromises } from "node:fs";
+import { z } from "zod";
+
+export const workflowFactory = async (reqBody: { messages: Message[] }) => {
+  const { messages } = reqBody;
+  // Extract the files from the messages
+  const files = extractFileAttachments(messages);
+  const fileIds = files.map((file) => file.id);
+
+  // Define a tool to read the file content using the id
+  const readFileTool = tool(
+    ({ fileId }) => {
+      if (!fileIds.includes(fileId)) {
+        throw new Error(`File with id ${fileId} not found`);
+      }
+
+      const filePath = getStoredFilePath({ id: fileId });
+      return fsPromises.readFile(filePath, "utf8");
+    },
+    {
+      name: "read_file",
+      description: `Use this tool with the id of the file to read the file content. Here are the available file ids: [${fileIds.join(", ")}]`,
+      parameters: z.object({
+        fileId: z.string(),
+      }),
+    },
+  );
+  return agent({
+    tools: [readFileTool],
+    systemPrompt: `
+      You are a helpful assistant that can help the user with their file.
+      You can use the read_file tool to read the file content.
+    `,
+  });
+};

packages/server/examples/private-file/custom-workflow.ts

@@ -0,0 +1,98 @@
+import { extractFileAttachments } from "@llamaindex/server";
+import { ChatMemoryBuffer, MessageContent, Settings } from "llamaindex";
+
+import {
+  agentStreamEvent,
+  createStatefulMiddleware,
+  createWorkflow,
+  startAgentEvent,
+  stopAgentEvent,
+  workflowEvent,
+} from "@llamaindex/workflow";
+import { Message } from "ai";
+import { promises as fsPromises } from "node:fs";
+
+const fileHelperEvent = workflowEvent<{
+  userInput: MessageContent;
+  fileContent: string;
+}>();
+
+/**
+ * This is an simple workflow to demonstrate how to use uploaded files in the workflow.
+ */
+export function workflowFactory(reqBody: { messages: Message[] }) {
+  const llm = Settings.llm;
+
+  // First, extract the uploaded file from the messages
+  const attachments = extractFileAttachments(reqBody.messages);
+
+  if (attachments.length === 0) {
+    throw new Error("Please upload a file to start");
+  }
+
+  // Then, add the uploaded file info to the workflow state
+  const { withState, getContext } = createStatefulMiddleware(() => {
+    return {
+      memory: new ChatMemoryBuffer({ llm }),
+      uploadedFile: attachments[attachments.length - 1],
+    };
+  });
+  const workflow = withState(createWorkflow());
+
+  // Handle the start of the workflow: read the file content
+  workflow.handle([startAgentEvent], async ({ data }) => {
+    const { userInput } = data;
+    // Prepare chat history
+    const { state } = getContext();
+    if (!userInput) {
+      throw new Error("Missing user input to start the workflow");
+    }
+    state.memory.put({ role: "user", content: userInput });
+
+    // Read file content
+    const fileContent = await fsPromises.readFile(
+      state.uploadedFile.path,
+      "utf8",
+    );
+
+    return fileHelperEvent.with({
+      userInput,
+      fileContent,
+    });
+  });
+
+  // Use LLM to help the user with the file content
+  workflow.handle([fileHelperEvent], async ({ data }) => {
+    const { sendEvent } = getContext();
+
+    const prompt = `
+You are a helpful assistant that can help the user with their file.
+
+Here is the provided file content:
+${data.fileContent}
+
+Now, let help the user with this request:
+${data.userInput}
+`;
+
+    const response = await llm.complete({
+      prompt,
+      stream: true,
+    });
+
+    // Stream the response
+    for await (const chunk of response) {
+      sendEvent(
+        agentStreamEvent.with({
+          delta: chunk.text,
+          response: chunk.text,
+          currentAgentName: "agent",
+          raw: chunk.raw,
+        }),
+      );
+    }
+    sendEvent(stopAgentEvent.with({ result: "" }));
+  });
+
+  return workflow;
+}

packages/server/examples/private-file/index.ts

@@ -0,0 +1,23 @@
+import { OpenAI, OpenAIEmbedding } from "@llamaindex/openai";
+import { LlamaIndexServer } from "@llamaindex/server";
+import { Settings } from "llamaindex";
+import { workflowFactory } from "./agent-workflow";
+// Uncomment this to use a custom workflow
+// import { workflowFactory } from "./custom-workflow";
+
+Settings.llm = new OpenAI({
+  model: "gpt-4o-mini",
+});
+
+Settings.embedModel = new OpenAIEmbedding({
+  model: "text-embedding-3-small",
+});
+
+new LlamaIndexServer({
+  workflow: workflowFactory,
+  suggestNextQuestions: false,
+  uiConfig: {
+    enableFileUpload: true,
+  },
+  port: 3000,
+}).start();

packages/server/examples/simple-workflow/calculator.ts

@@ -1,8 +1,13 @@
+import { OpenAI } from "@llamaindex/openai";
 import { LlamaIndexServer } from "@llamaindex/server";
 import { agent } from "@llamaindex/workflow";
-import { tool } from "llamaindex";
+import { Settings, tool } from "llamaindex";
 import { z } from "zod";
 
+Settings.llm = new OpenAI({
+  model: "gpt-4o-mini",
+});
+
 const calculatorAgent = agent({
   tools: [
     tool({

packages/server/next/app/api/files/helpers.ts

@@ -0,0 +1,57 @@
+import crypto from "node:crypto";
+import fs from "node:fs";
+import path from "node:path";
+
+import { type ServerFile } from "@llamaindex/server";
+
+export const UPLOADED_FOLDER = "output/uploaded";
+
+export async function storeFile(
+  name: string,
+  fileBuffer: Buffer,
+): Promise<ServerFile> {
+  const parts = name.split(".");
+  const fileName = parts[0];
+  const fileExt = parts[1];
+  if (!fileName) {
+    throw new Error("File name is required");
+  }
+  if (!fileExt) {
+    throw new Error("File extension is required");
+  }
+
+  const id = crypto.randomUUID();
+  const fileId = `${sanitizeFileName(fileName)}_${id}.${fileExt}`;
+  const filepath = path.join(UPLOADED_FOLDER, fileId);
+  const fileUrl = await saveFile(filepath, fileBuffer);
+  return {
+    id: fileId,
+    size: fileBuffer.length,
+    type: fileExt,
+    url: fileUrl,
+    path: filepath,
+  };
+}
+
+// Save document to file server and return the file url
+async function saveFile(filepath: string, content: string | Buffer) {
+  if (path.isAbsolute(filepath)) {
+    throw new Error("Absolute file paths are not allowed.");
+  }
+
+  const dirPath = path.dirname(filepath);
+  await fs.promises.mkdir(dirPath, { recursive: true });
+
+  if (typeof content === "string") {
+    await fs.promises.writeFile(filepath, content, "utf-8");
+  } else {
+    await fs.promises.writeFile(filepath, content);
+  }
+
+  const fileurl = `/api/files/${filepath}`;
+  return fileurl;
+}
+
+function sanitizeFileName(fileName: string) {
+  return fileName.replace(/[^a-zA-Z0-9_-]/g, "_");
+}

packages/server/next/app/api/files/route.ts

@@ -0,0 +1,49 @@
+import { type FileAnnotation } from "@llamaindex/server";
+import { NextRequest, NextResponse } from "next/server";
+import { storeFile } from "./helpers";
+
+export async function POST(request: NextRequest) {
+  try {
+    const {
+      name,
+      base64,
+    }: {
+      name: string;
+      base64: string;
+    } = await request.json();
+    if (!base64 || !name) {
+      return NextResponse.json(
+        { error: "base64 and name is required in the request body" },
+        { status: 400 },
+      );
+    }
+
+    const parts = base64.split(",");
+    if (parts.length !== 2) {
+      return NextResponse.json(
+        { error: "Invalid base64 format" },
+        { status: 400 },
+      );
+    }
+
+    const [header, content] = parts;
+    if (!header || !content) {
+      return NextResponse.json(
+        { error: "Invalid base64 format" },
+        { status: 400 },
+      );
+    }
+
+    const fileBuffer = Buffer.from(content, "base64");
+
+    const file = await storeFile(name, fileBuffer);
+
+    return NextResponse.json(file as FileAnnotation);
+  } catch (error) {
+    console.error("[Upload API]", error);
+    return NextResponse.json(
+      { error: (error as Error).message },
+      { status: 500 },
+    );
+  }
+}

packages/server/next/app/components/ui/chat/chat-message-content.tsx

@@ -19,7 +19,6 @@ export function ChatMessageContent({
       <ToolAnnotations />
       <ChatMessage.Content.Image />
       <DynamicEvents componentDefs={componentDefs} appendError={appendError} />
-      <ChatMessage.Content.Artifact />
       <ChatMessage.Content.Markdown />
       <ChatMessage.Content.DocumentFile />
       <ChatMessage.Content.Source />

packages/server/next/app/layout.tsx

@@ -1,6 +1,7 @@
 import type { Metadata } from "next";
 import { Inter } from "next/font/google";
 
+import "@llamaindex/chat-ui/styles/editor.css";
 import "@llamaindex/chat-ui/styles/markdown.css";
 import "@llamaindex/chat-ui/styles/pdf.css";
 import "./globals.css";

packages/server/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/server",
   "description": "LlamaIndex Server",
-  "version": "0.2.6",
+  "version": "0.2.7",
   "type": "module",
   "main": "./dist/index.cjs",
   "module": "./dist/index.js",
@@ -65,7 +65,7 @@
     "@babel/traverse": "^7.27.0",
     "@babel/types": "^7.27.0",
     "@hookform/resolvers": "^5.0.1",
-    "@llamaindex/chat-ui": "0.4.9",
+    "@llamaindex/chat-ui": "0.5.6",
     "@radix-ui/react-accordion": "^1.2.3",
     "@radix-ui/react-alert-dialog": "^1.1.7",
     "@radix-ui/react-aspect-ratio": "^1.1.3",

packages/server/project-config/eslint.config.mjs

@@ -18,6 +18,7 @@ const eslintConfig = [
       "react-hooks/exhaustive-deps": "off",
       "@next/next/no-img-element": "off",
       "@next/next/no-assign-module-variable": "off",
+      "@typescript-eslint/no-empty-object-type": "off",
     },
   },
   {

packages/server/project-config/package.json

@@ -41,7 +41,7 @@
     "@babel/traverse": "^7.27.0",
     "@babel/types": "^7.27.0",
     "@hookform/resolvers": "^5.0.1",
-    "@llamaindex/chat-ui": "0.4.9",
+    "@llamaindex/chat-ui": "0.5.6",
     "@llamaindex/env": "~0.1.30",
     "@llamaindex/openai": "~0.4.0",
     "@llamaindex/readers": "~3.1.4",

packages/server/src/index.ts

@@ -1,5 +1,7 @@
 export * from "./server";
 export * from "./types";
 export * from "./utils/events";
+export { getStoredFilePath } from "./utils/file";
 export { generateEventComponent } from "./utils/gen-ui";
+export * from "./utils/inline";
 export * from "./utils/prompts";

packages/server/src/server.ts

@@ -47,7 +47,7 @@ export class LlamaIndexServer {
     const componentsApi = this.componentsDir ? "/api/components" : undefined;
     const layoutApi = this.layoutDir ? "/api/layout" : undefined;
     const devMode = uiConfig?.devMode ?? false;
-
+    const enableFileUpload = uiConfig?.enableFileUpload ?? false;
     // content in javascript format
     const content = `
       window.LLAMAINDEX = {
@@ -57,7 +57,8 @@ export class LlamaIndexServer {
         COMPONENTS_API: ${JSON.stringify(componentsApi)},
         LAYOUT_API: ${JSON.stringify(layoutApi)},
         DEV_MODE: ${JSON.stringify(devMode)},
-        SUGGEST_NEXT_QUESTIONS: ${JSON.stringify(this.suggestNextQuestions)}
+        SUGGEST_NEXT_QUESTIONS: ${JSON.stringify(this.suggestNextQuestions)},
+        UPLOAD_API: ${JSON.stringify(enableFileUpload ? "/api/files" : undefined)}
       }
     `;
     fs.writeFileSync(configFile, content);

packages/server/src/types.ts

@@ -18,6 +18,7 @@ export type UIConfig = {
   layoutDir?: string;
   llamaCloudIndexSelector?: boolean;
   devMode?: boolean;
+  enableFileUpload?: boolean;
 };
 
 export type LlamaIndexServerOptions = NextAppOptions & {

packages/server/src/utils/events.ts

@@ -1,8 +1,15 @@
 import { randomUUID } from "@llamaindex/env";
 import { workflowEvent } from "@llamaindex/workflow";
 import type { Message } from "ai";
-import { MetadataMode, type Metadata, type NodeWithScore } from "llamaindex";
+import {
+  MetadataMode,
+  type ChatMessage,
+  type Metadata,
+  type NodeWithScore,
+} from "llamaindex";
 import { z } from "zod";
+import { getStoredFilePath } from "./file";
+import { getInlineAnnotations } from "./inline";
 
 // Events that appended to stream as annotations
 export type SourceEventNode = {
@@ -103,6 +110,7 @@ export type DocumentArtifactData = {
   title: string;
   content: string;
   type: string; // markdown, html,...
+  sources?: { id: string }[]; // sources that are used to render citation numbers in the document
 };
 
 export type CodeArtifact = Artifact<CodeArtifactData> & {
@@ -148,24 +156,22 @@ export const artifactAnnotationSchema = z.object({
   data: artifactSchema,
 });
 
-export function extractAllArtifacts(messages: Message[]): Artifact[] {
-  const allArtifacts: Artifact[] = [];
+export function extractArtifactsFromMessage(message: ChatMessage): Artifact[] {
+  const inlineAnnotations = getInlineAnnotations(message);
+  const artifacts = inlineAnnotations.filter(
+    (annotation): annotation is z.infer<typeof artifactAnnotationSchema> => {
+      return artifactAnnotationSchema.safeParse(annotation).success;
+    },
+  );
+  return artifacts.map((artifact) => artifact.data);
+}
 
-  for (const message of messages) {
-    const artifacts =
-      message.annotations
-        ?.filter(
-          (
-            annotation,
-          ): annotation is z.infer<typeof artifactAnnotationSchema> =>
-            artifactAnnotationSchema.safeParse(annotation).success,
-        )
-        .map((annotation) => annotation.data as Artifact) ?? [];
-
-    allArtifacts.push(...artifacts);
-  }
-
-  return allArtifacts;
+export function extractArtifactsFromAllMessages(
+  messages: ChatMessage[],
+): Artifact[] {
+  return messages
+    .flatMap((message) => extractArtifactsFromMessage(message))
+    .sort((a, b) => a.created_at - b.created_at);
 }
 
 export function extractLastArtifact(
@@ -187,10 +193,10 @@ export function extractLastArtifact(
   requestBody: unknown,
   type?: ArtifactType,
 ): CodeArtifact | DocumentArtifact | Artifact | undefined {
-  const { messages } = (requestBody as { messages?: Message[] }) ?? {};
+  const { messages } = (requestBody as { messages?: ChatMessage[] }) ?? {};
   if (!messages) return undefined;
 
-  const artifacts = extractAllArtifacts(messages);
+  const artifacts = extractArtifactsFromAllMessages(messages);
   if (!artifacts.length) return undefined;
 
   if (type) {
@@ -211,3 +217,64 @@ export function extractLastArtifact(
 
   return artifacts[artifacts.length - 1];
 }
+
+export const fileAnnotationSchema = z.object({
+  id: z.string(),
+  size: z.number(),
+  type: z.string(),
+  url: z.string(),
+});
+
+export const documentFileAnnotationSchema = z.object({
+  type: z.literal("document_file"),
+  data: z.object({
+    files: z.array(fileAnnotationSchema),
+  }),
+});
+type DocumentFileAnnotation = z.infer<typeof documentFileAnnotationSchema>;
+
+export type FileAnnotation = z.infer<typeof fileAnnotationSchema>;
+
+export type ServerFile = FileAnnotation & {
+  path: string;
+};
+
+/**
+ * Extract file attachments from an user message.
+ * @param message - The message to extract file attachments from.
+ * @returns The file attachments.
+ */
+export function extractFileAttachmentsFromMessage(
+  message: Message,
+): ServerFile[] {
+  const fileAttachments: ServerFile[] = [];
+  if (message.role === "user" && message.annotations) {
+    for (const annotation of message.annotations) {
+      if (documentFileAnnotationSchema.safeParse(annotation).success) {
+        const { data } = annotation as DocumentFileAnnotation;
+        for (const file of data.files) {
+          fileAttachments.push({
+            ...file,
+            path: getStoredFilePath({ id: file.id }),
+          });
+        }
+      }
+    }
+  }
+  return fileAttachments;
+}
+
+/**
+ * Extract file attachments from all user messages.
+ * @param messages - The messages to extract file attachments from.
+ * @returns The file attachments.
+ */
+export function extractFileAttachments(messages: Message[]): ServerFile[] {
+  const fileAttachments: ServerFile[] = [];
+
+  for (const message of messages) {
+    fileAttachments.push(...extractFileAttachmentsFromMessage(message));
+  }
+
+  return fileAttachments;
+}

packages/server/src/utils/file.ts

@@ -1,5 +1,6 @@
 import fs from "node:fs";
 import https from "node:https";
+import path from "node:path";
 
 export async function downloadFile(
   urlToDownload: string,
@@ -29,3 +30,41 @@ export async function downloadFile(
     throw new Error(`Error downloading file: ${error}`);
   }
 }
+
+/**
+ * Returns the full path to a stored file given its id and optional save directory.
+ * If saveDir is not provided, defaults to "output/uploaded".
+ *
+ * @param {Object} params - The parameters object.
+ * @param {string} params.id - The file identifier.
+ * @param {string} [params.saveDir] - Optional directory to save the file.
+ * @returns {string} The full file path.
+ */
+
+/**
+ * Constructs a stored file path from an ID and optional directory.
+ * Uses path.join for cross-platform safety and validates the ID to prevent path traversal.
+ *
+ * @param {Object} params - The parameters object.
+ * @param {string} params.id - The file identifier (must not contain path separators).
+ * @param {string} [params.saveDir] - Optional directory to save the file. Defaults to "output/uploaded".
+ * @returns {string} The full file path.
+ * @throws {Error} If the id contains invalid path characters.
+ */
+export function getStoredFilePath({
+  id,
+  saveDir,
+}: {
+  id: string;
+  saveDir?: string;
+}): string {
+  // Validate id to prevent path traversal and invalid characters
+  if (id.includes("/") || id.includes("\\") || id.includes("..")) {
+    throw new Error(
+      "Invalid file id: path traversal or separators are not allowed.",
+    );
+  }
+  // Use path.join to construct the default directory for cross-platform compatibility
+  const directory = saveDir ?? path.join("output", "uploaded");
+  return path.join(directory, id);
+}

packages/server/src/utils/index.ts

@@ -1,6 +1,7 @@
 export * from "./events";
 export * from "./file";
 export * from "./gen-ui";
+export * from "./inline";
 export * from "./prompts";
 export * from "./request";
 export * from "./stream";

packages/server/src/utils/inline.ts

@@ -0,0 +1,90 @@
+import { agentStreamEvent, type WorkflowEventData } from "@llamaindex/workflow";
+import { type ChatMessage } from "llamaindex";
+import { z } from "zod";
+
+const INLINE_ANNOTATION_KEY = "annotation"; // the language key to detect inline annotation code in markdown
+
+export const AnnotationSchema = z.object({
+  type: z.string(),
+  data: z.any(),
+});
+
+export type Annotation = z.infer<typeof AnnotationSchema>;
+
+export function getInlineAnnotations(message: ChatMessage): Annotation[] {
+  const markdownContent = getMessageMarkdownContent(message);
+
+  const inlineAnnotations: Annotation[] = [];
+
+  // Regex to match annotation code blocks
+  // Matches ```annotation followed by content until closing ```
+  const annotationRegex = new RegExp(
+    `\`\`\`${INLINE_ANNOTATION_KEY}\\s*\\n([\\s\\S]*?)\\n\`\`\``,
+    "g",
+  );
+
+  let match;
+  while ((match = annotationRegex.exec(markdownContent)) !== null) {
+    const jsonContent = match[1]?.trim();
+
+    if (!jsonContent) {
+      continue;
+    }
+
+    try {
+      // Parse the JSON content
+      const parsed = JSON.parse(jsonContent);
+
+      // Validate against the annotation schema
+      const validated = AnnotationSchema.parse(parsed);
+
+      // Extract the artifact data
+      inlineAnnotations.push(validated);
+    } catch (error) {
+      // Skip invalid annotations - they might be malformed JSON or invalid schema
+      console.warn("Failed to parse annotation:", error);
+    }
+  }
+
+  return inlineAnnotations;
+}
+
+/**
+ * To append inline annotations to the stream, we need to wrap the annotation in a code block with the language key.
+ * The language key is `annotation` and the code block is wrapped in backticks.
+ *
+ * \`\`\`annotation
+ * \{
+ *   "type": "artifact",
+ *   "data": \{...\}
+ * \}
+ * \`\`\`
+ */
+export function toInlineAnnotation(item: unknown) {
+  return `\n\`\`\`${INLINE_ANNOTATION_KEY}\n${JSON.stringify(item)}\n\`\`\`\n`;
+}
+
+export function toInlineAnnotationEvent(event: WorkflowEventData<unknown>) {
+  return agentStreamEvent.with({
+    delta: toInlineAnnotation(event.data),
+    response: "",
+    currentAgentName: "assistant",
+    raw: event.data,
+  });
+}
+
+function getMessageMarkdownContent(message: ChatMessage): string {
+  let markdownContent = "";
+
+  if (typeof message.content === "string") {
+    markdownContent = message.content;
+  } else {
+    message.content.forEach((item) => {
+      if (item.type === "text") {
+        markdownContent += item.text;
+      }
+    });
+  }
+
+  return markdownContent;
+}

packages/server/src/utils/workflow.ts

@@ -15,12 +15,14 @@ import {
   type NodeWithScore,
 } from "llamaindex";
 import {
+  artifactEvent,
   sourceEvent,
   toAgentRunEvent,
   toSourceEvent,
   type SourceEventNode,
 } from "./events";
 import { downloadFile } from "./file";
+import { toInlineAnnotationEvent } from "./inline";
 
 export async function runWorkflow(
   workflow: Workflow,
@@ -74,6 +76,10 @@ function processWorkflowStream(
               transformedEvent = toSourceEvent(sourceNodes);
             }
           }
+          // Handle artifact events, transform to agentStreamEvent
+          else if (artifactEvent.include(event)) {
+            transformedEvent = toInlineAnnotationEvent(event);
+          }
           // Post-process for llama-cloud files
           if (sourceEvent.include(transformedEvent)) {
             const sourceNodesForDownload = transformedEvent.data.data.nodes; // These are SourceEventNode[]

python/llama-index-server/CHANGELOG.md

@@ -1,5 +1,17 @@
 # @create-llama/llama-index-server
 
+## 0.1.21
+
+### Patch Changes
+
+- 1ff6eaf: Add support for upload file
+- af9ad3c: feat: show document artifact after generating report
+- a543a27: feat: bump chat-ui with inline artifact
+- Updated dependencies [af9ad3c]
+- Updated dependencies [a543a27]
+- Updated dependencies [1ff6eaf]
+  - @llamaindex/server@0.2.7
+
 ## 0.1.20
 
 ### Patch Changes

python/llama-index-server/README.md

@@ -78,6 +78,7 @@ The LlamaIndexServer accepts the following configuration parameters:
 - `env`: Environment setting ('dev' enables CORS and UI by default)
 - `ui_config`: UI configuration as a dictionary or UIConfig object with options:
   - `enabled`: Whether to enable the chat UI (default: True)
+  - `enable_file_upload`: Whether to enable file upload in the chat UI (default: False). Check [How to get the uploaded files in your workflow](https://github.com/run-llama/create-llama/blob/main/python/llama-index-server/examples/private_file/README.md#how-to-get-the-uploaded-files-in-your-workflow) for more details.
   - `starter_questions`: List of starter questions for the chat UI (default: None)
   - `ui_path`: Path for downloaded UI static files (default: ".ui")
   - `component_dir`: The directory for custom UI components rendering events emitted by the workflow. The default is None, which does not render custom UI components.
@@ -161,6 +162,7 @@ app = LlamaIndexServer(
 The server provides the following default endpoints:
 
 - `/api/chat`: Chat interaction endpoint
+- `/api/chat/file`: File upload endpoint (only available when `enable_file_upload` in `ui_config` is True)
 - `/api/files/data/*`: Access to data directory files
 - `/api/files/output/*`: Access to output directory files
 

python/llama-index-server/examples/README.md

@@ -0,0 +1,43 @@
+# Examples for llama-index-server
+
+This directory contains examples for llama-index-server.
+
+## How to run the examples
+
+1. Make sure you have [uv](https://docs.astral.sh/uv/) installed.
+
+2. Install the dependencies (with published packages) by running the following command:
+
+    ```bash
+    uv sync
+    ```
+
+3. Navigate to one of the example folders and follow the instructions in the example's README.md file:
+
+- [Simple Agent](./simple-agent/README.md)
+- [HITL](./hitl/README.md)
+- [Artifact](./artifact/README.md)
+- [LlamaCloud](./llamacloud/README.md)
+
+## Local Development
+
+1. For local development, you first need to build the UI resources for the server. At the root of the project, run the following command:
+
+    ```bash
+    pnpm install
+    pnpm build
+    ```
+
+2. Config to use the local llama-index-server package:
+
+    To run the examples with the local llama-index-server package, you need to tell uv to use the virtual environment of the root project
+    by setting the `UV_PROJECT` environment variable.
+
+    ```bash
+    export UV_PROJECT=<absolute path of the root project>
+    ```
+
+    Then continue with step 3 above.
+
+    > You can also use `--project <path to the root project>` instead of setting the `UV_PROJECT` environment variable.
+

python/llama-index-server/examples/artifact/README.md

@@ -4,7 +4,9 @@ This guide explains how to set up and use the LlamaIndex server with the artifac
 
 ## Prerequisites
 
-- [uv](https://github.com/astral-sh/uv) installed (a fast Python package manager and runner)
+Please follow the setup instructions in the [examples README](../README.md).
+
+You will also need:
 - An OpenAI API key
 
 ## Steps

python/llama-index-server/examples/artifact/main.py

@@ -1,9 +1,8 @@
+from code_workflow import ArtifactWorkflow
 from fastapi import FastAPI
 
-from examples.artifact.code_workflow import ArtifactWorkflow
-
 # To use document artifact workflow, uncomment the following line
-# from examples.artifact.document_workflow import ArtifactWorkflow
+# from document_workflow import ArtifactWorkflow
 from llama_index.core.workflow import Workflow
 from llama_index.llms.openai import OpenAI
 from llama_index.server import LlamaIndexServer, UIConfig

python/llama-index-server/examples/hitl/README.md

@@ -1,32 +1,77 @@
 # Human in the Loop
 
-This example shows how to use the LlamaIndexServer with a human in the loop.
+This example shows how to use the LlamaIndexServer with a human in the loop. It allows you to start CLI commands that are reviewed by a human before execution.
 
-## AgentWorkflow
+## Prerequisites
+
+Please follow the setup instructions in the [examples README](../README.md).
+
+## Getting Started
+
+### AgentWorkflow
+
+Using AgentWorkflow, you need to run the following command:
 
 ```bash
 uv run -- agent_workflow.py
 ```
 
-## Custom Workflow
+### Custom Workflow
 
 ```bash
 uv run -- custom_workflow.py
 ```
 
-## How does it work?
+### Access the Application
+
+Open your browser and go to:
+
+```
+http://localhost:8000
+```
+
+You will see the LlamaIndexServer UI, where you can interact with the HITL agent. Try "List all files in the current directory" and see how the agent pauses and waits for a human response before executing the command.
+
+## How does HITL it work?
+
+### Events
+
 The human-in-the-loop approach used here is based on a simple idea: the workflow pauses and waits for a human response before proceeding to the next step.
 
 To do this, you will need to implement two custom events: 
-+ [HumanInputEvent](../../llama_index/server/models/hitl.py#L10): This event is used to request input from the user.
-+ [HumanResponseEvent](../../llama_index/server/models/hitl.py#L43): This event is sent to the workflow to resume execution with input from the user.
++ [HumanInputEvent](../../llama_index/server/models/hitl.py#L21): This event is used to request input from the user.
++ [HumanResponseEvent](../../llama_index/server/models/hitl.py#L10): This event is sent to the workflow to resume execution with input from the user.
 
 In this example, we have implemented these two custom events:  
 
 - [CLIHumanInputEvent](events.py#L20) – to request input from the user for CLI command execution.
 - [CLIHumanResponseEvent](events.py#L8) – to resume the workflow with the response from the user.
 
-We also have a custom component, [cli_human_input.tsx](./components/cli_human_input.tsx), which displays a card that the user can update the command and choose to execute or cancel the command execution.
+### UI Component
+
+HITL also needs a custom UI component, that is shown when the LlamaIndexServer receives the `CLIHumanInputEvent`. The name of the component is defined in the `event_type` field of the `CLIHumanInputEvent` - in our case, it is `cli_human_input`, which corresponds to the [cli_human_input.tsx](./components/cli_human_input.tsx) component.
+
+The custom component must use `append` to send a message with a `human_response` annotation. The data of the annotation must be in the format of the response event `CLIHumanResponseEvent`, in our case, for sending to execute the command `ls -l`, we would send:
+
+```tsx
+append({
+    content: "Yes",
+    role: "user",
+    annotations: [
+    {
+        type: "human_response",
+        data: {
+            execute: true,
+            command: "ls -l" // The command to execute
+        },
+    },
+    ],
+});
+```
+
+This component displays the command to execute and the user can choose to execute or cancel the command execution.
+
+### AgentWorkflow
 
 To make the [AgentWorkflow](agent_workflow.py) work, we use the `wait_for_event()` method to wait for the human response when a tool is called.
 
@@ -51,6 +96,8 @@ async def cli_executor(ctx: Context, command: str) -> str:
 
 ```
 
+### LlamaIndex Workflows
+
 And for [Custom Workflow](custom_workflow.py), we can define a step that send the `CLIHumanInputEvent` and another step that wait for the `CLIHumanResponseEvent`.
 
 Example:

python/llama-index-server/examples/llamacloud/README.md

@@ -0,0 +1,68 @@
+# LlamaCloud Integration
+
+This guide explains how to set up and use the LlamaIndex server with LlamaCloud for retrieval-augmented generation (RAG) with citation support.
+
+## Prerequisites
+
+Please follow the setup instructions in the [examples README](../README.md).
+
+You will also need:
+- An OpenAI API key
+- A LlamaCloud account and API key
+- A LlamaCloud project with indexed documents
+
+## Steps
+
+1. **Set the Required Environment Variables**
+
+   Export your API keys and LlamaCloud configuration:
+
+   ```sh
+   export OPENAI_API_KEY=your_openai_api_key_here
+   export LLAMA_CLOUD_API_KEY=your_llamacloud_api_key_here
+   export LLAMA_CLOUD_PROJECT_NAME=your_project_name
+   export LLAMA_CLOUD_INDEX_NAME=your_index_name
+   ```
+
+2. **Run the Server Using uv**
+
+   Start the server with the following command:
+
+   ```sh
+   uv run main.py
+   ```
+
+   This will launch the FastAPI server using the LlamaCloud workflow defined in `main.py`.
+
+3. **Access the Application**
+
+   Open your browser and go to:
+
+   ```
+   http://localhost:8000
+   ```
+
+   You will see the LlamaIndex chat app UI with LlamaCloud integration, where you can query your indexed documents.
+
+## Features
+
+- **Document Retrieval**: Query your LlamaCloud indexed documents with two retrieval modes:
+  - **Chunk-level retrieval**: Best for specific, detailed questions requiring precise information
+  - **Document-level retrieval**: Best for high-level summarization and broader context questions
+- **Citation Support**: All responses include citations to the source documents, helping you verify and trace the information back to its origin.
+- **Index Selection**: The UI includes an index selector, allowing you to switch between different LlamaCloud indexes if you have multiple of them.
+
+## How it Works
+
+The workflow uses two specialized query engines:
+
+1. **Chunk Query Engine**: Retrieves specific chunks of documents for detailed, targeted questions
+2. **File Query Engine**: Retrieves entire documents as context for broader, summarization-type questions
+
+Both engines are enhanced with citation capabilities, ensuring transparency and traceability in the responses.
+
+## Notes
+
+- Make sure your LlamaCloud project has documents indexed before running the example
+- The server uses GPT-4.1 by default for optimal performance with citations
+- The workflow automatically selects the appropriate retrieval strategy based on your query type 

python/llama-index-server/examples/private_file/README.md

@@ -0,0 +1,102 @@
+# Uploaded File
+
+This example shows how to use the uploaded file (private file) from the user in the workflow.
+
+## Prerequisites
+
+Please follow the setup instructions in the [examples README](../README.md).
+
+You will also need:
+- An OpenAI API key
+- Text files for processing (the examples are optimized for smaller text files)
+
+## How to get the uploaded files in your workflow:
+
+The uploaded file information is included in the annotations of a [ChatAPIMessage](../../llama_index/server/models/chat.py#66). You can manually access it through the `chat_request` parameter in the workflow factory. We already provided a [get_file_attachments](../../llama_index/server/utils/chat_attachments.py) helper function to get the uploaded files from the chat request easier.
+
+```python
+from llama_index.server.api.utils.chat_attachments import get_file_attachments
+
+def create_workflow(chat_request: ChatRequest) -> Workflow:
+    uploaded_files = get_file_attachments(chat_request.messages)
+    ...
+```
+
+Each uploaded file item is a [ServerFile](../../llama_index/server/models/chat.py#9) object, which includes the file id, type, size, and url of the uploaded file. The `url` is an access url to the uploaded file that can be used to download or display the file from the browser, the `id` is used to manage the file in the server through the [FileService](../../llama_index/server/services/file.py).
+
+
+## Examples:
+
+### For agent workflow:
+   - We create a simple file reader tool that can read the uploaded file content.
+
+   ```python
+   def create_file_tool(chat_request: ChatRequest) -> Optional[FunctionTool]:
+      """
+      Create a tool to read file if the user uploads a file.
+      """
+      file_ids = []
+      # Get the uploaded file ids from the the chat messages
+      for file in get_file_attachments(chat_request.messages):
+         file_ids.append(file.id)
+      if len(file_ids) == 0:
+         return None
+
+      # Create a tool description that includes the file ids so the LLM knows which file it can access
+      file_tool_description = (
+         "Use this tool with a file id to read the content of the file."
+         f"\nYou only have access to the following file ids: {json.dumps(file_ids)}"
+      )
+
+      def read_file(file_id: str) -> str:
+         file_path = FileService.get_file_path(file_id)
+         try:
+               with open(file_path, "r") as file:
+                  return file.read()
+         except Exception as e:
+               return f"Error reading file {file_path}: {e}"
+      
+      # Create the tool
+      return FunctionTool.from_defaults(
+         fn=read_file,
+         name="read_file",
+         description=file_tool_description,
+      )
+   ```
+   - Check out the [agent-workflow.py](agent-workflow.py) for more details.
+
+   - You can run the agent workflow with file tool by running the following command:
+     ```bash
+     export OPENAI_API_KEY=your_openai_api_key_here
+     uv run agent-workflow.py
+     ```
+     then go to the UI at `http://localhost:8000` and upload the [example.txt](example.txt) file.
+
+### For custom workflow:
+   - The attachments are included in the `attachments` parameter of the `StartEvent` so you can easily access them in the workflow.
+
+   ```python
+   class MyWorkflow(Workflow):
+      @step
+      async def start_event_handler(self, ctx: Context, ev: StartEvent) -> StopEvent:
+         # Get attachments from the start event
+         attachments = ev.attachments
+         # Do something with the attachments
+         # e.g. read the file content
+         last_file = attachments[-1]
+         if last_file:
+            with open(last_file.path, "r") as f:
+               file_content = f.read()
+            ...
+         # or save it to the context for later use
+         await ctx.set("file_content", file_content)
+         return StopEvent()
+   ```
+   - Check out the [custom-workflow.py](custom-workflow.py) for more details.
+
+   - You can run the custom workflow by running the following command:
+     ```bash
+     export OPENAI_API_KEY=your_openai_api_key_here
+     uv run custom-workflow.py
+     ```
+     then go to the UI at `http://localhost:8000` and upload the [example.txt](example.txt) file.

python/llama-index-server/examples/private_file/agent-workflow.py

@@ -0,0 +1,78 @@
+import json
+from typing import List, Optional
+
+from fastapi import FastAPI
+
+from llama_index.core.agent.workflow import AgentWorkflow
+from llama_index.core.tools import FunctionTool
+from llama_index.llms.openai import OpenAI
+from llama_index.server import LlamaIndexServer, UIConfig
+from llama_index.server.api.utils.chat_attachments import get_file_attachments
+from llama_index.server.models.chat import ChatRequest
+from llama_index.server.models.file import ServerFile
+from llama_index.server.services.file import FileService
+
+
+def create_file_tool(file_attachments: List[ServerFile]) -> Optional[FunctionTool]:
+    """
+    Create a tool to read file if the user uploads a file.
+    """
+    file_ids = []
+    for file in file_attachments:
+        file_ids.append(file.id)
+    if len(file_ids) == 0:
+        return None
+
+    file_tool_description = (
+        "Use this tool with a file id to read the content of the file."
+        f"\nYou only have access to the following file ids: {json.dumps(file_ids)}"
+    )
+
+    def read_file(file_id: str) -> str:
+        # Validate if the file id is in the list of file ids
+        if file_id not in file_ids:
+            raise ValueError(f"I don't have access to file id {file_id}")
+
+        file_path = FileService.get_file_path(file_id)
+        try:
+            with open(file_path, "r") as file:
+                return file.read()
+        except Exception as e:
+            return f"Error reading file {file_path}: {e}"
+
+    return FunctionTool.from_defaults(
+        fn=read_file,
+        name="read_file",
+        description=file_tool_description,
+    )
+
+
+def create_workflow(chat_request: ChatRequest) -> AgentWorkflow:
+    file_attachments = get_file_attachments(chat_request.messages)
+    file_tool = create_file_tool(file_attachments)
+    return AgentWorkflow.from_tools_or_functions(
+        tools_or_functions=[file_tool] if file_tool else [],
+        llm=OpenAI(model="gpt-4.1-mini"),
+        system_prompt="You are a helpful assistant that can help users with their uploaded files.",
+    )
+
+
+def create_app() -> FastAPI:
+    app = LlamaIndexServer(
+        workflow_factory=create_workflow,
+        suggest_next_questions=False,
+        ui_config=UIConfig(
+            enable_file_upload=True,
+            component_dir="components",
+        ),
+    )
+    return app
+
+
+app = create_app()
+
+
+if __name__ == "__main__":
+    import uvicorn
+
+    uvicorn.run("agent-workflow:app", host="0.0.0.0", port=8000, reload=True)

python/llama-index-server/examples/private_file/custom-workflow.py

@@ -0,0 +1,126 @@
+from typing import Any, List
+
+from fastapi import FastAPI
+
+from llama_index.core.agent.workflow.workflow_events import AgentStream
+from llama_index.core.llms import LLM
+from llama_index.core.prompts import PromptTemplate
+from llama_index.core.workflow import (
+    Context,
+    Event,
+    StartEvent,
+    StopEvent,
+    Workflow,
+    WorkflowRuntimeError,
+    step,
+)
+from llama_index.llms.openai import OpenAI
+from llama_index.server import LlamaIndexServer, UIConfig
+from llama_index.server.api.utils.chat_attachments import get_file_attachments
+from llama_index.server.models.chat import ChatRequest
+from llama_index.server.models.file import ServerFile
+
+
+class FileHelpEvent(Event):
+    """
+    The event for helping the user with the an uploaded file.
+    """
+
+    file_content: str
+    user_request: str
+
+
+class FileHelpWorkflow(Workflow):
+    """
+    A simple workflow that helps the user with the an uploaded file.
+    Note: The workflow just simply feed all the file content to the LLM so it won't work for large files.
+    The purpose is just for demo how a workflow can work with the uploaded file from the user.
+    """
+
+    def __init__(
+        self,
+        llm: LLM,
+        file_attachments: List[ServerFile],
+        **kwargs: Any,
+    ):
+        super().__init__(**kwargs)
+        self.llm = llm
+        self.file_attachments = file_attachments
+
+    @step
+    async def read_files(self, ctx: Context, ev: StartEvent) -> FileHelpEvent:
+        user_msg = ev.user_msg
+        if len(self.file_attachments) == 0:
+            raise WorkflowRuntimeError("Please upload one file to start")
+
+        # Read the file content
+        last_file = self.file_attachments[-1]
+        with open(last_file.path, "r") as f:
+            file_content = f.read()
+
+        return FileHelpEvent(
+            file_content=file_content,
+            user_request=user_msg,
+        )
+
+    @step
+    async def help_user(self, ctx: Context, ev: FileHelpEvent) -> StopEvent:
+        default_prompt = PromptTemplate("""
+        You are a writing assistant.
+        You are given a file content and a user request.
+        Your task is to help the user with the file content.
+        
+        User request: {user_msg}
+
+        File content:
+        {file_content}
+        """)
+        prompt = default_prompt.format(
+            user_msg=ev.user_request,
+            file_content=ev.file_content,
+        )
+        stream = await self.llm.astream_complete(prompt)
+        async for chunk in stream:
+            ctx.write_event_to_stream(
+                AgentStream(
+                    response=chunk.text,
+                    delta=chunk.delta or "",
+                    current_agent_name="agent",
+                    tool_calls=[],
+                    raw=chunk.raw,
+                )
+            )
+
+        return StopEvent(
+            content=True,
+        )
+
+
+def create_workflow(chat_request: ChatRequest) -> Workflow:
+    # Use get_file_attachments to get the file attachments from the chat messages
+    file_attachments = get_file_attachments(chat_request.messages)
+    return FileHelpWorkflow(
+        llm=OpenAI(model="gpt-4.1-mini"),
+        file_attachments=file_attachments,
+    )
+
+
+def create_app() -> FastAPI:
+    app = LlamaIndexServer(
+        workflow_factory=create_workflow,
+        suggest_next_questions=False,
+        ui_config=UIConfig(
+            enable_file_upload=True,
+            component_dir="components",
+        ),
+    )
+    return app
+
+
+app = create_app()
+
+
+if __name__ == "__main__":
+    import uvicorn
+
+    uvicorn.run("custom-workflow:app", host="0.0.0.0", port=8000, reload=True)

python/llama-index-server/examples/private_file/example.txt

@@ -0,0 +1,142 @@
+# ACME Vendor Agreement
+
+**Effective Date:** January 1, 2024
+
+## Parties:
+
+- **Client:** LlamaCo ("Client")
+- **Vendor:** ACME Office Supply, Inc. ("Vendor")
+
+## 1. Overview
+
+This Vendor Agreement ("Agreement") sets forth the terms and conditions under which ACME Office Supply, Inc. will provide office supplies, consumables, related goods ("Products"), and associated data processing services to LlamaCo.
+
+## 2. Definitions
+
+- **Personal Data:** Any information relating to an identified or identifiable natural person ('data subject').
+- **Processing:** Any operation performed on Personal Data, including collection, storage, modification, transfer, or deletion.
+- **Data Controller:** LlamaCo, who determines the purposes and means of processing Personal Data.
+- **Data Processor:** ACME Office Supply, Inc., who processes Personal Data on behalf of the Controller.
+
+## 3. Data Protection and Privacy
+
+### 3.1 Scope of Processing
+
+Vendor shall process Personal Data only:
+
+- To fulfill orders and manage deliveries
+- To provide customer support services
+- To maintain business records
+- To comply with legal obligations
+
+### 3.2 Data Subject Rights
+
+Vendor shall:
+
+- Respond to data subject requests within 30 days
+- Provide data in a structured, commonly used format
+- Implement measures to facilitate data portability
+- Assist with data subject rights requests at no additional cost
+
+### 3.3 Data Transfers and Storage
+
+- Vendor maintains primary data centers in the United States
+- Vendor may transfer data to any country where it maintains operations
+- No prior notification required for new data storage locations
+- Vendor will rely on its standard data transfer mechanisms
+- Data may be processed by staff operating outside the EEA
+
+### 3.4 Subprocessors
+
+- Vendor may engage subprocessors without prior Client approval
+- Subprocessors may be located in any jurisdiction globally
+- Notice of new subprocessors provided within 30 days of engagement
+- Client has no right to object to new subprocessors
+
+## 4. Security Measures
+
+### 4.1 Technical and Organizational Measures
+
+Vendor shall implement appropriate measures including:
+
+- Encryption of Personal Data in transit and at rest
+- Access controls and authentication
+- Regular security testing and assessments
+- Employee training on data protection
+- Incident response procedures
+
+### 4.2 Data Breaches
+
+Vendor shall:
+
+- Notify Client of any Personal Data breach within 72 hours
+- Provide details necessary to meet regulatory requirements
+- Cooperate with Client's breach investigation
+- Maintain records of all data breaches
+
+## 5. Data Retention
+
+### 5.1 Retention Period
+
+- Personal Data retained only as long as necessary
+- Standard retention period of 3 years after last transaction
+- Deletion of Personal Data upon written request
+- Backup copies retained for maximum of 6 months
+
+### 5.2 Termination
+
+Upon termination of services:
+
+- Return all Personal Data in standard format
+- Delete existing copies within 30 days
+- Provide written confirmation of deletion
+- Cease all processing activities
+
+## 6. Compliance and Audit
+
+### 6.1 Documentation
+
+Vendor shall maintain:
+
+- Records of all processing activities
+- Security measure documentation
+- Data transfer mechanisms
+- Subprocessor agreements
+
+### 6.2 Audits
+
+- Annual compliance audits permitted
+- 30 days notice required for audits
+- Vendor to provide necessary documentation
+- Client bears reasonable audit costs
+
+## 7. Liability and Indemnification
+
+### 7.1 Liability
+
+- Vendor liable for data protection violations
+- Reasonable compensation for damages
+- Coverage for regulatory fines where applicable
+- Joint liability as required by law
+
+## 8. Governing Law
+
+This Agreement shall be governed by the laws of Ireland, without regard to its conflict of laws principles.
+
+---
+
+IN WITNESS WHEREOF, the parties have executed this Agreement as of the Effective Date.
+
+**LlamaCo**
+
+By: **_
+Name: [Authorized Representative]  
+Title: [Title]  
+Date: _**
+
+**ACME Office Supply, Inc.**
+
+By: **_  
+Name: [Authorized Representative]  
+Title: [Title]  
+Date: _**

python/llama-index-server/examples/pyproject.toml

@@ -0,0 +1,12 @@
+[project]
+name = "llama-index-server-examples"
+version = "0.1.0"
+requires-python = ">=3.10"
+dependencies = [
+    "llama-index-server",
+    "llama-index-llms-openai>=0.4.2",
+    "e2b-code-interpreter>=1.1.1,<2.0.0",
+    "llama-cloud>=0.1.17,<1.0.0",
+    "markdown>=3.7,<4.0",
+    "xhtml2pdf>=0.2.17,<1.0.0",
+]

python/llama-index-server/examples/simple-agent/README.md

@@ -4,7 +4,9 @@ This guide explains how to set up and use the LlamaIndex server with a simple ch
 
 ## Prerequisites
 
-- [uv](https://github.com/astral-sh/uv) installed (a fast Python package manager and runner)
+Please follow the setup instructions in the [examples README](../README.md).
+
+You will also need:
 - An OpenAI API key
 
 ## Steps
@@ -22,10 +24,10 @@ This guide explains how to set up and use the LlamaIndex server with a simple ch
    Start the server with the following command:
 
    ```sh
-   uv run workflow.py
+   uv run main.py
    ```
 
-   This will launch the FastAPI server using the workflow defined in `main.py`.
+   This will launch the FastAPI server using the workflow defined in `app/workflow.py`.
 
 3. **Access the Application**
 

python/llama-index-server/examples/simple-agent/app/workflow.py

@@ -1,7 +1,6 @@
 from typing import Optional
 
 from llama_index.core.agent.workflow import AgentWorkflow
-from llama_index.core.settings import Settings
 from llama_index.llms.openai import OpenAI
 from llama_index.server.models import ChatRequest
 
@@ -9,6 +8,6 @@ from llama_index.server.models import ChatRequest
 def create_workflow(chat_request: Optional[ChatRequest] = None) -> AgentWorkflow:
     return AgentWorkflow.from_tools_or_functions(
         tools_or_functions=[],
-        llm=Settings.llm or OpenAI(model="gpt-4o-mini"),
+        llm=OpenAI(model="gpt-4o-mini"),
         system_prompt="You are a helpful assistant that can tell a joke about Llama.",
     )

python/llama-index-server/llama_index/server/api/callbacks/__init__.py

@@ -1,4 +1,7 @@
 from llama_index.server.api.callbacks.agent_call_tool import AgentCallTool
+from llama_index.server.api.callbacks.artifact_transform import (
+    InlineAnnotationTransformer,
+)
 from llama_index.server.api.callbacks.base import EventCallback
 from llama_index.server.api.callbacks.llamacloud import LlamaCloudFileDownload
 from llama_index.server.api.callbacks.source_nodes import SourceNodesFromToolCall
@@ -12,4 +15,5 @@ __all__ = [
     "SuggestNextQuestions",
     "LlamaCloudFileDownload",
     "AgentCallTool",
+    "InlineAnnotationTransformer",
 ]

python/llama-index-server/llama_index/server/api/callbacks/artifact_transform.py

@@ -0,0 +1,24 @@
+import logging
+from typing import Any
+
+from llama_index.server.api.callbacks.base import EventCallback
+from llama_index.server.models.artifacts import ArtifactEvent
+from llama_index.server.utils.inline import to_inline_annotation_event
+
+logger = logging.getLogger("uvicorn")
+
+
+class InlineAnnotationTransformer(EventCallback):
+    """
+    Transforms an event to AgentStream with inline annotation format.
+    """
+
+    async def run(self, event: Any) -> Any:
+        # handle for ArtifactEvent specifically as it's only supported by inline annotation
+        if isinstance(event, ArtifactEvent):
+            return to_inline_annotation_event(event)
+        return event
+
+    @classmethod
+    def from_default(cls, *args: Any, **kwargs: Any) -> "InlineAnnotationTransformer":
+        return cls()

python/llama-index-server/llama_index/server/api/routers/chat.py

@@ -6,6 +6,7 @@ from typing import AsyncGenerator, Callable, Union
 
 from fastapi import APIRouter, BackgroundTasks, HTTPException
 from fastapi.responses import StreamingResponse
+
 from llama_index.core.agent.workflow.workflow_events import (
     AgentInput,
     AgentSetup,
@@ -18,14 +19,21 @@ from llama_index.core.workflow import (
 from llama_index.server.api.callbacks import (
     AgentCallTool,
     EventCallback,
+    InlineAnnotationTransformer,
     LlamaCloudFileDownload,
     SourceNodesFromToolCall,
     SuggestNextQuestions,
 )
 from llama_index.server.api.callbacks.stream_handler import StreamHandler
 from llama_index.server.api.utils.vercel_stream import VercelStreamResponse
-from llama_index.server.models.chat import ChatRequest
+from llama_index.server.models.chat import (
+    ChatRequest,
+    FileUpload,
+    MessageRole,
+)
+from llama_index.server.models.file import ServerFileResponse
 from llama_index.server.models.hitl import HumanInputEvent
+from llama_index.server.services.file import FileService
 from llama_index.server.services.llamacloud import LlamaCloudFileService
 from llama_index.server.services.workflow import HITLWorkflowService
 
@@ -44,7 +52,8 @@ def chat_router(
     ) -> StreamingResponse:
         try:
             last_message = request.messages[-1]
-            user_message = last_message.to_llamaindex_message()
+            if last_message.role != MessageRole.USER:
+                raise ValueError("Last message must be from user")
             chat_history = [
                 message.to_llamaindex_message() for message in request.messages[:-1]
             ]
@@ -66,12 +75,13 @@ def chat_router(
                 workflow_handler = workflow.run(ctx=ctx)
             else:
                 workflow_handler = workflow.run(
-                    user_msg=user_message.content,
+                    user_msg=last_message.content,
                     chat_history=chat_history,
                 )
 
             callbacks: list[EventCallback] = [
                 AgentCallTool(),
+                InlineAnnotationTransformer(),
                 SourceNodesFromToolCall(),
                 LlamaCloudFileDownload(background_tasks),
             ]
@@ -93,6 +103,21 @@ def chat_router(
             logger.error(e)
             raise HTTPException(status_code=500, detail=str(e))
 
+    # we just simply save the file to the server and don't index it
+    @router.post("/file")
+    async def upload_file(request: FileUpload) -> ServerFileResponse:
+        """
+        Upload a file to the server to be used in the chat session.
+        """
+        try:
+            save_dir = os.path.join("output", "private")
+            content, _ = FileService._preprocess_base64_file(request.base64)
+            file = FileService.save_file(content, request.name, save_dir)
+            return file.to_server_file_response()
+        except Exception:
+            raise HTTPException(status_code=500, detail="Error uploading file")
+
+    # Specific to LlamaCloud
     if LlamaCloudFileService.is_configured():
 
         @router.get("/config/llamacloud")

python/llama-index-server/llama_index/server/api/utils/chat_attachments.py

@@ -0,0 +1,40 @@
+from typing import List
+
+from llama_index.core.types import MessageRole
+from llama_index.server.models.chat import ChatAPIMessage, FileAnnotation
+from llama_index.server.models.file import ServerFile
+from llama_index.server.services.file import FileService
+
+
+def get_file_attachments(messages: List[ChatAPIMessage]) -> List[ServerFile]:
+    """
+    Extract all file attachments from user messages.
+
+    Args:
+        messages (List[ChatAPIMessage]): The list of messages.
+
+    Returns:
+        List[ServerFile]: The list of private files.
+    """
+    user_message_annotations = [
+        message.annotations
+        for message in messages
+        if message.annotations and message.role == MessageRole.USER
+    ]
+    files: List[ServerFile] = []
+    for annotation in user_message_annotations:
+        if isinstance(annotation, list):
+            for item in annotation:
+                if isinstance(item, FileAnnotation):
+                    server_files = [
+                        ServerFile(
+                            id=file.id,
+                            type=file.type,
+                            size=file.size,
+                            url=file.url,
+                            path=FileService.get_file_path(file.id),
+                        )
+                        for file in item.data.files
+                    ]
+                    files.extend(server_files)
+    return files

python/llama-index-server/llama_index/server/models/__init__.py

@@ -4,6 +4,7 @@ from llama_index.server.models.artifacts import (
     ArtifactType,
     CodeArtifactData,
     DocumentArtifactData,
+    DocumentArtifactSource,
 )
 from llama_index.server.models.chat import ChatAPIMessage, ChatRequest
 from llama_index.server.models.hitl import HumanInputEvent, HumanResponseEvent
@@ -20,6 +21,7 @@ __all__ = [
     "ArtifactEvent",
     "ArtifactType",
     "DocumentArtifactData",
+    "DocumentArtifactSource",
     "CodeArtifactData",
     "ChatAPIMessage",
     "ChatRequest",

python/llama-index-server/llama_index/server/models/artifacts.py

@@ -1,10 +1,11 @@
 import logging
 from enum import Enum
-from typing import Literal, Optional, Union
+from typing import List, Literal, Optional, Union
 
 from llama_index.core.workflow.events import Event
 from llama_index.server.models.chat import ChatAPIMessage
 from pydantic import BaseModel
+from llama_index.server.utils.inline import get_inline_annotations
 
 logger = logging.getLogger(__name__)
 
@@ -20,10 +21,16 @@ class CodeArtifactData(BaseModel):
     language: str
 
 
+class DocumentArtifactSource(BaseModel):
+    id: str
+    # we can add more fields here
+
+
 class DocumentArtifactData(BaseModel):
     title: str
     content: str
     type: Literal["markdown", "html"]
+    sources: Optional[List[DocumentArtifactSource]] = None
 
 
 class Artifact(BaseModel):
@@ -33,10 +40,9 @@ class Artifact(BaseModel):
 
     @classmethod
     def from_message(cls, message: ChatAPIMessage) -> Optional["Artifact"]:
-        if not message.annotations or not isinstance(message.annotations, list):
-            return None
+        inline_annotations = get_inline_annotations(message)
 
-        for annotation in message.annotations:
+        for annotation in inline_annotations:
             if isinstance(annotation, dict) and annotation.get("type") == "artifact":
                 try:
                     artifact = cls.model_validate(annotation.get("data"))

python/llama-index-server/llama_index/server/models/chat.py

@@ -1,17 +1,72 @@
 import re
-from typing import Any, List, Optional
+from typing import Any, List, Literal, Optional, Union
 
-from pydantic import BaseModel, field_validator
+from pydantic import BaseModel, Field, field_validator
 
 from llama_index.core.types import ChatMessage, MessageRole
+from llama_index.server.models.file import ServerFileResponse
+
+
+class FileData(BaseModel):
+    """
+    The data of a file.
+    """
+
+    files: List[ServerFileResponse]
+
+
+class FileAnnotation(BaseModel):
+    """
+    The annotation of a file.
+    """
+
+    type: Literal["document_file"]
+    data: FileData
+
+
+class FileUpload(BaseModel):
+    """
+    The file to be uploaded to the chat.
+    """
+
+    name: str
+    base64: str
+    params: Any = None
+
+
+class LlamaCloudPipeline(BaseModel):
+    """
+    The selected LlamaCloud pipeline to use for the chat.
+    (Only available when the app is configured to use LlamaCloud)
+    """
+
+    pipeline: str
+    project: str
+
+
+class ChatData(BaseModel):
+    """
+    The data of a chat.
+    """
+
+    llama_cloud_pipeline: Optional[LlamaCloudPipeline] = Field(
+        default=None,
+        description="The selected LlamaCloud pipeline to use for the chat",
+        alias="llamaCloudPipeline",
+        serialization_alias="llamaCloudPipeline",
+    )
 
 
 class ChatAPIMessage(BaseModel):
     role: MessageRole
     content: str
-    annotations: Optional[List[Any]] = None
+    annotations: Optional[List[Union[FileAnnotation, Any]]] = None
 
     def to_llamaindex_message(self) -> ChatMessage:
+        """
+        Simply convert text content of API message to llama_index's ChatMessage.
+        Annotations are not included.
+        """
         return ChatMessage(role=self.role, content=self.content)
 
     @property
@@ -27,9 +82,16 @@ class ChatAPIMessage(BaseModel):
 
 
 class ChatRequest(BaseModel):
+    """
+    The request to the chat API.
+    """
+
     id: str  # see https://ai-sdk.dev/docs/reference/ai-sdk-ui/use-chat#id - constant for the same chat session
     messages: List[ChatAPIMessage]
-    data: Optional[Any] = None
+    data: Optional[ChatData] = Field(
+        default=None,
+        description="The data of the chat",
+    )
 
     @field_validator("messages")
     def validate_messages(cls, v: List[ChatAPIMessage]) -> List[ChatAPIMessage]:

python/llama-index-server/llama_index/server/models/file.py

@@ -0,0 +1,26 @@
+from typing import Optional
+
+from pydantic import BaseModel, Field
+
+
+class ServerFileResponse(BaseModel):
+    id: str
+    type: Optional[str] = None
+    size: Optional[int] = None
+    url: Optional[str] = None
+
+
+class ServerFile(BaseModel):
+    id: str
+    path: str = Field(description="The path of the file in the server")
+    type: Optional[str] = None
+    size: Optional[int] = None
+    url: Optional[str] = None
+
+    def to_server_file_response(self) -> ServerFileResponse:
+        return ServerFileResponse(
+            id=self.id,
+            type=self.type,
+            size=self.size,
+            url=self.url,
+        )

python/llama-index-server/llama_index/server/server.py

@@ -7,8 +7,6 @@ from fastapi import FastAPI
 from fastapi.middleware.cors import CORSMiddleware
 from fastapi.routing import Mount
 from fastapi.staticfiles import StaticFiles
-from pydantic import BaseModel, Field
-
 from llama_index.core.workflow import Workflow
 from llama_index.server.api.routers import (
     chat_router,
@@ -18,6 +16,7 @@ from llama_index.server.api.routers import (
 )
 from llama_index.server.chat_ui import copy_bundled_chat_ui
 from llama_index.server.settings import server_settings
+from pydantic import BaseModel, Field
 
 
 class UIConfig(BaseModel):
@@ -29,6 +28,10 @@ class UIConfig(BaseModel):
         default=False,
         description="Whether to show the LlamaCloud index selector in the chat UI (need to set the LLAMA_CLOUD_API_KEY environment variable)",
     )
+    enable_file_upload: bool = Field(
+        default=False,
+        description="Whether to enable file upload in the chat UI",
+    )
     ui_path: str = Field(
         default=".ui", description="The path that stores static files for the chat UI"
     )
@@ -47,6 +50,11 @@ class UIConfig(BaseModel):
         return json.dumps(
             {
                 "CHAT_API": f"{server_settings.api_url}/chat",
+                "UPLOAD_API": (
+                    f"{server_settings.api_url}/chat/file"
+                    if self.enable_file_upload
+                    else None
+                ),
                 "STARTER_QUESTIONS": self.starter_questions or [],
                 "LLAMA_CLOUD_API": (
                     f"{server_settings.api_url}/chat/config/llamacloud"

python/llama-index-server/llama_index/server/services/file.py

@@ -1,39 +1,23 @@
+import base64
 import logging
+import mimetypes
 import os
 import re
 import uuid
 from pathlib import Path
-from typing import List, Optional, Union
+from typing import Optional, Tuple, Union
 
+from llama_index.server.models.file import ServerFile
 from llama_index.server.settings import server_settings
-from pydantic import BaseModel, Field
 
 logger = logging.getLogger(__name__)
 
-PRIVATE_STORE_PATH = str(Path("output", "uploaded"))
-TOOL_STORE_PATH = str(Path("output", "tools"))
-LLAMA_CLOUD_STORE_PATH = str(Path("output", "llamacloud"))
-
-
-class DocumentFile(BaseModel):
-    id: str
-    name: str  # Stored file name
-    type: Optional[str] = None
-    size: Optional[int] = None
-    url: Optional[str] = None
-    path: Optional[str] = Field(
-        None,
-        description="The stored file path. Used internally in the server.",
-        exclude=True,
-    )
-    refs: Optional[List[str]] = Field(
-        None, description="The document ids in the index."
-    )
+PRIVATE_STORE_PATH = str(Path("output", "private"))
 
 
 class FileService:
     """
-    To store the files uploaded by the user.
+    Store files to server
     """
 
     @classmethod
@@ -42,38 +26,31 @@ class FileService:
         content: Union[bytes, str],
         file_name: str,
         save_dir: Optional[str] = None,
-    ) -> DocumentFile:
+    ) -> ServerFile:
         """
         Save the content to a file in the local file server (accessible via URL).
 
         Args:
             content (bytes | str): The content to save, either bytes or string.
             file_name (str): The original name of the file.
-            save_dir (Optional[str]): The relative path from the current working directory. Defaults to the `output/uploaded` directory.
-
+            save_dir (Optional[str]): The path to store the file. Defaults is set to PRIVATE_STORE_PATH (output/private) if not provided.
         Returns:
             The metadata of the saved file.
         """
         if save_dir is None:
-            save_dir = os.path.join("output", "uploaded")
+            save_dir = PRIVATE_STORE_PATH
 
-        file_id = str(uuid.uuid4())
-        name, extension = os.path.splitext(file_name)
-        extension = extension.lstrip(".")
-        sanitized_name = _sanitize_file_name(name)
-        if extension == "":
-            raise ValueError("File is not supported!")
-        new_file_name = f"{sanitized_name}_{file_id}.{extension}"
-
-        file_path = os.path.join(save_dir, new_file_name)
-
-        if isinstance(content, str):
-            content = content.encode()
+        file_id, extension = cls._process_file_name(file_name)
+        file_path = os.path.join(save_dir, file_id)
 
+        # Write the file directly, handling both str and bytes
         try:
             os.makedirs(os.path.dirname(file_path), exist_ok=True)
-            with open(file_path, "wb") as file:
-                file.write(content)
+            with open(file_path, "wb") as f:
+                if isinstance(content, str):
+                    f.write(content.encode())
+                else:
+                    f.write(content)
         except PermissionError as e:
             logger.error(f"Permission denied when writing to file {file_path}: {e!s}")
             raise
@@ -87,31 +64,90 @@ class FileService:
         logger.info(f"Saved file to {file_path}")
 
         file_size = os.path.getsize(file_path)
-        file_url = (
-            f"{server_settings.file_server_url_prefix}/{save_dir}/{new_file_name}"
-        )
-        return DocumentFile(
+        file_url = cls._get_file_url(file_id, save_dir)
+        return ServerFile(
             id=file_id,
-            name=new_file_name,
             type=extension,
             size=file_size,
-            path=file_path,
             url=file_url,
-            refs=None,
+            path=file_path,
         )
 
     @classmethod
-    def get_file_url(cls, file_name: str, save_dir: Optional[str] = None) -> str:
+    def _process_file_name(cls, file_name: str) -> tuple[str, str]:
+        """
+        Process original file name to generate a unique file id and extension.
+        """
+        _id = str(uuid.uuid4())
+        name, extension = os.path.splitext(file_name)
+        extension = extension.lstrip(".")
+        if extension == "":
+            raise ValueError("File name is not valid! It must have an extension.")
+        # sanitize the name
+        name = re.sub(r"[^a-zA-Z0-9.]", "_", name)
+        file_id = f"{name}_{_id}.{extension}"
+        return file_id, extension
+
+    @classmethod
+    def _get_file_url(cls, file_id: str, save_dir: Optional[str] = None) -> str:
         """
         Get the URL of a file.
         """
         if save_dir is None:
-            save_dir = os.path.join("output", "uploaded")
-        return f"{server_settings.file_server_url_prefix}/{save_dir}/{file_name}"
+            save_dir = PRIVATE_STORE_PATH
+        # Ensure the path uses forward slashes for URLs
+        url_path = f"{save_dir}/{file_id}".replace("\\", "/")
+        return f"{server_settings.file_server_url_prefix}/{url_path}"
 
+    @classmethod
+    def get_file_path(cls, file_id: str, save_dir: Optional[str] = None) -> str:
+        """
+        Get the path of a private file.
 
-def _sanitize_file_name(file_name: str) -> str:
-    """
-    Sanitize the file name by replacing all non-alphanumeric characters with underscores.
-    """
-    return re.sub(r"[^a-zA-Z0-9.]", "_", file_name)
+        Args:
+            file_id (str): The ID of the file.
+            save_dir (Optional[str]): The path where the file is stored. Defaults to output/private if not provided.
+
+        Returns:
+            str: The full path to the file.
+        """
+        if save_dir is None:
+            save_dir = PRIVATE_STORE_PATH
+        return os.path.join(save_dir, file_id)
+
+    @classmethod
+    def get_file(cls, file_id: str, save_dir: Optional[str] = None) -> bytes:
+        """
+        Read and return the content of a file.
+
+        Args:
+            file_id (str): The ID of the file.
+            save_dir (Optional[str]): The path where the file is stored. Defaults to output/private if not provided.
+
+        Returns:
+            bytes: The content of the file.
+
+        Raises:
+            FileNotFoundError: If the file does not exist.
+        """
+        file_path = cls.get_file_path(file_id, save_dir)
+
+        try:
+            with open(file_path, "rb") as f:
+                return f.read()
+        except FileNotFoundError as e:
+            logger.error(f"File not found: {file_path}")
+            raise FileNotFoundError(f"File with ID '{file_id}' not found") from e
+        except Exception as e:
+            logger.error(f"Unexpected error when reading file {file_path}: {e!s}")
+            raise
+
+    @staticmethod
+    def _preprocess_base64_file(base64_content: str) -> Tuple[bytes, str]:
+        header, data = base64_content.split(",", 1)
+        mime_type = header.split(";")[0].split(":", 1)[1]
+        extension = mimetypes.guess_extension(mime_type)
+        if extension is None:
+            raise ValueError(f"Unsupported file type: {mime_type}")
+        extension = extension.lstrip(".")
+        return base64.b64decode(data), extension

python/llama-index-server/llama_index/server/services/llamacloud/index.py

@@ -92,14 +92,14 @@ class IndexConfig(BaseModel):
     def from_default(cls, chat_request: Optional[ChatRequest] = None) -> "IndexConfig":
         default_config = cls()
         if chat_request is not None and chat_request.data is not None:
-            llamacloud_config = chat_request.data.get("llamaCloudPipeline")
+            llamacloud_config = chat_request.data.llama_cloud_pipeline
             if llamacloud_config is not None:
-                default_config.llama_cloud_pipeline_config.pipeline = llamacloud_config[
-                    "pipeline"
-                ]
-                default_config.llama_cloud_pipeline_config.project = llamacloud_config[
-                    "project"
-                ]
+                default_config.llama_cloud_pipeline_config.pipeline = (
+                    llamacloud_config.pipeline
+                )
+                default_config.llama_cloud_pipeline_config.project = (
+                    llamacloud_config.project
+                )
         return default_config
 
 

python/llama-index-server/llama_index/server/tools/interpreter.py

@@ -4,10 +4,12 @@ import os
 import uuid
 from typing import Any, List, Optional
 
-from llama_index.core.tools import FunctionTool
-from llama_index.server.services.file import DocumentFile, FileService
 from pydantic import BaseModel
 
+from llama_index.core.tools import FunctionTool
+from llama_index.server.models.file import ServerFile
+from llama_index.server.services.file import FileService
+
 logger = logging.getLogger("uvicorn")
 
 
@@ -87,7 +89,7 @@ class E2BCodeInterpreter:
                         self.interpreter.files.write(file_path, content)
             logger.info(f"Uploaded {len(sandbox_files)} files to sandbox")
 
-    def _save_to_disk(self, base64_data: str, ext: str) -> DocumentFile:
+    def _save_to_disk(self, base64_data: str, ext: str) -> ServerFile:
         buffer = base64.b64decode(base64_data)
 
         # Output from e2b doesn't have a name. Create a random name for it.
@@ -117,7 +119,7 @@ class E2BCodeInterpreter:
                     output.append(
                         InterpreterExtraResult(
                             type=ext,
-                            filename=document_file.name,
+                            filename=document_file.id,
                             url=document_file.url,
                         )
                     )

python/llama-index-server/llama_index/server/utils/inline.py

@@ -0,0 +1,81 @@
+import json
+import re
+from typing import Any, List
+
+from pydantic import ValidationError
+
+from llama_index.core.workflow.events import Event
+from llama_index.server.models.chat import ChatAPIMessage
+from llama_index.core.agent.workflow.workflow_events import AgentStream
+
+INLINE_ANNOTATION_KEY = (
+    "annotation"  # the language key to detect inline annotation code in markdown
+)
+
+
+def get_inline_annotations(message: ChatAPIMessage) -> List[Any]:
+    """Extract inline annotations from a chat message."""
+    markdown_content = message.content
+
+    inline_annotations: List[Any] = []
+
+    # Regex to match annotation code blocks
+    # Matches ```annotation followed by content until closing ```
+    annotation_regex = re.compile(
+        rf"```{re.escape(INLINE_ANNOTATION_KEY)}\s*\n([\s\S]*?)\n```", re.MULTILINE
+    )
+
+    for match in annotation_regex.finditer(markdown_content):
+        json_content = match.group(1).strip() if match.group(1) else None
+
+        if not json_content:
+            continue
+
+        try:
+            # Parse the JSON content
+            parsed = json.loads(json_content)
+
+            # Check for required fields in the parsed annotation
+            if (
+                not isinstance(parsed, dict)
+                or "type" not in parsed
+                or "data" not in parsed
+            ):
+                continue
+
+            # Extract the annotation data
+            inline_annotations.append(parsed)
+        except (json.JSONDecodeError, ValidationError) as error:
+            # Skip invalid annotations - they might be malformed JSON or invalid schema
+            print(f"Failed to parse annotation: {error}")
+
+    return inline_annotations
+
+
+def to_inline_annotation(item: dict) -> str:
+    """
+    To append inline annotations to the stream, we need to wrap the annotation in a code block with the language key.
+    The language key is `annotation` and the code block is wrapped in backticks.
+
+    ```annotation
+    {
+      "type": "artifact",
+      "data": {...}
+    }
+    ```
+    """
+    return f"\n```{INLINE_ANNOTATION_KEY}\n{json.dumps(item)}\n```\n"
+
+
+def to_inline_annotation_event(event: Event) -> AgentStream:
+    """
+    Convert an event to an AgentStream with inline annotation format.
+    """
+    event_dict = event.model_dump()
+    return AgentStream(
+        delta=to_inline_annotation(event_dict),
+        response="",
+        current_agent_name="assistant",
+        tool_calls=[],
+        raw=event_dict,
+    )

python/llama-index-server/llama_index/server/utils/stream.py

@@ -0,0 +1,45 @@
+from typing import AsyncGenerator, Union
+from llama_index.core.base.llms.types import (
+    CompletionResponse,
+    CompletionResponseAsyncGen,
+)
+from llama_index.core.workflow import Context
+from llama_index.core.agent.workflow.workflow_events import AgentStream
+
+
+async def write_response_to_stream(
+    res: Union[CompletionResponse, CompletionResponseAsyncGen],
+    ctx: Context,
+    current_agent_name: str = "assistant",
+) -> str:
+    """
+    Handle both streaming and non-streaming LLM responses.
+
+    Args:
+        res: The LLM response (either streaming or non-streaming)
+        ctx: The workflow context for writing events to stream
+        current_agent_name: The name of the current agent (default: "assistant")
+
+    Returns:
+        The final response text as a string
+    """
+    final_response = ""
+
+    if isinstance(res, AsyncGenerator):
+        # Handle streaming response (CompletionResponseAsyncGen)
+        async for chunk in res:
+            ctx.write_event_to_stream(
+                AgentStream(
+                    delta=chunk.delta or "",
+                    response=final_response,
+                    current_agent_name=current_agent_name,
+                    tool_calls=[],
+                    raw=chunk.raw or "",
+                )
+            )
+            final_response = chunk.text
+    else:
+        # Handle non-streaming response (CompletionResponse)
+        final_response = res.text
+
+    return final_response

python/llama-index-server/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@create-llama/llama-index-server",
   "private": true,
-  "version": "0.1.20",
+  "version": "0.1.21",
   "type": "module",
   "scripts": {
     "prebuild": "uv run -- scripts/frontend.py --mode copy",

python/llama-index-server/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "llama-index-server"
-version = "0.1.20"
+version = "0.1.21"
 description = "llama-index fastapi server"
 readme = "README.md"
 license = "MIT"

python/llama-index-server/tests/api/test_chat_api.py

@@ -90,7 +90,6 @@ async def test_chat_router(
 
         # Verify the workflow was called with the correct arguments
         call_args = mock_workflow.run.call_args[1]
-        assert call_args["user_msg"] == "Hello, how are you?"
         assert isinstance(call_args["chat_history"], list)
         assert len(call_args["chat_history"]) == 0  # No history for first message
 
@@ -153,6 +152,5 @@ async def test_chat_with_agent_workflow(logger: logging.Logger) -> None:
 
         # Verify the workflow was called with the correct arguments
         call_args = mock_workflow.run.call_args[1]
-        assert call_args["user_msg"] == "What's the weather in New York?"
         assert isinstance(call_args["chat_history"], list)
         assert len(call_args["chat_history"]) == 0  # No history for first message

python/llama-index-server/tests/services/test_file_service.py

@@ -1,35 +1,24 @@
 import os
 import uuid
-from unittest.mock import mock_open, patch
+from unittest.mock import MagicMock, mock_open, patch
 
 import pytest
-from llama_index.server.services.file import FileService, _sanitize_file_name
+
+from llama_index.server.services.file import FileService
 
 
 class TestFileService:
-    def test_sanitize_file_name(self):
-        # Test with normal alphanumeric name
-        assert _sanitize_file_name("test123") == "test123"
-
-        # Test with spaces
-        assert _sanitize_file_name("test file") == "test_file"
-
-        # Test with special characters
-        assert _sanitize_file_name("test@file!name") == "test_file_name"
-
-        # Test with path-like characters
-        assert _sanitize_file_name("test/file/name") == "test_file_name"
-
-        # Test with dots (should be preserved)
-        assert _sanitize_file_name("test.file.name") == "test.file.name"
-
     @patch("uuid.uuid4")
     @patch("os.path.getsize")
     @patch("builtins.open", new_callable=mock_open)
     @patch("os.makedirs")
     def test_save_file_string_content(
-        self, mock_makedirs, mock_file_open, mock_getsize, mock_uuid
-    ):
+        self,
+        mock_makedirs: MagicMock,
+        mock_file_open: MagicMock,
+        mock_getsize: MagicMock,
+        mock_uuid: MagicMock,
+    ) -> None:
         # Setup
         test_uuid = "12345678-1234-5678-1234-567812345678"
         mock_uuid.return_value = uuid.UUID(test_uuid)
@@ -48,21 +37,23 @@ class TestFileService:
         mock_file_open.assert_called_once_with(expected_path, "wb")
         mock_file_open().write.assert_called_once_with(b"Hello World")
 
-        assert result.id == test_uuid
-        assert result.name == f"test_{test_uuid}.txt"
+        assert result.id == f"test_{test_uuid}.txt"
         assert result.type == "txt"
         assert result.size == 11
         assert result.path == expected_path
         assert result.url.endswith(expected_path.replace(os.path.sep, "/"))
-        assert result.refs is None
 
     @patch("uuid.uuid4")
     @patch("os.path.getsize")
     @patch("builtins.open", new_callable=mock_open)
     @patch("os.makedirs")
     def test_save_file_bytes_content(
-        self, mock_makedirs, mock_file_open, mock_getsize, mock_uuid
-    ):
+        self,
+        mock_makedirs: MagicMock,
+        mock_file_open: MagicMock,
+        mock_getsize: MagicMock,
+        mock_uuid: MagicMock,
+    ) -> None:
         # Setup
         test_uuid = "12345678-1234-5678-1234-567812345678"
         mock_uuid.return_value = uuid.UUID(test_uuid)
@@ -81,6 +72,7 @@ class TestFileService:
         mock_file_open.assert_called_once_with(expected_path, "wb")
         mock_file_open().write.assert_called_once_with(b"Hello World")
         assert result.path == expected_path
+        assert result.url.endswith(expected_path.replace(os.path.sep, "/"))
         assert result.type == "txt"
 
     @patch("uuid.uuid4")
@@ -88,8 +80,12 @@ class TestFileService:
     @patch("builtins.open", new_callable=mock_open)
     @patch("os.makedirs")
     def test_save_file_with_special_characters(
-        self, mock_makedirs, mock_file_open, mock_getsize, mock_uuid
-    ):
+        self,
+        mock_makedirs: MagicMock,
+        mock_file_open: MagicMock,
+        mock_getsize: MagicMock,
+        mock_uuid: MagicMock,
+    ) -> None:
         # Setup
         test_uuid = "12345678-1234-5678-1234-567812345678"
         mock_uuid.return_value = uuid.UUID(test_uuid)
@@ -107,15 +103,19 @@ class TestFileService:
         )
         mock_file_open.assert_called_once_with(expected_path, "wb")
         assert result.path == expected_path
-        assert result.name == f"test_file__{test_uuid}.txt"
+        assert result.url.endswith(expected_path.replace(os.path.sep, "/"))
 
     @patch("uuid.uuid4")
     @patch("os.path.getsize")
     @patch("builtins.open", new_callable=mock_open)
     @patch("os.makedirs")
     def test_save_file_default_directory(
-        self, mock_makedirs, mock_file_open, mock_getsize, mock_uuid
-    ):
+        self,
+        mock_makedirs: MagicMock,
+        mock_file_open: MagicMock,
+        mock_getsize: MagicMock,
+        mock_uuid: MagicMock,
+    ) -> None:
         # Setup
         test_uuid = "12345678-1234-5678-1234-567812345678"
         mock_uuid.return_value = uuid.UUID(test_uuid)
@@ -125,11 +125,12 @@ class TestFileService:
         result = FileService.save_file(content="Hello World", file_name="test.txt")
 
         # Assert
-        expected_path = os.path.join("output", "uploaded", f"test_{test_uuid}.txt")
+        expected_path = os.path.join("output", "private", f"test_{test_uuid}.txt")
         mock_makedirs.assert_called_once_with(
             os.path.dirname(expected_path), exist_ok=True
         )
         assert result.path == expected_path
+        assert result.url.endswith(expected_path.replace(os.path.sep, "/"))
 
     @patch("uuid.uuid4")
     @patch("os.getenv")
@@ -137,8 +138,13 @@ class TestFileService:
     @patch("builtins.open", new_callable=mock_open)
     @patch("os.makedirs")
     def test_save_file_custom_url_prefix(
-        self, mock_makedirs, mock_file_open, mock_getsize, mock_getenv, mock_uuid
-    ):
+        self,
+        mock_makedirs: MagicMock,
+        mock_file_open: MagicMock,
+        mock_getsize: MagicMock,
+        mock_getenv: MagicMock,
+        mock_uuid: MagicMock,
+    ) -> None:
         # Setup
         test_uuid = "12345678-1234-5678-1234-567812345678"
         mock_uuid.return_value = uuid.UUID(test_uuid)
@@ -157,13 +163,11 @@ class TestFileService:
         )
         mock_file_open.assert_called_once_with(expected_path, "wb")
         assert result.path == expected_path
-        # URL paths must use forward slashes, even on Windows
-        expected_url = f"/api/files/test_dir/test_{test_uuid}.txt"
-        assert result.url == expected_url
+        assert result.url.endswith(expected_path.replace(os.path.sep, "/"))
 
-    def test_save_file_no_extension(self):
+    def test_save_file_no_extension(self) -> None:
         # Test that saving a file without extension raises ValueError
-        with pytest.raises(ValueError, match="File is not supported!"):
+        with pytest.raises(ValueError, match="File name is not valid!"):
             FileService.save_file(
                 content="Hello World", file_name="test", save_dir="test_dir"
             )
@@ -173,8 +177,12 @@ class TestFileService:
     @patch("builtins.open")
     @patch("os.makedirs")
     def test_save_file_permission_error(
-        self, mock_makedirs, mock_file_open, mock_getsize, mock_uuid
-    ):
+        self,
+        mock_makedirs: MagicMock,
+        mock_file_open: MagicMock,
+        mock_getsize: MagicMock,
+        mock_uuid: MagicMock,
+    ) -> None:
         # Setup
         test_uuid = "12345678-1234-5678-1234-567812345678"
         mock_uuid.return_value = uuid.UUID(test_uuid)
@@ -191,8 +199,12 @@ class TestFileService:
     @patch("builtins.open")
     @patch("os.makedirs")
     def test_save_file_io_error(
-        self, mock_makedirs, mock_file_open, mock_getsize, mock_uuid
-    ):
+        self,
+        mock_makedirs: MagicMock,
+        mock_file_open: MagicMock,
+        mock_getsize: MagicMock,
+        mock_uuid: MagicMock,
+    ) -> None:
         # Setup
         test_uuid = "12345678-1234-5678-1234-567812345678"
         mock_uuid.return_value = uuid.UUID(test_uuid)