try fixed version

Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>

.changeset/warm-mugs-feel.md

@@ -0,0 +1,5 @@
+---
+"@create-llama/llama-index-server": patch
+---
+
+fix cannot catch the error raised from the workflow

.github/workflows/e2e.yml

@@ -12,100 +12,18 @@ on:
       - ".github/workflows/*llama_index_server.yml"
 
 jobs:
-  e2e-python:
-    name: python
-    timeout-minutes: 60
-    strategy:
-      fail-fast: true
-      matrix:
-        node-version: [20]
-        python-version: ["3.11"]
-        os: [macos-latest, windows-latest, ubuntu-22.04]
-        frameworks: ["fastapi"]
-        datasources: ["--no-files", "--example-file", "--llamacloud"]
-        template-types: ["streaming", "llamaindexserver"]
-    defaults:
-      run:
-        shell: bash
-    runs-on: ${{ matrix.os }}
-    steps:
-      - uses: actions/checkout@v4
-
-      - name: Set up python ${{ matrix.python-version }}
-        uses: actions/setup-python@v5
-        with:
-          python-version: ${{ matrix.python-version }}
-
-      - name: Install uv
-        run: curl -LsSf https://astral.sh/uv/install.sh | sh
-      - name: Add uv to PATH # Ensure uv is available in subsequent steps
-        run: echo "$HOME/.cargo/bin" >> $GITHUB_PATH
-
-      - uses: pnpm/action-setup@v3
-
-      - name: Setup Node.js ${{ matrix.node-version }}
-        uses: actions/setup-node@v4
-        with:
-          node-version: ${{ matrix.node-version }}
-          cache: "pnpm"
-
-      - name: Install dependencies
-        run: pnpm install
-
-      - name: Install Playwright Browsers
-        run: pnpm exec playwright install --with-deps
-        working-directory: packages/create-llama
-
-      - name: Build create-llama
-        run: pnpm run build
-        working-directory: packages/create-llama
-
-      - name: Install
-        run: pnpm run pack-install
-        working-directory: packages/create-llama
-
-      - name: Build and store server package
-        run: |
-          pnpm run build
-          wheel_file=$(ls dist/*.whl | head -n 1)
-          mkdir -p "${{ runner.temp }}"
-          cp "$wheel_file" "${{ runner.temp }}/"
-          echo "SERVER_PACKAGE_PATH=${{ runner.temp }}/$(basename "$wheel_file")" >> $GITHUB_ENV
-        working-directory: python/llama-index-server
-
-      - name: Run Playwright tests for Python
-        run: pnpm run e2e:python
-        env:
-          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
-          LLAMA_CLOUD_API_KEY: ${{ secrets.LLAMA_CLOUD_API_KEY }}
-          FRAMEWORK: ${{ matrix.frameworks }}
-          DATASOURCE: ${{ matrix.datasources }}
-          TEMPLATE_TYPE: ${{ matrix.template-types }}
-          PYTHONIOENCODING: utf-8
-          PYTHONLEGACYWINDOWSSTDIO: utf-8
-          SERVER_PACKAGE_PATH: ${{ env.SERVER_PACKAGE_PATH }}
-        working-directory: packages/create-llama
-
-      - uses: actions/upload-artifact@v4
-        if: always()
-        with:
-          name: playwright-report-python-${{ matrix.os }}-${{ matrix.frameworks }}-${{ matrix.datasources }}-${{ matrix.template-types }}
-          path: packages/create-llama/playwright-report/
-          overwrite: true
-          retention-days: 30
-
   e2e-typescript:
     name: typescript
     timeout-minutes: 60
     strategy:
       fail-fast: true
       matrix:
-        node-version: [20, 22]
+        node-version: [20]
         python-version: ["3.11"]
-        os: [macos-latest, windows-latest, ubuntu-22.04]
+        os: [macos-latest]
         frameworks: ["nextjs"]
-        datasources: ["--no-files", "--example-file", "--llamacloud"]
-        template-types: ["streaming", "llamaindexserver"]
+        datasources: ["--llamacloud"]
+        template-types: ["llamaindexserver"]
     defaults:
       run:
         shell: bash

packages/create-llama/CHANGELOG.md

@@ -1,5 +1,18 @@
 # create-llama
 
+## 0.5.21
+
+### Patch Changes
+
+- af9ad3c: feat: show document artifact after generating report
+- a543a27: feat: bump chat-ui with inline artifact
+
+## 0.5.20
+
+### Patch Changes
+
+- 3ff0a18: fix: default header padding
+
 ## 0.5.19
 
 ### Patch Changes

packages/create-llama/e2e/shared/llamaindexserver_template.spec.ts

@@ -24,9 +24,9 @@ const appType: AppType = "--frontend";
 const userMessage = "Write a blog post about physical standards for letters";
 const templateUseCases = [
   "agentic_rag",
-  "financial_report",
-  "deep_research",
-  "code_generator",
+  // "financial_report",
+  // "deep_research",
+  // "code_generator",
 ];
 const ejectDir = "next";
 

packages/create-llama/package.json

@@ -1,6 +1,6 @@
 {
   "name": "create-llama",
-  "version": "0.5.19",
+  "version": "0.5.21",
   "description": "Create LlamaIndex-powered apps with one command",
   "keywords": [
     "rag",

packages/create-llama/templates/components/ui/layout/header.tsx

@@ -4,7 +4,7 @@ import { Sparkles, Star } from "lucide-react";
 
 export default function Header() {
   return (
-    <div className="flex items-center justify-between px-4 pt-2">
+    <div className="flex items-center justify-between p-2 px-4">
       <div className="flex items-center gap-2">
         <Sparkles className="size-4" />
         <h1 className="font-semibold">LlamaIndex App</h1>

packages/create-llama/templates/components/use-cases/python/deep_research/workflow.py

@@ -23,7 +23,18 @@ from llama_index.core.workflow import (
     Workflow,
     step,
 )
-from llama_index.server.api.models import ChatRequest, SourceNodesEvent, UIEvent
+from llama_index.server.api.models import (
+    ArtifactEvent,
+    ArtifactType,
+    ChatRequest,
+    SourceNodesEvent,
+    UIEvent,
+    Artifact,
+    DocumentArtifactData,
+    DocumentArtifactSource,
+)
+import time
+from llama_index.server.utils.stream import write_response_to_stream
 from pydantic import BaseModel, Field
 
 logger = logging.getLogger("uvicorn")
@@ -365,8 +376,31 @@ class DeepResearchWorkflow(Workflow):
             user_request=self.user_request,
             stream=self.stream,
         )
+
+        final_response = await write_response_to_stream(res, ctx)
+
+        ctx.write_event_to_stream(
+            ArtifactEvent(
+                data=Artifact(
+                    type=ArtifactType.DOCUMENT,
+                    created_at=int(time.time()),
+                    data=DocumentArtifactData(
+                        title="DeepResearch Report",
+                        content=final_response,
+                        type="markdown",
+                        sources=[
+                            DocumentArtifactSource(
+                                id=node.id_,
+                            )
+                            for node in self.context_nodes
+                        ],
+                    ),
+                ),
+            )
+        )
+
         return StopEvent(
-            result=res,
+            result="",
         )
 
 

packages/create-llama/templates/components/use-cases/typescript/code_generator/src/app/workflow.ts

@@ -1,4 +1,4 @@
-import { extractLastArtifact } from "@llamaindex/server";
+import { artifactEvent, extractLastArtifact } from "@llamaindex/server";
 import { ChatMemoryBuffer, MessageContent, Settings } from "llamaindex";
 
 import {
@@ -52,19 +52,6 @@ const synthesizeAnswerEvent = workflowEvent<object>();
 
 const uiEvent = workflowEvent<UIEvent>();
 
-const artifactEvent = workflowEvent<{
-  type: "artifact";
-  data: {
-    type: "code";
-    created_at: number;
-    data: {
-      language: string;
-      file_name: string;
-      code: string;
-    };
-  };
-}>();
-
 export function workflowFactory(reqBody: any) {
   const llm = Settings.llm;
 

packages/create-llama/templates/components/use-cases/typescript/deep_research/src/app/workflow.ts

@@ -1,4 +1,4 @@
-import { toSourceEvent } from "@llamaindex/server";
+import { artifactEvent, toSourceEvent } from "@llamaindex/server";
 import {
   agentStreamEvent,
   createStatefulMiddleware,
@@ -339,6 +339,26 @@ export function getWorkflow(index: VectorStoreIndex | LlamaCloudIndex) {
         }),
       );
     }
+
+    // Open the generated report in Canvas
+    sendEvent(
+      artifactEvent.with({
+        type: "artifact",
+        data: {
+          type: "document",
+          created_at: Date.now(),
+          data: {
+            title: "DeepResearch Report",
+            content: response,
+            type: "markdown",
+            sources: state.contextNodes.map((node) => ({
+              id: node.node.id_,
+            })),
+          },
+        },
+      }),
+    );
+
     return stopAgentEvent.with({
       result: response,
     });

packages/create-llama/templates/components/use-cases/typescript/document_generator/src/app/workflow.ts

@@ -1,4 +1,4 @@
-import { extractLastArtifact } from "@llamaindex/server";
+import { artifactEvent, extractLastArtifact } from "@llamaindex/server";
 import { ChatMemoryBuffer, MessageContent, Settings } from "llamaindex";
 
 import {
@@ -55,19 +55,6 @@ const synthesizeAnswerEvent = workflowEvent<{
 
 const uiEvent = workflowEvent<UIEvent>();
 
-const artifactEvent = workflowEvent<{
-  type: "artifact";
-  data: {
-    type: "document";
-    created_at: number;
-    data: {
-      title: string;
-      content: string;
-      type: "markdown" | "html";
-    };
-  };
-}>();
-
 export function workflowFactory(reqBody: any) {
   const llm = Settings.llm;
 

packages/create-llama/templates/types/llamaindexserver/nextjs/package.json

@@ -20,7 +20,7 @@
   },
   "devDependencies": {
     "@types/node": "^20.10.3",
-    "tsx": "^4.7.2",
+    "tsx": "4.7.2",
     "typescript": "^5.3.2",
     "nodemon": "^3.1.10"
   }

packages/server/CHANGELOG.md

@@ -1,5 +1,27 @@
 # @llamaindex/server
 
+## 0.2.7
+
+### Patch Changes
+
+- af9ad3c: feat: show document artifact after generating report
+- a543a27: feat: bump chat-ui with inline artifact
+- 1ff6eaf: Add support for chat upload file
+
+## 0.2.6
+
+### Patch Changes
+
+- 3ff0a18: fix: default header padding
+- df10474: fix: missing cursor pointer for button
+- 087c961: Support zod and chat-ui hooks for custom components
+
+## 0.2.5
+
+### Patch Changes
+
+- 058b376: Fix generate script for ejected project
+
 ## 0.2.4
 
 ### Patch Changes

packages/server/README.md

@@ -60,6 +60,7 @@ The `LlamaIndexServer` accepts the following configuration options:
 - `workflow`: A callable function that creates a workflow instance for each request. See [Workflow factory contract](#workflow-factory-contract) for more details.
 - `uiConfig`: An object to configure the chat UI containing the following properties:
   - `starterQuestions`: List of starter questions for the chat UI (default: `[]`)
+  - `enableFileUpload`: Whether to enable file upload in the chat UI (default: `false`). See [Upload file example](./examples/private-file/README.md) for more details.
   - `componentsDir`: The directory for custom UI components rendering events emitted by the workflow. The default is undefined, which does not render custom UI components.
   - `layoutDir`: The directory for custom layout sections. The default value is `layout`. See [Custom Layout](#custom-layout) for more details.
   - `llamaCloudIndexSelector`: Whether to show the LlamaCloud index selector in the chat UI (requires `LLAMA_CLOUD_API_KEY` to be set in the environment variables) (default: `false`)
@@ -71,9 +72,18 @@ See all Nextjs Custom Server options [here](https://nextjs.org/docs/app/building
 
 ## Workflow factory contract
 
-The `workflow` provided will be called for each chat request to initialize a new workflow instance. The contract of the generated workflow must be the same as for the [Agent Workflow](https://ts.llamaindex.ai/docs/llamaindex/modules/agents/agent_workflow).
+The `workflow` provided will be called for each chat request to initialize a new workflow instance. For advanced use cases, you can define workflowFactory with a chatBody which include list of UI messages in the request body.
 
-This means that the workflow must handle a `startAgentEvent` event, which is the entry point of the workflow and contains the following information in it's `data` property:
+```typescript
+import { type Message } from "ai";
+import { agent } from "@llamaindex/workflow";
+
+const workflowFactory = (chatBody: { messages: Message[] }) => {
+  ...
+};
+```
+
+The contract of the generated workflow must be the same as for the [Agent Workflow](https://ts.llamaindex.ai/docs/llamaindex/modules/agents/agent_workflow). This means that the workflow must handle a `startAgentEvent` event, which is the entry point of the workflow and contains the following information in it's `data` property:
 
 ```typescript
 {

packages/server/examples/README.md

@@ -1,12 +1,38 @@
 # LlamaIndex Server Examples
 
-This directory contains examples of how to use the LlamaIndex Server.
+This directory provides example projects demonstrating how to use the LlamaIndex Server.
 
-## Running the examples
+## How to Run the Examples
 
-```bash
-export OPENAI_API_KEY=your_openai_api_key
-pnpm run dev
-```
+1. **Install dependencies**
 
-## Open browser at http://localhost:3000
+   In the root of this directory, run:
+
+   ```bash
+   pnpm install
+   ```
+
+2. **Set your OpenAI API key**
+
+   Export your OpenAI API key as an environment variable:
+
+   ```bash
+   export OPENAI_API_KEY=your_openai_api_key
+   ```
+
+3. **Start an example**
+
+   Replace `<example>` with the name of the example you want to run (e.g., `private-file`):
+
+   ```bash
+   pnpm nodemon --exec tsx <example>/index.ts
+   ```
+
+4. **Open the application in your browser**
+
+   Visit [http://localhost:3000](http://localhost:3000) to interact with the running example.
+
+## Notes
+
+- Make sure you have [pnpm](https://pnpm.io/) installed.
+- Each example may have its own specific instructions or requirements; check the individual example's index.ts for details.

packages/server/examples/agentic-rag/index.ts

@@ -1,12 +1,7 @@
+import { OpenAI, OpenAIEmbedding } from "@llamaindex/openai";
 import { LlamaIndexServer } from "@llamaindex/server";
 import { agent } from "@llamaindex/workflow";
-import {
-  Document,
-  OpenAI,
-  OpenAIEmbedding,
-  Settings,
-  VectorStoreIndex,
-} from "llamaindex";
+import { Document, Settings, VectorStoreIndex } from "llamaindex";
 
 Settings.llm = new OpenAI({
   model: "gpt-4o-mini",

packages/server/examples/code-gen/README.md

@@ -0,0 +1,22 @@
+This example demonstrates how to use the code generation workflow.
+
+```ts
+new LlamaIndexServer({
+  workflow: workflowFactory,
+  uiConfig: {
+    starterQuestions: [
+      "Generate a calculator app",
+      "Create a simple todo list app",
+    ],
+    componentsDir: "components",
+  },
+  port: 3000,
+}).start();
+```
+
+Export OpenAI API key and start the server in dev mode.
+
+```bash
+export OPENAI_API_KEY=<your-openai-api-key>
+npx nodemon --exec tsx index.ts
+```

packages/server/examples/code-gen/components/ui_event.jsx

@@ -0,0 +1,132 @@
+import { Badge } from "@/components/ui/badge";
+import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card";
+import { Progress } from "@/components/ui/progress";
+import { Skeleton } from "@/components/ui/skeleton";
+import { cn } from "@/lib/utils";
+import { Markdown } from "@llamaindex/chat-ui/widgets";
+import { ListChecks, Loader2, Wand2 } from "lucide-react";
+import { useEffect, useState } from "react";
+
+const STAGE_META = {
+  plan: {
+    icon: ListChecks,
+    badgeText: "Step 1/2: Planning",
+    gradient: "from-blue-100 via-blue-50 to-white",
+    progress: 33,
+    iconBg: "bg-blue-100 text-blue-600",
+    badge: "bg-blue-100 text-blue-700",
+  },
+  generate: {
+    icon: Wand2,
+    badgeText: "Step 2/2: Generating",
+    gradient: "from-violet-100 via-violet-50 to-white",
+    progress: 66,
+    iconBg: "bg-violet-100 text-violet-600",
+    badge: "bg-violet-100 text-violet-700",
+  },
+};
+
+function ArtifactWorkflowCard({ event }) {
+  const [visible, setVisible] = useState(event?.state !== "completed");
+  const [fade, setFade] = useState(false);
+
+  useEffect(() => {
+    if (event?.state === "completed") {
+      setVisible(false);
+    } else {
+      setVisible(true);
+      setFade(false);
+    }
+  }, [event?.state]);
+
+  if (!event || !visible) return null;
+
+  const { state, requirement } = event;
+  const meta = STAGE_META[state];
+
+  if (!meta) return null;
+
+  return (
+    <div className="flex min-h-[180px] w-full items-center justify-center py-2">
+      <Card
+        className={cn(
+          "w-full rounded-xl shadow-md transition-all duration-500",
+          "border-0",
+          fade && "pointer-events-none opacity-0",
+          `bg-gradient-to-br ${meta.gradient}`,
+        )}
+        style={{
+          boxShadow:
+            "0 2px 12px 0 rgba(80, 80, 120, 0.08), 0 1px 3px 0 rgba(80, 80, 120, 0.04)",
+        }}
+      >
+        <CardHeader className="flex flex-row items-center gap-2 px-3 pb-1 pt-2">
+          <div
+            className={cn(
+              "flex items-center justify-center rounded-full p-1",
+              meta.iconBg,
+            )}
+          >
+            <meta.icon className="h-5 w-5" />
+          </div>
+          <CardTitle className="flex items-center gap-2 text-base font-semibold">
+            <Badge className={cn("ml-1", meta.badge, "px-2 py-0.5 text-xs")}>
+              {meta.badgeText}
+            </Badge>
+          </CardTitle>
+        </CardHeader>
+        <CardContent className="px-3 py-1">
+          {state === "plan" && (
+            <div className="flex flex-col items-center gap-2 py-2">
+              <Loader2 className="mb-1 h-6 w-6 animate-spin text-blue-400" />
+              <div className="text-center text-sm font-medium text-blue-900">
+                Analyzing your request...
+              </div>
+              <Skeleton className="mt-1 h-3 w-1/2 rounded-full" />
+            </div>
+          )}
+          {state === "generate" && (
+            <div className="flex flex-col gap-2 py-2">
+              <div className="flex items-center gap-1">
+                <Loader2 className="h-4 w-4 animate-spin text-violet-400" />
+                <span className="text-sm font-medium text-violet-900">
+                  Working on the requirement:
+                </span>
+              </div>
+              <div className="max-h-24 overflow-auto rounded-lg border border-violet-200 bg-violet-50 px-2 py-1 text-xs">
+                {requirement ? (
+                  <Markdown content={requirement} />
+                ) : (
+                  <span className="italic text-violet-400">
+                    No requirements available yet.
+                  </span>
+                )}
+              </div>
+            </div>
+          )}
+        </CardContent>
+        <div className="px-3 pb-2 pt-1">
+          <Progress
+            value={meta.progress}
+            className={cn(
+              "h-1 rounded-full bg-gray-200",
+              state === "plan" && "bg-blue-200",
+              state === "generate" && "bg-violet-200",
+            )}
+          />
+        </div>
+      </Card>
+    </div>
+  );
+}
+
+export default function Component({ events }) {
+  const aggregateEvents = () => {
+    if (!events || events.length === 0) return null;
+    return events[events.length - 1];
+  };
+
+  const event = aggregateEvents();
+
+  return <ArtifactWorkflowCard event={event} />;
+}

packages/server/examples/code-gen/index.ts

@@ -0,0 +1,20 @@
+import { OpenAI } from "@llamaindex/openai";
+import { LlamaIndexServer } from "@llamaindex/server";
+import { Settings } from "llamaindex";
+import { workflowFactory } from "./src/app/workflow";
+
+Settings.llm = new OpenAI({
+  model: "gpt-4o-mini",
+});
+
+new LlamaIndexServer({
+  workflow: workflowFactory,
+  uiConfig: {
+    starterQuestions: [
+      "Generate a calculator app",
+      "Create a simple todo list app",
+    ],
+    componentsDir: "components",
+  },
+  port: 3000,
+}).start();

packages/server/examples/code-gen/src/app/workflow.ts

@@ -0,0 +1,337 @@
+import { artifactEvent, extractLastArtifact } from "@llamaindex/server";
+import { ChatMemoryBuffer, MessageContent, Settings } from "llamaindex";
+
+import {
+  agentStreamEvent,
+  createStatefulMiddleware,
+  createWorkflow,
+  startAgentEvent,
+  stopAgentEvent,
+  workflowEvent,
+} from "@llamaindex/workflow";
+
+import { z } from "zod";
+
+export const RequirementSchema = z.object({
+  next_step: z.enum(["answering", "coding"]),
+  language: z.string().nullable().optional(),
+  file_name: z.string().nullable().optional(),
+  requirement: z.string(),
+});
+
+export type Requirement = z.infer<typeof RequirementSchema>;
+
+export const UIEventSchema = z.object({
+  type: z.literal("ui_event"),
+  data: z.object({
+    state: z
+      .enum(["plan", "generate", "completed"])
+      .describe(
+        "The current state of the workflow: 'plan', 'generate', or 'completed'.",
+      ),
+    requirement: z
+      .string()
+      .optional()
+      .describe(
+        "An optional requirement creating or updating a code, if applicable.",
+      ),
+  }),
+});
+
+export type UIEvent = z.infer<typeof UIEventSchema>;
+const planEvent = workflowEvent<{
+  userInput: MessageContent;
+  context?: string | undefined;
+}>();
+
+const generateArtifactEvent = workflowEvent<{
+  requirement: Requirement;
+}>();
+
+const synthesizeAnswerEvent = workflowEvent<object>();
+
+const uiEvent = workflowEvent<UIEvent>();
+
+export function workflowFactory(reqBody: unknown) {
+  const llm = Settings.llm;
+
+  const { withState, getContext } = createStatefulMiddleware(() => {
+    return {
+      memory: new ChatMemoryBuffer({ llm }),
+      lastArtifact: extractLastArtifact(reqBody),
+    };
+  });
+  const workflow = withState(createWorkflow());
+
+  workflow.handle([startAgentEvent], async ({ data }) => {
+    const { userInput, chatHistory = [] } = data;
+    // Prepare chat history
+    const { state } = getContext();
+    // Put user input to the memory
+    if (!userInput) {
+      throw new Error("Missing user input to start the workflow");
+    }
+    state.memory.set(chatHistory);
+    state.memory.put({ role: "user", content: userInput });
+
+    return planEvent.with({
+      userInput: userInput,
+      context: state.lastArtifact
+        ? JSON.stringify(state.lastArtifact)
+        : undefined,
+    });
+  });
+
+  workflow.handle([planEvent], async ({ data: planData }) => {
+    const { sendEvent } = getContext();
+    const { state } = getContext();
+    sendEvent(
+      uiEvent.with({
+        type: "ui_event",
+        data: {
+          state: "plan",
+        },
+      }),
+    );
+    const user_msg = planData.userInput;
+    const context = planData.context
+      ? `## The context is: \n${planData.context}\n`
+      : "";
+    const prompt = `
+You are a product analyst responsible for analyzing the user's request and providing the next step for code or document generation.
+You are helping user with their code artifact. To update the code, you need to plan a coding step.
+
+Follow these instructions:
+1. Carefully analyze the conversation history and the user's request to determine what has been done and what the next step should be.
+2. The next step must be one of the following two options:
+    - "coding": To make the changes to the current code.
+    - "answering": If you don't need to update the current code or need clarification from the user.
+Important: Avoid telling the user to update the code themselves, you are the one who will update the code (by planning a coding step).
+3. If the next step is "coding", you may specify the language ("typescript" or "python") and file_name if known, otherwise set them to null. 
+4. The requirement must be provided clearly what is the user request and what need to be done for the next step in details
+    as precise and specific as possible, don't be stingy with in the requirement.
+5. If the next step is "answering", set language and file_name to null, and the requirement should describe what to answer or explain to the user.
+6. Be concise; only return the requirements for the next step.
+7. The requirements must be in the following format:
+    \`\`\`json
+    {
+        "next_step": "answering" | "coding",
+        "language": "typescript" | "python" | null,
+        "file_name": string | null,
+        "requirement": string
+    }
+    \`\`\`
+
+## Example 1:
+User request: Create a calculator app.
+You should return:
+\`\`\`json
+{
+    "next_step": "coding",
+    "language": "typescript",
+    "file_name": "calculator.tsx",
+    "requirement": "Generate code for a calculator app that has a simple UI with a display and button layout. The display should show the current input and the result. The buttons should include basic operators, numbers, clear, and equals. The calculation should work correctly."
+}
+\`\`\`
+
+## Example 2:
+User request: Explain how the game loop works.
+Context: You have already generated the code for a snake game.
+You should return:
+\`\`\`json
+{
+    "next_step": "answering",
+    "language": null,
+    "file_name": null,
+    "requirement": "The user is asking about the game loop. Explain how the game loop works."
+}
+\`\`\`
+
+${context}
+
+Now, plan the user's next step for this request:
+${user_msg}
+`;
+
+    const response = await llm.complete({
+      prompt,
+    });
+    // parse the response to Requirement
+    // 1. use regex to find the json block
+    const jsonBlock = response.text.match(/```json\s*([\s\S]*?)\s*```/);
+    if (!jsonBlock) {
+      throw new Error("No JSON block found in the response.");
+    }
+    const requirement = RequirementSchema.parse(JSON.parse(jsonBlock[1]));
+    state.memory.put({
+      role: "assistant",
+      content: `The plan for next step: \n${response.text}`,
+    });
+
+    if (requirement.next_step === "coding") {
+      return generateArtifactEvent.with({
+        requirement,
+      });
+    } else {
+      return synthesizeAnswerEvent.with({});
+    }
+  });
+
+  workflow.handle([generateArtifactEvent], async ({ data: planData }) => {
+    const { sendEvent } = getContext();
+    const { state } = getContext();
+
+    sendEvent(
+      uiEvent.with({
+        type: "ui_event",
+        data: {
+          state: "generate",
+          requirement: planData.requirement.requirement,
+        },
+      }),
+    );
+
+    const previousArtifact = state.lastArtifact
+      ? JSON.stringify(state.lastArtifact)
+      : "There is no previous artifact";
+    const requirementText = planData.requirement.requirement;
+
+    const prompt = `
+        You are a skilled developer who can help user with coding.
+        You are given a task to generate or update a code for a given requirement.
+
+        ## Follow these instructions:
+        **1. Carefully read the user's requirements.** 
+           If any details are ambiguous or missing, make reasonable assumptions and clearly reflect those in your output.
+           If the previous code is provided:
+           + Carefully analyze the code with the request to make the right changes.
+           + Avoid making a lot of changes from the previous code if the request is not to write the code from scratch again.
+        **2. For code requests:**
+           - If the user does not specify a framework or language, default to a React component using the Next.js framework.
+           - For Next.js, use Shadcn UI components, Typescript, @types/node, @types/react, @types/react-dom, PostCSS, and TailwindCSS.
+           The import pattern should be:
+           \`\`\`typescript
+           import { ComponentName } from "@/components/ui/component-name"
+           import { Markdown } from "@llamaindex/chat-ui"
+           import { cn } from "@/lib/utils"
+           \`\`\`
+           - Ensure the code is idiomatic, production-ready, and includes necessary imports.
+           - Only generate code relevant to the user's request—do not add extra boilerplate.
+        **3. Don't be verbose on response**
+           - No other text or comments only return the code which wrapped by \`\`\`language\`\`\` block.
+           - If the user's request is to update the code, only return the updated code.
+        **4. Only the following languages are allowed: "typescript", "python".**
+        **5. If there is no code to update, return the reason without any code block.**
+           
+        ## Example:
+        \`\`\`typescript
+        import React from "react";
+        import { Button } from "@/components/ui/button";
+        import { cn } from "@/lib/utils";
+
+        export default function MyComponent() {
+        return (
+           <div className="flex flex-col items-center justify-center h-screen">
+              <Button>Click me</Button>
+           </div>
+        );
+        }
+        \`\`\`
+
+        The previous code is:
+        {previousArtifact}
+
+        Now, i have to generate the code for the following requirement:
+        {requirement}
+      `
+      .replace("{previousArtifact}", previousArtifact)
+      .replace("{requirement}", requirementText);
+
+    const response = await llm.complete({
+      prompt,
+    });
+
+    // Extract the code from the response
+    const codeMatch = response.text.match(/```(\w+)([\s\S]*)```/);
+    if (!codeMatch) {
+      return synthesizeAnswerEvent.with({});
+    }
+
+    const code = codeMatch[2].trim();
+
+    // Put the generated code to the memory
+    state.memory.put({
+      role: "assistant",
+      content: `Updated the code: \n${response.text}`,
+    });
+
+    // To show the Canvas panel for the artifact
+    sendEvent(
+      artifactEvent.with({
+        type: "artifact",
+        data: {
+          type: "code",
+          created_at: Date.now(),
+          data: {
+            language: planData.requirement.language || "",
+            file_name: planData.requirement.file_name || "",
+            code,
+          },
+        },
+      }),
+    );
+
+    return synthesizeAnswerEvent.with({});
+  });
+
+  workflow.handle([synthesizeAnswerEvent], async () => {
+    const { sendEvent } = getContext();
+    const { state } = getContext();
+
+    const chatHistory = await state.memory.getMessages();
+    const messages = [
+      ...chatHistory,
+      {
+        role: "system" as const,
+        content: `
+        You are a helpful assistant who is responsible for explaining the work to the user.
+        Based on the conversation history, provide an answer to the user's question. 
+        The user has access to the code so avoid mentioning the whole code again in your response.
+      `,
+      },
+    ];
+
+    const responseStream = await llm.chat({
+      messages,
+      stream: true,
+    });
+
+    sendEvent(
+      uiEvent.with({
+        type: "ui_event",
+        data: {
+          state: "completed",
+        },
+      }),
+    );
+
+    let response = "";
+    for await (const chunk of responseStream) {
+      response += chunk.delta;
+      sendEvent(
+        agentStreamEvent.with({
+          delta: chunk.delta,
+          response: "",
+          currentAgentName: "assistant",
+          raw: chunk,
+        }),
+      );
+    }
+
+    return stopAgentEvent.with({
+      result: response,
+    });
+  });
+
+  return workflow;
+}

packages/server/examples/custom-layout/index.ts

@@ -1,8 +1,13 @@
+import { OpenAI } from "@llamaindex/openai";
 import { LlamaIndexServer } from "@llamaindex/server";
 import { agent } from "@llamaindex/workflow";
-import { tool } from "llamaindex";
+import { Settings, tool } from "llamaindex";
 import { z } from "zod";
 
+Settings.llm = new OpenAI({
+  model: "gpt-4o-mini",
+});
+
 const weatherAgent = agent({
   tools: [
     tool({

packages/server/examples/custom-layout/layout/header.tsx

@@ -4,7 +4,7 @@ import { Sparkles, Star } from "lucide-react";
 
 export default function Header() {
   return (
-    <div className="flex items-center justify-between px-4 pt-2">
+    <div className="flex items-center justify-between p-2 px-4">
       <div className="flex items-center gap-2">
         <Sparkles className="size-4" />
         <h1 className="font-semibold">LlamaIndex App</h1>

packages/server/examples/devmode/index.ts

@@ -1,6 +1,12 @@
+import { OpenAI } from "@llamaindex/openai";
 import { LlamaIndexServer } from "@llamaindex/server";
+import { Settings } from "llamaindex";
 import { workflowFactory } from "./src/app/workflow";
 
+Settings.llm = new OpenAI({
+  model: "gpt-4o-mini",
+});
+
 new LlamaIndexServer({
   workflow: workflowFactory,
   uiConfig: {

packages/server/examples/package.json

@@ -7,14 +7,13 @@
     "dev": "nodemon --exec tsx simple-workflow/calculator.ts"
   },
   "dependencies": {
-    "@llamaindex/openai": "^0.2.0",
-    "@llamaindex/readers": "^3.0.0",
+    "@llamaindex/openai": "~0.4.0",
+    "@llamaindex/readers": "~3.1.4",
     "@llamaindex/server": "workspace:*",
-    "@llamaindex/tools": "0.0.4",
-    "@llamaindex/workflow": "1.1.0",
+    "@llamaindex/tools": "~0.0.11",
     "dotenv": "^16.4.7",
-    "llamaindex": "0.10.2",
-    "zod": "^3.23.8"
+    "llamaindex": "~0.11.0",
+    "zod": "^3.24.2"
   },
   "devDependencies": {
     "@types/node": "^20.10.3",

packages/server/examples/private-file/README.md

@@ -0,0 +1,68 @@
+# Upload File Example
+
+This example shows how to use the uploaded file (private file) from the user in the workflow.
+
+## Prerequisites
+
+Please follow the setup instructions in the [examples README](../README.md).
+
+You will also need:
+
+- An OpenAI API key
+- The `enableFileUpload` option in the `uiConfig` is set to `true`.
+
+```typescript
+new LlamaIndexServer({
+  // ... other options
+  uiConfig: { enableFileUpload: true },
+}).start();
+```
+
+## How to get the uploaded files in your workflow:
+
+In LlamaIndexServer, the uploaded file is included in chat message annotations. You can easily get the uploaded files from chat messages using the [extractFileAttachments](https://github.com/llamaindex/llamaindex/blob/main/packages/server/src/utils/events.ts) function.
+
+```typescript
+import { type Message } from "ai";
+import { extractFileAttachments } from "@llamaindex/server";
+
+async function workflowFactory(reqBody: { messages: Message[] }) {
+  const attachments = extractFileAttachments(reqBody.messages);
+  // ...
+}
+```
+
+### AgentWorkflow
+
+If you are using AgentWorkflow, to provide file access to the agent, you can create a tool to read the file content. We recommend to use the `fileId` as the parameter of the tool instead of the `filePath` to avoid showing internal file path to the user. You can use the `getStoredFilePath` helper function to get the file path from the file id.
+
+```typescript
+import { getStoredFilePath, extractFileAttachments } from "@llamaindex/server";
+
+const readFileTool = tool(
+  ({ fileId }) => {
+    // Get the file path from the file id
+    const filePath = getStoredFilePath({ id: fileId });
+    return fsPromises.readFile(filePath, "utf8");
+  },
+  {
+    name: "read_file",
+    description: `Use this tool with the file id to read the file content. The available file are: [${attachments.map((file) => file.id).join(", ")}]`,
+    parameters: z.object({
+      fileId: z.string(),
+    }),
+  },
+);
+```
+
+**Tip:** You can either put the attachments file information to the tool description or agent's system prompt.
+
+Check: [agent-workflow.ts](./agent-workflow.ts) for the full example.
+
+### Custom Workflow
+
+In custom workflow, instead of defining a tool, you can use the helper functions (`extractFileAttachments` and `getStoredFilePath`) to work with file attachments in your workflow.
+
+Check: [custom-workflow.ts](./custom-workflow.ts) for the full example.
+
+> To run custom workflow example, update the `index.ts` file to use the `workflowFactory` from `custom-workflow.ts` instead of `agent-workflow.ts`.

packages/server/examples/private-file/agent-workflow.ts

@@ -0,0 +1,39 @@
+import { extractFileAttachments, getStoredFilePath } from "@llamaindex/server";
+import { agent } from "@llamaindex/workflow";
+import { type Message } from "ai";
+import { tool } from "llamaindex";
+import { promises as fsPromises } from "node:fs";
+import { z } from "zod";
+
+export const workflowFactory = async (reqBody: { messages: Message[] }) => {
+  const { messages } = reqBody;
+  // Extract the files from the messages
+  const files = extractFileAttachments(messages);
+  const fileIds = files.map((file) => file.id);
+
+  // Define a tool to read the file content using the id
+  const readFileTool = tool(
+    ({ fileId }) => {
+      if (!fileIds.includes(fileId)) {
+        throw new Error(`File with id ${fileId} not found`);
+      }
+
+      const filePath = getStoredFilePath({ id: fileId });
+      return fsPromises.readFile(filePath, "utf8");
+    },
+    {
+      name: "read_file",
+      description: `Use this tool with the id of the file to read the file content. Here are the available file ids: [${fileIds.join(", ")}]`,
+      parameters: z.object({
+        fileId: z.string(),
+      }),
+    },
+  );
+  return agent({
+    tools: [readFileTool],
+    systemPrompt: `
+      You are a helpful assistant that can help the user with their file.
+      You can use the read_file tool to read the file content.
+    `,
+  });
+};

packages/server/examples/private-file/custom-workflow.ts

@@ -0,0 +1,98 @@
+import { extractFileAttachments } from "@llamaindex/server";
+import { ChatMemoryBuffer, MessageContent, Settings } from "llamaindex";
+
+import {
+  agentStreamEvent,
+  createStatefulMiddleware,
+  createWorkflow,
+  startAgentEvent,
+  stopAgentEvent,
+  workflowEvent,
+} from "@llamaindex/workflow";
+import { Message } from "ai";
+import { promises as fsPromises } from "node:fs";
+
+const fileHelperEvent = workflowEvent<{
+  userInput: MessageContent;
+  fileContent: string;
+}>();
+
+/**
+ * This is an simple workflow to demonstrate how to use uploaded files in the workflow.
+ */
+export function workflowFactory(reqBody: { messages: Message[] }) {
+  const llm = Settings.llm;
+
+  // First, extract the uploaded file from the messages
+  const attachments = extractFileAttachments(reqBody.messages);
+
+  if (attachments.length === 0) {
+    throw new Error("Please upload a file to start");
+  }
+
+  // Then, add the uploaded file info to the workflow state
+  const { withState, getContext } = createStatefulMiddleware(() => {
+    return {
+      memory: new ChatMemoryBuffer({ llm }),
+      uploadedFile: attachments[attachments.length - 1],
+    };
+  });
+  const workflow = withState(createWorkflow());
+
+  // Handle the start of the workflow: read the file content
+  workflow.handle([startAgentEvent], async ({ data }) => {
+    const { userInput } = data;
+    // Prepare chat history
+    const { state } = getContext();
+    if (!userInput) {
+      throw new Error("Missing user input to start the workflow");
+    }
+    state.memory.put({ role: "user", content: userInput });
+
+    // Read file content
+    const fileContent = await fsPromises.readFile(
+      state.uploadedFile.path,
+      "utf8",
+    );
+
+    return fileHelperEvent.with({
+      userInput,
+      fileContent,
+    });
+  });
+
+  // Use LLM to help the user with the file content
+  workflow.handle([fileHelperEvent], async ({ data }) => {
+    const { sendEvent } = getContext();
+
+    const prompt = `
+You are a helpful assistant that can help the user with their file.
+
+Here is the provided file content:
+${data.fileContent}
+
+Now, let help the user with this request:
+${data.userInput}
+`;
+
+    const response = await llm.complete({
+      prompt,
+      stream: true,
+    });
+
+    // Stream the response
+    for await (const chunk of response) {
+      sendEvent(
+        agentStreamEvent.with({
+          delta: chunk.text,
+          response: chunk.text,
+          currentAgentName: "agent",
+          raw: chunk.raw,
+        }),
+      );
+    }
+    sendEvent(stopAgentEvent.with({ result: "" }));
+  });
+
+  return workflow;
+}

packages/server/examples/private-file/index.ts

@@ -0,0 +1,23 @@
+import { OpenAI, OpenAIEmbedding } from "@llamaindex/openai";
+import { LlamaIndexServer } from "@llamaindex/server";
+import { Settings } from "llamaindex";
+import { workflowFactory } from "./agent-workflow";
+// Uncomment this to use a custom workflow
+// import { workflowFactory } from "./custom-workflow";
+
+Settings.llm = new OpenAI({
+  model: "gpt-4o-mini",
+});
+
+Settings.embedModel = new OpenAIEmbedding({
+  model: "text-embedding-3-small",
+});
+
+new LlamaIndexServer({
+  workflow: workflowFactory,
+  suggestNextQuestions: false,
+  uiConfig: {
+    enableFileUpload: true,
+  },
+  port: 3000,
+}).start();

packages/server/examples/simple-workflow/calculator.ts

@@ -1,8 +1,13 @@
+import { OpenAI } from "@llamaindex/openai";
 import { LlamaIndexServer } from "@llamaindex/server";
 import { agent } from "@llamaindex/workflow";
-import { tool } from "llamaindex";
+import { Settings, tool } from "llamaindex";
 import { z } from "zod";
 
+Settings.llm = new OpenAI({
+  model: "gpt-4o-mini",
+});
+
 const calculatorAgent = agent({
   tools: [
     tool({

packages/server/next/app/api/files/helpers.ts

@@ -0,0 +1,57 @@
+import crypto from "node:crypto";
+import fs from "node:fs";
+import path from "node:path";
+
+import { type ServerFile } from "@llamaindex/server";
+
+export const UPLOADED_FOLDER = "output/uploaded";
+
+export async function storeFile(
+  name: string,
+  fileBuffer: Buffer,
+): Promise<ServerFile> {
+  const parts = name.split(".");
+  const fileName = parts[0];
+  const fileExt = parts[1];
+  if (!fileName) {
+    throw new Error("File name is required");
+  }
+  if (!fileExt) {
+    throw new Error("File extension is required");
+  }
+
+  const id = crypto.randomUUID();
+  const fileId = `${sanitizeFileName(fileName)}_${id}.${fileExt}`;
+  const filepath = path.join(UPLOADED_FOLDER, fileId);
+  const fileUrl = await saveFile(filepath, fileBuffer);
+  return {
+    id: fileId,
+    size: fileBuffer.length,
+    type: fileExt,
+    url: fileUrl,
+    path: filepath,
+  };
+}
+
+// Save document to file server and return the file url
+async function saveFile(filepath: string, content: string | Buffer) {
+  if (path.isAbsolute(filepath)) {
+    throw new Error("Absolute file paths are not allowed.");
+  }
+
+  const dirPath = path.dirname(filepath);
+  await fs.promises.mkdir(dirPath, { recursive: true });
+
+  if (typeof content === "string") {
+    await fs.promises.writeFile(filepath, content, "utf-8");
+  } else {
+    await fs.promises.writeFile(filepath, content);
+  }
+
+  const fileurl = `/api/files/${filepath}`;
+  return fileurl;
+}
+
+function sanitizeFileName(fileName: string) {
+  return fileName.replace(/[^a-zA-Z0-9_-]/g, "_");
+}

packages/server/next/app/api/files/route.ts

@@ -0,0 +1,49 @@
+import { type FileAnnotation } from "@llamaindex/server";
+import { NextRequest, NextResponse } from "next/server";
+import { storeFile } from "./helpers";
+
+export async function POST(request: NextRequest) {
+  try {
+    const {
+      name,
+      base64,
+    }: {
+      name: string;
+      base64: string;
+    } = await request.json();
+    if (!base64 || !name) {
+      return NextResponse.json(
+        { error: "base64 and name is required in the request body" },
+        { status: 400 },
+      );
+    }
+
+    const parts = base64.split(",");
+    if (parts.length !== 2) {
+      return NextResponse.json(
+        { error: "Invalid base64 format" },
+        { status: 400 },
+      );
+    }
+
+    const [header, content] = parts;
+    if (!header || !content) {
+      return NextResponse.json(
+        { error: "Invalid base64 format" },
+        { status: 400 },
+      );
+    }
+
+    const fileBuffer = Buffer.from(content, "base64");
+
+    const file = await storeFile(name, fileBuffer);
+
+    return NextResponse.json(file as FileAnnotation);
+  } catch (error) {
+    console.error("[Upload API]", error);
+    return NextResponse.json(
+      { error: (error as Error).message },
+      { status: 500 },
+    );
+  }
+}

packages/server/next/app/components/ui/chat/chat-message-content.tsx

@@ -19,7 +19,6 @@ export function ChatMessageContent({
       <ToolAnnotations />
       <ChatMessage.Content.Image />
       <DynamicEvents componentDefs={componentDefs} appendError={appendError} />
-      <ChatMessage.Content.Artifact />
       <ChatMessage.Content.Markdown />
       <ChatMessage.Content.DocumentFile />
       <ChatMessage.Content.Source />

packages/server/next/app/components/ui/chat/chat-messages.tsx

@@ -32,7 +32,10 @@ export default function CustomChatMessages({
             <ChatMessage.Actions />
           </ChatMessage>
         ))}
-        <ChatMessages.Empty />
+        <ChatMessages.Empty
+          heading="Hello there!"
+          subheading="I'm here to help you with your questions."
+        />
         <ChatMessages.Loading />
       </ChatMessages.List>
       <ChatStarter />

packages/server/next/app/components/ui/chat/custom/events/dynamic-events.tsx

@@ -1,7 +1,7 @@
 "use client";
 
 import {
-  getChatUIAnnotation,
+  getAnnotationData,
   JSONValue,
   MessageAnnotation,
   MessageAnnotationType,
@@ -25,9 +25,8 @@ export const DynamicEvents = ({
   componentDefs: ComponentDef[];
   appendError: (error: string) => void;
 }) => {
-  const {
-    message: { annotations },
-  } = useChatMessage();
+  const { message } = useChatMessage();
+  const annotations = message.annotations;
 
   const shownWarningsRef = useRef<Set<string>>(new Set()); // track warnings
   const [hasErrors, setHasErrors] = useState(false);
@@ -43,15 +42,16 @@ export const DynamicEvents = ({
 
     const availableComponents = new Set(componentDefs.map((comp) => comp.type));
 
-    annotations.forEach((annotation: MessageAnnotation) => {
+    annotations.forEach((item: JSONValue) => {
+      const annotation = item as MessageAnnotation;
       const type = annotation.type;
-      if (!type) return; // skip if annotation doesn't have a type
+      if (!type) return; // Skip if annotation doesn't have a type
 
-      const events = getChatUIAnnotation(annotations, type);
+      const events = getAnnotationData<JSONValue>(message, type);
 
       // Skip if it's a built-in component or if we've already shown the warning
       if (
-        BUILT_IN_CHATUI_COMPONENTS.includes(type) ||
+        BUILT_IN_CHATUI_COMPONENTS.includes(type as MessageAnnotationType) ||
         shownWarningsRef.current.has(type)
       ) {
         return;
@@ -69,7 +69,7 @@ export const DynamicEvents = ({
 
   const components: EventComponent[] = componentDefs
     .map((comp) => {
-      const events = getChatUIAnnotation(annotations, comp.type) as JSONValue[]; // get all event data by type
+      const events = getAnnotationData<JSONValue>(message, comp.type);
       if (!events?.length) return null;
       return { ...comp, events };
     })

packages/server/next/app/components/ui/chat/custom/events/import.ts

@@ -67,6 +67,9 @@ export const SOURCE_MAP: Record<string, () => Promise<any>> = {
     import("../../../toggle-group"),
   [`${SHADCN_IMPORT_PREFIX}/tooltip`]: () => import("../../../tooltip"),
 
+  ///// CHAT_UI GENERAL  /////
+  [`@llamaindex/chat-ui`]: () => import("@llamaindex/chat-ui"),
+
   ///// WIDGETS FROM CHAT_UI /////
   [`@llamaindex/chat-ui/widgets`]: () => import("@llamaindex/chat-ui/widgets"),
 
@@ -76,6 +79,9 @@ export const SOURCE_MAP: Record<string, () => Promise<any>> = {
   ///// UTILS /////
   [`@/components/lib/utils`]: () => import("../../../lib/utils"),
   [`@/lib/utils`]: () => import("../../../lib/utils"), // for v0 compatibility
+
+  ///// ZOD /////
+  [`zod`]: () => import("zod"),
 };
 
 // parse imports from code to get Function constructor arguments and component name
@@ -122,7 +128,7 @@ export async function parseImports(code: string) {
   const importPromises = imports.map(async ({ name, source }) => {
     if (!(source in SOURCE_MAP)) {
       throw new Error(
-        `Fail to import ${name} from ${source}. Reason: Module not found. \nCurrently we only support importing UI components from Shadcn components, widgets from "llamaindex/chat-ui/widgets" and icons from "lucide-react"`,
+        `Fail to import ${name} from ${source}. Reason: Module not found. \nCurrently we only support importing UI components from Shadcn components, widgets and hooks from "llamaindex/chat-ui", icons from "lucide-react" and zod for data validation.`,
       );
     }
     try {

packages/server/next/app/components/ui/chat/custom/markdown.tsx

@@ -1,7 +1,9 @@
 "use client";
 
-import { SourceData } from "@llamaindex/chat-ui";
-import { Markdown as MarkdownUI } from "@llamaindex/chat-ui/widgets";
+import {
+  Markdown as MarkdownUI,
+  SourceData,
+} from "@llamaindex/chat-ui/widgets";
 import { getConfig } from "../../lib/utils";
 const preprocessMedia = (content: string) => {
   // Remove `sandbox:` from the beginning of the URL before rendering markdown

packages/server/next/app/components/ui/chat/layout/header.tsx

@@ -4,7 +4,7 @@ import { Sparkles, Star } from "lucide-react";
 
 export function DefaultHeader() {
   return (
-    <div className="flex items-center justify-between px-4 pt-2">
+    <div className="flex items-center justify-between p-2 px-4">
       <div className="flex items-center gap-2">
         <Sparkles className="size-4" />
         <h1 className="font-semibold">LlamaIndex App</h1>

packages/server/next/app/components/ui/chat/tools/chat-tools.tsx

@@ -2,8 +2,7 @@
 
 import {
   Message,
-  MessageAnnotation,
-  getChatUIAnnotation,
+  getAnnotationData,
   useChatMessage,
   useChatUI,
 } from "@llamaindex/chat-ui";
@@ -21,13 +20,10 @@ export function ToolAnnotations() {
     [messages, message],
   );
   // Get the tool data from the message annotations
-  const annotations = message.annotations as MessageAnnotation[] | undefined;
-  const toolData = annotations
-    ? (getChatUIAnnotation(annotations, "tools") as unknown as ToolData[])
-    : null;
-  return toolData?.[0] ? (
-    <ChatTools data={toolData[0]} artifactVersion={artifactVersion} />
-  ) : null;
+  const toolData = getAnnotationData<ToolData>(message, "tools");
+  if (toolData.length === 0) return null;
+
+  return <ChatTools data={toolData[0]} artifactVersion={artifactVersion} />;
 }
 
 // TODO: Used to render outputs of tools. If needed, add more renderers here.
@@ -83,9 +79,7 @@ function getArtifactVersion(
   if (!messageId) return undefined;
   let versionIndex = 1;
   for (const m of messages) {
-    const toolData = m.annotations
-      ? (getChatUIAnnotation(m.annotations, "tools") as unknown as ToolData[])
-      : null;
+    const toolData = getAnnotationData<ToolData>(m, "tools");
 
     if (toolData?.some((t) => t.toolCall.name === "artifact")) {
       if ("id" in m && m.id === messageId) {

packages/server/next/app/globals.css

@@ -91,6 +91,13 @@
   ::file-selector-button {
     border-color: var(--color-gray-200, currentColor);
   }
+
+  /* Tailwind v4 removed cursor pointer of button and use default cursor */
+  /* https://github.com/shadcn-ui/ui/issues/6843#issuecomment-2696947980 */
+  button:not([disabled]),
+  [role="button"]:not([disabled]) {
+    cursor: pointer;
+  }
 }
 
 @layer base {

packages/server/next/app/layout.tsx

@@ -1,6 +1,7 @@
 import type { Metadata } from "next";
 import { Inter } from "next/font/google";
 
+import "@llamaindex/chat-ui/styles/editor.css";
 import "@llamaindex/chat-ui/styles/markdown.css";
 import "@llamaindex/chat-ui/styles/pdf.css";
 import "./globals.css";

packages/server/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/server",
   "description": "LlamaIndex Server",
-  "version": "0.2.4",
+  "version": "0.2.7",
   "type": "module",
   "main": "./dist/index.cjs",
   "module": "./dist/index.js",
@@ -65,7 +65,7 @@
     "@babel/traverse": "^7.27.0",
     "@babel/types": "^7.27.0",
     "@hookform/resolvers": "^5.0.1",
-    "@llamaindex/chat-ui": "0.4.6",
+    "@llamaindex/chat-ui": "0.5.6",
     "@radix-ui/react-accordion": "^1.2.3",
     "@radix-ui/react-alert-dialog": "^1.1.7",
     "@radix-ui/react-aspect-ratio": "^1.1.3",

packages/server/project-config/eslint.config.mjs

@@ -18,6 +18,7 @@ const eslintConfig = [
       "react-hooks/exhaustive-deps": "off",
       "@next/next/no-img-element": "off",
       "@next/next/no-assign-module-variable": "off",
+      "@typescript-eslint/no-empty-object-type": "off",
     },
   },
   {

packages/server/project-config/package.json

@@ -12,7 +12,7 @@
     "format": "prettier --ignore-unknown --cache --check .",
     "format:write": "prettier --ignore-unknown --write .",
     "typecheck": "tsc --noEmit",
-    "generate": "tsx app\\api\\chat\\generate.ts"
+    "generate": "tsx app/api/chat/generate.ts"
   },
   "devDependencies": {
     "@eslint/eslintrc": "^3",
@@ -41,7 +41,7 @@
     "@babel/traverse": "^7.27.0",
     "@babel/types": "^7.27.0",
     "@hookform/resolvers": "^5.0.1",
-    "@llamaindex/chat-ui": "0.4.5",
+    "@llamaindex/chat-ui": "0.5.6",
     "@llamaindex/env": "~0.1.30",
     "@llamaindex/openai": "~0.4.0",
     "@llamaindex/readers": "~3.1.4",

packages/server/src/index.ts

@@ -1,5 +1,7 @@
 export * from "./server";
 export * from "./types";
 export * from "./utils/events";
+export { getStoredFilePath } from "./utils/file";
 export { generateEventComponent } from "./utils/gen-ui";
+export * from "./utils/inline";
 export * from "./utils/prompts";

packages/server/src/server.ts

@@ -47,7 +47,7 @@ export class LlamaIndexServer {
     const componentsApi = this.componentsDir ? "/api/components" : undefined;
     const layoutApi = this.layoutDir ? "/api/layout" : undefined;
     const devMode = uiConfig?.devMode ?? false;
-
+    const enableFileUpload = uiConfig?.enableFileUpload ?? false;
     // content in javascript format
     const content = `
       window.LLAMAINDEX = {
@@ -57,7 +57,8 @@ export class LlamaIndexServer {
         COMPONENTS_API: ${JSON.stringify(componentsApi)},
         LAYOUT_API: ${JSON.stringify(layoutApi)},
         DEV_MODE: ${JSON.stringify(devMode)},
-        SUGGEST_NEXT_QUESTIONS: ${JSON.stringify(this.suggestNextQuestions)}
+        SUGGEST_NEXT_QUESTIONS: ${JSON.stringify(this.suggestNextQuestions)},
+        UPLOAD_API: ${JSON.stringify(enableFileUpload ? "/api/files" : undefined)}
       }
     `;
     fs.writeFileSync(configFile, content);

packages/server/src/types.ts

@@ -18,6 +18,7 @@ export type UIConfig = {
   layoutDir?: string;
   llamaCloudIndexSelector?: boolean;
   devMode?: boolean;
+  enableFileUpload?: boolean;
 };
 
 export type LlamaIndexServerOptions = NextAppOptions & {

packages/server/src/utils/events.ts

@@ -1,8 +1,15 @@
 import { randomUUID } from "@llamaindex/env";
 import { workflowEvent } from "@llamaindex/workflow";
 import type { Message } from "ai";
-import { MetadataMode, type Metadata, type NodeWithScore } from "llamaindex";
+import {
+  MetadataMode,
+  type ChatMessage,
+  type Metadata,
+  type NodeWithScore,
+} from "llamaindex";
 import { z } from "zod";
+import { getStoredFilePath } from "./file";
+import { getInlineAnnotations } from "./inline";
 
 // Events that appended to stream as annotations
 export type SourceEventNode = {
@@ -103,6 +110,7 @@ export type DocumentArtifactData = {
   title: string;
   content: string;
   type: string; // markdown, html,...
+  sources?: { id: string }[]; // sources that are used to render citation numbers in the document
 };
 
 export type CodeArtifact = Artifact<CodeArtifactData> & {
@@ -148,24 +156,22 @@ export const artifactAnnotationSchema = z.object({
   data: artifactSchema,
 });
 
-export function extractAllArtifacts(messages: Message[]): Artifact[] {
-  const allArtifacts: Artifact[] = [];
+export function extractArtifactsFromMessage(message: ChatMessage): Artifact[] {
+  const inlineAnnotations = getInlineAnnotations(message);
+  const artifacts = inlineAnnotations.filter(
+    (annotation): annotation is z.infer<typeof artifactAnnotationSchema> => {
+      return artifactAnnotationSchema.safeParse(annotation).success;
+    },
+  );
+  return artifacts.map((artifact) => artifact.data);
+}
 
-  for (const message of messages) {
-    const artifacts =
-      message.annotations
-        ?.filter(
-          (
-            annotation,
-          ): annotation is z.infer<typeof artifactAnnotationSchema> =>
-            artifactAnnotationSchema.safeParse(annotation).success,
-        )
-        .map((annotation) => annotation.data as Artifact) ?? [];
-
-    allArtifacts.push(...artifacts);
-  }
-
-  return allArtifacts;
+export function extractArtifactsFromAllMessages(
+  messages: ChatMessage[],
+): Artifact[] {
+  return messages
+    .flatMap((message) => extractArtifactsFromMessage(message))
+    .sort((a, b) => a.created_at - b.created_at);
 }
 
 export function extractLastArtifact(
@@ -187,10 +193,10 @@ export function extractLastArtifact(
   requestBody: unknown,
   type?: ArtifactType,
 ): CodeArtifact | DocumentArtifact | Artifact | undefined {
-  const { messages } = (requestBody as { messages?: Message[] }) ?? {};
+  const { messages } = (requestBody as { messages?: ChatMessage[] }) ?? {};
   if (!messages) return undefined;
 
-  const artifacts = extractAllArtifacts(messages);
+  const artifacts = extractArtifactsFromAllMessages(messages);
   if (!artifacts.length) return undefined;
 
   if (type) {
@@ -211,3 +217,64 @@ export function extractLastArtifact(
 
   return artifacts[artifacts.length - 1];
 }
+
+export const fileAnnotationSchema = z.object({
+  id: z.string(),
+  size: z.number(),
+  type: z.string(),
+  url: z.string(),
+});
+
+export const documentFileAnnotationSchema = z.object({
+  type: z.literal("document_file"),
+  data: z.object({
+    files: z.array(fileAnnotationSchema),
+  }),
+});
+type DocumentFileAnnotation = z.infer<typeof documentFileAnnotationSchema>;
+
+export type FileAnnotation = z.infer<typeof fileAnnotationSchema>;
+
+export type ServerFile = FileAnnotation & {
+  path: string;
+};
+
+/**
+ * Extract file attachments from an user message.
+ * @param message - The message to extract file attachments from.
+ * @returns The file attachments.
+ */
+export function extractFileAttachmentsFromMessage(
+  message: Message,
+): ServerFile[] {
+  const fileAttachments: ServerFile[] = [];
+  if (message.role === "user" && message.annotations) {
+    for (const annotation of message.annotations) {
+      if (documentFileAnnotationSchema.safeParse(annotation).success) {
+        const { data } = annotation as DocumentFileAnnotation;
+        for (const file of data.files) {
+          fileAttachments.push({
+            ...file,
+            path: getStoredFilePath({ id: file.id }),
+          });
+        }
+      }
+    }
+  }
+  return fileAttachments;
+}
+
+/**
+ * Extract file attachments from all user messages.
+ * @param messages - The messages to extract file attachments from.
+ * @returns The file attachments.
+ */
+export function extractFileAttachments(messages: Message[]): ServerFile[] {
+  const fileAttachments: ServerFile[] = [];
+
+  for (const message of messages) {
+    fileAttachments.push(...extractFileAttachmentsFromMessage(message));
+  }
+
+  return fileAttachments;
+}

packages/server/src/utils/file.ts

@@ -1,5 +1,6 @@
 import fs from "node:fs";
 import https from "node:https";
+import path from "node:path";
 
 export async function downloadFile(
   urlToDownload: string,
@@ -29,3 +30,41 @@ export async function downloadFile(
     throw new Error(`Error downloading file: ${error}`);
   }
 }
+
+/**
+ * Returns the full path to a stored file given its id and optional save directory.
+ * If saveDir is not provided, defaults to "output/uploaded".
+ *
+ * @param {Object} params - The parameters object.
+ * @param {string} params.id - The file identifier.
+ * @param {string} [params.saveDir] - Optional directory to save the file.
+ * @returns {string} The full file path.
+ */
+
+/**
+ * Constructs a stored file path from an ID and optional directory.
+ * Uses path.join for cross-platform safety and validates the ID to prevent path traversal.
+ *
+ * @param {Object} params - The parameters object.
+ * @param {string} params.id - The file identifier (must not contain path separators).
+ * @param {string} [params.saveDir] - Optional directory to save the file. Defaults to "output/uploaded".
+ * @returns {string} The full file path.
+ * @throws {Error} If the id contains invalid path characters.
+ */
+export function getStoredFilePath({
+  id,
+  saveDir,
+}: {
+  id: string;
+  saveDir?: string;
+}): string {
+  // Validate id to prevent path traversal and invalid characters
+  if (id.includes("/") || id.includes("\\") || id.includes("..")) {
+    throw new Error(
+      "Invalid file id: path traversal or separators are not allowed.",
+    );
+  }
+  // Use path.join to construct the default directory for cross-platform compatibility
+  const directory = saveDir ?? path.join("output", "uploaded");
+  return path.join(directory, id);
+}

packages/server/src/utils/index.ts

@@ -1,6 +1,7 @@
 export * from "./events";
 export * from "./file";
 export * from "./gen-ui";
+export * from "./inline";
 export * from "./prompts";
 export * from "./request";
 export * from "./stream";

packages/server/src/utils/inline.ts

@@ -0,0 +1,90 @@
+import { agentStreamEvent, type WorkflowEventData } from "@llamaindex/workflow";
+import { type ChatMessage } from "llamaindex";
+import { z } from "zod";
+
+const INLINE_ANNOTATION_KEY = "annotation"; // the language key to detect inline annotation code in markdown
+
+export const AnnotationSchema = z.object({
+  type: z.string(),
+  data: z.any(),
+});
+
+export type Annotation = z.infer<typeof AnnotationSchema>;
+
+export function getInlineAnnotations(message: ChatMessage): Annotation[] {
+  const markdownContent = getMessageMarkdownContent(message);
+
+  const inlineAnnotations: Annotation[] = [];
+
+  // Regex to match annotation code blocks
+  // Matches ```annotation followed by content until closing ```
+  const annotationRegex = new RegExp(
+    `\`\`\`${INLINE_ANNOTATION_KEY}\\s*\\n([\\s\\S]*?)\\n\`\`\``,
+    "g",
+  );
+
+  let match;
+  while ((match = annotationRegex.exec(markdownContent)) !== null) {
+    const jsonContent = match[1]?.trim();
+
+    if (!jsonContent) {
+      continue;
+    }
+
+    try {
+      // Parse the JSON content
+      const parsed = JSON.parse(jsonContent);
+
+      // Validate against the annotation schema
+      const validated = AnnotationSchema.parse(parsed);
+
+      // Extract the artifact data
+      inlineAnnotations.push(validated);
+    } catch (error) {
+      // Skip invalid annotations - they might be malformed JSON or invalid schema
+      console.warn("Failed to parse annotation:", error);
+    }
+  }
+
+  return inlineAnnotations;
+}
+
+/**
+ * To append inline annotations to the stream, we need to wrap the annotation in a code block with the language key.
+ * The language key is `annotation` and the code block is wrapped in backticks.
+ *
+ * \`\`\`annotation
+ * \{
+ *   "type": "artifact",
+ *   "data": \{...\}
+ * \}
+ * \`\`\`
+ */
+export function toInlineAnnotation(item: unknown) {
+  return `\n\`\`\`${INLINE_ANNOTATION_KEY}\n${JSON.stringify(item)}\n\`\`\`\n`;
+}
+
+export function toInlineAnnotationEvent(event: WorkflowEventData<unknown>) {
+  return agentStreamEvent.with({
+    delta: toInlineAnnotation(event.data),
+    response: "",
+    currentAgentName: "assistant",
+    raw: event.data,
+  });
+}
+
+function getMessageMarkdownContent(message: ChatMessage): string {
+  let markdownContent = "";
+
+  if (typeof message.content === "string") {
+    markdownContent = message.content;
+  } else {
+    message.content.forEach((item) => {
+      if (item.type === "text") {
+        markdownContent += item.text;
+      }
+    });
+  }
+
+  return markdownContent;
+}

packages/server/src/utils/workflow.ts

@@ -15,12 +15,14 @@ import {
   type NodeWithScore,
 } from "llamaindex";
 import {
+  artifactEvent,
   sourceEvent,
   toAgentRunEvent,
   toSourceEvent,
   type SourceEventNode,
 } from "./events";
 import { downloadFile } from "./file";
+import { toInlineAnnotationEvent } from "./inline";
 
 export async function runWorkflow(
   workflow: Workflow,
@@ -74,6 +76,10 @@ function processWorkflowStream(
               transformedEvent = toSourceEvent(sourceNodes);
             }
           }
+          // Handle artifact events, transform to agentStreamEvent
+          else if (artifactEvent.include(event)) {
+            transformedEvent = toInlineAnnotationEvent(event);
+          }
           // Post-process for llama-cloud files
           if (sourceEvent.include(transformedEvent)) {
             const sourceNodesForDownload = transformedEvent.data.data.nodes; // These are SourceEventNode[]

python/llama-index-server/CHANGELOG.md

@@ -1,5 +1,35 @@
 # @create-llama/llama-index-server
 
+## 0.1.21
+
+### Patch Changes
+
+- 1ff6eaf: Add support for upload file
+- af9ad3c: feat: show document artifact after generating report
+- a543a27: feat: bump chat-ui with inline artifact
+- Updated dependencies [af9ad3c]
+- Updated dependencies [a543a27]
+- Updated dependencies [1ff6eaf]
+  - @llamaindex/server@0.2.7
+
+## 0.1.20
+
+### Patch Changes
+
+- 087c961: Add support for human-in-the-loop
+- 087c961: Refactor models.py into a separate module
+- Updated dependencies [3ff0a18]
+- Updated dependencies [df10474]
+- Updated dependencies [087c961]
+  - @llamaindex/server@0.2.6
+
+## 0.1.19
+
+### Patch Changes
+
+- Updated dependencies [058b376]
+  - @llamaindex/server@0.2.5
+
 ## 0.1.18
 
 ### Patch Changes

python/llama-index-server/README.md

@@ -8,6 +8,7 @@ LlamaIndexServer is a FastAPI-based application that allows you to quickly launc
 - Built on FastAPI for high performance and easy API development
 - Optional built-in chat UI with extendable UI components
 - Prebuilt development code
+- Human-in-the-loop (HITL) support, check out the [Human-in-the-loop](https://github.com/run-llama/create-llama/blob/main/python/llama-index-server/examples/hitl/README.md) documentation for more details.
 
 ## Installation
 
@@ -77,6 +78,7 @@ The LlamaIndexServer accepts the following configuration parameters:
 - `env`: Environment setting ('dev' enables CORS and UI by default)
 - `ui_config`: UI configuration as a dictionary or UIConfig object with options:
   - `enabled`: Whether to enable the chat UI (default: True)
+  - `enable_file_upload`: Whether to enable file upload in the chat UI (default: False). Check [How to get the uploaded files in your workflow](https://github.com/run-llama/create-llama/blob/main/python/llama-index-server/examples/private_file/README.md#how-to-get-the-uploaded-files-in-your-workflow) for more details.
   - `starter_questions`: List of starter questions for the chat UI (default: None)
   - `ui_path`: Path for downloaded UI static files (default: ".ui")
   - `component_dir`: The directory for custom UI components rendering events emitted by the workflow. The default is None, which does not render custom UI components.
@@ -160,6 +162,7 @@ app = LlamaIndexServer(
 The server provides the following default endpoints:
 
 - `/api/chat`: Chat interaction endpoint
+- `/api/chat/file`: File upload endpoint (only available when `enable_file_upload` in `ui_config` is True)
 - `/api/files/data/*`: Access to data directory files
 - `/api/files/output/*`: Access to output directory files
 

python/llama-index-server/examples/README.md

@@ -0,0 +1,43 @@
+# Examples for llama-index-server
+
+This directory contains examples for llama-index-server.
+
+## How to run the examples
+
+1. Make sure you have [uv](https://docs.astral.sh/uv/) installed.
+
+2. Install the dependencies (with published packages) by running the following command:
+
+    ```bash
+    uv sync
+    ```
+
+3. Navigate to one of the example folders and follow the instructions in the example's README.md file:
+
+- [Simple Agent](./simple-agent/README.md)
+- [HITL](./hitl/README.md)
+- [Artifact](./artifact/README.md)
+- [LlamaCloud](./llamacloud/README.md)
+
+## Local Development
+
+1. For local development, you first need to build the UI resources for the server. At the root of the project, run the following command:
+
+    ```bash
+    pnpm install
+    pnpm build
+    ```
+
+2. Config to use the local llama-index-server package:
+
+    To run the examples with the local llama-index-server package, you need to tell uv to use the virtual environment of the root project
+    by setting the `UV_PROJECT` environment variable.
+
+    ```bash
+    export UV_PROJECT=<absolute path of the root project>
+    ```
+
+    Then continue with step 3 above.
+
+    > You can also use `--project <path to the root project>` instead of setting the `UV_PROJECT` environment variable.
+

python/llama-index-server/examples/artifact/README.md

@@ -4,7 +4,9 @@ This guide explains how to set up and use the LlamaIndex server with the artifac
 
 ## Prerequisites
 
-- [uv](https://github.com/astral-sh/uv) installed (a fast Python package manager and runner)
+Please follow the setup instructions in the [examples README](../README.md).
+
+You will also need:
 - An OpenAI API key
 
 ## Steps

python/llama-index-server/examples/artifact/code_workflow.py

@@ -16,7 +16,8 @@ from llama_index.core.workflow import (
     Workflow,
     step,
 )
-from llama_index.server.api.models import (
+from llama_index.server.api.utils import get_last_artifact
+from llama_index.server.models import (
     Artifact,
     ArtifactEvent,
     ArtifactType,
@@ -24,7 +25,6 @@ from llama_index.server.api.models import (
     CodeArtifactData,
     UIEvent,
 )
-from llama_index.server.api.utils import get_last_artifact
 
 
 class Requirement(BaseModel):

python/llama-index-server/examples/artifact/document_workflow.py

@@ -16,7 +16,8 @@ from llama_index.core.workflow import (
     Workflow,
     step,
 )
-from llama_index.server.api.models import (
+from llama_index.server.api.utils import get_last_artifact
+from llama_index.server.models import (
     Artifact,
     ArtifactEvent,
     ArtifactType,
@@ -24,7 +25,6 @@ from llama_index.server.api.models import (
     DocumentArtifactData,
     UIEvent,
 )
-from llama_index.server.api.utils import get_last_artifact
 
 
 class DocumentRequirement(BaseModel):

python/llama-index-server/examples/artifact/layout/header.tsx

@@ -4,7 +4,7 @@ import { Sparkles, Star } from "lucide-react";
 
 export default function Header() {
   return (
-    <div className="flex items-center justify-between px-4 pt-2">
+    <div className="flex items-center justify-between p-2 px-4">
       <div className="flex items-center gap-2">
         <Sparkles className="size-4" />
         <h1 className="font-semibold">Artifact Workflow</h1>

python/llama-index-server/examples/artifact/main.py

@@ -1,13 +1,12 @@
+from code_workflow import ArtifactWorkflow
 from fastapi import FastAPI
 
-from examples.artifact.code_workflow import ArtifactWorkflow
-
 # To use document artifact workflow, uncomment the following line
-# from examples.artifact.document_workflow import ArtifactWorkflow
+# from document_workflow import ArtifactWorkflow
 from llama_index.core.workflow import Workflow
 from llama_index.llms.openai import OpenAI
 from llama_index.server import LlamaIndexServer, UIConfig
-from llama_index.server.api.models import ChatRequest
+from llama_index.server.models import ChatRequest
 
 
 def create_workflow(chat_request: ChatRequest) -> Workflow:

python/llama-index-server/examples/hitl/README.md

@@ -0,0 +1,121 @@
+# Human in the Loop
+
+This example shows how to use the LlamaIndexServer with a human in the loop. It allows you to start CLI commands that are reviewed by a human before execution.
+
+## Prerequisites
+
+Please follow the setup instructions in the [examples README](../README.md).
+
+## Getting Started
+
+### AgentWorkflow
+
+Using AgentWorkflow, you need to run the following command:
+
+```bash
+uv run -- agent_workflow.py
+```
+
+### Custom Workflow
+
+```bash
+uv run -- custom_workflow.py
+```
+
+### Access the Application
+
+Open your browser and go to:
+
+```
+http://localhost:8000
+```
+
+You will see the LlamaIndexServer UI, where you can interact with the HITL agent. Try "List all files in the current directory" and see how the agent pauses and waits for a human response before executing the command.
+
+## How does HITL it work?
+
+### Events
+
+The human-in-the-loop approach used here is based on a simple idea: the workflow pauses and waits for a human response before proceeding to the next step.
+
+To do this, you will need to implement two custom events: 
++ [HumanInputEvent](../../llama_index/server/models/hitl.py#L21): This event is used to request input from the user.
++ [HumanResponseEvent](../../llama_index/server/models/hitl.py#L10): This event is sent to the workflow to resume execution with input from the user.
+
+In this example, we have implemented these two custom events:  
+
+- [CLIHumanInputEvent](events.py#L20) – to request input from the user for CLI command execution.
+- [CLIHumanResponseEvent](events.py#L8) – to resume the workflow with the response from the user.
+
+### UI Component
+
+HITL also needs a custom UI component, that is shown when the LlamaIndexServer receives the `CLIHumanInputEvent`. The name of the component is defined in the `event_type` field of the `CLIHumanInputEvent` - in our case, it is `cli_human_input`, which corresponds to the [cli_human_input.tsx](./components/cli_human_input.tsx) component.
+
+The custom component must use `append` to send a message with a `human_response` annotation. The data of the annotation must be in the format of the response event `CLIHumanResponseEvent`, in our case, for sending to execute the command `ls -l`, we would send:
+
+```tsx
+append({
+    content: "Yes",
+    role: "user",
+    annotations: [
+    {
+        type: "human_response",
+        data: {
+            execute: true,
+            command: "ls -l" // The command to execute
+        },
+    },
+    ],
+});
+```
+
+This component displays the command to execute and the user can choose to execute or cancel the command execution.
+
+### AgentWorkflow
+
+To make the [AgentWorkflow](agent_workflow.py) work, we use the `wait_for_event()` method to wait for the human response when a tool is called.
+
+Example:
+```python
+async def cli_executor(ctx: Context, command: str) -> str:
+    """
+    This tool carefully waits for user confirmation before executing a command.
+    """
+    confirmation = await ctx.wait_for_event(
+        CLIHumanResponseEvent,
+        waiter_event=CLIHumanInputEvent(
+            data=CLICommand(command=command),
+        ),
+    )
+    if confirmation.execute:
+        # Execute the command
+        ...
+    else:
+        # Cancel the command
+        ...
+
+```
+
+### LlamaIndex Workflows
+
+And for [Custom Workflow](custom_workflow.py), we can define a step that send the `CLIHumanInputEvent` and another step that wait for the `CLIHumanResponseEvent`.
+
+Example:
+```python
+@step
+async def request_input(self, ctx: Context, ev: StartEvent) -> CLIHumanInputEvent:
+    ...
+    return CLIHumanInputEvent(
+        data=CLICommand(command=command),
+        response_event_type=CLIHumanResponseEvent,
+    )
+
+@step
+async def handle_human_response(self, ctx: Context, ev: CLIHumanResponseEvent) -> StopEvent:
+    if ev.execute:
+        # Execute the command
+        ...
+    else:
+        # Cancel the command
+        ...
+```

python/llama-index-server/examples/hitl/agent_workflow.py

@@ -0,0 +1,60 @@
+import subprocess
+
+from events import CLICommand, CLIHumanInputEvent, CLIHumanResponseEvent
+from fastapi import FastAPI
+
+from llama_index.core.agent.workflow import AgentWorkflow
+from llama_index.core.workflow import Context
+from llama_index.llms.openai import OpenAI
+from llama_index.server import LlamaIndexServer, UIConfig
+
+
+async def cli_executor(ctx: Context, command: str) -> str:
+    """
+    This tool carefully waits for user confirmation before executing a command.
+    """
+    confirmation = await ctx.wait_for_event(
+        CLIHumanResponseEvent,
+        waiter_event=CLIHumanInputEvent(
+            data=CLICommand(command=command),
+        ),
+    )
+    if confirmation.execute:
+        return subprocess.check_output(confirmation.command, shell=True).decode("utf-8")
+    else:
+        return "Command execution cancelled."
+
+
+def create_workflow() -> AgentWorkflow:
+    return AgentWorkflow.from_tools_or_functions(
+        tools_or_functions=[cli_executor],
+        llm=OpenAI(model="gpt-4.1-mini"),
+        system_prompt="""
+        You are a helpful assistant that help the user execute commands.
+        You can execute commands using the cli_executor tool, don't need to ask for confirmation for triggering the tool.
+        """,
+    )
+
+
+def create_app() -> FastAPI:
+    app = LlamaIndexServer(
+        workflow_factory=create_workflow,
+        suggest_next_questions=False,
+        ui_config=UIConfig(
+            starter_questions=[
+                "List all files in the current directory",
+                "Fetch changes from the remote repository",
+            ],
+            component_dir="components",
+        ),
+    )
+    return app
+
+
+app = create_app()
+
+
+if __name__ == "__main__":
+    import uvicorn
+
+    uvicorn.run("agent_workflow:app", port=8000, reload=True)

python/llama-index-server/examples/hitl/components/cli_human_input.tsx

@@ -0,0 +1,96 @@
+import { JSONValue, useChatUI } from "@llamaindex/chat-ui";
+import React, { FC, useState } from "react";
+import { Button } from "@/components/ui/button";
+import { Card, CardContent, CardFooter } from "@/components/ui/card";
+import { z } from "zod";
+
+// This schema is equivalent to the CLICommand model defined in events.py
+const CLIInputEventSchema = z.object({
+  command: z.string(),
+});
+type CLIInputEvent = z.infer<typeof CLIInputEventSchema>;
+
+
+const CLIHumanInput: FC<{
+  events: JSONValue[];
+}> = ({ events }) => {
+  const inputEvent = (events || [])
+    .map((ev) => {
+      const parseResult = CLIInputEventSchema.safeParse(ev);
+      return parseResult.success ? parseResult.data : null;
+    })
+    .filter((ev): ev is CLIInputEvent => ev !== null)
+    .at(-1);
+
+  const { append } = useChatUI();
+  const [confirmedValue, setConfirmedValue] = useState<boolean | null>(null);
+  const [editableCommand, setEditableCommand] = useState<string | undefined>(
+    inputEvent?.command,
+  );
+
+  // Update editableCommand if inputEvent changes (e.g. new event comes in)
+  React.useEffect(() => {
+    setEditableCommand(inputEvent?.command);
+  }, [inputEvent?.command]);
+
+  const handleConfirm = () => {
+    append({
+      content: "Yes",
+      role: "user",
+      annotations: [
+        {
+          type: "human_response",
+          data: {
+            execute: true,
+            command: editableCommand, // Use editable command
+          },
+        },
+      ],
+    });
+    setConfirmedValue(true);
+  };
+
+  const handleCancel = () => {
+    append({
+      content: "No",
+      role: "user",
+      annotations: [
+        {
+          type: "human_response",
+          data: {
+            execute: false,
+            command: inputEvent?.command,
+          },
+        },
+      ],
+    });
+    setConfirmedValue(false);
+  };
+
+  return (
+    <Card className="my-4">
+      <CardContent className="pt-6">
+        <p className="text-sm text-gray-700">
+          Do you want to execute the following command?
+        </p>
+        <input
+          disabled
+          type="text"
+          value={editableCommand || ""}
+          onChange={(e) => setEditableCommand(e.target.value)}
+          className="bg-gray-100 rounded p-3 my-2 text-xs font-mono text-gray-800 overflow-x-auto w-full border border-gray-300"
+        />
+      </CardContent>
+      {confirmedValue === null ? (
+        <CardFooter className="flex justify-end gap-2">
+          <>
+            <Button onClick={handleConfirm}>Yes</Button>
+            <Button onClick={handleCancel}>No</Button>
+          </>
+        </CardFooter>
+      ) : null}
+    </Card>
+  );
+};
+
+export default CLIHumanInput;

python/llama-index-server/examples/hitl/custom_workflow.py

@@ -0,0 +1,109 @@
+import platform
+import subprocess
+from typing import Any
+
+from events import CLICommand, CLIHumanInputEvent, CLIHumanResponseEvent
+from fastapi import FastAPI
+
+from llama_index.core.prompts import PromptTemplate
+from llama_index.core.settings import Settings
+from llama_index.core.workflow import (
+    Context,
+    StartEvent,
+    StopEvent,
+    Workflow,
+    step,
+)
+from llama_index.server import LlamaIndexServer, UIConfig
+
+
+class CLIWorkflow(Workflow):
+    """
+    A workflow has ability to execute command line tool with human in the loop for confirmation.
+    """
+
+    default_prompt = PromptTemplate(
+        template="""
+        You are a helpful assistant who can write CLI commands to execute using {cli_language}.
+        Your task is to analyze the user's request and write a CLI command to execute.
+
+        ## User Request
+        {user_request}
+
+        Don't be verbose, only respond with the CLI command without any other text.
+        """
+    )
+
+    def __init__(self, **kwargs: Any) -> None:
+        # HITL Workflow should disable timeout otherwise, we will get a timeout error from callback
+        kwargs["timeout"] = None
+        super().__init__(**kwargs)
+
+    @step
+    async def start(self, ctx: Context, ev: StartEvent) -> CLIHumanInputEvent:
+        user_msg = ev.user_msg
+        if user_msg is None:
+            raise ValueError("Missing user_msg in StartEvent")
+        await ctx.set("user_msg", user_msg)
+        # Request LLM to generate a CLI command
+        os_name = platform.system()
+        if os_name == "Linux" or os_name == "Darwin":
+            cli_language = "bash"
+        else:
+            cli_language = "cmd"
+        prompt = self.default_prompt.format(
+            user_request=user_msg, cli_language=cli_language
+        )
+        llm = Settings.llm
+        if llm is None:
+            raise ValueError("Missing LLM in Settings")
+        response = await llm.acomplete(prompt, formatted=True)
+        command = response.text.strip()
+        if command == "":
+            raise ValueError("Couldn't generate a command")
+        # Send the command to the user for confirmation
+        await ctx.set("command", command)
+        return CLIHumanInputEvent(  # type: ignore
+            data=CLICommand(command=command),
+            response_event_type=CLIHumanResponseEvent,
+        )
+
+    @step
+    async def handle_human_response(
+        self,
+        ctx: Context,
+        ev: CLIHumanResponseEvent,  # This event is sent by LlamaIndexServer when user response
+    ) -> StopEvent:
+        # If we have human response, check the confirmation and execute the command
+        if ev.execute:
+            command = ev.command or ""
+            if command == "":
+                raise ValueError("Missing command in CLIExecutionEvent")
+            res = subprocess.run(command, shell=True, capture_output=True, text=True)
+            return StopEvent(result=res.stdout or res.stderr)
+        else:
+            return StopEvent(result=None)
+
+
+def create_app() -> FastAPI:
+    app = LlamaIndexServer(
+        workflow_factory=lambda: CLIWorkflow(),
+        suggest_next_questions=False,
+        ui_config=UIConfig(
+            starter_questions=[
+                "List all files in the current directory",
+                "Fetch changes from the remote repository",
+            ],
+            component_dir="components",
+        ),
+    )
+    return app
+
+
+app = create_app()
+
+
+if __name__ == "__main__":
+    import uvicorn
+
+    uvicorn.run("custom_workflow:app", port=8000, reload=True)

python/llama-index-server/examples/hitl/events.py

@@ -0,0 +1,34 @@
+from typing import Type
+
+from pydantic import BaseModel, Field
+
+from llama_index.server.models import HumanInputEvent, HumanResponseEvent
+
+
+class CLIHumanResponseEvent(HumanResponseEvent):
+    execute: bool = Field(
+        description="True if the human wants to execute the command, False otherwise."
+    )
+    command: str = Field(description="The command to execute.")
+
+
+class CLICommand(BaseModel):
+    command: str = Field(description="The command to execute.")
+
+
+# We need an event that extends from HumanInputEvent for HITL feature
+class CLIHumanInputEvent(HumanInputEvent):
+    """
+    CLIInputRequiredEvent is sent when the agent needs permission from the user to execute the CLI command or not.
+    Render this event by showing the command and a boolean button to execute the command or not.
+    """
+
+    event_type: str = (
+        "cli_human_input"  # used by UI to render with appropriate component
+    )
+    response_event_type: Type = (
+        CLIHumanResponseEvent  # used by workflow to resume with the correct event
+    )
+    data: CLICommand = Field(  # the data that sent to the UI for rendering
+        description="The command to execute.",
+    )

python/llama-index-server/examples/llamacloud/README.md

@@ -0,0 +1,68 @@
+# LlamaCloud Integration
+
+This guide explains how to set up and use the LlamaIndex server with LlamaCloud for retrieval-augmented generation (RAG) with citation support.
+
+## Prerequisites
+
+Please follow the setup instructions in the [examples README](../README.md).
+
+You will also need:
+- An OpenAI API key
+- A LlamaCloud account and API key
+- A LlamaCloud project with indexed documents
+
+## Steps
+
+1. **Set the Required Environment Variables**
+
+   Export your API keys and LlamaCloud configuration:
+
+   ```sh
+   export OPENAI_API_KEY=your_openai_api_key_here
+   export LLAMA_CLOUD_API_KEY=your_llamacloud_api_key_here
+   export LLAMA_CLOUD_PROJECT_NAME=your_project_name
+   export LLAMA_CLOUD_INDEX_NAME=your_index_name
+   ```
+
+2. **Run the Server Using uv**
+
+   Start the server with the following command:
+
+   ```sh
+   uv run main.py
+   ```
+
+   This will launch the FastAPI server using the LlamaCloud workflow defined in `main.py`.
+
+3. **Access the Application**
+
+   Open your browser and go to:
+
+   ```
+   http://localhost:8000
+   ```
+
+   You will see the LlamaIndex chat app UI with LlamaCloud integration, where you can query your indexed documents.
+
+## Features
+
+- **Document Retrieval**: Query your LlamaCloud indexed documents with two retrieval modes:
+  - **Chunk-level retrieval**: Best for specific, detailed questions requiring precise information
+  - **Document-level retrieval**: Best for high-level summarization and broader context questions
+- **Citation Support**: All responses include citations to the source documents, helping you verify and trace the information back to its origin.
+- **Index Selection**: The UI includes an index selector, allowing you to switch between different LlamaCloud indexes if you have multiple of them.
+
+## How it Works
+
+The workflow uses two specialized query engines:
+
+1. **Chunk Query Engine**: Retrieves specific chunks of documents for detailed, targeted questions
+2. **File Query Engine**: Retrieves entire documents as context for broader, summarization-type questions
+
+Both engines are enhanced with citation capabilities, ensuring transparency and traceability in the responses.
+
+## Notes
+
+- Make sure your LlamaCloud project has documents indexed before running the example
+- The server uses GPT-4.1 by default for optimal performance with citations
+- The workflow automatically selects the appropriate retrieval strategy based on your query type 

python/llama-index-server/examples/llamacloud/main.py

@@ -2,13 +2,14 @@ import os
 from typing import List, Optional
 
 from fastapi import FastAPI
+
 from llama_index.core.agent.workflow import AgentWorkflow
 from llama_index.core.query_engine.retriever_query_engine import RetrieverQueryEngine
 from llama_index.core.settings import Settings
 from llama_index.core.tools import QueryEngineTool, ToolMetadata
 from llama_index.llms.openai import OpenAI
 from llama_index.server import LlamaIndexServer, UIConfig
-from llama_index.server.api.models import ChatRequest
+from llama_index.server.models import ChatRequest
 from llama_index.server.services.llamacloud import LlamaCloudIndex, get_index
 from llama_index.server.tools.index.citation import (
     CITATION_SYSTEM_PROMPT,

python/llama-index-server/examples/private_file/README.md

@@ -0,0 +1,102 @@
+# Uploaded File
+
+This example shows how to use the uploaded file (private file) from the user in the workflow.
+
+## Prerequisites
+
+Please follow the setup instructions in the [examples README](../README.md).
+
+You will also need:
+- An OpenAI API key
+- Text files for processing (the examples are optimized for smaller text files)
+
+## How to get the uploaded files in your workflow:
+
+The uploaded file information is included in the annotations of a [ChatAPIMessage](../../llama_index/server/models/chat.py#66). You can manually access it through the `chat_request` parameter in the workflow factory. We already provided a [get_file_attachments](../../llama_index/server/utils/chat_attachments.py) helper function to get the uploaded files from the chat request easier.
+
+```python
+from llama_index.server.api.utils.chat_attachments import get_file_attachments
+
+def create_workflow(chat_request: ChatRequest) -> Workflow:
+    uploaded_files = get_file_attachments(chat_request.messages)
+    ...
+```
+
+Each uploaded file item is a [ServerFile](../../llama_index/server/models/chat.py#9) object, which includes the file id, type, size, and url of the uploaded file. The `url` is an access url to the uploaded file that can be used to download or display the file from the browser, the `id` is used to manage the file in the server through the [FileService](../../llama_index/server/services/file.py).
+
+
+## Examples:
+
+### For agent workflow:
+   - We create a simple file reader tool that can read the uploaded file content.
+
+   ```python
+   def create_file_tool(chat_request: ChatRequest) -> Optional[FunctionTool]:
+      """
+      Create a tool to read file if the user uploads a file.
+      """
+      file_ids = []
+      # Get the uploaded file ids from the the chat messages
+      for file in get_file_attachments(chat_request.messages):
+         file_ids.append(file.id)
+      if len(file_ids) == 0:
+         return None
+
+      # Create a tool description that includes the file ids so the LLM knows which file it can access
+      file_tool_description = (
+         "Use this tool with a file id to read the content of the file."
+         f"\nYou only have access to the following file ids: {json.dumps(file_ids)}"
+      )
+
+      def read_file(file_id: str) -> str:
+         file_path = FileService.get_file_path(file_id)
+         try:
+               with open(file_path, "r") as file:
+                  return file.read()
+         except Exception as e:
+               return f"Error reading file {file_path}: {e}"
+      
+      # Create the tool
+      return FunctionTool.from_defaults(
+         fn=read_file,
+         name="read_file",
+         description=file_tool_description,
+      )
+   ```
+   - Check out the [agent-workflow.py](agent-workflow.py) for more details.
+
+   - You can run the agent workflow with file tool by running the following command:
+     ```bash
+     export OPENAI_API_KEY=your_openai_api_key_here
+     uv run agent-workflow.py
+     ```
+     then go to the UI at `http://localhost:8000` and upload the [example.txt](example.txt) file.
+
+### For custom workflow:
+   - The attachments are included in the `attachments` parameter of the `StartEvent` so you can easily access them in the workflow.
+
+   ```python
+   class MyWorkflow(Workflow):
+      @step
+      async def start_event_handler(self, ctx: Context, ev: StartEvent) -> StopEvent:
+         # Get attachments from the start event
+         attachments = ev.attachments
+         # Do something with the attachments
+         # e.g. read the file content
+         last_file = attachments[-1]
+         if last_file:
+            with open(last_file.path, "r") as f:
+               file_content = f.read()
+            ...
+         # or save it to the context for later use
+         await ctx.set("file_content", file_content)
+         return StopEvent()
+   ```
+   - Check out the [custom-workflow.py](custom-workflow.py) for more details.
+
+   - You can run the custom workflow by running the following command:
+     ```bash
+     export OPENAI_API_KEY=your_openai_api_key_here
+     uv run custom-workflow.py
+     ```
+     then go to the UI at `http://localhost:8000` and upload the [example.txt](example.txt) file.

python/llama-index-server/examples/private_file/agent-workflow.py

@@ -0,0 +1,78 @@
+import json
+from typing import List, Optional
+
+from fastapi import FastAPI
+
+from llama_index.core.agent.workflow import AgentWorkflow
+from llama_index.core.tools import FunctionTool
+from llama_index.llms.openai import OpenAI
+from llama_index.server import LlamaIndexServer, UIConfig
+from llama_index.server.api.utils.chat_attachments import get_file_attachments
+from llama_index.server.models.chat import ChatRequest
+from llama_index.server.models.file import ServerFile
+from llama_index.server.services.file import FileService
+
+
+def create_file_tool(file_attachments: List[ServerFile]) -> Optional[FunctionTool]:
+    """
+    Create a tool to read file if the user uploads a file.
+    """
+    file_ids = []
+    for file in file_attachments:
+        file_ids.append(file.id)
+    if len(file_ids) == 0:
+        return None
+
+    file_tool_description = (
+        "Use this tool with a file id to read the content of the file."
+        f"\nYou only have access to the following file ids: {json.dumps(file_ids)}"
+    )
+
+    def read_file(file_id: str) -> str:
+        # Validate if the file id is in the list of file ids
+        if file_id not in file_ids:
+            raise ValueError(f"I don't have access to file id {file_id}")
+
+        file_path = FileService.get_file_path(file_id)
+        try:
+            with open(file_path, "r") as file:
+                return file.read()
+        except Exception as e:
+            return f"Error reading file {file_path}: {e}"
+
+    return FunctionTool.from_defaults(
+        fn=read_file,
+        name="read_file",
+        description=file_tool_description,
+    )
+
+
+def create_workflow(chat_request: ChatRequest) -> AgentWorkflow:
+    file_attachments = get_file_attachments(chat_request.messages)
+    file_tool = create_file_tool(file_attachments)
+    return AgentWorkflow.from_tools_or_functions(
+        tools_or_functions=[file_tool] if file_tool else [],
+        llm=OpenAI(model="gpt-4.1-mini"),
+        system_prompt="You are a helpful assistant that can help users with their uploaded files.",
+    )
+
+
+def create_app() -> FastAPI:
+    app = LlamaIndexServer(
+        workflow_factory=create_workflow,
+        suggest_next_questions=False,
+        ui_config=UIConfig(
+            enable_file_upload=True,
+            component_dir="components",
+        ),
+    )
+    return app
+
+
+app = create_app()
+
+
+if __name__ == "__main__":
+    import uvicorn
+
+    uvicorn.run("agent-workflow:app", host="0.0.0.0", port=8000, reload=True)

python/llama-index-server/examples/private_file/custom-workflow.py

@@ -0,0 +1,126 @@
+from typing import Any, List
+
+from fastapi import FastAPI
+
+from llama_index.core.agent.workflow.workflow_events import AgentStream
+from llama_index.core.llms import LLM
+from llama_index.core.prompts import PromptTemplate
+from llama_index.core.workflow import (
+    Context,
+    Event,
+    StartEvent,
+    StopEvent,
+    Workflow,
+    WorkflowRuntimeError,
+    step,
+)
+from llama_index.llms.openai import OpenAI
+from llama_index.server import LlamaIndexServer, UIConfig
+from llama_index.server.api.utils.chat_attachments import get_file_attachments
+from llama_index.server.models.chat import ChatRequest
+from llama_index.server.models.file import ServerFile
+
+
+class FileHelpEvent(Event):
+    """
+    The event for helping the user with the an uploaded file.
+    """
+
+    file_content: str
+    user_request: str
+
+
+class FileHelpWorkflow(Workflow):
+    """
+    A simple workflow that helps the user with the an uploaded file.
+    Note: The workflow just simply feed all the file content to the LLM so it won't work for large files.
+    The purpose is just for demo how a workflow can work with the uploaded file from the user.
+    """
+
+    def __init__(
+        self,
+        llm: LLM,
+        file_attachments: List[ServerFile],
+        **kwargs: Any,
+    ):
+        super().__init__(**kwargs)
+        self.llm = llm
+        self.file_attachments = file_attachments
+
+    @step
+    async def read_files(self, ctx: Context, ev: StartEvent) -> FileHelpEvent:
+        user_msg = ev.user_msg
+        if len(self.file_attachments) == 0:
+            raise WorkflowRuntimeError("Please upload one file to start")
+
+        # Read the file content
+        last_file = self.file_attachments[-1]
+        with open(last_file.path, "r") as f:
+            file_content = f.read()
+
+        return FileHelpEvent(
+            file_content=file_content,
+            user_request=user_msg,
+        )
+
+    @step
+    async def help_user(self, ctx: Context, ev: FileHelpEvent) -> StopEvent:
+        default_prompt = PromptTemplate("""
+        You are a writing assistant.
+        You are given a file content and a user request.
+        Your task is to help the user with the file content.
+        
+        User request: {user_msg}
+
+        File content:
+        {file_content}
+        """)
+        prompt = default_prompt.format(
+            user_msg=ev.user_request,
+            file_content=ev.file_content,
+        )
+        stream = await self.llm.astream_complete(prompt)
+        async for chunk in stream:
+            ctx.write_event_to_stream(
+                AgentStream(
+                    response=chunk.text,
+                    delta=chunk.delta or "",
+                    current_agent_name="agent",
+                    tool_calls=[],
+                    raw=chunk.raw,
+                )
+            )
+
+        return StopEvent(
+            content=True,
+        )
+
+
+def create_workflow(chat_request: ChatRequest) -> Workflow:
+    # Use get_file_attachments to get the file attachments from the chat messages
+    file_attachments = get_file_attachments(chat_request.messages)
+    return FileHelpWorkflow(
+        llm=OpenAI(model="gpt-4.1-mini"),
+        file_attachments=file_attachments,
+    )
+
+
+def create_app() -> FastAPI:
+    app = LlamaIndexServer(
+        workflow_factory=create_workflow,
+        suggest_next_questions=False,
+        ui_config=UIConfig(
+            enable_file_upload=True,
+            component_dir="components",
+        ),
+    )
+    return app
+
+
+app = create_app()
+
+
+if __name__ == "__main__":
+    import uvicorn
+
+    uvicorn.run("custom-workflow:app", host="0.0.0.0", port=8000, reload=True)

python/llama-index-server/examples/private_file/example.txt

@@ -0,0 +1,142 @@
+# ACME Vendor Agreement
+
+**Effective Date:** January 1, 2024
+
+## Parties:
+
+- **Client:** LlamaCo ("Client")
+- **Vendor:** ACME Office Supply, Inc. ("Vendor")
+
+## 1. Overview
+
+This Vendor Agreement ("Agreement") sets forth the terms and conditions under which ACME Office Supply, Inc. will provide office supplies, consumables, related goods ("Products"), and associated data processing services to LlamaCo.
+
+## 2. Definitions
+
+- **Personal Data:** Any information relating to an identified or identifiable natural person ('data subject').
+- **Processing:** Any operation performed on Personal Data, including collection, storage, modification, transfer, or deletion.
+- **Data Controller:** LlamaCo, who determines the purposes and means of processing Personal Data.
+- **Data Processor:** ACME Office Supply, Inc., who processes Personal Data on behalf of the Controller.
+
+## 3. Data Protection and Privacy
+
+### 3.1 Scope of Processing
+
+Vendor shall process Personal Data only:
+
+- To fulfill orders and manage deliveries
+- To provide customer support services
+- To maintain business records
+- To comply with legal obligations
+
+### 3.2 Data Subject Rights
+
+Vendor shall:
+
+- Respond to data subject requests within 30 days
+- Provide data in a structured, commonly used format
+- Implement measures to facilitate data portability
+- Assist with data subject rights requests at no additional cost
+
+### 3.3 Data Transfers and Storage
+
+- Vendor maintains primary data centers in the United States
+- Vendor may transfer data to any country where it maintains operations
+- No prior notification required for new data storage locations
+- Vendor will rely on its standard data transfer mechanisms
+- Data may be processed by staff operating outside the EEA
+
+### 3.4 Subprocessors
+
+- Vendor may engage subprocessors without prior Client approval
+- Subprocessors may be located in any jurisdiction globally
+- Notice of new subprocessors provided within 30 days of engagement
+- Client has no right to object to new subprocessors
+
+## 4. Security Measures
+
+### 4.1 Technical and Organizational Measures
+
+Vendor shall implement appropriate measures including:
+
+- Encryption of Personal Data in transit and at rest
+- Access controls and authentication
+- Regular security testing and assessments
+- Employee training on data protection
+- Incident response procedures
+
+### 4.2 Data Breaches
+
+Vendor shall:
+
+- Notify Client of any Personal Data breach within 72 hours
+- Provide details necessary to meet regulatory requirements
+- Cooperate with Client's breach investigation
+- Maintain records of all data breaches
+
+## 5. Data Retention
+
+### 5.1 Retention Period
+
+- Personal Data retained only as long as necessary
+- Standard retention period of 3 years after last transaction
+- Deletion of Personal Data upon written request
+- Backup copies retained for maximum of 6 months
+
+### 5.2 Termination
+
+Upon termination of services:
+
+- Return all Personal Data in standard format
+- Delete existing copies within 30 days
+- Provide written confirmation of deletion
+- Cease all processing activities
+
+## 6. Compliance and Audit
+
+### 6.1 Documentation
+
+Vendor shall maintain:
+
+- Records of all processing activities
+- Security measure documentation
+- Data transfer mechanisms
+- Subprocessor agreements
+
+### 6.2 Audits
+
+- Annual compliance audits permitted
+- 30 days notice required for audits
+- Vendor to provide necessary documentation
+- Client bears reasonable audit costs
+
+## 7. Liability and Indemnification
+
+### 7.1 Liability
+
+- Vendor liable for data protection violations
+- Reasonable compensation for damages
+- Coverage for regulatory fines where applicable
+- Joint liability as required by law
+
+## 8. Governing Law
+
+This Agreement shall be governed by the laws of Ireland, without regard to its conflict of laws principles.
+
+---
+
+IN WITNESS WHEREOF, the parties have executed this Agreement as of the Effective Date.
+
+**LlamaCo**
+
+By: **_
+Name: [Authorized Representative]  
+Title: [Title]  
+Date: _**
+
+**ACME Office Supply, Inc.**
+
+By: **_  
+Name: [Authorized Representative]  
+Title: [Title]  
+Date: _**

python/llama-index-server/examples/pyproject.toml

@@ -0,0 +1,12 @@
+[project]
+name = "llama-index-server-examples"
+version = "0.1.0"
+requires-python = ">=3.10"
+dependencies = [
+    "llama-index-server",
+    "llama-index-llms-openai>=0.4.2",
+    "e2b-code-interpreter>=1.1.1,<2.0.0",
+    "llama-cloud>=0.1.17,<1.0.0",
+    "markdown>=3.7,<4.0",
+    "xhtml2pdf>=0.2.17,<1.0.0",
+]

python/llama-index-server/examples/simple-agent/README.md

@@ -4,7 +4,9 @@ This guide explains how to set up and use the LlamaIndex server with a simple ch
 
 ## Prerequisites
 
-- [uv](https://github.com/astral-sh/uv) installed (a fast Python package manager and runner)
+Please follow the setup instructions in the [examples README](../README.md).
+
+You will also need:
 - An OpenAI API key
 
 ## Steps
@@ -22,10 +24,10 @@ This guide explains how to set up and use the LlamaIndex server with a simple ch
    Start the server with the following command:
 
    ```sh
-   uv run workflow.py
+   uv run main.py
    ```
 
-   This will launch the FastAPI server using the workflow defined in `main.py`.
+   This will launch the FastAPI server using the workflow defined in `app/workflow.py`.
 
 3. **Access the Application**
 

python/llama-index-server/examples/simple-agent/app/workflow.py

@@ -1,14 +1,13 @@
 from typing import Optional
 
 from llama_index.core.agent.workflow import AgentWorkflow
-from llama_index.core.settings import Settings
 from llama_index.llms.openai import OpenAI
-from llama_index.server.api.models import ChatRequest
+from llama_index.server.models import ChatRequest
 
 
 def create_workflow(chat_request: Optional[ChatRequest] = None) -> AgentWorkflow:
     return AgentWorkflow.from_tools_or_functions(
         tools_or_functions=[],
-        llm=Settings.llm or OpenAI(model="gpt-4o-mini"),
+        llm=OpenAI(model="gpt-4o-mini"),
         system_prompt="You are a helpful assistant that can tell a joke about Llama.",
     )

python/llama-index-server/llama_index/server/__init__.py

@@ -1,4 +1,4 @@
-from .api.models import UIEvent
+from .models.ui import UIEvent
 from .server import LlamaIndexServer, UIConfig
 
 __all__ = ["LlamaIndexServer", "UIConfig", "UIEvent"]

python/llama-index-server/llama_index/server/api/callbacks/__init__.py

@@ -1,4 +1,7 @@
 from llama_index.server.api.callbacks.agent_call_tool import AgentCallTool
+from llama_index.server.api.callbacks.artifact_transform import (
+    InlineAnnotationTransformer,
+)
 from llama_index.server.api.callbacks.base import EventCallback
 from llama_index.server.api.callbacks.llamacloud import LlamaCloudFileDownload
 from llama_index.server.api.callbacks.source_nodes import SourceNodesFromToolCall
@@ -12,4 +15,5 @@ __all__ = [
     "SuggestNextQuestions",
     "LlamaCloudFileDownload",
     "AgentCallTool",
+    "InlineAnnotationTransformer",
 ]

python/llama-index-server/llama_index/server/api/callbacks/agent_call_tool.py

@@ -3,7 +3,7 @@ from typing import Any
 
 from llama_index.core.agent.workflow.workflow_events import ToolCall, ToolCallResult
 from llama_index.server.api.callbacks.base import EventCallback
-from llama_index.server.api.models import AgentRunEvent
+from llama_index.server.models.ui import AgentRunEvent
 
 logger = logging.getLogger("uvicorn")
 

python/llama-index-server/llama_index/server/api/callbacks/artifact_transform.py

@@ -0,0 +1,24 @@
+import logging
+from typing import Any
+
+from llama_index.server.api.callbacks.base import EventCallback
+from llama_index.server.models.artifacts import ArtifactEvent
+from llama_index.server.utils.inline import to_inline_annotation_event
+
+logger = logging.getLogger("uvicorn")
+
+
+class InlineAnnotationTransformer(EventCallback):
+    """
+    Transforms an event to AgentStream with inline annotation format.
+    """
+
+    async def run(self, event: Any) -> Any:
+        # handle for ArtifactEvent specifically as it's only supported by inline annotation
+        if isinstance(event, ArtifactEvent):
+            return to_inline_annotation_event(event)
+        return event
+
+    @classmethod
+    def from_default(cls, *args: Any, **kwargs: Any) -> "InlineAnnotationTransformer":
+        return cls()

python/llama-index-server/llama_index/server/api/callbacks/source_nodes.py

@@ -4,7 +4,7 @@ from typing import Any, List, Optional
 from llama_index.core.agent.workflow.workflow_events import ToolCallResult
 from llama_index.core.schema import NodeWithScore
 from llama_index.server.api.callbacks.base import EventCallback
-from llama_index.server.api.models import SourceNodesEvent
+from llama_index.server.models.source_nodes import SourceNodesEvent
 
 logger = logging.getLogger(__name__)
 

python/llama-index-server/llama_index/server/api/callbacks/stream_handler.py

@@ -25,6 +25,10 @@ class StreamHandler:
         """Cancel the workflow handler."""
         await self.workflow_handler.cancel_run()
 
+    async def wait_for_completion(self) -> Any:
+        """Wait for the workflow to finish."""
+        await self.workflow_handler
+
     async def stream_events(self) -> AsyncGenerator[Any, None]:
         """Stream events through the processor chain."""
         try:

python/llama-index-server/llama_index/server/api/callbacks/suggest_next_questions.py

@@ -2,7 +2,7 @@ import logging
 from typing import Any, Optional
 
 from llama_index.server.api.callbacks.base import EventCallback
-from llama_index.server.api.models import ChatRequest
+from llama_index.server.models.chat import ChatRequest
 from llama_index.server.services.suggest_next_question import (
     SuggestNextQuestionsService,
 )

python/llama-index-server/llama_index/server/api/models.py

@@ -1,196 +1,2 @@
-import logging
-import os
-from enum import Enum
-from typing import Any, Dict, List, Literal, Optional, Union
-
-from pydantic import BaseModel, field_validator
-
-from llama_index.core.schema import NodeWithScore
-from llama_index.core.types import ChatMessage, MessageRole
-from llama_index.core.workflow import Event
-from llama_index.server.settings import server_settings
-from llama_index.server.utils import llamacloud
-
-logger = logging.getLogger("uvicorn")
-
-
-class ChatAPIMessage(BaseModel):
-    role: MessageRole
-    content: str
-    annotations: Optional[List[Any]] = None
-
-    def to_llamaindex_message(self) -> ChatMessage:
-        return ChatMessage(role=self.role, content=self.content)
-
-
-class ChatRequest(BaseModel):
-    messages: List[ChatAPIMessage]
-    data: Optional[Any] = None
-
-    @field_validator("messages")
-    def validate_messages(cls, v: List[ChatAPIMessage]) -> List[ChatAPIMessage]:
-        if v[-1].role != MessageRole.USER:
-            raise ValueError("Last message must be from user")
-        return v
-
-
-class AgentRunEventType(Enum):
-    TEXT = "text"
-    PROGRESS = "progress"
-
-
-class AgentRunEvent(Event):
-    name: str
-    msg: str
-    event_type: AgentRunEventType = AgentRunEventType.TEXT
-    data: Optional[dict] = None
-
-    def to_response(self) -> dict:
-        return {
-            "type": "agent",
-            "data": {
-                "agent": self.name,
-                "type": self.event_type.value,
-                "text": self.msg,
-                "data": self.data,
-            },
-        }
-
-
-class SourceNodesEvent(Event):
-    nodes: List[NodeWithScore]
-
-    def to_response(self) -> dict:
-        return {
-            "type": "sources",
-            "data": {
-                "nodes": [
-                    SourceNodes.from_source_node(node).model_dump()
-                    for node in self.nodes
-                ]
-            },
-        }
-
-
-class SourceNodes(BaseModel):
-    id: str
-    metadata: Dict[str, Any]
-    score: Optional[float]
-    text: str
-    url: Optional[str]
-
-    @classmethod
-    def from_source_node(cls, source_node: NodeWithScore) -> "SourceNodes":
-        metadata = source_node.node.metadata
-        url = cls.get_url_from_metadata(metadata)
-
-        return cls(
-            id=source_node.node.node_id,
-            metadata=metadata,
-            score=source_node.score,
-            text=source_node.node.text,  # type: ignore
-            url=url,
-        )
-
-    @classmethod
-    def get_url_from_metadata(
-        cls,
-        metadata: Dict[str, Any],
-        data_dir: Optional[str] = None,
-    ) -> Optional[str]:
-        url_prefix = server_settings.file_server_url_prefix
-        if data_dir is None:
-            data_dir = "data"
-        file_name = metadata.get("file_name")
-
-        if file_name and url_prefix:
-            if llamacloud.is_llamacloud_file(metadata):
-                file_name = llamacloud.get_local_file_name(metadata)
-                return f"{url_prefix}/output/llamacloud/{file_name}"
-            is_private = metadata.get("private", "false") == "true"
-            if is_private:
-                # file is a private upload
-                return f"{url_prefix}/output/uploaded/{file_name}"
-            # file is from calling the 'generate' script
-            # Get the relative path of file_path to data_dir
-            file_path = metadata.get("file_path")
-            data_dir = os.path.abspath(data_dir)
-            if file_path and data_dir:
-                relative_path = os.path.relpath(file_path, data_dir)
-                return f"{url_prefix}/data/{relative_path}"
-        # fallback to URL in metadata (e.g. for websites)
-        return metadata.get("URL")
-
-    @classmethod
-    def from_source_nodes(
-        cls, source_nodes: List[NodeWithScore]
-    ) -> List["SourceNodes"]:
-        return [cls.from_source_node(node) for node in source_nodes]
-
-
-class ComponentDefinition(BaseModel):
-    type: str
-    code: str
-    filename: str
-
-
-class UIEvent(Event):
-    type: str
-    data: BaseModel
-
-    def to_response(self) -> dict:
-        return {
-            "type": self.type,
-            "data": self.data.model_dump(),
-        }
-
-
-class ArtifactType(str, Enum):
-    CODE = "code"
-    DOCUMENT = "document"
-
-
-class CodeArtifactData(BaseModel):
-    file_name: str
-    code: str
-    language: str
-
-
-class DocumentArtifactData(BaseModel):
-    title: str
-    content: str
-    type: Literal["markdown", "html"]
-
-
-class Artifact(BaseModel):
-    created_at: Optional[int] = None
-    type: ArtifactType
-    data: Union[CodeArtifactData, DocumentArtifactData]
-
-    @classmethod
-    def from_message(cls, message: ChatAPIMessage) -> Optional["Artifact"]:
-        if not message.annotations or not isinstance(message.annotations, list):
-            return None
-
-        for annotation in message.annotations:
-            if isinstance(annotation, dict) and annotation.get("type") == "artifact":
-                try:
-                    artifact = cls.model_validate(annotation.get("data"))
-                    return artifact
-                except Exception as e:
-                    logger.warning(
-                        f"Failed to parse artifact from annotation: {annotation}. Error: {e}"
-                    )
-
-        return None
-
-
-class ArtifactEvent(Event):
-    type: str = "artifact"
-    data: Artifact
-
-    def to_response(self) -> dict:
-        return {
-            "type": self.type,
-            "data": self.data.model_dump(),
-        }
+# TODO: For backward compatibility, remove this in a minor release
+from llama_index.server.models import *  # noqa

python/llama-index-server/llama_index/server/api/routers/chat.py

@@ -6,23 +6,36 @@ from typing import AsyncGenerator, Callable, Union
 
 from fastapi import APIRouter, BackgroundTasks, HTTPException
 from fastapi.responses import StreamingResponse
+
 from llama_index.core.agent.workflow.workflow_events import (
     AgentInput,
     AgentSetup,
     AgentStream,
 )
-from llama_index.core.workflow import StopEvent, Workflow
+from llama_index.core.workflow import (
+    StopEvent,
+    Workflow,
+)
 from llama_index.server.api.callbacks import (
     AgentCallTool,
     EventCallback,
+    InlineAnnotationTransformer,
     LlamaCloudFileDownload,
     SourceNodesFromToolCall,
     SuggestNextQuestions,
 )
 from llama_index.server.api.callbacks.stream_handler import StreamHandler
-from llama_index.server.api.models import ChatRequest
 from llama_index.server.api.utils.vercel_stream import VercelStreamResponse
+from llama_index.server.models.chat import (
+    ChatRequest,
+    FileUpload,
+    MessageRole,
+)
+from llama_index.server.models.file import ServerFileResponse
+from llama_index.server.models.hitl import HumanInputEvent
+from llama_index.server.services.file import FileService
 from llama_index.server.services.llamacloud import LlamaCloudFileService
+from llama_index.server.services.workflow import HITLWorkflowService
 
 
 def chat_router(
@@ -38,7 +51,9 @@ def chat_router(
         background_tasks: BackgroundTasks,
     ) -> StreamingResponse:
         try:
-            user_message = request.messages[-1].to_llamaindex_message()
+            last_message = request.messages[-1]
+            if last_message.role != MessageRole.USER:
+                raise ValueError("Last message must be from user")
             chat_history = [
                 message.to_llamaindex_message() for message in request.messages[:-1]
             ]
@@ -48,13 +63,25 @@ def chat_router(
                 workflow = workflow_factory(chat_request=request)
             else:
                 workflow = workflow_factory()
-            workflow_handler = workflow.run(
-                user_msg=user_message.content,
-                chat_history=chat_history,
-            )
+
+            # Check if we should resume a chat with a human response
+            human_response = last_message.human_response
+            if human_response:
+                ctx = await HITLWorkflowService.load_context(
+                    id=request.id,
+                    workflow=workflow,
+                    data=human_response,
+                )
+                workflow_handler = workflow.run(ctx=ctx)
+            else:
+                workflow_handler = workflow.run(
+                    user_msg=last_message.content,
+                    chat_history=chat_history,
+                )
 
             callbacks: list[EventCallback] = [
                 AgentCallTool(),
+                InlineAnnotationTransformer(),
                 SourceNodesFromToolCall(),
                 LlamaCloudFileDownload(background_tasks),
             ]
@@ -66,12 +93,31 @@ def chat_router(
             )
 
             return VercelStreamResponse(
-                content_generator=_stream_content(stream_handler, request, logger),
+                content_generator=_stream_content(
+                    stream_handler,
+                    logger,
+                    request.id,
+                ),
             )
         except Exception as e:
             logger.error(e)
             raise HTTPException(status_code=500, detail=str(e))
 
+    # we just simply save the file to the server and don't index it
+    @router.post("/file")
+    async def upload_file(request: FileUpload) -> ServerFileResponse:
+        """
+        Upload a file to the server to be used in the chat session.
+        """
+        try:
+            save_dir = os.path.join("output", "private")
+            content, _ = FileService._preprocess_base64_file(request.base64)
+            file = FileService.save_file(content, request.name, save_dir)
+            return file.to_server_file_response()
+        except Exception:
+            raise HTTPException(status_code=500, detail="Error uploading file")
+
+    # Specific to LlamaCloud
     if LlamaCloudFileService.is_configured():
 
         @router.get("/config/llamacloud")
@@ -99,8 +145,8 @@ def chat_router(
 
 async def _stream_content(
     handler: StreamHandler,
-    request: ChatRequest,
     logger: logging.Logger,
+    chat_id: str,
 ) -> AsyncGenerator[str, None]:
     async def _text_stream(
         event: Union[AgentStream, StopEvent],
@@ -126,6 +172,19 @@ async def _stream_content(
                 async for chunk in _text_stream(event):
                     handler.accumulate_text(chunk)
                     yield VercelStreamResponse.convert_text(chunk)
+            elif isinstance(event, HumanInputEvent):
+                ctx = handler.workflow_handler.ctx
+                if ctx is None:
+                    raise RuntimeError("Context is None")
+                # Save the context with the HITL event
+                await HITLWorkflowService.save_context(
+                    id=chat_id,
+                    ctx=ctx,
+                    resume_event_type=event.response_event_type,
+                )
+                yield VercelStreamResponse.convert_data(event.to_response())
+                # Break to stop the stream
+                break
             elif isinstance(event, dict):
                 yield VercelStreamResponse.convert_data(event)
             elif hasattr(event, "to_response"):
@@ -136,6 +195,7 @@ async def _stream_content(
                 if not isinstance(event, (AgentInput, AgentSetup)):
                     yield VercelStreamResponse.convert_data(event.model_dump())
 
+        await handler.wait_for_completion()
     except asyncio.CancelledError:
         logger.warning("Client cancelled the request!")
         await handler.cancel_run()

python/llama-index-server/llama_index/server/api/routers/ui.py

@@ -2,7 +2,8 @@ import logging
 from typing import List
 
 from fastapi import APIRouter
-from llama_index.server.api.models import ComponentDefinition
+
+from llama_index.server.models.ui import ComponentDefinition
 from llama_index.server.services.custom_ui import CustomUI
 
 

python/llama-index-server/llama_index/server/api/utils/chat_attachments.py

@@ -0,0 +1,40 @@
+from typing import List
+
+from llama_index.core.types import MessageRole
+from llama_index.server.models.chat import ChatAPIMessage, FileAnnotation
+from llama_index.server.models.file import ServerFile
+from llama_index.server.services.file import FileService
+
+
+def get_file_attachments(messages: List[ChatAPIMessage]) -> List[ServerFile]:
+    """
+    Extract all file attachments from user messages.
+
+    Args:
+        messages (List[ChatAPIMessage]): The list of messages.
+
+    Returns:
+        List[ServerFile]: The list of private files.
+    """
+    user_message_annotations = [
+        message.annotations
+        for message in messages
+        if message.annotations and message.role == MessageRole.USER
+    ]
+    files: List[ServerFile] = []
+    for annotation in user_message_annotations:
+        if isinstance(annotation, list):
+            for item in annotation:
+                if isinstance(item, FileAnnotation):
+                    server_files = [
+                        ServerFile(
+                            id=file.id,
+                            type=file.type,
+                            size=file.size,
+                            url=file.url,
+                            path=FileService.get_file_path(file.id),
+                        )
+                        for file in item.data.files
+                    ]
+                    files.extend(server_files)
+    return files

python/llama-index-server/llama_index/server/api/utils/chat_request.py

@@ -1,6 +1,7 @@
 from typing import List, Optional
 
-from llama_index.server.api.models import Artifact, ChatRequest
+from llama_index.server.models.artifacts import Artifact
+from llama_index.server.models.chat import ChatRequest
 
 
 def get_artifacts(chat_request: ChatRequest) -> List[Artifact]:

python/llama-index-server/llama_index/server/models/__init__.py

@@ -0,0 +1,36 @@
+from llama_index.server.models.artifacts import (
+    Artifact,
+    ArtifactEvent,
+    ArtifactType,
+    CodeArtifactData,
+    DocumentArtifactData,
+    DocumentArtifactSource,
+)
+from llama_index.server.models.chat import ChatAPIMessage, ChatRequest
+from llama_index.server.models.hitl import HumanInputEvent, HumanResponseEvent
+from llama_index.server.models.source_nodes import SourceNodes, SourceNodesEvent
+from llama_index.server.models.ui import (
+    AgentRunEvent,
+    AgentRunEventType,
+    ComponentDefinition,
+    UIEvent,
+)
+
+__all__ = [
+    "Artifact",
+    "ArtifactEvent",
+    "ArtifactType",
+    "DocumentArtifactData",
+    "DocumentArtifactSource",
+    "CodeArtifactData",
+    "ChatAPIMessage",
+    "ChatRequest",
+    "UIEvent",
+    "ComponentDefinition",
+    "AgentRunEvent",
+    "AgentRunEventType",
+    "SourceNodes",
+    "SourceNodesEvent",
+    "HumanInputEvent",
+    "HumanResponseEvent",
+]

python/llama-index-server/llama_index/server/models/artifacts.py

@@ -0,0 +1,66 @@
+import logging
+from enum import Enum
+from typing import List, Literal, Optional, Union
+
+from llama_index.core.workflow.events import Event
+from llama_index.server.models.chat import ChatAPIMessage
+from pydantic import BaseModel
+from llama_index.server.utils.inline import get_inline_annotations
+
+logger = logging.getLogger(__name__)
+
+
+class ArtifactType(str, Enum):
+    CODE = "code"
+    DOCUMENT = "document"
+
+
+class CodeArtifactData(BaseModel):
+    file_name: str
+    code: str
+    language: str
+
+
+class DocumentArtifactSource(BaseModel):
+    id: str
+    # we can add more fields here
+
+
+class DocumentArtifactData(BaseModel):
+    title: str
+    content: str
+    type: Literal["markdown", "html"]
+    sources: Optional[List[DocumentArtifactSource]] = None
+
+
+class Artifact(BaseModel):
+    created_at: Optional[int] = None
+    type: ArtifactType
+    data: Union[CodeArtifactData, DocumentArtifactData]
+
+    @classmethod
+    def from_message(cls, message: ChatAPIMessage) -> Optional["Artifact"]:
+        inline_annotations = get_inline_annotations(message)
+
+        for annotation in inline_annotations:
+            if isinstance(annotation, dict) and annotation.get("type") == "artifact":
+                try:
+                    artifact = cls.model_validate(annotation.get("data"))
+                    return artifact
+                except Exception as e:
+                    logger.warning(
+                        f"Failed to parse artifact from annotation: {annotation}. Error: {e}"
+                    )
+
+        return None
+
+
+class ArtifactEvent(Event):
+    type: str = "artifact"
+    data: Artifact
+
+    def to_response(self) -> dict:
+        return {
+            "type": self.type,
+            "data": self.data.model_dump(),
+        }

python/llama-index-server/llama_index/server/models/chat.py

@@ -0,0 +1,106 @@
+import re
+from typing import Any, List, Literal, Optional, Union
+
+from pydantic import BaseModel, Field, field_validator
+
+from llama_index.core.types import ChatMessage, MessageRole
+from llama_index.server.models.file import ServerFileResponse
+
+
+class FileData(BaseModel):
+    """
+    The data of a file.
+    """
+
+    files: List[ServerFileResponse]
+
+
+class FileAnnotation(BaseModel):
+    """
+    The annotation of a file.
+    """
+
+    type: Literal["document_file"]
+    data: FileData
+
+
+class FileUpload(BaseModel):
+    """
+    The file to be uploaded to the chat.
+    """
+
+    name: str
+    base64: str
+    params: Any = None
+
+
+class LlamaCloudPipeline(BaseModel):
+    """
+    The selected LlamaCloud pipeline to use for the chat.
+    (Only available when the app is configured to use LlamaCloud)
+    """
+
+    pipeline: str
+    project: str
+
+
+class ChatData(BaseModel):
+    """
+    The data of a chat.
+    """
+
+    llama_cloud_pipeline: Optional[LlamaCloudPipeline] = Field(
+        default=None,
+        description="The selected LlamaCloud pipeline to use for the chat",
+        alias="llamaCloudPipeline",
+        serialization_alias="llamaCloudPipeline",
+    )
+
+
+class ChatAPIMessage(BaseModel):
+    role: MessageRole
+    content: str
+    annotations: Optional[List[Union[FileAnnotation, Any]]] = None
+
+    def to_llamaindex_message(self) -> ChatMessage:
+        """
+        Simply convert text content of API message to llama_index's ChatMessage.
+        Annotations are not included.
+        """
+        return ChatMessage(role=self.role, content=self.content)
+
+    @property
+    def human_response(self) -> Optional[Any]:
+        if self.annotations:
+            for annotation in self.annotations:
+                if (
+                    isinstance(annotation, dict)
+                    and annotation.get("type") == "human_response"
+                ):
+                    return annotation.get("data", {})
+        return None
+
+
+class ChatRequest(BaseModel):
+    """
+    The request to the chat API.
+    """
+
+    id: str  # see https://ai-sdk.dev/docs/reference/ai-sdk-ui/use-chat#id - constant for the same chat session
+    messages: List[ChatAPIMessage]
+    data: Optional[ChatData] = Field(
+        default=None,
+        description="The data of the chat",
+    )
+
+    @field_validator("messages")
+    def validate_messages(cls, v: List[ChatAPIMessage]) -> List[ChatAPIMessage]:
+        if v[-1].role != MessageRole.USER:
+            raise ValueError("Last message must be from user")
+        return v
+
+    @field_validator("id")
+    def validate_id(cls, v: str) -> str:
+        if re.search(r"[^a-zA-Z0-9_-]", v):
+            raise ValueError("ID contains special characters")
+        return v

python/llama-index-server/llama_index/server/models/file.py

@@ -0,0 +1,26 @@
+from typing import Optional
+
+from pydantic import BaseModel, Field
+
+
+class ServerFileResponse(BaseModel):
+    id: str
+    type: Optional[str] = None
+    size: Optional[int] = None
+    url: Optional[str] = None
+
+
+class ServerFile(BaseModel):
+    id: str
+    path: str = Field(description="The path of the file in the server")
+    type: Optional[str] = None
+    size: Optional[int] = None
+    url: Optional[str] = None
+
+    def to_server_file_response(self) -> ServerFileResponse:
+        return ServerFileResponse(
+            id=self.id,
+            type=self.type,
+            size=self.size,
+            url=self.url,
+        )

python/llama-index-server/llama_index/server/models/hitl.py

@@ -0,0 +1,51 @@
+from typing import Any, Dict, Type, Union
+
+from llama_index.core.workflow.events import (
+    HumanResponseEvent as FrameworkHumanResponseEvent,
+)
+from llama_index.core.workflow.events import InputRequiredEvent
+from pydantic import BaseModel, Field
+
+
+class HumanResponseEvent(FrameworkHumanResponseEvent):
+    """
+    Use this event to send a response from a human.
+    """
+
+    def __init__(self, **kwargs: Any) -> None:
+        if "response" not in kwargs:
+            kwargs["response"] = f"Human response with data: {kwargs.get('data', {})}"
+        super().__init__(**kwargs)
+
+
+class HumanInputEvent(InputRequiredEvent):
+    """
+    Use this event to request input from a human.
+    It will block the workflow execution until the human responds.
+    """
+
+    response_event_type: Type[HumanResponseEvent] = Field(
+        description="The type of event that the workflow is waiting for.",
+    )
+    event_type: str = Field(
+        description="An identifier for the UI component that will be used to render the input.",
+    )
+    data: Union[Dict[str, Any], BaseModel] = Field(
+        description="The data to be sent to the UI component that will be used to render the input.",
+    )
+
+    def __init__(self, **kwargs: Any) -> None:
+        # Construct the prefix for InputRequiredEvent
+        event_type = kwargs.get("event_type", None)
+        data = kwargs.get("data", None)
+        if "prefix" not in kwargs:
+            kwargs["prefix"] = f"Need input for {event_type} with data: {data}"
+        super().__init__(**kwargs)
+
+    def to_response(self) -> dict:
+        return {
+            "type": self.event_type,
+            "data": self.data
+            if isinstance(self.data, dict)
+            else self.data.model_dump(),
+        }

python/llama-index-server/llama_index/server/models/source_nodes.py

@@ -0,0 +1,49 @@
+from typing import Any, Dict, List, Optional
+
+from pydantic import BaseModel
+
+from llama_index.core.schema import NodeWithScore
+from llama_index.core.workflow.events import Event
+from llama_index.server.utils.chat_file import get_file_url_from_metadata
+
+
+class SourceNodesEvent(Event):
+    nodes: List[NodeWithScore]
+
+    def to_response(self) -> dict:
+        return {
+            "type": "sources",
+            "data": {
+                "nodes": [
+                    SourceNodes.from_source_node(node).model_dump()
+                    for node in self.nodes
+                ]
+            },
+        }
+
+
+class SourceNodes(BaseModel):
+    id: str
+    metadata: Dict[str, Any]
+    score: Optional[float]
+    text: str
+    url: Optional[str]
+
+    @classmethod
+    def from_source_node(cls, source_node: NodeWithScore) -> "SourceNodes":
+        metadata = source_node.node.metadata
+        url = get_file_url_from_metadata(metadata)
+
+        return cls(
+            id=source_node.node.node_id,
+            metadata=metadata,
+            score=source_node.score,
+            text=source_node.node.text,  # type: ignore
+            url=url,
+        )
+
+    @classmethod
+    def from_source_nodes(
+        cls, source_nodes: List[NodeWithScore]
+    ) -> List["SourceNodes"]:
+        return [cls.from_source_node(node) for node in source_nodes]

python/llama-index-server/llama_index/server/models/ui.py

@@ -0,0 +1,49 @@
+import logging
+from enum import Enum
+from typing import Optional
+
+from pydantic import BaseModel
+
+from llama_index.core.workflow import Event
+
+logger = logging.getLogger("uvicorn")
+
+
+class AgentRunEventType(Enum):
+    TEXT = "text"
+    PROGRESS = "progress"
+
+
+class AgentRunEvent(Event):
+    name: str
+    msg: str
+    event_type: AgentRunEventType = AgentRunEventType.TEXT
+    data: Optional[dict] = None
+
+    def to_response(self) -> dict:
+        return {
+            "type": "agent",
+            "data": {
+                "agent": self.name,
+                "type": self.event_type.value,
+                "text": self.msg,
+                "data": self.data,
+            },
+        }
+
+
+class ComponentDefinition(BaseModel):
+    type: str
+    code: str
+    filename: str
+
+
+class UIEvent(Event):
+    type: str
+    data: BaseModel
+
+    def to_response(self) -> dict:
+        return {
+            "type": self.type,
+            "data": self.data.model_dump(),
+        }

python/llama-index-server/llama_index/server/server.py

@@ -7,8 +7,6 @@ from fastapi import FastAPI
 from fastapi.middleware.cors import CORSMiddleware
 from fastapi.routing import Mount
 from fastapi.staticfiles import StaticFiles
-from pydantic import BaseModel, Field
-
 from llama_index.core.workflow import Workflow
 from llama_index.server.api.routers import (
     chat_router,
@@ -18,6 +16,7 @@ from llama_index.server.api.routers import (
 )
 from llama_index.server.chat_ui import copy_bundled_chat_ui
 from llama_index.server.settings import server_settings
+from pydantic import BaseModel, Field
 
 
 class UIConfig(BaseModel):
@@ -29,6 +28,10 @@ class UIConfig(BaseModel):
         default=False,
         description="Whether to show the LlamaCloud index selector in the chat UI (need to set the LLAMA_CLOUD_API_KEY environment variable)",
     )
+    enable_file_upload: bool = Field(
+        default=False,
+        description="Whether to enable file upload in the chat UI",
+    )
     ui_path: str = Field(
         default=".ui", description="The path that stores static files for the chat UI"
     )
@@ -47,6 +50,11 @@ class UIConfig(BaseModel):
         return json.dumps(
             {
                 "CHAT_API": f"{server_settings.api_url}/chat",
+                "UPLOAD_API": (
+                    f"{server_settings.api_url}/chat/file"
+                    if self.enable_file_upload
+                    else None
+                ),
                 "STARTER_QUESTIONS": self.starter_questions or [],
                 "LLAMA_CLOUD_API": (
                     f"{server_settings.api_url}/chat/config/llamacloud"

python/llama-index-server/llama_index/server/services/custom_ui.py

@@ -2,7 +2,7 @@ import logging
 import os
 from typing import List, Optional
 
-from llama_index.server.api.models import ComponentDefinition
+from llama_index.server.models.ui import ComponentDefinition
 
 
 class CustomUI:

python/llama-index-server/llama_index/server/services/file.py

@@ -1,39 +1,23 @@
+import base64
 import logging
+import mimetypes
 import os
 import re
 import uuid
 from pathlib import Path
-from typing import List, Optional, Union
+from typing import Optional, Tuple, Union
 
+from llama_index.server.models.file import ServerFile
 from llama_index.server.settings import server_settings
-from pydantic import BaseModel, Field
 
 logger = logging.getLogger(__name__)
 
-PRIVATE_STORE_PATH = str(Path("output", "uploaded"))
-TOOL_STORE_PATH = str(Path("output", "tools"))
-LLAMA_CLOUD_STORE_PATH = str(Path("output", "llamacloud"))
-
-
-class DocumentFile(BaseModel):
-    id: str
-    name: str  # Stored file name
-    type: Optional[str] = None
-    size: Optional[int] = None
-    url: Optional[str] = None
-    path: Optional[str] = Field(
-        None,
-        description="The stored file path. Used internally in the server.",
-        exclude=True,
-    )
-    refs: Optional[List[str]] = Field(
-        None, description="The document ids in the index."
-    )
+PRIVATE_STORE_PATH = str(Path("output", "private"))
 
 
 class FileService:
     """
-    To store the files uploaded by the user.
+    Store files to server
     """
 
     @classmethod
@@ -42,38 +26,31 @@ class FileService:
         content: Union[bytes, str],
         file_name: str,
         save_dir: Optional[str] = None,
-    ) -> DocumentFile:
+    ) -> ServerFile:
         """
         Save the content to a file in the local file server (accessible via URL).
 
         Args:
             content (bytes | str): The content to save, either bytes or string.
             file_name (str): The original name of the file.
-            save_dir (Optional[str]): The relative path from the current working directory. Defaults to the `output/uploaded` directory.
-
+            save_dir (Optional[str]): The path to store the file. Defaults is set to PRIVATE_STORE_PATH (output/private) if not provided.
         Returns:
             The metadata of the saved file.
         """
         if save_dir is None:
-            save_dir = os.path.join("output", "uploaded")
+            save_dir = PRIVATE_STORE_PATH
 
-        file_id = str(uuid.uuid4())
-        name, extension = os.path.splitext(file_name)
-        extension = extension.lstrip(".")
-        sanitized_name = _sanitize_file_name(name)
-        if extension == "":
-            raise ValueError("File is not supported!")
-        new_file_name = f"{sanitized_name}_{file_id}.{extension}"
-
-        file_path = os.path.join(save_dir, new_file_name)
-
-        if isinstance(content, str):
-            content = content.encode()
+        file_id, extension = cls._process_file_name(file_name)
+        file_path = os.path.join(save_dir, file_id)
 
+        # Write the file directly, handling both str and bytes
         try:
             os.makedirs(os.path.dirname(file_path), exist_ok=True)
-            with open(file_path, "wb") as file:
-                file.write(content)
+            with open(file_path, "wb") as f:
+                if isinstance(content, str):
+                    f.write(content.encode())
+                else:
+                    f.write(content)
         except PermissionError as e:
             logger.error(f"Permission denied when writing to file {file_path}: {e!s}")
             raise
@@ -87,31 +64,90 @@ class FileService:
         logger.info(f"Saved file to {file_path}")
 
         file_size = os.path.getsize(file_path)
-        file_url = (
-            f"{server_settings.file_server_url_prefix}/{save_dir}/{new_file_name}"
-        )
-        return DocumentFile(
+        file_url = cls._get_file_url(file_id, save_dir)
+        return ServerFile(
             id=file_id,
-            name=new_file_name,
             type=extension,
             size=file_size,
-            path=file_path,
             url=file_url,
-            refs=None,
+            path=file_path,
         )
 
     @classmethod
-    def get_file_url(cls, file_name: str, save_dir: Optional[str] = None) -> str:
+    def _process_file_name(cls, file_name: str) -> tuple[str, str]:
+        """
+        Process original file name to generate a unique file id and extension.
+        """
+        _id = str(uuid.uuid4())
+        name, extension = os.path.splitext(file_name)
+        extension = extension.lstrip(".")
+        if extension == "":
+            raise ValueError("File name is not valid! It must have an extension.")
+        # sanitize the name
+        name = re.sub(r"[^a-zA-Z0-9.]", "_", name)
+        file_id = f"{name}_{_id}.{extension}"
+        return file_id, extension
+
+    @classmethod
+    def _get_file_url(cls, file_id: str, save_dir: Optional[str] = None) -> str:
         """
         Get the URL of a file.
         """
         if save_dir is None:
-            save_dir = os.path.join("output", "uploaded")
-        return f"{server_settings.file_server_url_prefix}/{save_dir}/{file_name}"
+            save_dir = PRIVATE_STORE_PATH
+        # Ensure the path uses forward slashes for URLs
+        url_path = f"{save_dir}/{file_id}".replace("\\", "/")
+        return f"{server_settings.file_server_url_prefix}/{url_path}"
 
+    @classmethod
+    def get_file_path(cls, file_id: str, save_dir: Optional[str] = None) -> str:
+        """
+        Get the path of a private file.
 
-def _sanitize_file_name(file_name: str) -> str:
-    """
-    Sanitize the file name by replacing all non-alphanumeric characters with underscores.
-    """
-    return re.sub(r"[^a-zA-Z0-9.]", "_", file_name)
+        Args:
+            file_id (str): The ID of the file.
+            save_dir (Optional[str]): The path where the file is stored. Defaults to output/private if not provided.
+
+        Returns:
+            str: The full path to the file.
+        """
+        if save_dir is None:
+            save_dir = PRIVATE_STORE_PATH
+        return os.path.join(save_dir, file_id)
+
+    @classmethod
+    def get_file(cls, file_id: str, save_dir: Optional[str] = None) -> bytes:
+        """
+        Read and return the content of a file.
+
+        Args:
+            file_id (str): The ID of the file.
+            save_dir (Optional[str]): The path where the file is stored. Defaults to output/private if not provided.
+
+        Returns:
+            bytes: The content of the file.
+
+        Raises:
+            FileNotFoundError: If the file does not exist.
+        """
+        file_path = cls.get_file_path(file_id, save_dir)
+
+        try:
+            with open(file_path, "rb") as f:
+                return f.read()
+        except FileNotFoundError as e:
+            logger.error(f"File not found: {file_path}")
+            raise FileNotFoundError(f"File with ID '{file_id}' not found") from e
+        except Exception as e:
+            logger.error(f"Unexpected error when reading file {file_path}: {e!s}")
+            raise
+
+    @staticmethod
+    def _preprocess_base64_file(base64_content: str) -> Tuple[bytes, str]:
+        header, data = base64_content.split(",", 1)
+        mime_type = header.split(";")[0].split(":", 1)[1]
+        extension = mimetypes.guess_extension(mime_type)
+        if extension is None:
+            raise ValueError(f"Unsupported file type: {mime_type}")
+        extension = extension.lstrip(".")
+        return base64.b64decode(data), extension

python/llama-index-server/llama_index/server/services/llamacloud/file.py

@@ -11,7 +11,7 @@ from llama_cloud import ManagedIngestionStatus, PipelineFileCreateCustomMetadata
 from pydantic import BaseModel
 
 from llama_index.core.schema import NodeWithScore
-from llama_index.server.api.models import SourceNodes
+from llama_index.server.models.source_nodes import SourceNodes
 from llama_index.server.services.llamacloud.index import get_client
 from llama_index.server.utils import llamacloud
 

python/llama-index-server/llama_index/server/services/llamacloud/index.py

@@ -3,14 +3,15 @@ import os
 from typing import TYPE_CHECKING, Any, Optional
 
 from llama_cloud import PipelineType
+from pydantic import BaseModel, Field, field_validator
+
 from llama_index.core.callbacks import CallbackManager
 from llama_index.core.ingestion.api_utils import (
     get_client as llama_cloud_get_client,
 )
 from llama_index.core.settings import Settings
 from llama_index.indices.managed.llama_cloud import LlamaCloudIndex
-from llama_index.server.api.models import ChatRequest
-from pydantic import BaseModel, Field, field_validator
+from llama_index.server.models.chat import ChatRequest
 
 if TYPE_CHECKING:
     from llama_cloud.client import LlamaCloud
@@ -91,14 +92,14 @@ class IndexConfig(BaseModel):
     def from_default(cls, chat_request: Optional[ChatRequest] = None) -> "IndexConfig":
         default_config = cls()
         if chat_request is not None and chat_request.data is not None:
-            llamacloud_config = chat_request.data.get("llamaCloudPipeline")
+            llamacloud_config = chat_request.data.llama_cloud_pipeline
             if llamacloud_config is not None:
-                default_config.llama_cloud_pipeline_config.pipeline = llamacloud_config[
-                    "pipeline"
-                ]
-                default_config.llama_cloud_pipeline_config.project = llamacloud_config[
-                    "project"
-                ]
+                default_config.llama_cloud_pipeline_config.pipeline = (
+                    llamacloud_config.pipeline
+                )
+                default_config.llama_cloud_pipeline_config.project = (
+                    llamacloud_config.project
+                )
         return default_config
 
 

python/llama-index-server/llama_index/server/services/suggest_next_question.py

@@ -5,7 +5,7 @@ from typing import List, Optional, Union
 
 from llama_index.core.prompts import PromptTemplate
 from llama_index.core.settings import Settings
-from llama_index.server.api.models import ChatAPIMessage
+from llama_index.server.models.chat import ChatAPIMessage
 from llama_index.server.prompts import SUGGEST_NEXT_QUESTION_PROMPT
 
 logger = logging.getLogger("uvicorn")

python/llama-index-server/llama_index/server/services/workflow.py

@@ -0,0 +1,106 @@
+import json
+import logging
+from pathlib import Path
+from typing import Type
+
+from llama_index.core.workflow import (
+    Context,
+    JsonSerializer,
+    Workflow,
+)
+from llama_index.server.models.hitl import HumanResponseEvent
+from llama_index.server.utils.class_meta_serialization import (
+    type_from_identifier,
+    type_identifier,
+)
+
+logger = logging.getLogger(__name__)
+
+
+class HITLWorkflowService:
+    """
+    A service for helping pause and resume a HITL workflow.
+    """
+
+    # A key in context that stores the HITL event type
+    HITL_CONTEXT_KEY = "human_response_type"
+
+    @staticmethod
+    def get_storage_path(id: str) -> Path:
+        storage_dir = Path("output") / "checkpoints"
+        if not storage_dir.exists():
+            storage_dir.mkdir(parents=True, exist_ok=True)
+        return storage_dir / f"{id}.json"
+
+    @classmethod
+    async def save_context(
+        cls,
+        id: str,
+        ctx: Context,
+        resume_event_type: Type[HumanResponseEvent],
+    ) -> None:
+        """
+        Save the current checkpoint to a file and return the id
+
+        Args:
+            id: The id to save the context to.
+            ctx: The context to save.
+            resume_event_type [Optional]: Save workflow context with a resume event.
+        """
+        await ctx.set(
+            key=cls.HITL_CONTEXT_KEY,
+            value=type_identifier(resume_event_type),
+        )
+
+        ctx_data = ctx.to_dict(serializer=JsonSerializer())
+        with open(cls.get_storage_path(id), "w") as f:
+            json.dump(ctx_data, f)
+
+    @classmethod
+    async def load_context(
+        cls,
+        id: str,
+        workflow: Workflow,
+        data: dict,
+    ) -> Context:
+        file_path = cls.get_storage_path(id)
+        if not file_path.exists():
+            raise FileNotFoundError(f"No checkpoint found for id: {id}")
+        try:
+            with open(file_path, "r") as f:
+                ctx_data = json.load(f)
+        except json.JSONDecodeError as e:
+            raise ValueError(f"Invalid checkpoint data for id {id}: {e}")
+        ctx = Context.from_dict(
+            workflow=workflow,
+            data=ctx_data,
+            serializer=JsonSerializer(),
+        )
+        resume_event = await cls._construct_resume_event(ctx, data)
+        ctx.send_event(resume_event)
+        return ctx
+
+    @classmethod
+    async def _construct_resume_event(
+        cls, context: Context, data: dict
+    ) -> HumanResponseEvent:
+        """
+        Get the HITL event from the context.
+        """
+        event_type_str = await context.get(cls.HITL_CONTEXT_KEY)
+        if not event_type_str:
+            raise ValueError(
+                "Cannot resume the workflow because there is no resume event type in the context"
+            )
+        resume_event_type = type_from_identifier(event_type_str)
+        if not issubclass(resume_event_type, HumanResponseEvent):
+            raise ValueError(
+                f"Cannot resume the workflow because the resume event type {resume_event_type} is not a HumanResponseEvent"
+            )
+        try:
+            return resume_event_type(**data)
+        except Exception as e:
+            raise ValueError(
+                f"Error constructing resume event: {e}. "
+                f"Make sure the provided data is valid for the event type {resume_event_type}"
+            )