feat: update agent input interface to support files (#1938)

Co-authored-by: Marcus Schiesser <marcus.schiesser@googlemail.com>

.changeset/silver-sloths-tickle.md

@@ -0,0 +1,5 @@
+---
+"@llamaindex/workflow": patch
+---
+
+feat: update agent input interface to support any message content

packages/workflow/src/agent/agent-workflow.ts

@@ -44,7 +44,7 @@ const DEFAULT_HANDOFF_OUTPUT_PROMPT = new PromptTemplate({
 });
 
 export type AgentInputData = {
-  userInput?: string | undefined;
+  userInput?: MessageContent | undefined;
   chatHistory?: ChatMessage[] | undefined;
 };
 export const startAgentEvent = workflowEvent<
@@ -582,7 +582,7 @@ export class AgentWorkflow implements Workflow {
   }
 
   runStream(
-    userInput: string,
+    userInput: MessageContent,
     params?: {
       chatHistory?: ChatMessage[];
       state?: AgentWorkflowState;
@@ -604,7 +604,7 @@ export class AgentWorkflow implements Workflow {
   }
 
   async run(
-    userInput: string,
+    userInput: MessageContent,
     params?: {
       chatHistory?: ChatMessage[];
       state?: AgentWorkflowState;

packages/workflow/test/agent-workflow.test.ts

@@ -202,6 +202,46 @@ describe("AgentWorkflow", () => {
   });
 });
 
+describe("agent", () => {
+  test("agent run method handles file input correctly", async () => {
+    const pdfBuffer = Buffer.from("test PDF content");
+    const llm = new MockLLM();
+    llm.supportToolCall = true;
+    const addTool = FunctionTool.from(
+      (params: { x: number; y: number }) => params.x + params.y,
+      {
+        name: "add",
+        description: "Adds two numbers",
+        parameters: z.object({
+          x: z.number(),
+          y: z.number(),
+        }),
+      },
+    );
+    const testAgent = agent({
+      name: "TestAgent",
+      description: "Test agent",
+      tools: [addTool],
+      llm: llm,
+    });
+
+    const response = await testAgent.run([
+      {
+        type: "file",
+        data: pdfBuffer,
+        mimeType: "application/pdf",
+      },
+    ]);
+
+    const messages = response.data.state!.memory.getAllMessages();
+    const fileMessage = messages[0].content[0];
+
+    expect(fileMessage.type).toEqual("file");
+    expect(fileMessage.mimeType).toEqual("application/pdf");
+    expect(fileMessage.data).toEqual(pdfBuffer);
+  });
+});
+
 describe("Multiple agents", () => {
   test("multiple agents are set up correctly with handoff capabilities", () => {
     // Create mock LLM