feat: Add logger and callbacks to llm.exec (#2135)

.changeset/moody-otters-jump.md

@@ -0,0 +1,5 @@
+---
+"@llamaindex/core": patch
+---
+
+Add logger and callbacks to llm.exec

apps/next/src/content/docs/llamaindex/modules/agents/low-level.mdx

@@ -0,0 +1,164 @@
+---
+title: Low-Level LLM Execution
+---
+
+Sometimes your need more control over LLM interactions than what high-level agents provide. The `llm.exec` method makes it simple for you to make a single LLM call with tools but hides the complexity of executing the tools and generating the tool messages.
+
+## When to Use `llm.exec`
+
+Use `llm.exec` when you need to:
+- Build custom agent logic in [workflow](./workflows) steps
+- Have precise control over message handling and tool execution
+
+## Basic Usage
+
+The `llm.exec` method takes messages and tools as parameter and executes one LLM call.
+The LLM might either request to call one or more of the tools or generate an assistant message as result.
+For each tool call that is requested, `llm.exec` executes it and generates the two tool call messages (call and result). If no tool call is requested, just the assistant message is returned. 
+
+```ts
+import { openai } from "@llamaindex/openai";
+import { ChatMessage, tool } from "llamaindex";
+import z from "zod";
+
+const llm = openai({ model: "gpt-4.1-mini" });
+const messages = [
+  {
+    content: "What's the weather like in San Francisco?",
+    role: "user",
+  } as ChatMessage,
+];
+
+const { newMessages, toolCalls } = await llm.exec({
+  messages,
+  tools: [
+    tool({
+      name: "get_weather",
+      description: "Get the current weather for a location",
+      parameters: z.object({
+        address: z.string().describe("The address"),
+      }),
+      execute: ({ address }) => {
+        return `It's sunny in ${address}!`;
+      },
+    }),
+  ],
+});
+
+// Add the new messages (including tool calls and responses) to your conversation
+messages.push(...newMessages);
+```
+
+> `newMessages` is an array as each tool call generates two messages: a tool call message and the tool call result message.
+
+## Agent Loop Pattern
+
+A common pattern is to use `llm.exec` in a loop until the LLM stops making tool calls:
+
+```ts
+import { openai } from "@llamaindex/openai";
+import { ChatMessage, tool } from "llamaindex";
+import z from "zod";
+
+async function runAgentLoop() {
+  const llm = openai({ model: "gpt-4.1-mini" });
+  const messages = [
+    {
+      content: "What's the weather like in San Francisco?",
+      role: "user",
+    } as ChatMessage,
+  ];
+
+  let exit = false;
+  do {
+    const { newMessages, toolCalls } = await llm.exec({
+      messages,
+      tools: [
+        tool({
+          name: "get_weather",
+          description: "Get the current weather for a location",
+          parameters: z.object({
+            address: z.string().describe("The address"),
+          }),
+          execute: ({ address }) => {
+            return `It's sunny in ${address}!`;
+          },
+        }),
+      ],
+    });
+    
+    console.log(newMessages);
+    messages.push(...newMessages);
+    
+    // Exit when no more tool calls are made
+    exit = toolCalls.length === 0;
+  } while (!exit);
+}
+```
+
+## Streaming Support
+
+For real-time responses, use the `stream` option to get the assistant's response as streamed tokens:
+
+```ts
+import { openai } from "@llamaindex/openai";
+import { tool } from "llamaindex";
+import z from "zod";
+
+async function streamingAgentLoop() {
+  const llm = openai({ model: "gpt-4o-mini" });
+  const messages = [
+    {
+      content: "What's the weather like in San Francisco?",
+      role: "user",
+    } as ChatMessage,
+  ];
+
+  let exit = false;
+  do {
+    const { stream, newMessages, toolCalls } = await llm.exec({
+      messages,
+      tools: [
+        tool({
+          name: "get_weather",
+          description: "Get the current weather for a location",
+          parameters: z.object({
+            address: z.string().describe("The address"),
+          }),
+          execute: ({ address }) => {
+            return `It's sunny in ${address}!`;
+          },
+        }),
+      ],
+      stream: true,
+    });
+    
+    // Stream the response token by token
+    for await (const chunk of stream) {
+      process.stdout.write(chunk.delta);
+    }
+    
+    messages.push(...newMessages());
+    
+    exit = toolCalls.length === 0;
+  } while (!exit);
+}
+```
+
+> `newMessages` is a function when streaming. The reason is that the result only is available after streaming. Calling it before, will throw an error.
+
+## Return Values
+
+`llm.exec` returns an object with:
+
+- **`newMessages`**: Array of new chat messages including the LLM response and any tool call messages (call or result). This is a function return the array when streaming.
+- **`toolCalls`**: Array of tool calls made by the LLM
+- **`stream`**: Async iterable for streaming responses (only when `stream: true`)
+
+## Best Practices
+
+For using `llm.exec` in an agent loop, take care to:
+
+1. **Maintain message history**: Always add `newMessages` to your conversation history
+2. **Set exit conditions**: Implement proper logic to avoid infinite loops
+

apps/next/src/content/docs/llamaindex/modules/agents/meta.json

@@ -1,4 +1,10 @@
 {
   "title": "Agents",
-  "pages": ["tool", "agent_workflow", "workflows", "natural_language_workflow"]
+  "pages": [
+    "tool",
+    "agent_workflow",
+    "workflows",
+    "low-level",
+    "natural_language_workflow"
+  ]
 }

packages/core/src/llms/base.ts

@@ -1,6 +1,7 @@
+import { emptyLogger } from "@llamaindex/env";
 import { extractText } from "../utils/llms";
 import { streamConverter } from "../utils/stream";
-import { callTool, getToolCallsFromResponse } from "./tool-call";
+import { callToolToMessage, getToolCallsFromResponse } from "./tool-call";
 import type {
   ChatMessage,
   ChatResponse,
@@ -99,16 +100,19 @@ export abstract class BaseLLM<
     if (params.stream) {
       return this.streamExec(params);
     }
+    const logger = params.logger ?? emptyLogger;
     const newMessages: ChatMessage<AdditionalMessageOptions>[] = [];
     const response = await this.chat(params);
     newMessages.push(response.message);
     const toolCalls = getToolCallsFromResponse(response);
     if (params.tools && toolCalls.length > 0) {
       for (const toolCall of toolCalls) {
-        const toolResultMessage = await callTool<AdditionalMessageOptions>(
-          params.tools,
-          toolCall,
-        );
+        const toolResultMessage =
+          await callToolToMessage<AdditionalMessageOptions>(
+            params.tools,
+            toolCall,
+            logger,
+          );
         if (toolResultMessage) {
           newMessages.push(toolResultMessage);
         }
@@ -126,6 +130,7 @@ export abstract class BaseLLM<
       AdditionalMessageOptions
     >,
   ): Promise<ExecStreamResponse<AdditionalMessageOptions>> {
+    const logger = params.logger ?? emptyLogger;
     const responseStream = await this.chat(params);
     const iterator = responseStream[Symbol.asyncIterator]();
     const first = await iterator.next();
@@ -220,10 +225,12 @@ export abstract class BaseLLM<
         } as AdditionalMessageOptions,
       });
       for (const toolCall of toolCalls) {
-        const toolResultMessage = await callTool<AdditionalMessageOptions>(
-          params.tools,
-          toolCall,
-        );
+        const toolResultMessage =
+          await callToolToMessage<AdditionalMessageOptions>(
+            params.tools,
+            toolCall,
+            logger,
+          );
         if (toolResultMessage) {
           messages.push(toolResultMessage);
         }

packages/core/src/llms/tool-call.ts

@@ -1,3 +1,5 @@
+import { type Logger } from "@llamaindex/env";
+import { callTool } from "../agent/utils.js";
 import { stringifyJSONToMessageContent } from "../utils";
 import type {
   BaseTool,
@@ -35,27 +37,28 @@ export const getToolCallsFromResponse = (
   return [];
 };
 
-export const callTool = async <
+export const callToolToMessage = async <
   AdditionalMessageOptions extends object = object,
 >(
   tools: BaseTool[],
   toolCall: ToolCall,
+  logger: Logger,
 ): Promise<ChatMessage<AdditionalMessageOptions> | null> => {
   const tool = tools?.find((t) => t.metadata.name === toolCall.name);
-  // TODO: consider using BaseToolWithCall instead of BaseTool to avoid checking for tool.call
-  if (tool && tool.call) {
-    const result = await tool.call(toolCall.input);
-    const toolResultMessage: ChatMessage<AdditionalMessageOptions> = {
-      role: "user",
-      content: stringifyJSONToMessageContent(result),
-      options: {
-        toolResult: {
-          id: toolCall.id,
-          result,
-        },
-      } as AdditionalMessageOptions,
-    };
-    return toolResultMessage;
-  }
-  return null;
+
+  const toolOutput = await callTool(tool, toolCall, logger);
+
+  const toolResultMessage: ChatMessage<AdditionalMessageOptions> = {
+    role: "user",
+    content: stringifyJSONToMessageContent(toolOutput.output),
+    options: {
+      toolResult: {
+        id: toolCall.id,
+        result: toolOutput.output,
+        isError: toolOutput.isError,
+      },
+    } as AdditionalMessageOptions,
+  };
+
+  return toolResultMessage;
 };

packages/core/src/llms/type.ts

@@ -1,3 +1,4 @@
+import type { Logger } from "@llamaindex/env";
 import type { Tokenizers } from "@llamaindex/env/tokenizers";
 import type { JSONSchemaType } from "ajv";
 import { z } from "zod";
@@ -139,6 +140,7 @@ export interface LLMChatParamsBase<
   additionalChatOptions?: AdditionalChatOptions | undefined;
   tools?: BaseTool[] | undefined;
   responseFormat?: z.ZodType | object | undefined;
+  logger?: Logger | undefined;
 }
 
 export interface LLMChatParamsStreaming<