From 517881afa6e3d52eacf681bc3086f4b4cd81d6dc Mon Sep 17 00:00:00 2001 From: "David S. Batista" Date: Wed, 7 Oct 2026 17:19:42 +0100 Subject: [PATCH] feat: collecting agents tool in telemetry events (#13125) --- docs-website/docs/overview/telemetry.mdx | 2 + haystack/components/agents/agent.py | 47 +++++++++++++- haystack/core/pipeline/pipeline.py | 10 +-- ...gent-telemetry-tools-0ebbb696b62460de.yaml | 6 ++ test/components/agents/test_agent.py | 46 ++++++++++++- test/test_telemetry.py | 65 +++++++++++++++++++ 6 files changed, 170 insertions(+), 6 deletions(-) create mode 100644 releasenotes/notes/agent-telemetry-tools-0ebbb696b62460de.yaml diff --git a/docs-website/docs/overview/telemetry.mdx b/docs-website/docs/overview/telemetry.mdx index 206d12fa73..5fcc104ae5 100644 --- a/docs-website/docs/overview/telemetry.mdx +++ b/docs-website/docs/overview/telemetry.mdx @@ -13,6 +13,8 @@ Haystack relies on anonymous usage statistics to continuously improve. That's wh Telemetry in Haystack comprises anonymous usage statistics of base components, such as `DocumentStore`, `Retriever`, `Reader`, or any other pipeline component. We receive an event every time these components are initialized. This way, we know which components are most relevant to our community. For the same reason, an event is also sent when one of the tutorials is executed. +When a pipeline contains an `Agent`, the event also includes the Agent's chat generator type and model, and the type and name of each of its tools. Tool descriptions are never collected. + Each event contains an anonymous, randomly generated user ID (`uuid`) and a collection of properties about your execution environment. They **never** contain properties that can be used to identify you, such as: - IP addresses diff --git a/haystack/components/agents/agent.py b/haystack/components/agents/agent.py index 7335fb979d..1edd356844 100644 --- a/haystack/components/agents/agent.py +++ b/haystack/components/agents/agent.py @@ -3,6 +3,7 @@ # SPDX-License-Identifier: Apache-2.0 import inspect +from collections import Counter from dataclasses import dataclass from typing import Any, Literal, cast @@ -28,7 +29,12 @@ from haystack.components.agents.utils import ( ) from haystack.components.builders import ChatPromptBuilder from haystack.components.generators.chat.types import ChatGenerator -from haystack.core.serialization import component_to_dict, default_from_dict, default_to_dict +from haystack.core.serialization import ( + component_to_dict, + default_from_dict, + default_to_dict, + generate_qualified_class_name, +) from haystack.dataclasses import ChatMessage, ChatRole, StreamingCallbackT, select_streaming_callback from haystack.hooks.invocation import _run_hooks, _run_hooks_async from haystack.hooks.protocol import ( @@ -51,6 +57,8 @@ from haystack.hooks.utils import ( warm_up_hooks_async, ) from haystack.tools import ( + ComponentTool, + Tool, Toolset, ToolsType, _check_duplicate_tool_names, @@ -202,6 +210,34 @@ def _pending_tool_call_messages_from_state(state: State) -> list[ChatMessage]: return [last_message] if last_message.tool_calls else [] +def _get_tool_telemetry_data(tool: Tool) -> dict[str, Any]: + """ + Describe a single tool for telemetry. + + Descriptions are deliberately left out: they are free text that can hold personal or confidential data. + """ + data = {"type": generate_qualified_class_name(type(tool)), "name": tool.name} + if isinstance(tool, ComponentTool): + data["component"] = generate_qualified_class_name(type(tool._component)) + return data + + +def _get_tools_telemetry_data(tools: ToolsType) -> dict[str, Any]: + """ + Summarize the tools for telemetry. + + Call it only after the tools are warmed up: flattening iterates the Toolsets, and some of them (e.g. + SearchableToolset) warm up on iteration, while others (e.g. MCPToolset) only load their tools in `warm_up()`. + """ + toolsets = [tools] if isinstance(tools, Toolset) else [entry for entry in tools if isinstance(entry, Toolset)] + flat_tools = flatten_tools_or_toolsets(tools=tools) + return { + "count": len(flat_tools), + "tools": [_get_tool_telemetry_data(tool) for tool in flat_tools], + "toolset_types": dict(Counter(generate_qualified_class_name(type(toolset)) for toolset in toolsets)), + } + + @dataclass(kw_only=True) class _ExecutionContext: """ @@ -638,6 +674,15 @@ class Agent: params: dict[str, Any] = {name: getattr(self, name) for name in init_params if name != "self"} return type(self)(**{**params, **overrides}) + def _get_telemetry_data(self) -> dict[str, Any]: + """ + Data that is sent to Posthog for usage analytics. + """ + chat_generator_data: dict[str, Any] = {"type": generate_qualified_class_name(type(self.chat_generator))} + if hasattr(self.chat_generator, "_get_telemetry_data"): + chat_generator_data.update(self.chat_generator._get_telemetry_data()) + return {"chat_generator": chat_generator_data, "tools": _get_tools_telemetry_data(self.tools)} + def to_dict(self) -> dict[str, Any]: """ Serialize the component to a dictionary. diff --git a/haystack/core/pipeline/pipeline.py b/haystack/core/pipeline/pipeline.py index 26a79f65af..aaa399b0ef 100644 --- a/haystack/core/pipeline/pipeline.py +++ b/haystack/core/pipeline/pipeline.py @@ -325,8 +325,6 @@ class Pipeline(PipelineBase): :raises PipelineBreakpointException: When a pipeline_breakpoint is triggered. Contains the component name, state, and partial results. """ - pipeline_running(self) # telemetry - if ( break_point and pipeline_snapshot @@ -347,6 +345,9 @@ class Pipeline(PipelineBase): # warm up the pipeline by running each component's warm_up method self.warm_up() + # after warm-up, so toolsets that load their tools in warm_up() report them + pipeline_running(self) # telemetry + if include_outputs_from is None: include_outputs_from = set() @@ -900,11 +901,12 @@ class Pipeline(PipelineBase): if concurrency_limit < 1: raise ValueError("concurrency_limit must be greater than or equal to 1.") - pipeline_running(self) # telemetry - # warm up the pipeline by running each component's warm_up_async (or warm_up) method await self.warm_up_async() + # after warm-up, so toolsets that load their tools in warm_up() report them + pipeline_running(self) # telemetry + if include_outputs_from is None: include_outputs_from = set() diff --git a/releasenotes/notes/agent-telemetry-tools-0ebbb696b62460de.yaml b/releasenotes/notes/agent-telemetry-tools-0ebbb696b62460de.yaml new file mode 100644 index 0000000000..a30d5b5d98 --- /dev/null +++ b/releasenotes/notes/agent-telemetry-tools-0ebbb696b62460de.yaml @@ -0,0 +1,6 @@ +--- +enhancements: + - | + When an ``Agent`` runs inside a ``Pipeline``, the anonymous pipeline telemetry event now also reports the + chat generator class and its tools: the number of tools, the class and name of each tool, the component + wrapped by each ``ComponentTool``, and the classes of any ``Toolset`` objects. diff --git a/test/components/agents/test_agent.py b/test/components/agents/test_agent.py index 9d43934d16..d1c82f7418 100644 --- a/test/components/agents/test_agent.py +++ b/test/components/agents/test_agent.py @@ -34,7 +34,7 @@ from haystack.dataclasses.chat_message import ChatRole, TextContent from haystack.dataclasses.streaming_chunk import StreamingChunk from haystack.document_stores.in_memory import InMemoryDocumentStore from haystack.hooks import hook -from haystack.tools import ComponentTool, Tool, warm_up_tools, warm_up_tools_async +from haystack.tools import ComponentTool, SearchableToolset, Tool, warm_up_tools, warm_up_tools_async from haystack.tools.toolset import Toolset from haystack.utils import Secret @@ -620,6 +620,50 @@ class TestAgentClone: assert clone.state_schema == {"foo": {"type": str}, "notes": {"type": str}} +class TestAgentTelemetry: + def test_get_telemetry_data(self, weather_tool, component_tool): + chat_generator = OpenAIChatGenerator(api_key=Secret.from_token("test-api-key")) + agent = Agent(chat_generator=chat_generator, tools=[weather_tool, Toolset([component_tool])]) + + assert agent._get_telemetry_data() == { + "chat_generator": { + "type": "haystack.components.generators.chat.openai.OpenAIChatGenerator", + "model": chat_generator.model, + }, + "tools": { + "count": 2, + "tools": [ + {"type": "haystack.tools.tool.Tool", "name": "weather_tool"}, + { + "type": "haystack.tools.component_tool.ComponentTool", + "name": "parrot", + "component": "haystack.components.builders.prompt_builder.PromptBuilder", + }, + ], + "toolset_types": {"haystack.tools.toolset.Toolset": 1}, + }, + } + + def test_get_telemetry_data_without_tools(self): + agent = Agent(chat_generator=MockChatGenerator("Hello")) + + assert agent._get_telemetry_data() == { + "chat_generator": {"type": "haystack.components.generators.chat.mock.MockChatGenerator"}, + "tools": {"count": 0, "tools": [], "toolset_types": {}}, + } + + def test_get_telemetry_data_reports_searchable_toolset_catalog(self, weather_tool): + # below the search threshold a SearchableToolset is a passthrough exposing its whole catalog + agent = Agent(chat_generator=MockChatGenerator("Hello"), tools=SearchableToolset(catalog=[weather_tool])) + agent.warm_up() + + assert agent._get_telemetry_data()["tools"] == { + "count": 1, + "tools": [{"type": "haystack.tools.tool.Tool", "name": "weather_tool"}], + "toolset_types": {"haystack.tools.searchable_toolset.SearchableToolset": 1}, + } + + class TestGetModelExitReason: @pytest.mark.parametrize( ("message", "expected"), diff --git a/test/test_telemetry.py b/test/test_telemetry.py index 56e7b5f12f..82b1f75993 100644 --- a/test/test_telemetry.py +++ b/test/test_telemetry.py @@ -7,8 +7,12 @@ import logging from unittest.mock import Mock, patch from haystack import Pipeline, component +from haystack.components.agents import Agent +from haystack.components.generators.chat import MockChatGenerator from haystack.core.serialization import generate_qualified_class_name +from haystack.dataclasses import ChatMessage from haystack.telemetry._telemetry import pipeline_running, tutorial_running +from haystack.tools import Tool, Toolset from haystack.utils.auth import Secret, TokenSecret @@ -109,6 +113,67 @@ def test_pipeline_running_with_non_serializable_component(telemetry): ) +@patch("haystack.telemetry._telemetry.telemetry") +def test_pipeline_running_reports_agent_tools(telemetry): + telemetry.send_event = Mock() + tool = Tool(name="tool", description="A tool.", parameters={"type": "object", "properties": {}}, function=print) + + pipe = Pipeline() + pipe.add_component("agent", Agent(chat_generator=MockChatGenerator("Hello"), tools=[tool])) + pipeline_running(pipe) + + event_properties = telemetry.send_event.call_args.args[1] + assert event_properties["components"]["haystack.components.agents.agent.Agent"] == [ + { + "name": "agent", + "chat_generator": {"type": "haystack.components.generators.chat.mock.MockChatGenerator"}, + "tools": {"count": 1, "tools": [{"type": "haystack.tools.tool.Tool", "name": "tool"}], "toolset_types": {}}, + } + ] + + +class _ToolsetLoadingOnWarmUp(Toolset): + def __init__(self, tool: Tool) -> None: + super().__init__(tools=[]) + self._tool = tool + + def warm_up(self) -> None: + self.tools = [self._tool] + + +def _pipeline_with_agent_using_toolset_loading_on_warm_up() -> Pipeline: + tool = Tool(name="tool", description="A tool.", parameters={"type": "object", "properties": {}}, function=print) + pipe = Pipeline() + pipe.add_component( + "agent", Agent(chat_generator=MockChatGenerator("Hello"), tools=_ToolsetLoadingOnWarmUp(tool=tool)) + ) + return pipe + + +def _reported_agent_tools(capture: Mock) -> dict: + return capture.call_args.kwargs["properties"]["components"]["haystack.components.agents.agent.Agent"][0]["tools"] + + +def test_pipeline_run_reports_tools_loaded_on_warm_up(block_telemetry_network_calls: Mock) -> None: + pipe = _pipeline_with_agent_using_toolset_loading_on_warm_up() + + pipe.run({"agent": {"messages": [ChatMessage.from_user("Hi")]}}) + + assert _reported_agent_tools(block_telemetry_network_calls)["tools"] == [ + {"type": "haystack.tools.tool.Tool", "name": "tool"} + ] + + +async def test_pipeline_run_async_reports_tools_loaded_on_warm_up(block_telemetry_network_calls: Mock) -> None: + pipe = _pipeline_with_agent_using_toolset_loading_on_warm_up() + + await pipe.run_async({"agent": {"messages": [ChatMessage.from_user("Hi")]}}) + + assert _reported_agent_tools(block_telemetry_network_calls)["tools"] == [ + {"type": "haystack.tools.tool.Tool", "name": "tool"} + ] + + def test_pipeline_running_with_non_dict_telemetry_data(caplog): @component class Component: