feat: collecting agents tool in telemetry events (#13125)

docs-website/docs/overview/telemetry.mdx

@@ -13,6 +13,8 @@ Haystack relies on anonymous usage statistics to continuously improve. That's wh
 
 Telemetry in Haystack comprises anonymous usage statistics of base components, such as `DocumentStore`, `Retriever`, `Reader`, or any other pipeline component. We receive an event every time these components are initialized. This way, we know which components are most relevant to our community. For the same reason, an event is also sent when one of the tutorials is executed.
 
+When a pipeline contains an `Agent`, the event also includes the Agent's chat generator type and model, and the type and name of each of its tools. Tool descriptions are never collected.
+
 Each event contains an anonymous, randomly generated user ID (`uuid`)  and a collection of properties about your execution environment. They **never** contain properties that can be used to identify you, such as:
 
 - IP addresses

haystack/components/agents/agent.py

@@ -3,6 +3,7 @@
 # SPDX-License-Identifier: Apache-2.0
 
 import inspect
+from collections import Counter
 from dataclasses import dataclass
 from typing import Any, Literal, cast
 
@@ -28,7 +29,12 @@ from haystack.components.agents.utils import (
 )
 from haystack.components.builders import ChatPromptBuilder
 from haystack.components.generators.chat.types import ChatGenerator
-from haystack.core.serialization import component_to_dict, default_from_dict, default_to_dict
+from haystack.core.serialization import (
+    component_to_dict,
+    default_from_dict,
+    default_to_dict,
+    generate_qualified_class_name,
+)
 from haystack.dataclasses import ChatMessage, ChatRole, StreamingCallbackT, select_streaming_callback
 from haystack.hooks.invocation import _run_hooks, _run_hooks_async
 from haystack.hooks.protocol import (
@@ -51,6 +57,8 @@ from haystack.hooks.utils import (
     warm_up_hooks_async,
 )
 from haystack.tools import (
+    ComponentTool,
+    Tool,
     Toolset,
     ToolsType,
     _check_duplicate_tool_names,
@@ -202,6 +210,34 @@ def _pending_tool_call_messages_from_state(state: State) -> list[ChatMessage]:
     return [last_message] if last_message.tool_calls else []
 
 
+def _get_tool_telemetry_data(tool: Tool) -> dict[str, Any]:
+    """
+    Describe a single tool for telemetry.
+
+    Descriptions are deliberately left out: they are free text that can hold personal or confidential data.
+    """
+    data = {"type": generate_qualified_class_name(type(tool)), "name": tool.name}
+    if isinstance(tool, ComponentTool):
+        data["component"] = generate_qualified_class_name(type(tool._component))
+    return data
+
+
+def _get_tools_telemetry_data(tools: ToolsType) -> dict[str, Any]:
+    """
+    Summarize the tools for telemetry.
+
+    Call it only after the tools are warmed up: flattening iterates the Toolsets, and some of them (e.g.
+    SearchableToolset) warm up on iteration, while others (e.g. MCPToolset) only load their tools in `warm_up()`.
+    """
+    toolsets = [tools] if isinstance(tools, Toolset) else [entry for entry in tools if isinstance(entry, Toolset)]
+    flat_tools = flatten_tools_or_toolsets(tools=tools)
+    return {
+        "count": len(flat_tools),
+        "tools": [_get_tool_telemetry_data(tool) for tool in flat_tools],
+        "toolset_types": dict(Counter(generate_qualified_class_name(type(toolset)) for toolset in toolsets)),
+    }
+
+
 @dataclass(kw_only=True)
 class _ExecutionContext:
     """
@@ -638,6 +674,15 @@ class Agent:
         params: dict[str, Any] = {name: getattr(self, name) for name in init_params if name != "self"}
         return type(self)(**{**params, **overrides})
 
+    def _get_telemetry_data(self) -> dict[str, Any]:
+        """
+        Data that is sent to Posthog for usage analytics.
+        """
+        chat_generator_data: dict[str, Any] = {"type": generate_qualified_class_name(type(self.chat_generator))}
+        if hasattr(self.chat_generator, "_get_telemetry_data"):
+            chat_generator_data.update(self.chat_generator._get_telemetry_data())
+        return {"chat_generator": chat_generator_data, "tools": _get_tools_telemetry_data(self.tools)}
+
     def to_dict(self) -> dict[str, Any]:
         """
         Serialize the component to a dictionary.

haystack/core/pipeline/pipeline.py

@@ -325,8 +325,6 @@ class Pipeline(PipelineBase):
         :raises PipelineBreakpointException:
             When a pipeline_breakpoint is triggered. Contains the component name, state, and partial results.
         """
-        pipeline_running(self)  # telemetry
-
         if (
             break_point
             and pipeline_snapshot
@@ -347,6 +345,9 @@ class Pipeline(PipelineBase):
         # warm up the pipeline by running each component's warm_up method
         self.warm_up()
 
+        # after warm-up, so toolsets that load their tools in warm_up() report them
+        pipeline_running(self)  # telemetry
+
         if include_outputs_from is None:
             include_outputs_from = set()
 
@@ -900,11 +901,12 @@ class Pipeline(PipelineBase):
         if concurrency_limit < 1:
             raise ValueError("concurrency_limit must be greater than or equal to 1.")
 
-        pipeline_running(self)  # telemetry
-
         # warm up the pipeline by running each component's warm_up_async (or warm_up) method
         await self.warm_up_async()
 
+        # after warm-up, so toolsets that load their tools in warm_up() report them
+        pipeline_running(self)  # telemetry
+
         if include_outputs_from is None:
             include_outputs_from = set()
 

releasenotes/notes/agent-telemetry-tools-0ebbb696b62460de.yaml

@@ -0,0 +1,6 @@
+---
+enhancements:
+  - |
+    When an ``Agent`` runs inside a ``Pipeline``, the anonymous pipeline telemetry event now also reports the
+    chat generator class and its tools: the number of tools, the class and name of each tool, the component
+    wrapped by each ``ComponentTool``, and the classes of any ``Toolset`` objects.

test/components/agents/test_agent.py

@@ -34,7 +34,7 @@ from haystack.dataclasses.chat_message import ChatRole, TextContent
 from haystack.dataclasses.streaming_chunk import StreamingChunk
 from haystack.document_stores.in_memory import InMemoryDocumentStore
 from haystack.hooks import hook
-from haystack.tools import ComponentTool, Tool, warm_up_tools, warm_up_tools_async
+from haystack.tools import ComponentTool, SearchableToolset, Tool, warm_up_tools, warm_up_tools_async
 from haystack.tools.toolset import Toolset
 from haystack.utils import Secret
 
@@ -620,6 +620,50 @@ class TestAgentClone:
         assert clone.state_schema == {"foo": {"type": str}, "notes": {"type": str}}
 
 
+class TestAgentTelemetry:
+    def test_get_telemetry_data(self, weather_tool, component_tool):
+        chat_generator = OpenAIChatGenerator(api_key=Secret.from_token("test-api-key"))
+        agent = Agent(chat_generator=chat_generator, tools=[weather_tool, Toolset([component_tool])])
+
+        assert agent._get_telemetry_data() == {
+            "chat_generator": {
+                "type": "haystack.components.generators.chat.openai.OpenAIChatGenerator",
+                "model": chat_generator.model,
+            },
+            "tools": {
+                "count": 2,
+                "tools": [
+                    {"type": "haystack.tools.tool.Tool", "name": "weather_tool"},
+                    {
+                        "type": "haystack.tools.component_tool.ComponentTool",
+                        "name": "parrot",
+                        "component": "haystack.components.builders.prompt_builder.PromptBuilder",
+                    },
+                ],
+                "toolset_types": {"haystack.tools.toolset.Toolset": 1},
+            },
+        }
+
+    def test_get_telemetry_data_without_tools(self):
+        agent = Agent(chat_generator=MockChatGenerator("Hello"))
+
+        assert agent._get_telemetry_data() == {
+            "chat_generator": {"type": "haystack.components.generators.chat.mock.MockChatGenerator"},
+            "tools": {"count": 0, "tools": [], "toolset_types": {}},
+        }
+
+    def test_get_telemetry_data_reports_searchable_toolset_catalog(self, weather_tool):
+        # below the search threshold a SearchableToolset is a passthrough exposing its whole catalog
+        agent = Agent(chat_generator=MockChatGenerator("Hello"), tools=SearchableToolset(catalog=[weather_tool]))
+        agent.warm_up()
+
+        assert agent._get_telemetry_data()["tools"] == {
+            "count": 1,
+            "tools": [{"type": "haystack.tools.tool.Tool", "name": "weather_tool"}],
+            "toolset_types": {"haystack.tools.searchable_toolset.SearchableToolset": 1},
+        }
+
+
 class TestGetModelExitReason:
     @pytest.mark.parametrize(
         ("message", "expected"),

test/test_telemetry.py

@@ -7,8 +7,12 @@ import logging
 from unittest.mock import Mock, patch
 
 from haystack import Pipeline, component
+from haystack.components.agents import Agent
+from haystack.components.generators.chat import MockChatGenerator
 from haystack.core.serialization import generate_qualified_class_name
+from haystack.dataclasses import ChatMessage
 from haystack.telemetry._telemetry import pipeline_running, tutorial_running
+from haystack.tools import Tool, Toolset
 from haystack.utils.auth import Secret, TokenSecret
 
 
@@ -109,6 +113,67 @@ def test_pipeline_running_with_non_serializable_component(telemetry):
     )
 
 
+@patch("haystack.telemetry._telemetry.telemetry")
+def test_pipeline_running_reports_agent_tools(telemetry):
+    telemetry.send_event = Mock()
+    tool = Tool(name="tool", description="A tool.", parameters={"type": "object", "properties": {}}, function=print)
+
+    pipe = Pipeline()
+    pipe.add_component("agent", Agent(chat_generator=MockChatGenerator("Hello"), tools=[tool]))
+    pipeline_running(pipe)
+
+    event_properties = telemetry.send_event.call_args.args[1]
+    assert event_properties["components"]["haystack.components.agents.agent.Agent"] == [
+        {
+            "name": "agent",
+            "chat_generator": {"type": "haystack.components.generators.chat.mock.MockChatGenerator"},
+            "tools": {"count": 1, "tools": [{"type": "haystack.tools.tool.Tool", "name": "tool"}], "toolset_types": {}},
+        }
+    ]
+
+
+class _ToolsetLoadingOnWarmUp(Toolset):
+    def __init__(self, tool: Tool) -> None:
+        super().__init__(tools=[])
+        self._tool = tool
+
+    def warm_up(self) -> None:
+        self.tools = [self._tool]
+
+
+def _pipeline_with_agent_using_toolset_loading_on_warm_up() -> Pipeline:
+    tool = Tool(name="tool", description="A tool.", parameters={"type": "object", "properties": {}}, function=print)
+    pipe = Pipeline()
+    pipe.add_component(
+        "agent", Agent(chat_generator=MockChatGenerator("Hello"), tools=_ToolsetLoadingOnWarmUp(tool=tool))
+    )
+    return pipe
+
+
+def _reported_agent_tools(capture: Mock) -> dict:
+    return capture.call_args.kwargs["properties"]["components"]["haystack.components.agents.agent.Agent"][0]["tools"]
+
+
+def test_pipeline_run_reports_tools_loaded_on_warm_up(block_telemetry_network_calls: Mock) -> None:
+    pipe = _pipeline_with_agent_using_toolset_loading_on_warm_up()
+
+    pipe.run({"agent": {"messages": [ChatMessage.from_user("Hi")]}})
+
+    assert _reported_agent_tools(block_telemetry_network_calls)["tools"] == [
+        {"type": "haystack.tools.tool.Tool", "name": "tool"}
+    ]
+
+
+async def test_pipeline_run_async_reports_tools_loaded_on_warm_up(block_telemetry_network_calls: Mock) -> None:
+    pipe = _pipeline_with_agent_using_toolset_loading_on_warm_up()
+
+    await pipe.run_async({"agent": {"messages": [ChatMessage.from_user("Hi")]}})
+
+    assert _reported_agent_tools(block_telemetry_network_calls)["tools"] == [
+        {"type": "haystack.tools.tool.Tool", "name": "tool"}
+    ]
+
+
 def test_pipeline_running_with_non_dict_telemetry_data(caplog):
     @component
     class Component: