fix(agents): request the forced final answer as a user turn (#7450)

* fix(agents): request the forced final answer as a user turn When an agent reaches max_iter, handle_max_iterations_exceeded appended the "give your best final answer" instruction as an assistant message and relied on assistant prefill to make the model continue it. Current Claude models (Opus 5, Sonnet 5, Fable 5.x, the 4.6+ family) reject a request that ends on an assistant turn with a 400, after the whole iteration budget has already been spent. The instruction is now appended as a user turn, which every provider accepts. The handler's formatted_answer parameter is dropped: every caller had already appended that text as the last assistant message, so prefixing it again only duplicated history. Co-Authored-By: Claude Fable 5.1 <[email protected]> * fix(agents): stop the lite agent loop after the forced final answer LiteAgent._invoke_loop fell through after handle_max_iterations_exceeded and issued a regular LLM call on the same history, discarding the forced answer. Break out of the loop the way CrewAgentExecutor already does, and assert a single LLM call in the test. Co-Authored-By: Claude Fable 5.1 <[email protected]> --------- Co-authored-by: Claude Fable 5.1 <[email protected]> Co-authored-by: Vidit Ostwal <[email protected]>
2026-10-12 02:12:31 +00:00 · 2026-09-14 11:52:08 +05:30
parent 21678f8ac6
commit 9393a47f31
10 changed files with 450 additions and 18 deletions
@@ -364,7 +364,6 @@ class CrewAgentExecutor(BaseAgentExecutor):
            try:
                if has_reached_max_iterations(self.iterations, self.max_iter):
                    formatted_answer = handle_max_iterations_exceeded(
-                        formatted_answer,
                        printer=PRINTER,
                        messages=self.messages,
                        llm=cast("BaseLLM", self.llm),
@@ -524,7 +523,6 @@ class CrewAgentExecutor(BaseAgentExecutor):
            try:
                if has_reached_max_iterations(self.iterations, self.max_iter):
                    formatted_answer = handle_max_iterations_exceeded(
-                        None,
                        printer=PRINTER,
                        messages=self.messages,
                        llm=cast("BaseLLM", self.llm),
@@ -1178,7 +1176,6 @@ class CrewAgentExecutor(BaseAgentExecutor):
            try:
                if has_reached_max_iterations(self.iterations, self.max_iter):
                    formatted_answer = handle_max_iterations_exceeded(
-                        formatted_answer,
                        printer=PRINTER,
                        messages=self.messages,
                        llm=cast("BaseLLM", self.llm),
@@ -1324,7 +1321,6 @@ class CrewAgentExecutor(BaseAgentExecutor):
            try:
                if has_reached_max_iterations(self.iterations, self.max_iter):
                    formatted_answer = handle_max_iterations_exceeded(
-                        None,
                        printer=PRINTER,
                        messages=self.messages,
                        llm=cast("BaseLLM", self.llm),
@@ -1439,7 +1439,6 @@ class AgentExecutor(Flow[AgentExecutorState], BaseAgentExecutor):
            return "agent_finished"

        formatted_answer = handle_max_iterations_exceeded(
-            formatted_answer=None,
            printer=PRINTER,
            messages=list(self.state.messages),
            llm=self.llm,
@@ -929,13 +929,13 @@ class LiteAgent(FlowTrackable, BaseModel):
            try:
                if has_reached_max_iterations(self._iterations, self.max_iterations):
                    formatted_answer = handle_max_iterations_exceeded(
-                        formatted_answer,
                        printer=PRINTER,
                        messages=self._messages,
                        llm=cast(LLM, self.llm),
                        callbacks=self._callbacks,
                        verbose=self.verbose,
                    )
+                    break

                enforce_rpm_limit(self.request_within_rpm_limit)

@@ -374,7 +374,6 @@ def has_reached_max_iterations(iterations: int, max_iterations: int) -> bool:


 def handle_max_iterations_exceeded(
-    formatted_answer: AgentAction | AgentFinish | None,
    printer: Printer,
    messages: list[LLMMessage],
    llm: LLM | BaseLLM,
@@ -384,9 +383,9 @@ def handle_max_iterations_exceeded(
    """Handles the case when the maximum number of iterations is exceeded. Performs one more LLM call to get the final answer.

    Args:
-        formatted_answer: The last formatted answer from the agent.
        printer: Printer instance for output.
-        messages: List of messages to send to the LLM.
+        messages: Conversation so far; the forced-answer instruction is appended
+            to it as a user turn.
        llm: The LLM instance to call.
        callbacks: List of callbacks for the LLM call.
        verbose: Whether to print output.
@@ -400,14 +399,11 @@ def handle_max_iterations_exceeded(
            color="yellow",
        )

-    if formatted_answer and hasattr(formatted_answer, "text"):
-        assistant_message = (
-            formatted_answer.text + f"\n{I18N_DEFAULT.errors('force_final_answer')}"
-        )
-    else:
-        assistant_message = I18N_DEFAULT.errors("force_final_answer")
-
-    messages.append(format_message_for_llm(assistant_message, role="assistant"))
+    # A trailing assistant turn is a prefill request, which current Claude
+    # models reject with a 400; every provider accepts a trailing user turn.
+    messages.append(
+        format_message_for_llm(I18N_DEFAULT.errors("force_final_answer"), role="user")
+    )

    answer = llm.call(
        messages,
@@ -2889,3 +2889,101 @@ class TestSharedLLMStopWords:

        assert seen == [{"Original:", "Observation:"}]
        assert shared.stop == ["Original:"]
+
+
+class TestMaxIterationsForcedAnswer:
+    """Both sync loops request the forced final answer with a trailing user turn.
+
+    Current Claude models reject a request that ends on an assistant message,
+    so the nudge must never be sent as assistant prefill.
+    """
+
+    @staticmethod
+    def _make_executor(llm: MagicMock, original_tools: list) -> CrewAgentExecutor:
+        from crewai.agents.tools_handler import ToolsHandler
+
+        agent = Agent(role="r", goal="g", backstory="b", llm=llm, verbose=False)
+        task = Task(description="d", expected_output="o", agent=agent)
+        executor = CrewAgentExecutor(
+            agent=agent,
+            task=task,
+            llm=llm,
+            crew=None,
+            prompt={"prompt": "p {input} {tool_names} {tools}"},
+            max_iter=1,
+            tools=[],
+            original_tools=original_tools,
+            tools_names="",
+            stop_words=[],
+            tools_description="",
+            tools_handler=ToolsHandler(),
+        )
+        executor.iterations = 1
+        return executor
+
+    def test_react_loop_forces_final_answer_with_user_turn(self) -> None:
+        from crewai.utilities.i18n import I18N_DEFAULT
+
+        llm = MagicMock(spec=LLM)
+        llm.stop = []
+        llm.supports_stop_words.return_value = True
+        llm.supports_function_calling.return_value = False
+        llm.call.return_value = "Final Answer: forced"
+        executor = self._make_executor(llm, original_tools=[])
+        executor.messages = [
+            {"role": "user", "content": "Collect all the data."},
+            {"role": "assistant", "content": "Thought: I need data\nObservation: partial"},
+        ]
+
+        with patch.object(executor, "_show_logs"):
+            result = executor._invoke_loop()
+
+        sent = llm.call.call_args.args[0]
+        assert sent[-1] == {
+            "role": "user",
+            "content": I18N_DEFAULT.errors("force_final_answer"),
+        }
+        assert isinstance(result, AgentFinish)
+        assert result.output == "forced"
+
+    def test_native_tools_loop_forces_final_answer_with_user_turn(self) -> None:
+        from crewai.utilities.i18n import I18N_DEFAULT
+
+        @tool
+        def get_data(step: str) -> str:
+            """Get data for a step."""
+            return f"data for {step}"
+
+        llm = MagicMock(spec=LLM)
+        llm.stop = []
+        llm.supports_stop_words.return_value = True
+        llm.supports_function_calling.return_value = True
+        llm.call.return_value = "Final Answer: forced"
+        executor = self._make_executor(llm, original_tools=[get_data])
+        executor.messages = [
+            {"role": "user", "content": "Collect all the data."},
+            {
+                "role": "assistant",
+                "content": None,
+                "tool_calls": [
+                    {
+                        "id": "call_1",
+                        "type": "function",
+                        "function": {"name": "get_data", "arguments": "{}"},
+                    }
+                ],
+            },
+            {"role": "tool", "tool_call_id": "call_1", "name": "get_data", "content": "partial"},
+            {"role": "user", "content": I18N_DEFAULT.slice("post_tool_reasoning")},
+        ]
+
+        with patch.object(executor, "_show_logs"):
+            result = executor._invoke_loop()
+
+        sent = llm.call.call_args.args[0]
+        assert sent[-1] == {
+            "role": "user",
+            "content": I18N_DEFAULT.errors("force_final_answer"),
+        }
+        assert isinstance(result, AgentFinish)
+        assert result.output == "forced"
@@ -80,6 +80,7 @@ from crewai.utilities.step_execution_context import StepExecutionContext
 from crewai.utilities.planning_types import TodoItem, TodoList
 from crewai.utilities.prompts import StandardPromptResult, SystemPromptResult
 from crewai.utilities.file_store import clear_files, clear_task_files, store_files
+from crewai.utilities.i18n import I18N_DEFAULT
 from crewai_files import TextFile

 class TestAgentExecutorState:
@@ -2640,3 +2641,40 @@ class TestVisionImageFormatContract:
        assert hasattr(AnthropicCompletion, "_convert_image_blocks"), (
            "Anthropic provider must have _convert_image_blocks for auto-conversion"
        )
+
+
+class TestEnsureForceFinalAnswer:
+    """The forced final answer is requested with a trailing user turn."""
+
+    def test_forced_answer_request_ends_on_a_user_turn(self):
+        llm = Mock()
+        llm.call.return_value = "Final Answer: forced"
+        executor = _build_executor(
+            llm=llm, agent=SimpleNamespace(verbose=False), callbacks=[]
+        )
+        executor.state.messages = [
+            {"role": "user", "content": "Collect all the data."},
+            {"role": "assistant", "content": "Thought: I need data\nObservation: partial"},
+        ]
+
+        result = AgentExecutor.ensure_force_final_answer(executor)
+
+        assert result == "agent_finished"
+        sent = llm.call.call_args.args[0]
+        assert sent[-1] == {
+            "role": "user",
+            "content": I18N_DEFAULT.errors("force_final_answer"),
+        }
+        assert isinstance(executor.state.current_answer, AgentFinish)
+        assert executor.state.current_answer.output == "forced"
+        assert executor.state.is_finished is True
+
+    def test_skips_the_llm_call_once_finished(self):
+        llm = Mock()
+        executor = _build_executor(
+            llm=llm, agent=SimpleNamespace(verbose=False), callbacks=[]
+        )
+        executor.state.is_finished = True
+
+        assert AgentExecutor.ensure_force_final_answer(executor) == "agent_finished"
+        llm.call.assert_not_called()
@@ -10,9 +10,12 @@ from crewai.agent import Agent
 from crewai.agents.crew_agent_executor import CrewAgentExecutor
 from crewai.agents.parser import AgentAction, AgentFinish
 from crewai.agents.tools_handler import ToolsHandler
+from crewai.llm import LLM
 from crewai.llms.base_llm import BaseLLM
 from crewai.task import Task
+from crewai.tools import tool
 from crewai.tools.tool_types import ToolResult
+from crewai.utilities.i18n import I18N_DEFAULT


@pytest.fixture
@@ -451,4 +454,90 @@ class TestAsyncLLMResponseHelper:
                messages=[{"role": "user", "content": "test"}],
                callbacks=[],
                printer=Printer(),
-            )
+            )
+
+
+class TestAsyncMaxIterationsForcedAnswer:
+    """Both async loops request the forced final answer with a trailing user turn."""
+
+    @pytest.mark.asyncio
+    async def test_react_loop_forces_final_answer_with_user_turn(
+        self, executor: CrewAgentExecutor, mock_llm: MagicMock
+    ) -> None:
+        mock_llm.call.return_value = "Final Answer: forced"
+        executor.iterations = executor.max_iter
+        executor.messages = [
+            {"role": "user", "content": "Collect all the data."},
+            {"role": "assistant", "content": "Thought: I need data\nObservation: partial"},
+        ]
+
+        with patch.object(executor, "_show_logs"):
+            result = await executor._ainvoke_loop()
+
+        sent = mock_llm.call.call_args.args[0]
+        assert sent[-1] == {
+            "role": "user",
+            "content": I18N_DEFAULT.errors("force_final_answer"),
+        }
+        assert isinstance(result, AgentFinish)
+        assert result.output == "forced"
+
+    @pytest.mark.asyncio
+    async def test_native_tools_loop_forces_final_answer_with_user_turn(
+        self,
+        test_agent: Agent,
+        test_task: Task,
+        mock_tools_handler: MagicMock,
+    ) -> None:
+        @tool
+        def get_data(step: str) -> str:
+            """Get data for a step."""
+            return f"data for {step}"
+
+        # BaseLLM has no supports_function_calling; the native loop needs it.
+        llm = MagicMock(spec=LLM)
+        llm.stop = []
+        llm.supports_function_calling.return_value = True
+        llm.call.return_value = "Final Answer: forced"
+        executor = CrewAgentExecutor(
+            llm=llm,
+            task=test_task,
+            crew=None,
+            agent=test_agent,
+            prompt={"prompt": "Test prompt {input} {tool_names} {tools}"},
+            max_iter=1,
+            tools=[],
+            original_tools=[get_data],
+            tools_names="get_data",
+            stop_words=[],
+            tools_description="",
+            tools_handler=mock_tools_handler,
+        )
+        executor.iterations = 1
+        executor.messages = [
+            {"role": "user", "content": "Collect all the data."},
+            {
+                "role": "assistant",
+                "content": None,
+                "tool_calls": [
+                    {
+                        "id": "call_1",
+                        "type": "function",
+                        "function": {"name": "get_data", "arguments": "{}"},
+                    }
+                ],
+            },
+            {"role": "tool", "tool_call_id": "call_1", "name": "get_data", "content": "partial"},
+            {"role": "user", "content": I18N_DEFAULT.slice("post_tool_reasoning")},
+        ]
+
+        with patch.object(executor, "_show_logs"):
+            result = await executor._ainvoke_loop()
+
+        sent = llm.call.call_args.args[0]
+        assert sent[-1] == {
+            "role": "user",
+            "content": I18N_DEFAULT.errors("force_final_answer"),
+        }
+        assert isinstance(result, AgentFinish)
+        assert result.output == "forced"
@@ -1295,3 +1295,44 @@ class TestUsageMetricsDeltaSince:
        baseline = UsageMetrics(total_tokens=100, prompt_tokens=90, successful_requests=2)
        delta = UsageMetrics().delta_since(baseline)
        assert delta == UsageMetrics()
+
+
+@pytest.mark.filterwarnings("ignore:LiteAgent is deprecated")
+def test_lite_agent_forces_final_answer_with_user_turn():
+    """The forced final answer is requested with a trailing user turn, not assistant prefill."""
+    from crewai.utilities.i18n import I18N_DEFAULT
+
+    requests: list[list[dict]] = []
+
+    def record_request(messages, **_kwargs):
+        # Snapshot: the agent keeps appending to this same list after the call.
+        requests.append([dict(message) for message in messages])
+        return "Final Answer: forced"
+
+    mock_llm = Mock(spec=LLM)
+    mock_llm.call.side_effect = record_request
+    mock_llm.stop = []
+    mock_llm.get_token_usage_summary.return_value = UsageMetrics(
+        total_tokens=10,
+        prompt_tokens=5,
+        completion_tokens=5,
+        cached_prompt_tokens=0,
+        successful_requests=1,
+    )
+    agent = LiteAgent(
+        role="Test Agent",
+        goal="Test goal",
+        backstory="Test backstory",
+        llm=mock_llm,
+        max_iterations=0,
+        verbose=False,
+    )
+
+    result = agent.kickoff("Collect all the data.")
+
+    assert mock_llm.call.call_count == 1
+    assert requests[0][-1] == {
+        "role": "user",
+        "content": I18N_DEFAULT.errors("force_final_answer"),
+    }
+    assert result.raw == "forced"
@@ -1955,3 +1955,72 @@ def test_tool_fallback_still_used_for_models_without_native_support():
    assert kwargs["tool_choice"] == {"type": "tool", "name": "structured_output"}
    assert "betas" not in kwargs
    mock_client.beta.messages.create.assert_not_called()
+
+
+def _final_answer_response(text: str):
+    from anthropic.types import TextBlock
+
+    mock_response = MagicMock()
+    mock_response.content = [TextBlock(type="text", text=text, citations=None)]
+    mock_response.usage = MagicMock(input_tokens=10, output_tokens=5)
+    mock_response.stop_reason = "end_turn"
+    mock_response.id = "msg_forced_answer"
+    return mock_response
+
+
+@pytest.mark.parametrize(
+    "history_tail",
+    [
+        [
+            {
+                "role": "assistant",
+                "content": None,
+                "tool_calls": [
+                    {
+                        "id": "toolu_1",
+                        "type": "function",
+                        "function": {"name": "get_data", "arguments": "{}"},
+                    }
+                ],
+            },
+            {"role": "tool", "tool_call_id": "toolu_1", "name": "get_data", "content": "partial"},
+            {"role": "user", "content": "Analyze the tool result."},
+        ],
+        [
+            {
+                "role": "assistant",
+                "content": "Thought: I need data\nAction: get_data\nAction Input: {}\nObservation: partial",
+            },
+        ],
+    ],
+    ids=["native-tools", "react"],
+)
+def test_max_iterations_request_ends_on_a_user_turn(history_tail):
+    """Claude 4.6+ rejects a request whose last message is an assistant turn (no prefill).
+
+    Regression for the max_iter path: the forced final-answer instruction must
+    reach the Anthropic API as the trailing user message, on both loop shapes.
+    """
+    from crewai.llms.providers.anthropic.completion import AnthropicCompletion
+    from crewai.utilities.agent_utils import handle_max_iterations_exceeded
+    from crewai.utilities.i18n import I18N_DEFAULT
+
+    llm = AnthropicCompletion(model="claude-opus-5")
+    mock_client = MagicMock()
+    mock_client.messages.create.return_value = _final_answer_response("Final Answer: 42")
+    llm._client = mock_client
+
+    history = [
+        {"role": "system", "content": "You are an agent."},
+        {"role": "user", "content": "Collect all the data."},
+        *history_tail,
+    ]
+
+    result = handle_max_iterations_exceeded(
+        printer=MagicMock(), messages=history, llm=llm, callbacks=[], verbose=False
+    )
+
+    mock_client.messages.create.assert_called_once()
+    sent = mock_client.messages.create.call_args.kwargs["messages"]
+    assert sent[-1] == {"role": "user", "content": I18N_DEFAULT.errors("force_final_answer")}
+    assert result.output == "42"
@@ -16,6 +16,7 @@ from crewai.hooks.tool_hooks import (
    clear_before_tool_call_hooks,
    register_after_tool_call_hook,
 )
+from crewai.agents.parser import AgentFinish
 from crewai.tools.base_tool import BaseTool
 from crewai.llm import CONTEXT_WINDOW_USAGE_RATIO
 from crewai.utilities.agent_utils import (
@@ -30,6 +31,7 @@ from crewai.utilities.agent_utils import (
    _split_text_by_token_limit,
    format_message_for_llm,
    convert_tools_to_openai_schema,
+    handle_max_iterations_exceeded,
    execute_single_native_tool_call,
    extract_tool_call_info,
    is_tool_call_list,
@@ -1652,3 +1654,107 @@ class TestResolvePlusResponse:
                resolve_plus_response(future)

        asyncio.run(main())
+
+
+_FORCE_FINAL_ANSWER = I18N_DEFAULT.errors("force_final_answer")
+
+
+def _native_tool_history() -> list[dict[str, Any]]:
+    """History as the native tool-calling loop leaves it: ends on a user prompt."""
+    return [
+        {"role": "system", "content": "You are an agent."},
+        {"role": "user", "content": "Collect all the data."},
+        {
+            "role": "assistant",
+            "content": None,
+            "tool_calls": [
+                {
+                    "id": "call_1",
+                    "type": "function",
+                    "function": {"name": "get_data", "arguments": "{}"},
+                }
+            ],
+        },
+        {"role": "tool", "tool_call_id": "call_1", "name": "get_data", "content": "partial"},
+        {"role": "user", "content": I18N_DEFAULT.slice("post_tool_reasoning")},
+    ]
+
+
+def _react_history() -> list[dict[str, Any]]:
+    """History as the ReAct loop leaves it: ends on the assistant turn with the observation."""
+    return [
+        {"role": "system", "content": "You are an agent."},
+        {"role": "user", "content": "Collect all the data."},
+        {
+            "role": "assistant",
+            "content": "Thought: I need data\nAction: get_data\nAction Input: {}\nObservation: partial",
+        },
+    ]
+
+
+class TestHandleMaxIterationsExceeded:
+    """The forced final answer is requested with a user turn, never assistant prefill.
+
+    Current Claude models reject a request whose last message is an assistant
+    turn ("This model does not support assistant message prefill"), so the
+    nudge must go out as the user's instruction on every loop shape.
+    """
+
+    @pytest.mark.parametrize(
+        "make_history", [_native_tool_history, _react_history], ids=["native-tools", "react"]
+    )
+    def test_appends_the_instruction_as_a_user_turn(self, make_history) -> None:
+        history = make_history()
+        before = [dict(message) for message in history]
+        llm = MagicMock()
+        llm.call.return_value = "Final Answer: 42"
+
+        result = handle_max_iterations_exceeded(
+            printer=MagicMock(), messages=history, llm=llm, callbacks=[], verbose=False
+        )
+
+        assert history[:-1] == before
+        assert history[-1] == {"role": "user", "content": _FORCE_FINAL_ANSWER}
+        llm.call.assert_called_once_with(history, callbacks=[])
+        assert isinstance(result, AgentFinish)
+        assert result.output == "42"
+
+    def test_action_shaped_reply_still_becomes_a_final_answer(self) -> None:
+        reply = "Thought: one more\nAction: get_data\nAction Input: {}"
+        llm = MagicMock()
+        llm.call.return_value = reply
+
+        result = handle_max_iterations_exceeded(
+            printer=MagicMock(), messages=_react_history(), llm=llm, callbacks=[], verbose=False
+        )
+
+        assert isinstance(result, AgentFinish)
+        assert result.text == reply
+        assert result.output == reply
+
+    @pytest.mark.parametrize("reply", [None, ""], ids=["none", "empty"])
+    def test_empty_reply_raises(self, reply: str | None) -> None:
+        llm = MagicMock()
+        llm.call.return_value = reply
+
+        with pytest.raises(ValueError, match="Invalid response from LLM call - None or empty."):
+            handle_max_iterations_exceeded(
+                printer=MagicMock(), messages=_native_tool_history(), llm=llm, callbacks=[], verbose=False
+            )
+
+    @pytest.mark.parametrize("verbose", [True, False])
+    def test_notice_is_printed_only_when_verbose(self, verbose: bool) -> None:
+        printer = MagicMock()
+        llm = MagicMock()
+        llm.call.return_value = "Final Answer: 42"
+
+        handle_max_iterations_exceeded(
+            printer=printer, messages=_native_tool_history(), llm=llm, callbacks=[], verbose=verbose
+        )
+
+        if verbose:
+            printer.print.assert_called_once_with(
+                content="Maximum iterations reached. Requesting final answer.", color="yellow"
+            )
+        else:
+            printer.print.assert_not_called()