feat(tui): an Evaluate button beside View Traces and Deploy

A run finishes in the terminal UI and the next question is usually "was that
any good?" — which `crewai eval` answers, from the command line, about the run
that just happened. The button is there now, with `e` bound to it.

It behaves like Deploy rather than like View Traces: it leaves the UI and then
runs the command, because `crewai eval` talks — it prints the link to follow,
waits for the verdict and prints that — and none of it belongs inside a
full-screen app. It calls `eval_crew()` itself rather than reimplementing any
of it, so the button and the command can never say different things; a clean
SystemExit is the command deciding, and a failure is reported without failing
the run that already succeeded.

Declarative flows get it too — same button, same tail.

Co-Authored-By: Claude Opus 5 (1M context) <[email protected]>
This commit is contained in:
Joao Moura
2026-09-25 13:30:14 -07:00
co-authored by Claude Opus 5
parent 11abaa28bd
commit f4000251a5
6 changed files with 141 additions and 3 deletions
+33
View File
@@ -421,6 +421,19 @@ FooterKey .footer-key--key {
color: #888888;
}
#btn-eval {
background: #2b2b2b;
color: #e0e0e0;
border: none;
}
#btn-eval:hover {
background: #3d3d3d;
}
#btn-eval:disabled {
background: #202020;
color: #888888;
}
#btn-deploy {
background: #333333;
color: #e0e0e0;
@@ -437,6 +450,7 @@ FooterKey .footer-key--key {
Binding("s", "toggle_sidebar", "Sidebar"),
Binding("l", "toggle_logs", "Logs"),
Binding("t", "view_traces", "Traces", show=False),
Binding("e", "evaluate_crew", "Evaluate", show=False),
Binding("d", "deploy_crew", "Deploy", show=False),
Binding("down", "log_down", "Log ↓", show=False),
Binding("up", "log_up", "Log ↑", show=False),
@@ -529,6 +543,7 @@ FooterKey .footer-key--key {
self._current_method: str | None = None
self._elapsed_frozen: float | None = None
self._want_deploy: bool = False
self._want_eval: bool = False
self._consent_screen: TraceConsentScreen | None = None
self._trace_consent_pending: threading.Event | None = None
self._discard_trace_on_exit = False
@@ -557,6 +572,7 @@ FooterKey .footer-key--key {
yield Static(id="sidebar-content")
with Vertical(id="sidebar-actions"):
yield Button("View Traces", id="btn-traces", classes="action-btn")
yield Button("Evaluate", id="btn-eval", classes="action-btn")
yield Button("Deploy", id="btn-deploy", classes="action-btn")
with Vertical(id="main-panel"):
yield Static(id="task-header")
@@ -1066,6 +1082,21 @@ FooterKey .footer-key--key {
self._unsubscribe()
self.exit(self._crew_result)
def action_evaluate_crew(self) -> None:
"""Grade the run that just finished.
Like Deploy, this leaves the terminal UI first: `crewai eval` talks —
it prints the link to follow, waits for the verdict and prints it — and
that belongs in the terminal, not inside a full-screen app.
"""
if self._status != "completed":
return
self._record_tui_button_click("evaluate")
self._want_eval = True
self._unsubscribe()
self.exit(self._crew_result)
def _record_tui_button_click(self, button_name: str) -> None:
try:
if self._telemetry is None:
@@ -1078,6 +1109,8 @@ FooterKey .footer-key--key {
def on_button_pressed(self, event: Button.Pressed) -> None:
if event.button.id == "btn-traces":
self.action_view_traces()
elif event.button.id == "btn-eval":
self.action_evaluate_crew()
elif event.button.id == "btn-deploy":
self.action_deploy_crew()
+26
View File
@@ -370,6 +370,9 @@ def _run_json_crew(
if getattr(app, "_want_deploy", False):
_chain_deploy()
if getattr(app, "_want_eval", False):
_chain_eval()
return app._crew_result
@@ -493,6 +496,29 @@ def _run_json_crew_in_project_env(
return None
def _chain_eval() -> None:
"""Grade the run that just finished, in the terminal the TUI has left.
`crewai eval` is the whole command: it finds the run crewAI recorded, asks
AMP to evaluate it, prints the link to follow and then the verdict. Running
it here rather than reimplementing it means the button and the command can
never say different things.
"""
from rich.console import Console
console = Console()
try:
from crewai_cli.experimental.eval_crew import eval_crew
console.print("\nEvaluating this run…\n", style="bold #FF5A50")
eval_crew()
except SystemExit as exc: # the command says why and picks the code
if isinstance(exc.code, int) and exc.code not in (0, None):
raise
except Exception as error: # noqa: BLE001 - a failed evaluation never fails the run
console.print(f"\nEvaluation failed: {error}\n", style="bold red")
def _chain_deploy() -> None:
from rich.console import Console
@@ -218,6 +218,11 @@ def _run_declarative_flow_tui(
_chain_deploy()
if getattr(app, "_want_eval", False):
from crewai_cli.run_crew import _chain_eval
_chain_eval()
return app._crew_result
+60
View File
@@ -137,6 +137,37 @@ def test_chain_deploy_does_not_login_for_deploy_exit(monkeypatch, capsys) -> Non
assert "Deploy failed with exit code 42" in capsys.readouterr().out
def test_chain_eval_runs_the_command_itself(monkeypatch, capsys) -> None:
"""The button must not reimplement `crewai eval`, or the two can disagree."""
called: list[bool] = []
monkeypatch.setattr(
"crewai_cli.experimental.eval_crew.eval_crew", lambda: called.append(True)
)
run_crew._chain_eval()
assert called == [True]
assert "Evaluating this run" in capsys.readouterr().out
def test_chain_eval_lets_the_command_say_why_it_stopped(monkeypatch, capsys) -> None:
"""`crewai eval` exits with its own words and its own code; a clean exit is
not an error, and a failure is the command's to report."""
def refuse() -> None:
raise SystemExit(0)
monkeypatch.setattr("crewai_cli.experimental.eval_crew.eval_crew", refuse)
run_crew._chain_eval() # no traceback, nothing added
def blow_up() -> None:
raise RuntimeError("AMP said no")
monkeypatch.setattr("crewai_cli.experimental.eval_crew.eval_crew", blow_up)
run_crew._chain_eval() # a failed evaluation never fails the run
assert "Evaluation failed: AMP said no" in capsys.readouterr().out
def test_view_traces_button_click_records_telemetry(monkeypatch) -> None:
app = CrewRunApp()
app._status = "completed"
@@ -170,6 +201,35 @@ def test_deploy_button_click_records_telemetry() -> None:
assert exits == [app._crew_result]
def test_evaluate_button_leaves_the_tui_and_records_telemetry() -> None:
"""Evaluate behaves like Deploy: the app exits and the command takes over,
because `crewai eval` prints a link, waits, and prints a verdict."""
app = CrewRunApp()
app._status = "completed"
app._crew_result = object()
app._telemetry = Mock()
app._unsubscribe = lambda: None # type: ignore[method-assign]
exits: list[object] = []
app.exit = lambda result: exits.append(result) # type: ignore[method-assign]
app.on_button_pressed(SimpleNamespace(button=SimpleNamespace(id="btn-eval")))
app._telemetry.feature_usage_span.assert_called_once_with("cli_usage:evaluate")
assert app._want_eval is True
assert exits == [app._crew_result]
def test_evaluate_before_completion_records_nothing() -> None:
app = CrewRunApp()
app._status = "running"
app._telemetry = Mock()
app.action_evaluate_crew()
app._telemetry.feature_usage_span.assert_not_called()
assert app._want_eval is False
def test_conversation_turn_done_records_assistant_message() -> None:
class RawResult:
raw = "hello from the flow"
+15 -1
View File
@@ -418,7 +418,7 @@ def test_id_restore_still_drops_unknown_keys(
# ── TUI vs terminal (headless/deploy) routing ──────────────────────
def _install_fake_flow_app(monkeypatch, *, status, want_deploy=False):
def _install_fake_flow_app(monkeypatch, *, status, want_deploy=False, want_eval=False):
"""Replace CrewRunApp/EventListener/summary so _run_declarative_flow_tui is
driven by a controllable fake app."""
@@ -430,6 +430,7 @@ def _install_fake_flow_app(monkeypatch, *, status, want_deploy=False):
self._crew_name = crew_name
self._status = status
self._want_deploy = want_deploy
self._want_eval = want_eval
self._crew_result = "result"
def run(self):
@@ -541,6 +542,19 @@ def test_run_declarative_flow_tui_no_deploy_when_not_requested(
assert deploy_calls == []
def test_run_declarative_flow_tui_chains_eval(monkeypatch: pytest.MonkeyPatch) -> None:
"""A declarative flow has the button too, and the same tail runs it."""
_install_fake_flow_app(monkeypatch, status="completed", want_eval=True)
eval_calls: list[bool] = []
monkeypatch.setattr("crewai_cli.run_crew._chain_eval", lambda: eval_calls.append(True))
run_declarative_flow_module._run_declarative_flow_tui(
SimpleNamespace(name="Flow"), None
)
assert eval_calls == [True]
def test_run_declarative_flow_tui_enables_flow_events(
monkeypatch: pytest.MonkeyPatch,
) -> None:
Generated
+2 -2
View File
@@ -17,7 +17,7 @@ resolution-markers = [
]
[options]
exclude-newer = "2026-09-15T05:55:17.52834Z"
exclude-newer = "0001-01-01T00:00:00Z" # This has no effect and is included for backwards compatibility when using relative exclude-newer values.
exclude-newer-span = "P3D"
[options.exclude-newer-package]
@@ -1552,7 +1552,7 @@ requires-dist = [
{ name = "httpx-auth", marker = "extra == 'a2a'", specifier = "~=0.23.1" },
{ name = "httpx-sse", marker = "extra == 'a2a'", specifier = "~=0.4.0" },
{ name = "ibm-watsonx-ai", marker = "extra == 'watson'", specifier = "~=1.3.39" },
{ name = "instructor", specifier = ">=1.3.3" },
{ name = "instructor", specifier = ">=1.3.3,<1.16" },
{ name = "json-repair", specifier = "~=0.60.1" },
{ name = "json5", specifier = "~=0.10.0" },
{ name = "jsonref", specifier = "~=1.1.0" },