mirror of
https://github.com/crewAIInc/crewAI.git
synced 2026-10-12 02:12:31 +00:00
feat(tui): an Evaluate button beside View Traces and Deploy
A run finishes in the terminal UI and the next question is usually "was that any good?" — which `crewai eval` answers, from the command line, about the run that just happened. The button is there now, with `e` bound to it. It behaves like Deploy rather than like View Traces: it leaves the UI and then runs the command, because `crewai eval` talks — it prints the link to follow, waits for the verdict and prints that — and none of it belongs inside a full-screen app. It calls `eval_crew()` itself rather than reimplementing any of it, so the button and the command can never say different things; a clean SystemExit is the command deciding, and a failure is reported without failing the run that already succeeded. Declarative flows get it too — same button, same tail. Co-Authored-By: Claude Opus 5 (1M context) <[email protected]>
This commit is contained in:
co-authored by
Claude Opus 5
parent
11abaa28bd
commit
f4000251a5
@@ -421,6 +421,19 @@ FooterKey .footer-key--key {
|
||||
color: #888888;
|
||||
}
|
||||
|
||||
#btn-eval {
|
||||
background: #2b2b2b;
|
||||
color: #e0e0e0;
|
||||
border: none;
|
||||
}
|
||||
#btn-eval:hover {
|
||||
background: #3d3d3d;
|
||||
}
|
||||
#btn-eval:disabled {
|
||||
background: #202020;
|
||||
color: #888888;
|
||||
}
|
||||
|
||||
#btn-deploy {
|
||||
background: #333333;
|
||||
color: #e0e0e0;
|
||||
@@ -437,6 +450,7 @@ FooterKey .footer-key--key {
|
||||
Binding("s", "toggle_sidebar", "Sidebar"),
|
||||
Binding("l", "toggle_logs", "Logs"),
|
||||
Binding("t", "view_traces", "Traces", show=False),
|
||||
Binding("e", "evaluate_crew", "Evaluate", show=False),
|
||||
Binding("d", "deploy_crew", "Deploy", show=False),
|
||||
Binding("down", "log_down", "Log ↓", show=False),
|
||||
Binding("up", "log_up", "Log ↑", show=False),
|
||||
@@ -529,6 +543,7 @@ FooterKey .footer-key--key {
|
||||
self._current_method: str | None = None
|
||||
self._elapsed_frozen: float | None = None
|
||||
self._want_deploy: bool = False
|
||||
self._want_eval: bool = False
|
||||
self._consent_screen: TraceConsentScreen | None = None
|
||||
self._trace_consent_pending: threading.Event | None = None
|
||||
self._discard_trace_on_exit = False
|
||||
@@ -557,6 +572,7 @@ FooterKey .footer-key--key {
|
||||
yield Static(id="sidebar-content")
|
||||
with Vertical(id="sidebar-actions"):
|
||||
yield Button("View Traces", id="btn-traces", classes="action-btn")
|
||||
yield Button("Evaluate", id="btn-eval", classes="action-btn")
|
||||
yield Button("Deploy", id="btn-deploy", classes="action-btn")
|
||||
with Vertical(id="main-panel"):
|
||||
yield Static(id="task-header")
|
||||
@@ -1066,6 +1082,21 @@ FooterKey .footer-key--key {
|
||||
self._unsubscribe()
|
||||
self.exit(self._crew_result)
|
||||
|
||||
def action_evaluate_crew(self) -> None:
|
||||
"""Grade the run that just finished.
|
||||
|
||||
Like Deploy, this leaves the terminal UI first: `crewai eval` talks —
|
||||
it prints the link to follow, waits for the verdict and prints it — and
|
||||
that belongs in the terminal, not inside a full-screen app.
|
||||
"""
|
||||
if self._status != "completed":
|
||||
return
|
||||
|
||||
self._record_tui_button_click("evaluate")
|
||||
self._want_eval = True
|
||||
self._unsubscribe()
|
||||
self.exit(self._crew_result)
|
||||
|
||||
def _record_tui_button_click(self, button_name: str) -> None:
|
||||
try:
|
||||
if self._telemetry is None:
|
||||
@@ -1078,6 +1109,8 @@ FooterKey .footer-key--key {
|
||||
def on_button_pressed(self, event: Button.Pressed) -> None:
|
||||
if event.button.id == "btn-traces":
|
||||
self.action_view_traces()
|
||||
elif event.button.id == "btn-eval":
|
||||
self.action_evaluate_crew()
|
||||
elif event.button.id == "btn-deploy":
|
||||
self.action_deploy_crew()
|
||||
|
||||
|
||||
@@ -370,6 +370,9 @@ def _run_json_crew(
|
||||
if getattr(app, "_want_deploy", False):
|
||||
_chain_deploy()
|
||||
|
||||
if getattr(app, "_want_eval", False):
|
||||
_chain_eval()
|
||||
|
||||
return app._crew_result
|
||||
|
||||
|
||||
@@ -493,6 +496,29 @@ def _run_json_crew_in_project_env(
|
||||
return None
|
||||
|
||||
|
||||
def _chain_eval() -> None:
|
||||
"""Grade the run that just finished, in the terminal the TUI has left.
|
||||
|
||||
`crewai eval` is the whole command: it finds the run crewAI recorded, asks
|
||||
AMP to evaluate it, prints the link to follow and then the verdict. Running
|
||||
it here rather than reimplementing it means the button and the command can
|
||||
never say different things.
|
||||
"""
|
||||
from rich.console import Console
|
||||
|
||||
console = Console()
|
||||
try:
|
||||
from crewai_cli.experimental.eval_crew import eval_crew
|
||||
|
||||
console.print("\nEvaluating this run…\n", style="bold #FF5A50")
|
||||
eval_crew()
|
||||
except SystemExit as exc: # the command says why and picks the code
|
||||
if isinstance(exc.code, int) and exc.code not in (0, None):
|
||||
raise
|
||||
except Exception as error: # noqa: BLE001 - a failed evaluation never fails the run
|
||||
console.print(f"\nEvaluation failed: {error}\n", style="bold red")
|
||||
|
||||
|
||||
def _chain_deploy() -> None:
|
||||
from rich.console import Console
|
||||
|
||||
|
||||
@@ -218,6 +218,11 @@ def _run_declarative_flow_tui(
|
||||
|
||||
_chain_deploy()
|
||||
|
||||
if getattr(app, "_want_eval", False):
|
||||
from crewai_cli.run_crew import _chain_eval
|
||||
|
||||
_chain_eval()
|
||||
|
||||
return app._crew_result
|
||||
|
||||
|
||||
|
||||
@@ -137,6 +137,37 @@ def test_chain_deploy_does_not_login_for_deploy_exit(monkeypatch, capsys) -> Non
|
||||
assert "Deploy failed with exit code 42" in capsys.readouterr().out
|
||||
|
||||
|
||||
def test_chain_eval_runs_the_command_itself(monkeypatch, capsys) -> None:
|
||||
"""The button must not reimplement `crewai eval`, or the two can disagree."""
|
||||
called: list[bool] = []
|
||||
monkeypatch.setattr(
|
||||
"crewai_cli.experimental.eval_crew.eval_crew", lambda: called.append(True)
|
||||
)
|
||||
|
||||
run_crew._chain_eval()
|
||||
|
||||
assert called == [True]
|
||||
assert "Evaluating this run" in capsys.readouterr().out
|
||||
|
||||
|
||||
def test_chain_eval_lets_the_command_say_why_it_stopped(monkeypatch, capsys) -> None:
|
||||
"""`crewai eval` exits with its own words and its own code; a clean exit is
|
||||
not an error, and a failure is the command's to report."""
|
||||
def refuse() -> None:
|
||||
raise SystemExit(0)
|
||||
|
||||
monkeypatch.setattr("crewai_cli.experimental.eval_crew.eval_crew", refuse)
|
||||
run_crew._chain_eval() # no traceback, nothing added
|
||||
|
||||
def blow_up() -> None:
|
||||
raise RuntimeError("AMP said no")
|
||||
|
||||
monkeypatch.setattr("crewai_cli.experimental.eval_crew.eval_crew", blow_up)
|
||||
run_crew._chain_eval() # a failed evaluation never fails the run
|
||||
|
||||
assert "Evaluation failed: AMP said no" in capsys.readouterr().out
|
||||
|
||||
|
||||
def test_view_traces_button_click_records_telemetry(monkeypatch) -> None:
|
||||
app = CrewRunApp()
|
||||
app._status = "completed"
|
||||
@@ -170,6 +201,35 @@ def test_deploy_button_click_records_telemetry() -> None:
|
||||
assert exits == [app._crew_result]
|
||||
|
||||
|
||||
def test_evaluate_button_leaves_the_tui_and_records_telemetry() -> None:
|
||||
"""Evaluate behaves like Deploy: the app exits and the command takes over,
|
||||
because `crewai eval` prints a link, waits, and prints a verdict."""
|
||||
app = CrewRunApp()
|
||||
app._status = "completed"
|
||||
app._crew_result = object()
|
||||
app._telemetry = Mock()
|
||||
app._unsubscribe = lambda: None # type: ignore[method-assign]
|
||||
exits: list[object] = []
|
||||
app.exit = lambda result: exits.append(result) # type: ignore[method-assign]
|
||||
|
||||
app.on_button_pressed(SimpleNamespace(button=SimpleNamespace(id="btn-eval")))
|
||||
|
||||
app._telemetry.feature_usage_span.assert_called_once_with("cli_usage:evaluate")
|
||||
assert app._want_eval is True
|
||||
assert exits == [app._crew_result]
|
||||
|
||||
|
||||
def test_evaluate_before_completion_records_nothing() -> None:
|
||||
app = CrewRunApp()
|
||||
app._status = "running"
|
||||
app._telemetry = Mock()
|
||||
|
||||
app.action_evaluate_crew()
|
||||
|
||||
app._telemetry.feature_usage_span.assert_not_called()
|
||||
assert app._want_eval is False
|
||||
|
||||
|
||||
def test_conversation_turn_done_records_assistant_message() -> None:
|
||||
class RawResult:
|
||||
raw = "hello from the flow"
|
||||
|
||||
@@ -418,7 +418,7 @@ def test_id_restore_still_drops_unknown_keys(
|
||||
# ── TUI vs terminal (headless/deploy) routing ──────────────────────
|
||||
|
||||
|
||||
def _install_fake_flow_app(monkeypatch, *, status, want_deploy=False):
|
||||
def _install_fake_flow_app(monkeypatch, *, status, want_deploy=False, want_eval=False):
|
||||
"""Replace CrewRunApp/EventListener/summary so _run_declarative_flow_tui is
|
||||
driven by a controllable fake app."""
|
||||
|
||||
@@ -430,6 +430,7 @@ def _install_fake_flow_app(monkeypatch, *, status, want_deploy=False):
|
||||
self._crew_name = crew_name
|
||||
self._status = status
|
||||
self._want_deploy = want_deploy
|
||||
self._want_eval = want_eval
|
||||
self._crew_result = "result"
|
||||
|
||||
def run(self):
|
||||
@@ -541,6 +542,19 @@ def test_run_declarative_flow_tui_no_deploy_when_not_requested(
|
||||
assert deploy_calls == []
|
||||
|
||||
|
||||
def test_run_declarative_flow_tui_chains_eval(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""A declarative flow has the button too, and the same tail runs it."""
|
||||
_install_fake_flow_app(monkeypatch, status="completed", want_eval=True)
|
||||
eval_calls: list[bool] = []
|
||||
monkeypatch.setattr("crewai_cli.run_crew._chain_eval", lambda: eval_calls.append(True))
|
||||
|
||||
run_declarative_flow_module._run_declarative_flow_tui(
|
||||
SimpleNamespace(name="Flow"), None
|
||||
)
|
||||
|
||||
assert eval_calls == [True]
|
||||
|
||||
|
||||
def test_run_declarative_flow_tui_enables_flow_events(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
|
||||
@@ -17,7 +17,7 @@ resolution-markers = [
|
||||
]
|
||||
|
||||
[options]
|
||||
exclude-newer = "2026-09-15T05:55:17.52834Z"
|
||||
exclude-newer = "0001-01-01T00:00:00Z" # This has no effect and is included for backwards compatibility when using relative exclude-newer values.
|
||||
exclude-newer-span = "P3D"
|
||||
|
||||
[options.exclude-newer-package]
|
||||
@@ -1552,7 +1552,7 @@ requires-dist = [
|
||||
{ name = "httpx-auth", marker = "extra == 'a2a'", specifier = "~=0.23.1" },
|
||||
{ name = "httpx-sse", marker = "extra == 'a2a'", specifier = "~=0.4.0" },
|
||||
{ name = "ibm-watsonx-ai", marker = "extra == 'watson'", specifier = "~=1.3.39" },
|
||||
{ name = "instructor", specifier = ">=1.3.3" },
|
||||
{ name = "instructor", specifier = ">=1.3.3,<1.16" },
|
||||
{ name = "json-repair", specifier = "~=0.60.1" },
|
||||
{ name = "json5", specifier = "~=0.10.0" },
|
||||
{ name = "jsonref", specifier = "~=1.1.0" },
|
||||
|
||||
Reference in New Issue
Block a user