mirror of
https://github.com/crewAIInc/crewAI.git
synced 2026-10-12 02:12:31 +00:00
João MouraandClaude Opus 5.5
00e2ff98e9
* feat(cli): crewai eval prints a markdown brief when run by an agent When stdout is not a terminal (or CREWAI_DMN is on), `crewai eval` and `crewai eval --models` print the poll response's `brief_markdown` after the verdict and the report link — written straight to stdout, never through Rich markup — or "Brief (markdown): <brief_url>" when only the link came. An older AMP sends neither and nothing extra is printed. In a terminal nothing changes. Exit codes unchanged. `crewai eval --help` and the `--models` help say so, and the AGENTS.md the crew, flow and declarative-flow templates scaffold gains an "Evaluating" section telling a coding agent to run `crewai eval` after a change and act on the brief. Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]> * docs(cli): an agent re-runs before it re-evaluates; each command's exit rule; the brief may be a link Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]> * fix(cli): keep the eval guidance out of tool projects; the brief is conditional in --help `crewai create tool` copies the shared AGENTS.md too, but a tool project has no `crewai run` and no traced run for `crewai eval`. The evaluation blocks are marked in the template and dropped for a tool (`evaluation=False`); crew, flow and JSON-crew projects keep them, markers stripped. --help now says the brief prints when AMP provides it, else its link, else nothing more. Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]> --------- Co-authored-by: Claude Opus 5.5 (1M context) <[email protected]>
86 lines
3.4 KiB
Python
86 lines
3.4 KiB
Python
from __future__ import annotations
|
|
|
|
from pathlib import Path
|
|
|
|
from click.testing import CliRunner
|
|
from pytest import MonkeyPatch
|
|
import tomli
|
|
|
|
from crewai_cli.cli import crewai
|
|
from crewai_cli.create_flow import create_flow
|
|
|
|
|
|
def test_create_flow_declarative_project_can_run(
|
|
tmp_path: Path, monkeypatch: MonkeyPatch
|
|
):
|
|
monkeypatch.chdir(tmp_path)
|
|
create_flow("Research Flow", declarative=True)
|
|
|
|
project_root = tmp_path / "research_flow"
|
|
assert project_root.is_dir()
|
|
|
|
pyproject = tomli.loads(
|
|
(project_root / "pyproject.toml").read_text(encoding="utf-8")
|
|
)
|
|
assert pyproject["project"]["name"] == "research_flow"
|
|
assert pyproject["project"]["requires-python"]
|
|
assert pyproject["project"]["dependencies"]
|
|
assert (project_root / pyproject["tool"]["crewai"]["definition"]).is_file()
|
|
agents_md = (project_root / "AGENTS.md").read_text(encoding="utf-8")
|
|
assert "CrewAI Flow declaration" in agents_md
|
|
assert "schema: crewai.flow/v1" in agents_md
|
|
assert "do not assemble the string with CEL `+`" in agents_md
|
|
assert "Agent prompt template. Insert Flow values with `${...}`" in agents_md
|
|
assert "Runtime inputs passed to the Crew" in agents_md
|
|
assert "call: expression" in agents_md
|
|
assert "call: tool" not in agents_md
|
|
assert "call: script" not in agents_md
|
|
assert "call: each" not in agents_md
|
|
assert "human_feedback" not in agents_md
|
|
claude_md = (project_root / "CLAUDE.md").read_text(encoding="utf-8")
|
|
assert "@AGENTS.md" in claude_md.splitlines()
|
|
cursor_md = (project_root / "CURSOR.md").read_text(encoding="utf-8")
|
|
assert "@AGENTS.md" in cursor_md.splitlines()
|
|
gemini_md = (project_root / "GEMINI.md").read_text(encoding="utf-8")
|
|
assert "@./AGENTS.md" in gemini_md.splitlines()
|
|
|
|
monkeypatch.chdir(project_root)
|
|
result = CliRunner().invoke(crewai, ["run"], env={"UV_RUN_RECURSION_DEPTH": "1"})
|
|
|
|
assert result.exit_code == 0
|
|
assert "Running the Flow" not in result.output
|
|
assert "AI agents" in result.output
|
|
|
|
|
|
def test_create_flow_scaffolds_assistant_instructions(
|
|
tmp_path: Path, monkeypatch: MonkeyPatch
|
|
):
|
|
monkeypatch.chdir(tmp_path)
|
|
create_flow("Research Flow")
|
|
|
|
project_root = tmp_path / "research_flow"
|
|
agents_md = (project_root / "AGENTS.md").read_text(encoding="utf-8")
|
|
assert "CrewAI Reference for AI Coding Assistants" in agents_md
|
|
assert "Never disable, block, or silence CrewAI's built-in observability" in agents_md
|
|
claude_md = (project_root / "CLAUDE.md").read_text(encoding="utf-8")
|
|
assert "@AGENTS.md" in claude_md.splitlines()
|
|
gemini_md = (project_root / "GEMINI.md").read_text(encoding="utf-8")
|
|
assert "@./AGENTS.md" in gemini_md.splitlines()
|
|
|
|
|
|
def test_scaffolded_flow_agents_md_tells_assistants_to_evaluate_with_crewai_eval(
|
|
tmp_path: Path, monkeypatch: MonkeyPatch
|
|
):
|
|
monkeypatch.chdir(tmp_path)
|
|
create_flow("Research Flow")
|
|
create_flow("Declared Flow", declarative=True)
|
|
|
|
for project in ("research_flow", "declared_flow"):
|
|
agents_md = (tmp_path / project / "AGENTS.md").read_text(encoding="utf-8")
|
|
assert "## Evaluating this" in agents_md
|
|
assert "Run `crewai eval` after a change" in agents_md
|
|
assert "it prints a **markdown brief** after the verdict" in agents_md
|
|
assert 'crewai eval --models "provider/model,…"' in agents_md
|
|
assert "{{name}}" not in agents_md
|
|
assert "crewai-eval:" not in agents_md
|