From 917b9df6d70cc0389078c041c9a4373f9dc68570 Mon Sep 17 00:00:00 2001 From: Lorenze Jay <63378463+lorenzejay@users.noreply.github.com> Date: Tue, 1 Sep 2026 09:00:59 -0700 Subject: [PATCH] Validate JSON crews in project environments (#7171) * Validate JSON crews in project environments * fix(cli): address standalone deploy review feedback --------- Co-authored-by: Vidit Ostwal <110953813+Vidit-Ostwal@users.noreply.github.com> --- lib/cli/src/crewai_cli/deploy/validate.py | 174 ++++++++++++++++-- lib/cli/src/crewai_cli/install_crew.py | 3 +- lib/cli/src/crewai_cli/utils.py | 7 + lib/cli/tests/deploy/test_cli_only_install.py | 148 +++++++++++++++ lib/cli/tests/deploy/test_validate.py | 28 +++ 5 files changed, 339 insertions(+), 21 deletions(-) create mode 100644 lib/cli/tests/deploy/test_cli_only_install.py diff --git a/lib/cli/src/crewai_cli/deploy/validate.py b/lib/cli/src/crewai_cli/deploy/validate.py index 0b2564d24..d13562918 100644 --- a/lib/cli/src/crewai_cli/deploy/validate.py +++ b/lib/cli/src/crewai_cli/deploy/validate.py @@ -38,11 +38,6 @@ import subprocess import sys from typing import Any -from crewai.project.json_loader import ( - JSONProjectValidationError, - find_json_project_file, - validate_crew_project, -) from crewai_core.project import ( ProjectDefinitionError, configured_project_definition, @@ -52,6 +47,8 @@ from crewai_core.project import ( ) from rich.console import Console +from crewai_cli.utils import normalize_package_name + console = Console() logger = logging.getLogger(__name__) @@ -108,15 +105,143 @@ _KNOWN_API_KEY_HINTS: dict[str, str] = { } -def normalize_package_name(project_name: str) -> str: - """Normalize a pyproject project.name into a Python package directory name. +_JSON_VALIDATION_MARKER = "CREWAI_JSON_VALIDATION_RESULT=" +_JSON_VALIDATOR_SCRIPT = f""" +import json +import sys - Mirrors the rules in ``crewai.cli.create_crew.create_crew`` so the - validator agrees with the scaffolder about where ``src//`` should - live. - """ - folder = project_name.replace(" ", "_").replace("-", "_").lower() - return re.sub(r"[^a-zA-Z0-9_]", "", folder) +try: + from crewai.project.json_loader import validate_crew_project + project = validate_crew_project(sys.argv[1], agents_dir=sys.argv[2]) + payload = {{"ok": True, "agent_names": project.agent_names}} +except BaseException as exc: + errors = getattr(exc, "errors", None) + payload = {{ + "ok": False, + "error_type": type(exc).__name__, + "error": str(exc), + "errors": errors if isinstance(errors, list) else None, + }} + +print({_JSON_VALIDATION_MARKER!r} + json.dumps(payload)) +""".strip() + + +class _JSONProjectValidationError(ValueError): + def __init__(self, errors: list[str]) -> None: + self.errors = errors + super().__init__("\n".join(errors)) + + +class _JSONProjectEnvironmentError(RuntimeError): + """JSON validation could not run in the project's environment.""" + + hint = ( + "Install `uv` if needed, run `uv sync` in the project directory, " + "then retry with `uv run crewai deploy validate`." + ) + + +def _find_json_project_file(directory: Path, stem: str) -> Path | None: + for extension in (".jsonc", ".json"): + candidate = directory / f"{stem}{extension}" + if candidate.exists(): + return candidate + return None + + +def _validate_json_project_in_project_env( + crew_path: Path, agents_dir: Path, project_root: Path +) -> list[str]: + """Validate a JSON crew with the full CrewAI package from its project env.""" + uv_path = shutil.which("uv") + if uv_path is None: + raise _JSONProjectEnvironmentError( + "The `uv` executable is required to validate JSON crews from a " + "standalone CLI installation." + ) + + try: + proc = subprocess.run( # noqa: S603 - fixed command plus trusted paths + [ + uv_path, + "run", + "python", + "-c", + _JSON_VALIDATOR_SCRIPT, + str(crew_path), + str(agents_dir), + ], + cwd=project_root, + capture_output=True, + text=True, + timeout=120, + check=False, + ) + except subprocess.TimeoutExpired as exc: + raise _JSONProjectEnvironmentError( + "JSON crew validation timed out after 120s." + ) from exc + except OSError as exc: + raise _JSONProjectEnvironmentError( + f"Could not start JSON crew validation: {exc}" + ) from exc + + payload: dict[str, Any] | None = None + for line in reversed(proc.stdout.splitlines()): + if not line.startswith(_JSON_VALIDATION_MARKER): + continue + try: + payload = json.loads(line.removeprefix(_JSON_VALIDATION_MARKER)) + except json.JSONDecodeError: + pass + break + + if payload is None: + detail = (proc.stderr or proc.stdout or "").strip() + raise _JSONProjectEnvironmentError( + detail or "JSON crew validation produced no result." + ) + + if not payload.get("ok"): + errors = payload.get("errors") + if isinstance(errors, list) and all(isinstance(error, str) for error in errors): + raise _JSONProjectValidationError(errors) + error_type = payload.get("error_type", "Error") + error = payload.get("error", "JSON crew validation failed") + raise _JSONProjectEnvironmentError(f"{error_type}: {error}") + + agent_names = payload.get("agent_names") + if not isinstance(agent_names, list) or not all( + isinstance(name, str) for name in agent_names + ): + raise _JSONProjectEnvironmentError( + "JSON crew validation returned invalid agent names." + ) + return agent_names + + +def _validate_json_project( + crew_path: Path, agents_dir: Path, project_root: Path +) -> list[str]: + """Validate locally when possible, otherwise use the project's environment.""" + try: + from crewai.project.json_loader import ( + JSONProjectValidationError, + validate_crew_project, + ) + except ModuleNotFoundError as exc: + if exc.name and (exc.name == "crewai" or exc.name.startswith("crewai.")): + return _validate_json_project_in_project_env( + crew_path, agents_dir, project_root + ) + raise + + try: + project = validate_crew_project(crew_path, agents_dir) + except JSONProjectValidationError as exc: + raise _JSONProjectValidationError(exc.errors) from exc + return project.agent_names class DeployValidator: @@ -232,11 +357,13 @@ class DeployValidator: agents_dir = crew_path.parent / "agents" agents_dir_ok = self._check_json_agents_dir(agents_dir) - project = None + agent_names: list[str] | None = None try: if agents_dir_ok: - project = validate_crew_project(crew_path, agents_dir) - except JSONProjectValidationError as e: + agent_names = _validate_json_project( + crew_path, agents_dir, self.project_root + ) + except _JSONProjectValidationError as e: self._add( Severity.ERROR, "invalid_crew_json", @@ -245,6 +372,15 @@ class DeployValidator: hint="Fix the JSON crew, agent, and task references before deploying.", ) return self.results + except _JSONProjectEnvironmentError as e: + self._add( + Severity.ERROR, + "json_validation_environment_failed", + "Could not validate the JSON crew in the project environment", + detail=str(e), + hint=e.hint, + ) + return self.results except Exception as e: self._add( Severity.ERROR, @@ -254,8 +390,8 @@ class DeployValidator: ) return self.results - if project is not None: - self._check_env_vars_json(crew_path, agents_dir, project.agent_names) + if agent_names is not None: + self._check_env_vars_json(crew_path, agents_dir, agent_names) self._check_version_vs_lockfile() return self.results @@ -288,7 +424,7 @@ class DeployValidator: logger.debug("Skipping unreadable crew file %s: %s", crew_path, exc) for name in agent_names: - agent_path = find_json_project_file(agents_dir, name) + agent_path = _find_json_project_file(agents_dir, name) if agent_path is None: continue try: diff --git a/lib/cli/src/crewai_cli/install_crew.py b/lib/cli/src/crewai_cli/install_crew.py index e1944225f..808eeea7b 100644 --- a/lib/cli/src/crewai_cli/install_crew.py +++ b/lib/cli/src/crewai_cli/install_crew.py @@ -4,8 +4,7 @@ import subprocess import click from crewai_core.project import configured_project_definition, read_toml -from crewai_cli.deploy.validate import normalize_package_name -from crewai_cli.utils import build_env_with_all_tool_credentials +from crewai_cli.utils import build_env_with_all_tool_credentials, normalize_package_name def _is_json_crew_project(project_root: Path | None = None) -> bool: diff --git a/lib/cli/src/crewai_cli/utils.py b/lib/cli/src/crewai_cli/utils.py index df77341df..a4285f650 100644 --- a/lib/cli/src/crewai_cli/utils.py +++ b/lib/cli/src/crewai_cli/utils.py @@ -39,6 +39,7 @@ __all__ = [ "get_project_version", "is_dmn_mode_enabled", "load_env_vars", + "normalize_package_name", "parse_toml", "read_toml", "render_template", @@ -67,6 +68,12 @@ console = Console() _TEMPLATE_TOKEN_RE = re.compile(r"{{([a-zA-Z_][a-zA-Z0-9_]*)}}") +def normalize_package_name(project_name: str) -> str: + """Normalize a project name into its scaffolded Python package name.""" + folder = project_name.replace(" ", "_").replace("-", "_").lower() + return re.sub(r"[^a-zA-Z0-9_]", "", folder) + + def is_dmn_mode_enabled() -> bool: """Return True when the enterprise non-interactive mode is enabled.""" value = os.environ.get("CREWAI_DMN") diff --git a/lib/cli/tests/deploy/test_cli_only_install.py b/lib/cli/tests/deploy/test_cli_only_install.py new file mode 100644 index 000000000..e742d60db --- /dev/null +++ b/lib/cli/tests/deploy/test_cli_only_install.py @@ -0,0 +1,148 @@ +"""Regression coverage for crewai-cli installed without the full crewai package.""" + +import builtins +import json +from pathlib import Path +import shutil +import subprocess +import sys +from typing import Any + +import pytest + +import crewai_cli.deploy.validate as validate_module + + +def test_reported_commands_run_without_crewai() -> None: + script = r""" +import builtins +import os +from pathlib import Path +import subprocess + +from click.testing import CliRunner + +os.environ["CREWAI_DISABLE_TELEMETRY"] = "true" + +real_import = builtins.__import__ + +def import_without_crewai(name, *args, **kwargs): + if name == "crewai" or name.startswith("crewai."): + raise ModuleNotFoundError("No module named 'crewai'", name="crewai") + return real_import(name, *args, **kwargs) + +builtins.__import__ = import_without_crewai + +from crewai_cli.cli import crewai +import crewai_cli.command as command_module +import crewai_cli.install_crew as install_module + +def not_logged_in(): + raise RuntimeError("not logged in") + +command_module.get_auth_token = not_logged_in +install_module.build_env_with_all_tool_credentials = lambda: {} +install_module.subprocess.run = lambda *args, **kwargs: subprocess.CompletedProcess( + args[0], 0 +) + +runner = CliRunner() +with runner.isolated_filesystem(): + Path("pyproject.toml").write_text('[project]\nname = "demo"\n') + + deploy_result = runner.invoke(crewai, ["deploy", "list"]) + assert deploy_result.exit_code == 0, deploy_result.output + assert "Please sign up/login" in deploy_result.output + assert not isinstance(deploy_result.exception, ModuleNotFoundError) + + install_result = runner.invoke(crewai, ["install"]) + assert install_result.exit_code == 0, install_result.output + assert not isinstance(install_result.exception, ModuleNotFoundError) +""" + + proc = subprocess.run( # noqa: S603 + [sys.executable, "-c", script], + capture_output=True, + text=True, + check=False, + ) + + assert proc.returncode == 0, proc.stderr + + +def test_json_validation_uses_project_environment_without_crewai( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + real_import = builtins.__import__ + + def import_without_crewai(name: str, *args: Any, **kwargs: Any) -> Any: + if name == "crewai" or name.startswith("crewai."): + raise ModuleNotFoundError("No module named 'crewai'", name="crewai") + return real_import(name, *args, **kwargs) + + captured: dict[str, Any] = {} + + def fake_run( + command: list[str], **kwargs: Any + ) -> subprocess.CompletedProcess[str]: + captured["command"] = command + captured["kwargs"] = kwargs + payload = {"ok": True, "agent_names": ["researcher"]} + return subprocess.CompletedProcess( + command, + 0, + stdout=( + "uv output\n" + f"{validate_module._JSON_VALIDATION_MARKER}{json.dumps(payload)}\n" + ), + stderr="", + ) + + monkeypatch.setattr(builtins, "__import__", import_without_crewai) + monkeypatch.setattr(shutil, "which", lambda command: "/usr/bin/uv") + monkeypatch.setattr(subprocess, "run", fake_run) + + crew_path = tmp_path / "crew.jsonc" + agents_dir = tmp_path / "agents" + assert validate_module._validate_json_project( + crew_path, agents_dir, tmp_path + ) == ["researcher"] + + assert captured["command"][:4] == ["/usr/bin/uv", "run", "python", "-c"] + assert captured["kwargs"]["cwd"] == tmp_path + + +def test_project_environment_preserves_json_validation_errors( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + payload = {"ok": False, "errors": ["tasks[0] references missing_agent"]} + proc = subprocess.CompletedProcess( + [], + 0, + stdout=f"{validate_module._JSON_VALIDATION_MARKER}{json.dumps(payload)}\n", + stderr="", + ) + monkeypatch.setattr(shutil, "which", lambda command: "/usr/bin/uv") + monkeypatch.setattr(subprocess, "run", lambda *args, **kwargs: proc) + + with pytest.raises(validate_module._JSONProjectValidationError) as exc_info: + validate_module._validate_json_project_in_project_env( + tmp_path / "crew.jsonc", tmp_path / "agents", tmp_path + ) + + assert exc_info.value.errors == ["tasks[0] references missing_agent"] + + +def test_missing_uv_has_an_actionable_project_environment_error( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + monkeypatch.setattr(shutil, "which", lambda command: None) + + with pytest.raises(validate_module._JSONProjectEnvironmentError) as exc_info: + validate_module._validate_json_project_in_project_env( + tmp_path / "crew.jsonc", tmp_path / "agents", tmp_path + ) + + assert "required" in str(exc_info.value) + assert "Install `uv`" in exc_info.value.hint + assert "uv sync" in exc_info.value.hint diff --git a/lib/cli/tests/deploy/test_validate.py b/lib/cli/tests/deploy/test_validate.py index 31ecb0c8b..b79d1e035 100644 --- a/lib/cli/tests/deploy/test_validate.py +++ b/lib/cli/tests/deploy/test_validate.py @@ -16,6 +16,7 @@ import pytest from crewai_cli.deploy.validate import ( DeployValidator, Severity, + _JSONProjectEnvironmentError, normalize_package_name, ) @@ -205,6 +206,33 @@ def test_json_runtime_fields_are_deploy_errors(tmp_path: Path) -> None: assert "runtime-only" in finding.detail +def test_json_project_environment_failure_has_actionable_hint( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + _scaffold_json_crew(tmp_path) + + def fail_in_project_environment(*args: object) -> list[str]: + raise _JSONProjectEnvironmentError("uv run failed") + + monkeypatch.setattr( + "crewai_cli.deploy.validate._validate_json_project", + fail_in_project_environment, + ) + + validator = DeployValidator(project_root=tmp_path) + validator.run() + + finding = next( + result + for result in validator.results + if result.code == "json_validation_environment_failed" + ) + assert finding.title == "Could not validate the JSON crew in the project environment" + assert finding.detail == "uv run failed" + assert "Install `uv`" in finding.hint + assert "uv sync" in finding.hint + + def test_json_crew_requires_agents_dir_without_classic_errors(tmp_path: Path) -> None: _scaffold_json_crew(tmp_path) for path in (tmp_path / "agents").iterdir():