deepagents/libs/evals/scripts/generate_model_groups.py

"""Generate MODEL_GROUPS.md from the canonical model registry.

Usage:
    python scripts/generate_model_groups.py          # writes MODEL_GROUPS.md
    python scripts/generate_model_groups.py --check   # exits 1 if file is stale
"""

from __future__ import annotations

import argparse
import importlib.util
from pathlib import Path
from typing import TYPE_CHECKING

if TYPE_CHECKING:
    import types

_REPO_ROOT = Path(__file__).resolve().parents[3]
_EVALS_DIR = Path(__file__).resolve().parents[1]
_OUTPUT = _EVALS_DIR / "MODEL_GROUPS.md"

_HEADER = """\
<!-- AUTO-GENERATED by scripts/generate_model_groups.py — do not edit manually. -->
# Eval model groups

Quick reference for the model sets available in the
[evals workflow](../../.github/workflows/evals.yml).
Source of truth: [`.github/scripts/models.py`](../../.github/scripts/models.py).
"""


def _import_models() -> types.ModuleType:
    """Import `.github/scripts/models.py` by file path (it has no package structure)."""
    models_path = _REPO_ROOT / ".github" / "scripts" / "models.py"
    spec = importlib.util.spec_from_file_location("models", models_path)
    if spec is None or spec.loader is None:
        msg = f"Could not create import spec for {models_path}."
        raise ImportError(msg)
    mod = importlib.util.module_from_spec(spec)
    spec.loader.exec_module(mod)
    return mod


def _provider_heading(preset_name: str, registry: tuple) -> str:
    """Render the provider-section heading, prefixing the human label when distinct.

    Returns `Anthropic (anthropic)` when at least one registered model has a
    `provider_label` that differs case-insensitively from the preset name (the
    raw provider prefix), and `anthropic` otherwise. The check looks at the
    registry rather than a fixed mapping so a future provider whose label
    happens to lowercase to its prefix (e.g. `Ollama` → `ollama`) keeps
    the compact form automatically.
    """
    tag = f"eval:{preset_name}"
    for m in registry:
        if tag not in m.groups:
            continue
        label = m.provider_label
        if label and label.lower() != preset_name.lower():
            return f"{label} (`{preset_name}`)"
        # First match decides — provider_label is uniform within a provider.
        break
    return f"`{preset_name}`"


def generate() -> str:
    """Return the full markdown content for MODEL_GROUPS.md."""
    mod = _import_models()
    registry: tuple = mod.REGISTRY
    sections: list[tuple[str | None, list[tuple[str, str | None]]]] = mod._PRESET_SECTIONS  # noqa: SLF001

    lines: list[str] = [_HEADER]

    for section_name, presets in sections:
        if section_name is not None:
            lines.append(f"## {section_name}\n")

        for preset_name, tag_suffix in presets:
            if tag_suffix is not None:
                tag = f"eval:{tag_suffix}"
                models = [m.spec for m in registry if tag in m.groups]
            else:
                # "all" — every model with any eval: tag
                models = [m.spec for m in registry if any(g.startswith("eval:") for g in m.groups)]

            count = len(models)
            label = "model" if count == 1 else "models"
            heading = "##" if section_name is None else "###"
            if section_name == "Provider groups":
                title = _provider_heading(preset_name, registry)
            else:
                title = f"`{preset_name}`"
            lines.append(f"{heading} {title} ({count} {label})\n")
            # Sort alphabetically for human scannability — REGISTRY order is
            # preserved upstream where it matters (matrix routing).
            lines.extend(f"- `{spec}`" for spec in sorted(models))
            lines.append("")

    return "\n".join(lines)


def main() -> None:
    """Entry point."""
    parser = argparse.ArgumentParser()
    parser.add_argument(
        "--check",
        action="store_true",
        help="Check that MODEL_GROUPS.md is up-to-date (exit 1 if stale).",
    )
    args = parser.parse_args()

    expected = generate()

    if args.check:
        if not _OUTPUT.exists():
            print(f"MISSING: {_OUTPUT}")
            raise SystemExit(1)
        actual = _OUTPUT.read_text()
        if actual != expected:
            print(f"STALE: {_OUTPUT}")
            print("Run `make model-groups` from libs/evals/ to regenerate.")
            raise SystemExit(1)
        print(f"OK: {_OUTPUT} is up-to-date")
    else:
        _OUTPUT.write_text(expected)
        print(f"Wrote {_OUTPUT}")


if __name__ == "__main__":
    main()