.

.env.example

@@ -56,6 +56,31 @@ ANTHROPIC_API_KEY=
 # =========================
 OPENROUTER_API_KEY=
 
+# =========================
+# Speech providers (voice mode TTS/STT)
+# =========================
+# A USE_*_TTS / USE_*_STT flag picks the provider for each family, and the
+# DEEPEVAL_*_MODEL vars pick the model name. Both default to OpenAI.
+# ELEVENLABS_API_KEY=
+# DEEPGRAM_API_KEY=
+# CARTESIA_API_KEY=
+# ASSEMBLYAI_API_KEY=
+
+# Text-to-speech: pick at most one.
+# USE_OPENAI_TTS=
+# USE_ELEVENLABS_TTS=
+# USE_CARTESIA_TTS=
+# USE_DEEPGRAM_TTS=
+# DEEPEVAL_TTS_MODEL=
+
+# Speech-to-text: pick at most one.
+# USE_OPENAI_STT=
+# USE_ELEVENLABS_STT=
+# USE_CARTESIA_STT=
+# USE_DEEPGRAM_STT=
+# USE_ASSEMBLYAI_STT=
+# DEEPEVAL_STT_MODEL=
+
 # =========================
 # Confident
 # =========================

b.py

@@ -12,8 +12,8 @@ from deepeval.dataset import (
     Persona,
 )
 from deepeval.models.llms import OpenAIModel
-from deepeval.models.stt_models import OpenAISTTModel
-from deepeval.models.tts_models import OpenAITTSModel
+from deepeval.models.stt import OpenAISTTModel
+from deepeval.models.tts import OpenAITTSModel
 from deepeval.simulator import ConversationSimulator
 from deepeval.voice import CallbackVoiceConnector, VoiceConfig
 from deepeval.voice.timeline import build_audio_timeline

c.py

@@ -59,8 +59,8 @@ from contextlib import contextmanager
 from typing import Dict, List, Optional, Tuple
 
 from deepeval.models.llms import OpenAIModel
-from deepeval.models.stt_models import OpenAISTTModel
-from deepeval.models.tts_models import OpenAITTSModel
+from deepeval.models.stt import OpenAISTTModel
+from deepeval.models.tts import OpenAITTSModel
 from deepeval.test_case import Audio
 from deepeval.voice.connectors import audio_utils
 

deepeval/cli/app.py

@@ -0,0 +1,6 @@
+"""The root Typer app, kept apart from `main` so command modules can register
+onto it without importing `main` back."""
+
+import typer
+
+app = typer.Typer(name="deepeval", no_args_is_help=True)

deepeval/cli/diagnose/diagnose.py

@@ -40,6 +40,7 @@ from deepeval.key_handler import (
     EmbeddingKeyValues,
     KeyValues,
     ModelKeyValues,
+    SpeechKeyValues,
 )
 
 # Friendly provider labels for the model classes `initialize_model` /
@@ -63,6 +64,15 @@ _PROVIDER_BY_CLASS = {
     "AzureOpenAIEmbeddingModel": "Azure OpenAI",
     "OllamaEmbeddingModel": "Ollama",
     "LocalEmbeddingModel": "Local model",
+    "OpenAITTSModel": "OpenAI",
+    "ElevenLabsTTSModel": "ElevenLabs",
+    "CartesiaTTSModel": "Cartesia",
+    "DeepgramTTSModel": "Deepgram",
+    "OpenAISTTModel": "OpenAI",
+    "ElevenLabsSTTModel": "ElevenLabs",
+    "CartesiaSTTModel": "Cartesia",
+    "DeepgramSTTModel": "Deepgram",
+    "AssemblyAISTTModel": "AssemblyAI",
 }
 
 # Settings fields worth showing in the "configured settings" table when set.
@@ -78,6 +88,8 @@ _RELEVANT_MARKERS = (
     "DEEPEVAL_DEFAULT_SAVE",
     "DEEPEVAL_RESULTS_FOLDER",
     "DEEPEVAL_VOICE_FOLDER",
+    "DEEPEVAL_TTS_MODEL",
+    "DEEPEVAL_STT_MODEL",
 )
 
 
@@ -121,7 +133,12 @@ def _legacy_keystore() -> Tuple[Path, Dict[str, Any]]:
 
 def _env_key_to_json_key() -> Dict[str, str]:
     mapping: Dict[str, str] = {}
-    for enum in (KeyValues, ModelKeyValues, EmbeddingKeyValues):
+    for enum in (
+        KeyValues,
+        ModelKeyValues,
+        EmbeddingKeyValues,
+        SpeechKeyValues,
+    ):
         for member in enum:
             mapping[member.name] = member.value
     return mapping
@@ -312,6 +329,22 @@ def _models_section() -> Dict[str, Any]:
         "(override with embedder=... in ContextConstructionConfig)"
     )
 
+    # Reported from the selection alone: those constructors need an API key.
+    from deepeval.models.speech_selection import (
+        describe_stt_selection,
+        describe_tts_selection,
+    )
+
+    for key, (class_name, model_name) in (
+        ("tts", describe_tts_selection()),
+        ("stt", describe_stt_selection()),
+    ):
+        result[key] = {
+            "provider": _PROVIDER_BY_CLASS.get(class_name, class_name),
+            "model": model_name or "provider default",
+            "used_by": "voice simulations",
+        }
+
     return result
 
 
@@ -440,7 +473,12 @@ def diagnose_command(
     # Default models (global, overridable per class)
     models = report["default_models"]
     table = _kv_table("Default models")
-    for label, key in (("LLM", "llm"), ("Embeddings", "embeddings")):
+    for label, key in (
+        ("LLM", "llm"),
+        ("Embeddings", "embeddings"),
+        ("TTS", "tts"),
+        ("STT", "stt"),
+    ):
         info = models[key]
         if "error" in info:
             value = f"[red]⚠ unusable until fixed: {info['error']}[/red]"

deepeval/cli/providers/__init__.py

@@ -0,0 +1,3 @@
+"""Provider configuration commands, registered on import."""
+
+from deepeval.cli.providers import llms, tts, stt  # noqa: F401

deepeval/cli/providers/_speech.py

@@ -0,0 +1,122 @@
+"""Helpers shared by the `set-tts` / `set-stt` command families."""
+
+from typing import List, Optional
+
+import typer
+from rich import print
+from rich.markup import escape
+
+from deepeval.cli.utils import (
+    coerce_blank_to_none,
+    handle_save_result as _handle_save_result,
+    is_openai_configured,
+)
+from deepeval.config.settings import get_settings
+from deepeval.key_handler import SpeechKeyValues
+
+
+# provider -> (display name, API key setting)
+_SPEECH_PROVIDER_INFO = {
+    "openai": ("OpenAI", "OPENAI_API_KEY"),
+    "elevenlabs": ("ElevenLabs", "ELEVENLABS_API_KEY"),
+    "cartesia": ("Cartesia", "CARTESIA_API_KEY"),
+    "deepgram": ("Deepgram", "DEEPGRAM_API_KEY"),
+    "assemblyai": ("AssemblyAI", "ASSEMBLYAI_API_KEY"),
+}
+
+
+def _set_speech_provider(
+    *,
+    family: str,
+    provider: str,
+    model: Optional[str],
+    prompt_api_key: bool,
+    save: Optional[str],
+    quiet: bool,
+) -> None:
+    label, key_field = _SPEECH_PROVIDER_INFO[provider]
+    flag = SpeechKeyValues[f"USE_{provider.upper()}_{family}"]
+    model_field = f"DEEPEVAL_{family}_MODEL"
+
+    api_key = None
+    if prompt_api_key:
+        api_key = coerce_blank_to_none(
+            typer.prompt(f"{label} API key", hide_input=True)
+        )
+
+    model = coerce_blank_to_none(model)
+
+    settings = get_settings()
+    with settings.edit(save=save) as edit_ctx:
+        edit_ctx.switch_model_provider(flag)
+        if model is not None:
+            setattr(settings, model_field, model)
+        if api_key is not None:
+            setattr(settings, key_field, api_key)
+
+    handled, path, updates = edit_ctx.result
+
+    effective_model = getattr(settings, model_field)
+    model_phrase = (
+        f"`{escape(effective_model)}`"
+        if effective_model
+        else "with its default model"
+    )
+
+    _handle_save_result(
+        handled=handled,
+        path=path,
+        updates=updates,
+        save=save,
+        quiet=quiet,
+        success_msg=(
+            f":raising_hands: Congratulations! You're now using {label} "
+            f"{model_phrase} for {family} in voice simulations."
+        ),
+    )
+
+
+def _unset_speech_provider(
+    *,
+    family: str,
+    flags: List[str],
+    clear_secrets: bool,
+    save: Optional[str],
+    quiet: bool,
+) -> None:
+    settings = get_settings()
+    with settings.edit(save=save) as edit_ctx:
+        for flag in flags:
+            setattr(settings, flag, None)
+        setattr(settings, f"DEEPEVAL_{family}_MODEL", None)
+        if clear_secrets:
+            # OPENAI_API_KEY is deliberately left alone: it is shared with the
+            # LLM providers, so clearing it here would reach outside speech.
+            for flag in flags:
+                _, key_field = _SPEECH_PROVIDER_INFO[
+                    flag[len("USE_") : -len(f"_{family}")].lower()
+                ]
+                if key_field != "OPENAI_API_KEY":
+                    setattr(settings, key_field, None)
+
+    handled, path, updates = edit_ctx.result
+
+    if _handle_save_result(
+        handled=handled,
+        path=path,
+        updates=updates,
+        save=save,
+        quiet=quiet,
+        updated_msg=(f"Removed {family} environment variables from {{path}}."),
+        tip_msg=None,
+    ):
+        if is_openai_configured():
+            print(
+                f":raised_hands: OpenAI will still be used for {family} by "
+                "default because OPENAI_API_KEY is set."
+            )
+        else:
+            print(
+                f"The {family} provider has been unset. Voice simulations "
+                "fall back to OpenAI, which needs OPENAI_API_KEY."
+            )

deepeval/cli/providers/llms/__init__.py

@@ -0,0 +1,21 @@
+"""LLM commands, plus the embedding commands of providers that serve both.
+
+Import order is the order `deepeval --help` lists them in.
+"""
+
+from deepeval.cli.providers.llms import (  # noqa: F401
+    openai,
+    azure,
+    anthropic,
+    bedrock,
+    ollama,
+    local_model,
+    grok,
+    moonshot,
+    deepseek,
+    local_embeddings,
+    gemini,
+    litellm,
+    portkey,
+    openrouter,
+)

deepeval/cli/providers/llms/anthropic.py

@@ -0,0 +1,159 @@
+"""`deepeval set-anthropic, unset-anthropic` commands."""
+
+import typer
+from typing import Optional
+from rich import print
+from rich.markup import escape
+
+from deepeval.cli.app import app
+from deepeval.cli.utils import (
+    coerce_blank_to_none,
+    handle_save_result as _handle_save_result,
+    is_openai_configured,
+)
+from deepeval.config.settings import get_settings
+from deepeval.key_handler import (
+    ModelKeyValues,
+)
+
+
+@app.command(name="set-anthropic")
+def set_anthropic_model_env(
+    model: Optional[str] = typer.Option(
+        None,
+        "-m",
+        "--model",
+        help="Model identifier to use for this provider",
+    ),
+    prompt_api_key: bool = typer.Option(
+        False,
+        "-k",
+        "--prompt-api-key",
+        help=(
+            "Prompt for ANTHROPIC_API_KEY (input hidden). Not suitable for CI. "
+            "If --save (or DEEPEVAL_DEFAULT_SAVE) is used, the key is written to dotenv in plaintext."
+        ),
+    ),
+    cost_per_input_token: Optional[float] = typer.Option(
+        None,
+        "-i",
+        "--cost-per-input-token",
+        help=(
+            "USD per input token override used for cost tracking. Preconfigured for known models; "
+            "REQUIRED if you use a custom/unknown model."
+        ),
+    ),
+    cost_per_output_token: Optional[float] = typer.Option(
+        None,
+        "-o",
+        "--cost-per-output-token",
+        help=(
+            "USD per output token override used for cost tracking. Preconfigured for known models; "
+            "REQUIRED if you use a custom/unknown model."
+        ),
+    ),
+    save: Optional[str] = typer.Option(
+        None,
+        "-s",
+        "--save",
+        help="Persist CLI parameters as environment variables in a dotenv file. "
+        "Usage: --save=dotenv[:path] (default: .env.local)",
+    ),
+    quiet: bool = typer.Option(
+        False,
+        "-q",
+        "--quiet",
+        help="Suppress printing to the terminal (useful for CI).",
+    ),
+):
+    api_key = None
+    if prompt_api_key:
+        api_key = coerce_blank_to_none(
+            typer.prompt("Anthropic API key", hide_input=True)
+        )
+
+    model = coerce_blank_to_none(model)
+
+    settings = get_settings()
+    with settings.edit(save=save) as edit_ctx:
+        edit_ctx.switch_model_provider(ModelKeyValues.USE_ANTHROPIC_MODEL)
+        if api_key is not None:
+            settings.ANTHROPIC_API_KEY = api_key
+        if model is not None:
+            settings.ANTHROPIC_MODEL_NAME = model
+        if cost_per_input_token is not None:
+            settings.ANTHROPIC_COST_PER_INPUT_TOKEN = cost_per_input_token
+        if cost_per_output_token is not None:
+            settings.ANTHROPIC_COST_PER_OUTPUT_TOKEN = cost_per_output_token
+
+    handled, path, updates = edit_ctx.result
+
+    effective_model = settings.ANTHROPIC_MODEL_NAME
+    if not effective_model:
+        raise typer.BadParameter(
+            "Anthropic model name is not set. Pass --model (or set ANTHROPIC_MODEL_NAME).",
+            param_hint="--model",
+        )
+
+    _handle_save_result(
+        handled=handled,
+        path=path,
+        updates=updates,
+        save=save,
+        quiet=quiet,
+        success_msg=(
+            f":raising_hands: Congratulations! You're now using Anthropic `{escape(effective_model)}` for all evals that require an LLM."
+        ),
+    )
+
+
+@app.command(name="unset-anthropic")
+def unset_anthropic_model_env(
+    save: Optional[str] = typer.Option(
+        None,
+        "-s",
+        "--save",
+        help="Remove only the Anthropic model related environment variables from a dotenv file. "
+        "Usage: --save=dotenv[:path] (default: .env.local)",
+    ),
+    clear_secrets: bool = typer.Option(
+        False,
+        "-x",
+        "--clear-secrets",
+        help="Also remove ANTHROPIC_API_KEY from the dotenv store.",
+    ),
+    quiet: bool = typer.Option(
+        False,
+        "-q",
+        "--quiet",
+        help="Suppress printing to the terminal (useful for CI).",
+    ),
+):
+    settings = get_settings()
+    with settings.edit(save=save) as edit_ctx:
+        settings.USE_ANTHROPIC_MODEL = None
+        settings.ANTHROPIC_MODEL_NAME = None
+        settings.ANTHROPIC_COST_PER_INPUT_TOKEN = None
+        settings.ANTHROPIC_COST_PER_OUTPUT_TOKEN = None
+        if clear_secrets:
+            settings.ANTHROPIC_API_KEY = None
+
+    handled, path, updates = edit_ctx.result
+
+    if _handle_save_result(
+        handled=handled,
+        path=path,
+        updates=updates,
+        save=save,
+        quiet=quiet,
+        updated_msg="Removed Anthropic model environment variables from {path}.",
+        tip_msg=None,
+    ):
+        if is_openai_configured():
+            print(
+                ":raised_hands: OpenAI will still be used by default because OPENAI_API_KEY is set."
+            )
+        else:
+            print(
+                "The Anthropic model configuration has been removed. No model is currently configured, but you can set one with the CLI or add credentials to .env[.local]."
+            )

deepeval/cli/providers/llms/azure.py

@@ -0,0 +1,275 @@
+"""`deepeval set-azure-openai, set-azure-openai-embedding, unset-azure-openai, unset-azure-openai-embedding` commands."""
+
+import typer
+from typing import Optional
+from rich import print
+from rich.markup import escape
+
+from deepeval.cli.app import app
+from deepeval.cli.utils import (
+    coerce_blank_to_none,
+    handle_save_result as _handle_save_result,
+    is_openai_configured,
+)
+from deepeval.config.settings import get_settings
+from deepeval.key_handler import (
+    EmbeddingKeyValues,
+    ModelKeyValues,
+)
+
+
+@app.command(name="set-azure-openai")
+def set_azure_openai_env(
+    model: Optional[str] = typer.Option(
+        None,
+        "-m",
+        "--model",
+        help="Model identifier to use for this provider (e.g., `gpt-4.1`).",
+    ),
+    prompt_api_key: bool = typer.Option(
+        False,
+        "-k",
+        "--prompt-api-key",
+        help=(
+            "Prompt for AZURE_OPENAI_API_KEY (input hidden). Not suitable for CI. "
+            "If --save (or DEEPEVAL_DEFAULT_SAVE) is used, the key is written to dotenv in plaintext."
+        ),
+    ),
+    base_url: Optional[str] = typer.Option(
+        None,
+        "-u",
+        "--base-url",
+        help="Override the API endpoint/base URL used by this provider.",
+    ),
+    api_version: Optional[str] = typer.Option(
+        None,
+        "-v",
+        "--api-version",
+        help="Azure OpenAI API version (passed to the Azure OpenAI client).",
+    ),
+    model_version: Optional[str] = typer.Option(
+        None, "-V", "--model-version", help="Azure model version"
+    ),
+    deployment_name: Optional[str] = typer.Option(
+        None, "-d", "--deployment-name", help="Azure OpenAI deployment name"
+    ),
+    save: Optional[str] = typer.Option(
+        None,
+        "-s",
+        "--save",
+        help="Persist CLI parameters as environment variables in a dotenv file. "
+        "Usage: --save=dotenv[:path] (default: .env.local)",
+    ),
+    quiet: bool = typer.Option(
+        False,
+        "-q",
+        "--quiet",
+        help="Suppress printing to the terminal (useful for CI).",
+    ),
+):
+    api_key = None
+    if prompt_api_key:
+        api_key = coerce_blank_to_none(
+            typer.prompt("Azure OpenAI API key", hide_input=True)
+        )
+
+    model = coerce_blank_to_none(model)
+    base_url = coerce_blank_to_none(base_url)
+    api_version = coerce_blank_to_none(api_version)
+    deployment_name = coerce_blank_to_none(deployment_name)
+    model_version = coerce_blank_to_none(model_version)
+
+    settings = get_settings()
+    with settings.edit(save=save) as edit_ctx:
+        edit_ctx.switch_model_provider(ModelKeyValues.USE_AZURE_OPENAI)
+        if model is not None:
+            settings.AZURE_MODEL_NAME = model
+        if api_key is not None:
+            settings.AZURE_OPENAI_API_KEY = api_key
+        if base_url is not None:
+            settings.AZURE_OPENAI_ENDPOINT = base_url
+        if api_version is not None:
+            settings.OPENAI_API_VERSION = api_version
+        if deployment_name is not None:
+            settings.AZURE_DEPLOYMENT_NAME = deployment_name
+        if model_version is not None:
+            settings.AZURE_MODEL_VERSION = model_version
+
+    handled, path, updates = edit_ctx.result
+
+    effective_model = settings.AZURE_MODEL_NAME
+    if not effective_model:
+        raise typer.BadParameter(
+            "Azure OpenAI model name is not set. Pass --model (or set AZURE_MODEL_NAME).",
+            param_hint="--model",
+        )
+
+    _handle_save_result(
+        handled=handled,
+        path=path,
+        updates=updates,
+        save=save,
+        quiet=quiet,
+        success_msg=(
+            f":raising_hands: Congratulations! You're now using Azure OpenAI's `{escape(effective_model)}` for all evals that require an LLM."
+        ),
+    )
+
+
+@app.command(name="unset-azure-openai")
+def unset_azure_openai_env(
+    save: Optional[str] = typer.Option(
+        None,
+        "-s",
+        "--save",
+        help="Remove only the Azure OpenAI–related environment variables from a dotenv file. "
+        "Usage: --save=dotenv[:path] (default: .env.local)",
+    ),
+    clear_secrets: bool = typer.Option(
+        False,
+        "-x",
+        "--clear-secrets",
+        help="Also remove AZURE_OPENAI_API_KEY from the dotenv store.",
+    ),
+    quiet: bool = typer.Option(
+        False,
+        "-q",
+        "--quiet",
+        help="Suppress printing to the terminal (useful for CI).",
+    ),
+):
+    settings = get_settings()
+    with settings.edit(save=save) as edit_ctx:
+        settings.AZURE_OPENAI_ENDPOINT = None
+        settings.OPENAI_API_VERSION = None
+        settings.AZURE_DEPLOYMENT_NAME = None
+        settings.AZURE_MODEL_NAME = None
+        settings.AZURE_MODEL_VERSION = None
+        settings.USE_AZURE_OPENAI = None
+        if clear_secrets:
+            settings.AZURE_OPENAI_API_KEY = None
+
+    handled, path, updates = edit_ctx.result
+
+    if _handle_save_result(
+        handled=handled,
+        path=path,
+        updates=updates,
+        save=save,
+        quiet=quiet,
+        updated_msg="Removed Azure OpenAI environment variables from {path}.",
+        tip_msg=None,
+    ):
+        if is_openai_configured():
+            print(
+                ":raised_hands: OpenAI will still be used by default because OPENAI_API_KEY is set."
+            )
+        else:
+            print(
+                "Azure OpenAI has been unset. No active provider is configured. Set one with the CLI, or add credentials to .env[.local]."
+            )
+
+
+@app.command(name="set-azure-openai-embedding")
+def set_azure_openai_embedding_env(
+    model: Optional[str] = typer.Option(
+        None,
+        "-m",
+        "--model",
+        help="Model identifier to use for this provider (e.g., `gpt-4.1`).",
+    ),
+    deployment_name: Optional[str] = typer.Option(
+        None,
+        "-d",
+        "--deployment-name",
+        help="Azure embedding deployment name",
+    ),
+    save: Optional[str] = typer.Option(
+        None,
+        "-s",
+        "--save",
+        help="Persist CLI parameters as environment variables in a dotenv file. "
+        "Usage: --save=dotenv[:path] (default: .env.local)",
+    ),
+    quiet: bool = typer.Option(
+        False,
+        "-q",
+        "--quiet",
+        help="Suppress printing to the terminal (useful for CI).",
+    ),
+):
+    model = coerce_blank_to_none(model)
+    deployment_name = coerce_blank_to_none(deployment_name)
+
+    settings = get_settings()
+    with settings.edit(save=save) as edit_ctx:
+        edit_ctx.switch_model_provider(
+            EmbeddingKeyValues.USE_AZURE_OPENAI_EMBEDDING
+        )
+        if model is not None:
+            settings.AZURE_EMBEDDING_MODEL_NAME = model
+        if deployment_name is not None:
+            settings.AZURE_EMBEDDING_DEPLOYMENT_NAME = deployment_name
+
+    handled, path, updates = edit_ctx.result
+
+    effective_model = settings.AZURE_EMBEDDING_MODEL_NAME
+    if not effective_model:
+        raise typer.BadParameter(
+            "Azure OpenAI embedding model name is not set. Pass --model (or set AZURE_EMBEDDING_MODEL_NAME).",
+            param_hint="--model",
+        )
+
+    _handle_save_result(
+        handled=handled,
+        path=path,
+        updates=updates,
+        save=save,
+        quiet=quiet,
+        success_msg=(
+            f":raising_hands: Congratulations! You're now using Azure OpenAI embedding model `{escape(effective_model)}` for all evals that require text embeddings."
+        ),
+    )
+
+
+@app.command(name="unset-azure-openai-embedding")
+def unset_azure_openai_embedding_env(
+    save: Optional[str] = typer.Option(
+        None,
+        "-s",
+        "--save",
+        help="Remove only the Azure OpenAI embedding related environment variables from a dotenv file. "
+        "Usage: --save=dotenv[:path] (default: .env.local)",
+    ),
+    quiet: bool = typer.Option(
+        False,
+        "-q",
+        "--quiet",
+        help="Suppress printing to the terminal (useful for CI).",
+    ),
+):
+    settings = get_settings()
+    with settings.edit(save=save) as edit_ctx:
+        settings.AZURE_EMBEDDING_MODEL_NAME = None
+        settings.AZURE_EMBEDDING_DEPLOYMENT_NAME = None
+        settings.USE_AZURE_OPENAI_EMBEDDING = None
+
+    handled, path, updates = edit_ctx.result
+
+    if _handle_save_result(
+        handled=handled,
+        path=path,
+        updates=updates,
+        save=save,
+        quiet=quiet,
+        updated_msg="Removed Azure OpenAI embedding environment variables from {path}.",
+        tip_msg=None,
+    ):
+        if is_openai_configured():
+            print(
+                ":raised_hands: OpenAI will still be used by default because OPENAI_API_KEY is set."
+            )
+        else:
+            print(
+                "Azure OpenAI embedding has been unset. No active provider is configured. Set one with the CLI, or add credentials to .env[.local]."
+            )

deepeval/cli/providers/llms/bedrock.py

@@ -0,0 +1,174 @@
+"""`deepeval set-bedrock, unset-bedrock` commands."""
+
+import typer
+from typing import Optional
+from rich import print
+from rich.markup import escape
+
+from deepeval.cli.app import app
+from deepeval.cli.utils import (
+    coerce_blank_to_none,
+    handle_save_result as _handle_save_result,
+    is_openai_configured,
+)
+from deepeval.config.settings import get_settings
+from deepeval.key_handler import (
+    ModelKeyValues,
+)
+
+
+@app.command(name="set-bedrock")
+def set_bedrock_model_env(
+    model: Optional[str] = typer.Option(
+        None,
+        "-m",
+        "--model",
+        help="Model identifier to use for this provider",
+    ),
+    prompt_credentials: bool = typer.Option(
+        False,
+        "-a",
+        "--prompt-credentials",
+        help=(
+            "Prompt for AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY (secret access key input is hidden). Not suitable for CI. "
+            "If --save (or DEEPEVAL_DEFAULT_SAVE) is used, credentials are written to dotenv in plaintext."
+        ),
+    ),
+    region: Optional[str] = typer.Option(
+        None,
+        "-r",
+        "--region",
+        help="AWS region for bedrock (e.g., `us-east-1`).",
+    ),
+    cost_per_input_token: Optional[float] = typer.Option(
+        None,
+        "-i",
+        "--cost-per-input-token",
+        help=(
+            "USD per input token override used for cost tracking. Preconfigured for known models; "
+            "REQUIRED if you use a custom/unknown model."
+        ),
+    ),
+    cost_per_output_token: Optional[float] = typer.Option(
+        None,
+        "-o",
+        "--cost-per-output-token",
+        help=(
+            "USD per output token override used for cost tracking. Preconfigured for known models; "
+            "REQUIRED if you use a custom/unknown model."
+        ),
+    ),
+    save: Optional[str] = typer.Option(
+        None,
+        "-s",
+        "--save",
+        help="Persist CLI parameters as environment variables in a dotenv file. "
+        "Usage: --save=dotenv[:path] (default: .env.local)",
+    ),
+    quiet: bool = typer.Option(
+        False,
+        "-q",
+        "--quiet",
+        help="Suppress printing to the terminal (useful for CI).",
+    ),
+):
+    access_key_id = None
+    secret_access_key = None
+    if prompt_credentials:
+        access_key_id = coerce_blank_to_none(typer.prompt("AWS Access key Id"))
+        secret_access_key = coerce_blank_to_none(
+            typer.prompt("AWS Secret Access key", hide_input=True)
+        )
+
+    model = coerce_blank_to_none(model)
+    region = coerce_blank_to_none(region)
+
+    settings = get_settings()
+    with settings.edit(save=save) as edit_ctx:
+        edit_ctx.switch_model_provider(ModelKeyValues.USE_AWS_BEDROCK_MODEL)
+        if access_key_id is not None:
+            settings.AWS_ACCESS_KEY_ID = access_key_id
+        if secret_access_key is not None:
+            settings.AWS_SECRET_ACCESS_KEY = secret_access_key
+        if model is not None:
+            settings.AWS_BEDROCK_MODEL_NAME = model
+        if region is not None:
+            settings.AWS_BEDROCK_REGION = region
+        if cost_per_input_token is not None:
+            settings.AWS_BEDROCK_COST_PER_INPUT_TOKEN = cost_per_input_token
+        if cost_per_output_token is not None:
+            settings.AWS_BEDROCK_COST_PER_OUTPUT_TOKEN = cost_per_output_token
+
+    handled, path, updates = edit_ctx.result
+
+    effective_model = settings.AWS_BEDROCK_MODEL_NAME
+    if not effective_model:
+        raise typer.BadParameter(
+            "AWS Bedrock model name is not set. Pass --model (or set AWS_BEDROCK_MODEL_NAME).",
+            param_hint="--model",
+        )
+
+    _handle_save_result(
+        handled=handled,
+        path=path,
+        updates=updates,
+        save=save,
+        quiet=quiet,
+        success_msg=(
+            f":raising_hands: Congratulations! You're now using AWS Bedrock `{escape(effective_model)}` for all evals that require an LLM."
+        ),
+    )
+
+
+@app.command(name="unset-bedrock")
+def unset_bedrock_model_env(
+    save: Optional[str] = typer.Option(
+        None,
+        "-s",
+        "--save",
+        help="Remove only the AWS Bedrock model related environment variables from a dotenv file. "
+        "Usage: --save=dotenv[:path] (default: .env.local)",
+    ),
+    clear_secrets: bool = typer.Option(
+        False,
+        "-x",
+        "--clear-secrets",
+        help="Also remove AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY  from the dotenv store.",
+    ),
+    quiet: bool = typer.Option(
+        False,
+        "-q",
+        "--quiet",
+        help="Suppress printing to the terminal (useful for CI).",
+    ),
+):
+    settings = get_settings()
+    with settings.edit(save=save) as edit_ctx:
+        settings.USE_AWS_BEDROCK_MODEL = None
+        settings.AWS_BEDROCK_MODEL_NAME = None
+        settings.AWS_BEDROCK_REGION = None
+        settings.AWS_BEDROCK_COST_PER_INPUT_TOKEN = None
+        settings.AWS_BEDROCK_COST_PER_OUTPUT_TOKEN = None
+        if clear_secrets:
+            settings.AWS_ACCESS_KEY_ID = None
+            settings.AWS_SECRET_ACCESS_KEY = None
+
+    handled, path, updates = edit_ctx.result
+
+    if _handle_save_result(
+        handled=handled,
+        path=path,
+        updates=updates,
+        save=save,
+        quiet=quiet,
+        updated_msg="Removed AWS Bedrock model environment variables from {path}.",
+        tip_msg=None,
+    ):
+        if is_openai_configured():
+            print(
+                ":raised_hands: OpenAI will still be used by default because OPENAI_API_KEY is set."
+            )
+        else:
+            print(
+                "The AWS Bedrock model configuration has been removed. No model is currently configured, but you can set one with the CLI or add credentials to .env[.local]."
+            )

deepeval/cli/providers/llms/deepseek.py

@@ -0,0 +1,159 @@
+"""`deepeval set-deepseek, unset-deepseek` commands."""
+
+import typer
+from typing import Optional
+from rich import print
+from rich.markup import escape
+
+from deepeval.cli.app import app
+from deepeval.cli.utils import (
+    coerce_blank_to_none,
+    handle_save_result as _handle_save_result,
+    is_openai_configured,
+)
+from deepeval.config.settings import get_settings
+from deepeval.key_handler import (
+    ModelKeyValues,
+)
+
+
+@app.command(name="set-deepseek")
+def set_deepseek_model_env(
+    model: Optional[str] = typer.Option(
+        None,
+        "-m",
+        "--model",
+        help="Model identifier to use for this provider",
+    ),
+    prompt_api_key: bool = typer.Option(
+        False,
+        "-k",
+        "--prompt-api-key",
+        help=(
+            "Prompt for DEEPSEEK_API_KEY (input hidden). Not suitable for CI. "
+            "If --save (or DEEPEVAL_DEFAULT_SAVE) is used, the key is written to dotenv in plaintext."
+        ),
+    ),
+    cost_per_input_token: Optional[float] = typer.Option(
+        None,
+        "-i",
+        "--cost-per-input-token",
+        help=(
+            "USD per input token override used for cost tracking. Preconfigured for known models; "
+            "REQUIRED if you use a custom/unknown model."
+        ),
+    ),
+    cost_per_output_token: Optional[float] = typer.Option(
+        None,
+        "-o",
+        "--cost-per-output-token",
+        help=(
+            "USD per output token override used for cost tracking. Preconfigured for known models; "
+            "REQUIRED if you use a custom/unknown model."
+        ),
+    ),
+    save: Optional[str] = typer.Option(
+        None,
+        "-s",
+        "--save",
+        help="Persist CLI parameters as environment variables in a dotenv file. "
+        "Usage: --save=dotenv[:path] (default: .env.local)",
+    ),
+    quiet: bool = typer.Option(
+        False,
+        "-q",
+        "--quiet",
+        help="Suppress printing to the terminal (useful for CI).",
+    ),
+):
+    api_key = None
+    if prompt_api_key:
+        api_key = coerce_blank_to_none(
+            typer.prompt("DeepSeek API key", hide_input=True)
+        )
+
+    model = coerce_blank_to_none(model)
+
+    settings = get_settings()
+    with settings.edit(save=save) as edit_ctx:
+        edit_ctx.switch_model_provider(ModelKeyValues.USE_DEEPSEEK_MODEL)
+        if model is not None:
+            settings.DEEPSEEK_MODEL_NAME = model
+        if api_key is not None:
+            settings.DEEPSEEK_API_KEY = api_key
+        if cost_per_input_token is not None:
+            settings.DEEPSEEK_COST_PER_INPUT_TOKEN = cost_per_input_token
+        if cost_per_output_token is not None:
+            settings.DEEPSEEK_COST_PER_OUTPUT_TOKEN = cost_per_output_token
+
+    handled, path, updates = edit_ctx.result
+
+    effective_model = settings.DEEPSEEK_MODEL_NAME
+    if not effective_model:
+        raise typer.BadParameter(
+            "DeepSeek model name is not set. Pass --model (or set DEEPSEEK_MODEL_NAME).",
+            param_hint="--model",
+        )
+
+    _handle_save_result(
+        handled=handled,
+        path=path,
+        updates=updates,
+        save=save,
+        quiet=quiet,
+        success_msg=(
+            f":raising_hands: Congratulations! You're now using DeepSeek `{escape(effective_model)}` for all evals that require an LLM."
+        ),
+    )
+
+
+@app.command(name="unset-deepseek")
+def unset_deepseek_model_env(
+    save: Optional[str] = typer.Option(
+        None,
+        "-s",
+        "--save",
+        help="Remove only the DeepSeek model related environment variables from a dotenv file. "
+        "Usage: --save=dotenv[:path] (default: .env.local)",
+    ),
+    clear_secrets: bool = typer.Option(
+        False,
+        "-x",
+        "--clear-secrets",
+        help="Also remove DEEPSEEK_API_KEY from the dotenv store.",
+    ),
+    quiet: bool = typer.Option(
+        False,
+        "-q",
+        "--quiet",
+        help="Suppress printing to the terminal (useful for CI).",
+    ),
+):
+    settings = get_settings()
+    with settings.edit(save=save) as edit_ctx:
+        settings.USE_DEEPSEEK_MODEL = None
+        settings.DEEPSEEK_MODEL_NAME = None
+        settings.DEEPSEEK_COST_PER_INPUT_TOKEN = None
+        settings.DEEPSEEK_COST_PER_OUTPUT_TOKEN = None
+        if clear_secrets:
+            settings.DEEPSEEK_API_KEY = None
+
+    handled, path, updates = edit_ctx.result
+
+    if _handle_save_result(
+        handled=handled,
+        path=path,
+        updates=updates,
+        save=save,
+        quiet=quiet,
+        updated_msg="Removed DeepSeek model environment variables from {path}.",
+        tip_msg=None,
+    ):
+        if is_openai_configured():
+            print(
+                ":raised_hands: OpenAI will still be used by default because OPENAI_API_KEY is set."
+            )
+        else:
+            print(
+                "The DeepSeek model configuration has been removed. No model is currently configured, but you can set one with the CLI or add credentials to .env[.local]."
+            )

deepeval/cli/providers/llms/gemini.py

@@ -0,0 +1,179 @@
+"""`deepeval set-gemini, unset-gemini` commands."""
+
+import typer
+from pathlib import Path
+from typing import Optional
+from rich import print
+from rich.markup import escape
+
+from deepeval.cli.app import app
+from deepeval.cli.utils import (
+    coerce_blank_to_none,
+    handle_save_result as _handle_save_result,
+    is_openai_configured,
+    load_service_account_key_file,
+)
+from deepeval.config.settings import get_settings
+from deepeval.key_handler import (
+    ModelKeyValues,
+)
+
+
+@app.command(name="set-gemini")
+def set_gemini_model_env(
+    model: Optional[str] = typer.Option(
+        None,
+        "-m",
+        "--model",
+        help="Model identifier to use for this provider",
+    ),
+    prompt_api_key: bool = typer.Option(
+        False,
+        "-k",
+        "--prompt-api-key",
+        help=(
+            "Prompt for GOOGLE_API_KEY (input hidden). Not suitable for CI. "
+            "If --save (or DEEPEVAL_DEFAULT_SAVE) is used, the key is written to dotenv in plaintext."
+        ),
+    ),
+    project: Optional[str] = typer.Option(
+        None,
+        "-p",
+        "--project",
+        help="GCP project ID (used by Vertex AI / Gemini when applicable).",
+    ),
+    location: Optional[str] = typer.Option(
+        None,
+        "-l",
+        "--location",
+        help="GCP location/region for Vertex AI (e.g., `us-central1`).",
+    ),
+    service_account_file: Optional[Path] = typer.Option(
+        None,
+        "-S",
+        "--service-account-file",
+        help=("Path to a Google service account JSON key file."),
+        exists=True,
+        dir_okay=False,
+        readable=True,
+    ),
+    save: Optional[str] = typer.Option(
+        None,
+        "-s",
+        "--save",
+        help="Persist CLI parameters as environment variables in a dotenv file. "
+        "Usage: --save=dotenv[:path] (default: .env.local)",
+    ),
+    quiet: bool = typer.Option(
+        False,
+        "-q",
+        "--quiet",
+        help="Suppress printing to the terminal (useful for CI).",
+    ),
+):
+    api_key = None
+    if prompt_api_key:
+        api_key = coerce_blank_to_none(
+            typer.prompt("Google API key", hide_input=True)
+        )
+
+    model = coerce_blank_to_none(model)
+    project = coerce_blank_to_none(project)
+    location = coerce_blank_to_none(location)
+
+    settings = get_settings()
+    with settings.edit(save=save) as edit_ctx:
+        edit_ctx.switch_model_provider(ModelKeyValues.USE_GEMINI_MODEL)
+
+        if model is not None:
+            settings.GEMINI_MODEL_NAME = model
+        if project is not None:
+            settings.GOOGLE_CLOUD_PROJECT = project
+        if location is not None:
+            settings.GOOGLE_CLOUD_LOCATION = location
+        if service_account_file is not None:
+            settings.GOOGLE_SERVICE_ACCOUNT_KEY = load_service_account_key_file(
+                service_account_file
+            )
+        if api_key is not None:
+            settings.GOOGLE_API_KEY = api_key
+            settings.GOOGLE_GENAI_USE_VERTEXAI = False
+        elif (
+            project is not None
+            or location is not None
+            or service_account_file is not None
+        ):
+            settings.GOOGLE_GENAI_USE_VERTEXAI = True
+
+    handled, path, updates = edit_ctx.result
+
+    effective_model = settings.GEMINI_MODEL_NAME
+    if not effective_model:
+        raise typer.BadParameter(
+            "Gemini model name is not set. Pass --model (or set GEMINI_MODEL_NAME).",
+            param_hint="--model",
+        )
+    _handle_save_result(
+        handled=handled,
+        path=path,
+        updates=updates,
+        save=save,
+        quiet=quiet,
+        success_msg=(
+            f":raising_hands: Congratulations! You're now using Gemini `{escape(effective_model)}` for all evals that require an LLM."
+        ),
+    )
+
+
+@app.command(name="unset-gemini")
+def unset_gemini_model_env(
+    save: Optional[str] = typer.Option(
+        None,
+        "-s",
+        "--save",
+        help="Remove only the Gemini related environment variables from a dotenv file. "
+        "Usage: --save=dotenv[:path] (default: .env.local)",
+    ),
+    clear_secrets: bool = typer.Option(
+        False,
+        "-x",
+        "--clear-secrets",
+        help="Also remove GOOGLE_API_KEY and GOOGLE_SERVICE_ACCOUNT_KEY from the dotenv store.",
+    ),
+    quiet: bool = typer.Option(
+        False,
+        "-q",
+        "--quiet",
+        help="Suppress printing to the terminal (useful for CI).",
+    ),
+):
+    settings = get_settings()
+    with settings.edit(save=save) as edit_ctx:
+        settings.USE_GEMINI_MODEL = None
+        settings.GOOGLE_GENAI_USE_VERTEXAI = None
+        settings.GOOGLE_CLOUD_PROJECT = None
+        settings.GOOGLE_CLOUD_LOCATION = None
+        settings.GEMINI_MODEL_NAME = None
+        if clear_secrets:
+            settings.GOOGLE_API_KEY = None
+            settings.GOOGLE_SERVICE_ACCOUNT_KEY = None
+
+    handled, path, updates = edit_ctx.result
+
+    if _handle_save_result(
+        handled=handled,
+        path=path,
+        updates=updates,
+        save=save,
+        quiet=quiet,
+        updated_msg="Removed Gemini model environment variables from {path}.",
+        tip_msg=None,
+    ):
+        if is_openai_configured():
+            print(
+                ":raised_hands: OpenAI will still be used by default because OPENAI_API_KEY is set."
+            )
+        else:
+            print(
+                "The Gemini model configuration has been removed. No model is currently configured, but you can set one with the CLI or add credentials to .env[.local]."
+            )

deepeval/cli/providers/llms/grok.py

@@ -0,0 +1,158 @@
+"""`deepeval set-grok, unset-grok` commands."""
+
+import typer
+from typing import Optional
+from rich import print
+from rich.markup import escape
+
+from deepeval.cli.app import app
+from deepeval.cli.utils import (
+    coerce_blank_to_none,
+    handle_save_result as _handle_save_result,
+    is_openai_configured,
+)
+from deepeval.config.settings import get_settings
+from deepeval.key_handler import (
+    ModelKeyValues,
+)
+
+
+@app.command(name="set-grok")
+def set_grok_model_env(
+    model: Optional[str] = typer.Option(
+        None,
+        "-m",
+        "--model",
+        help="Model identifier to use for this provider",
+    ),
+    prompt_api_key: bool = typer.Option(
+        False,
+        "-k",
+        "--prompt-api-key",
+        help=(
+            "Prompt for GROK_API_KEY (input hidden). Not suitable for CI. "
+            "If --save (or DEEPEVAL_DEFAULT_SAVE) is used, the key is written to dotenv in plaintext."
+        ),
+    ),
+    cost_per_input_token: Optional[float] = typer.Option(
+        None,
+        "-i",
+        "--cost-per-input-token",
+        help=(
+            "USD per input token override used for cost tracking. Preconfigured for known models; "
+            "REQUIRED if you use a custom/unknown model."
+        ),
+    ),
+    cost_per_output_token: Optional[float] = typer.Option(
+        None,
+        "-o",
+        "--cost-per-output-token",
+        help=(
+            "USD per output token override used for cost tracking. Preconfigured for known models; "
+            "REQUIRED if you use a custom/unknown model."
+        ),
+    ),
+    save: Optional[str] = typer.Option(
+        None,
+        "-s",
+        "--save",
+        help="Persist CLI parameters as environment variables in a dotenv file. "
+        "Usage: --save=dotenv[:path] (default: .env.local)",
+    ),
+    quiet: bool = typer.Option(
+        False,
+        "-q",
+        "--quiet",
+        help="Suppress printing to the terminal (useful for CI).",
+    ),
+):
+    api_key = None
+    if prompt_api_key:
+        api_key = coerce_blank_to_none(
+            typer.prompt("Grok API key", hide_input=True)
+        )
+
+    model = coerce_blank_to_none(model)
+
+    settings = get_settings()
+    with settings.edit(save=save) as edit_ctx:
+        edit_ctx.switch_model_provider(ModelKeyValues.USE_GROK_MODEL)
+        if api_key is not None:
+            settings.GROK_API_KEY = api_key
+        if model is not None:
+            settings.GROK_MODEL_NAME = model
+        if cost_per_input_token is not None:
+            settings.GROK_COST_PER_INPUT_TOKEN = cost_per_input_token
+        if cost_per_output_token is not None:
+            settings.GROK_COST_PER_OUTPUT_TOKEN = cost_per_output_token
+
+    handled, path, updates = edit_ctx.result
+
+    effective_model = settings.GROK_MODEL_NAME
+    if not effective_model:
+        raise typer.BadParameter(
+            "Grok model name is not set. Pass --model (or set GROK_MODEL_NAME).",
+            param_hint="--model",
+        )
+    _handle_save_result(
+        handled=handled,
+        path=path,
+        updates=updates,
+        save=save,
+        quiet=quiet,
+        success_msg=(
+            f":raising_hands: Congratulations! You're now using Grok `{escape(effective_model)}` for all evals that require an LLM."
+        ),
+    )
+
+
+@app.command(name="unset-grok")
+def unset_grok_model_env(
+    save: Optional[str] = typer.Option(
+        None,
+        "-s",
+        "--save",
+        help="Remove only the Grok model related environment variables from a dotenv file. "
+        "Usage: --save=dotenv[:path] (default: .env.local)",
+    ),
+    clear_secrets: bool = typer.Option(
+        False,
+        "-x",
+        "--clear-secrets",
+        help="Also remove GROK_API_KEY from the dotenv store.",
+    ),
+    quiet: bool = typer.Option(
+        False,
+        "-q",
+        "--quiet",
+        help="Suppress printing to the terminal (useful for CI).",
+    ),
+):
+    settings = get_settings()
+    with settings.edit(save=save) as edit_ctx:
+        settings.USE_GROK_MODEL = None
+        settings.GROK_MODEL_NAME = None
+        settings.GROK_COST_PER_INPUT_TOKEN = None
+        settings.GROK_COST_PER_OUTPUT_TOKEN = None
+        if clear_secrets:
+            settings.GROK_API_KEY = None
+
+    handled, path, updates = edit_ctx.result
+
+    if _handle_save_result(
+        handled=handled,
+        path=path,
+        updates=updates,
+        save=save,
+        quiet=quiet,
+        updated_msg="Removed Grok model environment variables from {path}.",
+        tip_msg=None,
+    ):
+        if is_openai_configured():
+            print(
+                ":raised_hands: OpenAI will still be used by default because OPENAI_API_KEY is set."
+            )
+        else:
+            print(
+                "The Grok model configuration has been removed. No model is currently configured, but you can set one with the CLI or add credentials to .env[.local]."
+            )

deepeval/cli/providers/llms/litellm.py

@@ -0,0 +1,172 @@
+"""`deepeval set-litellm, unset-litellm` commands."""
+
+import typer
+from typing import Optional
+from rich import print
+from rich.markup import escape
+
+from deepeval.cli.app import app
+from deepeval.cli.utils import (
+    coerce_blank_to_none,
+    handle_save_result as _handle_save_result,
+    is_openai_configured,
+)
+from deepeval.config.settings import get_settings
+from deepeval.key_handler import (
+    ModelKeyValues,
+)
+
+
+@app.command(name="set-litellm")
+def set_litellm_model_env(
+    model: Optional[str] = typer.Option(
+        None,
+        "-m",
+        "--model",
+        help="Model identifier to use for this provider",
+    ),
+    prompt_api_key: bool = typer.Option(
+        False,
+        "-k",
+        "--prompt-api-key",
+        help=(
+            "Prompt for LITELLM_API_KEY (input hidden). Not suitable for CI. "
+            "If --save (or DEEPEVAL_DEFAULT_SAVE) is used, the key is written to dotenv in plaintext."
+        ),
+    ),
+    base_url: Optional[str] = typer.Option(
+        None,
+        "-u",
+        "--base-url",
+        help="Override the API endpoint/base URL used by this provider.",
+    ),
+    proxy_prompt_api_key: bool = typer.Option(
+        False,
+        "-K",
+        "--proxy-prompt-api-key",
+        help=(
+            "Prompt for LITELLM_PROXY_API_KEY (input hidden). Not suitable for CI. "
+            "If --save (or DEEPEVAL_DEFAULT_SAVE) is used, the key is written to dotenv in plaintext."
+        ),
+    ),
+    proxy_base_url: Optional[str] = typer.Option(
+        None,
+        "-U",
+        "--proxy-base-url",
+        help="Override the LITELLM_PROXY_API_BASE URL (useful for proxies, gateways, or self-hosted endpoints).",
+    ),
+    save: Optional[str] = typer.Option(
+        None,
+        "-s",
+        "--save",
+        help="Persist CLI parameters as environment variables in a dotenv file. "
+        "Usage: --save=dotenv[:path] (default: .env.local)",
+    ),
+    quiet: bool = typer.Option(
+        False,
+        "-q",
+        "--quiet",
+        help="Suppress printing to the terminal (useful for CI).",
+    ),
+):
+    api_key = None
+    if prompt_api_key:
+        api_key = coerce_blank_to_none(
+            typer.prompt("LiteLLM API key", hide_input=True)
+        )
+
+    proxy_api_key = None
+    if proxy_prompt_api_key:
+        proxy_api_key = coerce_blank_to_none(
+            typer.prompt("LiteLLM Proxy API key", hide_input=True)
+        )
+
+    model = coerce_blank_to_none(model)
+    base_url = coerce_blank_to_none(base_url)
+    proxy_base_url = coerce_blank_to_none(proxy_base_url)
+
+    settings = get_settings()
+    with settings.edit(save=save) as edit_ctx:
+        edit_ctx.switch_model_provider(ModelKeyValues.USE_LITELLM)
+        if model is not None:
+            settings.LITELLM_MODEL_NAME = model
+        if api_key is not None:
+            settings.LITELLM_API_KEY = api_key
+        if base_url is not None:
+            settings.LITELLM_API_BASE = base_url
+        if proxy_api_key is not None:
+            settings.LITELLM_PROXY_API_KEY = proxy_api_key
+        if proxy_base_url is not None:
+            settings.LITELLM_PROXY_API_BASE = proxy_base_url
+
+    handled, path, updates = edit_ctx.result
+
+    effective_model = settings.LITELLM_MODEL_NAME
+    if not effective_model:
+        raise typer.BadParameter(
+            "LiteLLM model name is not set. Pass --model (or set LITELLM_MODEL_NAME).",
+            param_hint="--model",
+        )
+    _handle_save_result(
+        handled=handled,
+        path=path,
+        updates=updates,
+        save=save,
+        quiet=quiet,
+        success_msg=(
+            f":raising_hands: Congratulations! You're now using LiteLLM `{escape(effective_model)}` for all evals that require an LLM."
+        ),
+    )
+
+
+@app.command(name="unset-litellm")
+def unset_litellm_model_env(
+    save: Optional[str] = typer.Option(
+        None,
+        "-s",
+        "--save",
+        help="Remove only the LiteLLM related environment variables from a dotenv file. "
+        "Usage: --save=dotenv[:path] (default: .env.local)",
+    ),
+    clear_secrets: bool = typer.Option(
+        False,
+        "-x",
+        "--clear-secrets",
+        help="Also remove LITELLM_API_KEY and LITELLM_PROXY_API_KEY from the dotenv store.",
+    ),
+    quiet: bool = typer.Option(
+        False,
+        "-q",
+        "--quiet",
+        help="Suppress printing to the terminal (useful for CI).",
+    ),
+):
+    settings = get_settings()
+    with settings.edit(save=save) as edit_ctx:
+        settings.USE_LITELLM = None
+        settings.LITELLM_MODEL_NAME = None
+        settings.LITELLM_API_BASE = None
+        settings.LITELLM_PROXY_API_BASE = None
+        if clear_secrets:
+            settings.LITELLM_API_KEY = None
+            settings.LITELLM_PROXY_API_KEY = None
+
+    handled, path, updates = edit_ctx.result
+
+    if _handle_save_result(
+        handled=handled,
+        path=path,
+        updates=updates,
+        save=save,
+        quiet=quiet,
+        updated_msg="Removed LiteLLM model environment variables from {path}.",
+        tip_msg=None,
+    ):
+        if is_openai_configured():
+            print(
+                ":raised_hands: OpenAI will still be used by default because OPENAI_API_KEY is set."
+            )
+        else:
+            print(
+                "The LiteLLM model configuration has been removed. No model is currently configured, but you can set one with the CLI or add credentials to .env[.local]."
+            )

deepeval/cli/providers/llms/local_embeddings.py

@@ -0,0 +1,144 @@
+"""`deepeval set-local-embeddings, unset-local-embeddings` commands."""
+
+import typer
+from typing import Optional
+from rich import print
+from rich.markup import escape
+
+from deepeval.cli.app import app
+from deepeval.cli.utils import (
+    coerce_blank_to_none,
+    handle_save_result as _handle_save_result,
+    is_openai_configured,
+)
+from deepeval.config.settings import get_settings
+from deepeval.key_handler import (
+    EmbeddingKeyValues,
+)
+
+
+@app.command(name="set-local-embeddings")
+def set_local_embeddings_env(
+    model: Optional[str] = typer.Option(
+        None,
+        "-m",
+        "--model",
+        help="Model identifier to use for this provider",
+    ),
+    prompt_api_key: bool = typer.Option(
+        False,
+        "-k",
+        "--prompt-api-key",
+        help=(
+            "Prompt for LOCAL_EMBEDDING_API_KEY (input hidden). Not suitable for CI. "
+            "If --save (or DEEPEVAL_DEFAULT_SAVE) is used, the key is written to dotenv in plaintext."
+        ),
+    ),
+    base_url: Optional[str] = typer.Option(
+        None,
+        "-u",
+        "--base-url",
+        help="Override the API endpoint/base URL used by this provider.",
+    ),
+    save: Optional[str] = typer.Option(
+        None,
+        "-s",
+        "--save",
+        help="Persist CLI parameters as environment variables in a dotenv file. "
+        "Usage: --save=dotenv[:path] (default: .env.local)",
+    ),
+    quiet: bool = typer.Option(
+        False,
+        "-q",
+        "--quiet",
+        help="Suppress printing to the terminal (useful for CI).",
+    ),
+):
+    api_key = None
+    if prompt_api_key:
+        api_key = coerce_blank_to_none(
+            typer.prompt("Local Embedding Model API key", hide_input=True)
+        )
+
+    model = coerce_blank_to_none(model)
+    base_url = coerce_blank_to_none(base_url)
+
+    settings = get_settings()
+    with settings.edit(save=save) as edit_ctx:
+        edit_ctx.switch_model_provider(EmbeddingKeyValues.USE_LOCAL_EMBEDDINGS)
+        if model is not None:
+            settings.LOCAL_EMBEDDING_MODEL_NAME = model
+        if base_url is not None:
+            settings.LOCAL_EMBEDDING_BASE_URL = base_url
+        if api_key is not None:
+            settings.LOCAL_EMBEDDING_API_KEY = api_key
+
+    handled, path, updates = edit_ctx.result
+
+    effective_model = settings.LOCAL_EMBEDDING_MODEL_NAME
+    if not effective_model:
+        raise typer.BadParameter(
+            "Local embedding model name is not set. Pass --model (or set LOCAL_EMBEDDING_MODEL_NAME).",
+            param_hint="--model",
+        )
+    _handle_save_result(
+        handled=handled,
+        path=path,
+        updates=updates,
+        save=save,
+        quiet=quiet,
+        success_msg=(
+            f":raising_hands: Congratulations! You're now using the local embedding model `{escape(effective_model)}` for all evals that require text embeddings."
+        ),
+    )
+
+
+@app.command(name="unset-local-embeddings")
+def unset_local_embeddings_env(
+    save: Optional[str] = typer.Option(
+        None,
+        "-s",
+        "--save",
+        help="Remove only the local embedding related environment variables from a dotenv file. "
+        "Usage: --save=dotenv[:path] (default: .env.local)",
+    ),
+    clear_secrets: bool = typer.Option(
+        False,
+        "-x",
+        "--clear-secrets",
+        help="Also remove LOCAL_MODEL_API_KEY from the dotenv store.",
+    ),
+    quiet: bool = typer.Option(
+        False,
+        "-q",
+        "--quiet",
+        help="Suppress printing to the terminal (useful for CI).",
+    ),
+):
+    settings = get_settings()
+    with settings.edit(save=save) as edit_ctx:
+        settings.LOCAL_EMBEDDING_MODEL_NAME = None
+        settings.LOCAL_EMBEDDING_BASE_URL = None
+        settings.USE_LOCAL_EMBEDDINGS = None
+        if clear_secrets:
+            settings.LOCAL_EMBEDDING_API_KEY = None
+
+    handled, path, updates = edit_ctx.result
+
+    if _handle_save_result(
+        handled=handled,
+        path=path,
+        updates=updates,
+        save=save,
+        quiet=quiet,
+        updated_msg="Removed local embedding environment variables from {path}.",
+        tip_msg=None,
+    ):
+        if is_openai_configured():
+            print(
+                ":raised_hands: OpenAI will still be used by default because OPENAI_API_KEY is set."
+            )
+        else:
+            print(
+                "The local embeddings model configuration has been removed. No model is currently configured, but you can set one with the CLI or add credentials to .env[.local]."
+            )

deepeval/cli/providers/llms/local_model.py

@@ -0,0 +1,154 @@
+"""`deepeval set-local-model, unset-local-model` commands."""
+
+import typer
+from typing import Optional
+from rich import print
+from rich.markup import escape
+
+from deepeval.cli.app import app
+from deepeval.cli.utils import (
+    coerce_blank_to_none,
+    handle_save_result as _handle_save_result,
+    is_openai_configured,
+)
+from deepeval.config.settings import get_settings
+from deepeval.key_handler import (
+    ModelKeyValues,
+)
+
+
+@app.command(name="set-local-model")
+def set_local_model_env(
+    model: Optional[str] = typer.Option(
+        None,
+        "-m",
+        "--model",
+        help="Model identifier to use for this provider",
+    ),
+    prompt_api_key: bool = typer.Option(
+        False,
+        "-k",
+        "--prompt-api-key",
+        help=(
+            "Prompt for LOCAL_MODEL_API_KEY (input hidden). Not suitable for CI. "
+            "If --save (or DEEPEVAL_DEFAULT_SAVE) is used, the key is written to dotenv in plaintext."
+        ),
+    ),
+    base_url: Optional[str] = typer.Option(
+        None,
+        "-u",
+        "--base-url",
+        help="Override the API endpoint/base URL used by this provider.",
+    ),
+    model_format: Optional[str] = typer.Option(
+        None,
+        "-f",
+        "--format",
+        help="Format of the response from the local model (default: json)",
+    ),
+    save: Optional[str] = typer.Option(
+        None,
+        "-s",
+        "--save",
+        help="Persist CLI parameters as environment variables in a dotenv file. "
+        "Usage: --save=dotenv[:path] (default: .env.local)",
+    ),
+    quiet: bool = typer.Option(
+        False,
+        "-q",
+        "--quiet",
+        help="Suppress printing to the terminal (useful for CI).",
+    ),
+):
+    api_key = None
+    if prompt_api_key:
+        api_key = coerce_blank_to_none(
+            typer.prompt("Local Model API key", hide_input=True)
+        )
+
+    model = coerce_blank_to_none(model)
+    base_url = coerce_blank_to_none(base_url)
+    model_format = coerce_blank_to_none(model_format)
+
+    settings = get_settings()
+    with settings.edit(save=save) as edit_ctx:
+        edit_ctx.switch_model_provider(ModelKeyValues.USE_LOCAL_MODEL)
+        if model is not None:
+            settings.LOCAL_MODEL_NAME = model
+        if base_url is not None:
+            settings.LOCAL_MODEL_BASE_URL = base_url
+        if api_key is not None:
+            settings.LOCAL_MODEL_API_KEY = api_key
+        if model_format is not None:
+            settings.LOCAL_MODEL_FORMAT = model_format
+
+    handled, path, updates = edit_ctx.result
+
+    effective_model = settings.LOCAL_MODEL_NAME
+    if not effective_model:
+        raise typer.BadParameter(
+            "Local model name is not set. Pass --model (or set LOCAL_MODEL_NAME).",
+            param_hint="--model",
+        )
+    _handle_save_result(
+        handled=handled,
+        path=path,
+        updates=updates,
+        save=save,
+        quiet=quiet,
+        success_msg=(
+            f":raising_hands: Congratulations! You're now using a local model `{escape(effective_model)}` for all evals that require an LLM."
+        ),
+    )
+
+
+@app.command(name="unset-local-model")
+def unset_local_model_env(
+    save: Optional[str] = typer.Option(
+        None,
+        "-s",
+        "--save",
+        help="Remove only the local model related environment variables from a dotenv file. "
+        "Usage: --save=dotenv[:path] (default: .env.local)",
+    ),
+    clear_secrets: bool = typer.Option(
+        False,
+        "-x",
+        "--clear-secrets",
+        help="Also remove LOCAL_MODEL_API_KEY from the dotenv store.",
+    ),
+    quiet: bool = typer.Option(
+        False,
+        "-q",
+        "--quiet",
+        help="Suppress printing to the terminal (useful for CI).",
+    ),
+):
+    settings = get_settings()
+    with settings.edit(save=save) as edit_ctx:
+        if clear_secrets:
+            settings.LOCAL_MODEL_API_KEY = None
+        settings.LOCAL_MODEL_NAME = None
+        settings.LOCAL_MODEL_BASE_URL = None
+        settings.LOCAL_MODEL_FORMAT = None
+        settings.USE_LOCAL_MODEL = None
+
+    handled, path, updates = edit_ctx.result
+
+    if _handle_save_result(
+        handled=handled,
+        path=path,
+        updates=updates,
+        save=save,
+        quiet=quiet,
+        updated_msg="Removed local model environment variables from {path}.",
+        tip_msg=None,
+    ):
+        if is_openai_configured():
+            print(
+                ":raised_hands: OpenAI will still be used by default because OPENAI_API_KEY is set."
+            )
+        else:
+            print(
+                "The local model configuration has been removed. No model is currently configured, but you can set one with the CLI or add credentials to .env[.local]."
+            )

deepeval/cli/providers/llms/moonshot.py

@@ -0,0 +1,158 @@
+"""`deepeval set-moonshot, unset-moonshot` commands."""
+
+import typer
+from typing import Optional
+from rich import print
+from rich.markup import escape
+
+from deepeval.cli.app import app
+from deepeval.cli.utils import (
+    coerce_blank_to_none,
+    handle_save_result as _handle_save_result,
+    is_openai_configured,
+)
+from deepeval.config.settings import get_settings
+from deepeval.key_handler import (
+    ModelKeyValues,
+)
+
+
+@app.command(name="set-moonshot")
+def set_moonshot_model_env(
+    model: Optional[str] = typer.Option(
+        None,
+        "-m",
+        "--model",
+        help="Model identifier to use for this provider",
+    ),
+    prompt_api_key: bool = typer.Option(
+        False,
+        "-k",
+        "--prompt-api-key",
+        help=(
+            "Prompt for MOONSHOT_API_KEY (input hidden). Not suitable for CI. "
+            "If --save (or DEEPEVAL_DEFAULT_SAVE) is used, the key is written to dotenv in plaintext."
+        ),
+    ),
+    cost_per_input_token: Optional[float] = typer.Option(
+        None,
+        "-i",
+        "--cost-per-input-token",
+        help=(
+            "USD per input token override used for cost tracking. Preconfigured for known models; "
+            "REQUIRED if you use a custom/unknown model."
+        ),
+    ),
+    cost_per_output_token: Optional[float] = typer.Option(
+        None,
+        "-o",
+        "--cost-per-output-token",
+        help=(
+            "USD per output token override used for cost tracking. Preconfigured for known models; "
+            "REQUIRED if you use a custom/unknown model."
+        ),
+    ),
+    save: Optional[str] = typer.Option(
+        None,
+        "-s",
+        "--save",
+        help="Persist CLI parameters as environment variables in a dotenv file. "
+        "Usage: --save=dotenv[:path] (default: .env.local)",
+    ),
+    quiet: bool = typer.Option(
+        False,
+        "-q",
+        "--quiet",
+        help="Suppress printing to the terminal (useful for CI).",
+    ),
+):
+    api_key = None
+    if prompt_api_key:
+        api_key = coerce_blank_to_none(
+            typer.prompt("Moonshot API key", hide_input=True)
+        )
+
+    model = coerce_blank_to_none(model)
+
+    settings = get_settings()
+    with settings.edit(save=save) as edit_ctx:
+        edit_ctx.switch_model_provider(ModelKeyValues.USE_MOONSHOT_MODEL)
+        if model is not None:
+            settings.MOONSHOT_MODEL_NAME = model
+        if api_key is not None:
+            settings.MOONSHOT_API_KEY = api_key
+        if cost_per_input_token is not None:
+            settings.MOONSHOT_COST_PER_INPUT_TOKEN = cost_per_input_token
+        if cost_per_output_token is not None:
+            settings.MOONSHOT_COST_PER_OUTPUT_TOKEN = cost_per_output_token
+
+    handled, path, updates = edit_ctx.result
+
+    effective_model = settings.MOONSHOT_MODEL_NAME
+    if not effective_model:
+        raise typer.BadParameter(
+            "Moonshot model name is not set. Pass --model (or set MOONSHOT_MODEL_NAME).",
+            param_hint="--model",
+        )
+    _handle_save_result(
+        handled=handled,
+        path=path,
+        updates=updates,
+        save=save,
+        quiet=quiet,
+        success_msg=(
+            f":raising_hands: Congratulations! You're now using Moonshot `{escape(effective_model)}` for all evals that require an LLM."
+        ),
+    )
+
+
+@app.command(name="unset-moonshot")
+def unset_moonshot_model_env(
+    save: Optional[str] = typer.Option(
+        None,
+        "-s",
+        "--save",
+        help="Remove only the Moonshot model related environment variables from a dotenv file. "
+        "Usage: --save=dotenv[:path] (default: .env.local)",
+    ),
+    clear_secrets: bool = typer.Option(
+        False,
+        "-x",
+        "--clear-secrets",
+        help="Also remove MOONSHOT_API_KEY from the dotenv store.",
+    ),
+    quiet: bool = typer.Option(
+        False,
+        "-q",
+        "--quiet",
+        help="Suppress printing to the terminal (useful for CI).",
+    ),
+):
+    settings = get_settings()
+    with settings.edit(save=save) as edit_ctx:
+        settings.USE_MOONSHOT_MODEL = None
+        settings.MOONSHOT_MODEL_NAME = None
+        settings.MOONSHOT_COST_PER_INPUT_TOKEN = None
+        settings.MOONSHOT_COST_PER_OUTPUT_TOKEN = None
+        if clear_secrets:
+            settings.MOONSHOT_API_KEY = None
+
+    handled, path, updates = edit_ctx.result
+
+    if _handle_save_result(
+        handled=handled,
+        path=path,
+        updates=updates,
+        save=save,
+        quiet=quiet,
+        updated_msg="Removed Moonshot model environment variables from {path}.",
+        tip_msg=None,
+    ):
+        if is_openai_configured():
+            print(
+                ":raised_hands: OpenAI will still be used by default because OPENAI_API_KEY is set."
+            )
+        else:
+            print(
+                "The Moonshot model configuration has been removed. No model is currently configured, but you can set one with the CLI or add credentials to .env[.local]."
+            )

deepeval/cli/providers/llms/ollama.py

@@ -0,0 +1,240 @@
+"""`deepeval set-ollama, set-ollama-embeddings, unset-ollama, unset-ollama-embeddings` commands."""
+
+import typer
+from typing import Optional
+from rich import print
+from rich.markup import escape
+
+from deepeval.cli.app import app
+from deepeval.cli.utils import (
+    coerce_blank_to_none,
+    handle_save_result as _handle_save_result,
+    is_openai_configured,
+)
+from deepeval.config.settings import get_settings
+from deepeval.key_handler import (
+    EmbeddingKeyValues,
+    ModelKeyValues,
+)
+
+
+@app.command(name="set-ollama")
+def set_ollama_model_env(
+    model: Optional[str] = typer.Option(
+        None,
+        "-m",
+        "--model",
+        help="Model identifier to use for this provider",
+    ),
+    base_url: str = typer.Option(
+        "http://localhost:11434",
+        "-u",
+        "--base-url",
+        help="Override the API endpoint/base URL used by this provider.",
+    ),
+    save: Optional[str] = typer.Option(
+        None,
+        "-s",
+        "--save",
+        help="Persist CLI parameters as environment variables in a dotenv file. "
+        "Usage: --save=dotenv[:path] (default: .env.local)",
+    ),
+    quiet: bool = typer.Option(
+        False,
+        "-q",
+        "--quiet",
+        help="Suppress printing to the terminal (useful for CI).",
+    ),
+):
+    model = coerce_blank_to_none(model)
+    base_url = coerce_blank_to_none(base_url)
+
+    settings = get_settings()
+    with settings.edit(save=save) as edit_ctx:
+        edit_ctx.switch_model_provider(ModelKeyValues.USE_LOCAL_MODEL)
+        settings.LOCAL_MODEL_API_KEY = "ollama"
+        if model is not None:
+            settings.OLLAMA_MODEL_NAME = model
+        if base_url is not None:
+            settings.LOCAL_MODEL_BASE_URL = base_url
+
+    handled, path, updates = edit_ctx.result
+
+    effective_model = settings.OLLAMA_MODEL_NAME
+    if not effective_model:
+        raise typer.BadParameter(
+            "Ollama model name is not set. Pass --model (or set OLLAMA_MODEL_NAME).",
+            param_hint="--model",
+        )
+    _handle_save_result(
+        handled=handled,
+        path=path,
+        updates=updates,
+        save=save,
+        quiet=quiet,
+        success_msg=(
+            f":raising_hands: Congratulations! You're now using a local Ollama model `{escape(effective_model)}` for all evals that require an LLM."
+        ),
+    )
+
+
+@app.command(name="unset-ollama")
+def unset_ollama_model_env(
+    save: Optional[str] = typer.Option(
+        None,
+        "-s",
+        "--save",
+        help="Remove only the Ollama related environment variables from a dotenv file. "
+        "Usage: --save=dotenv[:path] (default: .env.local)",
+    ),
+    clear_secrets: bool = typer.Option(
+        False,
+        "-x",
+        "--clear-secrets",
+        help="Also remove LOCAL_MODEL_API_KEY from the dotenv store.",
+    ),
+    quiet: bool = typer.Option(
+        False,
+        "-q",
+        "--quiet",
+        help="Suppress printing to the terminal (useful for CI).",
+    ),
+):
+    settings = get_settings()
+    with settings.edit(save=save) as edit_ctx:
+        if clear_secrets:
+            settings.LOCAL_MODEL_API_KEY = None
+        settings.OLLAMA_MODEL_NAME = None
+        settings.LOCAL_MODEL_BASE_URL = None
+        settings.USE_LOCAL_MODEL = None
+
+    handled, path, updates = edit_ctx.result
+
+    if _handle_save_result(
+        handled=handled,
+        path=path,
+        updates=updates,
+        save=save,
+        quiet=quiet,
+        updated_msg="Removed local Ollama environment variables from {path}.",
+        tip_msg=None,
+    ):
+        if is_openai_configured():
+            print(
+                ":raised_hands: OpenAI will still be used by default because OPENAI_API_KEY is set."
+            )
+        else:
+            print(
+                "The local Ollama model configuration has been removed. No model is currently configured, but you can set one with the CLI or add credentials to .env[.local]."
+            )
+
+
+@app.command(name="set-ollama-embeddings")
+def set_ollama_embeddings_env(
+    model: Optional[str] = typer.Option(
+        None,
+        "-m",
+        "--model",
+        help="Model identifier to use for this provider.",
+    ),
+    base_url: str = typer.Option(
+        "http://localhost:11434",
+        "-u",
+        "--base-url",
+        help="Override the API endpoint/base URL used by this provider.",
+    ),
+    save: Optional[str] = typer.Option(
+        None,
+        "-s",
+        "--save",
+        help="Persist CLI parameters as environment variables in a dotenv file. "
+        "Usage: --save=dotenv[:path] (default: .env.local)",
+    ),
+    quiet: bool = typer.Option(
+        False,
+        "-q",
+        "--quiet",
+        help="Suppress printing to the terminal (useful for CI).",
+    ),
+):
+    model = coerce_blank_to_none(model)
+    base_url = coerce_blank_to_none(base_url)
+
+    settings = get_settings()
+    with settings.edit(save=save) as edit_ctx:
+        edit_ctx.switch_model_provider(EmbeddingKeyValues.USE_LOCAL_EMBEDDINGS)
+        settings.LOCAL_EMBEDDING_API_KEY = "ollama"
+        if model is not None:
+            settings.LOCAL_EMBEDDING_MODEL_NAME = model
+        if base_url is not None:
+            settings.LOCAL_EMBEDDING_BASE_URL = base_url
+
+    handled, path, updates = edit_ctx.result
+
+    effective_model = settings.LOCAL_EMBEDDING_MODEL_NAME
+    if not effective_model:
+        raise typer.BadParameter(
+            "Ollama embedding model name is not set. Pass --model (or set LOCAL_EMBEDDING_MODEL_NAME).",
+            param_hint="--model",
+        )
+    _handle_save_result(
+        handled=handled,
+        path=path,
+        updates=updates,
+        save=save,
+        quiet=quiet,
+        success_msg=(
+            f":raising_hands: Congratulations! You're now using the Ollama embedding model `{escape(effective_model)}` for all evals that require text embeddings."
+        ),
+    )
+
+
+@app.command(name="unset-ollama-embeddings")
+def unset_ollama_embeddings_env(
+    save: Optional[str] = typer.Option(
+        None,
+        "-s",
+        "--save",
+        help="Remove only the Ollama embedding related environment variables from a dotenv file. "
+        "Usage: --save=dotenv[:path] (default: .env.local)",
+    ),
+    clear_secrets: bool = typer.Option(
+        False,
+        "-x",
+        "--clear-secrets",
+        help="Also remove LOCAL_EMBEDDING_API_KEY from the dotenv store.",
+    ),
+    quiet: bool = typer.Option(
+        False,
+        "-q",
+        "--quiet",
+        help="Suppress printing to the terminal (useful for CI).",
+    ),
+):
+    settings = get_settings()
+    with settings.edit(save=save) as edit_ctx:
+        if clear_secrets:
+            settings.LOCAL_EMBEDDING_API_KEY = None
+        settings.LOCAL_EMBEDDING_MODEL_NAME = None
+        settings.LOCAL_EMBEDDING_BASE_URL = None
+        settings.USE_LOCAL_EMBEDDINGS = None
+
+    handled, path, updates = edit_ctx.result
+
+    if _handle_save_result(
+        handled=handled,
+        path=path,
+        updates=updates,
+        save=save,
+        quiet=quiet,
+        updated_msg="Removed local Ollama embedding environment variables from {path}.",
+        tip_msg=None,
+    ):
+        if is_openai_configured():
+            print(
+                ":raised_hands: Regular OpenAI embeddings will still be used by default because OPENAI_API_KEY is set."
+            )
+        else:
+            print(
+                "The local Ollama embedding model configuration has been removed. No model is currently configured, but you can set one with the CLI or add credentials to .env[.local]."
+            )

deepeval/cli/providers/llms/openai.py

@@ -0,0 +1,214 @@
+"""`deepeval set-openai, unset-openai` commands."""
+
+import typer
+from typing import Optional
+from rich import print
+from rich.markup import escape
+
+from deepeval.cli.app import app
+from deepeval.cli.utils import (
+    coerce_blank_to_none,
+    handle_save_result as _handle_save_result,
+    is_openai_configured,
+)
+from deepeval.config.settings import get_settings
+from deepeval.key_handler import (
+    ModelKeyValues,
+)
+
+
+@app.command(name="set-openai")
+def set_openai_env(
+    model: Optional[str] = typer.Option(
+        None,
+        "-m",
+        "--model",
+        help="Model identifier to use for this provider (e.g., `gpt-4.1`).",
+    ),
+    prompt_api_key: bool = typer.Option(
+        False,
+        "-k",
+        "--prompt-api-key",
+        help=(
+            "Prompt for OPENAI_API_KEY (input hidden). Not suitable for CI. "
+            "If --save (or DEEPEVAL_DEFAULT_SAVE) is used, the key is written to dotenv in plaintext."
+        ),
+    ),
+    cost_per_input_token: Optional[float] = typer.Option(
+        None,
+        "-i",
+        "--cost-per-input-token",
+        help=(
+            "USD per input token override used for cost tracking. Preconfigured for known models; "
+            "REQUIRED if you use a custom/unknown model."
+        ),
+    ),
+    cost_per_output_token: Optional[float] = typer.Option(
+        None,
+        "-o",
+        "--cost-per-output-token",
+        help=(
+            "USD per output token override used for cost tracking. Preconfigured for known models; "
+            "REQUIRED if you use a custom/unknown model."
+        ),
+    ),
+    save: Optional[str] = typer.Option(
+        None,
+        "-s",
+        "--save",
+        help="Persist CLI parameters as environment variables in a dotenv file. "
+        "Usage: --save=dotenv[:path] (default: .env.local)",
+    ),
+    quiet: bool = typer.Option(
+        False,
+        "-q",
+        "--quiet",
+        help="Suppress printing to the terminal (useful for CI).",
+    ),
+):
+    """
+    Configure OpenAI as the active LLM provider.
+
+    What this does:
+    - Sets the active provider flag to `USE_OPENAI_MODEL`.
+    - Persists the selected model name and any cost overrides in the JSON store.
+    - secrets are never written to `.deepeval/.deepeval` (JSON).
+
+    Pricing rules:
+    - If `model` is a known OpenAI model, you may omit costs (built‑in pricing is used).
+    - If `model` is custom/unsupported, you must provide both
+      `--cost-per-input-token` and `--cost-per-output-token`.
+
+    Secrets & saving:
+
+    - If you run with --prompt-api-key, DeepEval will set OPENAI_API_KEY for this session.
+    - If --save=dotenv[:path] is used (or DEEPEVAL_DEFAULT_SAVE is set), the key will be written to that dotenv file (plaintext).
+
+    Secrets are never written to .deepeval/.deepeval (legacy JSON store).
+
+    Args:
+        --model: OpenAI model name, such as `gpt-4o-mini`.
+        --prompt-api-key: Prompt interactively for OPENAI_API_KEY (input hidden). Avoids putting secrets on the command line (shell history/process args). Not suitable for CI.
+        --cost-per-input-token: USD per input token (optional for known models).
+        --cost-per-output-token: USD per output token (optional for known models).
+        --save: Persist config (and supported secrets) to a dotenv file; format `dotenv[:path]`.
+        --quiet: Suppress printing to the terminal.
+
+    Example:
+        deepeval set-openai \\
+          --model gpt-4o-mini \\
+          --cost-per-input-token 0.0005 \\
+          --cost-per-output-token 0.0015 \\
+          --save dotenv:.env.local
+    """
+    api_key = None
+    if prompt_api_key:
+        api_key = coerce_blank_to_none(
+            typer.prompt("OpenAI API key", hide_input=True)
+        )
+
+    model = coerce_blank_to_none(model)
+
+    settings = get_settings()
+    with settings.edit(save=save) as edit_ctx:
+        edit_ctx.switch_model_provider(ModelKeyValues.USE_OPENAI_MODEL)
+        if model is not None:
+            settings.OPENAI_MODEL_NAME = model
+        if api_key is not None:
+            settings.OPENAI_API_KEY = api_key
+        if cost_per_input_token is not None:
+            settings.OPENAI_COST_PER_INPUT_TOKEN = cost_per_input_token
+        if cost_per_output_token is not None:
+            settings.OPENAI_COST_PER_OUTPUT_TOKEN = cost_per_output_token
+
+    handled, path, updates = edit_ctx.result
+
+    effective_model = settings.OPENAI_MODEL_NAME
+    if not effective_model:
+        raise typer.BadParameter(
+            "OpenAI model name is not set. Pass --model (or set OPENAI_MODEL_NAME).",
+            param_hint="--model",
+        )
+
+    _handle_save_result(
+        handled=handled,
+        path=path,
+        updates=updates,
+        save=save,
+        quiet=quiet,
+        success_msg=(
+            f":raising_hands: Congratulations! You're now using OpenAI's `{escape(effective_model)}` "
+            "for all evals that require an LLM."
+        ),
+    )
+
+
+@app.command(name="unset-openai")
+def unset_openai_env(
+    save: Optional[str] = typer.Option(
+        None,
+        "-s",
+        "--save",
+        help="Remove only the OpenAI related environment variables from a dotenv file. "
+        "Usage: --save=dotenv[:path] (default: .env.local)",
+    ),
+    clear_secrets: bool = typer.Option(
+        False,
+        "-x",
+        "--clear-secrets",
+        help="Also remove OPENAI_API_KEY from the dotenv store.",
+    ),
+    quiet: bool = typer.Option(
+        False,
+        "-q",
+        "--quiet",
+        help="Suppress printing to the terminal (useful for CI).",
+    ),
+):
+    """
+    Unset OpenAI as the active provider.
+
+    Behavior:
+    - Removes OpenAI keys (model, costs, toggle) from the JSON store.
+    - If `--save` is provided, removes those keys from the specified dotenv file.
+    - After unsetting, if `OPENAI_API_KEY` is still set in the environment,
+      OpenAI may still be usable by default. Otherwise, no active provider is configured.
+
+    Args:
+        --save: Remove OpenAI keys from the given dotenv file as well.
+        --clear-secrets: Removes OPENAI_API_KEY from the dotenv store
+        --quiet: Suppress printing to the terminal
+
+    Example:
+        deepeval unset-openai --save dotenv:.env.local
+    """
+
+    settings = get_settings()
+    with settings.edit(save=save) as edit_ctx:
+        settings.OPENAI_MODEL_NAME = None
+        settings.OPENAI_COST_PER_INPUT_TOKEN = None
+        settings.OPENAI_COST_PER_OUTPUT_TOKEN = None
+        settings.USE_OPENAI_MODEL = None
+        if clear_secrets:
+            settings.OPENAI_API_KEY = None
+
+    handled, path, updates = edit_ctx.result
+
+    if _handle_save_result(
+        handled=handled,
+        path=path,
+        updates=updates,
+        save=save,
+        quiet=quiet,
+        updated_msg="Removed OpenAI environment variables from {path}.",
+        tip_msg=None,
+    ):
+        if is_openai_configured():
+            print(
+                ":raised_hands: OpenAI will still be used by default because OPENAI_API_KEY is set."
+            )
+        else:
+            print(
+                "OpenAI has been unset. No active provider is configured. "
+                "Set one with the CLI, or add credentials to .env[.local]."
+            )

deepeval/cli/providers/llms/openrouter.py

@@ -0,0 +1,180 @@
+"""`deepeval set-openrouter, unset-openrouter` commands."""
+
+import typer
+from typing import Optional
+from rich import print
+from rich.markup import escape
+
+from deepeval.cli.app import app
+from deepeval.cli.utils import (
+    coerce_blank_to_none,
+    handle_save_result as _handle_save_result,
+    is_openai_configured,
+)
+from deepeval.config.settings import get_settings
+from deepeval.key_handler import (
+    ModelKeyValues,
+)
+
+
+@app.command(name="set-openrouter")
+def set_openrouter_model_env(
+    model: Optional[str] = typer.Option(
+        None,
+        "-m",
+        "--model",
+        help="Model identifier to use for this provider (e.g., `openai/gpt-4.1`).",
+    ),
+    prompt_api_key: bool = typer.Option(
+        False,
+        "-k",
+        "--prompt-api-key",
+        help=(
+            "Prompt for OPENROUTER_API_KEY (input hidden). Not suitable for CI. "
+            "If --save (or DEEPEVAL_DEFAULT_SAVE) is used, the key is written to dotenv in plaintext."
+        ),
+    ),
+    base_url: Optional[str] = typer.Option(
+        None,
+        "-u",
+        "--base-url",
+        help="Override the API endpoint/base URL used by this provider (default: https://openrouter.ai/api/v1).",
+    ),
+    temperature: Optional[float] = typer.Option(
+        None,
+        "-t",
+        "--temperature",
+        help="Override the global TEMPERATURE used by LLM providers (e.g., 0.0 for deterministic behavior).",
+    ),
+    cost_per_input_token: Optional[float] = typer.Option(
+        None,
+        "-i",
+        "--cost-per-input-token",
+        help=(
+            "USD per input token used for cost tracking. "
+            "If unset and OpenRouter does not return pricing metadata, "
+            "costs will not be calculated."
+        ),
+    ),
+    cost_per_output_token: Optional[float] = typer.Option(
+        None,
+        "-o",
+        "--cost-per-output-token",
+        help=(
+            "USD per output token used for cost tracking. "
+            "If unset and OpenRouter does not return pricing metadata, "
+            "costs will not be calculated."
+        ),
+    ),
+    save: Optional[str] = typer.Option(
+        None,
+        "-s",
+        "--save",
+        help="Persist CLI parameters as environment variables in a dotenv file. "
+        "Usage: --save=dotenv[:path] (default: .env.local)",
+    ),
+    quiet: bool = typer.Option(
+        False,
+        "-q",
+        "--quiet",
+        help="Suppress printing to the terminal (useful for CI).",
+    ),
+):
+    api_key = None
+    if prompt_api_key:
+        api_key = coerce_blank_to_none(
+            typer.prompt("OpenRouter API key", hide_input=True)
+        )
+
+    model = coerce_blank_to_none(model)
+    base_url = coerce_blank_to_none(base_url)
+
+    settings = get_settings()
+    with settings.edit(save=save) as edit_ctx:
+        edit_ctx.switch_model_provider(ModelKeyValues.USE_OPENROUTER_MODEL)
+        if model is not None:
+            settings.OPENROUTER_MODEL_NAME = model
+        if api_key is not None:
+            settings.OPENROUTER_API_KEY = api_key
+        if base_url is not None:
+            settings.OPENROUTER_BASE_URL = base_url
+        if temperature is not None:
+            settings.TEMPERATURE = temperature
+        if cost_per_input_token is not None:
+            settings.OPENROUTER_COST_PER_INPUT_TOKEN = cost_per_input_token
+        if cost_per_output_token is not None:
+            settings.OPENROUTER_COST_PER_OUTPUT_TOKEN = cost_per_output_token
+
+    handled, path, updates = edit_ctx.result
+
+    effective_model = settings.OPENROUTER_MODEL_NAME
+    if not effective_model:
+        raise typer.BadParameter(
+            "OpenRouter model name is not set. Pass --model (or set OPENROUTER_MODEL_NAME).",
+            param_hint="--model",
+        )
+
+    _handle_save_result(
+        handled=handled,
+        path=path,
+        updates=updates,
+        save=save,
+        quiet=quiet,
+        success_msg=(
+            f":raising_hands: Congratulations! You're now using OpenRouter `{escape(effective_model)}` for all evals that require an LLM."
+        ),
+    )
+
+
+@app.command(name="unset-openrouter")
+def unset_openrouter_model_env(
+    save: Optional[str] = typer.Option(
+        None,
+        "-s",
+        "--save",
+        help="Remove only the OpenRouter model related environment variables from a dotenv file. "
+        "Usage: --save=dotenv[:path] (default: .env.local)",
+    ),
+    clear_secrets: bool = typer.Option(
+        False,
+        "-x",
+        "--clear-secrets",
+        help="Also remove OPENROUTER_API_KEY from the dotenv store.",
+    ),
+    quiet: bool = typer.Option(
+        False,
+        "-q",
+        "--quiet",
+        help="Suppress printing to the terminal (useful for CI).",
+    ),
+):
+    settings = get_settings()
+    with settings.edit(save=save) as edit_ctx:
+        settings.USE_OPENROUTER_MODEL = None
+        settings.OPENROUTER_MODEL_NAME = None
+        settings.OPENROUTER_BASE_URL = None
+        settings.OPENROUTER_COST_PER_INPUT_TOKEN = None
+        settings.OPENROUTER_COST_PER_OUTPUT_TOKEN = None
+        # Intentionally do NOT touch TEMPERATURE here; it's a global dial.
+        if clear_secrets:
+            settings.OPENROUTER_API_KEY = None
+
+    handled, path, updates = edit_ctx.result
+
+    if _handle_save_result(
+        handled=handled,
+        path=path,
+        updates=updates,
+        save=save,
+        quiet=quiet,
+        updated_msg="Removed OpenRouter model environment variables from {path}.",
+        tip_msg=None,
+    ):
+        if is_openai_configured():
+            print(
+                ":raised_hands: OpenAI will still be used by default because OPENAI_API_KEY is set."
+            )
+        else:
+            print(
+                "The OpenRouter model configuration has been removed. No model is currently configured, but you can set one with the CLI or add credentials to .env[.local]."
+            )

deepeval/cli/providers/llms/portkey.py

@@ -0,0 +1,154 @@
+"""`deepeval set-portkey, unset-portkey` commands."""
+
+import typer
+from typing import Optional
+from rich import print
+from rich.markup import escape
+
+from deepeval.cli.app import app
+from deepeval.cli.utils import (
+    coerce_blank_to_none,
+    handle_save_result as _handle_save_result,
+    is_openai_configured,
+)
+from deepeval.config.settings import get_settings
+from deepeval.key_handler import (
+    ModelKeyValues,
+)
+
+
+@app.command(name="set-portkey")
+def set_portkey_model_env(
+    model: Optional[str] = typer.Option(
+        None,
+        "-m",
+        "--model",
+        help="Model identifier to use for this provider",
+    ),
+    prompt_api_key: bool = typer.Option(
+        False,
+        "-k",
+        "--prompt-api-key",
+        help=(
+            "Prompt for PORTKEY_API_KEY (input hidden). Not suitable for CI. "
+            "If --save (or DEEPEVAL_DEFAULT_SAVE) is used, the key is written to dotenv in plaintext."
+        ),
+    ),
+    base_url: Optional[str] = typer.Option(
+        None,
+        "-u",
+        "--base-url",
+        help="Override the API endpoint/base URL used by this provider.",
+    ),
+    provider: Optional[str] = typer.Option(
+        None,
+        "-P",
+        "--provider",
+        help="Override the PORTKEY_PROVIDER_NAME.",
+    ),
+    save: Optional[str] = typer.Option(
+        None,
+        "-s",
+        "--save",
+        help="Persist CLI parameters as environment variables in a dotenv file. "
+        "Usage: --save=dotenv[:path] (default: .env.local)",
+    ),
+    quiet: bool = typer.Option(
+        False,
+        "-q",
+        "--quiet",
+        help="Suppress printing to the terminal (useful for CI).",
+    ),
+):
+    api_key = None
+    if prompt_api_key:
+        api_key = coerce_blank_to_none(
+            typer.prompt("Portkey API key", hide_input=True)
+        )
+
+    model = coerce_blank_to_none(model)
+    base_url = coerce_blank_to_none(base_url)
+    provider = coerce_blank_to_none(provider)
+
+    settings = get_settings()
+    with settings.edit(save=save) as edit_ctx:
+        edit_ctx.switch_model_provider(ModelKeyValues.USE_PORTKEY_MODEL)
+        if model is not None:
+            settings.PORTKEY_MODEL_NAME = model
+        if api_key is not None:
+            settings.PORTKEY_API_KEY = api_key
+        if base_url is not None:
+            settings.PORTKEY_BASE_URL = base_url
+        if provider is not None:
+            settings.PORTKEY_PROVIDER_NAME = provider
+
+    handled, path, updates = edit_ctx.result
+
+    effective_model = settings.PORTKEY_MODEL_NAME
+    if not effective_model:
+        raise typer.BadParameter(
+            "Portkey model name is not set. Pass --model (or set PORTKEY_MODEL_NAME).",
+            param_hint="--model",
+        )
+    _handle_save_result(
+        handled=handled,
+        path=path,
+        updates=updates,
+        save=save,
+        quiet=quiet,
+        success_msg=(
+            f":raising_hands: Congratulations! You're now using Portkey `{escape(effective_model)}` for all evals that require an LLM."
+        ),
+    )
+
+
+@app.command(name="unset-portkey")
+def unset_portkey_model_env(
+    save: Optional[str] = typer.Option(
+        None,
+        "-s",
+        "--save",
+        help="Remove only the Portkey related environment variables from a dotenv file. "
+        "Usage: --save=dotenv[:path] (default: .env.local)",
+    ),
+    clear_secrets: bool = typer.Option(
+        False,
+        "-x",
+        "--clear-secrets",
+        help="Also remove PORTKEY_API_KEY from the dotenv store.",
+    ),
+    quiet: bool = typer.Option(
+        False,
+        "-q",
+        "--quiet",
+        help="Suppress printing to the terminal (useful for CI).",
+    ),
+):
+    settings = get_settings()
+    with settings.edit(save=save) as edit_ctx:
+        settings.USE_PORTKEY_MODEL = None
+        settings.PORTKEY_MODEL_NAME = None
+        settings.PORTKEY_BASE_URL = None
+        settings.PORTKEY_PROVIDER_NAME = None
+        if clear_secrets:
+            settings.PORTKEY_API_KEY = None
+
+    handled, path, updates = edit_ctx.result
+
+    if _handle_save_result(
+        handled=handled,
+        path=path,
+        updates=updates,
+        save=save,
+        quiet=quiet,
+        updated_msg="Removed Portkey model environment variables from {path}.",
+        tip_msg=None,
+    ):
+        if is_openai_configured():
+            print(
+                ":raised_hands: OpenAI will still be used by default because OPENAI_API_KEY is set."
+            )
+        else:
+            print(
+                "The Portkey model configuration has been removed. No model is currently configured, but you can set one with the CLI or add credentials to .env[.local]."
+            )

deepeval/cli/providers/stt/__init__.py

@@ -0,0 +1,3 @@
+"""Speech-to-text provider commands."""
+
+from deepeval.cli.providers.stt import command  # noqa: F401

deepeval/cli/providers/stt/command.py

@@ -0,0 +1,117 @@
+"""`deepeval set-stt / unset-stt` commands."""
+
+from enum import Enum
+from typing import Optional
+
+import typer
+
+from deepeval.cli.app import app
+from deepeval.cli.utils import USE_STT_KEYS
+from deepeval.cli.providers._speech import (
+    _set_speech_provider,
+    _unset_speech_provider,
+)
+
+
+class STTProviders(str, Enum):
+    OPENAI = "openai"
+    ELEVENLABS = "elevenlabs"
+    CARTESIA = "cartesia"
+    DEEPGRAM = "deepgram"
+    ASSEMBLYAI = "assemblyai"
+
+
+@app.command(name="set-stt")
+def set_stt_env(
+    provider: STTProviders = typer.Argument(
+        ...,
+        help="Speech-to-text provider to use in voice simulations.",
+    ),
+    model: Optional[str] = typer.Option(
+        None,
+        "-m",
+        "--model",
+        help="Model identifier for this provider (e.g., `nova-3`). Defaults to the provider's own default.",
+    ),
+    prompt_api_key: bool = typer.Option(
+        False,
+        "-k",
+        "--prompt-api-key",
+        help=(
+            "Prompt for the provider's API key (input hidden). Not suitable for CI. "
+            "If --save (or DEEPEVAL_DEFAULT_SAVE) is used, the key is written to dotenv in plaintext."
+        ),
+    ),
+    save: Optional[str] = typer.Option(
+        None,
+        "-s",
+        "--save",
+        help="Persist CLI parameters as environment variables in a dotenv file. "
+        "Usage: --save=dotenv[:path] (default: .env.local)",
+    ),
+    quiet: bool = typer.Option(
+        False,
+        "-q",
+        "--quiet",
+        help="Suppress printing to the terminal (useful for CI).",
+    ),
+):
+    """
+    Configure the active speech-to-text provider.
+
+    What this does:
+    - Sets the active provider flag (e.g. `USE_DEEPGRAM_STT`), clearing the other STT flags.
+    - Persists `DEEPEVAL_STT_MODEL` in the JSON store when `--model` is given.
+    - Leaves the LLM, embedding and TTS selections alone.
+
+    Example:
+        deepeval set-stt assemblyai --model universal-3-5-pro --save dotenv:.env.local
+    """
+    _set_speech_provider(
+        family="STT",
+        provider=provider.value,
+        model=model,
+        prompt_api_key=prompt_api_key,
+        save=save,
+        quiet=quiet,
+    )
+
+
+@app.command(name="unset-stt")
+def unset_stt_env(
+    save: Optional[str] = typer.Option(
+        None,
+        "-s",
+        "--save",
+        help="Remove only the STT related environment variables from a dotenv file. "
+        "Usage: --save=dotenv[:path] (default: .env.local)",
+    ),
+    clear_secrets: bool = typer.Option(
+        False,
+        "-x",
+        "--clear-secrets",
+        help="Also remove the speech provider API keys from the dotenv store (OPENAI_API_KEY is left alone).",
+    ),
+    quiet: bool = typer.Option(
+        False,
+        "-q",
+        "--quiet",
+        help="Suppress printing to the terminal (useful for CI).",
+    ),
+):
+    """
+    Unset the active speech-to-text provider.
+
+    Voice simulations fall back to OpenAI, exactly as they do when no STT
+    provider was ever configured.
+
+    Example:
+        deepeval unset-stt --save dotenv:.env.local
+    """
+    _unset_speech_provider(
+        family="STT",
+        flags=USE_STT_KEYS,
+        clear_secrets=clear_secrets,
+        save=save,
+        quiet=quiet,
+    )

deepeval/cli/providers/tts/__init__.py

@@ -0,0 +1,3 @@
+"""Text-to-speech provider commands."""
+
+from deepeval.cli.providers.tts import command  # noqa: F401

deepeval/cli/providers/tts/command.py

@@ -0,0 +1,116 @@
+"""`deepeval set-tts / unset-tts` commands."""
+
+from enum import Enum
+from typing import Optional
+
+import typer
+
+from deepeval.cli.app import app
+from deepeval.cli.utils import USE_TTS_KEYS
+from deepeval.cli.providers._speech import (
+    _set_speech_provider,
+    _unset_speech_provider,
+)
+
+
+class TTSProviders(str, Enum):
+    OPENAI = "openai"
+    ELEVENLABS = "elevenlabs"
+    CARTESIA = "cartesia"
+    DEEPGRAM = "deepgram"
+
+
+@app.command(name="set-tts")
+def set_tts_env(
+    provider: TTSProviders = typer.Argument(
+        ...,
+        help="Text-to-speech provider to use in voice simulations.",
+    ),
+    model: Optional[str] = typer.Option(
+        None,
+        "-m",
+        "--model",
+        help="Model identifier for this provider (e.g., `aura-2-thalia-en`). Defaults to the provider's own default.",
+    ),
+    prompt_api_key: bool = typer.Option(
+        False,
+        "-k",
+        "--prompt-api-key",
+        help=(
+            "Prompt for the provider's API key (input hidden). Not suitable for CI. "
+            "If --save (or DEEPEVAL_DEFAULT_SAVE) is used, the key is written to dotenv in plaintext."
+        ),
+    ),
+    save: Optional[str] = typer.Option(
+        None,
+        "-s",
+        "--save",
+        help="Persist CLI parameters as environment variables in a dotenv file. "
+        "Usage: --save=dotenv[:path] (default: .env.local)",
+    ),
+    quiet: bool = typer.Option(
+        False,
+        "-q",
+        "--quiet",
+        help="Suppress printing to the terminal (useful for CI).",
+    ),
+):
+    """
+    Configure the active text-to-speech provider.
+
+    What this does:
+    - Sets the active provider flag (e.g. `USE_DEEPGRAM_TTS`), clearing the other TTS flags.
+    - Persists `DEEPEVAL_TTS_MODEL` in the JSON store when `--model` is given.
+    - Leaves the LLM, embedding and STT selections alone.
+
+    Example:
+        deepeval set-tts deepgram --model aura-2-thalia-en --save dotenv:.env.local
+    """
+    _set_speech_provider(
+        family="TTS",
+        provider=provider.value,
+        model=model,
+        prompt_api_key=prompt_api_key,
+        save=save,
+        quiet=quiet,
+    )
+
+
+@app.command(name="unset-tts")
+def unset_tts_env(
+    save: Optional[str] = typer.Option(
+        None,
+        "-s",
+        "--save",
+        help="Remove only the TTS related environment variables from a dotenv file. "
+        "Usage: --save=dotenv[:path] (default: .env.local)",
+    ),
+    clear_secrets: bool = typer.Option(
+        False,
+        "-x",
+        "--clear-secrets",
+        help="Also remove the speech provider API keys from the dotenv store (OPENAI_API_KEY is left alone).",
+    ),
+    quiet: bool = typer.Option(
+        False,
+        "-q",
+        "--quiet",
+        help="Suppress printing to the terminal (useful for CI).",
+    ),
+):
+    """
+    Unset the active text-to-speech provider.
+
+    Voice simulations fall back to OpenAI, exactly as they do when no TTS
+    provider was ever configured.
+
+    Example:
+        deepeval unset-tts --save dotenv:.env.local
+    """
+    _unset_speech_provider(
+        family="TTS",
+        flags=USE_TTS_KEYS,
+        clear_secrets=clear_secrets,
+        save=save,
+        quiet=quiet,
+    )

deepeval/cli/utils.py

@@ -7,7 +7,7 @@ import typer
 import webbrowser
 from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit
 
-from pydantic import ValidationError
+from pydantic import SecretStr, ValidationError
 from pydantic.fields import FieldInfo
 from enum import Enum
 from pathlib import Path
@@ -34,6 +34,7 @@ from deepeval.key_handler import (
     KEY_FILE_HANDLER,
     ModelKeyValues,
     EmbeddingKeyValues,
+    SpeechKeyValues,
 )
 from deepeval.test_run.test_run import (
     global_test_run_manager,
@@ -102,6 +103,31 @@ USE_EMBED_KEYS = [
     for key in Settings.model_fields
     if key.startswith("USE_") and key in EmbeddingKeyValues.__members__
 ]
+USE_TTS_KEYS = [
+    key
+    for key in Settings.model_fields
+    if key.startswith("USE_") and key.endswith("_TTS")
+]
+USE_STT_KEYS = [
+    key
+    for key in Settings.model_fields
+    if key.startswith("USE_") and key.endswith("_STT")
+]
+
+
+def is_openai_configured() -> bool:
+    s = get_settings()
+    v = s.OPENAI_API_KEY
+    if isinstance(v, SecretStr):
+        try:
+            if v.get_secret_value().strip():
+                return True
+        except Exception:
+            pass
+    elif v and str(v).strip():
+        return True
+    env = os.getenv("OPENAI_API_KEY")
+    return bool(env and env.strip())
 
 
 def handle_save_result(
@@ -286,29 +312,37 @@ def unset_environ_in_store(
 
 def _as_legacy_use_key(
     k: str,
-) -> Union[ModelKeyValues, EmbeddingKeyValues, None]:
+) -> Union[ModelKeyValues, EmbeddingKeyValues, SpeechKeyValues, None]:
     if k in ModelKeyValues.__members__:
         return ModelKeyValues[k]
     if k in EmbeddingKeyValues.__members__:
         return EmbeddingKeyValues[k]
+    if k in SpeechKeyValues.__members__:
+        return SpeechKeyValues[k]
     return None
 
 
 def switch_model_provider(
-    target: Union[ModelKeyValues, EmbeddingKeyValues],
+    target: Union[ModelKeyValues, EmbeddingKeyValues, SpeechKeyValues],
     save: Optional[str] = None,
 ) -> Tuple[bool, Optional[str]]:
     """
-    Ensure exactly one USE_* flag is enabled.
+    Ensure exactly one USE_* flag is enabled within the target's family.
     We *unset* all other USE_* keys (instead of writing explicit "NO") to:
       - keep dotenv clean
       - preserve Optional[bool] semantics (unset vs explicit false)
     """
-    keys_to_clear = (
-        USE_LLM_KEYS if isinstance(target, ModelKeyValues) else USE_EMBED_KEYS
-    )
     target_key = target.name  # or _to_str_key(target)
 
+    if target_key.endswith("_TTS"):
+        keys_to_clear = USE_TTS_KEYS
+    elif target_key.endswith("_STT"):
+        keys_to_clear = USE_STT_KEYS
+    elif isinstance(target, ModelKeyValues):
+        keys_to_clear = USE_LLM_KEYS
+    else:
+        keys_to_clear = USE_EMBED_KEYS
+
     if target_key not in keys_to_clear:
         raise ValueError(f"{target} is not a recognized USE_* model key")
 

deepeval/config/settings.py

@@ -74,14 +74,31 @@ _DEPRECATED_TO_OVERRIDE = {
 _LEGACY_KEYFILE_SECRET_WARNED: set[str] = set()
 
 
+def _family_of_use_flag(env_key: str) -> str:
+    """The provider family a USE_* flag belongs to.
+
+    Families are mutually exclusive within themselves but independent of each
+    other, so selecting an LLM provider leaves the embedding, TTS and STT
+    selections alone.
+    """
+    if env_key.endswith("_TTS"):
+        return "tts"
+    if env_key.endswith("_STT"):
+        return "stt"
+    if "EMBEDDING" in env_key:
+        return "embedding"
+    return "llm"
+
+
 def _find_legacy_enum(env_key: str):
     from deepeval.key_handler import (
         ModelKeyValues,
         EmbeddingKeyValues,
+        SpeechKeyValues,
         KeyValues,
     )
 
-    enums = (ModelKeyValues, EmbeddingKeyValues, KeyValues)
+    enums = (ModelKeyValues, EmbeddingKeyValues, SpeechKeyValues, KeyValues)
 
     for enum in enums:
         try:
@@ -129,6 +146,7 @@ def _merge_legacy_keyfile_into_env() -> None:
         KeyValues,
         ModelKeyValues,
         EmbeddingKeyValues,
+        SpeechKeyValues,
     )
 
     key_path = Path(HIDDEN_DIR) / KEY_FILE
@@ -149,7 +167,7 @@ def _merge_legacy_keyfile_into_env() -> None:
 
     # Map JSON keys (enum .value) -> env keys (enum .name)
     mapping: Dict[str, str] = {}
-    for enum in (KeyValues, ModelKeyValues, EmbeddingKeyValues):
+    for enum in (KeyValues, ModelKeyValues, EmbeddingKeyValues, SpeechKeyValues):
         for member in enum:
             mapping[member.value] = member.name
 
@@ -711,6 +729,69 @@ class Settings(BaseSettings):
     )
     VLLM_MODEL_NAME: Optional[str] = Field(None, description="vLLM model name.")
 
+    #
+    # Speech Keys (TTS/STT)
+    #
+    # TTS and STT are two independent families: a USE_*_TTS flag and a
+    # USE_*_STT flag can both be on, and each family's model name comes from
+    # DEEPEVAL_TTS_MODEL / DEEPEVAL_STT_MODEL. With no flag set, voice mode
+    # falls back to OpenAI, the same way `initialize_model()` does for LLMs.
+
+    # TTS
+    USE_OPENAI_TTS: Optional[bool] = Field(
+        None, description="Use OpenAI for text-to-speech."
+    )
+    USE_ELEVENLABS_TTS: Optional[bool] = Field(
+        None, description="Use ElevenLabs for text-to-speech."
+    )
+    USE_CARTESIA_TTS: Optional[bool] = Field(
+        None, description="Use Cartesia for text-to-speech."
+    )
+    USE_DEEPGRAM_TTS: Optional[bool] = Field(
+        None, description="Use Deepgram for text-to-speech."
+    )
+    DEEPEVAL_TTS_MODEL: Optional[str] = Field(
+        None,
+        description="Model name for the selected TTS provider (defaults to that provider's own default).",
+    )
+
+    # STT
+    USE_OPENAI_STT: Optional[bool] = Field(
+        None, description="Use OpenAI for speech-to-text."
+    )
+    USE_ELEVENLABS_STT: Optional[bool] = Field(
+        None, description="Use ElevenLabs for speech-to-text."
+    )
+    USE_CARTESIA_STT: Optional[bool] = Field(
+        None, description="Use Cartesia for speech-to-text."
+    )
+    USE_DEEPGRAM_STT: Optional[bool] = Field(
+        None, description="Use Deepgram for speech-to-text."
+    )
+    USE_ASSEMBLYAI_STT: Optional[bool] = Field(
+        None, description="Use AssemblyAI for speech-to-text."
+    )
+    DEEPEVAL_STT_MODEL: Optional[str] = Field(
+        None,
+        description="Model name for the selected STT provider (defaults to that provider's own default).",
+    )
+
+    ASSEMBLYAI_API_KEY: Optional[SecretStr] = Field(
+        None, description="AssemblyAI API key (speech-to-text)."
+    )
+    CARTESIA_API_KEY: Optional[SecretStr] = Field(
+        None,
+        description="Cartesia API key (text-to-speech and speech-to-text).",
+    )
+    DEEPGRAM_API_KEY: Optional[SecretStr] = Field(
+        None,
+        description="Deepgram API key (text-to-speech and speech-to-text).",
+    )
+    ELEVENLABS_API_KEY: Optional[SecretStr] = Field(
+        None,
+        description="ElevenLabs API key (text-to-speech and speech-to-text).",
+    )
+
     #
     # Embedding Keys
     #
@@ -1063,6 +1144,15 @@ class Settings(BaseSettings):
         "USE_AZURE_OPENAI_EMBEDDING",
         "USE_LOCAL_EMBEDDINGS",
         "USE_PORTKEY_MODEL",
+        "USE_OPENAI_TTS",
+        "USE_ELEVENLABS_TTS",
+        "USE_CARTESIA_TTS",
+        "USE_DEEPGRAM_TTS",
+        "USE_OPENAI_STT",
+        "USE_ELEVENLABS_STT",
+        "USE_CARTESIA_STT",
+        "USE_DEEPGRAM_STT",
+        "USE_ASSEMBLYAI_STT",
         mode="before",
     )
     @classmethod
@@ -1520,22 +1610,19 @@ class Settings(BaseSettings):
 
         def switch_model_provider(self, target) -> None:
             """
-            Flip USE_* settings within the target's provider family (LLM vs embeddings).
+            Flip USE_* settings within the target's provider family
+            (LLM, embeddings, TTS or STT).
             """
             from deepeval.key_handler import KEY_FILE_HANDLER
 
             target_key = getattr(target, "value", str(target))
-
-            def _is_embedding_flag(k: str) -> bool:
-                return "EMBEDDING" in k
-
-            target_is_embedding = _is_embedding_flag(target_key)
+            target_family = _family_of_use_flag(target_key)
 
             use_fields = [
                 field
                 for field in type(self._s).model_fields
                 if field.startswith("USE_")
-                and _is_embedding_flag(field) == target_is_embedding
+                and _family_of_use_flag(field) == target_family
             ]
 
             if target_key not in use_fields:

deepeval/constants.py

@@ -42,6 +42,12 @@ class ProviderSlug(str, Enum):
     OLLAMA = "ollama"
     OPENROUTER = "openrouter"
     PORTKEY = "portkey"
+    # Speech (TTS/STT) providers. They serve no LLMs, so they appear here only
+    # to carry a retry policy, not as selectable LLM providers.
+    ASSEMBLYAI = "assemblyai"
+    CARTESIA = "cartesia"
+    DEEPGRAM = "deepgram"
+    ELEVENLABS = "elevenlabs"
 
 
 def slugify(value: Union[str, ProviderSlug]) -> str:

deepeval/errors.py

@@ -1,3 +1,6 @@
+from typing import Optional
+
+
 class DeepEvalError(Exception):
     """Base class for framework-originated errors.
     If raised and not handled, it will abort the current operation.
@@ -12,6 +15,34 @@ class UserAppError(Exception):
     """
 
 
+class SpeechHTTPError(DeepEvalError):
+    """A speech (TTS/STT) provider returned a non-2xx response.
+
+    Defined here rather than beside the transport that raises it so
+    `deepeval.models.retry_policy` can classify it without importing
+    `deepeval.models.speech`, whose providers import the retry policy.
+    """
+
+    def __init__(
+        self,
+        message: str,
+        *,
+        status_code: Optional[int] = None,
+        provider_label: Optional[str] = None,
+    ):
+        super().__init__(message)
+        self.status_code = status_code
+        self.provider_label = provider_label
+
+
+class SpeechAuthError(SpeechHTTPError):
+    """Speech provider credentials were rejected (401/403). Never retryable."""
+
+
+class SpeechRateLimitError(SpeechHTTPError):
+    """A speech provider is rate limiting (429). Usually worth retrying."""
+
+
 class MissingTestCaseParamsError(DeepEvalError):
     """Required test case fields are missing."""
 

deepeval/key_handler.py

@@ -180,6 +180,30 @@ class EmbeddingKeyValues(Enum):
     LOCAL_EMBEDDING_API_KEY = ("LOCAL_EMBEDDING_API_KEY",)
 
 
+class SpeechKeyValues(Enum):
+    # TTS
+    USE_OPENAI_TTS = "USE_OPENAI_TTS"
+    USE_ELEVENLABS_TTS = "USE_ELEVENLABS_TTS"
+    USE_CARTESIA_TTS = "USE_CARTESIA_TTS"
+    USE_DEEPGRAM_TTS = "USE_DEEPGRAM_TTS"
+    DEEPEVAL_TTS_MODEL = "DEEPEVAL_TTS_MODEL"
+
+    # STT
+    USE_OPENAI_STT = "USE_OPENAI_STT"
+    USE_ELEVENLABS_STT = "USE_ELEVENLABS_STT"
+    USE_CARTESIA_STT = "USE_CARTESIA_STT"
+    USE_DEEPGRAM_STT = "USE_DEEPGRAM_STT"
+    USE_ASSEMBLYAI_STT = "USE_ASSEMBLYAI_STT"
+    DEEPEVAL_STT_MODEL = "DEEPEVAL_STT_MODEL"
+
+    # Provider API keys. `write_key` refuses secrets, so these never reach the
+    # JSON store; they are here so the CLI can name them in one place.
+    ASSEMBLYAI_API_KEY = "ASSEMBLYAI_API_KEY"
+    CARTESIA_API_KEY = "CARTESIA_API_KEY"
+    DEEPGRAM_API_KEY = "DEEPGRAM_API_KEY"
+    ELEVENLABS_API_KEY = "ELEVENLABS_API_KEY"
+
+
 class KeyFileHandler:
     def __init__(self):
         self.data = {}
@@ -188,7 +212,7 @@ class KeyFileHandler:
         os.makedirs(HIDDEN_DIR, exist_ok=True)
 
     def write_key(
-        self, key: Union[KeyValues, ModelKeyValues, EmbeddingKeyValues], value
+        self, key: Union[KeyValues, ModelKeyValues, EmbeddingKeyValues, SpeechKeyValues], value
     ):
         """Appends or updates data in the hidden file"""
 
@@ -222,7 +246,7 @@ class KeyFileHandler:
             json.dump(self.data, f)
 
     def fetch_data(
-        self, key: Union[KeyValues, ModelKeyValues, EmbeddingKeyValues]
+        self, key: Union[KeyValues, ModelKeyValues, EmbeddingKeyValues, SpeechKeyValues]
     ):
         """Fetches the data from the hidden file.
         NOTE: secrets in this file are deprecated; prefer env/.env."""
@@ -257,7 +281,7 @@ class KeyFileHandler:
         return value
 
     def remove_key(
-        self, key: Union[KeyValues, ModelKeyValues, EmbeddingKeyValues]
+        self, key: Union[KeyValues, ModelKeyValues, EmbeddingKeyValues, SpeechKeyValues]
     ):
         """Removes the specified key from the data."""
         try:

deepeval/metrics/utils.py

@@ -3,6 +3,7 @@ import json
 import re
 import sys
 import warnings
+from pydantic import SecretStr
 from typing import (
     Any,
     Callable,
@@ -621,8 +622,13 @@ def should_use_local_model():
 
 
 def should_use_ollama_model():
-    if SETTINGS.LOCAL_MODEL_API_KEY:
-        return SETTINGS.LOCAL_MODEL_API_KEY == "ollama"
+    local_key = SETTINGS.LOCAL_MODEL_API_KEY
+    if local_key:
+        # `LOCAL_MODEL_API_KEY` is a `SecretStr`, which never compares equal to
+        # a plain string, so the sentinel is read out before comparing.
+        if isinstance(local_key, SecretStr):
+            local_key = local_key.get_secret_value()
+        return local_key == "ollama"
     value = KEY_FILE_HANDLER.fetch_data(ModelKeyValues.LOCAL_MODEL_API_KEY)
     return value == "ollama"
 

deepeval/models/__init__.py

@@ -26,8 +26,19 @@ from deepeval.models.embedding_models import (
     LocalEmbeddingModel,
     OllamaEmbeddingModel,
 )
-from deepeval.models.tts_models import OpenAITTSModel
-from deepeval.models.stt_models import OpenAISTTModel
+from deepeval.models.tts import (
+    CartesiaTTSModel,
+    DeepgramTTSModel,
+    ElevenLabsTTSModel,
+    OpenAITTSModel,
+)
+from deepeval.models.stt import (
+    AssemblyAISTTModel,
+    CartesiaSTTModel,
+    DeepgramSTTModel,
+    ElevenLabsSTTModel,
+    OpenAISTTModel,
+)
 
 __all__ = [
     "DeepEvalBaseModel",
@@ -54,6 +65,13 @@ __all__ = [
     "OpenRouterModel",
     "OpenAITTSModel",
     "OpenAISTTModel",
+    "CartesiaTTSModel",
+    "DeepgramTTSModel",
+    "ElevenLabsTTSModel",
+    "AssemblyAISTTModel",
+    "CartesiaSTTModel",
+    "DeepgramSTTModel",
+    "ElevenLabsSTTModel",
 ]
 
 

deepeval/models/retry_policy.py

@@ -987,6 +987,50 @@ except Exception:
     OLLAMA_ERROR_POLICY = None
 
 
+####################
+# Speech providers #
+####################
+# The speech providers (ElevenLabs, Deepgram, Cartesia, AssemblyAI) are reached
+# over plain HTTP rather than through vendor SDKs, so there are no provider
+# exception classes to classify. The speech transport raises its own hierarchy
+# instead, already split along the lines this policy needs, so one policy serves
+# all four. Imported from `deepeval.errors` rather than from the transport that
+# raises them, which would import the providers, which import this module.
+
+try:
+    from deepeval.errors import (
+        SpeechAuthError,
+        SpeechHTTPError,
+        SpeechRateLimitError,
+    )
+
+    def _aiohttp_net_excs() -> tuple[type, ...]:
+        try:
+            import aiohttp
+        except Exception:
+            return ()
+        names = (
+            "ClientConnectionError",
+            "ClientPayloadError",
+            "ServerTimeoutError",
+            "ServerDisconnectedError",
+        )
+        return tuple(getattr(aiohttp, n) for n in names if hasattr(aiohttp, n))
+
+    SPEECH_ERROR_POLICY = ErrorPolicy(
+        auth_excs=(SpeechAuthError,),
+        rate_limit_excs=(SpeechRateLimitError,),
+        network_excs=_aiohttp_net_excs() + _requests_net_excs(),
+        # Checked after the two above, so the base class here only ever sees
+        # statuses that are neither auth nor rate limit.
+        http_excs=(SpeechHTTPError,),
+        non_retryable_codes=frozenset(),
+        message_markers={},
+    )
+except Exception:  # pragma: no cover - aiohttp/requests are core deps
+    SPEECH_ERROR_POLICY = None
+
+
 # Map provider slugs to their policy objects.
 # It is OK if some are None, we'll treat that as no Error Policy / Tenacity
 _POLICY_BY_SLUG: dict[str, Optional[ErrorPolicy]] = {
@@ -1003,6 +1047,10 @@ _POLICY_BY_SLUG: dict[str, Optional[ErrorPolicy]] = {
     PS.OLLAMA.value: OLLAMA_ERROR_POLICY,
     PS.OPENROUTER.value: OPENROUTER_ERROR_POLICY,
     PS.PORTKEY.value: PORTKEY_ERROR_POLICY,
+    PS.ASSEMBLYAI.value: SPEECH_ERROR_POLICY,
+    PS.CARTESIA.value: SPEECH_ERROR_POLICY,
+    PS.DEEPGRAM.value: SPEECH_ERROR_POLICY,
+    PS.ELEVENLABS.value: SPEECH_ERROR_POLICY,
 }
 
 
@@ -1026,6 +1074,10 @@ _STATIC_PRED_BY_SLUG: dict[str, Optional[Callable[[Exception], bool]]] = {
     PS.OLLAMA.value: _opt_pred(OLLAMA_ERROR_POLICY),
     PS.OPENROUTER.value: _opt_pred(OPENROUTER_ERROR_POLICY),
     PS.PORTKEY.value: _opt_pred(PORTKEY_ERROR_POLICY),
+    PS.ASSEMBLYAI.value: _opt_pred(SPEECH_ERROR_POLICY),
+    PS.CARTESIA.value: _opt_pred(SPEECH_ERROR_POLICY),
+    PS.DEEPGRAM.value: _opt_pred(SPEECH_ERROR_POLICY),
+    PS.ELEVENLABS.value: _opt_pred(SPEECH_ERROR_POLICY),
 }
 
 
@@ -1050,4 +1102,5 @@ __all__ = [
     "GOOGLE_ERROR_POLICY",
     "GROK_ERROR_POLICY",
     "LOCAL_ERROR_POLICY",
+    "SPEECH_ERROR_POLICY",
 ]

deepeval/models/speech.py

@@ -0,0 +1,423 @@
+"""Transport and schema plumbing shared by the TTS and STT providers.
+
+The models live in `deepeval.models.tts` and `deepeval.models.stt`, one
+subpackage per provider. Every provider except OpenAI is reached over plain HTTP
+rather than through a vendor SDK: each is a handful of REST calls, and `requests`
+and `aiohttp` are already core dependencies, so `pip install deepeval` is enough
+to reach any of them without a per-provider install and version constraint to
+track. Rather than have nine models each grow their own session handling and
+error translation, that lives here once.
+
+Two HTTP clients are used rather than one: `requests` for the synchronous
+`synthesize`/`transcribe` methods and `aiohttp` for their `a_` counterparts.
+Using each in its native mode avoids driving a sync client from an event loop
+(which blocks it) or bridging an async one into sync code (which needs a loop
+that may already be running).
+"""
+
+from __future__ import annotations
+
+import json as jsonlib
+from dataclasses import dataclass, field
+from typing import (
+    Any,
+    AsyncGenerator,
+    Dict,
+    List,
+    Optional,
+    Tuple,
+    Type,
+    TypeVar,
+)
+
+import aiohttp
+import requests
+from pydantic import BaseModel, ValidationError
+
+# Raised from here, defined in `deepeval.errors`, re-exported for callers.
+from deepeval.errors import (
+    SpeechAuthError,
+    SpeechHTTPError,
+    SpeechRateLimitError,
+)
+
+T = TypeVar("T", bound=BaseModel)
+
+# Long enough that a slow batch transcription of a whole conversation does not
+# trip it, short enough that a hung connection does not stall a run forever.
+DEFAULT_TIMEOUT_SECONDS = 120.0
+
+# Error bodies go into exception messages, so they are truncated: providers can
+# return a wall of JSON, and the useful part is always at the front.
+_MAX_ERROR_BODY_CHARS = 500
+
+
+def _raise_for_status(
+    status: int, body: bytes, *, provider_label: str, url: str
+) -> None:
+    if 200 <= status < 300:
+        return
+
+    detail = body.decode("utf-8", errors="replace").strip()
+    if len(detail) > _MAX_ERROR_BODY_CHARS:
+        detail = detail[:_MAX_ERROR_BODY_CHARS] + "..."
+    message = f"{provider_label} request to {url} failed with HTTP {status}"
+    if detail:
+        message = f"{message}: {detail}"
+
+    if status in (401, 403):
+        raise SpeechAuthError(
+            f"{message}. Check the API key you configured.",
+            status_code=status,
+            provider_label=provider_label,
+        )
+    if status == 429:
+        raise SpeechRateLimitError(
+            message, status_code=status, provider_label=provider_label
+        )
+    raise SpeechHTTPError(
+        message, status_code=status, provider_label=provider_label
+    )
+
+
+@dataclass
+class Multipart:
+    """A multipart/form-data body: one audio file plus scalar fields.
+
+    Every speech endpoint that takes multipart follows this exact shape, so the
+    two HTTP clients' quite different form APIs can be hidden behind it.
+    """
+
+    file_field: str
+    filename: str
+    content: bytes
+    content_type: str
+    fields: Dict[str, Any] = field(default_factory=dict)
+
+    def _scalar_fields(self) -> Dict[str, str]:
+        # Both clients want strings; booleans have to go over the wire in the
+        # JSON spelling ("true"), not Python's ("True").
+        out: Dict[str, str] = {}
+        for key, value in self.fields.items():
+            if value is None:
+                continue
+            if isinstance(value, bool):
+                out[key] = "true" if value else "false"
+            elif isinstance(value, (list, tuple, dict)):
+                out[key] = jsonlib.dumps(value)
+            else:
+                out[key] = str(value)
+        return out
+
+    def as_requests_kwargs(self) -> Dict[str, Any]:
+        return {
+            "files": {
+                self.file_field: (
+                    self.filename,
+                    self.content,
+                    self.content_type,
+                )
+            },
+            "data": self._scalar_fields(),
+        }
+
+    def as_aiohttp_form(self) -> aiohttp.FormData:
+        form = aiohttp.FormData()
+        for key, value in self._scalar_fields().items():
+            form.add_field(key, value)
+        form.add_field(
+            self.file_field,
+            self.content,
+            filename=self.filename,
+            content_type=self.content_type,
+        )
+        return form
+
+
+class SpeechTransport:
+    """Issues HTTP requests to one provider, sync or async.
+
+    Each async request opens its own `aiohttp.ClientSession`. Caching one would
+    save a handshake per call, but an `aiohttp.ClientSession` is bound to the
+    event loop that created it and has to be closed on that same loop — and a
+    speech model has no teardown hook to close it from, since `VoiceConfig`
+    holds the model for the life of the process. A cached session would
+    therefore outlive its loop and be reported as an unclosed session at
+    interpreter shutdown. A streaming response still holds one connection for
+    its whole body, which is where reuse actually matters here.
+    """
+
+    def __init__(
+        self,
+        *,
+        provider_label: str,
+        base_url: str,
+        headers: Dict[str, str],
+        timeout: float = DEFAULT_TIMEOUT_SECONDS,
+    ):
+        self.provider_label = provider_label
+        self.base_url = base_url.rstrip("/")
+        self.headers = {k: v for k, v in headers.items() if v is not None}
+        self.timeout = timeout
+        self._sync_session: Optional[requests.Session] = None
+
+    def url(self, path: str) -> str:
+        if path.startswith(("http://", "https://")):
+            return path
+        return f"{self.base_url}/{path.lstrip('/')}"
+
+    def _merged_headers(
+        self, extra: Optional[Dict[str, str]] = None
+    ) -> Dict[str, str]:
+        headers = dict(self.headers)
+        if extra:
+            headers.update({k: v for k, v in extra.items() if v is not None})
+        return headers
+
+    #
+    # Synchronous
+    #
+
+    def _requests_session(self) -> requests.Session:
+        if self._sync_session is None:
+            self._sync_session = requests.Session()
+        return self._sync_session
+
+    def _request(
+        self,
+        method: str,
+        path: str,
+        *,
+        params: Optional[Dict[str, Any]] = None,
+        json: Optional[Dict[str, Any]] = None,
+        content: Optional[bytes] = None,
+        multipart: Optional[Multipart] = None,
+        headers: Optional[Dict[str, str]] = None,
+    ) -> Tuple[int, bytes]:
+        url = self.url(path)
+        kwargs: Dict[str, Any] = {
+            "params": _clean_params(params),
+            "headers": self._merged_headers(headers),
+            "timeout": self.timeout,
+        }
+        if json is not None:
+            kwargs["json"] = json
+        if content is not None:
+            kwargs["data"] = content
+        if multipart is not None:
+            kwargs.update(multipart.as_requests_kwargs())
+
+        response = self._requests_session().request(method, url, **kwargs)
+        body = response.content
+        _raise_for_status(
+            response.status_code,
+            body,
+            provider_label=self.provider_label,
+            url=url,
+        )
+        return response.status_code, body
+
+    def request_bytes(self, method: str, path: str, **kwargs) -> bytes:
+        _, body = self._request(method, path, **kwargs)
+        return body
+
+    def request_json(self, method: str, path: str, **kwargs) -> Any:
+        _, body = self._request(method, path, **kwargs)
+        return _decode_json(body, provider_label=self.provider_label)
+
+    #
+    # Asynchronous
+    #
+
+    def _session(self) -> aiohttp.ClientSession:
+        return aiohttp.ClientSession(
+            timeout=aiohttp.ClientTimeout(total=self.timeout)
+        )
+
+    def _a_request_kwargs(
+        self,
+        *,
+        params: Optional[Dict[str, Any]],
+        json: Optional[Dict[str, Any]],
+        content: Optional[bytes],
+        multipart: Optional[Multipart],
+        headers: Optional[Dict[str, str]],
+    ) -> Dict[str, Any]:
+        kwargs: Dict[str, Any] = {
+            "params": _clean_params(params),
+            "headers": self._merged_headers(headers),
+        }
+        if json is not None:
+            kwargs["json"] = json
+        if content is not None:
+            kwargs["data"] = content
+        if multipart is not None:
+            kwargs["data"] = multipart.as_aiohttp_form()
+        return kwargs
+
+    async def _a_request(
+        self,
+        method: str,
+        path: str,
+        *,
+        params: Optional[Dict[str, Any]] = None,
+        json: Optional[Dict[str, Any]] = None,
+        content: Optional[bytes] = None,
+        multipart: Optional[Multipart] = None,
+        headers: Optional[Dict[str, str]] = None,
+    ) -> bytes:
+        url = self.url(path)
+        kwargs = self._a_request_kwargs(
+            params=params,
+            json=json,
+            content=content,
+            multipart=multipart,
+            headers=headers,
+        )
+        async with self._session() as session:
+            async with session.request(method, url, **kwargs) as response:
+                body = await response.read()
+                _raise_for_status(
+                    response.status,
+                    body,
+                    provider_label=self.provider_label,
+                    url=url,
+                )
+                return body
+
+    async def a_request_bytes(self, method: str, path: str, **kwargs) -> bytes:
+        return await self._a_request(method, path, **kwargs)
+
+    async def a_request_json(self, method: str, path: str, **kwargs) -> Any:
+        body = await self._a_request(method, path, **kwargs)
+        return _decode_json(body, provider_label=self.provider_label)
+
+    async def a_stream_bytes(
+        self,
+        method: str,
+        path: str,
+        *,
+        params: Optional[Dict[str, Any]] = None,
+        json: Optional[Dict[str, Any]] = None,
+        content: Optional[bytes] = None,
+        multipart: Optional[Multipart] = None,
+        headers: Optional[Dict[str, str]] = None,
+        chunk_size: int = 4096,
+    ) -> AsyncGenerator[bytes, None]:
+        """Yield the response body as it arrives.
+
+        An error response is read in full before raising, since the body is the
+        only place the provider explains what went wrong.
+        """
+        url = self.url(path)
+        kwargs = self._a_request_kwargs(
+            params=params,
+            json=json,
+            content=content,
+            multipart=multipart,
+            headers=headers,
+        )
+        async with self._session() as session:
+            async with session.request(method, url, **kwargs) as response:
+                if response.status < 200 or response.status >= 300:
+                    _raise_for_status(
+                        response.status,
+                        await response.read(),
+                        provider_label=self.provider_label,
+                        url=url,
+                    )
+                async for data in response.content.iter_chunked(chunk_size):
+                    if data:
+                        yield data
+
+    def close(self) -> None:
+        if self._sync_session is not None:
+            self._sync_session.close()
+            self._sync_session = None
+
+
+def _clean_params(
+    params: Optional[Dict[str, Any]],
+) -> Optional[List[Tuple[str, str]]]:
+    """Drop unset params and spell values the way query strings expect.
+
+    Returns pairs rather than a dict for two reasons: providers spell a
+    repeated param (Deepgram's `keyterm`) as the same key several times, which a
+    dict cannot hold, and `aiohttp` rejects a dict whose values are lists while
+    accepting pairs. Booleans are normalized here too, since `?smart_format=True`
+    is not the `true` these APIs look for.
+    """
+    if not params:
+        return None
+    out: List[Tuple[str, str]] = []
+    for key, value in params.items():
+        if value is None:
+            continue
+        if isinstance(value, (list, tuple)):
+            out.extend(
+                (key, _param_value(item)) for item in value if item is not None
+            )
+        else:
+            out.append((key, _param_value(value)))
+    return out or None
+
+
+def _param_value(value: Any) -> str:
+    if isinstance(value, bool):
+        return "true" if value else "false"
+    return str(value)
+
+
+def _decode_json(body: bytes, *, provider_label: str) -> Any:
+    try:
+        return jsonlib.loads(body)
+    except ValueError as exc:
+        snippet = body.decode("utf-8", errors="replace")[:200]
+        raise SpeechHTTPError(
+            f"{provider_label} returned a response that is not valid JSON: "
+            f"{snippet}",
+            provider_label=provider_label,
+        ) from exc
+
+
+#
+# Schema
+#
+# Each provider's `schema.py` declares the request and response shapes deepeval
+# relies on. These two helpers are the only places those models cross the wire,
+# so serialization rules and validation failures are handled the same way
+# everywhere.
+#
+
+
+def dump_request(model: BaseModel) -> Dict[str, Any]:
+    """Serialize a request model for the wire.
+
+    `exclude_none` is what lets a schema declare every optional parameter a
+    provider accepts without sending the ones nobody set — an explicit `null`
+    is not the same as an absent field to these APIs, and several read it as an
+    error. Extras survive, which is how `generation_kwargs` reaches a provider
+    parameter that has no field of its own yet.
+    """
+    return model.model_dump(exclude_none=True, by_alias=True)
+
+
+def parse_response(schema: Type[T], payload: Any, *, provider_label: str) -> T:
+    """Validate a provider response, or fail with something readable.
+
+    Response models allow unknown fields, so this only rejects a payload whose
+    declared fields are the wrong shape entirely. When that happens the useful
+    thing to report is that the provider's response no longer matches what
+    deepeval expects — a raw `ValidationError` names pydantic models the reader
+    has never heard of and says nothing about which provider broke.
+    """
+    try:
+        return schema.model_validate(payload)
+    except ValidationError as exc:
+        raise SpeechHTTPError(
+            f"{provider_label} returned a response deepeval could not read. "
+            f"This usually means the provider changed its response format: "
+            f"{exc.error_count()} field(s) did not match "
+            f"{schema.__name__} ({exc.errors(include_url=False)}).",
+            provider_label=provider_label,
+        ) from exc

deepeval/models/speech_selection.py

@@ -0,0 +1,111 @@
+"""Provider selection for speech models.
+
+TTS and STT mirror the LLM mechanism in `deepeval.metrics.utils`: a `USE_*`
+flag picks the provider, and the model name comes from the family-level
+`DEEPEVAL_TTS_MODEL` / `DEEPEVAL_STT_MODEL` unless one is passed in code. With
+no flag set, both families fall back to OpenAI.
+"""
+
+from __future__ import annotations
+
+from typing import Optional, Tuple, Union
+
+from deepeval.config.settings import get_settings
+from deepeval.key_handler import KEY_FILE_HANDLER, SpeechKeyValues
+from deepeval.models.base_model import DeepEvalBaseSTT, DeepEvalBaseTTS
+
+# Flag -> class name, in the order they are checked. Named rather than
+# imported because `deepeval.models.tts` reaches back into `deepeval.voice`.
+_TTS_PROVIDERS: Tuple[Tuple[str, str], ...] = (
+    ("USE_OPENAI_TTS", "OpenAITTSModel"),
+    ("USE_ELEVENLABS_TTS", "ElevenLabsTTSModel"),
+    ("USE_CARTESIA_TTS", "CartesiaTTSModel"),
+    ("USE_DEEPGRAM_TTS", "DeepgramTTSModel"),
+)
+_STT_PROVIDERS: Tuple[Tuple[str, str], ...] = (
+    ("USE_OPENAI_STT", "OpenAISTTModel"),
+    ("USE_ELEVENLABS_STT", "ElevenLabsSTTModel"),
+    ("USE_CARTESIA_STT", "CartesiaSTTModel"),
+    ("USE_DEEPGRAM_STT", "DeepgramSTTModel"),
+    ("USE_ASSEMBLYAI_STT", "AssemblyAISTTModel"),
+)
+
+
+def _from_keystore(field: str) -> Optional[str]:
+    member = SpeechKeyValues.__members__.get(field)
+    if member is None:
+        return None
+    return KEY_FILE_HANDLER.fetch_data(member)
+
+
+def _flag_enabled(field: str) -> bool:
+    if getattr(get_settings(), field, None):
+        return True
+    value = _from_keystore(field)
+    return value.lower() == "yes" if value is not None else False
+
+
+def _model_name(field: str, override: Optional[str]) -> Optional[str]:
+    if override is not None:
+        return override
+    return getattr(get_settings(), field, None) or _from_keystore(field)
+
+
+def _selected(providers: Tuple[Tuple[str, str], ...]) -> str:
+    for flag, class_name in providers:
+        if _flag_enabled(flag):
+            return class_name
+    # No flag set: OpenAI, matching `initialize_model()`'s LLM fallback.
+    return providers[0][1]
+
+
+def describe_tts_selection() -> Tuple[str, Optional[str]]:
+    """The TTS class name and model name selected by the environment.
+
+    Reported without constructing anything, so it stays answerable on a
+    machine with no speech API key set.
+    """
+    return _selected(_TTS_PROVIDERS), _model_name("DEEPEVAL_TTS_MODEL", None)
+
+
+def describe_stt_selection() -> Tuple[str, Optional[str]]:
+    """The STT class name and model name selected by the environment."""
+    return _selected(_STT_PROVIDERS), _model_name("DEEPEVAL_STT_MODEL", None)
+
+
+def initialize_tts_model(
+    model: Optional[Union[str, DeepEvalBaseTTS]] = None,
+) -> DeepEvalBaseTTS:
+    """The TTS model for this run.
+
+    A model object is used as-is, a string is the model name for the
+    flag-selected provider, and `None` falls back to `DEEPEVAL_TTS_MODEL`.
+    """
+    if model is not None and not isinstance(model, str):
+        return model
+
+    from deepeval.models import tts
+
+    class_name = _selected(_TTS_PROVIDERS)
+    return getattr(tts, class_name)(
+        model=_model_name("DEEPEVAL_TTS_MODEL", model)
+    )
+
+
+def initialize_stt_model(
+    model: Optional[Union[str, DeepEvalBaseSTT]] = None,
+) -> DeepEvalBaseSTT:
+    """The STT model for this run.
+
+    A model object is used as-is, a string is the model name for the
+    flag-selected provider, and `None` falls back to `DEEPEVAL_STT_MODEL`.
+    """
+    if model is not None and not isinstance(model, str):
+        return model
+
+    from deepeval.models import stt
+
+    class_name = _selected(_STT_PROVIDERS)
+    return getattr(stt, class_name)(
+        model=_model_name("DEEPEVAL_STT_MODEL", model)
+    )

deepeval/models/stt/__init__.py

@@ -0,0 +1,13 @@
+from deepeval.models.stt.assemblyai import AssemblyAISTTModel
+from deepeval.models.stt.cartesia import CartesiaSTTModel
+from deepeval.models.stt.deepgram import DeepgramSTTModel
+from deepeval.models.stt.elevenlabs import ElevenLabsSTTModel
+from deepeval.models.stt.openai import OpenAISTTModel
+
+__all__ = [
+    "AssemblyAISTTModel",
+    "CartesiaSTTModel",
+    "DeepgramSTTModel",
+    "ElevenLabsSTTModel",
+    "OpenAISTTModel",
+]

deepeval/models/stt/_audio.py

@@ -0,0 +1,31 @@
+"""Audio inspection shared by the STT models."""
+
+from __future__ import annotations
+
+import io
+import wave
+from typing import Optional, TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from deepeval.test_case import Audio
+
+
+def audio_duration_seconds(audio: "Audio") -> Optional[float]:
+    """How long `audio` runs, or `None` when it cannot be determined.
+
+    Needed because several providers price transcription by the minute but
+    report usage only sometimes, leaving the duration to be measured locally.
+    Anything that is not a readable WAV yields `None` rather than a guess, so a
+    cost is never invented from a number we do not have.
+    """
+    if audio.duration is not None:
+        return audio.duration
+    try:
+        with wave.open(io.BytesIO(audio.get_bytes()), "rb") as wf:
+            frames = wf.getnframes()
+            rate = wf.getframerate()
+            if rate:
+                return frames / float(rate)
+    except (wave.Error, EOFError, ValueError):
+        pass
+    return None

deepeval/models/stt/_stream.py

@@ -0,0 +1,75 @@
+"""Partial transcripts for providers whose batch API is all we use."""
+
+from __future__ import annotations
+
+from typing import AsyncGenerator, AsyncIterable, Optional
+
+from deepeval.test_case import Audio, AudioChunk
+from deepeval.voice.connectors import audio_utils
+
+DEFAULT_STREAM_SAMPLE_RATE = 24000
+
+
+class BufferedTranscribeMixin:
+    """Approximates streaming transcription on top of a batch endpoint.
+
+    `DeepEvalBaseSTT.a_transcribe_stream` raises by default, so a model without
+    it cannot follow an agent as it speaks — a caller wanting a transcript
+    before the utterance ends has nothing to consume. Every provider here has a
+    realtime WebSocket API, but the models talk to the batch endpoint, so this
+    buffers the incoming PCM and re-transcribes the accumulated audio whenever
+    roughly `partial_every_seconds` of new audio has arrived, yielding the
+    transcript each time it changes.
+
+    Re-transcribing from the top is wasteful, and deliberately so: it keeps a
+    provider's streaming protocol out of the models, and the audio involved is a
+    single conversational turn. Mixing this in gives a model working partials
+    from `a_transcribe` alone; a provider whose realtime API is worth the
+    complexity can override `a_transcribe_stream` instead.
+
+    Must be mixed in ahead of `DeepEvalBaseSTT` so it takes precedence over the
+    base's `NotImplementedError`.
+    """
+
+    async def a_transcribe_stream(
+        self,
+        audio_stream: AsyncIterable[AudioChunk],
+        *args,
+        language: Optional[str] = None,
+        partial_every_seconds: float = 1.0,
+        **kwargs,
+    ) -> AsyncGenerator[str, None]:
+        pcm = bytearray()
+        sample_rate = DEFAULT_STREAM_SAMPLE_RATE
+        bytes_since_partial = 0
+        last_text = ""
+
+        async for chunk in audio_stream:
+            if chunk.sampleRate:
+                sample_rate = chunk.sampleRate
+            frame = chunk.get_bytes()
+            pcm.extend(frame)
+            bytes_since_partial += len(frame)
+
+            threshold = int(sample_rate * 2 * partial_every_seconds)
+            if bytes_since_partial < threshold and not chunk.final:
+                continue
+
+            audio = Audio.from_bytes(
+                audio_utils.pcm16_to_wav_bytes(bytes(pcm), sample_rate, 1),
+                "audio/wav",
+                sampleRate=sample_rate,
+                encoding="wav",
+            )
+            text, _ = await self.a_transcribe(
+                audio, language=language, **kwargs
+            )
+            if text and text != last_text:
+                last_text = text
+                yield text
+            bytes_since_partial = 0
+            if chunk.final:
+                break
+
+    def supports_streaming(self) -> bool:
+        return True

deepeval/models/stt/assemblyai/__init__.py

@@ -0,0 +1,3 @@
+from deepeval.models.stt.assemblyai.assemblyai import AssemblyAISTTModel
+
+__all__ = ["AssemblyAISTTModel"]

deepeval/models/stt/assemblyai/assemblyai.py

@@ -0,0 +1,330 @@
+import asyncio
+import json as jsonlib
+import time
+from typing import Any, Dict, Optional, Tuple
+
+from pydantic import SecretStr
+
+from deepeval.config.settings import get_settings
+from deepeval.constants import ProviderSlug as PS
+from deepeval.models.base_model import DeepEvalBaseSTT
+from deepeval.models.retry_policy import create_retry_decorator
+from deepeval.models.speech import (
+    DEFAULT_TIMEOUT_SECONDS,
+    Multipart,
+    SpeechHTTPError,
+    SpeechTransport,
+    dump_request,
+    parse_response,
+)
+from deepeval.models.stt._audio import audio_duration_seconds
+from deepeval.models.stt._stream import BufferedTranscribeMixin
+from deepeval.models.stt.assemblyai.schema import (
+    AssemblyAISyncConfig,
+    AssemblyAITranscriptRequest,
+    AssemblyAITranscriptResponse,
+    AssemblyAIUploadResponse,
+)
+from deepeval.models.utils import parse_model_name, require_secret_api_key
+from deepeval.test_case import Audio
+
+retry_assemblyai = create_retry_decorator(PS.ASSEMBLYAI)
+
+DEFAULT_BASE_URL = "https://api.assemblyai.com"
+
+# The synchronous endpoint lives on its own host and answers in one round trip.
+SYNC_URL = "https://sync.assemblyai.com/transcribe"
+
+# `slam-1` is deprecated in favour of this.
+DEFAULT_STT_MODEL = "universal-3-5-pro"
+
+# The synchronous endpoint's ceiling. Conversation turns sit far below it, so it
+# is the normal path; anything longer falls back to upload-and-poll.
+SYNC_MAX_SECONDS = 120.0
+
+_SYNC_PRICE_PER_HOUR = 0.45
+_ASYNC_PRICE_PER_HOUR = {
+    "universal-3-5-pro": 0.21,
+    "universal-2": 0.15,
+}
+
+DEFAULT_POLL_INTERVAL_SECONDS = 1.0
+
+
+class AssemblyAISTTModel(BufferedTranscribeMixin, DeepEvalBaseSTT):
+    """AssemblyAI speech-to-text.
+
+    Prefers the synchronous endpoint, which returns a transcript in a single
+    request (~134 ms p50) instead of the three-step upload, create and poll
+    flow. That endpoint caps clips at 120 seconds, which no simulated
+    conversation turn approaches, so the slower flow is used only when the audio
+    really is longer.
+    """
+
+    # Universal does not complete a word clipped by a barge-in, so the audio is
+    # transcribed exactly as it was heard.
+    truncated_audio_pad_seconds: float = 0.0
+
+    def __init__(
+        self,
+        model: Optional[str] = None,
+        api_key: Optional[str] = None,
+        base_url: Optional[str] = None,
+        language: Optional[str] = None,
+        cost_per_hour: Optional[float] = None,
+        transcription_kwargs: Optional[Dict] = None,
+        timeout: float = DEFAULT_TIMEOUT_SECONDS,
+        poll_interval_seconds: float = DEFAULT_POLL_INTERVAL_SECONDS,
+    ):
+        settings = get_settings()
+        self.name = parse_model_name(model or DEFAULT_STT_MODEL)
+        self.language = language
+        self.transcription_kwargs = transcription_kwargs or {}
+        self.cost_per_hour = cost_per_hour
+        # Only consulted on the upload-and-poll path, which a conversation turn
+        # never takes.
+        self.poll_interval_seconds = poll_interval_seconds
+        self.api_key = (
+            SecretStr(api_key)
+            if api_key is not None
+            else settings.ASSEMBLYAI_API_KEY
+        )
+        self.base_url = (
+            str(base_url).rstrip("/")
+            if base_url is not None
+            else DEFAULT_BASE_URL
+        )
+        self.timeout = timeout
+        self.model = self.load_model()
+
+    def load_model(self) -> SpeechTransport:
+        api_key = require_secret_api_key(
+            self.api_key,
+            provider_label="AssemblyAI",
+            env_var_name="ASSEMBLYAI_API_KEY",
+            param_hint="`api_key` to AssemblyAISTTModel(...)",
+        )
+        return SpeechTransport(
+            provider_label="AssemblyAI",
+            base_url=self.base_url,
+            # Note the bare key: AssemblyAI does not use a `Bearer` prefix.
+            headers={"Authorization": api_key},
+            timeout=self.timeout,
+        )
+
+    #
+    # Routing
+    #
+
+    def _use_sync_endpoint(self, audio: Audio) -> bool:
+        seconds = audio_duration_seconds(audio)
+        # An unmeasurable duration takes the slower path: it always works,
+        # where the fast one would reject audio that turns out to be too long.
+        if seconds is None:
+            return False
+        return seconds <= SYNC_MAX_SECONDS
+
+    def _cost(self, audio: Audio, *, used_sync: bool) -> Optional[float]:
+        rate = self.cost_per_hour
+        if rate is None:
+            rate = (
+                _SYNC_PRICE_PER_HOUR
+                if used_sync
+                else _ASYNC_PRICE_PER_HOUR.get(self.name)
+            )
+        if rate is None:
+            return None
+        seconds = audio_duration_seconds(audio)
+        if seconds is None:
+            return None
+        return seconds / 3600.0 * rate
+
+    #
+    # Request building
+    #
+
+    def _sync_multipart(
+        self, audio: Audio, language: Optional[str], kwargs: Dict[str, Any]
+    ) -> Multipart:
+        lang = language or self.language
+        config = AssemblyAISyncConfig(
+            sample_rate=audio.sampleRate or None,
+            language_code=None if lang == "auto" else lang,
+            **{**self.transcription_kwargs, **kwargs},
+        )
+        dumped = dump_request(config)
+
+        fields: Dict[str, Any] = {}
+        if dumped:
+            # The whole config rides as one JSON-encoded form field.
+            fields["config"] = jsonlib.dumps(dumped)
+
+        return Multipart(
+            file_field="audio",
+            filename=audio.filename or "audio.wav",
+            content=audio.get_bytes(),
+            content_type=audio.mimeType or "audio/wav",
+            fields=fields,
+        )
+
+    def _create_body(
+        self,
+        upload_url: str,
+        language: Optional[str],
+        kwargs: Dict[str, Any],
+    ) -> Dict[str, Any]:
+        lang = language or self.language
+        auto = lang == "auto"
+        return dump_request(
+            AssemblyAITranscriptRequest(
+                audio_url=upload_url,
+                speech_models=[self.name],
+                language_code=None if auto else lang,
+                language_detection=True if auto else None,
+                **{**self.transcription_kwargs, **kwargs},
+            )
+        )
+
+    #
+    # Response handling
+    #
+
+    def _parse(self, payload: Any) -> AssemblyAITranscriptResponse:
+        return parse_response(
+            AssemblyAITranscriptResponse,
+            payload,
+            provider_label="AssemblyAI",
+        )
+
+    def _raise_if_failed(self, payload: AssemblyAITranscriptResponse) -> None:
+        if payload.failed:
+            raise SpeechHTTPError(
+                "AssemblyAI failed to transcribe the audio: "
+                f"{payload.error or 'no reason given'}",
+                provider_label="AssemblyAI",
+            )
+
+    def _raise_if_timed_out(self, deadline: float, transcript_id: str) -> None:
+        if time.monotonic() >= deadline:
+            raise SpeechHTTPError(
+                "AssemblyAI did not finish transcribing within "
+                f"{self.timeout:g}s (transcript {transcript_id}).",
+                provider_label="AssemblyAI",
+            )
+
+    #
+    # DeepEvalBaseSTT
+    #
+
+    @retry_assemblyai
+    def transcribe(
+        self, audio: Audio, *, language: Optional[str] = None, **kwargs
+    ) -> Tuple[str, Optional[float]]:
+        used_sync = self._use_sync_endpoint(audio)
+        if used_sync:
+            payload = self._parse(
+                self.model.request_json(
+                    "POST",
+                    SYNC_URL,
+                    multipart=self._sync_multipart(audio, language, kwargs),
+                    headers={"X-AAI-Model": self.name},
+                )
+            )
+        else:
+            payload = self._transcribe_by_upload(audio, language, kwargs)
+        return payload.transcript() or "", self._cost(
+            audio, used_sync=used_sync
+        )
+
+    def _transcribe_by_upload(
+        self, audio: Audio, language: Optional[str], kwargs: Dict[str, Any]
+    ) -> AssemblyAITranscriptResponse:
+        uploaded = parse_response(
+            AssemblyAIUploadResponse,
+            self.model.request_json(
+                "POST",
+                "/v2/upload",
+                content=audio.get_bytes(),
+                headers={"Content-Type": "application/octet-stream"},
+            ),
+            provider_label="AssemblyAI",
+        )
+        payload = self._parse(
+            self.model.request_json(
+                "POST",
+                "/v2/transcript",
+                json=self._create_body(uploaded.upload_url, language, kwargs),
+            )
+        )
+        transcript_id = payload.id or ""
+
+        deadline = time.monotonic() + self.timeout
+        while not payload.finished:
+            self._raise_if_timed_out(deadline, transcript_id)
+            time.sleep(self.poll_interval_seconds)
+            payload = self._parse(
+                self.model.request_json(
+                    "GET", f"/v2/transcript/{transcript_id}"
+                )
+            )
+        self._raise_if_failed(payload)
+        return payload
+
+    @retry_assemblyai
+    async def a_transcribe(
+        self, audio: Audio, *, language: Optional[str] = None, **kwargs
+    ) -> Tuple[str, Optional[float]]:
+        used_sync = self._use_sync_endpoint(audio)
+        if used_sync:
+            payload = self._parse(
+                await self.model.a_request_json(
+                    "POST",
+                    SYNC_URL,
+                    multipart=self._sync_multipart(audio, language, kwargs),
+                    headers={"X-AAI-Model": self.name},
+                )
+            )
+        else:
+            payload = await self._a_transcribe_by_upload(
+                audio, language, kwargs
+            )
+        return payload.transcript() or "", self._cost(
+            audio, used_sync=used_sync
+        )
+
+    async def _a_transcribe_by_upload(
+        self, audio: Audio, language: Optional[str], kwargs: Dict[str, Any]
+    ) -> AssemblyAITranscriptResponse:
+        uploaded = parse_response(
+            AssemblyAIUploadResponse,
+            await self.model.a_request_json(
+                "POST",
+                "/v2/upload",
+                content=audio.get_bytes(),
+                headers={"Content-Type": "application/octet-stream"},
+            ),
+            provider_label="AssemblyAI",
+        )
+        payload = self._parse(
+            await self.model.a_request_json(
+                "POST",
+                "/v2/transcript",
+                json=self._create_body(uploaded.upload_url, language, kwargs),
+            )
+        )
+        transcript_id = payload.id or ""
+
+        deadline = time.monotonic() + self.timeout
+        while not payload.finished:
+            self._raise_if_timed_out(deadline, transcript_id)
+            await asyncio.sleep(self.poll_interval_seconds)
+            payload = self._parse(
+                await self.model.a_request_json(
+                    "GET", f"/v2/transcript/{transcript_id}"
+                )
+            )
+        self._raise_if_failed(payload)
+        return payload
+
+    def get_model_name(self) -> str:
+        return self.name

deepeval/models/stt/assemblyai/schema.py

@@ -0,0 +1,87 @@
+"""AssemblyAI request and response shapes.
+
+AssemblyAI has two transcription paths with different request shapes, so both
+are modelled: a synchronous endpoint that takes its settings as one
+JSON-encoded multipart field, and the upload-create-poll flow that takes a JSON
+body.
+"""
+
+from typing import List, Optional
+
+from pydantic import BaseModel, ConfigDict
+
+# Requests allow extras so `transcription_kwargs` can set a provider parameter
+# deepeval has no field for. Responses allow them so a provider adding a field
+# never breaks a run.
+_REQUEST = ConfigDict(extra="allow", protected_namespaces=())
+_RESPONSE = ConfigDict(extra="allow", protected_namespaces=())
+
+# Statuses that end the poll loop.
+TERMINAL_STATUSES = ("completed", "error")
+
+
+class AssemblyAISyncConfig(BaseModel):
+    """The `config` multipart field on `POST sync.assemblyai.com/transcribe`.
+
+    Unlike the async flow, this endpoint takes the model in an `X-AAI-Model`
+    header rather than in the config, so there is no model field here.
+    """
+
+    model_config = _REQUEST
+
+    sample_rate: Optional[int] = None
+    language_code: Optional[str] = None
+
+
+class AssemblyAITranscriptRequest(BaseModel):
+    """Body for `POST /v2/transcript`.
+
+    `speech_models` is an ordered fallback list; the older singular
+    `speech_model` field is deprecated.
+    """
+
+    model_config = _REQUEST
+
+    audio_url: str
+    speech_models: List[str]
+    language_code: Optional[str] = None
+    language_detection: Optional[bool] = None
+
+
+class AssemblyAIUploadResponse(BaseModel):
+    """Response from `POST /v2/upload`."""
+
+    model_config = _RESPONSE
+
+    upload_url: str
+
+
+class AssemblyAITranscriptResponse(BaseModel):
+    """Response from the sync endpoint and from `/v2/transcript` (create + poll).
+
+    `status` is absent on the synchronous endpoint's response, which returns a
+    finished transcript directly and has no queue to report on.
+    """
+
+    model_config = _RESPONSE
+
+    id: Optional[str] = None
+    status: Optional[str] = None
+    text: Optional[str] = None
+    error: Optional[str] = None
+
+    @property
+    def finished(self) -> bool:
+        return self.status is None or self.status in TERMINAL_STATUSES
+
+    @property
+    def failed(self) -> bool:
+        return self.status == "error"
+
+    def transcript(self) -> Optional[str]:
+        """The spoken text, or `None` when the response carries no `text`.
+
+        `None` and `""` mean different things: `""` is silence, which is a
+        perfectly good answer, while `None` means the field was absent.
+        """
+        return self.text

deepeval/models/stt/cartesia/__init__.py

@@ -0,0 +1,3 @@
+from deepeval.models.stt.cartesia.cartesia import CartesiaSTTModel
+
+__all__ = ["CartesiaSTTModel"]

deepeval/models/stt/cartesia/cartesia.py

@@ -0,0 +1,154 @@
+from typing import Any, Dict, Optional, Tuple
+
+from pydantic import SecretStr
+
+from deepeval.config.settings import get_settings
+from deepeval.constants import ProviderSlug as PS
+from deepeval.models.base_model import DeepEvalBaseSTT
+from deepeval.models.retry_policy import create_retry_decorator
+from deepeval.models.speech import (
+    DEFAULT_TIMEOUT_SECONDS,
+    Multipart,
+    SpeechTransport,
+    dump_request,
+    parse_response,
+)
+from deepeval.models.stt._audio import audio_duration_seconds
+from deepeval.models.stt._stream import BufferedTranscribeMixin
+from deepeval.models.stt.cartesia.schema import (
+    CartesiaSTTRequest,
+    CartesiaSTTResponse,
+)
+from deepeval.models.utils import parse_model_name, require_secret_api_key
+from deepeval.test_case import Audio
+
+retry_cartesia = create_retry_decorator(PS.CARTESIA)
+
+DEFAULT_BASE_URL = "https://api.cartesia.ai"
+DEFAULT_API_VERSION = "2026-08-14"
+
+# The batch endpoint only serves the ink-whisper family; `ink-2` is realtime
+# only, so it is not a valid default here.
+DEFAULT_STT_MODEL = "ink-whisper"
+
+
+class CartesiaSTTModel(BufferedTranscribeMixin, DeepEvalBaseSTT):
+    """Cartesia Ink speech-to-text."""
+
+    # Whisper-derived, so it finishes a word the barge-in cut off unless the
+    # clip is presented as a complete utterance.
+    truncated_audio_pad_seconds: float = 0.3
+
+    def __init__(
+        self,
+        model: Optional[str] = None,
+        api_key: Optional[str] = None,
+        base_url: Optional[str] = None,
+        language: Optional[str] = None,
+        api_version: str = DEFAULT_API_VERSION,
+        cost_per_minute: Optional[float] = None,
+        transcription_kwargs: Optional[Dict] = None,
+        timeout: float = DEFAULT_TIMEOUT_SECONDS,
+    ):
+        settings = get_settings()
+        self.name = parse_model_name(model or DEFAULT_STT_MODEL)
+        self.language = language
+        self.api_version = api_version
+        self.transcription_kwargs = transcription_kwargs or {}
+        # Cartesia meters in credits with no published dollar rate, so there is
+        # nothing to default to. Pass `cost_per_minute` to account for spend.
+        self.cost_per_minute = cost_per_minute
+        self.api_key = (
+            SecretStr(api_key)
+            if api_key is not None
+            else settings.CARTESIA_API_KEY
+        )
+        self.base_url = (
+            str(base_url).rstrip("/")
+            if base_url is not None
+            else DEFAULT_BASE_URL
+        )
+        self.timeout = timeout
+        self.model = self.load_model()
+
+    def load_model(self) -> SpeechTransport:
+        api_key = require_secret_api_key(
+            self.api_key,
+            provider_label="Cartesia",
+            env_var_name="CARTESIA_API_KEY",
+            param_hint="`api_key` to CartesiaSTTModel(...)",
+        )
+        return SpeechTransport(
+            provider_label="Cartesia",
+            base_url=self.base_url,
+            headers={
+                "X-Api-Key": api_key,
+                "Cartesia-Version": self.api_version,
+            },
+            timeout=self.timeout,
+        )
+
+    def _multipart(
+        self, audio: Audio, language: Optional[str], kwargs: Dict[str, Any]
+    ) -> Multipart:
+        lang = language or self.language
+        request = CartesiaSTTRequest(
+            model=self.name,
+            # Cartesia defaults to English when `language` is absent rather
+            # than detecting, so "auto" has to be dropped instead of forwarded.
+            language=None if lang == "auto" else lang,
+            **{**self.transcription_kwargs, **kwargs},
+        )
+        return Multipart(
+            file_field="file",
+            filename=audio.filename or "audio.wav",
+            content=audio.get_bytes(),
+            content_type=audio.mimeType or "audio/wav",
+            fields=dump_request(request),
+        )
+
+    def _cost(
+        self, audio: Audio, payload: CartesiaSTTResponse
+    ) -> Optional[float]:
+        if self.cost_per_minute is None:
+            return None
+        seconds = payload.duration
+        if seconds is None:
+            seconds = audio_duration_seconds(audio)
+        if seconds is None:
+            return None
+        return seconds / 60.0 * self.cost_per_minute
+
+    def _parse(self, payload: Any) -> CartesiaSTTResponse:
+        return parse_response(
+            CartesiaSTTResponse, payload, provider_label="Cartesia"
+        )
+
+    @retry_cartesia
+    def transcribe(
+        self, audio: Audio, *, language: Optional[str] = None, **kwargs
+    ) -> Tuple[str, Optional[float]]:
+        payload = self._parse(
+            self.model.request_json(
+                "POST",
+                "/stt",
+                multipart=self._multipart(audio, language, kwargs),
+            )
+        )
+        return payload.transcript() or "", self._cost(audio, payload)
+
+    @retry_cartesia
+    async def a_transcribe(
+        self, audio: Audio, *, language: Optional[str] = None, **kwargs
+    ) -> Tuple[str, Optional[float]]:
+        payload = self._parse(
+            await self.model.a_request_json(
+                "POST",
+                "/stt",
+                multipart=self._multipart(audio, language, kwargs),
+            )
+        )
+        return payload.transcript() or "", self._cost(audio, payload)
+
+    def get_model_name(self) -> str:
+        return self.name

deepeval/models/stt/cartesia/schema.py

@@ -0,0 +1,43 @@
+"""Cartesia speech-to-text request and response shapes."""
+
+from typing import Optional
+
+from pydantic import BaseModel, ConfigDict
+
+# Requests allow extras so `transcription_kwargs` can set a provider parameter
+# deepeval has no field for. Responses allow them so a provider adding a field
+# never breaks a run. `protected_namespaces` is cleared because pydantic
+# reserves the `model_` prefix and Cartesia's field is called `model`.
+_REQUEST = ConfigDict(extra="allow", protected_namespaces=())
+_RESPONSE = ConfigDict(extra="allow", protected_namespaces=())
+
+
+class CartesiaSTTRequest(BaseModel):
+    """Scalar multipart fields for `POST /stt`.
+
+    Cartesia defaults to English when `language` is absent rather than
+    detecting, so there is no way to ask for detection here.
+    """
+
+    model_config = _REQUEST
+
+    model: str
+    language: Optional[str] = None
+
+
+class CartesiaSTTResponse(BaseModel):
+    """Response from `POST /stt`."""
+
+    model_config = _RESPONSE
+
+    text: Optional[str] = None
+    duration: Optional[float] = None
+    language: Optional[str] = None
+
+    def transcript(self) -> Optional[str]:
+        """The spoken text, or `None` when the response carries no `text`.
+
+        `None` and `""` mean different things: `""` is silence, which is a
+        perfectly good answer, while `None` means the field was absent.
+        """
+        return self.text

deepeval/models/stt/deepgram/__init__.py

@@ -0,0 +1,3 @@
+from deepeval.models.stt.deepgram.deepgram import DeepgramSTTModel
+
+__all__ = ["DeepgramSTTModel"]

deepeval/models/stt/deepgram/deepgram.py

@@ -0,0 +1,151 @@
+from typing import Any, Dict, Optional, Tuple
+
+from pydantic import SecretStr
+
+from deepeval.config.settings import get_settings
+from deepeval.constants import ProviderSlug as PS
+from deepeval.models.base_model import DeepEvalBaseSTT
+from deepeval.models.retry_policy import create_retry_decorator
+from deepeval.models.speech import (
+    DEFAULT_TIMEOUT_SECONDS,
+    SpeechTransport,
+    dump_request,
+    parse_response,
+)
+from deepeval.models.stt._audio import audio_duration_seconds
+from deepeval.models.stt._stream import BufferedTranscribeMixin
+from deepeval.models.stt.deepgram.schema import (
+    DeepgramListenParams,
+    DeepgramListenResponse,
+)
+from deepeval.models.utils import parse_model_name, require_secret_api_key
+from deepeval.test_case import Audio
+
+retry_deepgram = create_retry_decorator(PS.DEEPGRAM)
+
+DEFAULT_BASE_URL = "https://api.deepgram.com"
+DEFAULT_STT_MODEL = "nova-3"
+
+_STT_PRICE_PER_MINUTE = {
+    "nova-3": 0.0043,
+    "nova-3-general": 0.0043,
+}
+
+
+class DeepgramSTTModel(BufferedTranscribeMixin, DeepEvalBaseSTT):
+    """Deepgram Nova speech-to-text."""
+
+    # Nova is not autoregressive, so it transcribes what it hears and stops
+    # rather than completing a word the barge-in cut off. No padding needed.
+    truncated_audio_pad_seconds: float = 0.0
+
+    def __init__(
+        self,
+        model: Optional[str] = None,
+        api_key: Optional[str] = None,
+        base_url: Optional[str] = None,
+        language: Optional[str] = None,
+        smart_format: bool = True,
+        cost_per_minute: Optional[float] = None,
+        transcription_kwargs: Optional[Dict] = None,
+        timeout: float = DEFAULT_TIMEOUT_SECONDS,
+    ):
+        settings = get_settings()
+        self.name = parse_model_name(model or DEFAULT_STT_MODEL)
+        self.language = language
+        self.smart_format = smart_format
+        self.transcription_kwargs = transcription_kwargs or {}
+        self.cost_per_minute = (
+            cost_per_minute
+            if cost_per_minute is not None
+            else _STT_PRICE_PER_MINUTE.get(self.name)
+        )
+        self.api_key = (
+            SecretStr(api_key)
+            if api_key is not None
+            else settings.DEEPGRAM_API_KEY
+        )
+        self.base_url = (
+            str(base_url).rstrip("/")
+            if base_url is not None
+            else DEFAULT_BASE_URL
+        )
+        self.timeout = timeout
+        self.model = self.load_model()
+
+    def load_model(self) -> SpeechTransport:
+        api_key = require_secret_api_key(
+            self.api_key,
+            provider_label="Deepgram",
+            env_var_name="DEEPGRAM_API_KEY",
+            param_hint="`api_key` to DeepgramSTTModel(...)",
+        )
+        return SpeechTransport(
+            provider_label="Deepgram",
+            base_url=self.base_url,
+            headers={"Authorization": f"Token {api_key}"},
+            timeout=self.timeout,
+        )
+
+    def _params(
+        self, language: Optional[str], kwargs: Dict[str, Any]
+    ) -> Dict[str, Any]:
+        lang = language or self.language
+        auto = lang == "auto"
+        return dump_request(
+            DeepgramListenParams(
+                model=self.name,
+                smart_format=self.smart_format,
+                # Deepgram detects only when asked to, so "auto" turns
+                # detection on instead of omitting the field.
+                language=None if auto else lang,
+                detect_language=True if auto else None,
+                **{**self.transcription_kwargs, **kwargs},
+            )
+        )
+
+    def _cost(self, audio: Audio) -> Optional[float]:
+        if self.cost_per_minute is None:
+            return None
+        seconds = audio_duration_seconds(audio)
+        if seconds is None:
+            return None
+        return seconds / 60.0 * self.cost_per_minute
+
+    def _parse(self, payload: Any) -> DeepgramListenResponse:
+        return parse_response(
+            DeepgramListenResponse, payload, provider_label="Deepgram"
+        )
+
+    @retry_deepgram
+    def transcribe(
+        self, audio: Audio, *, language: Optional[str] = None, **kwargs
+    ) -> Tuple[str, Optional[float]]:
+        payload = self._parse(
+            self.model.request_json(
+                "POST",
+                "/v1/listen",
+                params=self._params(language, kwargs),
+                content=audio.get_bytes(),
+                headers={"Content-Type": audio.mimeType or "audio/wav"},
+            )
+        )
+        return payload.transcript() or "", self._cost(audio)
+
+    @retry_deepgram
+    async def a_transcribe(
+        self, audio: Audio, *, language: Optional[str] = None, **kwargs
+    ) -> Tuple[str, Optional[float]]:
+        payload = self._parse(
+            await self.model.a_request_json(
+                "POST",
+                "/v1/listen",
+                params=self._params(language, kwargs),
+                content=audio.get_bytes(),
+                headers={"Content-Type": audio.mimeType or "audio/wav"},
+            )
+        )
+        return payload.transcript() or "", self._cost(audio)
+
+    def get_model_name(self) -> str:
+        return self.name

deepeval/models/stt/deepgram/schema.py

@@ -0,0 +1,73 @@
+"""Deepgram speech-to-text request and response shapes.
+
+Deepgram takes its transcription parameters in the query string rather than a
+body, so the request side is a params model.
+"""
+
+from typing import List, Optional
+
+from pydantic import BaseModel, ConfigDict
+
+# Requests allow extras so `transcription_kwargs` can set a provider parameter
+# deepeval has no field for. Responses allow them so a provider adding a field
+# never breaks a run. `protected_namespaces` is cleared because pydantic
+# reserves the `model_` prefix and Deepgram's parameter is called `model`.
+_REQUEST = ConfigDict(extra="allow", protected_namespaces=())
+_RESPONSE = ConfigDict(extra="allow", protected_namespaces=())
+
+
+class DeepgramListenParams(BaseModel):
+    """Query string for `POST /v1/listen`.
+
+    Deepgram detects the language only when asked to. An absent `language`
+    means English rather than detection, so `detect_language` has to be set
+    explicitly.
+    """
+
+    model_config = _REQUEST
+
+    model: str
+    smart_format: bool = True
+    language: Optional[str] = None
+    detect_language: Optional[bool] = None
+
+
+class DeepgramAlternative(BaseModel):
+    model_config = _RESPONSE
+
+    transcript: Optional[str] = None
+
+
+class DeepgramChannel(BaseModel):
+    model_config = _RESPONSE
+
+    alternatives: List[DeepgramAlternative] = []
+
+
+class DeepgramResults(BaseModel):
+    model_config = _RESPONSE
+
+    channels: List[DeepgramChannel] = []
+
+
+class DeepgramListenResponse(BaseModel):
+    """Response from `POST /v1/listen`."""
+
+    model_config = _RESPONSE
+
+    results: Optional[DeepgramResults] = None
+
+    def transcript(self) -> Optional[str]:
+        """The spoken text, or `None` when the response carries no transcript.
+
+        `None` and `""` mean different things: silence comes back as a present
+        but empty `transcript`, where `None` means the nesting this reads
+        through was not there at all.
+        """
+        if self.results is None:
+            return None
+        for channel in self.results.channels:
+            for alternative in channel.alternatives:
+                if alternative.transcript is not None:
+                    return alternative.transcript
+        return None

deepeval/models/stt/elevenlabs/__init__.py

@@ -0,0 +1,3 @@
+from deepeval.models.stt.elevenlabs.elevenlabs import ElevenLabsSTTModel
+
+__all__ = ["ElevenLabsSTTModel"]

deepeval/models/stt/elevenlabs/elevenlabs.py

@@ -0,0 +1,154 @@
+from typing import Any, Dict, Optional, Tuple
+
+from pydantic import SecretStr
+
+from deepeval.config.settings import get_settings
+from deepeval.constants import ProviderSlug as PS
+from deepeval.models.base_model import DeepEvalBaseSTT
+from deepeval.models.retry_policy import create_retry_decorator
+from deepeval.models.speech import (
+    DEFAULT_TIMEOUT_SECONDS,
+    Multipart,
+    SpeechTransport,
+    dump_request,
+    parse_response,
+)
+from deepeval.models.stt._audio import audio_duration_seconds
+from deepeval.models.stt._stream import BufferedTranscribeMixin
+from deepeval.models.stt.elevenlabs.schema import (
+    ElevenLabsSTTRequest,
+    ElevenLabsSTTResponse,
+)
+from deepeval.models.utils import parse_model_name, require_secret_api_key
+from deepeval.test_case import Audio
+
+retry_elevenlabs = create_retry_decorator(PS.ELEVENLABS)
+
+DEFAULT_BASE_URL = "https://api.elevenlabs.io"
+
+# `scribe_v1` is deprecated in favour of v2.
+DEFAULT_STT_MODEL = "scribe_v2"
+
+_STT_PRICE_PER_HOUR = {
+    "scribe_v2": 0.22,
+    "scribe_v1": 0.22,
+}
+
+
+class ElevenLabsSTTModel(BufferedTranscribeMixin, DeepEvalBaseSTT):
+    """ElevenLabs Scribe speech-to-text."""
+
+    # Scribe completes a word clipped mid-syllable, inventing speech the caller
+    # never heard, so interrupted audio gets a tail of silence to mark the end.
+    truncated_audio_pad_seconds: float = 0.3
+
+    def __init__(
+        self,
+        model: Optional[str] = None,
+        api_key: Optional[str] = None,
+        base_url: Optional[str] = None,
+        language: Optional[str] = None,
+        cost_per_hour: Optional[float] = None,
+        transcription_kwargs: Optional[Dict] = None,
+        timeout: float = DEFAULT_TIMEOUT_SECONDS,
+    ):
+        settings = get_settings()
+        self.name = parse_model_name(model or DEFAULT_STT_MODEL)
+        self.language = language
+        self.transcription_kwargs = transcription_kwargs or {}
+        self.cost_per_hour = (
+            cost_per_hour
+            if cost_per_hour is not None
+            else _STT_PRICE_PER_HOUR.get(self.name)
+        )
+        self.api_key = (
+            SecretStr(api_key)
+            if api_key is not None
+            else settings.ELEVENLABS_API_KEY
+        )
+        self.base_url = (
+            str(base_url).rstrip("/")
+            if base_url is not None
+            else DEFAULT_BASE_URL
+        )
+        self.timeout = timeout
+        self.model = self.load_model()
+
+    def load_model(self) -> SpeechTransport:
+        api_key = require_secret_api_key(
+            self.api_key,
+            provider_label="ElevenLabs",
+            env_var_name="ELEVENLABS_API_KEY",
+            param_hint="`api_key` to ElevenLabsSTTModel(...)",
+        )
+        return SpeechTransport(
+            provider_label="ElevenLabs",
+            base_url=self.base_url,
+            headers={"xi-api-key": api_key},
+            timeout=self.timeout,
+        )
+
+    def _multipart(
+        self, audio: Audio, language: Optional[str], kwargs: Dict[str, Any]
+    ) -> Multipart:
+        lang = language or self.language
+        request = ElevenLabsSTTRequest(
+            model_id=self.name,
+            # Scribe detects the language whenever `language_code` is absent,
+            # so "auto" means leaving it out.
+            language_code=None if lang == "auto" else lang,
+            **{**self.transcription_kwargs, **kwargs},
+        )
+        return Multipart(
+            file_field="file",
+            filename=audio.filename or "audio.wav",
+            content=audio.get_bytes(),
+            content_type=audio.mimeType or "audio/wav",
+            fields=dump_request(request),
+        )
+
+    def _cost(
+        self, audio: Audio, payload: ElevenLabsSTTResponse
+    ) -> Optional[float]:
+        if self.cost_per_hour is None:
+            return None
+        seconds = payload.audio_duration_secs
+        if seconds is None:
+            seconds = audio_duration_seconds(audio)
+        if seconds is None:
+            return None
+        return seconds / 3600.0 * self.cost_per_hour
+
+    def _parse(self, payload: Any) -> ElevenLabsSTTResponse:
+        return parse_response(
+            ElevenLabsSTTResponse, payload, provider_label="ElevenLabs"
+        )
+
+    @retry_elevenlabs
+    def transcribe(
+        self, audio: Audio, *, language: Optional[str] = None, **kwargs
+    ) -> Tuple[str, Optional[float]]:
+        payload = self._parse(
+            self.model.request_json(
+                "POST",
+                "/v1/speech-to-text",
+                multipart=self._multipart(audio, language, kwargs),
+            )
+        )
+        return payload.transcript() or "", self._cost(audio, payload)
+
+    @retry_elevenlabs
+    async def a_transcribe(
+        self, audio: Audio, *, language: Optional[str] = None, **kwargs
+    ) -> Tuple[str, Optional[float]]:
+        payload = self._parse(
+            await self.model.a_request_json(
+                "POST",
+                "/v1/speech-to-text",
+                multipart=self._multipart(audio, language, kwargs),
+            )
+        )
+        return payload.transcript() or "", self._cost(audio, payload)
+
+    def get_model_name(self) -> str:
+        return self.name

deepeval/models/stt/elevenlabs/schema.py

@@ -0,0 +1,64 @@
+"""ElevenLabs speech-to-text request and response shapes.
+
+`protected_namespaces=()` is cleared because ElevenLabs names its field
+`model_id` and pydantic reserves the `model_` prefix for its own methods, so
+without it every import warns about a conflict that does not matter here.
+"""
+
+from typing import List, Optional
+
+from pydantic import BaseModel, ConfigDict
+
+# Requests allow extras so `transcription_kwargs` can set a provider parameter
+# deepeval has no field for. Responses allow them so a provider adding a field
+# never breaks a run.
+_REQUEST = ConfigDict(extra="allow", protected_namespaces=())
+_RESPONSE = ConfigDict(extra="allow", protected_namespaces=())
+
+
+class ElevenLabsSTTRequest(BaseModel):
+    """Scalar multipart fields for `POST /v1/speech-to-text`.
+
+    Scribe detects the language whenever `language_code` is absent, so omitting
+    it is how automatic detection is requested.
+    """
+
+    model_config = _REQUEST
+
+    model_id: str
+    language_code: Optional[str] = None
+
+
+class ElevenLabsTranscriptChannel(BaseModel):
+    model_config = _RESPONSE
+
+    text: Optional[str] = None
+
+
+class ElevenLabsSTTResponse(BaseModel):
+    """Response from `POST /v1/speech-to-text`.
+
+    A single-channel request answers with `text`; a multi-channel one answers
+    with one entry per channel in `transcripts` and no top-level `text`.
+    """
+
+    model_config = _RESPONSE
+
+    text: Optional[str] = None
+    audio_duration_secs: Optional[float] = None
+    transcripts: Optional[List[ElevenLabsTranscriptChannel]] = None
+
+    def transcript(self) -> Optional[str]:
+        """The spoken text, or `None` when the response carries no transcript.
+
+        `None` and `""` mean different things: `""` is silence, which is a
+        perfectly good answer, while `None` means neither field was present.
+        """
+        if self.text is not None:
+            return self.text
+        if self.transcripts is not None:
+            parts = [
+                channel.text for channel in self.transcripts if channel.text
+            ]
+            return " ".join(parts).strip()
+        return None

deepeval/models/stt/openai/__init__.py

@@ -0,0 +1,3 @@
+from deepeval.models.stt.openai.openai import OpenAISTTModel
+
+__all__ = ["OpenAISTTModel"]

deepeval/models/stt/openai/openai.py

@@ -1,8 +1,4 @@
-import io
-import wave
 from typing import (
-    AsyncGenerator,
-    AsyncIterable,
     Optional,
     Dict,
     Tuple,
@@ -10,13 +6,14 @@ from typing import (
 from pydantic import SecretStr
 from openai import OpenAI, AsyncOpenAI
 
-from deepeval.test_case import Audio, AudioChunk
+from deepeval.test_case import Audio
 from deepeval.models.base_model import DeepEvalBaseSTT
+from deepeval.models.stt._audio import audio_duration_seconds
+from deepeval.models.stt._stream import BufferedTranscribeMixin
 from deepeval.models.utils import parse_model_name, require_secret_api_key
 from deepeval.models.retry_policy import create_retry_decorator
 from deepeval.constants import ProviderSlug as PS
 from deepeval.config.settings import get_settings
-from deepeval.voice.connectors import audio_utils
 
 retry_openai = create_retry_decorator(PS.OPENAI)
 
@@ -32,7 +29,7 @@ _STT_PRICE_PER_MINUTE = {
 }
 
 
-class OpenAISTTModel(DeepEvalBaseSTT):
+class OpenAISTTModel(BufferedTranscribeMixin, DeepEvalBaseSTT):
     # Whisper and the gpt-4o transcribe family both finish clipped words, so
     # they need a tail of silence to be told the utterance ended where it did.
     truncated_audio_pad_seconds: float = 0.3
@@ -115,20 +112,6 @@ class OpenAISTTModel(DeepEvalBaseSTT):
             merged["language"] = lang
         return merged
 
-    @staticmethod
-    def _audio_duration_seconds(audio: Audio) -> Optional[float]:
-        if audio.duration is not None:
-            return audio.duration
-        try:
-            with wave.open(io.BytesIO(audio.get_bytes()), "rb") as wf:
-                frames = wf.getnframes()
-                rate = wf.getframerate()
-                if rate:
-                    return frames / float(rate)
-        except (wave.Error, EOFError, ValueError):
-            pass
-        return None
-
     def _calculate_cost(self, audio: Audio, response) -> Optional[float]:
         usage = getattr(response, "usage", None)
         usage_type = getattr(usage, "type", None)
@@ -152,7 +135,7 @@ class OpenAISTTModel(DeepEvalBaseSTT):
                 else None
             )
             if seconds is None:
-                seconds = self._audio_duration_seconds(audio)
+                seconds = audio_duration_seconds(audio)
             if seconds is not None:
                 return seconds / 60.0 * self.cost_per_minute
 
@@ -180,52 +163,5 @@ class OpenAISTTModel(DeepEvalBaseSTT):
         )
         return response.text, self._calculate_cost(audio, response)
 
-    async def a_transcribe_stream(
-        self,
-        audio_stream: AsyncIterable[AudioChunk],
-        *args,
-        language: Optional[str] = None,
-        partial_every_seconds: float = 1.0,
-        **kwargs,
-    ) -> AsyncGenerator[str, None]:
-        """Yield progressively updated transcripts from a live PCM stream.
-
-        OpenAI's batch transcription API has no true partial streaming, so
-        this buffers chunks and re-transcribes the accumulated WAV whenever
-        roughly `partial_every_seconds` of new audio arrives. Used as a
-        duplex barge-in fallback when the transport has no text partials
-        (e.g. LiveKit).
-        """
-        pcm = bytearray()
-        sample_rate = 24000
-        bytes_since_partial = 0
-        last_text = ""
-        async for chunk in audio_stream:
-            if chunk.sampleRate:
-                sample_rate = chunk.sampleRate
-            frame = chunk.get_bytes()
-            pcm.extend(frame)
-            bytes_since_partial += len(frame)
-            threshold = int(sample_rate * 2 * partial_every_seconds)
-            if bytes_since_partial >= threshold or chunk.final:
-                audio = Audio.from_bytes(
-                    audio_utils.pcm16_to_wav_bytes(bytes(pcm), sample_rate, 1),
-                    "audio/wav",
-                    sampleRate=sample_rate,
-                    encoding="wav",
-                )
-                text, _ = await self.a_transcribe(
-                    audio, language=language, **kwargs
-                )
-                if text and text != last_text:
-                    last_text = text
-                    yield text
-                bytes_since_partial = 0
-                if chunk.final:
-                    break
-
-    def supports_streaming(self) -> bool:
-        return True
-
     def get_model_name(self) -> str:
         return self.name

deepeval/models/stt_models/__init__.py

@@ -1,3 +0,0 @@
-from deepeval.models.stt_models.openai_stt import OpenAISTTModel
-
-__all__ = ["OpenAISTTModel"]

deepeval/models/tts/__init__.py

@@ -0,0 +1,11 @@
+from deepeval.models.tts.cartesia import CartesiaTTSModel
+from deepeval.models.tts.deepgram import DeepgramTTSModel
+from deepeval.models.tts.elevenlabs import ElevenLabsTTSModel
+from deepeval.models.tts.openai import OpenAITTSModel
+
+__all__ = [
+    "CartesiaTTSModel",
+    "DeepgramTTSModel",
+    "ElevenLabsTTSModel",
+    "OpenAITTSModel",
+]

deepeval/models/tts/_frames.py

@@ -0,0 +1,72 @@
+"""Turning a stream of PCM bytes into `AudioChunk`s."""
+
+from __future__ import annotations
+
+from typing import AsyncGenerator, AsyncIterable, Optional
+
+from deepeval.test_case import AudioChunk
+
+# A frame has to be playable on its own, which rules out every container format
+# (their headers declare a length the first frame does not know yet), so a
+# stream is always raw 16-bit mono PCM.
+STREAM_MIME = "audio/pcm"
+STREAM_ENCODING = "pcm"
+DEFAULT_FRAME_SECONDS = 0.1
+
+
+def pcm_chunk(
+    data: bytes, *, sample_rate: int, final: bool = False
+) -> AudioChunk:
+    return AudioChunk.from_bytes(
+        data,
+        STREAM_MIME,
+        sampleRate=sample_rate,
+        encoding=STREAM_ENCODING,
+        duration=len(data) / 2 / sample_rate,
+        final=final,
+    )
+
+
+def frame_size_bytes(
+    sample_rate: int, frame_seconds: float = DEFAULT_FRAME_SECONDS
+) -> int:
+    """Bytes in one frame, always even so a frame never splits a sample."""
+    samples = max(1, int(sample_rate * frame_seconds))
+    return samples * 2
+
+
+async def frame_pcm_stream(
+    source: AsyncIterable[bytes],
+    *,
+    sample_rate: int,
+    frame_seconds: float = DEFAULT_FRAME_SECONDS,
+) -> AsyncGenerator[AudioChunk, None]:
+    """Re-cut arbitrary byte reads from an HTTP body into fixed audio frames.
+
+    A provider's chunked response boundaries follow its network writes, not the
+    audio, so reads arrive at unhelpful sizes and can split a sample in half.
+
+    The last frame is held back so it can be flagged `final`, which is how a
+    consumer knows the utterance is complete without also tracking the stream.
+    """
+    frame_bytes = frame_size_bytes(sample_rate, frame_seconds)
+    buffer = bytearray()
+    held: Optional[bytes] = None
+
+    async for data in source:
+        if not data:
+            continue
+        buffer.extend(data)
+        while len(buffer) >= frame_bytes:
+            frame = bytes(buffer[:frame_bytes])
+            del buffer[:frame_bytes]
+            if held is not None:
+                yield pcm_chunk(held, sample_rate=sample_rate)
+            held = frame
+
+    if buffer:
+        if held is not None:
+            yield pcm_chunk(held, sample_rate=sample_rate)
+        held = bytes(buffer)
+    if held is not None:
+        yield pcm_chunk(held, sample_rate=sample_rate, final=True)

deepeval/models/tts/cartesia/__init__.py

@@ -0,0 +1,3 @@
+from deepeval.models.tts.cartesia.cartesia import CartesiaTTSModel
+
+__all__ = ["CartesiaTTSModel"]

deepeval/models/tts/cartesia/cartesia.py

@@ -0,0 +1,195 @@
+from typing import Any, AsyncGenerator, Dict, Optional, Tuple
+
+from pydantic import SecretStr
+
+from deepeval.config.settings import get_settings
+from deepeval.constants import ProviderSlug as PS
+from deepeval.errors import DeepEvalError
+from deepeval.models.base_model import DeepEvalBaseTTS
+from deepeval.models.retry_policy import create_retry_decorator
+from deepeval.models.speech import (
+    DEFAULT_TIMEOUT_SECONDS,
+    SpeechTransport,
+    dump_request,
+)
+from deepeval.models.tts._frames import frame_pcm_stream
+from deepeval.models.tts.cartesia.schema import (
+    CartesiaOutputFormat,
+    CartesiaTTSRequest,
+)
+from deepeval.models.utils import parse_model_name, require_secret_api_key
+from deepeval.test_case import Audio, AudioChunk
+
+retry_cartesia = create_retry_decorator(PS.CARTESIA)
+
+DEFAULT_BASE_URL = "https://api.cartesia.ai"
+
+# Cartesia dates its API and requires the version on every request. Pinning it
+# here rather than tracking "latest" means a server-side release cannot change
+# the response shape underneath us.
+DEFAULT_API_VERSION = "2026-08-14"
+
+# `sonic-2` and `sonic-turbo` are gone from the current model enum and now fail
+# with model_not_found, so neither is a safe default.
+DEFAULT_TTS_MODEL = "sonic-3.6"
+
+DEFAULT_SAMPLE_RATE = 24000
+_SAMPLE_RATES = frozenset({8000, 16000, 22050, 24000, 44100, 48000})
+
+
+class CartesiaTTSModel(DeepEvalBaseTTS):
+    """Cartesia Sonic text-to-speech.
+
+    Cartesia has no default voice, so `voice` is required: it is a voice ID from
+    your Cartesia account rather than a name from a fixed cast.
+    """
+
+    def __init__(
+        self,
+        model: Optional[str] = None,
+        api_key: Optional[str] = None,
+        base_url: Optional[str] = None,
+        voice: Optional[str] = None,
+        sample_rate: int = DEFAULT_SAMPLE_RATE,
+        language: Optional[str] = None,
+        api_version: str = DEFAULT_API_VERSION,
+        cost_per_1m_chars: Optional[float] = None,
+        generation_kwargs: Optional[Dict] = None,
+        timeout: float = DEFAULT_TIMEOUT_SECONDS,
+    ):
+        settings = get_settings()
+        self.name = parse_model_name(model or DEFAULT_TTS_MODEL)
+        self.voice = voice
+        self.language = language
+        self.api_version = api_version
+        self.generation_kwargs = generation_kwargs or {}
+        # Cartesia meters in credits and publishes no credit-to-dollar rate, so
+        # there is no price table to default to. Pass `cost_per_1m_chars` to
+        # have synthesis costs accounted for.
+        self.cost_per_1m_chars = cost_per_1m_chars
+
+        if sample_rate not in _SAMPLE_RATES:
+            raise ValueError(
+                f"Cartesia cannot synthesize at {sample_rate} Hz. "
+                f"Supported rates: {sorted(_SAMPLE_RATES)}."
+            )
+        # Instance attribute, not the class default: connectors read
+        # `tts.sample_rate` off the instance to configure uplink resampling.
+        self.sample_rate = sample_rate
+
+        self.api_key = (
+            SecretStr(api_key)
+            if api_key is not None
+            else settings.CARTESIA_API_KEY
+        )
+        self.base_url = (
+            str(base_url).rstrip("/")
+            if base_url is not None
+            else DEFAULT_BASE_URL
+        )
+        self.timeout = timeout
+        self.model = self.load_model()
+
+    def load_model(self) -> SpeechTransport:
+        api_key = require_secret_api_key(
+            self.api_key,
+            provider_label="Cartesia",
+            env_var_name="CARTESIA_API_KEY",
+            param_hint="`api_key` to CartesiaTTSModel(...)",
+        )
+        return SpeechTransport(
+            provider_label="Cartesia",
+            base_url=self.base_url,
+            headers={
+                "X-Api-Key": api_key,
+                "Cartesia-Version": self.api_version,
+            },
+            timeout=self.timeout,
+        )
+
+    def _resolve_voice(self, voice: Optional[str]) -> str:
+        resolved = voice or self.voice
+        if not resolved:
+            raise DeepEvalError(
+                "Cartesia has no default voice. Pass `voice` to "
+                "CartesiaTTSModel(...) with a voice ID from your Cartesia "
+                "account, or set it on the golden's `Persona`."
+            )
+        return resolved
+
+    def _body(
+        self,
+        text: str,
+        voice: Optional[str],
+        *,
+        container: str,
+        extra: Dict[str, Any],
+    ) -> Dict[str, Any]:
+        return dump_request(
+            CartesiaTTSRequest(
+                model_id=self.name,
+                transcript=text,
+                voice=self._resolve_voice(voice),
+                output_format=CartesiaOutputFormat(
+                    container=container, sample_rate=self.sample_rate
+                ),
+                language=self.language,
+                **{**self.generation_kwargs, **extra},
+            )
+        )
+
+    def synthesis_cost(self, text: str) -> Optional[float]:
+        if self.cost_per_1m_chars is None:
+            return None
+        return len(text) / 1e6 * self.cost_per_1m_chars
+
+    @retry_cartesia
+    def synthesize(
+        self, text: str, *, voice: Optional[str] = None, **kwargs
+    ) -> Tuple[Audio, Optional[float]]:
+        data = self.model.request_bytes(
+            "POST",
+            "/tts/bytes",
+            json=self._body(text, voice, container="wav", extra=kwargs),
+        )
+        return self._to_audio(data), self.synthesis_cost(text)
+
+    @retry_cartesia
+    async def a_synthesize(
+        self, text: str, *, voice: Optional[str] = None, **kwargs
+    ) -> Tuple[Audio, Optional[float]]:
+        data = await self.model.a_request_bytes(
+            "POST",
+            "/tts/bytes",
+            json=self._body(text, voice, container="wav", extra=kwargs),
+        )
+        return self._to_audio(data), self.synthesis_cost(text)
+
+    def _to_audio(self, data: bytes) -> Audio:
+        return Audio.from_bytes(
+            data,
+            "audio/wav",
+            sampleRate=self.sample_rate,
+            encoding="wav",
+        )
+
+    async def a_synthesize_stream(
+        self, text: str, *args, voice: Optional[str] = None, **kwargs
+    ) -> AsyncGenerator[AudioChunk, None]:
+        source = self.model.a_stream_bytes(
+            "POST",
+            "/tts/bytes",
+            # A raw container is headerless PCM, which is what makes each frame
+            # playable the moment it arrives.
+            json=self._body(text, voice, container="raw", extra=kwargs),
+        )
+        async for chunk in frame_pcm_stream(
+            source, sample_rate=self.sample_rate
+        ):
+            yield chunk
+
+    def supports_streaming(self) -> bool:
+        return True
+
+    def get_model_name(self) -> str:
+        return self.name

deepeval/models/tts/cartesia/schema.py

@@ -0,0 +1,45 @@
+"""Cartesia text-to-speech request shapes.
+
+Cartesia is the only provider here that nests its audio settings, in an
+`output_format` object rather than flat fields, so that nesting gets a model of
+its own.
+"""
+
+from typing import Optional
+
+from pydantic import BaseModel, ConfigDict
+
+# Requests allow extras so `generation_kwargs` can set a provider parameter
+# deepeval has no field for. `protected_namespaces` is cleared because pydantic
+# reserves the `model_` prefix and Cartesia's field is called `model_id`.
+_REQUEST = ConfigDict(extra="allow", protected_namespaces=())
+
+
+class CartesiaOutputFormat(BaseModel):
+    """The `output_format` object on a TTS request.
+
+    `container` is `wav` for a batch call and `raw` for a streamed one: a WAV
+    header declares a length the first frame of a stream does not know yet.
+    """
+
+    model_config = _REQUEST
+
+    container: str
+    sample_rate: int
+    encoding: str = "pcm_s16le"
+
+
+class CartesiaTTSRequest(BaseModel):
+    """Body for `POST /tts/bytes`.
+
+    `voice` is a voice ID from your Cartesia account. There is no default, which
+    is why the model requires one rather than falling back.
+    """
+
+    model_config = _REQUEST
+
+    model_id: str
+    transcript: str
+    voice: str
+    output_format: CartesiaOutputFormat
+    language: Optional[str] = None

deepeval/models/tts/deepgram/__init__.py

@@ -0,0 +1,3 @@
+from deepeval.models.tts.deepgram.deepgram import DeepgramTTSModel
+
+__all__ = ["DeepgramTTSModel"]

deepeval/models/tts/deepgram/deepgram.py

@@ -0,0 +1,202 @@
+from typing import Any, AsyncGenerator, Dict, Optional, Tuple
+
+from pydantic import SecretStr
+
+from deepeval.config.settings import get_settings
+from deepeval.constants import ProviderSlug as PS
+from deepeval.models.base_model import DeepEvalBaseTTS
+from deepeval.models.retry_policy import create_retry_decorator
+from deepeval.models.speech import (
+    DEFAULT_TIMEOUT_SECONDS,
+    SpeechTransport,
+    dump_request,
+)
+from deepeval.models.tts._frames import frame_pcm_stream
+from deepeval.models.tts.deepgram.schema import (
+    DeepgramSpeakParams,
+    DeepgramSpeakRequest,
+)
+from deepeval.models.utils import parse_model_name, require_secret_api_key
+from deepeval.test_case import Audio, AudioChunk
+
+retry_deepgram = create_retry_decorator(PS.DEEPGRAM)
+
+DEFAULT_BASE_URL = "https://api.deepgram.com"
+DEFAULT_TTS_MODEL = "aura-2-thalia-en"
+DEFAULT_SAMPLE_RATE = 24000
+
+# Rates Deepgram accepts for linear16.
+_PCM_SAMPLE_RATES = frozenset({8000, 16000, 24000, 32000, 48000})
+
+_AURA_2_PRICE_PER_1M_CHARS = 30.0
+_AURA_1_PRICE_PER_1M_CHARS = 15.0
+
+
+class DeepgramTTSModel(DeepEvalBaseTTS):
+    """Deepgram Aura text-to-speech.
+
+    Aura model names carry the voice and the language in the identifier
+    (`aura-2-thalia-en`), so the `voice` argument rewrites that middle segment
+    rather than being sent as a separate field.
+    """
+
+    def __init__(
+        self,
+        model: Optional[str] = None,
+        api_key: Optional[str] = None,
+        base_url: Optional[str] = None,
+        voice: Optional[str] = None,
+        sample_rate: int = DEFAULT_SAMPLE_RATE,
+        cost_per_1m_chars: Optional[float] = None,
+        generation_kwargs: Optional[Dict] = None,
+        timeout: float = DEFAULT_TIMEOUT_SECONDS,
+    ):
+        settings = get_settings()
+        self.name = parse_model_name(model or DEFAULT_TTS_MODEL)
+        # A voice given here is folded into the model name up front, so
+        # `get_model_name()` reports what will actually be synthesized.
+        if voice:
+            self.name = apply_voice(self.name, voice)
+        self.generation_kwargs = generation_kwargs or {}
+        self.cost_per_1m_chars = (
+            cost_per_1m_chars
+            if cost_per_1m_chars is not None
+            else _default_price(self.name)
+        )
+
+        if sample_rate not in _PCM_SAMPLE_RATES:
+            raise ValueError(
+                f"Deepgram cannot synthesize linear16 at {sample_rate} Hz. "
+                f"Supported rates: {sorted(_PCM_SAMPLE_RATES)}."
+            )
+        # Instance attribute, not the class default: connectors read
+        # `tts.sample_rate` off the instance to configure uplink resampling.
+        self.sample_rate = sample_rate
+
+        self.api_key = (
+            SecretStr(api_key)
+            if api_key is not None
+            else settings.DEEPGRAM_API_KEY
+        )
+        self.base_url = (
+            str(base_url).rstrip("/")
+            if base_url is not None
+            else DEFAULT_BASE_URL
+        )
+        self.timeout = timeout
+        self.model = self.load_model()
+
+    def load_model(self) -> SpeechTransport:
+        api_key = require_secret_api_key(
+            self.api_key,
+            provider_label="Deepgram",
+            env_var_name="DEEPGRAM_API_KEY",
+            param_hint="`api_key` to DeepgramTTSModel(...)",
+        )
+        return SpeechTransport(
+            provider_label="Deepgram",
+            base_url=self.base_url,
+            headers={"Authorization": f"Token {api_key}"},
+            timeout=self.timeout,
+        )
+
+    def _params(
+        self, voice: Optional[str], *, container: str, extra: Dict[str, Any]
+    ) -> Dict[str, Any]:
+        return dump_request(
+            DeepgramSpeakParams(
+                model=apply_voice(self.name, voice) if voice else self.name,
+                container=container,
+                sample_rate=self.sample_rate,
+                **{**self.generation_kwargs, **extra},
+            )
+        )
+
+    def _body(self, text: str) -> Dict[str, Any]:
+        return dump_request(DeepgramSpeakRequest(text=text))
+
+    def synthesis_cost(self, text: str) -> Optional[float]:
+        if self.cost_per_1m_chars is None:
+            return None
+        return len(text) / 1e6 * self.cost_per_1m_chars
+
+    @retry_deepgram
+    def synthesize(
+        self, text: str, *, voice: Optional[str] = None, **kwargs
+    ) -> Tuple[Audio, Optional[float]]:
+        data = self.model.request_bytes(
+            "POST",
+            "/v1/speak",
+            params=self._params(voice, container="wav", extra=kwargs),
+            json=self._body(text),
+        )
+        return self._to_audio(data), self.synthesis_cost(text)
+
+    @retry_deepgram
+    async def a_synthesize(
+        self, text: str, *, voice: Optional[str] = None, **kwargs
+    ) -> Tuple[Audio, Optional[float]]:
+        data = await self.model.a_request_bytes(
+            "POST",
+            "/v1/speak",
+            params=self._params(voice, container="wav", extra=kwargs),
+            json=self._body(text),
+        )
+        return self._to_audio(data), self.synthesis_cost(text)
+
+    def _to_audio(self, data: bytes) -> Audio:
+        return Audio.from_bytes(
+            data,
+            "audio/wav",
+            sampleRate=self.sample_rate,
+            encoding="wav",
+        )
+
+    async def a_synthesize_stream(
+        self, text: str, *args, voice: Optional[str] = None, **kwargs
+    ) -> AsyncGenerator[AudioChunk, None]:
+        source = self.model.a_stream_bytes(
+            "POST",
+            "/v1/speak",
+            # `container=none` is what makes this headerless PCM; a WAV header
+            # would arrive declaring a length the first frame cannot know.
+            params=self._params(voice, container="none", extra=kwargs),
+            json=self._body(text),
+        )
+        async for chunk in frame_pcm_stream(
+            source, sample_rate=self.sample_rate
+        ):
+            yield chunk
+
+    def supports_streaming(self) -> bool:
+        return True
+
+    def get_model_name(self) -> str:
+        return self.name
+
+
+def apply_voice(model_name: str, voice: str) -> str:
+    """Swap the voice segment of an Aura model name.
+
+    Accepts either a bare voice (`zeus`) or a full model name
+    (`aura-2-zeus-en`), since a `Persona.voice` written for another provider is
+    just a name while one written for Deepgram may well be the whole thing.
+    """
+    if voice.startswith("aura"):
+        return voice
+
+    parts = model_name.split("-")
+    # aura-2-thalia-en -> [aura, 2, thalia, en]; aura-asteria-en -> [aura, asteria, en]
+    if len(parts) >= 4:
+        return "-".join([parts[0], parts[1], voice, *parts[3:]])
+    if len(parts) == 3:
+        return "-".join([parts[0], voice, parts[2]])
+    return voice
+
+
+def _default_price(model_name: str) -> Optional[float]:
+    if model_name.startswith("aura-2"):
+        return _AURA_2_PRICE_PER_1M_CHARS
+    if model_name.startswith("aura"):
+        return _AURA_1_PRICE_PER_1M_CHARS
+    return None

deepeval/models/tts/deepgram/schema.py

@@ -0,0 +1,36 @@
+"""Deepgram text-to-speech request shapes.
+
+Deepgram is the odd one here: the text goes in the body and every other
+parameter goes in the query string, which is the reverse of the other providers.
+Hence a params model alongside the request body model.
+"""
+
+from pydantic import BaseModel, ConfigDict
+
+# Requests allow extras so `generation_kwargs` can set a provider parameter
+# deepeval has no field for. `protected_namespaces` is cleared because pydantic
+# reserves the `model_` prefix and Deepgram's parameter is called `model`.
+_REQUEST = ConfigDict(extra="allow", protected_namespaces=())
+
+
+class DeepgramSpeakParams(BaseModel):
+    """Query string for `POST /v1/speak`.
+
+    `container` is `wav` for a batch call and `none` for a streamed one: a WAV
+    header declares a length the first frame of a stream does not know yet.
+    """
+
+    model_config = _REQUEST
+
+    model: str
+    container: str
+    sample_rate: int
+    encoding: str = "linear16"
+
+
+class DeepgramSpeakRequest(BaseModel):
+    """Body for `POST /v1/speak`."""
+
+    model_config = _REQUEST
+
+    text: str

deepeval/models/tts/elevenlabs/__init__.py

@@ -0,0 +1,3 @@
+from deepeval.models.tts.elevenlabs.elevenlabs import ElevenLabsTTSModel
+
+__all__ = ["ElevenLabsTTSModel"]

deepeval/models/tts/elevenlabs/elevenlabs.py

@@ -0,0 +1,200 @@
+from typing import Any, AsyncGenerator, Dict, Optional, Tuple
+
+from pydantic import SecretStr
+
+from deepeval.config.settings import get_settings
+from deepeval.constants import ProviderSlug as PS
+from deepeval.models.base_model import DeepEvalBaseTTS
+from deepeval.models.retry_policy import create_retry_decorator
+from deepeval.models.speech import (
+    DEFAULT_TIMEOUT_SECONDS,
+    SpeechTransport,
+    dump_request,
+)
+from deepeval.models.tts._frames import frame_pcm_stream
+from deepeval.models.tts.elevenlabs.schema import (
+    ElevenLabsTTSParams,
+    ElevenLabsTTSRequest,
+)
+from deepeval.models.utils import parse_model_name, require_secret_api_key
+from deepeval.test_case import Audio, AudioChunk
+
+retry_elevenlabs = create_retry_decorator(PS.ELEVENLABS)
+
+DEFAULT_BASE_URL = "https://api.elevenlabs.io"
+
+# Flash is the model built for realtime agents (~75 ms). The Turbo family is
+# deprecated in favour of it, and the expressive v3 models cost twice as much
+# and are slower, which is the wrong trade for a simulated caller.
+DEFAULT_TTS_MODEL = "eleven_flash_v2_5"
+
+# Deliberately not one of the legacy "Default" voices (Rachel, Adam, Bella...):
+# those are unavailable to accounts created after March 2026 and stop resolving
+# entirely on 2027-01-01. This is George, from the Voice Library.
+DEFAULT_VOICE = "JBFqnCBsd6RMkjVDRZzb"
+
+DEFAULT_SAMPLE_RATE = 24000
+
+# Rates ElevenLabs will emit as PCM. Anything else has to be requested in a
+# container format, which cannot be streamed frame by frame.
+_PCM_SAMPLE_RATES = frozenset({8000, 16000, 22050, 24000, 32000, 44100, 48000})
+
+_TTS_PRICE_PER_1M_CHARS = {
+    "eleven_flash_v2_5": 50.0,
+    "eleven_flash_v2": 50.0,
+    "eleven_v3": 100.0,
+    "eleven_v3_conversational": 100.0,
+    "eleven_multilingual_v2": 100.0,
+    "eleven_turbo_v2_5": 50.0,
+    "eleven_turbo_v2": 50.0,
+}
+
+
+class ElevenLabsTTSModel(DeepEvalBaseTTS):
+    """ElevenLabs text-to-speech.
+
+    Note this is unrelated to `ElevenLabsConnector`, which bridges to a voice
+    agent hosted on ElevenLabs. This is the model that speaks the simulated
+    user's side of the conversation, and it can be pointed at an agent on any
+    platform.
+    """
+
+    def __init__(
+        self,
+        model: Optional[str] = None,
+        api_key: Optional[str] = None,
+        base_url: Optional[str] = None,
+        voice: Optional[str] = None,
+        sample_rate: int = DEFAULT_SAMPLE_RATE,
+        language_code: Optional[str] = None,
+        voice_settings: Optional[Dict[str, Any]] = None,
+        cost_per_1m_chars: Optional[float] = None,
+        generation_kwargs: Optional[Dict] = None,
+        timeout: float = DEFAULT_TIMEOUT_SECONDS,
+    ):
+        settings = get_settings()
+        self.name = parse_model_name(model or DEFAULT_TTS_MODEL)
+        self.voice = voice or DEFAULT_VOICE
+        self.language_code = language_code
+        self.voice_settings = voice_settings
+        self.generation_kwargs = generation_kwargs or {}
+        self.cost_per_1m_chars = (
+            cost_per_1m_chars
+            if cost_per_1m_chars is not None
+            else _TTS_PRICE_PER_1M_CHARS.get(self.name)
+        )
+
+        if sample_rate not in _PCM_SAMPLE_RATES:
+            raise ValueError(
+                f"ElevenLabs cannot synthesize at {sample_rate} Hz. "
+                f"Supported rates: {sorted(_PCM_SAMPLE_RATES)}."
+            )
+        # Instance attribute, not the class default: connectors read
+        # `tts.sample_rate` off the instance to configure uplink resampling.
+        self.sample_rate = sample_rate
+
+        self.api_key = (
+            SecretStr(api_key)
+            if api_key is not None
+            else settings.ELEVENLABS_API_KEY
+        )
+        self.base_url = (
+            str(base_url).rstrip("/")
+            if base_url is not None
+            else DEFAULT_BASE_URL
+        )
+        self.timeout = timeout
+        self.model = self.load_model()
+
+    def load_model(self) -> SpeechTransport:
+        api_key = require_secret_api_key(
+            self.api_key,
+            provider_label="ElevenLabs",
+            env_var_name="ELEVENLABS_API_KEY",
+            param_hint="`api_key` to ElevenLabsTTSModel(...)",
+        )
+        return SpeechTransport(
+            provider_label="ElevenLabs",
+            base_url=self.base_url,
+            headers={"xi-api-key": api_key},
+            timeout=self.timeout,
+        )
+
+    def _body(self, text: str, extra: Dict[str, Any]) -> Dict[str, Any]:
+        return dump_request(
+            ElevenLabsTTSRequest(
+                text=text,
+                model_id=self.name,
+                language_code=self.language_code,
+                voice_settings=self.voice_settings,
+                **{**self.generation_kwargs, **extra},
+            )
+        )
+
+    def _params(self, *, stream: bool) -> Dict[str, Any]:
+        prefix = "pcm" if stream else "wav"
+        return dump_request(
+            ElevenLabsTTSParams(output_format=f"{prefix}_{self.sample_rate}")
+        )
+
+    def _path(self, voice: Optional[str], *, stream: bool) -> str:
+        voice_id = voice or self.voice
+        suffix = "/stream" if stream else ""
+        return f"/v1/text-to-speech/{voice_id}{suffix}"
+
+    def synthesis_cost(self, text: str) -> Optional[float]:
+        if self.cost_per_1m_chars is None:
+            return None
+        return len(text) / 1e6 * self.cost_per_1m_chars
+
+    @retry_elevenlabs
+    def synthesize(
+        self, text: str, *, voice: Optional[str] = None, **kwargs
+    ) -> Tuple[Audio, Optional[float]]:
+        data = self.model.request_bytes(
+            "POST",
+            self._path(voice, stream=False),
+            params=self._params(stream=False),
+            json=self._body(text, kwargs),
+        )
+        return self._to_audio(data), self.synthesis_cost(text)
+
+    @retry_elevenlabs
+    async def a_synthesize(
+        self, text: str, *, voice: Optional[str] = None, **kwargs
+    ) -> Tuple[Audio, Optional[float]]:
+        data = await self.model.a_request_bytes(
+            "POST",
+            self._path(voice, stream=False),
+            params=self._params(stream=False),
+            json=self._body(text, kwargs),
+        )
+        return self._to_audio(data), self.synthesis_cost(text)
+
+    def _to_audio(self, data: bytes) -> Audio:
+        return Audio.from_bytes(
+            data,
+            "audio/wav",
+            sampleRate=self.sample_rate,
+            encoding="wav",
+        )
+
+    async def a_synthesize_stream(
+        self, text: str, *args, voice: Optional[str] = None, **kwargs
+    ) -> AsyncGenerator[AudioChunk, None]:
+        source = self.model.a_stream_bytes(
+            "POST",
+            self._path(voice, stream=True),
+            params=self._params(stream=True),
+            json=self._body(text, kwargs),
+        )
+        async for chunk in frame_pcm_stream(
+            source, sample_rate=self.sample_rate
+        ):
+            yield chunk
+
+    def supports_streaming(self) -> bool:
+        return True
+
+    def get_model_name(self) -> str:
+        return self.name

deepeval/models/tts/elevenlabs/schema.py

@@ -0,0 +1,38 @@
+"""ElevenLabs text-to-speech request shapes.
+
+`protected_namespaces=()` is cleared because ElevenLabs names its field
+`model_id` and pydantic reserves the `model_` prefix for its own methods, so
+without it every import warns about a conflict that does not matter here.
+"""
+
+from typing import Any, Dict, Optional
+
+from pydantic import BaseModel, ConfigDict
+
+# Requests allow extras so `generation_kwargs` can set a provider parameter
+# deepeval has no field for.
+_REQUEST = ConfigDict(extra="allow", protected_namespaces=())
+
+
+class ElevenLabsTTSParams(BaseModel):
+    """Query string for `POST /v1/text-to-speech/{voice_id}`.
+
+    `output_format` is `wav_{rate}` for a batch call and `pcm_{rate}` for a
+    streamed one: a WAV header declares a length the first frame of a stream
+    does not know yet.
+    """
+
+    model_config = _REQUEST
+
+    output_format: str
+
+
+class ElevenLabsTTSRequest(BaseModel):
+    """Body for `POST /v1/text-to-speech/{voice_id}`."""
+
+    model_config = _REQUEST
+
+    text: str
+    model_id: str
+    language_code: Optional[str] = None
+    voice_settings: Optional[Dict[str, Any]] = None

deepeval/models/tts/openai/__init__.py

@@ -0,0 +1,3 @@
+from deepeval.models.tts.openai.openai import OpenAITTSModel
+
+__all__ = ["OpenAITTSModel"]

deepeval/models/tts/openai/openai.py

@@ -4,6 +4,7 @@ from openai import OpenAI, AsyncOpenAI
 
 from deepeval.test_case import Audio, AudioChunk
 from deepeval.models.base_model import DeepEvalBaseTTS
+from deepeval.models.tts._frames import frame_pcm_stream, frame_size_bytes
 from deepeval.models.utils import parse_model_name, require_secret_api_key
 from deepeval.models.retry_policy import create_retry_decorator
 from deepeval.constants import ProviderSlug as PS
@@ -27,8 +28,7 @@ _FORMAT_MIME = {
 # format above: their headers declare a length the first chunk does not have
 # yet. Streaming is therefore always raw 16-bit mono PCM at 24 kHz.
 _STREAM_FORMAT = "pcm"
-_STREAM_FRAME_SECONDS = 0.1
-_STREAM_FRAME_BYTES = int(_OPENAI_TTS_SAMPLE_RATE * 2 * _STREAM_FRAME_SECONDS)
+_STREAM_FRAME_BYTES = frame_size_bytes(_OPENAI_TTS_SAMPLE_RATE)
 _TTS_PRICE_PER_1M_CHARS = {
     "tts-1": 15.0,
     "tts-1-hd": 30.0,
@@ -138,16 +138,6 @@ class OpenAITTSModel(DeepEvalBaseTTS):
         )
         return self._to_audio(response.content, fmt), self.synthesis_cost(text)
 
-    def _to_chunk(self, data: bytes, *, final: bool) -> AudioChunk:
-        return AudioChunk.from_bytes(
-            data,
-            _FORMAT_MIME[_STREAM_FORMAT],
-            sampleRate=_OPENAI_TTS_SAMPLE_RATE,
-            encoding=_STREAM_FORMAT,
-            duration=len(data) / 2 / _OPENAI_TTS_SAMPLE_RATE,
-            final=final,
-        )
-
     async def a_synthesize_stream(
         self,
         text: str,
@@ -161,34 +151,25 @@ class OpenAITTSModel(DeepEvalBaseTTS):
         entire synthesis time in front of the first word. Streaming lets a
         caller start speaking after the first frame, so what is left to
         synthesize is spoken over.
-
-        The last frame is held back so it can be flagged `final`, which is how
-        a consumer knows the utterance is complete without also tracking the
-        stream itself.
         """
-        buffer = bytearray()
-        held: Optional[bytes] = None
-        async with self._async().audio.speech.with_streaming_response.create(
-            model=self.name,
-            voice=voice or self.voice,
-            input=text,
-            response_format=_STREAM_FORMAT,
-            **{**self.generation_kwargs, **kwargs},
-        ) as response:
-            async for data in response.iter_bytes():
-                buffer.extend(data)
-                while len(buffer) >= _STREAM_FRAME_BYTES:
-                    frame = bytes(buffer[:_STREAM_FRAME_BYTES])
-                    del buffer[:_STREAM_FRAME_BYTES]
-                    if held is not None:
-                        yield self._to_chunk(held, final=False)
-                    held = frame
-        if buffer:
-            if held is not None:
-                yield self._to_chunk(held, final=False)
-            held = bytes(buffer)
-        if held is not None:
-            yield self._to_chunk(held, final=True)
+
+        async def _reads():
+            async with (
+                self._async().audio.speech.with_streaming_response.create(
+                    model=self.name,
+                    voice=voice or self.voice,
+                    input=text,
+                    response_format=_STREAM_FORMAT,
+                    **{**self.generation_kwargs, **kwargs},
+                )
+            ) as response:
+                async for data in response.iter_bytes():
+                    yield data
+
+        async for chunk in frame_pcm_stream(
+            _reads(), sample_rate=_OPENAI_TTS_SAMPLE_RATE
+        ):
+            yield chunk
 
     def supports_streaming(self) -> bool:
         return True

deepeval/models/tts_models/__init__.py

@@ -1,3 +0,0 @@
-from deepeval.models.tts_models.openai_tts import OpenAITTSModel
-
-__all__ = ["OpenAITTSModel"]

deepeval/simulator/conversation_simulator.py

@@ -196,14 +196,11 @@ class ConversationSimulator:
             # A single connector holds a single live call; concurrent
             # conversations would interleave audio on the same session.
             max_concurrent = 1
-            from deepeval.models.tts_models import OpenAITTSModel
-            from deepeval.models.stt_models import OpenAISTTModel
-
             self._voice = _VoiceRun(
                 config=voice_config,
                 connector=voice_config.connector,
-                tts_model=voice_config.tts_model or OpenAITTSModel(),
-                stt_model=voice_config.stt_model or OpenAISTTModel(),
+                tts_model=voice_config.tts_model,
+                stt_model=voice_config.stt_model,
             )
             model_callback = self._voice_model_callback
             if voice_config.interruption_settings is not None:

deepeval/voice/__init__.py

@@ -12,6 +12,8 @@ from deepeval.voice.connectors import (
     BaseWebSocketConnector,
     ElevenLabsConnector,
     WebSocketConnector,
+    PipecatConnector,
+    VapiConnector,
 )
 
 __all__ = [
@@ -31,6 +33,8 @@ __all__ = [
     "BaseWebSocketConnector",
     "ElevenLabsConnector",
     "WebSocketConnector",
+    "PipecatConnector",
+    "VapiConnector",
 ]
 
 # Resolved lazily: these live with `Persona` in `deepeval.dataset`, which

deepeval/voice/config.py

@@ -12,6 +12,36 @@ if TYPE_CHECKING:
     from deepeval.dataset.golden import InterruptionBehavior
 
 
+def resolve_tts_model(
+    model: Optional[Union[str, DeepEvalBaseTTS]],
+) -> DeepEvalBaseTTS:
+    """The TTS model to speak with.
+
+    A model name is the name for the provider selected by `USE_*_TTS`, and
+    `None` leaves both the provider and the name to the environment. Model
+    objects are left alone, including ones that only duck-type the base class.
+    """
+    # Imported here because `deepeval.models.tts` reaches back into
+    # `deepeval.voice`, so importing it at module scope would cycle.
+    from deepeval.models.speech_selection import initialize_tts_model
+
+    return initialize_tts_model(model)
+
+
+def resolve_stt_model(
+    model: Optional[Union[str, DeepEvalBaseSTT]],
+) -> DeepEvalBaseSTT:
+    """The STT model to transcribe with.
+
+    A model name is the name for the provider selected by `USE_*_STT`, and
+    `None` leaves both the provider and the name to the environment. Model
+    objects are left alone, including ones that only duck-type the base class.
+    """
+    from deepeval.models.speech_selection import initialize_stt_model
+
+    return initialize_stt_model(model)
+
+
 @dataclass
 class VoiceConfig:
     """Voice-mode settings for `ConversationSimulator`.
@@ -19,7 +49,13 @@ class VoiceConfig:
     Passing a `VoiceConfig` puts the simulator in voice mode: simulated user
     turns are spoken (TTS), sent to the agent over `connector`, and the
     agent's spoken replies are transcribed (STT). `tts_model` / `stt_model`
-    default to the OpenAI implementations when left as None.
+    come from the environment when left as None — the `USE_*_TTS` /
+    `USE_*_STT` flag picks the provider and `DEEPEVAL_TTS_MODEL` /
+    `DEEPEVAL_STT_MODEL` picks the model, both defaulting to OpenAI. A bare
+    model name is the name for that same selected provider, so
+    `stt_model="whisper-1"` is shorthand for `OpenAISTTModel(model="whisper-1")`
+    unless another `USE_*_STT` flag is set. Pass the model object itself to
+    reach a specific provider or set anything beyond the name.
 
     `connector` is the connector the session runs over — `ElevenLabsConnector`,
     `LiveKitConnector`, `WebSocketConnector`, `CallbackVoiceConnector`, or your
@@ -34,8 +70,8 @@ class VoiceConfig:
     """
 
     connector: Union[BaseVoiceConnector, Callable[[], BaseVoiceConnector]]
-    tts_model: Optional[DeepEvalBaseTTS] = None
-    stt_model: Optional[DeepEvalBaseSTT] = None
+    tts_model: Optional[Union[str, DeepEvalBaseTTS]] = None
+    stt_model: Optional[Union[str, DeepEvalBaseSTT]] = None
     # Directory to write per-turn and combined audio files into. Left alone,
     # this resolves through `DEEPEVAL_VOICE_FOLDER` to a default folder; pass
     # `None` to skip writing audio to disk. Resolved once, here, so that by the
@@ -53,6 +89,8 @@ class VoiceConfig:
             self.connector, BaseVoiceConnector
         ):
             self.connector = validate_connector(self.connector)
+        self.tts_model = resolve_tts_model(self.tts_model)
+        self.stt_model = resolve_stt_model(self.stt_model)
         self.output_dir = resolve_output_dir(self.output_dir)
 
     def make_connector(self) -> BaseVoiceConnector:

deepeval/voice/connectors/__init__.py

@@ -8,6 +8,8 @@ from deepeval.voice.connectors.transports import (
 from deepeval.voice.connectors.providers import (
     ElevenLabsConnector,
     LiveKitConnector,
+    PipecatConnector,
+    VapiConnector,
 )
 
 __all__ = [
@@ -19,4 +21,6 @@ __all__ = [
     "BaseWebSocketConnector",
     "ElevenLabsConnector",
     "WebSocketConnector",
+    "PipecatConnector",
+    "VapiConnector",
 ]

deepeval/voice/connectors/providers/__init__.py

@@ -1,7 +1,11 @@
 from deepeval.voice.connectors.providers.elevenlabs import ElevenLabsConnector
 from deepeval.voice.connectors.providers.livekit import LiveKitConnector
+from deepeval.voice.connectors.providers.pipecat import PipecatConnector
+from deepeval.voice.connectors.providers.vapi import VapiConnector
 
 __all__ = [
     "ElevenLabsConnector",
     "LiveKitConnector",
+    "PipecatConnector",
+    "VapiConnector",
 ]

deepeval/voice/connectors/providers/pipecat/README.md

@@ -0,0 +1,92 @@
+# Pipecat
+
+Contributor reference and TODO list for the Pipecat integration. This file
+tracks which of Pipecat's entry points deepeval can reach; it deliberately does
+not restate their parameters, because those are defined by Pipecat and go stale
+here. Each row links to the page that owns the answer.
+
+## One connector, no dependency on `pipecat-ai`
+
+`PipecatConnector` (`connector.py`) connects to a pipeline running behind one of
+Pipecat's [WebSocket transports](https://docs.pipecat.ai/server/services/transport/websocket-server)
+the way any other client would: it is given a URL, and the pipeline is the
+user's to host.
+
+`protobuf.py` is our own encoder for the five messages in Pipecat's
+[`frames.proto`](https://github.com/pipecat-ai/pipecat/blob/main/src/pipecat/serializers/protobuf.py),
+which is what `ProtobufFrameSerializer` — the default on those transports —
+puts on the wire. Encoding them directly is about 150 lines and keeps
+`pipecat-ai`, a server framework with a large dependency tree, out of the
+requirements of testing something built with it.
+
+The schema is `oneof { text, audio, transcription, message, interruption }`.
+Only `audio` and `message` are written; `message` carries a JSON string, which
+is how RTVI rides the same socket.
+
+## What carries the turn-taking
+
+Pipecat has no end-of-turn frame in the protobuf schema, so the turn-taking
+comes from [RTVI](https://docs.pipecat.ai/server/frameworks/rtvi/introduction)
+messages, which a pipeline only sends if it runs an `RTVIProcessor` and its
+observer:
+
+| Message | Used for |
+| --- | --- |
+| `bot-stopped-speaking` | Ending the agent's turn outright, instead of inferring it from silence |
+| `bot-transcription` | The agent's own words, which skip STT for that turn |
+| `bot-interrupted` | Recording that a barge landed |
+| `client-ready` (sent by us) | Pipelines that hold their greeting, or their audio input, until the client is up |
+
+`signals_turn_complete` is therefore not a constant: it starts False and flips
+once a `bot-stopped-speaking` has actually been seen. Claiming it up front
+would leave a non-RTVI pipeline's turns waiting for a signal that is never
+coming; claiming it never would truncate replies that pause mid-sentence.
+
+RTVI emits one `bot-transcription` per sentence, so `_on_transcription`
+accumulates them and `bot-stopped-speaking` clears the buffer.
+
+## Entry point coverage
+
+| Entry point | Status | Reference |
+| --- | --- | --- |
+| `WebsocketServerTransport` | Covered | [WebSocket server](https://docs.pipecat.ai/server/services/transport/websocket-server) |
+| `FastAPIWebsocketTransport` | Covered | [FastAPI WebSocket](https://docs.pipecat.ai/server/services/transport/fastapi-websocket) |
+| `ProtobufFrameSerializer` | Covered | [Serializers](https://docs.pipecat.ai/server/utilities/serializers/introduction) |
+| `add_wav_header=True` on the transport | Covered | [WebSocket server](https://docs.pipecat.ai/server/services/transport/websocket-server) |
+| RTVI turn signals and bot transcripts | Covered | [RTVI](https://docs.pipecat.ai/server/frameworks/rtvi/introduction) |
+| `client-ready` handshake | Covered | [RTVI](https://docs.pipecat.ai/server/frameworks/rtvi/introduction) |
+| `bot-output` (the successor to `bot-transcription`) | Not covered | [RTVI messages](https://docs.pipecat.ai/client/js/api-reference/callbacks) |
+| Client-side function calls over `client-message` | Not covered | [RTVI](https://docs.pipecat.ai/server/frameworks/rtvi/introduction) |
+| `TwilioFrameSerializer`, `ExotelFrameSerializer`, and the other telephony dialects | Not covered | [Serializers](https://docs.pipecat.ai/server/utilities/serializers/introduction) |
+| Daily, LiveKit, and the other WebRTC transports | Not covered here | [Transports](https://docs.pipecat.ai/server/services/transport/daily) |
+| SmallWebRTC transport | Not covered | [SmallWebRTC](https://docs.pipecat.ai/server/services/transport/small-webrtc) |
+
+A pipeline on a WebRTC transport is reached through the connector for that
+service — a Pipecat pipeline on LiveKit is a LiveKit room, and
+`LiveKitConnector` joins it. The telephony serializers are the agent taking a
+real phone call; reaching such an agent means running the same pipeline behind
+a WebSocket transport instead.
+
+## TODO
+
+**Smoke test against a live pipeline.** The frame encoding is checked against
+Pipecat's own generated `frames_pb2` in both directions, but three things could
+only be confirmed against a running pipeline: that a pipeline gated on
+`on_client_ready` does open on our `client-ready`, that `bot-stopped-speaking`
+arrives after the last audio frame rather than before it, and how a pipeline
+whose `audio_in_sample_rate` differs from `agent_sample_rate` degrades (Pipecat
+does not resample what it is handed).
+
+**`bot-output`.** RTVI is deprecating `bot-transcription` in favour of
+`bot-output`, which carries progress updates for the same text and would double
+up the transcript if both were read. When `bot-transcription` goes, the
+accumulation has to move to `bot-output` and de-duplicate by segment id.
+
+**What the agent heard.** `transcription` frames carry Pipecat's STT of
+deepeval's synthesized audio, and are dropped. As with the other connectors, it
+is the signal that separates "the agent answered badly" from "the agent misheard
+the question", and there is nowhere on `ConnectorTurn` to put it today.
+
+**Ending the pipeline.** `disconnect()` closes the socket and nothing more.
+RTVI's `disconnect-bot` would end the pipeline task outright, which is more
+than a client should decide for a server it does not own.

deepeval/voice/connectors/providers/pipecat/__init__.py

@@ -0,0 +1,5 @@
+from deepeval.voice.connectors.providers.pipecat.connector import (
+    PipecatConnector,
+)
+
+__all__ = ["PipecatConnector"]

deepeval/voice/connectors/providers/pipecat/connector.py

@@ -0,0 +1,205 @@
+import json
+import logging
+import uuid
+from typing import List, Optional, Union
+
+from deepeval.voice.connectors import audio_utils
+from deepeval.voice.connectors.providers.pipecat import protobuf
+from deepeval.voice.connectors.transports.websocket import (
+    BaseWebSocketConnector,
+    InboundEvent,
+)
+
+logger = logging.getLogger(__name__)
+
+# Pipecat's default input rate; its output default is 24000, which is also the
+# rate deepeval records at, so only the uplink needs stating.
+DEFAULT_AGENT_SAMPLE_RATE = 16000
+
+RTVI_LABEL = "rtvi-ai"
+RTVI_PROTOCOL_VERSION = "2.1.0"
+
+
+class PipecatConnector(BaseWebSocketConnector):
+    """Connect to a Pipecat pipeline over its WebSocket transport.
+
+    The pipeline is yours and self-hosted, so the connector takes a URL rather
+    than credentials, and speaks the protobuf frames Pipecat's WebSocket
+    transports serialize by default.
+    """
+
+    def __init__(
+        self,
+        url: str,
+        *,
+        headers: Optional[dict] = None,
+        agent_sample_rate: int = DEFAULT_AGENT_SAMPLE_RATE,
+        client_ready: bool = True,
+        **base_kwargs,
+    ):
+        super().__init__(**base_kwargs)
+        self.url = url
+        self.headers = headers
+        # Pipecat does not resample what it is handed: the frames have to
+        # arrive at the rate the pipeline's VAD and STT were started with.
+        self.agent_sample_rate = agent_sample_rate
+        self._send_rate = agent_sample_rate
+        self.client_ready = client_ready
+        self._transcript_parts: List[str] = []
+        self._seen_turn_signal = False
+        self._warned_non_protobuf = False
+
+    @property
+    def signals_turn_complete(self) -> bool:
+        """True once the pipeline has been seen to announce an end of turn.
+
+        Whether it does is a property of the pipeline rather than of Pipecat:
+        the announcement is an RTVI `bot-stopped-speaking` message, and only a
+        pipeline running an `RTVIProcessor` sends one. Reading it off the wire
+        rather than taking it on trust means a pipeline without RTVI falls back
+        to ending turns on silence instead of waiting for a signal that is
+        never coming.
+        """
+        return self._seen_turn_signal
+
+    async def connect(self) -> None:
+        self._transcript_parts = []
+        self._seen_turn_signal = False
+        await super().connect()
+
+    async def _open_session(self) -> str:
+        return self.url
+
+    def _connect_headers(self) -> Optional[dict]:
+        return self.headers
+
+    def _ready_on_connect(self) -> bool:
+        # A Pipecat server accepts audio as soon as the socket is up; nothing
+        # is sent back to acknowledge the client.
+        return True
+
+    def _initial_messages(self) -> List[Union[str, bytes]]:
+        """Announce the client, which some pipelines wait for before greeting.
+
+        Pipelines commonly open on `on_client_ready` — the greeting, and
+        sometimes the audio input itself, is gated on it. A pipeline without an
+        `RTVIProcessor` has nothing listening for the message and ignores it.
+        """
+        if not self.client_ready:
+            return []
+        return [
+            protobuf.encode_message_frame(
+                json.dumps(
+                    {
+                        "label": RTVI_LABEL,
+                        "type": "client-ready",
+                        "id": uuid.uuid4().hex,
+                        "data": {
+                            "version": RTVI_PROTOCOL_VERSION,
+                            "about": {"library": "deepeval"},
+                        },
+                    }
+                )
+            )
+        ]
+
+    def _encode_outbound(self, pcm: bytes) -> Union[str, bytes]:
+        return protobuf.encode_audio_frame(pcm, self._send_rate, 1)
+
+    def _decode_inbound(self, raw: Union[str, bytes]) -> Optional[InboundEvent]:
+        if isinstance(raw, str):
+            self._warn_non_protobuf("a text message")
+            return None
+        try:
+            frame = protobuf.decode_frame(bytes(raw))
+        except ValueError:
+            self._warn_non_protobuf("bytes that are not protobuf frames")
+            return None
+        if frame is None:
+            return None
+
+        if frame.kind == "audio":
+            return self._on_audio(frame)
+        if frame.kind == "message":
+            return self._on_transport_message(frame.data)
+        if frame.kind == "interruption":
+            self._interrupted = True
+        # A `transcription` frame is Pipecat's STT of deepeval's own audio, and
+        # a `text` frame is unattributed pipeline text; neither is the agent's
+        # reply.
+        return None
+
+    def _warn_non_protobuf(self, what: str) -> None:
+        if self._warned_non_protobuf:
+            return
+        self._warned_non_protobuf = True
+        logger.warning(
+            "PipecatConnector received %s. It speaks the protobuf frames of "
+            "`ProtobufFrameSerializer`, which is what Pipecat's WebSocket "
+            "transports use by default — a pipeline serializing something "
+            "else needs a `WebSocketConnector` describing that dialect "
+            "instead.",
+            what,
+        )
+
+    def _on_audio(self, frame: protobuf.PipecatFrame) -> Optional[InboundEvent]:
+        pcm = frame.audio or b""
+        if not pcm:
+            return None
+        sample_rate, channels = frame.sample_rate, frame.num_channels or 1
+        if pcm[:4] == b"RIFF":
+            # `add_wav_header=True` on the transport wraps every frame.
+            pcm, sample_rate, channels = audio_utils.wav_bytes_to_pcm16(pcm)
+        if sample_rate and sample_rate != self._recv_rate:
+            logger.debug(
+                "Pipecat is sending %dHz audio; reading the downlink at that "
+                "rate instead of %dHz.",
+                sample_rate,
+                self._recv_rate,
+            )
+            self._recv_rate = sample_rate
+        return InboundEvent(audio=audio_utils.downmix_to_mono(pcm, channels))
+
+    def _on_transport_message(
+        self, data: Optional[str]
+    ) -> Optional[InboundEvent]:
+        try:
+            message = json.loads(data or "")
+        except ValueError:
+            return None
+        if not isinstance(message, dict) or message.get("label") != RTVI_LABEL:
+            return None
+
+        message_type = message.get("type")
+
+        if message_type == "bot-stopped-speaking":
+            self._seen_turn_signal = True
+            self._transcript_parts = []
+            return InboundEvent(turn_complete=True)
+
+        if message_type == "bot-interrupted":
+            self._interrupted = True
+            return None
+
+        if message_type == "bot-transcription":
+            return self._on_transcription(message)
+
+        return None
+
+    def _on_transcription(self, message: dict) -> Optional[InboundEvent]:
+        """Collect the agent's transcript for the turn in progress.
+
+        RTVI emits one `bot-transcription` per sentence, so the last one alone
+        would record the end of a multi-sentence reply and nothing before it.
+        """
+        data = message.get("data")
+        text = (data or {}).get("text") if isinstance(data, dict) else None
+        text = (text or "").strip()
+        if not text:
+            return None
+        self._transcript_parts.append(text)
+        return InboundEvent(transcript=" ".join(self._transcript_parts))
+
+    def drain_downlink(self) -> None:
+        self._transcript_parts = []
+        super().drain_downlink()

deepeval/voice/connectors/providers/pipecat/protobuf.py

@@ -0,0 +1,178 @@
+"""The part of Pipecat's `frames.proto` that a caller has to speak.
+
+Pipecat's WebSocket transports serialize with `ProtobufFrameSerializer`, whose
+schema is five messages wide and carries only what a client sends or reads:
+audio, text, a transcription, a JSON transport message, and an interruption.
+Encoding those directly keeps `pipecat-ai` — a server-side framework with a
+large dependency tree — out of the requirements of running a simulation
+against it.
+
+https://github.com/pipecat-ai/pipecat/blob/main/src/pipecat/serializers/protobuf.py
+"""
+
+from dataclasses import dataclass
+from typing import Iterator, Optional, Tuple
+
+_WIRE_VARINT = 0
+_WIRE_64BIT = 1
+_WIRE_LEN = 2
+_WIRE_32BIT = 5
+
+# Fields of the `Frame` oneof.
+_FRAME_TEXT = 1
+_FRAME_AUDIO = 2
+_FRAME_TRANSCRIPTION = 3
+_FRAME_MESSAGE = 4
+_FRAME_INTERRUPTION = 5
+
+# Fields within each of those messages.
+_TEXT_TEXT = 3
+_AUDIO_AUDIO = 3
+_AUDIO_SAMPLE_RATE = 4
+_AUDIO_NUM_CHANNELS = 5
+_TRANSCRIPTION_TEXT = 3
+_MESSAGE_DATA = 1
+
+
+@dataclass
+class PipecatFrame:
+    """One decoded frame, named by which arm of the oneof it arrived on."""
+
+    kind: str  # text | audio | transcription | message | interruption
+    audio: Optional[bytes] = None
+    sample_rate: Optional[int] = None
+    num_channels: Optional[int] = None
+    text: Optional[str] = None
+    data: Optional[str] = None  # JSON, for `message`
+
+
+def _varint(value: int) -> bytes:
+    out = bytearray()
+    while True:
+        byte = value & 0x7F
+        value >>= 7
+        out.append(byte | (0x80 if value else 0))
+        if not value:
+            return bytes(out)
+
+
+def _tag(field: int, wire: int) -> bytes:
+    return _varint(field << 3 | wire)
+
+
+def _len_field(field: int, payload: bytes) -> bytes:
+    return _tag(field, _WIRE_LEN) + _varint(len(payload)) + payload
+
+
+def _varint_field(field: int, value: int) -> bytes:
+    return _tag(field, _WIRE_VARINT) + _varint(value)
+
+
+def encode_audio_frame(
+    pcm: bytes, sample_rate: int, num_channels: int = 1
+) -> bytes:
+    """A `Frame` carrying PCM16, which Pipecat reads as an `InputAudioRawFrame`."""
+    audio = (
+        _len_field(_AUDIO_AUDIO, pcm)
+        + _varint_field(_AUDIO_SAMPLE_RATE, sample_rate)
+        + _varint_field(_AUDIO_NUM_CHANNELS, num_channels)
+    )
+    return _len_field(_FRAME_AUDIO, audio)
+
+
+def encode_message_frame(data: str) -> bytes:
+    """A `Frame` carrying a JSON transport message, such as RTVI's."""
+    return _len_field(
+        _FRAME_MESSAGE, _len_field(_MESSAGE_DATA, data.encode("utf-8"))
+    )
+
+
+def _read_varint(buf: bytes, i: int) -> Tuple[int, int]:
+    value = shift = 0
+    while True:
+        if i >= len(buf):
+            raise ValueError("truncated varint")
+        byte = buf[i]
+        i += 1
+        value |= (byte & 0x7F) << shift
+        if not byte & 0x80:
+            return value, i
+        shift += 7
+
+
+def _iter_fields(buf: bytes) -> Iterator[Tuple[int, int, object]]:
+    i = 0
+    while i < len(buf):
+        key, i = _read_varint(buf, i)
+        field, wire = key >> 3, key & 0x07
+        if wire == _WIRE_VARINT:
+            value, i = _read_varint(buf, i)
+        elif wire == _WIRE_LEN:
+            length, i = _read_varint(buf, i)
+            value, i = buf[i : i + length], i + length
+            if len(value) != length:
+                raise ValueError("truncated length-delimited field")
+        elif wire in (_WIRE_64BIT, _WIRE_32BIT):
+            width = 8 if wire == _WIRE_64BIT else 4
+            value, i = buf[i : i + width], i + width
+        else:
+            raise ValueError(f"unsupported protobuf wire type {wire}")
+        yield field, wire, value
+
+
+def _string(value: bytes) -> str:
+    return value.decode("utf-8", "replace")
+
+
+def _audio_frame(payload: bytes) -> PipecatFrame:
+    frame = PipecatFrame(kind="audio", audio=b"")
+    for field, _, value in _iter_fields(payload):
+        if field == _AUDIO_AUDIO:
+            frame.audio = bytes(value)
+        elif field == _AUDIO_SAMPLE_RATE:
+            frame.sample_rate = value
+        elif field == _AUDIO_NUM_CHANNELS:
+            frame.num_channels = value
+    return frame
+
+
+def _one_string_frame(
+    payload: bytes, kind: str, field_number: int
+) -> PipecatFrame:
+    frame = PipecatFrame(kind=kind, text="")
+    for field, _, value in _iter_fields(payload):
+        if field == field_number:
+            frame.text = _string(value)
+    return frame
+
+
+def _message_frame(payload: bytes) -> PipecatFrame:
+    frame = PipecatFrame(kind="message", data="")
+    for field, _, value in _iter_fields(payload):
+        if field == _MESSAGE_DATA:
+            frame.data = _string(value)
+    return frame
+
+
+def decode_frame(raw: bytes) -> Optional[PipecatFrame]:
+    """Read a serialized `Frame`, or None if it carries nothing we know.
+
+    Raises `ValueError` on bytes that are not protobuf at all, which is how a
+    pipeline serializing something else announces itself.
+    """
+    for field, wire, value in _iter_fields(raw):
+        if wire != _WIRE_LEN:
+            continue
+        if field == _FRAME_AUDIO:
+            return _audio_frame(value)
+        if field == _FRAME_MESSAGE:
+            return _message_frame(value)
+        if field == _FRAME_TRANSCRIPTION:
+            return _one_string_frame(
+                value, "transcription", _TRANSCRIPTION_TEXT
+            )
+        if field == _FRAME_TEXT:
+            return _one_string_frame(value, "text", _TEXT_TEXT)
+        if field == _FRAME_INTERRUPTION:
+            return PipecatFrame(kind="interruption")
+    return None

deepeval/voice/connectors/providers/vapi/README.md

@@ -0,0 +1,85 @@
+# Vapi
+
+Contributor reference and TODO list for the Vapi integration. This file tracks
+which of Vapi's entry points deepeval can reach; it deliberately does not
+restate their parameters, because those are defined by Vapi and go stale here.
+Each row links to the page that owns the answer.
+
+## One connector
+
+`VapiConnector` (`connector.py`) is our own client for Vapi's
+[WebSocket transport](https://docs.vapi.ai/calls/websocket-transport). The
+caller passes an `assistant_id` plus an API key, deepeval creates the call and
+owns the socket, and there is no dependency on any Vapi SDK.
+
+The call is minted over REST (`POST /call` with
+`transport.provider = "vapi.websocket"`), which returns
+`transport.websocketCallUrl`. Phone parameters are rejected on that transport,
+so this path never touches telephony — the simulator is the only caller.
+
+## Client messages deepeval depends on
+
+Two of Vapi's [client messages](https://docs.vapi.ai/api-reference/webhooks/client-message)
+carry the turn-taking:
+
+| Message | Shape deepeval reads | Used for |
+| --- | --- | --- |
+| `speech-update` | `status` (`started`/`stopped`), `role` | Ending the agent's turn outright, instead of inferring it from silence |
+| `transcript` | `role`, `transcriptType` (`partial`/`final`), `transcript` | The agent's own words, which skip STT for that turn |
+
+Both are in Vapi's default `clientMessages` set, so nothing is sent unless the
+caller narrows the set themselves — `_assistant_overrides` then adds the two
+back, because losing them costs end-of-turn detection and free transcripts
+silently rather than loudly.
+
+Vapi emits one `final` transcript per utterance rather than one per turn, so
+`_on_transcript` accumulates them and `speech-update: stopped` clears the
+buffer. Taking only the last one would record half of a two-sentence reply.
+
+## Entry point coverage
+
+| Entry point | Status | Reference |
+| --- | --- | --- |
+| Assistant over WebSocket transport | Covered | [WebSocket transport](https://docs.vapi.ai/calls/websocket-transport) |
+| Per-call `assistantOverrides` (incl. `variableValues`) | Covered | [Assistant overrides](https://docs.vapi.ai/assistants/dynamic-variables) |
+| Explicit end of turn (`speech-update`) | Covered | [Client messages](https://docs.vapi.ai/api-reference/webhooks/client-message) |
+| Agent transcript (`transcript`, final) | Covered | [Client messages](https://docs.vapi.ai/api-reference/webhooks/client-message) |
+| `user-interrupted` | Covered | [Client messages](https://docs.vapi.ai/api-reference/webhooks/client-message) |
+| Ending the call (`end-call`) | Covered | [WebSocket transport](https://docs.vapi.ai/calls/websocket-transport) |
+| mu-law audio format | Not covered | [WebSocket transport](https://docs.vapi.ai/calls/websocket-transport) |
+| Inline assistant config instead of `assistantId` | Not covered | [Create call](https://docs.vapi.ai/api-reference/calls/create) |
+| Squads | Not covered | [Squads](https://docs.vapi.ai/squads) |
+| `tool-calls` / client-side tools | Not covered | [Client messages](https://docs.vapi.ai/api-reference/webhooks/client-message) |
+| `conversation-update`, `model-output` | Not covered | [Client messages](https://docs.vapi.ai/api-reference/webhooks/client-message) |
+| Call Listen (`monitor.listenUrl`) | Not covered, by choice | [WebSocket transport](https://docs.vapi.ai/calls/websocket-transport) |
+| Phone numbers, campaigns, SIP | Out of scope | [Calls](https://docs.vapi.ai/calls) |
+
+Call Listen is listen-only and supplements a call that already exists, so it
+cannot carry a simulation. "Out of scope" means the entry point is the agent
+placing or receiving a real phone call; reaching such an agent means pointing
+deepeval at the same assistant over the WebSocket transport instead.
+
+## TODO
+
+**Smoke test against a live assistant.** Everything here is written against
+Vapi's documented schemas. Three things could only be confirmed on a real call:
+whether the socket wraps client messages in a `message` envelope the way
+webhooks do (`_decode_inbound` accepts both shapes, so either works), whether a
+non-default `sampleRate` is honoured or quietly converted
+(`_apply_audio_format` adopts whatever comes back), and how the socket behaves
+before the first audio frame (`_ready_on_connect` assumes no handshake).
+`vapi_capture.py` at the repo root records one conversation for this.
+
+**What the agent heard.** `transcript` messages with `role: "user"` carry
+Vapi's transcription of deepeval's synthesized audio, and are dropped. As with
+ElevenLabs, it is the signal that separates "the agent answered badly" from
+"the agent misheard the question", and there is nowhere on `ConnectorTurn` to
+put it today.
+
+**Interruption detail.** `user-interrupted` sets `_interrupted` for the turn,
+but Vapi's own barge-in handling and deepeval's floor control both act on the
+same moment, and the interaction has not been exercised.
+
+**Call id.** `call_id` is kept from the create-call response but never
+surfaced, so a simulated conversation cannot be cross-referenced with the same
+call in Vapi's dashboard.

deepeval/voice/connectors/providers/vapi/__init__.py

@@ -0,0 +1,3 @@
+from deepeval.voice.connectors.providers.vapi.connector import VapiConnector
+
+__all__ = ["VapiConnector"]

deepeval/voice/connectors/providers/vapi/connector.py

@@ -0,0 +1,213 @@
+import os
+import json
+import logging
+from typing import Any, Dict, List, Optional, Union
+
+from deepeval.errors import DeepEvalError
+from deepeval.voice.connectors.transports.websocket import (
+    BaseWebSocketConnector,
+    InboundEvent,
+)
+
+logger = logging.getLogger(__name__)
+
+DEFAULT_BASE_URL = "https://api.vapi.ai"
+
+# The client messages deepeval reads off the socket. Both are in Vapi's default
+# set, so they are only ever re-stated when a caller narrows the set themselves.
+# https://docs.vapi.ai/api-reference/webhooks/client-message
+REQUIRED_CLIENT_MESSAGES = ("speech-update", "transcript")
+
+
+class VapiConnector(BaseWebSocketConnector):
+
+    def __init__(
+        self,
+        assistant_id: str,
+        api_key: Optional[str] = None,
+        assistant_overrides: Optional[dict] = None,
+        base_url: str = DEFAULT_BASE_URL,
+        **base_kwargs,
+    ):
+        super().__init__(**base_kwargs)
+        self.assistant_id = assistant_id
+        self.api_key = api_key or os.getenv("VAPI_API_KEY")
+        self.assistant_overrides = assistant_overrides
+        self.base_url = base_url.rstrip("/")
+        self.call_id: Optional[str] = None
+        self._transcript_parts: List[str] = []
+
+    @property
+    def signals_turn_complete(self) -> bool:
+        # `speech-update` closes every turn.
+        return True
+
+    def _ready_on_connect(self) -> bool:
+        # The socket is minted per call and carries no handshake; audio may go
+        # out as soon as it opens.
+        return True
+
+    def _assistant_overrides(self) -> Optional[dict]:
+        """The caller's overrides, plus the client messages deepeval needs.
+
+        Narrowing `clientMessages` is how a Vapi assistant turns events off, so
+        a caller who sets it without `speech-update` would silently remove
+        end-of-turn detection, and one without `transcript` would pay for STT
+        on every turn.
+        """
+        overrides = dict(self.assistant_overrides or {})
+        requested = overrides.get("clientMessages")
+        if requested is None:
+            return overrides or None
+        missing = [
+            message
+            for message in REQUIRED_CLIENT_MESSAGES
+            if message not in requested
+        ]
+        if missing:
+            overrides["clientMessages"] = list(requested) + missing
+        return overrides
+
+    def _call_payload(self) -> Dict[str, Any]:
+        payload: Dict[str, Any] = {
+            "assistantId": self.assistant_id,
+            "transport": {
+                "provider": "vapi.websocket",
+                "audioFormat": {
+                    "format": "pcm_s16le",
+                    "container": "raw",
+                    "sampleRate": self.sample_rate,
+                },
+            },
+        }
+        overrides = self._assistant_overrides()
+        if overrides:
+            payload["assistantOverrides"] = overrides
+        return payload
+
+    async def _open_session(self) -> str:
+        if not self.api_key:
+            raise DeepEvalError(
+                "VapiConnector requires a Vapi API key (pass `api_key=...` or "
+                "set VAPI_API_KEY). The key mints the call the WebSocket "
+                "carries."
+            )
+        async with self._session.post(
+            f"{self.base_url}/call",
+            headers={"authorization": f"Bearer {self.api_key}"},
+            json=self._call_payload(),
+        ) as resp:
+            body = await resp.text()
+            if resp.status >= 300:
+                raise DeepEvalError(
+                    f"Vapi call creation failed ({resp.status}): {body}"
+                )
+            try:
+                data = json.loads(body)
+            except ValueError:
+                raise DeepEvalError(
+                    f"Vapi call creation returned a non-JSON body: {body[:200]}"
+                )
+
+        transport = data.get("transport") or {}
+        url = transport.get("websocketCallUrl")
+        if not url:
+            raise DeepEvalError(
+                "Vapi call response had no `transport.websocketCallUrl`. Check "
+                "that the assistant exists and the key may create calls."
+            )
+        self.call_id = data.get("id")
+        self._apply_audio_format(transport.get("audioFormat"))
+        return url
+
+    def _apply_audio_format(self, audio_format: Optional[dict]) -> None:
+        """Adopt the format Vapi settled on, which need not be the one asked for."""
+        if not isinstance(audio_format, dict):
+            return
+        encoding = audio_format.get("format")
+        if encoding and encoding != "pcm_s16le":
+            logger.warning(
+                "Vapi negotiated audio format %r; deepeval sends and reads "
+                "PCM16, so the audio will be garbled. Request pcm_s16le.",
+                encoding,
+            )
+        rate = audio_format.get("sampleRate")
+        if isinstance(rate, int) and rate > 0:
+            self._send_rate = rate
+            self._recv_rate = rate
+
+    def _encode_outbound(self, pcm: bytes) -> Union[str, bytes]:
+        return pcm
+
+    def _decode_inbound(self, raw: Union[str, bytes]) -> Optional[InboundEvent]:
+        if isinstance(raw, (bytes, bytearray)):
+            return InboundEvent(audio=bytes(raw))
+
+        try:
+            payload = json.loads(raw)
+        except (ValueError, TypeError):
+            return None
+        if not isinstance(payload, dict):
+            return None
+        # Vapi's webhook deliveries wrap the event in `message`; the socket
+        # sends the same events, so accept either shape.
+        message = payload.get("message")
+        if not isinstance(message, dict):
+            message = payload
+
+        message_type = message.get("type")
+
+        if message_type == "transcript":
+            return self._on_transcript(message)
+
+        if message_type == "speech-update":
+            if (
+                message.get("role") == "assistant"
+                and message.get("status") == "stopped"
+            ):
+                self._transcript_parts = []
+                return InboundEvent(turn_complete=True)
+            return None
+
+        if message_type == "user-interrupted":
+            self._interrupted = True
+            return None
+
+        return None
+
+    def _on_transcript(self, message: dict) -> Optional[InboundEvent]:
+        """Collect the agent's final transcripts for the turn in progress.
+
+        Vapi emits one per utterance rather than one per turn, so a two
+        sentence reply arrives as two finals and the last one on its own would
+        record half of what the agent said.
+        """
+        if message.get("role") != "assistant":
+            return None
+        if message.get("transcriptType") != "final":
+            return None
+        text = (message.get("transcript") or "").strip()
+        if not text:
+            return None
+        self._transcript_parts.append(text)
+        return InboundEvent(transcript=" ".join(self._transcript_parts))
+
+    def drain_downlink(self) -> None:
+        self._transcript_parts = []
+        super().drain_downlink()
+
+    async def disconnect(self) -> None:
+        """End the call before dropping the socket.
+
+        Closing the socket alone leaves the call live on Vapi's side until it
+        times out, still billing and still holding a concurrency slot.
+        """
+        if self._ws is not None and not self._ws.closed:
+            try:
+                await self._send(json.dumps({"type": "end-call"}))
+            except Exception:
+                logger.debug(
+                    "Failed to send Vapi end-call; closing the socket anyway.",
+                    exc_info=True,
+                )
+        await super().disconnect()

deepeval/voice/connectors/utils.py

@@ -16,6 +16,20 @@ class LiveKitIdentity(Enum):
     ROOM = "Room"
 
 
+class VapiIdentity(Enum):
+    CLIENT = "Vapi"
+    ASYNC_CLIENT = "AsyncVapi"
+
+
+class PipecatIdentity(Enum):
+    PIPELINE = "Pipeline"
+    TASK = "PipelineTask"
+    WORKER = "PipelineWorker"
+    SERVER_TRANSPORT = "WebsocketServerTransport"
+    SINGLE_CLIENT_TRANSPORT = "SingleClientWebsocketServerTransport"
+    FASTAPI_TRANSPORT = "FastAPIWebsocketTransport"
+
+
 # What to tell someone who passed the vendor object instead of a connector.
 # Every voice session runs through a connector, so these are redirections
 # rather than alternative entry points.
@@ -36,8 +50,33 @@ _REDIRECTS = {
         "`LiveKitConnector(room=your_room)` instead, which joins the room you "
         "already have."
     ),
+    VapiIdentity.CLIENT: (
+        "VoiceConfig received a Vapi client. Pass "
+        "`VapiConnector(assistant_id=...)` instead — it creates the call "
+        "itself, and takes `assistant_overrides` for anything you would "
+        "configure per call."
+    ),
+    VapiIdentity.ASYNC_CLIENT: (
+        "VoiceConfig received a Vapi client. Pass "
+        "`VapiConnector(assistant_id=...)` instead."
+    ),
 }
 
+# Pipecat's pipeline objects and its WebSocket transports all mean the same
+# thing here: the agent is reached over the socket its server is listening on,
+# never in-process.
+_REDIRECTS.update(
+    {
+        identity: (
+            f"VoiceConfig received a Pipecat `{identity.value}`. Run the "
+            "pipeline behind its WebSocket transport and pass "
+            "`PipecatConnector(url=...)` instead, which connects to it the "
+            "way any other client would."
+        )
+        for identity in PipecatIdentity
+    }
+)
+
 
 def _root_module(cls: type) -> str:
     return (getattr(cls, "__module__", "") or "").split(".")[0]
@@ -45,7 +84,9 @@ def _root_module(cls: type) -> str:
 
 def _identify(
     obj: Any,
-) -> Optional[Union[ElevenLabsIdentity, LiveKitIdentity]]:
+) -> Optional[
+    Union[ElevenLabsIdentity, LiveKitIdentity, PipecatIdentity, VapiIdentity]
+]:
     """Name the vendor class `obj` is, without importing the vendor package.
 
     Matching walks the MRO by `(root module, class name)` rather than using
@@ -63,6 +104,14 @@ def _identify(
             for identity in LiveKitIdentity:
                 if name == identity.value:
                     return identity
+        if root in ("vapi", "vapi_server_sdk"):
+            for identity in VapiIdentity:
+                if name == identity.value:
+                    return identity
+        if root == "pipecat":
+            for identity in PipecatIdentity:
+                if name == identity.value:
+                    return identity
     return None
 
 
@@ -82,7 +131,7 @@ def validate_connector(connector: Any) -> BaseVoiceConnector:
 
     raise DeepEvalError(
         f"VoiceConfig cannot use {type(connector).__name__!r} as a connector. "
-        "Pass an `ElevenLabsConnector`, `LiveKitConnector`, "
-        "`WebSocketConnector`, `CallbackVoiceConnector`, or your own "
-        "`BaseVoiceConnector` subclass."
+        "Pass an `ElevenLabsConnector`, `LiveKitConnector`, `VapiConnector`, "
+        "`PipecatConnector`, `WebSocketConnector`, `CallbackVoiceConnector`, "
+        "or your own `BaseVoiceConnector` subclass."
     )

docs/content/docs/command-line-interface.mdx

@@ -372,7 +372,7 @@ npx deepeval diagnose
 The report shows:
 
 - **Versions**: deepeval and the runtime it's running on.
-- **Evaluation model**: the model deepeval will use when a metric<Only id="python"> (or synthesizer, simulator, etc.)</Only> is constructed without an explicit model, plus the reason it won.<Only id="python"> The default embedding model is listed alongside it, with what each is used by.</Only>
+- **Evaluation model**: the model deepeval will use when a metric<Only id="python"> (or synthesizer, simulator, etc.)</Only> is constructed without an explicit model, plus the reason it won.<Only id="python"> The default embedding model is listed alongside it, and the selected text-to-speech and speech-to-text models below those, with what each is used by.</Only>
 - **Configuration sources**: the precedence order every variable is resolved against, independently (process environment → `.env.local` → `.env.<APP_ENV>` → `.env` → JSON keystore → built-in defaults), which dotenv files were loaded, and where the keystore lives.
 - **Configured settings**: every deepeval setting that is explicitly set, with secrets masked (only the last 6 characters shown) and the **winning source** for each — the report uses the same resolvers as the runtime, so what you see is what your evals get.
 - **Confident AI**: login status, masked API key, data region, and the API endpoint.<Only id="python"> The OTEL endpoint is shown too, with warnings if either endpoint looks like it belongs to a different region than your API key.</Only>
@@ -680,12 +680,24 @@ All provider commands come in pairs:
 
 </Switch>
 
-This switches the active provider:
+This switches the active provider **within its family**:
 
-- It sets `USE_<PROVIDER>_MODEL = True` for the chosen provider, and
-- Turns all other `USE_*` flags off so that only one provider is enabled at a time.
+- It sets that provider's `USE_*` flag, such as `USE_ANTHROPIC_MODEL`, and
+- Turns off the other `USE_*` flags in the same family, so only one provider per family is enabled at a time.
 
-When you **set** a provider, the CLI enables that provider’s `USE_<PROVIDER>_MODEL` flag and disables all other `USE_*` flags. When you **unset** a provider, it disables only that provider’s `USE_*` flag and leaves all others untouched. If you manually set env vars (or edit dotenv files) it’s possible to end up with multiple `USE_*` flags enabled.
+<Only id="python">
+
+The four families are the LLM judge, embeddings, text-to-speech and speech-to-text. They are independent, so `deepeval set-openai` leaves a configured embedding or speech provider exactly as it was, and each family falls back to OpenAI when none of its flags are set.
+
+</Only>
+
+<Only id="typescript">
+
+Every `USE_*` flag here belongs to the LLM family, so setting one provider turns off all the others. With none set, `deepeval` falls back to OpenAI.
+
+</Only>
+
+When you **unset** a provider, it disables only that provider’s `USE_*` flag and leaves all others untouched. If you manually set env vars (or edit dotenv files) it’s possible to end up with multiple `USE_*` flags enabled within one family.
 
 :::caution
 Because of how `deepeval` manages your model related environment variables, **using the CLI is 100% the recommended way to configure evaluation models in `deepeval`.** It handles all the necessary environment variables for you, ensuring consistent and correct setup across different providers.
@@ -723,6 +735,21 @@ If you want to see what environment variables `deepeval` manages under the hood,
 | Local (HTTP)          | `set-local-embeddings`       | `unset-local-embeddings`       |
 | Ollama                | `set-ollama-embeddings`      | `unset-ollama-embeddings`      |
 
+**Speech (voice simulation):**
+
+Speech takes the provider as an argument instead of a command per provider, and text-to-speech and speech-to-text are selected independently of each other and of the LLM:
+
+| Family         | Set                                                            | Unset        |
+| -------------- | -------------------------------------------------------------- | ------------ |
+| Text-to-speech | `set-tts <openai\|elevenlabs\|cartesia\|deepgram>`              | `unset-tts`  |
+| Speech-to-text | `set-stt <openai\|elevenlabs\|cartesia\|deepgram\|assemblyai>`  | `unset-stt`  |
+
+```bash
+deepeval set-stt deepgram --model nova-3
+```
+
+The model name is optional and defaults to that provider's own default. `unset-tts` and `unset-stt` clear their whole family rather than one provider, since the provider is an argument rather than part of the command name.
+
 </Case>
 
 <Case id="typescript">
@@ -755,7 +782,7 @@ For provider-specific flags, run any of the `set-*` commands above with `--help`
 ## Common Issues
 
 - **Nothing printed?** For `set-*` / `unset-*` / `set-debug`, a clean exit with no output often means you are passing the `--quiet` / `-q` flag.
-- **Provider still active after unsetting?** Unsetting turns off that provider's `USE_*` flag only; if another provider's flag is still on, it becomes the active provider. With no `USE_*` flag enabled, deepeval falls back to OpenAI. To force a provider, run the corresponding `set-<provider>` command.
+- **Provider still active after unsetting?** Unsetting turns off that provider's `USE_*` flag only; if another provider in the same family still has its flag on, it becomes the active one for that family. With no flag enabled in a family, deepeval falls back to OpenAI for it. To force a provider, run the corresponding `set-<provider>` command.
 - **Dotenv edits not picked up?** deepeval loads dotenv files from the current working directory by default, or `ENV_DIR_PATH` if set. Ensure your process runs in that context.
 
 If you’re still stuck, the dedicated [Troubleshooting](/docs/troubleshooting) page covers deeper debugging (TLS errors, logging, timeouts, dotenv loading, and config caching).

docs/content/docs/conversation-simulator-voice-connectors.mdx

@@ -12,8 +12,10 @@ You won't need to write one. Pick whichever matches where your agent runs, tell
 
 | Connector                | Transport  | Use it for                                                           |
 | ------------------------ | ---------- | -------------------------------------------------------------------- |
+| `VapiConnector`          | WebSocket  | Vapi assistants, addressed by `assistant_id`.                        |
 | `ElevenLabsConnector`    | WebSocket  | ElevenLabs agents, addressed by `agent_id`.                          |
 | `LiveKitConnector`       | WebRTC     | Agents dispatched into LiveKit rooms.                                |
+| `PipecatConnector`       | WebSocket  | Self-hosted Pipecat pipelines, addressed by URL.                     |
 | `WebSocketConnector`     | WebSocket  | Custom agents that speak raw audio over a WebSocket.                 |
 | `CallbackVoiceConnector` | In-process | A Python callable, no network — handy for testing your own pipeline. |
 
@@ -25,133 +27,12 @@ See [Voice concepts](/docs/evaluation-voice#transports-and-connectors) for where
 Provider not listed? If your agent exposes a raw-audio WebSocket, `WebSocketConnector` needs only the URL and the message shape. If it's callable in the same Python process, use `CallbackVoiceConnector`. Subclass `BaseVoiceConnector` only for a transport the others can't express.
 ::::
 
-## ElevenLabs
-
-Your agent lives in the ElevenLabs dashboard, so the connector only needs its `agent_id`. It speaks ElevenLabs' WebSocket API directly, so the `elevenlabs` package isn't required. Private agents also need an API key.
-
-```python
-from deepeval.voice import ElevenLabsConnector
-
-connector = ElevenLabsConnector(agent_id="your-agent-id")
-```
-
-There are **ONE** mandatory and **SEVEN** optional parameters when creating an `ElevenLabsConnector`:
-
-- `agent_id`: a string identifying the ElevenLabs conversational agent.
-- [Optional] `api_key`: a string API key. When omitted, reads `ELEVENLABS_API_KEY` from the environment. Defaulted to `None`.
-- [Optional] `region`: a string naming an ElevenLabs [data residency](https://elevenlabs.io/docs/overview/administration/data-residency) region — one of `"us"`, `"eu"`, `"in"`, or `"sg"`. Any other value raises. Defaulted to `None` (global host).
-- [Optional] `client_tools`: a dictionary mapping [client tool](https://elevenlabs.io/docs/eleven-agents/customization/tools/client-tools) names to the functions that run them. Defaulted to `None`.
-- [Optional] `dynamic_variables`: a dictionary of [dynamic variables](https://elevenlabs.io/docs/eleven-agents/customization/personalization/dynamic-variables) for this conversation. Defaulted to `None`.
-- [Optional] `conversation_config_override`: a dictionary of [overrides](https://elevenlabs.io/docs/eleven-agents/customization/personalization/overrides) for this conversation, such as prompt, first message, language, or voice. Defaulted to `None`.
-- [Optional] `client_tool_timeout_s`: how long to wait for one of your tool functions before answering the agent with an error. Defaulted to `30.0`.
-- [Optional] `turn_detection`: how long to wait before deciding your agent has finished speaking — `"eager"`, `"balanced"`, or `"patient"`. See [Turn Detection](#turn-detection). Defaulted to `"balanced"`.
-
-The connector reads ElevenLabs' [`agent_response_complete`](https://elevenlabs.io/docs/eleven-agents/api-reference/eleven-agents/websocket) message off the wire, so it knows exactly when a turn ended instead of inferring it from silence.
-
-### Client tools
-
-If your agent calls client tools, give the connector a function for each one. Declare the tool in the ElevenLabs dashboard as usual; the name here has to match.
-
-```python
-async def get_customer_details(parameters):
-    return {"name": "Alice", "subscription": "Pro"}
-
-connector = ElevenLabsConnector(
-    agent_id="your-agent-id",
-    client_tools={"getCustomerDetails": get_customer_details},
-)
-```
-
-Handlers can be `async def` or plain functions — a plain one runs in a thread so it can block without stalling the call. Each receives one dictionary of the tool's parameters, plus `tool_call_id`, and whatever it returns is given to the agent as context. Returning realistic data is what keeps the rest of the conversation realistic.
-
-Your agent is waiting on that return value, so a handler is never allowed to leave it hanging. An unregistered name, a handler that raises, and one that outlives `client_tool_timeout_s` are all answered with an error result, and time spent inside your handler is excluded from the reported latency and from end-of-turn detection — otherwise a slow tool would read as a slow agent, or cut its turn short.
-
-### Personalizing a run
-
-`dynamic_variables` and `conversation_config_override` are sent per conversation, so one dashboard agent can be simulated as many different callers — a different customer, tier, or language per golden.
-
-```python
-connector = ElevenLabsConnector(
-    agent_id="your-agent-id",
-    dynamic_variables={"customer_name": "Alice", "tier": "enterprise"},
-    conversation_config_override={"agent": {"language": "fr"}},
-)
-```
-
-::::note
-Overrides only take effect if you've enabled the matching fields on the agent in ElevenLabs first. Left disabled, they're ignored rather than rejected, which looks like the override silently doing nothing.
-::::
-
-## LiveKit
-
-`deepeval` joins your agent's room as another participant and trades audio over WebRTC, so the agent needs to be deployed and able to join. Install LiveKit's SDKs:
-
-```bash
-pip install livekit livekit-api
-```
-
-A LiveKit agent has no id you can dial — it's a worker [dispatched](https://docs.livekit.io/agents/server/agent-dispatch/) into a [room](https://docs.livekit.io/intro/basics/connect/). So instead of naming the agent, you tell the connector how to reach the room. Give it credentials and it mints a token and creates one:
-
-```python
-from deepeval.voice import LiveKitConnector
-
-connector = LiveKitConnector(
-    url="wss://your-project.livekit.cloud",
-    api_key="your-api-key",
-    api_secret="your-api-secret",
-    agent_name="restaurant-agent",
-)
-```
-
-There are **ZERO** mandatory and **ELEVEN** optional parameters when creating a `LiveKitConnector`. None is mandatory on its own, but the connector needs some way to reach the room — either the three connection values, or a `token`, or a connected `room`:
-
-- [Optional] `url`: the LiveKit server URL. Required unless `LIVEKIT_URL` is set, or `room` is already connected. Defaulted to `None`.
-- [Optional] `api_key`: the LiveKit API key. Required to mint a token, unless `LIVEKIT_API_KEY` is set. Defaulted to `None`.
-- [Optional] `api_secret`: the LiveKit API secret. Required to mint a token, unless `LIVEKIT_API_SECRET` is set. Defaulted to `None`.
-- [Optional] `room`: an [`rtc.Room`](https://docs.livekit.io/reference/python/livekit/rtc/room.html) to use instead of creating one. Connected or not. Defaulted to `None`.
-- [Optional] `token`: a LiveKit [access token](https://docs.livekit.io/frontends/reference/tokens-grants/) you minted yourself, used instead of `api_key` / `api_secret`. Defaulted to `None`.
-- [Optional] `room_name`: the room to join. When omitted, a room is created for the session. Defaulted to `None`.
-- [Optional] `identity`: the participant identity for the simulator. Defaulted to `"deepeval-test"`.
-- [Optional] `agent_name`: the name of the agent to [dispatch](https://docs.livekit.io/agents/server/agent-dispatch/) into the room. Defaulted to `None`.
-- [Optional] `turn_detection`: how long to wait before deciding your agent has finished speaking — `"eager"`, `"balanced"`, or `"patient"`. See [Turn Detection](#turn-detection). Defaulted to `"balanced"`.
-- [Optional] `connect_timeout_s`: timeout (seconds) for establishing the room connection. Defaulted to `15.0`.
-- [Optional] `transcript_grace_s`: how long to wait after your agent stops speaking for its transcript to arrive. Defaulted to `1.5`.
-
-`room_name` and `agent_name` are what actually find your agent. Omit `room_name` and the connector creates a room. Set `agent_name` and LiveKit dispatches that agent into it; leave it unset and your existing routing has to put the agent there.
-
-If your test setup already joined a room, hand that over instead and skip the credentials:
-
-```python
-from livekit import rtc
-from deepeval.voice import LiveKitConnector
-
-room = rtc.Room()
-await room.connect("wss://your-project.livekit.cloud", your_token)
-
-connector = LiveKitConnector(room=room)
-```
-
-`deepeval` publishes a microphone track, listens for the agent, and leaves the room as it found it — a room you connected stays connected afterwards, with just the simulated microphone removed. An unconnected room gets connected using the parameters above.
-
-::::note
-`deepeval` reads your agent's own transcript from LiveKit's [`lk.transcription`](https://docs.livekit.io/agents/multimodality/text/) text stream, so STT is usually unnecessary here — the words the agent actually spoke become the assistant `Turn.content`. STT is the fallback when your agent publishes no transcript.
-::::
+<include cwd>snippets/voice/connector-tabs.mdx</include>
 
 ## Generic WebSocket
 
 For custom agents that accept and emit raw audio over a WebSocket, `WebSocketConnector` lets you describe your agent's message shape — which JSON keys carry audio in and out, whether frames are binary, and what marks the end of a turn.
 
-```python
-from deepeval.voice import WebSocketConnector
-
-connector = WebSocketConnector(
-    url="wss://your-agent.example.com/audio",
-    send_key="audio",
-    receive_audio_key="audio",
-    turn_complete_type="turn_end",
-)
-```
-
 There are **ONE** mandatory and **TWELVE** optional parameters when creating a `WebSocketConnector`:
 
 - `url`: a string WebSocket URL for your agent (e.g. `"wss://your-agent.example.com/audio"`).
@@ -172,12 +53,6 @@ There are **ONE** mandatory and **TWELVE** optional parameters when creating a `
 
 Wraps an in-process Python callable, so you can exercise the full TTS → agent → STT pipeline without any network transport — useful for testing your simulation setup before pointing it at a deployed agent.
 
-```python
-from deepeval.voice import CallbackVoiceConnector
-
-connector = CallbackVoiceConnector(my_agent)
-```
-
 There are **ONE** mandatory and **THREE** optional parameters when creating a `CallbackVoiceConnector`:
 
 - `agent`: a sync or async callable that accepts user `Audio` and returns a `ConnectorTurn` or plain `Audio`.
@@ -247,9 +122,9 @@ connector = CallbackVoiceConnector(my_agent, turn_detection="patient")
 
 Pick by listening to your agent's longest natural pause. Too eager and its turn ends at that pause, cutting the reply short and discarding the rest of it; too patient and the simulator sits through dead air before responding, which inflates the gaps in the recording and slows the run.
 
-When your transport closes each turn explicitly, silence is only the fallback: `ElevenLabsConnector`, `CallbackVoiceConnector`, and a `WebSocketConnector` given a `turn_complete_type` are all believed over a pause, so the reply is heard out in full however long your agent stops for. Only the ceiling still applies.
+When your transport closes each turn explicitly, silence is only the fallback: `ElevenLabsConnector`, `VapiConnector`, `CallbackVoiceConnector`, and a `WebSocketConnector` given a `turn_complete_type` are all believed over a pause, so the reply is heard out in full however long your agent stops for. Only the ceiling still applies.
 
-The preset matters more everywhere else, because silence is the only evidence there is. `LiveKitConnector` gets a bare audio track with nothing to say when a turn is over, and so does a `WebSocketConnector` whose agent sends no end-of-turn message to point at.
+The preset matters more everywhere else, because silence is the only evidence there is. `LiveKitConnector` gets a bare audio track with nothing to say when a turn is over, and so does a `WebSocketConnector` whose agent sends no end-of-turn message to point at. `PipecatConnector` falls on either side depending on your pipeline: it's believed over a pause once your pipeline has been seen to announce an end of turn, and reads silence until then.
 
 ::::note
 The underlying millisecond values are chosen as a coordinated preset rather than exposed individually, for the same reason [interruption timing](/docs/conversation-simulator-voice-interruptions#when-does-it-interrupt) is: they are interdependent, and picking them apart is guesswork without knowing how the turn-taking loop uses them.

docs/content/docs/environment-variables.mdx

@@ -437,4 +437,29 @@ Azure reuses `OPENAI_COST_PER_INPUT_TOKEN` and `OPENAI_COST_PER_OUTPUT_TOKEN` fo
 
 </details>
 
+<details>
+<summary>Speech models (TTS/STT)</summary>
+
+Text-to-speech and speech-to-text are two independent families used by [voice simulation](/docs/multi-turn-conversation-simulation). One `USE_*_TTS` flag and one `USE_*_STT` flag can be on at the same time, and each family falls back to OpenAI when no flag is set. Set them with [`deepeval set-tts`](/docs/command-line-interface#model-provider-configs) and `deepeval set-stt`.
+
+| Variable               | Values            | Effect                                                                                     |
+| ---------------------- | ----------------- | -------------------------------------------------------------------------------------------- |
+| `USE_OPENAI_TTS`       | `1` / `0` / unset | Prefer OpenAI for text-to-speech.                                                          |
+| `USE_ELEVENLABS_TTS`   | `1` / `0` / unset | Prefer ElevenLabs for text-to-speech.                                                      |
+| `USE_CARTESIA_TTS`     | `1` / `0` / unset | Prefer Cartesia for text-to-speech.                                                        |
+| `USE_DEEPGRAM_TTS`     | `1` / `0` / unset | Prefer Deepgram for text-to-speech.                                                        |
+| `DEEPEVAL_TTS_MODEL`   | `string` / unset  | Model name for the selected TTS provider (defaults to that provider's own default model).  |
+| `USE_OPENAI_STT`       | `1` / `0` / unset | Prefer OpenAI for speech-to-text.                                                          |
+| `USE_ELEVENLABS_STT`   | `1` / `0` / unset | Prefer ElevenLabs for speech-to-text.                                                      |
+| `USE_CARTESIA_STT`     | `1` / `0` / unset | Prefer Cartesia for speech-to-text.                                                        |
+| `USE_DEEPGRAM_STT`     | `1` / `0` / unset | Prefer Deepgram for speech-to-text.                                                        |
+| `USE_ASSEMBLYAI_STT`   | `1` / `0` / unset | Prefer AssemblyAI for speech-to-text.                                                      |
+| `DEEPEVAL_STT_MODEL`   | `string` / unset  | Model name for the selected STT provider (defaults to that provider's own default model).  |
+| `ELEVENLABS_API_KEY`   | `string` / unset  | ElevenLabs API key.                                                                        |
+| `CARTESIA_API_KEY`     | `string` / unset  | Cartesia API key.                                                                          |
+| `DEEPGRAM_API_KEY`     | `string` / unset  | Deepgram API key.                                                                          |
+| `ASSEMBLYAI_API_KEY`   | `string` / unset  | AssemblyAI API key.                                                                        |
+
+</details>
+
 </Only>

docs/content/integrations/index.mdx

@@ -238,6 +238,70 @@ Evaluation model integrations configure the LLM provider DeepEval uses for LLM-a
 
 <Only id="python">
 
+## Speech Models
+
+Speech model integrations configure the text-to-speech and speech-to-text providers DeepEval uses for [voice simulation](/docs/conversation-simulator-voice-mode). Every provider apart from OpenAI is reached over plain HTTP, so none of them need a package beyond `deepeval` itself.
+
+### Speech-To-Text (STT)
+
+STT transcribes your agent's spoken replies into the `Turn.content` your metrics judge, so its accuracy bounds how faithfully every multi-turn metric sees the conversation.
+
+<Cards className="md:grid-cols-3">
+  <Card title="OpenAI" href="/integrations/stt/openai" />
+  <Card title="Deepgram" href="/integrations/stt/deepgram" />
+  <Card title="AssemblyAI" href="/integrations/stt/assemblyai" />
+  <Card title="ElevenLabs" href="/integrations/stt/elevenlabs" />
+  <Card title="Cartesia" href="/integrations/stt/cartesia" />
+</Cards>
+
+### Text-To-Speech (TTS)
+
+TTS speaks the simulated user's messages to your agent, so its clarity is what your agent's own speech recognition has to work with.
+
+<Cards className="md:grid-cols-3">
+  <Card title="OpenAI" href="/integrations/tts/openai" />
+  <Card title="ElevenLabs" href="/integrations/tts/elevenlabs" />
+  <Card title="Cartesia" href="/integrations/tts/cartesia" />
+  <Card title="Deepgram" href="/integrations/tts/deepgram" />
+</Cards>
+
+</Only>
+
+<Only id="python">
+
+## Voice Agent Connectors
+
+Voice agent connectors are how DeepEval reaches the agent under test during a [voice simulation](/docs/conversation-simulator-voice-mode): they open a live session, play the simulated user's audio, and capture the spoken reply. Pick the one matching where your agent runs.
+
+<Cards className="md:grid-cols-3">
+  <Card
+    title="Vapi"
+    href="/integrations/voice-agent-connectors/vapi"
+    description="Call a Vapi assistant over their WebSocket transport."
+  />
+  <Card
+    title="ElevenLabs"
+    href="/integrations/voice-agent-connectors/elevenlabs"
+    description="Simulate against a conversational agent in your ElevenLabs dashboard."
+  />
+  <Card
+    title="LiveKit"
+    href="/integrations/voice-agent-connectors/livekit"
+    description="Join your agent's LiveKit room and trade audio over WebRTC."
+  />
+  <Card
+    title="Pipecat"
+    href="/integrations/voice-agent-connectors/pipecat"
+    description="Connect to your self-hosted Pipecat pipeline over its WebSocket transport."
+  />
+</Cards>
+
+Custom agents are covered by `WebSocketConnector` and `CallbackVoiceConnector` — see [Voice Connectors](/docs/conversation-simulator-voice-connectors).
+
+</Only>
+
+<Only id="python">
+
 ## Vector DBs
 
 Vector database integrations show how to connect retrieval systems to DeepEval so RAG metrics can evaluate the context your application actually retrieves. Use these examples to benchmark retrieval quality and end-to-end RAG behavior.

docs/content/integrations/meta.json

@@ -4,6 +4,9 @@
     "index",
     "frameworks",
     "models",
+    "stt",
+    "tts",
+    "voice-agent-connectors",
     "vector-databases",
     "others"
   ]

docs/content/integrations/models/amazon-bedrock.mdx

@@ -67,24 +67,6 @@ model = AmazonBedrockModel(
 answer_relevancy = AnswerRelevancyMetric(model=model)
 ```
 
-</Tab>
-<Tab value="ENV">
-
-To use any Amazon Bedrock model directly in `deepeval`, set the `USE_AWS_BEDROCK_MODEL=1` in your `env` and simply pass the name of your desired model in your metric initialization:
-
-```python
-from deepeval.metrics import AnswerRelevancyMetric
-
-answer_relevancy = AnswerRelevancyMetric(
-    model="anthropic.claude-sonnet-4-5-20250929-v1:0",
-)
-```
-
-You should also set the other necessary vars like `AWS_ACCESS_KEY_ID`, `AWS_SESSION_TOKEN`, ..etc. to be able to use the Amazon Bedrock models as shown above.
-
-</Tab>
-</Tabs>
-
 There are **ZERO** mandatory and **SEVEN** optional parameters when creating an `AmazonBedrockModel`:
 
 - [Optional] `model`: A string specifying the bedrock model identifier to call (e.g. `anthropic.claude-sonnet-4-5-20250929-v1:0`). Defaults to `AWS_BEDROCK_MODEL_NAME` if not passed; raises an error at runtime if unset.
@@ -103,6 +85,24 @@ Pass generation parameters like `temperature`, `topP`, or `maxTokens` via `gener
 Extra `**kwargs` passed to `AmazonBedrockModel(...)` are forwarded to the underlying Bedrock client (aiobotocore/botocore) and are **not** treated as generation parameters.
 :::
 
+</Tab>
+<Tab value="ENV">
+
+To use any Amazon Bedrock model directly in `deepeval`, set the `USE_AWS_BEDROCK_MODEL=1` in your `env` and simply pass the name of your desired model in your metric initialization:
+
+```python
+from deepeval.metrics import AnswerRelevancyMetric
+
+answer_relevancy = AnswerRelevancyMetric(
+    model="anthropic.claude-sonnet-4-5-20250929-v1:0",
+)
+```
+
+You should also set the other necessary vars like `AWS_ACCESS_KEY_ID`, `AWS_SESSION_TOKEN`, ..etc. to be able to use the Amazon Bedrock models as shown above.
+
+</Tab>
+</Tabs>
+
 </Case>
 
 <Case id="typescript">
@@ -124,23 +124,6 @@ const model = new AmazonBedrockModel({
 const answerRelevancy = new AnswerRelevancyMetric({ model });
 ```
 
-</Tab>
-<Tab value="ENV">
-
-Every option is backed by an environment variable, so you can set `AWS_BEDROCK_MODEL_NAME`, `AWS_BEDROCK_REGION`, `AWS_ACCESS_KEY_ID`, and `AWS_SECRET_ACCESS_KEY` in your `env` and construct the model with no arguments:
-
-```typescript
-import { AnswerRelevancyMetric } from "deepeval/metrics";
-import { AmazonBedrockModel } from "deepeval/models";
-
-const answerRelevancy = new AnswerRelevancyMetric({
-  model: new AmazonBedrockModel(),
-});
-```
-
-</Tab>
-</Tabs>
-
 There are **ZERO** mandatory and **EIGHT** optional parameters when creating an `AmazonBedrockModel`:
 
 - [Optional] `model`: A string specifying the bedrock model identifier to call (e.g. `anthropic.claude-sonnet-4-5-20250929-v1:0`). Defaults to `AWS_BEDROCK_MODEL_NAME` if not passed; raises an error at runtime if unset.
@@ -164,6 +147,23 @@ const model = new AmazonBedrockModel({
 
 Available keys vary by the Bedrock model you choose — see the [AWS Bedrock inference parameters docs](https://docs.aws.amazon.com/bedrock/latest/userguide/inference-parameters.html).
 
+</Tab>
+<Tab value="ENV">
+
+Every option is backed by an environment variable, so you can set `AWS_BEDROCK_MODEL_NAME`, `AWS_BEDROCK_REGION`, `AWS_ACCESS_KEY_ID`, and `AWS_SECRET_ACCESS_KEY` in your `env` and construct the model with no arguments:
+
+```typescript
+import { AnswerRelevancyMetric } from "deepeval/metrics";
+import { AmazonBedrockModel } from "deepeval/models";
+
+const answerRelevancy = new AnswerRelevancyMetric({
+  model: new AmazonBedrockModel(),
+});
+```
+
+</Tab>
+</Tabs>
+
 </Case>
 
 </Switch>

docs/content/integrations/models/anthropic.mdx

@@ -44,6 +44,23 @@ model = AnthropicModel(
 answer_relevancy = AnswerRelevancyMetric(model=model)
 ```
 
+There are **ZERO** mandatory and **SIX** optional parameters when creating an `AnthropicModel`:
+
+- [Optional] `model`: A string specifying which Claude model to use. Defaults to `ANTHROPIC_MODEL_NAME` if not passed; falls back to <DefaultLLMModel provider="anthropic" /> if unset.
+- [Optional] `api_key`: A string specifying your Anthropic API key. Defaults to `ANTHROPIC_API_KEY` if not passed; raises an error at runtime if unset.
+- [Optional] `temperature`: A float specifying the model temperature. Defaults to `TEMPERATURE` if not passed; falls back to `0.0` if unset and raises if < 0.
+- [Optional] `cost_per_input_token`: A float specifying the cost for each input token for the provided model. Defaults to `ANTHROPIC_COST_PER_INPUT_TOKEN` if available in `deepeval`'s model cost registry, else `None`.
+- [Optional] `cost_per_output_token`: A float specifying the cost for each output token for the provided model. Defaults to `ANTHROPIC_COST_PER_OUTPUT_TOKEN` if available in `deepeval`'s model cost registry, else `None`.
+- [Optional] `generation_kwargs`: A dictionary of additional generation parameters forwarded to the Anthropic `messages.create(...)` call.
+
+Parameters may be explicitly passed to the model at initialization time, or configured with optional settings. The **mandatory** parameters are required at runtime, but you can provide them either explicitly as constructor arguments, **or** via `deepeval` settings / environment variables (constructor args take precedence). See [Environment variables and settings](/docs/evaluation-flags-and-configs#model-settings-anthropic) for the Anthropic-related environment variables.
+
+:::tip
+Pass generation parameters, such as `max_tokens`, via `generation_kwargs` (they are forwarded to `messages.create(...)`).
+
+Extra `**kwargs` passed to `AnthropicModel(...)` are forwarded to the underlying Anthropic client and are **not** treated as generation parameters.
+:::
+
 </Tab>
 <Tab value="ENV">
 
@@ -62,23 +79,6 @@ You should also set the other necessary vars like `ANTHROPIC_API_KEY` to be able
 </Tab>
 </Tabs>
 
-There are **ZERO** mandatory and **SIX** optional parameters when creating an `AnthropicModel`:
-
-- [Optional] `model`: A string specifying which Claude model to use. Defaults to `ANTHROPIC_MODEL_NAME` if not passed; falls back to <DefaultLLMModel provider="anthropic" /> if unset.
-- [Optional] `api_key`: A string specifying your Anthropic API key. Defaults to `ANTHROPIC_API_KEY` if not passed; raises an error at runtime if unset.
-- [Optional] `temperature`: A float specifying the model temperature. Defaults to `TEMPERATURE` if not passed; falls back to `0.0` if unset and raises if < 0.
-- [Optional] `cost_per_input_token`: A float specifying the cost for each input token for the provided model. Defaults to `ANTHROPIC_COST_PER_INPUT_TOKEN` if available in `deepeval`'s model cost registry, else `None`.
-- [Optional] `cost_per_output_token`: A float specifying the cost for each output token for the provided model. Defaults to `ANTHROPIC_COST_PER_OUTPUT_TOKEN` if available in `deepeval`'s model cost registry, else `None`.
-- [Optional] `generation_kwargs`: A dictionary of additional generation parameters forwarded to the Anthropic `messages.create(...)` call.
-
-Parameters may be explicitly passed to the model at initialization time, or configured with optional settings. The **mandatory** parameters are required at runtime, but you can provide them either explicitly as constructor arguments, **or** via `deepeval` settings / environment variables (constructor args take precedence). See [Environment variables and settings](/docs/evaluation-flags-and-configs#model-settings-anthropic) for the Anthropic-related environment variables.
-
-:::tip
-Pass generation parameters, such as `max_tokens`, via `generation_kwargs` (they are forwarded to `messages.create(...)`).
-
-Extra `**kwargs` passed to `AnthropicModel(...)` are forwarded to the underlying Anthropic client and are **not** treated as generation parameters.
-:::
-
 </Case>
 
 <Case id="typescript">
@@ -99,23 +99,6 @@ const model = new AnthropicModel({
 const answerRelevancy = new AnswerRelevancyMetric({ model });
 ```
 
-</Tab>
-<Tab value="ENV">
-
-Every option is backed by an environment variable, so you can set `ANTHROPIC_MODEL_NAME` and `ANTHROPIC_API_KEY` in your `env` and construct the model with no arguments:
-
-```typescript
-import { AnswerRelevancyMetric } from "deepeval/metrics";
-import { AnthropicModel } from "deepeval/models";
-
-const answerRelevancy = new AnswerRelevancyMetric({
-  model: new AnthropicModel(),
-});
-```
-
-</Tab>
-</Tabs>
-
 There are **ZERO** mandatory and **SIX** optional parameters when creating an `AnthropicModel`:
 
 - [Optional] `model`: A string specifying which Claude model to use. Defaults to `ANTHROPIC_MODEL_NAME` if not passed; falls back to <DefaultLLMModel provider="anthropic" /> if unset.
@@ -135,6 +118,23 @@ const model = new AnthropicModel({
 });
 ```
 
+</Tab>
+<Tab value="ENV">
+
+Every option is backed by an environment variable, so you can set `ANTHROPIC_MODEL_NAME` and `ANTHROPIC_API_KEY` in your `env` and construct the model with no arguments:
+
+```typescript
+import { AnswerRelevancyMetric } from "deepeval/metrics";
+import { AnthropicModel } from "deepeval/models";
+
+const answerRelevancy = new AnswerRelevancyMetric({
+  model: new AnthropicModel(),
+});
+```
+
+</Tab>
+</Tabs>
+
 </Case>
 
 </Switch>

docs/content/integrations/models/azure-openai.mdx

@@ -64,6 +64,22 @@ model = AzureOpenAIModel(
 answer_relevancy = AnswerRelevancyMetric(model=model)
 ```
 
+There are **ZERO** mandatory and **NINE** optional parameters when creating an `AzureOpenAIModel`:
+
+- [Optional] `model`: A string specifying the name of the Azure OpenAI model to use. Defaults to `AZURE_MODEL_NAME` if not passed; raises an error at runtime if unset.
+- [Optional] `api_key`: A string specifying your Azure OpenAI API key. Defaults to `AZURE_OPENAI_API_KEY` if not passed; raises an error at runtime if `azure_ad_token` and `azure_ad_token_provider` are also unset.
+- [Optional] `azure_ad_token`: A string specifying your Azure Ad Token. Defaults to `AZURE_OPENAI_AD_TOKEN` if not passed; raises an error at runtime if `api_key` and `azure_ad_token_provider` are also unset.
+- [Optional] `azure_ad_token_provider`: A callback of either `AsyncAzureADTokenProvider` or `AzureADTokenProvider` that can be used for credentials [(see example usage)](https://github.com/openai/openai-python/blob/main/examples/azure_ad.py#L20). Raises an error at runtime if `api_key` and `azure_ad_token` are also unset.
+- [Optional] `base_url`: A string specifying your Azure OpenAI endpoint URL. Defaults to `AZURE_OPENAI_ENDPOINT` if not passed; raises an error at runtime if unset.
+- [Optional] `temperature`: A float specifying the model temperature. Defaults to `TEMPERATURE` if not passed; falls back to `0.0` if unset.
+- [Optional] `cost_per_input_token`: A float specifying the cost for each input token for the provided model. Defaults to `OPENAI_COST_PER_INPUT_TOKEN` if available in `deepeval`'s model cost registry, else `None`.
+- [Optional] `cost_per_output_token`: A float specifying the cost for each output token for the provided model. Defaults to `OPENAI_COST_PER_OUTPUT_TOKEN` if available in `deepeval`'s model cost registry, else `None`.
+- [Optional] `deployment_name`: A string specifying the name of your Azure OpenAI deployment. Defaults to `AZURE_DEPLOYMENT_NAME` if not passed; raises an error at runtime if unset.
+- [Optional] `api_version`: A string specifying the OpenAI API version used in your deployment. Defaults to `OPENAI_API_VERSION` if not passed; raises an error at runtime if unset.
+- [Optional] `generation_kwargs`: A dictionary of additional generation parameters forwarded to the Azure OpenAI `chat.completions.create(...)` and `beta.chat.completions.parse(...)` calls.
+
+Parameters may be explicitly passed to the model at initialization time, or configured with optional settings. The **mandatory** parameters are required at runtime, but you can provide them either explicitly as constructor arguments, **or** via `deepeval` settings / environment variables (constructor args take precedence). See [Environment variables and settings](/docs/evaluation-flags-and-configs#model-settings-azure-openai) for the Azure OpenAI-related environment variables.
+
 </Tab>
 <Tab value="ENV">
 
@@ -82,22 +98,6 @@ You should also set the other necessary vars like `AZURE_OPENAI_API_KEY` to be a
 </Tab>
 </Tabs>
 
-There are **ZERO** mandatory and **NINE** optional parameters when creating an `AzureOpenAIModel`:
-
-- [Optional] `model`: A string specifying the name of the Azure OpenAI model to use. Defaults to `AZURE_MODEL_NAME` if not passed; raises an error at runtime if unset.
-- [Optional] `api_key`: A string specifying your Azure OpenAI API key. Defaults to `AZURE_OPENAI_API_KEY` if not passed; raises an error at runtime if `azure_ad_token` and `azure_ad_token_provider` are also unset.
-- [Optional] `azure_ad_token`: A string specifying your Azure Ad Token. Defaults to `AZURE_OPENAI_AD_TOKEN` if not passed; raises an error at runtime if `api_key` and `azure_ad_token_provider` are also unset.
-- [Optional] `azure_ad_token_provider`: A callback of either `AsyncAzureADTokenProvider` or `AzureADTokenProvider` that can be used for credentials [(see example usage)](https://github.com/openai/openai-python/blob/main/examples/azure_ad.py#L20). Raises an error at runtime if `api_key` and `azure_ad_token` are also unset.
-- [Optional] `base_url`: A string specifying your Azure OpenAI endpoint URL. Defaults to `AZURE_OPENAI_ENDPOINT` if not passed; raises an error at runtime if unset.
-- [Optional] `temperature`: A float specifying the model temperature. Defaults to `TEMPERATURE` if not passed; falls back to `0.0` if unset.
-- [Optional] `cost_per_input_token`: A float specifying the cost for each input token for the provided model. Defaults to `OPENAI_COST_PER_INPUT_TOKEN` if available in `deepeval`'s model cost registry, else `None`.
-- [Optional] `cost_per_output_token`: A float specifying the cost for each output token for the provided model. Defaults to `OPENAI_COST_PER_OUTPUT_TOKEN` if available in `deepeval`'s model cost registry, else `None`.
-- [Optional] `deployment_name`: A string specifying the name of your Azure OpenAI deployment. Defaults to `AZURE_DEPLOYMENT_NAME` if not passed; raises an error at runtime if unset.
-- [Optional] `api_version`: A string specifying the OpenAI API version used in your deployment. Defaults to `OPENAI_API_VERSION` if not passed; raises an error at runtime if unset.
-- [Optional] `generation_kwargs`: A dictionary of additional generation parameters forwarded to the Azure OpenAI `chat.completions.create(...)` and `beta.chat.completions.parse(...)` calls.
-
-Parameters may be explicitly passed to the model at initialization time, or configured with optional settings. The **mandatory** parameters are required at runtime, but you can provide them either explicitly as constructor arguments, **or** via `deepeval` settings / environment variables (constructor args take precedence). See [Environment variables and settings](/docs/evaluation-flags-and-configs#model-settings-azure-openai) for the Azure OpenAI-related environment variables.
-
 </Case>
 
 <Case id="typescript">
@@ -121,6 +121,27 @@ const model = new AzureOpenAIModel({
 const answerRelevancy = new AnswerRelevancyMetric({ model });
 ```
 
+There are **ZERO** mandatory and **EIGHT** optional parameters when creating an `AzureOpenAIModel`:
+
+- [Optional] `model`: A string specifying the name of the Azure OpenAI model your deployment serves. Used to look up pricing and capabilities in `deepeval`'s model registry, and used as the deployment name when `deployment` is not passed.
+- [Optional] `deployment`: A string specifying the name of your Azure OpenAI deployment, which requests are routed by. Defaults to `model`, then `AZURE_DEPLOYMENT_NAME`, then `AZURE_MODEL_NAME` if not passed; raises an error if unset.
+- [Optional] `endpoint`: A string specifying your Azure OpenAI endpoint URL. Defaults to `AZURE_OPENAI_ENDPOINT` if not passed; raises an error if unset.
+- [Optional] `apiKey`: A string specifying your Azure OpenAI API key. Defaults to `AZURE_OPENAI_API_KEY` if not passed; raises an error at runtime if unset.
+- [Optional] `apiVersion`: A string specifying the OpenAI API version used in your deployment. Defaults to `OPENAI_API_VERSION` if not passed.
+- [Optional] `temperature`: A number specifying the model temperature. Defaults to `0`. Pass `null` to omit it from the request entirely, which is required by reasoning models that reject it.
+- [Optional] `costPerInputToken`: A number specifying the cost for each input token for the provided model. Defaults to the price in `deepeval`'s model cost registry, else `null`.
+- [Optional] `costPerOutputToken`: A number specifying the cost for each output token for the provided model. Defaults to the price in `deepeval`'s model cost registry, else `null`.
+
+Any other key you pass is forwarded as-is to the Azure OpenAI `chat.completions.create(...)` call, so generation parameters `deepeval` does not name go inline alongside the rest:
+
+```typescript
+const model = new AzureOpenAIModel({
+  model: "gpt-4.1",
+  top_p: 0.9,
+  seed: 42,
+});
+```
+
 </Tab>
 <Tab value="ENV">
 
@@ -149,27 +170,6 @@ const answerRelevancy = new AnswerRelevancyMetric({
 </Tab>
 </Tabs>
 
-There are **ZERO** mandatory and **EIGHT** optional parameters when creating an `AzureOpenAIModel`:
-
-- [Optional] `model`: A string specifying the name of the Azure OpenAI model your deployment serves. Used to look up pricing and capabilities in `deepeval`'s model registry, and used as the deployment name when `deployment` is not passed.
-- [Optional] `deployment`: A string specifying the name of your Azure OpenAI deployment, which requests are routed by. Defaults to `model`, then `AZURE_DEPLOYMENT_NAME`, then `AZURE_MODEL_NAME` if not passed; raises an error if unset.
-- [Optional] `endpoint`: A string specifying your Azure OpenAI endpoint URL. Defaults to `AZURE_OPENAI_ENDPOINT` if not passed; raises an error if unset.
-- [Optional] `apiKey`: A string specifying your Azure OpenAI API key. Defaults to `AZURE_OPENAI_API_KEY` if not passed; raises an error at runtime if unset.
-- [Optional] `apiVersion`: A string specifying the OpenAI API version used in your deployment. Defaults to `OPENAI_API_VERSION` if not passed.
-- [Optional] `temperature`: A number specifying the model temperature. Defaults to `0`. Pass `null` to omit it from the request entirely, which is required by reasoning models that reject it.
-- [Optional] `costPerInputToken`: A number specifying the cost for each input token for the provided model. Defaults to the price in `deepeval`'s model cost registry, else `null`.
-- [Optional] `costPerOutputToken`: A number specifying the cost for each output token for the provided model. Defaults to the price in `deepeval`'s model cost registry, else `null`.
-
-Any other key you pass is forwarded as-is to the Azure OpenAI `chat.completions.create(...)` call, so generation parameters `deepeval` does not name go inline alongside the rest:
-
-```typescript
-const model = new AzureOpenAIModel({
-  model: "gpt-4.1",
-  top_p: 0.9,
-  seed: 42,
-});
-```
-
 </Case>
 
 </Switch>

docs/content/integrations/models/deepseek.mdx

@@ -51,6 +51,17 @@ model = DeepSeekModel(
 answer_relevancy = AnswerRelevancyMetric(model=model)
 ```
 
+There are **ZERO** mandatory and **SIX** optional parameters when creating a `DeepSeekModel`:
+
+- [Optional] `model`: A string specifying the name of the DeepSeek model to use. Defaults to `DEEPSEEK_MODEL_NAME` if not passed; raises an error at runtime if unset.
+- [Optional] `api_key`: A string specifying your DeepSeek API key for authentication. Defaults to `DEEPSEEK_API_KEY` if not passed; raises an error at runtime if unset.
+- [Optional] `temperature`: A float specifying the model temperature. Defaults to `TEMPERATURE` if not passed; falls back to `0.0` if unset.
+- [Optional] `cost_per_input_token`: A float specifying the cost for each input token for the provided model. Defaults to `DEEPSEEK_COST_PER_INPUT_TOKEN` if available in `deepeval`'s model cost registry, else `None`.
+- [Optional] `cost_per_output_token`: A float specifying the cost for each output token for the provided model. Defaults to `DEEPSEEK_COST_PER_OUTPUT_TOKEN` if available in `deepeval`'s model cost registry, else `None`.
+- [Optional] `generation_kwargs`: A dictionary of additional generation forwarded to the OpenAI `chat.completions.create(...)` call.
+
+Parameters may be explicitly passed to the model at initialization time, or configured with optional settings. The **mandatory** parameters are required at runtime, but you can provide them either explicitly as constructor arguments, **or** via `deepeval` settings / environment variables (constructor args take precedence). See [Environment variables and settings](/docs/evaluation-flags-and-configs#model-settings-deep-seek) for the DeepSeek-related environment variables.
+
 </Tab>
 <Tab value="ENV">
 
@@ -69,17 +80,6 @@ You should also set the other necessary vars like `DEEPSEEK_API_KEY` to be able
 </Tab>
 </Tabs>
 
-There are **ZERO** mandatory and **SIX** optional parameters when creating a `DeepSeekModel`:
-
-- [Optional] `model`: A string specifying the name of the DeepSeek model to use. Defaults to `DEEPSEEK_MODEL_NAME` if not passed; raises an error at runtime if unset.
-- [Optional] `api_key`: A string specifying your DeepSeek API key for authentication. Defaults to `DEEPSEEK_API_KEY` if not passed; raises an error at runtime if unset.
-- [Optional] `temperature`: A float specifying the model temperature. Defaults to `TEMPERATURE` if not passed; falls back to `0.0` if unset.
-- [Optional] `cost_per_input_token`: A float specifying the cost for each input token for the provided model. Defaults to `DEEPSEEK_COST_PER_INPUT_TOKEN` if available in `deepeval`'s model cost registry, else `None`.
-- [Optional] `cost_per_output_token`: A float specifying the cost for each output token for the provided model. Defaults to `DEEPSEEK_COST_PER_OUTPUT_TOKEN` if available in `deepeval`'s model cost registry, else `None`.
-- [Optional] `generation_kwargs`: A dictionary of additional generation forwarded to the OpenAI `chat.completions.create(...)` call.
-
-Parameters may be explicitly passed to the model at initialization time, or configured with optional settings. The **mandatory** parameters are required at runtime, but you can provide them either explicitly as constructor arguments, **or** via `deepeval` settings / environment variables (constructor args take precedence). See [Environment variables and settings](/docs/evaluation-flags-and-configs#model-settings-deep-seek) for the DeepSeek-related environment variables.
-
 </Case>
 
 <Case id="typescript">
@@ -100,23 +100,6 @@ const model = new DeepSeekModel({
 const answerRelevancy = new AnswerRelevancyMetric({ model });
 ```
 
-</Tab>
-<Tab value="ENV">
-
-To use any DeepSeek model directly in `deepeval`, set `DEEPSEEK_MODEL_NAME` and `DEEPSEEK_API_KEY` in your `env` and create a `DeepSeekModel` with no arguments:
-
-```typescript
-import { AnswerRelevancyMetric } from "deepeval/metrics";
-import { DeepSeekModel } from "deepeval/models";
-
-const answerRelevancy = new AnswerRelevancyMetric({
-  model: new DeepSeekModel(),
-});
-```
-
-</Tab>
-</Tabs>
-
 There are **ZERO** mandatory and **SEVEN** optional parameters when creating a `DeepSeekModel`:
 
 - [Optional] `model`: A string specifying the name of the DeepSeek model to use. Defaults to `DEEPSEEK_MODEL_NAME` if not passed; falls back to `deepseek-chat` if unset.
@@ -137,6 +120,23 @@ const model = new DeepSeekModel({
 });
 ```
 
+</Tab>
+<Tab value="ENV">
+
+To use any DeepSeek model directly in `deepeval`, set `DEEPSEEK_MODEL_NAME` and `DEEPSEEK_API_KEY` in your `env` and create a `DeepSeekModel` with no arguments:
+
+```typescript
+import { AnswerRelevancyMetric } from "deepeval/metrics";
+import { DeepSeekModel } from "deepeval/models";
+
+const answerRelevancy = new AnswerRelevancyMetric({
+  model: new DeepSeekModel(),
+});
+```
+
+</Tab>
+</Tabs>
+
 </Case>
 
 </Switch>

docs/content/integrations/models/gemini.mdx

@@ -56,6 +56,21 @@ model = GeminiModel(
 answer_relevancy = AnswerRelevancyMetric(model=model)
 ```
 
+There are **ZERO** mandatory and **SIX** optional parameters when creating a `GeminiModel`:
+
+- [Optional] `model`: A string specifying the name of the Gemini model to use. Defaults to `GEMINI_MODEL_NAME` if not passed; raises an error at runtime if unset.
+- [Optional] `api_key`: A string specifying the Google API key for authentication. Defaults to `GOOGLE_API_KEY` if not passed; raises an error at runtime if unset.
+- [Optional] `temperature`: A float specifying the model temperature. Defaults to `TEMPERATURE` if not passed; falls back to `0.0` if unset.
+- [Optional] `cost_per_input_token`: A float specifying the cost per input token for the provided model. Defaults to `GEMINI_COST_PER_INPUT_TOKEN` if set; falls back to `deepeval`'s model cost registry, else `None`.
+- [Optional] `cost_per_output_token`: A float specifying the cost per output token for the provided model. Defaults to `GEMINI_COST_PER_OUTPUT_TOKEN` if set; falls back to `deepeval`'s model cost registry, else `None`.
+- [Optional] `generation_kwargs`: A dictionary of additional generation parameters forwarded to the Gemini API `generate_content(...)` call.
+
+Parameters may be explicitly passed to the model at initialization time, or configured with optional settings. The **mandatory** parameters are required at runtime, but you can provide them either explicitly as constructor arguments, **or** via `deepeval` settings / environment variables (constructor args take precedence). See [Environment variables and settings](/docs/evaluation-flags-and-configs#model-settings-gemini) for the Gemini-related environment variables.
+
+:::note
+At runtime, you must provide an API key (via `api_key` or `GOOGLE_API_KEY`) unless you’re using Vertex AI. See [Vertex AI](/docs/integrations/models/vertex-ai).
+:::
+
 </Tab>
 <Tab value="ENV">
 
@@ -74,21 +89,6 @@ You should also set the other necessary vars like `GOOGLE_API_KEY` to be able to
 </Tab>
 </Tabs>
 
-There are **ZERO** mandatory and **SIX** optional parameters when creating a `GeminiModel`:
-
-- [Optional] `model`: A string specifying the name of the Gemini model to use. Defaults to `GEMINI_MODEL_NAME` if not passed; raises an error at runtime if unset.
-- [Optional] `api_key`: A string specifying the Google API key for authentication. Defaults to `GOOGLE_API_KEY` if not passed; raises an error at runtime if unset.
-- [Optional] `temperature`: A float specifying the model temperature. Defaults to `TEMPERATURE` if not passed; falls back to `0.0` if unset.
-- [Optional] `cost_per_input_token`: A float specifying the cost per input token for the provided model. Defaults to `GEMINI_COST_PER_INPUT_TOKEN` if set; falls back to `deepeval`'s model cost registry, else `None`.
-- [Optional] `cost_per_output_token`: A float specifying the cost per output token for the provided model. Defaults to `GEMINI_COST_PER_OUTPUT_TOKEN` if set; falls back to `deepeval`'s model cost registry, else `None`.
-- [Optional] `generation_kwargs`: A dictionary of additional generation parameters forwarded to the Gemini API `generate_content(...)` call.
-
-Parameters may be explicitly passed to the model at initialization time, or configured with optional settings. The **mandatory** parameters are required at runtime, but you can provide them either explicitly as constructor arguments, **or** via `deepeval` settings / environment variables (constructor args take precedence). See [Environment variables and settings](/docs/evaluation-flags-and-configs#model-settings-gemini) for the Gemini-related environment variables.
-
-:::note
-At runtime, you must provide an API key (via `api_key` or `GOOGLE_API_KEY`) unless you’re using Vertex AI. See [Vertex AI](/docs/integrations/models/vertex-ai).
-:::
-
 </Case>
 
 <Case id="typescript">
@@ -113,23 +113,6 @@ const model = new GeminiModel({
 const answerRelevancy = new AnswerRelevancyMetric({ model });
 ```
 
-</Tab>
-<Tab value="ENV">
-
-Every option is backed by an environment variable, so you can set `GEMINI_MODEL_NAME` and `GOOGLE_API_KEY` in your `env` and construct the model with no arguments:
-
-```typescript
-import { AnswerRelevancyMetric } from "deepeval/metrics";
-import { GeminiModel } from "deepeval/models";
-
-const answerRelevancy = new AnswerRelevancyMetric({
-  model: new GeminiModel(),
-});
-```
-
-</Tab>
-</Tabs>
-
 There are **ZERO** mandatory and **EIGHT** optional parameters when creating a `GeminiModel`:
 
 - [Optional] `model`: A string specifying the name of the Gemini model to use. Defaults to `GEMINI_MODEL_NAME` if not passed; falls back to <DefaultLLMModel provider="gemini" /> if unset.
@@ -155,6 +138,23 @@ const model = new GeminiModel({
 At runtime, you must provide an API key (via `apiKey`, `GOOGLE_API_KEY`, or `GEMINI_API_KEY`) unless you’re using Vertex AI. See [Vertex AI](/docs/integrations/models/vertex-ai).
 :::
 
+</Tab>
+<Tab value="ENV">
+
+Every option is backed by an environment variable, so you can set `GEMINI_MODEL_NAME` and `GOOGLE_API_KEY` in your `env` and construct the model with no arguments:
+
+```typescript
+import { AnswerRelevancyMetric } from "deepeval/metrics";
+import { GeminiModel } from "deepeval/models";
+
+const answerRelevancy = new AnswerRelevancyMetric({
+  model: new GeminiModel(),
+});
+```
+
+</Tab>
+</Tabs>
+
 </Case>
 
 </Switch>

docs/content/integrations/models/grok.mdx

@@ -74,6 +74,15 @@ model = GrokModel(
 answer_relevancy = AnswerRelevancyMetric(model=model)
 ```
 
+There are **ZERO** mandatory and **SIX** optional parameters when creating a `GrokModel`:
+
+- [Optional] `model`: A string specifying the name of the Grok model to use. Defaults to `GROK_MODEL_NAME` if not passed; raises an error at runtime if unset.
+- [Optional] `api_key`: A string specifying your Grok API key for authentication. Defaults to `GROK_API_KEY` if not passed; raises an error at runtime if unset.
+- [Optional] `temperature`: A float specifying the model temperature. Defaults to `TEMPERATURE` if not passed; falls back to `0.0` if unset.
+- [Optional] `cost_per_input_token`: A float specifying the cost for each input token for the provided model. Defaults to `GROK_COST_PER_INPUT_TOKEN` if available in `deepeval`'s model cost registry, else `None`.
+- [Optional] `cost_per_output_token`: A float specifying the cost for each output token for the provided model. Defaults to `GROK_COST_PER_OUTPUT_TOKEN` if available in `deepeval`'s model cost registry, else `None`.
+- [Optional] `generation_kwargs`: A dictionary of additional generation parameters forwarded to the xAI SDK `client.chat.create(...)` call.
+
 </Tab>
 <Tab value="ENV">
 
@@ -92,15 +101,6 @@ You should also set the other necessary vars like `GROK_API_KEY` to be able to u
 </Tab>
 </Tabs>
 
-There are **ZERO** mandatory and **SIX** optional parameters when creating a `GrokModel`:
-
-- [Optional] `model`: A string specifying the name of the Grok model to use. Defaults to `GROK_MODEL_NAME` if not passed; raises an error at runtime if unset.
-- [Optional] `api_key`: A string specifying your Grok API key for authentication. Defaults to `GROK_API_KEY` if not passed; raises an error at runtime if unset.
-- [Optional] `temperature`: A float specifying the model temperature. Defaults to `TEMPERATURE` if not passed; falls back to `0.0` if unset.
-- [Optional] `cost_per_input_token`: A float specifying the cost for each input token for the provided model. Defaults to `GROK_COST_PER_INPUT_TOKEN` if available in `deepeval`'s model cost registry, else `None`.
-- [Optional] `cost_per_output_token`: A float specifying the cost for each output token for the provided model. Defaults to `GROK_COST_PER_OUTPUT_TOKEN` if available in `deepeval`'s model cost registry, else `None`.
-- [Optional] `generation_kwargs`: A dictionary of additional generation parameters forwarded to the xAI SDK `client.chat.create(...)` call.
-
 </Case>
 
 <Case id="typescript">
@@ -121,23 +121,6 @@ const model = new GrokModel({
 const answerRelevancy = new AnswerRelevancyMetric({ model });
 ```
 
-</Tab>
-<Tab value="ENV">
-
-To use any Grok model directly in `deepeval`, set `GROK_MODEL_NAME` and `GROK_API_KEY` in your `env` and create a `GrokModel` with no arguments:
-
-```typescript
-import { AnswerRelevancyMetric } from "deepeval/metrics";
-import { GrokModel } from "deepeval/models";
-
-const answerRelevancy = new AnswerRelevancyMetric({
-  model: new GrokModel(),
-});
-```
-
-</Tab>
-</Tabs>
-
 There are **ZERO** mandatory and **SEVEN** optional parameters when creating a `GrokModel`:
 
 - [Optional] `model`: A string specifying the name of the Grok model to use. Defaults to `GROK_MODEL_NAME` if not passed; falls back to `grok-3` if unset.
@@ -158,6 +141,23 @@ const model = new GrokModel({
 });
 ```
 
+</Tab>
+<Tab value="ENV">
+
+To use any Grok model directly in `deepeval`, set `GROK_MODEL_NAME` and `GROK_API_KEY` in your `env` and create a `GrokModel` with no arguments:
+
+```typescript
+import { AnswerRelevancyMetric } from "deepeval/metrics";
+import { GrokModel } from "deepeval/models";
+
+const answerRelevancy = new AnswerRelevancyMetric({
+  model: new GrokModel(),
+});
+```
+
+</Tab>
+</Tabs>
+
 </Case>
 
 </Switch>

docs/content/integrations/models/litellm.mdx

@@ -74,6 +74,20 @@ model = LiteLLMModel(
 answer_relevancy = AnswerRelevancyMetric(model=model)
 ```
 
+There are **ZERO** mandatory and **SEVEN** optional parameters when creating a `LiteLLMModel`:
+
+- [Optional] `model`: A string specifying the provider and model name (e.g. `openai/gpt-3.5-turbo`, `anthropic/claude-3-opus`). Defaults to `LITELLM_MODEL_NAME` if not passed; raises an error at runtime if unset.
+- [Optional] `api_key`: A string specifying the API key for the model. If not passed, `deepeval` attempts (in order) `LITELLM_API_KEY`, `LITELLM_PROXY_API_KEY`, `OPENAI_API_KEY`, `ANTHROPIC_API_KEY`, then `GOOGLE_API_KEY`. If none are set, the key is left unset and the underlying LiteLLM/provider behavior applies.
+- [Optional] `base_url`: A string specifying the base URL for the model API. Defaults to `LITELLM_API_BASE`, then `LITELLM_PROXY_API_BASE` if not passed.
+- [Optional] `temperature`: A float specifying the model temperature. Defaults to `TEMPERATURE` if not passed; falls back to `0.0` if unset.
+- [Optional] `cost_per_input_token`: A float specifying the cost for each input token for the provided model. Defaults to `None` if not passed; when unset, the cost is taken from LiteLLM's response if it reports one, otherwise reported as unknown.
+- [Optional] `cost_per_output_token`: A float specifying the cost for each output token for the provided model. Defaults to `None` if not passed; when unset, the cost is taken from LiteLLM's response if it reports one, otherwise reported as unknown.
+- [Optional] `generation_kwargs`: A dictionary of additional generation parameters forwarded to LiteLLM's `completion(...)` / `acompletion(...)` call.
+
+:::tip
+Any `**kwargs` you would like to use for your model can be passed through the `generation_kwargs` parameter. However, we request you to double check the params supported by the model and your model provider in their [official docs](https://docs.litellm.ai/docs/providers/custom_llm_server).
+:::
+
 </Tab>
 <Tab value="ENV">
 
@@ -92,20 +106,6 @@ You should also set the other necessary vars like `LITELLM_API_KEY` to be able t
 </Tab>
 </Tabs>
 
-There are **ZERO** mandatory and **SEVEN** optional parameters when creating a `LiteLLMModel`:
-
-- [Optional] `model`: A string specifying the provider and model name (e.g. `openai/gpt-3.5-turbo`, `anthropic/claude-3-opus`). Defaults to `LITELLM_MODEL_NAME` if not passed; raises an error at runtime if unset.
-- [Optional] `api_key`: A string specifying the API key for the model. If not passed, `deepeval` attempts (in order) `LITELLM_API_KEY`, `LITELLM_PROXY_API_KEY`, `OPENAI_API_KEY`, `ANTHROPIC_API_KEY`, then `GOOGLE_API_KEY`. If none are set, the key is left unset and the underlying LiteLLM/provider behavior applies.
-- [Optional] `base_url`: A string specifying the base URL for the model API. Defaults to `LITELLM_API_BASE`, then `LITELLM_PROXY_API_BASE` if not passed.
-- [Optional] `temperature`: A float specifying the model temperature. Defaults to `TEMPERATURE` if not passed; falls back to `0.0` if unset.
-- [Optional] `cost_per_input_token`: A float specifying the cost for each input token for the provided model. Defaults to `None` if not passed; when unset, the cost is taken from LiteLLM's response if it reports one, otherwise reported as unknown.
-- [Optional] `cost_per_output_token`: A float specifying the cost for each output token for the provided model. Defaults to `None` if not passed; when unset, the cost is taken from LiteLLM's response if it reports one, otherwise reported as unknown.
-- [Optional] `generation_kwargs`: A dictionary of additional generation parameters forwarded to LiteLLM's `completion(...)` / `acompletion(...)` call.
-
-:::tip
-Any `**kwargs` you would like to use for your model can be passed through the `generation_kwargs` parameter. However, we request you to double check the params supported by the model and your model provider in their [official docs](https://docs.litellm.ai/docs/providers/custom_llm_server).
-:::
-
 ### Available Models
 
 :::note

docs/content/integrations/models/lmstudio.mdx

@@ -54,6 +54,19 @@ model = LocalModel(
 answer_relevancy = AnswerRelevancyMetric(model=model)
 ```
 
+There are **ZERO** mandatory and **SIX** optional parameters when creating a `LocalModel`:
+
+- [Optional] `model`: A string specifying the local model to use. Defaults to `LOCAL_MODEL_NAME` if not passed; raises an error at runtime if unset.
+- [Optional] `api_key`: A string specifying the API key for your local server. Defaults to `LOCAL_MODEL_API_KEY` if not passed; raises an error at runtime if unset. Local servers without authentication accept any placeholder string.
+- [Optional] `base_url`: A string specifying the base URL of your local server. Defaults to `LOCAL_MODEL_BASE_URL` if not passed.
+- [Optional] `temperature`: A float specifying the model temperature. Defaults to `TEMPERATURE` if not passed; falls back to `0.0` if unset.
+- [Optional] `format`: A string specifying the structured-output response format. Defaults to `LOCAL_MODEL_FORMAT` if not passed; falls back to `"json"` if unset.
+- [Optional] `generation_kwargs`: A dictionary of additional generation parameters forwarded to the local server's `chat.completions.create(...)` call.
+
+:::tip
+Any `**kwargs` you would like to use for your model can be passed directly to `LocalModel(...)`; these are forwarded to the underlying OpenAI client constructor.
+:::
+
 </Tab>
 <Tab value="ENV">
 
@@ -72,19 +85,6 @@ You should also set the other necessary vars like `LOCAL_MODEL_BASE_URL` and `LO
 </Tab>
 </Tabs>
 
-There are **ZERO** mandatory and **SIX** optional parameters when creating a `LocalModel`:
-
-- [Optional] `model`: A string specifying the local model to use. Defaults to `LOCAL_MODEL_NAME` if not passed; raises an error at runtime if unset.
-- [Optional] `api_key`: A string specifying the API key for your local server. Defaults to `LOCAL_MODEL_API_KEY` if not passed; raises an error at runtime if unset. Local servers without authentication accept any placeholder string.
-- [Optional] `base_url`: A string specifying the base URL of your local server. Defaults to `LOCAL_MODEL_BASE_URL` if not passed.
-- [Optional] `temperature`: A float specifying the model temperature. Defaults to `TEMPERATURE` if not passed; falls back to `0.0` if unset.
-- [Optional] `format`: A string specifying the structured-output response format. Defaults to `LOCAL_MODEL_FORMAT` if not passed; falls back to `"json"` if unset.
-- [Optional] `generation_kwargs`: A dictionary of additional generation parameters forwarded to the local server's `chat.completions.create(...)` call.
-
-:::tip
-Any `**kwargs` you would like to use for your model can be passed directly to `LocalModel(...)`; these are forwarded to the underlying OpenAI client constructor.
-:::
-
 </Case>
 
 <Case id="typescript">
@@ -106,6 +106,27 @@ const model = new LocalModel({
 const answerRelevancy = new AnswerRelevancyMetric({ model });
 ```
 
+There are **ZERO** mandatory and **SEVEN** optional parameters when creating a `LocalModel`:
+
+- [Optional] `model`: A string specifying the local model to use. Defaults to `LOCAL_MODEL_NAME` if not passed; raises an error if unset.
+- [Optional] `apiKey`: A string specifying the API key for your local server. Defaults to `LOCAL_MODEL_API_KEY` if not passed; falls back to a placeholder string, so local servers without authentication work out of the box.
+- [Optional] `baseURL`: A string specifying the base URL of your local server. Defaults to `LOCAL_MODEL_BASE_URL` if not passed; raises an error if unset.
+- [Optional] `temperature`: A number specifying the model temperature. Defaults to `0`. Pass `null` to omit it from the request entirely, which is required by reasoning models that reject it.
+- [Optional] `defaultHeaders`: An object of extra HTTP headers sent with every request.
+- [Optional] `costPerInputToken`: A number specifying the cost for each input token for the provided model. Defaults to the price in `deepeval`'s model cost registry, else `null`.
+- [Optional] `costPerOutputToken`: A number specifying the cost for each output token for the provided model. Defaults to the price in `deepeval`'s model cost registry, else `null`.
+
+Any other key you pass is forwarded as-is to the local server's `chat.completions.create(...)` call, so generation parameters `deepeval` does not name go inline alongside the rest:
+
+```typescript
+const model = new LocalModel({
+  model: "<model_name>",
+  baseURL: "http://localhost:1234/v1/",
+  top_p: 0.9,
+  max_tokens: 512,
+});
+```
+
 </Tab>
 <Tab value="ENV">
 
@@ -132,27 +153,6 @@ const answerRelevancy = new AnswerRelevancyMetric({
 </Tab>
 </Tabs>
 
-There are **ZERO** mandatory and **SEVEN** optional parameters when creating a `LocalModel`:
-
-- [Optional] `model`: A string specifying the local model to use. Defaults to `LOCAL_MODEL_NAME` if not passed; raises an error if unset.
-- [Optional] `apiKey`: A string specifying the API key for your local server. Defaults to `LOCAL_MODEL_API_KEY` if not passed; falls back to a placeholder string, so local servers without authentication work out of the box.
-- [Optional] `baseURL`: A string specifying the base URL of your local server. Defaults to `LOCAL_MODEL_BASE_URL` if not passed; raises an error if unset.
-- [Optional] `temperature`: A number specifying the model temperature. Defaults to `0`. Pass `null` to omit it from the request entirely, which is required by reasoning models that reject it.
-- [Optional] `defaultHeaders`: An object of extra HTTP headers sent with every request.
-- [Optional] `costPerInputToken`: A number specifying the cost for each input token for the provided model. Defaults to the price in `deepeval`'s model cost registry, else `null`.
-- [Optional] `costPerOutputToken`: A number specifying the cost for each output token for the provided model. Defaults to the price in `deepeval`'s model cost registry, else `null`.
-
-Any other key you pass is forwarded as-is to the local server's `chat.completions.create(...)` call, so generation parameters `deepeval` does not name go inline alongside the rest:
-
-```typescript
-const model = new LocalModel({
-  model: "<model_name>",
-  baseURL: "http://localhost:1234/v1/",
-  top_p: 0.9,
-  max_tokens: 512,
-});
-```
-
 </Case>
 
 </Switch>

docs/content/integrations/models/moonshot.mdx

@@ -55,6 +55,15 @@ model = KimiModel(
 answer_relevancy = AnswerRelevancyMetric(model=model)
 ```
 
+There are **ZERO** mandatory and **SIX** optional parameters when creating an `KimiModel`:
+
+- [Optional] `model`: A string specifying the name of the Kimi model to use. Defaults to `MOONSHOT_MODEL_NAME` if not passed; raises an error at runtime if unset.
+- [Optional] `api_key`: A string specifying your Kimi API key for authentication. Defaults to `MOONSHOT_API_KEY` if not passed; raises an error at runtime if unset.
+- [Optional] `temperature`: A float specifying the model temperature. Defaults to `TEMPERATURE` if not passed; falls back to `0.0` if unset and raises if < 0.
+- [Optional] `cost_per_input_token`: A float specifying the cost for each input token for the provided model. Defaults to `MOONSHOT_COST_PER_INPUT_TOKEN` if available in `deepeval`'s model cost registry, else `None`.
+- [Optional] `cost_per_output_token`: A float specifying the cost for each output token for the provided model. Defaults to `MOONSHOT_COST_PER_OUTPUT_TOKEN` if available in `deepeval`'s model cost registry, else `None`.
+- [Optional] `generation_kwargs`: A dictionary of additional generation parameters forwarded to the OpenAI `chat.completions.create(...)` call.
+
 </Tab>
 <Tab value="ENV">
 
@@ -73,15 +82,6 @@ You should also set the other necessary vars like `MOONSHOT_API_KEY` to be able
 </Tab>
 </Tabs>
 
-There are **ZERO** mandatory and **SIX** optional parameters when creating an `KimiModel`:
-
-- [Optional] `model`: A string specifying the name of the Kimi model to use. Defaults to `MOONSHOT_MODEL_NAME` if not passed; raises an error at runtime if unset.
-- [Optional] `api_key`: A string specifying your Kimi API key for authentication. Defaults to `MOONSHOT_API_KEY` if not passed; raises an error at runtime if unset.
-- [Optional] `temperature`: A float specifying the model temperature. Defaults to `TEMPERATURE` if not passed; falls back to `0.0` if unset and raises if < 0.
-- [Optional] `cost_per_input_token`: A float specifying the cost for each input token for the provided model. Defaults to `MOONSHOT_COST_PER_INPUT_TOKEN` if available in `deepeval`'s model cost registry, else `None`.
-- [Optional] `cost_per_output_token`: A float specifying the cost for each output token for the provided model. Defaults to `MOONSHOT_COST_PER_OUTPUT_TOKEN` if available in `deepeval`'s model cost registry, else `None`.
-- [Optional] `generation_kwargs`: A dictionary of additional generation parameters forwarded to the OpenAI `chat.completions.create(...)` call.
-
 </Case>
 
 <Case id="typescript">
@@ -102,23 +102,6 @@ const model = new KimiModel({
 const answerRelevancy = new AnswerRelevancyMetric({ model });
 ```
 
-</Tab>
-<Tab value="ENV">
-
-To use any Moonshot model directly in `deepeval`, set `MOONSHOT_MODEL_NAME` and `MOONSHOT_API_KEY` in your `env` and create a `KimiModel` with no arguments:
-
-```typescript
-import { AnswerRelevancyMetric } from "deepeval/metrics";
-import { KimiModel } from "deepeval/models";
-
-const answerRelevancy = new AnswerRelevancyMetric({
-  model: new KimiModel(),
-});
-```
-
-</Tab>
-</Tabs>
-
 There are **ZERO** mandatory and **SEVEN** optional parameters when creating a `KimiModel`:
 
 - [Optional] `model`: A string specifying the name of the Kimi model to use. Defaults to `MOONSHOT_MODEL_NAME` if not passed; falls back to `moonshot-v1-8k` if unset.
@@ -139,6 +122,23 @@ const model = new KimiModel({
 });
 ```
 
+</Tab>
+<Tab value="ENV">
+
+To use any Moonshot model directly in `deepeval`, set `MOONSHOT_MODEL_NAME` and `MOONSHOT_API_KEY` in your `env` and create a `KimiModel` with no arguments:
+
+```typescript
+import { AnswerRelevancyMetric } from "deepeval/metrics";
+import { KimiModel } from "deepeval/models";
+
+const answerRelevancy = new AnswerRelevancyMetric({
+  model: new KimiModel(),
+});
+```
+
+</Tab>
+</Tabs>
+
 </Case>
 
 </Switch>

docs/content/integrations/models/ollama.mdx

@@ -81,7 +81,15 @@ answer_relevancy = AnswerRelevancyMetric(model=model)
 </Tab>
 <Tab value="ENV">
 
-To use any Ollama model directly in `deepeval`, set the `LOCAL_MODEL_API_KEY` in your `env` and simply pass the name of your desired model in your metric initialization:
+First point `deepeval` at Ollama. `LOCAL_MODEL_API_KEY` must be the literal string `ollama` — that value is what routes a plain model name to your Ollama server instead of a hosted provider:
+
+```bash
+# .env.local
+LOCAL_MODEL_API_KEY=ollama
+LOCAL_MODEL_BASE_URL=http://localhost:11434
+```
+
+Then pass the name of your desired model as a string in your metric initialization:
 
 ```python
 from deepeval.metrics import AnswerRelevancyMetric
@@ -91,6 +99,8 @@ answer_relevancy = AnswerRelevancyMetric(
 )
 ```
 
+Running `deepeval set-ollama --model=deepseek-r1:1.5b --save` writes these same variables for you.
+
 </Tab>
 </Tabs>
 
@@ -121,31 +131,6 @@ const model = new OllamaModel({
 const answerRelevancy = new AnswerRelevancyMetric({ model });
 ```
 
-</Tab>
-<Tab value="ENV">
-
-`deepeval` autoloads `.env.local` then `.env` at import time, so you can keep your Ollama configuration out of your code entirely:
-
-```bash
-# .env.local
-OLLAMA_MODEL_NAME=deepseek-r1:1.5b
-LOCAL_MODEL_BASE_URL=http://localhost:11434
-```
-
-Then create the model with no arguments and pass it to any metric:
-
-```typescript
-import { AnswerRelevancyMetric } from "deepeval/metrics";
-import { OllamaModel } from "deepeval/models";
-
-const answerRelevancy = new AnswerRelevancyMetric({
-  model: new OllamaModel(),
-});
-```
-
-</Tab>
-</Tabs>
-
 There are **ZERO** mandatory and **FIVE** optional parameters when creating an `OllamaModel`:
 
 - [Optional] `model`: A string specifying the name of the Ollama model to use. Defaults to `OLLAMA_MODEL_NAME` if not passed; raises an error at runtime if unset.
@@ -164,6 +149,32 @@ const model = new OllamaModel({
 });
 ```
 
+</Tab>
+<Tab value="ENV">
+
+First point `deepeval` at Ollama. `LOCAL_MODEL_API_KEY` must be the literal string `ollama` — that value is what routes a plain model name to your Ollama server instead of a hosted provider:
+
+```bash
+# .env.local
+LOCAL_MODEL_API_KEY=ollama
+LOCAL_MODEL_BASE_URL=http://localhost:11434
+```
+
+Then pass the name of your desired model as a string in your metric initialization:
+
+```typescript
+import { AnswerRelevancyMetric } from "deepeval/metrics";
+
+const answerRelevancy = new AnswerRelevancyMetric({
+  model: "deepseek-r1:1.5b",
+});
+```
+
+Running `deepeval set-ollama --model=deepseek-r1:1.5b --save` writes these same variables for you.
+
+</Tab>
+</Tabs>
+
 </Case>
 
 </Switch>

docs/content/integrations/models/openai.mdx

@@ -18,13 +18,13 @@ DeepEval autoloads `.env.local` then `.env` at import time (process env -> `.env
 OPENAI_API_KEY=<your-openai-api-key>
 ```
 
-Alternative (Shell/CI)
+**Alternative (Shell/CI):**
 
 ```bash
 export OPENAI_API_KEY=<your-openai-api-key>
 ```
 
-Alternative (notebook)
+**Alternative (notebook):**
 
 If you're working in a notebook environment (Jupyter or Colab), set your `OPENAI_API_KEY` in a cell:
 
@@ -85,6 +85,16 @@ model = OpenAIModel(
 answer_relevancy = AnswerRelevancyMetric(model=model)
 ```
 
+There are **ZERO** mandatory and **SEVEN** optional parameters when creating an `OpenAIModel`:
+
+- [Optional] `model`: A string specifying the name of the GPT model to use. Defaulted to `OPENAI_MODEL_NAME` if not set; falls back to <DefaultLLMModel />.
+- [Optional] `api_key`: A string specifying the OpenAI API key for authentication. Defaults to `OPENAI_API_KEY` if not passed; raises an error at runtime if unset.
+- [Optional] `base_url`: A string specifying your OpenAI URL.
+- [Optional] `temperature`: A float specifying the model temperature. Defaults to `TEMPERATURE` if not passed; falls back to `0.0` if unset.
+- [Optional] `cost_per_input_token`: A float specifying the cost for each input token for the provided model. Defaults to `OPENAI_COST_PER_INPUT_TOKEN` if available in `deepeval`'s model cost registry, else `None`.
+- [Optional] `cost_per_output_token`: A float specifying the cost for each output token for the provided model. Defaults to `OPENAI_COST_PER_OUTPUT_TOKEN` if available in `deepeval`'s model cost registry, else `None`.
+- [Optional] `generation_kwargs`: A dictionary of additional generation parameters forwarded to the OpenAI `chat.completions.create(...)` and `beta.chat.completions.parse(...)` calls.
+
 </Tab>
 <Tab value="ENV">
 
@@ -101,16 +111,6 @@ answer_relevancy = AnswerRelevancyMetric(
 </Tab>
 </Tabs>
 
-There are **ZERO** mandatory and **SEVEN** optional parameters when creating an `OpenAIModel`:
-
-- [Optional] `model`: A string specifying the name of the GPT model to use. Defaulted to `OPENAI_MODEL_NAME` if not set; falls back to <DefaultLLMModel />.
-- [Optional] `api_key`: A string specifying the OpenAI API key for authentication. Defaults to `OPENAI_API_KEY` if not passed; raises an error at runtime if unset.
-- [Optional] `base_url`: A string specifying your OpenAI URL.
-- [Optional] `temperature`: A float specifying the model temperature. Defaults to `TEMPERATURE` if not passed; falls back to `0.0` if unset.
-- [Optional] `cost_per_input_token`: A float specifying the cost for each input token for the provided model. Defaults to `OPENAI_COST_PER_INPUT_TOKEN` if available in `deepeval`'s model cost registry, else `None`.
-- [Optional] `cost_per_output_token`: A float specifying the cost for each output token for the provided model. Defaults to `OPENAI_COST_PER_OUTPUT_TOKEN` if available in `deepeval`'s model cost registry, else `None`.
-- [Optional] `generation_kwargs`: A dictionary of additional generation parameters forwarded to the OpenAI `chat.completions.create(...)` and `beta.chat.completions.parse(...)` calls.
-
 </Case>
 
 <Case id="typescript">
@@ -131,22 +131,6 @@ const model = new OpenAIModel({
 const answerRelevancy = new AnswerRelevancyMetric({ model });
 ```
 
-</Tab>
-<Tab value="ENV">
-
-`deepeval` by default uses OpenAI models for evaluations, you can simply pass the name of your desired model in metric initialization and set the `OPENAI_API_KEY` to use OpenAI models:
-
-```typescript
-import { AnswerRelevancyMetric } from "deepeval/metrics";
-
-const answerRelevancy = new AnswerRelevancyMetric({
-  model: "gpt-4.1",
-});
-```
-
-</Tab>
-</Tabs>
-
 There are **ZERO** mandatory and **SEVEN** optional parameters when creating an `OpenAIModel`:
 
 - [Optional] `model`: A string specifying the name of the OpenAI model to use. Defaulted to `OPENAI_MODEL_NAME` if not set; falls back to <DefaultLLMModel />.
@@ -168,6 +152,22 @@ const model = new OpenAIModel({
 });
 ```
 
+</Tab>
+<Tab value="ENV">
+
+`deepeval` by default uses OpenAI models for evaluations, you can simply pass the name of your desired model in metric initialization and set the `OPENAI_API_KEY` to use OpenAI models:
+
+```typescript
+import { AnswerRelevancyMetric } from "deepeval/metrics";
+
+const answerRelevancy = new AnswerRelevancyMetric({
+  model: "gpt-4.1",
+});
+```
+
+</Tab>
+</Tabs>
+
 </Case>
 
 </Switch>

docs/content/integrations/models/openrouter.mdx

@@ -62,24 +62,6 @@ model = OpenRouterModel(
 answer_relevancy = AnswerRelevancyMetric(model=model)
 ```
 
-</Tab>
-<Tab value="ENV">
-
-To use any OpenRouter model directly in `deepeval`, set the `USE_OPENROUTER_MODEL=1` in your `env` and simply pass the name of your desired model in your metric initialization:
-
-```python
-from deepeval.metrics import AnswerRelevancyMetric
-
-answer_relevancy = AnswerRelevancyMetric(
-    model="openai/gpt-4.1",
-)
-```
-
-You should also set the other necessary vars like `OPENROUTER_API_KEY` to be able to use the OpenRouter models as shown above.
-
-</Tab>
-</Tabs>
-
 There are **ZERO** mandatory and **SEVEN** optional parameters when creating an `OpenRouterModel`:
 
 - [Optional] `model`: A string specifying the OpenRouter model to use. Defaults to `OPENROUTER_MODEL_NAME` if set; otherwise falls back to <DefaultLLMModel provider="openrouter" />.
@@ -107,6 +89,24 @@ model = OpenRouterModel(
 ```
 :::
 
+</Tab>
+<Tab value="ENV">
+
+To use any OpenRouter model directly in `deepeval`, set the `USE_OPENROUTER_MODEL=1` in your `env` and simply pass the name of your desired model in your metric initialization:
+
+```python
+from deepeval.metrics import AnswerRelevancyMetric
+
+answer_relevancy = AnswerRelevancyMetric(
+    model="openai/gpt-4.1",
+)
+```
+
+You should also set the other necessary vars like `OPENROUTER_API_KEY` to be able to use the OpenRouter models as shown above.
+
+</Tab>
+</Tabs>
+
 </Case>
 
 <Case id="typescript">
@@ -133,23 +133,6 @@ const model = new OpenRouterModel({
 const answerRelevancy = new AnswerRelevancyMetric({ model });
 ```
 
-</Tab>
-<Tab value="ENV">
-
-To use any OpenRouter model directly in `deepeval`, set `OPENROUTER_MODEL_NAME` and `OPENROUTER_API_KEY` in your `env` and create an `OpenRouterModel` with no arguments:
-
-```typescript
-import { AnswerRelevancyMetric } from "deepeval/metrics";
-import { OpenRouterModel } from "deepeval/models";
-
-const answerRelevancy = new AnswerRelevancyMetric({
-  model: new OpenRouterModel(),
-});
-```
-
-</Tab>
-</Tabs>
-
 There are **ZERO** mandatory and **SEVEN** optional parameters when creating an `OpenRouterModel`:
 
 - [Optional] `model`: A string specifying the OpenRouter model to use. Defaults to `OPENROUTER_MODEL_NAME` if set; otherwise falls back to <DefaultLLMModel provider="openrouter" />.
@@ -187,6 +170,23 @@ const model = new OpenRouterModel({
 ```
 :::
 
+</Tab>
+<Tab value="ENV">
+
+To use any OpenRouter model directly in `deepeval`, set `OPENROUTER_MODEL_NAME` and `OPENROUTER_API_KEY` in your `env` and create an `OpenRouterModel` with no arguments:
+
+```typescript
+import { AnswerRelevancyMetric } from "deepeval/metrics";
+import { OpenRouterModel } from "deepeval/models";
+
+const answerRelevancy = new AnswerRelevancyMetric({
+  model: new OpenRouterModel(),
+});
+```
+
+</Tab>
+</Tabs>
+
 </Case>
 
 </Switch>

docs/content/integrations/models/portkey.mdx

@@ -58,6 +58,21 @@ model = PortkeyModel(
 answer_relevancy = AnswerRelevancyMetric(model=model)
 ```
 
+There are **ZERO** mandatory and **SIX** optional parameters when creating a `PortkeyModel`:
+
+- [Optional] `model`: A string specifying the name of the Portkey model to use. Defaults to `PORTKEY_MODEL_NAME` if not passed; raises an error at runtime if unset.
+- [Optional] `api_key`: A string specifying your Portkey API key for authentication. Defaults to `PORTKEY_API_KEY` if not passed; raises an error at runtime if unset.
+- [Optional] `provider`: A string specifying the Portkey provider of your model. Defaults to `PORTKEY_PROVIDER_NAME` if not passed; raises an error at runtime if unset.
+- [Optional] `base_url`: A string specifying the base URL for the model API. Defaults to `PORTKEY_BASE_URL` if not passed; raises an error at runtime if unset.
+- [Optional] `temperature`: A float specifying the model temperature. Defaults to `TEMPERATURE` if not passed; falls back to `0.0` if unset.
+- [Optional] `generation_kwargs`: A dictionary of additional generation parameters forwarded to Portkey's `chat.completions.create(...)` call.
+
+Any additional `**kwargs` you would like to use for your `Portkey` client can be passed directly to `PortkeyModel(...)`. These are forwarded to the underlying OpenAI client constructor. We recommend double-checking the parameters supported by your chosen model and provider in the [official Portkey docs](https://portkey.ai/docs).
+
+:::tip
+Pass generation parameters such as `max_tokens` via `generation_kwargs` (they are forwarded to `chat.completions.create(...)`).
+:::
+
 </Tab>
 <Tab value="ENV">
 
@@ -76,21 +91,6 @@ You should also set the other necessary vars like `PORTKEY_API_KEY` to be able t
 </Tab>
 </Tabs>
 
-There are **ZERO** mandatory and **SIX** optional parameters when creating a `PortkeyModel`:
-
-- [Optional] `model`: A string specifying the name of the Portkey model to use. Defaults to `PORTKEY_MODEL_NAME` if not passed; raises an error at runtime if unset.
-- [Optional] `api_key`: A string specifying your Portkey API key for authentication. Defaults to `PORTKEY_API_KEY` if not passed; raises an error at runtime if unset.
-- [Optional] `provider`: A string specifying the Portkey provider of your model. Defaults to `PORTKEY_PROVIDER_NAME` if not passed; raises an error at runtime if unset.
-- [Optional] `base_url`: A string specifying the base URL for the model API. Defaults to `PORTKEY_BASE_URL` if not passed; raises an error at runtime if unset.
-- [Optional] `temperature`: A float specifying the model temperature. Defaults to `TEMPERATURE` if not passed; falls back to `0.0` if unset.
-- [Optional] `generation_kwargs`: A dictionary of additional generation parameters forwarded to Portkey's `chat.completions.create(...)` call.
-
-Any additional `**kwargs` you would like to use for your `Portkey` client can be passed directly to `PortkeyModel(...)`. These are forwarded to the underlying OpenAI client constructor. We recommend double-checking the parameters supported by your chosen model and provider in the [official Portkey docs](https://portkey.ai/docs).
-
-:::tip
-Pass generation parameters such as `max_tokens` via `generation_kwargs` (they are forwarded to `chat.completions.create(...)`).
-:::
-
 </Case>
 
 <Case id="typescript">
@@ -112,23 +112,6 @@ const model = new PortkeyModel({
 const answerRelevancy = new AnswerRelevancyMetric({ model });
 ```
 
-</Tab>
-<Tab value="ENV">
-
-To use any Portkey model directly in `deepeval`, set `PORTKEY_MODEL_NAME`, `PORTKEY_PROVIDER_NAME`, and `PORTKEY_API_KEY` in your `env` and create a `PortkeyModel` with no arguments:
-
-```typescript
-import { AnswerRelevancyMetric } from "deepeval/metrics";
-import { PortkeyModel } from "deepeval/models";
-
-const answerRelevancy = new AnswerRelevancyMetric({
-  model: new PortkeyModel(),
-});
-```
-
-</Tab>
-</Tabs>
-
 There are **ZERO** mandatory and **EIGHT** optional parameters when creating a `PortkeyModel`:
 
 - [Optional] `model`: A string specifying the name of the Portkey model to use. Defaults to `PORTKEY_MODEL_NAME` if not passed; raises an error at runtime if unset.
@@ -152,6 +135,23 @@ const model = new PortkeyModel({
 
 We recommend double-checking the parameters supported by your chosen model and provider in the [official Portkey docs](https://portkey.ai/docs).
 
+</Tab>
+<Tab value="ENV">
+
+To use any Portkey model directly in `deepeval`, set `PORTKEY_MODEL_NAME`, `PORTKEY_PROVIDER_NAME`, and `PORTKEY_API_KEY` in your `env` and create a `PortkeyModel` with no arguments:
+
+```typescript
+import { AnswerRelevancyMetric } from "deepeval/metrics";
+import { PortkeyModel } from "deepeval/models";
+
+const answerRelevancy = new AnswerRelevancyMetric({
+  model: new PortkeyModel(),
+});
+```
+
+</Tab>
+</Tabs>
+
 </Case>
 
 </Switch>