diff --git a/.env.example b/.env.example
index a0d783b0b..79810c276 100644
--- a/.env.example
+++ b/.env.example
@@ -56,6 +56,31 @@ ANTHROPIC_API_KEY=
# =========================
OPENROUTER_API_KEY=
+# =========================
+# Speech providers (voice mode TTS/STT)
+# =========================
+# A USE_*_TTS / USE_*_STT flag picks the provider for each family, and the
+# DEEPEVAL_*_MODEL vars pick the model name. Both default to OpenAI.
+# ELEVENLABS_API_KEY=
+# DEEPGRAM_API_KEY=
+# CARTESIA_API_KEY=
+# ASSEMBLYAI_API_KEY=
+
+# Text-to-speech: pick at most one.
+# USE_OPENAI_TTS=
+# USE_ELEVENLABS_TTS=
+# USE_CARTESIA_TTS=
+# USE_DEEPGRAM_TTS=
+# DEEPEVAL_TTS_MODEL=
+
+# Speech-to-text: pick at most one.
+# USE_OPENAI_STT=
+# USE_ELEVENLABS_STT=
+# USE_CARTESIA_STT=
+# USE_DEEPGRAM_STT=
+# USE_ASSEMBLYAI_STT=
+# DEEPEVAL_STT_MODEL=
+
# =========================
# Confident
# =========================
diff --git a/b.py b/b.py
index 2ad4e3adc..2d55ceff1 100644
--- a/b.py
+++ b/b.py
@@ -12,8 +12,8 @@ from deepeval.dataset import (
Persona,
)
from deepeval.models.llms import OpenAIModel
-from deepeval.models.stt_models import OpenAISTTModel
-from deepeval.models.tts_models import OpenAITTSModel
+from deepeval.models.stt import OpenAISTTModel
+from deepeval.models.tts import OpenAITTSModel
from deepeval.simulator import ConversationSimulator
from deepeval.voice import CallbackVoiceConnector, VoiceConfig
from deepeval.voice.timeline import build_audio_timeline
diff --git a/c.py b/c.py
index a121a1c56..e444c2887 100644
--- a/c.py
+++ b/c.py
@@ -59,8 +59,8 @@ from contextlib import contextmanager
from typing import Dict, List, Optional, Tuple
from deepeval.models.llms import OpenAIModel
-from deepeval.models.stt_models import OpenAISTTModel
-from deepeval.models.tts_models import OpenAITTSModel
+from deepeval.models.stt import OpenAISTTModel
+from deepeval.models.tts import OpenAITTSModel
from deepeval.test_case import Audio
from deepeval.voice.connectors import audio_utils
diff --git a/deepeval/cli/app.py b/deepeval/cli/app.py
new file mode 100644
index 000000000..bd5cefaee
--- /dev/null
+++ b/deepeval/cli/app.py
@@ -0,0 +1,6 @@
+"""The root Typer app, kept apart from `main` so command modules can register
+onto it without importing `main` back."""
+
+import typer
+
+app = typer.Typer(name="deepeval", no_args_is_help=True)
diff --git a/deepeval/cli/diagnose/diagnose.py b/deepeval/cli/diagnose/diagnose.py
index 5c063391c..d3bc36d2b 100644
--- a/deepeval/cli/diagnose/diagnose.py
+++ b/deepeval/cli/diagnose/diagnose.py
@@ -40,6 +40,7 @@ from deepeval.key_handler import (
EmbeddingKeyValues,
KeyValues,
ModelKeyValues,
+ SpeechKeyValues,
)
# Friendly provider labels for the model classes `initialize_model` /
@@ -63,6 +64,15 @@ _PROVIDER_BY_CLASS = {
"AzureOpenAIEmbeddingModel": "Azure OpenAI",
"OllamaEmbeddingModel": "Ollama",
"LocalEmbeddingModel": "Local model",
+ "OpenAITTSModel": "OpenAI",
+ "ElevenLabsTTSModel": "ElevenLabs",
+ "CartesiaTTSModel": "Cartesia",
+ "DeepgramTTSModel": "Deepgram",
+ "OpenAISTTModel": "OpenAI",
+ "ElevenLabsSTTModel": "ElevenLabs",
+ "CartesiaSTTModel": "Cartesia",
+ "DeepgramSTTModel": "Deepgram",
+ "AssemblyAISTTModel": "AssemblyAI",
}
# Settings fields worth showing in the "configured settings" table when set.
@@ -78,6 +88,8 @@ _RELEVANT_MARKERS = (
"DEEPEVAL_DEFAULT_SAVE",
"DEEPEVAL_RESULTS_FOLDER",
"DEEPEVAL_VOICE_FOLDER",
+ "DEEPEVAL_TTS_MODEL",
+ "DEEPEVAL_STT_MODEL",
)
@@ -121,7 +133,12 @@ def _legacy_keystore() -> Tuple[Path, Dict[str, Any]]:
def _env_key_to_json_key() -> Dict[str, str]:
mapping: Dict[str, str] = {}
- for enum in (KeyValues, ModelKeyValues, EmbeddingKeyValues):
+ for enum in (
+ KeyValues,
+ ModelKeyValues,
+ EmbeddingKeyValues,
+ SpeechKeyValues,
+ ):
for member in enum:
mapping[member.name] = member.value
return mapping
@@ -312,6 +329,22 @@ def _models_section() -> Dict[str, Any]:
"(override with embedder=... in ContextConstructionConfig)"
)
+ # Reported from the selection alone: those constructors need an API key.
+ from deepeval.models.speech_selection import (
+ describe_stt_selection,
+ describe_tts_selection,
+ )
+
+ for key, (class_name, model_name) in (
+ ("tts", describe_tts_selection()),
+ ("stt", describe_stt_selection()),
+ ):
+ result[key] = {
+ "provider": _PROVIDER_BY_CLASS.get(class_name, class_name),
+ "model": model_name or "provider default",
+ "used_by": "voice simulations",
+ }
+
return result
@@ -440,7 +473,12 @@ def diagnose_command(
# Default models (global, overridable per class)
models = report["default_models"]
table = _kv_table("Default models")
- for label, key in (("LLM", "llm"), ("Embeddings", "embeddings")):
+ for label, key in (
+ ("LLM", "llm"),
+ ("Embeddings", "embeddings"),
+ ("TTS", "tts"),
+ ("STT", "stt"),
+ ):
info = models[key]
if "error" in info:
value = f"[red]⚠ unusable until fixed: {info['error']}[/red]"
diff --git a/deepeval/cli/main.py b/deepeval/cli/main.py
index 89a61375f..34ecad689 100644
--- a/deepeval/cli/main.py
+++ b/deepeval/cli/main.py
@@ -1,7 +1,8 @@
"""
-DeepEval CLI: Model Provider Configuration Commands
+DeepEval CLI.
-General behavior for all `set-*` / `unset-*` commands:
+The `set-*` / `unset-*` provider commands live in `deepeval.cli.providers`,
+split into `llms`, `tts` and `stt`. General behavior for all of them:
- Non-secret settings (model name, endpoint, deployment, toggles) are always
persisted in the hidden `.deepeval/.deepeval` JSON store.
@@ -15,7 +16,6 @@ General behavior for all `set-*` / `unset-*` commands:
may still be selected as the default.
"""
-import os
import typer
import importlib.metadata
from typing import List, Optional
@@ -24,29 +24,23 @@ from rich.markup import escape
from rich.console import Console
from rich.table import Table
from enum import Enum
-from pathlib import Path
from pydantic import SecretStr
from pydantic_core import PydanticUndefined
-from deepeval.key_handler import (
- EmbeddingKeyValues,
- ModelKeyValues,
-)
from deepeval.telemetry import capture_cli_command
from deepeval.config.settings import get_settings
from deepeval.utils import open_browser
from deepeval.test_run.test_run import (
global_test_run_manager,
)
+from deepeval.cli.app import app
from deepeval.cli.auth import LOGIN_HELP, login_command, logout_command
from deepeval.cli.diagnose import diagnose_command
from deepeval.cli.generate.command import generate_command
from deepeval.cli.inspect import inspect_command
from deepeval.cli.test.command import app as test_app
from deepeval.cli.utils import (
- coerce_blank_to_none,
- handle_save_result,
+ handle_save_result as _handle_save_result,
is_optional,
- load_service_account_key_file,
parse_and_validate,
resolve_field_names,
upload_and_open_link,
@@ -58,7 +52,6 @@ from deepeval.confident.api import (
HttpMethods,
)
-app = typer.Typer(name="deepeval", no_args_is_help=True)
app.add_typer(test_app, name="test")
app.command(name="generate")(generate_command)
app.command(name="inspect")(inspect_command)
@@ -83,24 +76,6 @@ def version_callback(value: Optional[bool] = None) -> None:
raise typer.Exit()
-def is_openai_configured() -> bool:
- s = get_settings()
- v = s.OPENAI_API_KEY
- if isinstance(v, SecretStr):
- try:
- if v.get_secret_value().strip():
- return True
- except Exception:
- pass
- elif v and str(v).strip():
- return True
- env = os.getenv("OPENAI_API_KEY")
- return bool(env and env.strip())
-
-
-_handle_save_result = handle_save_result
-
-
@app.callback()
def main(
ctx: typer.Context,
@@ -610,2347 +585,8 @@ def unset_debug(
)
-#############################################
-# OpenAI Integration ########################
-#############################################
-
-
-@app.command(name="set-openai")
-def set_openai_env(
- model: Optional[str] = typer.Option(
- None,
- "-m",
- "--model",
- help="Model identifier to use for this provider (e.g., `gpt-4.1`).",
- ),
- prompt_api_key: bool = typer.Option(
- False,
- "-k",
- "--prompt-api-key",
- help=(
- "Prompt for OPENAI_API_KEY (input hidden). Not suitable for CI. "
- "If --save (or DEEPEVAL_DEFAULT_SAVE) is used, the key is written to dotenv in plaintext."
- ),
- ),
- cost_per_input_token: Optional[float] = typer.Option(
- None,
- "-i",
- "--cost-per-input-token",
- help=(
- "USD per input token override used for cost tracking. Preconfigured for known models; "
- "REQUIRED if you use a custom/unknown model."
- ),
- ),
- cost_per_output_token: Optional[float] = typer.Option(
- None,
- "-o",
- "--cost-per-output-token",
- help=(
- "USD per output token override used for cost tracking. Preconfigured for known models; "
- "REQUIRED if you use a custom/unknown model."
- ),
- ),
- save: Optional[str] = typer.Option(
- None,
- "-s",
- "--save",
- help="Persist CLI parameters as environment variables in a dotenv file. "
- "Usage: --save=dotenv[:path] (default: .env.local)",
- ),
- quiet: bool = typer.Option(
- False,
- "-q",
- "--quiet",
- help="Suppress printing to the terminal (useful for CI).",
- ),
-):
- """
- Configure OpenAI as the active LLM provider.
-
- What this does:
- - Sets the active provider flag to `USE_OPENAI_MODEL`.
- - Persists the selected model name and any cost overrides in the JSON store.
- - secrets are never written to `.deepeval/.deepeval` (JSON).
-
- Pricing rules:
- - If `model` is a known OpenAI model, you may omit costs (built‑in pricing is used).
- - If `model` is custom/unsupported, you must provide both
- `--cost-per-input-token` and `--cost-per-output-token`.
-
- Secrets & saving:
-
- - If you run with --prompt-api-key, DeepEval will set OPENAI_API_KEY for this session.
- - If --save=dotenv[:path] is used (or DEEPEVAL_DEFAULT_SAVE is set), the key will be written to that dotenv file (plaintext).
-
- Secrets are never written to .deepeval/.deepeval (legacy JSON store).
-
- Args:
- --model: OpenAI model name, such as `gpt-4o-mini`.
- --prompt-api-key: Prompt interactively for OPENAI_API_KEY (input hidden). Avoids putting secrets on the command line (shell history/process args). Not suitable for CI.
- --cost-per-input-token: USD per input token (optional for known models).
- --cost-per-output-token: USD per output token (optional for known models).
- --save: Persist config (and supported secrets) to a dotenv file; format `dotenv[:path]`.
- --quiet: Suppress printing to the terminal.
-
- Example:
- deepeval set-openai \\
- --model gpt-4o-mini \\
- --cost-per-input-token 0.0005 \\
- --cost-per-output-token 0.0015 \\
- --save dotenv:.env.local
- """
- api_key = None
- if prompt_api_key:
- api_key = coerce_blank_to_none(
- typer.prompt("OpenAI API key", hide_input=True)
- )
-
- model = coerce_blank_to_none(model)
-
- settings = get_settings()
- with settings.edit(save=save) as edit_ctx:
- edit_ctx.switch_model_provider(ModelKeyValues.USE_OPENAI_MODEL)
- if model is not None:
- settings.OPENAI_MODEL_NAME = model
- if api_key is not None:
- settings.OPENAI_API_KEY = api_key
- if cost_per_input_token is not None:
- settings.OPENAI_COST_PER_INPUT_TOKEN = cost_per_input_token
- if cost_per_output_token is not None:
- settings.OPENAI_COST_PER_OUTPUT_TOKEN = cost_per_output_token
-
- handled, path, updates = edit_ctx.result
-
- effective_model = settings.OPENAI_MODEL_NAME
- if not effective_model:
- raise typer.BadParameter(
- "OpenAI model name is not set. Pass --model (or set OPENAI_MODEL_NAME).",
- param_hint="--model",
- )
-
- _handle_save_result(
- handled=handled,
- path=path,
- updates=updates,
- save=save,
- quiet=quiet,
- success_msg=(
- f":raising_hands: Congratulations! You're now using OpenAI's `{escape(effective_model)}` "
- "for all evals that require an LLM."
- ),
- )
-
-
-@app.command(name="unset-openai")
-def unset_openai_env(
- save: Optional[str] = typer.Option(
- None,
- "-s",
- "--save",
- help="Remove only the OpenAI related environment variables from a dotenv file. "
- "Usage: --save=dotenv[:path] (default: .env.local)",
- ),
- clear_secrets: bool = typer.Option(
- False,
- "-x",
- "--clear-secrets",
- help="Also remove OPENAI_API_KEY from the dotenv store.",
- ),
- quiet: bool = typer.Option(
- False,
- "-q",
- "--quiet",
- help="Suppress printing to the terminal (useful for CI).",
- ),
-):
- """
- Unset OpenAI as the active provider.
-
- Behavior:
- - Removes OpenAI keys (model, costs, toggle) from the JSON store.
- - If `--save` is provided, removes those keys from the specified dotenv file.
- - After unsetting, if `OPENAI_API_KEY` is still set in the environment,
- OpenAI may still be usable by default. Otherwise, no active provider is configured.
-
- Args:
- --save: Remove OpenAI keys from the given dotenv file as well.
- --clear-secrets: Removes OPENAI_API_KEY from the dotenv store
- --quiet: Suppress printing to the terminal
-
- Example:
- deepeval unset-openai --save dotenv:.env.local
- """
-
- settings = get_settings()
- with settings.edit(save=save) as edit_ctx:
- settings.OPENAI_MODEL_NAME = None
- settings.OPENAI_COST_PER_INPUT_TOKEN = None
- settings.OPENAI_COST_PER_OUTPUT_TOKEN = None
- settings.USE_OPENAI_MODEL = None
- if clear_secrets:
- settings.OPENAI_API_KEY = None
-
- handled, path, updates = edit_ctx.result
-
- if _handle_save_result(
- handled=handled,
- path=path,
- updates=updates,
- save=save,
- quiet=quiet,
- updated_msg="Removed OpenAI environment variables from {path}.",
- tip_msg=None,
- ):
- if is_openai_configured():
- print(
- ":raised_hands: OpenAI will still be used by default because OPENAI_API_KEY is set."
- )
- else:
- print(
- "OpenAI has been unset. No active provider is configured. "
- "Set one with the CLI, or add credentials to .env[.local]."
- )
-
-
-#############################################
-# Azure Integration ########################
-#############################################
-
-
-@app.command(name="set-azure-openai")
-def set_azure_openai_env(
- model: Optional[str] = typer.Option(
- None,
- "-m",
- "--model",
- help="Model identifier to use for this provider (e.g., `gpt-4.1`).",
- ),
- prompt_api_key: bool = typer.Option(
- False,
- "-k",
- "--prompt-api-key",
- help=(
- "Prompt for AZURE_OPENAI_API_KEY (input hidden). Not suitable for CI. "
- "If --save (or DEEPEVAL_DEFAULT_SAVE) is used, the key is written to dotenv in plaintext."
- ),
- ),
- base_url: Optional[str] = typer.Option(
- None,
- "-u",
- "--base-url",
- help="Override the API endpoint/base URL used by this provider.",
- ),
- api_version: Optional[str] = typer.Option(
- None,
- "-v",
- "--api-version",
- help="Azure OpenAI API version (passed to the Azure OpenAI client).",
- ),
- model_version: Optional[str] = typer.Option(
- None, "-V", "--model-version", help="Azure model version"
- ),
- deployment_name: Optional[str] = typer.Option(
- None, "-d", "--deployment-name", help="Azure OpenAI deployment name"
- ),
- save: Optional[str] = typer.Option(
- None,
- "-s",
- "--save",
- help="Persist CLI parameters as environment variables in a dotenv file. "
- "Usage: --save=dotenv[:path] (default: .env.local)",
- ),
- quiet: bool = typer.Option(
- False,
- "-q",
- "--quiet",
- help="Suppress printing to the terminal (useful for CI).",
- ),
-):
- api_key = None
- if prompt_api_key:
- api_key = coerce_blank_to_none(
- typer.prompt("Azure OpenAI API key", hide_input=True)
- )
-
- model = coerce_blank_to_none(model)
- base_url = coerce_blank_to_none(base_url)
- api_version = coerce_blank_to_none(api_version)
- deployment_name = coerce_blank_to_none(deployment_name)
- model_version = coerce_blank_to_none(model_version)
-
- settings = get_settings()
- with settings.edit(save=save) as edit_ctx:
- edit_ctx.switch_model_provider(ModelKeyValues.USE_AZURE_OPENAI)
- if model is not None:
- settings.AZURE_MODEL_NAME = model
- if api_key is not None:
- settings.AZURE_OPENAI_API_KEY = api_key
- if base_url is not None:
- settings.AZURE_OPENAI_ENDPOINT = base_url
- if api_version is not None:
- settings.OPENAI_API_VERSION = api_version
- if deployment_name is not None:
- settings.AZURE_DEPLOYMENT_NAME = deployment_name
- if model_version is not None:
- settings.AZURE_MODEL_VERSION = model_version
-
- handled, path, updates = edit_ctx.result
-
- effective_model = settings.AZURE_MODEL_NAME
- if not effective_model:
- raise typer.BadParameter(
- "Azure OpenAI model name is not set. Pass --model (or set AZURE_MODEL_NAME).",
- param_hint="--model",
- )
-
- _handle_save_result(
- handled=handled,
- path=path,
- updates=updates,
- save=save,
- quiet=quiet,
- success_msg=(
- f":raising_hands: Congratulations! You're now using Azure OpenAI's `{escape(effective_model)}` for all evals that require an LLM."
- ),
- )
-
-
-@app.command(name="unset-azure-openai")
-def unset_azure_openai_env(
- save: Optional[str] = typer.Option(
- None,
- "-s",
- "--save",
- help="Remove only the Azure OpenAI–related environment variables from a dotenv file. "
- "Usage: --save=dotenv[:path] (default: .env.local)",
- ),
- clear_secrets: bool = typer.Option(
- False,
- "-x",
- "--clear-secrets",
- help="Also remove AZURE_OPENAI_API_KEY from the dotenv store.",
- ),
- quiet: bool = typer.Option(
- False,
- "-q",
- "--quiet",
- help="Suppress printing to the terminal (useful for CI).",
- ),
-):
- settings = get_settings()
- with settings.edit(save=save) as edit_ctx:
- settings.AZURE_OPENAI_ENDPOINT = None
- settings.OPENAI_API_VERSION = None
- settings.AZURE_DEPLOYMENT_NAME = None
- settings.AZURE_MODEL_NAME = None
- settings.AZURE_MODEL_VERSION = None
- settings.USE_AZURE_OPENAI = None
- if clear_secrets:
- settings.AZURE_OPENAI_API_KEY = None
-
- handled, path, updates = edit_ctx.result
-
- if _handle_save_result(
- handled=handled,
- path=path,
- updates=updates,
- save=save,
- quiet=quiet,
- updated_msg="Removed Azure OpenAI environment variables from {path}.",
- tip_msg=None,
- ):
- if is_openai_configured():
- print(
- ":raised_hands: OpenAI will still be used by default because OPENAI_API_KEY is set."
- )
- else:
- print(
- "Azure OpenAI has been unset. No active provider is configured. Set one with the CLI, or add credentials to .env[.local]."
- )
-
-
-@app.command(name="set-azure-openai-embedding")
-def set_azure_openai_embedding_env(
- model: Optional[str] = typer.Option(
- None,
- "-m",
- "--model",
- help="Model identifier to use for this provider (e.g., `gpt-4.1`).",
- ),
- deployment_name: Optional[str] = typer.Option(
- None,
- "-d",
- "--deployment-name",
- help="Azure embedding deployment name",
- ),
- save: Optional[str] = typer.Option(
- None,
- "-s",
- "--save",
- help="Persist CLI parameters as environment variables in a dotenv file. "
- "Usage: --save=dotenv[:path] (default: .env.local)",
- ),
- quiet: bool = typer.Option(
- False,
- "-q",
- "--quiet",
- help="Suppress printing to the terminal (useful for CI).",
- ),
-):
- model = coerce_blank_to_none(model)
- deployment_name = coerce_blank_to_none(deployment_name)
-
- settings = get_settings()
- with settings.edit(save=save) as edit_ctx:
- edit_ctx.switch_model_provider(
- EmbeddingKeyValues.USE_AZURE_OPENAI_EMBEDDING
- )
- if model is not None:
- settings.AZURE_EMBEDDING_MODEL_NAME = model
- if deployment_name is not None:
- settings.AZURE_EMBEDDING_DEPLOYMENT_NAME = deployment_name
-
- handled, path, updates = edit_ctx.result
-
- effective_model = settings.AZURE_EMBEDDING_MODEL_NAME
- if not effective_model:
- raise typer.BadParameter(
- "Azure OpenAI embedding model name is not set. Pass --model (or set AZURE_EMBEDDING_MODEL_NAME).",
- param_hint="--model",
- )
-
- _handle_save_result(
- handled=handled,
- path=path,
- updates=updates,
- save=save,
- quiet=quiet,
- success_msg=(
- f":raising_hands: Congratulations! You're now using Azure OpenAI embedding model `{escape(effective_model)}` for all evals that require text embeddings."
- ),
- )
-
-
-@app.command(name="unset-azure-openai-embedding")
-def unset_azure_openai_embedding_env(
- save: Optional[str] = typer.Option(
- None,
- "-s",
- "--save",
- help="Remove only the Azure OpenAI embedding related environment variables from a dotenv file. "
- "Usage: --save=dotenv[:path] (default: .env.local)",
- ),
- quiet: bool = typer.Option(
- False,
- "-q",
- "--quiet",
- help="Suppress printing to the terminal (useful for CI).",
- ),
-):
- settings = get_settings()
- with settings.edit(save=save) as edit_ctx:
- settings.AZURE_EMBEDDING_MODEL_NAME = None
- settings.AZURE_EMBEDDING_DEPLOYMENT_NAME = None
- settings.USE_AZURE_OPENAI_EMBEDDING = None
-
- handled, path, updates = edit_ctx.result
-
- if _handle_save_result(
- handled=handled,
- path=path,
- updates=updates,
- save=save,
- quiet=quiet,
- updated_msg="Removed Azure OpenAI embedding environment variables from {path}.",
- tip_msg=None,
- ):
- if is_openai_configured():
- print(
- ":raised_hands: OpenAI will still be used by default because OPENAI_API_KEY is set."
- )
- else:
- print(
- "Azure OpenAI embedding has been unset. No active provider is configured. Set one with the CLI, or add credentials to .env[.local]."
- )
-
-
-#############################################
-# Anthropic Model Integration ###############
-#############################################
-
-
-@app.command(name="set-anthropic")
-def set_anthropic_model_env(
- model: Optional[str] = typer.Option(
- None,
- "-m",
- "--model",
- help="Model identifier to use for this provider",
- ),
- prompt_api_key: bool = typer.Option(
- False,
- "-k",
- "--prompt-api-key",
- help=(
- "Prompt for ANTHROPIC_API_KEY (input hidden). Not suitable for CI. "
- "If --save (or DEEPEVAL_DEFAULT_SAVE) is used, the key is written to dotenv in plaintext."
- ),
- ),
- cost_per_input_token: Optional[float] = typer.Option(
- None,
- "-i",
- "--cost-per-input-token",
- help=(
- "USD per input token override used for cost tracking. Preconfigured for known models; "
- "REQUIRED if you use a custom/unknown model."
- ),
- ),
- cost_per_output_token: Optional[float] = typer.Option(
- None,
- "-o",
- "--cost-per-output-token",
- help=(
- "USD per output token override used for cost tracking. Preconfigured for known models; "
- "REQUIRED if you use a custom/unknown model."
- ),
- ),
- save: Optional[str] = typer.Option(
- None,
- "-s",
- "--save",
- help="Persist CLI parameters as environment variables in a dotenv file. "
- "Usage: --save=dotenv[:path] (default: .env.local)",
- ),
- quiet: bool = typer.Option(
- False,
- "-q",
- "--quiet",
- help="Suppress printing to the terminal (useful for CI).",
- ),
-):
- api_key = None
- if prompt_api_key:
- api_key = coerce_blank_to_none(
- typer.prompt("Anthropic API key", hide_input=True)
- )
-
- model = coerce_blank_to_none(model)
-
- settings = get_settings()
- with settings.edit(save=save) as edit_ctx:
- edit_ctx.switch_model_provider(ModelKeyValues.USE_ANTHROPIC_MODEL)
- if api_key is not None:
- settings.ANTHROPIC_API_KEY = api_key
- if model is not None:
- settings.ANTHROPIC_MODEL_NAME = model
- if cost_per_input_token is not None:
- settings.ANTHROPIC_COST_PER_INPUT_TOKEN = cost_per_input_token
- if cost_per_output_token is not None:
- settings.ANTHROPIC_COST_PER_OUTPUT_TOKEN = cost_per_output_token
-
- handled, path, updates = edit_ctx.result
-
- effective_model = settings.ANTHROPIC_MODEL_NAME
- if not effective_model:
- raise typer.BadParameter(
- "Anthropic model name is not set. Pass --model (or set ANTHROPIC_MODEL_NAME).",
- param_hint="--model",
- )
-
- _handle_save_result(
- handled=handled,
- path=path,
- updates=updates,
- save=save,
- quiet=quiet,
- success_msg=(
- f":raising_hands: Congratulations! You're now using Anthropic `{escape(effective_model)}` for all evals that require an LLM."
- ),
- )
-
-
-@app.command(name="unset-anthropic")
-def unset_anthropic_model_env(
- save: Optional[str] = typer.Option(
- None,
- "-s",
- "--save",
- help="Remove only the Anthropic model related environment variables from a dotenv file. "
- "Usage: --save=dotenv[:path] (default: .env.local)",
- ),
- clear_secrets: bool = typer.Option(
- False,
- "-x",
- "--clear-secrets",
- help="Also remove ANTHROPIC_API_KEY from the dotenv store.",
- ),
- quiet: bool = typer.Option(
- False,
- "-q",
- "--quiet",
- help="Suppress printing to the terminal (useful for CI).",
- ),
-):
- settings = get_settings()
- with settings.edit(save=save) as edit_ctx:
- settings.USE_ANTHROPIC_MODEL = None
- settings.ANTHROPIC_MODEL_NAME = None
- settings.ANTHROPIC_COST_PER_INPUT_TOKEN = None
- settings.ANTHROPIC_COST_PER_OUTPUT_TOKEN = None
- if clear_secrets:
- settings.ANTHROPIC_API_KEY = None
-
- handled, path, updates = edit_ctx.result
-
- if _handle_save_result(
- handled=handled,
- path=path,
- updates=updates,
- save=save,
- quiet=quiet,
- updated_msg="Removed Anthropic model environment variables from {path}.",
- tip_msg=None,
- ):
- if is_openai_configured():
- print(
- ":raised_hands: OpenAI will still be used by default because OPENAI_API_KEY is set."
- )
- else:
- print(
- "The Anthropic model configuration has been removed. No model is currently configured, but you can set one with the CLI or add credentials to .env[.local]."
- )
-
-
-#############################################
-# AWS Bedrock Model Integration #############
-#############################################
-
-
-@app.command(name="set-bedrock")
-def set_bedrock_model_env(
- model: Optional[str] = typer.Option(
- None,
- "-m",
- "--model",
- help="Model identifier to use for this provider",
- ),
- prompt_credentials: bool = typer.Option(
- False,
- "-a",
- "--prompt-credentials",
- help=(
- "Prompt for AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY (secret access key input is hidden). Not suitable for CI. "
- "If --save (or DEEPEVAL_DEFAULT_SAVE) is used, credentials are written to dotenv in plaintext."
- ),
- ),
- region: Optional[str] = typer.Option(
- None,
- "-r",
- "--region",
- help="AWS region for bedrock (e.g., `us-east-1`).",
- ),
- cost_per_input_token: Optional[float] = typer.Option(
- None,
- "-i",
- "--cost-per-input-token",
- help=(
- "USD per input token override used for cost tracking. Preconfigured for known models; "
- "REQUIRED if you use a custom/unknown model."
- ),
- ),
- cost_per_output_token: Optional[float] = typer.Option(
- None,
- "-o",
- "--cost-per-output-token",
- help=(
- "USD per output token override used for cost tracking. Preconfigured for known models; "
- "REQUIRED if you use a custom/unknown model."
- ),
- ),
- save: Optional[str] = typer.Option(
- None,
- "-s",
- "--save",
- help="Persist CLI parameters as environment variables in a dotenv file. "
- "Usage: --save=dotenv[:path] (default: .env.local)",
- ),
- quiet: bool = typer.Option(
- False,
- "-q",
- "--quiet",
- help="Suppress printing to the terminal (useful for CI).",
- ),
-):
- access_key_id = None
- secret_access_key = None
- if prompt_credentials:
- access_key_id = coerce_blank_to_none(typer.prompt("AWS Access key Id"))
- secret_access_key = coerce_blank_to_none(
- typer.prompt("AWS Secret Access key", hide_input=True)
- )
-
- model = coerce_blank_to_none(model)
- region = coerce_blank_to_none(region)
-
- settings = get_settings()
- with settings.edit(save=save) as edit_ctx:
- edit_ctx.switch_model_provider(ModelKeyValues.USE_AWS_BEDROCK_MODEL)
- if access_key_id is not None:
- settings.AWS_ACCESS_KEY_ID = access_key_id
- if secret_access_key is not None:
- settings.AWS_SECRET_ACCESS_KEY = secret_access_key
- if model is not None:
- settings.AWS_BEDROCK_MODEL_NAME = model
- if region is not None:
- settings.AWS_BEDROCK_REGION = region
- if cost_per_input_token is not None:
- settings.AWS_BEDROCK_COST_PER_INPUT_TOKEN = cost_per_input_token
- if cost_per_output_token is not None:
- settings.AWS_BEDROCK_COST_PER_OUTPUT_TOKEN = cost_per_output_token
-
- handled, path, updates = edit_ctx.result
-
- effective_model = settings.AWS_BEDROCK_MODEL_NAME
- if not effective_model:
- raise typer.BadParameter(
- "AWS Bedrock model name is not set. Pass --model (or set AWS_BEDROCK_MODEL_NAME).",
- param_hint="--model",
- )
-
- _handle_save_result(
- handled=handled,
- path=path,
- updates=updates,
- save=save,
- quiet=quiet,
- success_msg=(
- f":raising_hands: Congratulations! You're now using AWS Bedrock `{escape(effective_model)}` for all evals that require an LLM."
- ),
- )
-
-
-@app.command(name="unset-bedrock")
-def unset_bedrock_model_env(
- save: Optional[str] = typer.Option(
- None,
- "-s",
- "--save",
- help="Remove only the AWS Bedrock model related environment variables from a dotenv file. "
- "Usage: --save=dotenv[:path] (default: .env.local)",
- ),
- clear_secrets: bool = typer.Option(
- False,
- "-x",
- "--clear-secrets",
- help="Also remove AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY from the dotenv store.",
- ),
- quiet: bool = typer.Option(
- False,
- "-q",
- "--quiet",
- help="Suppress printing to the terminal (useful for CI).",
- ),
-):
- settings = get_settings()
- with settings.edit(save=save) as edit_ctx:
- settings.USE_AWS_BEDROCK_MODEL = None
- settings.AWS_BEDROCK_MODEL_NAME = None
- settings.AWS_BEDROCK_REGION = None
- settings.AWS_BEDROCK_COST_PER_INPUT_TOKEN = None
- settings.AWS_BEDROCK_COST_PER_OUTPUT_TOKEN = None
- if clear_secrets:
- settings.AWS_ACCESS_KEY_ID = None
- settings.AWS_SECRET_ACCESS_KEY = None
-
- handled, path, updates = edit_ctx.result
-
- if _handle_save_result(
- handled=handled,
- path=path,
- updates=updates,
- save=save,
- quiet=quiet,
- updated_msg="Removed AWS Bedrock model environment variables from {path}.",
- tip_msg=None,
- ):
- if is_openai_configured():
- print(
- ":raised_hands: OpenAI will still be used by default because OPENAI_API_KEY is set."
- )
- else:
- print(
- "The AWS Bedrock model configuration has been removed. No model is currently configured, but you can set one with the CLI or add credentials to .env[.local]."
- )
-
-
-#############################################
-# Ollama Integration ########################
-#############################################
-
-
-@app.command(name="set-ollama")
-def set_ollama_model_env(
- model: Optional[str] = typer.Option(
- None,
- "-m",
- "--model",
- help="Model identifier to use for this provider",
- ),
- base_url: str = typer.Option(
- "http://localhost:11434",
- "-u",
- "--base-url",
- help="Override the API endpoint/base URL used by this provider.",
- ),
- save: Optional[str] = typer.Option(
- None,
- "-s",
- "--save",
- help="Persist CLI parameters as environment variables in a dotenv file. "
- "Usage: --save=dotenv[:path] (default: .env.local)",
- ),
- quiet: bool = typer.Option(
- False,
- "-q",
- "--quiet",
- help="Suppress printing to the terminal (useful for CI).",
- ),
-):
- model = coerce_blank_to_none(model)
- base_url = coerce_blank_to_none(base_url)
-
- settings = get_settings()
- with settings.edit(save=save) as edit_ctx:
- edit_ctx.switch_model_provider(ModelKeyValues.USE_LOCAL_MODEL)
- settings.LOCAL_MODEL_API_KEY = "ollama"
- if model is not None:
- settings.OLLAMA_MODEL_NAME = model
- if base_url is not None:
- settings.LOCAL_MODEL_BASE_URL = base_url
-
- handled, path, updates = edit_ctx.result
-
- effective_model = settings.OLLAMA_MODEL_NAME
- if not effective_model:
- raise typer.BadParameter(
- "Ollama model name is not set. Pass --model (or set OLLAMA_MODEL_NAME).",
- param_hint="--model",
- )
- _handle_save_result(
- handled=handled,
- path=path,
- updates=updates,
- save=save,
- quiet=quiet,
- success_msg=(
- f":raising_hands: Congratulations! You're now using a local Ollama model `{escape(effective_model)}` for all evals that require an LLM."
- ),
- )
-
-
-@app.command(name="unset-ollama")
-def unset_ollama_model_env(
- save: Optional[str] = typer.Option(
- None,
- "-s",
- "--save",
- help="Remove only the Ollama related environment variables from a dotenv file. "
- "Usage: --save=dotenv[:path] (default: .env.local)",
- ),
- clear_secrets: bool = typer.Option(
- False,
- "-x",
- "--clear-secrets",
- help="Also remove LOCAL_MODEL_API_KEY from the dotenv store.",
- ),
- quiet: bool = typer.Option(
- False,
- "-q",
- "--quiet",
- help="Suppress printing to the terminal (useful for CI).",
- ),
-):
- settings = get_settings()
- with settings.edit(save=save) as edit_ctx:
- if clear_secrets:
- settings.LOCAL_MODEL_API_KEY = None
- settings.OLLAMA_MODEL_NAME = None
- settings.LOCAL_MODEL_BASE_URL = None
- settings.USE_LOCAL_MODEL = None
-
- handled, path, updates = edit_ctx.result
-
- if _handle_save_result(
- handled=handled,
- path=path,
- updates=updates,
- save=save,
- quiet=quiet,
- updated_msg="Removed local Ollama environment variables from {path}.",
- tip_msg=None,
- ):
- if is_openai_configured():
- print(
- ":raised_hands: OpenAI will still be used by default because OPENAI_API_KEY is set."
- )
- else:
- print(
- "The local Ollama model configuration has been removed. No model is currently configured, but you can set one with the CLI or add credentials to .env[.local]."
- )
-
-
-@app.command(name="set-ollama-embeddings")
-def set_ollama_embeddings_env(
- model: Optional[str] = typer.Option(
- None,
- "-m",
- "--model",
- help="Model identifier to use for this provider.",
- ),
- base_url: str = typer.Option(
- "http://localhost:11434",
- "-u",
- "--base-url",
- help="Override the API endpoint/base URL used by this provider.",
- ),
- save: Optional[str] = typer.Option(
- None,
- "-s",
- "--save",
- help="Persist CLI parameters as environment variables in a dotenv file. "
- "Usage: --save=dotenv[:path] (default: .env.local)",
- ),
- quiet: bool = typer.Option(
- False,
- "-q",
- "--quiet",
- help="Suppress printing to the terminal (useful for CI).",
- ),
-):
- model = coerce_blank_to_none(model)
- base_url = coerce_blank_to_none(base_url)
-
- settings = get_settings()
- with settings.edit(save=save) as edit_ctx:
- edit_ctx.switch_model_provider(EmbeddingKeyValues.USE_LOCAL_EMBEDDINGS)
- settings.LOCAL_EMBEDDING_API_KEY = "ollama"
- if model is not None:
- settings.LOCAL_EMBEDDING_MODEL_NAME = model
- if base_url is not None:
- settings.LOCAL_EMBEDDING_BASE_URL = base_url
-
- handled, path, updates = edit_ctx.result
-
- effective_model = settings.LOCAL_EMBEDDING_MODEL_NAME
- if not effective_model:
- raise typer.BadParameter(
- "Ollama embedding model name is not set. Pass --model (or set LOCAL_EMBEDDING_MODEL_NAME).",
- param_hint="--model",
- )
- _handle_save_result(
- handled=handled,
- path=path,
- updates=updates,
- save=save,
- quiet=quiet,
- success_msg=(
- f":raising_hands: Congratulations! You're now using the Ollama embedding model `{escape(effective_model)}` for all evals that require text embeddings."
- ),
- )
-
-
-@app.command(name="unset-ollama-embeddings")
-def unset_ollama_embeddings_env(
- save: Optional[str] = typer.Option(
- None,
- "-s",
- "--save",
- help="Remove only the Ollama embedding related environment variables from a dotenv file. "
- "Usage: --save=dotenv[:path] (default: .env.local)",
- ),
- clear_secrets: bool = typer.Option(
- False,
- "-x",
- "--clear-secrets",
- help="Also remove LOCAL_EMBEDDING_API_KEY from the dotenv store.",
- ),
- quiet: bool = typer.Option(
- False,
- "-q",
- "--quiet",
- help="Suppress printing to the terminal (useful for CI).",
- ),
-):
- settings = get_settings()
- with settings.edit(save=save) as edit_ctx:
- if clear_secrets:
- settings.LOCAL_EMBEDDING_API_KEY = None
- settings.LOCAL_EMBEDDING_MODEL_NAME = None
- settings.LOCAL_EMBEDDING_BASE_URL = None
- settings.USE_LOCAL_EMBEDDINGS = None
-
- handled, path, updates = edit_ctx.result
-
- if _handle_save_result(
- handled=handled,
- path=path,
- updates=updates,
- save=save,
- quiet=quiet,
- updated_msg="Removed local Ollama embedding environment variables from {path}.",
- tip_msg=None,
- ):
- if is_openai_configured():
- print(
- ":raised_hands: Regular OpenAI embeddings will still be used by default because OPENAI_API_KEY is set."
- )
- else:
- print(
- "The local Ollama embedding model configuration has been removed. No model is currently configured, but you can set one with the CLI or add credentials to .env[.local]."
- )
-
-
-#############################################
-# Local Model Integration ###################
-#############################################
-
-
-@app.command(name="set-local-model")
-def set_local_model_env(
- model: Optional[str] = typer.Option(
- None,
- "-m",
- "--model",
- help="Model identifier to use for this provider",
- ),
- prompt_api_key: bool = typer.Option(
- False,
- "-k",
- "--prompt-api-key",
- help=(
- "Prompt for LOCAL_MODEL_API_KEY (input hidden). Not suitable for CI. "
- "If --save (or DEEPEVAL_DEFAULT_SAVE) is used, the key is written to dotenv in plaintext."
- ),
- ),
- base_url: Optional[str] = typer.Option(
- None,
- "-u",
- "--base-url",
- help="Override the API endpoint/base URL used by this provider.",
- ),
- model_format: Optional[str] = typer.Option(
- None,
- "-f",
- "--format",
- help="Format of the response from the local model (default: json)",
- ),
- save: Optional[str] = typer.Option(
- None,
- "-s",
- "--save",
- help="Persist CLI parameters as environment variables in a dotenv file. "
- "Usage: --save=dotenv[:path] (default: .env.local)",
- ),
- quiet: bool = typer.Option(
- False,
- "-q",
- "--quiet",
- help="Suppress printing to the terminal (useful for CI).",
- ),
-):
- api_key = None
- if prompt_api_key:
- api_key = coerce_blank_to_none(
- typer.prompt("Local Model API key", hide_input=True)
- )
-
- model = coerce_blank_to_none(model)
- base_url = coerce_blank_to_none(base_url)
- model_format = coerce_blank_to_none(model_format)
-
- settings = get_settings()
- with settings.edit(save=save) as edit_ctx:
- edit_ctx.switch_model_provider(ModelKeyValues.USE_LOCAL_MODEL)
- if model is not None:
- settings.LOCAL_MODEL_NAME = model
- if base_url is not None:
- settings.LOCAL_MODEL_BASE_URL = base_url
- if api_key is not None:
- settings.LOCAL_MODEL_API_KEY = api_key
- if model_format is not None:
- settings.LOCAL_MODEL_FORMAT = model_format
-
- handled, path, updates = edit_ctx.result
-
- effective_model = settings.LOCAL_MODEL_NAME
- if not effective_model:
- raise typer.BadParameter(
- "Local model name is not set. Pass --model (or set LOCAL_MODEL_NAME).",
- param_hint="--model",
- )
- _handle_save_result(
- handled=handled,
- path=path,
- updates=updates,
- save=save,
- quiet=quiet,
- success_msg=(
- f":raising_hands: Congratulations! You're now using a local model `{escape(effective_model)}` for all evals that require an LLM."
- ),
- )
-
-
-@app.command(name="unset-local-model")
-def unset_local_model_env(
- save: Optional[str] = typer.Option(
- None,
- "-s",
- "--save",
- help="Remove only the local model related environment variables from a dotenv file. "
- "Usage: --save=dotenv[:path] (default: .env.local)",
- ),
- clear_secrets: bool = typer.Option(
- False,
- "-x",
- "--clear-secrets",
- help="Also remove LOCAL_MODEL_API_KEY from the dotenv store.",
- ),
- quiet: bool = typer.Option(
- False,
- "-q",
- "--quiet",
- help="Suppress printing to the terminal (useful for CI).",
- ),
-):
- settings = get_settings()
- with settings.edit(save=save) as edit_ctx:
- if clear_secrets:
- settings.LOCAL_MODEL_API_KEY = None
- settings.LOCAL_MODEL_NAME = None
- settings.LOCAL_MODEL_BASE_URL = None
- settings.LOCAL_MODEL_FORMAT = None
- settings.USE_LOCAL_MODEL = None
-
- handled, path, updates = edit_ctx.result
-
- if _handle_save_result(
- handled=handled,
- path=path,
- updates=updates,
- save=save,
- quiet=quiet,
- updated_msg="Removed local model environment variables from {path}.",
- tip_msg=None,
- ):
- if is_openai_configured():
- print(
- ":raised_hands: OpenAI will still be used by default because OPENAI_API_KEY is set."
- )
- else:
- print(
- "The local model configuration has been removed. No model is currently configured, but you can set one with the CLI or add credentials to .env[.local]."
- )
-
-
-#############################################
-# Grok Model Integration ####################
-#############################################
-
-
-@app.command(name="set-grok")
-def set_grok_model_env(
- model: Optional[str] = typer.Option(
- None,
- "-m",
- "--model",
- help="Model identifier to use for this provider",
- ),
- prompt_api_key: bool = typer.Option(
- False,
- "-k",
- "--prompt-api-key",
- help=(
- "Prompt for GROK_API_KEY (input hidden). Not suitable for CI. "
- "If --save (or DEEPEVAL_DEFAULT_SAVE) is used, the key is written to dotenv in plaintext."
- ),
- ),
- cost_per_input_token: Optional[float] = typer.Option(
- None,
- "-i",
- "--cost-per-input-token",
- help=(
- "USD per input token override used for cost tracking. Preconfigured for known models; "
- "REQUIRED if you use a custom/unknown model."
- ),
- ),
- cost_per_output_token: Optional[float] = typer.Option(
- None,
- "-o",
- "--cost-per-output-token",
- help=(
- "USD per output token override used for cost tracking. Preconfigured for known models; "
- "REQUIRED if you use a custom/unknown model."
- ),
- ),
- save: Optional[str] = typer.Option(
- None,
- "-s",
- "--save",
- help="Persist CLI parameters as environment variables in a dotenv file. "
- "Usage: --save=dotenv[:path] (default: .env.local)",
- ),
- quiet: bool = typer.Option(
- False,
- "-q",
- "--quiet",
- help="Suppress printing to the terminal (useful for CI).",
- ),
-):
- api_key = None
- if prompt_api_key:
- api_key = coerce_blank_to_none(
- typer.prompt("Grok API key", hide_input=True)
- )
-
- model = coerce_blank_to_none(model)
-
- settings = get_settings()
- with settings.edit(save=save) as edit_ctx:
- edit_ctx.switch_model_provider(ModelKeyValues.USE_GROK_MODEL)
- if api_key is not None:
- settings.GROK_API_KEY = api_key
- if model is not None:
- settings.GROK_MODEL_NAME = model
- if cost_per_input_token is not None:
- settings.GROK_COST_PER_INPUT_TOKEN = cost_per_input_token
- if cost_per_output_token is not None:
- settings.GROK_COST_PER_OUTPUT_TOKEN = cost_per_output_token
-
- handled, path, updates = edit_ctx.result
-
- effective_model = settings.GROK_MODEL_NAME
- if not effective_model:
- raise typer.BadParameter(
- "Grok model name is not set. Pass --model (or set GROK_MODEL_NAME).",
- param_hint="--model",
- )
- _handle_save_result(
- handled=handled,
- path=path,
- updates=updates,
- save=save,
- quiet=quiet,
- success_msg=(
- f":raising_hands: Congratulations! You're now using Grok `{escape(effective_model)}` for all evals that require an LLM."
- ),
- )
-
-
-@app.command(name="unset-grok")
-def unset_grok_model_env(
- save: Optional[str] = typer.Option(
- None,
- "-s",
- "--save",
- help="Remove only the Grok model related environment variables from a dotenv file. "
- "Usage: --save=dotenv[:path] (default: .env.local)",
- ),
- clear_secrets: bool = typer.Option(
- False,
- "-x",
- "--clear-secrets",
- help="Also remove GROK_API_KEY from the dotenv store.",
- ),
- quiet: bool = typer.Option(
- False,
- "-q",
- "--quiet",
- help="Suppress printing to the terminal (useful for CI).",
- ),
-):
- settings = get_settings()
- with settings.edit(save=save) as edit_ctx:
- settings.USE_GROK_MODEL = None
- settings.GROK_MODEL_NAME = None
- settings.GROK_COST_PER_INPUT_TOKEN = None
- settings.GROK_COST_PER_OUTPUT_TOKEN = None
- if clear_secrets:
- settings.GROK_API_KEY = None
-
- handled, path, updates = edit_ctx.result
-
- if _handle_save_result(
- handled=handled,
- path=path,
- updates=updates,
- save=save,
- quiet=quiet,
- updated_msg="Removed Grok model environment variables from {path}.",
- tip_msg=None,
- ):
- if is_openai_configured():
- print(
- ":raised_hands: OpenAI will still be used by default because OPENAI_API_KEY is set."
- )
- else:
- print(
- "The Grok model configuration has been removed. No model is currently configured, but you can set one with the CLI or add credentials to .env[.local]."
- )
-
-
-#############################################
-# Moonshot Model Integration ################
-#############################################
-
-
-@app.command(name="set-moonshot")
-def set_moonshot_model_env(
- model: Optional[str] = typer.Option(
- None,
- "-m",
- "--model",
- help="Model identifier to use for this provider",
- ),
- prompt_api_key: bool = typer.Option(
- False,
- "-k",
- "--prompt-api-key",
- help=(
- "Prompt for MOONSHOT_API_KEY (input hidden). Not suitable for CI. "
- "If --save (or DEEPEVAL_DEFAULT_SAVE) is used, the key is written to dotenv in plaintext."
- ),
- ),
- cost_per_input_token: Optional[float] = typer.Option(
- None,
- "-i",
- "--cost-per-input-token",
- help=(
- "USD per input token override used for cost tracking. Preconfigured for known models; "
- "REQUIRED if you use a custom/unknown model."
- ),
- ),
- cost_per_output_token: Optional[float] = typer.Option(
- None,
- "-o",
- "--cost-per-output-token",
- help=(
- "USD per output token override used for cost tracking. Preconfigured for known models; "
- "REQUIRED if you use a custom/unknown model."
- ),
- ),
- save: Optional[str] = typer.Option(
- None,
- "-s",
- "--save",
- help="Persist CLI parameters as environment variables in a dotenv file. "
- "Usage: --save=dotenv[:path] (default: .env.local)",
- ),
- quiet: bool = typer.Option(
- False,
- "-q",
- "--quiet",
- help="Suppress printing to the terminal (useful for CI).",
- ),
-):
- api_key = None
- if prompt_api_key:
- api_key = coerce_blank_to_none(
- typer.prompt("Moonshot API key", hide_input=True)
- )
-
- model = coerce_blank_to_none(model)
-
- settings = get_settings()
- with settings.edit(save=save) as edit_ctx:
- edit_ctx.switch_model_provider(ModelKeyValues.USE_MOONSHOT_MODEL)
- if model is not None:
- settings.MOONSHOT_MODEL_NAME = model
- if api_key is not None:
- settings.MOONSHOT_API_KEY = api_key
- if cost_per_input_token is not None:
- settings.MOONSHOT_COST_PER_INPUT_TOKEN = cost_per_input_token
- if cost_per_output_token is not None:
- settings.MOONSHOT_COST_PER_OUTPUT_TOKEN = cost_per_output_token
-
- handled, path, updates = edit_ctx.result
-
- effective_model = settings.MOONSHOT_MODEL_NAME
- if not effective_model:
- raise typer.BadParameter(
- "Moonshot model name is not set. Pass --model (or set MOONSHOT_MODEL_NAME).",
- param_hint="--model",
- )
- _handle_save_result(
- handled=handled,
- path=path,
- updates=updates,
- save=save,
- quiet=quiet,
- success_msg=(
- f":raising_hands: Congratulations! You're now using Moonshot `{escape(effective_model)}` for all evals that require an LLM."
- ),
- )
-
-
-@app.command(name="unset-moonshot")
-def unset_moonshot_model_env(
- save: Optional[str] = typer.Option(
- None,
- "-s",
- "--save",
- help="Remove only the Moonshot model related environment variables from a dotenv file. "
- "Usage: --save=dotenv[:path] (default: .env.local)",
- ),
- clear_secrets: bool = typer.Option(
- False,
- "-x",
- "--clear-secrets",
- help="Also remove MOONSHOT_API_KEY from the dotenv store.",
- ),
- quiet: bool = typer.Option(
- False,
- "-q",
- "--quiet",
- help="Suppress printing to the terminal (useful for CI).",
- ),
-):
- settings = get_settings()
- with settings.edit(save=save) as edit_ctx:
- settings.USE_MOONSHOT_MODEL = None
- settings.MOONSHOT_MODEL_NAME = None
- settings.MOONSHOT_COST_PER_INPUT_TOKEN = None
- settings.MOONSHOT_COST_PER_OUTPUT_TOKEN = None
- if clear_secrets:
- settings.MOONSHOT_API_KEY = None
-
- handled, path, updates = edit_ctx.result
-
- if _handle_save_result(
- handled=handled,
- path=path,
- updates=updates,
- save=save,
- quiet=quiet,
- updated_msg="Removed Moonshot model environment variables from {path}.",
- tip_msg=None,
- ):
- if is_openai_configured():
- print(
- ":raised_hands: OpenAI will still be used by default because OPENAI_API_KEY is set."
- )
- else:
- print(
- "The Moonshot model configuration has been removed. No model is currently configured, but you can set one with the CLI or add credentials to .env[.local]."
- )
-
-
-#############################################
-# DeepSeek Model Integration ################
-#############################################
-
-
-@app.command(name="set-deepseek")
-def set_deepseek_model_env(
- model: Optional[str] = typer.Option(
- None,
- "-m",
- "--model",
- help="Model identifier to use for this provider",
- ),
- prompt_api_key: bool = typer.Option(
- False,
- "-k",
- "--prompt-api-key",
- help=(
- "Prompt for DEEPSEEK_API_KEY (input hidden). Not suitable for CI. "
- "If --save (or DEEPEVAL_DEFAULT_SAVE) is used, the key is written to dotenv in plaintext."
- ),
- ),
- cost_per_input_token: Optional[float] = typer.Option(
- None,
- "-i",
- "--cost-per-input-token",
- help=(
- "USD per input token override used for cost tracking. Preconfigured for known models; "
- "REQUIRED if you use a custom/unknown model."
- ),
- ),
- cost_per_output_token: Optional[float] = typer.Option(
- None,
- "-o",
- "--cost-per-output-token",
- help=(
- "USD per output token override used for cost tracking. Preconfigured for known models; "
- "REQUIRED if you use a custom/unknown model."
- ),
- ),
- save: Optional[str] = typer.Option(
- None,
- "-s",
- "--save",
- help="Persist CLI parameters as environment variables in a dotenv file. "
- "Usage: --save=dotenv[:path] (default: .env.local)",
- ),
- quiet: bool = typer.Option(
- False,
- "-q",
- "--quiet",
- help="Suppress printing to the terminal (useful for CI).",
- ),
-):
- api_key = None
- if prompt_api_key:
- api_key = coerce_blank_to_none(
- typer.prompt("DeepSeek API key", hide_input=True)
- )
-
- model = coerce_blank_to_none(model)
-
- settings = get_settings()
- with settings.edit(save=save) as edit_ctx:
- edit_ctx.switch_model_provider(ModelKeyValues.USE_DEEPSEEK_MODEL)
- if model is not None:
- settings.DEEPSEEK_MODEL_NAME = model
- if api_key is not None:
- settings.DEEPSEEK_API_KEY = api_key
- if cost_per_input_token is not None:
- settings.DEEPSEEK_COST_PER_INPUT_TOKEN = cost_per_input_token
- if cost_per_output_token is not None:
- settings.DEEPSEEK_COST_PER_OUTPUT_TOKEN = cost_per_output_token
-
- handled, path, updates = edit_ctx.result
-
- effective_model = settings.DEEPSEEK_MODEL_NAME
- if not effective_model:
- raise typer.BadParameter(
- "DeepSeek model name is not set. Pass --model (or set DEEPSEEK_MODEL_NAME).",
- param_hint="--model",
- )
-
- _handle_save_result(
- handled=handled,
- path=path,
- updates=updates,
- save=save,
- quiet=quiet,
- success_msg=(
- f":raising_hands: Congratulations! You're now using DeepSeek `{escape(effective_model)}` for all evals that require an LLM."
- ),
- )
-
-
-@app.command(name="unset-deepseek")
-def unset_deepseek_model_env(
- save: Optional[str] = typer.Option(
- None,
- "-s",
- "--save",
- help="Remove only the DeepSeek model related environment variables from a dotenv file. "
- "Usage: --save=dotenv[:path] (default: .env.local)",
- ),
- clear_secrets: bool = typer.Option(
- False,
- "-x",
- "--clear-secrets",
- help="Also remove DEEPSEEK_API_KEY from the dotenv store.",
- ),
- quiet: bool = typer.Option(
- False,
- "-q",
- "--quiet",
- help="Suppress printing to the terminal (useful for CI).",
- ),
-):
- settings = get_settings()
- with settings.edit(save=save) as edit_ctx:
- settings.USE_DEEPSEEK_MODEL = None
- settings.DEEPSEEK_MODEL_NAME = None
- settings.DEEPSEEK_COST_PER_INPUT_TOKEN = None
- settings.DEEPSEEK_COST_PER_OUTPUT_TOKEN = None
- if clear_secrets:
- settings.DEEPSEEK_API_KEY = None
-
- handled, path, updates = edit_ctx.result
-
- if _handle_save_result(
- handled=handled,
- path=path,
- updates=updates,
- save=save,
- quiet=quiet,
- updated_msg="Removed DeepSeek model environment variables from {path}.",
- tip_msg=None,
- ):
- if is_openai_configured():
- print(
- ":raised_hands: OpenAI will still be used by default because OPENAI_API_KEY is set."
- )
- else:
- print(
- "The DeepSeek model configuration has been removed. No model is currently configured, but you can set one with the CLI or add credentials to .env[.local]."
- )
-
-
-#############################################
-# Local Embedding Model Integration #########
-#############################################
-
-
-@app.command(name="set-local-embeddings")
-def set_local_embeddings_env(
- model: Optional[str] = typer.Option(
- None,
- "-m",
- "--model",
- help="Model identifier to use for this provider",
- ),
- prompt_api_key: bool = typer.Option(
- False,
- "-k",
- "--prompt-api-key",
- help=(
- "Prompt for LOCAL_EMBEDDING_API_KEY (input hidden). Not suitable for CI. "
- "If --save (or DEEPEVAL_DEFAULT_SAVE) is used, the key is written to dotenv in plaintext."
- ),
- ),
- base_url: Optional[str] = typer.Option(
- None,
- "-u",
- "--base-url",
- help="Override the API endpoint/base URL used by this provider.",
- ),
- save: Optional[str] = typer.Option(
- None,
- "-s",
- "--save",
- help="Persist CLI parameters as environment variables in a dotenv file. "
- "Usage: --save=dotenv[:path] (default: .env.local)",
- ),
- quiet: bool = typer.Option(
- False,
- "-q",
- "--quiet",
- help="Suppress printing to the terminal (useful for CI).",
- ),
-):
- api_key = None
- if prompt_api_key:
- api_key = coerce_blank_to_none(
- typer.prompt("Local Embedding Model API key", hide_input=True)
- )
-
- model = coerce_blank_to_none(model)
- base_url = coerce_blank_to_none(base_url)
-
- settings = get_settings()
- with settings.edit(save=save) as edit_ctx:
- edit_ctx.switch_model_provider(EmbeddingKeyValues.USE_LOCAL_EMBEDDINGS)
- if model is not None:
- settings.LOCAL_EMBEDDING_MODEL_NAME = model
- if base_url is not None:
- settings.LOCAL_EMBEDDING_BASE_URL = base_url
- if api_key is not None:
- settings.LOCAL_EMBEDDING_API_KEY = api_key
-
- handled, path, updates = edit_ctx.result
-
- effective_model = settings.LOCAL_EMBEDDING_MODEL_NAME
- if not effective_model:
- raise typer.BadParameter(
- "Local embedding model name is not set. Pass --model (or set LOCAL_EMBEDDING_MODEL_NAME).",
- param_hint="--model",
- )
- _handle_save_result(
- handled=handled,
- path=path,
- updates=updates,
- save=save,
- quiet=quiet,
- success_msg=(
- f":raising_hands: Congratulations! You're now using the local embedding model `{escape(effective_model)}` for all evals that require text embeddings."
- ),
- )
-
-
-@app.command(name="unset-local-embeddings")
-def unset_local_embeddings_env(
- save: Optional[str] = typer.Option(
- None,
- "-s",
- "--save",
- help="Remove only the local embedding related environment variables from a dotenv file. "
- "Usage: --save=dotenv[:path] (default: .env.local)",
- ),
- clear_secrets: bool = typer.Option(
- False,
- "-x",
- "--clear-secrets",
- help="Also remove LOCAL_MODEL_API_KEY from the dotenv store.",
- ),
- quiet: bool = typer.Option(
- False,
- "-q",
- "--quiet",
- help="Suppress printing to the terminal (useful for CI).",
- ),
-):
- settings = get_settings()
- with settings.edit(save=save) as edit_ctx:
- settings.LOCAL_EMBEDDING_MODEL_NAME = None
- settings.LOCAL_EMBEDDING_BASE_URL = None
- settings.USE_LOCAL_EMBEDDINGS = None
- if clear_secrets:
- settings.LOCAL_EMBEDDING_API_KEY = None
-
- handled, path, updates = edit_ctx.result
-
- if _handle_save_result(
- handled=handled,
- path=path,
- updates=updates,
- save=save,
- quiet=quiet,
- updated_msg="Removed local embedding environment variables from {path}.",
- tip_msg=None,
- ):
- if is_openai_configured():
- print(
- ":raised_hands: OpenAI will still be used by default because OPENAI_API_KEY is set."
- )
- else:
- print(
- "The local embeddings model configuration has been removed. No model is currently configured, but you can set one with the CLI or add credentials to .env[.local]."
- )
-
-
-#############################################
-# Gemini Integration ########################
-#############################################
-
-
-@app.command(name="set-gemini")
-def set_gemini_model_env(
- model: Optional[str] = typer.Option(
- None,
- "-m",
- "--model",
- help="Model identifier to use for this provider",
- ),
- prompt_api_key: bool = typer.Option(
- False,
- "-k",
- "--prompt-api-key",
- help=(
- "Prompt for GOOGLE_API_KEY (input hidden). Not suitable for CI. "
- "If --save (or DEEPEVAL_DEFAULT_SAVE) is used, the key is written to dotenv in plaintext."
- ),
- ),
- project: Optional[str] = typer.Option(
- None,
- "-p",
- "--project",
- help="GCP project ID (used by Vertex AI / Gemini when applicable).",
- ),
- location: Optional[str] = typer.Option(
- None,
- "-l",
- "--location",
- help="GCP location/region for Vertex AI (e.g., `us-central1`).",
- ),
- service_account_file: Optional[Path] = typer.Option(
- None,
- "-S",
- "--service-account-file",
- help=("Path to a Google service account JSON key file."),
- exists=True,
- dir_okay=False,
- readable=True,
- ),
- save: Optional[str] = typer.Option(
- None,
- "-s",
- "--save",
- help="Persist CLI parameters as environment variables in a dotenv file. "
- "Usage: --save=dotenv[:path] (default: .env.local)",
- ),
- quiet: bool = typer.Option(
- False,
- "-q",
- "--quiet",
- help="Suppress printing to the terminal (useful for CI).",
- ),
-):
- api_key = None
- if prompt_api_key:
- api_key = coerce_blank_to_none(
- typer.prompt("Google API key", hide_input=True)
- )
-
- model = coerce_blank_to_none(model)
- project = coerce_blank_to_none(project)
- location = coerce_blank_to_none(location)
-
- settings = get_settings()
- with settings.edit(save=save) as edit_ctx:
- edit_ctx.switch_model_provider(ModelKeyValues.USE_GEMINI_MODEL)
-
- if model is not None:
- settings.GEMINI_MODEL_NAME = model
- if project is not None:
- settings.GOOGLE_CLOUD_PROJECT = project
- if location is not None:
- settings.GOOGLE_CLOUD_LOCATION = location
- if service_account_file is not None:
- settings.GOOGLE_SERVICE_ACCOUNT_KEY = load_service_account_key_file(
- service_account_file
- )
- if api_key is not None:
- settings.GOOGLE_API_KEY = api_key
- settings.GOOGLE_GENAI_USE_VERTEXAI = False
- elif (
- project is not None
- or location is not None
- or service_account_file is not None
- ):
- settings.GOOGLE_GENAI_USE_VERTEXAI = True
-
- handled, path, updates = edit_ctx.result
-
- effective_model = settings.GEMINI_MODEL_NAME
- if not effective_model:
- raise typer.BadParameter(
- "Gemini model name is not set. Pass --model (or set GEMINI_MODEL_NAME).",
- param_hint="--model",
- )
- _handle_save_result(
- handled=handled,
- path=path,
- updates=updates,
- save=save,
- quiet=quiet,
- success_msg=(
- f":raising_hands: Congratulations! You're now using Gemini `{escape(effective_model)}` for all evals that require an LLM."
- ),
- )
-
-
-@app.command(name="unset-gemini")
-def unset_gemini_model_env(
- save: Optional[str] = typer.Option(
- None,
- "-s",
- "--save",
- help="Remove only the Gemini related environment variables from a dotenv file. "
- "Usage: --save=dotenv[:path] (default: .env.local)",
- ),
- clear_secrets: bool = typer.Option(
- False,
- "-x",
- "--clear-secrets",
- help="Also remove GOOGLE_API_KEY and GOOGLE_SERVICE_ACCOUNT_KEY from the dotenv store.",
- ),
- quiet: bool = typer.Option(
- False,
- "-q",
- "--quiet",
- help="Suppress printing to the terminal (useful for CI).",
- ),
-):
- settings = get_settings()
- with settings.edit(save=save) as edit_ctx:
- settings.USE_GEMINI_MODEL = None
- settings.GOOGLE_GENAI_USE_VERTEXAI = None
- settings.GOOGLE_CLOUD_PROJECT = None
- settings.GOOGLE_CLOUD_LOCATION = None
- settings.GEMINI_MODEL_NAME = None
- if clear_secrets:
- settings.GOOGLE_API_KEY = None
- settings.GOOGLE_SERVICE_ACCOUNT_KEY = None
-
- handled, path, updates = edit_ctx.result
-
- if _handle_save_result(
- handled=handled,
- path=path,
- updates=updates,
- save=save,
- quiet=quiet,
- updated_msg="Removed Gemini model environment variables from {path}.",
- tip_msg=None,
- ):
- if is_openai_configured():
- print(
- ":raised_hands: OpenAI will still be used by default because OPENAI_API_KEY is set."
- )
- else:
- print(
- "The Gemini model configuration has been removed. No model is currently configured, but you can set one with the CLI or add credentials to .env[.local]."
- )
-
-
-@app.command(name="set-litellm")
-def set_litellm_model_env(
- model: Optional[str] = typer.Option(
- None,
- "-m",
- "--model",
- help="Model identifier to use for this provider",
- ),
- prompt_api_key: bool = typer.Option(
- False,
- "-k",
- "--prompt-api-key",
- help=(
- "Prompt for LITELLM_API_KEY (input hidden). Not suitable for CI. "
- "If --save (or DEEPEVAL_DEFAULT_SAVE) is used, the key is written to dotenv in plaintext."
- ),
- ),
- base_url: Optional[str] = typer.Option(
- None,
- "-u",
- "--base-url",
- help="Override the API endpoint/base URL used by this provider.",
- ),
- proxy_prompt_api_key: bool = typer.Option(
- False,
- "-K",
- "--proxy-prompt-api-key",
- help=(
- "Prompt for LITELLM_PROXY_API_KEY (input hidden). Not suitable for CI. "
- "If --save (or DEEPEVAL_DEFAULT_SAVE) is used, the key is written to dotenv in plaintext."
- ),
- ),
- proxy_base_url: Optional[str] = typer.Option(
- None,
- "-U",
- "--proxy-base-url",
- help="Override the LITELLM_PROXY_API_BASE URL (useful for proxies, gateways, or self-hosted endpoints).",
- ),
- save: Optional[str] = typer.Option(
- None,
- "-s",
- "--save",
- help="Persist CLI parameters as environment variables in a dotenv file. "
- "Usage: --save=dotenv[:path] (default: .env.local)",
- ),
- quiet: bool = typer.Option(
- False,
- "-q",
- "--quiet",
- help="Suppress printing to the terminal (useful for CI).",
- ),
-):
- api_key = None
- if prompt_api_key:
- api_key = coerce_blank_to_none(
- typer.prompt("LiteLLM API key", hide_input=True)
- )
-
- proxy_api_key = None
- if proxy_prompt_api_key:
- proxy_api_key = coerce_blank_to_none(
- typer.prompt("LiteLLM Proxy API key", hide_input=True)
- )
-
- model = coerce_blank_to_none(model)
- base_url = coerce_blank_to_none(base_url)
- proxy_base_url = coerce_blank_to_none(proxy_base_url)
-
- settings = get_settings()
- with settings.edit(save=save) as edit_ctx:
- edit_ctx.switch_model_provider(ModelKeyValues.USE_LITELLM)
- if model is not None:
- settings.LITELLM_MODEL_NAME = model
- if api_key is not None:
- settings.LITELLM_API_KEY = api_key
- if base_url is not None:
- settings.LITELLM_API_BASE = base_url
- if proxy_api_key is not None:
- settings.LITELLM_PROXY_API_KEY = proxy_api_key
- if proxy_base_url is not None:
- settings.LITELLM_PROXY_API_BASE = proxy_base_url
-
- handled, path, updates = edit_ctx.result
-
- effective_model = settings.LITELLM_MODEL_NAME
- if not effective_model:
- raise typer.BadParameter(
- "LiteLLM model name is not set. Pass --model (or set LITELLM_MODEL_NAME).",
- param_hint="--model",
- )
- _handle_save_result(
- handled=handled,
- path=path,
- updates=updates,
- save=save,
- quiet=quiet,
- success_msg=(
- f":raising_hands: Congratulations! You're now using LiteLLM `{escape(effective_model)}` for all evals that require an LLM."
- ),
- )
-
-
-@app.command(name="unset-litellm")
-def unset_litellm_model_env(
- save: Optional[str] = typer.Option(
- None,
- "-s",
- "--save",
- help="Remove only the LiteLLM related environment variables from a dotenv file. "
- "Usage: --save=dotenv[:path] (default: .env.local)",
- ),
- clear_secrets: bool = typer.Option(
- False,
- "-x",
- "--clear-secrets",
- help="Also remove LITELLM_API_KEY and LITELLM_PROXY_API_KEY from the dotenv store.",
- ),
- quiet: bool = typer.Option(
- False,
- "-q",
- "--quiet",
- help="Suppress printing to the terminal (useful for CI).",
- ),
-):
- settings = get_settings()
- with settings.edit(save=save) as edit_ctx:
- settings.USE_LITELLM = None
- settings.LITELLM_MODEL_NAME = None
- settings.LITELLM_API_BASE = None
- settings.LITELLM_PROXY_API_BASE = None
- if clear_secrets:
- settings.LITELLM_API_KEY = None
- settings.LITELLM_PROXY_API_KEY = None
-
- handled, path, updates = edit_ctx.result
-
- if _handle_save_result(
- handled=handled,
- path=path,
- updates=updates,
- save=save,
- quiet=quiet,
- updated_msg="Removed LiteLLM model environment variables from {path}.",
- tip_msg=None,
- ):
- if is_openai_configured():
- print(
- ":raised_hands: OpenAI will still be used by default because OPENAI_API_KEY is set."
- )
- else:
- print(
- "The LiteLLM model configuration has been removed. No model is currently configured, but you can set one with the CLI or add credentials to .env[.local]."
- )
-
-
-#############################################
-# Portkey #############
-#############################################
-
-
-@app.command(name="set-portkey")
-def set_portkey_model_env(
- model: Optional[str] = typer.Option(
- None,
- "-m",
- "--model",
- help="Model identifier to use for this provider",
- ),
- prompt_api_key: bool = typer.Option(
- False,
- "-k",
- "--prompt-api-key",
- help=(
- "Prompt for PORTKEY_API_KEY (input hidden). Not suitable for CI. "
- "If --save (or DEEPEVAL_DEFAULT_SAVE) is used, the key is written to dotenv in plaintext."
- ),
- ),
- base_url: Optional[str] = typer.Option(
- None,
- "-u",
- "--base-url",
- help="Override the API endpoint/base URL used by this provider.",
- ),
- provider: Optional[str] = typer.Option(
- None,
- "-P",
- "--provider",
- help="Override the PORTKEY_PROVIDER_NAME.",
- ),
- save: Optional[str] = typer.Option(
- None,
- "-s",
- "--save",
- help="Persist CLI parameters as environment variables in a dotenv file. "
- "Usage: --save=dotenv[:path] (default: .env.local)",
- ),
- quiet: bool = typer.Option(
- False,
- "-q",
- "--quiet",
- help="Suppress printing to the terminal (useful for CI).",
- ),
-):
- api_key = None
- if prompt_api_key:
- api_key = coerce_blank_to_none(
- typer.prompt("Portkey API key", hide_input=True)
- )
-
- model = coerce_blank_to_none(model)
- base_url = coerce_blank_to_none(base_url)
- provider = coerce_blank_to_none(provider)
-
- settings = get_settings()
- with settings.edit(save=save) as edit_ctx:
- edit_ctx.switch_model_provider(ModelKeyValues.USE_PORTKEY_MODEL)
- if model is not None:
- settings.PORTKEY_MODEL_NAME = model
- if api_key is not None:
- settings.PORTKEY_API_KEY = api_key
- if base_url is not None:
- settings.PORTKEY_BASE_URL = base_url
- if provider is not None:
- settings.PORTKEY_PROVIDER_NAME = provider
-
- handled, path, updates = edit_ctx.result
-
- effective_model = settings.PORTKEY_MODEL_NAME
- if not effective_model:
- raise typer.BadParameter(
- "Portkey model name is not set. Pass --model (or set PORTKEY_MODEL_NAME).",
- param_hint="--model",
- )
- _handle_save_result(
- handled=handled,
- path=path,
- updates=updates,
- save=save,
- quiet=quiet,
- success_msg=(
- f":raising_hands: Congratulations! You're now using Portkey `{escape(effective_model)}` for all evals that require an LLM."
- ),
- )
-
-
-@app.command(name="unset-portkey")
-def unset_portkey_model_env(
- save: Optional[str] = typer.Option(
- None,
- "-s",
- "--save",
- help="Remove only the Portkey related environment variables from a dotenv file. "
- "Usage: --save=dotenv[:path] (default: .env.local)",
- ),
- clear_secrets: bool = typer.Option(
- False,
- "-x",
- "--clear-secrets",
- help="Also remove PORTKEY_API_KEY from the dotenv store.",
- ),
- quiet: bool = typer.Option(
- False,
- "-q",
- "--quiet",
- help="Suppress printing to the terminal (useful for CI).",
- ),
-):
- settings = get_settings()
- with settings.edit(save=save) as edit_ctx:
- settings.USE_PORTKEY_MODEL = None
- settings.PORTKEY_MODEL_NAME = None
- settings.PORTKEY_BASE_URL = None
- settings.PORTKEY_PROVIDER_NAME = None
- if clear_secrets:
- settings.PORTKEY_API_KEY = None
-
- handled, path, updates = edit_ctx.result
-
- if _handle_save_result(
- handled=handled,
- path=path,
- updates=updates,
- save=save,
- quiet=quiet,
- updated_msg="Removed Portkey model environment variables from {path}.",
- tip_msg=None,
- ):
- if is_openai_configured():
- print(
- ":raised_hands: OpenAI will still be used by default because OPENAI_API_KEY is set."
- )
- else:
- print(
- "The Portkey model configuration has been removed. No model is currently configured, but you can set one with the CLI or add credentials to .env[.local]."
- )
-
-
-#############################################
-# OpenRouter Integration ####################
-#############################################
-
-
-@app.command(name="set-openrouter")
-def set_openrouter_model_env(
- model: Optional[str] = typer.Option(
- None,
- "-m",
- "--model",
- help="Model identifier to use for this provider (e.g., `openai/gpt-4.1`).",
- ),
- prompt_api_key: bool = typer.Option(
- False,
- "-k",
- "--prompt-api-key",
- help=(
- "Prompt for OPENROUTER_API_KEY (input hidden). Not suitable for CI. "
- "If --save (or DEEPEVAL_DEFAULT_SAVE) is used, the key is written to dotenv in plaintext."
- ),
- ),
- base_url: Optional[str] = typer.Option(
- None,
- "-u",
- "--base-url",
- help="Override the API endpoint/base URL used by this provider (default: https://openrouter.ai/api/v1).",
- ),
- temperature: Optional[float] = typer.Option(
- None,
- "-t",
- "--temperature",
- help="Override the global TEMPERATURE used by LLM providers (e.g., 0.0 for deterministic behavior).",
- ),
- cost_per_input_token: Optional[float] = typer.Option(
- None,
- "-i",
- "--cost-per-input-token",
- help=(
- "USD per input token used for cost tracking. "
- "If unset and OpenRouter does not return pricing metadata, "
- "costs will not be calculated."
- ),
- ),
- cost_per_output_token: Optional[float] = typer.Option(
- None,
- "-o",
- "--cost-per-output-token",
- help=(
- "USD per output token used for cost tracking. "
- "If unset and OpenRouter does not return pricing metadata, "
- "costs will not be calculated."
- ),
- ),
- save: Optional[str] = typer.Option(
- None,
- "-s",
- "--save",
- help="Persist CLI parameters as environment variables in a dotenv file. "
- "Usage: --save=dotenv[:path] (default: .env.local)",
- ),
- quiet: bool = typer.Option(
- False,
- "-q",
- "--quiet",
- help="Suppress printing to the terminal (useful for CI).",
- ),
-):
- api_key = None
- if prompt_api_key:
- api_key = coerce_blank_to_none(
- typer.prompt("OpenRouter API key", hide_input=True)
- )
-
- model = coerce_blank_to_none(model)
- base_url = coerce_blank_to_none(base_url)
-
- settings = get_settings()
- with settings.edit(save=save) as edit_ctx:
- edit_ctx.switch_model_provider(ModelKeyValues.USE_OPENROUTER_MODEL)
- if model is not None:
- settings.OPENROUTER_MODEL_NAME = model
- if api_key is not None:
- settings.OPENROUTER_API_KEY = api_key
- if base_url is not None:
- settings.OPENROUTER_BASE_URL = base_url
- if temperature is not None:
- settings.TEMPERATURE = temperature
- if cost_per_input_token is not None:
- settings.OPENROUTER_COST_PER_INPUT_TOKEN = cost_per_input_token
- if cost_per_output_token is not None:
- settings.OPENROUTER_COST_PER_OUTPUT_TOKEN = cost_per_output_token
-
- handled, path, updates = edit_ctx.result
-
- effective_model = settings.OPENROUTER_MODEL_NAME
- if not effective_model:
- raise typer.BadParameter(
- "OpenRouter model name is not set. Pass --model (or set OPENROUTER_MODEL_NAME).",
- param_hint="--model",
- )
-
- _handle_save_result(
- handled=handled,
- path=path,
- updates=updates,
- save=save,
- quiet=quiet,
- success_msg=(
- f":raising_hands: Congratulations! You're now using OpenRouter `{escape(effective_model)}` for all evals that require an LLM."
- ),
- )
-
-
-@app.command(name="unset-openrouter")
-def unset_openrouter_model_env(
- save: Optional[str] = typer.Option(
- None,
- "-s",
- "--save",
- help="Remove only the OpenRouter model related environment variables from a dotenv file. "
- "Usage: --save=dotenv[:path] (default: .env.local)",
- ),
- clear_secrets: bool = typer.Option(
- False,
- "-x",
- "--clear-secrets",
- help="Also remove OPENROUTER_API_KEY from the dotenv store.",
- ),
- quiet: bool = typer.Option(
- False,
- "-q",
- "--quiet",
- help="Suppress printing to the terminal (useful for CI).",
- ),
-):
- settings = get_settings()
- with settings.edit(save=save) as edit_ctx:
- settings.USE_OPENROUTER_MODEL = None
- settings.OPENROUTER_MODEL_NAME = None
- settings.OPENROUTER_BASE_URL = None
- settings.OPENROUTER_COST_PER_INPUT_TOKEN = None
- settings.OPENROUTER_COST_PER_OUTPUT_TOKEN = None
- # Intentionally do NOT touch TEMPERATURE here; it's a global dial.
- if clear_secrets:
- settings.OPENROUTER_API_KEY = None
-
- handled, path, updates = edit_ctx.result
-
- if _handle_save_result(
- handled=handled,
- path=path,
- updates=updates,
- save=save,
- quiet=quiet,
- updated_msg="Removed OpenRouter model environment variables from {path}.",
- tip_msg=None,
- ):
- if is_openai_configured():
- print(
- ":raised_hands: OpenAI will still be used by default because OPENAI_API_KEY is set."
- )
- else:
- print(
- "The OpenRouter model configuration has been removed. No model is currently configured, but you can set one with the CLI or add credentials to .env[.local]."
- )
+# Last, so provider commands list after the core ones in `deepeval --help`.
+from deepeval.cli import providers # noqa: E402,F401
if __name__ == "__main__":
diff --git a/deepeval/cli/providers/__init__.py b/deepeval/cli/providers/__init__.py
new file mode 100644
index 000000000..a7f2b43b0
--- /dev/null
+++ b/deepeval/cli/providers/__init__.py
@@ -0,0 +1,3 @@
+"""Provider configuration commands, registered on import."""
+
+from deepeval.cli.providers import llms, tts, stt # noqa: F401
diff --git a/deepeval/cli/providers/_speech.py b/deepeval/cli/providers/_speech.py
new file mode 100644
index 000000000..966089745
--- /dev/null
+++ b/deepeval/cli/providers/_speech.py
@@ -0,0 +1,122 @@
+"""Helpers shared by the `set-tts` / `set-stt` command families."""
+
+from typing import List, Optional
+
+import typer
+from rich import print
+from rich.markup import escape
+
+from deepeval.cli.utils import (
+ coerce_blank_to_none,
+ handle_save_result as _handle_save_result,
+ is_openai_configured,
+)
+from deepeval.config.settings import get_settings
+from deepeval.key_handler import SpeechKeyValues
+
+
+# provider -> (display name, API key setting)
+_SPEECH_PROVIDER_INFO = {
+ "openai": ("OpenAI", "OPENAI_API_KEY"),
+ "elevenlabs": ("ElevenLabs", "ELEVENLABS_API_KEY"),
+ "cartesia": ("Cartesia", "CARTESIA_API_KEY"),
+ "deepgram": ("Deepgram", "DEEPGRAM_API_KEY"),
+ "assemblyai": ("AssemblyAI", "ASSEMBLYAI_API_KEY"),
+}
+
+
+def _set_speech_provider(
+ *,
+ family: str,
+ provider: str,
+ model: Optional[str],
+ prompt_api_key: bool,
+ save: Optional[str],
+ quiet: bool,
+) -> None:
+ label, key_field = _SPEECH_PROVIDER_INFO[provider]
+ flag = SpeechKeyValues[f"USE_{provider.upper()}_{family}"]
+ model_field = f"DEEPEVAL_{family}_MODEL"
+
+ api_key = None
+ if prompt_api_key:
+ api_key = coerce_blank_to_none(
+ typer.prompt(f"{label} API key", hide_input=True)
+ )
+
+ model = coerce_blank_to_none(model)
+
+ settings = get_settings()
+ with settings.edit(save=save) as edit_ctx:
+ edit_ctx.switch_model_provider(flag)
+ if model is not None:
+ setattr(settings, model_field, model)
+ if api_key is not None:
+ setattr(settings, key_field, api_key)
+
+ handled, path, updates = edit_ctx.result
+
+ effective_model = getattr(settings, model_field)
+ model_phrase = (
+ f"`{escape(effective_model)}`"
+ if effective_model
+ else "with its default model"
+ )
+
+ _handle_save_result(
+ handled=handled,
+ path=path,
+ updates=updates,
+ save=save,
+ quiet=quiet,
+ success_msg=(
+ f":raising_hands: Congratulations! You're now using {label} "
+ f"{model_phrase} for {family} in voice simulations."
+ ),
+ )
+
+
+def _unset_speech_provider(
+ *,
+ family: str,
+ flags: List[str],
+ clear_secrets: bool,
+ save: Optional[str],
+ quiet: bool,
+) -> None:
+ settings = get_settings()
+ with settings.edit(save=save) as edit_ctx:
+ for flag in flags:
+ setattr(settings, flag, None)
+ setattr(settings, f"DEEPEVAL_{family}_MODEL", None)
+ if clear_secrets:
+ # OPENAI_API_KEY is deliberately left alone: it is shared with the
+ # LLM providers, so clearing it here would reach outside speech.
+ for flag in flags:
+ _, key_field = _SPEECH_PROVIDER_INFO[
+ flag[len("USE_") : -len(f"_{family}")].lower()
+ ]
+ if key_field != "OPENAI_API_KEY":
+ setattr(settings, key_field, None)
+
+ handled, path, updates = edit_ctx.result
+
+ if _handle_save_result(
+ handled=handled,
+ path=path,
+ updates=updates,
+ save=save,
+ quiet=quiet,
+ updated_msg=(f"Removed {family} environment variables from {{path}}."),
+ tip_msg=None,
+ ):
+ if is_openai_configured():
+ print(
+ f":raised_hands: OpenAI will still be used for {family} by "
+ "default because OPENAI_API_KEY is set."
+ )
+ else:
+ print(
+ f"The {family} provider has been unset. Voice simulations "
+ "fall back to OpenAI, which needs OPENAI_API_KEY."
+ )
diff --git a/deepeval/cli/providers/llms/__init__.py b/deepeval/cli/providers/llms/__init__.py
new file mode 100644
index 000000000..c9ef1c96f
--- /dev/null
+++ b/deepeval/cli/providers/llms/__init__.py
@@ -0,0 +1,21 @@
+"""LLM commands, plus the embedding commands of providers that serve both.
+
+Import order is the order `deepeval --help` lists them in.
+"""
+
+from deepeval.cli.providers.llms import ( # noqa: F401
+ openai,
+ azure,
+ anthropic,
+ bedrock,
+ ollama,
+ local_model,
+ grok,
+ moonshot,
+ deepseek,
+ local_embeddings,
+ gemini,
+ litellm,
+ portkey,
+ openrouter,
+)
diff --git a/deepeval/cli/providers/llms/anthropic.py b/deepeval/cli/providers/llms/anthropic.py
new file mode 100644
index 000000000..43cc8c138
--- /dev/null
+++ b/deepeval/cli/providers/llms/anthropic.py
@@ -0,0 +1,159 @@
+"""`deepeval set-anthropic, unset-anthropic` commands."""
+
+import typer
+from typing import Optional
+from rich import print
+from rich.markup import escape
+
+from deepeval.cli.app import app
+from deepeval.cli.utils import (
+ coerce_blank_to_none,
+ handle_save_result as _handle_save_result,
+ is_openai_configured,
+)
+from deepeval.config.settings import get_settings
+from deepeval.key_handler import (
+ ModelKeyValues,
+)
+
+
+@app.command(name="set-anthropic")
+def set_anthropic_model_env(
+ model: Optional[str] = typer.Option(
+ None,
+ "-m",
+ "--model",
+ help="Model identifier to use for this provider",
+ ),
+ prompt_api_key: bool = typer.Option(
+ False,
+ "-k",
+ "--prompt-api-key",
+ help=(
+ "Prompt for ANTHROPIC_API_KEY (input hidden). Not suitable for CI. "
+ "If --save (or DEEPEVAL_DEFAULT_SAVE) is used, the key is written to dotenv in plaintext."
+ ),
+ ),
+ cost_per_input_token: Optional[float] = typer.Option(
+ None,
+ "-i",
+ "--cost-per-input-token",
+ help=(
+ "USD per input token override used for cost tracking. Preconfigured for known models; "
+ "REQUIRED if you use a custom/unknown model."
+ ),
+ ),
+ cost_per_output_token: Optional[float] = typer.Option(
+ None,
+ "-o",
+ "--cost-per-output-token",
+ help=(
+ "USD per output token override used for cost tracking. Preconfigured for known models; "
+ "REQUIRED if you use a custom/unknown model."
+ ),
+ ),
+ save: Optional[str] = typer.Option(
+ None,
+ "-s",
+ "--save",
+ help="Persist CLI parameters as environment variables in a dotenv file. "
+ "Usage: --save=dotenv[:path] (default: .env.local)",
+ ),
+ quiet: bool = typer.Option(
+ False,
+ "-q",
+ "--quiet",
+ help="Suppress printing to the terminal (useful for CI).",
+ ),
+):
+ api_key = None
+ if prompt_api_key:
+ api_key = coerce_blank_to_none(
+ typer.prompt("Anthropic API key", hide_input=True)
+ )
+
+ model = coerce_blank_to_none(model)
+
+ settings = get_settings()
+ with settings.edit(save=save) as edit_ctx:
+ edit_ctx.switch_model_provider(ModelKeyValues.USE_ANTHROPIC_MODEL)
+ if api_key is not None:
+ settings.ANTHROPIC_API_KEY = api_key
+ if model is not None:
+ settings.ANTHROPIC_MODEL_NAME = model
+ if cost_per_input_token is not None:
+ settings.ANTHROPIC_COST_PER_INPUT_TOKEN = cost_per_input_token
+ if cost_per_output_token is not None:
+ settings.ANTHROPIC_COST_PER_OUTPUT_TOKEN = cost_per_output_token
+
+ handled, path, updates = edit_ctx.result
+
+ effective_model = settings.ANTHROPIC_MODEL_NAME
+ if not effective_model:
+ raise typer.BadParameter(
+ "Anthropic model name is not set. Pass --model (or set ANTHROPIC_MODEL_NAME).",
+ param_hint="--model",
+ )
+
+ _handle_save_result(
+ handled=handled,
+ path=path,
+ updates=updates,
+ save=save,
+ quiet=quiet,
+ success_msg=(
+ f":raising_hands: Congratulations! You're now using Anthropic `{escape(effective_model)}` for all evals that require an LLM."
+ ),
+ )
+
+
+@app.command(name="unset-anthropic")
+def unset_anthropic_model_env(
+ save: Optional[str] = typer.Option(
+ None,
+ "-s",
+ "--save",
+ help="Remove only the Anthropic model related environment variables from a dotenv file. "
+ "Usage: --save=dotenv[:path] (default: .env.local)",
+ ),
+ clear_secrets: bool = typer.Option(
+ False,
+ "-x",
+ "--clear-secrets",
+ help="Also remove ANTHROPIC_API_KEY from the dotenv store.",
+ ),
+ quiet: bool = typer.Option(
+ False,
+ "-q",
+ "--quiet",
+ help="Suppress printing to the terminal (useful for CI).",
+ ),
+):
+ settings = get_settings()
+ with settings.edit(save=save) as edit_ctx:
+ settings.USE_ANTHROPIC_MODEL = None
+ settings.ANTHROPIC_MODEL_NAME = None
+ settings.ANTHROPIC_COST_PER_INPUT_TOKEN = None
+ settings.ANTHROPIC_COST_PER_OUTPUT_TOKEN = None
+ if clear_secrets:
+ settings.ANTHROPIC_API_KEY = None
+
+ handled, path, updates = edit_ctx.result
+
+ if _handle_save_result(
+ handled=handled,
+ path=path,
+ updates=updates,
+ save=save,
+ quiet=quiet,
+ updated_msg="Removed Anthropic model environment variables from {path}.",
+ tip_msg=None,
+ ):
+ if is_openai_configured():
+ print(
+ ":raised_hands: OpenAI will still be used by default because OPENAI_API_KEY is set."
+ )
+ else:
+ print(
+ "The Anthropic model configuration has been removed. No model is currently configured, but you can set one with the CLI or add credentials to .env[.local]."
+ )
diff --git a/deepeval/cli/providers/llms/azure.py b/deepeval/cli/providers/llms/azure.py
new file mode 100644
index 000000000..bb59ee119
--- /dev/null
+++ b/deepeval/cli/providers/llms/azure.py
@@ -0,0 +1,275 @@
+"""`deepeval set-azure-openai, set-azure-openai-embedding, unset-azure-openai, unset-azure-openai-embedding` commands."""
+
+import typer
+from typing import Optional
+from rich import print
+from rich.markup import escape
+
+from deepeval.cli.app import app
+from deepeval.cli.utils import (
+ coerce_blank_to_none,
+ handle_save_result as _handle_save_result,
+ is_openai_configured,
+)
+from deepeval.config.settings import get_settings
+from deepeval.key_handler import (
+ EmbeddingKeyValues,
+ ModelKeyValues,
+)
+
+
+@app.command(name="set-azure-openai")
+def set_azure_openai_env(
+ model: Optional[str] = typer.Option(
+ None,
+ "-m",
+ "--model",
+ help="Model identifier to use for this provider (e.g., `gpt-4.1`).",
+ ),
+ prompt_api_key: bool = typer.Option(
+ False,
+ "-k",
+ "--prompt-api-key",
+ help=(
+ "Prompt for AZURE_OPENAI_API_KEY (input hidden). Not suitable for CI. "
+ "If --save (or DEEPEVAL_DEFAULT_SAVE) is used, the key is written to dotenv in plaintext."
+ ),
+ ),
+ base_url: Optional[str] = typer.Option(
+ None,
+ "-u",
+ "--base-url",
+ help="Override the API endpoint/base URL used by this provider.",
+ ),
+ api_version: Optional[str] = typer.Option(
+ None,
+ "-v",
+ "--api-version",
+ help="Azure OpenAI API version (passed to the Azure OpenAI client).",
+ ),
+ model_version: Optional[str] = typer.Option(
+ None, "-V", "--model-version", help="Azure model version"
+ ),
+ deployment_name: Optional[str] = typer.Option(
+ None, "-d", "--deployment-name", help="Azure OpenAI deployment name"
+ ),
+ save: Optional[str] = typer.Option(
+ None,
+ "-s",
+ "--save",
+ help="Persist CLI parameters as environment variables in a dotenv file. "
+ "Usage: --save=dotenv[:path] (default: .env.local)",
+ ),
+ quiet: bool = typer.Option(
+ False,
+ "-q",
+ "--quiet",
+ help="Suppress printing to the terminal (useful for CI).",
+ ),
+):
+ api_key = None
+ if prompt_api_key:
+ api_key = coerce_blank_to_none(
+ typer.prompt("Azure OpenAI API key", hide_input=True)
+ )
+
+ model = coerce_blank_to_none(model)
+ base_url = coerce_blank_to_none(base_url)
+ api_version = coerce_blank_to_none(api_version)
+ deployment_name = coerce_blank_to_none(deployment_name)
+ model_version = coerce_blank_to_none(model_version)
+
+ settings = get_settings()
+ with settings.edit(save=save) as edit_ctx:
+ edit_ctx.switch_model_provider(ModelKeyValues.USE_AZURE_OPENAI)
+ if model is not None:
+ settings.AZURE_MODEL_NAME = model
+ if api_key is not None:
+ settings.AZURE_OPENAI_API_KEY = api_key
+ if base_url is not None:
+ settings.AZURE_OPENAI_ENDPOINT = base_url
+ if api_version is not None:
+ settings.OPENAI_API_VERSION = api_version
+ if deployment_name is not None:
+ settings.AZURE_DEPLOYMENT_NAME = deployment_name
+ if model_version is not None:
+ settings.AZURE_MODEL_VERSION = model_version
+
+ handled, path, updates = edit_ctx.result
+
+ effective_model = settings.AZURE_MODEL_NAME
+ if not effective_model:
+ raise typer.BadParameter(
+ "Azure OpenAI model name is not set. Pass --model (or set AZURE_MODEL_NAME).",
+ param_hint="--model",
+ )
+
+ _handle_save_result(
+ handled=handled,
+ path=path,
+ updates=updates,
+ save=save,
+ quiet=quiet,
+ success_msg=(
+ f":raising_hands: Congratulations! You're now using Azure OpenAI's `{escape(effective_model)}` for all evals that require an LLM."
+ ),
+ )
+
+
+@app.command(name="unset-azure-openai")
+def unset_azure_openai_env(
+ save: Optional[str] = typer.Option(
+ None,
+ "-s",
+ "--save",
+ help="Remove only the Azure OpenAI–related environment variables from a dotenv file. "
+ "Usage: --save=dotenv[:path] (default: .env.local)",
+ ),
+ clear_secrets: bool = typer.Option(
+ False,
+ "-x",
+ "--clear-secrets",
+ help="Also remove AZURE_OPENAI_API_KEY from the dotenv store.",
+ ),
+ quiet: bool = typer.Option(
+ False,
+ "-q",
+ "--quiet",
+ help="Suppress printing to the terminal (useful for CI).",
+ ),
+):
+ settings = get_settings()
+ with settings.edit(save=save) as edit_ctx:
+ settings.AZURE_OPENAI_ENDPOINT = None
+ settings.OPENAI_API_VERSION = None
+ settings.AZURE_DEPLOYMENT_NAME = None
+ settings.AZURE_MODEL_NAME = None
+ settings.AZURE_MODEL_VERSION = None
+ settings.USE_AZURE_OPENAI = None
+ if clear_secrets:
+ settings.AZURE_OPENAI_API_KEY = None
+
+ handled, path, updates = edit_ctx.result
+
+ if _handle_save_result(
+ handled=handled,
+ path=path,
+ updates=updates,
+ save=save,
+ quiet=quiet,
+ updated_msg="Removed Azure OpenAI environment variables from {path}.",
+ tip_msg=None,
+ ):
+ if is_openai_configured():
+ print(
+ ":raised_hands: OpenAI will still be used by default because OPENAI_API_KEY is set."
+ )
+ else:
+ print(
+ "Azure OpenAI has been unset. No active provider is configured. Set one with the CLI, or add credentials to .env[.local]."
+ )
+
+
+@app.command(name="set-azure-openai-embedding")
+def set_azure_openai_embedding_env(
+ model: Optional[str] = typer.Option(
+ None,
+ "-m",
+ "--model",
+ help="Model identifier to use for this provider (e.g., `gpt-4.1`).",
+ ),
+ deployment_name: Optional[str] = typer.Option(
+ None,
+ "-d",
+ "--deployment-name",
+ help="Azure embedding deployment name",
+ ),
+ save: Optional[str] = typer.Option(
+ None,
+ "-s",
+ "--save",
+ help="Persist CLI parameters as environment variables in a dotenv file. "
+ "Usage: --save=dotenv[:path] (default: .env.local)",
+ ),
+ quiet: bool = typer.Option(
+ False,
+ "-q",
+ "--quiet",
+ help="Suppress printing to the terminal (useful for CI).",
+ ),
+):
+ model = coerce_blank_to_none(model)
+ deployment_name = coerce_blank_to_none(deployment_name)
+
+ settings = get_settings()
+ with settings.edit(save=save) as edit_ctx:
+ edit_ctx.switch_model_provider(
+ EmbeddingKeyValues.USE_AZURE_OPENAI_EMBEDDING
+ )
+ if model is not None:
+ settings.AZURE_EMBEDDING_MODEL_NAME = model
+ if deployment_name is not None:
+ settings.AZURE_EMBEDDING_DEPLOYMENT_NAME = deployment_name
+
+ handled, path, updates = edit_ctx.result
+
+ effective_model = settings.AZURE_EMBEDDING_MODEL_NAME
+ if not effective_model:
+ raise typer.BadParameter(
+ "Azure OpenAI embedding model name is not set. Pass --model (or set AZURE_EMBEDDING_MODEL_NAME).",
+ param_hint="--model",
+ )
+
+ _handle_save_result(
+ handled=handled,
+ path=path,
+ updates=updates,
+ save=save,
+ quiet=quiet,
+ success_msg=(
+ f":raising_hands: Congratulations! You're now using Azure OpenAI embedding model `{escape(effective_model)}` for all evals that require text embeddings."
+ ),
+ )
+
+
+@app.command(name="unset-azure-openai-embedding")
+def unset_azure_openai_embedding_env(
+ save: Optional[str] = typer.Option(
+ None,
+ "-s",
+ "--save",
+ help="Remove only the Azure OpenAI embedding related environment variables from a dotenv file. "
+ "Usage: --save=dotenv[:path] (default: .env.local)",
+ ),
+ quiet: bool = typer.Option(
+ False,
+ "-q",
+ "--quiet",
+ help="Suppress printing to the terminal (useful for CI).",
+ ),
+):
+ settings = get_settings()
+ with settings.edit(save=save) as edit_ctx:
+ settings.AZURE_EMBEDDING_MODEL_NAME = None
+ settings.AZURE_EMBEDDING_DEPLOYMENT_NAME = None
+ settings.USE_AZURE_OPENAI_EMBEDDING = None
+
+ handled, path, updates = edit_ctx.result
+
+ if _handle_save_result(
+ handled=handled,
+ path=path,
+ updates=updates,
+ save=save,
+ quiet=quiet,
+ updated_msg="Removed Azure OpenAI embedding environment variables from {path}.",
+ tip_msg=None,
+ ):
+ if is_openai_configured():
+ print(
+ ":raised_hands: OpenAI will still be used by default because OPENAI_API_KEY is set."
+ )
+ else:
+ print(
+ "Azure OpenAI embedding has been unset. No active provider is configured. Set one with the CLI, or add credentials to .env[.local]."
+ )
diff --git a/deepeval/cli/providers/llms/bedrock.py b/deepeval/cli/providers/llms/bedrock.py
new file mode 100644
index 000000000..42f0e59aa
--- /dev/null
+++ b/deepeval/cli/providers/llms/bedrock.py
@@ -0,0 +1,174 @@
+"""`deepeval set-bedrock, unset-bedrock` commands."""
+
+import typer
+from typing import Optional
+from rich import print
+from rich.markup import escape
+
+from deepeval.cli.app import app
+from deepeval.cli.utils import (
+ coerce_blank_to_none,
+ handle_save_result as _handle_save_result,
+ is_openai_configured,
+)
+from deepeval.config.settings import get_settings
+from deepeval.key_handler import (
+ ModelKeyValues,
+)
+
+
+@app.command(name="set-bedrock")
+def set_bedrock_model_env(
+ model: Optional[str] = typer.Option(
+ None,
+ "-m",
+ "--model",
+ help="Model identifier to use for this provider",
+ ),
+ prompt_credentials: bool = typer.Option(
+ False,
+ "-a",
+ "--prompt-credentials",
+ help=(
+ "Prompt for AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY (secret access key input is hidden). Not suitable for CI. "
+ "If --save (or DEEPEVAL_DEFAULT_SAVE) is used, credentials are written to dotenv in plaintext."
+ ),
+ ),
+ region: Optional[str] = typer.Option(
+ None,
+ "-r",
+ "--region",
+ help="AWS region for bedrock (e.g., `us-east-1`).",
+ ),
+ cost_per_input_token: Optional[float] = typer.Option(
+ None,
+ "-i",
+ "--cost-per-input-token",
+ help=(
+ "USD per input token override used for cost tracking. Preconfigured for known models; "
+ "REQUIRED if you use a custom/unknown model."
+ ),
+ ),
+ cost_per_output_token: Optional[float] = typer.Option(
+ None,
+ "-o",
+ "--cost-per-output-token",
+ help=(
+ "USD per output token override used for cost tracking. Preconfigured for known models; "
+ "REQUIRED if you use a custom/unknown model."
+ ),
+ ),
+ save: Optional[str] = typer.Option(
+ None,
+ "-s",
+ "--save",
+ help="Persist CLI parameters as environment variables in a dotenv file. "
+ "Usage: --save=dotenv[:path] (default: .env.local)",
+ ),
+ quiet: bool = typer.Option(
+ False,
+ "-q",
+ "--quiet",
+ help="Suppress printing to the terminal (useful for CI).",
+ ),
+):
+ access_key_id = None
+ secret_access_key = None
+ if prompt_credentials:
+ access_key_id = coerce_blank_to_none(typer.prompt("AWS Access key Id"))
+ secret_access_key = coerce_blank_to_none(
+ typer.prompt("AWS Secret Access key", hide_input=True)
+ )
+
+ model = coerce_blank_to_none(model)
+ region = coerce_blank_to_none(region)
+
+ settings = get_settings()
+ with settings.edit(save=save) as edit_ctx:
+ edit_ctx.switch_model_provider(ModelKeyValues.USE_AWS_BEDROCK_MODEL)
+ if access_key_id is not None:
+ settings.AWS_ACCESS_KEY_ID = access_key_id
+ if secret_access_key is not None:
+ settings.AWS_SECRET_ACCESS_KEY = secret_access_key
+ if model is not None:
+ settings.AWS_BEDROCK_MODEL_NAME = model
+ if region is not None:
+ settings.AWS_BEDROCK_REGION = region
+ if cost_per_input_token is not None:
+ settings.AWS_BEDROCK_COST_PER_INPUT_TOKEN = cost_per_input_token
+ if cost_per_output_token is not None:
+ settings.AWS_BEDROCK_COST_PER_OUTPUT_TOKEN = cost_per_output_token
+
+ handled, path, updates = edit_ctx.result
+
+ effective_model = settings.AWS_BEDROCK_MODEL_NAME
+ if not effective_model:
+ raise typer.BadParameter(
+ "AWS Bedrock model name is not set. Pass --model (or set AWS_BEDROCK_MODEL_NAME).",
+ param_hint="--model",
+ )
+
+ _handle_save_result(
+ handled=handled,
+ path=path,
+ updates=updates,
+ save=save,
+ quiet=quiet,
+ success_msg=(
+ f":raising_hands: Congratulations! You're now using AWS Bedrock `{escape(effective_model)}` for all evals that require an LLM."
+ ),
+ )
+
+
+@app.command(name="unset-bedrock")
+def unset_bedrock_model_env(
+ save: Optional[str] = typer.Option(
+ None,
+ "-s",
+ "--save",
+ help="Remove only the AWS Bedrock model related environment variables from a dotenv file. "
+ "Usage: --save=dotenv[:path] (default: .env.local)",
+ ),
+ clear_secrets: bool = typer.Option(
+ False,
+ "-x",
+ "--clear-secrets",
+ help="Also remove AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY from the dotenv store.",
+ ),
+ quiet: bool = typer.Option(
+ False,
+ "-q",
+ "--quiet",
+ help="Suppress printing to the terminal (useful for CI).",
+ ),
+):
+ settings = get_settings()
+ with settings.edit(save=save) as edit_ctx:
+ settings.USE_AWS_BEDROCK_MODEL = None
+ settings.AWS_BEDROCK_MODEL_NAME = None
+ settings.AWS_BEDROCK_REGION = None
+ settings.AWS_BEDROCK_COST_PER_INPUT_TOKEN = None
+ settings.AWS_BEDROCK_COST_PER_OUTPUT_TOKEN = None
+ if clear_secrets:
+ settings.AWS_ACCESS_KEY_ID = None
+ settings.AWS_SECRET_ACCESS_KEY = None
+
+ handled, path, updates = edit_ctx.result
+
+ if _handle_save_result(
+ handled=handled,
+ path=path,
+ updates=updates,
+ save=save,
+ quiet=quiet,
+ updated_msg="Removed AWS Bedrock model environment variables from {path}.",
+ tip_msg=None,
+ ):
+ if is_openai_configured():
+ print(
+ ":raised_hands: OpenAI will still be used by default because OPENAI_API_KEY is set."
+ )
+ else:
+ print(
+ "The AWS Bedrock model configuration has been removed. No model is currently configured, but you can set one with the CLI or add credentials to .env[.local]."
+ )
diff --git a/deepeval/cli/providers/llms/deepseek.py b/deepeval/cli/providers/llms/deepseek.py
new file mode 100644
index 000000000..5139f15dc
--- /dev/null
+++ b/deepeval/cli/providers/llms/deepseek.py
@@ -0,0 +1,159 @@
+"""`deepeval set-deepseek, unset-deepseek` commands."""
+
+import typer
+from typing import Optional
+from rich import print
+from rich.markup import escape
+
+from deepeval.cli.app import app
+from deepeval.cli.utils import (
+ coerce_blank_to_none,
+ handle_save_result as _handle_save_result,
+ is_openai_configured,
+)
+from deepeval.config.settings import get_settings
+from deepeval.key_handler import (
+ ModelKeyValues,
+)
+
+
+@app.command(name="set-deepseek")
+def set_deepseek_model_env(
+ model: Optional[str] = typer.Option(
+ None,
+ "-m",
+ "--model",
+ help="Model identifier to use for this provider",
+ ),
+ prompt_api_key: bool = typer.Option(
+ False,
+ "-k",
+ "--prompt-api-key",
+ help=(
+ "Prompt for DEEPSEEK_API_KEY (input hidden). Not suitable for CI. "
+ "If --save (or DEEPEVAL_DEFAULT_SAVE) is used, the key is written to dotenv in plaintext."
+ ),
+ ),
+ cost_per_input_token: Optional[float] = typer.Option(
+ None,
+ "-i",
+ "--cost-per-input-token",
+ help=(
+ "USD per input token override used for cost tracking. Preconfigured for known models; "
+ "REQUIRED if you use a custom/unknown model."
+ ),
+ ),
+ cost_per_output_token: Optional[float] = typer.Option(
+ None,
+ "-o",
+ "--cost-per-output-token",
+ help=(
+ "USD per output token override used for cost tracking. Preconfigured for known models; "
+ "REQUIRED if you use a custom/unknown model."
+ ),
+ ),
+ save: Optional[str] = typer.Option(
+ None,
+ "-s",
+ "--save",
+ help="Persist CLI parameters as environment variables in a dotenv file. "
+ "Usage: --save=dotenv[:path] (default: .env.local)",
+ ),
+ quiet: bool = typer.Option(
+ False,
+ "-q",
+ "--quiet",
+ help="Suppress printing to the terminal (useful for CI).",
+ ),
+):
+ api_key = None
+ if prompt_api_key:
+ api_key = coerce_blank_to_none(
+ typer.prompt("DeepSeek API key", hide_input=True)
+ )
+
+ model = coerce_blank_to_none(model)
+
+ settings = get_settings()
+ with settings.edit(save=save) as edit_ctx:
+ edit_ctx.switch_model_provider(ModelKeyValues.USE_DEEPSEEK_MODEL)
+ if model is not None:
+ settings.DEEPSEEK_MODEL_NAME = model
+ if api_key is not None:
+ settings.DEEPSEEK_API_KEY = api_key
+ if cost_per_input_token is not None:
+ settings.DEEPSEEK_COST_PER_INPUT_TOKEN = cost_per_input_token
+ if cost_per_output_token is not None:
+ settings.DEEPSEEK_COST_PER_OUTPUT_TOKEN = cost_per_output_token
+
+ handled, path, updates = edit_ctx.result
+
+ effective_model = settings.DEEPSEEK_MODEL_NAME
+ if not effective_model:
+ raise typer.BadParameter(
+ "DeepSeek model name is not set. Pass --model (or set DEEPSEEK_MODEL_NAME).",
+ param_hint="--model",
+ )
+
+ _handle_save_result(
+ handled=handled,
+ path=path,
+ updates=updates,
+ save=save,
+ quiet=quiet,
+ success_msg=(
+ f":raising_hands: Congratulations! You're now using DeepSeek `{escape(effective_model)}` for all evals that require an LLM."
+ ),
+ )
+
+
+@app.command(name="unset-deepseek")
+def unset_deepseek_model_env(
+ save: Optional[str] = typer.Option(
+ None,
+ "-s",
+ "--save",
+ help="Remove only the DeepSeek model related environment variables from a dotenv file. "
+ "Usage: --save=dotenv[:path] (default: .env.local)",
+ ),
+ clear_secrets: bool = typer.Option(
+ False,
+ "-x",
+ "--clear-secrets",
+ help="Also remove DEEPSEEK_API_KEY from the dotenv store.",
+ ),
+ quiet: bool = typer.Option(
+ False,
+ "-q",
+ "--quiet",
+ help="Suppress printing to the terminal (useful for CI).",
+ ),
+):
+ settings = get_settings()
+ with settings.edit(save=save) as edit_ctx:
+ settings.USE_DEEPSEEK_MODEL = None
+ settings.DEEPSEEK_MODEL_NAME = None
+ settings.DEEPSEEK_COST_PER_INPUT_TOKEN = None
+ settings.DEEPSEEK_COST_PER_OUTPUT_TOKEN = None
+ if clear_secrets:
+ settings.DEEPSEEK_API_KEY = None
+
+ handled, path, updates = edit_ctx.result
+
+ if _handle_save_result(
+ handled=handled,
+ path=path,
+ updates=updates,
+ save=save,
+ quiet=quiet,
+ updated_msg="Removed DeepSeek model environment variables from {path}.",
+ tip_msg=None,
+ ):
+ if is_openai_configured():
+ print(
+ ":raised_hands: OpenAI will still be used by default because OPENAI_API_KEY is set."
+ )
+ else:
+ print(
+ "The DeepSeek model configuration has been removed. No model is currently configured, but you can set one with the CLI or add credentials to .env[.local]."
+ )
diff --git a/deepeval/cli/providers/llms/gemini.py b/deepeval/cli/providers/llms/gemini.py
new file mode 100644
index 000000000..c7b0d542d
--- /dev/null
+++ b/deepeval/cli/providers/llms/gemini.py
@@ -0,0 +1,179 @@
+"""`deepeval set-gemini, unset-gemini` commands."""
+
+import typer
+from pathlib import Path
+from typing import Optional
+from rich import print
+from rich.markup import escape
+
+from deepeval.cli.app import app
+from deepeval.cli.utils import (
+ coerce_blank_to_none,
+ handle_save_result as _handle_save_result,
+ is_openai_configured,
+ load_service_account_key_file,
+)
+from deepeval.config.settings import get_settings
+from deepeval.key_handler import (
+ ModelKeyValues,
+)
+
+
+@app.command(name="set-gemini")
+def set_gemini_model_env(
+ model: Optional[str] = typer.Option(
+ None,
+ "-m",
+ "--model",
+ help="Model identifier to use for this provider",
+ ),
+ prompt_api_key: bool = typer.Option(
+ False,
+ "-k",
+ "--prompt-api-key",
+ help=(
+ "Prompt for GOOGLE_API_KEY (input hidden). Not suitable for CI. "
+ "If --save (or DEEPEVAL_DEFAULT_SAVE) is used, the key is written to dotenv in plaintext."
+ ),
+ ),
+ project: Optional[str] = typer.Option(
+ None,
+ "-p",
+ "--project",
+ help="GCP project ID (used by Vertex AI / Gemini when applicable).",
+ ),
+ location: Optional[str] = typer.Option(
+ None,
+ "-l",
+ "--location",
+ help="GCP location/region for Vertex AI (e.g., `us-central1`).",
+ ),
+ service_account_file: Optional[Path] = typer.Option(
+ None,
+ "-S",
+ "--service-account-file",
+ help=("Path to a Google service account JSON key file."),
+ exists=True,
+ dir_okay=False,
+ readable=True,
+ ),
+ save: Optional[str] = typer.Option(
+ None,
+ "-s",
+ "--save",
+ help="Persist CLI parameters as environment variables in a dotenv file. "
+ "Usage: --save=dotenv[:path] (default: .env.local)",
+ ),
+ quiet: bool = typer.Option(
+ False,
+ "-q",
+ "--quiet",
+ help="Suppress printing to the terminal (useful for CI).",
+ ),
+):
+ api_key = None
+ if prompt_api_key:
+ api_key = coerce_blank_to_none(
+ typer.prompt("Google API key", hide_input=True)
+ )
+
+ model = coerce_blank_to_none(model)
+ project = coerce_blank_to_none(project)
+ location = coerce_blank_to_none(location)
+
+ settings = get_settings()
+ with settings.edit(save=save) as edit_ctx:
+ edit_ctx.switch_model_provider(ModelKeyValues.USE_GEMINI_MODEL)
+
+ if model is not None:
+ settings.GEMINI_MODEL_NAME = model
+ if project is not None:
+ settings.GOOGLE_CLOUD_PROJECT = project
+ if location is not None:
+ settings.GOOGLE_CLOUD_LOCATION = location
+ if service_account_file is not None:
+ settings.GOOGLE_SERVICE_ACCOUNT_KEY = load_service_account_key_file(
+ service_account_file
+ )
+ if api_key is not None:
+ settings.GOOGLE_API_KEY = api_key
+ settings.GOOGLE_GENAI_USE_VERTEXAI = False
+ elif (
+ project is not None
+ or location is not None
+ or service_account_file is not None
+ ):
+ settings.GOOGLE_GENAI_USE_VERTEXAI = True
+
+ handled, path, updates = edit_ctx.result
+
+ effective_model = settings.GEMINI_MODEL_NAME
+ if not effective_model:
+ raise typer.BadParameter(
+ "Gemini model name is not set. Pass --model (or set GEMINI_MODEL_NAME).",
+ param_hint="--model",
+ )
+ _handle_save_result(
+ handled=handled,
+ path=path,
+ updates=updates,
+ save=save,
+ quiet=quiet,
+ success_msg=(
+ f":raising_hands: Congratulations! You're now using Gemini `{escape(effective_model)}` for all evals that require an LLM."
+ ),
+ )
+
+
+@app.command(name="unset-gemini")
+def unset_gemini_model_env(
+ save: Optional[str] = typer.Option(
+ None,
+ "-s",
+ "--save",
+ help="Remove only the Gemini related environment variables from a dotenv file. "
+ "Usage: --save=dotenv[:path] (default: .env.local)",
+ ),
+ clear_secrets: bool = typer.Option(
+ False,
+ "-x",
+ "--clear-secrets",
+ help="Also remove GOOGLE_API_KEY and GOOGLE_SERVICE_ACCOUNT_KEY from the dotenv store.",
+ ),
+ quiet: bool = typer.Option(
+ False,
+ "-q",
+ "--quiet",
+ help="Suppress printing to the terminal (useful for CI).",
+ ),
+):
+ settings = get_settings()
+ with settings.edit(save=save) as edit_ctx:
+ settings.USE_GEMINI_MODEL = None
+ settings.GOOGLE_GENAI_USE_VERTEXAI = None
+ settings.GOOGLE_CLOUD_PROJECT = None
+ settings.GOOGLE_CLOUD_LOCATION = None
+ settings.GEMINI_MODEL_NAME = None
+ if clear_secrets:
+ settings.GOOGLE_API_KEY = None
+ settings.GOOGLE_SERVICE_ACCOUNT_KEY = None
+
+ handled, path, updates = edit_ctx.result
+
+ if _handle_save_result(
+ handled=handled,
+ path=path,
+ updates=updates,
+ save=save,
+ quiet=quiet,
+ updated_msg="Removed Gemini model environment variables from {path}.",
+ tip_msg=None,
+ ):
+ if is_openai_configured():
+ print(
+ ":raised_hands: OpenAI will still be used by default because OPENAI_API_KEY is set."
+ )
+ else:
+ print(
+ "The Gemini model configuration has been removed. No model is currently configured, but you can set one with the CLI or add credentials to .env[.local]."
+ )
diff --git a/deepeval/cli/providers/llms/grok.py b/deepeval/cli/providers/llms/grok.py
new file mode 100644
index 000000000..cea4312c9
--- /dev/null
+++ b/deepeval/cli/providers/llms/grok.py
@@ -0,0 +1,158 @@
+"""`deepeval set-grok, unset-grok` commands."""
+
+import typer
+from typing import Optional
+from rich import print
+from rich.markup import escape
+
+from deepeval.cli.app import app
+from deepeval.cli.utils import (
+ coerce_blank_to_none,
+ handle_save_result as _handle_save_result,
+ is_openai_configured,
+)
+from deepeval.config.settings import get_settings
+from deepeval.key_handler import (
+ ModelKeyValues,
+)
+
+
+@app.command(name="set-grok")
+def set_grok_model_env(
+ model: Optional[str] = typer.Option(
+ None,
+ "-m",
+ "--model",
+ help="Model identifier to use for this provider",
+ ),
+ prompt_api_key: bool = typer.Option(
+ False,
+ "-k",
+ "--prompt-api-key",
+ help=(
+ "Prompt for GROK_API_KEY (input hidden). Not suitable for CI. "
+ "If --save (or DEEPEVAL_DEFAULT_SAVE) is used, the key is written to dotenv in plaintext."
+ ),
+ ),
+ cost_per_input_token: Optional[float] = typer.Option(
+ None,
+ "-i",
+ "--cost-per-input-token",
+ help=(
+ "USD per input token override used for cost tracking. Preconfigured for known models; "
+ "REQUIRED if you use a custom/unknown model."
+ ),
+ ),
+ cost_per_output_token: Optional[float] = typer.Option(
+ None,
+ "-o",
+ "--cost-per-output-token",
+ help=(
+ "USD per output token override used for cost tracking. Preconfigured for known models; "
+ "REQUIRED if you use a custom/unknown model."
+ ),
+ ),
+ save: Optional[str] = typer.Option(
+ None,
+ "-s",
+ "--save",
+ help="Persist CLI parameters as environment variables in a dotenv file. "
+ "Usage: --save=dotenv[:path] (default: .env.local)",
+ ),
+ quiet: bool = typer.Option(
+ False,
+ "-q",
+ "--quiet",
+ help="Suppress printing to the terminal (useful for CI).",
+ ),
+):
+ api_key = None
+ if prompt_api_key:
+ api_key = coerce_blank_to_none(
+ typer.prompt("Grok API key", hide_input=True)
+ )
+
+ model = coerce_blank_to_none(model)
+
+ settings = get_settings()
+ with settings.edit(save=save) as edit_ctx:
+ edit_ctx.switch_model_provider(ModelKeyValues.USE_GROK_MODEL)
+ if api_key is not None:
+ settings.GROK_API_KEY = api_key
+ if model is not None:
+ settings.GROK_MODEL_NAME = model
+ if cost_per_input_token is not None:
+ settings.GROK_COST_PER_INPUT_TOKEN = cost_per_input_token
+ if cost_per_output_token is not None:
+ settings.GROK_COST_PER_OUTPUT_TOKEN = cost_per_output_token
+
+ handled, path, updates = edit_ctx.result
+
+ effective_model = settings.GROK_MODEL_NAME
+ if not effective_model:
+ raise typer.BadParameter(
+ "Grok model name is not set. Pass --model (or set GROK_MODEL_NAME).",
+ param_hint="--model",
+ )
+ _handle_save_result(
+ handled=handled,
+ path=path,
+ updates=updates,
+ save=save,
+ quiet=quiet,
+ success_msg=(
+ f":raising_hands: Congratulations! You're now using Grok `{escape(effective_model)}` for all evals that require an LLM."
+ ),
+ )
+
+
+@app.command(name="unset-grok")
+def unset_grok_model_env(
+ save: Optional[str] = typer.Option(
+ None,
+ "-s",
+ "--save",
+ help="Remove only the Grok model related environment variables from a dotenv file. "
+ "Usage: --save=dotenv[:path] (default: .env.local)",
+ ),
+ clear_secrets: bool = typer.Option(
+ False,
+ "-x",
+ "--clear-secrets",
+ help="Also remove GROK_API_KEY from the dotenv store.",
+ ),
+ quiet: bool = typer.Option(
+ False,
+ "-q",
+ "--quiet",
+ help="Suppress printing to the terminal (useful for CI).",
+ ),
+):
+ settings = get_settings()
+ with settings.edit(save=save) as edit_ctx:
+ settings.USE_GROK_MODEL = None
+ settings.GROK_MODEL_NAME = None
+ settings.GROK_COST_PER_INPUT_TOKEN = None
+ settings.GROK_COST_PER_OUTPUT_TOKEN = None
+ if clear_secrets:
+ settings.GROK_API_KEY = None
+
+ handled, path, updates = edit_ctx.result
+
+ if _handle_save_result(
+ handled=handled,
+ path=path,
+ updates=updates,
+ save=save,
+ quiet=quiet,
+ updated_msg="Removed Grok model environment variables from {path}.",
+ tip_msg=None,
+ ):
+ if is_openai_configured():
+ print(
+ ":raised_hands: OpenAI will still be used by default because OPENAI_API_KEY is set."
+ )
+ else:
+ print(
+ "The Grok model configuration has been removed. No model is currently configured, but you can set one with the CLI or add credentials to .env[.local]."
+ )
diff --git a/deepeval/cli/providers/llms/litellm.py b/deepeval/cli/providers/llms/litellm.py
new file mode 100644
index 000000000..2d36f83de
--- /dev/null
+++ b/deepeval/cli/providers/llms/litellm.py
@@ -0,0 +1,172 @@
+"""`deepeval set-litellm, unset-litellm` commands."""
+
+import typer
+from typing import Optional
+from rich import print
+from rich.markup import escape
+
+from deepeval.cli.app import app
+from deepeval.cli.utils import (
+ coerce_blank_to_none,
+ handle_save_result as _handle_save_result,
+ is_openai_configured,
+)
+from deepeval.config.settings import get_settings
+from deepeval.key_handler import (
+ ModelKeyValues,
+)
+
+
+@app.command(name="set-litellm")
+def set_litellm_model_env(
+ model: Optional[str] = typer.Option(
+ None,
+ "-m",
+ "--model",
+ help="Model identifier to use for this provider",
+ ),
+ prompt_api_key: bool = typer.Option(
+ False,
+ "-k",
+ "--prompt-api-key",
+ help=(
+ "Prompt for LITELLM_API_KEY (input hidden). Not suitable for CI. "
+ "If --save (or DEEPEVAL_DEFAULT_SAVE) is used, the key is written to dotenv in plaintext."
+ ),
+ ),
+ base_url: Optional[str] = typer.Option(
+ None,
+ "-u",
+ "--base-url",
+ help="Override the API endpoint/base URL used by this provider.",
+ ),
+ proxy_prompt_api_key: bool = typer.Option(
+ False,
+ "-K",
+ "--proxy-prompt-api-key",
+ help=(
+ "Prompt for LITELLM_PROXY_API_KEY (input hidden). Not suitable for CI. "
+ "If --save (or DEEPEVAL_DEFAULT_SAVE) is used, the key is written to dotenv in plaintext."
+ ),
+ ),
+ proxy_base_url: Optional[str] = typer.Option(
+ None,
+ "-U",
+ "--proxy-base-url",
+ help="Override the LITELLM_PROXY_API_BASE URL (useful for proxies, gateways, or self-hosted endpoints).",
+ ),
+ save: Optional[str] = typer.Option(
+ None,
+ "-s",
+ "--save",
+ help="Persist CLI parameters as environment variables in a dotenv file. "
+ "Usage: --save=dotenv[:path] (default: .env.local)",
+ ),
+ quiet: bool = typer.Option(
+ False,
+ "-q",
+ "--quiet",
+ help="Suppress printing to the terminal (useful for CI).",
+ ),
+):
+ api_key = None
+ if prompt_api_key:
+ api_key = coerce_blank_to_none(
+ typer.prompt("LiteLLM API key", hide_input=True)
+ )
+
+ proxy_api_key = None
+ if proxy_prompt_api_key:
+ proxy_api_key = coerce_blank_to_none(
+ typer.prompt("LiteLLM Proxy API key", hide_input=True)
+ )
+
+ model = coerce_blank_to_none(model)
+ base_url = coerce_blank_to_none(base_url)
+ proxy_base_url = coerce_blank_to_none(proxy_base_url)
+
+ settings = get_settings()
+ with settings.edit(save=save) as edit_ctx:
+ edit_ctx.switch_model_provider(ModelKeyValues.USE_LITELLM)
+ if model is not None:
+ settings.LITELLM_MODEL_NAME = model
+ if api_key is not None:
+ settings.LITELLM_API_KEY = api_key
+ if base_url is not None:
+ settings.LITELLM_API_BASE = base_url
+ if proxy_api_key is not None:
+ settings.LITELLM_PROXY_API_KEY = proxy_api_key
+ if proxy_base_url is not None:
+ settings.LITELLM_PROXY_API_BASE = proxy_base_url
+
+ handled, path, updates = edit_ctx.result
+
+ effective_model = settings.LITELLM_MODEL_NAME
+ if not effective_model:
+ raise typer.BadParameter(
+ "LiteLLM model name is not set. Pass --model (or set LITELLM_MODEL_NAME).",
+ param_hint="--model",
+ )
+ _handle_save_result(
+ handled=handled,
+ path=path,
+ updates=updates,
+ save=save,
+ quiet=quiet,
+ success_msg=(
+ f":raising_hands: Congratulations! You're now using LiteLLM `{escape(effective_model)}` for all evals that require an LLM."
+ ),
+ )
+
+
+@app.command(name="unset-litellm")
+def unset_litellm_model_env(
+ save: Optional[str] = typer.Option(
+ None,
+ "-s",
+ "--save",
+ help="Remove only the LiteLLM related environment variables from a dotenv file. "
+ "Usage: --save=dotenv[:path] (default: .env.local)",
+ ),
+ clear_secrets: bool = typer.Option(
+ False,
+ "-x",
+ "--clear-secrets",
+ help="Also remove LITELLM_API_KEY and LITELLM_PROXY_API_KEY from the dotenv store.",
+ ),
+ quiet: bool = typer.Option(
+ False,
+ "-q",
+ "--quiet",
+ help="Suppress printing to the terminal (useful for CI).",
+ ),
+):
+ settings = get_settings()
+ with settings.edit(save=save) as edit_ctx:
+ settings.USE_LITELLM = None
+ settings.LITELLM_MODEL_NAME = None
+ settings.LITELLM_API_BASE = None
+ settings.LITELLM_PROXY_API_BASE = None
+ if clear_secrets:
+ settings.LITELLM_API_KEY = None
+ settings.LITELLM_PROXY_API_KEY = None
+
+ handled, path, updates = edit_ctx.result
+
+ if _handle_save_result(
+ handled=handled,
+ path=path,
+ updates=updates,
+ save=save,
+ quiet=quiet,
+ updated_msg="Removed LiteLLM model environment variables from {path}.",
+ tip_msg=None,
+ ):
+ if is_openai_configured():
+ print(
+ ":raised_hands: OpenAI will still be used by default because OPENAI_API_KEY is set."
+ )
+ else:
+ print(
+ "The LiteLLM model configuration has been removed. No model is currently configured, but you can set one with the CLI or add credentials to .env[.local]."
+ )
diff --git a/deepeval/cli/providers/llms/local_embeddings.py b/deepeval/cli/providers/llms/local_embeddings.py
new file mode 100644
index 000000000..cc2fcea09
--- /dev/null
+++ b/deepeval/cli/providers/llms/local_embeddings.py
@@ -0,0 +1,144 @@
+"""`deepeval set-local-embeddings, unset-local-embeddings` commands."""
+
+import typer
+from typing import Optional
+from rich import print
+from rich.markup import escape
+
+from deepeval.cli.app import app
+from deepeval.cli.utils import (
+ coerce_blank_to_none,
+ handle_save_result as _handle_save_result,
+ is_openai_configured,
+)
+from deepeval.config.settings import get_settings
+from deepeval.key_handler import (
+ EmbeddingKeyValues,
+)
+
+
+@app.command(name="set-local-embeddings")
+def set_local_embeddings_env(
+ model: Optional[str] = typer.Option(
+ None,
+ "-m",
+ "--model",
+ help="Model identifier to use for this provider",
+ ),
+ prompt_api_key: bool = typer.Option(
+ False,
+ "-k",
+ "--prompt-api-key",
+ help=(
+ "Prompt for LOCAL_EMBEDDING_API_KEY (input hidden). Not suitable for CI. "
+ "If --save (or DEEPEVAL_DEFAULT_SAVE) is used, the key is written to dotenv in plaintext."
+ ),
+ ),
+ base_url: Optional[str] = typer.Option(
+ None,
+ "-u",
+ "--base-url",
+ help="Override the API endpoint/base URL used by this provider.",
+ ),
+ save: Optional[str] = typer.Option(
+ None,
+ "-s",
+ "--save",
+ help="Persist CLI parameters as environment variables in a dotenv file. "
+ "Usage: --save=dotenv[:path] (default: .env.local)",
+ ),
+ quiet: bool = typer.Option(
+ False,
+ "-q",
+ "--quiet",
+ help="Suppress printing to the terminal (useful for CI).",
+ ),
+):
+ api_key = None
+ if prompt_api_key:
+ api_key = coerce_blank_to_none(
+ typer.prompt("Local Embedding Model API key", hide_input=True)
+ )
+
+ model = coerce_blank_to_none(model)
+ base_url = coerce_blank_to_none(base_url)
+
+ settings = get_settings()
+ with settings.edit(save=save) as edit_ctx:
+ edit_ctx.switch_model_provider(EmbeddingKeyValues.USE_LOCAL_EMBEDDINGS)
+ if model is not None:
+ settings.LOCAL_EMBEDDING_MODEL_NAME = model
+ if base_url is not None:
+ settings.LOCAL_EMBEDDING_BASE_URL = base_url
+ if api_key is not None:
+ settings.LOCAL_EMBEDDING_API_KEY = api_key
+
+ handled, path, updates = edit_ctx.result
+
+ effective_model = settings.LOCAL_EMBEDDING_MODEL_NAME
+ if not effective_model:
+ raise typer.BadParameter(
+ "Local embedding model name is not set. Pass --model (or set LOCAL_EMBEDDING_MODEL_NAME).",
+ param_hint="--model",
+ )
+ _handle_save_result(
+ handled=handled,
+ path=path,
+ updates=updates,
+ save=save,
+ quiet=quiet,
+ success_msg=(
+ f":raising_hands: Congratulations! You're now using the local embedding model `{escape(effective_model)}` for all evals that require text embeddings."
+ ),
+ )
+
+
+@app.command(name="unset-local-embeddings")
+def unset_local_embeddings_env(
+ save: Optional[str] = typer.Option(
+ None,
+ "-s",
+ "--save",
+ help="Remove only the local embedding related environment variables from a dotenv file. "
+ "Usage: --save=dotenv[:path] (default: .env.local)",
+ ),
+ clear_secrets: bool = typer.Option(
+ False,
+ "-x",
+ "--clear-secrets",
+ help="Also remove LOCAL_MODEL_API_KEY from the dotenv store.",
+ ),
+ quiet: bool = typer.Option(
+ False,
+ "-q",
+ "--quiet",
+ help="Suppress printing to the terminal (useful for CI).",
+ ),
+):
+ settings = get_settings()
+ with settings.edit(save=save) as edit_ctx:
+ settings.LOCAL_EMBEDDING_MODEL_NAME = None
+ settings.LOCAL_EMBEDDING_BASE_URL = None
+ settings.USE_LOCAL_EMBEDDINGS = None
+ if clear_secrets:
+ settings.LOCAL_EMBEDDING_API_KEY = None
+
+ handled, path, updates = edit_ctx.result
+
+ if _handle_save_result(
+ handled=handled,
+ path=path,
+ updates=updates,
+ save=save,
+ quiet=quiet,
+ updated_msg="Removed local embedding environment variables from {path}.",
+ tip_msg=None,
+ ):
+ if is_openai_configured():
+ print(
+ ":raised_hands: OpenAI will still be used by default because OPENAI_API_KEY is set."
+ )
+ else:
+ print(
+ "The local embeddings model configuration has been removed. No model is currently configured, but you can set one with the CLI or add credentials to .env[.local]."
+ )
diff --git a/deepeval/cli/providers/llms/local_model.py b/deepeval/cli/providers/llms/local_model.py
new file mode 100644
index 000000000..c6700f9cf
--- /dev/null
+++ b/deepeval/cli/providers/llms/local_model.py
@@ -0,0 +1,154 @@
+"""`deepeval set-local-model, unset-local-model` commands."""
+
+import typer
+from typing import Optional
+from rich import print
+from rich.markup import escape
+
+from deepeval.cli.app import app
+from deepeval.cli.utils import (
+ coerce_blank_to_none,
+ handle_save_result as _handle_save_result,
+ is_openai_configured,
+)
+from deepeval.config.settings import get_settings
+from deepeval.key_handler import (
+ ModelKeyValues,
+)
+
+
+@app.command(name="set-local-model")
+def set_local_model_env(
+ model: Optional[str] = typer.Option(
+ None,
+ "-m",
+ "--model",
+ help="Model identifier to use for this provider",
+ ),
+ prompt_api_key: bool = typer.Option(
+ False,
+ "-k",
+ "--prompt-api-key",
+ help=(
+ "Prompt for LOCAL_MODEL_API_KEY (input hidden). Not suitable for CI. "
+ "If --save (or DEEPEVAL_DEFAULT_SAVE) is used, the key is written to dotenv in plaintext."
+ ),
+ ),
+ base_url: Optional[str] = typer.Option(
+ None,
+ "-u",
+ "--base-url",
+ help="Override the API endpoint/base URL used by this provider.",
+ ),
+ model_format: Optional[str] = typer.Option(
+ None,
+ "-f",
+ "--format",
+ help="Format of the response from the local model (default: json)",
+ ),
+ save: Optional[str] = typer.Option(
+ None,
+ "-s",
+ "--save",
+ help="Persist CLI parameters as environment variables in a dotenv file. "
+ "Usage: --save=dotenv[:path] (default: .env.local)",
+ ),
+ quiet: bool = typer.Option(
+ False,
+ "-q",
+ "--quiet",
+ help="Suppress printing to the terminal (useful for CI).",
+ ),
+):
+ api_key = None
+ if prompt_api_key:
+ api_key = coerce_blank_to_none(
+ typer.prompt("Local Model API key", hide_input=True)
+ )
+
+ model = coerce_blank_to_none(model)
+ base_url = coerce_blank_to_none(base_url)
+ model_format = coerce_blank_to_none(model_format)
+
+ settings = get_settings()
+ with settings.edit(save=save) as edit_ctx:
+ edit_ctx.switch_model_provider(ModelKeyValues.USE_LOCAL_MODEL)
+ if model is not None:
+ settings.LOCAL_MODEL_NAME = model
+ if base_url is not None:
+ settings.LOCAL_MODEL_BASE_URL = base_url
+ if api_key is not None:
+ settings.LOCAL_MODEL_API_KEY = api_key
+ if model_format is not None:
+ settings.LOCAL_MODEL_FORMAT = model_format
+
+ handled, path, updates = edit_ctx.result
+
+ effective_model = settings.LOCAL_MODEL_NAME
+ if not effective_model:
+ raise typer.BadParameter(
+ "Local model name is not set. Pass --model (or set LOCAL_MODEL_NAME).",
+ param_hint="--model",
+ )
+ _handle_save_result(
+ handled=handled,
+ path=path,
+ updates=updates,
+ save=save,
+ quiet=quiet,
+ success_msg=(
+ f":raising_hands: Congratulations! You're now using a local model `{escape(effective_model)}` for all evals that require an LLM."
+ ),
+ )
+
+
+@app.command(name="unset-local-model")
+def unset_local_model_env(
+ save: Optional[str] = typer.Option(
+ None,
+ "-s",
+ "--save",
+ help="Remove only the local model related environment variables from a dotenv file. "
+ "Usage: --save=dotenv[:path] (default: .env.local)",
+ ),
+ clear_secrets: bool = typer.Option(
+ False,
+ "-x",
+ "--clear-secrets",
+ help="Also remove LOCAL_MODEL_API_KEY from the dotenv store.",
+ ),
+ quiet: bool = typer.Option(
+ False,
+ "-q",
+ "--quiet",
+ help="Suppress printing to the terminal (useful for CI).",
+ ),
+):
+ settings = get_settings()
+ with settings.edit(save=save) as edit_ctx:
+ if clear_secrets:
+ settings.LOCAL_MODEL_API_KEY = None
+ settings.LOCAL_MODEL_NAME = None
+ settings.LOCAL_MODEL_BASE_URL = None
+ settings.LOCAL_MODEL_FORMAT = None
+ settings.USE_LOCAL_MODEL = None
+
+ handled, path, updates = edit_ctx.result
+
+ if _handle_save_result(
+ handled=handled,
+ path=path,
+ updates=updates,
+ save=save,
+ quiet=quiet,
+ updated_msg="Removed local model environment variables from {path}.",
+ tip_msg=None,
+ ):
+ if is_openai_configured():
+ print(
+ ":raised_hands: OpenAI will still be used by default because OPENAI_API_KEY is set."
+ )
+ else:
+ print(
+ "The local model configuration has been removed. No model is currently configured, but you can set one with the CLI or add credentials to .env[.local]."
+ )
diff --git a/deepeval/cli/providers/llms/moonshot.py b/deepeval/cli/providers/llms/moonshot.py
new file mode 100644
index 000000000..c1111c6c3
--- /dev/null
+++ b/deepeval/cli/providers/llms/moonshot.py
@@ -0,0 +1,158 @@
+"""`deepeval set-moonshot, unset-moonshot` commands."""
+
+import typer
+from typing import Optional
+from rich import print
+from rich.markup import escape
+
+from deepeval.cli.app import app
+from deepeval.cli.utils import (
+ coerce_blank_to_none,
+ handle_save_result as _handle_save_result,
+ is_openai_configured,
+)
+from deepeval.config.settings import get_settings
+from deepeval.key_handler import (
+ ModelKeyValues,
+)
+
+
+@app.command(name="set-moonshot")
+def set_moonshot_model_env(
+ model: Optional[str] = typer.Option(
+ None,
+ "-m",
+ "--model",
+ help="Model identifier to use for this provider",
+ ),
+ prompt_api_key: bool = typer.Option(
+ False,
+ "-k",
+ "--prompt-api-key",
+ help=(
+ "Prompt for MOONSHOT_API_KEY (input hidden). Not suitable for CI. "
+ "If --save (or DEEPEVAL_DEFAULT_SAVE) is used, the key is written to dotenv in plaintext."
+ ),
+ ),
+ cost_per_input_token: Optional[float] = typer.Option(
+ None,
+ "-i",
+ "--cost-per-input-token",
+ help=(
+ "USD per input token override used for cost tracking. Preconfigured for known models; "
+ "REQUIRED if you use a custom/unknown model."
+ ),
+ ),
+ cost_per_output_token: Optional[float] = typer.Option(
+ None,
+ "-o",
+ "--cost-per-output-token",
+ help=(
+ "USD per output token override used for cost tracking. Preconfigured for known models; "
+ "REQUIRED if you use a custom/unknown model."
+ ),
+ ),
+ save: Optional[str] = typer.Option(
+ None,
+ "-s",
+ "--save",
+ help="Persist CLI parameters as environment variables in a dotenv file. "
+ "Usage: --save=dotenv[:path] (default: .env.local)",
+ ),
+ quiet: bool = typer.Option(
+ False,
+ "-q",
+ "--quiet",
+ help="Suppress printing to the terminal (useful for CI).",
+ ),
+):
+ api_key = None
+ if prompt_api_key:
+ api_key = coerce_blank_to_none(
+ typer.prompt("Moonshot API key", hide_input=True)
+ )
+
+ model = coerce_blank_to_none(model)
+
+ settings = get_settings()
+ with settings.edit(save=save) as edit_ctx:
+ edit_ctx.switch_model_provider(ModelKeyValues.USE_MOONSHOT_MODEL)
+ if model is not None:
+ settings.MOONSHOT_MODEL_NAME = model
+ if api_key is not None:
+ settings.MOONSHOT_API_KEY = api_key
+ if cost_per_input_token is not None:
+ settings.MOONSHOT_COST_PER_INPUT_TOKEN = cost_per_input_token
+ if cost_per_output_token is not None:
+ settings.MOONSHOT_COST_PER_OUTPUT_TOKEN = cost_per_output_token
+
+ handled, path, updates = edit_ctx.result
+
+ effective_model = settings.MOONSHOT_MODEL_NAME
+ if not effective_model:
+ raise typer.BadParameter(
+ "Moonshot model name is not set. Pass --model (or set MOONSHOT_MODEL_NAME).",
+ param_hint="--model",
+ )
+ _handle_save_result(
+ handled=handled,
+ path=path,
+ updates=updates,
+ save=save,
+ quiet=quiet,
+ success_msg=(
+ f":raising_hands: Congratulations! You're now using Moonshot `{escape(effective_model)}` for all evals that require an LLM."
+ ),
+ )
+
+
+@app.command(name="unset-moonshot")
+def unset_moonshot_model_env(
+ save: Optional[str] = typer.Option(
+ None,
+ "-s",
+ "--save",
+ help="Remove only the Moonshot model related environment variables from a dotenv file. "
+ "Usage: --save=dotenv[:path] (default: .env.local)",
+ ),
+ clear_secrets: bool = typer.Option(
+ False,
+ "-x",
+ "--clear-secrets",
+ help="Also remove MOONSHOT_API_KEY from the dotenv store.",
+ ),
+ quiet: bool = typer.Option(
+ False,
+ "-q",
+ "--quiet",
+ help="Suppress printing to the terminal (useful for CI).",
+ ),
+):
+ settings = get_settings()
+ with settings.edit(save=save) as edit_ctx:
+ settings.USE_MOONSHOT_MODEL = None
+ settings.MOONSHOT_MODEL_NAME = None
+ settings.MOONSHOT_COST_PER_INPUT_TOKEN = None
+ settings.MOONSHOT_COST_PER_OUTPUT_TOKEN = None
+ if clear_secrets:
+ settings.MOONSHOT_API_KEY = None
+
+ handled, path, updates = edit_ctx.result
+
+ if _handle_save_result(
+ handled=handled,
+ path=path,
+ updates=updates,
+ save=save,
+ quiet=quiet,
+ updated_msg="Removed Moonshot model environment variables from {path}.",
+ tip_msg=None,
+ ):
+ if is_openai_configured():
+ print(
+ ":raised_hands: OpenAI will still be used by default because OPENAI_API_KEY is set."
+ )
+ else:
+ print(
+ "The Moonshot model configuration has been removed. No model is currently configured, but you can set one with the CLI or add credentials to .env[.local]."
+ )
diff --git a/deepeval/cli/providers/llms/ollama.py b/deepeval/cli/providers/llms/ollama.py
new file mode 100644
index 000000000..114eef748
--- /dev/null
+++ b/deepeval/cli/providers/llms/ollama.py
@@ -0,0 +1,240 @@
+"""`deepeval set-ollama, set-ollama-embeddings, unset-ollama, unset-ollama-embeddings` commands."""
+
+import typer
+from typing import Optional
+from rich import print
+from rich.markup import escape
+
+from deepeval.cli.app import app
+from deepeval.cli.utils import (
+ coerce_blank_to_none,
+ handle_save_result as _handle_save_result,
+ is_openai_configured,
+)
+from deepeval.config.settings import get_settings
+from deepeval.key_handler import (
+ EmbeddingKeyValues,
+ ModelKeyValues,
+)
+
+
+@app.command(name="set-ollama")
+def set_ollama_model_env(
+ model: Optional[str] = typer.Option(
+ None,
+ "-m",
+ "--model",
+ help="Model identifier to use for this provider",
+ ),
+ base_url: str = typer.Option(
+ "http://localhost:11434",
+ "-u",
+ "--base-url",
+ help="Override the API endpoint/base URL used by this provider.",
+ ),
+ save: Optional[str] = typer.Option(
+ None,
+ "-s",
+ "--save",
+ help="Persist CLI parameters as environment variables in a dotenv file. "
+ "Usage: --save=dotenv[:path] (default: .env.local)",
+ ),
+ quiet: bool = typer.Option(
+ False,
+ "-q",
+ "--quiet",
+ help="Suppress printing to the terminal (useful for CI).",
+ ),
+):
+ model = coerce_blank_to_none(model)
+ base_url = coerce_blank_to_none(base_url)
+
+ settings = get_settings()
+ with settings.edit(save=save) as edit_ctx:
+ edit_ctx.switch_model_provider(ModelKeyValues.USE_LOCAL_MODEL)
+ settings.LOCAL_MODEL_API_KEY = "ollama"
+ if model is not None:
+ settings.OLLAMA_MODEL_NAME = model
+ if base_url is not None:
+ settings.LOCAL_MODEL_BASE_URL = base_url
+
+ handled, path, updates = edit_ctx.result
+
+ effective_model = settings.OLLAMA_MODEL_NAME
+ if not effective_model:
+ raise typer.BadParameter(
+ "Ollama model name is not set. Pass --model (or set OLLAMA_MODEL_NAME).",
+ param_hint="--model",
+ )
+ _handle_save_result(
+ handled=handled,
+ path=path,
+ updates=updates,
+ save=save,
+ quiet=quiet,
+ success_msg=(
+ f":raising_hands: Congratulations! You're now using a local Ollama model `{escape(effective_model)}` for all evals that require an LLM."
+ ),
+ )
+
+
+@app.command(name="unset-ollama")
+def unset_ollama_model_env(
+ save: Optional[str] = typer.Option(
+ None,
+ "-s",
+ "--save",
+ help="Remove only the Ollama related environment variables from a dotenv file. "
+ "Usage: --save=dotenv[:path] (default: .env.local)",
+ ),
+ clear_secrets: bool = typer.Option(
+ False,
+ "-x",
+ "--clear-secrets",
+ help="Also remove LOCAL_MODEL_API_KEY from the dotenv store.",
+ ),
+ quiet: bool = typer.Option(
+ False,
+ "-q",
+ "--quiet",
+ help="Suppress printing to the terminal (useful for CI).",
+ ),
+):
+ settings = get_settings()
+ with settings.edit(save=save) as edit_ctx:
+ if clear_secrets:
+ settings.LOCAL_MODEL_API_KEY = None
+ settings.OLLAMA_MODEL_NAME = None
+ settings.LOCAL_MODEL_BASE_URL = None
+ settings.USE_LOCAL_MODEL = None
+
+ handled, path, updates = edit_ctx.result
+
+ if _handle_save_result(
+ handled=handled,
+ path=path,
+ updates=updates,
+ save=save,
+ quiet=quiet,
+ updated_msg="Removed local Ollama environment variables from {path}.",
+ tip_msg=None,
+ ):
+ if is_openai_configured():
+ print(
+ ":raised_hands: OpenAI will still be used by default because OPENAI_API_KEY is set."
+ )
+ else:
+ print(
+ "The local Ollama model configuration has been removed. No model is currently configured, but you can set one with the CLI or add credentials to .env[.local]."
+ )
+
+
+@app.command(name="set-ollama-embeddings")
+def set_ollama_embeddings_env(
+ model: Optional[str] = typer.Option(
+ None,
+ "-m",
+ "--model",
+ help="Model identifier to use for this provider.",
+ ),
+ base_url: str = typer.Option(
+ "http://localhost:11434",
+ "-u",
+ "--base-url",
+ help="Override the API endpoint/base URL used by this provider.",
+ ),
+ save: Optional[str] = typer.Option(
+ None,
+ "-s",
+ "--save",
+ help="Persist CLI parameters as environment variables in a dotenv file. "
+ "Usage: --save=dotenv[:path] (default: .env.local)",
+ ),
+ quiet: bool = typer.Option(
+ False,
+ "-q",
+ "--quiet",
+ help="Suppress printing to the terminal (useful for CI).",
+ ),
+):
+ model = coerce_blank_to_none(model)
+ base_url = coerce_blank_to_none(base_url)
+
+ settings = get_settings()
+ with settings.edit(save=save) as edit_ctx:
+ edit_ctx.switch_model_provider(EmbeddingKeyValues.USE_LOCAL_EMBEDDINGS)
+ settings.LOCAL_EMBEDDING_API_KEY = "ollama"
+ if model is not None:
+ settings.LOCAL_EMBEDDING_MODEL_NAME = model
+ if base_url is not None:
+ settings.LOCAL_EMBEDDING_BASE_URL = base_url
+
+ handled, path, updates = edit_ctx.result
+
+ effective_model = settings.LOCAL_EMBEDDING_MODEL_NAME
+ if not effective_model:
+ raise typer.BadParameter(
+ "Ollama embedding model name is not set. Pass --model (or set LOCAL_EMBEDDING_MODEL_NAME).",
+ param_hint="--model",
+ )
+ _handle_save_result(
+ handled=handled,
+ path=path,
+ updates=updates,
+ save=save,
+ quiet=quiet,
+ success_msg=(
+ f":raising_hands: Congratulations! You're now using the Ollama embedding model `{escape(effective_model)}` for all evals that require text embeddings."
+ ),
+ )
+
+
+@app.command(name="unset-ollama-embeddings")
+def unset_ollama_embeddings_env(
+ save: Optional[str] = typer.Option(
+ None,
+ "-s",
+ "--save",
+ help="Remove only the Ollama embedding related environment variables from a dotenv file. "
+ "Usage: --save=dotenv[:path] (default: .env.local)",
+ ),
+ clear_secrets: bool = typer.Option(
+ False,
+ "-x",
+ "--clear-secrets",
+ help="Also remove LOCAL_EMBEDDING_API_KEY from the dotenv store.",
+ ),
+ quiet: bool = typer.Option(
+ False,
+ "-q",
+ "--quiet",
+ help="Suppress printing to the terminal (useful for CI).",
+ ),
+):
+ settings = get_settings()
+ with settings.edit(save=save) as edit_ctx:
+ if clear_secrets:
+ settings.LOCAL_EMBEDDING_API_KEY = None
+ settings.LOCAL_EMBEDDING_MODEL_NAME = None
+ settings.LOCAL_EMBEDDING_BASE_URL = None
+ settings.USE_LOCAL_EMBEDDINGS = None
+
+ handled, path, updates = edit_ctx.result
+
+ if _handle_save_result(
+ handled=handled,
+ path=path,
+ updates=updates,
+ save=save,
+ quiet=quiet,
+ updated_msg="Removed local Ollama embedding environment variables from {path}.",
+ tip_msg=None,
+ ):
+ if is_openai_configured():
+ print(
+ ":raised_hands: Regular OpenAI embeddings will still be used by default because OPENAI_API_KEY is set."
+ )
+ else:
+ print(
+ "The local Ollama embedding model configuration has been removed. No model is currently configured, but you can set one with the CLI or add credentials to .env[.local]."
+ )
diff --git a/deepeval/cli/providers/llms/openai.py b/deepeval/cli/providers/llms/openai.py
new file mode 100644
index 000000000..36a07ce0a
--- /dev/null
+++ b/deepeval/cli/providers/llms/openai.py
@@ -0,0 +1,214 @@
+"""`deepeval set-openai, unset-openai` commands."""
+
+import typer
+from typing import Optional
+from rich import print
+from rich.markup import escape
+
+from deepeval.cli.app import app
+from deepeval.cli.utils import (
+ coerce_blank_to_none,
+ handle_save_result as _handle_save_result,
+ is_openai_configured,
+)
+from deepeval.config.settings import get_settings
+from deepeval.key_handler import (
+ ModelKeyValues,
+)
+
+
+@app.command(name="set-openai")
+def set_openai_env(
+ model: Optional[str] = typer.Option(
+ None,
+ "-m",
+ "--model",
+ help="Model identifier to use for this provider (e.g., `gpt-4.1`).",
+ ),
+ prompt_api_key: bool = typer.Option(
+ False,
+ "-k",
+ "--prompt-api-key",
+ help=(
+ "Prompt for OPENAI_API_KEY (input hidden). Not suitable for CI. "
+ "If --save (or DEEPEVAL_DEFAULT_SAVE) is used, the key is written to dotenv in plaintext."
+ ),
+ ),
+ cost_per_input_token: Optional[float] = typer.Option(
+ None,
+ "-i",
+ "--cost-per-input-token",
+ help=(
+ "USD per input token override used for cost tracking. Preconfigured for known models; "
+ "REQUIRED if you use a custom/unknown model."
+ ),
+ ),
+ cost_per_output_token: Optional[float] = typer.Option(
+ None,
+ "-o",
+ "--cost-per-output-token",
+ help=(
+ "USD per output token override used for cost tracking. Preconfigured for known models; "
+ "REQUIRED if you use a custom/unknown model."
+ ),
+ ),
+ save: Optional[str] = typer.Option(
+ None,
+ "-s",
+ "--save",
+ help="Persist CLI parameters as environment variables in a dotenv file. "
+ "Usage: --save=dotenv[:path] (default: .env.local)",
+ ),
+ quiet: bool = typer.Option(
+ False,
+ "-q",
+ "--quiet",
+ help="Suppress printing to the terminal (useful for CI).",
+ ),
+):
+ """
+ Configure OpenAI as the active LLM provider.
+
+ What this does:
+ - Sets the active provider flag to `USE_OPENAI_MODEL`.
+ - Persists the selected model name and any cost overrides in the JSON store.
+ - secrets are never written to `.deepeval/.deepeval` (JSON).
+
+ Pricing rules:
+ - If `model` is a known OpenAI model, you may omit costs (built‑in pricing is used).
+ - If `model` is custom/unsupported, you must provide both
+ `--cost-per-input-token` and `--cost-per-output-token`.
+
+ Secrets & saving:
+
+ - If you run with --prompt-api-key, DeepEval will set OPENAI_API_KEY for this session.
+ - If --save=dotenv[:path] is used (or DEEPEVAL_DEFAULT_SAVE is set), the key will be written to that dotenv file (plaintext).
+
+ Secrets are never written to .deepeval/.deepeval (legacy JSON store).
+
+ Args:
+ --model: OpenAI model name, such as `gpt-4o-mini`.
+ --prompt-api-key: Prompt interactively for OPENAI_API_KEY (input hidden). Avoids putting secrets on the command line (shell history/process args). Not suitable for CI.
+ --cost-per-input-token: USD per input token (optional for known models).
+ --cost-per-output-token: USD per output token (optional for known models).
+ --save: Persist config (and supported secrets) to a dotenv file; format `dotenv[:path]`.
+ --quiet: Suppress printing to the terminal.
+
+ Example:
+ deepeval set-openai \\
+ --model gpt-4o-mini \\
+ --cost-per-input-token 0.0005 \\
+ --cost-per-output-token 0.0015 \\
+ --save dotenv:.env.local
+ """
+ api_key = None
+ if prompt_api_key:
+ api_key = coerce_blank_to_none(
+ typer.prompt("OpenAI API key", hide_input=True)
+ )
+
+ model = coerce_blank_to_none(model)
+
+ settings = get_settings()
+ with settings.edit(save=save) as edit_ctx:
+ edit_ctx.switch_model_provider(ModelKeyValues.USE_OPENAI_MODEL)
+ if model is not None:
+ settings.OPENAI_MODEL_NAME = model
+ if api_key is not None:
+ settings.OPENAI_API_KEY = api_key
+ if cost_per_input_token is not None:
+ settings.OPENAI_COST_PER_INPUT_TOKEN = cost_per_input_token
+ if cost_per_output_token is not None:
+ settings.OPENAI_COST_PER_OUTPUT_TOKEN = cost_per_output_token
+
+ handled, path, updates = edit_ctx.result
+
+ effective_model = settings.OPENAI_MODEL_NAME
+ if not effective_model:
+ raise typer.BadParameter(
+ "OpenAI model name is not set. Pass --model (or set OPENAI_MODEL_NAME).",
+ param_hint="--model",
+ )
+
+ _handle_save_result(
+ handled=handled,
+ path=path,
+ updates=updates,
+ save=save,
+ quiet=quiet,
+ success_msg=(
+ f":raising_hands: Congratulations! You're now using OpenAI's `{escape(effective_model)}` "
+ "for all evals that require an LLM."
+ ),
+ )
+
+
+@app.command(name="unset-openai")
+def unset_openai_env(
+ save: Optional[str] = typer.Option(
+ None,
+ "-s",
+ "--save",
+ help="Remove only the OpenAI related environment variables from a dotenv file. "
+ "Usage: --save=dotenv[:path] (default: .env.local)",
+ ),
+ clear_secrets: bool = typer.Option(
+ False,
+ "-x",
+ "--clear-secrets",
+ help="Also remove OPENAI_API_KEY from the dotenv store.",
+ ),
+ quiet: bool = typer.Option(
+ False,
+ "-q",
+ "--quiet",
+ help="Suppress printing to the terminal (useful for CI).",
+ ),
+):
+ """
+ Unset OpenAI as the active provider.
+
+ Behavior:
+ - Removes OpenAI keys (model, costs, toggle) from the JSON store.
+ - If `--save` is provided, removes those keys from the specified dotenv file.
+ - After unsetting, if `OPENAI_API_KEY` is still set in the environment,
+ OpenAI may still be usable by default. Otherwise, no active provider is configured.
+
+ Args:
+ --save: Remove OpenAI keys from the given dotenv file as well.
+ --clear-secrets: Removes OPENAI_API_KEY from the dotenv store
+ --quiet: Suppress printing to the terminal
+
+ Example:
+ deepeval unset-openai --save dotenv:.env.local
+ """
+
+ settings = get_settings()
+ with settings.edit(save=save) as edit_ctx:
+ settings.OPENAI_MODEL_NAME = None
+ settings.OPENAI_COST_PER_INPUT_TOKEN = None
+ settings.OPENAI_COST_PER_OUTPUT_TOKEN = None
+ settings.USE_OPENAI_MODEL = None
+ if clear_secrets:
+ settings.OPENAI_API_KEY = None
+
+ handled, path, updates = edit_ctx.result
+
+ if _handle_save_result(
+ handled=handled,
+ path=path,
+ updates=updates,
+ save=save,
+ quiet=quiet,
+ updated_msg="Removed OpenAI environment variables from {path}.",
+ tip_msg=None,
+ ):
+ if is_openai_configured():
+ print(
+ ":raised_hands: OpenAI will still be used by default because OPENAI_API_KEY is set."
+ )
+ else:
+ print(
+ "OpenAI has been unset. No active provider is configured. "
+ "Set one with the CLI, or add credentials to .env[.local]."
+ )
diff --git a/deepeval/cli/providers/llms/openrouter.py b/deepeval/cli/providers/llms/openrouter.py
new file mode 100644
index 000000000..f55c181b2
--- /dev/null
+++ b/deepeval/cli/providers/llms/openrouter.py
@@ -0,0 +1,180 @@
+"""`deepeval set-openrouter, unset-openrouter` commands."""
+
+import typer
+from typing import Optional
+from rich import print
+from rich.markup import escape
+
+from deepeval.cli.app import app
+from deepeval.cli.utils import (
+ coerce_blank_to_none,
+ handle_save_result as _handle_save_result,
+ is_openai_configured,
+)
+from deepeval.config.settings import get_settings
+from deepeval.key_handler import (
+ ModelKeyValues,
+)
+
+
+@app.command(name="set-openrouter")
+def set_openrouter_model_env(
+ model: Optional[str] = typer.Option(
+ None,
+ "-m",
+ "--model",
+ help="Model identifier to use for this provider (e.g., `openai/gpt-4.1`).",
+ ),
+ prompt_api_key: bool = typer.Option(
+ False,
+ "-k",
+ "--prompt-api-key",
+ help=(
+ "Prompt for OPENROUTER_API_KEY (input hidden). Not suitable for CI. "
+ "If --save (or DEEPEVAL_DEFAULT_SAVE) is used, the key is written to dotenv in plaintext."
+ ),
+ ),
+ base_url: Optional[str] = typer.Option(
+ None,
+ "-u",
+ "--base-url",
+ help="Override the API endpoint/base URL used by this provider (default: https://openrouter.ai/api/v1).",
+ ),
+ temperature: Optional[float] = typer.Option(
+ None,
+ "-t",
+ "--temperature",
+ help="Override the global TEMPERATURE used by LLM providers (e.g., 0.0 for deterministic behavior).",
+ ),
+ cost_per_input_token: Optional[float] = typer.Option(
+ None,
+ "-i",
+ "--cost-per-input-token",
+ help=(
+ "USD per input token used for cost tracking. "
+ "If unset and OpenRouter does not return pricing metadata, "
+ "costs will not be calculated."
+ ),
+ ),
+ cost_per_output_token: Optional[float] = typer.Option(
+ None,
+ "-o",
+ "--cost-per-output-token",
+ help=(
+ "USD per output token used for cost tracking. "
+ "If unset and OpenRouter does not return pricing metadata, "
+ "costs will not be calculated."
+ ),
+ ),
+ save: Optional[str] = typer.Option(
+ None,
+ "-s",
+ "--save",
+ help="Persist CLI parameters as environment variables in a dotenv file. "
+ "Usage: --save=dotenv[:path] (default: .env.local)",
+ ),
+ quiet: bool = typer.Option(
+ False,
+ "-q",
+ "--quiet",
+ help="Suppress printing to the terminal (useful for CI).",
+ ),
+):
+ api_key = None
+ if prompt_api_key:
+ api_key = coerce_blank_to_none(
+ typer.prompt("OpenRouter API key", hide_input=True)
+ )
+
+ model = coerce_blank_to_none(model)
+ base_url = coerce_blank_to_none(base_url)
+
+ settings = get_settings()
+ with settings.edit(save=save) as edit_ctx:
+ edit_ctx.switch_model_provider(ModelKeyValues.USE_OPENROUTER_MODEL)
+ if model is not None:
+ settings.OPENROUTER_MODEL_NAME = model
+ if api_key is not None:
+ settings.OPENROUTER_API_KEY = api_key
+ if base_url is not None:
+ settings.OPENROUTER_BASE_URL = base_url
+ if temperature is not None:
+ settings.TEMPERATURE = temperature
+ if cost_per_input_token is not None:
+ settings.OPENROUTER_COST_PER_INPUT_TOKEN = cost_per_input_token
+ if cost_per_output_token is not None:
+ settings.OPENROUTER_COST_PER_OUTPUT_TOKEN = cost_per_output_token
+
+ handled, path, updates = edit_ctx.result
+
+ effective_model = settings.OPENROUTER_MODEL_NAME
+ if not effective_model:
+ raise typer.BadParameter(
+ "OpenRouter model name is not set. Pass --model (or set OPENROUTER_MODEL_NAME).",
+ param_hint="--model",
+ )
+
+ _handle_save_result(
+ handled=handled,
+ path=path,
+ updates=updates,
+ save=save,
+ quiet=quiet,
+ success_msg=(
+ f":raising_hands: Congratulations! You're now using OpenRouter `{escape(effective_model)}` for all evals that require an LLM."
+ ),
+ )
+
+
+@app.command(name="unset-openrouter")
+def unset_openrouter_model_env(
+ save: Optional[str] = typer.Option(
+ None,
+ "-s",
+ "--save",
+ help="Remove only the OpenRouter model related environment variables from a dotenv file. "
+ "Usage: --save=dotenv[:path] (default: .env.local)",
+ ),
+ clear_secrets: bool = typer.Option(
+ False,
+ "-x",
+ "--clear-secrets",
+ help="Also remove OPENROUTER_API_KEY from the dotenv store.",
+ ),
+ quiet: bool = typer.Option(
+ False,
+ "-q",
+ "--quiet",
+ help="Suppress printing to the terminal (useful for CI).",
+ ),
+):
+ settings = get_settings()
+ with settings.edit(save=save) as edit_ctx:
+ settings.USE_OPENROUTER_MODEL = None
+ settings.OPENROUTER_MODEL_NAME = None
+ settings.OPENROUTER_BASE_URL = None
+ settings.OPENROUTER_COST_PER_INPUT_TOKEN = None
+ settings.OPENROUTER_COST_PER_OUTPUT_TOKEN = None
+ # Intentionally do NOT touch TEMPERATURE here; it's a global dial.
+ if clear_secrets:
+ settings.OPENROUTER_API_KEY = None
+
+ handled, path, updates = edit_ctx.result
+
+ if _handle_save_result(
+ handled=handled,
+ path=path,
+ updates=updates,
+ save=save,
+ quiet=quiet,
+ updated_msg="Removed OpenRouter model environment variables from {path}.",
+ tip_msg=None,
+ ):
+ if is_openai_configured():
+ print(
+ ":raised_hands: OpenAI will still be used by default because OPENAI_API_KEY is set."
+ )
+ else:
+ print(
+ "The OpenRouter model configuration has been removed. No model is currently configured, but you can set one with the CLI or add credentials to .env[.local]."
+ )
diff --git a/deepeval/cli/providers/llms/portkey.py b/deepeval/cli/providers/llms/portkey.py
new file mode 100644
index 000000000..f18cdb89d
--- /dev/null
+++ b/deepeval/cli/providers/llms/portkey.py
@@ -0,0 +1,154 @@
+"""`deepeval set-portkey, unset-portkey` commands."""
+
+import typer
+from typing import Optional
+from rich import print
+from rich.markup import escape
+
+from deepeval.cli.app import app
+from deepeval.cli.utils import (
+ coerce_blank_to_none,
+ handle_save_result as _handle_save_result,
+ is_openai_configured,
+)
+from deepeval.config.settings import get_settings
+from deepeval.key_handler import (
+ ModelKeyValues,
+)
+
+
+@app.command(name="set-portkey")
+def set_portkey_model_env(
+ model: Optional[str] = typer.Option(
+ None,
+ "-m",
+ "--model",
+ help="Model identifier to use for this provider",
+ ),
+ prompt_api_key: bool = typer.Option(
+ False,
+ "-k",
+ "--prompt-api-key",
+ help=(
+ "Prompt for PORTKEY_API_KEY (input hidden). Not suitable for CI. "
+ "If --save (or DEEPEVAL_DEFAULT_SAVE) is used, the key is written to dotenv in plaintext."
+ ),
+ ),
+ base_url: Optional[str] = typer.Option(
+ None,
+ "-u",
+ "--base-url",
+ help="Override the API endpoint/base URL used by this provider.",
+ ),
+ provider: Optional[str] = typer.Option(
+ None,
+ "-P",
+ "--provider",
+ help="Override the PORTKEY_PROVIDER_NAME.",
+ ),
+ save: Optional[str] = typer.Option(
+ None,
+ "-s",
+ "--save",
+ help="Persist CLI parameters as environment variables in a dotenv file. "
+ "Usage: --save=dotenv[:path] (default: .env.local)",
+ ),
+ quiet: bool = typer.Option(
+ False,
+ "-q",
+ "--quiet",
+ help="Suppress printing to the terminal (useful for CI).",
+ ),
+):
+ api_key = None
+ if prompt_api_key:
+ api_key = coerce_blank_to_none(
+ typer.prompt("Portkey API key", hide_input=True)
+ )
+
+ model = coerce_blank_to_none(model)
+ base_url = coerce_blank_to_none(base_url)
+ provider = coerce_blank_to_none(provider)
+
+ settings = get_settings()
+ with settings.edit(save=save) as edit_ctx:
+ edit_ctx.switch_model_provider(ModelKeyValues.USE_PORTKEY_MODEL)
+ if model is not None:
+ settings.PORTKEY_MODEL_NAME = model
+ if api_key is not None:
+ settings.PORTKEY_API_KEY = api_key
+ if base_url is not None:
+ settings.PORTKEY_BASE_URL = base_url
+ if provider is not None:
+ settings.PORTKEY_PROVIDER_NAME = provider
+
+ handled, path, updates = edit_ctx.result
+
+ effective_model = settings.PORTKEY_MODEL_NAME
+ if not effective_model:
+ raise typer.BadParameter(
+ "Portkey model name is not set. Pass --model (or set PORTKEY_MODEL_NAME).",
+ param_hint="--model",
+ )
+ _handle_save_result(
+ handled=handled,
+ path=path,
+ updates=updates,
+ save=save,
+ quiet=quiet,
+ success_msg=(
+ f":raising_hands: Congratulations! You're now using Portkey `{escape(effective_model)}` for all evals that require an LLM."
+ ),
+ )
+
+
+@app.command(name="unset-portkey")
+def unset_portkey_model_env(
+ save: Optional[str] = typer.Option(
+ None,
+ "-s",
+ "--save",
+ help="Remove only the Portkey related environment variables from a dotenv file. "
+ "Usage: --save=dotenv[:path] (default: .env.local)",
+ ),
+ clear_secrets: bool = typer.Option(
+ False,
+ "-x",
+ "--clear-secrets",
+ help="Also remove PORTKEY_API_KEY from the dotenv store.",
+ ),
+ quiet: bool = typer.Option(
+ False,
+ "-q",
+ "--quiet",
+ help="Suppress printing to the terminal (useful for CI).",
+ ),
+):
+ settings = get_settings()
+ with settings.edit(save=save) as edit_ctx:
+ settings.USE_PORTKEY_MODEL = None
+ settings.PORTKEY_MODEL_NAME = None
+ settings.PORTKEY_BASE_URL = None
+ settings.PORTKEY_PROVIDER_NAME = None
+ if clear_secrets:
+ settings.PORTKEY_API_KEY = None
+
+ handled, path, updates = edit_ctx.result
+
+ if _handle_save_result(
+ handled=handled,
+ path=path,
+ updates=updates,
+ save=save,
+ quiet=quiet,
+ updated_msg="Removed Portkey model environment variables from {path}.",
+ tip_msg=None,
+ ):
+ if is_openai_configured():
+ print(
+ ":raised_hands: OpenAI will still be used by default because OPENAI_API_KEY is set."
+ )
+ else:
+ print(
+ "The Portkey model configuration has been removed. No model is currently configured, but you can set one with the CLI or add credentials to .env[.local]."
+ )
diff --git a/deepeval/cli/providers/stt/__init__.py b/deepeval/cli/providers/stt/__init__.py
new file mode 100644
index 000000000..a086817cc
--- /dev/null
+++ b/deepeval/cli/providers/stt/__init__.py
@@ -0,0 +1,3 @@
+"""Speech-to-text provider commands."""
+
+from deepeval.cli.providers.stt import command # noqa: F401
diff --git a/deepeval/cli/providers/stt/command.py b/deepeval/cli/providers/stt/command.py
new file mode 100644
index 000000000..d16b972ef
--- /dev/null
+++ b/deepeval/cli/providers/stt/command.py
@@ -0,0 +1,117 @@
+"""`deepeval set-stt / unset-stt` commands."""
+
+from enum import Enum
+from typing import Optional
+
+import typer
+
+from deepeval.cli.app import app
+from deepeval.cli.utils import USE_STT_KEYS
+from deepeval.cli.providers._speech import (
+ _set_speech_provider,
+ _unset_speech_provider,
+)
+
+
+class STTProviders(str, Enum):
+ OPENAI = "openai"
+ ELEVENLABS = "elevenlabs"
+ CARTESIA = "cartesia"
+ DEEPGRAM = "deepgram"
+ ASSEMBLYAI = "assemblyai"
+
+
+@app.command(name="set-stt")
+def set_stt_env(
+ provider: STTProviders = typer.Argument(
+ ...,
+ help="Speech-to-text provider to use in voice simulations.",
+ ),
+ model: Optional[str] = typer.Option(
+ None,
+ "-m",
+ "--model",
+ help="Model identifier for this provider (e.g., `nova-3`). Defaults to the provider's own default.",
+ ),
+ prompt_api_key: bool = typer.Option(
+ False,
+ "-k",
+ "--prompt-api-key",
+ help=(
+ "Prompt for the provider's API key (input hidden). Not suitable for CI. "
+ "If --save (or DEEPEVAL_DEFAULT_SAVE) is used, the key is written to dotenv in plaintext."
+ ),
+ ),
+ save: Optional[str] = typer.Option(
+ None,
+ "-s",
+ "--save",
+ help="Persist CLI parameters as environment variables in a dotenv file. "
+ "Usage: --save=dotenv[:path] (default: .env.local)",
+ ),
+ quiet: bool = typer.Option(
+ False,
+ "-q",
+ "--quiet",
+ help="Suppress printing to the terminal (useful for CI).",
+ ),
+):
+ """
+ Configure the active speech-to-text provider.
+
+ What this does:
+ - Sets the active provider flag (e.g. `USE_DEEPGRAM_STT`), clearing the other STT flags.
+ - Persists `DEEPEVAL_STT_MODEL` in the JSON store when `--model` is given.
+ - Leaves the LLM, embedding and TTS selections alone.
+
+ Example:
+ deepeval set-stt assemblyai --model universal-3-5-pro --save dotenv:.env.local
+ """
+ _set_speech_provider(
+ family="STT",
+ provider=provider.value,
+ model=model,
+ prompt_api_key=prompt_api_key,
+ save=save,
+ quiet=quiet,
+ )
+
+
+@app.command(name="unset-stt")
+def unset_stt_env(
+ save: Optional[str] = typer.Option(
+ None,
+ "-s",
+ "--save",
+ help="Remove only the STT related environment variables from a dotenv file. "
+ "Usage: --save=dotenv[:path] (default: .env.local)",
+ ),
+ clear_secrets: bool = typer.Option(
+ False,
+ "-x",
+ "--clear-secrets",
+ help="Also remove the speech provider API keys from the dotenv store (OPENAI_API_KEY is left alone).",
+ ),
+ quiet: bool = typer.Option(
+ False,
+ "-q",
+ "--quiet",
+ help="Suppress printing to the terminal (useful for CI).",
+ ),
+):
+ """
+ Unset the active speech-to-text provider.
+
+ Voice simulations fall back to OpenAI, exactly as they do when no STT
+ provider was ever configured.
+
+ Example:
+ deepeval unset-stt --save dotenv:.env.local
+ """
+ _unset_speech_provider(
+ family="STT",
+ flags=USE_STT_KEYS,
+ clear_secrets=clear_secrets,
+ save=save,
+ quiet=quiet,
+ )
diff --git a/deepeval/cli/providers/tts/__init__.py b/deepeval/cli/providers/tts/__init__.py
new file mode 100644
index 000000000..30dc86345
--- /dev/null
+++ b/deepeval/cli/providers/tts/__init__.py
@@ -0,0 +1,3 @@
+"""Text-to-speech provider commands."""
+
+from deepeval.cli.providers.tts import command # noqa: F401
diff --git a/deepeval/cli/providers/tts/command.py b/deepeval/cli/providers/tts/command.py
new file mode 100644
index 000000000..52925c89e
--- /dev/null
+++ b/deepeval/cli/providers/tts/command.py
@@ -0,0 +1,116 @@
+"""`deepeval set-tts / unset-tts` commands."""
+
+from enum import Enum
+from typing import Optional
+
+import typer
+
+from deepeval.cli.app import app
+from deepeval.cli.utils import USE_TTS_KEYS
+from deepeval.cli.providers._speech import (
+ _set_speech_provider,
+ _unset_speech_provider,
+)
+
+
+class TTSProviders(str, Enum):
+ OPENAI = "openai"
+ ELEVENLABS = "elevenlabs"
+ CARTESIA = "cartesia"
+ DEEPGRAM = "deepgram"
+
+
+@app.command(name="set-tts")
+def set_tts_env(
+ provider: TTSProviders = typer.Argument(
+ ...,
+ help="Text-to-speech provider to use in voice simulations.",
+ ),
+ model: Optional[str] = typer.Option(
+ None,
+ "-m",
+ "--model",
+ help="Model identifier for this provider (e.g., `aura-2-thalia-en`). Defaults to the provider's own default.",
+ ),
+ prompt_api_key: bool = typer.Option(
+ False,
+ "-k",
+ "--prompt-api-key",
+ help=(
+ "Prompt for the provider's API key (input hidden). Not suitable for CI. "
+ "If --save (or DEEPEVAL_DEFAULT_SAVE) is used, the key is written to dotenv in plaintext."
+ ),
+ ),
+ save: Optional[str] = typer.Option(
+ None,
+ "-s",
+ "--save",
+ help="Persist CLI parameters as environment variables in a dotenv file. "
+ "Usage: --save=dotenv[:path] (default: .env.local)",
+ ),
+ quiet: bool = typer.Option(
+ False,
+ "-q",
+ "--quiet",
+ help="Suppress printing to the terminal (useful for CI).",
+ ),
+):
+ """
+ Configure the active text-to-speech provider.
+
+ What this does:
+ - Sets the active provider flag (e.g. `USE_DEEPGRAM_TTS`), clearing the other TTS flags.
+ - Persists `DEEPEVAL_TTS_MODEL` in the JSON store when `--model` is given.
+ - Leaves the LLM, embedding and STT selections alone.
+
+ Example:
+ deepeval set-tts deepgram --model aura-2-thalia-en --save dotenv:.env.local
+ """
+ _set_speech_provider(
+ family="TTS",
+ provider=provider.value,
+ model=model,
+ prompt_api_key=prompt_api_key,
+ save=save,
+ quiet=quiet,
+ )
+
+
+@app.command(name="unset-tts")
+def unset_tts_env(
+ save: Optional[str] = typer.Option(
+ None,
+ "-s",
+ "--save",
+ help="Remove only the TTS related environment variables from a dotenv file. "
+ "Usage: --save=dotenv[:path] (default: .env.local)",
+ ),
+ clear_secrets: bool = typer.Option(
+ False,
+ "-x",
+ "--clear-secrets",
+ help="Also remove the speech provider API keys from the dotenv store (OPENAI_API_KEY is left alone).",
+ ),
+ quiet: bool = typer.Option(
+ False,
+ "-q",
+ "--quiet",
+ help="Suppress printing to the terminal (useful for CI).",
+ ),
+):
+ """
+ Unset the active text-to-speech provider.
+
+ Voice simulations fall back to OpenAI, exactly as they do when no TTS
+ provider was ever configured.
+
+ Example:
+ deepeval unset-tts --save dotenv:.env.local
+ """
+ _unset_speech_provider(
+ family="TTS",
+ flags=USE_TTS_KEYS,
+ clear_secrets=clear_secrets,
+ save=save,
+ quiet=quiet,
+ )
diff --git a/deepeval/cli/utils.py b/deepeval/cli/utils.py
index e0513eaa7..ce94e150f 100644
--- a/deepeval/cli/utils.py
+++ b/deepeval/cli/utils.py
@@ -7,7 +7,7 @@ import typer
import webbrowser
from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit
-from pydantic import ValidationError
+from pydantic import SecretStr, ValidationError
from pydantic.fields import FieldInfo
from enum import Enum
from pathlib import Path
@@ -34,6 +34,7 @@ from deepeval.key_handler import (
KEY_FILE_HANDLER,
ModelKeyValues,
EmbeddingKeyValues,
+ SpeechKeyValues,
)
from deepeval.test_run.test_run import (
global_test_run_manager,
@@ -102,6 +103,31 @@ USE_EMBED_KEYS = [
for key in Settings.model_fields
if key.startswith("USE_") and key in EmbeddingKeyValues.__members__
]
+USE_TTS_KEYS = [
+ key
+ for key in Settings.model_fields
+ if key.startswith("USE_") and key.endswith("_TTS")
+]
+USE_STT_KEYS = [
+ key
+ for key in Settings.model_fields
+ if key.startswith("USE_") and key.endswith("_STT")
+]
+
+
+def is_openai_configured() -> bool:
+ s = get_settings()
+ v = s.OPENAI_API_KEY
+ if isinstance(v, SecretStr):
+ try:
+ if v.get_secret_value().strip():
+ return True
+ except Exception:
+ pass
+ elif v and str(v).strip():
+ return True
+ env = os.getenv("OPENAI_API_KEY")
+ return bool(env and env.strip())
def handle_save_result(
@@ -286,29 +312,37 @@ def unset_environ_in_store(
def _as_legacy_use_key(
k: str,
-) -> Union[ModelKeyValues, EmbeddingKeyValues, None]:
+) -> Union[ModelKeyValues, EmbeddingKeyValues, SpeechKeyValues, None]:
if k in ModelKeyValues.__members__:
return ModelKeyValues[k]
if k in EmbeddingKeyValues.__members__:
return EmbeddingKeyValues[k]
+ if k in SpeechKeyValues.__members__:
+ return SpeechKeyValues[k]
return None
def switch_model_provider(
- target: Union[ModelKeyValues, EmbeddingKeyValues],
+ target: Union[ModelKeyValues, EmbeddingKeyValues, SpeechKeyValues],
save: Optional[str] = None,
) -> Tuple[bool, Optional[str]]:
"""
- Ensure exactly one USE_* flag is enabled.
+ Ensure exactly one USE_* flag is enabled within the target's family.
We *unset* all other USE_* keys (instead of writing explicit "NO") to:
- keep dotenv clean
- preserve Optional[bool] semantics (unset vs explicit false)
"""
- keys_to_clear = (
- USE_LLM_KEYS if isinstance(target, ModelKeyValues) else USE_EMBED_KEYS
- )
target_key = target.name # or _to_str_key(target)
+ if target_key.endswith("_TTS"):
+ keys_to_clear = USE_TTS_KEYS
+ elif target_key.endswith("_STT"):
+ keys_to_clear = USE_STT_KEYS
+ elif isinstance(target, ModelKeyValues):
+ keys_to_clear = USE_LLM_KEYS
+ else:
+ keys_to_clear = USE_EMBED_KEYS
+
if target_key not in keys_to_clear:
raise ValueError(f"{target} is not a recognized USE_* model key")
diff --git a/deepeval/config/settings.py b/deepeval/config/settings.py
index 9a15fde2a..06e85b333 100644
--- a/deepeval/config/settings.py
+++ b/deepeval/config/settings.py
@@ -74,14 +74,31 @@ _DEPRECATED_TO_OVERRIDE = {
_LEGACY_KEYFILE_SECRET_WARNED: set[str] = set()
+def _family_of_use_flag(env_key: str) -> str:
+ """The provider family a USE_* flag belongs to.
+
+ Families are mutually exclusive within themselves but independent of each
+ other, so selecting an LLM provider leaves the embedding, TTS and STT
+ selections alone.
+ """
+ if env_key.endswith("_TTS"):
+ return "tts"
+ if env_key.endswith("_STT"):
+ return "stt"
+ if "EMBEDDING" in env_key:
+ return "embedding"
+ return "llm"
+
+
def _find_legacy_enum(env_key: str):
from deepeval.key_handler import (
ModelKeyValues,
EmbeddingKeyValues,
+ SpeechKeyValues,
KeyValues,
)
- enums = (ModelKeyValues, EmbeddingKeyValues, KeyValues)
+ enums = (ModelKeyValues, EmbeddingKeyValues, SpeechKeyValues, KeyValues)
for enum in enums:
try:
@@ -129,6 +146,7 @@ def _merge_legacy_keyfile_into_env() -> None:
KeyValues,
ModelKeyValues,
EmbeddingKeyValues,
+ SpeechKeyValues,
)
key_path = Path(HIDDEN_DIR) / KEY_FILE
@@ -149,7 +167,7 @@ def _merge_legacy_keyfile_into_env() -> None:
# Map JSON keys (enum .value) -> env keys (enum .name)
mapping: Dict[str, str] = {}
- for enum in (KeyValues, ModelKeyValues, EmbeddingKeyValues):
+ for enum in (KeyValues, ModelKeyValues, EmbeddingKeyValues, SpeechKeyValues):
for member in enum:
mapping[member.value] = member.name
@@ -711,6 +729,69 @@ class Settings(BaseSettings):
)
VLLM_MODEL_NAME: Optional[str] = Field(None, description="vLLM model name.")
+ #
+ # Speech Keys (TTS/STT)
+ #
+ # TTS and STT are two independent families: a USE_*_TTS flag and a
+ # USE_*_STT flag can both be on, and each family's model name comes from
+ # DEEPEVAL_TTS_MODEL / DEEPEVAL_STT_MODEL. With no flag set, voice mode
+ # falls back to OpenAI, the same way `initialize_model()` does for LLMs.
+
+ # TTS
+ USE_OPENAI_TTS: Optional[bool] = Field(
+ None, description="Use OpenAI for text-to-speech."
+ )
+ USE_ELEVENLABS_TTS: Optional[bool] = Field(
+ None, description="Use ElevenLabs for text-to-speech."
+ )
+ USE_CARTESIA_TTS: Optional[bool] = Field(
+ None, description="Use Cartesia for text-to-speech."
+ )
+ USE_DEEPGRAM_TTS: Optional[bool] = Field(
+ None, description="Use Deepgram for text-to-speech."
+ )
+ DEEPEVAL_TTS_MODEL: Optional[str] = Field(
+ None,
+ description="Model name for the selected TTS provider (defaults to that provider's own default).",
+ )
+
+ # STT
+ USE_OPENAI_STT: Optional[bool] = Field(
+ None, description="Use OpenAI for speech-to-text."
+ )
+ USE_ELEVENLABS_STT: Optional[bool] = Field(
+ None, description="Use ElevenLabs for speech-to-text."
+ )
+ USE_CARTESIA_STT: Optional[bool] = Field(
+ None, description="Use Cartesia for speech-to-text."
+ )
+ USE_DEEPGRAM_STT: Optional[bool] = Field(
+ None, description="Use Deepgram for speech-to-text."
+ )
+ USE_ASSEMBLYAI_STT: Optional[bool] = Field(
+ None, description="Use AssemblyAI for speech-to-text."
+ )
+ DEEPEVAL_STT_MODEL: Optional[str] = Field(
+ None,
+ description="Model name for the selected STT provider (defaults to that provider's own default).",
+ )
+
+ ASSEMBLYAI_API_KEY: Optional[SecretStr] = Field(
+ None, description="AssemblyAI API key (speech-to-text)."
+ )
+ CARTESIA_API_KEY: Optional[SecretStr] = Field(
+ None,
+ description="Cartesia API key (text-to-speech and speech-to-text).",
+ )
+ DEEPGRAM_API_KEY: Optional[SecretStr] = Field(
+ None,
+ description="Deepgram API key (text-to-speech and speech-to-text).",
+ )
+ ELEVENLABS_API_KEY: Optional[SecretStr] = Field(
+ None,
+ description="ElevenLabs API key (text-to-speech and speech-to-text).",
+ )
+
#
# Embedding Keys
#
@@ -1063,6 +1144,15 @@ class Settings(BaseSettings):
"USE_AZURE_OPENAI_EMBEDDING",
"USE_LOCAL_EMBEDDINGS",
"USE_PORTKEY_MODEL",
+ "USE_OPENAI_TTS",
+ "USE_ELEVENLABS_TTS",
+ "USE_CARTESIA_TTS",
+ "USE_DEEPGRAM_TTS",
+ "USE_OPENAI_STT",
+ "USE_ELEVENLABS_STT",
+ "USE_CARTESIA_STT",
+ "USE_DEEPGRAM_STT",
+ "USE_ASSEMBLYAI_STT",
mode="before",
)
@classmethod
@@ -1520,22 +1610,19 @@ class Settings(BaseSettings):
def switch_model_provider(self, target) -> None:
"""
- Flip USE_* settings within the target's provider family (LLM vs embeddings).
+ Flip USE_* settings within the target's provider family
+ (LLM, embeddings, TTS or STT).
"""
from deepeval.key_handler import KEY_FILE_HANDLER
target_key = getattr(target, "value", str(target))
-
- def _is_embedding_flag(k: str) -> bool:
- return "EMBEDDING" in k
-
- target_is_embedding = _is_embedding_flag(target_key)
+ target_family = _family_of_use_flag(target_key)
use_fields = [
field
for field in type(self._s).model_fields
if field.startswith("USE_")
- and _is_embedding_flag(field) == target_is_embedding
+ and _family_of_use_flag(field) == target_family
]
if target_key not in use_fields:
diff --git a/deepeval/constants.py b/deepeval/constants.py
index cb4cbff4d..53f62ec20 100644
--- a/deepeval/constants.py
+++ b/deepeval/constants.py
@@ -42,6 +42,12 @@ class ProviderSlug(str, Enum):
OLLAMA = "ollama"
OPENROUTER = "openrouter"
PORTKEY = "portkey"
+ # Speech (TTS/STT) providers. They serve no LLMs, so they appear here only
+ # to carry a retry policy, not as selectable LLM providers.
+ ASSEMBLYAI = "assemblyai"
+ CARTESIA = "cartesia"
+ DEEPGRAM = "deepgram"
+ ELEVENLABS = "elevenlabs"
def slugify(value: Union[str, ProviderSlug]) -> str:
diff --git a/deepeval/errors.py b/deepeval/errors.py
index 515a5845c..4ac24faa0 100644
--- a/deepeval/errors.py
+++ b/deepeval/errors.py
@@ -1,3 +1,6 @@
+from typing import Optional
+
+
class DeepEvalError(Exception):
"""Base class for framework-originated errors.
If raised and not handled, it will abort the current operation.
@@ -12,6 +15,34 @@ class UserAppError(Exception):
"""
+class SpeechHTTPError(DeepEvalError):
+ """A speech (TTS/STT) provider returned a non-2xx response.
+
+ Defined here rather than beside the transport that raises it so
+ `deepeval.models.retry_policy` can classify it without importing
+ `deepeval.models.speech`, whose providers import the retry policy.
+ """
+
+ def __init__(
+ self,
+ message: str,
+ *,
+ status_code: Optional[int] = None,
+ provider_label: Optional[str] = None,
+ ):
+ super().__init__(message)
+ self.status_code = status_code
+ self.provider_label = provider_label
+
+
+class SpeechAuthError(SpeechHTTPError):
+ """Speech provider credentials were rejected (401/403). Never retryable."""
+
+
+class SpeechRateLimitError(SpeechHTTPError):
+ """A speech provider is rate limiting (429). Usually worth retrying."""
+
+
class MissingTestCaseParamsError(DeepEvalError):
"""Required test case fields are missing."""
diff --git a/deepeval/key_handler.py b/deepeval/key_handler.py
index 41507f8d6..5d8b3aac5 100644
--- a/deepeval/key_handler.py
+++ b/deepeval/key_handler.py
@@ -180,6 +180,30 @@ class EmbeddingKeyValues(Enum):
LOCAL_EMBEDDING_API_KEY = ("LOCAL_EMBEDDING_API_KEY",)
+class SpeechKeyValues(Enum):
+ # TTS
+ USE_OPENAI_TTS = "USE_OPENAI_TTS"
+ USE_ELEVENLABS_TTS = "USE_ELEVENLABS_TTS"
+ USE_CARTESIA_TTS = "USE_CARTESIA_TTS"
+ USE_DEEPGRAM_TTS = "USE_DEEPGRAM_TTS"
+ DEEPEVAL_TTS_MODEL = "DEEPEVAL_TTS_MODEL"
+
+ # STT
+ USE_OPENAI_STT = "USE_OPENAI_STT"
+ USE_ELEVENLABS_STT = "USE_ELEVENLABS_STT"
+ USE_CARTESIA_STT = "USE_CARTESIA_STT"
+ USE_DEEPGRAM_STT = "USE_DEEPGRAM_STT"
+ USE_ASSEMBLYAI_STT = "USE_ASSEMBLYAI_STT"
+ DEEPEVAL_STT_MODEL = "DEEPEVAL_STT_MODEL"
+
+ # Provider API keys. `write_key` refuses secrets, so these never reach the
+ # JSON store; they are here so the CLI can name them in one place.
+ ASSEMBLYAI_API_KEY = "ASSEMBLYAI_API_KEY"
+ CARTESIA_API_KEY = "CARTESIA_API_KEY"
+ DEEPGRAM_API_KEY = "DEEPGRAM_API_KEY"
+ ELEVENLABS_API_KEY = "ELEVENLABS_API_KEY"
+
+
class KeyFileHandler:
def __init__(self):
self.data = {}
@@ -188,7 +212,7 @@ class KeyFileHandler:
os.makedirs(HIDDEN_DIR, exist_ok=True)
def write_key(
- self, key: Union[KeyValues, ModelKeyValues, EmbeddingKeyValues], value
+ self, key: Union[KeyValues, ModelKeyValues, EmbeddingKeyValues, SpeechKeyValues], value
):
"""Appends or updates data in the hidden file"""
@@ -222,7 +246,7 @@ class KeyFileHandler:
json.dump(self.data, f)
def fetch_data(
- self, key: Union[KeyValues, ModelKeyValues, EmbeddingKeyValues]
+ self, key: Union[KeyValues, ModelKeyValues, EmbeddingKeyValues, SpeechKeyValues]
):
"""Fetches the data from the hidden file.
NOTE: secrets in this file are deprecated; prefer env/.env."""
@@ -257,7 +281,7 @@ class KeyFileHandler:
return value
def remove_key(
- self, key: Union[KeyValues, ModelKeyValues, EmbeddingKeyValues]
+ self, key: Union[KeyValues, ModelKeyValues, EmbeddingKeyValues, SpeechKeyValues]
):
"""Removes the specified key from the data."""
try:
diff --git a/deepeval/metrics/utils.py b/deepeval/metrics/utils.py
index fe9d51309..49021f990 100644
--- a/deepeval/metrics/utils.py
+++ b/deepeval/metrics/utils.py
@@ -3,6 +3,7 @@ import json
import re
import sys
import warnings
+from pydantic import SecretStr
from typing import (
Any,
Callable,
@@ -621,8 +622,13 @@ def should_use_local_model():
def should_use_ollama_model():
- if SETTINGS.LOCAL_MODEL_API_KEY:
- return SETTINGS.LOCAL_MODEL_API_KEY == "ollama"
+ local_key = SETTINGS.LOCAL_MODEL_API_KEY
+ if local_key:
+ # `LOCAL_MODEL_API_KEY` is a `SecretStr`, which never compares equal to
+ # a plain string, so the sentinel is read out before comparing.
+ if isinstance(local_key, SecretStr):
+ local_key = local_key.get_secret_value()
+ return local_key == "ollama"
value = KEY_FILE_HANDLER.fetch_data(ModelKeyValues.LOCAL_MODEL_API_KEY)
return value == "ollama"
diff --git a/deepeval/models/__init__.py b/deepeval/models/__init__.py
index a5815cfbe..5c0914975 100644
--- a/deepeval/models/__init__.py
+++ b/deepeval/models/__init__.py
@@ -26,8 +26,19 @@ from deepeval.models.embedding_models import (
LocalEmbeddingModel,
OllamaEmbeddingModel,
)
-from deepeval.models.tts_models import OpenAITTSModel
-from deepeval.models.stt_models import OpenAISTTModel
+from deepeval.models.tts import (
+ CartesiaTTSModel,
+ DeepgramTTSModel,
+ ElevenLabsTTSModel,
+ OpenAITTSModel,
+)
+from deepeval.models.stt import (
+ AssemblyAISTTModel,
+ CartesiaSTTModel,
+ DeepgramSTTModel,
+ ElevenLabsSTTModel,
+ OpenAISTTModel,
+)
__all__ = [
"DeepEvalBaseModel",
@@ -54,6 +65,13 @@ __all__ = [
"OpenRouterModel",
"OpenAITTSModel",
"OpenAISTTModel",
+ "CartesiaTTSModel",
+ "DeepgramTTSModel",
+ "ElevenLabsTTSModel",
+ "AssemblyAISTTModel",
+ "CartesiaSTTModel",
+ "DeepgramSTTModel",
+ "ElevenLabsSTTModel",
]
diff --git a/deepeval/models/retry_policy.py b/deepeval/models/retry_policy.py
index 6afc2e834..8e1d858d4 100644
--- a/deepeval/models/retry_policy.py
+++ b/deepeval/models/retry_policy.py
@@ -987,6 +987,50 @@ except Exception:
OLLAMA_ERROR_POLICY = None
+####################
+# Speech providers #
+####################
+# The speech providers (ElevenLabs, Deepgram, Cartesia, AssemblyAI) are reached
+# over plain HTTP rather than through vendor SDKs, so there are no provider
+# exception classes to classify. The speech transport raises its own hierarchy
+# instead, already split along the lines this policy needs, so one policy serves
+# all four. Imported from `deepeval.errors` rather than from the transport that
+# raises them, which would import the providers, which import this module.
+
+try:
+ from deepeval.errors import (
+ SpeechAuthError,
+ SpeechHTTPError,
+ SpeechRateLimitError,
+ )
+
+ def _aiohttp_net_excs() -> tuple[type, ...]:
+ try:
+ import aiohttp
+ except Exception:
+ return ()
+ names = (
+ "ClientConnectionError",
+ "ClientPayloadError",
+ "ServerTimeoutError",
+ "ServerDisconnectedError",
+ )
+ return tuple(getattr(aiohttp, n) for n in names if hasattr(aiohttp, n))
+
+ SPEECH_ERROR_POLICY = ErrorPolicy(
+ auth_excs=(SpeechAuthError,),
+ rate_limit_excs=(SpeechRateLimitError,),
+ network_excs=_aiohttp_net_excs() + _requests_net_excs(),
+ # Checked after the two above, so the base class here only ever sees
+ # statuses that are neither auth nor rate limit.
+ http_excs=(SpeechHTTPError,),
+ non_retryable_codes=frozenset(),
+ message_markers={},
+ )
+except Exception: # pragma: no cover - aiohttp/requests are core deps
+ SPEECH_ERROR_POLICY = None
+
+
# Map provider slugs to their policy objects.
# It is OK if some are None, we'll treat that as no Error Policy / Tenacity
_POLICY_BY_SLUG: dict[str, Optional[ErrorPolicy]] = {
@@ -1003,6 +1047,10 @@ _POLICY_BY_SLUG: dict[str, Optional[ErrorPolicy]] = {
PS.OLLAMA.value: OLLAMA_ERROR_POLICY,
PS.OPENROUTER.value: OPENROUTER_ERROR_POLICY,
PS.PORTKEY.value: PORTKEY_ERROR_POLICY,
+ PS.ASSEMBLYAI.value: SPEECH_ERROR_POLICY,
+ PS.CARTESIA.value: SPEECH_ERROR_POLICY,
+ PS.DEEPGRAM.value: SPEECH_ERROR_POLICY,
+ PS.ELEVENLABS.value: SPEECH_ERROR_POLICY,
}
@@ -1026,6 +1074,10 @@ _STATIC_PRED_BY_SLUG: dict[str, Optional[Callable[[Exception], bool]]] = {
PS.OLLAMA.value: _opt_pred(OLLAMA_ERROR_POLICY),
PS.OPENROUTER.value: _opt_pred(OPENROUTER_ERROR_POLICY),
PS.PORTKEY.value: _opt_pred(PORTKEY_ERROR_POLICY),
+ PS.ASSEMBLYAI.value: _opt_pred(SPEECH_ERROR_POLICY),
+ PS.CARTESIA.value: _opt_pred(SPEECH_ERROR_POLICY),
+ PS.DEEPGRAM.value: _opt_pred(SPEECH_ERROR_POLICY),
+ PS.ELEVENLABS.value: _opt_pred(SPEECH_ERROR_POLICY),
}
@@ -1050,4 +1102,5 @@ __all__ = [
"GOOGLE_ERROR_POLICY",
"GROK_ERROR_POLICY",
"LOCAL_ERROR_POLICY",
+ "SPEECH_ERROR_POLICY",
]
diff --git a/deepeval/models/speech.py b/deepeval/models/speech.py
new file mode 100644
index 000000000..91852c6f2
--- /dev/null
+++ b/deepeval/models/speech.py
@@ -0,0 +1,423 @@
+"""Transport and schema plumbing shared by the TTS and STT providers.
+
+The models live in `deepeval.models.tts` and `deepeval.models.stt`, one
+subpackage per provider. Every provider except OpenAI is reached over plain HTTP
+rather than through a vendor SDK: each is a handful of REST calls, and `requests`
+and `aiohttp` are already core dependencies, so `pip install deepeval` is enough
+to reach any of them without a per-provider install and version constraint to
+track. Rather than have nine models each grow their own session handling and
+error translation, that lives here once.
+
+Two HTTP clients are used rather than one: `requests` for the synchronous
+`synthesize`/`transcribe` methods and `aiohttp` for their `a_` counterparts.
+Using each in its native mode avoids driving a sync client from an event loop
+(which blocks it) or bridging an async one into sync code (which needs a loop
+that may already be running).
+"""
+
+from __future__ import annotations
+
+import json as jsonlib
+from dataclasses import dataclass, field
+from typing import (
+ Any,
+ AsyncGenerator,
+ Dict,
+ List,
+ Optional,
+ Tuple,
+ Type,
+ TypeVar,
+)
+
+import aiohttp
+import requests
+from pydantic import BaseModel, ValidationError
+
+# Raised from here, defined in `deepeval.errors`, re-exported for callers.
+from deepeval.errors import (
+ SpeechAuthError,
+ SpeechHTTPError,
+ SpeechRateLimitError,
+)
+
+T = TypeVar("T", bound=BaseModel)
+
+# Long enough that a slow batch transcription of a whole conversation does not
+# trip it, short enough that a hung connection does not stall a run forever.
+DEFAULT_TIMEOUT_SECONDS = 120.0
+
+# Error bodies go into exception messages, so they are truncated: providers can
+# return a wall of JSON, and the useful part is always at the front.
+_MAX_ERROR_BODY_CHARS = 500
+
+
+def _raise_for_status(
+ status: int, body: bytes, *, provider_label: str, url: str
+) -> None:
+ if 200 <= status < 300:
+ return
+
+ detail = body.decode("utf-8", errors="replace").strip()
+ if len(detail) > _MAX_ERROR_BODY_CHARS:
+ detail = detail[:_MAX_ERROR_BODY_CHARS] + "..."
+ message = f"{provider_label} request to {url} failed with HTTP {status}"
+ if detail:
+ message = f"{message}: {detail}"
+
+ if status in (401, 403):
+ raise SpeechAuthError(
+ f"{message}. Check the API key you configured.",
+ status_code=status,
+ provider_label=provider_label,
+ )
+ if status == 429:
+ raise SpeechRateLimitError(
+ message, status_code=status, provider_label=provider_label
+ )
+ raise SpeechHTTPError(
+ message, status_code=status, provider_label=provider_label
+ )
+
+
+@dataclass
+class Multipart:
+ """A multipart/form-data body: one audio file plus scalar fields.
+
+ Every speech endpoint that takes multipart follows this exact shape, so the
+ two HTTP clients' quite different form APIs can be hidden behind it.
+ """
+
+ file_field: str
+ filename: str
+ content: bytes
+ content_type: str
+ fields: Dict[str, Any] = field(default_factory=dict)
+
+ def _scalar_fields(self) -> Dict[str, str]:
+ # Both clients want strings; booleans have to go over the wire in the
+ # JSON spelling ("true"), not Python's ("True").
+ out: Dict[str, str] = {}
+ for key, value in self.fields.items():
+ if value is None:
+ continue
+ if isinstance(value, bool):
+ out[key] = "true" if value else "false"
+ elif isinstance(value, (list, tuple, dict)):
+ out[key] = jsonlib.dumps(value)
+ else:
+ out[key] = str(value)
+ return out
+
+ def as_requests_kwargs(self) -> Dict[str, Any]:
+ return {
+ "files": {
+ self.file_field: (
+ self.filename,
+ self.content,
+ self.content_type,
+ )
+ },
+ "data": self._scalar_fields(),
+ }
+
+ def as_aiohttp_form(self) -> aiohttp.FormData:
+ form = aiohttp.FormData()
+ for key, value in self._scalar_fields().items():
+ form.add_field(key, value)
+ form.add_field(
+ self.file_field,
+ self.content,
+ filename=self.filename,
+ content_type=self.content_type,
+ )
+ return form
+
+
+class SpeechTransport:
+ """Issues HTTP requests to one provider, sync or async.
+
+ Each async request opens its own `aiohttp.ClientSession`. Caching one would
+ save a handshake per call, but an `aiohttp.ClientSession` is bound to the
+ event loop that created it and has to be closed on that same loop — and a
+ speech model has no teardown hook to close it from, since `VoiceConfig`
+ holds the model for the life of the process. A cached session would
+ therefore outlive its loop and be reported as an unclosed session at
+ interpreter shutdown. A streaming response still holds one connection for
+ its whole body, which is where reuse actually matters here.
+ """
+
+ def __init__(
+ self,
+ *,
+ provider_label: str,
+ base_url: str,
+ headers: Dict[str, str],
+ timeout: float = DEFAULT_TIMEOUT_SECONDS,
+ ):
+ self.provider_label = provider_label
+ self.base_url = base_url.rstrip("/")
+ self.headers = {k: v for k, v in headers.items() if v is not None}
+ self.timeout = timeout
+ self._sync_session: Optional[requests.Session] = None
+
+ def url(self, path: str) -> str:
+ if path.startswith(("http://", "https://")):
+ return path
+ return f"{self.base_url}/{path.lstrip('/')}"
+
+ def _merged_headers(
+ self, extra: Optional[Dict[str, str]] = None
+ ) -> Dict[str, str]:
+ headers = dict(self.headers)
+ if extra:
+ headers.update({k: v for k, v in extra.items() if v is not None})
+ return headers
+
+ #
+ # Synchronous
+ #
+
+ def _requests_session(self) -> requests.Session:
+ if self._sync_session is None:
+ self._sync_session = requests.Session()
+ return self._sync_session
+
+ def _request(
+ self,
+ method: str,
+ path: str,
+ *,
+ params: Optional[Dict[str, Any]] = None,
+ json: Optional[Dict[str, Any]] = None,
+ content: Optional[bytes] = None,
+ multipart: Optional[Multipart] = None,
+ headers: Optional[Dict[str, str]] = None,
+ ) -> Tuple[int, bytes]:
+ url = self.url(path)
+ kwargs: Dict[str, Any] = {
+ "params": _clean_params(params),
+ "headers": self._merged_headers(headers),
+ "timeout": self.timeout,
+ }
+ if json is not None:
+ kwargs["json"] = json
+ if content is not None:
+ kwargs["data"] = content
+ if multipart is not None:
+ kwargs.update(multipart.as_requests_kwargs())
+
+ response = self._requests_session().request(method, url, **kwargs)
+ body = response.content
+ _raise_for_status(
+ response.status_code,
+ body,
+ provider_label=self.provider_label,
+ url=url,
+ )
+ return response.status_code, body
+
+ def request_bytes(self, method: str, path: str, **kwargs) -> bytes:
+ _, body = self._request(method, path, **kwargs)
+ return body
+
+ def request_json(self, method: str, path: str, **kwargs) -> Any:
+ _, body = self._request(method, path, **kwargs)
+ return _decode_json(body, provider_label=self.provider_label)
+
+ #
+ # Asynchronous
+ #
+
+ def _session(self) -> aiohttp.ClientSession:
+ return aiohttp.ClientSession(
+ timeout=aiohttp.ClientTimeout(total=self.timeout)
+ )
+
+ def _a_request_kwargs(
+ self,
+ *,
+ params: Optional[Dict[str, Any]],
+ json: Optional[Dict[str, Any]],
+ content: Optional[bytes],
+ multipart: Optional[Multipart],
+ headers: Optional[Dict[str, str]],
+ ) -> Dict[str, Any]:
+ kwargs: Dict[str, Any] = {
+ "params": _clean_params(params),
+ "headers": self._merged_headers(headers),
+ }
+ if json is not None:
+ kwargs["json"] = json
+ if content is not None:
+ kwargs["data"] = content
+ if multipart is not None:
+ kwargs["data"] = multipart.as_aiohttp_form()
+ return kwargs
+
+ async def _a_request(
+ self,
+ method: str,
+ path: str,
+ *,
+ params: Optional[Dict[str, Any]] = None,
+ json: Optional[Dict[str, Any]] = None,
+ content: Optional[bytes] = None,
+ multipart: Optional[Multipart] = None,
+ headers: Optional[Dict[str, str]] = None,
+ ) -> bytes:
+ url = self.url(path)
+ kwargs = self._a_request_kwargs(
+ params=params,
+ json=json,
+ content=content,
+ multipart=multipart,
+ headers=headers,
+ )
+ async with self._session() as session:
+ async with session.request(method, url, **kwargs) as response:
+ body = await response.read()
+ _raise_for_status(
+ response.status,
+ body,
+ provider_label=self.provider_label,
+ url=url,
+ )
+ return body
+
+ async def a_request_bytes(self, method: str, path: str, **kwargs) -> bytes:
+ return await self._a_request(method, path, **kwargs)
+
+ async def a_request_json(self, method: str, path: str, **kwargs) -> Any:
+ body = await self._a_request(method, path, **kwargs)
+ return _decode_json(body, provider_label=self.provider_label)
+
+ async def a_stream_bytes(
+ self,
+ method: str,
+ path: str,
+ *,
+ params: Optional[Dict[str, Any]] = None,
+ json: Optional[Dict[str, Any]] = None,
+ content: Optional[bytes] = None,
+ multipart: Optional[Multipart] = None,
+ headers: Optional[Dict[str, str]] = None,
+ chunk_size: int = 4096,
+ ) -> AsyncGenerator[bytes, None]:
+ """Yield the response body as it arrives.
+
+ An error response is read in full before raising, since the body is the
+ only place the provider explains what went wrong.
+ """
+ url = self.url(path)
+ kwargs = self._a_request_kwargs(
+ params=params,
+ json=json,
+ content=content,
+ multipart=multipart,
+ headers=headers,
+ )
+ async with self._session() as session:
+ async with session.request(method, url, **kwargs) as response:
+ if response.status < 200 or response.status >= 300:
+ _raise_for_status(
+ response.status,
+ await response.read(),
+ provider_label=self.provider_label,
+ url=url,
+ )
+ async for data in response.content.iter_chunked(chunk_size):
+ if data:
+ yield data
+
+ def close(self) -> None:
+ if self._sync_session is not None:
+ self._sync_session.close()
+ self._sync_session = None
+
+
+def _clean_params(
+ params: Optional[Dict[str, Any]],
+) -> Optional[List[Tuple[str, str]]]:
+ """Drop unset params and spell values the way query strings expect.
+
+ Returns pairs rather than a dict for two reasons: providers spell a
+ repeated param (Deepgram's `keyterm`) as the same key several times, which a
+ dict cannot hold, and `aiohttp` rejects a dict whose values are lists while
+ accepting pairs. Booleans are normalized here too, since `?smart_format=True`
+ is not the `true` these APIs look for.
+ """
+ if not params:
+ return None
+ out: List[Tuple[str, str]] = []
+ for key, value in params.items():
+ if value is None:
+ continue
+ if isinstance(value, (list, tuple)):
+ out.extend(
+ (key, _param_value(item)) for item in value if item is not None
+ )
+ else:
+ out.append((key, _param_value(value)))
+ return out or None
+
+
+def _param_value(value: Any) -> str:
+ if isinstance(value, bool):
+ return "true" if value else "false"
+ return str(value)
+
+
+def _decode_json(body: bytes, *, provider_label: str) -> Any:
+ try:
+ return jsonlib.loads(body)
+ except ValueError as exc:
+ snippet = body.decode("utf-8", errors="replace")[:200]
+ raise SpeechHTTPError(
+ f"{provider_label} returned a response that is not valid JSON: "
+ f"{snippet}",
+ provider_label=provider_label,
+ ) from exc
+
+
+#
+# Schema
+#
+# Each provider's `schema.py` declares the request and response shapes deepeval
+# relies on. These two helpers are the only places those models cross the wire,
+# so serialization rules and validation failures are handled the same way
+# everywhere.
+#
+
+
+def dump_request(model: BaseModel) -> Dict[str, Any]:
+ """Serialize a request model for the wire.
+
+ `exclude_none` is what lets a schema declare every optional parameter a
+ provider accepts without sending the ones nobody set — an explicit `null`
+ is not the same as an absent field to these APIs, and several read it as an
+ error. Extras survive, which is how `generation_kwargs` reaches a provider
+ parameter that has no field of its own yet.
+ """
+ return model.model_dump(exclude_none=True, by_alias=True)
+
+
+def parse_response(schema: Type[T], payload: Any, *, provider_label: str) -> T:
+ """Validate a provider response, or fail with something readable.
+
+ Response models allow unknown fields, so this only rejects a payload whose
+ declared fields are the wrong shape entirely. When that happens the useful
+ thing to report is that the provider's response no longer matches what
+ deepeval expects — a raw `ValidationError` names pydantic models the reader
+ has never heard of and says nothing about which provider broke.
+ """
+ try:
+ return schema.model_validate(payload)
+ except ValidationError as exc:
+ raise SpeechHTTPError(
+ f"{provider_label} returned a response deepeval could not read. "
+ f"This usually means the provider changed its response format: "
+ f"{exc.error_count()} field(s) did not match "
+ f"{schema.__name__} ({exc.errors(include_url=False)}).",
+ provider_label=provider_label,
+ ) from exc
diff --git a/deepeval/models/speech_selection.py b/deepeval/models/speech_selection.py
new file mode 100644
index 000000000..731620c73
--- /dev/null
+++ b/deepeval/models/speech_selection.py
@@ -0,0 +1,111 @@
+"""Provider selection for speech models.
+
+TTS and STT mirror the LLM mechanism in `deepeval.metrics.utils`: a `USE_*`
+flag picks the provider, and the model name comes from the family-level
+`DEEPEVAL_TTS_MODEL` / `DEEPEVAL_STT_MODEL` unless one is passed in code. With
+no flag set, both families fall back to OpenAI.
+"""
+
+from __future__ import annotations
+
+from typing import Optional, Tuple, Union
+
+from deepeval.config.settings import get_settings
+from deepeval.key_handler import KEY_FILE_HANDLER, SpeechKeyValues
+from deepeval.models.base_model import DeepEvalBaseSTT, DeepEvalBaseTTS
+
+# Flag -> class name, in the order they are checked. Named rather than
+# imported because `deepeval.models.tts` reaches back into `deepeval.voice`.
+_TTS_PROVIDERS: Tuple[Tuple[str, str], ...] = (
+ ("USE_OPENAI_TTS", "OpenAITTSModel"),
+ ("USE_ELEVENLABS_TTS", "ElevenLabsTTSModel"),
+ ("USE_CARTESIA_TTS", "CartesiaTTSModel"),
+ ("USE_DEEPGRAM_TTS", "DeepgramTTSModel"),
+)
+_STT_PROVIDERS: Tuple[Tuple[str, str], ...] = (
+ ("USE_OPENAI_STT", "OpenAISTTModel"),
+ ("USE_ELEVENLABS_STT", "ElevenLabsSTTModel"),
+ ("USE_CARTESIA_STT", "CartesiaSTTModel"),
+ ("USE_DEEPGRAM_STT", "DeepgramSTTModel"),
+ ("USE_ASSEMBLYAI_STT", "AssemblyAISTTModel"),
+)
+
+
+def _from_keystore(field: str) -> Optional[str]:
+ member = SpeechKeyValues.__members__.get(field)
+ if member is None:
+ return None
+ return KEY_FILE_HANDLER.fetch_data(member)
+
+
+def _flag_enabled(field: str) -> bool:
+ if getattr(get_settings(), field, None):
+ return True
+ value = _from_keystore(field)
+ return value.lower() == "yes" if value is not None else False
+
+
+def _model_name(field: str, override: Optional[str]) -> Optional[str]:
+ if override is not None:
+ return override
+ return getattr(get_settings(), field, None) or _from_keystore(field)
+
+
+def _selected(providers: Tuple[Tuple[str, str], ...]) -> str:
+ for flag, class_name in providers:
+ if _flag_enabled(flag):
+ return class_name
+ # No flag set: OpenAI, matching `initialize_model()`'s LLM fallback.
+ return providers[0][1]
+
+
+def describe_tts_selection() -> Tuple[str, Optional[str]]:
+ """The TTS class name and model name selected by the environment.
+
+ Reported without constructing anything, so it stays answerable on a
+ machine with no speech API key set.
+ """
+ return _selected(_TTS_PROVIDERS), _model_name("DEEPEVAL_TTS_MODEL", None)
+
+
+def describe_stt_selection() -> Tuple[str, Optional[str]]:
+ """The STT class name and model name selected by the environment."""
+ return _selected(_STT_PROVIDERS), _model_name("DEEPEVAL_STT_MODEL", None)
+
+
+def initialize_tts_model(
+ model: Optional[Union[str, DeepEvalBaseTTS]] = None,
+) -> DeepEvalBaseTTS:
+ """The TTS model for this run.
+
+ A model object is used as-is, a string is the model name for the
+ flag-selected provider, and `None` falls back to `DEEPEVAL_TTS_MODEL`.
+ """
+ if model is not None and not isinstance(model, str):
+ return model
+
+ from deepeval.models import tts
+
+ class_name = _selected(_TTS_PROVIDERS)
+ return getattr(tts, class_name)(
+ model=_model_name("DEEPEVAL_TTS_MODEL", model)
+ )
+
+
+def initialize_stt_model(
+ model: Optional[Union[str, DeepEvalBaseSTT]] = None,
+) -> DeepEvalBaseSTT:
+ """The STT model for this run.
+
+ A model object is used as-is, a string is the model name for the
+ flag-selected provider, and `None` falls back to `DEEPEVAL_STT_MODEL`.
+ """
+ if model is not None and not isinstance(model, str):
+ return model
+
+ from deepeval.models import stt
+
+ class_name = _selected(_STT_PROVIDERS)
+ return getattr(stt, class_name)(
+ model=_model_name("DEEPEVAL_STT_MODEL", model)
+ )
diff --git a/deepeval/models/stt/__init__.py b/deepeval/models/stt/__init__.py
new file mode 100644
index 000000000..f0d5095b9
--- /dev/null
+++ b/deepeval/models/stt/__init__.py
@@ -0,0 +1,13 @@
+from deepeval.models.stt.assemblyai import AssemblyAISTTModel
+from deepeval.models.stt.cartesia import CartesiaSTTModel
+from deepeval.models.stt.deepgram import DeepgramSTTModel
+from deepeval.models.stt.elevenlabs import ElevenLabsSTTModel
+from deepeval.models.stt.openai import OpenAISTTModel
+
+__all__ = [
+ "AssemblyAISTTModel",
+ "CartesiaSTTModel",
+ "DeepgramSTTModel",
+ "ElevenLabsSTTModel",
+ "OpenAISTTModel",
+]
diff --git a/deepeval/models/stt/_audio.py b/deepeval/models/stt/_audio.py
new file mode 100644
index 000000000..f8ceb5d91
--- /dev/null
+++ b/deepeval/models/stt/_audio.py
@@ -0,0 +1,31 @@
+"""Audio inspection shared by the STT models."""
+
+from __future__ import annotations
+
+import io
+import wave
+from typing import Optional, TYPE_CHECKING
+
+if TYPE_CHECKING:
+ from deepeval.test_case import Audio
+
+
+def audio_duration_seconds(audio: "Audio") -> Optional[float]:
+ """How long `audio` runs, or `None` when it cannot be determined.
+
+ Needed because several providers price transcription by the minute but
+ report usage only sometimes, leaving the duration to be measured locally.
+ Anything that is not a readable WAV yields `None` rather than a guess, so a
+ cost is never invented from a number we do not have.
+ """
+ if audio.duration is not None:
+ return audio.duration
+ try:
+ with wave.open(io.BytesIO(audio.get_bytes()), "rb") as wf:
+ frames = wf.getnframes()
+ rate = wf.getframerate()
+ if rate:
+ return frames / float(rate)
+ except (wave.Error, EOFError, ValueError):
+ pass
+ return None
diff --git a/deepeval/models/stt/_stream.py b/deepeval/models/stt/_stream.py
new file mode 100644
index 000000000..3264d723c
--- /dev/null
+++ b/deepeval/models/stt/_stream.py
@@ -0,0 +1,75 @@
+"""Partial transcripts for providers whose batch API is all we use."""
+
+from __future__ import annotations
+
+from typing import AsyncGenerator, AsyncIterable, Optional
+
+from deepeval.test_case import Audio, AudioChunk
+from deepeval.voice.connectors import audio_utils
+
+DEFAULT_STREAM_SAMPLE_RATE = 24000
+
+
+class BufferedTranscribeMixin:
+ """Approximates streaming transcription on top of a batch endpoint.
+
+ `DeepEvalBaseSTT.a_transcribe_stream` raises by default, so a model without
+ it cannot follow an agent as it speaks — a caller wanting a transcript
+ before the utterance ends has nothing to consume. Every provider here has a
+ realtime WebSocket API, but the models talk to the batch endpoint, so this
+ buffers the incoming PCM and re-transcribes the accumulated audio whenever
+ roughly `partial_every_seconds` of new audio has arrived, yielding the
+ transcript each time it changes.
+
+ Re-transcribing from the top is wasteful, and deliberately so: it keeps a
+ provider's streaming protocol out of the models, and the audio involved is a
+ single conversational turn. Mixing this in gives a model working partials
+ from `a_transcribe` alone; a provider whose realtime API is worth the
+ complexity can override `a_transcribe_stream` instead.
+
+ Must be mixed in ahead of `DeepEvalBaseSTT` so it takes precedence over the
+ base's `NotImplementedError`.
+ """
+
+ async def a_transcribe_stream(
+ self,
+ audio_stream: AsyncIterable[AudioChunk],
+ *args,
+ language: Optional[str] = None,
+ partial_every_seconds: float = 1.0,
+ **kwargs,
+ ) -> AsyncGenerator[str, None]:
+ pcm = bytearray()
+ sample_rate = DEFAULT_STREAM_SAMPLE_RATE
+ bytes_since_partial = 0
+ last_text = ""
+
+ async for chunk in audio_stream:
+ if chunk.sampleRate:
+ sample_rate = chunk.sampleRate
+ frame = chunk.get_bytes()
+ pcm.extend(frame)
+ bytes_since_partial += len(frame)
+
+ threshold = int(sample_rate * 2 * partial_every_seconds)
+ if bytes_since_partial < threshold and not chunk.final:
+ continue
+
+ audio = Audio.from_bytes(
+ audio_utils.pcm16_to_wav_bytes(bytes(pcm), sample_rate, 1),
+ "audio/wav",
+ sampleRate=sample_rate,
+ encoding="wav",
+ )
+ text, _ = await self.a_transcribe(
+ audio, language=language, **kwargs
+ )
+ if text and text != last_text:
+ last_text = text
+ yield text
+ bytes_since_partial = 0
+ if chunk.final:
+ break
+
+ def supports_streaming(self) -> bool:
+ return True
diff --git a/deepeval/models/stt/assemblyai/__init__.py b/deepeval/models/stt/assemblyai/__init__.py
new file mode 100644
index 000000000..ed393af07
--- /dev/null
+++ b/deepeval/models/stt/assemblyai/__init__.py
@@ -0,0 +1,3 @@
+from deepeval.models.stt.assemblyai.assemblyai import AssemblyAISTTModel
+
+__all__ = ["AssemblyAISTTModel"]
diff --git a/deepeval/models/stt/assemblyai/assemblyai.py b/deepeval/models/stt/assemblyai/assemblyai.py
new file mode 100644
index 000000000..ccd2e8c71
--- /dev/null
+++ b/deepeval/models/stt/assemblyai/assemblyai.py
@@ -0,0 +1,330 @@
+import asyncio
+import json as jsonlib
+import time
+from typing import Any, Dict, Optional, Tuple
+
+from pydantic import SecretStr
+
+from deepeval.config.settings import get_settings
+from deepeval.constants import ProviderSlug as PS
+from deepeval.models.base_model import DeepEvalBaseSTT
+from deepeval.models.retry_policy import create_retry_decorator
+from deepeval.models.speech import (
+ DEFAULT_TIMEOUT_SECONDS,
+ Multipart,
+ SpeechHTTPError,
+ SpeechTransport,
+ dump_request,
+ parse_response,
+)
+from deepeval.models.stt._audio import audio_duration_seconds
+from deepeval.models.stt._stream import BufferedTranscribeMixin
+from deepeval.models.stt.assemblyai.schema import (
+ AssemblyAISyncConfig,
+ AssemblyAITranscriptRequest,
+ AssemblyAITranscriptResponse,
+ AssemblyAIUploadResponse,
+)
+from deepeval.models.utils import parse_model_name, require_secret_api_key
+from deepeval.test_case import Audio
+
+retry_assemblyai = create_retry_decorator(PS.ASSEMBLYAI)
+
+DEFAULT_BASE_URL = "https://api.assemblyai.com"
+
+# The synchronous endpoint lives on its own host and answers in one round trip.
+SYNC_URL = "https://sync.assemblyai.com/transcribe"
+
+# `slam-1` is deprecated in favour of this.
+DEFAULT_STT_MODEL = "universal-3-5-pro"
+
+# The synchronous endpoint's ceiling. Conversation turns sit far below it, so it
+# is the normal path; anything longer falls back to upload-and-poll.
+SYNC_MAX_SECONDS = 120.0
+
+_SYNC_PRICE_PER_HOUR = 0.45
+_ASYNC_PRICE_PER_HOUR = {
+ "universal-3-5-pro": 0.21,
+ "universal-2": 0.15,
+}
+
+DEFAULT_POLL_INTERVAL_SECONDS = 1.0
+
+
+class AssemblyAISTTModel(BufferedTranscribeMixin, DeepEvalBaseSTT):
+ """AssemblyAI speech-to-text.
+
+ Prefers the synchronous endpoint, which returns a transcript in a single
+ request (~134 ms p50) instead of the three-step upload, create and poll
+ flow. That endpoint caps clips at 120 seconds, which no simulated
+ conversation turn approaches, so the slower flow is used only when the audio
+ really is longer.
+ """
+
+ # Universal does not complete a word clipped by a barge-in, so the audio is
+ # transcribed exactly as it was heard.
+ truncated_audio_pad_seconds: float = 0.0
+
+ def __init__(
+ self,
+ model: Optional[str] = None,
+ api_key: Optional[str] = None,
+ base_url: Optional[str] = None,
+ language: Optional[str] = None,
+ cost_per_hour: Optional[float] = None,
+ transcription_kwargs: Optional[Dict] = None,
+ timeout: float = DEFAULT_TIMEOUT_SECONDS,
+ poll_interval_seconds: float = DEFAULT_POLL_INTERVAL_SECONDS,
+ ):
+ settings = get_settings()
+ self.name = parse_model_name(model or DEFAULT_STT_MODEL)
+ self.language = language
+ self.transcription_kwargs = transcription_kwargs or {}
+ self.cost_per_hour = cost_per_hour
+ # Only consulted on the upload-and-poll path, which a conversation turn
+ # never takes.
+ self.poll_interval_seconds = poll_interval_seconds
+ self.api_key = (
+ SecretStr(api_key)
+ if api_key is not None
+ else settings.ASSEMBLYAI_API_KEY
+ )
+ self.base_url = (
+ str(base_url).rstrip("/")
+ if base_url is not None
+ else DEFAULT_BASE_URL
+ )
+ self.timeout = timeout
+ self.model = self.load_model()
+
+ def load_model(self) -> SpeechTransport:
+ api_key = require_secret_api_key(
+ self.api_key,
+ provider_label="AssemblyAI",
+ env_var_name="ASSEMBLYAI_API_KEY",
+ param_hint="`api_key` to AssemblyAISTTModel(...)",
+ )
+ return SpeechTransport(
+ provider_label="AssemblyAI",
+ base_url=self.base_url,
+ # Note the bare key: AssemblyAI does not use a `Bearer` prefix.
+ headers={"Authorization": api_key},
+ timeout=self.timeout,
+ )
+
+ #
+ # Routing
+ #
+
+ def _use_sync_endpoint(self, audio: Audio) -> bool:
+ seconds = audio_duration_seconds(audio)
+ # An unmeasurable duration takes the slower path: it always works,
+ # where the fast one would reject audio that turns out to be too long.
+ if seconds is None:
+ return False
+ return seconds <= SYNC_MAX_SECONDS
+
+ def _cost(self, audio: Audio, *, used_sync: bool) -> Optional[float]:
+ rate = self.cost_per_hour
+ if rate is None:
+ rate = (
+ _SYNC_PRICE_PER_HOUR
+ if used_sync
+ else _ASYNC_PRICE_PER_HOUR.get(self.name)
+ )
+ if rate is None:
+ return None
+ seconds = audio_duration_seconds(audio)
+ if seconds is None:
+ return None
+ return seconds / 3600.0 * rate
+
+ #
+ # Request building
+ #
+
+ def _sync_multipart(
+ self, audio: Audio, language: Optional[str], kwargs: Dict[str, Any]
+ ) -> Multipart:
+ lang = language or self.language
+ config = AssemblyAISyncConfig(
+ sample_rate=audio.sampleRate or None,
+ language_code=None if lang == "auto" else lang,
+ **{**self.transcription_kwargs, **kwargs},
+ )
+ dumped = dump_request(config)
+
+ fields: Dict[str, Any] = {}
+ if dumped:
+ # The whole config rides as one JSON-encoded form field.
+ fields["config"] = jsonlib.dumps(dumped)
+
+ return Multipart(
+ file_field="audio",
+ filename=audio.filename or "audio.wav",
+ content=audio.get_bytes(),
+ content_type=audio.mimeType or "audio/wav",
+ fields=fields,
+ )
+
+ def _create_body(
+ self,
+ upload_url: str,
+ language: Optional[str],
+ kwargs: Dict[str, Any],
+ ) -> Dict[str, Any]:
+ lang = language or self.language
+ auto = lang == "auto"
+ return dump_request(
+ AssemblyAITranscriptRequest(
+ audio_url=upload_url,
+ speech_models=[self.name],
+ language_code=None if auto else lang,
+ language_detection=True if auto else None,
+ **{**self.transcription_kwargs, **kwargs},
+ )
+ )
+
+ #
+ # Response handling
+ #
+
+ def _parse(self, payload: Any) -> AssemblyAITranscriptResponse:
+ return parse_response(
+ AssemblyAITranscriptResponse,
+ payload,
+ provider_label="AssemblyAI",
+ )
+
+ def _raise_if_failed(self, payload: AssemblyAITranscriptResponse) -> None:
+ if payload.failed:
+ raise SpeechHTTPError(
+ "AssemblyAI failed to transcribe the audio: "
+ f"{payload.error or 'no reason given'}",
+ provider_label="AssemblyAI",
+ )
+
+ def _raise_if_timed_out(self, deadline: float, transcript_id: str) -> None:
+ if time.monotonic() >= deadline:
+ raise SpeechHTTPError(
+ "AssemblyAI did not finish transcribing within "
+ f"{self.timeout:g}s (transcript {transcript_id}).",
+ provider_label="AssemblyAI",
+ )
+
+ #
+ # DeepEvalBaseSTT
+ #
+
+ @retry_assemblyai
+ def transcribe(
+ self, audio: Audio, *, language: Optional[str] = None, **kwargs
+ ) -> Tuple[str, Optional[float]]:
+ used_sync = self._use_sync_endpoint(audio)
+ if used_sync:
+ payload = self._parse(
+ self.model.request_json(
+ "POST",
+ SYNC_URL,
+ multipart=self._sync_multipart(audio, language, kwargs),
+ headers={"X-AAI-Model": self.name},
+ )
+ )
+ else:
+ payload = self._transcribe_by_upload(audio, language, kwargs)
+ return payload.transcript() or "", self._cost(
+ audio, used_sync=used_sync
+ )
+
+ def _transcribe_by_upload(
+ self, audio: Audio, language: Optional[str], kwargs: Dict[str, Any]
+ ) -> AssemblyAITranscriptResponse:
+ uploaded = parse_response(
+ AssemblyAIUploadResponse,
+ self.model.request_json(
+ "POST",
+ "/v2/upload",
+ content=audio.get_bytes(),
+ headers={"Content-Type": "application/octet-stream"},
+ ),
+ provider_label="AssemblyAI",
+ )
+ payload = self._parse(
+ self.model.request_json(
+ "POST",
+ "/v2/transcript",
+ json=self._create_body(uploaded.upload_url, language, kwargs),
+ )
+ )
+ transcript_id = payload.id or ""
+
+ deadline = time.monotonic() + self.timeout
+ while not payload.finished:
+ self._raise_if_timed_out(deadline, transcript_id)
+ time.sleep(self.poll_interval_seconds)
+ payload = self._parse(
+ self.model.request_json(
+ "GET", f"/v2/transcript/{transcript_id}"
+ )
+ )
+ self._raise_if_failed(payload)
+ return payload
+
+ @retry_assemblyai
+ async def a_transcribe(
+ self, audio: Audio, *, language: Optional[str] = None, **kwargs
+ ) -> Tuple[str, Optional[float]]:
+ used_sync = self._use_sync_endpoint(audio)
+ if used_sync:
+ payload = self._parse(
+ await self.model.a_request_json(
+ "POST",
+ SYNC_URL,
+ multipart=self._sync_multipart(audio, language, kwargs),
+ headers={"X-AAI-Model": self.name},
+ )
+ )
+ else:
+ payload = await self._a_transcribe_by_upload(
+ audio, language, kwargs
+ )
+ return payload.transcript() or "", self._cost(
+ audio, used_sync=used_sync
+ )
+
+ async def _a_transcribe_by_upload(
+ self, audio: Audio, language: Optional[str], kwargs: Dict[str, Any]
+ ) -> AssemblyAITranscriptResponse:
+ uploaded = parse_response(
+ AssemblyAIUploadResponse,
+ await self.model.a_request_json(
+ "POST",
+ "/v2/upload",
+ content=audio.get_bytes(),
+ headers={"Content-Type": "application/octet-stream"},
+ ),
+ provider_label="AssemblyAI",
+ )
+ payload = self._parse(
+ await self.model.a_request_json(
+ "POST",
+ "/v2/transcript",
+ json=self._create_body(uploaded.upload_url, language, kwargs),
+ )
+ )
+ transcript_id = payload.id or ""
+
+ deadline = time.monotonic() + self.timeout
+ while not payload.finished:
+ self._raise_if_timed_out(deadline, transcript_id)
+ await asyncio.sleep(self.poll_interval_seconds)
+ payload = self._parse(
+ await self.model.a_request_json(
+ "GET", f"/v2/transcript/{transcript_id}"
+ )
+ )
+ self._raise_if_failed(payload)
+ return payload
+
+ def get_model_name(self) -> str:
+ return self.name
diff --git a/deepeval/models/stt/assemblyai/schema.py b/deepeval/models/stt/assemblyai/schema.py
new file mode 100644
index 000000000..a7878ae00
--- /dev/null
+++ b/deepeval/models/stt/assemblyai/schema.py
@@ -0,0 +1,87 @@
+"""AssemblyAI request and response shapes.
+
+AssemblyAI has two transcription paths with different request shapes, so both
+are modelled: a synchronous endpoint that takes its settings as one
+JSON-encoded multipart field, and the upload-create-poll flow that takes a JSON
+body.
+"""
+
+from typing import List, Optional
+
+from pydantic import BaseModel, ConfigDict
+
+# Requests allow extras so `transcription_kwargs` can set a provider parameter
+# deepeval has no field for. Responses allow them so a provider adding a field
+# never breaks a run.
+_REQUEST = ConfigDict(extra="allow", protected_namespaces=())
+_RESPONSE = ConfigDict(extra="allow", protected_namespaces=())
+
+# Statuses that end the poll loop.
+TERMINAL_STATUSES = ("completed", "error")
+
+
+class AssemblyAISyncConfig(BaseModel):
+ """The `config` multipart field on `POST sync.assemblyai.com/transcribe`.
+
+ Unlike the async flow, this endpoint takes the model in an `X-AAI-Model`
+ header rather than in the config, so there is no model field here.
+ """
+
+ model_config = _REQUEST
+
+ sample_rate: Optional[int] = None
+ language_code: Optional[str] = None
+
+
+class AssemblyAITranscriptRequest(BaseModel):
+ """Body for `POST /v2/transcript`.
+
+ `speech_models` is an ordered fallback list; the older singular
+ `speech_model` field is deprecated.
+ """
+
+ model_config = _REQUEST
+
+ audio_url: str
+ speech_models: List[str]
+ language_code: Optional[str] = None
+ language_detection: Optional[bool] = None
+
+
+class AssemblyAIUploadResponse(BaseModel):
+ """Response from `POST /v2/upload`."""
+
+ model_config = _RESPONSE
+
+ upload_url: str
+
+
+class AssemblyAITranscriptResponse(BaseModel):
+ """Response from the sync endpoint and from `/v2/transcript` (create + poll).
+
+ `status` is absent on the synchronous endpoint's response, which returns a
+ finished transcript directly and has no queue to report on.
+ """
+
+ model_config = _RESPONSE
+
+ id: Optional[str] = None
+ status: Optional[str] = None
+ text: Optional[str] = None
+ error: Optional[str] = None
+
+ @property
+ def finished(self) -> bool:
+ return self.status is None or self.status in TERMINAL_STATUSES
+
+ @property
+ def failed(self) -> bool:
+ return self.status == "error"
+
+ def transcript(self) -> Optional[str]:
+ """The spoken text, or `None` when the response carries no `text`.
+
+ `None` and `""` mean different things: `""` is silence, which is a
+ perfectly good answer, while `None` means the field was absent.
+ """
+ return self.text
diff --git a/deepeval/models/stt/cartesia/__init__.py b/deepeval/models/stt/cartesia/__init__.py
new file mode 100644
index 000000000..771acd4a7
--- /dev/null
+++ b/deepeval/models/stt/cartesia/__init__.py
@@ -0,0 +1,3 @@
+from deepeval.models.stt.cartesia.cartesia import CartesiaSTTModel
+
+__all__ = ["CartesiaSTTModel"]
diff --git a/deepeval/models/stt/cartesia/cartesia.py b/deepeval/models/stt/cartesia/cartesia.py
new file mode 100644
index 000000000..50490a6db
--- /dev/null
+++ b/deepeval/models/stt/cartesia/cartesia.py
@@ -0,0 +1,154 @@
+from typing import Any, Dict, Optional, Tuple
+
+from pydantic import SecretStr
+
+from deepeval.config.settings import get_settings
+from deepeval.constants import ProviderSlug as PS
+from deepeval.models.base_model import DeepEvalBaseSTT
+from deepeval.models.retry_policy import create_retry_decorator
+from deepeval.models.speech import (
+ DEFAULT_TIMEOUT_SECONDS,
+ Multipart,
+ SpeechTransport,
+ dump_request,
+ parse_response,
+)
+from deepeval.models.stt._audio import audio_duration_seconds
+from deepeval.models.stt._stream import BufferedTranscribeMixin
+from deepeval.models.stt.cartesia.schema import (
+ CartesiaSTTRequest,
+ CartesiaSTTResponse,
+)
+from deepeval.models.utils import parse_model_name, require_secret_api_key
+from deepeval.test_case import Audio
+
+retry_cartesia = create_retry_decorator(PS.CARTESIA)
+
+DEFAULT_BASE_URL = "https://api.cartesia.ai"
+DEFAULT_API_VERSION = "2026-08-14"
+
+# The batch endpoint only serves the ink-whisper family; `ink-2` is realtime
+# only, so it is not a valid default here.
+DEFAULT_STT_MODEL = "ink-whisper"
+
+
+class CartesiaSTTModel(BufferedTranscribeMixin, DeepEvalBaseSTT):
+ """Cartesia Ink speech-to-text."""
+
+ # Whisper-derived, so it finishes a word the barge-in cut off unless the
+ # clip is presented as a complete utterance.
+ truncated_audio_pad_seconds: float = 0.3
+
+ def __init__(
+ self,
+ model: Optional[str] = None,
+ api_key: Optional[str] = None,
+ base_url: Optional[str] = None,
+ language: Optional[str] = None,
+ api_version: str = DEFAULT_API_VERSION,
+ cost_per_minute: Optional[float] = None,
+ transcription_kwargs: Optional[Dict] = None,
+ timeout: float = DEFAULT_TIMEOUT_SECONDS,
+ ):
+ settings = get_settings()
+ self.name = parse_model_name(model or DEFAULT_STT_MODEL)
+ self.language = language
+ self.api_version = api_version
+ self.transcription_kwargs = transcription_kwargs or {}
+ # Cartesia meters in credits with no published dollar rate, so there is
+ # nothing to default to. Pass `cost_per_minute` to account for spend.
+ self.cost_per_minute = cost_per_minute
+ self.api_key = (
+ SecretStr(api_key)
+ if api_key is not None
+ else settings.CARTESIA_API_KEY
+ )
+ self.base_url = (
+ str(base_url).rstrip("/")
+ if base_url is not None
+ else DEFAULT_BASE_URL
+ )
+ self.timeout = timeout
+ self.model = self.load_model()
+
+ def load_model(self) -> SpeechTransport:
+ api_key = require_secret_api_key(
+ self.api_key,
+ provider_label="Cartesia",
+ env_var_name="CARTESIA_API_KEY",
+ param_hint="`api_key` to CartesiaSTTModel(...)",
+ )
+ return SpeechTransport(
+ provider_label="Cartesia",
+ base_url=self.base_url,
+ headers={
+ "X-Api-Key": api_key,
+ "Cartesia-Version": self.api_version,
+ },
+ timeout=self.timeout,
+ )
+
+ def _multipart(
+ self, audio: Audio, language: Optional[str], kwargs: Dict[str, Any]
+ ) -> Multipart:
+ lang = language or self.language
+ request = CartesiaSTTRequest(
+ model=self.name,
+ # Cartesia defaults to English when `language` is absent rather
+ # than detecting, so "auto" has to be dropped instead of forwarded.
+ language=None if lang == "auto" else lang,
+ **{**self.transcription_kwargs, **kwargs},
+ )
+ return Multipart(
+ file_field="file",
+ filename=audio.filename or "audio.wav",
+ content=audio.get_bytes(),
+ content_type=audio.mimeType or "audio/wav",
+ fields=dump_request(request),
+ )
+
+ def _cost(
+ self, audio: Audio, payload: CartesiaSTTResponse
+ ) -> Optional[float]:
+ if self.cost_per_minute is None:
+ return None
+ seconds = payload.duration
+ if seconds is None:
+ seconds = audio_duration_seconds(audio)
+ if seconds is None:
+ return None
+ return seconds / 60.0 * self.cost_per_minute
+
+ def _parse(self, payload: Any) -> CartesiaSTTResponse:
+ return parse_response(
+ CartesiaSTTResponse, payload, provider_label="Cartesia"
+ )
+
+ @retry_cartesia
+ def transcribe(
+ self, audio: Audio, *, language: Optional[str] = None, **kwargs
+ ) -> Tuple[str, Optional[float]]:
+ payload = self._parse(
+ self.model.request_json(
+ "POST",
+ "/stt",
+ multipart=self._multipart(audio, language, kwargs),
+ )
+ )
+ return payload.transcript() or "", self._cost(audio, payload)
+
+ @retry_cartesia
+ async def a_transcribe(
+ self, audio: Audio, *, language: Optional[str] = None, **kwargs
+ ) -> Tuple[str, Optional[float]]:
+ payload = self._parse(
+ await self.model.a_request_json(
+ "POST",
+ "/stt",
+ multipart=self._multipart(audio, language, kwargs),
+ )
+ )
+ return payload.transcript() or "", self._cost(audio, payload)
+
+ def get_model_name(self) -> str:
+ return self.name
diff --git a/deepeval/models/stt/cartesia/schema.py b/deepeval/models/stt/cartesia/schema.py
new file mode 100644
index 000000000..d35e0f2d3
--- /dev/null
+++ b/deepeval/models/stt/cartesia/schema.py
@@ -0,0 +1,43 @@
+"""Cartesia speech-to-text request and response shapes."""
+
+from typing import Optional
+
+from pydantic import BaseModel, ConfigDict
+
+# Requests allow extras so `transcription_kwargs` can set a provider parameter
+# deepeval has no field for. Responses allow them so a provider adding a field
+# never breaks a run. `protected_namespaces` is cleared because pydantic
+# reserves the `model_` prefix and Cartesia's field is called `model`.
+_REQUEST = ConfigDict(extra="allow", protected_namespaces=())
+_RESPONSE = ConfigDict(extra="allow", protected_namespaces=())
+
+
+class CartesiaSTTRequest(BaseModel):
+ """Scalar multipart fields for `POST /stt`.
+
+ Cartesia defaults to English when `language` is absent rather than
+ detecting, so there is no way to ask for detection here.
+ """
+
+ model_config = _REQUEST
+
+ model: str
+ language: Optional[str] = None
+
+
+class CartesiaSTTResponse(BaseModel):
+ """Response from `POST /stt`."""
+
+ model_config = _RESPONSE
+
+ text: Optional[str] = None
+ duration: Optional[float] = None
+ language: Optional[str] = None
+
+ def transcript(self) -> Optional[str]:
+ """The spoken text, or `None` when the response carries no `text`.
+
+ `None` and `""` mean different things: `""` is silence, which is a
+ perfectly good answer, while `None` means the field was absent.
+ """
+ return self.text
diff --git a/deepeval/models/stt/deepgram/__init__.py b/deepeval/models/stt/deepgram/__init__.py
new file mode 100644
index 000000000..73508b8f9
--- /dev/null
+++ b/deepeval/models/stt/deepgram/__init__.py
@@ -0,0 +1,3 @@
+from deepeval.models.stt.deepgram.deepgram import DeepgramSTTModel
+
+__all__ = ["DeepgramSTTModel"]
diff --git a/deepeval/models/stt/deepgram/deepgram.py b/deepeval/models/stt/deepgram/deepgram.py
new file mode 100644
index 000000000..2db7bdc7f
--- /dev/null
+++ b/deepeval/models/stt/deepgram/deepgram.py
@@ -0,0 +1,151 @@
+from typing import Any, Dict, Optional, Tuple
+
+from pydantic import SecretStr
+
+from deepeval.config.settings import get_settings
+from deepeval.constants import ProviderSlug as PS
+from deepeval.models.base_model import DeepEvalBaseSTT
+from deepeval.models.retry_policy import create_retry_decorator
+from deepeval.models.speech import (
+ DEFAULT_TIMEOUT_SECONDS,
+ SpeechTransport,
+ dump_request,
+ parse_response,
+)
+from deepeval.models.stt._audio import audio_duration_seconds
+from deepeval.models.stt._stream import BufferedTranscribeMixin
+from deepeval.models.stt.deepgram.schema import (
+ DeepgramListenParams,
+ DeepgramListenResponse,
+)
+from deepeval.models.utils import parse_model_name, require_secret_api_key
+from deepeval.test_case import Audio
+
+retry_deepgram = create_retry_decorator(PS.DEEPGRAM)
+
+DEFAULT_BASE_URL = "https://api.deepgram.com"
+DEFAULT_STT_MODEL = "nova-3"
+
+_STT_PRICE_PER_MINUTE = {
+ "nova-3": 0.0043,
+ "nova-3-general": 0.0043,
+}
+
+
+class DeepgramSTTModel(BufferedTranscribeMixin, DeepEvalBaseSTT):
+ """Deepgram Nova speech-to-text."""
+
+ # Nova is not autoregressive, so it transcribes what it hears and stops
+ # rather than completing a word the barge-in cut off. No padding needed.
+ truncated_audio_pad_seconds: float = 0.0
+
+ def __init__(
+ self,
+ model: Optional[str] = None,
+ api_key: Optional[str] = None,
+ base_url: Optional[str] = None,
+ language: Optional[str] = None,
+ smart_format: bool = True,
+ cost_per_minute: Optional[float] = None,
+ transcription_kwargs: Optional[Dict] = None,
+ timeout: float = DEFAULT_TIMEOUT_SECONDS,
+ ):
+ settings = get_settings()
+ self.name = parse_model_name(model or DEFAULT_STT_MODEL)
+ self.language = language
+ self.smart_format = smart_format
+ self.transcription_kwargs = transcription_kwargs or {}
+ self.cost_per_minute = (
+ cost_per_minute
+ if cost_per_minute is not None
+ else _STT_PRICE_PER_MINUTE.get(self.name)
+ )
+ self.api_key = (
+ SecretStr(api_key)
+ if api_key is not None
+ else settings.DEEPGRAM_API_KEY
+ )
+ self.base_url = (
+ str(base_url).rstrip("/")
+ if base_url is not None
+ else DEFAULT_BASE_URL
+ )
+ self.timeout = timeout
+ self.model = self.load_model()
+
+ def load_model(self) -> SpeechTransport:
+ api_key = require_secret_api_key(
+ self.api_key,
+ provider_label="Deepgram",
+ env_var_name="DEEPGRAM_API_KEY",
+ param_hint="`api_key` to DeepgramSTTModel(...)",
+ )
+ return SpeechTransport(
+ provider_label="Deepgram",
+ base_url=self.base_url,
+ headers={"Authorization": f"Token {api_key}"},
+ timeout=self.timeout,
+ )
+
+ def _params(
+ self, language: Optional[str], kwargs: Dict[str, Any]
+ ) -> Dict[str, Any]:
+ lang = language or self.language
+ auto = lang == "auto"
+ return dump_request(
+ DeepgramListenParams(
+ model=self.name,
+ smart_format=self.smart_format,
+ # Deepgram detects only when asked to, so "auto" turns
+ # detection on instead of omitting the field.
+ language=None if auto else lang,
+ detect_language=True if auto else None,
+ **{**self.transcription_kwargs, **kwargs},
+ )
+ )
+
+ def _cost(self, audio: Audio) -> Optional[float]:
+ if self.cost_per_minute is None:
+ return None
+ seconds = audio_duration_seconds(audio)
+ if seconds is None:
+ return None
+ return seconds / 60.0 * self.cost_per_minute
+
+ def _parse(self, payload: Any) -> DeepgramListenResponse:
+ return parse_response(
+ DeepgramListenResponse, payload, provider_label="Deepgram"
+ )
+
+ @retry_deepgram
+ def transcribe(
+ self, audio: Audio, *, language: Optional[str] = None, **kwargs
+ ) -> Tuple[str, Optional[float]]:
+ payload = self._parse(
+ self.model.request_json(
+ "POST",
+ "/v1/listen",
+ params=self._params(language, kwargs),
+ content=audio.get_bytes(),
+ headers={"Content-Type": audio.mimeType or "audio/wav"},
+ )
+ )
+ return payload.transcript() or "", self._cost(audio)
+
+ @retry_deepgram
+ async def a_transcribe(
+ self, audio: Audio, *, language: Optional[str] = None, **kwargs
+ ) -> Tuple[str, Optional[float]]:
+ payload = self._parse(
+ await self.model.a_request_json(
+ "POST",
+ "/v1/listen",
+ params=self._params(language, kwargs),
+ content=audio.get_bytes(),
+ headers={"Content-Type": audio.mimeType or "audio/wav"},
+ )
+ )
+ return payload.transcript() or "", self._cost(audio)
+
+ def get_model_name(self) -> str:
+ return self.name
diff --git a/deepeval/models/stt/deepgram/schema.py b/deepeval/models/stt/deepgram/schema.py
new file mode 100644
index 000000000..f749821d4
--- /dev/null
+++ b/deepeval/models/stt/deepgram/schema.py
@@ -0,0 +1,73 @@
+"""Deepgram speech-to-text request and response shapes.
+
+Deepgram takes its transcription parameters in the query string rather than a
+body, so the request side is a params model.
+"""
+
+from typing import List, Optional
+
+from pydantic import BaseModel, ConfigDict
+
+# Requests allow extras so `transcription_kwargs` can set a provider parameter
+# deepeval has no field for. Responses allow them so a provider adding a field
+# never breaks a run. `protected_namespaces` is cleared because pydantic
+# reserves the `model_` prefix and Deepgram's parameter is called `model`.
+_REQUEST = ConfigDict(extra="allow", protected_namespaces=())
+_RESPONSE = ConfigDict(extra="allow", protected_namespaces=())
+
+
+class DeepgramListenParams(BaseModel):
+ """Query string for `POST /v1/listen`.
+
+ Deepgram detects the language only when asked to. An absent `language`
+ means English rather than detection, so `detect_language` has to be set
+ explicitly.
+ """
+
+ model_config = _REQUEST
+
+ model: str
+ smart_format: bool = True
+ language: Optional[str] = None
+ detect_language: Optional[bool] = None
+
+
+class DeepgramAlternative(BaseModel):
+ model_config = _RESPONSE
+
+ transcript: Optional[str] = None
+
+
+class DeepgramChannel(BaseModel):
+ model_config = _RESPONSE
+
+ alternatives: List[DeepgramAlternative] = []
+
+
+class DeepgramResults(BaseModel):
+ model_config = _RESPONSE
+
+ channels: List[DeepgramChannel] = []
+
+
+class DeepgramListenResponse(BaseModel):
+ """Response from `POST /v1/listen`."""
+
+ model_config = _RESPONSE
+
+ results: Optional[DeepgramResults] = None
+
+ def transcript(self) -> Optional[str]:
+ """The spoken text, or `None` when the response carries no transcript.
+
+ `None` and `""` mean different things: silence comes back as a present
+ but empty `transcript`, where `None` means the nesting this reads
+ through was not there at all.
+ """
+ if self.results is None:
+ return None
+ for channel in self.results.channels:
+ for alternative in channel.alternatives:
+ if alternative.transcript is not None:
+ return alternative.transcript
+ return None
diff --git a/deepeval/models/stt/elevenlabs/__init__.py b/deepeval/models/stt/elevenlabs/__init__.py
new file mode 100644
index 000000000..567e3409d
--- /dev/null
+++ b/deepeval/models/stt/elevenlabs/__init__.py
@@ -0,0 +1,3 @@
+from deepeval.models.stt.elevenlabs.elevenlabs import ElevenLabsSTTModel
+
+__all__ = ["ElevenLabsSTTModel"]
diff --git a/deepeval/models/stt/elevenlabs/elevenlabs.py b/deepeval/models/stt/elevenlabs/elevenlabs.py
new file mode 100644
index 000000000..6a4a91f36
--- /dev/null
+++ b/deepeval/models/stt/elevenlabs/elevenlabs.py
@@ -0,0 +1,154 @@
+from typing import Any, Dict, Optional, Tuple
+
+from pydantic import SecretStr
+
+from deepeval.config.settings import get_settings
+from deepeval.constants import ProviderSlug as PS
+from deepeval.models.base_model import DeepEvalBaseSTT
+from deepeval.models.retry_policy import create_retry_decorator
+from deepeval.models.speech import (
+ DEFAULT_TIMEOUT_SECONDS,
+ Multipart,
+ SpeechTransport,
+ dump_request,
+ parse_response,
+)
+from deepeval.models.stt._audio import audio_duration_seconds
+from deepeval.models.stt._stream import BufferedTranscribeMixin
+from deepeval.models.stt.elevenlabs.schema import (
+ ElevenLabsSTTRequest,
+ ElevenLabsSTTResponse,
+)
+from deepeval.models.utils import parse_model_name, require_secret_api_key
+from deepeval.test_case import Audio
+
+retry_elevenlabs = create_retry_decorator(PS.ELEVENLABS)
+
+DEFAULT_BASE_URL = "https://api.elevenlabs.io"
+
+# `scribe_v1` is deprecated in favour of v2.
+DEFAULT_STT_MODEL = "scribe_v2"
+
+_STT_PRICE_PER_HOUR = {
+ "scribe_v2": 0.22,
+ "scribe_v1": 0.22,
+}
+
+
+class ElevenLabsSTTModel(BufferedTranscribeMixin, DeepEvalBaseSTT):
+ """ElevenLabs Scribe speech-to-text."""
+
+ # Scribe completes a word clipped mid-syllable, inventing speech the caller
+ # never heard, so interrupted audio gets a tail of silence to mark the end.
+ truncated_audio_pad_seconds: float = 0.3
+
+ def __init__(
+ self,
+ model: Optional[str] = None,
+ api_key: Optional[str] = None,
+ base_url: Optional[str] = None,
+ language: Optional[str] = None,
+ cost_per_hour: Optional[float] = None,
+ transcription_kwargs: Optional[Dict] = None,
+ timeout: float = DEFAULT_TIMEOUT_SECONDS,
+ ):
+ settings = get_settings()
+ self.name = parse_model_name(model or DEFAULT_STT_MODEL)
+ self.language = language
+ self.transcription_kwargs = transcription_kwargs or {}
+ self.cost_per_hour = (
+ cost_per_hour
+ if cost_per_hour is not None
+ else _STT_PRICE_PER_HOUR.get(self.name)
+ )
+ self.api_key = (
+ SecretStr(api_key)
+ if api_key is not None
+ else settings.ELEVENLABS_API_KEY
+ )
+ self.base_url = (
+ str(base_url).rstrip("/")
+ if base_url is not None
+ else DEFAULT_BASE_URL
+ )
+ self.timeout = timeout
+ self.model = self.load_model()
+
+ def load_model(self) -> SpeechTransport:
+ api_key = require_secret_api_key(
+ self.api_key,
+ provider_label="ElevenLabs",
+ env_var_name="ELEVENLABS_API_KEY",
+ param_hint="`api_key` to ElevenLabsSTTModel(...)",
+ )
+ return SpeechTransport(
+ provider_label="ElevenLabs",
+ base_url=self.base_url,
+ headers={"xi-api-key": api_key},
+ timeout=self.timeout,
+ )
+
+ def _multipart(
+ self, audio: Audio, language: Optional[str], kwargs: Dict[str, Any]
+ ) -> Multipart:
+ lang = language or self.language
+ request = ElevenLabsSTTRequest(
+ model_id=self.name,
+ # Scribe detects the language whenever `language_code` is absent,
+ # so "auto" means leaving it out.
+ language_code=None if lang == "auto" else lang,
+ **{**self.transcription_kwargs, **kwargs},
+ )
+ return Multipart(
+ file_field="file",
+ filename=audio.filename or "audio.wav",
+ content=audio.get_bytes(),
+ content_type=audio.mimeType or "audio/wav",
+ fields=dump_request(request),
+ )
+
+ def _cost(
+ self, audio: Audio, payload: ElevenLabsSTTResponse
+ ) -> Optional[float]:
+ if self.cost_per_hour is None:
+ return None
+ seconds = payload.audio_duration_secs
+ if seconds is None:
+ seconds = audio_duration_seconds(audio)
+ if seconds is None:
+ return None
+ return seconds / 3600.0 * self.cost_per_hour
+
+ def _parse(self, payload: Any) -> ElevenLabsSTTResponse:
+ return parse_response(
+ ElevenLabsSTTResponse, payload, provider_label="ElevenLabs"
+ )
+
+ @retry_elevenlabs
+ def transcribe(
+ self, audio: Audio, *, language: Optional[str] = None, **kwargs
+ ) -> Tuple[str, Optional[float]]:
+ payload = self._parse(
+ self.model.request_json(
+ "POST",
+ "/v1/speech-to-text",
+ multipart=self._multipart(audio, language, kwargs),
+ )
+ )
+ return payload.transcript() or "", self._cost(audio, payload)
+
+ @retry_elevenlabs
+ async def a_transcribe(
+ self, audio: Audio, *, language: Optional[str] = None, **kwargs
+ ) -> Tuple[str, Optional[float]]:
+ payload = self._parse(
+ await self.model.a_request_json(
+ "POST",
+ "/v1/speech-to-text",
+ multipart=self._multipart(audio, language, kwargs),
+ )
+ )
+ return payload.transcript() or "", self._cost(audio, payload)
+
+ def get_model_name(self) -> str:
+ return self.name
diff --git a/deepeval/models/stt/elevenlabs/schema.py b/deepeval/models/stt/elevenlabs/schema.py
new file mode 100644
index 000000000..e13c1e19a
--- /dev/null
+++ b/deepeval/models/stt/elevenlabs/schema.py
@@ -0,0 +1,64 @@
+"""ElevenLabs speech-to-text request and response shapes.
+
+`protected_namespaces=()` is cleared because ElevenLabs names its field
+`model_id` and pydantic reserves the `model_` prefix for its own methods, so
+without it every import warns about a conflict that does not matter here.
+"""
+
+from typing import List, Optional
+
+from pydantic import BaseModel, ConfigDict
+
+# Requests allow extras so `transcription_kwargs` can set a provider parameter
+# deepeval has no field for. Responses allow them so a provider adding a field
+# never breaks a run.
+_REQUEST = ConfigDict(extra="allow", protected_namespaces=())
+_RESPONSE = ConfigDict(extra="allow", protected_namespaces=())
+
+
+class ElevenLabsSTTRequest(BaseModel):
+ """Scalar multipart fields for `POST /v1/speech-to-text`.
+
+ Scribe detects the language whenever `language_code` is absent, so omitting
+ it is how automatic detection is requested.
+ """
+
+ model_config = _REQUEST
+
+ model_id: str
+ language_code: Optional[str] = None
+
+
+class ElevenLabsTranscriptChannel(BaseModel):
+ model_config = _RESPONSE
+
+ text: Optional[str] = None
+
+
+class ElevenLabsSTTResponse(BaseModel):
+ """Response from `POST /v1/speech-to-text`.
+
+ A single-channel request answers with `text`; a multi-channel one answers
+ with one entry per channel in `transcripts` and no top-level `text`.
+ """
+
+ model_config = _RESPONSE
+
+ text: Optional[str] = None
+ audio_duration_secs: Optional[float] = None
+ transcripts: Optional[List[ElevenLabsTranscriptChannel]] = None
+
+ def transcript(self) -> Optional[str]:
+ """The spoken text, or `None` when the response carries no transcript.
+
+ `None` and `""` mean different things: `""` is silence, which is a
+ perfectly good answer, while `None` means neither field was present.
+ """
+ if self.text is not None:
+ return self.text
+ if self.transcripts is not None:
+ parts = [
+ channel.text for channel in self.transcripts if channel.text
+ ]
+ return " ".join(parts).strip()
+ return None
diff --git a/deepeval/models/stt/openai/__init__.py b/deepeval/models/stt/openai/__init__.py
new file mode 100644
index 000000000..84fd655fb
--- /dev/null
+++ b/deepeval/models/stt/openai/__init__.py
@@ -0,0 +1,3 @@
+from deepeval.models.stt.openai.openai import OpenAISTTModel
+
+__all__ = ["OpenAISTTModel"]
diff --git a/deepeval/models/stt_models/openai_stt.py b/deepeval/models/stt/openai/openai.py
similarity index 68%
rename from deepeval/models/stt_models/openai_stt.py
rename to deepeval/models/stt/openai/openai.py
index fe1f1de02..c8aa2f0ba 100644
--- a/deepeval/models/stt_models/openai_stt.py
+++ b/deepeval/models/stt/openai/openai.py
@@ -1,8 +1,4 @@
-import io
-import wave
from typing import (
- AsyncGenerator,
- AsyncIterable,
Optional,
Dict,
Tuple,
@@ -10,13 +6,14 @@ from typing import (
from pydantic import SecretStr
from openai import OpenAI, AsyncOpenAI
-from deepeval.test_case import Audio, AudioChunk
+from deepeval.test_case import Audio
from deepeval.models.base_model import DeepEvalBaseSTT
+from deepeval.models.stt._audio import audio_duration_seconds
+from deepeval.models.stt._stream import BufferedTranscribeMixin
from deepeval.models.utils import parse_model_name, require_secret_api_key
from deepeval.models.retry_policy import create_retry_decorator
from deepeval.constants import ProviderSlug as PS
from deepeval.config.settings import get_settings
-from deepeval.voice.connectors import audio_utils
retry_openai = create_retry_decorator(PS.OPENAI)
@@ -32,7 +29,7 @@ _STT_PRICE_PER_MINUTE = {
}
-class OpenAISTTModel(DeepEvalBaseSTT):
+class OpenAISTTModel(BufferedTranscribeMixin, DeepEvalBaseSTT):
# Whisper and the gpt-4o transcribe family both finish clipped words, so
# they need a tail of silence to be told the utterance ended where it did.
truncated_audio_pad_seconds: float = 0.3
@@ -115,20 +112,6 @@ class OpenAISTTModel(DeepEvalBaseSTT):
merged["language"] = lang
return merged
- @staticmethod
- def _audio_duration_seconds(audio: Audio) -> Optional[float]:
- if audio.duration is not None:
- return audio.duration
- try:
- with wave.open(io.BytesIO(audio.get_bytes()), "rb") as wf:
- frames = wf.getnframes()
- rate = wf.getframerate()
- if rate:
- return frames / float(rate)
- except (wave.Error, EOFError, ValueError):
- pass
- return None
-
def _calculate_cost(self, audio: Audio, response) -> Optional[float]:
usage = getattr(response, "usage", None)
usage_type = getattr(usage, "type", None)
@@ -152,7 +135,7 @@ class OpenAISTTModel(DeepEvalBaseSTT):
else None
)
if seconds is None:
- seconds = self._audio_duration_seconds(audio)
+ seconds = audio_duration_seconds(audio)
if seconds is not None:
return seconds / 60.0 * self.cost_per_minute
@@ -180,52 +163,5 @@ class OpenAISTTModel(DeepEvalBaseSTT):
)
return response.text, self._calculate_cost(audio, response)
- async def a_transcribe_stream(
- self,
- audio_stream: AsyncIterable[AudioChunk],
- *args,
- language: Optional[str] = None,
- partial_every_seconds: float = 1.0,
- **kwargs,
- ) -> AsyncGenerator[str, None]:
- """Yield progressively updated transcripts from a live PCM stream.
-
- OpenAI's batch transcription API has no true partial streaming, so
- this buffers chunks and re-transcribes the accumulated WAV whenever
- roughly `partial_every_seconds` of new audio arrives. Used as a
- duplex barge-in fallback when the transport has no text partials
- (e.g. LiveKit).
- """
- pcm = bytearray()
- sample_rate = 24000
- bytes_since_partial = 0
- last_text = ""
- async for chunk in audio_stream:
- if chunk.sampleRate:
- sample_rate = chunk.sampleRate
- frame = chunk.get_bytes()
- pcm.extend(frame)
- bytes_since_partial += len(frame)
- threshold = int(sample_rate * 2 * partial_every_seconds)
- if bytes_since_partial >= threshold or chunk.final:
- audio = Audio.from_bytes(
- audio_utils.pcm16_to_wav_bytes(bytes(pcm), sample_rate, 1),
- "audio/wav",
- sampleRate=sample_rate,
- encoding="wav",
- )
- text, _ = await self.a_transcribe(
- audio, language=language, **kwargs
- )
- if text and text != last_text:
- last_text = text
- yield text
- bytes_since_partial = 0
- if chunk.final:
- break
-
- def supports_streaming(self) -> bool:
- return True
-
def get_model_name(self) -> str:
return self.name
diff --git a/deepeval/models/stt_models/__init__.py b/deepeval/models/stt_models/__init__.py
deleted file mode 100644
index fa95415da..000000000
--- a/deepeval/models/stt_models/__init__.py
+++ /dev/null
@@ -1,3 +0,0 @@
-from deepeval.models.stt_models.openai_stt import OpenAISTTModel
-
-__all__ = ["OpenAISTTModel"]
diff --git a/deepeval/models/tts/__init__.py b/deepeval/models/tts/__init__.py
new file mode 100644
index 000000000..f2450b028
--- /dev/null
+++ b/deepeval/models/tts/__init__.py
@@ -0,0 +1,11 @@
+from deepeval.models.tts.cartesia import CartesiaTTSModel
+from deepeval.models.tts.deepgram import DeepgramTTSModel
+from deepeval.models.tts.elevenlabs import ElevenLabsTTSModel
+from deepeval.models.tts.openai import OpenAITTSModel
+
+__all__ = [
+ "CartesiaTTSModel",
+ "DeepgramTTSModel",
+ "ElevenLabsTTSModel",
+ "OpenAITTSModel",
+]
diff --git a/deepeval/models/tts/_frames.py b/deepeval/models/tts/_frames.py
new file mode 100644
index 000000000..551faa839
--- /dev/null
+++ b/deepeval/models/tts/_frames.py
@@ -0,0 +1,72 @@
+"""Turning a stream of PCM bytes into `AudioChunk`s."""
+
+from __future__ import annotations
+
+from typing import AsyncGenerator, AsyncIterable, Optional
+
+from deepeval.test_case import AudioChunk
+
+# A frame has to be playable on its own, which rules out every container format
+# (their headers declare a length the first frame does not know yet), so a
+# stream is always raw 16-bit mono PCM.
+STREAM_MIME = "audio/pcm"
+STREAM_ENCODING = "pcm"
+DEFAULT_FRAME_SECONDS = 0.1
+
+
+def pcm_chunk(
+ data: bytes, *, sample_rate: int, final: bool = False
+) -> AudioChunk:
+ return AudioChunk.from_bytes(
+ data,
+ STREAM_MIME,
+ sampleRate=sample_rate,
+ encoding=STREAM_ENCODING,
+ duration=len(data) / 2 / sample_rate,
+ final=final,
+ )
+
+
+def frame_size_bytes(
+ sample_rate: int, frame_seconds: float = DEFAULT_FRAME_SECONDS
+) -> int:
+ """Bytes in one frame, always even so a frame never splits a sample."""
+ samples = max(1, int(sample_rate * frame_seconds))
+ return samples * 2
+
+
+async def frame_pcm_stream(
+ source: AsyncIterable[bytes],
+ *,
+ sample_rate: int,
+ frame_seconds: float = DEFAULT_FRAME_SECONDS,
+) -> AsyncGenerator[AudioChunk, None]:
+ """Re-cut arbitrary byte reads from an HTTP body into fixed audio frames.
+
+ A provider's chunked response boundaries follow its network writes, not the
+ audio, so reads arrive at unhelpful sizes and can split a sample in half.
+
+ The last frame is held back so it can be flagged `final`, which is how a
+ consumer knows the utterance is complete without also tracking the stream.
+ """
+ frame_bytes = frame_size_bytes(sample_rate, frame_seconds)
+ buffer = bytearray()
+ held: Optional[bytes] = None
+
+ async for data in source:
+ if not data:
+ continue
+ buffer.extend(data)
+ while len(buffer) >= frame_bytes:
+ frame = bytes(buffer[:frame_bytes])
+ del buffer[:frame_bytes]
+ if held is not None:
+ yield pcm_chunk(held, sample_rate=sample_rate)
+ held = frame
+
+ if buffer:
+ if held is not None:
+ yield pcm_chunk(held, sample_rate=sample_rate)
+ held = bytes(buffer)
+ if held is not None:
+ yield pcm_chunk(held, sample_rate=sample_rate, final=True)
diff --git a/deepeval/models/tts/cartesia/__init__.py b/deepeval/models/tts/cartesia/__init__.py
new file mode 100644
index 000000000..870276ee8
--- /dev/null
+++ b/deepeval/models/tts/cartesia/__init__.py
@@ -0,0 +1,3 @@
+from deepeval.models.tts.cartesia.cartesia import CartesiaTTSModel
+
+__all__ = ["CartesiaTTSModel"]
diff --git a/deepeval/models/tts/cartesia/cartesia.py b/deepeval/models/tts/cartesia/cartesia.py
new file mode 100644
index 000000000..a38a6462b
--- /dev/null
+++ b/deepeval/models/tts/cartesia/cartesia.py
@@ -0,0 +1,195 @@
+from typing import Any, AsyncGenerator, Dict, Optional, Tuple
+
+from pydantic import SecretStr
+
+from deepeval.config.settings import get_settings
+from deepeval.constants import ProviderSlug as PS
+from deepeval.errors import DeepEvalError
+from deepeval.models.base_model import DeepEvalBaseTTS
+from deepeval.models.retry_policy import create_retry_decorator
+from deepeval.models.speech import (
+ DEFAULT_TIMEOUT_SECONDS,
+ SpeechTransport,
+ dump_request,
+)
+from deepeval.models.tts._frames import frame_pcm_stream
+from deepeval.models.tts.cartesia.schema import (
+ CartesiaOutputFormat,
+ CartesiaTTSRequest,
+)
+from deepeval.models.utils import parse_model_name, require_secret_api_key
+from deepeval.test_case import Audio, AudioChunk
+
+retry_cartesia = create_retry_decorator(PS.CARTESIA)
+
+DEFAULT_BASE_URL = "https://api.cartesia.ai"
+
+# Cartesia dates its API and requires the version on every request. Pinning it
+# here rather than tracking "latest" means a server-side release cannot change
+# the response shape underneath us.
+DEFAULT_API_VERSION = "2026-08-14"
+
+# `sonic-2` and `sonic-turbo` are gone from the current model enum and now fail
+# with model_not_found, so neither is a safe default.
+DEFAULT_TTS_MODEL = "sonic-3.6"
+
+DEFAULT_SAMPLE_RATE = 24000
+_SAMPLE_RATES = frozenset({8000, 16000, 22050, 24000, 44100, 48000})
+
+
+class CartesiaTTSModel(DeepEvalBaseTTS):
+ """Cartesia Sonic text-to-speech.
+
+ Cartesia has no default voice, so `voice` is required: it is a voice ID from
+ your Cartesia account rather than a name from a fixed cast.
+ """
+
+ def __init__(
+ self,
+ model: Optional[str] = None,
+ api_key: Optional[str] = None,
+ base_url: Optional[str] = None,
+ voice: Optional[str] = None,
+ sample_rate: int = DEFAULT_SAMPLE_RATE,
+ language: Optional[str] = None,
+ api_version: str = DEFAULT_API_VERSION,
+ cost_per_1m_chars: Optional[float] = None,
+ generation_kwargs: Optional[Dict] = None,
+ timeout: float = DEFAULT_TIMEOUT_SECONDS,
+ ):
+ settings = get_settings()
+ self.name = parse_model_name(model or DEFAULT_TTS_MODEL)
+ self.voice = voice
+ self.language = language
+ self.api_version = api_version
+ self.generation_kwargs = generation_kwargs or {}
+ # Cartesia meters in credits and publishes no credit-to-dollar rate, so
+ # there is no price table to default to. Pass `cost_per_1m_chars` to
+ # have synthesis costs accounted for.
+ self.cost_per_1m_chars = cost_per_1m_chars
+
+ if sample_rate not in _SAMPLE_RATES:
+ raise ValueError(
+ f"Cartesia cannot synthesize at {sample_rate} Hz. "
+ f"Supported rates: {sorted(_SAMPLE_RATES)}."
+ )
+ # Instance attribute, not the class default: connectors read
+ # `tts.sample_rate` off the instance to configure uplink resampling.
+ self.sample_rate = sample_rate
+
+ self.api_key = (
+ SecretStr(api_key)
+ if api_key is not None
+ else settings.CARTESIA_API_KEY
+ )
+ self.base_url = (
+ str(base_url).rstrip("/")
+ if base_url is not None
+ else DEFAULT_BASE_URL
+ )
+ self.timeout = timeout
+ self.model = self.load_model()
+
+ def load_model(self) -> SpeechTransport:
+ api_key = require_secret_api_key(
+ self.api_key,
+ provider_label="Cartesia",
+ env_var_name="CARTESIA_API_KEY",
+ param_hint="`api_key` to CartesiaTTSModel(...)",
+ )
+ return SpeechTransport(
+ provider_label="Cartesia",
+ base_url=self.base_url,
+ headers={
+ "X-Api-Key": api_key,
+ "Cartesia-Version": self.api_version,
+ },
+ timeout=self.timeout,
+ )
+
+ def _resolve_voice(self, voice: Optional[str]) -> str:
+ resolved = voice or self.voice
+ if not resolved:
+ raise DeepEvalError(
+ "Cartesia has no default voice. Pass `voice` to "
+ "CartesiaTTSModel(...) with a voice ID from your Cartesia "
+ "account, or set it on the golden's `Persona`."
+ )
+ return resolved
+
+ def _body(
+ self,
+ text: str,
+ voice: Optional[str],
+ *,
+ container: str,
+ extra: Dict[str, Any],
+ ) -> Dict[str, Any]:
+ return dump_request(
+ CartesiaTTSRequest(
+ model_id=self.name,
+ transcript=text,
+ voice=self._resolve_voice(voice),
+ output_format=CartesiaOutputFormat(
+ container=container, sample_rate=self.sample_rate
+ ),
+ language=self.language,
+ **{**self.generation_kwargs, **extra},
+ )
+ )
+
+ def synthesis_cost(self, text: str) -> Optional[float]:
+ if self.cost_per_1m_chars is None:
+ return None
+ return len(text) / 1e6 * self.cost_per_1m_chars
+
+ @retry_cartesia
+ def synthesize(
+ self, text: str, *, voice: Optional[str] = None, **kwargs
+ ) -> Tuple[Audio, Optional[float]]:
+ data = self.model.request_bytes(
+ "POST",
+ "/tts/bytes",
+ json=self._body(text, voice, container="wav", extra=kwargs),
+ )
+ return self._to_audio(data), self.synthesis_cost(text)
+
+ @retry_cartesia
+ async def a_synthesize(
+ self, text: str, *, voice: Optional[str] = None, **kwargs
+ ) -> Tuple[Audio, Optional[float]]:
+ data = await self.model.a_request_bytes(
+ "POST",
+ "/tts/bytes",
+ json=self._body(text, voice, container="wav", extra=kwargs),
+ )
+ return self._to_audio(data), self.synthesis_cost(text)
+
+ def _to_audio(self, data: bytes) -> Audio:
+ return Audio.from_bytes(
+ data,
+ "audio/wav",
+ sampleRate=self.sample_rate,
+ encoding="wav",
+ )
+
+ async def a_synthesize_stream(
+ self, text: str, *args, voice: Optional[str] = None, **kwargs
+ ) -> AsyncGenerator[AudioChunk, None]:
+ source = self.model.a_stream_bytes(
+ "POST",
+ "/tts/bytes",
+ # A raw container is headerless PCM, which is what makes each frame
+ # playable the moment it arrives.
+ json=self._body(text, voice, container="raw", extra=kwargs),
+ )
+ async for chunk in frame_pcm_stream(
+ source, sample_rate=self.sample_rate
+ ):
+ yield chunk
+
+ def supports_streaming(self) -> bool:
+ return True
+
+ def get_model_name(self) -> str:
+ return self.name
diff --git a/deepeval/models/tts/cartesia/schema.py b/deepeval/models/tts/cartesia/schema.py
new file mode 100644
index 000000000..6b93fef74
--- /dev/null
+++ b/deepeval/models/tts/cartesia/schema.py
@@ -0,0 +1,45 @@
+"""Cartesia text-to-speech request shapes.
+
+Cartesia is the only provider here that nests its audio settings, in an
+`output_format` object rather than flat fields, so that nesting gets a model of
+its own.
+"""
+
+from typing import Optional
+
+from pydantic import BaseModel, ConfigDict
+
+# Requests allow extras so `generation_kwargs` can set a provider parameter
+# deepeval has no field for. `protected_namespaces` is cleared because pydantic
+# reserves the `model_` prefix and Cartesia's field is called `model_id`.
+_REQUEST = ConfigDict(extra="allow", protected_namespaces=())
+
+
+class CartesiaOutputFormat(BaseModel):
+ """The `output_format` object on a TTS request.
+
+ `container` is `wav` for a batch call and `raw` for a streamed one: a WAV
+ header declares a length the first frame of a stream does not know yet.
+ """
+
+ model_config = _REQUEST
+
+ container: str
+ sample_rate: int
+ encoding: str = "pcm_s16le"
+
+
+class CartesiaTTSRequest(BaseModel):
+ """Body for `POST /tts/bytes`.
+
+ `voice` is a voice ID from your Cartesia account. There is no default, which
+ is why the model requires one rather than falling back.
+ """
+
+ model_config = _REQUEST
+
+ model_id: str
+ transcript: str
+ voice: str
+ output_format: CartesiaOutputFormat
+ language: Optional[str] = None
diff --git a/deepeval/models/tts/deepgram/__init__.py b/deepeval/models/tts/deepgram/__init__.py
new file mode 100644
index 000000000..dafba2411
--- /dev/null
+++ b/deepeval/models/tts/deepgram/__init__.py
@@ -0,0 +1,3 @@
+from deepeval.models.tts.deepgram.deepgram import DeepgramTTSModel
+
+__all__ = ["DeepgramTTSModel"]
diff --git a/deepeval/models/tts/deepgram/deepgram.py b/deepeval/models/tts/deepgram/deepgram.py
new file mode 100644
index 000000000..79af86c5c
--- /dev/null
+++ b/deepeval/models/tts/deepgram/deepgram.py
@@ -0,0 +1,202 @@
+from typing import Any, AsyncGenerator, Dict, Optional, Tuple
+
+from pydantic import SecretStr
+
+from deepeval.config.settings import get_settings
+from deepeval.constants import ProviderSlug as PS
+from deepeval.models.base_model import DeepEvalBaseTTS
+from deepeval.models.retry_policy import create_retry_decorator
+from deepeval.models.speech import (
+ DEFAULT_TIMEOUT_SECONDS,
+ SpeechTransport,
+ dump_request,
+)
+from deepeval.models.tts._frames import frame_pcm_stream
+from deepeval.models.tts.deepgram.schema import (
+ DeepgramSpeakParams,
+ DeepgramSpeakRequest,
+)
+from deepeval.models.utils import parse_model_name, require_secret_api_key
+from deepeval.test_case import Audio, AudioChunk
+
+retry_deepgram = create_retry_decorator(PS.DEEPGRAM)
+
+DEFAULT_BASE_URL = "https://api.deepgram.com"
+DEFAULT_TTS_MODEL = "aura-2-thalia-en"
+DEFAULT_SAMPLE_RATE = 24000
+
+# Rates Deepgram accepts for linear16.
+_PCM_SAMPLE_RATES = frozenset({8000, 16000, 24000, 32000, 48000})
+
+_AURA_2_PRICE_PER_1M_CHARS = 30.0
+_AURA_1_PRICE_PER_1M_CHARS = 15.0
+
+
+class DeepgramTTSModel(DeepEvalBaseTTS):
+ """Deepgram Aura text-to-speech.
+
+ Aura model names carry the voice and the language in the identifier
+ (`aura-2-thalia-en`), so the `voice` argument rewrites that middle segment
+ rather than being sent as a separate field.
+ """
+
+ def __init__(
+ self,
+ model: Optional[str] = None,
+ api_key: Optional[str] = None,
+ base_url: Optional[str] = None,
+ voice: Optional[str] = None,
+ sample_rate: int = DEFAULT_SAMPLE_RATE,
+ cost_per_1m_chars: Optional[float] = None,
+ generation_kwargs: Optional[Dict] = None,
+ timeout: float = DEFAULT_TIMEOUT_SECONDS,
+ ):
+ settings = get_settings()
+ self.name = parse_model_name(model or DEFAULT_TTS_MODEL)
+ # A voice given here is folded into the model name up front, so
+ # `get_model_name()` reports what will actually be synthesized.
+ if voice:
+ self.name = apply_voice(self.name, voice)
+ self.generation_kwargs = generation_kwargs or {}
+ self.cost_per_1m_chars = (
+ cost_per_1m_chars
+ if cost_per_1m_chars is not None
+ else _default_price(self.name)
+ )
+
+ if sample_rate not in _PCM_SAMPLE_RATES:
+ raise ValueError(
+ f"Deepgram cannot synthesize linear16 at {sample_rate} Hz. "
+ f"Supported rates: {sorted(_PCM_SAMPLE_RATES)}."
+ )
+ # Instance attribute, not the class default: connectors read
+ # `tts.sample_rate` off the instance to configure uplink resampling.
+ self.sample_rate = sample_rate
+
+ self.api_key = (
+ SecretStr(api_key)
+ if api_key is not None
+ else settings.DEEPGRAM_API_KEY
+ )
+ self.base_url = (
+ str(base_url).rstrip("/")
+ if base_url is not None
+ else DEFAULT_BASE_URL
+ )
+ self.timeout = timeout
+ self.model = self.load_model()
+
+ def load_model(self) -> SpeechTransport:
+ api_key = require_secret_api_key(
+ self.api_key,
+ provider_label="Deepgram",
+ env_var_name="DEEPGRAM_API_KEY",
+ param_hint="`api_key` to DeepgramTTSModel(...)",
+ )
+ return SpeechTransport(
+ provider_label="Deepgram",
+ base_url=self.base_url,
+ headers={"Authorization": f"Token {api_key}"},
+ timeout=self.timeout,
+ )
+
+ def _params(
+ self, voice: Optional[str], *, container: str, extra: Dict[str, Any]
+ ) -> Dict[str, Any]:
+ return dump_request(
+ DeepgramSpeakParams(
+ model=apply_voice(self.name, voice) if voice else self.name,
+ container=container,
+ sample_rate=self.sample_rate,
+ **{**self.generation_kwargs, **extra},
+ )
+ )
+
+ def _body(self, text: str) -> Dict[str, Any]:
+ return dump_request(DeepgramSpeakRequest(text=text))
+
+ def synthesis_cost(self, text: str) -> Optional[float]:
+ if self.cost_per_1m_chars is None:
+ return None
+ return len(text) / 1e6 * self.cost_per_1m_chars
+
+ @retry_deepgram
+ def synthesize(
+ self, text: str, *, voice: Optional[str] = None, **kwargs
+ ) -> Tuple[Audio, Optional[float]]:
+ data = self.model.request_bytes(
+ "POST",
+ "/v1/speak",
+ params=self._params(voice, container="wav", extra=kwargs),
+ json=self._body(text),
+ )
+ return self._to_audio(data), self.synthesis_cost(text)
+
+ @retry_deepgram
+ async def a_synthesize(
+ self, text: str, *, voice: Optional[str] = None, **kwargs
+ ) -> Tuple[Audio, Optional[float]]:
+ data = await self.model.a_request_bytes(
+ "POST",
+ "/v1/speak",
+ params=self._params(voice, container="wav", extra=kwargs),
+ json=self._body(text),
+ )
+ return self._to_audio(data), self.synthesis_cost(text)
+
+ def _to_audio(self, data: bytes) -> Audio:
+ return Audio.from_bytes(
+ data,
+ "audio/wav",
+ sampleRate=self.sample_rate,
+ encoding="wav",
+ )
+
+ async def a_synthesize_stream(
+ self, text: str, *args, voice: Optional[str] = None, **kwargs
+ ) -> AsyncGenerator[AudioChunk, None]:
+ source = self.model.a_stream_bytes(
+ "POST",
+ "/v1/speak",
+ # `container=none` is what makes this headerless PCM; a WAV header
+ # would arrive declaring a length the first frame cannot know.
+ params=self._params(voice, container="none", extra=kwargs),
+ json=self._body(text),
+ )
+ async for chunk in frame_pcm_stream(
+ source, sample_rate=self.sample_rate
+ ):
+ yield chunk
+
+ def supports_streaming(self) -> bool:
+ return True
+
+ def get_model_name(self) -> str:
+ return self.name
+
+
+def apply_voice(model_name: str, voice: str) -> str:
+ """Swap the voice segment of an Aura model name.
+
+ Accepts either a bare voice (`zeus`) or a full model name
+ (`aura-2-zeus-en`), since a `Persona.voice` written for another provider is
+ just a name while one written for Deepgram may well be the whole thing.
+ """
+ if voice.startswith("aura"):
+ return voice
+
+ parts = model_name.split("-")
+ # aura-2-thalia-en -> [aura, 2, thalia, en]; aura-asteria-en -> [aura, asteria, en]
+ if len(parts) >= 4:
+ return "-".join([parts[0], parts[1], voice, *parts[3:]])
+ if len(parts) == 3:
+ return "-".join([parts[0], voice, parts[2]])
+ return voice
+
+
+def _default_price(model_name: str) -> Optional[float]:
+ if model_name.startswith("aura-2"):
+ return _AURA_2_PRICE_PER_1M_CHARS
+ if model_name.startswith("aura"):
+ return _AURA_1_PRICE_PER_1M_CHARS
+ return None
diff --git a/deepeval/models/tts/deepgram/schema.py b/deepeval/models/tts/deepgram/schema.py
new file mode 100644
index 000000000..0d0910731
--- /dev/null
+++ b/deepeval/models/tts/deepgram/schema.py
@@ -0,0 +1,36 @@
+"""Deepgram text-to-speech request shapes.
+
+Deepgram is the odd one here: the text goes in the body and every other
+parameter goes in the query string, which is the reverse of the other providers.
+Hence a params model alongside the request body model.
+"""
+
+from pydantic import BaseModel, ConfigDict
+
+# Requests allow extras so `generation_kwargs` can set a provider parameter
+# deepeval has no field for. `protected_namespaces` is cleared because pydantic
+# reserves the `model_` prefix and Deepgram's parameter is called `model`.
+_REQUEST = ConfigDict(extra="allow", protected_namespaces=())
+
+
+class DeepgramSpeakParams(BaseModel):
+ """Query string for `POST /v1/speak`.
+
+ `container` is `wav` for a batch call and `none` for a streamed one: a WAV
+ header declares a length the first frame of a stream does not know yet.
+ """
+
+ model_config = _REQUEST
+
+ model: str
+ container: str
+ sample_rate: int
+ encoding: str = "linear16"
+
+
+class DeepgramSpeakRequest(BaseModel):
+ """Body for `POST /v1/speak`."""
+
+ model_config = _REQUEST
+
+ text: str
diff --git a/deepeval/models/tts/elevenlabs/__init__.py b/deepeval/models/tts/elevenlabs/__init__.py
new file mode 100644
index 000000000..83e56c1e5
--- /dev/null
+++ b/deepeval/models/tts/elevenlabs/__init__.py
@@ -0,0 +1,3 @@
+from deepeval.models.tts.elevenlabs.elevenlabs import ElevenLabsTTSModel
+
+__all__ = ["ElevenLabsTTSModel"]
diff --git a/deepeval/models/tts/elevenlabs/elevenlabs.py b/deepeval/models/tts/elevenlabs/elevenlabs.py
new file mode 100644
index 000000000..1cd2567fd
--- /dev/null
+++ b/deepeval/models/tts/elevenlabs/elevenlabs.py
@@ -0,0 +1,200 @@
+from typing import Any, AsyncGenerator, Dict, Optional, Tuple
+
+from pydantic import SecretStr
+
+from deepeval.config.settings import get_settings
+from deepeval.constants import ProviderSlug as PS
+from deepeval.models.base_model import DeepEvalBaseTTS
+from deepeval.models.retry_policy import create_retry_decorator
+from deepeval.models.speech import (
+ DEFAULT_TIMEOUT_SECONDS,
+ SpeechTransport,
+ dump_request,
+)
+from deepeval.models.tts._frames import frame_pcm_stream
+from deepeval.models.tts.elevenlabs.schema import (
+ ElevenLabsTTSParams,
+ ElevenLabsTTSRequest,
+)
+from deepeval.models.utils import parse_model_name, require_secret_api_key
+from deepeval.test_case import Audio, AudioChunk
+
+retry_elevenlabs = create_retry_decorator(PS.ELEVENLABS)
+
+DEFAULT_BASE_URL = "https://api.elevenlabs.io"
+
+# Flash is the model built for realtime agents (~75 ms). The Turbo family is
+# deprecated in favour of it, and the expressive v3 models cost twice as much
+# and are slower, which is the wrong trade for a simulated caller.
+DEFAULT_TTS_MODEL = "eleven_flash_v2_5"
+
+# Deliberately not one of the legacy "Default" voices (Rachel, Adam, Bella...):
+# those are unavailable to accounts created after March 2026 and stop resolving
+# entirely on 2027-01-01. This is George, from the Voice Library.
+DEFAULT_VOICE = "JBFqnCBsd6RMkjVDRZzb"
+
+DEFAULT_SAMPLE_RATE = 24000
+
+# Rates ElevenLabs will emit as PCM. Anything else has to be requested in a
+# container format, which cannot be streamed frame by frame.
+_PCM_SAMPLE_RATES = frozenset({8000, 16000, 22050, 24000, 32000, 44100, 48000})
+
+_TTS_PRICE_PER_1M_CHARS = {
+ "eleven_flash_v2_5": 50.0,
+ "eleven_flash_v2": 50.0,
+ "eleven_v3": 100.0,
+ "eleven_v3_conversational": 100.0,
+ "eleven_multilingual_v2": 100.0,
+ "eleven_turbo_v2_5": 50.0,
+ "eleven_turbo_v2": 50.0,
+}
+
+
+class ElevenLabsTTSModel(DeepEvalBaseTTS):
+ """ElevenLabs text-to-speech.
+
+ Note this is unrelated to `ElevenLabsConnector`, which bridges to a voice
+ agent hosted on ElevenLabs. This is the model that speaks the simulated
+ user's side of the conversation, and it can be pointed at an agent on any
+ platform.
+ """
+
+ def __init__(
+ self,
+ model: Optional[str] = None,
+ api_key: Optional[str] = None,
+ base_url: Optional[str] = None,
+ voice: Optional[str] = None,
+ sample_rate: int = DEFAULT_SAMPLE_RATE,
+ language_code: Optional[str] = None,
+ voice_settings: Optional[Dict[str, Any]] = None,
+ cost_per_1m_chars: Optional[float] = None,
+ generation_kwargs: Optional[Dict] = None,
+ timeout: float = DEFAULT_TIMEOUT_SECONDS,
+ ):
+ settings = get_settings()
+ self.name = parse_model_name(model or DEFAULT_TTS_MODEL)
+ self.voice = voice or DEFAULT_VOICE
+ self.language_code = language_code
+ self.voice_settings = voice_settings
+ self.generation_kwargs = generation_kwargs or {}
+ self.cost_per_1m_chars = (
+ cost_per_1m_chars
+ if cost_per_1m_chars is not None
+ else _TTS_PRICE_PER_1M_CHARS.get(self.name)
+ )
+
+ if sample_rate not in _PCM_SAMPLE_RATES:
+ raise ValueError(
+ f"ElevenLabs cannot synthesize at {sample_rate} Hz. "
+ f"Supported rates: {sorted(_PCM_SAMPLE_RATES)}."
+ )
+ # Instance attribute, not the class default: connectors read
+ # `tts.sample_rate` off the instance to configure uplink resampling.
+ self.sample_rate = sample_rate
+
+ self.api_key = (
+ SecretStr(api_key)
+ if api_key is not None
+ else settings.ELEVENLABS_API_KEY
+ )
+ self.base_url = (
+ str(base_url).rstrip("/")
+ if base_url is not None
+ else DEFAULT_BASE_URL
+ )
+ self.timeout = timeout
+ self.model = self.load_model()
+
+ def load_model(self) -> SpeechTransport:
+ api_key = require_secret_api_key(
+ self.api_key,
+ provider_label="ElevenLabs",
+ env_var_name="ELEVENLABS_API_KEY",
+ param_hint="`api_key` to ElevenLabsTTSModel(...)",
+ )
+ return SpeechTransport(
+ provider_label="ElevenLabs",
+ base_url=self.base_url,
+ headers={"xi-api-key": api_key},
+ timeout=self.timeout,
+ )
+
+ def _body(self, text: str, extra: Dict[str, Any]) -> Dict[str, Any]:
+ return dump_request(
+ ElevenLabsTTSRequest(
+ text=text,
+ model_id=self.name,
+ language_code=self.language_code,
+ voice_settings=self.voice_settings,
+ **{**self.generation_kwargs, **extra},
+ )
+ )
+
+ def _params(self, *, stream: bool) -> Dict[str, Any]:
+ prefix = "pcm" if stream else "wav"
+ return dump_request(
+ ElevenLabsTTSParams(output_format=f"{prefix}_{self.sample_rate}")
+ )
+
+ def _path(self, voice: Optional[str], *, stream: bool) -> str:
+ voice_id = voice or self.voice
+ suffix = "/stream" if stream else ""
+ return f"/v1/text-to-speech/{voice_id}{suffix}"
+
+ def synthesis_cost(self, text: str) -> Optional[float]:
+ if self.cost_per_1m_chars is None:
+ return None
+ return len(text) / 1e6 * self.cost_per_1m_chars
+
+ @retry_elevenlabs
+ def synthesize(
+ self, text: str, *, voice: Optional[str] = None, **kwargs
+ ) -> Tuple[Audio, Optional[float]]:
+ data = self.model.request_bytes(
+ "POST",
+ self._path(voice, stream=False),
+ params=self._params(stream=False),
+ json=self._body(text, kwargs),
+ )
+ return self._to_audio(data), self.synthesis_cost(text)
+
+ @retry_elevenlabs
+ async def a_synthesize(
+ self, text: str, *, voice: Optional[str] = None, **kwargs
+ ) -> Tuple[Audio, Optional[float]]:
+ data = await self.model.a_request_bytes(
+ "POST",
+ self._path(voice, stream=False),
+ params=self._params(stream=False),
+ json=self._body(text, kwargs),
+ )
+ return self._to_audio(data), self.synthesis_cost(text)
+
+ def _to_audio(self, data: bytes) -> Audio:
+ return Audio.from_bytes(
+ data,
+ "audio/wav",
+ sampleRate=self.sample_rate,
+ encoding="wav",
+ )
+
+ async def a_synthesize_stream(
+ self, text: str, *args, voice: Optional[str] = None, **kwargs
+ ) -> AsyncGenerator[AudioChunk, None]:
+ source = self.model.a_stream_bytes(
+ "POST",
+ self._path(voice, stream=True),
+ params=self._params(stream=True),
+ json=self._body(text, kwargs),
+ )
+ async for chunk in frame_pcm_stream(
+ source, sample_rate=self.sample_rate
+ ):
+ yield chunk
+
+ def supports_streaming(self) -> bool:
+ return True
+
+ def get_model_name(self) -> str:
+ return self.name
diff --git a/deepeval/models/tts/elevenlabs/schema.py b/deepeval/models/tts/elevenlabs/schema.py
new file mode 100644
index 000000000..07dada24b
--- /dev/null
+++ b/deepeval/models/tts/elevenlabs/schema.py
@@ -0,0 +1,38 @@
+"""ElevenLabs text-to-speech request shapes.
+
+`protected_namespaces=()` is cleared because ElevenLabs names its field
+`model_id` and pydantic reserves the `model_` prefix for its own methods, so
+without it every import warns about a conflict that does not matter here.
+"""
+
+from typing import Any, Dict, Optional
+
+from pydantic import BaseModel, ConfigDict
+
+# Requests allow extras so `generation_kwargs` can set a provider parameter
+# deepeval has no field for.
+_REQUEST = ConfigDict(extra="allow", protected_namespaces=())
+
+
+class ElevenLabsTTSParams(BaseModel):
+ """Query string for `POST /v1/text-to-speech/{voice_id}`.
+
+ `output_format` is `wav_{rate}` for a batch call and `pcm_{rate}` for a
+ streamed one: a WAV header declares a length the first frame of a stream
+ does not know yet.
+ """
+
+ model_config = _REQUEST
+
+ output_format: str
+
+
+class ElevenLabsTTSRequest(BaseModel):
+ """Body for `POST /v1/text-to-speech/{voice_id}`."""
+
+ model_config = _REQUEST
+
+ text: str
+ model_id: str
+ language_code: Optional[str] = None
+ voice_settings: Optional[Dict[str, Any]] = None
diff --git a/deepeval/models/tts/openai/__init__.py b/deepeval/models/tts/openai/__init__.py
new file mode 100644
index 000000000..d3cd47e62
--- /dev/null
+++ b/deepeval/models/tts/openai/__init__.py
@@ -0,0 +1,3 @@
+from deepeval.models.tts.openai.openai import OpenAITTSModel
+
+__all__ = ["OpenAITTSModel"]
diff --git a/deepeval/models/tts_models/openai_tts.py b/deepeval/models/tts/openai/openai.py
similarity index 76%
rename from deepeval/models/tts_models/openai_tts.py
rename to deepeval/models/tts/openai/openai.py
index f3605e7a1..2afe874a0 100644
--- a/deepeval/models/tts_models/openai_tts.py
+++ b/deepeval/models/tts/openai/openai.py
@@ -4,6 +4,7 @@ from openai import OpenAI, AsyncOpenAI
from deepeval.test_case import Audio, AudioChunk
from deepeval.models.base_model import DeepEvalBaseTTS
+from deepeval.models.tts._frames import frame_pcm_stream, frame_size_bytes
from deepeval.models.utils import parse_model_name, require_secret_api_key
from deepeval.models.retry_policy import create_retry_decorator
from deepeval.constants import ProviderSlug as PS
@@ -27,8 +28,7 @@ _FORMAT_MIME = {
# format above: their headers declare a length the first chunk does not have
# yet. Streaming is therefore always raw 16-bit mono PCM at 24 kHz.
_STREAM_FORMAT = "pcm"
-_STREAM_FRAME_SECONDS = 0.1
-_STREAM_FRAME_BYTES = int(_OPENAI_TTS_SAMPLE_RATE * 2 * _STREAM_FRAME_SECONDS)
+_STREAM_FRAME_BYTES = frame_size_bytes(_OPENAI_TTS_SAMPLE_RATE)
_TTS_PRICE_PER_1M_CHARS = {
"tts-1": 15.0,
"tts-1-hd": 30.0,
@@ -138,16 +138,6 @@ class OpenAITTSModel(DeepEvalBaseTTS):
)
return self._to_audio(response.content, fmt), self.synthesis_cost(text)
- def _to_chunk(self, data: bytes, *, final: bool) -> AudioChunk:
- return AudioChunk.from_bytes(
- data,
- _FORMAT_MIME[_STREAM_FORMAT],
- sampleRate=_OPENAI_TTS_SAMPLE_RATE,
- encoding=_STREAM_FORMAT,
- duration=len(data) / 2 / _OPENAI_TTS_SAMPLE_RATE,
- final=final,
- )
-
async def a_synthesize_stream(
self,
text: str,
@@ -161,34 +151,25 @@ class OpenAITTSModel(DeepEvalBaseTTS):
entire synthesis time in front of the first word. Streaming lets a
caller start speaking after the first frame, so what is left to
synthesize is spoken over.
-
- The last frame is held back so it can be flagged `final`, which is how
- a consumer knows the utterance is complete without also tracking the
- stream itself.
"""
- buffer = bytearray()
- held: Optional[bytes] = None
- async with self._async().audio.speech.with_streaming_response.create(
- model=self.name,
- voice=voice or self.voice,
- input=text,
- response_format=_STREAM_FORMAT,
- **{**self.generation_kwargs, **kwargs},
- ) as response:
- async for data in response.iter_bytes():
- buffer.extend(data)
- while len(buffer) >= _STREAM_FRAME_BYTES:
- frame = bytes(buffer[:_STREAM_FRAME_BYTES])
- del buffer[:_STREAM_FRAME_BYTES]
- if held is not None:
- yield self._to_chunk(held, final=False)
- held = frame
- if buffer:
- if held is not None:
- yield self._to_chunk(held, final=False)
- held = bytes(buffer)
- if held is not None:
- yield self._to_chunk(held, final=True)
+
+ async def _reads():
+ async with (
+ self._async().audio.speech.with_streaming_response.create(
+ model=self.name,
+ voice=voice or self.voice,
+ input=text,
+ response_format=_STREAM_FORMAT,
+ **{**self.generation_kwargs, **kwargs},
+ )
+ ) as response:
+ async for data in response.iter_bytes():
+ yield data
+
+ async for chunk in frame_pcm_stream(
+ _reads(), sample_rate=_OPENAI_TTS_SAMPLE_RATE
+ ):
+ yield chunk
def supports_streaming(self) -> bool:
return True
diff --git a/deepeval/models/tts_models/__init__.py b/deepeval/models/tts_models/__init__.py
deleted file mode 100644
index b2b4b39f6..000000000
--- a/deepeval/models/tts_models/__init__.py
+++ /dev/null
@@ -1,3 +0,0 @@
-from deepeval.models.tts_models.openai_tts import OpenAITTSModel
-
-__all__ = ["OpenAITTSModel"]
diff --git a/deepeval/simulator/conversation_simulator.py b/deepeval/simulator/conversation_simulator.py
index 687cfc72e..8b197b111 100644
--- a/deepeval/simulator/conversation_simulator.py
+++ b/deepeval/simulator/conversation_simulator.py
@@ -196,14 +196,11 @@ class ConversationSimulator:
# A single connector holds a single live call; concurrent
# conversations would interleave audio on the same session.
max_concurrent = 1
- from deepeval.models.tts_models import OpenAITTSModel
- from deepeval.models.stt_models import OpenAISTTModel
-
self._voice = _VoiceRun(
config=voice_config,
connector=voice_config.connector,
- tts_model=voice_config.tts_model or OpenAITTSModel(),
- stt_model=voice_config.stt_model or OpenAISTTModel(),
+ tts_model=voice_config.tts_model,
+ stt_model=voice_config.stt_model,
)
model_callback = self._voice_model_callback
if voice_config.interruption_settings is not None:
diff --git a/deepeval/voice/__init__.py b/deepeval/voice/__init__.py
index cce93d0e0..b0e319fb8 100644
--- a/deepeval/voice/__init__.py
+++ b/deepeval/voice/__init__.py
@@ -12,6 +12,8 @@ from deepeval.voice.connectors import (
BaseWebSocketConnector,
ElevenLabsConnector,
WebSocketConnector,
+ PipecatConnector,
+ VapiConnector,
)
__all__ = [
@@ -31,6 +33,8 @@ __all__ = [
"BaseWebSocketConnector",
"ElevenLabsConnector",
"WebSocketConnector",
+ "PipecatConnector",
+ "VapiConnector",
]
# Resolved lazily: these live with `Persona` in `deepeval.dataset`, which
diff --git a/deepeval/voice/config.py b/deepeval/voice/config.py
index 7add2aaba..d4eaece4c 100644
--- a/deepeval/voice/config.py
+++ b/deepeval/voice/config.py
@@ -12,6 +12,36 @@ if TYPE_CHECKING:
from deepeval.dataset.golden import InterruptionBehavior
+def resolve_tts_model(
+ model: Optional[Union[str, DeepEvalBaseTTS]],
+) -> DeepEvalBaseTTS:
+ """The TTS model to speak with.
+
+ A model name is the name for the provider selected by `USE_*_TTS`, and
+ `None` leaves both the provider and the name to the environment. Model
+ objects are left alone, including ones that only duck-type the base class.
+ """
+ # Imported here because `deepeval.models.tts` reaches back into
+ # `deepeval.voice`, so importing it at module scope would cycle.
+ from deepeval.models.speech_selection import initialize_tts_model
+
+ return initialize_tts_model(model)
+
+
+def resolve_stt_model(
+ model: Optional[Union[str, DeepEvalBaseSTT]],
+) -> DeepEvalBaseSTT:
+ """The STT model to transcribe with.
+
+ A model name is the name for the provider selected by `USE_*_STT`, and
+ `None` leaves both the provider and the name to the environment. Model
+ objects are left alone, including ones that only duck-type the base class.
+ """
+ from deepeval.models.speech_selection import initialize_stt_model
+
+ return initialize_stt_model(model)
+
+
@dataclass
class VoiceConfig:
"""Voice-mode settings for `ConversationSimulator`.
@@ -19,7 +49,13 @@ class VoiceConfig:
Passing a `VoiceConfig` puts the simulator in voice mode: simulated user
turns are spoken (TTS), sent to the agent over `connector`, and the
agent's spoken replies are transcribed (STT). `tts_model` / `stt_model`
- default to the OpenAI implementations when left as None.
+ come from the environment when left as None — the `USE_*_TTS` /
+ `USE_*_STT` flag picks the provider and `DEEPEVAL_TTS_MODEL` /
+ `DEEPEVAL_STT_MODEL` picks the model, both defaulting to OpenAI. A bare
+ model name is the name for that same selected provider, so
+ `stt_model="whisper-1"` is shorthand for `OpenAISTTModel(model="whisper-1")`
+ unless another `USE_*_STT` flag is set. Pass the model object itself to
+ reach a specific provider or set anything beyond the name.
`connector` is the connector the session runs over — `ElevenLabsConnector`,
`LiveKitConnector`, `WebSocketConnector`, `CallbackVoiceConnector`, or your
@@ -34,8 +70,8 @@ class VoiceConfig:
"""
connector: Union[BaseVoiceConnector, Callable[[], BaseVoiceConnector]]
- tts_model: Optional[DeepEvalBaseTTS] = None
- stt_model: Optional[DeepEvalBaseSTT] = None
+ tts_model: Optional[Union[str, DeepEvalBaseTTS]] = None
+ stt_model: Optional[Union[str, DeepEvalBaseSTT]] = None
# Directory to write per-turn and combined audio files into. Left alone,
# this resolves through `DEEPEVAL_VOICE_FOLDER` to a default folder; pass
# `None` to skip writing audio to disk. Resolved once, here, so that by the
@@ -53,6 +89,8 @@ class VoiceConfig:
self.connector, BaseVoiceConnector
):
self.connector = validate_connector(self.connector)
+ self.tts_model = resolve_tts_model(self.tts_model)
+ self.stt_model = resolve_stt_model(self.stt_model)
self.output_dir = resolve_output_dir(self.output_dir)
def make_connector(self) -> BaseVoiceConnector:
diff --git a/deepeval/voice/connectors/__init__.py b/deepeval/voice/connectors/__init__.py
index 1538da111..2f3997ede 100644
--- a/deepeval/voice/connectors/__init__.py
+++ b/deepeval/voice/connectors/__init__.py
@@ -8,6 +8,8 @@ from deepeval.voice.connectors.transports import (
from deepeval.voice.connectors.providers import (
ElevenLabsConnector,
LiveKitConnector,
+ PipecatConnector,
+ VapiConnector,
)
__all__ = [
@@ -19,4 +21,6 @@ __all__ = [
"BaseWebSocketConnector",
"ElevenLabsConnector",
"WebSocketConnector",
+ "PipecatConnector",
+ "VapiConnector",
]
diff --git a/deepeval/voice/connectors/providers/__init__.py b/deepeval/voice/connectors/providers/__init__.py
index 4d1d9098d..6577336c6 100644
--- a/deepeval/voice/connectors/providers/__init__.py
+++ b/deepeval/voice/connectors/providers/__init__.py
@@ -1,7 +1,11 @@
from deepeval.voice.connectors.providers.elevenlabs import ElevenLabsConnector
from deepeval.voice.connectors.providers.livekit import LiveKitConnector
+from deepeval.voice.connectors.providers.pipecat import PipecatConnector
+from deepeval.voice.connectors.providers.vapi import VapiConnector
__all__ = [
"ElevenLabsConnector",
"LiveKitConnector",
+ "PipecatConnector",
+ "VapiConnector",
]
diff --git a/deepeval/voice/connectors/providers/pipecat/README.md b/deepeval/voice/connectors/providers/pipecat/README.md
new file mode 100644
index 000000000..4fdb14d1b
--- /dev/null
+++ b/deepeval/voice/connectors/providers/pipecat/README.md
@@ -0,0 +1,92 @@
+# Pipecat
+
+Contributor reference and TODO list for the Pipecat integration. This file
+tracks which of Pipecat's entry points deepeval can reach; it deliberately does
+not restate their parameters, because those are defined by Pipecat and go stale
+here. Each row links to the page that owns the answer.
+
+## One connector, no dependency on `pipecat-ai`
+
+`PipecatConnector` (`connector.py`) connects to a pipeline running behind one of
+Pipecat's [WebSocket transports](https://docs.pipecat.ai/server/services/transport/websocket-server)
+the way any other client would: it is given a URL, and the pipeline is the
+user's to host.
+
+`protobuf.py` is our own encoder for the five messages in Pipecat's
+[`frames.proto`](https://github.com/pipecat-ai/pipecat/blob/main/src/pipecat/serializers/protobuf.py),
+which is what `ProtobufFrameSerializer` — the default on those transports —
+puts on the wire. Encoding them directly is about 150 lines and keeps
+`pipecat-ai`, a server framework with a large dependency tree, out of the
+requirements of testing something built with it.
+
+The schema is `oneof { text, audio, transcription, message, interruption }`.
+Only `audio` and `message` are written; `message` carries a JSON string, which
+is how RTVI rides the same socket.
+
+## What carries the turn-taking
+
+Pipecat has no end-of-turn frame in the protobuf schema, so the turn-taking
+comes from [RTVI](https://docs.pipecat.ai/server/frameworks/rtvi/introduction)
+messages, which a pipeline only sends if it runs an `RTVIProcessor` and its
+observer:
+
+| Message | Used for |
+| --- | --- |
+| `bot-stopped-speaking` | Ending the agent's turn outright, instead of inferring it from silence |
+| `bot-transcription` | The agent's own words, which skip STT for that turn |
+| `bot-interrupted` | Recording that a barge landed |
+| `client-ready` (sent by us) | Pipelines that hold their greeting, or their audio input, until the client is up |
+
+`signals_turn_complete` is therefore not a constant: it starts False and flips
+once a `bot-stopped-speaking` has actually been seen. Claiming it up front
+would leave a non-RTVI pipeline's turns waiting for a signal that is never
+coming; claiming it never would truncate replies that pause mid-sentence.
+
+RTVI emits one `bot-transcription` per sentence, so `_on_transcription`
+accumulates them and `bot-stopped-speaking` clears the buffer.
+
+## Entry point coverage
+
+| Entry point | Status | Reference |
+| --- | --- | --- |
+| `WebsocketServerTransport` | Covered | [WebSocket server](https://docs.pipecat.ai/server/services/transport/websocket-server) |
+| `FastAPIWebsocketTransport` | Covered | [FastAPI WebSocket](https://docs.pipecat.ai/server/services/transport/fastapi-websocket) |
+| `ProtobufFrameSerializer` | Covered | [Serializers](https://docs.pipecat.ai/server/utilities/serializers/introduction) |
+| `add_wav_header=True` on the transport | Covered | [WebSocket server](https://docs.pipecat.ai/server/services/transport/websocket-server) |
+| RTVI turn signals and bot transcripts | Covered | [RTVI](https://docs.pipecat.ai/server/frameworks/rtvi/introduction) |
+| `client-ready` handshake | Covered | [RTVI](https://docs.pipecat.ai/server/frameworks/rtvi/introduction) |
+| `bot-output` (the successor to `bot-transcription`) | Not covered | [RTVI messages](https://docs.pipecat.ai/client/js/api-reference/callbacks) |
+| Client-side function calls over `client-message` | Not covered | [RTVI](https://docs.pipecat.ai/server/frameworks/rtvi/introduction) |
+| `TwilioFrameSerializer`, `ExotelFrameSerializer`, and the other telephony dialects | Not covered | [Serializers](https://docs.pipecat.ai/server/utilities/serializers/introduction) |
+| Daily, LiveKit, and the other WebRTC transports | Not covered here | [Transports](https://docs.pipecat.ai/server/services/transport/daily) |
+| SmallWebRTC transport | Not covered | [SmallWebRTC](https://docs.pipecat.ai/server/services/transport/small-webrtc) |
+
+A pipeline on a WebRTC transport is reached through the connector for that
+service — a Pipecat pipeline on LiveKit is a LiveKit room, and
+`LiveKitConnector` joins it. The telephony serializers are the agent taking a
+real phone call; reaching such an agent means running the same pipeline behind
+a WebSocket transport instead.
+
+## TODO
+
+**Smoke test against a live pipeline.** The frame encoding is checked against
+Pipecat's own generated `frames_pb2` in both directions, but three things could
+only be confirmed against a running pipeline: that a pipeline gated on
+`on_client_ready` does open on our `client-ready`, that `bot-stopped-speaking`
+arrives after the last audio frame rather than before it, and how a pipeline
+whose `audio_in_sample_rate` differs from `agent_sample_rate` degrades (Pipecat
+does not resample what it is handed).
+
+**`bot-output`.** RTVI is deprecating `bot-transcription` in favour of
+`bot-output`, which carries progress updates for the same text and would double
+up the transcript if both were read. When `bot-transcription` goes, the
+accumulation has to move to `bot-output` and de-duplicate by segment id.
+
+**What the agent heard.** `transcription` frames carry Pipecat's STT of
+deepeval's synthesized audio, and are dropped. As with the other connectors, it
+is the signal that separates "the agent answered badly" from "the agent misheard
+the question", and there is nowhere on `ConnectorTurn` to put it today.
+
+**Ending the pipeline.** `disconnect()` closes the socket and nothing more.
+RTVI's `disconnect-bot` would end the pipeline task outright, which is more
+than a client should decide for a server it does not own.
diff --git a/deepeval/voice/connectors/providers/pipecat/__init__.py b/deepeval/voice/connectors/providers/pipecat/__init__.py
new file mode 100644
index 000000000..165d0ea79
--- /dev/null
+++ b/deepeval/voice/connectors/providers/pipecat/__init__.py
@@ -0,0 +1,5 @@
+from deepeval.voice.connectors.providers.pipecat.connector import (
+ PipecatConnector,
+)
+
+__all__ = ["PipecatConnector"]
diff --git a/deepeval/voice/connectors/providers/pipecat/connector.py b/deepeval/voice/connectors/providers/pipecat/connector.py
new file mode 100644
index 000000000..a667d6079
--- /dev/null
+++ b/deepeval/voice/connectors/providers/pipecat/connector.py
@@ -0,0 +1,205 @@
+import json
+import logging
+import uuid
+from typing import List, Optional, Union
+
+from deepeval.voice.connectors import audio_utils
+from deepeval.voice.connectors.providers.pipecat import protobuf
+from deepeval.voice.connectors.transports.websocket import (
+ BaseWebSocketConnector,
+ InboundEvent,
+)
+
+logger = logging.getLogger(__name__)
+
+# Pipecat's default input rate; its output default is 24000, which is also the
+# rate deepeval records at, so only the uplink needs stating.
+DEFAULT_AGENT_SAMPLE_RATE = 16000
+
+RTVI_LABEL = "rtvi-ai"
+RTVI_PROTOCOL_VERSION = "2.1.0"
+
+
+class PipecatConnector(BaseWebSocketConnector):
+ """Connect to a Pipecat pipeline over its WebSocket transport.
+
+ The pipeline is yours and self-hosted, so the connector takes a URL rather
+ than credentials, and speaks the protobuf frames Pipecat's WebSocket
+ transports serialize by default.
+ """
+
+ def __init__(
+ self,
+ url: str,
+ *,
+ headers: Optional[dict] = None,
+ agent_sample_rate: int = DEFAULT_AGENT_SAMPLE_RATE,
+ client_ready: bool = True,
+ **base_kwargs,
+ ):
+ super().__init__(**base_kwargs)
+ self.url = url
+ self.headers = headers
+ # Pipecat does not resample what it is handed: the frames have to
+ # arrive at the rate the pipeline's VAD and STT were started with.
+ self.agent_sample_rate = agent_sample_rate
+ self._send_rate = agent_sample_rate
+ self.client_ready = client_ready
+ self._transcript_parts: List[str] = []
+ self._seen_turn_signal = False
+ self._warned_non_protobuf = False
+
+ @property
+ def signals_turn_complete(self) -> bool:
+ """True once the pipeline has been seen to announce an end of turn.
+
+ Whether it does is a property of the pipeline rather than of Pipecat:
+ the announcement is an RTVI `bot-stopped-speaking` message, and only a
+ pipeline running an `RTVIProcessor` sends one. Reading it off the wire
+ rather than taking it on trust means a pipeline without RTVI falls back
+ to ending turns on silence instead of waiting for a signal that is
+ never coming.
+ """
+ return self._seen_turn_signal
+
+ async def connect(self) -> None:
+ self._transcript_parts = []
+ self._seen_turn_signal = False
+ await super().connect()
+
+ async def _open_session(self) -> str:
+ return self.url
+
+ def _connect_headers(self) -> Optional[dict]:
+ return self.headers
+
+ def _ready_on_connect(self) -> bool:
+ # A Pipecat server accepts audio as soon as the socket is up; nothing
+ # is sent back to acknowledge the client.
+ return True
+
+ def _initial_messages(self) -> List[Union[str, bytes]]:
+ """Announce the client, which some pipelines wait for before greeting.
+
+ Pipelines commonly open on `on_client_ready` — the greeting, and
+ sometimes the audio input itself, is gated on it. A pipeline without an
+ `RTVIProcessor` has nothing listening for the message and ignores it.
+ """
+ if not self.client_ready:
+ return []
+ return [
+ protobuf.encode_message_frame(
+ json.dumps(
+ {
+ "label": RTVI_LABEL,
+ "type": "client-ready",
+ "id": uuid.uuid4().hex,
+ "data": {
+ "version": RTVI_PROTOCOL_VERSION,
+ "about": {"library": "deepeval"},
+ },
+ }
+ )
+ )
+ ]
+
+ def _encode_outbound(self, pcm: bytes) -> Union[str, bytes]:
+ return protobuf.encode_audio_frame(pcm, self._send_rate, 1)
+
+ def _decode_inbound(self, raw: Union[str, bytes]) -> Optional[InboundEvent]:
+ if isinstance(raw, str):
+ self._warn_non_protobuf("a text message")
+ return None
+ try:
+ frame = protobuf.decode_frame(bytes(raw))
+ except ValueError:
+ self._warn_non_protobuf("bytes that are not protobuf frames")
+ return None
+ if frame is None:
+ return None
+
+ if frame.kind == "audio":
+ return self._on_audio(frame)
+ if frame.kind == "message":
+ return self._on_transport_message(frame.data)
+ if frame.kind == "interruption":
+ self._interrupted = True
+ # A `transcription` frame is Pipecat's STT of deepeval's own audio, and
+ # a `text` frame is unattributed pipeline text; neither is the agent's
+ # reply.
+ return None
+
+ def _warn_non_protobuf(self, what: str) -> None:
+ if self._warned_non_protobuf:
+ return
+ self._warned_non_protobuf = True
+ logger.warning(
+ "PipecatConnector received %s. It speaks the protobuf frames of "
+ "`ProtobufFrameSerializer`, which is what Pipecat's WebSocket "
+ "transports use by default — a pipeline serializing something "
+ "else needs a `WebSocketConnector` describing that dialect "
+ "instead.",
+ what,
+ )
+
+ def _on_audio(self, frame: protobuf.PipecatFrame) -> Optional[InboundEvent]:
+ pcm = frame.audio or b""
+ if not pcm:
+ return None
+ sample_rate, channels = frame.sample_rate, frame.num_channels or 1
+ if pcm[:4] == b"RIFF":
+ # `add_wav_header=True` on the transport wraps every frame.
+ pcm, sample_rate, channels = audio_utils.wav_bytes_to_pcm16(pcm)
+ if sample_rate and sample_rate != self._recv_rate:
+ logger.debug(
+ "Pipecat is sending %dHz audio; reading the downlink at that "
+ "rate instead of %dHz.",
+ sample_rate,
+ self._recv_rate,
+ )
+ self._recv_rate = sample_rate
+ return InboundEvent(audio=audio_utils.downmix_to_mono(pcm, channels))
+
+ def _on_transport_message(
+ self, data: Optional[str]
+ ) -> Optional[InboundEvent]:
+ try:
+ message = json.loads(data or "")
+ except ValueError:
+ return None
+ if not isinstance(message, dict) or message.get("label") != RTVI_LABEL:
+ return None
+
+ message_type = message.get("type")
+
+ if message_type == "bot-stopped-speaking":
+ self._seen_turn_signal = True
+ self._transcript_parts = []
+ return InboundEvent(turn_complete=True)
+
+ if message_type == "bot-interrupted":
+ self._interrupted = True
+ return None
+
+ if message_type == "bot-transcription":
+ return self._on_transcription(message)
+
+ return None
+
+ def _on_transcription(self, message: dict) -> Optional[InboundEvent]:
+ """Collect the agent's transcript for the turn in progress.
+
+ RTVI emits one `bot-transcription` per sentence, so the last one alone
+ would record the end of a multi-sentence reply and nothing before it.
+ """
+ data = message.get("data")
+ text = (data or {}).get("text") if isinstance(data, dict) else None
+ text = (text or "").strip()
+ if not text:
+ return None
+ self._transcript_parts.append(text)
+ return InboundEvent(transcript=" ".join(self._transcript_parts))
+
+ def drain_downlink(self) -> None:
+ self._transcript_parts = []
+ super().drain_downlink()
diff --git a/deepeval/voice/connectors/providers/pipecat/protobuf.py b/deepeval/voice/connectors/providers/pipecat/protobuf.py
new file mode 100644
index 000000000..1b7691e8f
--- /dev/null
+++ b/deepeval/voice/connectors/providers/pipecat/protobuf.py
@@ -0,0 +1,178 @@
+"""The part of Pipecat's `frames.proto` that a caller has to speak.
+
+Pipecat's WebSocket transports serialize with `ProtobufFrameSerializer`, whose
+schema is five messages wide and carries only what a client sends or reads:
+audio, text, a transcription, a JSON transport message, and an interruption.
+Encoding those directly keeps `pipecat-ai` — a server-side framework with a
+large dependency tree — out of the requirements of running a simulation
+against it.
+
+https://github.com/pipecat-ai/pipecat/blob/main/src/pipecat/serializers/protobuf.py
+"""
+
+from dataclasses import dataclass
+from typing import Iterator, Optional, Tuple
+
+_WIRE_VARINT = 0
+_WIRE_64BIT = 1
+_WIRE_LEN = 2
+_WIRE_32BIT = 5
+
+# Fields of the `Frame` oneof.
+_FRAME_TEXT = 1
+_FRAME_AUDIO = 2
+_FRAME_TRANSCRIPTION = 3
+_FRAME_MESSAGE = 4
+_FRAME_INTERRUPTION = 5
+
+# Fields within each of those messages.
+_TEXT_TEXT = 3
+_AUDIO_AUDIO = 3
+_AUDIO_SAMPLE_RATE = 4
+_AUDIO_NUM_CHANNELS = 5
+_TRANSCRIPTION_TEXT = 3
+_MESSAGE_DATA = 1
+
+
+@dataclass
+class PipecatFrame:
+ """One decoded frame, named by which arm of the oneof it arrived on."""
+
+ kind: str # text | audio | transcription | message | interruption
+ audio: Optional[bytes] = None
+ sample_rate: Optional[int] = None
+ num_channels: Optional[int] = None
+ text: Optional[str] = None
+ data: Optional[str] = None # JSON, for `message`
+
+
+def _varint(value: int) -> bytes:
+ out = bytearray()
+ while True:
+ byte = value & 0x7F
+ value >>= 7
+ out.append(byte | (0x80 if value else 0))
+ if not value:
+ return bytes(out)
+
+
+def _tag(field: int, wire: int) -> bytes:
+ return _varint(field << 3 | wire)
+
+
+def _len_field(field: int, payload: bytes) -> bytes:
+ return _tag(field, _WIRE_LEN) + _varint(len(payload)) + payload
+
+
+def _varint_field(field: int, value: int) -> bytes:
+ return _tag(field, _WIRE_VARINT) + _varint(value)
+
+
+def encode_audio_frame(
+ pcm: bytes, sample_rate: int, num_channels: int = 1
+) -> bytes:
+ """A `Frame` carrying PCM16, which Pipecat reads as an `InputAudioRawFrame`."""
+ audio = (
+ _len_field(_AUDIO_AUDIO, pcm)
+ + _varint_field(_AUDIO_SAMPLE_RATE, sample_rate)
+ + _varint_field(_AUDIO_NUM_CHANNELS, num_channels)
+ )
+ return _len_field(_FRAME_AUDIO, audio)
+
+
+def encode_message_frame(data: str) -> bytes:
+ """A `Frame` carrying a JSON transport message, such as RTVI's."""
+ return _len_field(
+ _FRAME_MESSAGE, _len_field(_MESSAGE_DATA, data.encode("utf-8"))
+ )
+
+
+def _read_varint(buf: bytes, i: int) -> Tuple[int, int]:
+ value = shift = 0
+ while True:
+ if i >= len(buf):
+ raise ValueError("truncated varint")
+ byte = buf[i]
+ i += 1
+ value |= (byte & 0x7F) << shift
+ if not byte & 0x80:
+ return value, i
+ shift += 7
+
+
+def _iter_fields(buf: bytes) -> Iterator[Tuple[int, int, object]]:
+ i = 0
+ while i < len(buf):
+ key, i = _read_varint(buf, i)
+ field, wire = key >> 3, key & 0x07
+ if wire == _WIRE_VARINT:
+ value, i = _read_varint(buf, i)
+ elif wire == _WIRE_LEN:
+ length, i = _read_varint(buf, i)
+ value, i = buf[i : i + length], i + length
+ if len(value) != length:
+ raise ValueError("truncated length-delimited field")
+ elif wire in (_WIRE_64BIT, _WIRE_32BIT):
+ width = 8 if wire == _WIRE_64BIT else 4
+ value, i = buf[i : i + width], i + width
+ else:
+ raise ValueError(f"unsupported protobuf wire type {wire}")
+ yield field, wire, value
+
+
+def _string(value: bytes) -> str:
+ return value.decode("utf-8", "replace")
+
+
+def _audio_frame(payload: bytes) -> PipecatFrame:
+ frame = PipecatFrame(kind="audio", audio=b"")
+ for field, _, value in _iter_fields(payload):
+ if field == _AUDIO_AUDIO:
+ frame.audio = bytes(value)
+ elif field == _AUDIO_SAMPLE_RATE:
+ frame.sample_rate = value
+ elif field == _AUDIO_NUM_CHANNELS:
+ frame.num_channels = value
+ return frame
+
+
+def _one_string_frame(
+ payload: bytes, kind: str, field_number: int
+) -> PipecatFrame:
+ frame = PipecatFrame(kind=kind, text="")
+ for field, _, value in _iter_fields(payload):
+ if field == field_number:
+ frame.text = _string(value)
+ return frame
+
+
+def _message_frame(payload: bytes) -> PipecatFrame:
+ frame = PipecatFrame(kind="message", data="")
+ for field, _, value in _iter_fields(payload):
+ if field == _MESSAGE_DATA:
+ frame.data = _string(value)
+ return frame
+
+
+def decode_frame(raw: bytes) -> Optional[PipecatFrame]:
+ """Read a serialized `Frame`, or None if it carries nothing we know.
+
+ Raises `ValueError` on bytes that are not protobuf at all, which is how a
+ pipeline serializing something else announces itself.
+ """
+ for field, wire, value in _iter_fields(raw):
+ if wire != _WIRE_LEN:
+ continue
+ if field == _FRAME_AUDIO:
+ return _audio_frame(value)
+ if field == _FRAME_MESSAGE:
+ return _message_frame(value)
+ if field == _FRAME_TRANSCRIPTION:
+ return _one_string_frame(
+ value, "transcription", _TRANSCRIPTION_TEXT
+ )
+ if field == _FRAME_TEXT:
+ return _one_string_frame(value, "text", _TEXT_TEXT)
+ if field == _FRAME_INTERRUPTION:
+ return PipecatFrame(kind="interruption")
+ return None
diff --git a/deepeval/voice/connectors/providers/vapi/README.md b/deepeval/voice/connectors/providers/vapi/README.md
new file mode 100644
index 000000000..9e67c5c73
--- /dev/null
+++ b/deepeval/voice/connectors/providers/vapi/README.md
@@ -0,0 +1,85 @@
+# Vapi
+
+Contributor reference and TODO list for the Vapi integration. This file tracks
+which of Vapi's entry points deepeval can reach; it deliberately does not
+restate their parameters, because those are defined by Vapi and go stale here.
+Each row links to the page that owns the answer.
+
+## One connector
+
+`VapiConnector` (`connector.py`) is our own client for Vapi's
+[WebSocket transport](https://docs.vapi.ai/calls/websocket-transport). The
+caller passes an `assistant_id` plus an API key, deepeval creates the call and
+owns the socket, and there is no dependency on any Vapi SDK.
+
+The call is minted over REST (`POST /call` with
+`transport.provider = "vapi.websocket"`), which returns
+`transport.websocketCallUrl`. Phone parameters are rejected on that transport,
+so this path never touches telephony — the simulator is the only caller.
+
+## Client messages deepeval depends on
+
+Two of Vapi's [client messages](https://docs.vapi.ai/api-reference/webhooks/client-message)
+carry the turn-taking:
+
+| Message | Shape deepeval reads | Used for |
+| --- | --- | --- |
+| `speech-update` | `status` (`started`/`stopped`), `role` | Ending the agent's turn outright, instead of inferring it from silence |
+| `transcript` | `role`, `transcriptType` (`partial`/`final`), `transcript` | The agent's own words, which skip STT for that turn |
+
+Both are in Vapi's default `clientMessages` set, so nothing is sent unless the
+caller narrows the set themselves — `_assistant_overrides` then adds the two
+back, because losing them costs end-of-turn detection and free transcripts
+silently rather than loudly.
+
+Vapi emits one `final` transcript per utterance rather than one per turn, so
+`_on_transcript` accumulates them and `speech-update: stopped` clears the
+buffer. Taking only the last one would record half of a two-sentence reply.
+
+## Entry point coverage
+
+| Entry point | Status | Reference |
+| --- | --- | --- |
+| Assistant over WebSocket transport | Covered | [WebSocket transport](https://docs.vapi.ai/calls/websocket-transport) |
+| Per-call `assistantOverrides` (incl. `variableValues`) | Covered | [Assistant overrides](https://docs.vapi.ai/assistants/dynamic-variables) |
+| Explicit end of turn (`speech-update`) | Covered | [Client messages](https://docs.vapi.ai/api-reference/webhooks/client-message) |
+| Agent transcript (`transcript`, final) | Covered | [Client messages](https://docs.vapi.ai/api-reference/webhooks/client-message) |
+| `user-interrupted` | Covered | [Client messages](https://docs.vapi.ai/api-reference/webhooks/client-message) |
+| Ending the call (`end-call`) | Covered | [WebSocket transport](https://docs.vapi.ai/calls/websocket-transport) |
+| mu-law audio format | Not covered | [WebSocket transport](https://docs.vapi.ai/calls/websocket-transport) |
+| Inline assistant config instead of `assistantId` | Not covered | [Create call](https://docs.vapi.ai/api-reference/calls/create) |
+| Squads | Not covered | [Squads](https://docs.vapi.ai/squads) |
+| `tool-calls` / client-side tools | Not covered | [Client messages](https://docs.vapi.ai/api-reference/webhooks/client-message) |
+| `conversation-update`, `model-output` | Not covered | [Client messages](https://docs.vapi.ai/api-reference/webhooks/client-message) |
+| Call Listen (`monitor.listenUrl`) | Not covered, by choice | [WebSocket transport](https://docs.vapi.ai/calls/websocket-transport) |
+| Phone numbers, campaigns, SIP | Out of scope | [Calls](https://docs.vapi.ai/calls) |
+
+Call Listen is listen-only and supplements a call that already exists, so it
+cannot carry a simulation. "Out of scope" means the entry point is the agent
+placing or receiving a real phone call; reaching such an agent means pointing
+deepeval at the same assistant over the WebSocket transport instead.
+
+## TODO
+
+**Smoke test against a live assistant.** Everything here is written against
+Vapi's documented schemas. Three things could only be confirmed on a real call:
+whether the socket wraps client messages in a `message` envelope the way
+webhooks do (`_decode_inbound` accepts both shapes, so either works), whether a
+non-default `sampleRate` is honoured or quietly converted
+(`_apply_audio_format` adopts whatever comes back), and how the socket behaves
+before the first audio frame (`_ready_on_connect` assumes no handshake).
+`vapi_capture.py` at the repo root records one conversation for this.
+
+**What the agent heard.** `transcript` messages with `role: "user"` carry
+Vapi's transcription of deepeval's synthesized audio, and are dropped. As with
+ElevenLabs, it is the signal that separates "the agent answered badly" from
+"the agent misheard the question", and there is nowhere on `ConnectorTurn` to
+put it today.
+
+**Interruption detail.** `user-interrupted` sets `_interrupted` for the turn,
+but Vapi's own barge-in handling and deepeval's floor control both act on the
+same moment, and the interaction has not been exercised.
+
+**Call id.** `call_id` is kept from the create-call response but never
+surfaced, so a simulated conversation cannot be cross-referenced with the same
+call in Vapi's dashboard.
diff --git a/deepeval/voice/connectors/providers/vapi/__init__.py b/deepeval/voice/connectors/providers/vapi/__init__.py
new file mode 100644
index 000000000..187b7e2a6
--- /dev/null
+++ b/deepeval/voice/connectors/providers/vapi/__init__.py
@@ -0,0 +1,3 @@
+from deepeval.voice.connectors.providers.vapi.connector import VapiConnector
+
+__all__ = ["VapiConnector"]
diff --git a/deepeval/voice/connectors/providers/vapi/connector.py b/deepeval/voice/connectors/providers/vapi/connector.py
new file mode 100644
index 000000000..91c7e458a
--- /dev/null
+++ b/deepeval/voice/connectors/providers/vapi/connector.py
@@ -0,0 +1,213 @@
+import os
+import json
+import logging
+from typing import Any, Dict, List, Optional, Union
+
+from deepeval.errors import DeepEvalError
+from deepeval.voice.connectors.transports.websocket import (
+ BaseWebSocketConnector,
+ InboundEvent,
+)
+
+logger = logging.getLogger(__name__)
+
+DEFAULT_BASE_URL = "https://api.vapi.ai"
+
+# The client messages deepeval reads off the socket. Both are in Vapi's default
+# set, so they are only ever re-stated when a caller narrows the set themselves.
+# https://docs.vapi.ai/api-reference/webhooks/client-message
+REQUIRED_CLIENT_MESSAGES = ("speech-update", "transcript")
+
+
+class VapiConnector(BaseWebSocketConnector):
+
+ def __init__(
+ self,
+ assistant_id: str,
+ api_key: Optional[str] = None,
+ assistant_overrides: Optional[dict] = None,
+ base_url: str = DEFAULT_BASE_URL,
+ **base_kwargs,
+ ):
+ super().__init__(**base_kwargs)
+ self.assistant_id = assistant_id
+ self.api_key = api_key or os.getenv("VAPI_API_KEY")
+ self.assistant_overrides = assistant_overrides
+ self.base_url = base_url.rstrip("/")
+ self.call_id: Optional[str] = None
+ self._transcript_parts: List[str] = []
+
+ @property
+ def signals_turn_complete(self) -> bool:
+ # `speech-update` closes every turn.
+ return True
+
+ def _ready_on_connect(self) -> bool:
+ # The socket is minted per call and carries no handshake; audio may go
+ # out as soon as it opens.
+ return True
+
+ def _assistant_overrides(self) -> Optional[dict]:
+ """The caller's overrides, plus the client messages deepeval needs.
+
+ Narrowing `clientMessages` is how a Vapi assistant turns events off, so
+ a caller who sets it without `speech-update` would silently remove
+ end-of-turn detection, and one without `transcript` would pay for STT
+ on every turn.
+ """
+ overrides = dict(self.assistant_overrides or {})
+ requested = overrides.get("clientMessages")
+ if requested is None:
+ return overrides or None
+ missing = [
+ message
+ for message in REQUIRED_CLIENT_MESSAGES
+ if message not in requested
+ ]
+ if missing:
+ overrides["clientMessages"] = list(requested) + missing
+ return overrides
+
+ def _call_payload(self) -> Dict[str, Any]:
+ payload: Dict[str, Any] = {
+ "assistantId": self.assistant_id,
+ "transport": {
+ "provider": "vapi.websocket",
+ "audioFormat": {
+ "format": "pcm_s16le",
+ "container": "raw",
+ "sampleRate": self.sample_rate,
+ },
+ },
+ }
+ overrides = self._assistant_overrides()
+ if overrides:
+ payload["assistantOverrides"] = overrides
+ return payload
+
+ async def _open_session(self) -> str:
+ if not self.api_key:
+ raise DeepEvalError(
+ "VapiConnector requires a Vapi API key (pass `api_key=...` or "
+ "set VAPI_API_KEY). The key mints the call the WebSocket "
+ "carries."
+ )
+ async with self._session.post(
+ f"{self.base_url}/call",
+ headers={"authorization": f"Bearer {self.api_key}"},
+ json=self._call_payload(),
+ ) as resp:
+ body = await resp.text()
+ if resp.status >= 300:
+ raise DeepEvalError(
+ f"Vapi call creation failed ({resp.status}): {body}"
+ )
+ try:
+ data = json.loads(body)
+ except ValueError:
+ raise DeepEvalError(
+ f"Vapi call creation returned a non-JSON body: {body[:200]}"
+ )
+
+ transport = data.get("transport") or {}
+ url = transport.get("websocketCallUrl")
+ if not url:
+ raise DeepEvalError(
+ "Vapi call response had no `transport.websocketCallUrl`. Check "
+ "that the assistant exists and the key may create calls."
+ )
+ self.call_id = data.get("id")
+ self._apply_audio_format(transport.get("audioFormat"))
+ return url
+
+ def _apply_audio_format(self, audio_format: Optional[dict]) -> None:
+ """Adopt the format Vapi settled on, which need not be the one asked for."""
+ if not isinstance(audio_format, dict):
+ return
+ encoding = audio_format.get("format")
+ if encoding and encoding != "pcm_s16le":
+ logger.warning(
+ "Vapi negotiated audio format %r; deepeval sends and reads "
+ "PCM16, so the audio will be garbled. Request pcm_s16le.",
+ encoding,
+ )
+ rate = audio_format.get("sampleRate")
+ if isinstance(rate, int) and rate > 0:
+ self._send_rate = rate
+ self._recv_rate = rate
+
+ def _encode_outbound(self, pcm: bytes) -> Union[str, bytes]:
+ return pcm
+
+ def _decode_inbound(self, raw: Union[str, bytes]) -> Optional[InboundEvent]:
+ if isinstance(raw, (bytes, bytearray)):
+ return InboundEvent(audio=bytes(raw))
+
+ try:
+ payload = json.loads(raw)
+ except (ValueError, TypeError):
+ return None
+ if not isinstance(payload, dict):
+ return None
+ # Vapi's webhook deliveries wrap the event in `message`; the socket
+ # sends the same events, so accept either shape.
+ message = payload.get("message")
+ if not isinstance(message, dict):
+ message = payload
+
+ message_type = message.get("type")
+
+ if message_type == "transcript":
+ return self._on_transcript(message)
+
+ if message_type == "speech-update":
+ if (
+ message.get("role") == "assistant"
+ and message.get("status") == "stopped"
+ ):
+ self._transcript_parts = []
+ return InboundEvent(turn_complete=True)
+ return None
+
+ if message_type == "user-interrupted":
+ self._interrupted = True
+ return None
+
+ return None
+
+ def _on_transcript(self, message: dict) -> Optional[InboundEvent]:
+ """Collect the agent's final transcripts for the turn in progress.
+
+ Vapi emits one per utterance rather than one per turn, so a two
+ sentence reply arrives as two finals and the last one on its own would
+ record half of what the agent said.
+ """
+ if message.get("role") != "assistant":
+ return None
+ if message.get("transcriptType") != "final":
+ return None
+ text = (message.get("transcript") or "").strip()
+ if not text:
+ return None
+ self._transcript_parts.append(text)
+ return InboundEvent(transcript=" ".join(self._transcript_parts))
+
+ def drain_downlink(self) -> None:
+ self._transcript_parts = []
+ super().drain_downlink()
+
+ async def disconnect(self) -> None:
+ """End the call before dropping the socket.
+
+ Closing the socket alone leaves the call live on Vapi's side until it
+ times out, still billing and still holding a concurrency slot.
+ """
+ if self._ws is not None and not self._ws.closed:
+ try:
+ await self._send(json.dumps({"type": "end-call"}))
+ except Exception:
+ logger.debug(
+ "Failed to send Vapi end-call; closing the socket anyway.",
+ exc_info=True,
+ )
+ await super().disconnect()
diff --git a/deepeval/voice/connectors/utils.py b/deepeval/voice/connectors/utils.py
index 3b477da80..f6e4b2ba2 100644
--- a/deepeval/voice/connectors/utils.py
+++ b/deepeval/voice/connectors/utils.py
@@ -16,6 +16,20 @@ class LiveKitIdentity(Enum):
ROOM = "Room"
+class VapiIdentity(Enum):
+ CLIENT = "Vapi"
+ ASYNC_CLIENT = "AsyncVapi"
+
+
+class PipecatIdentity(Enum):
+ PIPELINE = "Pipeline"
+ TASK = "PipelineTask"
+ WORKER = "PipelineWorker"
+ SERVER_TRANSPORT = "WebsocketServerTransport"
+ SINGLE_CLIENT_TRANSPORT = "SingleClientWebsocketServerTransport"
+ FASTAPI_TRANSPORT = "FastAPIWebsocketTransport"
+
+
# What to tell someone who passed the vendor object instead of a connector.
# Every voice session runs through a connector, so these are redirections
# rather than alternative entry points.
@@ -36,8 +50,33 @@ _REDIRECTS = {
"`LiveKitConnector(room=your_room)` instead, which joins the room you "
"already have."
),
+ VapiIdentity.CLIENT: (
+ "VoiceConfig received a Vapi client. Pass "
+ "`VapiConnector(assistant_id=...)` instead — it creates the call "
+ "itself, and takes `assistant_overrides` for anything you would "
+ "configure per call."
+ ),
+ VapiIdentity.ASYNC_CLIENT: (
+ "VoiceConfig received a Vapi client. Pass "
+ "`VapiConnector(assistant_id=...)` instead."
+ ),
}
+# Pipecat's pipeline objects and its WebSocket transports all mean the same
+# thing here: the agent is reached over the socket its server is listening on,
+# never in-process.
+_REDIRECTS.update(
+ {
+ identity: (
+ f"VoiceConfig received a Pipecat `{identity.value}`. Run the "
+ "pipeline behind its WebSocket transport and pass "
+ "`PipecatConnector(url=...)` instead, which connects to it the "
+ "way any other client would."
+ )
+ for identity in PipecatIdentity
+ }
+)
+
def _root_module(cls: type) -> str:
return (getattr(cls, "__module__", "") or "").split(".")[0]
@@ -45,7 +84,9 @@ def _root_module(cls: type) -> str:
def _identify(
obj: Any,
-) -> Optional[Union[ElevenLabsIdentity, LiveKitIdentity]]:
+) -> Optional[
+ Union[ElevenLabsIdentity, LiveKitIdentity, PipecatIdentity, VapiIdentity]
+]:
"""Name the vendor class `obj` is, without importing the vendor package.
Matching walks the MRO by `(root module, class name)` rather than using
@@ -63,6 +104,14 @@ def _identify(
for identity in LiveKitIdentity:
if name == identity.value:
return identity
+ if root in ("vapi", "vapi_server_sdk"):
+ for identity in VapiIdentity:
+ if name == identity.value:
+ return identity
+ if root == "pipecat":
+ for identity in PipecatIdentity:
+ if name == identity.value:
+ return identity
return None
@@ -82,7 +131,7 @@ def validate_connector(connector: Any) -> BaseVoiceConnector:
raise DeepEvalError(
f"VoiceConfig cannot use {type(connector).__name__!r} as a connector. "
- "Pass an `ElevenLabsConnector`, `LiveKitConnector`, "
- "`WebSocketConnector`, `CallbackVoiceConnector`, or your own "
- "`BaseVoiceConnector` subclass."
+ "Pass an `ElevenLabsConnector`, `LiveKitConnector`, `VapiConnector`, "
+ "`PipecatConnector`, `WebSocketConnector`, `CallbackVoiceConnector`, "
+ "or your own `BaseVoiceConnector` subclass."
)
diff --git a/docs/content/docs/command-line-interface.mdx b/docs/content/docs/command-line-interface.mdx
index c8cd73a21..e5fc8aadf 100644
--- a/docs/content/docs/command-line-interface.mdx
+++ b/docs/content/docs/command-line-interface.mdx
@@ -372,7 +372,7 @@ npx deepeval diagnose
The report shows:
- **Versions**: deepeval and the runtime it's running on.
-- **Evaluation model**: the model deepeval will use when a metric (or synthesizer, simulator, etc.) is constructed without an explicit model, plus the reason it won. The default embedding model is listed alongside it, with what each is used by.
+- **Evaluation model**: the model deepeval will use when a metric (or synthesizer, simulator, etc.) is constructed without an explicit model, plus the reason it won. The default embedding model is listed alongside it, and the selected text-to-speech and speech-to-text models below those, with what each is used by.
- **Configuration sources**: the precedence order every variable is resolved against, independently (process environment → `.env.local` → `.env.` → `.env` → JSON keystore → built-in defaults), which dotenv files were loaded, and where the keystore lives.
- **Configured settings**: every deepeval setting that is explicitly set, with secrets masked (only the last 6 characters shown) and the **winning source** for each — the report uses the same resolvers as the runtime, so what you see is what your evals get.
- **Confident AI**: login status, masked API key, data region, and the API endpoint. The OTEL endpoint is shown too, with warnings if either endpoint looks like it belongs to a different region than your API key.
@@ -680,12 +680,24 @@ All provider commands come in pairs:
-This switches the active provider:
+This switches the active provider **within its family**:
-- It sets `USE__MODEL = True` for the chosen provider, and
-- Turns all other `USE_*` flags off so that only one provider is enabled at a time.
+- It sets that provider's `USE_*` flag, such as `USE_ANTHROPIC_MODEL`, and
+- Turns off the other `USE_*` flags in the same family, so only one provider per family is enabled at a time.
-When you **set** a provider, the CLI enables that provider’s `USE__MODEL` flag and disables all other `USE_*` flags. When you **unset** a provider, it disables only that provider’s `USE_*` flag and leaves all others untouched. If you manually set env vars (or edit dotenv files) it’s possible to end up with multiple `USE_*` flags enabled.
+
+
+The four families are the LLM judge, embeddings, text-to-speech and speech-to-text. They are independent, so `deepeval set-openai` leaves a configured embedding or speech provider exactly as it was, and each family falls back to OpenAI when none of its flags are set.
+
+
+
+
+
+Every `USE_*` flag here belongs to the LLM family, so setting one provider turns off all the others. With none set, `deepeval` falls back to OpenAI.
+
+
+
+When you **unset** a provider, it disables only that provider’s `USE_*` flag and leaves all others untouched. If you manually set env vars (or edit dotenv files) it’s possible to end up with multiple `USE_*` flags enabled within one family.
:::caution
Because of how `deepeval` manages your model related environment variables, **using the CLI is 100% the recommended way to configure evaluation models in `deepeval`.** It handles all the necessary environment variables for you, ensuring consistent and correct setup across different providers.
@@ -723,6 +735,21 @@ If you want to see what environment variables `deepeval` manages under the hood,
| Local (HTTP) | `set-local-embeddings` | `unset-local-embeddings` |
| Ollama | `set-ollama-embeddings` | `unset-ollama-embeddings` |
+**Speech (voice simulation):**
+
+Speech takes the provider as an argument instead of a command per provider, and text-to-speech and speech-to-text are selected independently of each other and of the LLM:
+
+| Family | Set | Unset |
+| -------------- | -------------------------------------------------------------- | ------------ |
+| Text-to-speech | `set-tts ` | `unset-tts` |
+| Speech-to-text | `set-stt ` | `unset-stt` |
+
+```bash
+deepeval set-stt deepgram --model nova-3
+```
+
+The model name is optional and defaults to that provider's own default. `unset-tts` and `unset-stt` clear their whole family rather than one provider, since the provider is an argument rather than part of the command name.
+
@@ -755,7 +782,7 @@ For provider-specific flags, run any of the `set-*` commands above with `--help`
## Common Issues
- **Nothing printed?** For `set-*` / `unset-*` / `set-debug`, a clean exit with no output often means you are passing the `--quiet` / `-q` flag.
-- **Provider still active after unsetting?** Unsetting turns off that provider's `USE_*` flag only; if another provider's flag is still on, it becomes the active provider. With no `USE_*` flag enabled, deepeval falls back to OpenAI. To force a provider, run the corresponding `set-` command.
+- **Provider still active after unsetting?** Unsetting turns off that provider's `USE_*` flag only; if another provider in the same family still has its flag on, it becomes the active one for that family. With no flag enabled in a family, deepeval falls back to OpenAI for it. To force a provider, run the corresponding `set-` command.
- **Dotenv edits not picked up?** deepeval loads dotenv files from the current working directory by default, or `ENV_DIR_PATH` if set. Ensure your process runs in that context.
If you’re still stuck, the dedicated [Troubleshooting](/docs/troubleshooting) page covers deeper debugging (TLS errors, logging, timeouts, dotenv loading, and config caching).
diff --git a/docs/content/docs/conversation-simulator-voice-connectors.mdx b/docs/content/docs/conversation-simulator-voice-connectors.mdx
index 31de783fb..329c22cb7 100644
--- a/docs/content/docs/conversation-simulator-voice-connectors.mdx
+++ b/docs/content/docs/conversation-simulator-voice-connectors.mdx
@@ -12,8 +12,10 @@ You won't need to write one. Pick whichever matches where your agent runs, tell
| Connector | Transport | Use it for |
| ------------------------ | ---------- | -------------------------------------------------------------------- |
+| `VapiConnector` | WebSocket | Vapi assistants, addressed by `assistant_id`. |
| `ElevenLabsConnector` | WebSocket | ElevenLabs agents, addressed by `agent_id`. |
| `LiveKitConnector` | WebRTC | Agents dispatched into LiveKit rooms. |
+| `PipecatConnector` | WebSocket | Self-hosted Pipecat pipelines, addressed by URL. |
| `WebSocketConnector` | WebSocket | Custom agents that speak raw audio over a WebSocket. |
| `CallbackVoiceConnector` | In-process | A Python callable, no network — handy for testing your own pipeline. |
@@ -25,133 +27,12 @@ See [Voice concepts](/docs/evaluation-voice#transports-and-connectors) for where
Provider not listed? If your agent exposes a raw-audio WebSocket, `WebSocketConnector` needs only the URL and the message shape. If it's callable in the same Python process, use `CallbackVoiceConnector`. Subclass `BaseVoiceConnector` only for a transport the others can't express.
::::
-## ElevenLabs
-
-Your agent lives in the ElevenLabs dashboard, so the connector only needs its `agent_id`. It speaks ElevenLabs' WebSocket API directly, so the `elevenlabs` package isn't required. Private agents also need an API key.
-
-```python
-from deepeval.voice import ElevenLabsConnector
-
-connector = ElevenLabsConnector(agent_id="your-agent-id")
-```
-
-There are **ONE** mandatory and **SEVEN** optional parameters when creating an `ElevenLabsConnector`:
-
-- `agent_id`: a string identifying the ElevenLabs conversational agent.
-- [Optional] `api_key`: a string API key. When omitted, reads `ELEVENLABS_API_KEY` from the environment. Defaulted to `None`.
-- [Optional] `region`: a string naming an ElevenLabs [data residency](https://elevenlabs.io/docs/overview/administration/data-residency) region — one of `"us"`, `"eu"`, `"in"`, or `"sg"`. Any other value raises. Defaulted to `None` (global host).
-- [Optional] `client_tools`: a dictionary mapping [client tool](https://elevenlabs.io/docs/eleven-agents/customization/tools/client-tools) names to the functions that run them. Defaulted to `None`.
-- [Optional] `dynamic_variables`: a dictionary of [dynamic variables](https://elevenlabs.io/docs/eleven-agents/customization/personalization/dynamic-variables) for this conversation. Defaulted to `None`.
-- [Optional] `conversation_config_override`: a dictionary of [overrides](https://elevenlabs.io/docs/eleven-agents/customization/personalization/overrides) for this conversation, such as prompt, first message, language, or voice. Defaulted to `None`.
-- [Optional] `client_tool_timeout_s`: how long to wait for one of your tool functions before answering the agent with an error. Defaulted to `30.0`.
-- [Optional] `turn_detection`: how long to wait before deciding your agent has finished speaking — `"eager"`, `"balanced"`, or `"patient"`. See [Turn Detection](#turn-detection). Defaulted to `"balanced"`.
-
-The connector reads ElevenLabs' [`agent_response_complete`](https://elevenlabs.io/docs/eleven-agents/api-reference/eleven-agents/websocket) message off the wire, so it knows exactly when a turn ended instead of inferring it from silence.
-
-### Client tools
-
-If your agent calls client tools, give the connector a function for each one. Declare the tool in the ElevenLabs dashboard as usual; the name here has to match.
-
-```python
-async def get_customer_details(parameters):
- return {"name": "Alice", "subscription": "Pro"}
-
-connector = ElevenLabsConnector(
- agent_id="your-agent-id",
- client_tools={"getCustomerDetails": get_customer_details},
-)
-```
-
-Handlers can be `async def` or plain functions — a plain one runs in a thread so it can block without stalling the call. Each receives one dictionary of the tool's parameters, plus `tool_call_id`, and whatever it returns is given to the agent as context. Returning realistic data is what keeps the rest of the conversation realistic.
-
-Your agent is waiting on that return value, so a handler is never allowed to leave it hanging. An unregistered name, a handler that raises, and one that outlives `client_tool_timeout_s` are all answered with an error result, and time spent inside your handler is excluded from the reported latency and from end-of-turn detection — otherwise a slow tool would read as a slow agent, or cut its turn short.
-
-### Personalizing a run
-
-`dynamic_variables` and `conversation_config_override` are sent per conversation, so one dashboard agent can be simulated as many different callers — a different customer, tier, or language per golden.
-
-```python
-connector = ElevenLabsConnector(
- agent_id="your-agent-id",
- dynamic_variables={"customer_name": "Alice", "tier": "enterprise"},
- conversation_config_override={"agent": {"language": "fr"}},
-)
-```
-
-::::note
-Overrides only take effect if you've enabled the matching fields on the agent in ElevenLabs first. Left disabled, they're ignored rather than rejected, which looks like the override silently doing nothing.
-::::
-
-## LiveKit
-
-`deepeval` joins your agent's room as another participant and trades audio over WebRTC, so the agent needs to be deployed and able to join. Install LiveKit's SDKs:
-
-```bash
-pip install livekit livekit-api
-```
-
-A LiveKit agent has no id you can dial — it's a worker [dispatched](https://docs.livekit.io/agents/server/agent-dispatch/) into a [room](https://docs.livekit.io/intro/basics/connect/). So instead of naming the agent, you tell the connector how to reach the room. Give it credentials and it mints a token and creates one:
-
-```python
-from deepeval.voice import LiveKitConnector
-
-connector = LiveKitConnector(
- url="wss://your-project.livekit.cloud",
- api_key="your-api-key",
- api_secret="your-api-secret",
- agent_name="restaurant-agent",
-)
-```
-
-There are **ZERO** mandatory and **ELEVEN** optional parameters when creating a `LiveKitConnector`. None is mandatory on its own, but the connector needs some way to reach the room — either the three connection values, or a `token`, or a connected `room`:
-
-- [Optional] `url`: the LiveKit server URL. Required unless `LIVEKIT_URL` is set, or `room` is already connected. Defaulted to `None`.
-- [Optional] `api_key`: the LiveKit API key. Required to mint a token, unless `LIVEKIT_API_KEY` is set. Defaulted to `None`.
-- [Optional] `api_secret`: the LiveKit API secret. Required to mint a token, unless `LIVEKIT_API_SECRET` is set. Defaulted to `None`.
-- [Optional] `room`: an [`rtc.Room`](https://docs.livekit.io/reference/python/livekit/rtc/room.html) to use instead of creating one. Connected or not. Defaulted to `None`.
-- [Optional] `token`: a LiveKit [access token](https://docs.livekit.io/frontends/reference/tokens-grants/) you minted yourself, used instead of `api_key` / `api_secret`. Defaulted to `None`.
-- [Optional] `room_name`: the room to join. When omitted, a room is created for the session. Defaulted to `None`.
-- [Optional] `identity`: the participant identity for the simulator. Defaulted to `"deepeval-test"`.
-- [Optional] `agent_name`: the name of the agent to [dispatch](https://docs.livekit.io/agents/server/agent-dispatch/) into the room. Defaulted to `None`.
-- [Optional] `turn_detection`: how long to wait before deciding your agent has finished speaking — `"eager"`, `"balanced"`, or `"patient"`. See [Turn Detection](#turn-detection). Defaulted to `"balanced"`.
-- [Optional] `connect_timeout_s`: timeout (seconds) for establishing the room connection. Defaulted to `15.0`.
-- [Optional] `transcript_grace_s`: how long to wait after your agent stops speaking for its transcript to arrive. Defaulted to `1.5`.
-
-`room_name` and `agent_name` are what actually find your agent. Omit `room_name` and the connector creates a room. Set `agent_name` and LiveKit dispatches that agent into it; leave it unset and your existing routing has to put the agent there.
-
-If your test setup already joined a room, hand that over instead and skip the credentials:
-
-```python
-from livekit import rtc
-from deepeval.voice import LiveKitConnector
-
-room = rtc.Room()
-await room.connect("wss://your-project.livekit.cloud", your_token)
-
-connector = LiveKitConnector(room=room)
-```
-
-`deepeval` publishes a microphone track, listens for the agent, and leaves the room as it found it — a room you connected stays connected afterwards, with just the simulated microphone removed. An unconnected room gets connected using the parameters above.
-
-::::note
-`deepeval` reads your agent's own transcript from LiveKit's [`lk.transcription`](https://docs.livekit.io/agents/multimodality/text/) text stream, so STT is usually unnecessary here — the words the agent actually spoke become the assistant `Turn.content`. STT is the fallback when your agent publishes no transcript.
-::::
+snippets/voice/connector-tabs.mdx
## Generic WebSocket
For custom agents that accept and emit raw audio over a WebSocket, `WebSocketConnector` lets you describe your agent's message shape — which JSON keys carry audio in and out, whether frames are binary, and what marks the end of a turn.
-```python
-from deepeval.voice import WebSocketConnector
-
-connector = WebSocketConnector(
- url="wss://your-agent.example.com/audio",
- send_key="audio",
- receive_audio_key="audio",
- turn_complete_type="turn_end",
-)
-```
-
There are **ONE** mandatory and **TWELVE** optional parameters when creating a `WebSocketConnector`:
- `url`: a string WebSocket URL for your agent (e.g. `"wss://your-agent.example.com/audio"`).
@@ -172,12 +53,6 @@ There are **ONE** mandatory and **TWELVE** optional parameters when creating a `
Wraps an in-process Python callable, so you can exercise the full TTS → agent → STT pipeline without any network transport — useful for testing your simulation setup before pointing it at a deployed agent.
-```python
-from deepeval.voice import CallbackVoiceConnector
-
-connector = CallbackVoiceConnector(my_agent)
-```
-
There are **ONE** mandatory and **THREE** optional parameters when creating a `CallbackVoiceConnector`:
- `agent`: a sync or async callable that accepts user `Audio` and returns a `ConnectorTurn` or plain `Audio`.
@@ -247,9 +122,9 @@ connector = CallbackVoiceConnector(my_agent, turn_detection="patient")
Pick by listening to your agent's longest natural pause. Too eager and its turn ends at that pause, cutting the reply short and discarding the rest of it; too patient and the simulator sits through dead air before responding, which inflates the gaps in the recording and slows the run.
-When your transport closes each turn explicitly, silence is only the fallback: `ElevenLabsConnector`, `CallbackVoiceConnector`, and a `WebSocketConnector` given a `turn_complete_type` are all believed over a pause, so the reply is heard out in full however long your agent stops for. Only the ceiling still applies.
+When your transport closes each turn explicitly, silence is only the fallback: `ElevenLabsConnector`, `VapiConnector`, `CallbackVoiceConnector`, and a `WebSocketConnector` given a `turn_complete_type` are all believed over a pause, so the reply is heard out in full however long your agent stops for. Only the ceiling still applies.
-The preset matters more everywhere else, because silence is the only evidence there is. `LiveKitConnector` gets a bare audio track with nothing to say when a turn is over, and so does a `WebSocketConnector` whose agent sends no end-of-turn message to point at.
+The preset matters more everywhere else, because silence is the only evidence there is. `LiveKitConnector` gets a bare audio track with nothing to say when a turn is over, and so does a `WebSocketConnector` whose agent sends no end-of-turn message to point at. `PipecatConnector` falls on either side depending on your pipeline: it's believed over a pause once your pipeline has been seen to announce an end of turn, and reads silence until then.
::::note
The underlying millisecond values are chosen as a coordinated preset rather than exposed individually, for the same reason [interruption timing](/docs/conversation-simulator-voice-interruptions#when-does-it-interrupt) is: they are interdependent, and picking them apart is guesswork without knowing how the turn-taking loop uses them.
diff --git a/docs/content/docs/environment-variables.mdx b/docs/content/docs/environment-variables.mdx
index 542b571ef..049810e0a 100644
--- a/docs/content/docs/environment-variables.mdx
+++ b/docs/content/docs/environment-variables.mdx
@@ -437,4 +437,29 @@ Azure reuses `OPENAI_COST_PER_INPUT_TOKEN` and `OPENAI_COST_PER_OUTPUT_TOKEN` fo
+
+Speech models (TTS/STT)
+
+Text-to-speech and speech-to-text are two independent families used by [voice simulation](/docs/multi-turn-conversation-simulation). One `USE_*_TTS` flag and one `USE_*_STT` flag can be on at the same time, and each family falls back to OpenAI when no flag is set. Set them with [`deepeval set-tts`](/docs/command-line-interface#model-provider-configs) and `deepeval set-stt`.
+
+| Variable | Values | Effect |
+| ---------------------- | ----------------- | -------------------------------------------------------------------------------------------- |
+| `USE_OPENAI_TTS` | `1` / `0` / unset | Prefer OpenAI for text-to-speech. |
+| `USE_ELEVENLABS_TTS` | `1` / `0` / unset | Prefer ElevenLabs for text-to-speech. |
+| `USE_CARTESIA_TTS` | `1` / `0` / unset | Prefer Cartesia for text-to-speech. |
+| `USE_DEEPGRAM_TTS` | `1` / `0` / unset | Prefer Deepgram for text-to-speech. |
+| `DEEPEVAL_TTS_MODEL` | `string` / unset | Model name for the selected TTS provider (defaults to that provider's own default model). |
+| `USE_OPENAI_STT` | `1` / `0` / unset | Prefer OpenAI for speech-to-text. |
+| `USE_ELEVENLABS_STT` | `1` / `0` / unset | Prefer ElevenLabs for speech-to-text. |
+| `USE_CARTESIA_STT` | `1` / `0` / unset | Prefer Cartesia for speech-to-text. |
+| `USE_DEEPGRAM_STT` | `1` / `0` / unset | Prefer Deepgram for speech-to-text. |
+| `USE_ASSEMBLYAI_STT` | `1` / `0` / unset | Prefer AssemblyAI for speech-to-text. |
+| `DEEPEVAL_STT_MODEL` | `string` / unset | Model name for the selected STT provider (defaults to that provider's own default model). |
+| `ELEVENLABS_API_KEY` | `string` / unset | ElevenLabs API key. |
+| `CARTESIA_API_KEY` | `string` / unset | Cartesia API key. |
+| `DEEPGRAM_API_KEY` | `string` / unset | Deepgram API key. |
+| `ASSEMBLYAI_API_KEY` | `string` / unset | AssemblyAI API key. |
+
+
+
diff --git a/docs/content/integrations/index.mdx b/docs/content/integrations/index.mdx
index 1c505fb85..08f738a28 100644
--- a/docs/content/integrations/index.mdx
+++ b/docs/content/integrations/index.mdx
@@ -238,6 +238,70 @@ Evaluation model integrations configure the LLM provider DeepEval uses for LLM-a
+## Speech Models
+
+Speech model integrations configure the text-to-speech and speech-to-text providers DeepEval uses for [voice simulation](/docs/conversation-simulator-voice-mode). Every provider apart from OpenAI is reached over plain HTTP, so none of them need a package beyond `deepeval` itself.
+
+### Speech-To-Text (STT)
+
+STT transcribes your agent's spoken replies into the `Turn.content` your metrics judge, so its accuracy bounds how faithfully every multi-turn metric sees the conversation.
+
+
+
+
+
+
+
+
+
+### Text-To-Speech (TTS)
+
+TTS speaks the simulated user's messages to your agent, so its clarity is what your agent's own speech recognition has to work with.
+
+
+
+
+
+
+
+
+
+
+
+
+## Voice Agent Connectors
+
+Voice agent connectors are how DeepEval reaches the agent under test during a [voice simulation](/docs/conversation-simulator-voice-mode): they open a live session, play the simulated user's audio, and capture the spoken reply. Pick the one matching where your agent runs.
+
+
+
+
+
+
+
+
+Custom agents are covered by `WebSocketConnector` and `CallbackVoiceConnector` — see [Voice Connectors](/docs/conversation-simulator-voice-connectors).
+
+
+
+
+
## Vector DBs
Vector database integrations show how to connect retrieval systems to DeepEval so RAG metrics can evaluate the context your application actually retrieves. Use these examples to benchmark retrieval quality and end-to-end RAG behavior.
diff --git a/docs/content/integrations/meta.json b/docs/content/integrations/meta.json
index 000da5947..d765aae5e 100644
--- a/docs/content/integrations/meta.json
+++ b/docs/content/integrations/meta.json
@@ -4,6 +4,9 @@
"index",
"frameworks",
"models",
+ "stt",
+ "tts",
+ "voice-agent-connectors",
"vector-databases",
"others"
]
diff --git a/docs/content/integrations/models/amazon-bedrock.mdx b/docs/content/integrations/models/amazon-bedrock.mdx
index 007c48ccb..051eafb94 100644
--- a/docs/content/integrations/models/amazon-bedrock.mdx
+++ b/docs/content/integrations/models/amazon-bedrock.mdx
@@ -67,24 +67,6 @@ model = AmazonBedrockModel(
answer_relevancy = AnswerRelevancyMetric(model=model)
```
-
-
-
-To use any Amazon Bedrock model directly in `deepeval`, set the `USE_AWS_BEDROCK_MODEL=1` in your `env` and simply pass the name of your desired model in your metric initialization:
-
-```python
-from deepeval.metrics import AnswerRelevancyMetric
-
-answer_relevancy = AnswerRelevancyMetric(
- model="anthropic.claude-sonnet-4-5-20250929-v1:0",
-)
-```
-
-You should also set the other necessary vars like `AWS_ACCESS_KEY_ID`, `AWS_SESSION_TOKEN`, ..etc. to be able to use the Amazon Bedrock models as shown above.
-
-
-
-
There are **ZERO** mandatory and **SEVEN** optional parameters when creating an `AmazonBedrockModel`:
- [Optional] `model`: A string specifying the bedrock model identifier to call (e.g. `anthropic.claude-sonnet-4-5-20250929-v1:0`). Defaults to `AWS_BEDROCK_MODEL_NAME` if not passed; raises an error at runtime if unset.
@@ -103,6 +85,24 @@ Pass generation parameters like `temperature`, `topP`, or `maxTokens` via `gener
Extra `**kwargs` passed to `AmazonBedrockModel(...)` are forwarded to the underlying Bedrock client (aiobotocore/botocore) and are **not** treated as generation parameters.
:::
+
+
+
+To use any Amazon Bedrock model directly in `deepeval`, set the `USE_AWS_BEDROCK_MODEL=1` in your `env` and simply pass the name of your desired model in your metric initialization:
+
+```python
+from deepeval.metrics import AnswerRelevancyMetric
+
+answer_relevancy = AnswerRelevancyMetric(
+ model="anthropic.claude-sonnet-4-5-20250929-v1:0",
+)
+```
+
+You should also set the other necessary vars like `AWS_ACCESS_KEY_ID`, `AWS_SESSION_TOKEN`, ..etc. to be able to use the Amazon Bedrock models as shown above.
+
+
+
+
@@ -124,23 +124,6 @@ const model = new AmazonBedrockModel({
const answerRelevancy = new AnswerRelevancyMetric({ model });
```
-
-
-
-Every option is backed by an environment variable, so you can set `AWS_BEDROCK_MODEL_NAME`, `AWS_BEDROCK_REGION`, `AWS_ACCESS_KEY_ID`, and `AWS_SECRET_ACCESS_KEY` in your `env` and construct the model with no arguments:
-
-```typescript
-import { AnswerRelevancyMetric } from "deepeval/metrics";
-import { AmazonBedrockModel } from "deepeval/models";
-
-const answerRelevancy = new AnswerRelevancyMetric({
- model: new AmazonBedrockModel(),
-});
-```
-
-
-
-
There are **ZERO** mandatory and **EIGHT** optional parameters when creating an `AmazonBedrockModel`:
- [Optional] `model`: A string specifying the bedrock model identifier to call (e.g. `anthropic.claude-sonnet-4-5-20250929-v1:0`). Defaults to `AWS_BEDROCK_MODEL_NAME` if not passed; raises an error at runtime if unset.
@@ -164,6 +147,23 @@ const model = new AmazonBedrockModel({
Available keys vary by the Bedrock model you choose — see the [AWS Bedrock inference parameters docs](https://docs.aws.amazon.com/bedrock/latest/userguide/inference-parameters.html).
+
+
+
+Every option is backed by an environment variable, so you can set `AWS_BEDROCK_MODEL_NAME`, `AWS_BEDROCK_REGION`, `AWS_ACCESS_KEY_ID`, and `AWS_SECRET_ACCESS_KEY` in your `env` and construct the model with no arguments:
+
+```typescript
+import { AnswerRelevancyMetric } from "deepeval/metrics";
+import { AmazonBedrockModel } from "deepeval/models";
+
+const answerRelevancy = new AnswerRelevancyMetric({
+ model: new AmazonBedrockModel(),
+});
+```
+
+
+
+
diff --git a/docs/content/integrations/models/anthropic.mdx b/docs/content/integrations/models/anthropic.mdx
index 01ae99d37..a262cf8c0 100644
--- a/docs/content/integrations/models/anthropic.mdx
+++ b/docs/content/integrations/models/anthropic.mdx
@@ -44,6 +44,23 @@ model = AnthropicModel(
answer_relevancy = AnswerRelevancyMetric(model=model)
```
+There are **ZERO** mandatory and **SIX** optional parameters when creating an `AnthropicModel`:
+
+- [Optional] `model`: A string specifying which Claude model to use. Defaults to `ANTHROPIC_MODEL_NAME` if not passed; falls back to if unset.
+- [Optional] `api_key`: A string specifying your Anthropic API key. Defaults to `ANTHROPIC_API_KEY` if not passed; raises an error at runtime if unset.
+- [Optional] `temperature`: A float specifying the model temperature. Defaults to `TEMPERATURE` if not passed; falls back to `0.0` if unset and raises if < 0.
+- [Optional] `cost_per_input_token`: A float specifying the cost for each input token for the provided model. Defaults to `ANTHROPIC_COST_PER_INPUT_TOKEN` if available in `deepeval`'s model cost registry, else `None`.
+- [Optional] `cost_per_output_token`: A float specifying the cost for each output token for the provided model. Defaults to `ANTHROPIC_COST_PER_OUTPUT_TOKEN` if available in `deepeval`'s model cost registry, else `None`.
+- [Optional] `generation_kwargs`: A dictionary of additional generation parameters forwarded to the Anthropic `messages.create(...)` call.
+
+Parameters may be explicitly passed to the model at initialization time, or configured with optional settings. The **mandatory** parameters are required at runtime, but you can provide them either explicitly as constructor arguments, **or** via `deepeval` settings / environment variables (constructor args take precedence). See [Environment variables and settings](/docs/evaluation-flags-and-configs#model-settings-anthropic) for the Anthropic-related environment variables.
+
+:::tip
+Pass generation parameters, such as `max_tokens`, via `generation_kwargs` (they are forwarded to `messages.create(...)`).
+
+Extra `**kwargs` passed to `AnthropicModel(...)` are forwarded to the underlying Anthropic client and are **not** treated as generation parameters.
+:::
+
@@ -62,23 +79,6 @@ You should also set the other necessary vars like `ANTHROPIC_API_KEY` to be able
-There are **ZERO** mandatory and **SIX** optional parameters when creating an `AnthropicModel`:
-
-- [Optional] `model`: A string specifying which Claude model to use. Defaults to `ANTHROPIC_MODEL_NAME` if not passed; falls back to if unset.
-- [Optional] `api_key`: A string specifying your Anthropic API key. Defaults to `ANTHROPIC_API_KEY` if not passed; raises an error at runtime if unset.
-- [Optional] `temperature`: A float specifying the model temperature. Defaults to `TEMPERATURE` if not passed; falls back to `0.0` if unset and raises if < 0.
-- [Optional] `cost_per_input_token`: A float specifying the cost for each input token for the provided model. Defaults to `ANTHROPIC_COST_PER_INPUT_TOKEN` if available in `deepeval`'s model cost registry, else `None`.
-- [Optional] `cost_per_output_token`: A float specifying the cost for each output token for the provided model. Defaults to `ANTHROPIC_COST_PER_OUTPUT_TOKEN` if available in `deepeval`'s model cost registry, else `None`.
-- [Optional] `generation_kwargs`: A dictionary of additional generation parameters forwarded to the Anthropic `messages.create(...)` call.
-
-Parameters may be explicitly passed to the model at initialization time, or configured with optional settings. The **mandatory** parameters are required at runtime, but you can provide them either explicitly as constructor arguments, **or** via `deepeval` settings / environment variables (constructor args take precedence). See [Environment variables and settings](/docs/evaluation-flags-and-configs#model-settings-anthropic) for the Anthropic-related environment variables.
-
-:::tip
-Pass generation parameters, such as `max_tokens`, via `generation_kwargs` (they are forwarded to `messages.create(...)`).
-
-Extra `**kwargs` passed to `AnthropicModel(...)` are forwarded to the underlying Anthropic client and are **not** treated as generation parameters.
-:::
-
@@ -99,23 +99,6 @@ const model = new AnthropicModel({
const answerRelevancy = new AnswerRelevancyMetric({ model });
```
-
-
-
-Every option is backed by an environment variable, so you can set `ANTHROPIC_MODEL_NAME` and `ANTHROPIC_API_KEY` in your `env` and construct the model with no arguments:
-
-```typescript
-import { AnswerRelevancyMetric } from "deepeval/metrics";
-import { AnthropicModel } from "deepeval/models";
-
-const answerRelevancy = new AnswerRelevancyMetric({
- model: new AnthropicModel(),
-});
-```
-
-
-
-
There are **ZERO** mandatory and **SIX** optional parameters when creating an `AnthropicModel`:
- [Optional] `model`: A string specifying which Claude model to use. Defaults to `ANTHROPIC_MODEL_NAME` if not passed; falls back to if unset.
@@ -135,6 +118,23 @@ const model = new AnthropicModel({
});
```
+
+
+
+Every option is backed by an environment variable, so you can set `ANTHROPIC_MODEL_NAME` and `ANTHROPIC_API_KEY` in your `env` and construct the model with no arguments:
+
+```typescript
+import { AnswerRelevancyMetric } from "deepeval/metrics";
+import { AnthropicModel } from "deepeval/models";
+
+const answerRelevancy = new AnswerRelevancyMetric({
+ model: new AnthropicModel(),
+});
+```
+
+
+
+
diff --git a/docs/content/integrations/models/azure-openai.mdx b/docs/content/integrations/models/azure-openai.mdx
index 331f9a8c0..1af706dde 100644
--- a/docs/content/integrations/models/azure-openai.mdx
+++ b/docs/content/integrations/models/azure-openai.mdx
@@ -64,6 +64,22 @@ model = AzureOpenAIModel(
answer_relevancy = AnswerRelevancyMetric(model=model)
```
+There are **ZERO** mandatory and **NINE** optional parameters when creating an `AzureOpenAIModel`:
+
+- [Optional] `model`: A string specifying the name of the Azure OpenAI model to use. Defaults to `AZURE_MODEL_NAME` if not passed; raises an error at runtime if unset.
+- [Optional] `api_key`: A string specifying your Azure OpenAI API key. Defaults to `AZURE_OPENAI_API_KEY` if not passed; raises an error at runtime if `azure_ad_token` and `azure_ad_token_provider` are also unset.
+- [Optional] `azure_ad_token`: A string specifying your Azure Ad Token. Defaults to `AZURE_OPENAI_AD_TOKEN` if not passed; raises an error at runtime if `api_key` and `azure_ad_token_provider` are also unset.
+- [Optional] `azure_ad_token_provider`: A callback of either `AsyncAzureADTokenProvider` or `AzureADTokenProvider` that can be used for credentials [(see example usage)](https://github.com/openai/openai-python/blob/main/examples/azure_ad.py#L20). Raises an error at runtime if `api_key` and `azure_ad_token` are also unset.
+- [Optional] `base_url`: A string specifying your Azure OpenAI endpoint URL. Defaults to `AZURE_OPENAI_ENDPOINT` if not passed; raises an error at runtime if unset.
+- [Optional] `temperature`: A float specifying the model temperature. Defaults to `TEMPERATURE` if not passed; falls back to `0.0` if unset.
+- [Optional] `cost_per_input_token`: A float specifying the cost for each input token for the provided model. Defaults to `OPENAI_COST_PER_INPUT_TOKEN` if available in `deepeval`'s model cost registry, else `None`.
+- [Optional] `cost_per_output_token`: A float specifying the cost for each output token for the provided model. Defaults to `OPENAI_COST_PER_OUTPUT_TOKEN` if available in `deepeval`'s model cost registry, else `None`.
+- [Optional] `deployment_name`: A string specifying the name of your Azure OpenAI deployment. Defaults to `AZURE_DEPLOYMENT_NAME` if not passed; raises an error at runtime if unset.
+- [Optional] `api_version`: A string specifying the OpenAI API version used in your deployment. Defaults to `OPENAI_API_VERSION` if not passed; raises an error at runtime if unset.
+- [Optional] `generation_kwargs`: A dictionary of additional generation parameters forwarded to the Azure OpenAI `chat.completions.create(...)` and `beta.chat.completions.parse(...)` calls.
+
+Parameters may be explicitly passed to the model at initialization time, or configured with optional settings. The **mandatory** parameters are required at runtime, but you can provide them either explicitly as constructor arguments, **or** via `deepeval` settings / environment variables (constructor args take precedence). See [Environment variables and settings](/docs/evaluation-flags-and-configs#model-settings-azure-openai) for the Azure OpenAI-related environment variables.
+
@@ -82,22 +98,6 @@ You should also set the other necessary vars like `AZURE_OPENAI_API_KEY` to be a
-There are **ZERO** mandatory and **NINE** optional parameters when creating an `AzureOpenAIModel`:
-
-- [Optional] `model`: A string specifying the name of the Azure OpenAI model to use. Defaults to `AZURE_MODEL_NAME` if not passed; raises an error at runtime if unset.
-- [Optional] `api_key`: A string specifying your Azure OpenAI API key. Defaults to `AZURE_OPENAI_API_KEY` if not passed; raises an error at runtime if `azure_ad_token` and `azure_ad_token_provider` are also unset.
-- [Optional] `azure_ad_token`: A string specifying your Azure Ad Token. Defaults to `AZURE_OPENAI_AD_TOKEN` if not passed; raises an error at runtime if `api_key` and `azure_ad_token_provider` are also unset.
-- [Optional] `azure_ad_token_provider`: A callback of either `AsyncAzureADTokenProvider` or `AzureADTokenProvider` that can be used for credentials [(see example usage)](https://github.com/openai/openai-python/blob/main/examples/azure_ad.py#L20). Raises an error at runtime if `api_key` and `azure_ad_token` are also unset.
-- [Optional] `base_url`: A string specifying your Azure OpenAI endpoint URL. Defaults to `AZURE_OPENAI_ENDPOINT` if not passed; raises an error at runtime if unset.
-- [Optional] `temperature`: A float specifying the model temperature. Defaults to `TEMPERATURE` if not passed; falls back to `0.0` if unset.
-- [Optional] `cost_per_input_token`: A float specifying the cost for each input token for the provided model. Defaults to `OPENAI_COST_PER_INPUT_TOKEN` if available in `deepeval`'s model cost registry, else `None`.
-- [Optional] `cost_per_output_token`: A float specifying the cost for each output token for the provided model. Defaults to `OPENAI_COST_PER_OUTPUT_TOKEN` if available in `deepeval`'s model cost registry, else `None`.
-- [Optional] `deployment_name`: A string specifying the name of your Azure OpenAI deployment. Defaults to `AZURE_DEPLOYMENT_NAME` if not passed; raises an error at runtime if unset.
-- [Optional] `api_version`: A string specifying the OpenAI API version used in your deployment. Defaults to `OPENAI_API_VERSION` if not passed; raises an error at runtime if unset.
-- [Optional] `generation_kwargs`: A dictionary of additional generation parameters forwarded to the Azure OpenAI `chat.completions.create(...)` and `beta.chat.completions.parse(...)` calls.
-
-Parameters may be explicitly passed to the model at initialization time, or configured with optional settings. The **mandatory** parameters are required at runtime, but you can provide them either explicitly as constructor arguments, **or** via `deepeval` settings / environment variables (constructor args take precedence). See [Environment variables and settings](/docs/evaluation-flags-and-configs#model-settings-azure-openai) for the Azure OpenAI-related environment variables.
-
@@ -121,6 +121,27 @@ const model = new AzureOpenAIModel({
const answerRelevancy = new AnswerRelevancyMetric({ model });
```
+There are **ZERO** mandatory and **EIGHT** optional parameters when creating an `AzureOpenAIModel`:
+
+- [Optional] `model`: A string specifying the name of the Azure OpenAI model your deployment serves. Used to look up pricing and capabilities in `deepeval`'s model registry, and used as the deployment name when `deployment` is not passed.
+- [Optional] `deployment`: A string specifying the name of your Azure OpenAI deployment, which requests are routed by. Defaults to `model`, then `AZURE_DEPLOYMENT_NAME`, then `AZURE_MODEL_NAME` if not passed; raises an error if unset.
+- [Optional] `endpoint`: A string specifying your Azure OpenAI endpoint URL. Defaults to `AZURE_OPENAI_ENDPOINT` if not passed; raises an error if unset.
+- [Optional] `apiKey`: A string specifying your Azure OpenAI API key. Defaults to `AZURE_OPENAI_API_KEY` if not passed; raises an error at runtime if unset.
+- [Optional] `apiVersion`: A string specifying the OpenAI API version used in your deployment. Defaults to `OPENAI_API_VERSION` if not passed.
+- [Optional] `temperature`: A number specifying the model temperature. Defaults to `0`. Pass `null` to omit it from the request entirely, which is required by reasoning models that reject it.
+- [Optional] `costPerInputToken`: A number specifying the cost for each input token for the provided model. Defaults to the price in `deepeval`'s model cost registry, else `null`.
+- [Optional] `costPerOutputToken`: A number specifying the cost for each output token for the provided model. Defaults to the price in `deepeval`'s model cost registry, else `null`.
+
+Any other key you pass is forwarded as-is to the Azure OpenAI `chat.completions.create(...)` call, so generation parameters `deepeval` does not name go inline alongside the rest:
+
+```typescript
+const model = new AzureOpenAIModel({
+ model: "gpt-4.1",
+ top_p: 0.9,
+ seed: 42,
+});
+```
+
@@ -149,27 +170,6 @@ const answerRelevancy = new AnswerRelevancyMetric({
-There are **ZERO** mandatory and **EIGHT** optional parameters when creating an `AzureOpenAIModel`:
-
-- [Optional] `model`: A string specifying the name of the Azure OpenAI model your deployment serves. Used to look up pricing and capabilities in `deepeval`'s model registry, and used as the deployment name when `deployment` is not passed.
-- [Optional] `deployment`: A string specifying the name of your Azure OpenAI deployment, which requests are routed by. Defaults to `model`, then `AZURE_DEPLOYMENT_NAME`, then `AZURE_MODEL_NAME` if not passed; raises an error if unset.
-- [Optional] `endpoint`: A string specifying your Azure OpenAI endpoint URL. Defaults to `AZURE_OPENAI_ENDPOINT` if not passed; raises an error if unset.
-- [Optional] `apiKey`: A string specifying your Azure OpenAI API key. Defaults to `AZURE_OPENAI_API_KEY` if not passed; raises an error at runtime if unset.
-- [Optional] `apiVersion`: A string specifying the OpenAI API version used in your deployment. Defaults to `OPENAI_API_VERSION` if not passed.
-- [Optional] `temperature`: A number specifying the model temperature. Defaults to `0`. Pass `null` to omit it from the request entirely, which is required by reasoning models that reject it.
-- [Optional] `costPerInputToken`: A number specifying the cost for each input token for the provided model. Defaults to the price in `deepeval`'s model cost registry, else `null`.
-- [Optional] `costPerOutputToken`: A number specifying the cost for each output token for the provided model. Defaults to the price in `deepeval`'s model cost registry, else `null`.
-
-Any other key you pass is forwarded as-is to the Azure OpenAI `chat.completions.create(...)` call, so generation parameters `deepeval` does not name go inline alongside the rest:
-
-```typescript
-const model = new AzureOpenAIModel({
- model: "gpt-4.1",
- top_p: 0.9,
- seed: 42,
-});
-```
-
diff --git a/docs/content/integrations/models/deepseek.mdx b/docs/content/integrations/models/deepseek.mdx
index 74a3b0f70..b4a3698ea 100644
--- a/docs/content/integrations/models/deepseek.mdx
+++ b/docs/content/integrations/models/deepseek.mdx
@@ -51,6 +51,17 @@ model = DeepSeekModel(
answer_relevancy = AnswerRelevancyMetric(model=model)
```
+There are **ZERO** mandatory and **SIX** optional parameters when creating a `DeepSeekModel`:
+
+- [Optional] `model`: A string specifying the name of the DeepSeek model to use. Defaults to `DEEPSEEK_MODEL_NAME` if not passed; raises an error at runtime if unset.
+- [Optional] `api_key`: A string specifying your DeepSeek API key for authentication. Defaults to `DEEPSEEK_API_KEY` if not passed; raises an error at runtime if unset.
+- [Optional] `temperature`: A float specifying the model temperature. Defaults to `TEMPERATURE` if not passed; falls back to `0.0` if unset.
+- [Optional] `cost_per_input_token`: A float specifying the cost for each input token for the provided model. Defaults to `DEEPSEEK_COST_PER_INPUT_TOKEN` if available in `deepeval`'s model cost registry, else `None`.
+- [Optional] `cost_per_output_token`: A float specifying the cost for each output token for the provided model. Defaults to `DEEPSEEK_COST_PER_OUTPUT_TOKEN` if available in `deepeval`'s model cost registry, else `None`.
+- [Optional] `generation_kwargs`: A dictionary of additional generation forwarded to the OpenAI `chat.completions.create(...)` call.
+
+Parameters may be explicitly passed to the model at initialization time, or configured with optional settings. The **mandatory** parameters are required at runtime, but you can provide them either explicitly as constructor arguments, **or** via `deepeval` settings / environment variables (constructor args take precedence). See [Environment variables and settings](/docs/evaluation-flags-and-configs#model-settings-deep-seek) for the DeepSeek-related environment variables.
+
@@ -69,17 +80,6 @@ You should also set the other necessary vars like `DEEPSEEK_API_KEY` to be able
-There are **ZERO** mandatory and **SIX** optional parameters when creating a `DeepSeekModel`:
-
-- [Optional] `model`: A string specifying the name of the DeepSeek model to use. Defaults to `DEEPSEEK_MODEL_NAME` if not passed; raises an error at runtime if unset.
-- [Optional] `api_key`: A string specifying your DeepSeek API key for authentication. Defaults to `DEEPSEEK_API_KEY` if not passed; raises an error at runtime if unset.
-- [Optional] `temperature`: A float specifying the model temperature. Defaults to `TEMPERATURE` if not passed; falls back to `0.0` if unset.
-- [Optional] `cost_per_input_token`: A float specifying the cost for each input token for the provided model. Defaults to `DEEPSEEK_COST_PER_INPUT_TOKEN` if available in `deepeval`'s model cost registry, else `None`.
-- [Optional] `cost_per_output_token`: A float specifying the cost for each output token for the provided model. Defaults to `DEEPSEEK_COST_PER_OUTPUT_TOKEN` if available in `deepeval`'s model cost registry, else `None`.
-- [Optional] `generation_kwargs`: A dictionary of additional generation forwarded to the OpenAI `chat.completions.create(...)` call.
-
-Parameters may be explicitly passed to the model at initialization time, or configured with optional settings. The **mandatory** parameters are required at runtime, but you can provide them either explicitly as constructor arguments, **or** via `deepeval` settings / environment variables (constructor args take precedence). See [Environment variables and settings](/docs/evaluation-flags-and-configs#model-settings-deep-seek) for the DeepSeek-related environment variables.
-
@@ -100,23 +100,6 @@ const model = new DeepSeekModel({
const answerRelevancy = new AnswerRelevancyMetric({ model });
```
-
-
-
-To use any DeepSeek model directly in `deepeval`, set `DEEPSEEK_MODEL_NAME` and `DEEPSEEK_API_KEY` in your `env` and create a `DeepSeekModel` with no arguments:
-
-```typescript
-import { AnswerRelevancyMetric } from "deepeval/metrics";
-import { DeepSeekModel } from "deepeval/models";
-
-const answerRelevancy = new AnswerRelevancyMetric({
- model: new DeepSeekModel(),
-});
-```
-
-
-
-
There are **ZERO** mandatory and **SEVEN** optional parameters when creating a `DeepSeekModel`:
- [Optional] `model`: A string specifying the name of the DeepSeek model to use. Defaults to `DEEPSEEK_MODEL_NAME` if not passed; falls back to `deepseek-chat` if unset.
@@ -137,6 +120,23 @@ const model = new DeepSeekModel({
});
```
+
+
+
+To use any DeepSeek model directly in `deepeval`, set `DEEPSEEK_MODEL_NAME` and `DEEPSEEK_API_KEY` in your `env` and create a `DeepSeekModel` with no arguments:
+
+```typescript
+import { AnswerRelevancyMetric } from "deepeval/metrics";
+import { DeepSeekModel } from "deepeval/models";
+
+const answerRelevancy = new AnswerRelevancyMetric({
+ model: new DeepSeekModel(),
+});
+```
+
+
+
+
diff --git a/docs/content/integrations/models/gemini.mdx b/docs/content/integrations/models/gemini.mdx
index 490ce9230..807ebd676 100644
--- a/docs/content/integrations/models/gemini.mdx
+++ b/docs/content/integrations/models/gemini.mdx
@@ -56,6 +56,21 @@ model = GeminiModel(
answer_relevancy = AnswerRelevancyMetric(model=model)
```
+There are **ZERO** mandatory and **SIX** optional parameters when creating a `GeminiModel`:
+
+- [Optional] `model`: A string specifying the name of the Gemini model to use. Defaults to `GEMINI_MODEL_NAME` if not passed; raises an error at runtime if unset.
+- [Optional] `api_key`: A string specifying the Google API key for authentication. Defaults to `GOOGLE_API_KEY` if not passed; raises an error at runtime if unset.
+- [Optional] `temperature`: A float specifying the model temperature. Defaults to `TEMPERATURE` if not passed; falls back to `0.0` if unset.
+- [Optional] `cost_per_input_token`: A float specifying the cost per input token for the provided model. Defaults to `GEMINI_COST_PER_INPUT_TOKEN` if set; falls back to `deepeval`'s model cost registry, else `None`.
+- [Optional] `cost_per_output_token`: A float specifying the cost per output token for the provided model. Defaults to `GEMINI_COST_PER_OUTPUT_TOKEN` if set; falls back to `deepeval`'s model cost registry, else `None`.
+- [Optional] `generation_kwargs`: A dictionary of additional generation parameters forwarded to the Gemini API `generate_content(...)` call.
+
+Parameters may be explicitly passed to the model at initialization time, or configured with optional settings. The **mandatory** parameters are required at runtime, but you can provide them either explicitly as constructor arguments, **or** via `deepeval` settings / environment variables (constructor args take precedence). See [Environment variables and settings](/docs/evaluation-flags-and-configs#model-settings-gemini) for the Gemini-related environment variables.
+
+:::note
+At runtime, you must provide an API key (via `api_key` or `GOOGLE_API_KEY`) unless you’re using Vertex AI. See [Vertex AI](/docs/integrations/models/vertex-ai).
+:::
+
@@ -74,21 +89,6 @@ You should also set the other necessary vars like `GOOGLE_API_KEY` to be able to
-There are **ZERO** mandatory and **SIX** optional parameters when creating a `GeminiModel`:
-
-- [Optional] `model`: A string specifying the name of the Gemini model to use. Defaults to `GEMINI_MODEL_NAME` if not passed; raises an error at runtime if unset.
-- [Optional] `api_key`: A string specifying the Google API key for authentication. Defaults to `GOOGLE_API_KEY` if not passed; raises an error at runtime if unset.
-- [Optional] `temperature`: A float specifying the model temperature. Defaults to `TEMPERATURE` if not passed; falls back to `0.0` if unset.
-- [Optional] `cost_per_input_token`: A float specifying the cost per input token for the provided model. Defaults to `GEMINI_COST_PER_INPUT_TOKEN` if set; falls back to `deepeval`'s model cost registry, else `None`.
-- [Optional] `cost_per_output_token`: A float specifying the cost per output token for the provided model. Defaults to `GEMINI_COST_PER_OUTPUT_TOKEN` if set; falls back to `deepeval`'s model cost registry, else `None`.
-- [Optional] `generation_kwargs`: A dictionary of additional generation parameters forwarded to the Gemini API `generate_content(...)` call.
-
-Parameters may be explicitly passed to the model at initialization time, or configured with optional settings. The **mandatory** parameters are required at runtime, but you can provide them either explicitly as constructor arguments, **or** via `deepeval` settings / environment variables (constructor args take precedence). See [Environment variables and settings](/docs/evaluation-flags-and-configs#model-settings-gemini) for the Gemini-related environment variables.
-
-:::note
-At runtime, you must provide an API key (via `api_key` or `GOOGLE_API_KEY`) unless you’re using Vertex AI. See [Vertex AI](/docs/integrations/models/vertex-ai).
-:::
-
@@ -113,23 +113,6 @@ const model = new GeminiModel({
const answerRelevancy = new AnswerRelevancyMetric({ model });
```
-
-
-
-Every option is backed by an environment variable, so you can set `GEMINI_MODEL_NAME` and `GOOGLE_API_KEY` in your `env` and construct the model with no arguments:
-
-```typescript
-import { AnswerRelevancyMetric } from "deepeval/metrics";
-import { GeminiModel } from "deepeval/models";
-
-const answerRelevancy = new AnswerRelevancyMetric({
- model: new GeminiModel(),
-});
-```
-
-
-
-
There are **ZERO** mandatory and **EIGHT** optional parameters when creating a `GeminiModel`:
- [Optional] `model`: A string specifying the name of the Gemini model to use. Defaults to `GEMINI_MODEL_NAME` if not passed; falls back to if unset.
@@ -155,6 +138,23 @@ const model = new GeminiModel({
At runtime, you must provide an API key (via `apiKey`, `GOOGLE_API_KEY`, or `GEMINI_API_KEY`) unless you’re using Vertex AI. See [Vertex AI](/docs/integrations/models/vertex-ai).
:::
+
+
+
+Every option is backed by an environment variable, so you can set `GEMINI_MODEL_NAME` and `GOOGLE_API_KEY` in your `env` and construct the model with no arguments:
+
+```typescript
+import { AnswerRelevancyMetric } from "deepeval/metrics";
+import { GeminiModel } from "deepeval/models";
+
+const answerRelevancy = new AnswerRelevancyMetric({
+ model: new GeminiModel(),
+});
+```
+
+
+
+
diff --git a/docs/content/integrations/models/grok.mdx b/docs/content/integrations/models/grok.mdx
index d090b6200..3b5c8c4f5 100644
--- a/docs/content/integrations/models/grok.mdx
+++ b/docs/content/integrations/models/grok.mdx
@@ -74,6 +74,15 @@ model = GrokModel(
answer_relevancy = AnswerRelevancyMetric(model=model)
```
+There are **ZERO** mandatory and **SIX** optional parameters when creating a `GrokModel`:
+
+- [Optional] `model`: A string specifying the name of the Grok model to use. Defaults to `GROK_MODEL_NAME` if not passed; raises an error at runtime if unset.
+- [Optional] `api_key`: A string specifying your Grok API key for authentication. Defaults to `GROK_API_KEY` if not passed; raises an error at runtime if unset.
+- [Optional] `temperature`: A float specifying the model temperature. Defaults to `TEMPERATURE` if not passed; falls back to `0.0` if unset.
+- [Optional] `cost_per_input_token`: A float specifying the cost for each input token for the provided model. Defaults to `GROK_COST_PER_INPUT_TOKEN` if available in `deepeval`'s model cost registry, else `None`.
+- [Optional] `cost_per_output_token`: A float specifying the cost for each output token for the provided model. Defaults to `GROK_COST_PER_OUTPUT_TOKEN` if available in `deepeval`'s model cost registry, else `None`.
+- [Optional] `generation_kwargs`: A dictionary of additional generation parameters forwarded to the xAI SDK `client.chat.create(...)` call.
+
@@ -92,15 +101,6 @@ You should also set the other necessary vars like `GROK_API_KEY` to be able to u
-There are **ZERO** mandatory and **SIX** optional parameters when creating a `GrokModel`:
-
-- [Optional] `model`: A string specifying the name of the Grok model to use. Defaults to `GROK_MODEL_NAME` if not passed; raises an error at runtime if unset.
-- [Optional] `api_key`: A string specifying your Grok API key for authentication. Defaults to `GROK_API_KEY` if not passed; raises an error at runtime if unset.
-- [Optional] `temperature`: A float specifying the model temperature. Defaults to `TEMPERATURE` if not passed; falls back to `0.0` if unset.
-- [Optional] `cost_per_input_token`: A float specifying the cost for each input token for the provided model. Defaults to `GROK_COST_PER_INPUT_TOKEN` if available in `deepeval`'s model cost registry, else `None`.
-- [Optional] `cost_per_output_token`: A float specifying the cost for each output token for the provided model. Defaults to `GROK_COST_PER_OUTPUT_TOKEN` if available in `deepeval`'s model cost registry, else `None`.
-- [Optional] `generation_kwargs`: A dictionary of additional generation parameters forwarded to the xAI SDK `client.chat.create(...)` call.
-
@@ -121,23 +121,6 @@ const model = new GrokModel({
const answerRelevancy = new AnswerRelevancyMetric({ model });
```
-
-
-
-To use any Grok model directly in `deepeval`, set `GROK_MODEL_NAME` and `GROK_API_KEY` in your `env` and create a `GrokModel` with no arguments:
-
-```typescript
-import { AnswerRelevancyMetric } from "deepeval/metrics";
-import { GrokModel } from "deepeval/models";
-
-const answerRelevancy = new AnswerRelevancyMetric({
- model: new GrokModel(),
-});
-```
-
-
-
-
There are **ZERO** mandatory and **SEVEN** optional parameters when creating a `GrokModel`:
- [Optional] `model`: A string specifying the name of the Grok model to use. Defaults to `GROK_MODEL_NAME` if not passed; falls back to `grok-3` if unset.
@@ -158,6 +141,23 @@ const model = new GrokModel({
});
```
+
+
+
+To use any Grok model directly in `deepeval`, set `GROK_MODEL_NAME` and `GROK_API_KEY` in your `env` and create a `GrokModel` with no arguments:
+
+```typescript
+import { AnswerRelevancyMetric } from "deepeval/metrics";
+import { GrokModel } from "deepeval/models";
+
+const answerRelevancy = new AnswerRelevancyMetric({
+ model: new GrokModel(),
+});
+```
+
+
+
+
diff --git a/docs/content/integrations/models/litellm.mdx b/docs/content/integrations/models/litellm.mdx
index dbffa11ca..9693c3162 100644
--- a/docs/content/integrations/models/litellm.mdx
+++ b/docs/content/integrations/models/litellm.mdx
@@ -74,6 +74,20 @@ model = LiteLLMModel(
answer_relevancy = AnswerRelevancyMetric(model=model)
```
+There are **ZERO** mandatory and **SEVEN** optional parameters when creating a `LiteLLMModel`:
+
+- [Optional] `model`: A string specifying the provider and model name (e.g. `openai/gpt-3.5-turbo`, `anthropic/claude-3-opus`). Defaults to `LITELLM_MODEL_NAME` if not passed; raises an error at runtime if unset.
+- [Optional] `api_key`: A string specifying the API key for the model. If not passed, `deepeval` attempts (in order) `LITELLM_API_KEY`, `LITELLM_PROXY_API_KEY`, `OPENAI_API_KEY`, `ANTHROPIC_API_KEY`, then `GOOGLE_API_KEY`. If none are set, the key is left unset and the underlying LiteLLM/provider behavior applies.
+- [Optional] `base_url`: A string specifying the base URL for the model API. Defaults to `LITELLM_API_BASE`, then `LITELLM_PROXY_API_BASE` if not passed.
+- [Optional] `temperature`: A float specifying the model temperature. Defaults to `TEMPERATURE` if not passed; falls back to `0.0` if unset.
+- [Optional] `cost_per_input_token`: A float specifying the cost for each input token for the provided model. Defaults to `None` if not passed; when unset, the cost is taken from LiteLLM's response if it reports one, otherwise reported as unknown.
+- [Optional] `cost_per_output_token`: A float specifying the cost for each output token for the provided model. Defaults to `None` if not passed; when unset, the cost is taken from LiteLLM's response if it reports one, otherwise reported as unknown.
+- [Optional] `generation_kwargs`: A dictionary of additional generation parameters forwarded to LiteLLM's `completion(...)` / `acompletion(...)` call.
+
+:::tip
+Any `**kwargs` you would like to use for your model can be passed through the `generation_kwargs` parameter. However, we request you to double check the params supported by the model and your model provider in their [official docs](https://docs.litellm.ai/docs/providers/custom_llm_server).
+:::
+
@@ -92,20 +106,6 @@ You should also set the other necessary vars like `LITELLM_API_KEY` to be able t
-There are **ZERO** mandatory and **SEVEN** optional parameters when creating a `LiteLLMModel`:
-
-- [Optional] `model`: A string specifying the provider and model name (e.g. `openai/gpt-3.5-turbo`, `anthropic/claude-3-opus`). Defaults to `LITELLM_MODEL_NAME` if not passed; raises an error at runtime if unset.
-- [Optional] `api_key`: A string specifying the API key for the model. If not passed, `deepeval` attempts (in order) `LITELLM_API_KEY`, `LITELLM_PROXY_API_KEY`, `OPENAI_API_KEY`, `ANTHROPIC_API_KEY`, then `GOOGLE_API_KEY`. If none are set, the key is left unset and the underlying LiteLLM/provider behavior applies.
-- [Optional] `base_url`: A string specifying the base URL for the model API. Defaults to `LITELLM_API_BASE`, then `LITELLM_PROXY_API_BASE` if not passed.
-- [Optional] `temperature`: A float specifying the model temperature. Defaults to `TEMPERATURE` if not passed; falls back to `0.0` if unset.
-- [Optional] `cost_per_input_token`: A float specifying the cost for each input token for the provided model. Defaults to `None` if not passed; when unset, the cost is taken from LiteLLM's response if it reports one, otherwise reported as unknown.
-- [Optional] `cost_per_output_token`: A float specifying the cost for each output token for the provided model. Defaults to `None` if not passed; when unset, the cost is taken from LiteLLM's response if it reports one, otherwise reported as unknown.
-- [Optional] `generation_kwargs`: A dictionary of additional generation parameters forwarded to LiteLLM's `completion(...)` / `acompletion(...)` call.
-
-:::tip
-Any `**kwargs` you would like to use for your model can be passed through the `generation_kwargs` parameter. However, we request you to double check the params supported by the model and your model provider in their [official docs](https://docs.litellm.ai/docs/providers/custom_llm_server).
-:::
-
### Available Models
:::note
diff --git a/docs/content/integrations/models/lmstudio.mdx b/docs/content/integrations/models/lmstudio.mdx
index 508d95d77..8a911b5fc 100644
--- a/docs/content/integrations/models/lmstudio.mdx
+++ b/docs/content/integrations/models/lmstudio.mdx
@@ -54,6 +54,19 @@ model = LocalModel(
answer_relevancy = AnswerRelevancyMetric(model=model)
```
+There are **ZERO** mandatory and **SIX** optional parameters when creating a `LocalModel`:
+
+- [Optional] `model`: A string specifying the local model to use. Defaults to `LOCAL_MODEL_NAME` if not passed; raises an error at runtime if unset.
+- [Optional] `api_key`: A string specifying the API key for your local server. Defaults to `LOCAL_MODEL_API_KEY` if not passed; raises an error at runtime if unset. Local servers without authentication accept any placeholder string.
+- [Optional] `base_url`: A string specifying the base URL of your local server. Defaults to `LOCAL_MODEL_BASE_URL` if not passed.
+- [Optional] `temperature`: A float specifying the model temperature. Defaults to `TEMPERATURE` if not passed; falls back to `0.0` if unset.
+- [Optional] `format`: A string specifying the structured-output response format. Defaults to `LOCAL_MODEL_FORMAT` if not passed; falls back to `"json"` if unset.
+- [Optional] `generation_kwargs`: A dictionary of additional generation parameters forwarded to the local server's `chat.completions.create(...)` call.
+
+:::tip
+Any `**kwargs` you would like to use for your model can be passed directly to `LocalModel(...)`; these are forwarded to the underlying OpenAI client constructor.
+:::
+
@@ -72,19 +85,6 @@ You should also set the other necessary vars like `LOCAL_MODEL_BASE_URL` and `LO
-There are **ZERO** mandatory and **SIX** optional parameters when creating a `LocalModel`:
-
-- [Optional] `model`: A string specifying the local model to use. Defaults to `LOCAL_MODEL_NAME` if not passed; raises an error at runtime if unset.
-- [Optional] `api_key`: A string specifying the API key for your local server. Defaults to `LOCAL_MODEL_API_KEY` if not passed; raises an error at runtime if unset. Local servers without authentication accept any placeholder string.
-- [Optional] `base_url`: A string specifying the base URL of your local server. Defaults to `LOCAL_MODEL_BASE_URL` if not passed.
-- [Optional] `temperature`: A float specifying the model temperature. Defaults to `TEMPERATURE` if not passed; falls back to `0.0` if unset.
-- [Optional] `format`: A string specifying the structured-output response format. Defaults to `LOCAL_MODEL_FORMAT` if not passed; falls back to `"json"` if unset.
-- [Optional] `generation_kwargs`: A dictionary of additional generation parameters forwarded to the local server's `chat.completions.create(...)` call.
-
-:::tip
-Any `**kwargs` you would like to use for your model can be passed directly to `LocalModel(...)`; these are forwarded to the underlying OpenAI client constructor.
-:::
-
@@ -106,6 +106,27 @@ const model = new LocalModel({
const answerRelevancy = new AnswerRelevancyMetric({ model });
```
+There are **ZERO** mandatory and **SEVEN** optional parameters when creating a `LocalModel`:
+
+- [Optional] `model`: A string specifying the local model to use. Defaults to `LOCAL_MODEL_NAME` if not passed; raises an error if unset.
+- [Optional] `apiKey`: A string specifying the API key for your local server. Defaults to `LOCAL_MODEL_API_KEY` if not passed; falls back to a placeholder string, so local servers without authentication work out of the box.
+- [Optional] `baseURL`: A string specifying the base URL of your local server. Defaults to `LOCAL_MODEL_BASE_URL` if not passed; raises an error if unset.
+- [Optional] `temperature`: A number specifying the model temperature. Defaults to `0`. Pass `null` to omit it from the request entirely, which is required by reasoning models that reject it.
+- [Optional] `defaultHeaders`: An object of extra HTTP headers sent with every request.
+- [Optional] `costPerInputToken`: A number specifying the cost for each input token for the provided model. Defaults to the price in `deepeval`'s model cost registry, else `null`.
+- [Optional] `costPerOutputToken`: A number specifying the cost for each output token for the provided model. Defaults to the price in `deepeval`'s model cost registry, else `null`.
+
+Any other key you pass is forwarded as-is to the local server's `chat.completions.create(...)` call, so generation parameters `deepeval` does not name go inline alongside the rest:
+
+```typescript
+const model = new LocalModel({
+ model: "",
+ baseURL: "http://localhost:1234/v1/",
+ top_p: 0.9,
+ max_tokens: 512,
+});
+```
+
@@ -132,27 +153,6 @@ const answerRelevancy = new AnswerRelevancyMetric({
-There are **ZERO** mandatory and **SEVEN** optional parameters when creating a `LocalModel`:
-
-- [Optional] `model`: A string specifying the local model to use. Defaults to `LOCAL_MODEL_NAME` if not passed; raises an error if unset.
-- [Optional] `apiKey`: A string specifying the API key for your local server. Defaults to `LOCAL_MODEL_API_KEY` if not passed; falls back to a placeholder string, so local servers without authentication work out of the box.
-- [Optional] `baseURL`: A string specifying the base URL of your local server. Defaults to `LOCAL_MODEL_BASE_URL` if not passed; raises an error if unset.
-- [Optional] `temperature`: A number specifying the model temperature. Defaults to `0`. Pass `null` to omit it from the request entirely, which is required by reasoning models that reject it.
-- [Optional] `defaultHeaders`: An object of extra HTTP headers sent with every request.
-- [Optional] `costPerInputToken`: A number specifying the cost for each input token for the provided model. Defaults to the price in `deepeval`'s model cost registry, else `null`.
-- [Optional] `costPerOutputToken`: A number specifying the cost for each output token for the provided model. Defaults to the price in `deepeval`'s model cost registry, else `null`.
-
-Any other key you pass is forwarded as-is to the local server's `chat.completions.create(...)` call, so generation parameters `deepeval` does not name go inline alongside the rest:
-
-```typescript
-const model = new LocalModel({
- model: "",
- baseURL: "http://localhost:1234/v1/",
- top_p: 0.9,
- max_tokens: 512,
-});
-```
-
diff --git a/docs/content/integrations/models/moonshot.mdx b/docs/content/integrations/models/moonshot.mdx
index ff33f5f95..ba63c0751 100644
--- a/docs/content/integrations/models/moonshot.mdx
+++ b/docs/content/integrations/models/moonshot.mdx
@@ -55,6 +55,15 @@ model = KimiModel(
answer_relevancy = AnswerRelevancyMetric(model=model)
```
+There are **ZERO** mandatory and **SIX** optional parameters when creating an `KimiModel`:
+
+- [Optional] `model`: A string specifying the name of the Kimi model to use. Defaults to `MOONSHOT_MODEL_NAME` if not passed; raises an error at runtime if unset.
+- [Optional] `api_key`: A string specifying your Kimi API key for authentication. Defaults to `MOONSHOT_API_KEY` if not passed; raises an error at runtime if unset.
+- [Optional] `temperature`: A float specifying the model temperature. Defaults to `TEMPERATURE` if not passed; falls back to `0.0` if unset and raises if < 0.
+- [Optional] `cost_per_input_token`: A float specifying the cost for each input token for the provided model. Defaults to `MOONSHOT_COST_PER_INPUT_TOKEN` if available in `deepeval`'s model cost registry, else `None`.
+- [Optional] `cost_per_output_token`: A float specifying the cost for each output token for the provided model. Defaults to `MOONSHOT_COST_PER_OUTPUT_TOKEN` if available in `deepeval`'s model cost registry, else `None`.
+- [Optional] `generation_kwargs`: A dictionary of additional generation parameters forwarded to the OpenAI `chat.completions.create(...)` call.
+
@@ -73,15 +82,6 @@ You should also set the other necessary vars like `MOONSHOT_API_KEY` to be able
-There are **ZERO** mandatory and **SIX** optional parameters when creating an `KimiModel`:
-
-- [Optional] `model`: A string specifying the name of the Kimi model to use. Defaults to `MOONSHOT_MODEL_NAME` if not passed; raises an error at runtime if unset.
-- [Optional] `api_key`: A string specifying your Kimi API key for authentication. Defaults to `MOONSHOT_API_KEY` if not passed; raises an error at runtime if unset.
-- [Optional] `temperature`: A float specifying the model temperature. Defaults to `TEMPERATURE` if not passed; falls back to `0.0` if unset and raises if < 0.
-- [Optional] `cost_per_input_token`: A float specifying the cost for each input token for the provided model. Defaults to `MOONSHOT_COST_PER_INPUT_TOKEN` if available in `deepeval`'s model cost registry, else `None`.
-- [Optional] `cost_per_output_token`: A float specifying the cost for each output token for the provided model. Defaults to `MOONSHOT_COST_PER_OUTPUT_TOKEN` if available in `deepeval`'s model cost registry, else `None`.
-- [Optional] `generation_kwargs`: A dictionary of additional generation parameters forwarded to the OpenAI `chat.completions.create(...)` call.
-
@@ -102,23 +102,6 @@ const model = new KimiModel({
const answerRelevancy = new AnswerRelevancyMetric({ model });
```
-
-
-
-To use any Moonshot model directly in `deepeval`, set `MOONSHOT_MODEL_NAME` and `MOONSHOT_API_KEY` in your `env` and create a `KimiModel` with no arguments:
-
-```typescript
-import { AnswerRelevancyMetric } from "deepeval/metrics";
-import { KimiModel } from "deepeval/models";
-
-const answerRelevancy = new AnswerRelevancyMetric({
- model: new KimiModel(),
-});
-```
-
-
-
-
There are **ZERO** mandatory and **SEVEN** optional parameters when creating a `KimiModel`:
- [Optional] `model`: A string specifying the name of the Kimi model to use. Defaults to `MOONSHOT_MODEL_NAME` if not passed; falls back to `moonshot-v1-8k` if unset.
@@ -139,6 +122,23 @@ const model = new KimiModel({
});
```
+
+
+
+To use any Moonshot model directly in `deepeval`, set `MOONSHOT_MODEL_NAME` and `MOONSHOT_API_KEY` in your `env` and create a `KimiModel` with no arguments:
+
+```typescript
+import { AnswerRelevancyMetric } from "deepeval/metrics";
+import { KimiModel } from "deepeval/models";
+
+const answerRelevancy = new AnswerRelevancyMetric({
+ model: new KimiModel(),
+});
+```
+
+
+
+
diff --git a/docs/content/integrations/models/ollama.mdx b/docs/content/integrations/models/ollama.mdx
index c4f6f0f94..c445e6754 100644
--- a/docs/content/integrations/models/ollama.mdx
+++ b/docs/content/integrations/models/ollama.mdx
@@ -81,7 +81,15 @@ answer_relevancy = AnswerRelevancyMetric(model=model)
-To use any Ollama model directly in `deepeval`, set the `LOCAL_MODEL_API_KEY` in your `env` and simply pass the name of your desired model in your metric initialization:
+First point `deepeval` at Ollama. `LOCAL_MODEL_API_KEY` must be the literal string `ollama` — that value is what routes a plain model name to your Ollama server instead of a hosted provider:
+
+```bash
+# .env.local
+LOCAL_MODEL_API_KEY=ollama
+LOCAL_MODEL_BASE_URL=http://localhost:11434
+```
+
+Then pass the name of your desired model as a string in your metric initialization:
```python
from deepeval.metrics import AnswerRelevancyMetric
@@ -91,6 +99,8 @@ answer_relevancy = AnswerRelevancyMetric(
)
```
+Running `deepeval set-ollama --model=deepseek-r1:1.5b --save` writes these same variables for you.
+
@@ -121,31 +131,6 @@ const model = new OllamaModel({
const answerRelevancy = new AnswerRelevancyMetric({ model });
```
-
-
-
-`deepeval` autoloads `.env.local` then `.env` at import time, so you can keep your Ollama configuration out of your code entirely:
-
-```bash
-# .env.local
-OLLAMA_MODEL_NAME=deepseek-r1:1.5b
-LOCAL_MODEL_BASE_URL=http://localhost:11434
-```
-
-Then create the model with no arguments and pass it to any metric:
-
-```typescript
-import { AnswerRelevancyMetric } from "deepeval/metrics";
-import { OllamaModel } from "deepeval/models";
-
-const answerRelevancy = new AnswerRelevancyMetric({
- model: new OllamaModel(),
-});
-```
-
-
-
-
There are **ZERO** mandatory and **FIVE** optional parameters when creating an `OllamaModel`:
- [Optional] `model`: A string specifying the name of the Ollama model to use. Defaults to `OLLAMA_MODEL_NAME` if not passed; raises an error at runtime if unset.
@@ -164,6 +149,32 @@ const model = new OllamaModel({
});
```
+
+
+
+First point `deepeval` at Ollama. `LOCAL_MODEL_API_KEY` must be the literal string `ollama` — that value is what routes a plain model name to your Ollama server instead of a hosted provider:
+
+```bash
+# .env.local
+LOCAL_MODEL_API_KEY=ollama
+LOCAL_MODEL_BASE_URL=http://localhost:11434
+```
+
+Then pass the name of your desired model as a string in your metric initialization:
+
+```typescript
+import { AnswerRelevancyMetric } from "deepeval/metrics";
+
+const answerRelevancy = new AnswerRelevancyMetric({
+ model: "deepseek-r1:1.5b",
+});
+```
+
+Running `deepeval set-ollama --model=deepseek-r1:1.5b --save` writes these same variables for you.
+
+
+
+
diff --git a/docs/content/integrations/models/openai.mdx b/docs/content/integrations/models/openai.mdx
index 501c5e98f..09ba585c2 100644
--- a/docs/content/integrations/models/openai.mdx
+++ b/docs/content/integrations/models/openai.mdx
@@ -18,13 +18,13 @@ DeepEval autoloads `.env.local` then `.env` at import time (process env -> `.env
OPENAI_API_KEY=
```
-Alternative (Shell/CI)
+**Alternative (Shell/CI):**
```bash
export OPENAI_API_KEY=
```
-Alternative (notebook)
+**Alternative (notebook):**
If you're working in a notebook environment (Jupyter or Colab), set your `OPENAI_API_KEY` in a cell:
@@ -85,6 +85,16 @@ model = OpenAIModel(
answer_relevancy = AnswerRelevancyMetric(model=model)
```
+There are **ZERO** mandatory and **SEVEN** optional parameters when creating an `OpenAIModel`:
+
+- [Optional] `model`: A string specifying the name of the GPT model to use. Defaulted to `OPENAI_MODEL_NAME` if not set; falls back to .
+- [Optional] `api_key`: A string specifying the OpenAI API key for authentication. Defaults to `OPENAI_API_KEY` if not passed; raises an error at runtime if unset.
+- [Optional] `base_url`: A string specifying your OpenAI URL.
+- [Optional] `temperature`: A float specifying the model temperature. Defaults to `TEMPERATURE` if not passed; falls back to `0.0` if unset.
+- [Optional] `cost_per_input_token`: A float specifying the cost for each input token for the provided model. Defaults to `OPENAI_COST_PER_INPUT_TOKEN` if available in `deepeval`'s model cost registry, else `None`.
+- [Optional] `cost_per_output_token`: A float specifying the cost for each output token for the provided model. Defaults to `OPENAI_COST_PER_OUTPUT_TOKEN` if available in `deepeval`'s model cost registry, else `None`.
+- [Optional] `generation_kwargs`: A dictionary of additional generation parameters forwarded to the OpenAI `chat.completions.create(...)` and `beta.chat.completions.parse(...)` calls.
+
@@ -101,16 +111,6 @@ answer_relevancy = AnswerRelevancyMetric(
-There are **ZERO** mandatory and **SEVEN** optional parameters when creating an `OpenAIModel`:
-
-- [Optional] `model`: A string specifying the name of the GPT model to use. Defaulted to `OPENAI_MODEL_NAME` if not set; falls back to .
-- [Optional] `api_key`: A string specifying the OpenAI API key for authentication. Defaults to `OPENAI_API_KEY` if not passed; raises an error at runtime if unset.
-- [Optional] `base_url`: A string specifying your OpenAI URL.
-- [Optional] `temperature`: A float specifying the model temperature. Defaults to `TEMPERATURE` if not passed; falls back to `0.0` if unset.
-- [Optional] `cost_per_input_token`: A float specifying the cost for each input token for the provided model. Defaults to `OPENAI_COST_PER_INPUT_TOKEN` if available in `deepeval`'s model cost registry, else `None`.
-- [Optional] `cost_per_output_token`: A float specifying the cost for each output token for the provided model. Defaults to `OPENAI_COST_PER_OUTPUT_TOKEN` if available in `deepeval`'s model cost registry, else `None`.
-- [Optional] `generation_kwargs`: A dictionary of additional generation parameters forwarded to the OpenAI `chat.completions.create(...)` and `beta.chat.completions.parse(...)` calls.
-
@@ -131,22 +131,6 @@ const model = new OpenAIModel({
const answerRelevancy = new AnswerRelevancyMetric({ model });
```
-
-
-
-`deepeval` by default uses OpenAI models for evaluations, you can simply pass the name of your desired model in metric initialization and set the `OPENAI_API_KEY` to use OpenAI models:
-
-```typescript
-import { AnswerRelevancyMetric } from "deepeval/metrics";
-
-const answerRelevancy = new AnswerRelevancyMetric({
- model: "gpt-4.1",
-});
-```
-
-
-
-
There are **ZERO** mandatory and **SEVEN** optional parameters when creating an `OpenAIModel`:
- [Optional] `model`: A string specifying the name of the OpenAI model to use. Defaulted to `OPENAI_MODEL_NAME` if not set; falls back to .
@@ -168,6 +152,22 @@ const model = new OpenAIModel({
});
```
+
+
+
+`deepeval` by default uses OpenAI models for evaluations, you can simply pass the name of your desired model in metric initialization and set the `OPENAI_API_KEY` to use OpenAI models:
+
+```typescript
+import { AnswerRelevancyMetric } from "deepeval/metrics";
+
+const answerRelevancy = new AnswerRelevancyMetric({
+ model: "gpt-4.1",
+});
+```
+
+
+
+
diff --git a/docs/content/integrations/models/openrouter.mdx b/docs/content/integrations/models/openrouter.mdx
index 3eea98e1d..647950ba3 100644
--- a/docs/content/integrations/models/openrouter.mdx
+++ b/docs/content/integrations/models/openrouter.mdx
@@ -62,24 +62,6 @@ model = OpenRouterModel(
answer_relevancy = AnswerRelevancyMetric(model=model)
```
-
-
-
-To use any OpenRouter model directly in `deepeval`, set the `USE_OPENROUTER_MODEL=1` in your `env` and simply pass the name of your desired model in your metric initialization:
-
-```python
-from deepeval.metrics import AnswerRelevancyMetric
-
-answer_relevancy = AnswerRelevancyMetric(
- model="openai/gpt-4.1",
-)
-```
-
-You should also set the other necessary vars like `OPENROUTER_API_KEY` to be able to use the OpenRouter models as shown above.
-
-
-
-
There are **ZERO** mandatory and **SEVEN** optional parameters when creating an `OpenRouterModel`:
- [Optional] `model`: A string specifying the OpenRouter model to use. Defaults to `OPENROUTER_MODEL_NAME` if set; otherwise falls back to .
@@ -107,6 +89,24 @@ model = OpenRouterModel(
```
:::
+
+
+
+To use any OpenRouter model directly in `deepeval`, set the `USE_OPENROUTER_MODEL=1` in your `env` and simply pass the name of your desired model in your metric initialization:
+
+```python
+from deepeval.metrics import AnswerRelevancyMetric
+
+answer_relevancy = AnswerRelevancyMetric(
+ model="openai/gpt-4.1",
+)
+```
+
+You should also set the other necessary vars like `OPENROUTER_API_KEY` to be able to use the OpenRouter models as shown above.
+
+
+
+
@@ -133,23 +133,6 @@ const model = new OpenRouterModel({
const answerRelevancy = new AnswerRelevancyMetric({ model });
```
-
-
-
-To use any OpenRouter model directly in `deepeval`, set `OPENROUTER_MODEL_NAME` and `OPENROUTER_API_KEY` in your `env` and create an `OpenRouterModel` with no arguments:
-
-```typescript
-import { AnswerRelevancyMetric } from "deepeval/metrics";
-import { OpenRouterModel } from "deepeval/models";
-
-const answerRelevancy = new AnswerRelevancyMetric({
- model: new OpenRouterModel(),
-});
-```
-
-
-
-
There are **ZERO** mandatory and **SEVEN** optional parameters when creating an `OpenRouterModel`:
- [Optional] `model`: A string specifying the OpenRouter model to use. Defaults to `OPENROUTER_MODEL_NAME` if set; otherwise falls back to .
@@ -187,6 +170,23 @@ const model = new OpenRouterModel({
```
:::
+
+
+
+To use any OpenRouter model directly in `deepeval`, set `OPENROUTER_MODEL_NAME` and `OPENROUTER_API_KEY` in your `env` and create an `OpenRouterModel` with no arguments:
+
+```typescript
+import { AnswerRelevancyMetric } from "deepeval/metrics";
+import { OpenRouterModel } from "deepeval/models";
+
+const answerRelevancy = new AnswerRelevancyMetric({
+ model: new OpenRouterModel(),
+});
+```
+
+
+
+
diff --git a/docs/content/integrations/models/portkey.mdx b/docs/content/integrations/models/portkey.mdx
index b5d9ced95..165da9167 100644
--- a/docs/content/integrations/models/portkey.mdx
+++ b/docs/content/integrations/models/portkey.mdx
@@ -58,6 +58,21 @@ model = PortkeyModel(
answer_relevancy = AnswerRelevancyMetric(model=model)
```
+There are **ZERO** mandatory and **SIX** optional parameters when creating a `PortkeyModel`:
+
+- [Optional] `model`: A string specifying the name of the Portkey model to use. Defaults to `PORTKEY_MODEL_NAME` if not passed; raises an error at runtime if unset.
+- [Optional] `api_key`: A string specifying your Portkey API key for authentication. Defaults to `PORTKEY_API_KEY` if not passed; raises an error at runtime if unset.
+- [Optional] `provider`: A string specifying the Portkey provider of your model. Defaults to `PORTKEY_PROVIDER_NAME` if not passed; raises an error at runtime if unset.
+- [Optional] `base_url`: A string specifying the base URL for the model API. Defaults to `PORTKEY_BASE_URL` if not passed; raises an error at runtime if unset.
+- [Optional] `temperature`: A float specifying the model temperature. Defaults to `TEMPERATURE` if not passed; falls back to `0.0` if unset.
+- [Optional] `generation_kwargs`: A dictionary of additional generation parameters forwarded to Portkey's `chat.completions.create(...)` call.
+
+Any additional `**kwargs` you would like to use for your `Portkey` client can be passed directly to `PortkeyModel(...)`. These are forwarded to the underlying OpenAI client constructor. We recommend double-checking the parameters supported by your chosen model and provider in the [official Portkey docs](https://portkey.ai/docs).
+
+:::tip
+Pass generation parameters such as `max_tokens` via `generation_kwargs` (they are forwarded to `chat.completions.create(...)`).
+:::
+
@@ -76,21 +91,6 @@ You should also set the other necessary vars like `PORTKEY_API_KEY` to be able t
-There are **ZERO** mandatory and **SIX** optional parameters when creating a `PortkeyModel`:
-
-- [Optional] `model`: A string specifying the name of the Portkey model to use. Defaults to `PORTKEY_MODEL_NAME` if not passed; raises an error at runtime if unset.
-- [Optional] `api_key`: A string specifying your Portkey API key for authentication. Defaults to `PORTKEY_API_KEY` if not passed; raises an error at runtime if unset.
-- [Optional] `provider`: A string specifying the Portkey provider of your model. Defaults to `PORTKEY_PROVIDER_NAME` if not passed; raises an error at runtime if unset.
-- [Optional] `base_url`: A string specifying the base URL for the model API. Defaults to `PORTKEY_BASE_URL` if not passed; raises an error at runtime if unset.
-- [Optional] `temperature`: A float specifying the model temperature. Defaults to `TEMPERATURE` if not passed; falls back to `0.0` if unset.
-- [Optional] `generation_kwargs`: A dictionary of additional generation parameters forwarded to Portkey's `chat.completions.create(...)` call.
-
-Any additional `**kwargs` you would like to use for your `Portkey` client can be passed directly to `PortkeyModel(...)`. These are forwarded to the underlying OpenAI client constructor. We recommend double-checking the parameters supported by your chosen model and provider in the [official Portkey docs](https://portkey.ai/docs).
-
-:::tip
-Pass generation parameters such as `max_tokens` via `generation_kwargs` (they are forwarded to `chat.completions.create(...)`).
-:::
-
@@ -112,23 +112,6 @@ const model = new PortkeyModel({
const answerRelevancy = new AnswerRelevancyMetric({ model });
```
-
-
-
-To use any Portkey model directly in `deepeval`, set `PORTKEY_MODEL_NAME`, `PORTKEY_PROVIDER_NAME`, and `PORTKEY_API_KEY` in your `env` and create a `PortkeyModel` with no arguments:
-
-```typescript
-import { AnswerRelevancyMetric } from "deepeval/metrics";
-import { PortkeyModel } from "deepeval/models";
-
-const answerRelevancy = new AnswerRelevancyMetric({
- model: new PortkeyModel(),
-});
-```
-
-
-
-
There are **ZERO** mandatory and **EIGHT** optional parameters when creating a `PortkeyModel`:
- [Optional] `model`: A string specifying the name of the Portkey model to use. Defaults to `PORTKEY_MODEL_NAME` if not passed; raises an error at runtime if unset.
@@ -152,6 +135,23 @@ const model = new PortkeyModel({
We recommend double-checking the parameters supported by your chosen model and provider in the [official Portkey docs](https://portkey.ai/docs).
+
+
+
+To use any Portkey model directly in `deepeval`, set `PORTKEY_MODEL_NAME`, `PORTKEY_PROVIDER_NAME`, and `PORTKEY_API_KEY` in your `env` and create a `PortkeyModel` with no arguments:
+
+```typescript
+import { AnswerRelevancyMetric } from "deepeval/metrics";
+import { PortkeyModel } from "deepeval/models";
+
+const answerRelevancy = new AnswerRelevancyMetric({
+ model: new PortkeyModel(),
+});
+```
+
+
+
+
diff --git a/docs/content/integrations/models/vllm.mdx b/docs/content/integrations/models/vllm.mdx
index 49efc8d1d..47720b48a 100644
--- a/docs/content/integrations/models/vllm.mdx
+++ b/docs/content/integrations/models/vllm.mdx
@@ -52,6 +52,19 @@ model = LocalModel(
answer_relevancy = AnswerRelevancyMetric(model=model)
```
+There are **ZERO** mandatory and **SIX** optional parameters when creating a `LocalModel`:
+
+- [Optional] `model`: A string specifying the local model to use. Defaults to `LOCAL_MODEL_NAME` if not passed; raises an error at runtime if unset.
+- [Optional] `api_key`: A string specifying the API key for your local server. Defaults to `LOCAL_MODEL_API_KEY` if not passed; raises an error at runtime if unset. Local servers without authentication accept any placeholder string.
+- [Optional] `base_url`: A string specifying the base URL of your local server. Defaults to `LOCAL_MODEL_BASE_URL` if not passed.
+- [Optional] `temperature`: A float specifying the model temperature. Defaults to `TEMPERATURE` if not passed; falls back to `0.0` if unset.
+- [Optional] `format`: A string specifying the structured-output response format. Defaults to `LOCAL_MODEL_FORMAT` if not passed; falls back to `"json"` if unset.
+- [Optional] `generation_kwargs`: A dictionary of additional generation parameters forwarded to the local server's `chat.completions.create(...)` call.
+
+:::tip
+Any `**kwargs` you would like to use for your model can be passed directly to `LocalModel(...)`; these are forwarded to the underlying OpenAI client constructor.
+:::
+
@@ -70,19 +83,6 @@ You should also set the other necessary vars like `LOCAL_MODEL_BASE_URL` and `LO
-There are **ZERO** mandatory and **SIX** optional parameters when creating a `LocalModel`:
-
-- [Optional] `model`: A string specifying the local model to use. Defaults to `LOCAL_MODEL_NAME` if not passed; raises an error at runtime if unset.
-- [Optional] `api_key`: A string specifying the API key for your local server. Defaults to `LOCAL_MODEL_API_KEY` if not passed; raises an error at runtime if unset. Local servers without authentication accept any placeholder string.
-- [Optional] `base_url`: A string specifying the base URL of your local server. Defaults to `LOCAL_MODEL_BASE_URL` if not passed.
-- [Optional] `temperature`: A float specifying the model temperature. Defaults to `TEMPERATURE` if not passed; falls back to `0.0` if unset.
-- [Optional] `format`: A string specifying the structured-output response format. Defaults to `LOCAL_MODEL_FORMAT` if not passed; falls back to `"json"` if unset.
-- [Optional] `generation_kwargs`: A dictionary of additional generation parameters forwarded to the local server's `chat.completions.create(...)` call.
-
-:::tip
-Any `**kwargs` you would like to use for your model can be passed directly to `LocalModel(...)`; these are forwarded to the underlying OpenAI client constructor.
-:::
-
@@ -104,6 +104,27 @@ const model = new LocalModel({
const answerRelevancy = new AnswerRelevancyMetric({ model });
```
+There are **ZERO** mandatory and **SEVEN** optional parameters when creating a `LocalModel`:
+
+- [Optional] `model`: A string specifying the local model to use. Defaults to `LOCAL_MODEL_NAME` if not passed; raises an error if unset.
+- [Optional] `apiKey`: A string specifying the API key for your local server. Defaults to `LOCAL_MODEL_API_KEY` if not passed; falls back to a placeholder string, so local servers without authentication work out of the box.
+- [Optional] `baseURL`: A string specifying the base URL of your local server. Defaults to `LOCAL_MODEL_BASE_URL` if not passed; raises an error if unset.
+- [Optional] `temperature`: A number specifying the model temperature. Defaults to `0`. Pass `null` to omit it from the request entirely, which is required by reasoning models that reject it.
+- [Optional] `defaultHeaders`: An object of extra HTTP headers sent with every request.
+- [Optional] `costPerInputToken`: A number specifying the cost for each input token for the provided model. Defaults to the price in `deepeval`'s model cost registry, else `null`.
+- [Optional] `costPerOutputToken`: A number specifying the cost for each output token for the provided model. Defaults to the price in `deepeval`'s model cost registry, else `null`.
+
+Any other key you pass is forwarded as-is to the local server's `chat.completions.create(...)` call, so generation parameters `deepeval` does not name go inline alongside the rest:
+
+```typescript
+const model = new LocalModel({
+ model: "",
+ baseURL: "http://localhost:8000/v1/",
+ top_p: 0.9,
+ max_tokens: 512,
+});
+```
+
@@ -130,27 +151,6 @@ const answerRelevancy = new AnswerRelevancyMetric({
-There are **ZERO** mandatory and **SEVEN** optional parameters when creating a `LocalModel`:
-
-- [Optional] `model`: A string specifying the local model to use. Defaults to `LOCAL_MODEL_NAME` if not passed; raises an error if unset.
-- [Optional] `apiKey`: A string specifying the API key for your local server. Defaults to `LOCAL_MODEL_API_KEY` if not passed; falls back to a placeholder string, so local servers without authentication work out of the box.
-- [Optional] `baseURL`: A string specifying the base URL of your local server. Defaults to `LOCAL_MODEL_BASE_URL` if not passed; raises an error if unset.
-- [Optional] `temperature`: A number specifying the model temperature. Defaults to `0`. Pass `null` to omit it from the request entirely, which is required by reasoning models that reject it.
-- [Optional] `defaultHeaders`: An object of extra HTTP headers sent with every request.
-- [Optional] `costPerInputToken`: A number specifying the cost for each input token for the provided model. Defaults to the price in `deepeval`'s model cost registry, else `null`.
-- [Optional] `costPerOutputToken`: A number specifying the cost for each output token for the provided model. Defaults to the price in `deepeval`'s model cost registry, else `null`.
-
-Any other key you pass is forwarded as-is to the local server's `chat.completions.create(...)` call, so generation parameters `deepeval` does not name go inline alongside the rest:
-
-```typescript
-const model = new LocalModel({
- model: "",
- baseURL: "http://localhost:8000/v1/",
- top_p: 0.9,
- max_tokens: 512,
-});
-```
-
diff --git a/docs/content/integrations/stt/assemblyai.mdx b/docs/content/integrations/stt/assemblyai.mdx
new file mode 100644
index 000000000..c4d822d30
--- /dev/null
+++ b/docs/content/integrations/stt/assemblyai.mdx
@@ -0,0 +1,135 @@
+---
+title: AssemblyAI STT
+sidebar_label: AssemblyAI
+languages: [python]
+---
+
+In `deepeval`, you can use AssemblyAI to transcribe speech into text when running [voice simulations](/docs/conversation-simulator-voice-mode). Transcription is what turns your agent's spoken replies into the `Turn.content` your multi-turn metrics judge, so its accuracy directly bounds how faithfully those metrics see the conversation.
+
+:::info
+AssemblyAI is reached over plain HTTP rather than through a vendor SDK, so `pip install deepeval` is all you need — there is no extra package to install or version to pin.
+:::
+
+### Setting Up Your API Key
+
+DeepEval autoloads `.env.local` then `.env` at import time (process env -> `.env.local` -> `.env`).
+
+**Recommended (local dev):**
+
+```bash
+# .env.local
+ASSEMBLYAI_API_KEY=
+```
+
+**Alternative (Shell/CI):**
+
+```bash
+export ASSEMBLYAI_API_KEY=
+```
+
+**Alternative (notebook):**
+
+If you're working in a notebook environment (Jupyter or Colab), set your `ASSEMBLYAI_API_KEY` in a cell:
+
+```bash
+%env ASSEMBLYAI_API_KEY=
+```
+
+### Command Line
+
+To transcribe with AssemblyAI in every voice simulation, run:
+
+```bash
+deepeval set-stt assemblyai --model=universal-3-5-pro
+```
+
+`--model` is optional and defaults to AssemblyAI's own default model.
+
+:::info
+The command above sets AssemblyAI as the default speech-to-text provider, unless overridden in Python code. To use a different one, run `deepeval set-stt` for that provider, or drop back to the OpenAI default:
+
+```bash
+deepeval unset-stt
+```
+
+:::
+
+:::tip[Persisting settings]
+You can persist CLI settings with the optional `--save` flag.
+See [Flags and Configs -> Persisting CLI settings](/docs/evaluation-flags-and-configs#persisting-cli-settings-with---save).
+:::
+
+### In Code
+
+Pass an `AssemblyAISTTModel` to `VoiceConfig` to transcribe with AssemblyAI instead of the default OpenAI model:
+
+
+
+
+```python
+from deepeval.models import AssemblyAISTTModel
+from deepeval.voice import VoiceConfig
+
+stt_model = AssemblyAISTTModel(
+ model="universal-3-5-pro",
+ language="en",
+)
+voice_config = VoiceConfig(
+ stt_model=stt_model,
+ ...,
+)
+```
+
+There are **ZERO** mandatory and **EIGHT** optional parameters when creating an `AssemblyAISTTModel`:
+
+- [Optional] `model`: A string specifying the name of the transcription model to use. Defaulted to `universal-3-5-pro`.
+- [Optional] `api_key`: A string specifying your AssemblyAI API key. Defaults to `ASSEMBLYAI_API_KEY` if not passed; raises an error at runtime if neither is set.
+- [Optional] `base_url`: A string specifying a custom endpoint to reach the AssemblyAI API through. Defaulted to `https://api.assemblyai.com`.
+- [Optional] `language`: A string [ISO-639-1](https://en.wikipedia.org/wiki/List_of_ISO_639_language_codes) code (e.g. `"en"`) pinning the spoken language. Pass `"auto"` to have AssemblyAI detect it instead. Defaulted to `None`.
+- [Optional] `cost_per_hour`: A float overriding the hourly price used for cost accounting. Defaults to the built-in price for whichever endpoint the clip took.
+- [Optional] `transcription_kwargs`: A dictionary of additional parameters forwarded to AssemblyAI's transcription request.
+- [Optional] `timeout`: A float number of seconds to wait on a request before giving up. Defaulted to `120.0`.
+- [Optional] `poll_interval_seconds`: A float number of seconds between status checks on the upload-and-poll path. Defaulted to `1.0`, and never consulted for clips short enough to take the synchronous endpoint.
+
+:::tip
+Any AssemblyAI parameter `deepeval` has no field for — `speaker_labels`, `keyterms_prompt`, `redact_pii` — can be passed through `transcription_kwargs` and is sent as-is.
+:::
+
+
+
+
+First set your API key and select AssemblyAI as the transcription provider:
+
+```bash
+# .env.local
+ASSEMBLYAI_API_KEY=
+USE_ASSEMBLYAI_STT=1
+DEEPEVAL_STT_MODEL=universal-3-5-pro
+```
+
+`deepeval set-stt assemblyai --model universal-3-5-pro` writes those same variables for you. `DEEPEVAL_STT_MODEL` is optional: leave it out and transcription runs on `universal-3-5-pro` anyway, since that is AssemblyAI's default here.
+
+You can also pass the model name in code, which wins over `DEEPEVAL_STT_MODEL`:
+
+```python
+from deepeval.voice import VoiceConfig
+
+voice_config = VoiceConfig(
+ stt_model="universal-3-5-pro",
+ ...,
+)
+```
+
+With `USE_ASSEMBLYAI_STT` set, a bare model name is shorthand for `AssemblyAISTTModel(model=...)`. Pass the model object itself to pin a language.
+
+
+
+
+### Available AssemblyAI Models
+
+- `universal-3-5-pro` (default, replaces the deprecated `slam-1`)
+- `universal-2`
+
+:::info
+AssemblyAI has a synchronous endpoint that answers with a finished transcript in a single request, and an older upload-create-poll flow that takes three. `deepeval` prefers the synchronous one, which is dramatically faster and is what every conversation turn will use in practice. It caps clips at 120 seconds, so audio longer than that — or audio whose duration can't be measured — falls back to upload-and-poll automatically.
+:::
diff --git a/docs/content/integrations/stt/cartesia.mdx b/docs/content/integrations/stt/cartesia.mdx
new file mode 100644
index 000000000..5ae213c12
--- /dev/null
+++ b/docs/content/integrations/stt/cartesia.mdx
@@ -0,0 +1,132 @@
+---
+title: Cartesia STT
+sidebar_label: Cartesia
+languages: [python]
+---
+
+In `deepeval`, you can use Cartesia to transcribe speech into text when running [voice simulations](/docs/conversation-simulator-voice-mode). Transcription is what turns your agent's spoken replies into the `Turn.content` your multi-turn metrics judge, so its accuracy directly bounds how faithfully those metrics see the conversation.
+
+:::info
+Cartesia is reached over plain HTTP rather than through a vendor SDK, so `pip install deepeval` is all you need — there is no extra package to install or version to pin.
+:::
+
+### Setting Up Your API Key
+
+DeepEval autoloads `.env.local` then `.env` at import time (process env -> `.env.local` -> `.env`).
+
+**Recommended (local dev):**
+
+```bash
+# .env.local
+CARTESIA_API_KEY=
+```
+
+**Alternative (Shell/CI):**
+
+```bash
+export CARTESIA_API_KEY=
+```
+
+**Alternative (notebook):**
+
+If you're working in a notebook environment (Jupyter or Colab), set your `CARTESIA_API_KEY` in a cell:
+
+```bash
+%env CARTESIA_API_KEY=
+```
+
+### Command Line
+
+To transcribe with Cartesia in every voice simulation, run:
+
+```bash
+deepeval set-stt cartesia --model=ink-whisper
+```
+
+`--model` is optional and defaults to Cartesia's own default model.
+
+:::info
+The command above sets Cartesia as the default speech-to-text provider, unless overridden in Python code. To use a different one, run `deepeval set-stt` for that provider, or drop back to the OpenAI default:
+
+```bash
+deepeval unset-stt
+```
+
+:::
+
+:::tip[Persisting settings]
+You can persist CLI settings with the optional `--save` flag.
+See [Flags and Configs -> Persisting CLI settings](/docs/evaluation-flags-and-configs#persisting-cli-settings-with---save).
+:::
+
+### In Code
+
+Pass a `CartesiaSTTModel` to `VoiceConfig` to transcribe with Ink instead of the default OpenAI model:
+
+
+
+
+```python
+from deepeval.models import CartesiaSTTModel
+from deepeval.voice import VoiceConfig
+
+stt_model = CartesiaSTTModel(
+ model="ink-whisper",
+ language="en",
+)
+voice_config = VoiceConfig(
+ stt_model=stt_model,
+ ...,
+)
+```
+
+There are **ZERO** mandatory and **EIGHT** optional parameters when creating a `CartesiaSTTModel`:
+
+- [Optional] `model`: A string specifying the name of the transcription model to use. Defaulted to `ink-whisper`.
+- [Optional] `api_key`: A string specifying your Cartesia API key. Defaults to `CARTESIA_API_KEY` if not passed; raises an error at runtime if neither is set.
+- [Optional] `base_url`: A string specifying a custom endpoint to reach the Cartesia API through. Defaulted to `https://api.cartesia.ai`.
+- [Optional] `language`: A string [ISO-639-1](https://en.wikipedia.org/wiki/List_of_ISO_639_language_codes) code (e.g. `"en"`) pinning the spoken language. Defaulted to `None`, which Cartesia reads as English.
+- [Optional] `api_version`: A string dating the Cartesia API to call, sent on every request. Defaulted to `2026-08-14`, pinned so a server-side release can't change the response shape underneath a run.
+- [Optional] `cost_per_minute`: A float setting the per-minute price used for cost accounting. Defaulted to `None`.
+- [Optional] `transcription_kwargs`: A dictionary of additional multipart fields forwarded to Cartesia's `/stt` endpoint.
+- [Optional] `timeout`: A float number of seconds to wait on a request before giving up. Defaulted to `120.0`.
+
+:::tip
+Any Cartesia parameter `deepeval` has no field for can be passed through `transcription_kwargs` and is sent as-is.
+:::
+
+
+
+
+First set your API key and select Cartesia as the transcription provider:
+
+```bash
+# .env.local
+CARTESIA_API_KEY=
+USE_CARTESIA_STT=1
+DEEPEVAL_STT_MODEL=ink-whisper
+```
+
+`deepeval set-stt cartesia --model ink-whisper` writes those same variables for you. `DEEPEVAL_STT_MODEL` is optional: leave it out and transcription runs on `ink-whisper` anyway, since that is Cartesia's default here.
+
+You can also pass the model name in code, which wins over `DEEPEVAL_STT_MODEL`:
+
+```python
+from deepeval.voice import VoiceConfig
+
+voice_config = VoiceConfig(
+ stt_model="ink-whisper",
+ ...,
+)
+```
+
+With `USE_CARTESIA_STT` set, a bare model name is shorthand for `CartesiaSTTModel(model=...)`. Pass the model object itself to pin a language.
+
+
+
+
+### Available Cartesia Models
+
+- `ink-whisper` (default)
+
+Cartesia's batch endpoint only serves the `ink-whisper` family. `ink-2` is realtime-only and will not transcribe here.
diff --git a/docs/content/integrations/stt/deepgram.mdx b/docs/content/integrations/stt/deepgram.mdx
new file mode 100644
index 000000000..055077d8f
--- /dev/null
+++ b/docs/content/integrations/stt/deepgram.mdx
@@ -0,0 +1,133 @@
+---
+title: Deepgram STT
+sidebar_label: Deepgram
+languages: [python]
+---
+
+In `deepeval`, you can use Deepgram to transcribe speech into text when running [voice simulations](/docs/conversation-simulator-voice-mode). Transcription is what turns your agent's spoken replies into the `Turn.content` your multi-turn metrics judge, so its accuracy directly bounds how faithfully those metrics see the conversation.
+
+:::info
+Deepgram is reached over plain HTTP rather than through a vendor SDK, so `pip install deepeval` is all you need — there is no extra package to install or version to pin.
+:::
+
+### Setting Up Your API Key
+
+DeepEval autoloads `.env.local` then `.env` at import time (process env -> `.env.local` -> `.env`).
+
+**Recommended (local dev):**
+
+```bash
+# .env.local
+DEEPGRAM_API_KEY=
+```
+
+**Alternative (Shell/CI):**
+
+```bash
+export DEEPGRAM_API_KEY=
+```
+
+**Alternative (notebook):**
+
+If you're working in a notebook environment (Jupyter or Colab), set your `DEEPGRAM_API_KEY` in a cell:
+
+```bash
+%env DEEPGRAM_API_KEY=
+```
+
+### Command Line
+
+To transcribe with Deepgram in every voice simulation, run:
+
+```bash
+deepeval set-stt deepgram --model=nova-3
+```
+
+`--model` is optional and defaults to Deepgram's own default model.
+
+:::info
+The command above sets Deepgram as the default speech-to-text provider, unless overridden in Python code. To use a different one, run `deepeval set-stt` for that provider, or drop back to the OpenAI default:
+
+```bash
+deepeval unset-stt
+```
+
+:::
+
+:::tip[Persisting settings]
+You can persist CLI settings with the optional `--save` flag.
+See [Flags and Configs -> Persisting CLI settings](/docs/evaluation-flags-and-configs#persisting-cli-settings-with---save).
+:::
+
+### In Code
+
+Pass a `DeepgramSTTModel` to `VoiceConfig` to transcribe with Deepgram instead of the default OpenAI model:
+
+
+
+
+```python
+from deepeval.models import DeepgramSTTModel
+from deepeval.voice import VoiceConfig
+
+stt_model = DeepgramSTTModel(
+ model="nova-3",
+ language="en",
+)
+voice_config = VoiceConfig(
+ stt_model=stt_model,
+ ...,
+)
+```
+
+There are **ZERO** mandatory and **EIGHT** optional parameters when creating a `DeepgramSTTModel`:
+
+- [Optional] `model`: A string specifying the name of the transcription model to use. Defaulted to `nova-3`.
+- [Optional] `api_key`: A string specifying your Deepgram API key. Defaults to `DEEPGRAM_API_KEY` if not passed; raises an error at runtime if neither is set.
+- [Optional] `base_url`: A string specifying a custom endpoint to reach the Deepgram API through. Defaulted to `https://api.deepgram.com`.
+- [Optional] `language`: A string [BCP-47](https://en.wikipedia.org/wiki/IETF_language_tag) code (e.g. `"en"`) pinning the spoken language. Pass `"auto"` to have Deepgram detect it per request instead. Defaulted to `None`, which Deepgram reads as English.
+- [Optional] `smart_format`: A boolean asking Deepgram to punctuate, capitalize and format numbers in the transcript. Defaulted to `True`.
+- [Optional] `cost_per_minute`: A float overriding the per-minute price used for cost accounting. Defaults to the built-in price for the model.
+- [Optional] `transcription_kwargs`: A dictionary of additional query parameters forwarded to Deepgram's `/v1/listen` endpoint.
+- [Optional] `timeout`: A float number of seconds to wait on a request before giving up. Defaulted to `120.0`.
+
+:::tip
+Any Deepgram parameter `deepeval` has no field for — `keyterm`, `diarize`, `redact` — can be passed through `transcription_kwargs` and is sent as-is.
+:::
+
+
+
+
+First set your API key and select Deepgram as the transcription provider:
+
+```bash
+# .env.local
+DEEPGRAM_API_KEY=
+USE_DEEPGRAM_STT=1
+DEEPEVAL_STT_MODEL=nova-3
+```
+
+`deepeval set-stt deepgram --model nova-3` writes those same variables for you. `DEEPEVAL_STT_MODEL` is optional: leave it out and transcription runs on `nova-3` anyway, since that is Deepgram's default here.
+
+You can also pass the model name in code, which wins over `DEEPEVAL_STT_MODEL`:
+
+```python
+from deepeval.voice import VoiceConfig
+
+voice_config = VoiceConfig(
+ stt_model="nova-3",
+ ...,
+)
+```
+
+With `USE_DEEPGRAM_STT` set, a bare model name is shorthand for `DeepgramSTTModel(model=...)`. Pass the model object itself to pin a language or turn off smart formatting.
+
+
+
+
+### Available Deepgram Models
+
+- `nova-3` (default)
+- `nova-3-general`
+
+Any model Deepgram's `/v1/listen` endpoint accepts will work.
diff --git a/docs/content/integrations/stt/elevenlabs.mdx b/docs/content/integrations/stt/elevenlabs.mdx
new file mode 100644
index 000000000..f01682688
--- /dev/null
+++ b/docs/content/integrations/stt/elevenlabs.mdx
@@ -0,0 +1,130 @@
+---
+title: ElevenLabs STT
+sidebar_label: ElevenLabs
+languages: [python]
+---
+
+In `deepeval`, you can use ElevenLabs to transcribe speech into text when running [voice simulations](/docs/conversation-simulator-voice-mode). Transcription is what turns your agent's spoken replies into the `Turn.content` your multi-turn metrics judge, so its accuracy directly bounds how faithfully those metrics see the conversation.
+
+:::caution
+This is unrelated to [`ElevenLabsConnector`](/integrations/voice-agent-connectors/elevenlabs), which bridges to a voice agent hosted on ElevenLabs. This page is about the model that transcribes your agent's replies, and it can be pointed at an agent on any platform.
+:::
+
+### Setting Up Your API Key
+
+DeepEval autoloads `.env.local` then `.env` at import time (process env -> `.env.local` -> `.env`).
+
+**Recommended (local dev):**
+
+```bash
+# .env.local
+ELEVENLABS_API_KEY=
+```
+
+**Alternative (Shell/CI):**
+
+```bash
+export ELEVENLABS_API_KEY=
+```
+
+**Alternative (notebook):**
+
+If you're working in a notebook environment (Jupyter or Colab), set your `ELEVENLABS_API_KEY` in a cell:
+
+```bash
+%env ELEVENLABS_API_KEY=
+```
+
+### Command Line
+
+To transcribe with ElevenLabs in every voice simulation, run:
+
+```bash
+deepeval set-stt elevenlabs --model=scribe_v2
+```
+
+`--model` is optional and defaults to ElevenLabs's own default model.
+
+:::info
+The command above sets ElevenLabs as the default speech-to-text provider, unless overridden in Python code. To use a different one, run `deepeval set-stt` for that provider, or drop back to the OpenAI default:
+
+```bash
+deepeval unset-stt
+```
+
+:::
+
+:::tip[Persisting settings]
+You can persist CLI settings with the optional `--save` flag.
+See [Flags and Configs -> Persisting CLI settings](/docs/evaluation-flags-and-configs#persisting-cli-settings-with---save).
+:::
+
+### In Code
+
+Pass an `ElevenLabsSTTModel` to `VoiceConfig` to transcribe with Scribe instead of the default OpenAI model:
+
+
+
+
+```python
+from deepeval.models import ElevenLabsSTTModel
+from deepeval.voice import VoiceConfig
+
+stt_model = ElevenLabsSTTModel(
+ model="scribe_v2",
+ language="en",
+)
+voice_config = VoiceConfig(
+ stt_model=stt_model,
+ ...,
+)
+```
+
+There are **ZERO** mandatory and **SEVEN** optional parameters when creating an `ElevenLabsSTTModel`:
+
+- [Optional] `model`: A string specifying the name of the transcription model to use. Defaulted to `scribe_v2`.
+- [Optional] `api_key`: A string specifying your ElevenLabs API key. Defaults to `ELEVENLABS_API_KEY` if not passed; raises an error at runtime if neither is set.
+- [Optional] `base_url`: A string specifying a custom endpoint to reach the ElevenLabs API through. Defaulted to `https://api.elevenlabs.io`.
+- [Optional] `language`: A string [ISO-639-1](https://en.wikipedia.org/wiki/List_of_ISO_639_language_codes) code (e.g. `"en"`) pinning the spoken language. Pass `"auto"` to have Scribe detect it instead. Defaulted to `None`, which also detects.
+- [Optional] `cost_per_hour`: A float overriding the hourly price used for cost accounting. Defaults to the built-in price for the model.
+- [Optional] `transcription_kwargs`: A dictionary of additional multipart fields forwarded to ElevenLabs' `/v1/speech-to-text` endpoint.
+- [Optional] `timeout`: A float number of seconds to wait on a request before giving up. Defaulted to `120.0`.
+
+:::tip
+Any ElevenLabs parameter `deepeval` has no field for — `diarize`, `tag_audio_events`, `num_speakers` — can be passed through `transcription_kwargs` and is sent as-is.
+:::
+
+
+
+
+First set your API key and select ElevenLabs as the transcription provider:
+
+```bash
+# .env.local
+ELEVENLABS_API_KEY=
+USE_ELEVENLABS_STT=1
+DEEPEVAL_STT_MODEL=scribe_v2
+```
+
+`deepeval set-stt elevenlabs --model scribe_v2` writes those same variables for you. `DEEPEVAL_STT_MODEL` is optional: leave it out and transcription runs on ElevenLabs' default model.
+
+You can also pass the model name in code, which wins over `DEEPEVAL_STT_MODEL`:
+
+```python
+from deepeval.voice import VoiceConfig
+
+voice_config = VoiceConfig(
+ stt_model="scribe_v2",
+ ...,
+)
+```
+
+With `USE_ELEVENLABS_STT` set, a bare model name is shorthand for `ElevenLabsSTTModel(model=...)`. Pass the model object itself to pin a language.
+
+
+
+
+### Available ElevenLabs Models
+
+- `scribe_v2` (default)
+- `scribe_v1` (deprecated in favour of v2)
diff --git a/docs/content/integrations/stt/meta.json b/docs/content/integrations/stt/meta.json
new file mode 100644
index 000000000..ec26c3b02
--- /dev/null
+++ b/docs/content/integrations/stt/meta.json
@@ -0,0 +1,4 @@
+{
+ "title": "Speech-To-Text (STT)",
+ "pages": ["openai", "deepgram", "assemblyai", "elevenlabs", "cartesia"]
+}
diff --git a/docs/content/integrations/stt/openai.mdx b/docs/content/integrations/stt/openai.mdx
new file mode 100644
index 000000000..84c17a1a4
--- /dev/null
+++ b/docs/content/integrations/stt/openai.mdx
@@ -0,0 +1,133 @@
+---
+title: OpenAI STT
+sidebar_label: OpenAI
+languages: [python]
+---
+
+In `deepeval`, you can use OpenAI to transcribe speech into text when running [voice simulations](/docs/conversation-simulator-voice-mode). Transcription is what turns your agent's spoken replies into the `Turn.content` your multi-turn metrics judge, so its accuracy directly bounds how faithfully those metrics see the conversation.
+
+:::tip
+OpenAI is the default STT model for voice simuations, so if you're happy with `gpt-4o-transcribe` you only need an `OPENAI_API_KEY`.
+:::
+
+### Setting Up Your API Key
+
+DeepEval autoloads `.env.local` then `.env` at import time (process env -> `.env.local` -> `.env`).
+
+**Recommended (local dev):**
+
+```bash
+# .env.local
+OPENAI_API_KEY=
+```
+
+**Alternative (Shell/CI):**
+
+```bash
+export OPENAI_API_KEY=
+```
+
+**Alternative (notebook):**
+
+If you're working in a notebook environment (Jupyter or Colab), set your `OPENAI_API_KEY` in a cell:
+
+```bash
+%env OPENAI_API_KEY=
+```
+
+### Command Line
+
+To transcribe with OpenAI in every voice simulation, run:
+
+```bash
+deepeval set-stt openai --model=whisper-1
+```
+
+`--model` is optional and defaults to OpenAI's own default model.
+
+:::info
+OpenAI is already the speech-to-text provider when nothing is selected, so this command matters mainly for pinning a model, or for switching back after another provider was set. Clearing the selection entirely also lands on OpenAI:
+
+```bash
+deepeval unset-stt
+```
+
+:::
+
+:::tip[Persisting settings]
+You can persist CLI settings with the optional `--save` flag.
+See [Flags and Configs -> Persisting CLI settings](/docs/evaluation-flags-and-configs#persisting-cli-settings-with---save).
+:::
+
+### In Code
+
+Pass an `OpenAISTTModel` to `VoiceConfig` to transcribe with a model other than the default, or to pin a language:
+
+
+
+
+```python
+from deepeval.models import OpenAISTTModel
+from deepeval.voice import VoiceConfig
+
+stt_model = OpenAISTTModel(
+ model="gpt-4o-transcribe",
+ language="en",
+)
+voice_config = VoiceConfig(
+ stt_model=stt_model,
+ ...,
+)
+```
+
+There are **ZERO** mandatory and **EIGHT** optional parameters when creating an `OpenAISTTModel`:
+
+- [Optional] `model`: A string specifying the name of the transcription model to use. Defaulted to `gpt-4o-transcribe`.
+- [Optional] `api_key`: A string specifying your OpenAI API key. Defaults to `OPENAI_API_KEY` if not passed; raises an error at runtime if neither is set.
+- [Optional] `base_url`: A string specifying a custom endpoint to reach the OpenAI API through. Defaulted to `None`.
+- [Optional] `language`: A string [ISO-639-1](https://en.wikipedia.org/wiki/List_of_ISO_639_language_codes) code (e.g. `"en"`) pinning the spoken language, which improves accuracy and latency. Pass `"auto"` to detect the language per utterance instead. Defaulted to `None`, which leaves the decision to OpenAI.
+- [Optional] `cost_per_1m_input_tokens`: A float overriding the input token price used for cost accounting. Defaults to the built-in price for the model.
+- [Optional] `cost_per_1m_output_tokens`: A float overriding the output token price used for cost accounting. Defaults to the built-in price for the model.
+- [Optional] `cost_per_minute`: A float overriding the per-minute price used for cost accounting. Only applies to models priced by duration, such as `whisper-1`.
+- [Optional] `transcription_kwargs`: A dictionary of additional parameters forwarded to OpenAI's `audio.transcriptions.create(...)` call.
+
+:::tip
+Any `**kwargs` you would like to send with each transcription request can be passed through the `transcription_kwargs` parameter. Anything else you pass goes to the OpenAI client constructor instead, which is where options like `organization` or `timeout` belong.
+:::
+
+
+
+
+First set your API key:
+
+```bash
+# .env.local
+OPENAI_API_KEY=
+DEEPEVAL_STT_MODEL=whisper-1
+```
+
+OpenAI is the transcription provider when no `USE_*_STT` flag is set, so `USE_OPENAI_STT=1` is only needed to switch back after selecting another provider. `deepeval set-stt openai --model whisper-1` writes both for you.
+
+You can also pass the model name in code, which wins over `DEEPEVAL_STT_MODEL`:
+
+```python
+from deepeval.voice import VoiceConfig
+
+voice_config = VoiceConfig(
+ stt_model="whisper-1",
+ ...,
+)
+```
+
+A bare model name is shorthand for `OpenAISTTModel(model=...)`, so you can leave `stt_model` out entirely and transcription still runs on `gpt-4o-transcribe`. Pass the model object itself to pin a language.
+
+
+
+
+### Available OpenAI Models
+
+- `gpt-4o-transcribe` (default)
+- `gpt-4o-mini-transcribe`
+- `whisper-1`
+
+The `gpt-4o` transcription models are billed per token, while `whisper-1` is billed per minute of audio.
diff --git a/docs/content/integrations/tts/cartesia.mdx b/docs/content/integrations/tts/cartesia.mdx
new file mode 100644
index 000000000..2e54dc542
--- /dev/null
+++ b/docs/content/integrations/tts/cartesia.mdx
@@ -0,0 +1,130 @@
+---
+title: Cartesia TTS
+sidebar_label: Cartesia
+languages: [python]
+---
+
+In `deepeval`, you can use Cartesia to convert text into speech when running [voice simulations](/docs/conversation-simulator-voice-mode). Synthesis is what speaks the simulated user's messages to your agent, so its clarity is what your agent's own speech recognition has to work with.
+
+Cartesia is reached over plain HTTP rather than through a vendor SDK, so `pip install deepeval` is all you need — there is no extra package to install or version to pin.
+
+### Setting Up Your API Key
+
+DeepEval autoloads `.env.local` then `.env` at import time (process env -> `.env.local` -> `.env`).
+
+**Recommended (local dev):**
+
+```bash
+# .env.local
+CARTESIA_API_KEY=
+```
+
+**Alternative (Shell/CI):**
+
+```bash
+export CARTESIA_API_KEY=
+```
+
+**Alternative (notebook):**
+
+If you're working in a notebook environment (Jupyter or Colab), set your `CARTESIA_API_KEY` in a cell:
+
+```bash
+%env CARTESIA_API_KEY=
+```
+
+### Command Line
+
+To speak with Cartesia in every voice simulation, run:
+
+```bash
+deepeval set-tts cartesia --model=sonic-3.6
+```
+
+`--model` is optional and defaults to Cartesia's own default model.
+
+:::info
+The command above sets Cartesia as the default text-to-speech provider, unless overridden in Python code. To use a different one, run `deepeval set-tts` for that provider, or drop back to the OpenAI default:
+
+```bash
+deepeval unset-tts
+```
+
+:::
+
+:::tip[Persisting settings]
+You can persist CLI settings with the optional `--save` flag.
+See [Flags and Configs -> Persisting CLI settings](/docs/evaluation-flags-and-configs#persisting-cli-settings-with---save).
+:::
+
+### In Code
+
+Pass a `CartesiaTTSModel` to `VoiceConfig` to synthesize with Sonic instead of the default OpenAI model. Cartesia has no default voice, so `voice` is the one thing you have to supply:
+
+
+
+
+```python
+from deepeval.models import CartesiaTTSModel
+from deepeval.voice import VoiceConfig
+
+tts_model = CartesiaTTSModel(
+ model="sonic-3.6",
+ voice="",
+)
+voice_config = VoiceConfig(
+ tts_model=tts_model,
+ ...,
+)
+```
+
+There are **ZERO** mandatory and **TEN** optional parameters when creating a `CartesiaTTSModel`:
+
+- [Optional] `model`: A string specifying the name of the speech model to use. Defaulted to `sonic-3.6`.
+- [Optional] `api_key`: A string specifying your Cartesia API key. Defaults to `CARTESIA_API_KEY` if not passed; raises an error at runtime if neither is set.
+- [Optional] `base_url`: A string specifying a custom endpoint to reach the Cartesia API through. Defaulted to `https://api.cartesia.ai`.
+- [Optional] `voice`: A string voice ID from your Cartesia account. Defaulted to `None`, which raises at synthesis time unless a `Persona` supplies one.
+- [Optional] `sample_rate`: An integer sample rate to synthesize at — one of `8000`, `16000`, `22050`, `24000`, `44100`, or `48000`. Anything else raises immediately. Defaulted to `24000`.
+- [Optional] `language`: A string [ISO-639-1](https://en.wikipedia.org/wiki/List_of_ISO_639_language_codes) code for the language spoken. Defaulted to `None`.
+- [Optional] `api_version`: A string dating the Cartesia API to call, sent on every request. Defaulted to `2026-08-14`, pinned so a server-side release can't change the response shape underneath a run.
+- [Optional] `cost_per_1m_chars`: A float setting the per-character price used for cost accounting. Defaulted to `None`.
+- [Optional] `generation_kwargs`: A dictionary of additional parameters forwarded to Cartesia's `/tts/bytes` request body.
+- [Optional] `timeout`: A float number of seconds to wait on a request before giving up. Defaulted to `120.0`.
+
+:::tip
+Any Cartesia parameter `deepeval` has no field for — `speed`, `emotion`, `duration` — can be passed through `generation_kwargs` and is sent as-is.
+:::
+
+
+
+
+First set your API key and select Cartesia as the speech provider:
+
+```bash
+# .env.local
+CARTESIA_API_KEY=
+USE_CARTESIA_TTS=1
+DEEPEVAL_TTS_MODEL=sonic-3.6
+```
+
+`deepeval set-tts cartesia --model sonic-3.6` writes those same variables for you. `DEEPEVAL_TTS_MODEL` is optional: leave it out and synthesis runs on `sonic-3.6` anyway, since that is Cartesia's default here.
+
+You can also pass the model name in code, which wins over `DEEPEVAL_TTS_MODEL`:
+
+```python
+from deepeval.voice import VoiceConfig
+
+voice_config = VoiceConfig(
+ tts_model="sonic-3.6",
+ ...,
+)
+```
+
+With `USE_CARTESIA_TTS` set, a bare model name is shorthand for `CartesiaTTSModel(model=...)`, which has no voice of its own, so every golden's `Persona` has to carry a `voice`. Pass the model object itself to set one default voice for the whole run.
+
+
+
+
+### Available Cartesia Models
+
+- `sonic-3.6` (default)
diff --git a/docs/content/integrations/tts/deepgram.mdx b/docs/content/integrations/tts/deepgram.mdx
new file mode 100644
index 000000000..177d0f758
--- /dev/null
+++ b/docs/content/integrations/tts/deepgram.mdx
@@ -0,0 +1,135 @@
+---
+title: Deepgram TTS
+sidebar_label: Deepgram
+languages: [python]
+---
+
+In `deepeval`, you can use Deepgram to convert text into speech when running [voice simulations](/docs/conversation-simulator-voice-mode). Synthesis is what speaks the simulated user's messages to your agent, so its clarity is what your agent's own speech recognition has to work with.
+
+Deepgram is reached over plain HTTP rather than through a vendor SDK, so `pip install deepeval` is all you need — there is no extra package to install or version to pin.
+
+### Setting Up Your API Key
+
+DeepEval autoloads `.env.local` then `.env` at import time (process env -> `.env.local` -> `.env`).
+
+**Recommended (local dev):**
+
+```bash
+# .env.local
+DEEPGRAM_API_KEY=
+```
+
+**Alternative (Shell/CI):**
+
+```bash
+export DEEPGRAM_API_KEY=
+```
+
+**Alternative (notebook):**
+
+If you're working in a notebook environment (Jupyter or Colab), set your `DEEPGRAM_API_KEY` in a cell:
+
+```bash
+%env DEEPGRAM_API_KEY=
+```
+
+### Command Line
+
+To speak with Deepgram in every voice simulation, run:
+
+```bash
+deepeval set-tts deepgram --model=aura-2-thalia-en
+```
+
+`--model` is optional and defaults to Deepgram's own default model.
+
+:::info
+The command above sets Deepgram as the default text-to-speech provider, unless overridden in Python code. To use a different one, run `deepeval set-tts` for that provider, or drop back to the OpenAI default:
+
+```bash
+deepeval unset-tts
+```
+
+:::
+
+:::tip[Persisting settings]
+You can persist CLI settings with the optional `--save` flag.
+See [Flags and Configs -> Persisting CLI settings](/docs/evaluation-flags-and-configs#persisting-cli-settings-with---save).
+:::
+
+### In Code
+
+Pass a `DeepgramTTSModel` to `VoiceConfig` to synthesize with Aura instead of the default OpenAI model:
+
+
+
+
+```python
+from deepeval.models import DeepgramTTSModel
+from deepeval.voice import VoiceConfig
+
+tts_model = DeepgramTTSModel(
+ model="aura-2-thalia-en",
+ voice="zeus",
+)
+voice_config = VoiceConfig(
+ tts_model=tts_model,
+ ...,
+)
+```
+
+There are **ZERO** mandatory and **EIGHT** optional parameters when creating a `DeepgramTTSModel`:
+
+- [Optional] `model`: A string specifying the name of the speech model to use. Defaulted to `aura-2-thalia-en`.
+- [Optional] `api_key`: A string specifying your Deepgram API key. Defaults to `DEEPGRAM_API_KEY` if not passed; raises an error at runtime if neither is set.
+- [Optional] `base_url`: A string specifying a custom endpoint to reach the Deepgram API through. Defaulted to `https://api.deepgram.com`.
+- [Optional] `voice`: A string naming the Aura voice to speak in, either bare (`"zeus"`) or as a whole model name (`"aura-2-zeus-en"`). Defaulted to `None`, which leaves the voice named in `model` alone.
+- [Optional] `sample_rate`: An integer sample rate to synthesize at — one of `8000`, `16000`, `24000`, `32000`, or `48000`. Anything else raises immediately. Defaulted to `24000`.
+- [Optional] `cost_per_1m_chars`: A float overriding the per-character price used for cost accounting. Defaults to the built-in price for the model family.
+- [Optional] `generation_kwargs`: A dictionary of additional query parameters forwarded to Deepgram's `/v1/speak` endpoint.
+- [Optional] `timeout`: A float number of seconds to wait on a request before giving up. Defaulted to `120.0`.
+
+:::tip
+Any Deepgram parameter `deepeval` has no field for can be passed through `generation_kwargs` and is sent as-is.
+:::
+
+
+
+
+First set your API key and select Deepgram as the speech provider:
+
+```bash
+# .env.local
+DEEPGRAM_API_KEY=
+USE_DEEPGRAM_TTS=1
+DEEPEVAL_TTS_MODEL=aura-2-thalia-en
+```
+
+`deepeval set-tts deepgram --model aura-2-thalia-en` writes those same variables for you. `DEEPEVAL_TTS_MODEL` is optional: leave it out and synthesis runs on `aura-2-thalia-en` anyway, since that is Deepgram's default here.
+
+You can also pass the model name in code, which wins over `DEEPEVAL_TTS_MODEL`:
+
+```python
+from deepeval.voice import VoiceConfig
+
+voice_config = VoiceConfig(
+ tts_model="aura-2-thalia-en",
+ ...,
+)
+```
+
+With `USE_DEEPGRAM_TTS` set, a bare model name is shorthand for `DeepgramTTSModel(model=...)`, and since an Aura name carries its own voice, picking a voice here is just naming a different model. Pass the model object itself to change the sample rate.
+
+
+
+
+### Available Deepgram Models
+
+- `aura-2-thalia-en` (default)
+- `aura-2-andromeda-en`
+- `aura-2-apollo-en`
+- `aura-2-arcas-en`
+- `aura-2-zeus-en`
+- `aura-asteria-en`
+
+An Aura name carries the family, the voice, and the language in one identifier — `aura-2-thalia-en` is Thalia from Aura 2 speaking English. Any name Deepgram's `/v1/speak` endpoint accepts will work, so the list above is a starting point rather than the full [Aura voice catalog](https://developers.deepgram.com/docs/tts-models).
diff --git a/docs/content/integrations/tts/elevenlabs.mdx b/docs/content/integrations/tts/elevenlabs.mdx
new file mode 100644
index 000000000..0c2200c17
--- /dev/null
+++ b/docs/content/integrations/tts/elevenlabs.mdx
@@ -0,0 +1,140 @@
+---
+title: ElevenLabs TTS
+sidebar_label: ElevenLabs
+languages: [python]
+---
+
+In `deepeval`, you can use ElevenLabs to convert text into speech when running [voice simulations](/docs/conversation-simulator-voice-mode). Synthesis is what speaks the simulated user's messages to your agent, so its clarity is what your agent's own speech recognition has to work with.
+
+ElevenLabs is reached over plain HTTP rather than through a vendor SDK, so `pip install deepeval` is all you need — there is no extra package to install or version to pin.
+
+:::note
+This is unrelated to [`ElevenLabsConnector`](/integrations/voice-agent-connectors/elevenlabs), which bridges to a voice agent hosted on ElevenLabs. This page is about the model that speaks the simulated user's side of the conversation, and it can be pointed at an agent on any platform.
+:::
+
+### Setting Up Your API Key
+
+DeepEval autoloads `.env.local` then `.env` at import time (process env -> `.env.local` -> `.env`).
+
+**Recommended (local dev):**
+
+```bash
+# .env.local
+ELEVENLABS_API_KEY=
+```
+
+**Alternative (Shell/CI):**
+
+```bash
+export ELEVENLABS_API_KEY=
+```
+
+**Alternative (notebook):**
+
+If you're working in a notebook environment (Jupyter or Colab), set your `ELEVENLABS_API_KEY` in a cell:
+
+```bash
+%env ELEVENLABS_API_KEY=
+```
+
+### Command Line
+
+To speak with ElevenLabs in every voice simulation, run:
+
+```bash
+deepeval set-tts elevenlabs --model=eleven_flash_v2_5
+```
+
+`--model` is optional and defaults to ElevenLabs's own default model.
+
+:::info
+The command above sets ElevenLabs as the default text-to-speech provider, unless overridden in Python code. To use a different one, run `deepeval set-tts` for that provider, or drop back to the OpenAI default:
+
+```bash
+deepeval unset-tts
+```
+
+:::
+
+:::tip[Persisting settings]
+You can persist CLI settings with the optional `--save` flag.
+See [Flags and Configs -> Persisting CLI settings](/docs/evaluation-flags-and-configs#persisting-cli-settings-with---save).
+:::
+
+### In Code
+
+Pass an `ElevenLabsTTSModel` to `VoiceConfig` to synthesize with ElevenLabs instead of the default OpenAI model:
+
+
+
+
+```python
+from deepeval.models import ElevenLabsTTSModel
+from deepeval.voice import VoiceConfig
+
+tts_model = ElevenLabsTTSModel(
+ model="eleven_flash_v2_5",
+ voice="JBFqnCBsd6RMkjVDRZzb",
+)
+voice_config = VoiceConfig(
+ tts_model=tts_model,
+ ...,
+)
+```
+
+There are **ZERO** mandatory and **TEN** optional parameters when creating an `ElevenLabsTTSModel`:
+
+- [Optional] `model`: A string specifying the name of the speech model to use. Defaulted to `eleven_flash_v2_5`.
+- [Optional] `api_key`: A string specifying your ElevenLabs API key. Defaults to `ELEVENLABS_API_KEY` if not passed; raises an error at runtime if neither is set.
+- [Optional] `base_url`: A string specifying a custom endpoint to reach the ElevenLabs API through. Defaulted to `https://api.elevenlabs.io`.
+- [Optional] `voice`: A string voice ID from the Voice Library or your own account. Defaulted to `JBFqnCBsd6RMkjVDRZzb` (George).
+- [Optional] `sample_rate`: An integer sample rate to synthesize at — one of `8000`, `16000`, `22050`, `24000`, `32000`, `44100`, or `48000`. Anything else raises immediately. Defaulted to `24000`.
+- [Optional] `language_code`: A string [ISO-639-1](https://en.wikipedia.org/wiki/List_of_ISO_639_language_codes) code enforcing the language spoken. Defaulted to `None`.
+- [Optional] `voice_settings`: A dictionary of ElevenLabs voice settings such as `stability`, `similarity_boost`, and `speed`. Defaulted to `None`.
+- [Optional] `cost_per_1m_chars`: A float overriding the per-character price used for cost accounting. Defaults to the built-in price for the model.
+- [Optional] `generation_kwargs`: A dictionary of additional parameters forwarded to ElevenLabs' text-to-speech request body.
+- [Optional] `timeout`: A float number of seconds to wait on a request before giving up. Defaulted to `120.0`.
+
+:::tip
+Any ElevenLabs parameter `deepeval` has no field for — `seed`, `previous_text`, `apply_text_normalization` — can be passed through `generation_kwargs` and is sent as-is.
+:::
+
+
+
+
+First set your API key and select ElevenLabs as the speech provider:
+
+```bash
+# .env.local
+ELEVENLABS_API_KEY=
+USE_ELEVENLABS_TTS=1
+DEEPEVAL_TTS_MODEL=eleven_flash_v2_5
+```
+
+`deepeval set-tts elevenlabs --model eleven_flash_v2_5` writes those same variables for you. `DEEPEVAL_TTS_MODEL` is optional: leave it out and synthesis runs on `eleven_flash_v2_5` anyway, since that is ElevenLabs' default here.
+
+You can also pass the model name in code, which wins over `DEEPEVAL_TTS_MODEL`:
+
+```python
+from deepeval.voice import VoiceConfig
+
+voice_config = VoiceConfig(
+ tts_model="eleven_flash_v2_5",
+ ...,
+)
+```
+
+With `USE_ELEVENLABS_TTS` set, a bare model name is shorthand for `ElevenLabsTTSModel(model=...)`, which speaks in the default George voice. Pass the model object itself to pick another voice or change the sample rate.
+
+
+
+
+### Available ElevenLabs Models
+
+- `eleven_flash_v2_5` (default)
+- `eleven_flash_v2`
+- `eleven_turbo_v2_5`
+- `eleven_turbo_v2`
+- `eleven_multilingual_v2`
+- `eleven_v3`
+- `eleven_v3_conversational`
diff --git a/docs/content/integrations/tts/meta.json b/docs/content/integrations/tts/meta.json
new file mode 100644
index 000000000..e047c1093
--- /dev/null
+++ b/docs/content/integrations/tts/meta.json
@@ -0,0 +1,4 @@
+{
+ "title": "Text-To-Speech (TTS)",
+ "pages": ["openai", "elevenlabs", "cartesia", "deepgram"]
+}
diff --git a/docs/content/integrations/tts/openai.mdx b/docs/content/integrations/tts/openai.mdx
new file mode 100644
index 000000000..96848c224
--- /dev/null
+++ b/docs/content/integrations/tts/openai.mdx
@@ -0,0 +1,130 @@
+---
+title: OpenAI TTS
+sidebar_label: OpenAI
+languages: [python]
+---
+
+In `deepeval`, you can use OpenAI to convert text into speech when running [voice simulations](/docs/conversation-simulator-voice-mode). Synthesis is what speaks the simulated user's messages to your agent, so its clarity is what your agent's own speech recognition has to work with.
+
+:::tip
+OpenAI is the default TTS for voice simulations, so if you're happy with `gpt-4o-mini-tts` speaking in the `alloy` voice you only need an `OPENAI_API_KEY`.
+:::
+
+### Setting Up Your API Key
+
+DeepEval autoloads `.env.local` then `.env` at import time (process env -> `.env.local` -> `.env`).
+
+**Recommended (local dev):**
+
+```bash
+# .env.local
+OPENAI_API_KEY=
+```
+
+**Alternative (Shell/CI):**
+
+```bash
+export OPENAI_API_KEY=
+```
+
+**Alternative (notebook):**
+
+If you're working in a notebook environment (Jupyter or Colab), set your `OPENAI_API_KEY` in a cell:
+
+```bash
+%env OPENAI_API_KEY=
+```
+
+### Command Line
+
+To speak with OpenAI in every voice simulation, run:
+
+```bash
+deepeval set-tts openai --model=tts-1-hd
+```
+
+`--model` is optional and defaults to OpenAI's own default model.
+
+:::info
+OpenAI is already the text-to-speech provider when nothing is selected, so this command matters mainly for pinning a model, or for switching back after another provider was set. Clearing the selection entirely also lands on OpenAI:
+
+```bash
+deepeval unset-tts
+```
+
+:::
+
+:::tip[Persisting settings]
+You can persist CLI settings with the optional `--save` flag.
+See [Flags and Configs -> Persisting CLI settings](/docs/evaluation-flags-and-configs#persisting-cli-settings-with---save).
+:::
+
+### In Code
+
+Pass an `OpenAITTSModel` to `VoiceConfig` to synthesize with a model other than the default, or to pick a voice:
+
+
+
+
+```python
+from deepeval.models import OpenAITTSModel
+from deepeval.voice import VoiceConfig
+
+tts_model = OpenAITTSModel(
+ model="gpt-4o-mini-tts",
+ voice="coral",
+)
+voice_config = VoiceConfig(
+ tts_model=tts_model,
+ ...,
+)
+```
+
+There are **ZERO** mandatory and **SEVEN** optional parameters when creating an `OpenAITTSModel`:
+
+- [Optional] `model`: A string specifying the name of the speech model to use. Defaulted to `gpt-4o-mini-tts`.
+- [Optional] `api_key`: A string specifying your OpenAI API key. Defaults to `OPENAI_API_KEY` if not passed; raises an error at runtime if neither is set.
+- [Optional] `base_url`: A string specifying a custom endpoint to reach the OpenAI API through. Defaulted to `None`.
+- [Optional] `voice`: A string naming the OpenAI voice to speak in, such as `"alloy"`, `"onyx"`, or `"shimmer"`. Defaulted to `alloy`.
+- [Optional] `response_format`: A string audio format for synthesized turns — one of `"wav"`, `"mp3"`, `"opus"`, `"aac"`, `"flac"`, or `"pcm"`. Defaulted to `"wav"`.
+- [Optional] `cost_per_1m_chars`: A float overriding the per-character price used for cost accounting. Defaults to the built-in price for the model.
+- [Optional] `generation_kwargs`: A dictionary of additional parameters forwarded to OpenAI's `audio.speech.create(...)` call.
+
+:::tip
+Any `**kwargs` you would like to send with each synthesis request can be passed through the `generation_kwargs` parameter. Anything else you pass goes to the OpenAI client constructor instead, which is where options like `organization` or `timeout` belong.
+:::
+
+
+
+
+First set your API key:
+
+```bash
+# .env.local
+OPENAI_API_KEY=
+DEEPEVAL_TTS_MODEL=tts-1-hd
+```
+
+OpenAI is the speech provider when no `USE_*_TTS` flag is set, so `USE_OPENAI_TTS=1` is only needed to switch back after selecting another provider. `deepeval set-tts openai --model tts-1-hd` writes both for you.
+
+You can also pass the model name in code, which wins over `DEEPEVAL_TTS_MODEL`:
+
+```python
+from deepeval.voice import VoiceConfig
+
+voice_config = VoiceConfig(
+ tts_model="tts-1-hd",
+ ...,
+)
+```
+
+A bare model name is shorthand for `OpenAITTSModel(model=...)`, so you can leave `tts_model` out entirely and synthesis still runs on `gpt-4o-mini-tts`. Pass the model object itself to pick a voice.
+
+
+
+
+### Available OpenAI Models
+
+- `gpt-4o-mini-tts` (default)
+- `tts-1`
+- `tts-1-hd`
diff --git a/docs/content/integrations/voice-agent-connectors/elevenlabs.mdx b/docs/content/integrations/voice-agent-connectors/elevenlabs.mdx
new file mode 100644
index 000000000..37de08abb
--- /dev/null
+++ b/docs/content/integrations/voice-agent-connectors/elevenlabs.mdx
@@ -0,0 +1,101 @@
+---
+title: ElevenLabs Agents
+sidebar_label: ElevenLabs
+languages: [python]
+---
+
+In `deepeval`, you can run a [voice simulation](/docs/conversation-simulator-voice-mode) against a conversational agent hosted on ElevenLabs. The connector opens a live session with your agent, plays the simulated user's audio into it, and captures the spoken reply along with how long it took to start.
+
+:::caution
+This is unrelated to [`ElevenLabsTTSModel`](/integrations/tts/elevenlabs) and [`ElevenLabsSTTModel`](/integrations/stt/elevenlabs), which synthesize and transcribe audio for the simulation. This page is about reaching an agent that lives in your ElevenLabs dashboard, and it works no matter which speech models you run the simulation with.
+:::
+
+Your agent is addressed over ElevenLabs' WebSocket API directly, so the `elevenlabs` package isn't required.
+
+### Setting Up Your API Key
+
+Public agents can be dialed without credentials. Private ones need an API key, which `deepeval` autoloads from `.env.local` then `.env` at import time (process env -> `.env.local` -> `.env`).
+
+**Recommended (local dev):**
+
+```bash
+# .env.local
+ELEVENLABS_API_KEY=
+```
+
+**Alternative (Shell/CI):**
+
+```bash
+export ELEVENLABS_API_KEY=
+```
+
+**Alternative (notebook):**
+
+If you're working in a notebook environment (Jupyter or Colab), set your `ELEVENLABS_API_KEY` in a cell:
+
+```bash
+%env ELEVENLABS_API_KEY=
+```
+
+### In Code
+
+Pass an `ElevenLabsConnector` to `VoiceConfig` to simulate against your dashboard agent:
+
+```python
+from deepeval.voice import ElevenLabsConnector, VoiceConfig
+
+connector = ElevenLabsConnector(agent_id="your-agent-id")
+voice_config = VoiceConfig(
+ connector=connector,
+ ...,
+)
+```
+
+There are **ONE** mandatory and **SEVEN** optional parameters when creating an `ElevenLabsConnector`:
+
+- `agent_id`: A string identifying the ElevenLabs conversational agent.
+- [Optional] `api_key`: A string specifying your ElevenLabs API key. Defaults to `ELEVENLABS_API_KEY` if not passed, and is required for private agents.
+- [Optional] `region`: A string naming an ElevenLabs [data residency](https://elevenlabs.io/docs/overview/administration/data-residency) region — one of `"us"`, `"eu"`, `"in"`, or `"sg"`. Any other value raises. Defaulted to `None`, which uses the global host.
+- [Optional] `client_tools`: A dictionary mapping [client tool](https://elevenlabs.io/docs/eleven-agents/customization/tools/client-tools) names to the functions that run them. Defaulted to `None`.
+- [Optional] `dynamic_variables`: A dictionary of [dynamic variables](https://elevenlabs.io/docs/eleven-agents/customization/personalization/dynamic-variables) for this conversation. Defaulted to `None`.
+- [Optional] `conversation_config_override`: A dictionary of [overrides](https://elevenlabs.io/docs/eleven-agents/customization/personalization/overrides) for this conversation, such as prompt, first message, language, or voice. Defaulted to `None`.
+- [Optional] `client_tool_timeout_s`: A float number of seconds to wait for one of your tool functions before answering the agent with an error. Defaulted to `30.0`.
+- [Optional] `turn_detection`: A string preset — `"eager"`, `"balanced"`, or `"patient"` — controlling how long a pause has to last before your agent is considered finished speaking. Defaulted to `"balanced"`.
+
+:::tip
+ElevenLabs sends an explicit `agent_response_complete` message at the end of each turn, and `deepeval` reads it off the wire, so end-of-turn is known rather than inferred from silence. `turn_detection` only sets the ceiling that stops an agent which never stops talking from hanging the run.
+:::
+
+### Client Tools
+
+If your agent calls client tools, give the connector a function for each one. Declare the tool in the ElevenLabs dashboard as usual, then match its name here:
+
+```python
+async def get_customer_details(parameters):
+ return {"name": "Alice", "subscription": "Pro"}
+
+connector = ElevenLabsConnector(
+ agent_id="your-agent-id",
+ client_tools={"getCustomerDetails": get_customer_details},
+)
+```
+
+Handlers can be `async def` or plain functions, and each receives one dictionary of the tool's parameters plus `tool_call_id`. Whatever a handler returns is given to the agent as context, so realistic return values are what keep the rest of the conversation realistic. Time spent inside your handler is excluded from the reported latency, and a handler that raises or times out is answered with an error result rather than leaving the agent hanging.
+
+### Personalizing a Run
+
+`dynamic_variables` and `conversation_config_override` are sent per conversation, so one dashboard agent can be simulated as many different callers — a different customer, tier, or language per golden:
+
+```python
+connector = ElevenLabsConnector(
+ agent_id="your-agent-id",
+ dynamic_variables={"customer_name": "Alice", "tier": "enterprise"},
+ conversation_config_override={"agent": {"language": "fr"}},
+)
+```
+
+:::note
+Overrides only take effect if you've enabled the matching fields on the agent in ElevenLabs first. Left disabled, they're ignored rather than rejected, which looks like the override silently doing nothing.
+:::
+
+For how connectors fit into the simulation loop, and for turn detection in full, see [Voice Connectors](/docs/conversation-simulator-voice-connectors).
diff --git a/docs/content/integrations/voice-agent-connectors/livekit.mdx b/docs/content/integrations/voice-agent-connectors/livekit.mdx
new file mode 100644
index 000000000..ec947da07
--- /dev/null
+++ b/docs/content/integrations/voice-agent-connectors/livekit.mdx
@@ -0,0 +1,117 @@
+---
+title: LiveKit Agents
+sidebar_label: LiveKit
+languages: [python]
+---
+
+In `deepeval`, you can run a [voice simulation](/docs/conversation-simulator-voice-mode) against a LiveKit agent. `deepeval` joins your agent's room as another participant and trades audio with it over WebRTC, so the agent has to be deployed and able to join a room.
+
+:::note
+Before getting started, make sure you have LiveKit's SDKs installed. They will not be installed automatically with DeepEval, you need to install them separately:
+
+```bash
+pip install livekit livekit-api
+```
+
+:::
+
+### Setting Up Your Credentials
+
+A LiveKit agent has no id you can dial — it's a worker [dispatched](https://docs.livekit.io/agents/server/agent-dispatch/) into a [room](https://docs.livekit.io/intro/basics/connect/). So instead of naming the agent, you give `deepeval` credentials for your LiveKit project and it mints a token and creates a room. DeepEval autoloads `.env.local` then `.env` at import time (process env -> `.env.local` -> `.env`).
+
+**Recommended (local dev):**
+
+```bash
+# .env.local
+LIVEKIT_URL=wss://your-project.livekit.cloud
+LIVEKIT_API_KEY=
+LIVEKIT_API_SECRET=
+```
+
+**Alternative (Shell/CI):**
+
+```bash
+export LIVEKIT_URL=wss://your-project.livekit.cloud
+export LIVEKIT_API_KEY=
+export LIVEKIT_API_SECRET=
+```
+
+**Alternative (notebook):**
+
+If you're working in a notebook environment (Jupyter or Colab), set your credentials in a cell:
+
+```bash
+%env LIVEKIT_URL=wss://your-project.livekit.cloud
+%env LIVEKIT_API_KEY=
+%env LIVEKIT_API_SECRET=
+```
+
+### In Code
+
+Pass a `LiveKitConnector` to `VoiceConfig`, naming the agent to dispatch into the room:
+
+```python
+from deepeval.voice import LiveKitConnector, VoiceConfig
+
+connector = LiveKitConnector(
+ url="wss://your-project.livekit.cloud",
+ api_key="your-api-key",
+ api_secret="your-api-secret",
+ agent_name="restaurant-agent",
+)
+voice_config = VoiceConfig(
+ connector=connector,
+ ...,
+)
+```
+
+:::caution
+No argument is mandatory on its own, but the connector needs some way to reach the room — either the three connection values, or a `token`, or a connected `room`:
+:::
+
+There are **ZERO** mandatory and **ELEVEN** optional parameters when creating a `LiveKitConnector`.
+
+- [Optional] `url`: A string LiveKit server URL. Defaults to `LIVEKIT_URL`, and is required unless `room` is already connected.
+- [Optional] `api_key`: A string LiveKit API key. Defaults to `LIVEKIT_API_KEY`, and is required to mint a token.
+- [Optional] `api_secret`: A string LiveKit API secret. Defaults to `LIVEKIT_API_SECRET`, and is required to mint a token.
+- [Optional] `room`: An [`rtc.Room`](https://docs.livekit.io/reference/python/livekit/rtc/room.html) to use instead of creating one, connected or not. Defaulted to `None`.
+- [Optional] `token`: A LiveKit [access token](https://docs.livekit.io/frontends/reference/tokens-grants/) you minted yourself, used instead of `api_key` and `api_secret`. Defaulted to `None`.
+- [Optional] `room_name`: A string naming the room to join. Defaulted to `None`, which creates a room for the session.
+- [Optional] `identity`: A string participant identity for the simulator. Defaulted to `"deepeval-test"`.
+- [Optional] `agent_name`: A string naming the agent to [dispatch](https://docs.livekit.io/agents/server/agent-dispatch/) into the room. Defaulted to `None`, which leaves your existing routing to place the agent there.
+- [Optional] `turn_detection`: A string preset — `"eager"`, `"balanced"`, or `"patient"` — controlling how long a pause has to last before your agent is considered finished speaking. Defaulted to `"balanced"`.
+- [Optional] `connect_timeout_s`: A float number of seconds to wait when establishing the room connection. Defaulted to `15.0`.
+- [Optional] `transcript_grace_s`: A float number of seconds to wait after your agent stops speaking for its transcript to arrive. Defaulted to `1.5`.
+
+`room_name` and `agent_name` are what actually find your agent. Omit `room_name` and the connector creates a room; set `agent_name` and LiveKit dispatches that agent into it.
+
+:::tip
+A LiveKit room carries a bare audio track with nothing to say when a turn is over, so `turn_detection` matters more here than on transports that close each turn explicitly. Pick the preset by listening to your agent's longest natural pause.
+:::
+
+### Bringing Your Own Room
+
+If your test setup already joined a room, hand that over instead and skip the credentials:
+
+```python
+from livekit import rtc
+from deepeval.voice import LiveKitConnector, VoiceConfig
+
+room = rtc.Room()
+await room.connect("wss://your-project.livekit.cloud", your_token)
+
+voice_config = VoiceConfig(
+ connector=LiveKitConnector(room=room),
+ ...,
+)
+```
+
+Concurrent conversations each get their own session, cloned from the connector you passed. A connector holding a room you connected yourself can't be cloned, so give `VoiceConfig` a zero-argument callable that builds a fresh connector instead of the connector itself.
+
+`deepeval` publishes a microphone track, listens for the agent, and leaves the room as it found it — a room you connected stays connected afterwards, with just the simulated microphone removed. An unconnected room gets connected using the parameters above.
+
+:::note
+`deepeval` reads your agent's own transcript from LiveKit's [`lk.transcription`](https://docs.livekit.io/agents/multimodality/text/) text stream, so on turns where one arrives within `transcript_grace_s` the words the agent actually spoke become the assistant `Turn.content` and no transcription runs. Turns without one still fall back to your [speech-to-text model](/integrations/stt/openai), which `VoiceConfig` resolves either way — so it still needs to be configured, credentials included, even for an agent that publishes every transcript.
+:::
+
+For how connectors fit into the simulation loop, and for turn detection in full, see [Voice Connectors](/docs/conversation-simulator-voice-connectors).
diff --git a/docs/content/integrations/voice-agent-connectors/meta.json b/docs/content/integrations/voice-agent-connectors/meta.json
new file mode 100644
index 000000000..b36f50d5d
--- /dev/null
+++ b/docs/content/integrations/voice-agent-connectors/meta.json
@@ -0,0 +1,4 @@
+{
+ "title": "Voice Agent Connectors",
+ "pages": ["vapi", "elevenlabs", "livekit", "pipecat"]
+}
diff --git a/docs/content/integrations/voice-agent-connectors/pipecat.mdx b/docs/content/integrations/voice-agent-connectors/pipecat.mdx
new file mode 100644
index 000000000..50dee943e
--- /dev/null
+++ b/docs/content/integrations/voice-agent-connectors/pipecat.mdx
@@ -0,0 +1,80 @@
+---
+title: Pipecat Pipelines
+sidebar_label: Pipecat
+languages: [python]
+---
+
+In `deepeval`, you can run a [voice simulation](/docs/conversation-simulator-voice-mode) against a Pipecat pipeline. The pipeline is yours and self-hosted, so the connector takes the URL its WebSocket transport is listening on, connects the way any other client would, plays the simulated user's audio into it, and captures the spoken reply along with how long it took to start.
+
+No credentials are involved, and `pipecat-ai` isn't required on the machine running the simulation — `deepeval` speaks the [protobuf frames](https://docs.pipecat.ai/server/utilities/serializers/introduction) that Pipecat's WebSocket transports serialize by default.
+
+### Serving Your Pipeline
+
+The connector needs a pipeline behind a [`WebsocketServerTransport`](https://docs.pipecat.ai/server/services/transport/websocket-server) or [`FastAPIWebsocketTransport`](https://docs.pipecat.ai/server/services/transport/fastapi-websocket), which is the same setup a browser client would connect to:
+
+```python
+from pipecat.serializers.protobuf import ProtobufFrameSerializer
+from pipecat.transports.websocket.server import (
+ SingleClientWebsocketServerParams,
+ SingleClientWebsocketServerTransport,
+)
+
+transport = SingleClientWebsocketServerTransport(
+ host="localhost",
+ port=8765,
+ params=SingleClientWebsocketServerParams(
+ audio_in_enabled=True,
+ audio_out_enabled=True,
+ serializer=ProtobufFrameSerializer(),
+ ),
+)
+```
+
+### In Code
+
+Pass a `PipecatConnector` to `VoiceConfig`, pointing at that server:
+
+```python
+from deepeval.voice import PipecatConnector, VoiceConfig
+
+connector = PipecatConnector(url="ws://localhost:8765/ws")
+voice_config = VoiceConfig(
+ connector=connector,
+ ...,
+)
+```
+
+There are **ONE** mandatory and **THREE** optional parameters when creating a `PipecatConnector`, alongside the shared connector settings:
+
+- `url`: A string specifying the WebSocket URL your pipeline is serving on.
+- [Optional] `headers`: A dictionary of HTTP headers sent with the upgrade request, for a pipeline behind auth. Defaulted to `None`.
+- [Optional] `agent_sample_rate`: An integer specifying the rate audio is sent to your pipeline at. Must match the `audio_in_sample_rate` your pipeline runs with, since Pipecat doesn't resample what it's handed. Defaulted to `16000`, which is Pipecat's own default.
+- [Optional] `client_ready`: A boolean which, if `True`, sends an RTVI `client-ready` message once connected. Defaulted to `True`.
+- [Optional] `turn_detection`: A string preset — `"eager"`, `"balanced"`, or `"patient"` — controlling how long a pause has to last before your agent is considered finished speaking. Defaulted to `"balanced"`.
+
+The reply's sample rate is read from the frames your pipeline sends, so `audio_out_sample_rate` needs no configuring here.
+
+### Turn Detection and Transcripts
+
+Pipecat's frame schema carries audio, not turn boundaries — what says a turn is over is an [RTVI](https://docs.pipecat.ai/server/frameworks/rtvi/introduction) `bot-stopped-speaking` message, and only a pipeline running an `RTVIProcessor` sends one:
+
+```python
+from pipecat.processors.frameworks.rtvi import RTVIProcessor
+
+rtvi = RTVIProcessor()
+pipeline = Pipeline([transport.input(), rtvi, ..., transport.output()])
+```
+
+With RTVI in the pipeline, `deepeval` ends each turn on the signal instead of inferring it from silence, and takes your agent's own words from `bot-transcription` — so no transcription runs on turns where one arrives. Without it, turns end on silence and every turn is transcribed. Either way `VoiceConfig` resolves a [speech-to-text model](/integrations/stt/openai) for the turns that need one, so it still needs to be configured, credentials included.
+
+:::note
+`deepeval` doesn't take the signal on trust: it ends turns on silence until it has actually seen a `bot-stopped-speaking`, so a pipeline without RTVI is never left waiting for one.
+:::
+
+Some pipelines hold their greeting — or their audio input — until the client announces itself. `client_ready` sends that announcement on connect, and a pipeline with no `RTVIProcessor` ignores it.
+
+### Pipelines on Other Transports
+
+A Pipecat pipeline served over WebRTC is reached through the connector for that service rather than this one: a pipeline on LiveKit is a LiveKit room, so [`LiveKitConnector`](/integrations/voice-agent-connectors/livekit) joins it. Pipelines on a telephony serializer such as `TwilioFrameSerializer` are the agent taking a real phone call, which the simulator can't place — run the same pipeline behind a WebSocket transport to test it.
+
+For how connectors fit into the simulation loop, and for turn detection in full, see [Voice Connectors](/docs/conversation-simulator-voice-connectors).
diff --git a/docs/content/integrations/voice-agent-connectors/vapi.mdx b/docs/content/integrations/voice-agent-connectors/vapi.mdx
new file mode 100644
index 000000000..903db4a0b
--- /dev/null
+++ b/docs/content/integrations/voice-agent-connectors/vapi.mdx
@@ -0,0 +1,86 @@
+---
+title: Vapi Assistants
+sidebar_label: Vapi
+languages: [python]
+---
+
+In `deepeval`, you can run a [voice simulation](/docs/conversation-simulator-voice-mode) against an assistant hosted on Vapi. The connector creates a call on Vapi's WebSocket transport, plays the simulated user's audio into it, and captures the spoken reply along with how long it took to start.
+
+No phone number is involved. Vapi rejects phone parameters on this transport, so the simulator is the only caller and nothing routes through telephony. The connector speaks Vapi's API directly, so no Vapi SDK is required.
+
+### Setting Up Your API Key
+
+Your key mints the call that the WebSocket carries, so it's required. DeepEval autoloads `.env.local` then `.env` at import time (process env -> `.env.local` -> `.env`).
+
+**Recommended (local dev):**
+
+```bash
+# .env.local
+VAPI_API_KEY=
+```
+
+**Alternative (Shell/CI):**
+
+```bash
+export VAPI_API_KEY=
+```
+
+**Alternative (notebook):**
+
+If you're working in a notebook environment (Jupyter or Colab), set your `VAPI_API_KEY` in a cell:
+
+```bash
+%env VAPI_API_KEY=
+```
+
+### In Code
+
+Pass a `VapiConnector` to `VoiceConfig`, naming the assistant to call:
+
+```python
+from deepeval.voice import VapiConnector, VoiceConfig
+
+connector = VapiConnector(assistant_id="your-assistant-id")
+voice_config = VoiceConfig(
+ connector=connector,
+ ...,
+)
+```
+
+There are **ONE** mandatory and **THREE** optional parameters when creating a `VapiConnector`, alongside the shared connector settings:
+
+- `assistant_id`: A string identifying the Vapi assistant to call.
+- [Optional] `api_key`: A string specifying your Vapi API key. Defaults to `VAPI_API_KEY` if not passed; raises at connect time if neither is set.
+- [Optional] `assistant_overrides`: A dictionary of [assistant overrides](https://docs.vapi.ai/assistants/dynamic-variables) for this call, such as `variableValues`, `firstMessage`, or `model`. Defaulted to `None`.
+- [Optional] `base_url`: A string specifying a custom endpoint to reach the Vapi API through. Defaulted to `https://api.vapi.ai`.
+- [Optional] `turn_detection`: A string preset — `"eager"`, `"balanced"`, or `"patient"` — controlling how long a pause has to last before your assistant is considered finished speaking. Defaulted to `"balanced"`.
+
+:::tip
+Vapi sends a `speech-update` message when your assistant stops speaking, and `deepeval` reads it off the wire, so end-of-turn is known rather than inferred from silence. `turn_detection` only sets the ceiling that stops an assistant which never stops talking from hanging the run.
+:::
+
+### Personalizing a Call
+
+`assistant_overrides` is sent per call, so one assistant can be simulated as many different callers — a different customer, tier, or opening line per golden:
+
+```python
+connector = VapiConnector(
+ assistant_id="your-assistant-id",
+ assistant_overrides={
+ "variableValues": {"customer_name": "Alice", "tier": "enterprise"},
+ "firstMessage": "Hi, thanks for calling. How can I help?",
+ },
+)
+```
+
+:::note
+`deepeval` reads two of Vapi's [client messages](https://docs.vapi.ai/api-reference/webhooks/client-message): `speech-update` to know when a turn ended, and `transcript` for what your assistant said. Both are on by default. If you narrow `clientMessages` in your overrides, the connector adds those two back, since losing them would silently cost end-of-turn detection and make every turn pay for transcription.
+:::
+
+### Transcripts
+
+Your assistant's own final transcripts arrive over the same socket and become the assistant `Turn.content`, so no transcription runs on turns where they land. Vapi sends one final per utterance rather than one per turn, and the connector joins them, so a reply made of several sentences is recorded whole.
+
+Turns without a transcript still fall back to your [speech-to-text model](/integrations/stt/openai), which `VoiceConfig` resolves either way — so it still needs to be configured, credentials included.
+
+For how connectors fit into the simulation loop, and for turn detection in full, see [Voice Connectors](/docs/conversation-simulator-voice-connectors).
diff --git a/docs/lib/generated/page-languages.json b/docs/lib/generated/page-languages.json
index 89854cc47..ae98b4846 100644
--- a/docs/lib/generated/page-languages.json
+++ b/docs/lib/generated/page-languages.json
@@ -87,6 +87,18 @@
"/docs/conversation-simulator-stopping-logic": [
"python"
],
+ "/docs/conversation-simulator-voice-connectors": [
+ "python"
+ ],
+ "/docs/conversation-simulator-voice-interruptions": [
+ "python"
+ ],
+ "/docs/conversation-simulator-voice-mode": [
+ "python"
+ ],
+ "/docs/conversation-simulator-voice-personas": [
+ "python"
+ ],
"/docs/data-privacy": [
"python",
"typescript"
@@ -134,10 +146,17 @@
"/docs/evaluation-prompts": [
"python"
],
+ "/docs/evaluation-trajectory-based-llm-evals": [
+ "python",
+ "typescript"
+ ],
"/docs/evaluation-unit-testing-in-ci-cd": [
"python",
"typescript"
],
+ "/docs/evaluation-voice": [
+ "python"
+ ],
"/docs/faq": [
"python",
"typescript"
@@ -184,6 +203,9 @@
"/docs/metrics-agent-loop-detection": [
"python"
],
+ "/docs/metrics-agent-responsiveness": [
+ "python"
+ ],
"/docs/metrics-answer-relevancy": [
"python",
"typescript"
@@ -196,6 +218,9 @@
"python",
"typescript"
],
+ "/docs/metrics-audio-integrity": [
+ "python"
+ ],
"/docs/metrics-bias": [
"python",
"typescript"
@@ -317,6 +342,9 @@
"python",
"typescript"
],
+ "/docs/metrics-speech-intelligibility": [
+ "python"
+ ],
"/docs/metrics-step-efficiency": [
"python",
"typescript"
@@ -368,6 +396,18 @@
"python",
"typescript"
],
+ "/docs/metrics-turn-taking-naturalness": [
+ "python"
+ ],
+ "/docs/metrics-voice-consistency": [
+ "python"
+ ],
+ "/docs/metrics-voice-naturalness": [
+ "python"
+ ],
+ "/docs/metrics-voice-reliability": [
+ "python"
+ ],
"/docs/miscellaneous": [
"python"
],
@@ -517,7 +557,8 @@
"typescript"
],
"/integrations/frameworks/langgraph": [
- "python"
+ "python",
+ "typescript"
],
"/integrations/frameworks/llamaindex": [
"python"
@@ -526,7 +567,8 @@
"typescript"
],
"/integrations/frameworks/openai": [
- "python"
+ "python",
+ "typescript"
],
"/integrations/frameworks/openai-agents": [
"python",
@@ -600,6 +642,33 @@
"python",
"typescript"
],
+ "/integrations/stt/assemblyai": [
+ "python"
+ ],
+ "/integrations/stt/cartesia": [
+ "python"
+ ],
+ "/integrations/stt/deepgram": [
+ "python"
+ ],
+ "/integrations/stt/elevenlabs": [
+ "python"
+ ],
+ "/integrations/stt/openai": [
+ "python"
+ ],
+ "/integrations/tts/cartesia": [
+ "python"
+ ],
+ "/integrations/tts/deepgram": [
+ "python"
+ ],
+ "/integrations/tts/elevenlabs": [
+ "python"
+ ],
+ "/integrations/tts/openai": [
+ "python"
+ ],
"/integrations/vector-databases/chroma": [
"python"
],
@@ -618,6 +687,18 @@
"/integrations/vector-databases/weaviate": [
"python"
],
+ "/integrations/voice-agent-connectors/elevenlabs": [
+ "python"
+ ],
+ "/integrations/voice-agent-connectors/livekit": [
+ "python"
+ ],
+ "/integrations/voice-agent-connectors/pipecat": [
+ "python"
+ ],
+ "/integrations/voice-agent-connectors/vapi": [
+ "python"
+ ],
"/tutorials/medical-chatbot/development": [
"python"
],
diff --git a/docs/snippets/voice/connector-tabs.mdx b/docs/snippets/voice/connector-tabs.mdx
new file mode 100644
index 000000000..c4600c3be
--- /dev/null
+++ b/docs/snippets/voice/connector-tabs.mdx
@@ -0,0 +1,94 @@
+
+
+
+Your agent lives in the ElevenLabs dashboard, so the connector only needs its `agent_id`. It speaks ElevenLabs' WebSocket API directly, so the `elevenlabs` package isn't required. Private agents also need an API key.
+
+```python
+from deepeval.voice import ElevenLabsConnector
+
+connector = ElevenLabsConnector(agent_id="your-agent-id")
+```
+
+Client tools, per-conversation overrides, and every parameter are on the [ElevenLabs integration page](/integrations/voice-agent-connectors/elevenlabs).
+
+
+
+
+Your assistant lives in Vapi, so the connector needs its `assistant_id` and an API key to create the call with. Phone parameters are rejected on Vapi's WebSocket transport, so the simulator is the only caller and no Vapi SDK is required.
+
+```python
+from deepeval.voice import VapiConnector
+
+connector = VapiConnector(assistant_id="your-assistant-id")
+```
+
+Per-call overrides and every parameter are on the [Vapi integration page](/integrations/voice-agent-connectors/vapi).
+
+
+
+
+A LiveKit agent has no id to dial — it's a worker dispatched into a room. So you give the connector credentials, and it mints a token, creates a room, and joins as another participant. This one needs LiveKit's SDKs:
+
+```bash
+pip install livekit livekit-api
+```
+
+```python
+from deepeval.voice import LiveKitConnector
+
+connector = LiveKitConnector(
+ url="wss://your-project.livekit.cloud",
+ api_key="your-api-key",
+ api_secret="your-api-secret",
+ agent_name="restaurant-agent",
+)
+```
+
+Bringing your own room, transcripts, and every parameter are on the [LiveKit integration page](/integrations/voice-agent-connectors/livekit).
+
+
+
+
+Your pipeline is self-hosted, so the connector needs the URL its WebSocket transport is serving on and nothing else. It speaks the protobuf frames Pipecat serializes by default, so `pipecat-ai` isn't required to run the simulation.
+
+```python
+from deepeval.voice import PipecatConnector
+
+connector = PipecatConnector(url="ws://localhost:8765/ws")
+```
+
+Serving your pipeline, RTVI turn signals, and every parameter are on the [Pipecat integration page](/integrations/voice-agent-connectors/pipecat).
+
+
+
+
+For a custom agent that accepts and emits raw audio over a WebSocket, describe its message shape — which JSON keys carry audio in and out, whether frames are binary, and what marks the end of a turn.
+
+```python
+from deepeval.voice import WebSocketConnector
+
+connector = WebSocketConnector(
+ url="wss://your-agent.example.com/audio",
+ send_key="audio",
+ receive_audio_key="audio",
+ turn_complete_type="turn_end",
+)
+```
+
+Every parameter is under [Generic WebSocket](#generic-websocket) below.
+
+
+
+
+Wraps an in-process Python callable, so you can exercise the full TTS to agent to STT pipeline with no network transport at all.
+
+```python
+from deepeval.voice import CallbackVoiceConnector
+
+connector = CallbackVoiceConnector(my_agent)
+```
+
+What your callable may return is under [Callback](#callback) below.
+
+
+
diff --git a/tests/test_core/test_cli/test_cli.py b/tests/test_core/test_cli/test_cli.py
index 55b037d82..dbe0a6d2c 100644
--- a/tests/test_core/test_cli/test_cli.py
+++ b/tests/test_core/test_cli/test_cli.py
@@ -10,7 +10,12 @@ from dataclasses import dataclass
import deepeval.cli.generate.command as generate_cli
from deepeval.cli.main import app as cli_app
-from deepeval.cli.utils import USE_EMBED_KEYS, USE_LLM_KEYS
+from deepeval.cli.utils import (
+ USE_EMBED_KEYS,
+ USE_LLM_KEYS,
+ USE_STT_KEYS,
+ USE_TTS_KEYS,
+)
from deepeval.config.settings import Settings, reset_settings # noqa: E402
from deepeval.config.utils import parse_bool
@@ -640,6 +645,104 @@ def test_set_unset_embedding_provider_roundtrip(
], "Expected no embedding USE_* key to remain YES after unset"
+SPEECH_PROVIDER_CASES: List[_ProviderCase] = [
+ _ProviderCase(
+ set_cmd="set-tts",
+ unset_cmd="unset-tts",
+ use_key="USE_DEEPGRAM_TTS",
+ set_flags=("deepgram", "--model", "aura-2-thalia-en"),
+ expected_env={"DEEPEVAL_TTS_MODEL": "aura-2-thalia-en"},
+ expected_store={"DEEPEVAL_TTS_MODEL": "aura-2-thalia-en"},
+ ),
+ _ProviderCase(
+ set_cmd="set-stt",
+ unset_cmd="unset-stt",
+ use_key="USE_ASSEMBLYAI_STT",
+ set_flags=("assemblyai", "--model", "universal-3-5-pro"),
+ expected_env={"DEEPEVAL_STT_MODEL": "universal-3-5-pro"},
+ expected_store={"DEEPEVAL_STT_MODEL": "universal-3-5-pro"},
+ ),
+]
+
+
+@pytest.mark.parametrize(
+ "case", SPEECH_PROVIDER_CASES, ids=lambda c: c.set_cmd
+)
+def test_set_unset_speech_provider_roundtrip(
+ case: _ProviderCase,
+ hidden_store_dir: Path,
+ env_path: Path,
+) -> None:
+ runner = CliRunner()
+ save = f"dotenv:{env_path}"
+ family_keys = (
+ USE_TTS_KEYS if case.use_key.endswith("_TTS") else USE_STT_KEYS
+ )
+
+ result = runner.invoke(
+ cli_app, [case.set_cmd, *case.set_flags, "--save", save]
+ )
+ assert result.exit_code == 0, result.output
+
+ env = _read_dotenv(env_path)
+ _assert_no_dupes(env_path, list(case.expected_env.keys()) + family_keys)
+ for k, v in case.expected_env.items():
+ assert env.get(k) == v
+ _assert_use_flags_exclusive_env(env_path, case.use_key, family_keys)
+
+ store = _read_hidden_store_json(hidden_store_dir)
+ _assert_use_flags_exclusive_store(store, case.use_key, family_keys)
+ for k, v in case.expected_store.items():
+ assert store.get(k) == v
+
+ result2 = runner.invoke(cli_app, [case.unset_cmd, "--save", save])
+ assert result2.exit_code == 0, result2.output
+
+ env2 = _read_dotenv(env_path)
+ for k in case.expected_env.keys():
+ assert k not in env2
+ assert not parse_bool(env2.get(case.use_key))
+
+ store2 = _read_hidden_store_json(hidden_store_dir)
+ for k in case.expected_store.keys():
+ assert k not in store2
+ assert "YES" not in [store2.get(k) for k in family_keys]
+
+
+def test_speech_and_llm_selections_do_not_clear_each_other(
+ hidden_store_dir: Path,
+ env_path: Path,
+) -> None:
+ runner = CliRunner()
+ save = f"dotenv:{env_path}"
+
+ for args in (
+ ["set-stt", "deepgram", "--model", "nova-3", "--save", save],
+ ["set-tts", "elevenlabs", "--save", save],
+ ["set-openai", "--model", "gpt-4o-mini", "--save", save],
+ ):
+ result = runner.invoke(cli_app, args)
+ assert result.exit_code == 0, result.output
+
+ store = _read_hidden_store_json(hidden_store_dir)
+ env = _read_dotenv(env_path)
+
+ # Selecting an LLM provider leaves both speech families alone.
+ for key in ("USE_DEEPGRAM_STT", "USE_ELEVENLABS_TTS", "USE_OPENAI_MODEL"):
+ assert store.get(key) == "YES", store
+ assert parse_bool(env.get(key)), env
+ assert store.get("DEEPEVAL_STT_MODEL") == "nova-3"
+
+ # And unsetting one speech family leaves the other one alone.
+ result = runner.invoke(cli_app, ["unset-stt", "--save", save])
+ assert result.exit_code == 0, result.output
+
+ store2 = _read_hidden_store_json(hidden_store_dir)
+ assert store2.get("USE_DEEPGRAM_STT") is None
+ assert store2.get("USE_ELEVENLABS_TTS") == "YES"
+ assert store2.get("USE_OPENAI_MODEL") == "YES"
+
+
def test_set_unset_gemini_service_account_file_roundtrip_dotenv_only(
tmp_path,
hidden_store_dir,
diff --git a/tests/test_core/test_config/test_settings.py b/tests/test_core/test_config/test_settings.py
index 07e37c45f..164b08b27 100644
--- a/tests/test_core/test_config/test_settings.py
+++ b/tests/test_core/test_config/test_settings.py
@@ -5,6 +5,7 @@ from pathlib import Path
from deepeval.config.utils import parse_bool
from deepeval.config.settings import (
+ _family_of_use_flag,
autoload_dotenv,
get_settings,
reset_settings,
@@ -309,50 +310,37 @@ def test_sample_rate_invalid_raises(monkeypatch, val):
def test_switch_model_provider_flips_use_flags_within_family_only(settings):
- # Split USE_* flags into "llm family" vs "embedding family"
+ # Split USE_* flags into their provider families: llm, embedding, tts, stt
all_use = [
field
for field in type(settings).model_fields
if field.startswith("USE_")
]
- llm_flags = [field for field in all_use if "EMBEDDING" not in field]
- emb_flags = [field for field in all_use if "EMBEDDING" in field]
+ families: dict[str, list[str]] = {}
+ for field in all_use:
+ families.setdefault(_family_of_use_flag(field), []).append(field)
- # Assert both families exist
- assert llm_flags, "No LLM USE_* flags found on Settings"
- assert emb_flags, "No embedding USE_* flags found on Settings"
-
- target_llm = llm_flags[0]
- target_emb = emb_flags[0]
+ assert set(families) == {"llm", "embedding", "tts", "stt"}
with settings.edit(persist=False) as ctx:
- # Seed all flags to True so we can observe which ones get flipped
- for field in all_use:
- setattr(settings, field, True)
+ for family, flags in families.items():
+ target = flags[0]
- # Flip LLM family only
- ctx.switch_model_provider(target_llm)
+ # Seed all flags to True so we can observe which ones get flipped
+ for field in all_use:
+ setattr(settings, field, True)
- for field in llm_flags:
- expected = True if field == target_llm else None
- assert getattr(settings, field) is expected
- for field in emb_flags:
- # Embeddings should be untouched by LLM switch (remain True)
- assert getattr(settings, field) is True
+ ctx.switch_model_provider(target)
- # Reset everything to True again
- for field in all_use:
- setattr(settings, field, True)
-
- # Flip embedding family only
- ctx.switch_model_provider(target_emb)
-
- for field in emb_flags:
- expected = True if field == target_emb else None
- assert getattr(settings, field) is expected
- for field in llm_flags:
- # LLMs should be untouched by embedding switch (remain True)
- assert getattr(settings, field) is True
+ for field in flags:
+ expected = True if field == target else None
+ assert getattr(settings, field) is expected
+ for other_family, other_flags in families.items():
+ if other_family == family:
+ continue
+ for field in other_flags:
+ # Other families are untouched by this switch
+ assert getattr(settings, field) is True
############################################################
diff --git a/tests/test_core/test_models/speech_stubs.py b/tests/test_core/test_models/speech_stubs.py
new file mode 100644
index 000000000..8aeed137e
--- /dev/null
+++ b/tests/test_core/test_models/speech_stubs.py
@@ -0,0 +1,102 @@
+"""A stand-in for `SpeechTransport` that records calls instead of making them."""
+
+from typing import Any, Dict, List, Optional
+
+from deepeval.models.speech import SpeechHTTPError
+from deepeval.test_case import Audio
+from deepeval.voice.connectors import audio_utils
+
+
+class FakeTransport:
+ """Replaces the transport on a speech model.
+
+ Recording the request each model builds is the whole point: the interesting
+ behaviour is which endpoint, query parameters and body a provider needs, and
+ that is exactly what a real HTTP call would hide.
+ """
+
+ def __init__(
+ self,
+ *,
+ content: bytes = b"",
+ json: Any = None,
+ json_sequence: Optional[List[Any]] = None,
+ stream_reads: Optional[List[bytes]] = None,
+ ):
+ self._content = content
+ self._json_sequence = (
+ list(json_sequence) if json_sequence is not None else None
+ )
+ self._json = json
+ self._stream_reads = stream_reads or []
+ self.calls: List[Dict[str, Any]] = []
+
+ #
+ # Recording
+ #
+
+ def _record(self, method: str, path: str, kwargs: Dict[str, Any]) -> None:
+ self.calls.append({"method": method, "path": path, **kwargs})
+
+ @property
+ def last(self) -> Dict[str, Any]:
+ return self.calls[-1]
+
+ def params_of(self, index: int = -1) -> Dict[str, str]:
+ """Query params as a dict, flattening the pair list the transport uses."""
+ params = self.calls[index].get("params") or {}
+ if isinstance(params, dict):
+ return {k: str(v) for k, v in params.items()}
+ return {k: str(v) for k, v in params}
+
+ def paths(self) -> List[str]:
+ return [call["path"] for call in self.calls]
+
+ #
+ # Responses
+ #
+
+ def _next_json(self) -> Any:
+ if self._json_sequence is not None:
+ if not self._json_sequence:
+ raise SpeechHTTPError("FakeTransport ran out of responses")
+ return self._json_sequence.pop(0)
+ return self._json
+
+ def request_bytes(self, method: str, path: str, **kwargs) -> bytes:
+ self._record(method, path, kwargs)
+ return self._content
+
+ def request_json(self, method: str, path: str, **kwargs) -> Any:
+ self._record(method, path, kwargs)
+ return self._next_json()
+
+ async def a_request_bytes(self, method: str, path: str, **kwargs) -> bytes:
+ self._record(method, path, kwargs)
+ return self._content
+
+ async def a_request_json(self, method: str, path: str, **kwargs) -> Any:
+ self._record(method, path, kwargs)
+ return self._next_json()
+
+ async def a_stream_bytes(self, method: str, path: str, **kwargs):
+ self._record(method, path, kwargs)
+ for read in self._stream_reads:
+ yield read
+
+
+def wav_audio(
+ seconds: float = 1.0,
+ sample_rate: int = 24000,
+ *,
+ duration: Optional[float] = None,
+) -> Audio:
+ """A real (silent) WAV, so duration-based cost and routing are exercised."""
+ pcm = b"\x00\x00" * int(sample_rate * seconds)
+ return Audio.from_bytes(
+ audio_utils.pcm16_to_wav_bytes(pcm, sample_rate, 1),
+ "audio/wav",
+ sampleRate=sample_rate,
+ encoding="wav",
+ duration=duration,
+ )
diff --git a/tests/test_core/test_models/test_openai_tts.py b/tests/test_core/test_models/test_openai_tts.py
index 2d27237ce..52b59b4e9 100644
--- a/tests/test_core/test_models/test_openai_tts.py
+++ b/tests/test_core/test_models/test_openai_tts.py
@@ -4,7 +4,7 @@ from typing import List
import pytest
-from deepeval.models.tts_models.openai_tts import (
+from deepeval.models.tts.openai.openai import (
_STREAM_FRAME_BYTES,
OpenAITTSModel,
)
diff --git a/tests/test_core/test_models/test_speech_schema.py b/tests/test_core/test_models/test_speech_schema.py
new file mode 100644
index 000000000..935fb4b12
--- /dev/null
+++ b/tests/test_core/test_models/test_speech_schema.py
@@ -0,0 +1,315 @@
+"""The per-provider schema models and the two helpers that cross the wire."""
+
+import warnings
+
+import pytest
+
+from deepeval.models import (
+ CartesiaSTTModel,
+ CartesiaTTSModel,
+ DeepgramSTTModel,
+ DeepgramTTSModel,
+ ElevenLabsSTTModel,
+ ElevenLabsTTSModel,
+)
+from deepeval.models.speech import (
+ SpeechHTTPError,
+ dump_request,
+ parse_response,
+)
+from deepeval.models.stt.assemblyai.schema import (
+ AssemblyAITranscriptRequest,
+ AssemblyAITranscriptResponse,
+ AssemblyAIUploadResponse,
+)
+from deepeval.models.stt.cartesia.schema import CartesiaSTTResponse
+from deepeval.models.stt.deepgram.schema import (
+ DeepgramListenParams,
+ DeepgramListenResponse,
+)
+from deepeval.models.stt.elevenlabs.schema import ElevenLabsSTTResponse
+from deepeval.models.tts.cartesia.schema import (
+ CartesiaOutputFormat,
+ CartesiaTTSRequest,
+)
+from deepeval.models.tts.elevenlabs.schema import ElevenLabsTTSRequest
+from tests.test_core.test_models.speech_stubs import FakeTransport, wav_audio
+
+
+#
+# dump_request
+#
+
+
+def test_unset_optional_fields_are_left_off_the_wire():
+ # Several of these APIs read an explicit null as an error, so a schema can
+ # only declare every optional parameter if unset ones are dropped.
+ dumped = dump_request(
+ ElevenLabsTTSRequest(text="hi", model_id="eleven_flash_v2_5")
+ )
+
+ assert dumped == {"text": "hi", "model_id": "eleven_flash_v2_5"}
+
+
+def test_a_set_optional_field_is_kept():
+ dumped = dump_request(
+ ElevenLabsTTSRequest(
+ text="hi", model_id="m", language_code="es", voice_settings={"s": 1}
+ )
+ )
+
+ assert dumped["language_code"] == "es"
+ assert dumped["voice_settings"] == {"s": 1}
+
+
+def test_unknown_fields_survive_so_new_provider_params_are_reachable():
+ # This is what makes generation_kwargs useful before deepeval has a field
+ # for a newly shipped provider parameter.
+ dumped = dump_request(
+ ElevenLabsTTSRequest(
+ text="hi", model_id="m", some_brand_new_flag=True, seed=7
+ )
+ )
+
+ assert dumped["some_brand_new_flag"] is True
+ assert dumped["seed"] == 7
+
+
+def test_nested_objects_are_serialized_as_objects():
+ dumped = dump_request(
+ CartesiaTTSRequest(
+ model_id="sonic-3.6",
+ transcript="hi",
+ voice="v",
+ output_format=CartesiaOutputFormat(
+ container="raw", sample_rate=24000
+ ),
+ )
+ )
+
+ assert dumped["output_format"] == {
+ "container": "raw",
+ "sample_rate": 24000,
+ "encoding": "pcm_s16le",
+ }
+
+
+def test_a_declared_field_given_the_wrong_type_is_rejected():
+ # Modelling requests is what buys this: a typo in generation_kwargs that
+ # collides with a real field fails here rather than at the provider.
+ with pytest.raises(Exception, match="sample_rate"):
+ CartesiaOutputFormat(container="raw", sample_rate="not-a-number")
+
+
+def test_model_prefixed_fields_do_not_warn_about_pydantics_namespace():
+ # ElevenLabs and Cartesia both name a field `model_id`, which collides with
+ # pydantic's reserved `model_` prefix unless the schema clears it.
+ with warnings.catch_warnings():
+ warnings.simplefilter("error")
+ ElevenLabsTTSRequest(text="hi", model_id="m")
+ CartesiaTTSRequest(
+ model_id="m",
+ transcript="hi",
+ voice="v",
+ output_format=CartesiaOutputFormat(
+ container="raw", sample_rate=8000
+ ),
+ )
+ DeepgramListenParams(model="nova-3")
+
+
+#
+# parse_response
+#
+
+
+def test_unknown_response_fields_are_tolerated():
+ # Providers add fields without warning; a run should not care.
+ payload = parse_response(
+ CartesiaSTTResponse,
+ {"text": "hi", "duration": 1.0, "a_field_from_next_year": {"x": 1}},
+ provider_label="Cartesia",
+ )
+
+ assert payload.text == "hi"
+
+
+def test_a_response_of_the_wrong_shape_names_the_provider_and_the_cause():
+ with pytest.raises(SpeechHTTPError) as excinfo:
+ parse_response(
+ CartesiaSTTResponse,
+ {"duration": "not-a-number"},
+ provider_label="Cartesia",
+ )
+
+ message = str(excinfo.value)
+ assert "Cartesia" in message
+ assert "changed its response format" in message
+ assert "duration" in message
+
+
+def test_a_missing_required_response_field_is_reported():
+ with pytest.raises(SpeechHTTPError, match="AssemblyAI"):
+ parse_response(
+ AssemblyAIUploadResponse, {}, provider_label="AssemblyAI"
+ )
+
+
+#
+# Silence vs. a changed response shape
+#
+
+
+@pytest.mark.parametrize(
+ "schema,silent,absent",
+ [
+ (CartesiaSTTResponse, {"text": ""}, {"duration": 1.0}),
+ (ElevenLabsSTTResponse, {"text": ""}, {"audio_duration_secs": 1.0}),
+ (AssemblyAITranscriptResponse, {"text": ""}, {"status": "completed"}),
+ (
+ DeepgramListenResponse,
+ {"results": {"channels": [{"alternatives": [{"transcript": ""}]}]}},
+ {"results": {"channels": []}},
+ ),
+ ],
+ ids=["cartesia", "11labs", "assemblyai", "deepgram"],
+)
+def test_transcript_tells_silence_apart_from_a_missing_field(
+ schema, silent, absent
+):
+ # Silence is a legitimate answer and comes back as an empty string. `None`
+ # means the field deepeval reads was not in the response at all, which is
+ # what a changed response format would look like.
+ assert schema.model_validate(silent).transcript() == ""
+ assert schema.model_validate(absent).transcript() is None
+
+
+def test_elevenlabs_falls_back_to_per_channel_transcripts():
+ payload = ElevenLabsSTTResponse.model_validate(
+ {"transcripts": [{"text": "left"}, {"text": "right"}]}
+ )
+
+ assert payload.transcript() == "left right"
+
+
+def test_deepgram_reads_the_first_alternative_it_finds():
+ payload = DeepgramListenResponse.model_validate(
+ {
+ "results": {
+ "channels": [
+ {"alternatives": []},
+ {"alternatives": [{"transcript": "second channel"}]},
+ ]
+ }
+ }
+ )
+
+ assert payload.transcript() == "second channel"
+
+
+#
+# AssemblyAI poll-loop helpers
+#
+
+
+@pytest.mark.parametrize(
+ "status,finished,failed",
+ [
+ ("queued", False, False),
+ ("processing", False, False),
+ ("completed", True, False),
+ ("error", True, True),
+ # The synchronous endpoint returns a finished transcript with no queue
+ # to report on, so it sends no status at all.
+ (None, True, False),
+ ],
+)
+def test_status_drives_the_poll_loop(status, finished, failed):
+ payload = AssemblyAITranscriptResponse.model_validate(
+ {"status": status} if status else {}
+ )
+
+ assert payload.finished is finished
+ assert payload.failed is failed
+
+
+def test_assemblyai_asks_for_models_as_an_ordered_list():
+ # The singular `speech_model` field is deprecated.
+ dumped = dump_request(
+ AssemblyAITranscriptRequest(
+ audio_url="https://cdn/x", speech_models=["universal-3-5-pro"]
+ )
+ )
+
+ assert dumped == {
+ "audio_url": "https://cdn/x",
+ "speech_models": ["universal-3-5-pro"],
+ }
+
+
+#
+# The models actually route their kwargs through the schemas
+#
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+ "factory,kwarg,where",
+ [
+ (
+ lambda **kw: ElevenLabsTTSModel(api_key="k", **kw),
+ "generation_kwargs",
+ "json",
+ ),
+ (
+ lambda **kw: DeepgramTTSModel(api_key="k", **kw),
+ "generation_kwargs",
+ "params",
+ ),
+ (
+ lambda **kw: CartesiaTTSModel(api_key="k", voice="v", **kw),
+ "generation_kwargs",
+ "json",
+ ),
+ ],
+ ids=["11labs", "deepgram", "cartesia"],
+)
+async def test_generation_kwargs_reach_the_wire_through_the_schema(
+ factory, kwarg, where
+):
+ model = factory(**{kwarg: {"an_unmodelled_provider_flag": True}})
+ transport = FakeTransport(content=b"RIFF")
+ model.model = transport
+
+ await model.a_synthesize("hi")
+
+ assert transport.last[where]["an_unmodelled_provider_flag"] is True
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+ "factory,where",
+ [
+ (lambda **kw: DeepgramSTTModel(api_key="k", **kw), "params"),
+ (lambda **kw: ElevenLabsSTTModel(api_key="k", **kw), "multipart"),
+ (lambda **kw: CartesiaSTTModel(api_key="k", **kw), "multipart"),
+ ],
+ ids=["deepgram", "11labs", "cartesia"],
+)
+async def test_transcription_kwargs_reach_the_wire_through_the_schema(
+ factory, where
+):
+ model = factory(transcription_kwargs={"an_unmodelled_provider_flag": True})
+ transport = FakeTransport(
+ json={
+ "text": "hi",
+ "results": {"channels": [{"alternatives": [{"transcript": "hi"}]}]},
+ }
+ )
+ model.model = transport
+
+ await model.a_transcribe(wav_audio())
+
+ sent = transport.last[where]
+ fields = sent.fields if where == "multipart" else sent
+ assert fields["an_unmodelled_provider_flag"] is True
diff --git a/tests/test_core/test_models/test_speech_stt.py b/tests/test_core/test_models/test_speech_stt.py
new file mode 100644
index 000000000..ed33648d6
--- /dev/null
+++ b/tests/test_core/test_models/test_speech_stt.py
@@ -0,0 +1,438 @@
+"""The HTTP-based STT providers: transcript extraction, routing and cost."""
+
+import json as jsonlib
+from typing import List
+
+import pytest
+
+from deepeval.models import (
+ AssemblyAISTTModel,
+ CartesiaSTTModel,
+ DeepgramSTTModel,
+ ElevenLabsSTTModel,
+ OpenAISTTModel,
+)
+from deepeval.models.speech import SpeechHTTPError
+from deepeval.models.stt.assemblyai.assemblyai import SYNC_URL
+from deepeval.test_case import AudioChunk
+from tests.test_core.test_models.speech_stubs import FakeTransport, wav_audio
+
+
+def _deepgram(**kwargs):
+ return DeepgramSTTModel(api_key="test-key", **kwargs)
+
+
+def _elevenlabs(**kwargs):
+ return ElevenLabsSTTModel(api_key="test-key", **kwargs)
+
+
+def _cartesia(**kwargs):
+ return CartesiaSTTModel(api_key="test-key", **kwargs)
+
+
+def _assemblyai(**kwargs):
+ kwargs.setdefault("poll_interval_seconds", 0.0)
+ return AssemblyAISTTModel(api_key="test-key", **kwargs)
+
+
+def _with_transport(model, **transport_kwargs):
+ transport = FakeTransport(**transport_kwargs)
+ model.model = transport
+ return model, transport
+
+
+def _deepgram_payload(transcript: str):
+ return {
+ "results": {
+ "channels": [{"alternatives": [{"transcript": transcript}]}]
+ }
+ }
+
+
+#
+# Truncated-audio padding
+#
+
+
+@pytest.mark.parametrize(
+ "factory,expected",
+ [
+ # Whisper-derived and autoregressive: they finish a clipped word,
+ # inventing speech the caller never heard.
+ (_elevenlabs, 0.3),
+ (_cartesia, 0.3),
+ # These stop where the audio stops, so padding would only add silence.
+ (_deepgram, 0.0),
+ (_assemblyai, 0.0),
+ ],
+ ids=["11labs", "cartesia", "deepgram", "assemblyai"],
+)
+def test_padding_follows_whether_the_model_completes_clipped_words(
+ factory, expected
+):
+ assert factory().truncated_audio_pad_seconds == expected
+
+
+#
+# Deepgram
+#
+
+
+@pytest.mark.asyncio
+async def test_deepgram_reads_the_transcript_out_of_its_nested_response():
+ model, transport = _with_transport(
+ _deepgram(), json=_deepgram_payload("hello there")
+ )
+
+ text, _ = await model.a_transcribe(wav_audio())
+
+ assert text == "hello there"
+ assert transport.last["path"] == "/v1/listen"
+ assert transport.params_of()["model"] == "nova-3"
+ assert transport.params_of()["smart_format"] == "True"
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+ "payload",
+ [
+ {"results": {"channels": []}},
+ {"results": {"channels": [{"alternatives": []}]}},
+ {"results": {}},
+ {},
+ ],
+)
+async def test_deepgram_treats_a_transcriptless_response_as_silence(payload):
+ # Silence is a successful response with nothing in it, not an error.
+ model, _ = _with_transport(_deepgram(), json=payload)
+
+ text, _ = await model.a_transcribe(wav_audio())
+
+ assert text == ""
+
+
+@pytest.mark.asyncio
+async def test_deepgram_turns_auto_into_explicit_language_detection():
+ # Deepgram detects only when asked, so an omitted language means English,
+ # not detection.
+ model, transport = _with_transport(
+ _deepgram(language="en"), json=_deepgram_payload("hola")
+ )
+
+ await model.a_transcribe(wav_audio(), language="auto")
+
+ params = transport.params_of()
+ assert params["detect_language"] == "True"
+ assert "language" not in params
+
+
+@pytest.mark.asyncio
+async def test_deepgram_forwards_a_configured_language():
+ model, transport = _with_transport(
+ _deepgram(language="es"), json=_deepgram_payload("hola")
+ )
+
+ await model.a_transcribe(wav_audio())
+
+ assert transport.params_of()["language"] == "es"
+
+
+@pytest.mark.asyncio
+async def test_deepgram_prices_by_the_minute_of_audio():
+ model, _ = _with_transport(_deepgram(), json=_deepgram_payload("hi"))
+
+ _, cost = await model.a_transcribe(wav_audio(seconds=30))
+
+ assert cost == pytest.approx(0.5 * 0.0043)
+
+
+#
+# ElevenLabs
+#
+
+
+@pytest.mark.asyncio
+async def test_elevenlabs_reads_a_flat_text_field():
+ model, transport = _with_transport(
+ _elevenlabs(), json={"text": "hello", "audio_duration_secs": 3600.0}
+ )
+
+ text, cost = await model.a_transcribe(wav_audio())
+
+ assert text == "hello"
+ assert transport.last["path"] == "/v1/speech-to-text"
+ assert transport.last["multipart"].fields["model_id"] == "scribe_v2"
+ # Priced per hour, and the response reports exactly one hour of audio.
+ assert cost == pytest.approx(0.22)
+
+
+@pytest.mark.asyncio
+async def test_elevenlabs_joins_a_multi_channel_response():
+ model, _ = _with_transport(
+ _elevenlabs(),
+ json={"transcripts": [{"text": "left"}, {"text": "right"}]},
+ )
+
+ text, _ = await model.a_transcribe(wav_audio())
+
+ assert text == "left right"
+
+
+@pytest.mark.asyncio
+async def test_elevenlabs_omits_the_language_to_ask_for_detection():
+ # Scribe detects whenever language_code is absent.
+ model, transport = _with_transport(
+ _elevenlabs(language="en"), json={"text": "hola"}
+ )
+
+ await model.a_transcribe(wav_audio(), language="auto")
+
+ assert "language_code" not in transport.last["multipart"].fields
+
+
+def test_elevenlabs_does_not_default_to_the_deprecated_scribe_v1():
+ assert _elevenlabs().get_model_name() == "scribe_v2"
+
+
+#
+# Cartesia
+#
+
+
+@pytest.mark.asyncio
+async def test_cartesia_reads_text_and_prices_from_the_reported_duration():
+ model, transport = _with_transport(
+ _cartesia(cost_per_minute=0.01),
+ json={"text": "hello", "duration": 120.0},
+ )
+
+ text, cost = await model.a_transcribe(wav_audio())
+
+ assert text == "hello"
+ assert transport.last["path"] == "/stt"
+ assert transport.last["multipart"].fields["model"] == "ink-whisper"
+ assert cost == pytest.approx(0.02)
+
+
+@pytest.mark.asyncio
+async def test_cartesia_reports_no_cost_without_an_explicit_rate():
+ model, _ = _with_transport(_cartesia(), json={"text": "hi", "duration": 60})
+
+ _, cost = await model.a_transcribe(wav_audio())
+
+ assert cost is None
+
+
+def test_cartesia_stt_defaults_to_a_model_the_batch_endpoint_serves():
+ # ink-2 is realtime only; the batch endpoint rejects it.
+ assert _cartesia().get_model_name() == "ink-whisper"
+
+
+def test_cartesia_stt_pins_the_dated_api_version():
+ assert _cartesia().model.headers["Cartesia-Version"] == "2026-08-14"
+
+
+#
+# AssemblyAI
+#
+
+
+@pytest.mark.asyncio
+async def test_assemblyai_uses_the_one_shot_endpoint_for_a_short_turn():
+ model, transport = _with_transport(_assemblyai(), json={"text": "hello"})
+
+ text, cost = await model.a_transcribe(wav_audio(seconds=2))
+
+ assert text == "hello"
+ assert transport.last["path"] == SYNC_URL
+ assert transport.last["headers"] == {"X-AAI-Model": "universal-3-5-pro"}
+ assert transport.last["multipart"].file_field == "audio"
+ assert cost == pytest.approx(2 / 3600 * 0.45)
+
+
+@pytest.mark.asyncio
+async def test_assemblyai_falls_back_to_upload_and_poll_beyond_the_sync_ceiling():
+ model, transport = _with_transport(
+ _assemblyai(),
+ json_sequence=[
+ {"upload_url": "https://cdn.assemblyai.com/upload/abc"},
+ {"id": "t-1", "status": "queued"},
+ {"id": "t-1", "status": "processing"},
+ {"id": "t-1", "status": "completed", "text": "a long one"},
+ ],
+ )
+
+ text, cost = await model.a_transcribe(wav_audio(seconds=1, duration=3600.0))
+
+ assert text == "a long one"
+ assert transport.paths() == [
+ "/v2/upload",
+ "/v2/transcript",
+ "/v2/transcript/t-1",
+ "/v2/transcript/t-1",
+ ]
+ # The async flow is priced per hour, well below the sync endpoint's rate.
+ assert cost == pytest.approx(0.21)
+
+
+@pytest.mark.asyncio
+async def test_assemblyai_asks_for_the_model_as_a_fallback_list():
+ model, transport = _with_transport(
+ _assemblyai(),
+ json_sequence=[
+ {"upload_url": "https://cdn/x"},
+ {"id": "t", "status": "completed", "text": "ok"},
+ ],
+ )
+
+ await model.a_transcribe(wav_audio(duration=3600.0))
+
+ # `speech_model` is deprecated in favour of the ordered list.
+ body = transport.calls[1]["json"]
+ assert body["speech_models"] == ["universal-3-5-pro"]
+ assert "speech_model" not in body
+
+
+@pytest.mark.asyncio
+async def test_assemblyai_surfaces_a_failed_transcription():
+ model, _ = _with_transport(
+ _assemblyai(),
+ json_sequence=[
+ {"upload_url": "https://cdn/x"},
+ {"id": "t", "status": "error", "error": "audio too quiet"},
+ ],
+ )
+
+ with pytest.raises(SpeechHTTPError, match="audio too quiet"):
+ await model.a_transcribe(wav_audio(duration=3600.0))
+
+
+@pytest.mark.asyncio
+async def test_assemblyai_takes_the_reliable_path_when_duration_is_unknown():
+ # An unmeasurable clip could be longer than the sync ceiling, and the slow
+ # path always works.
+ model, transport = _with_transport(
+ _assemblyai(),
+ json_sequence=[
+ {"upload_url": "https://cdn/x"},
+ {"id": "t", "status": "completed", "text": "ok"},
+ ],
+ )
+ audio = wav_audio()
+ audio.dataBase64 = ""
+
+ text, _ = await model.a_transcribe(audio)
+
+ assert text == "ok"
+ assert transport.paths()[0] == "/v2/upload"
+
+
+def test_assemblyai_does_not_default_to_the_deprecated_slam_model():
+ assert _assemblyai().get_model_name() == "universal-3-5-pro"
+
+
+#
+# The buffered partial-transcript stream, shared by every STT model
+#
+
+
+async def _chunks(count: int, sample_rate: int = 24000):
+ # Half a second each, so two chunks cross the one-second partial threshold.
+ payload = b"\x00\x00" * (sample_rate // 2)
+ for index in range(count):
+ yield AudioChunk.from_bytes(
+ payload,
+ "audio/pcm",
+ sampleRate=sample_rate,
+ encoding="pcm",
+ final=index == count - 1,
+ )
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+ "factory",
+ [_deepgram, _elevenlabs, _cartesia, _assemblyai],
+ ids=["deepgram", "11labs", "cartesia", "assemblyai"],
+)
+async def test_every_stt_model_can_produce_partial_transcripts(factory):
+ # The base class raises instead of streaming, so a model that does not opt
+ # in cannot be followed while the agent is still speaking.
+ model = factory()
+ assert model.supports_streaming() is True
+
+ seen: List[str] = []
+ calls = {"n": 0}
+
+ async def fake_transcribe(audio, **kwargs):
+ calls["n"] += 1
+ return f"partial {calls['n']}", None
+
+ model.a_transcribe = fake_transcribe
+ async for text in model.a_transcribe_stream(_chunks(4)):
+ seen.append(text)
+
+ assert seen == ["partial 1", "partial 2"]
+
+
+@pytest.mark.asyncio
+async def test_a_partial_that_has_not_changed_is_not_re_emitted():
+ model = _deepgram()
+
+ async def fake_transcribe(audio, **kwargs):
+ return "same text", None
+
+ model.a_transcribe = fake_transcribe
+ seen = [text async for text in model.a_transcribe_stream(_chunks(6))]
+
+ assert seen == ["same text"]
+
+
+@pytest.mark.asyncio
+async def test_openai_still_streams_partials_through_the_shared_mixin():
+ # OpenAISTTModel gave up its own copy of this loop; check it kept the
+ # behaviour.
+ model = OpenAISTTModel(api_key="test-key")
+
+ async def fake_transcribe(audio, **kwargs):
+ return "openai partial", None
+
+ model.a_transcribe = fake_transcribe
+ seen = [text async for text in model.a_transcribe_stream(_chunks(2))]
+
+ assert seen == ["openai partial"]
+ assert model.supports_streaming() is True
+
+
+#
+# Sync parity
+#
+
+
+@pytest.mark.parametrize(
+ "factory,payload",
+ [
+ (_deepgram, _deepgram_payload("hi")),
+ (_elevenlabs, {"text": "hi"}),
+ (_cartesia, {"text": "hi"}),
+ (_assemblyai, {"text": "hi"}),
+ ],
+ ids=["deepgram", "11labs", "cartesia", "assemblyai"],
+)
+def test_the_sync_path_requests_the_same_thing_as_the_async_one(
+ factory, payload
+):
+ import asyncio
+
+ audio = wav_audio(seconds=2)
+
+ sync_model, sync_transport = _with_transport(factory(), json=payload)
+ sync_text, _ = sync_model.transcribe(audio)
+
+ async_model, async_transport = _with_transport(factory(), json=payload)
+ async_text, _ = asyncio.run(async_model.a_transcribe(audio))
+
+ assert sync_text == async_text
+ assert jsonlib.dumps(
+ sync_transport.last, sort_keys=True, default=str
+ ) == jsonlib.dumps(async_transport.last, sort_keys=True, default=str)
diff --git a/tests/test_core/test_models/test_speech_transport.py b/tests/test_core/test_models/test_speech_transport.py
new file mode 100644
index 000000000..8debc642b
--- /dev/null
+++ b/tests/test_core/test_models/test_speech_transport.py
@@ -0,0 +1,340 @@
+"""`SpeechTransport` against a real local HTTP server.
+
+The provider tests stub the transport out, which leaves the transport itself
+unexercised: whether it actually reaches the network, encodes a multipart body
+the way a server can parse it, and turns status codes into the exception classes
+the retry policy classifies on.
+"""
+
+import asyncio
+import json as jsonlib
+import threading
+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
+from typing import Iterator
+
+import pytest
+
+from deepeval.models.retry_policy import SPEECH_ERROR_POLICY, make_is_transient
+from deepeval.models.speech import (
+ Multipart,
+ SpeechAuthError,
+ SpeechHTTPError,
+ SpeechRateLimitError,
+ SpeechTransport,
+)
+
+_received = {}
+
+
+class _Handler(BaseHTTPRequestHandler):
+ def log_message(self, *args): # keep the test output clean
+ pass
+
+ def _respond(
+ self, status: int, body: bytes, content_type="application/json"
+ ):
+ self.send_response(status)
+ self.send_header("Content-Type", content_type)
+ self.send_header("Content-Length", str(len(body)))
+ self.end_headers()
+ self.wfile.write(body)
+
+ def do_GET(self):
+ if self.path.startswith("/status/"):
+ status = int(self.path.rsplit("/", 1)[1])
+ self._respond(status, b'{"error":"boom"}')
+ return
+ if self.path == "/not-json":
+ self._respond(200, b"nope", "text/html")
+ return
+ _received["query"] = self.path
+ self._respond(200, jsonlib.dumps({"path": self.path}).encode())
+
+ def do_POST(self):
+ length = int(self.headers.get("Content-Length") or 0)
+ body = self.rfile.read(length)
+ _received["headers"] = dict(self.headers)
+ _received["body"] = body
+ if self.path == "/stream":
+ self.send_response(200)
+ self.send_header("Content-Type", "audio/pcm")
+ self.end_headers()
+ for _ in range(4):
+ self.wfile.write(b"\x01\x02" * 512)
+ return
+ self._respond(200, jsonlib.dumps({"ok": True}).encode())
+
+
+@pytest.fixture(scope="module")
+def server() -> Iterator[str]:
+ httpd = ThreadingHTTPServer(("127.0.0.1", 0), _Handler)
+ thread = threading.Thread(target=httpd.serve_forever, daemon=True)
+ thread.start()
+ host, port = httpd.server_address[:2]
+ yield f"http://{host}:{port}"
+ httpd.shutdown()
+ httpd.server_close()
+
+
+def _transport(base_url: str, **kwargs) -> SpeechTransport:
+ return SpeechTransport(
+ provider_label="Test",
+ base_url=base_url,
+ headers={"X-Api-Key": "secret", "X-Skipped": None},
+ **kwargs,
+ )
+
+
+def test_none_valued_headers_are_dropped(server):
+ assert "X-Skipped" not in _transport(server).headers
+
+
+def test_absolute_urls_bypass_the_base_url(server):
+ transport = _transport(server)
+
+ assert transport.url("/v1/x") == f"{server}/v1/x"
+ assert transport.url("https://other.example/y") == "https://other.example/y"
+
+
+#
+# Encoding
+#
+
+
+@pytest.mark.parametrize(
+ "params,expected",
+ [
+ ({"flag": True}, "flag=true"),
+ ({"flag": False}, "flag=false"),
+ ({"skip": None, "keep": "1"}, "keep=1"),
+ ({"n": 24000}, "n=24000"),
+ ],
+)
+def test_booleans_and_none_are_encoded_the_way_these_apis_expect(
+ server, params, expected
+):
+ # `?flag=True` is not the `true` any of these providers look for.
+ transport = _transport(server)
+
+ payload = transport.request_json("GET", "/echo", params=params)
+
+ assert expected in payload["path"]
+ if "skip" in params:
+ assert "skip" not in payload["path"]
+
+
+def test_a_repeated_param_is_sent_once_per_value(server):
+ # Deepgram spells multiple keyterms as the same key repeated, which a dict
+ # cannot express.
+ transport = _transport(server)
+
+ payload = transport.request_json(
+ "GET", "/echo", params={"keyterm": ["alpha", "beta"]}
+ )
+
+ assert "keyterm=alpha" in payload["path"]
+ assert "keyterm=beta" in payload["path"]
+
+
+def test_multipart_bodies_are_parseable_and_carry_scalar_fields(server):
+ transport = _transport(server)
+
+ transport.request_json(
+ "POST",
+ "/upload",
+ multipart=Multipart(
+ file_field="audio",
+ filename="turn.wav",
+ content=b"RIFFDATA",
+ content_type="audio/wav",
+ fields={"model": "m-1", "diarize": True, "skip": None},
+ ),
+ )
+
+ body = _received["body"]
+ assert b'name="audio"' in body
+ assert b'filename="turn.wav"' in body
+ assert b"RIFFDATA" in body
+ assert b'name="model"' in body and b"m-1" in body
+ # Booleans have to reach the server in the JSON spelling, not Python's.
+ assert b"true" in body and b"True" not in body
+ assert b'name="skip"' not in body
+
+
+@pytest.mark.asyncio
+async def test_the_async_client_encodes_multipart_the_same_way(server):
+ transport = _transport(server)
+
+ await transport.a_request_json(
+ "POST",
+ "/upload",
+ multipart=Multipart(
+ file_field="audio",
+ filename="turn.wav",
+ content=b"RIFFDATA",
+ content_type="audio/wav",
+ fields={"model": "m-1"},
+ ),
+ )
+
+ body = _received["body"]
+ assert b'filename="turn.wav"' in body
+ assert b"RIFFDATA" in body
+ assert b'name="model"' in body
+
+
+def test_configured_headers_reach_the_server(server):
+ _transport(server).request_json("POST", "/upload", content=b"x")
+
+ assert _received["headers"]["X-Api-Key"] == "secret"
+
+
+def test_per_request_headers_are_merged_over_the_defaults(server):
+ _transport(server).request_json(
+ "POST", "/upload", content=b"x", headers={"X-AAI-Model": "m"}
+ )
+
+ assert _received["headers"]["X-AAI-Model"] == "m"
+ assert _received["headers"]["X-Api-Key"] == "secret"
+
+
+#
+# Streaming
+#
+
+
+@pytest.mark.asyncio
+async def test_streaming_yields_the_body_in_pieces(server):
+ transport = _transport(server)
+
+ reads = [
+ data
+ async for data in transport.a_stream_bytes(
+ "POST", "/stream", content=b"go", chunk_size=256
+ )
+ ]
+
+ assert len(reads) > 1
+ assert b"".join(reads) == b"\x01\x02" * 512 * 4
+
+
+@pytest.mark.asyncio
+async def test_many_sequential_requests_do_not_leak_sessions(server):
+ # A cached session would be bound to a loop nobody closes; this checks the
+ # per-request session actually works under repetition.
+ transport = _transport(server)
+
+ for _ in range(5):
+ assert await transport.a_request_json("POST", "/upload", content=b"x")
+
+
+def test_a_model_can_be_used_from_more_than_one_event_loop(server):
+ # Each `asyncio.run` makes a fresh loop, and an aiohttp session bound to a
+ # previous one would raise here.
+ transport = _transport(server)
+
+ for _ in range(3):
+ payload = asyncio.run(
+ transport.a_request_json("POST", "/upload", content=b"x")
+ )
+ assert payload == {"ok": True}
+
+
+#
+# Error translation
+#
+
+
+@pytest.mark.parametrize(
+ "status,expected",
+ [
+ (401, SpeechAuthError),
+ (403, SpeechAuthError),
+ (429, SpeechRateLimitError),
+ (400, SpeechHTTPError),
+ (500, SpeechHTTPError),
+ (503, SpeechHTTPError),
+ ],
+)
+def test_status_codes_become_the_classes_the_retry_policy_reads(
+ server, status, expected
+):
+ transport = _transport(server)
+
+ with pytest.raises(expected) as excinfo:
+ transport.request_json("GET", f"/status/{status}")
+
+ assert excinfo.value.status_code == status
+ # The body explains what went wrong, so it belongs in the message.
+ assert "boom" in str(excinfo.value)
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+ "status,expected",
+ [
+ (401, SpeechAuthError),
+ (429, SpeechRateLimitError),
+ (500, SpeechHTTPError),
+ ],
+)
+async def test_the_async_client_translates_errors_the_same_way(
+ server, status, expected
+):
+ with pytest.raises(expected):
+ await _transport(server).a_request_json("GET", f"/status/{status}")
+
+
+@pytest.mark.asyncio
+async def test_a_streaming_error_is_raised_with_its_body(server):
+ with pytest.raises(SpeechHTTPError, match="boom"):
+ async for _ in _transport(server).a_stream_bytes("GET", "/status/500"):
+ pass
+
+
+def test_an_auth_failure_says_to_check_the_key(server):
+ with pytest.raises(SpeechAuthError, match="API key"):
+ _transport(server).request_json("GET", "/status/401")
+
+
+def test_a_non_json_response_is_reported_as_such(server):
+ with pytest.raises(SpeechHTTPError, match="not valid JSON"):
+ _transport(server).request_json("GET", "/not-json")
+
+
+@pytest.mark.parametrize(
+ "status,retryable",
+ [
+ # Bad credentials will be just as bad next time.
+ (401, False),
+ (403, False),
+ # Rate limits and provider faults are worth another attempt.
+ (429, True),
+ (500, True),
+ (503, True),
+ # A malformed request is permanent.
+ (400, False),
+ (404, False),
+ ],
+)
+def test_the_retry_policy_classifies_speech_failures(server, status, retryable):
+ is_transient = make_is_transient(SPEECH_ERROR_POLICY)
+ transport = _transport(server)
+
+ try:
+ transport.request_json("GET", f"/status/{status}")
+ raise AssertionError("expected a failure")
+ except SpeechHTTPError as exc:
+ assert is_transient(exc) is retryable
+
+
+def test_a_connection_failure_is_treated_as_transient():
+ is_transient = make_is_transient(SPEECH_ERROR_POLICY)
+ # Port 1 is not listening, so this is a genuine connection error rather
+ # than a simulated one.
+ transport = _transport("http://127.0.0.1:1", timeout=2.0)
+
+ with pytest.raises(Exception) as excinfo:
+ transport.request_json("GET", "/echo")
+
+ assert is_transient(excinfo.value) is True
diff --git a/tests/test_core/test_models/test_speech_tts.py b/tests/test_core/test_models/test_speech_tts.py
new file mode 100644
index 000000000..f3b325a62
--- /dev/null
+++ b/tests/test_core/test_models/test_speech_tts.py
@@ -0,0 +1,322 @@
+"""The HTTP-based TTS providers: framing, request shape and cost."""
+
+import json as jsonlib
+from typing import List
+
+import pytest
+
+from deepeval.errors import DeepEvalError
+from deepeval.models import (
+ CartesiaTTSModel,
+ DeepgramTTSModel,
+ ElevenLabsTTSModel,
+)
+from deepeval.models.tts._frames import frame_size_bytes
+from deepeval.models.tts.deepgram.deepgram import apply_voice
+from tests.test_core.test_models.speech_stubs import FakeTransport
+
+_RATE = 24000
+_FRAME = frame_size_bytes(_RATE)
+
+
+def _elevenlabs(**kwargs):
+ model = ElevenLabsTTSModel(api_key="test-key", **kwargs)
+ return model
+
+
+def _deepgram(**kwargs):
+ return DeepgramTTSModel(api_key="test-key", **kwargs)
+
+
+def _cartesia(**kwargs):
+ return CartesiaTTSModel(api_key="test-key", voice="voice-id", **kwargs)
+
+
+def _with_transport(model, **transport_kwargs):
+ transport = FakeTransport(**transport_kwargs)
+ model.model = transport
+ return model, transport
+
+
+async def _collect(model, text: str = "Hello there.", **kwargs) -> List:
+ return [chunk async for chunk in model.a_synthesize_stream(text, **kwargs)]
+
+
+#
+# Framing (shared by all three providers)
+#
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+ "factory", [_elevenlabs, _deepgram, _cartesia], ids=["11labs", "dg", "ct"]
+)
+async def test_stream_yields_fixed_frames_with_a_final_flag(factory):
+ # Three and a half frames, arriving in reads that straddle frame edges.
+ audio = bytes(range(256)) * 66
+ reads = [audio[i : i + 1000] for i in range(0, len(audio), 1000)]
+ model, _ = _with_transport(factory(), stream_reads=reads)
+
+ chunks = await _collect(model)
+
+ sizes = [len(chunk.get_bytes()) for chunk in chunks]
+ assert sizes == [_FRAME] * (len(audio) // _FRAME) + [len(audio) % _FRAME]
+ assert [chunk.final for chunk in chunks] == [False, False, False, True]
+ assert b"".join(chunk.get_bytes() for chunk in chunks) == audio
+
+
+@pytest.mark.asyncio
+async def test_an_exact_multiple_of_a_frame_still_ends_with_a_final_chunk():
+ model, _ = _with_transport(
+ _elevenlabs(), stream_reads=[b"\x07\x00" * _FRAME]
+ )
+
+ chunks = await _collect(model)
+
+ assert [len(c.get_bytes()) for c in chunks] == [_FRAME] * 2
+ assert [c.final for c in chunks] == [False, True]
+
+
+@pytest.mark.asyncio
+async def test_stream_describes_its_frames_as_pcm():
+ model, _ = _with_transport(_elevenlabs(), stream_reads=[b"\x01\x02" * 100])
+
+ (chunk,) = await _collect(model)
+
+ assert chunk.mimeType == "audio/pcm"
+ assert chunk.encoding == "pcm"
+ assert chunk.sampleRate == _RATE
+ assert chunk.duration == pytest.approx(200 / 2 / _RATE)
+ assert chunk.final is True
+
+
+@pytest.mark.asyncio
+async def test_a_silent_stream_yields_nothing():
+ model, _ = _with_transport(_elevenlabs(), stream_reads=[])
+
+ assert await _collect(model) == []
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+ "factory", [_elevenlabs, _deepgram, _cartesia], ids=["11labs", "dg", "ct"]
+)
+async def test_streaming_is_advertised(factory):
+ assert factory().supports_streaming() is True
+
+
+#
+# ElevenLabs
+#
+
+
+@pytest.mark.asyncio
+async def test_elevenlabs_batch_asks_for_a_wav_and_streams_pcm():
+ model, transport = _with_transport(
+ _elevenlabs(), content=b"RIFF", stream_reads=[b"\x00\x00"]
+ )
+
+ audio, _ = await model.a_synthesize("Hi")
+ assert transport.params_of()["output_format"] == f"wav_{_RATE}"
+ assert audio.mimeType == "audio/wav"
+ assert audio.sampleRate == _RATE
+
+ await _collect(model)
+ # Streaming cannot use a container: its header declares a length the first
+ # frame does not know yet.
+ assert transport.params_of()["output_format"] == f"pcm_{_RATE}"
+
+
+@pytest.mark.asyncio
+async def test_elevenlabs_puts_the_voice_in_the_path_and_the_model_in_the_body():
+ model, transport = _with_transport(_elevenlabs(), content=b"RIFF")
+
+ await model.a_synthesize("Hi", voice="custom-voice")
+
+ assert transport.last["path"] == "/v1/text-to-speech/custom-voice"
+ assert transport.last["json"] == {
+ "text": "Hi",
+ "model_id": "eleven_flash_v2_5",
+ }
+
+
+@pytest.mark.asyncio
+async def test_elevenlabs_streams_from_the_stream_endpoint():
+ model, transport = _with_transport(
+ _elevenlabs(), stream_reads=[b"\x00\x00"]
+ )
+
+ await _collect(model)
+
+ assert transport.last["path"].endswith("/stream")
+
+
+def test_elevenlabs_does_not_default_to_a_sunsetting_voice():
+ # The legacy "Default" voices (Rachel, Adam, Bella, ...) stop resolving on
+ # 2027-01-01 and already 404 for accounts created after March 2026.
+ retired = {
+ "21m00Tcm4TlvDq8ikWAM",
+ "pNInz6obpgDQGcFmaJgB",
+ "EXAVITQu4vr4xnSDxMaL",
+ "TxGEqnHWrfWFTfGW9XjX",
+ "yoZ06aMxZJJ28mfd3POQ",
+ "ErXwobaYiN019PkySvjV",
+ }
+ assert _elevenlabs().voice not in retired
+
+
+def test_elevenlabs_rejects_a_sample_rate_it_cannot_emit():
+ with pytest.raises(ValueError, match="16000"):
+ _elevenlabs(sample_rate=11111)
+
+
+def test_elevenlabs_prices_flash_below_the_expressive_models():
+ text = "a" * 1_000_000
+ assert _elevenlabs().synthesis_cost(text) == pytest.approx(50.0)
+ assert _elevenlabs(model="eleven_v3").synthesis_cost(text) == pytest.approx(
+ 100.0
+ )
+ assert _elevenlabs(cost_per_1m_chars=7.5).synthesis_cost(
+ text
+ ) == pytest.approx(7.5)
+
+
+def test_an_unknown_elevenlabs_model_has_no_price_rather_than_a_wrong_one():
+ assert (
+ _elevenlabs(model="eleven_something_new").synthesis_cost("hi") is None
+ )
+
+
+#
+# Deepgram
+#
+
+
+@pytest.mark.asyncio
+async def test_deepgram_sends_text_in_the_body_and_everything_else_in_the_query():
+ model, transport = _with_transport(_deepgram(), content=b"RIFF")
+
+ await model.a_synthesize("Hi")
+
+ assert transport.last["path"] == "/v1/speak"
+ assert transport.last["json"] == {"text": "Hi"}
+ assert transport.params_of() == {
+ "model": "aura-2-thalia-en",
+ "encoding": "linear16",
+ "container": "wav",
+ "sample_rate": str(_RATE),
+ }
+
+
+@pytest.mark.asyncio
+async def test_deepgram_drops_the_container_when_streaming():
+ model, transport = _with_transport(_deepgram(), stream_reads=[b"\x00\x00"])
+
+ await _collect(model)
+
+ assert transport.params_of()["container"] == "none"
+
+
+@pytest.mark.parametrize(
+ "model_name,voice,expected",
+ [
+ ("aura-2-thalia-en", "zeus", "aura-2-zeus-en"),
+ ("aura-2-thalia-en", "aura-2-luna-es", "aura-2-luna-es"),
+ ("aura-asteria-en", "orion", "aura-orion-en"),
+ ("aura-2-thalia-ja", "fujin", "aura-2-fujin-ja"),
+ ],
+)
+def test_deepgram_rewrites_the_voice_segment_of_the_model_name(
+ model_name, voice, expected
+):
+ # An Aura model name carries voice and language, so selecting a voice means
+ # editing the identifier rather than sending a separate field.
+ assert apply_voice(model_name, voice) == expected
+
+
+def test_deepgram_folds_a_constructor_voice_into_the_reported_model_name():
+ assert _deepgram(voice="zeus").get_model_name() == "aura-2-zeus-en"
+
+
+def test_deepgram_prices_aura_2_above_aura_1():
+ text = "a" * 1_000_000
+ assert _deepgram().synthesis_cost(text) == pytest.approx(30.0)
+ assert _deepgram(model="aura-asteria-en").synthesis_cost(
+ text
+ ) == pytest.approx(15.0)
+
+
+#
+# Cartesia
+#
+
+
+@pytest.mark.asyncio
+async def test_cartesia_nests_the_output_format_and_switches_container():
+ model, transport = _with_transport(
+ _cartesia(), content=b"RIFF", stream_reads=[b"\x00\x00"]
+ )
+
+ await model.a_synthesize("Hi")
+ body = transport.last["json"]
+ assert body["model_id"] == "sonic-3.6"
+ assert body["transcript"] == "Hi"
+ assert body["voice"] == "voice-id"
+ assert body["output_format"] == {
+ "container": "wav",
+ "encoding": "pcm_s16le",
+ "sample_rate": _RATE,
+ }
+
+ await _collect(model)
+ assert transport.last["json"]["output_format"]["container"] == "raw"
+
+
+def test_cartesia_does_not_default_to_a_retired_model():
+ # sonic-2 and sonic-turbo have left the model enum and now fail with
+ # model_not_found.
+ assert _cartesia().get_model_name() not in {"sonic-2", "sonic-turbo"}
+
+
+def test_cartesia_pins_the_dated_api_version_it_was_written_against():
+ model = _cartesia()
+ assert model.model.headers["Cartesia-Version"] == "2026-08-14"
+
+
+def test_cartesia_requires_a_voice():
+ model = CartesiaTTSModel(api_key="test-key")
+ model.model = FakeTransport(content=b"RIFF")
+
+ with pytest.raises(DeepEvalError, match="no default voice"):
+ model.synthesize("Hi")
+
+
+def test_cartesia_reports_no_cost_without_an_explicit_rate():
+ # Cartesia bills in credits and publishes no dollar conversion, so a
+ # default price would be invented.
+ assert _cartesia().synthesis_cost("a" * 1000) is None
+ assert _cartesia(cost_per_1m_chars=25.0).synthesis_cost(
+ "a" * 1_000_000
+ ) == pytest.approx(25.0)
+
+
+#
+# Sync parity
+#
+
+
+@pytest.mark.parametrize(
+ "factory", [_elevenlabs, _deepgram, _cartesia], ids=["11labs", "dg", "ct"]
+)
+def test_the_sync_path_requests_the_same_thing_as_the_async_one(factory):
+ import asyncio
+
+ sync_model, sync_transport = _with_transport(factory(), content=b"RIFF")
+ sync_model.synthesize("Hi")
+
+ async_model, async_transport = _with_transport(factory(), content=b"RIFF")
+ asyncio.run(async_model.a_synthesize("Hi"))
+
+ assert jsonlib.dumps(
+ sync_transport.last, sort_keys=True, default=str
+ ) == jsonlib.dumps(async_transport.last, sort_keys=True, default=str)
diff --git a/tests/test_core/test_voice/test_connectors/__init__.py b/tests/test_core/test_voice/test_connectors/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/tests/test_core/test_voice/test_connectors/test_pipecat.py b/tests/test_core/test_voice/test_connectors/test_pipecat.py
new file mode 100644
index 000000000..bbfd50c9c
--- /dev/null
+++ b/tests/test_core/test_voice/test_connectors/test_pipecat.py
@@ -0,0 +1,162 @@
+"""Speaking Pipecat's protobuf frames, and reading RTVI off the same socket."""
+
+import json
+
+from deepeval.voice import PipecatConnector
+from deepeval.voice.connectors import audio_utils
+from deepeval.voice.connectors.providers.pipecat import protobuf
+
+
+def _connector(**kwargs) -> PipecatConnector:
+ kwargs.setdefault("url", "ws://localhost:8765/ws")
+ return PipecatConnector(**kwargs)
+
+
+def _rtvi(message_type: str, **data) -> bytes:
+ payload = {"label": "rtvi-ai", "type": message_type, "id": "1"}
+ if data:
+ payload["data"] = data
+ return protobuf.encode_message_frame(json.dumps(payload))
+
+
+def _string_frame(frame_field: int, text: str) -> bytes:
+ """A `Frame` whose only content is a string in field 3, as text frames are."""
+ inner = protobuf._len_field(3, text.encode("utf-8"))
+ return protobuf._len_field(frame_field, inner)
+
+
+def test_audio_frames_match_the_wire_schema():
+ """Byte-for-byte, since the pipeline on the other end is the only reader."""
+ assert protobuf.encode_audio_frame(b"\x01\x02", 16000, 1) == (
+ b"\x12\x09" # Frame.audio, 9 bytes
+ b"\x1a\x02\x01\x02" # AudioRawFrame.audio
+ b"\x20\x80\x7d" # AudioRawFrame.sample_rate = 16000
+ b"\x28\x01" # AudioRawFrame.num_channels = 1
+ )
+
+
+def test_an_encoded_audio_frame_decodes_back():
+ frame = protobuf.decode_frame(
+ protobuf.encode_audio_frame(b"\x00\x01\x02\x03", 24000, 2)
+ )
+
+ assert frame.kind == "audio"
+ assert frame.audio == b"\x00\x01\x02\x03"
+ assert frame.sample_rate == 24000
+ assert frame.num_channels == 2
+
+
+def test_unknown_frames_and_junk_are_told_apart():
+ """An unread frame is None; bytes that are not protobuf at all raise."""
+ assert protobuf.decode_frame(protobuf._varint_field(9, 1)) is None
+
+ connector = _connector()
+ assert connector._decode_inbound(b"\xff\xff\xff") is None
+ assert connector._decode_inbound("a text message") is None
+
+
+def test_audio_goes_out_at_the_rate_the_pipeline_listens_at():
+ connector = _connector(agent_sample_rate=8000)
+
+ frame = protobuf.decode_frame(connector._encode_outbound(b"\x01\x02"))
+
+ assert frame.sample_rate == 8000
+
+
+def test_a_client_ready_message_opens_the_session():
+ """Pipelines commonly hold their greeting until the client says it is up."""
+ frame = protobuf.decode_frame(_connector()._initial_messages()[0])
+
+ message = json.loads(frame.data)
+ assert message["label"] == "rtvi-ai"
+ assert message["type"] == "client-ready"
+ assert message["data"]["about"]["library"] == "deepeval"
+
+
+def test_client_ready_can_be_turned_off():
+ assert _connector(client_ready=False)._initial_messages() == []
+
+
+def test_the_turn_end_signal_is_claimed_only_once_seen():
+ """A pipeline without RTVI never sends one, and must not be waited on."""
+ connector = _connector()
+ assert connector.signals_turn_complete is False
+
+ event = connector._decode_inbound(_rtvi("bot-stopped-speaking"))
+
+ assert event.turn_complete is True
+ assert connector.signals_turn_complete is True
+
+
+def test_bot_transcriptions_accumulate_across_one_turn():
+ """RTVI sends one per sentence, not one per turn."""
+ connector = _connector()
+
+ first = connector._decode_inbound(
+ _rtvi("bot-transcription", text="Sure, one moment.")
+ )
+ second = connector._decode_inbound(
+ _rtvi("bot-transcription", text="Your table is booked.")
+ )
+
+ assert first.transcript == "Sure, one moment."
+ assert second.transcript == "Sure, one moment. Your table is booked."
+
+
+def test_a_new_turn_starts_from_an_empty_transcript():
+ connector = _connector()
+ connector._decode_inbound(_rtvi("bot-transcription", text="First turn."))
+
+ connector._decode_inbound(_rtvi("bot-stopped-speaking"))
+ event = connector._decode_inbound(
+ _rtvi("bot-transcription", text="Second turn.")
+ )
+
+ assert event.transcript == "Second turn."
+
+
+def test_an_interruption_is_recorded_on_the_turn():
+ connector = _connector()
+
+ assert connector._decode_inbound(_rtvi("bot-interrupted")) is None
+ assert connector._interrupted is True
+
+
+def test_other_rtvi_messages_are_ignored():
+ connector = _connector()
+
+ assert connector._decode_inbound(_rtvi("user-started-speaking")) is None
+ assert connector._decode_inbound(_rtvi("bot-llm-text", text="Hi")) is None
+
+
+def test_pipeline_text_is_not_mistaken_for_the_agents_reply():
+ """A `transcription` frame is the pipeline's STT of deepeval's own audio."""
+ connector = _connector()
+
+ assert connector._decode_inbound(_string_frame(3, "book a table")) is None
+ assert connector._decode_inbound(_string_frame(1, "some text")) is None
+ assert connector._current_transcript is None
+
+
+def test_the_rate_the_pipeline_speaks_at_wins_over_the_default():
+ connector = _connector(sample_rate=24000)
+
+ connector._decode_inbound(
+ protobuf.encode_audio_frame(b"\x01\x02", 16000, 1)
+ )
+
+ assert connector._recv_rate == 16000
+
+
+def test_wav_wrapped_audio_is_unwrapped():
+ """`add_wav_header=True` on the transport wraps every audio frame."""
+ pcm = b"\x01\x02\x03\x04"
+ wav = audio_utils.pcm16_to_wav_bytes(pcm, 16000, 1)
+ connector = _connector()
+
+ event = connector._decode_inbound(
+ protobuf.encode_audio_frame(wav, 16000, 1)
+ )
+
+ assert event.audio == pcm
+ assert connector._recv_rate == 16000
diff --git a/tests/test_core/test_voice/test_connectors/test_vapi.py b/tests/test_core/test_voice/test_connectors/test_vapi.py
new file mode 100644
index 000000000..986216bd6
--- /dev/null
+++ b/tests/test_core/test_voice/test_connectors/test_vapi.py
@@ -0,0 +1,176 @@
+"""Reading Vapi's client messages off the call socket."""
+
+import json
+
+import pytest
+
+from deepeval.errors import DeepEvalError
+from deepeval.voice import VapiConnector
+
+
+def _connector(**kwargs) -> VapiConnector:
+ kwargs.setdefault("assistant_id", "assistant-1")
+ kwargs.setdefault("api_key", "key-1")
+ return VapiConnector(**kwargs)
+
+
+def _speech_update(status: str, role: str = "assistant") -> str:
+ return json.dumps({"type": "speech-update", "status": status, "role": role})
+
+
+def _transcript(text: str, *, role="assistant", kind="final") -> str:
+ return json.dumps(
+ {
+ "type": "transcript",
+ "role": role,
+ "transcriptType": kind,
+ "transcript": text,
+ }
+ )
+
+
+def test_call_payload_asks_for_pcm_at_the_connector_rate():
+ payload = _connector(sample_rate=24000)._call_payload()
+
+ assert payload["assistantId"] == "assistant-1"
+ assert payload["transport"] == {
+ "provider": "vapi.websocket",
+ "audioFormat": {
+ "format": "pcm_s16le",
+ "container": "raw",
+ "sampleRate": 24000,
+ },
+ }
+ assert "assistantOverrides" not in payload
+
+
+def test_overrides_are_passed_through_untouched():
+ overrides = {"variableValues": {"customer_name": "Alice"}}
+
+ payload = _connector(assistant_overrides=overrides)._call_payload()
+
+ assert payload["assistantOverrides"] == overrides
+
+
+def test_narrowed_client_messages_regain_the_ones_deepeval_reads():
+ """Without these, turns end on silence and every turn pays for STT."""
+ connector = _connector(assistant_overrides={"clientMessages": ["hang"]})
+
+ messages = connector._call_payload()["assistantOverrides"]["clientMessages"]
+
+ assert messages == ["hang", "speech-update", "transcript"]
+
+
+def test_client_messages_are_left_alone_when_already_complete():
+ connector = _connector(
+ assistant_overrides={"clientMessages": ["transcript", "speech-update"]}
+ )
+
+ messages = connector._call_payload()["assistantOverrides"]["clientMessages"]
+
+ assert messages == ["transcript", "speech-update"]
+
+
+def test_binary_frames_are_agent_audio():
+ event = _connector()._decode_inbound(b"\x01\x02")
+
+ assert event.audio == b"\x01\x02"
+
+
+def test_a_stopped_assistant_closes_the_turn():
+ connector = _connector()
+
+ assert connector.signals_turn_complete is True
+ assert connector._decode_inbound(_speech_update("stopped")).turn_complete
+
+
+def test_the_caller_speaking_does_not_close_the_agents_turn():
+ connector = _connector()
+
+ assert connector._decode_inbound(_speech_update("started")) is None
+ assert connector._decode_inbound(_speech_update("stopped", "user")) is None
+
+
+def test_final_transcripts_accumulate_across_one_turn():
+ """Vapi sends one final per utterance, not one per turn."""
+ connector = _connector()
+
+ first = connector._decode_inbound(_transcript("Sure, one moment."))
+ second = connector._decode_inbound(_transcript("Your table is booked."))
+
+ assert first.transcript == "Sure, one moment."
+ assert second.transcript == "Sure, one moment. Your table is booked."
+
+
+def test_partial_and_caller_transcripts_are_ignored():
+ connector = _connector()
+
+ assert (
+ connector._decode_inbound(_transcript("Sure", kind="partial")) is None
+ )
+ assert connector._decode_inbound(_transcript("Hi", role="user")) is None
+
+
+def test_a_new_turn_starts_from_an_empty_transcript():
+ connector = _connector()
+ connector._decode_inbound(_transcript("First turn."))
+
+ connector._decode_inbound(_speech_update("stopped"))
+ event = connector._decode_inbound(_transcript("Second turn."))
+
+ assert event.transcript == "Second turn."
+
+
+def test_events_wrapped_in_a_message_envelope_are_understood():
+ """Webhook deliveries wrap the event; the socket may do either."""
+ connector = _connector()
+
+ raw = json.dumps(
+ {
+ "message": {
+ "type": "speech-update",
+ "status": "stopped",
+ "role": "assistant",
+ }
+ }
+ )
+
+ assert connector._decode_inbound(raw).turn_complete
+
+
+def test_an_interruption_is_recorded_on_the_turn():
+ connector = _connector()
+
+ assert (
+ connector._decode_inbound(json.dumps({"type": "user-interrupted"}))
+ is None
+ )
+ assert connector._interrupted is True
+
+
+def test_unknown_events_and_junk_are_ignored():
+ connector = _connector()
+
+ assert (
+ connector._decode_inbound(json.dumps({"type": "voice-input"})) is None
+ )
+ assert connector._decode_inbound("not json") is None
+ assert connector._decode_inbound(json.dumps([1, 2])) is None
+
+
+def test_the_rate_vapi_settles_on_wins_over_the_one_requested():
+ connector = _connector(sample_rate=24000)
+
+ connector._apply_audio_format({"format": "pcm_s16le", "sampleRate": 16000})
+
+ assert connector._send_rate == 16000
+ assert connector._recv_rate == 16000
+
+
+@pytest.mark.asyncio
+async def test_a_missing_api_key_is_reported_before_any_request(monkeypatch):
+ monkeypatch.delenv("VAPI_API_KEY", raising=False)
+ connector = VapiConnector(assistant_id="assistant-1", api_key=None)
+
+ with pytest.raises(DeepEvalError, match="VAPI_API_KEY"):
+ await connector._open_session()
diff --git a/tests/test_core/test_voice/test_speech_selection.py b/tests/test_core/test_voice/test_speech_selection.py
new file mode 100644
index 000000000..a7f669f0d
--- /dev/null
+++ b/tests/test_core/test_voice/test_speech_selection.py
@@ -0,0 +1,108 @@
+"""Which speech provider and model a run ends up with.
+
+TTS and STT are picked the same way LLM providers are: a `USE_*` flag chooses
+the provider, `DEEPEVAL_TTS_MODEL` / `DEEPEVAL_STT_MODEL` choose the model
+name, and anything passed in code outranks both. These pin down that order,
+and that the two families never reach into each other.
+"""
+
+import pytest
+
+from deepeval.models.speech_selection import (
+ initialize_stt_model,
+ initialize_tts_model,
+)
+from deepeval.models.stt import (
+ AssemblyAISTTModel,
+ DeepgramSTTModel,
+ OpenAISTTModel,
+)
+from deepeval.models.tts import (
+ DeepgramTTSModel,
+ ElevenLabsTTSModel,
+ OpenAITTSModel,
+)
+
+
+@pytest.fixture(autouse=True)
+def speech_api_keys(monkeypatch):
+ # Every provider raises on construction without a key, and none of these
+ # tests make a request.
+ for name in (
+ "OPENAI_API_KEY",
+ "ELEVENLABS_API_KEY",
+ "CARTESIA_API_KEY",
+ "DEEPGRAM_API_KEY",
+ "ASSEMBLYAI_API_KEY",
+ ):
+ monkeypatch.setenv(name, "test-key")
+
+
+def test_no_flag_falls_back_to_openai():
+ assert isinstance(initialize_tts_model(), OpenAITTSModel)
+ assert isinstance(initialize_stt_model(), OpenAISTTModel)
+
+
+def test_flag_picks_the_provider(monkeypatch):
+ monkeypatch.setenv("USE_DEEPGRAM_TTS", "1")
+ monkeypatch.setenv("USE_ASSEMBLYAI_STT", "1")
+
+ assert isinstance(initialize_tts_model(), DeepgramTTSModel)
+ assert isinstance(initialize_stt_model(), AssemblyAISTTModel)
+
+
+def test_family_env_var_picks_the_model(monkeypatch):
+ monkeypatch.setenv("USE_DEEPGRAM_STT", "1")
+ monkeypatch.setenv("DEEPEVAL_STT_MODEL", "nova-3-general")
+
+ model = initialize_stt_model()
+
+ assert isinstance(model, DeepgramSTTModel)
+ assert model.get_model_name() == "nova-3-general"
+
+
+def test_a_string_names_the_model_for_the_selected_provider(monkeypatch):
+ monkeypatch.setenv("USE_ELEVENLABS_TTS", "1")
+ monkeypatch.setenv("DEEPEVAL_TTS_MODEL", "eleven_flash_v2_5")
+
+ model = initialize_tts_model("eleven_turbo_v2_5")
+
+ # The string is the model name only: the provider still comes from the
+ # flag, and the name beats the env var.
+ assert isinstance(model, ElevenLabsTTSModel)
+ assert model.get_model_name() == "eleven_turbo_v2_5"
+
+
+def test_an_instance_passes_through_untouched(monkeypatch):
+ monkeypatch.setenv("USE_DEEPGRAM_TTS", "1")
+ passed = OpenAITTSModel(model="tts-1-hd")
+
+ assert initialize_tts_model(passed) is passed
+
+
+def test_the_two_families_are_independent(monkeypatch):
+ monkeypatch.setenv("USE_DEEPGRAM_TTS", "1")
+ monkeypatch.setenv("DEEPEVAL_TTS_MODEL", "aura-2-thalia-en")
+
+ # No STT flag, so STT stays on OpenAI and ignores the TTS model name.
+ stt = initialize_stt_model()
+
+ assert isinstance(initialize_tts_model(), DeepgramTTSModel)
+ assert isinstance(stt, OpenAISTTModel)
+ assert stt.get_model_name() != "aura-2-thalia-en"
+
+
+def test_voice_config_resolves_through_the_same_chain(monkeypatch):
+ monkeypatch.setenv("USE_DEEPGRAM_STT", "1")
+ from deepeval.voice import VoiceConfig
+ from deepeval.voice.connectors.transports.callback import (
+ CallbackVoiceConnector,
+ )
+
+ async def agent(audio):
+ raise AssertionError("the agent is never called in this test")
+
+ config = VoiceConfig(connector=CallbackVoiceConnector(agent))
+
+ assert isinstance(config.stt_model, DeepgramSTTModel)
+ assert isinstance(config.tts_model, OpenAITTSModel)
diff --git a/tests/test_core/test_voice/test_voice_output_dir.py b/tests/test_core/test_voice/test_voice_output_dir.py
index b863674bb..a42f313a4 100644
--- a/tests/test_core/test_voice/test_voice_output_dir.py
+++ b/tests/test_core/test_voice/test_voice_output_dir.py
@@ -15,6 +15,13 @@ from deepeval.voice.connectors.transports.callback import (
from deepeval.voice.output import DEFAULT_VOICE_FOLDER
+@pytest.fixture(autouse=True)
+def openai_api_key(monkeypatch):
+ # `VoiceConfig` resolves its speech models, which default to OpenAI and
+ # refuse to be constructed without a key. None of these tests speak.
+ monkeypatch.setenv("OPENAI_API_KEY", "test-key")
+
+
@pytest.fixture
def connector():
async def agent(audio):
diff --git a/typescript/scripts/env-parity-expected.json b/typescript/scripts/env-parity-expected.json
index 50ce9279c..48c41209a 100644
--- a/typescript/scripts/env-parity-expected.json
+++ b/typescript/scripts/env-parity-expected.json
@@ -57,8 +57,25 @@
]
},
"voiceSimulation": {
- "$comment": "Voice mode is Python-only: there is no VoiceConfig, connector, or TTS/STT model in typescript/src, so there is no audio for TS to write anywhere.",
- "names": ["DEEPEVAL_VOICE_FOLDER"]
+ "$comment": "Voice mode is Python-only: there is no VoiceConfig, connector, or TTS/STT model in typescript/src, so there is no audio for TS to write anywhere and no speech provider for it to authenticate against.",
+ "names": [
+ "ASSEMBLYAI_API_KEY",
+ "CARTESIA_API_KEY",
+ "DEEPEVAL_STT_MODEL",
+ "DEEPEVAL_TTS_MODEL",
+ "DEEPEVAL_VOICE_FOLDER",
+ "DEEPGRAM_API_KEY",
+ "ELEVENLABS_API_KEY",
+ "USE_ASSEMBLYAI_STT",
+ "USE_CARTESIA_STT",
+ "USE_CARTESIA_TTS",
+ "USE_DEEPGRAM_STT",
+ "USE_DEEPGRAM_TTS",
+ "USE_ELEVENLABS_STT",
+ "USE_ELEVENLABS_TTS",
+ "USE_OPENAI_STT",
+ "USE_OPENAI_TTS"
+ ]
},
"other": ["DEEPEVAL_MAX_CONCURRENT_DOC_PROCESSING"]
},
diff --git a/vapi_capture.py b/vapi_capture.py
new file mode 100644
index 000000000..781c4c563
--- /dev/null
+++ b/vapi_capture.py
@@ -0,0 +1,193 @@
+"""Capture Vapi's WebSocket event vocabulary for one short conversation.
+
+Run this once against a scratch assistant. It creates a `vapi.websocket` call,
+speaks two utterances into it, and writes every inbound message to
+`vapi_events.jsonl` — binary audio as a summary line, JSON control messages
+verbatim. The log answers the four questions VapiConnector needs:
+
+ 1. Is there a per-turn completion event to trust instead of silence?
+ 2. Do final transcripts arrive, so STT can be skipped?
+ 3. Is there a ready/handshake message before audio is accepted?
+ 4. Does the assistant speak first, before we send anything?
+
+Usage:
+ VAPI_API_KEY=... VAPI_ASSISTANT_ID=... OPENAI_API_KEY=... python vapi_capture.py
+"""
+
+import asyncio
+import json
+import os
+import sys
+import time
+
+import aiohttp
+
+from deepeval.models import OpenAITTSModel
+from deepeval.voice.connectors import audio_utils
+
+API_BASE = os.getenv("VAPI_API_URL", "https://api.vapi.ai")
+SAMPLE_RATE = 24000
+CHUNK_MS = 20
+LOG_PATH = "vapi_events.jsonl"
+
+UTTERANCES = [
+ "Hi, I'd like to book a table for two people tomorrow at seven pm.",
+ "Actually, could you make that eight pm instead?",
+]
+
+# How long to listen after each utterance before speaking again.
+LISTEN_SECONDS = 15.0
+# How long to listen before the first utterance, to see whether the assistant
+# opens the conversation on its own.
+GREETING_SECONDS = 8.0
+
+
+def log(record: dict, started: float) -> None:
+ record = {"at_s": round(time.perf_counter() - started, 3), **record}
+ with open(LOG_PATH, "a") as f:
+ f.write(json.dumps(record) + "\n")
+ if record.get("kind") != "audio":
+ print(json.dumps(record)[:400])
+
+
+async def create_call(
+ session: aiohttp.ClientSession, api_key: str, assistant_id: str
+) -> str:
+ payload = {
+ "assistantId": assistant_id,
+ "transport": {
+ "provider": "vapi.websocket",
+ "audioFormat": {
+ "format": "pcm_s16le",
+ "container": "raw",
+ "sampleRate": SAMPLE_RATE,
+ },
+ },
+ }
+ async with session.post(
+ f"{API_BASE}/call",
+ headers={"authorization": f"Bearer {api_key}"},
+ json=payload,
+ ) as resp:
+ body = await resp.text()
+ if resp.status >= 300:
+ raise SystemExit(
+ f"Vapi call creation failed ({resp.status}): {body}"
+ )
+ data = json.loads(body)
+ print(json.dumps({"created_call": data}, indent=2)[:2000])
+ url = (data.get("transport") or {}).get("websocketCallUrl")
+ if not url:
+ raise SystemExit("Response had no transport.websocketCallUrl")
+ return url
+
+
+async def synthesize_pcm(tts: OpenAITTSModel, text: str) -> bytes:
+ audio, _cost = await tts.a_synthesize(text)
+ pcm, rate, channels = audio_utils.wav_bytes_to_pcm16(audio.get_bytes())
+ pcm = audio_utils.downmix_to_mono(pcm, channels)
+ if rate != SAMPLE_RATE:
+ pcm = audio_utils.resample_pcm16(pcm, rate, SAMPLE_RATE)
+ return pcm
+
+
+async def send_pcm(ws: aiohttp.ClientWebSocketResponse, pcm: bytes) -> None:
+ """Stream in real time, as a caller's microphone would."""
+ frame_bytes = int(SAMPLE_RATE * 2 * CHUNK_MS / 1000)
+ for offset in range(0, len(pcm), frame_bytes):
+ await ws.send_bytes(pcm[offset : offset + frame_bytes])
+ await asyncio.sleep(CHUNK_MS / 1000)
+
+
+async def reader(ws: aiohttp.ClientWebSocketResponse, started: float) -> None:
+ audio_bytes = 0
+ audio_messages = 0
+ async for msg in ws:
+ if msg.type == aiohttp.WSMsgType.BINARY:
+ audio_bytes += len(msg.data)
+ audio_messages += 1
+ # One line per second of audio keeps the log readable.
+ if audio_messages % 50 == 0:
+ log(
+ {
+ "kind": "audio",
+ "messages": audio_messages,
+ "total_bytes": audio_bytes,
+ },
+ started,
+ )
+ elif msg.type == aiohttp.WSMsgType.TEXT:
+ try:
+ parsed = json.loads(msg.data)
+ except ValueError:
+ log({"kind": "text_unparsed", "raw": msg.data[:2000]}, started)
+ continue
+ log({"kind": "event", "message": parsed}, started)
+ elif msg.type in (aiohttp.WSMsgType.CLOSED, aiohttp.WSMsgType.ERROR):
+ log({"kind": "socket_closed", "type": str(msg.type)}, started)
+ return
+
+
+async def main() -> None:
+ api_key = os.getenv("VAPI_API_KEY")
+ assistant_id = os.getenv("VAPI_ASSISTANT_ID")
+ if not api_key or not assistant_id:
+ raise SystemExit("Set VAPI_API_KEY and VAPI_ASSISTANT_ID")
+
+ open(LOG_PATH, "w").close()
+ tts = OpenAITTSModel()
+ print("Synthesizing utterances...")
+ utterance_pcm = [await synthesize_pcm(tts, text) for text in UTTERANCES]
+
+ async with aiohttp.ClientSession() as session:
+ url = await create_call(session, api_key, assistant_id)
+ print(f"Connecting to {url}")
+ async with session.ws_connect(url) as ws:
+ started = time.perf_counter()
+ read_task = asyncio.create_task(reader(ws, started))
+
+ log(
+ {
+ "kind": "note",
+ "note": "listening for an unprompted greeting",
+ },
+ started,
+ )
+ await asyncio.sleep(GREETING_SECONDS)
+
+ for index, pcm in enumerate(utterance_pcm):
+ log(
+ {
+ "kind": "note",
+ "note": f"sending utterance {index + 1}",
+ "text": UTTERANCES[index],
+ "pcm_bytes": len(pcm),
+ },
+ started,
+ )
+ await send_pcm(ws, pcm)
+ log(
+ {"kind": "note", "note": f"utterance {index + 1} sent"},
+ started,
+ )
+ await asyncio.sleep(LISTEN_SECONDS)
+
+ await ws.send_str(json.dumps({"type": "end-call"}))
+ log({"kind": "note", "note": "sent end-call"}, started)
+ await asyncio.sleep(2.0)
+ await ws.close()
+ await read_task
+
+ print(f"\nWrote {LOG_PATH}")
+ print("Paste it back, or grep it for distinct event types:")
+ print(
+ " jq -r 'select(.kind==\"event\") | .message.type' "
+ f"{LOG_PATH} | sort | uniq -c"
+ )
+
+
+if __name__ == "__main__":
+ try:
+ asyncio.run(main())
+ except KeyboardInterrupt:
+ sys.exit(1)