pydantic/pydantic-ai
1
0
Fork 0
mirror of https://github.com/pydantic/pydantic-ai.git synced 2026-10-12 02:21:31 +00:00
Code Issues Packages Projects Releases Wiki Activity

Bump anyio dev floor to 4.11 and run async tests via anyio_mode = "auto" (#8874)

Browse Source
This commit is contained in:
Marcelo Trylesinski
2026-09-28 08:45:29 +00:00
committed by GitHub
parent f4d0111ba0
commit c897465bf9
399 changed files with 69 additions and 1286 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pydantic_ai_slim/pydantic_ai/durable_exec/temporal/__init__.py
+1 -1
View File
@@ -62,7 +62,7 @@ __all__ = [
# We need eagerly import the anyio backends or it will happens inside workflow code and temporal has issues
# Note: It's difficult to add a test that covers this because pytest presumably does these imports itself
# when you have a @pytest.mark.anyio somewhere.
# when running async tests via the anyio pytest plugin.
# I suppose we could add a test that runs a python script in a separate process, but I have not done that...
import anyio._backends._asyncio # pyright: ignore[reportUnusedImport] #noqa: F401
pyproject.toml
+4 -1
View File
@@ -224,7 +224,8 @@ google-genai = "2026-09-22T17:23:02Z"
[dependency-groups]
dev = [
"anyio>=4.7.0",
# `anyio_mode = "auto"` (pytest plugin) was added in 4.11.0.
"anyio>=4.11.0",
"trio>=0.34.0",
"asgi-lifespan>=2.1.0",
"httpx>=0.27",
@@ -451,6 +452,8 @@ strict = true
[tool.pytest.ini_options]
addopts = "-p no:cassetter"
# Every `async def` test runs via anyio; no `@pytest.mark.anyio` needed.
anyio_mode = "auto"
testpaths = ["tests", ".github/scripts/test_ci_duration.py"]
xfail_strict = true
markers = ["modal_live: runs against a real Modal control plane; needs credentials"]
src/pydantic_ai_harness/agent_docs/concurrency.md
+6 -8
View File
@@ -184,14 +184,12 @@ before writing cleanup.
- Reach the real trigger. Level-cancellation behavior needs a real outer
`anyio` cancel scope, not a bare `CancelledError` raise; Trio behavior needs
the `trio` parametrization this suite already runs, not a mental model of it.
- Know which backend a suite runs under before you trust it. anyio's pytest
plugin parametrizes `anyio_backend` over every installed backend, and
`anyio[trio]` is in the dev group, so the default is asyncio and Trio:
`tests/shell` and `tests/filesystem` take that default (collect them and
every test id ends in `[asyncio]` or `[trio]`), with individual tests opting
out via `@pytest.mark.anyio(backends=['asyncio'])`. A suite goes asyncio-only
by overriding the `anyio_backend` fixture at module level to return
`'asyncio'`, which `tests/subagents` and `tests/filesystem/test_events.py`
- Know which backend a suite runs under before you trust it. Every `async def`
test runs via anyio's pytest plugin (`anyio_mode = "auto"`), and the root
`anyio_backend` fixture picks the backend from `--anyio-backend` (asyncio by
default; pass `--anyio-backend=trio` to run under Trio). A suite goes asyncio-only by
overriding the `anyio_backend` fixture at module level to return `'asyncio'`,
which `tests/harness/subagents` and `tests/harness/filesystem/test_events.py`
do. That override is why `SubAgentToolset.delegate_task` can use
`asyncio.wait_for` today. Reach for `anyio` primitives by default
(`fail_after`, `Lock`, task groups); before you remove an `anyio_backend`
src/pydantic_ai_harness/integration_tests/localstack/test_live_localstack.py
-3
View File
@@ -211,7 +211,6 @@ async def _s3_round_trip(endpoint_url: str, aws_cli: str, tmp_path: Path) -> Non
assert '[error:' not in health
@pytest.mark.anyio(backends=['asyncio'])
async def test_existing_localstack_s3_round_trip(tmp_path: Path) -> None:
"""Drive S3 through a LocalStack service started by the test environment."""
endpoint_url = _external_endpoint_url()
@@ -221,7 +220,6 @@ async def test_existing_localstack_s3_round_trip(tmp_path: Path) -> None:
await _s3_round_trip(endpoint_url, _aws_cli_path(), tmp_path)
@pytest.mark.anyio(backends=['asyncio'])
async def test_external_container_s3_round_trip(tmp_path: Path) -> None:
"""Drive S3 through an unmanaged capability against a harness-started container."""
if _skip_managed_container_tests():
@@ -240,7 +238,6 @@ async def test_external_container_s3_round_trip(tmp_path: Path) -> None:
await _s3_round_trip(localstack.endpoint_url, aws_cli, tmp_path)
@pytest.mark.anyio(backends=['asyncio'])
async def test_managed_container_sqs_round_trip_and_cleanup() -> None:
"""Drive SQS through `manage_container=True` and verify cleanup."""
if _skip_managed_container_tests():
src/pydantic_ai_harness/integration_tests/mongodb/test_live_mongodb.py
-7
View File
@@ -187,7 +187,6 @@ def test_live_tests_run_on_asyncio_backend(anyio_backend: str) -> None:
pytest.param(_DRIVER_REFUSED_BYTES, DocumentTooLarge, id='driver-refused'),
],
)
@pytest.mark.anyio(backends=['asyncio'])
async def test_oversized_snapshot_fails_loudly_without_media_externalization(
payload_bytes: int, expected: type[PyMongoError]
) -> None:
@@ -202,7 +201,6 @@ async def test_oversized_snapshot_fails_loudly_without_media_externalization(
assert await store.latest_snapshot(run_id='run-too-large') is None
@pytest.mark.anyio(backends=['asyncio'])
async def test_media_externalization_saves_a_snapshot_past_the_bson_cap() -> None:
"""The same payload round-trips once large text is offloaded to the media store."""
text = 'x' * _SERVER_REFUSED_BYTES
@@ -226,7 +224,6 @@ async def test_media_externalization_saves_a_snapshot_past_the_bson_cap() -> Non
assert part.content == text
@pytest.mark.anyio(backends=['asyncio'])
async def test_multi_chunk_blob_round_trips_at_the_default_chunk_size() -> None:
"""A blob larger than one BSON document survives the default 8MB chunking."""
data = os.urandom(_MULTI_CHUNK_BLOB_BYTES)
@@ -244,7 +241,6 @@ async def test_multi_chunk_blob_round_trips_at_the_default_chunk_size() -> None:
assert await store.get(uri) == data
@pytest.mark.anyio(backends=['asyncio'])
async def test_chunk_reassembly_ignores_collection_scan_order() -> None:
"""Reassembly follows the sort on `n`, not the order mongod stores chunks in."""
data = os.urandom(5 * 1024)
@@ -269,7 +265,6 @@ async def test_chunk_reassembly_ignores_collection_scan_order() -> None:
assert await store.get(uri) == data
@pytest.mark.anyio(backends=['asyncio'])
async def test_awkward_metadata_keys_round_trip_through_real_bson() -> None:
"""Dotted, `$`-prefixed, and non-ASCII metadata keys are stored verbatim."""
async with _live_database() as db:
@@ -292,7 +287,6 @@ async def test_awkward_metadata_keys_round_trip_through_real_bson() -> None:
assert raw['metadata'] == _AWKWARD_METADATA
@pytest.mark.anyio(backends=['asyncio'])
async def test_null_byte_metadata_key_is_rejected_by_the_bson_encoder() -> None:
"""The one metadata key shape BSON cannot carry fails at encode time, before the wire."""
async with _live_database() as db:
@@ -302,7 +296,6 @@ async def test_null_byte_metadata_key_is_rejected_by_the_bson_encoder() -> None:
await store.register_run(RunRecord(run_id='run-null-key', metadata={'a\x00b': 'v'}))
@pytest.mark.anyio(backends=['asyncio'])
async def test_tool_effect_unique_index_is_created_and_enforced() -> None:
"""`(run_id, tool_call_id)` is a real server-side constraint, not a convention."""
async with _live_database() as db:
src/pydantic_ai_harness/integration_tests/redis/test_live_redis.py
-2
View File
@@ -59,8 +59,6 @@ from redis.exceptions import RedisError, ResponseError
from pydantic_ai_harness.spend import RedisSpendStore, SpendEntry, Spent
pytestmark = pytest.mark.anyio
_NANOS = Decimal(10) ** 9
"""Billionths of a dollar to the dollar, which is how the store keeps money."""
tests/AGENTS.md
+4 -3
View File
@@ -2,8 +2,9 @@
## Testing philosophy
The async backend defaults to asyncio. Use `uv run pytest <test-path> --anyio-backend=trio --record-mode=none`
to run selected portable tests on Trio without duplicating the default suite. During the Trio migration,
Every `async def` test runs via the anyio pytest plugin automatically (`anyio_mode = "auto"` in
`pyproject.toml`); don't add `@pytest.mark.anyio`. The async backend defaults to asyncio. Use
`uv run pytest <test-path> --anyio-backend=trio --record-mode=none` to run selected portable tests on Trio without duplicating the default suite. During the Trio migration,
run affected concurrency tests once per backend. Keep broad Trio runs manual or periodic; do not add a
second backend to every ordinary CI matrix. The selector does not imply that every test or integration
already supports Trio.
@@ -28,7 +29,7 @@ from pydantic_ai import Agent
from pydantic_ai.models import Model
# ... other imports
pytestmark = [pytest.mark.anyio, pytest.mark.vcr]
pytestmark = pytest.mark.vcr
# fixtures/helpers immediately before their test
tests/benchmarks/test_agent.py
+1 -1
View File
@@ -17,7 +17,7 @@ from pydantic_ai.models import CompletedStreamedResponse, ModelRequestParameters
from pydantic_ai.models.function import AgentInfo, FunctionModel
from pydantic_ai.models.test import TestModel
pytestmark = [pytest.mark.anyio, pytest.mark.benchmark]
pytestmark = [pytest.mark.benchmark]
@pytest.fixture
tests/clai2/test_api_keys.py
-2
View File
@@ -13,8 +13,6 @@ from pydantic_clai2.commands import set_completions
from pydantic_clai2.credential_store import credentials_path, save_codex_credentials
from tests.clai2.menu_script import make_context
pytestmark = pytest.mark.anyio
@pytest.fixture
def anyio_backend() -> str:
tests/clai2/test_app_edges.py
-2
View File
@@ -28,8 +28,6 @@ from pydantic_clai2.model_menu import ModelSettingsSource, model_settings_comman
from pydantic_clai2.settings_store import SettingsStore
from tests.clai2.menu_script import Script, pick, typed
pytestmark = pytest.mark.anyio
PromptT = TypeVar('PromptT')
tests/clai2/test_ask_user_menu.py
-2
View File
@@ -34,8 +34,6 @@ from pydantic_clai2.plugins import PluginHost
from pydantic_clai2.prompt_surface import PromptSurface
from pydantic_clai2.question_input import Paste
pytestmark = pytest.mark.anyio
@pytest.fixture
def anyio_backend() -> str:
tests/clai2/test_auth.py
-2
View File
@@ -16,8 +16,6 @@ from pydantic_clai2.auth import CodexAuth, CodexCredentials, code_from_paste, lo
from pydantic_clai2.commands import Command, Commands
from pydantic_clai2.config import Settings
pytestmark = pytest.mark.anyio
CREDENTIALS = OpenAICodexCredentials(
access_token='fake-access', refresh_token='fake-refresh', account_id='fake-account'
)
tests/clai2/test_cancelled_history.py
-2
View File
@@ -12,8 +12,6 @@ from pydantic_ai.models.test import TestModel
from pydantic_clai2 import Session
from pydantic_clai2.interrupts import Interrupts
pytestmark = pytest.mark.anyio
@pytest.fixture
def anyio_backend() -> str:
tests/clai2/test_clai.py
-2
View File
@@ -31,8 +31,6 @@ from pydantic_clai2.commands import Command, Commands, config_command, config_co
from pydantic_clai2.settings_store import SettingsStore
from pydantic_clai2.splash import Splash
pytestmark = pytest.mark.anyio
@pytest.fixture
def anyio_backend() -> str:
tests/clai2/test_code_highlighting.py
-2
View File
@@ -11,8 +11,6 @@ from termflow.themes import PALETTES
from pydantic_ai import PartDeltaEvent, PartStartEvent, TextPart, TextPartDelta, ThinkingPart
from pydantic_clai2 import StreamRenderer, theme
pytestmark = pytest.mark.anyio
@pytest.mark.parametrize('chunk_size', [1, 1000])
async def test_multiline_strings_keep_their_highlight(chunk_size: int) -> None:
tests/clai2/test_codex_fast_mode.py
-2
View File
@@ -17,8 +17,6 @@ from pydantic_clai2.field_menu import FieldMenu
from pydantic_clai2.model_menu import ModelSettingsSource, model_settings_command
from tests.clai2.menu_script import Script, make_context, pick
pytestmark = pytest.mark.anyio
@pytest.fixture
def anyio_backend() -> str:
tests/clai2/test_compact_tools.py
-2
View File
@@ -13,8 +13,6 @@ from pydantic_ai_harness.shell import CommandFinishedEvent, CommandOutputEvent,
from pydantic_clai2 import StreamRenderer, theme
from pydantic_clai2.config import Settings, resolve_settings
pytestmark = pytest.mark.anyio
@pytest.fixture
def anyio_backend() -> str:
tests/clai2/test_compaction.py
-2
View File
@@ -20,8 +20,6 @@ from pydantic_clai2.plugins import PluginHost, SessionEnd, Transcript
from pydantic_clai2.settings_store import SettingsStore
from tests.clai2.test_app_edges import inputs
pytestmark = pytest.mark.anyio
@pytest.fixture
def anyio_backend() -> str:
tests/clai2/test_credential_store.py
-2
View File
@@ -22,8 +22,6 @@ from pydantic_clai2.credential_store import (
save_codex_credentials,
)
pytestmark = pytest.mark.anyio
@pytest.fixture
def anyio_backend() -> str:
tests/clai2/test_customization.py
-2
View File
@@ -17,8 +17,6 @@ from pydantic_clai2 import Session
from pydantic_clai2._app import create_agent
from pydantic_clai2.customization import customization_guide, read_clai_customization_guide
pytestmark = pytest.mark.anyio
async def test_default_agent_does_not_read_guide_for_normal_turn(monkeypatch: pytest.MonkeyPatch) -> None:
def unexpected_read(*args: object, **kwargs: object) -> str:
tests/clai2/test_error_messages.py
-2
View File
@@ -20,8 +20,6 @@ from pydantic_clai2.project_settings import ProjectSettings
from pydantic_clai2.settings_store import SettingsStore
from tests.clai2.test_app_edges import inputs
pytestmark = pytest.mark.anyio
@pytest.mark.parametrize('mode', ['interactive', 'headless'])
@pytest.mark.parametrize('chain', ['direct', 'cause', 'context', 'suppressed', 'cycle', 'network', 'persistence'])
tests/clai2/test_forks.py
-2
View File
@@ -33,8 +33,6 @@ from pydantic_clai2.project_settings import ProjectSettings
from pydantic_clai2.settings_store import SettingsStore
from pydantic_clai2.spinners import BUILTIN_SPINNERS, DEFAULT_SPINNER
pytestmark = pytest.mark.anyio
PromptT = TypeVar('PromptT')
tests/clai2/test_github_copilot.py
-2
View File
@@ -31,8 +31,6 @@ from pydantic_clai2.settings_store import SettingsStore
from tests.clai2.menu_script import Script, make_context, pick, typed
from tests.clai2.test_app_edges import inputs
pytestmark = pytest.mark.anyio
CREDENTIALS = GitHubCopilotCredentials(access_token='fake-access', token_type='bearer', scope='')
tests/clai2/test_grep_output.py
-2
View File
@@ -9,8 +9,6 @@ from pydantic_ai import FunctionToolCallEvent, FunctionToolResultEvent
from pydantic_ai.messages import ToolCallPart, ToolReturnPart
from pydantic_clai2 import StreamRenderer
pytestmark = pytest.mark.anyio
@pytest.fixture
def anyio_backend() -> str:
tests/clai2/test_headless.py
-2
View File
@@ -16,8 +16,6 @@ from pydantic_clai2.config import PluginSettings, Settings
from pydantic_clai2.project_settings import ProjectSettings
from pydantic_clai2.settings_store import SettingsStore
pytestmark = pytest.mark.anyio
async def test_answer_resume_and_no_ask_user(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]
tests/clai2/test_image_input.py
-2
View File
@@ -33,8 +33,6 @@ from pydantic_clai2.image_input import (
from pydantic_clai2.prompt_surface import PromptSurface
from pydantic_clai2.settings_store import SettingsStore
pytestmark = pytest.mark.anyio
@pytest.fixture
def anyio_backend() -> str:
tests/clai2/test_input_history.py
-2
View File
@@ -16,8 +16,6 @@ from pydantic_clai2 import chat
from pydantic_clai2.input_history import input_history
from pydantic_clai2.settings_store import SettingsStore
pytestmark = pytest.mark.anyio
@pytest.fixture
def anyio_backend() -> str:
tests/clai2/test_interrupts.py
-2
View File
@@ -8,8 +8,6 @@ import pytest
from pydantic_clai2.interrupts import Interrupts
pytestmark = pytest.mark.anyio
@pytest.fixture
def anyio_backend() -> str:
tests/clai2/test_key_menu.py
-2
View File
@@ -13,8 +13,6 @@ from pydantic_clai2.field_menu import FieldMenu
from pydantic_clai2.key_menu import KeyAction, KeysSource, build_keys_menu, keys_command, run_keys_flow
from tests.clai2.menu_script import Script, pick, typed
pytestmark = pytest.mark.anyio
@pytest.fixture
def anyio_backend() -> str:
tests/clai2/test_key_references.py
-2
View File
@@ -10,8 +10,6 @@ from pydantic_ai.exceptions import UserError
from pydantic_clai2 import api_keys, openrouter, vllm
from pydantic_clai2.credential_store import load_codex_credentials
pytestmark = pytest.mark.anyio
@pytest.fixture
def anyio_backend() -> str:
tests/clai2/test_live_prompt.py
-2
View File
@@ -31,8 +31,6 @@ from pydantic_clai2.live_prompt import LivePrompt
from pydantic_clai2.prompt_completion import CompletionWorker
from tests.clai2.surface_terminal import SurfaceTerminal
pytestmark = pytest.mark.anyio
@pytest.fixture
def anyio_backend() -> str:
tests/clai2/test_logfire.py
-2
View File
@@ -28,8 +28,6 @@ from pydantic_clai2.plugin_loader import PluginError, PluginLoader
from pydantic_clai2.plugins import PluginHost, SessionEnd, SessionStart
from pydantic_clai2.settings_store import SettingsStore
pytestmark = pytest.mark.anyio
class Exporter(InMemorySpanExporter):
closed = False
tests/clai2/test_markdown_links.py
-2
View File
@@ -13,8 +13,6 @@ from pydantic_clai2 import StreamRenderer
from pydantic_clai2._rendering import LinkOutput
from pydantic_clai2.prompt_surface import PromptSurface
pytestmark = pytest.mark.anyio
URL = 'https://github.com/pydantic/pydantic-ai-harness/pull/1006'
OPEN = f'\x1b]8;;{URL}\x1b\\'
CLOSE = '\x1b]8;;\x1b\\'
tests/clai2/test_mcp.py
-2
View File
@@ -34,8 +34,6 @@ from pydantic_clai2.plugin_loader import PluginLoader
from pydantic_clai2.plugins import PluginHost, SessionEnd, SessionStart
from pydantic_clai2.settings_store import SettingsStore
pytestmark = pytest.mark.anyio
def make_host(settings: dict[str, JsonValue], store: MCPStore | None = None) -> PluginHost[None]:
host: PluginHost[None] = PluginHost(name='mcp', console=Console(file=io.StringIO()), settings=settings)
tests/clai2/test_mcp_command.py
-2
View File
@@ -33,8 +33,6 @@ from pydantic_clai2.mcp import (
from pydantic_clai2.plugins import PluginHost
from tests.clai2.menu_script import Script, pick, typed
pytestmark = pytest.mark.anyio
ESC = MenuResult(cancelled=True)
tests/clai2/test_mcp_tokens.py
-2
View File
@@ -14,8 +14,6 @@ from pydantic import AnyUrl, HttpUrl
from pydantic_clai2.mcp import HTTPServer, MCPCommand, MCPServers, MCPStore, SSEServer, StdioServer, TokenStore, oauth
from tests.clai2.menu_script import Script, pick, typed
pytestmark = pytest.mark.anyio
URL = 'https://mcp.example.com/mcp'
Vault = dict[tuple[str, str], str]
tests/clai2/test_mid_turn_menus.py
-2
View File
@@ -28,8 +28,6 @@ from pydantic_clai2.screen import Screen
from pydantic_clai2.session_settings import SessionSettings
from pydantic_clai2.settings_store import SettingsStore
pytestmark = pytest.mark.anyio
@pytest.fixture
def anyio_backend() -> str:
tests/clai2/test_model_configuration.py
-2
View File
@@ -22,8 +22,6 @@ from pydantic_clai2.model_options import model_options, validate_model_options
from pydantic_clai2.model_settings import ModelSettingsForm, model_defaults, model_settings_from_json
from tests.clai2.menu_script import Script, make_context, pick, typed
pytestmark = pytest.mark.anyio
@pytest.fixture
def anyio_backend() -> str:
tests/clai2/test_model_defaults.py
-2
View File
@@ -20,8 +20,6 @@ from pydantic_clai2.model_settings import model_defaults
from pydantic_clai2.settings_store import SettingsStore
from tests.clai2.menu_script import Script, make_context, pick
pytestmark = pytest.mark.anyio
@pytest.fixture
def anyio_backend() -> str:
tests/clai2/test_model_menu.py
-2
View File
@@ -22,8 +22,6 @@ from pydantic_clai2.model_settings import ModelSettingsForm, model_settings_from
from pydantic_clai2.settings_store import SettingsStore
from tests.clai2.menu_script import Script, make_context, pick, typed
pytestmark = pytest.mark.anyio
@pytest.fixture
def anyio_backend() -> str:
tests/clai2/test_notifications.py
-2
View File
@@ -19,8 +19,6 @@ from pydantic_clai2.plugins import PluginHost, TurnEnd, TurnOutcome, TurnStart
from pydantic_clai2.project_settings import ProjectSettings
from pydantic_clai2.settings_store import SettingsStore
pytestmark = pytest.mark.anyio
@pytest.fixture
def sent(monkeypatch: pytest.MonkeyPatch) -> list[str]:
tests/clai2/test_openrouter.py
-2
View File
@@ -11,8 +11,6 @@ from pydantic_clai2 import openrouter
from pydantic_clai2.model_menu import open_add_model_menu
from tests.clai2.menu_script import make_context
pytestmark = pytest.mark.anyio
@pytest.fixture
def anyio_backend() -> str:
tests/clai2/test_openrouter_auth.py
-2
View File
@@ -20,8 +20,6 @@ from pydantic_clai2 import openrouter
from pydantic_clai2.openrouter_auth import OpenRouterAuth, authorization_code
from tests.clai2.menu_script import make_context
pytestmark = pytest.mark.anyio
@pytest.fixture
def anyio_backend() -> str:
tests/clai2/test_plugin_loader.py
-2
View File
@@ -19,8 +19,6 @@ from pydantic_clai2.plugin_loader import PluginError, PluginLoader
from pydantic_clai2.plugins import PluginHost, SessionEnd, SessionStart, TurnEnd, TurnStart
from pydantic_clai2.settings_store import SettingsStore
pytestmark = pytest.mark.anyio
@pytest.fixture
def anyio_backend() -> str:
tests/clai2/test_plugin_menu.py
-2
View File
@@ -17,8 +17,6 @@ from pydantic_clai2.plugin_menu import PluginMenu, open_plugins_menu
from pydantic_clai2.plugins import SessionStart
from pydantic_clai2.settings_store import SettingsStore
pytestmark = pytest.mark.anyio
PLUGIN = 'from pydantic_clai2.plugins import PluginHost\ndef activate(host: PluginHost) -> None:\n pass\n'
tests/clai2/test_plugins.py
-2
View File
@@ -24,8 +24,6 @@ from pydantic_clai2.commands import Command
from pydantic_clai2.plugins import CoreHookName, PluginHost, SessionEnd, SessionStart, Transcript, TurnEnd, TurnStart
from pydantic_clai2.settings_store import SettingsStore
pytestmark = pytest.mark.anyio
@pytest.fixture
def anyio_backend() -> str:
tests/clai2/test_project_settings.py
-2
View File
@@ -19,8 +19,6 @@ from pydantic_clai2.config import PluginSettings, resolve_settings
from pydantic_clai2.project_settings import PROJECT_FILE, ProjectSettings, find_project_file, load_project_settings
from pydantic_clai2.settings_store import SettingsStore
pytestmark = pytest.mark.anyio
def write(directory: Path, content: dict[str, JsonValue]) -> Path:
path = directory / PROJECT_FILE
tests/clai2/test_prompt_completion.py
-2
View File
@@ -11,8 +11,6 @@ from termflow.tui.completion import Completion
from pydantic_clai2.prompt_completion import CompletionWorker
pytestmark = pytest.mark.anyio
@pytest.fixture
def anyio_backend() -> str:
tests/clai2/test_prompt_controls.py
-2
View File
@@ -15,8 +15,6 @@ from pydantic_ai.models.test import TestModel
from pydantic_clai2 import chat
from pydantic_clai2.settings_store import SettingsStore
pytestmark = pytest.mark.anyio
@pytest.fixture
def anyio_backend() -> str:
tests/clai2/test_prompt_keys.py
-2
View File
@@ -10,8 +10,6 @@ from prompt_toolkit.keys import Keys
from pydantic_clai2.prompt_keys import PromptKeys
pytestmark = pytest.mark.anyio
@pytest.fixture
def anyio_backend() -> str:
tests/clai2/test_prompt_startup.py
-2
View File
@@ -18,8 +18,6 @@ from pydantic_clai2.prompt_surface import PromptSurface
from pydantic_clai2.prompt_transcript import TranscriptBuffer
from pydantic_clai2.settings_store import SettingsStore
pytestmark = pytest.mark.anyio
@pytest.fixture
def anyio_backend() -> str:
tests/clai2/test_prompt_surface.py
-2
View File
@@ -9,8 +9,6 @@ from pydantic_clai2 import prompt_surface
from pydantic_clai2.prompt_surface import PromptSurface
from tests.clai2.surface_terminal import SurfaceTerminal
pytestmark = pytest.mark.anyio
ROWS = ('TOP', 'DRAFT', 'BOTTOM', 'FOOTER')
tests/clai2/test_render_edges.py
-2
View File
@@ -18,8 +18,6 @@ from pydantic_ai.messages import (
from pydantic_ai_harness.filesystem import FileChangeRequestEvent, FileEditedEvent, FileWrittenEvent
from pydantic_clai2 import StreamRenderer
pytestmark = pytest.mark.anyio
@pytest.fixture
def anyio_backend() -> str:
tests/clai2/test_resume_integration.py
-2
View File
@@ -32,8 +32,6 @@ from pydantic_clai2.settings_store import SettingsStore
if sys.version_info < (3, 11):
from exceptiongroup import BaseExceptionGroup
pytestmark = pytest.mark.anyio
async def test_reload_keeps_saved_conversation(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
def rebuild(factory: Callable[[], object]) -> object:
tests/clai2/test_review_regressions.py
-2
View File
@@ -22,8 +22,6 @@ from pydantic_clai2.plugins import PluginHost
from pydantic_clai2.settings_store import SettingsStore
from tests.clai2.test_plugin_loader import Harness
pytestmark = pytest.mark.anyio
@pytest.fixture
def anyio_backend() -> str:
tests/clai2/test_screen.py
-2
View File
@@ -21,8 +21,6 @@ from pydantic_clai2.screen import Screen
from pydantic_clai2.settings_store import SettingsStore
from pydantic_clai2.status import Status, StatusLine
pytestmark = pytest.mark.anyio
@pytest.fixture
def anyio_backend() -> str:
tests/clai2/test_sessions.py
-2
View File
@@ -23,8 +23,6 @@ from pydantic_clai2._session import Session
from pydantic_clai2.plugins import PluginHost
from pydantic_clai2.sessions import activate
pytestmark = pytest.mark.anyio
def saved_session(tmp_path: Path) -> Session[None, str]:
return Session(
tests/clai2/test_set_menu.py
-2
View File
@@ -12,8 +12,6 @@ from pydantic_clai2.project_settings import ProjectSettings
from pydantic_clai2.set_menu import SettingsSource, open_settings_menu
from tests.clai2.menu_script import Script, make_context, pick, typed
pytestmark = pytest.mark.anyio
@pytest.fixture
def anyio_backend() -> str:
tests/clai2/test_shell_passthrough.py
-2
View File
@@ -15,8 +15,6 @@ from pydantic_clai2.settings_store import SettingsStore
from pydantic_clai2.shell_passthrough import shell_command
from tests.clai2.test_app_edges import inputs
pytestmark = pytest.mark.anyio
@pytest.mark.parametrize(
('text', 'command'),
tests/clai2/test_speculation.py
-2
View File
@@ -32,8 +32,6 @@ from pydantic_clai2.sandbox_calls import SandboxCallOrder, SandboxCallStartedEve
from pydantic_clai2.settings_store import SettingsStore
from pydantic_clai2.speculation import Speculation, SpeculationCounters
pytestmark = pytest.mark.anyio
@pytest.fixture
def anyio_backend() -> str:
tests/clai2/test_speculative_mode.py
-2
View File
@@ -44,8 +44,6 @@ from pydantic_clai2.speculative_mode import (
workspace_mount,
)
pytestmark = pytest.mark.anyio
@pytest.fixture
def anyio_backend() -> str:
tests/clai2/test_spinners.py
-2
View File
@@ -41,8 +41,6 @@ from pydantic_clai2.spinners import (
)
from tests.clai2.menu_script import Script, make_context, pick
pytestmark = pytest.mark.anyio
PromptT = TypeVar('PromptT')
CODE_PUPPY_BUILTINS = (
'puppy',
tests/clai2/test_status.py
-2
View File
@@ -16,8 +16,6 @@ from pydantic_clai2._app import _reset_status # pyright: ignore[reportPrivateUs
from pydantic_clai2.status import Status, StatusLine
from pydantic_clai2.theme import MUTED, WARNING, sgr
pytestmark = pytest.mark.anyio
@pytest.fixture
def anyio_backend() -> str:
tests/clai2/test_steering.py
-2
View File
@@ -21,8 +21,6 @@ from pydantic_clai2.project_settings import ProjectSettings
from pydantic_clai2.settings_store import SettingsStore
from tests.clai2.test_live_prompt import editor
pytestmark = pytest.mark.anyio
@pytest.mark.parametrize('supplied_handler', [False, True])
async def test_enter_during_run_teardown_queues_follow_up(supplied_handler: bool) -> None:
tests/clai2/test_streaming.py
-2
View File
@@ -14,8 +14,6 @@ from pydantic_ai.messages import ThinkingPart, ThinkingPartDelta, ToolCallPart,
from pydantic_clai2 import StreamRenderer
from pydantic_clai2.config import Settings
pytestmark = pytest.mark.anyio
@pytest.fixture
def anyio_backend() -> str:
tests/clai2/test_terminal.py
-2
View File
@@ -30,8 +30,6 @@ from pydantic_clai2.prompt_surface import PromptSurface
from pydantic_clai2.settings_store import SettingsStore
from pydantic_clai2.splash import Splash
pytestmark = pytest.mark.anyio
@pytest.fixture
def anyio_backend() -> str:
tests/clai2/test_theme_picker.py
-2
View File
@@ -26,8 +26,6 @@ from pydantic_clai2.settings_store import SettingsStore
from pydantic_clai2.theme_picker import build_theme_picker, theme_command
from tests.clai2.menu_script import Script, make_context, pick
pytestmark = pytest.mark.anyio
async def test_picker_and_settings_share_registry_and_persistence(tmp_path: Path) -> None:
context, applied = make_context(tmp_path)
tests/clai2/test_theme_preview.py
-2
View File
@@ -14,8 +14,6 @@ from pydantic_ai_harness.filesystem import FileEditedEvent
from pydantic_clai2 import StreamRenderer, theme
from pydantic_clai2.theme_picker import theme_preview
pytestmark = pytest.mark.anyio
@pytest.mark.parametrize('name', theme.names())
@pytest.mark.parametrize('width', [46, 86])
tests/clai2/test_tool_output.py
-2
View File
@@ -12,8 +12,6 @@ from pydantic_ai_harness.filesystem import FileEditedEvent
from pydantic_ai_harness.shell import CommandFinishedEvent, CommandOutputEvent, CommandStartedEvent
from pydantic_clai2 import StreamRenderer
pytestmark = pytest.mark.anyio
@pytest.fixture
def anyio_backend() -> str:
tests/clai2/test_tool_retries.py
-2
View File
@@ -18,8 +18,6 @@ from pydantic_clai2.settings_store import SettingsStore
from tests.clai2.menu_script import make_context
from tests.clai2.test_app_edges import inputs
pytestmark = pytest.mark.anyio
@pytest.fixture
def anyio_backend() -> str:
tests/clai2/test_usage_report.py
-2
View File
@@ -27,8 +27,6 @@ from pydantic_clai2.settings_store import SettingsStore
from pydantic_clai2.status import Status
from pydantic_clai2.usage_report import cost_line, format_cost, session_usage, usage_command
pytestmark = pytest.mark.anyio
PromptT = TypeVar('PromptT')
tests/clai2/test_vllm.py
-2
View File
@@ -11,8 +11,6 @@ from pydantic_clai2 import vllm
from pydantic_clai2.model_menu import open_add_model_menu
from tests.clai2.menu_script import make_context
pytestmark = pytest.mark.anyio
@pytest.fixture
def anyio_backend() -> str:
tests/clai2/test_worktrees.py
-1
View File
@@ -136,7 +136,6 @@ def test_unwritable_exclude_is_a_parser_error(repository: Path) -> None:
assert not (repository / '.worktrees').exists()
@pytest.mark.anyio
async def test_session_belongs_to_worktree_and_can_be_resumed(repository: Path) -> None:
result = launch(
repository,
tests/clai2/test_write_diff.py
-2
View File
@@ -13,8 +13,6 @@ from pydantic_ai.models.function import AgentInfo, DeltaToolCall, FunctionModel
from pydantic_ai_harness.coder import Coder
from pydantic_clai2 import Session, StreamRenderer
pytestmark = pytest.mark.anyio
@pytest.fixture
def anyio_backend() -> str:
tests/durable_exec/temporal/test_agent.py
-1
View File
@@ -199,7 +199,6 @@ with workflow.unsafe.imports_passed_through():
warnings.filterwarnings('ignore', message='`TemporalAgent` is deprecated', category=PydanticAIDeprecationWarning)
pytestmark = [
pytest.mark.anyio,
pytest.mark.vcr,
pytest.mark.xdist_group(name='temporal-agent'),
pytest.mark.filterwarnings(
tests/durable_exec/temporal/test_durability.py
-1
View File
@@ -211,7 +211,6 @@ with workflow.unsafe.imports_passed_through():
warnings.filterwarnings('ignore', message='`TemporalAgent` is deprecated', category=PydanticAIDeprecationWarning)
pytestmark = [
pytest.mark.anyio,
pytest.mark.vcr,
pytest.mark.xdist_group(name='temporal-durability'),
pytest.mark.filterwarnings(
tests/durable_exec/temporal/test_model_and_serialization.py
-1
View File
@@ -199,7 +199,6 @@ with try_import() as anthropic_imports_successful:
warnings.filterwarnings('ignore', message='`TemporalAgent` is deprecated', category=PydanticAIDeprecationWarning)
pytestmark = [
pytest.mark.anyio,
pytest.mark.vcr,
pytest.mark.xdist_group(name='temporal-model'),
pytest.mark.filterwarnings(
tests/durable_exec/temporal/test_toolsets.py
-1
View File
@@ -176,7 +176,6 @@ with workflow.unsafe.imports_passed_through():
warnings.filterwarnings('ignore', message='`TemporalAgent` is deprecated', category=PydanticAIDeprecationWarning)
pytestmark = [
pytest.mark.anyio,
pytest.mark.vcr,
pytest.mark.xdist_group(name='temporal-toolsets'),
pytest.mark.filterwarnings(
tests/durable_exec/temporal/test_workflow_streams.py
+1 -1
View File
@@ -67,7 +67,7 @@ if sys.version_info >= (3, 14): # pragma: lax no cover
with workflow.unsafe.imports_passed_through():
from ._shared import BASE_ACTIVITY_CONFIG, TASK_QUEUE
pytestmark = [pytest.mark.anyio, pytest.mark.xdist_group(name='temporal-durability')]
pytestmark = [pytest.mark.xdist_group(name='temporal-durability')]
TOPIC = 'agent_events'
tests/durable_exec/test_capability_durable_operation.py
-1
View File
@@ -104,7 +104,6 @@ else:
ActivityDefinition = TemporalDurability = cast(Any, None)
temporal_available = False
pytestmark = pytest.mark.anyio
requires_dbos = pytest.mark.skipif(not dbos_available, reason='DBOS is not installed')
requires_prefect = pytest.mark.skipif(not prefect_available, reason='Prefect is not installed')
tests/durable_exec/test_dbos.py
-1
View File
@@ -137,7 +137,6 @@ from .decision_spans import ShipIt, ShipItDecisionModel, decide_span_lineage
warnings.filterwarnings('ignore', message='`DBOSAgent` is deprecated', category=PydanticAIDeprecationWarning)
pytestmark = [
pytest.mark.anyio,
pytest.mark.vcr,
pytest.mark.xdist_group(name='dbos'),
pytest.mark.filterwarnings('ignore:`DBOSAgent` is deprecated:pydantic_ai._warnings.PydanticAIDeprecationWarning'),
tests/durable_exec/test_prefect.py
-1
View File
@@ -372,7 +372,6 @@ def test_prefect_operation_config_routes_roles_and_tool_kinds() -> None:
warnings.filterwarnings('ignore', message='`PrefectAgent` is deprecated', category=PydanticAIDeprecationWarning)
pytestmark = [
pytest.mark.anyio,
pytest.mark.vcr,
pytest.mark.xdist_group(name='prefect'),
pytest.mark.filterwarnings(
tests/evals/test_agentic_evaluators.py
+1 -1
View File
@@ -31,7 +31,7 @@ with try_import() as imports_successful:
from pydantic_evals.otel.span_tree import SpanNode, SpanStatus, SpanTree
pytestmark = [pytest.mark.skipif(not imports_successful(), reason='pydantic-evals not installed'), pytest.mark.anyio]
pytestmark = [pytest.mark.skipif(not imports_successful(), reason='pydantic-evals not installed')]
_EPOCH = datetime(2025, 1, 1, tzinfo=timezone.utc)
tests/evals/test_dataset.py
+1 -3
View File
@@ -65,7 +65,7 @@ with try_import() as tenacity_import_successful:
from pydantic_ai.retries import RetryConfig
pytestmark = [pytest.mark.skipif(not imports_successful(), reason='pydantic-evals not installed'), pytest.mark.anyio]
pytestmark = [pytest.mark.skipif(not imports_successful(), reason='pydantic-evals not installed')]
needs_logfire = pytest.mark.skipif(not logfire_import_successful(), reason='logfire not installed')
@@ -1612,7 +1612,6 @@ async def test_dataset_evaluate_with_multiple_evaluators(example_dataset: Datase
assert len(report.cases[0].scores) == 2
@pytest.mark.anyio
async def test_unnamed_cases():
dataset = Dataset[TaskInput, TaskOutput, TaskMetadata](
name='unnamed_cases',
@@ -1640,7 +1639,6 @@ async def test_unnamed_cases():
assert [case.name for case in result.cases] == ['Case 1', 'My Case', 'Case 3']
@pytest.mark.anyio
async def test_duplicate_case_names():
with pytest.raises(ValueError) as exc_info:
Dataset[TaskInput, TaskOutput, TaskMetadata](
tests/evals/test_evaluator_base.py
+1 -1
View File
@@ -22,7 +22,7 @@ with try_import() as imports_successful:
)
from pydantic_evals.otel._errors import SpanTreeRecordingError
pytestmark = [pytest.mark.skipif(not imports_successful(), reason='pydantic-evals not installed'), pytest.mark.anyio]
pytestmark = [pytest.mark.skipif(not imports_successful(), reason='pydantic-evals not installed')]
def test_evaluation_reason():
tests/evals/test_evaluator_common.py
+1 -3
View File
@@ -42,7 +42,7 @@ with try_import() as logfire_import_successful:
from pydantic_evals.otel._context_in_memory_span_exporter import context_subtree
pytestmark = [pytest.mark.skipif(not imports_successful(), reason='pydantic-evals not installed'), pytest.mark.anyio]
pytestmark = [pytest.mark.skipif(not imports_successful(), reason='pydantic-evals not installed')]
needs_logfire = pytest.mark.skipif(not logfire_import_successful(), reason='logfire not installed')
@@ -259,7 +259,6 @@ async def test_max_duration():
assert evaluator.evaluate(MockContext(duration=1.5)) is False
@pytest.mark.anyio
async def test_llm_judge_evaluator(mocker: MockerFixture):
"""Test LLMJudge evaluator."""
# Create a mock GradingOutput
@@ -389,7 +388,6 @@ async def test_llm_judge_omits_an_unavailable_reason(mocker: MockerFixture):
assert to_jsonable_python(await evaluator.evaluate(MockContext(output='Hello'))) == snapshot({'LLMJudge': True})
@pytest.mark.anyio
async def test_llm_judge_evaluator_with_model_settings(mocker: MockerFixture):
"""Test LLMJudge evaluator with specific model_settings."""
mock_grading_output = mocker.MagicMock()
tests/evals/test_evaluator_context.py
+1 -1
View File
@@ -14,7 +14,7 @@ with try_import() as imports_successful:
from pydantic_evals.otel._errors import SpanTreeRecordingError
from pydantic_evals.otel.span_tree import SpanNode, SpanTree
pytestmark = [pytest.mark.skipif(not imports_successful(), reason='pydantic-evals not installed'), pytest.mark.anyio]
pytestmark = [pytest.mark.skipif(not imports_successful(), reason='pydantic-evals not installed')]
def test_evaluator_context_basic():
tests/evals/test_evaluator_spec.py
+1 -1
View File
@@ -11,7 +11,7 @@ with try_import() as imports_successful:
from pydantic_evals.evaluators.spec import EvaluatorSpec
from pydantic_evals.otel.span_tree import SpanQuery
pytestmark = [pytest.mark.skipif(not imports_successful(), reason='pydantic-evals not installed'), pytest.mark.anyio]
pytestmark = [pytest.mark.skipif(not imports_successful(), reason='pydantic-evals not installed')]
def test_evaluator_spec_basic():
tests/evals/test_evaluators.py
+1 -1
View File
@@ -43,7 +43,7 @@ with try_import() as logfire_import_successful:
from pydantic_evals.otel._context_in_memory_span_exporter import context_subtree
pytestmark = [pytest.mark.skipif(not imports_successful(), reason='pydantic-evals not installed'), pytest.mark.anyio]
pytestmark = [pytest.mark.skipif(not imports_successful(), reason='pydantic-evals not installed')]
needs_logfire = pytest.mark.skipif(not logfire_import_successful(), reason='logfire not installed')
tests/evals/test_llm_as_a_judge.py
+1 -10
View File
@@ -39,7 +39,7 @@ with try_import() as imports_successful:
judge_output_expected,
)
pytestmark = [pytest.mark.skipif(not imports_successful(), reason='pydantic-evals not installed'), pytest.mark.anyio]
pytestmark = [pytest.mark.skipif(not imports_successful(), reason='pydantic-evals not installed')]
@pytest.fixture(autouse=True)
@@ -72,7 +72,6 @@ def test_grading_output():
assert schema['properties']['reason']['description'] == ('A concise 1-2 sentence justification for the verdict.')
@pytest.mark.anyio
async def test_judge_prompts_constrain_reason():
"""Every judge sends the concise-reason instruction in its system prompt (#5034).
@@ -433,7 +432,6 @@ def test_build_prompt_section_order_matches_few_shot_examples(
assert example_tags[start : start + len(expected_tags)] == expected_tags
@pytest.mark.anyio
async def test_judge_output_mock(mocker: MockerFixture):
"""Test judge_output function with mocked agent."""
# Mock the agent run method
@@ -455,7 +453,6 @@ async def test_judge_output_mock(mocker: MockerFixture):
assert '<Rubric>\nContent contains a greeting\n</Rubric>' in call_args[0]
@pytest.mark.anyio
async def test_judge_output_with_model_settings_mock(mocker: MockerFixture):
"""Test judge_output function with model_settings and mocked agent."""
mock_result = mocker.MagicMock()
@@ -483,7 +480,6 @@ async def test_judge_output_with_model_settings_mock(mocker: MockerFixture):
assert 'model' in call_kwargs
@pytest.mark.anyio
async def test_judge_input_output_mock(mocker: MockerFixture):
"""Test judge_input_output function with mocked agent."""
# Mock the agent run method
@@ -574,7 +570,6 @@ async def test_judge_input_output_binary_content_mock(mocker: MockerFixture, ima
assert image_content in raw_prompt, 'Expected the exact BinaryContent instance to be in the prompt list'
@pytest.mark.anyio
async def test_judge_input_output_with_model_settings_mock(mocker: MockerFixture):
"""Test judge_input_output function with model_settings and mocked agent."""
mock_result = mocker.MagicMock()
@@ -604,7 +599,6 @@ async def test_judge_input_output_with_model_settings_mock(mocker: MockerFixture
assert 'model' in call_kwargs
@pytest.mark.anyio
async def test_judge_input_output_expected_mock(mocker: MockerFixture, image_content: BinaryContent):
"""Test judge_input_output_expected function with mocked agent."""
# Mock the agent run method
@@ -667,7 +661,6 @@ Output contains input
)
@pytest.mark.anyio
async def test_judge_input_output_expected_with_model_settings_mock(
mocker: MockerFixture, image_content: BinaryContent
):
@@ -818,7 +811,6 @@ Output contains input with settings
)
@pytest.mark.anyio
async def test_judge_output_expected_mock(mocker: MockerFixture):
"""Test judge_output_expected function with mocked agent."""
# Mock the agent run method
@@ -841,7 +833,6 @@ async def test_judge_output_expected_mock(mocker: MockerFixture):
assert '<Rubric>\nOutput contains input\n</Rubric>' in call_args[0]
@pytest.mark.anyio
async def test_judge_output_expected_with_model_settings_mock(mocker: MockerFixture, image_content: BinaryContent):
"""Test judge_output_expected function with model_settings and mocked agent."""
mock_result = mocker.MagicMock()
tests/evals/test_multi_run.py
+1 -1
View File
@@ -40,7 +40,7 @@ with try_import() as logfire_import_successful:
from logfire.testing import CaptureLogfire
pytestmark = [pytest.mark.skipif(not imports_successful(), reason='pydantic-evals not installed'), pytest.mark.anyio]
pytestmark = [pytest.mark.skipif(not imports_successful(), reason='pydantic-evals not installed')]
needs_logfire = pytest.mark.skipif(not logfire_import_successful(), reason='logfire not installed')
tests/evals/test_online.py
-99
View File
@@ -146,7 +146,6 @@ if TYPE_CHECKING or imports_successful():
return [self._data[s.span_id] for s in spans]
@pytest.mark.anyio
async def test_callback_sink_sync():
"""CallbackSink works with sync callbacks."""
collected: list[tuple[list[Any], list[Any], Any]] = []
@@ -172,7 +171,6 @@ async def test_callback_sink_sync():
assert collected[0][2] is ctx
@pytest.mark.anyio
async def test_callback_sink_async():
"""CallbackSink works with async callbacks."""
collector = Collector()
@@ -186,7 +184,6 @@ async def test_callback_sink_async():
assert len(collector.calls) == 1
@pytest.mark.anyio
async def test_callback_sink_ignores_span_reference():
"""CallbackSink does not pass span_reference to the callback."""
collector = Collector()
@@ -201,7 +198,6 @@ async def test_callback_sink_ignores_span_reference():
assert collector.result_count == 0
@pytest.mark.anyio
async def test_span_reference():
"""SpanReference stores trace and span IDs."""
ref = SpanReference(trace_id='abc123', span_id='def456')
@@ -209,7 +205,6 @@ async def test_span_reference():
assert ref.span_id == 'def456'
@pytest.mark.anyio
async def test_online_evaluator_defaults():
"""OnlineEvaluator has sensible defaults."""
evaluator = AlwaysTrue()
@@ -220,7 +215,6 @@ async def test_online_evaluator_defaults():
assert online.max_concurrency == 10
@pytest.mark.anyio
async def test_online_evaluator_custom_config():
"""OnlineEvaluator accepts custom configuration."""
evaluator = AlwaysTrue()
@@ -244,7 +238,6 @@ def test_online_evaluator_invalid_max_concurrency(max_concurrency: int):
OnlineEvaluator(evaluator=AlwaysTrue(), max_concurrency=max_concurrency)
@pytest.mark.anyio
async def test_run_evaluators_success():
"""run_evaluators returns results from all evaluators."""
ctx = _make_context(output=42)
@@ -256,7 +249,6 @@ async def test_run_evaluators_success():
assert results[1].value is True
@pytest.mark.anyio
async def test_run_evaluators_with_failure():
"""run_evaluators collects failures separately from results."""
ctx = _make_context(output=42)
@@ -268,7 +260,6 @@ async def test_run_evaluators_with_failure():
assert 'Simulated evaluator failure' in failures[0].error_message
@pytest.mark.anyio
async def test_run_evaluators_empty():
"""run_evaluators handles empty evaluator list."""
ctx = _make_context(output=42)
@@ -277,7 +268,6 @@ async def test_run_evaluators_empty():
assert failures == []
@pytest.mark.anyio
async def test_run_evaluators_multi_result():
"""run_evaluators handles evaluators that return multiple results."""
ctx = _make_context(output=42)
@@ -289,7 +279,6 @@ async def test_run_evaluators_multi_result():
assert result_names == {'accuracy', 'score', 'label'}
@pytest.mark.anyio
async def test_run_evaluators_async_evaluator():
"""run_evaluators works with async evaluators."""
ctx = _make_context(output=42)
@@ -300,7 +289,6 @@ async def test_run_evaluators_async_evaluator():
assert len(failures) == 0
@pytest.mark.anyio
async def test_evaluate_decorator_async_basic():
"""evaluate() decorator runs evaluators on async function calls."""
collector = Collector()
@@ -324,7 +312,6 @@ async def test_evaluate_decorator_async_basic():
assert ctx.inputs == {'x': 21}
@pytest.mark.anyio
async def test_evaluate_decorator_async_callable_instance():
"""An async callable instance is awaited; this local wrapper dispatch has no provider boundary to record."""
collector = Collector()
@@ -353,7 +340,6 @@ async def test_evaluate_decorator_async_callable_instance():
assert inspect.isawaitable(ctx.output) is False
@pytest.mark.anyio
async def test_evaluate_decorator_async_preserves_signature():
"""evaluate() decorator preserves the function's name and docs."""
@@ -367,7 +353,6 @@ async def test_evaluate_decorator_async_preserves_signature():
assert await my_func(42) == 42
@pytest.mark.anyio
async def test_evaluate_decorator_multiple_evaluators():
"""evaluate() decorator runs multiple evaluators."""
collector = Collector()
@@ -386,7 +371,6 @@ async def test_evaluate_decorator_multiple_evaluators():
assert collector.result_count == 2
@pytest.mark.anyio
async def test_evaluate_decorator_async_default_skips_dispatch_on_exception():
"""By default, evaluators are not dispatched when the decorated async function raises."""
collector = Collector()
@@ -403,7 +387,6 @@ async def test_evaluate_decorator_async_default_skips_dispatch_on_exception():
assert collector.calls == []
@pytest.mark.anyio
async def test_evaluate_decorator_async_run_on_errors_dispatches():
"""`run_on_errors=True` dispatches the evaluator with the raised exception as `output`."""
collector = Collector()
@@ -426,7 +409,6 @@ async def test_evaluate_decorator_async_run_on_errors_dispatches():
assert ctx.inputs == {'x': 42}
@pytest.mark.anyio
async def test_evaluate_decorator_async_run_on_errors_filters_evaluators():
"""When some evaluators opt in and some don't, only the opted-in ones run on error."""
collector = Collector()
@@ -449,7 +431,6 @@ async def test_evaluate_decorator_async_run_on_errors_filters_evaluators():
assert results[0].value is True
@pytest.mark.anyio
async def test_evaluate_decorator_sync_run_on_errors_dispatches():
"""Sync decorator: `run_on_errors=True` dispatches with the exception as `output`."""
collector = Collector()
@@ -469,7 +450,6 @@ async def test_evaluate_decorator_sync_run_on_errors_dispatches():
assert isinstance(ctx.output, RuntimeError)
@pytest.mark.anyio
async def test_evaluate_decorator_sync_run_on_errors_no_event_loop():
"""Sync `run_on_errors=True` without a running loop dispatches via background thread."""
collector = Collector()
@@ -496,7 +476,6 @@ async def test_evaluate_decorator_sync_run_on_errors_no_event_loop():
assert isinstance(ctx.output, RuntimeError)
@pytest.mark.anyio
async def test_evaluate_decorator_with_failure():
"""evaluate() decorator handles evaluator failures gracefully."""
collector = Collector()
@@ -518,7 +497,6 @@ async def test_evaluate_decorator_with_failure():
assert 'Simulated evaluator failure' in failures[0].error_message
@pytest.mark.anyio
async def test_sample_rate_zero_skips_evaluation():
"""sample_rate=0.0 skips all evaluations."""
collector = Collector()
@@ -535,7 +513,6 @@ async def test_sample_rate_zero_skips_evaluation():
assert len(collector.calls) == 0
@pytest.mark.anyio
async def test_sample_rate_one_always_evaluates():
"""sample_rate=1.0 always evaluates."""
collector = Collector()
@@ -552,7 +529,6 @@ async def test_sample_rate_one_always_evaluates():
assert len(collector.calls) == 5
@pytest.mark.anyio
async def test_sample_rate_callable():
"""sample_rate as a callable is evaluated each time."""
call_count = 0
@@ -576,7 +552,6 @@ async def test_sample_rate_callable():
assert len(collector.calls) == 1
@pytest.mark.anyio
async def test_sample_rate_callable_returning_bool():
"""sample_rate callable can return bool."""
collector = Collector()
@@ -591,7 +566,6 @@ async def test_sample_rate_callable_returning_bool():
assert len(collector.calls) == 0
@pytest.mark.anyio
async def test_disable_evaluation_context_manager():
"""disable_evaluation() suppresses all evaluators."""
collector = Collector()
@@ -609,7 +583,6 @@ async def test_disable_evaluation_context_manager():
assert len(collector.calls) == 0
@pytest.mark.anyio
async def test_disable_evaluation_restores():
"""disable_evaluation() restores evaluation after exiting."""
collector = Collector()
@@ -628,7 +601,6 @@ async def test_disable_evaluation_restores():
assert len(collector.calls) == 1
@pytest.mark.anyio
async def test_config_enabled_false():
"""OnlineEvalConfig.enabled=False disables all evaluation."""
collector = Collector()
@@ -645,7 +617,6 @@ async def test_config_enabled_false():
assert len(collector.calls) == 0
@pytest.mark.anyio
async def test_per_evaluator_sink_override():
"""OnlineEvaluator.sink overrides config's default_sink."""
default_collector = Collector()
@@ -667,7 +638,6 @@ async def test_per_evaluator_sink_override():
assert len(override_collector.calls) == 1
@pytest.mark.anyio
@needs_logfire
async def test_no_sink_still_emits_otel_events(capfire: CaptureLogfire):
"""When no sink is configured, evaluators still run and emit OTel events.
@@ -692,7 +662,6 @@ async def test_no_sink_still_emits_otel_events(capfire: CaptureLogfire):
assert attrs['gen_ai.evaluation.score.label'] == 'pass'
@pytest.mark.anyio
@needs_logfire
async def test_emit_otel_events_false_disables_emission(capfire: CaptureLogfire):
"""`emit_otel_events=False` suppresses the default OTel emission."""
@@ -720,7 +689,6 @@ async def test_emit_otel_events_false_disables_emission(capfire: CaptureLogfire)
assert list(capfire.log_exporter.get_finished_logs()) == []
@pytest.mark.anyio
async def test_configure_updates_default_config():
"""configure() updates the global DEFAULT_CONFIG."""
original_enabled = DEFAULT_CONFIG.enabled
@@ -757,7 +725,6 @@ async def test_configure_updates_default_config():
DEFAULT_CONFIG.include_baggage = True
@pytest.mark.anyio
async def test_configure_can_reset_to_none():
"""configure() can explicitly set fields to None to clear them."""
collector = Collector()
@@ -778,7 +745,6 @@ async def test_configure_can_reset_to_none():
DEFAULT_CONFIG.metadata = original_metadata
@pytest.mark.anyio
async def test_module_level_evaluate():
"""Module-level evaluate() delegates to DEFAULT_CONFIG."""
collector = Collector()
@@ -799,7 +765,6 @@ async def test_module_level_evaluate():
DEFAULT_CONFIG.default_sink = original_sink
@pytest.mark.anyio
async def test_context_source_fetch():
"""EvaluatorContextSource.fetch retrieves stored context data."""
source = MockContextSource(
@@ -822,7 +787,6 @@ async def test_context_source_fetch():
assert ctx.duration == 1.5
@pytest.mark.anyio
async def test_context_source_fetch_many():
"""EvaluatorContextSource.fetch_many retrieves multiple contexts in batch."""
source = MockContextSource(
@@ -845,7 +809,6 @@ async def test_context_source_fetch_many():
assert contexts[1].output == 'b'
@pytest.mark.anyio
async def test_fetch_and_run_evaluators():
"""EvaluatorContextSource.fetch + run_evaluators works end-to-end."""
source = MockContextSource(
@@ -862,7 +825,6 @@ async def test_fetch_and_run_evaluators():
assert all(r.value is True for r in results)
@pytest.mark.anyio
async def test_config_metadata_passed_to_context():
"""OnlineEvalConfig.metadata is included in the EvaluatorContext."""
collected_contexts: list[EvaluatorContext[Any, Any, Any]] = []
@@ -890,7 +852,6 @@ async def test_config_metadata_passed_to_context():
assert collected_contexts[0].metadata == {'service': 'test-app', 'version': '1.0'}
@pytest.mark.anyio
async def test_max_concurrency_respected():
"""OnlineEvaluator.max_concurrency limits concurrent evaluations."""
active = 0
@@ -926,7 +887,6 @@ async def test_max_concurrency_respected():
assert max_active <= 2
@pytest.mark.anyio
async def test_custom_sink_protocol():
"""Custom EvaluationSink implementations work."""
@@ -953,7 +913,6 @@ async def test_custom_sink_protocol():
assert results[0].value is True
@pytest.mark.anyio
async def test_bare_evaluator_uses_config_defaults():
"""Bare Evaluator passed to evaluate() uses config's default_sample_rate."""
collector = Collector()
@@ -968,7 +927,6 @@ async def test_bare_evaluator_uses_config_defaults():
assert len(collector.calls) == 0
@pytest.mark.anyio
async def test_bare_evaluator_late_binds_config_defaults():
"""Config defaults are resolved at call time, not decoration time."""
collector = Collector()
@@ -1003,7 +961,6 @@ async def test_bare_evaluator_late_binds_config_defaults():
assert len(collector2.calls) == 0 # still 0 because OnlineEvaluator has explicit sample_rate=0.0
@pytest.mark.anyio
async def test_multiple_sinks():
"""Multiple sinks receive all results."""
collector1 = Collector()
@@ -1022,7 +979,6 @@ async def test_multiple_sinks():
assert len(collector2.calls) == 1
@pytest.mark.anyio
async def test_fractional_sample_rate():
"""Fractional sample_rate evaluates a subset of calls."""
collector = Collector()
@@ -1041,7 +997,6 @@ async def test_fractional_sample_rate():
assert 5 < len(collector.calls) < 45
@pytest.mark.anyio
async def test_sample_rate_callable_exception_propagates():
"""Exception in sample_rate callable propagates — it's the user's responsibility."""
collector = Collector()
@@ -1059,7 +1014,6 @@ async def test_sample_rate_callable_exception_propagates():
await my_func(42)
@pytest.mark.anyio
async def test_sample_rate_callable_exception_calls_on_sampling_error():
"""When on_sampling_error is set, sample_rate exceptions are routed there instead of propagating."""
errors: list[tuple[Exception, Evaluator]] = []
@@ -1086,7 +1040,6 @@ async def test_sample_rate_callable_exception_calls_on_sampling_error():
assert str(errors[0][0]) == 'rate error'
@pytest.mark.anyio
async def test_on_sampling_error_handler_exception_suppressed():
"""If on_sampling_error itself raises, the exception is suppressed and the evaluator is skipped."""
collector = Collector()
@@ -1108,7 +1061,6 @@ async def test_on_sampling_error_handler_exception_suppressed():
assert len(collector.calls) == 0
@pytest.mark.anyio
async def test_sink_exception_does_not_propagate():
"""Exception in a sink is logged but does not break other sinks."""
@@ -1131,7 +1083,6 @@ async def test_sink_exception_does_not_propagate():
assert len(collector.calls) == 1
@pytest.mark.anyio
async def test_on_max_concurrency_exception_suppressed_when_no_on_error():
"""`on_max_concurrency` raising with no `on_error` configured is silently swallowed."""
@@ -1165,7 +1116,6 @@ async def test_on_max_concurrency_exception_suppressed_when_no_on_error():
await wait_for_evaluations()
@pytest.mark.anyio
async def test_sink_exception_suppressed_when_no_on_error():
"""A sink raising with no `on_error` configured is silently swallowed."""
@@ -1185,7 +1135,6 @@ async def test_sink_exception_suppressed_when_no_on_error():
await wait_for_evaluations()
@pytest.mark.anyio
async def test_shared_on_error_across_evaluators_fires_once_per_sink_failure():
"""When multiple evaluators in a group share the same `on_error` handler, a single
sink failure still only fires it once — dedup by handler identity."""
@@ -1217,7 +1166,6 @@ async def test_shared_on_error_across_evaluators_fires_once_per_sink_failure():
assert fires == ['sink']
@pytest.mark.anyio
async def test_evaluator_returning_empty_mapping_emits_nothing():
"""An evaluator returning `{}` produces no results — the empty-batch branch skips the submit."""
collector = Collector()
@@ -1239,7 +1187,6 @@ async def test_evaluator_returning_empty_mapping_emits_nothing():
assert collector.calls == []
@pytest.mark.anyio
async def test_sync_function_from_async_context():
"""Sync decorated function called from async context dispatches via background thread."""
collector = Collector()
@@ -1262,7 +1209,6 @@ async def test_sync_function_from_async_context():
@needs_logfire
@pytest.mark.anyio
async def test_span_reference_with_configured_logfire(capfire: CaptureLogfire):
"""Decorator produces valid SpanReference when logfire is configured."""
span_refs: list[SpanReference | None] = []
@@ -1290,7 +1236,6 @@ async def test_span_reference_with_configured_logfire(capfire: CaptureLogfire):
assert int(ref.span_id, 16) != 0
@pytest.mark.anyio
async def test_sync_decorated_function_dispatch():
"""Sync decorated function dispatches evaluators when called from async context."""
collector = Collector()
@@ -1313,7 +1258,6 @@ async def test_sync_decorated_function_dispatch():
assert ctx.inputs == {'x': 21}
@pytest.mark.anyio
async def test_sync_decorated_function_disabled():
"""Disabled config doesn't dispatch evaluators for sync decorated functions."""
collector = Collector()
@@ -1330,7 +1274,6 @@ async def test_sync_decorated_function_disabled():
assert len(collector.calls) == 0
@pytest.mark.anyio
async def test_sync_decorated_function_sample_rate_zero():
"""sample_rate=0 doesn't dispatch evaluators for sync decorated functions."""
collector = Collector()
@@ -1347,7 +1290,6 @@ async def test_sync_decorated_function_sample_rate_zero():
assert len(collector.calls) == 0
@pytest.mark.anyio
async def test_sync_function_no_event_loop():
"""Sync decorated function called without an event loop dispatches via background thread."""
collector = Collector()
@@ -1373,7 +1315,6 @@ async def test_sync_function_no_event_loop():
assert ctx.inputs == {'x': 21}
@pytest.mark.anyio
async def test_wait_for_evaluations_joins_background_thread():
"""wait_for_evaluations() joins a background thread that is still running when the wait begins."""
started = threading.Event()
@@ -1426,7 +1367,6 @@ async def test_wait_for_evaluations_joins_background_thread():
assert ctx.inputs == {'x': 21}
@pytest.mark.anyio
async def test_wait_for_evaluations_warns_on_background_thread_timeout():
"""wait_for_evaluations(timeout=...) warns when a background thread outlives the timeout."""
started = threading.Event()
@@ -1473,7 +1413,6 @@ async def test_wait_for_evaluations_warns_on_background_thread_timeout():
assert results[0].value is True
@pytest.mark.anyio
async def test_mixed_list_sink():
"""A list containing both a bare callable and a CallbackSink exercises _normalize_single_sink."""
collector1 = Collector()
@@ -1494,7 +1433,6 @@ async def test_mixed_list_sink():
assert len(collector2.calls) == 1
@pytest.mark.anyio
async def test_on_max_concurrency_callback():
"""on_max_concurrency is called when evaluations are dropped."""
dropped_contexts: list[EvaluatorContext[Any, Any, Any]] = []
@@ -1531,7 +1469,6 @@ async def test_on_max_concurrency_callback():
assert len(dropped_contexts) + len(collector.calls) == 5
@pytest.mark.anyio
async def test_on_max_concurrency_async_callback():
"""on_max_concurrency works with async callbacks."""
dropped_count = 0
@@ -1569,7 +1506,6 @@ async def test_on_max_concurrency_async_callback():
assert dropped_count + len(collector.calls) == 5
@pytest.mark.anyio
async def test_on_max_concurrency_config_default():
"""OnlineEvalConfig.on_max_concurrency is used when OnlineEvaluator doesn't set one."""
dropped_contexts: list[EvaluatorContext[Any, Any, Any]] = []
@@ -1599,7 +1535,6 @@ async def test_on_max_concurrency_config_default():
assert len(dropped_contexts) + len(collector.calls) == 5
@pytest.mark.anyio
async def test_on_max_concurrency_evaluator_overrides_config():
"""OnlineEvaluator.on_max_concurrency overrides the config default."""
config_drops: list[EvaluatorContext[Any, Any, Any]] = []
@@ -1642,7 +1577,6 @@ async def test_on_max_concurrency_evaluator_overrides_config():
# --- on_error tests ---
@pytest.mark.anyio
async def test_on_error_sink_exception():
"""on_error is called with 'sink' location when sink raises."""
errors: list[tuple[Exception, OnErrorLocation]] = []
@@ -1675,7 +1609,6 @@ async def test_on_error_sink_exception():
assert len(good_collector.calls) == 1
@pytest.mark.anyio
async def test_on_error_on_max_concurrency_exception():
"""on_error is called with 'on_max_concurrency' when on_max_concurrency callback raises."""
errors: list[tuple[Exception, OnErrorLocation]] = []
@@ -1720,7 +1653,6 @@ async def test_on_error_on_max_concurrency_exception():
assert all(loc == 'on_max_concurrency' for _, loc in errors)
@pytest.mark.anyio
async def test_on_error_handler_exception_suppressed():
"""on_error handler that raises is silently suppressed."""
@@ -1753,7 +1685,6 @@ async def test_on_error_handler_exception_suppressed():
assert len(good_collector.calls) == 1
@pytest.mark.anyio
async def test_on_error_per_evaluator_overrides_config():
"""Per-evaluator on_error overrides the config default."""
evaluator_errors: list[OnErrorLocation] = []
@@ -1793,7 +1724,6 @@ async def test_on_error_per_evaluator_overrides_config():
assert evaluator_errors[0] == 'sink'
@pytest.mark.anyio
async def test_on_error_async_callback():
"""Async on_error callback works."""
errors: list[OnErrorLocation] = []
@@ -1824,7 +1754,6 @@ async def test_on_error_async_callback():
assert errors[0] == 'sink'
@pytest.mark.anyio
async def test_configure_on_sampling_error():
"""configure() can set on_sampling_error on DEFAULT_CONFIG."""
original = DEFAULT_CONFIG.on_sampling_error
@@ -1842,7 +1771,6 @@ async def test_configure_on_sampling_error():
DEFAULT_CONFIG.on_sampling_error = original
@pytest.mark.anyio
async def test_configure_on_error():
"""configure() can set on_error on DEFAULT_CONFIG."""
original = DEFAULT_CONFIG.on_error
@@ -1868,7 +1796,6 @@ async def test_configure_on_error():
# --- SamplingContext tests ---
@pytest.mark.anyio
async def test_sampling_context_passed_to_callable():
"""SamplingContext is passed to sample_rate callables with correct data."""
captured_contexts: list[SamplingContext] = []
@@ -1898,7 +1825,6 @@ async def test_sampling_context_passed_to_callable():
assert 0.0 <= ctx.call_seed < 1.0
@pytest.mark.anyio
async def test_sampling_context_call_seed_shared_across_evaluators():
"""call_seed is the same for all evaluators in a single call."""
captured_seeds: list[float] = []
@@ -1924,7 +1850,6 @@ async def test_sampling_context_call_seed_shared_across_evaluators():
assert captured_seeds[0] == captured_seeds[1]
@pytest.mark.anyio
async def test_sampling_context_input_based_sampling():
"""sample_rate callable can use inputs to decide whether to evaluate."""
collector = Collector()
@@ -1949,7 +1874,6 @@ async def test_sampling_context_input_based_sampling():
# --- correlated sampling tests ---
@pytest.mark.anyio
async def test_correlated_sampling_subset_property(monkeypatch: pytest.MonkeyPatch):
"""In correlated mode, lower-rate evaluator calls are a subset of higher-rate ones."""
# Seed the sampler so the test is deterministic. Without a seed this test
@@ -1981,7 +1905,6 @@ async def test_correlated_sampling_subset_property(monkeypatch: pytest.MonkeyPat
assert len(collector_low.calls) > 0
@pytest.mark.anyio
async def test_correlated_sampling_max_overhead(monkeypatch: pytest.MonkeyPatch):
"""In correlated mode, total overhead probability equals max(rate_i)."""
# Seed the sampler so the `5 < calls < 40` window is deterministic.
@@ -2011,7 +1934,6 @@ async def test_correlated_sampling_max_overhead(monkeypatch: pytest.MonkeyPatch)
assert 5 < len(collector1.calls) < 40
@pytest.mark.anyio
async def test_independent_sampling_is_default(monkeypatch: pytest.MonkeyPatch):
"""Independent mode is the default — evaluators sample independently."""
# Seed the sampler so `>0` for both collectors is deterministic.
@@ -2039,7 +1961,6 @@ async def test_independent_sampling_is_default(monkeypatch: pytest.MonkeyPatch):
assert len(collector2.calls) > 0
@pytest.mark.anyio
async def test_configure_sampling_mode():
"""configure() can set sampling_mode on DEFAULT_CONFIG."""
original = DEFAULT_CONFIG.sampling_mode
@@ -2056,7 +1977,6 @@ async def test_configure_sampling_mode():
# --- attributes/metrics tests ---
@pytest.mark.anyio
async def test_set_eval_attribute_in_async_function():
"""set_eval_attribute in an async decorated function propagates to EvaluatorContext."""
collector = Collector()
@@ -2078,7 +1998,6 @@ async def test_set_eval_attribute_in_async_function():
assert ctx.attributes == {'model': 'gpt-4o', 'region': 'us-east-1'}
@pytest.mark.anyio
async def test_increment_eval_metric_in_async_function():
"""increment_eval_metric in an async decorated function propagates to EvaluatorContext."""
collector = Collector()
@@ -2101,7 +2020,6 @@ async def test_increment_eval_metric_in_async_function():
assert ctx.metrics == {'tokens': 200, 'requests': 1}
@pytest.mark.anyio
async def test_set_eval_attribute_in_sync_function():
"""set_eval_attribute in a sync decorated function propagates to EvaluatorContext."""
collector = Collector()
@@ -2122,7 +2040,6 @@ async def test_set_eval_attribute_in_sync_function():
assert ctx.attributes == {'model': 'gpt-4o'}
@pytest.mark.anyio
async def test_increment_eval_metric_in_sync_function():
"""increment_eval_metric in a sync decorated function propagates to EvaluatorContext."""
collector = Collector()
@@ -2143,7 +2060,6 @@ async def test_increment_eval_metric_in_sync_function():
assert ctx.metrics == {'tokens': 100}
@pytest.mark.anyio
async def test_attributes_and_metrics_empty_by_default():
"""When no attributes/metrics are set, context has empty dicts."""
collector = Collector()
@@ -2162,7 +2078,6 @@ async def test_attributes_and_metrics_empty_by_default():
assert ctx.metrics == {}
@pytest.mark.anyio
async def test_online_eval_suppressed_inside_task_run():
"""Online evaluation is suppressed when already inside `CURRENT_TASK_RUN`."""
from pydantic_evals._task_run import CURRENT_TASK_RUN, TaskRun
@@ -2188,7 +2103,6 @@ async def test_online_eval_suppressed_inside_task_run():
assert len(collector.calls) == 0
@pytest.mark.anyio
async def test_metadata_not_shared_between_contexts():
"""config.metadata is copied so sinks can't corrupt the config."""
collected_contexts: list[EvaluatorContext[Any, Any, Any]] = []
@@ -2229,7 +2143,6 @@ async def test_metadata_not_shared_between_contexts():
@needs_logfire
@pytest.mark.anyio
async def test_call_span_default_name_and_no_args(capfire: CaptureLogfire):
"""Each decorated call opens a span named after the function; args/return are not recorded by default."""
config = OnlineEvalConfig(emit_otel_events=False)
@@ -2250,7 +2163,6 @@ async def test_call_span_default_name_and_no_args(capfire: CaptureLogfire):
@needs_logfire
@pytest.mark.anyio
async def test_call_span_extract_args_true_records_all(capfire: CaptureLogfire):
"""`extract_args=True` records every bound argument."""
config = OnlineEvalConfig(emit_otel_events=False)
@@ -2271,7 +2183,6 @@ async def test_call_span_extract_args_true_records_all(capfire: CaptureLogfire):
@needs_logfire
@pytest.mark.anyio
async def test_call_span_extract_args_subset(capfire: CaptureLogfire):
"""Passing a list to `extract_args` records only the named arguments."""
config = OnlineEvalConfig(emit_otel_events=False)
@@ -2292,7 +2203,6 @@ async def test_call_span_extract_args_subset(capfire: CaptureLogfire):
@needs_logfire
@pytest.mark.anyio
async def test_call_span_record_return(capfire: CaptureLogfire):
"""`record_return=True` records the function's return value on the span."""
config = OnlineEvalConfig(emit_otel_events=False)
@@ -2311,7 +2221,6 @@ async def test_call_span_record_return(capfire: CaptureLogfire):
@needs_logfire
@pytest.mark.anyio
async def test_call_span_msg_template_and_span_name(capfire: CaptureLogfire):
"""`msg_template` formats against call args (logfire convention); `span_name` overrides the span name."""
config = OnlineEvalConfig(emit_otel_events=False)
@@ -2333,7 +2242,6 @@ async def test_call_span_msg_template_and_span_name(capfire: CaptureLogfire):
@needs_logfire
@pytest.mark.anyio
async def test_evaluation_events_parented_to_call_span(capfire: CaptureLogfire):
"""Emitted `gen_ai.evaluation.result` events parent to the decorated call's span."""
config = OnlineEvalConfig() # default emit_otel_events=True
@@ -2387,7 +2295,6 @@ def test_extract_args_unknown_parameter_raises():
@needs_logfire
@pytest.mark.anyio
async def test_extract_args_accepts_single_string(capfire: CaptureLogfire):
"""A bare string is treated as a one-element list of arg names."""
config = OnlineEvalConfig(default_sink=Collector(), emit_otel_events=False)
@@ -2408,7 +2315,6 @@ async def test_extract_args_accepts_single_string(capfire: CaptureLogfire):
@needs_logfire
@pytest.mark.anyio
async def test_extract_args_empty_iterable_records_nothing(capfire: CaptureLogfire):
"""An empty iterable for `extract_args` is treated as `False`."""
config = OnlineEvalConfig(default_sink=Collector(), emit_otel_events=False)
@@ -2427,7 +2333,6 @@ async def test_extract_args_empty_iterable_records_nothing(capfire: CaptureLogfi
@needs_logfire
@pytest.mark.anyio
async def test_sync_call_span_with_extract_args(capfire: CaptureLogfire):
"""Sync decorated functions also open a span and honour `extract_args`."""
config = OnlineEvalConfig(emit_otel_events=False)
@@ -2451,7 +2356,6 @@ async def test_sync_call_span_with_extract_args(capfire: CaptureLogfire):
@needs_logfire
@pytest.mark.anyio
async def test_dispatch_skipped_when_emit_off_and_no_sinks(capfire: CaptureLogfire):
"""Skip evaluator dispatch entirely when results would have nowhere to go."""
config = OnlineEvalConfig(emit_otel_events=False) # no sinks either
@@ -2472,7 +2376,6 @@ async def test_dispatch_skipped_when_emit_off_and_no_sinks(capfire: CaptureLogfi
@needs_logfire
@pytest.mark.anyio
async def test_evaluator_span_nested_under_call_span(capfire: CaptureLogfire):
"""The `evaluator: {name}` span created in `run_evaluator` parents to the call span."""
# Need a sink (or `emit_otel_events=True`) to keep dispatch active — see
@@ -2498,7 +2401,6 @@ async def test_evaluator_span_nested_under_call_span(capfire: CaptureLogfire):
@needs_logfire
@pytest.mark.anyio
async def test_baggage_attached_to_evaluation_event(capfire: CaptureLogfire):
"""Baggage set in the calling context propagates onto emitted evaluation events."""
from opentelemetry import baggage as ot_baggage, context as ot_context
@@ -2524,7 +2426,6 @@ async def test_baggage_attached_to_evaluation_event(capfire: CaptureLogfire):
@needs_logfire
@pytest.mark.anyio
async def test_baggage_disabled_via_config(capfire: CaptureLogfire):
"""`include_baggage=False` keeps baggage out of emitted events."""
from opentelemetry import baggage as ot_baggage, context as ot_context
tests/evals/test_online_capability.py
-20
View File
@@ -80,7 +80,6 @@ if TYPE_CHECKING or imports_successful():
self.span_refs.append(payload.span_reference)
@pytest.mark.anyio
async def test_basic_dispatch():
"""OnlineEvaluation dispatches evaluators after agent.run()."""
collector = Collector()
@@ -103,7 +102,6 @@ async def test_basic_dispatch():
assert ctx.output == 'success (no tool calls)'
@pytest.mark.anyio
async def test_evaluator_context_fields():
"""EvaluatorContext is populated with correct agent run data."""
collector = Collector()
@@ -133,7 +131,6 @@ async def test_evaluator_context_fields():
@needs_logfire
@pytest.mark.anyio
async def test_usage_metrics(capfire: CaptureLogfire):
"""Span tree metrics are extracted into EvaluatorContext when instrumented."""
collector = Collector()
@@ -155,7 +152,6 @@ async def test_usage_metrics(capfire: CaptureLogfire):
assert ctx.metrics.get('requests', 0) > 0
@pytest.mark.anyio
async def test_sampling_zero_rate():
"""Evaluators with sample_rate=0.0 are never dispatched."""
collector = Collector()
@@ -177,7 +173,6 @@ async def test_sampling_zero_rate():
assert len(collector.calls) == 0
@pytest.mark.anyio
async def test_sampling_context_inputs_match_evaluator_inputs():
"""Sampling and evaluation see the same agent inputs value."""
collector = Collector()
@@ -215,7 +210,6 @@ async def test_sampling_context_inputs_match_evaluator_inputs():
assert ctx.inputs == 'hello'
@pytest.mark.anyio
async def test_disable_evaluation():
"""disable_evaluation() context manager prevents dispatch."""
collector = Collector()
@@ -233,7 +227,6 @@ async def test_disable_evaluation():
assert len(collector.calls) == 0
@pytest.mark.anyio
async def test_config_disabled():
"""Config with enabled=False prevents dispatch."""
collector = Collector()
@@ -251,7 +244,6 @@ async def test_config_disabled():
assert len(collector.calls) == 0
@pytest.mark.anyio
async def test_multiple_evaluators():
"""Multiple evaluators all dispatch concurrently."""
collector = Collector()
@@ -283,7 +275,6 @@ if TYPE_CHECKING or imports_successful():
raise RuntimeError('model exploded')
@pytest.mark.anyio
async def test_run_on_errors_default_skips_dispatch_on_exception():
"""By default, sampled evaluators are not dispatched when the agent run raises."""
collector = Collector()
@@ -301,7 +292,6 @@ async def test_run_on_errors_default_skips_dispatch_on_exception():
assert collector.calls == []
@pytest.mark.anyio
async def test_run_on_errors_dispatches_with_exception_as_output():
"""`run_on_errors=True` dispatches the evaluator with the raised exception as `output`."""
collector = Collector()
@@ -329,7 +319,6 @@ async def test_run_on_errors_dispatches_with_exception_as_output():
assert str(ctx.output) == 'model exploded'
@pytest.mark.anyio
async def test_run_on_errors_only_dispatches_opted_in_evaluators():
"""When some evaluators opt in and some don't, only the opted-in ones run on error."""
collector = Collector()
@@ -357,7 +346,6 @@ async def test_run_on_errors_only_dispatches_opted_in_evaluators():
assert len(results) == 1
@pytest.mark.anyio
async def test_failing_evaluator_does_not_crash_agent():
"""Evaluator exceptions don't crash the agent run."""
collector = Collector()
@@ -380,7 +368,6 @@ async def test_failing_evaluator_does_not_crash_agent():
assert len(failures) == 1
@pytest.mark.anyio
async def test_metadata_merging():
"""Config metadata and run metadata are merged."""
collector = Collector()
@@ -401,7 +388,6 @@ async def test_metadata_merging():
assert ctx.metadata['run_key'] == 'run_val'
@pytest.mark.anyio
async def test_empty_config_metadata_is_preserved():
"""An empty config metadata dict stays an empty dict instead of becoming None."""
collector = Collector()
@@ -420,7 +406,6 @@ async def test_empty_config_metadata_is_preserved():
assert ctx.metadata == {}
@pytest.mark.anyio
async def test_name_defaults_to_run_id():
"""EvaluatorContext name defaults to run_id when no name is provided."""
collector = Collector()
@@ -439,7 +424,6 @@ async def test_name_defaults_to_run_id():
assert ctx.name == result.run_id
@pytest.mark.anyio
async def test_default_config_fallback():
"""OnlineEvaluation uses DEFAULT_CONFIG when no config is provided."""
collector = Collector()
@@ -464,7 +448,6 @@ def test_serialization_name_is_none():
assert OnlineEvaluation.get_serialization_name() is None
@pytest.mark.anyio
async def test_streaming_dispatches_after_context_exit():
"""Streaming runs dispatch evaluations only after the stream context exits."""
collector = Collector()
@@ -488,7 +471,6 @@ async def test_streaming_dispatches_after_context_exit():
assert ctx.output == 'success (no tool calls)'
@pytest.mark.anyio
async def test_iter_dispatches_after_context_exit():
"""Iter-based runs dispatch evaluations only after the context exits."""
collector = Collector()
@@ -516,7 +498,6 @@ async def test_iter_dispatches_after_context_exit():
@needs_logfire
@pytest.mark.anyio
async def test_span_reference_with_logfire(capfire: CaptureLogfire):
"""OnlineEvaluation produces a valid SpanReference when logfire is configured."""
collector = SpanCollector()
@@ -549,7 +530,6 @@ async def test_span_reference_with_logfire(capfire: CaptureLogfire):
f'00-1234567890abcdef1234567890abcdef-{"0" * 16}-01',
),
)
@pytest.mark.anyio
async def test_malformed_traceparent_yields_no_span_reference(monkeypatch: pytest.MonkeyPatch, traceparent: str):
"""Malformed traceparents do not produce span references."""
monkeypatch.setattr(
tests/evals/test_otel.py
-1
View File
@@ -34,7 +34,6 @@ with try_import() as logfire_import_successful:
pytestmark = [
pytest.mark.skipif(not imports_successful(), reason='pydantic-evals not installed'),
pytest.mark.skipif(not logfire_import_successful(), reason='logfire not installed'),
pytest.mark.anyio,
]
tests/evals/test_render_numbers.py
+1 -1
View File
@@ -13,7 +13,7 @@ with try_import() as imports_successful:
default_render_number_diff,
)
pytestmark = [pytest.mark.skipif(not imports_successful(), reason='pydantic-evals not installed'), pytest.mark.anyio]
pytestmark = [pytest.mark.skipif(not imports_successful(), reason='pydantic-evals not installed')]
@pytest.mark.parametrize(
tests/evals/test_report_evaluators.py
-1
View File
@@ -39,7 +39,6 @@ with try_import() as logfire_import_successful:
pytestmark = [
pytest.mark.skipif(not imports_successful(), reason='pydantic-evals not installed'),
pytest.mark.anyio,
]
needs_logfire = pytest.mark.skipif(not logfire_import_successful(), reason='logfire not installed')
tests/evals/test_reporting.py
+1 -1
View File
@@ -21,7 +21,7 @@ with try_import() as imports_successful:
ReportCaseAggregate,
)
pytestmark = [pytest.mark.skipif(not imports_successful(), reason='pydantic-evals not installed'), pytest.mark.anyio]
pytestmark = [pytest.mark.skipif(not imports_successful(), reason='pydantic-evals not installed')]
class TaskInput(BaseModel):
tests/evals/test_reports.py
+1 -1
View File
@@ -20,7 +20,7 @@ with try_import() as imports_successful:
ReportCaseAggregate,
)
pytestmark = [pytest.mark.skipif(not imports_successful(), reason='pydantic-evals not installed'), pytest.mark.anyio]
pytestmark = [pytest.mark.skipif(not imports_successful(), reason='pydantic-evals not installed')]
class TaskInput(BaseModel):
tests/evals/test_typesafe_judges.py
-1
View File
@@ -12,7 +12,6 @@ with try_import() as imports_successful:
pytestmark = [
pytest.mark.skipif(not imports_successful(), reason='pydantic-evals or typesafe-sdk not installed'),
pytest.mark.anyio,
pytest.mark.vcr,
]
tests/evals/test_utils.py
+1 -1
View File
@@ -29,7 +29,7 @@ with try_import() as imports_successful:
task_group_gather,
)
pytestmark = [pytest.mark.skipif(not imports_successful(), reason='pydantic-evals not installed'), pytest.mark.anyio]
pytestmark = [pytest.mark.skipif(not imports_successful(), reason='pydantic-evals not installed')]
def test_run_until_complete_cleans_up_own_task_on_interrupt():

Some files were not shown because too many files have changed in this diff Show More