Cache prompts by default on every clai2 model with the unified cache setting (#10041)

2026-10-12 02:21:31 +00:00 · 2026-10-09 13:10:02 -06:00
parent 7db7a4d8c4
commit c0f6aad37b
11 changed files with 123 additions and 90 deletions
@@ -522,7 +522,7 @@ takes its place, and a run's own `workspace=` replaces it for that run. Commands
 this process's environment minus LLM provider API keys. Model names resolve as in
 `clai2`, the agent's own and any a run passes, so `openai-codex:` and `github-copilot:`
 use the sign-ins saved with `/login`. CLAI's per-model defaults apply too, such as
-Anthropic prompt caching. Without `model`, every run must pass one.
+prompt caching. Without `model`, every run must pass one.

 Nothing else you saved for `clai2` applies: no saved, drop-in, or project plugins, no
 `.clai/settings.json`, no `/model settings`, and no `chain:` fallback chains. `ask_user`
@@ -991,10 +991,12 @@ from the model's next request, even in a running turn. `r` resets a field; Esc
 or Ctrl-C goes back. Fixed choices
 open a picker; numeric fields accept typed values, and empty input resets.

-CLAI2 enables Anthropic conversation, static-instruction, and tool-schema caching by default:
-`anthropic:` and `gateway/anthropic:` use a 5-minute TTL, while `claude-code:` uses 1 hour.
-These are CLI defaults only; plain Pydantic AI agents are unchanged. Saved cache settings override
-the defaults. Automatic caching advances to the last cacheable block, including tool results.
+CLAI2 enables prompt caching by default for every model, with the unified `Prompt Caching`
+setting (`cache`). It covers Claude on the Anthropic API, `claude-code:`, Amazon Bedrock and
+OpenRouter, Nova on Bedrock, OpenRouter's Gemini routes, and OpenAI's GPT-5.6 and later; providers
+that cache implicitly or not at all ignore it. Set it to `false` to turn caching off for a model, or
+to `1h` to keep the cache longer. Saved `anthropic_cache*` settings still apply and take precedence
+over it. These are CLI defaults only; plain Pydantic AI agents are unchanged.

 `/effort` shows the active model's configured reasoning effort and supported values.
 `/effort high` (or another listed value) saves it for that model; `/effort reset`
@@ -4,7 +4,7 @@ import re

 from pydantic_ai.profiles.anthropic import anthropic_model_profile
 from pydantic_ai.profiles.openai import openai_model_profile
-from pydantic_clai2.models.model_settings import ModelSettingsForm, model_defaults
+from pydantic_clai2.models.model_settings import ModelSettingsForm, gpt_defaults
 from pydantic_clai2.models.profiles import base_model, provider_of


@@ -12,7 +12,7 @@ def model_options(*, model: str) -> dict[str, tuple[str, ...]]:
    """Offer native controls only for the provider that will consume them, whichever profile runs it."""
    provider, _, name = base_model(model).partition(':')
    name = name.rsplit('/', 1)[-1]
-    family_defaults = bool(model_defaults(model=model))
+    family_defaults = bool(gpt_defaults(model=model))
    options: dict[str, tuple[str, ...]] = {key: () for key in ('max_tokens', 'temperature', 'seed', 'custom_params')}
    if provider in ('openai', 'openai-chat', 'openai-responses', 'openai-codex'):
        options = _openai_options(provider=provider, name=name, family_defaults=family_defaults)
@@ -28,6 +28,7 @@ def model_options(*, model: str) -> dict[str, tuple[str, ...]]:
        options['thinking'] = ()
        if provider in ('openrouter', 'vllm'):
            options = _openai_options(provider='openai-chat', name=name, family_defaults=family_defaults)
+    options['cache'] = ()
    glm = _glm_version(name=name)
    if glm is not None and glm >= (4, 5):
        options['glm_thinking'] = ()
@@ -14,7 +14,7 @@ from pydantic import BaseModel, ConfigDict, Field, JsonValue, field_validator

 from pydantic_ai.models.anthropic import AnthropicModelSettings
 from pydantic_ai.models.openai import OpenAIResponsesModelSettings
-from pydantic_ai.settings import ModelSettings
+from pydantic_ai.settings import CacheRetention, ModelSettings
 from pydantic_clai2.models.custom_params import expand_params


@@ -45,6 +45,10 @@ class ModelSettingsForm(BaseModel):
    service_tier: Literal['auto', 'default', 'flex', 'priority'] | None = Field(
        default=None, description='Provider service tier (OpenAI).'
    )
+    cache: bool | CacheRetention | None = Field(
+        default=None,
+        description='Prompt caching: off, on, or a retention (where the provider supports configuring it).',
+    )

    openai_reasoning_effort: Literal['none', 'minimal', 'low', 'medium', 'high', 'xhigh', 'max'] | None = Field(
        default=None, description='OpenAI reasoning effort. Overrides generic thinking.'
@@ -148,6 +152,8 @@ class ModelSettingsForm(BaseModel):
            settings['thinking'] = self.thinking
        if self.service_tier is not None:
            settings['service_tier'] = self.service_tier
+        if self.cache is not None:
+            settings['cache'] = self.cache
        settings.update(self._openai_settings())
        settings.update(self._anthropic_settings())
        body = self._glm_body()
@@ -247,15 +253,17 @@ class ModelSettingsForm(BaseModel):


 def model_defaults(*, model: str) -> dict[str, JsonValue]:
-    """CLAI caching defaults for Anthropic and reasoning defaults for GPT families."""
-    provider = model.partition(':')[0]
-    if provider in ('anthropic', 'gateway/anthropic', 'claude-code'):
-        ttl = '1h' if provider == 'claude-code' else '5m'
-        return {
-            'anthropic_cache': ttl,
-            'anthropic_cache_instructions': ttl,
-            'anthropic_cache_tool_definitions': ttl,
-        }
+    """CLAI's prompt caching default for every model, and reasoning defaults for GPT families.
+
+    Caching uses the unified `cache` setting, which providers without configurable caching ignore. A
+    saved `anthropic_cache*` override still applies, since provider-specific cache settings take
+    precedence over it.
+    """
+    return {'cache': True, **gpt_defaults(model=model)}
+
+
+def gpt_defaults(*, model: str) -> dict[str, JsonValue]:
+    """CLAI's reasoning defaults for GPT-6 and GPT-5.6 families, whichever provider serves them."""
    name = model.partition(':')[2] if ':' in model else model
    name = name.rsplit('/', 1)[-1]
    if not re.match(r'^gpt-(?:6(?:\.\d+)?|5\.6)(?:$|[-:])', name):
@@ -281,9 +289,11 @@ def default_model_settings(*, model: str, saved: Mapping[str, JsonValue]) -> Mod
    return ModelSettingsForm.model_validate(defaults).to_model_settings()


-def model_settings_from_json(values: dict[str, JsonValue], *, model: str = '') -> ModelSettingsForm:
+def model_settings_from_json(values: dict[str, JsonValue], *, model: str | None = None) -> ModelSettingsForm:
    """Read shared preferences, ignoring keys from newer versions without changing the store.

-    Known fields still validate normally. New edits use the strict form directly.
+    Known fields still validate normally. New edits use the strict form directly. With `model`, its
+    family defaults fill the fields `values` leaves unset; without it, only `values` are read.
    """
-    return ModelSettingsForm.model_validate({**model_defaults(model=model), **values}, extra='ignore')
+    defaults = model_defaults(model=model) if model is not None else {}
+    return ModelSettingsForm.model_validate({**defaults, **values}, extra='ignore')
@@ -136,6 +136,7 @@ def _setting_label(key: str) -> str:
    labels = {
        'max_tokens': 'Max Output Tokens',
        'top_p': 'Top-P (Nucleus Sampling)',
+        'cache': 'Prompt Caching',
        'openai_text_verbosity': 'Verbosity',
        'anthropic_thinking_mode': 'Extended Thinking',
        'anthropic_thinking_budget': 'Thinking Budget',
@@ -410,7 +411,8 @@ def build_model_settings_picker(*, context: CommandContext, current: str | None)
 def model_settings_summary(*, store: SettingsStore, model: str, settings_as: str | None = None) -> str:
    """Preview effective settings without mutating the model or its saved overrides."""
    values = {**model_defaults(model=settings_as or model), **store.model_settings(model)}
-    lines = [model, '', 'Configured settings:' if values else 'No custom settings (model defaults).']
+    # Never empty: every model has at least CLAI's prompt caching default.
+    lines = [model, '', 'Configured settings:']
    lines.extend(f'{_setting_label(key)}: {shown(value)}' for key, value in values.items())
    return '\n'.join(lines)

@@ -138,8 +138,8 @@ async def test_effort_for_supplied_model_instance(
    # A bare supplied name has the same limitations as /model settings, not command-only inference.
    assert shell.context.settings_model('gpt-5') == 'gpt-5'
    assert 'openai_reasoning_effort' not in {row.key for row in ModelSettingsSource(store, 'gpt-5').rows()}
-    assert shell.context.model_settings('gpt-5') is None
-    assert shell.context.model_defaults('gpt-5')('gpt-5') is None
+    assert shell.context.model_settings('gpt-5') == {'cache': True}
+    assert shell.context.model_defaults('gpt-5')('gpt-5') == {'cache': True}
    assert await shell.commands.execute_async('/effort') == snapshot(
        'No reasoning effort control for gpt-5. Use /model settings for available controls.'
    )
@@ -233,7 +233,7 @@ async def test_model_settings_saved_mid_turn_reach_the_running_models_next_reque
            await streamed.wait()
            pipe.send_text('/exit\r')
            await done.wait()
-    assert seen == [{'seed': 1}, {'seed': 1, 'max_tokens': 42}]
+    assert seen == [{'cache': True, 'seed': 1}, {'cache': True, 'seed': 1, 'max_tokens': 42}]
    assert 'Saved max_tokens for test.' in output.getvalue()


@@ -3,6 +3,7 @@
 from pathlib import Path

 import pytest
+from pydantic import JsonValue
 from termflow.tui import MenuItem
 from termflow.tui.menu import MenuResult

@@ -27,6 +28,7 @@ def test_defaults_apply_without_saved_preferences(tmp_path: Path, provider: str,
    context, _ = make_context(tmp_path)
    model = f'{provider}:{name}'
    assert context.model_settings(model) == {
+        'cache': True,
        'thinking': True,
        'service_tier': 'default',
        'openai_reasoning_effort': 'medium',
@@ -39,35 +41,68 @@ def test_defaults_apply_without_saved_preferences(tmp_path: Path, provider: str,


@pytest.mark.parametrize('model', ['openai:gpt-5.5', 'openai:gpt-5.60', 'gpt-60', 'gpt-5.6ish', 'test', ''])
-def test_other_models_unchanged(model: str) -> None:
-    assert model_defaults(model=model) == {}
+def test_other_models_only_cache(model: str) -> None:
+    assert model_defaults(model=model) == {'cache': True}


-@pytest.mark.parametrize(('provider', 'ttl'), [('anthropic', '5m'), ('gateway/anthropic', '5m'), ('claude-code', '1h')])
-def test_anthropic_cache_defaults(tmp_path: Path, provider: str, ttl: str) -> None:
+@pytest.mark.parametrize(
+    'model',
+    [
+        'anthropic:claude-sonnet-4-6',
+        'claude-code:claude-sonnet-4-6',
+        'bedrock:us.anthropic.claude-sonnet-4-6',
+        'openrouter:anthropic/claude-sonnet-4.6',
+        'openai:gpt-5.6',
+    ],
+)
+def test_unified_cache_default_can_be_overridden(tmp_path: Path, model: str) -> None:
+    """Every model caches through the unified setting, which the user can change per model."""
    context, _ = make_context(tmp_path)
-    model = f'{provider}:claude-sonnet-4-6'
-    expected = {
-        'anthropic_cache': ttl,
-        'anthropic_cache_instructions': ttl,
-        'anthropic_cache_tool_definitions': ttl,
-    }
-    assert context.model_settings(model) == expected
-    assert context.store.model_settings(model) == {}
+    assert (context.model_settings(model) or {}).get('cache') is True
    source = ModelSettingsSource(context.store, model)
    menu = FieldMenu(source, searchable=False)
-    for key in expected:
+    row = menu.row_for('cache')
+    assert row is not None
+    assert source.current(row) == row.default == 'true'
+    assert source.apply(row, 'false').startswith('Saved')
+    assert context.store.model_settings(model) == {'cache': False}
+    assert (context.model_settings(model) or {}).get('cache') is False
+    assert source.apply(row, '1h').startswith('Saved')
+    assert (context.model_settings(model) or {}).get('cache') == '1h'
+    source.reset(row)
+    assert (context.model_settings(model) or {}).get('cache') is True
+
+
+@pytest.mark.parametrize('provider', ['anthropic', 'gateway/anthropic', 'claude-code'])
+def test_saved_anthropic_cache_settings_still_apply(tmp_path: Path, provider: str) -> None:
+    """Native Anthropic cache settings are no longer defaults, but saved ones still reach the run."""
+    context, _ = make_context(tmp_path)
+    model = f'{provider}:claude-sonnet-4-6'
+    assert context.model_settings(model) == {'cache': True}
+    native: dict[str, JsonValue] = {
+        'anthropic_cache': False,
+        'anthropic_cache_instructions': '1h',
+        'anthropic_cache_tool_definitions': '1h',
+    }
+    source = ModelSettingsSource(context.store, model)
+    menu = FieldMenu(source, searchable=False)
+    for key in native:
        row = menu.row_for(key)
        assert row is not None
-        assert source.current(row) == ttl
-    context.store.save_model_settings(model, {key: False for key in expected})
-    assert context.model_settings(model) == {key: False for key in expected}
+        assert source.current(row) == '(not set)'
+    context.store.save_model_settings(model, native)
+    assert context.model_settings(model) == {'cache': True, **native}


-@pytest.mark.parametrize('provider', ['anthropic', 'claude-code'])
+@pytest.mark.parametrize(
+    'saved',
+    [{}, {'anthropic_cache': False, 'anthropic_cache_instructions': '1h'}],
+    ids=['unified', 'saved-native'],
+)
 async def test_cache_defaults_reach_anthropic_request(
-    tmp_path: Path, provider: str, allow_model_requests: None
+    tmp_path: Path, saved: dict[str, JsonValue], allow_model_requests: None
 ) -> None:
+    from anthropic import omit as OMIT
    from anthropic.types.beta import BetaTextBlock, BetaUsage

    from pydantic_ai import Agent, ModelRequest, ModelResponse, ToolCallPart, ToolReturnPart, UserPromptPart
@@ -76,6 +111,8 @@ async def test_cache_defaults_reach_anthropic_request(
    from tests.models.test_anthropic import MockAnthropic, completion_message, get_mock_chat_completion_kwargs

    context, _ = make_context(tmp_path)
+    name = 'anthropic:claude-sonnet-4-6'
+    context.store.save_model_settings(name, saved)
    client = MockAnthropic.create_mock(
        completion_message([BetaTextBlock(type='text', text='Done')], BetaUsage(input_tokens=1, output_tokens=1))
    )
@@ -89,7 +126,7 @@ async def test_cache_defaults_reach_anthropic_request(

    await agent.run(
        'Continue',
-        model_settings=context.model_settings(f'{provider}:claude-sonnet-4-6'),
+        model_settings=context.model_settings(name),
        message_history=[
            ModelRequest(parts=[UserPromptPart('Start')]),
            ModelResponse(parts=[ToolCallPart('lookup', {}, tool_call_id='call')]),
@@ -97,10 +134,12 @@ async def test_cache_defaults_reach_anthropic_request(
        ],
    )
    request = get_mock_chat_completion_kwargs(client)[0]
-    expected = {'type': 'ephemeral', 'ttl': '1h' if provider == 'claude-code' else '5m'}
-    assert request['cache_control'] == expected
-    assert request['system'][-1]['cache_control'] == expected
-    assert request['tools'][-1]['cache_control'] == expected
+    if saved:
+        # A saved native setting takes precedence over the unified default.
+        assert request['cache_control'] is OMIT
+        assert request['system'][-1]['cache_control'] == {'type': 'ephemeral', 'ttl': '1h'}
+    else:
+        assert request['cache_control']['type'] == 'ephemeral'
    assert model.resolve_cache_retention(None) is None  # Library defaults remain opt-in.


@@ -193,7 +232,7 @@ def test_unknown_and_nonreasoning_fallbacks(model: str) -> None:

 def test_model_preview(tmp_path: Path) -> None:
    context, _ = make_context(tmp_path)
-    assert 'No custom settings' in model_settings_summary(store=context.store, model='test')
+    assert 'Prompt Caching: true' in model_settings_summary(store=context.store, model='test')
    context.store.save_model_settings('openai:gpt-6', {'openai_reasoning_effort': 'high'})
    text = model_settings_summary(store=context.store, model='openai:gpt-6')
    assert 'Reasoning Effort: high' in text and 'Verbosity: low' in text
@@ -225,4 +264,4 @@ def test_chat_compatible_reasoning_defaults_can_be_overridden(tmp_path: Path, pr

@pytest.mark.parametrize('model', ['custom:gpt-6', 'custom:o3'])
 def test_unknown_provider_does_not_claim_chat_protocol_support(model: str) -> None:
-    assert set(model_options(model=model)) == {'max_tokens', 'thinking', 'custom_params'}
+    assert set(model_options(model=model)) == {'max_tokens', 'thinking', 'custom_params', 'cache'}
@@ -71,6 +71,7 @@ def test_settings_form_validates_and_converts() -> None:
        'parallel_tool_calls': True,
        'thinking': True,
        'service_tier': 'flex',
+        'cache': '1h',
    }
    assert model_settings_from_json(everything).to_model_settings() == everything
    with pytest.raises(ValidationError):
@@ -197,8 +198,8 @@ async def test_open_add_model_menu_and_settings_reach_the_run(tmp_path: Path) ->
    assert await open_add_model_menu(context, run=lambda menu: []) == 'No changes.'
    assert await open_add_model_menu(context, run=lambda menu: [menu.choose('test')]) == 'Saved model. Applied.'
    context.store.save_model_settings('test', {'max_tokens': 3, 'seed': 7})
-    assert context.model_settings('test') == {'max_tokens': 3, 'seed': 7}
-    assert context.model_settings('other') is None
+    assert context.model_settings('test') == {'max_tokens': 3, 'seed': 7, 'cache': True}
+    assert context.model_settings('other') == {'cache': True}
    seen: list[ModelSettings | None] = []
    hooks = Hooks[None]()

@@ -210,7 +211,7 @@ async def test_open_add_model_menu_and_settings_reach_the_run(tmp_path: Path) ->
    session = Session(Agent(TestModel(custom_output_text='ok')), deps=None, plugins=[hooks])
    session.model_settings = context.model_settings('test')
    assert (await session.prompt('hi')).output == 'ok'
-    assert seen == [{'max_tokens': 3, 'seed': 7}]
+    assert seen == [{'max_tokens': 3, 'seed': 7, 'cache': True}]


 def test_live_model_overrides_follow_saves_and_keep_the_last_valid_ones(tmp_path: Path) -> None:
@@ -32,10 +32,10 @@ def test_reads_keep_known_overrides_and_ignore_future_fields(tmp_path: Path) ->
    context, _ = make_context(tmp_path)
    saved: dict[str, JsonValue] = {'temperature': 0.5, 'future_setting': {'nested': [1, None, False]}}
    context.store.save_model_settings('test', saved)
-    assert context.model_settings('test') == {'temperature': 0.5}
+    assert context.model_settings('test') == {'cache': True, 'temperature': 0.5}
    assert context.store.model_settings('test') == saved
    context.store.save_model_settings('test', {'future_setting': True})
-    assert context.model_settings('test') is None
+    assert context.model_settings('test') == {'cache': True}


 def test_known_values_and_new_edits_still_validate() -> None:
@@ -167,9 +167,6 @@ CASES = [
        },
        effective=snapshot(
            {
-                'anthropic_cache': '5m',
-                'anthropic_cache_instructions': '5m',
-                'anthropic_cache_tool_definitions': '5m',
                'anthropic_effort': 'high',
                'anthropic_thinking': {'type': 'adaptive', 'display': 'updates'},
                'extra_headers': {'anthropic-beta': 'thinking-display-updates-2026-08-18'},
@@ -343,7 +340,7 @@ async def test_outermost_capability_settings_beat_defaults(tmp_path: Path, stock
 async def test_capability_model_beats_default_model(tmp_path: Path) -> None:
    """A capability's model replaces CLAI's default and resolves through CLAI; the GPT defaults do not follow it.

-    The capability's Claude model gets its own family defaults, the prompt-caching TTLs, instead.
+    The capability's Claude model gets only the prompt caching default, which core moves out of the settings.

    `resolved_model()`, which `/compact` and session naming run on, still names the selected model,
    as `PLUGINS.md` documents.
@@ -351,18 +348,7 @@ async def test_capability_model_beats_default_model(tmp_path: Path) -> None:
    recorder, session = await run_turn(
        tmp_path, settings=Settings(), plugins=(Published(model='anthropic:claude-sonnet-4-6'),)
    )
-    assert recorder.calls == snapshot(
-        [
-            (
-                'anthropic:claude-sonnet-4-6',
-                {
-                    'anthropic_cache': '5m',
-                    'anthropic_cache_instructions': '5m',
-                    'anthropic_cache_tool_definitions': '5m',
-                },
-            )
-        ]
-    )
+    assert recorder.calls == snapshot([('anthropic:claude-sonnet-4-6', {})])
    selected = await session.resolved_model()
    assert isinstance(selected, Model) and selected.model_name == 'openai-codex:gpt-6-astra'

@@ -383,9 +369,8 @@ async def test_saved_settings_stay_with_their_model(tmp_path: Path, stock: bool)
        agent=None if stock else Agent(recorder.resolve(name), deps_type=type(None), capabilities=[published]),
        recorder=recorder,
    )
-    # Only the Claude model's own caching defaults; nothing saved for the GPT model.
-    cache = {'anthropic_cache': '5m', 'anthropic_cache_instructions': '5m', 'anthropic_cache_tool_definitions': '5m'}
-    assert recorder.calls == [(other, cache)]
+    # Nothing saved for the GPT model; the caching default is moved out of the settings by core.
+    assert recorder.calls == [(other, {})]


 async def test_agent_capability_settings_beat_family_defaults(tmp_path: Path) -> None:
@@ -565,6 +550,10 @@ async def test_settings_saved_mid_turn_match_the_merged_settings(tmp_path: Path)
        thinking = info.model_request_parameters.thinking
        assert thinking is not None
        settings['thinking'] = thinking
+        # Core moves `cache` out of the settings too, into the request parameters.
+        cache = info.model_request_parameters.cache
+        assert cache is not None
+        settings['cache'] = cache
        recorded.append(settings)
        if edits:
            store.save_model_settings(name, edits.pop(0))
@@ -575,7 +564,10 @@ async def test_settings_saved_mid_turn_match_the_merged_settings(tmp_path: Path)

    recorded: list[ModelSettings] = []
    model = FunctionModel(
-        respond, stream_function=stream, model_name=name, profile=ModelProfile(supports_thinking=True)
+        respond,
+        stream_function=stream,
+        model_name=name,
+        profile=ModelProfile(supports_thinking=True, supports_cache=True),
    )
    tool = Capability[None](tools=[Tool(lambda: 'ok', name='noop', takes_ctx=False)])
    await run_turn(tmp_path, settings=resolve_settings({'model': name}), plugins=(tool,), resolve=lambda _: model)
@@ -177,14 +177,7 @@ async def test_model_names_resolve_through_clai(tmp_path: Path, monkeypatch: pyt
        await agent.run('hello', model='openai:gpt-6')
    assert requests == snapshot(
        [
-            (
-                'anthropic:claude-sonnet-4-6',
-                {
-                    'anthropic_cache': '5m',
-                    'anthropic_cache_instructions': '5m',
-                    'anthropic_cache_tool_definitions': '5m',
-                },
-            ),
+            ('anthropic:claude-sonnet-4-6', {}),
            (
                'openai:gpt-6',
                {
@@ -273,14 +266,7 @@ async def test_approval_gated_host_configuration(tmp_path: Path, monkeypatch: py
        )
    [(name, settings, tools)] = requests
    assert name == 'gateway/anthropic:claude-opus-5-5'
-    assert settings == snapshot(
-        {
-            'anthropic_cache': '5m',
-            'anthropic_cache_instructions': '5m',
-            'anthropic_cache_tool_definitions': '5m',
-            'thinking': 'high',
-        }
-    )
+    assert settings == snapshot({'thinking': 'high'})
    assert 'delegate_task' not in tools
    assert isinstance(result.output, str)
    assert 'HOST_VARIABLE=forwarded' in result.output