fix(sdk): compare provider in model_matches_spec (#3943)

`model_matches_spec` is the SDK primitive callers use to decide whether
a model instance already satisfies a string spec — most notably to gate
whether a runtime model swap is needed. Provider-prefixed specs
(`provider:model`) were matched on the model-name portion alone, so two
providers that expose the *same* model identifier were treated as equal.

This is a latent correctness bug: `foo:model` and `bar:model` resolve to
different backends but share the identifier `model`. A caller asking
"does the current model match `bar:model`?" while running on `foo:model`
would get `True` and skip the swap, silently staying on the wrong
backend.

Provider-prefixed specs now also compare the provider reported by the
model via `_get_ls_params`. Hyphen/underscore spellings are normalized.
When the provider cannot be inspected — custom models that do not
implement `_get_ls_params` — the check falls back to identifier-only
matching, so existing behavior for those models is preserved.

Bare specs (no `provider:` prefix) are unchanged: they still match on
identifier alone.
This commit is contained in:
Mason Daugherty
2026-06-13 01:18:05 -04:00
committed by GitHub
parent 7a451b60c9
commit 34244b6aeb
2 changed files with 124 additions and 6 deletions
+61 -6
View File
@@ -3,6 +3,7 @@
from __future__ import annotations
import logging
from collections.abc import Mapping
from langchain.chat_models import init_chat_model
from langchain_core.language_models import BaseChatModel
@@ -11,6 +12,13 @@ from deepagents.profiles.provider.provider_profiles import apply_provider_profil
logger = logging.getLogger(__name__)
# LangChain specs and LangSmith params use different provider names for some
# integrations. Canonicalize only known aliases before comparing providers.
_PROVIDER_ALIASES = {
"azure_openai": "azure",
"mistralai": "mistral",
}
def resolve_model(model: str | BaseChatModel) -> BaseChatModel:
"""Resolve a model string to a `BaseChatModel`.
@@ -78,6 +86,20 @@ def get_model_provider(model: BaseChatModel) -> str | None:
exc,
)
return None
if not isinstance(ls_params, Mapping):
# A custom integration may return `None` (or another non-mapping)
# instead of raising. Treat that as "provider unavailable" rather than
# letting the subsequent `.get` raise `AttributeError`. Logged at INFO
# for the same reason as the `except` branch above: the user-visible
# outcome is identical (provider silently unavailable), so this path
# must be just as discoverable at default log levels.
logger.info(
"Could not extract provider from %s.%s: _get_ls_params returned %s, not a mapping",
type(model).__module__,
type(model).__name__,
type(ls_params).__name__,
)
return None
provider = ls_params.get("ls_provider")
if isinstance(provider, str) and provider:
return provider
@@ -87,10 +109,12 @@ def get_model_provider(model: BaseChatModel) -> str | None:
def model_matches_spec(model: BaseChatModel, spec: str) -> bool:
"""Check whether a model instance already matches a string model spec.
Matching is performed in two ways: first by exact string equality between
`spec` and the model identifier, then by comparing only the model-name
portion of a `provider:model` spec against the identifier. For example,
`"openai:gpt-5"` matches a model with identifier `"gpt-5"`.
Bare specs match by model identifier. Provider-prefixed specs match by both
model identifier and provider when the current model exposes a provider via
`_get_ls_params`; if the provider cannot be inspected, the check falls back
to identifier-only matching for backwards compatibility with custom models.
Provider comparison is normalized, so case, hyphen/underscore spelling, and
known aliases do not read as a mismatch (see `_normalize_provider`).
Assumes the `provider:model` convention (single colon separator).
@@ -107,8 +131,39 @@ def model_matches_spec(model: BaseChatModel, spec: str) -> bool:
if spec == current:
return True
_, separator, model_name = spec.partition(":")
return bool(separator) and model_name == current
provider, separator, model_name = spec.partition(":")
if not separator or model_name != current:
return False
current_provider = get_model_provider(model)
if current_provider is None:
# Provider could not be inspected, so the spec's provider cannot be
# confirmed. Fall back to the identifier-only match. Logged at DEBUG so
# that a consumer skipping a model swap on the strength of this match
# (e.g. the runtime model override) is traceable when it surprises.
logger.debug(
"Matched spec %r on identifier alone; provider for %s.%s is uninspectable, so the spec's %r provider was not verified",
spec,
type(model).__module__,
type(model).__name__,
provider,
)
return True
return _normalize_provider(provider) == _normalize_provider(current_provider)
def _normalize_provider(provider: str) -> str:
"""Canonicalize a provider name so equal providers compare equal.
Specs use the `provider:model` spelling (lowercase, underscore-separated,
e.g. `azure_openai`), while the `ls_provider` reported by `_get_ls_params`
may differ in case, use hyphens (`openai-codex`), or use an entirely
different name (`mistralai` vs `mistral`). Folding both sides through this
function before comparison keeps those spellings from reading as a
mismatch.
"""
normalized = provider.lower().replace("-", "_")
return _PROVIDER_ALIASES.get(normalized, normalized)
def _string_attr(obj: object, attr: str) -> str | None:
@@ -218,6 +218,13 @@ class TestGetModelProvider:
model._get_ls_params = MagicMock(side_effect=TypeError("unexpected"))
assert get_model_provider(model) is None
def test_returns_none_when_get_ls_params_returns_non_mapping(self) -> None:
# A custom integration may return `None` instead of a mapping; this
# must not raise `AttributeError` on the subsequent `.get`.
model = _make_model({})
model._get_ls_params = MagicMock(return_value=None)
assert get_model_provider(model) is None
class TestModelMatchesSpec:
"""Tests for `model_matches_spec`."""
@@ -227,9 +234,65 @@ class TestModelMatchesSpec:
assert model_matches_spec(model, "claude-sonnet-4-6") is True
def test_provider_prefixed_match(self) -> None:
# Set `ls_provider` explicitly so this exercises the provider-match
# path rather than the identifier-only fallback (an unset mock returns
# a non-mapping, which would route through the fallback instead).
model = _make_model({"model_name": "claude-sonnet-4-6"})
model._get_ls_params = MagicMock(return_value={"ls_provider": "anthropic"})
assert model_matches_spec(model, "anthropic:claude-sonnet-4-6") is True
def test_provider_prefixed_match_checks_provider_when_available(self) -> None:
model = _make_model({"model_name": "gpt-5.5"})
model._get_ls_params = MagicMock(return_value={"ls_provider": "openai"})
assert model_matches_spec(model, "openai:gpt-5.5") is True
assert model_matches_spec(model, "openai_codex:gpt-5.5") is False
def test_provider_match_normalizes_langsmith_provider_spelling(self) -> None:
model = _make_model({"model_name": "gpt-5.5"})
model._get_ls_params = MagicMock(return_value={"ls_provider": "openai-codex"})
assert model_matches_spec(model, "openai_codex:gpt-5.5") is True
def test_provider_match_normalizes_spec_provider_spelling(self) -> None:
# The reverse of the case above: a hyphenated spec must match an
# underscored `ls_provider`. Normalization is applied to both operands,
# so neither spelling direction should read as a mismatch.
model = _make_model({"model_name": "gpt-5.5"})
model._get_ls_params = MagicMock(return_value={"ls_provider": "openai_codex"})
assert model_matches_spec(model, "openai-codex:gpt-5.5") is True
@pytest.mark.parametrize(
("spec_provider", "ls_provider"),
[
("azure_openai", "azure"),
("mistralai", "mistral"),
("nvidia", "NVIDIA"),
],
)
def test_provider_match_normalizes_langchain_provider_aliases(self, spec_provider: str, ls_provider: str) -> None:
model = _make_model({"model_name": "provider-model"})
model._get_ls_params = MagicMock(return_value={"ls_provider": ls_provider})
assert model_matches_spec(model, f"{spec_provider}:provider-model") is True
def test_provider_prefixed_match_falls_back_when_provider_unknown(self) -> None:
model = _make_model({"model_name": "claude-sonnet-4-6"})
model._get_ls_params = MagicMock(return_value={})
assert model_matches_spec(model, "anthropic:claude-sonnet-4-6") is True
def test_provider_prefixed_match_falls_back_when_ls_params_non_mapping(
self,
) -> None:
# `_get_ls_params` returning `None` must fall back to identifier-only
# matching rather than raising `AttributeError` out of the match.
model = _make_model({"model_name": "gpt-5.5"})
model._get_ls_params = MagicMock(return_value=None)
assert model_matches_spec(model, "openai:gpt-5.5") is True
def test_no_match(self) -> None:
model = _make_model({"model_name": "claude-sonnet-4-6"})
assert model_matches_spec(model, "openai:gpt-5") is False