mirror of
https://github.com/langchain-ai/deepagents.git
synced 2026-07-22 01:35:28 -04:00
fix(sdk): compare provider in model_matches_spec (#3943)
`model_matches_spec` is the SDK primitive callers use to decide whether a model instance already satisfies a string spec — most notably to gate whether a runtime model swap is needed. Provider-prefixed specs (`provider:model`) were matched on the model-name portion alone, so two providers that expose the *same* model identifier were treated as equal. This is a latent correctness bug: `foo:model` and `bar:model` resolve to different backends but share the identifier `model`. A caller asking "does the current model match `bar:model`?" while running on `foo:model` would get `True` and skip the swap, silently staying on the wrong backend. Provider-prefixed specs now also compare the provider reported by the model via `_get_ls_params`. Hyphen/underscore spellings are normalized. When the provider cannot be inspected — custom models that do not implement `_get_ls_params` — the check falls back to identifier-only matching, so existing behavior for those models is preserved. Bare specs (no `provider:` prefix) are unchanged: they still match on identifier alone.
This commit is contained in:
@@ -3,6 +3,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from collections.abc import Mapping
|
||||
|
||||
from langchain.chat_models import init_chat_model
|
||||
from langchain_core.language_models import BaseChatModel
|
||||
@@ -11,6 +12,13 @@ from deepagents.profiles.provider.provider_profiles import apply_provider_profil
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# LangChain specs and LangSmith params use different provider names for some
|
||||
# integrations. Canonicalize only known aliases before comparing providers.
|
||||
_PROVIDER_ALIASES = {
|
||||
"azure_openai": "azure",
|
||||
"mistralai": "mistral",
|
||||
}
|
||||
|
||||
|
||||
def resolve_model(model: str | BaseChatModel) -> BaseChatModel:
|
||||
"""Resolve a model string to a `BaseChatModel`.
|
||||
@@ -78,6 +86,20 @@ def get_model_provider(model: BaseChatModel) -> str | None:
|
||||
exc,
|
||||
)
|
||||
return None
|
||||
if not isinstance(ls_params, Mapping):
|
||||
# A custom integration may return `None` (or another non-mapping)
|
||||
# instead of raising. Treat that as "provider unavailable" rather than
|
||||
# letting the subsequent `.get` raise `AttributeError`. Logged at INFO
|
||||
# for the same reason as the `except` branch above: the user-visible
|
||||
# outcome is identical (provider silently unavailable), so this path
|
||||
# must be just as discoverable at default log levels.
|
||||
logger.info(
|
||||
"Could not extract provider from %s.%s: _get_ls_params returned %s, not a mapping",
|
||||
type(model).__module__,
|
||||
type(model).__name__,
|
||||
type(ls_params).__name__,
|
||||
)
|
||||
return None
|
||||
provider = ls_params.get("ls_provider")
|
||||
if isinstance(provider, str) and provider:
|
||||
return provider
|
||||
@@ -87,10 +109,12 @@ def get_model_provider(model: BaseChatModel) -> str | None:
|
||||
def model_matches_spec(model: BaseChatModel, spec: str) -> bool:
|
||||
"""Check whether a model instance already matches a string model spec.
|
||||
|
||||
Matching is performed in two ways: first by exact string equality between
|
||||
`spec` and the model identifier, then by comparing only the model-name
|
||||
portion of a `provider:model` spec against the identifier. For example,
|
||||
`"openai:gpt-5"` matches a model with identifier `"gpt-5"`.
|
||||
Bare specs match by model identifier. Provider-prefixed specs match by both
|
||||
model identifier and provider when the current model exposes a provider via
|
||||
`_get_ls_params`; if the provider cannot be inspected, the check falls back
|
||||
to identifier-only matching for backwards compatibility with custom models.
|
||||
Provider comparison is normalized, so case, hyphen/underscore spelling, and
|
||||
known aliases do not read as a mismatch (see `_normalize_provider`).
|
||||
|
||||
Assumes the `provider:model` convention (single colon separator).
|
||||
|
||||
@@ -107,8 +131,39 @@ def model_matches_spec(model: BaseChatModel, spec: str) -> bool:
|
||||
if spec == current:
|
||||
return True
|
||||
|
||||
_, separator, model_name = spec.partition(":")
|
||||
return bool(separator) and model_name == current
|
||||
provider, separator, model_name = spec.partition(":")
|
||||
if not separator or model_name != current:
|
||||
return False
|
||||
|
||||
current_provider = get_model_provider(model)
|
||||
if current_provider is None:
|
||||
# Provider could not be inspected, so the spec's provider cannot be
|
||||
# confirmed. Fall back to the identifier-only match. Logged at DEBUG so
|
||||
# that a consumer skipping a model swap on the strength of this match
|
||||
# (e.g. the runtime model override) is traceable when it surprises.
|
||||
logger.debug(
|
||||
"Matched spec %r on identifier alone; provider for %s.%s is uninspectable, so the spec's %r provider was not verified",
|
||||
spec,
|
||||
type(model).__module__,
|
||||
type(model).__name__,
|
||||
provider,
|
||||
)
|
||||
return True
|
||||
return _normalize_provider(provider) == _normalize_provider(current_provider)
|
||||
|
||||
|
||||
def _normalize_provider(provider: str) -> str:
|
||||
"""Canonicalize a provider name so equal providers compare equal.
|
||||
|
||||
Specs use the `provider:model` spelling (lowercase, underscore-separated,
|
||||
e.g. `azure_openai`), while the `ls_provider` reported by `_get_ls_params`
|
||||
may differ in case, use hyphens (`openai-codex`), or use an entirely
|
||||
different name (`mistralai` vs `mistral`). Folding both sides through this
|
||||
function before comparison keeps those spellings from reading as a
|
||||
mismatch.
|
||||
"""
|
||||
normalized = provider.lower().replace("-", "_")
|
||||
return _PROVIDER_ALIASES.get(normalized, normalized)
|
||||
|
||||
|
||||
def _string_attr(obj: object, attr: str) -> str | None:
|
||||
|
||||
@@ -218,6 +218,13 @@ class TestGetModelProvider:
|
||||
model._get_ls_params = MagicMock(side_effect=TypeError("unexpected"))
|
||||
assert get_model_provider(model) is None
|
||||
|
||||
def test_returns_none_when_get_ls_params_returns_non_mapping(self) -> None:
|
||||
# A custom integration may return `None` instead of a mapping; this
|
||||
# must not raise `AttributeError` on the subsequent `.get`.
|
||||
model = _make_model({})
|
||||
model._get_ls_params = MagicMock(return_value=None)
|
||||
assert get_model_provider(model) is None
|
||||
|
||||
|
||||
class TestModelMatchesSpec:
|
||||
"""Tests for `model_matches_spec`."""
|
||||
@@ -227,9 +234,65 @@ class TestModelMatchesSpec:
|
||||
assert model_matches_spec(model, "claude-sonnet-4-6") is True
|
||||
|
||||
def test_provider_prefixed_match(self) -> None:
|
||||
# Set `ls_provider` explicitly so this exercises the provider-match
|
||||
# path rather than the identifier-only fallback (an unset mock returns
|
||||
# a non-mapping, which would route through the fallback instead).
|
||||
model = _make_model({"model_name": "claude-sonnet-4-6"})
|
||||
model._get_ls_params = MagicMock(return_value={"ls_provider": "anthropic"})
|
||||
assert model_matches_spec(model, "anthropic:claude-sonnet-4-6") is True
|
||||
|
||||
def test_provider_prefixed_match_checks_provider_when_available(self) -> None:
|
||||
model = _make_model({"model_name": "gpt-5.5"})
|
||||
model._get_ls_params = MagicMock(return_value={"ls_provider": "openai"})
|
||||
|
||||
assert model_matches_spec(model, "openai:gpt-5.5") is True
|
||||
assert model_matches_spec(model, "openai_codex:gpt-5.5") is False
|
||||
|
||||
def test_provider_match_normalizes_langsmith_provider_spelling(self) -> None:
|
||||
model = _make_model({"model_name": "gpt-5.5"})
|
||||
model._get_ls_params = MagicMock(return_value={"ls_provider": "openai-codex"})
|
||||
|
||||
assert model_matches_spec(model, "openai_codex:gpt-5.5") is True
|
||||
|
||||
def test_provider_match_normalizes_spec_provider_spelling(self) -> None:
|
||||
# The reverse of the case above: a hyphenated spec must match an
|
||||
# underscored `ls_provider`. Normalization is applied to both operands,
|
||||
# so neither spelling direction should read as a mismatch.
|
||||
model = _make_model({"model_name": "gpt-5.5"})
|
||||
model._get_ls_params = MagicMock(return_value={"ls_provider": "openai_codex"})
|
||||
|
||||
assert model_matches_spec(model, "openai-codex:gpt-5.5") is True
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("spec_provider", "ls_provider"),
|
||||
[
|
||||
("azure_openai", "azure"),
|
||||
("mistralai", "mistral"),
|
||||
("nvidia", "NVIDIA"),
|
||||
],
|
||||
)
|
||||
def test_provider_match_normalizes_langchain_provider_aliases(self, spec_provider: str, ls_provider: str) -> None:
|
||||
model = _make_model({"model_name": "provider-model"})
|
||||
model._get_ls_params = MagicMock(return_value={"ls_provider": ls_provider})
|
||||
|
||||
assert model_matches_spec(model, f"{spec_provider}:provider-model") is True
|
||||
|
||||
def test_provider_prefixed_match_falls_back_when_provider_unknown(self) -> None:
|
||||
model = _make_model({"model_name": "claude-sonnet-4-6"})
|
||||
model._get_ls_params = MagicMock(return_value={})
|
||||
|
||||
assert model_matches_spec(model, "anthropic:claude-sonnet-4-6") is True
|
||||
|
||||
def test_provider_prefixed_match_falls_back_when_ls_params_non_mapping(
|
||||
self,
|
||||
) -> None:
|
||||
# `_get_ls_params` returning `None` must fall back to identifier-only
|
||||
# matching rather than raising `AttributeError` out of the match.
|
||||
model = _make_model({"model_name": "gpt-5.5"})
|
||||
model._get_ls_params = MagicMock(return_value=None)
|
||||
|
||||
assert model_matches_spec(model, "openai:gpt-5.5") is True
|
||||
|
||||
def test_no_match(self) -> None:
|
||||
model = _make_model({"model_name": "claude-sonnet-4-6"})
|
||||
assert model_matches_spec(model, "openai:gpt-5") is False
|
||||
|
||||
Reference in New Issue
Block a user