mirror of
https://github.com/usestrix/strix.git
synced 2026-11-09 13:31:41 -10:00
Compare commits
2
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
e011253ebc | ||
|
|
5227960e99 |
No files matched your search
+1
-33
@@ -5,7 +5,6 @@ from __future__ import annotations
|
||||
import asyncio
|
||||
import contextlib
|
||||
import inspect
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import time
|
||||
@@ -43,7 +42,7 @@ from strix.config import codex
|
||||
from strix.config.loader import load_settings
|
||||
from strix.config.tool_call_ids import TurnCallIdRewriter, dedupe_input
|
||||
from strix.config.tool_call_limits import TurnToolCallLimiter
|
||||
from strix.llm import provider_bans, request_log
|
||||
from strix.llm import request_log
|
||||
|
||||
|
||||
if TYPE_CHECKING:
|
||||
@@ -746,11 +745,6 @@ def _install_openrouter_stream_cost_capture() -> None:
|
||||
|
||||
class _StrixOpenRouterStreamingHandler(OpenRouterChatCompletionStreamingHandler):
|
||||
def chunk_parser(self, chunk: dict[str, Any]) -> Any:
|
||||
# Before parsing: LiteLLM raises on an error chunk and drops its
|
||||
# top-level ``provider``.
|
||||
request_log.record_upstream_provider(
|
||||
chunk.get("provider"), _openrouter_error_type(chunk.get("error"))
|
||||
)
|
||||
stream = super().chunk_parser(chunk)
|
||||
usage = chunk.get("usage")
|
||||
response_id = chunk.get("id") or getattr(stream, "id", None)
|
||||
@@ -769,24 +763,12 @@ def _install_openrouter_stream_cost_capture() -> None:
|
||||
json_mode=json_mode,
|
||||
)
|
||||
|
||||
def get_error_class(self, error_message: str, status_code: int, headers: Any) -> Any:
|
||||
# A non-2xx reply names the provider in ``error.metadata``.
|
||||
with contextlib.suppress(Exception):
|
||||
error = json.loads(error_message)["error"]
|
||||
request_log.record_upstream_provider(
|
||||
error["metadata"].get("provider_name"),
|
||||
_openrouter_error_type(error),
|
||||
rejected=True,
|
||||
)
|
||||
return super().get_error_class(error_message, status_code, headers)
|
||||
|
||||
def transform_response(self, *args: Any, **kwargs: Any) -> Any:
|
||||
# Non-streamed replies (LLM_DISABLE_STREAMING) skip the chunk parser.
|
||||
response = super().transform_response(*args, **kwargs)
|
||||
raw_response = kwargs.get("raw_response", args[1] if len(args) > 1 else None)
|
||||
with contextlib.suppress(Exception):
|
||||
body = raw_response.json() # type: ignore[union-attr]
|
||||
request_log.record_upstream_provider(body.get("provider"))
|
||||
if body.get("usage"):
|
||||
record_openrouter_provider(body.get("provider"), body["usage"])
|
||||
return response
|
||||
@@ -799,12 +781,6 @@ def _install_openrouter_stream_cost_capture() -> None:
|
||||
if agent_id and load_settings().llm.openrouter_sticky_sessions:
|
||||
session_id = _OPENROUTER_SESSION_IDS.setdefault(agent_id, str(uuid.uuid4()))
|
||||
body.setdefault("session_id", session_id)
|
||||
model = kwargs.get("model", args[0] if args else "")
|
||||
if banned := provider_bans.banned_providers(model):
|
||||
# Copy: strix-pro's hook sets ``provider`` to a dict it caches.
|
||||
provider = dict(body.get("provider") or {})
|
||||
provider["ignore"] = sorted({*provider.get("ignore", ()), *banned})
|
||||
body["provider"] = provider
|
||||
return body
|
||||
|
||||
# LiteLLM's provider-config factory reads litellm.OpenrouterConfig at call
|
||||
@@ -813,14 +789,6 @@ def _install_openrouter_stream_cost_capture() -> None:
|
||||
litellm.OpenrouterConfig = _StrixOpenrouterConfig # type: ignore[misc]
|
||||
|
||||
|
||||
def _openrouter_error_type(error: object) -> object:
|
||||
if isinstance(error, dict):
|
||||
metadata = error.get("metadata")
|
||||
if isinstance(metadata, dict):
|
||||
return metadata.get("error_type")
|
||||
return None
|
||||
|
||||
|
||||
OPENROUTER_ATTRIBUTION_HEADERS = {
|
||||
"HTTP-Referer": "https://strix.ai",
|
||||
"X-Title": "Strix",
|
||||
|
||||
@@ -72,8 +72,6 @@ class LlmSettings(BaseSettings):
|
||||
default=False,
|
||||
alias="STRIX_OPENROUTER_STICKY_SESSIONS",
|
||||
)
|
||||
openrouter_provider_bans: bool = Field(default=True, alias="STRIX_OPENROUTER_PROVIDER_BANS")
|
||||
openrouter_ban_after_4xx: int = Field(default=2, ge=1, alias="STRIX_OPENROUTER_BAN_AFTER_4XX")
|
||||
disable_streaming: bool = Field(
|
||||
default=False,
|
||||
alias="LLM_DISABLE_STREAMING",
|
||||
|
||||
+2
-34
@@ -6,7 +6,6 @@ import asyncio
|
||||
import contextlib
|
||||
import logging
|
||||
import uuid
|
||||
from collections import Counter
|
||||
from collections.abc import Callable
|
||||
from functools import cache
|
||||
from typing import TYPE_CHECKING, Any, cast
|
||||
@@ -21,7 +20,6 @@ from openai import (
|
||||
)
|
||||
|
||||
from strix.config import codex
|
||||
from strix.config.loader import load_settings
|
||||
from strix.core.hooks import (
|
||||
BudgetExceededError,
|
||||
BudgetPausedError,
|
||||
@@ -35,7 +33,7 @@ from strix.core.sessions import (
|
||||
seed_initial_input,
|
||||
strip_all_images_from_session,
|
||||
)
|
||||
from strix.llm import provider_bans, request_log
|
||||
from strix.llm import request_log
|
||||
from strix.llm.compaction import is_context_overflow, maybe_compact
|
||||
|
||||
|
||||
@@ -125,7 +123,6 @@ async def _compact_session(
|
||||
_MAX_TRANSIENT_MODEL_RETRIES = 5
|
||||
_TRANSIENT_MODEL_RETRY_BASE_DELAY_S = 2.0
|
||||
_TRANSIENT_MODEL_RETRY_MAX_DELAY_S = 90.0
|
||||
_provider_4xx_counts: Counter[tuple[str, str]] = Counter()
|
||||
|
||||
|
||||
def _model_error_status_code(exc: BaseException) -> int | None:
|
||||
@@ -141,41 +138,13 @@ def _is_transient_model_error(exc: BaseException) -> bool:
|
||||
):
|
||||
return True
|
||||
code = _model_error_status_code(exc)
|
||||
if code in (401, 402, 403):
|
||||
# Auth, credit and policy errors from the gateway itself (e.g. OpenRouter's
|
||||
# "User not found" 401) name no upstream provider; retrying cannot help.
|
||||
reply = request_log.failed_reply(exc)
|
||||
return reply is not None and reply.upstream_provider is not None
|
||||
if code is not None:
|
||||
import litellm
|
||||
|
||||
return 400 <= code < 500 or bool(litellm._should_retry(code))
|
||||
return bool(litellm._should_retry(code))
|
||||
return isinstance(exc, APIError)
|
||||
|
||||
|
||||
async def _maybe_ban_provider(exc: BaseException, run_config: RunConfig) -> None:
|
||||
reply = request_log.failed_reply(exc)
|
||||
if reply is None or reply.upstream_provider is None:
|
||||
return
|
||||
model = _run_config_model(run_config)
|
||||
if model is None:
|
||||
return
|
||||
provider = reply.upstream_provider
|
||||
code = _model_error_status_code(exc)
|
||||
if reply.upstream_error_type == "provider_unavailable":
|
||||
reason = "provider_unavailable"
|
||||
elif reply.upstream_rejected:
|
||||
reason = f"upstream_rejected_{code}"
|
||||
elif code is not None and 400 <= code < 500:
|
||||
_provider_4xx_counts[model, provider] += 1
|
||||
if _provider_4xx_counts[model, provider] < load_settings().llm.openrouter_ban_after_4xx:
|
||||
return
|
||||
reason = "repeated_4xx"
|
||||
else:
|
||||
return
|
||||
await asyncio.to_thread(provider_bans.ban_provider, model, provider, reason)
|
||||
|
||||
|
||||
def _transient_model_retry_delay(attempt: int) -> float:
|
||||
delay = _TRANSIENT_MODEL_RETRY_BASE_DELAY_S * float(2 ** (attempt - 1))
|
||||
return min(delay, _TRANSIENT_MODEL_RETRY_MAX_DELAY_S)
|
||||
@@ -875,7 +844,6 @@ async def _run_cycle( # noqa: PLR0912, PLR0915
|
||||
input_data = []
|
||||
continue
|
||||
if model_retries < _MAX_TRANSIENT_MODEL_RETRIES and _is_transient_model_error(exc):
|
||||
await _maybe_ban_provider(exc, run_config)
|
||||
model_retries += 1
|
||||
delay = _transient_model_retry_delay(model_retries)
|
||||
logger.warning(
|
||||
|
||||
+20
-9
@@ -15,7 +15,11 @@ from typing import TYPE_CHECKING, Any
|
||||
|
||||
from agents.model_settings import ModelSettings
|
||||
from agents.models.interface import ModelTracing
|
||||
from openai.types.responses import ResponseOutputMessage, ResponseOutputText
|
||||
from openai.types.responses import (
|
||||
ResponseOutputMessage,
|
||||
ResponseOutputText,
|
||||
ResponseReasoningItem,
|
||||
)
|
||||
|
||||
from strix.config import load_settings
|
||||
from strix.config.models import StrixProvider
|
||||
@@ -288,15 +292,22 @@ def _checkpoint_item(summary: str) -> dict[str, Any]:
|
||||
|
||||
def _extract_text(response: ModelResponse) -> str:
|
||||
parts: list[str] = []
|
||||
reasoning: list[str] = []
|
||||
for item in response.output:
|
||||
if not isinstance(item, ResponseOutputMessage):
|
||||
continue
|
||||
parts.extend(
|
||||
chunk.text
|
||||
for chunk in item.content
|
||||
if isinstance(chunk, ResponseOutputText) and chunk.text
|
||||
)
|
||||
return "".join(parts)
|
||||
if isinstance(item, ResponseOutputMessage):
|
||||
parts.extend(
|
||||
chunk.text
|
||||
for chunk in item.content
|
||||
if isinstance(chunk, ResponseOutputText) and chunk.text
|
||||
)
|
||||
elif isinstance(item, ResponseReasoningItem):
|
||||
reasoning.extend(summary.text for summary in item.summary if summary.text)
|
||||
if parts or not reasoning:
|
||||
return "".join(parts)
|
||||
# Reasoning models can spend the whole output budget thinking and return no
|
||||
# message; their reasoning is a degraded but usable summary.
|
||||
logger.warning("compaction summary had no output text; using reasoning text")
|
||||
return "\n".join(reasoning)
|
||||
|
||||
|
||||
async def _summarize(model: str, prompt: str, max_tokens: int) -> str | None:
|
||||
|
||||
@@ -1,74 +0,0 @@
|
||||
"""Per-run bans on the OpenRouter upstream providers behind a model."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import threading
|
||||
|
||||
import httpx
|
||||
|
||||
from strix.config.loader import load_settings
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_ENDPOINTS_URL = "https://openrouter.ai/api/v1/models/{model}/endpoints"
|
||||
|
||||
_LOCK = threading.Lock()
|
||||
# OpenRouter reports display names ("InferenceNet"); provider.ignore takes slugs ("inference-net").
|
||||
_SLUGS: dict[str, dict[str, str]] = {}
|
||||
_BANNED: dict[str, set[str]] = {}
|
||||
|
||||
|
||||
def _model_id(model: str) -> str:
|
||||
model = model.strip()
|
||||
for prefix in ("litellm/", "openrouter/"):
|
||||
model = model.removeprefix(prefix)
|
||||
return model
|
||||
|
||||
|
||||
def _provider_slugs(model: str) -> dict[str, str]:
|
||||
if model not in _SLUGS:
|
||||
response = httpx.get(_ENDPOINTS_URL.format(model=model), timeout=5)
|
||||
response.raise_for_status()
|
||||
slugs: dict[str, str] = {}
|
||||
for endpoint in response.json()["data"]["endpoints"]:
|
||||
if not (tag := endpoint.get("tag")):
|
||||
continue
|
||||
# A tag is the provider slug plus an optional variant, e.g. "inference-net/fp4".
|
||||
slug = str(tag).split("/")[0]
|
||||
slugs[slug] = slug
|
||||
slugs[str(endpoint.get("provider_name") or slug).lower()] = slug
|
||||
_SLUGS[model] = slugs
|
||||
return _SLUGS[model]
|
||||
|
||||
|
||||
def ban_provider(model: str, provider: str, reason: str) -> None:
|
||||
"""Ban for the rest of the run; ``provider`` may be a display name or slug."""
|
||||
if not load_settings().llm.openrouter_provider_bans:
|
||||
return
|
||||
model = _model_id(model)
|
||||
try:
|
||||
slugs = _provider_slugs(model)
|
||||
slug = slugs[provider.strip().lower()]
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.warning(
|
||||
"provider_ban_skipped upstream=%s model=%s reason=%s: %r", provider, model, reason, exc
|
||||
)
|
||||
return
|
||||
with _LOCK:
|
||||
banned = _BANNED.setdefault(model, set())
|
||||
if slug in banned:
|
||||
return
|
||||
banned.add(slug)
|
||||
logger.warning(
|
||||
"provider_banned upstream=%s slug=%s model=%s reason=%s", provider, slug, model, reason
|
||||
)
|
||||
if banned >= set(slugs.values()):
|
||||
banned.clear()
|
||||
logger.warning("provider_bans_reset model=%s: every provider was banned", model)
|
||||
|
||||
|
||||
def banned_providers(model: str) -> list[str]:
|
||||
with _LOCK:
|
||||
return sorted(_BANNED.get(_model_id(model), ()))
|
||||
@@ -159,43 +159,9 @@ class HttpReply:
|
||||
|
||||
status_code: int | None = None
|
||||
headers: dict[str, str] | None = None
|
||||
upstream_provider: str | None = None
|
||||
upstream_error_type: str | None = None
|
||||
# The provider rejected the request (named in the gateway's error body).
|
||||
upstream_rejected: bool = False
|
||||
|
||||
|
||||
_http_reply: ContextVar[HttpReply | None] = ContextVar("strix_llm_http_reply", default=None)
|
||||
_FAILED_REPLY_ATTR = "_strix_http_reply"
|
||||
|
||||
|
||||
def record_upstream_provider(
|
||||
provider: object, error_type: object = None, *, rejected: bool = False
|
||||
) -> None:
|
||||
"""Remember which provider behind a gateway served the attempt in flight."""
|
||||
reply = _http_reply.get()
|
||||
if reply is None:
|
||||
return
|
||||
if isinstance(provider, str) and provider:
|
||||
reply.upstream_provider = provider
|
||||
reply.upstream_rejected = rejected
|
||||
if isinstance(error_type, str) and error_type:
|
||||
reply.upstream_error_type = error_type
|
||||
|
||||
|
||||
def _upstream_details(reply: HttpReply | None) -> list[tuple[str, object]]:
|
||||
if reply is None:
|
||||
return []
|
||||
return [
|
||||
("upstream_provider", reply.upstream_provider),
|
||||
("upstream_error_type", reply.upstream_error_type),
|
||||
]
|
||||
|
||||
|
||||
def failed_reply(exc: BaseException) -> HttpReply | None:
|
||||
"""The reply of the attempt that raised ``exc``."""
|
||||
reply = getattr(exc, _FAILED_REPLY_ATTR, None)
|
||||
return reply if isinstance(reply, HttpReply) else None
|
||||
|
||||
|
||||
async def record_http_reply(response: Response) -> None:
|
||||
@@ -806,7 +772,6 @@ def _litellm_success(
|
||||
("request", _litellm_request_details(kwargs)),
|
||||
("response", _litellm_response_details(response)),
|
||||
("litellm", _litellm_hidden_details(hidden, slo)),
|
||||
*_upstream_details(_http_reply.get()),
|
||||
),
|
||||
)
|
||||
|
||||
@@ -856,7 +821,6 @@ def _litellm_failure(
|
||||
("request", _litellm_request_details(kwargs)),
|
||||
("error", error or None),
|
||||
("litellm", _litellm_hidden_details(hidden, slo)),
|
||||
*_upstream_details(_http_reply.get()),
|
||||
),
|
||||
)
|
||||
|
||||
@@ -930,18 +894,15 @@ def _observe_sdk_shared_http_client() -> None:
|
||||
|
||||
|
||||
def _log_line_sink(event: LlmRequestEvent) -> None:
|
||||
details = event.details or {}
|
||||
level = logging.DEBUG if event.outcome == "success" else logging.WARNING
|
||||
logger.log(
|
||||
level,
|
||||
"llm_request route=%s provider=%s upstream=%s upstream_error=%s model=%s host=%s "
|
||||
"outcome=%s status=%s request_id=%s response_id=%s stream=%s duration_ms=%d ttft_ms=%s "
|
||||
"llm_request route=%s provider=%s model=%s host=%s outcome=%s status=%s "
|
||||
"request_id=%s response_id=%s stream=%s duration_ms=%d ttft_ms=%s "
|
||||
"req_bytes=%s res_bytes=%s finish=%s "
|
||||
"in=%s out=%s cached=%s cost=%s agent=%s attempt=%d%s",
|
||||
event.route,
|
||||
event.provider or "-",
|
||||
details.get("upstream_provider") or "-",
|
||||
details.get("upstream_error_type") or "-",
|
||||
event.model,
|
||||
event.api_host or "-",
|
||||
event.outcome,
|
||||
@@ -1109,7 +1070,6 @@ class RequestLoggingModel(Model):
|
||||
details=merge_details(
|
||||
("request", request.details),
|
||||
("response", _openai_response_details(response, raw_response)),
|
||||
*_upstream_details(reply),
|
||||
),
|
||||
)
|
||||
status, request_id = _openai_error_fields(exc)
|
||||
@@ -1129,7 +1089,6 @@ class RequestLoggingModel(Model):
|
||||
details=merge_details(
|
||||
("request", request.details),
|
||||
("error", _exception_details(exc)),
|
||||
*_upstream_details(reply),
|
||||
),
|
||||
)
|
||||
|
||||
@@ -1219,7 +1178,6 @@ class RequestLoggingModel(Model):
|
||||
prompt=prompt,
|
||||
)
|
||||
except BaseException as exc:
|
||||
setattr(exc, _FAILED_REPLY_ATTR, reply)
|
||||
if self._should_emit(exc):
|
||||
emit(
|
||||
self._event(
|
||||
@@ -1314,7 +1272,6 @@ class RequestLoggingModel(Model):
|
||||
raw_response = event.response
|
||||
yield event
|
||||
except BaseException as exc:
|
||||
setattr(exc, _FAILED_REPLY_ATTR, reply)
|
||||
if self._should_emit(exc):
|
||||
emit(
|
||||
self._event(
|
||||
|
||||
@@ -6,6 +6,8 @@ from types import SimpleNamespace
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import pytest
|
||||
from agents.extensions.models.litellm_model import InternalChatCompletionMessage
|
||||
from agents.models.chatcmpl_converter import Converter
|
||||
from litellm.exceptions import BadRequestError, ContextWindowExceededError, RateLimitError
|
||||
from openai.types.responses import ResponseOutputMessage, ResponseOutputText
|
||||
|
||||
@@ -242,6 +244,29 @@ async def test_summarize_routes_through_provider_with_settings(
|
||||
assert settings.max_tokens == 64
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_summarize_falls_back_to_reasoning_when_no_output_text(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
_patch_budget(monkeypatch, keep_tokens=30, window=4_000)
|
||||
message = InternalChatCompletionMessage(
|
||||
role="assistant", content=None, reasoning_content="REASONED SUMMARY"
|
||||
)
|
||||
response = SimpleNamespace(output=Converter.message_to_output_items(message))
|
||||
|
||||
class FakeModel:
|
||||
async def get_response(self, **_kwargs: Any) -> Any:
|
||||
return response
|
||||
|
||||
class FakeProvider:
|
||||
def get_model(self, _model_name: str | None) -> Any:
|
||||
return FakeModel()
|
||||
|
||||
monkeypatch.setattr(compaction, "StrixProvider", FakeProvider)
|
||||
|
||||
assert await compaction._summarize("m", "p", 64) == "REASONED SUMMARY"
|
||||
|
||||
|
||||
def test_fit_to_tokens_truncates_oversized_text(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setattr(compaction, "count_tokens", lambda _m, t: len(t))
|
||||
text = "x" * 10_000
|
||||
|
||||
@@ -2,7 +2,6 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import uuid
|
||||
from types import SimpleNamespace
|
||||
from typing import TYPE_CHECKING, Any
|
||||
@@ -390,100 +389,3 @@ def test_openrouter_request_carries_agent_session_id() -> None:
|
||||
assert body()["session_id"] == session_id
|
||||
finally:
|
||||
request_log.reset_call_context(token)
|
||||
|
||||
|
||||
def test_openrouter_request_ignores_banned_providers() -> None:
|
||||
_install_openrouter_stream_cost_capture()
|
||||
config = ProviderConfigManager.get_provider_chat_config(
|
||||
model="moonshotai/kimi-k3", provider=LlmProviders.OPENROUTER
|
||||
)
|
||||
assert config is not None
|
||||
messages: list[AllMessageValues] = [{"role": "user", "content": "hi"}]
|
||||
pro_preferences = {"order": ["novita", "relace"], "allow_fallbacks": True, "ignore": ["a"]}
|
||||
with (
|
||||
patch(
|
||||
"litellm.llms.openrouter.chat.transformation.OpenrouterConfig.transform_request",
|
||||
return_value={"provider": pro_preferences},
|
||||
),
|
||||
patch("strix.llm.provider_bans.banned_providers", return_value=["relace"]),
|
||||
):
|
||||
body = config.transform_request(
|
||||
model="moonshotai/kimi-k3",
|
||||
messages=messages,
|
||||
optional_params={},
|
||||
litellm_params={},
|
||||
headers={},
|
||||
)
|
||||
|
||||
assert body["provider"] == {
|
||||
"order": ["novita", "relace"],
|
||||
"allow_fallbacks": True,
|
||||
"ignore": ["a", "relace"],
|
||||
}
|
||||
assert pro_preferences["ignore"] == ["a"]
|
||||
|
||||
|
||||
def _openrouter_config() -> Any:
|
||||
_install_openrouter_stream_cost_capture()
|
||||
config = ProviderConfigManager.get_provider_chat_config(
|
||||
model="z-ai/glm-5.3", provider=LlmProviders.OPENROUTER
|
||||
)
|
||||
assert config is not None
|
||||
return config
|
||||
|
||||
|
||||
def test_openrouter_records_upstream_of_stream_and_mid_stream_error() -> None:
|
||||
handler = _openrouter_config().get_model_response_iterator(
|
||||
streaming_response=iter([]), sync_stream=True
|
||||
)
|
||||
reply = request_log.HttpReply()
|
||||
token = request_log._http_reply.set(reply)
|
||||
try:
|
||||
handler.chunk_parser(
|
||||
{
|
||||
"id": "gen-a",
|
||||
"created": 1,
|
||||
"model": "z-ai/glm-5.3",
|
||||
"provider": "Relace",
|
||||
"choices": [{"index": 0, "delta": {"content": "x"}}],
|
||||
}
|
||||
)
|
||||
assert reply.upstream_provider == "Relace"
|
||||
assert reply.upstream_error_type is None
|
||||
with pytest.raises(Exception, match="incomplete tool call"):
|
||||
handler.chunk_parser(
|
||||
{
|
||||
"id": "gen-a",
|
||||
"provider": "Relace",
|
||||
"error": {
|
||||
"code": 400,
|
||||
"message": "Generation stopped with an incomplete tool call.",
|
||||
"metadata": {"error_type": "invalid_request"},
|
||||
},
|
||||
}
|
||||
)
|
||||
finally:
|
||||
request_log._http_reply.reset(token)
|
||||
assert reply.upstream_provider == "Relace"
|
||||
assert reply.upstream_error_type == "invalid_request"
|
||||
assert reply.upstream_rejected is False
|
||||
|
||||
|
||||
def test_openrouter_records_upstream_of_pre_stream_error() -> None:
|
||||
body = {
|
||||
"error": {
|
||||
"message": "Provider returned error",
|
||||
"code": 400,
|
||||
"metadata": {"raw": "...", "provider_name": "InferenceNet"},
|
||||
}
|
||||
}
|
||||
reply = request_log.HttpReply()
|
||||
token = request_log._http_reply.set(reply)
|
||||
try:
|
||||
_openrouter_config().get_error_class(json.dumps(body), 400, {})
|
||||
_openrouter_config().get_error_class("<html>502 Bad Gateway</html>", 502, {})
|
||||
finally:
|
||||
request_log._http_reply.reset(token)
|
||||
assert reply.upstream_provider == "InferenceNet"
|
||||
assert reply.upstream_error_type is None
|
||||
assert reply.upstream_rejected is True
|
||||
@@ -1,7 +1,6 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from collections import Counter
|
||||
from typing import Any
|
||||
from typing import Any, cast
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
@@ -19,7 +18,6 @@ from openai import (
|
||||
from strix.config import codex
|
||||
from strix.core import execution
|
||||
from strix.core.agents import AgentCoordinator
|
||||
from strix.llm import provider_bans, request_log
|
||||
|
||||
|
||||
def _request() -> httpx.Request:
|
||||
@@ -81,74 +79,15 @@ def test_content_guardrail_is_not_retried() -> None:
|
||||
assert execution._is_transient_model_error(guardrail) is False
|
||||
|
||||
|
||||
def _upstream_error(
|
||||
status: int, provider: str, *, error_type: str | None = None, rejected: bool = False
|
||||
) -> APIStatusError:
|
||||
exc = _status_error(status)
|
||||
reply = request_log.HttpReply(
|
||||
upstream_provider=provider, upstream_error_type=error_type, upstream_rejected=rejected
|
||||
)
|
||||
setattr(exc, request_log._FAILED_REPLY_ATTR, reply)
|
||||
return exc
|
||||
|
||||
|
||||
def test_client_errors_are_transient() -> None:
|
||||
def test_client_errors_are_not_transient() -> None:
|
||||
bad_request = BadRequestError(
|
||||
"bad", response=httpx.Response(400, request=_request()), body=None
|
||||
)
|
||||
assert execution._is_transient_model_error(bad_request) is True
|
||||
assert execution._is_transient_model_error(_status_error(404)) is True
|
||||
assert execution._is_transient_model_error(bad_request) is False
|
||||
assert execution._is_transient_model_error(_status_error(404)) is False
|
||||
assert execution._is_transient_model_error(ValueError("nope")) is False
|
||||
|
||||
|
||||
def test_gateway_auth_credit_and_policy_errors_are_not_transient() -> None:
|
||||
for status in (401, 402, 403):
|
||||
assert execution._is_transient_model_error(_status_error(status)) is False
|
||||
assert execution._is_transient_model_error(_upstream_error(401, "Relace")) is True
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def bans(monkeypatch: pytest.MonkeyPatch) -> list[tuple[str, str, str]]:
|
||||
calls: list[tuple[str, str, str]] = []
|
||||
|
||||
def _ban(model: str, provider: str, reason: str) -> None:
|
||||
calls.append((model, provider, reason))
|
||||
|
||||
monkeypatch.setattr(provider_bans, "ban_provider", _ban)
|
||||
monkeypatch.setattr(execution, "_provider_4xx_counts", Counter())
|
||||
return calls
|
||||
|
||||
|
||||
_OPENROUTER_RUN_CONFIG = RunConfig(model="openrouter/z-ai/glm-5.3")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_bans_upstream_right_away_on_provider_unavailable_or_rejection(
|
||||
bans: list[tuple[str, str, str]],
|
||||
) -> None:
|
||||
unavailable = _upstream_error(502, "Relace", error_type="provider_unavailable")
|
||||
await execution._maybe_ban_provider(unavailable, _OPENROUTER_RUN_CONFIG)
|
||||
rejected = _upstream_error(400, "InferenceNet", rejected=True)
|
||||
await execution._maybe_ban_provider(rejected, _OPENROUTER_RUN_CONFIG)
|
||||
assert bans == [
|
||||
("openrouter/z-ai/glm-5.3", "Relace", "provider_unavailable"),
|
||||
("openrouter/z-ai/glm-5.3", "InferenceNet", "upstream_rejected_400"),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_bans_upstream_after_repeated_mid_stream_4xx(
|
||||
bans: list[tuple[str, str, str]],
|
||||
) -> None:
|
||||
for _ in range(2):
|
||||
await execution._maybe_ban_provider(_upstream_error(400, "Relace"), _OPENROUTER_RUN_CONFIG)
|
||||
await execution._maybe_ban_provider(
|
||||
_upstream_error(504, "Together"), _OPENROUTER_RUN_CONFIG
|
||||
)
|
||||
await execution._maybe_ban_provider(_status_error(400), _OPENROUTER_RUN_CONFIG)
|
||||
assert bans == [("openrouter/z-ai/glm-5.3", "Relace", "repeated_4xx")]
|
||||
|
||||
|
||||
class _FakeStream:
|
||||
def __init__(self, exc: BaseException | None = None) -> None:
|
||||
self._exc = exc
|
||||
@@ -189,7 +128,7 @@ async def _run_once(
|
||||
coordinator,
|
||||
"root",
|
||||
input_data="task",
|
||||
run_config=RunConfig(),
|
||||
run_config=cast("RunConfig", object()),
|
||||
context={},
|
||||
max_turns=5,
|
||||
session=None,
|
||||
@@ -227,6 +166,9 @@ async def test_run_cycle_gives_up_after_max_retries(
|
||||
async def test_run_cycle_does_not_retry_permanent_error(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
streams = [_FakeStream(exc=_status_error(401)), _FakeStream()]
|
||||
with pytest.raises(APIStatusError):
|
||||
bad_request = BadRequestError(
|
||||
"bad", response=httpx.Response(400, request=_request()), body=None
|
||||
)
|
||||
streams = [_FakeStream(exc=bad_request), _FakeStream()]
|
||||
with pytest.raises(BadRequestError):
|
||||
await _run_once(monkeypatch, streams)
|
||||
@@ -394,30 +394,9 @@ def test_log_line_sink_formats_without_content(caplog: pytest.LogCaptureFixture)
|
||||
assert "request_id=req_line01" in line
|
||||
assert "status=400" in line
|
||||
assert "provider=anthropic" in line
|
||||
assert "upstream=- upstream_error=-" in line
|
||||
assert "SECRET PROMPT" not in line
|
||||
|
||||
|
||||
def test_log_line_names_upstream_recorded_during_attempt(
|
||||
caplog: pytest.LogCaptureFixture,
|
||||
) -> None:
|
||||
exc = _anthropic_error(400, ANTHROPIC_BLOCK_BODY, {})
|
||||
token = request_log._http_reply.set(request_log.HttpReply())
|
||||
try:
|
||||
request_log.record_upstream_provider("Together", "provider_unavailable")
|
||||
event = request_log.event_from_litellm(
|
||||
_anthropic_kwargs(exc), None, None, None, outcome="error"
|
||||
)
|
||||
finally:
|
||||
request_log._http_reply.reset(token)
|
||||
assert event.details is not None
|
||||
assert event.details["upstream_provider"] == "Together"
|
||||
with caplog.at_level(logging.DEBUG, logger="strix.llm.request_log"):
|
||||
request_log._log_line_sink(event)
|
||||
line = caplog.records[-1].getMessage()
|
||||
assert "upstream=Together upstream_error=provider_unavailable" in line
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# sizes, timing, finish reason, free-form headers and details #
|
||||
# --------------------------------------------------------------------------- #
|
||||
@@ -1235,23 +1214,6 @@ async def test_litellm_route_wrapper_only_logs_abandoned_attempts(
|
||||
assert event.request_bytes is not None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_failed_attempt_reply_rides_on_its_exception() -> None:
|
||||
exc = _openai_status_error(400, None)
|
||||
model = request_log.RequestLoggingModel(
|
||||
_Inner(exc=exc),
|
||||
model_name="openrouter/z-ai/glm-5.3",
|
||||
provider="openrouter",
|
||||
base_url=None,
|
||||
route="litellm",
|
||||
abandoned_only=True,
|
||||
)
|
||||
with pytest.raises(APIStatusError):
|
||||
await model.get_response(*_CALL_ARGS, **_CALL_KWARGS)
|
||||
assert isinstance(request_log.failed_reply(exc), request_log.HttpReply)
|
||||
assert request_log.failed_reply(ValueError()) is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stream_closed_early_is_logged_as_abandoned(captured: list[LlmRequestEvent]) -> None:
|
||||
created = ResponseCreatedEvent(
|
||||
|
||||
@@ -1,75 +0,0 @@
|
||||
"""Tests for per-run OpenRouter provider bans."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
from strix.llm import provider_bans
|
||||
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Iterator
|
||||
|
||||
|
||||
MODEL = "openrouter/z-ai/glm-5.3"
|
||||
ENDPOINTS = [
|
||||
{"provider_name": "Relace", "tag": "relace"},
|
||||
{"provider_name": "InferenceNet", "tag": "inference-net/fp4"},
|
||||
{"provider_name": "Google", "tag": "google-vertex/us-east5"},
|
||||
{"provider_name": "Google", "tag": "google-vertex"},
|
||||
]
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def endpoints() -> Iterator[MagicMock]:
|
||||
for state in (provider_bans._SLUGS, provider_bans._BANNED):
|
||||
state.clear()
|
||||
response = httpx.Response(
|
||||
200,
|
||||
json={"data": {"endpoints": ENDPOINTS}},
|
||||
request=httpx.Request("GET", "https://openrouter.ai"),
|
||||
)
|
||||
with patch("strix.llm.provider_bans.httpx.get", return_value=response) as get:
|
||||
yield get
|
||||
|
||||
|
||||
def test_ban_maps_provider_names_to_slugs(endpoints: MagicMock) -> None:
|
||||
provider_bans.ban_provider(MODEL, "InferenceNet", "context_length")
|
||||
provider_bans.ban_provider("z-ai/glm-5.3", "google", "provider_unavailable")
|
||||
provider_bans.ban_provider(MODEL, "InferenceNet", "context_length")
|
||||
|
||||
assert provider_bans.banned_providers(MODEL) == ["google-vertex", "inference-net"]
|
||||
assert provider_bans.banned_providers("openrouter/other/model") == []
|
||||
endpoints.assert_called_once_with(
|
||||
"https://openrouter.ai/api/v1/models/z-ai/glm-5.3/endpoints", timeout=5
|
||||
)
|
||||
|
||||
|
||||
def test_banning_every_provider_unbans_them_all() -> None:
|
||||
provider_bans.ban_provider(MODEL, "Relace", "incomplete_tool_call")
|
||||
provider_bans.ban_provider(MODEL, "InferenceNet", "context_length")
|
||||
assert provider_bans.banned_providers(MODEL) == ["inference-net", "relace"]
|
||||
provider_bans.ban_provider(MODEL, "Google", "provider_unavailable")
|
||||
|
||||
assert provider_bans.banned_providers(MODEL) == []
|
||||
|
||||
|
||||
def test_ban_is_skipped_for_an_unknown_provider_or_failed_lookup(endpoints: MagicMock) -> None:
|
||||
provider_bans.ban_provider(MODEL, "Nobody", "provider_unavailable")
|
||||
endpoints.side_effect = httpx.ConnectError("offline")
|
||||
provider_bans.ban_provider("openrouter/other/model", "Relace", "x")
|
||||
|
||||
assert provider_bans.banned_providers(MODEL) == []
|
||||
|
||||
|
||||
def test_bans_can_be_turned_off(endpoints: MagicMock) -> None:
|
||||
with patch("strix.llm.provider_bans.load_settings") as settings:
|
||||
settings.return_value.llm.openrouter_provider_bans = False
|
||||
provider_bans.ban_provider(MODEL, "Relace", "incomplete_tool_call")
|
||||
|
||||
assert provider_bans.banned_providers(MODEL) == []
|
||||
endpoints.assert_not_called()
|
||||
Reference in new issue
Block a user